diff options
Diffstat (limited to 'nikola/plugins/task')
31 files changed, 451 insertions, 1454 deletions
diff --git a/nikola/plugins/task/archive.py b/nikola/plugins/task/archive.py index a65a63f..4f1ab19 100644 --- a/nikola/plugins/task/archive.py +++ b/nikola/plugins/task/archive.py @@ -73,16 +73,15 @@ class Archive(Task): context["permalink"] = self.site.link("archive", year, lang) if not kw["create_monthly_archive"]: template_name = "list_post.tmpl" - post_list = [self.site.global_data[post] for post in posts] - post_list.sort(key=lambda a: a.date) + post_list = sorted(posts, key=lambda a: a.date) post_list.reverse() context["posts"] = post_list else: # Monthly archives, just list the months - months = set([m.split('/')[1] for m in self.site.posts_per_month.keys() if m.startswith(str(year))]) + months = set([(m.split('/')[1], self.site.link("archive", m, lang)) for m in self.site.posts_per_month.keys() if m.startswith(str(year))]) months = sorted(list(months)) months.reverse() template_name = "list.tmpl" - context["items"] = [[nikola.utils.LocaleBorg().get_month_name(int(month), lang), month] for month in months] + context["items"] = [[nikola.utils.LocaleBorg().get_month_name(int(month), lang), link] for month, link in months] post_list = [] task = self.site.generic_post_list_renderer( lang, @@ -93,7 +92,12 @@ class Archive(Task): context, ) n = len(post_list) if 'posts' in context else len(months) - task_cfg = {1: task['uptodate'][0].config, 2: kw, 3: n} + + deps_translatable = {} + for k in self.site._GLOBAL_CONTEXT_TRANSLATABLE: + deps_translatable[k] = self.site.GLOBAL_CONTEXT[k](lang) + + task_cfg = {1: task['uptodate'][0].config, 2: kw, 3: n, 4: deps_translatable} task['uptodate'] = [config_changed(task_cfg)] task['basename'] = self.name yield task @@ -106,8 +110,7 @@ class Archive(Task): kw['output_folder'], self.site.path("archive", yearmonth, lang)) year, month = yearmonth.split('/') - post_list = [self.site.global_data[post] for post in posts] - post_list.sort(key=lambda a: a.date) + post_list = sorted(posts, key=lambda a: a.date) post_list.reverse() context = {} context["lang"] = lang @@ -141,8 +144,8 @@ class Archive(Task): kw['output_folder'], self.site.path("archive", None, lang)) context["title"] = kw["messages"][lang]["Archive"] - context["items"] = [(year, self.site.link("archive", year, lang)) - for year in years] + context["items"] = [(y, self.site.link("archive", y, lang)) + for y in years] context["permalink"] = self.site.link("archive", None, lang) task = self.site.generic_post_list_renderer( lang, diff --git a/nikola/plugins/task/build_less.plugin b/nikola/plugins/task/build_less.plugin deleted file mode 100644 index 27ca8cd..0000000 --- a/nikola/plugins/task/build_less.plugin +++ /dev/null @@ -1,10 +0,0 @@ -[Core] -Name = build_less -Module = build_less - -[Documentation] -Author = Roberto Alsina -Version = 0.1 -Website = http://getnikola.com -Description = Build CSS out of LESS sources - diff --git a/nikola/plugins/task/build_less.py b/nikola/plugins/task/build_less.py deleted file mode 100644 index a672282..0000000 --- a/nikola/plugins/task/build_less.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2012-2014 Roberto Alsina and others. - -# Permission is hereby granted, free of charge, to any -# person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the -# Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the -# Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice -# shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from __future__ import unicode_literals - -import codecs -import glob -import os -import sys -import subprocess - -from nikola.plugin_categories import Task -from nikola import utils - - -class BuildLess(Task): - """Generate CSS out of LESS sources.""" - - name = "build_less" - sources_folder = "less" - sources_ext = ".less" - - def gen_tasks(self): - """Generate CSS out of LESS sources.""" - self.compiler_name = self.site.config['LESS_COMPILER'] - self.compiler_options = self.site.config['LESS_OPTIONS'] - - kw = { - 'cache_folder': self.site.config['CACHE_FOLDER'], - 'themes': self.site.THEMES, - } - tasks = {} - - # Find where in the theme chain we define the LESS targets - # There can be many *.less in the folder, but we only will build - # the ones listed in less/targets - if os.path.isfile(os.path.join(self.sources_folder, "targets")): - targets_path = os.path.join(self.sources_folder, "targets") - else: - targets_path = utils.get_asset_path(os.path.join(self.sources_folder, "targets"), self.site.THEMES) - try: - with codecs.open(targets_path, "rb", "utf-8") as inf: - targets = [x.strip() for x in inf.readlines()] - except Exception: - targets = [] - - for task in utils.copy_tree(self.sources_folder, os.path.join(kw['cache_folder'], self.sources_folder)): - if task['name'] in tasks: - continue - task['basename'] = 'prepare_less_sources' - tasks[task['name']] = task - yield task - - for theme_name in kw['themes']: - src = os.path.join(utils.get_theme_path(theme_name), self.sources_folder) - for task in utils.copy_tree(src, os.path.join(kw['cache_folder'], self.sources_folder)): - task['basename'] = 'prepare_less_sources' - yield task - - # Build targets and write CSS files - base_path = utils.get_theme_path(self.site.THEMES[0]) - dst_dir = os.path.join(self.site.config['OUTPUT_FOLDER'], 'assets', 'css') - # Make everything depend on all sources, rough but enough - deps = glob.glob(os.path.join( - base_path, - self.sources_folder, - "*{0}".format(self.sources_ext))) - - def compile_target(target, dst): - utils.makedirs(dst_dir) - src = os.path.join(kw['cache_folder'], self.sources_folder, target) - run_in_shell = sys.platform == 'win32' - try: - compiled = subprocess.check_output([self.compiler_name] + self.compiler_options + [src], shell=run_in_shell) - except OSError: - utils.req_missing([self.compiler_name], - 'build LESS files (and use this theme)', - False, False) - with open(dst, "wb+") as outf: - outf.write(compiled) - - yield self.group_task() - - for target in targets: - dst = os.path.join(dst_dir, target.replace(self.sources_ext, ".css")) - yield { - 'basename': self.name, - 'name': dst, - 'targets': [dst], - 'file_dep': deps, - 'task_dep': ['prepare_less_sources'], - 'actions': ((compile_target, [target, dst]), ), - 'uptodate': [utils.config_changed(kw)], - 'clean': True - } diff --git a/nikola/plugins/task/build_sass.plugin b/nikola/plugins/task/build_sass.plugin deleted file mode 100644 index 746c1df..0000000 --- a/nikola/plugins/task/build_sass.plugin +++ /dev/null @@ -1,9 +0,0 @@ -[Core] -Name = build_sass -Module = build_sass - -[Documentation] -Author = Roberto Alsina, Chris “Kwpolska” Warrick -Version = 0.1 -Website = http://getnikola.com -Description = Build CSS out of Sass sources diff --git a/nikola/plugins/task/build_sass.py b/nikola/plugins/task/build_sass.py deleted file mode 100644 index becc843..0000000 --- a/nikola/plugins/task/build_sass.py +++ /dev/null @@ -1,139 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2012-2014 Roberto Alsina and others. - -# Permission is hereby granted, free of charge, to any -# person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the -# Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the -# Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice -# shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from __future__ import unicode_literals - -import codecs -import glob -import os -import sys -import subprocess - -from nikola.plugin_categories import Task -from nikola import utils - - -class BuildSass(Task): - """Generate CSS out of Sass sources.""" - - name = "build_sass" - sources_folder = "sass" - sources_ext = (".sass", ".scss") - - def gen_tasks(self): - """Generate CSS out of Sass sources.""" - self.logger = utils.get_logger('build_sass', self.site.loghandlers) - self.compiler_name = self.site.config['SASS_COMPILER'] - self.compiler_options = self.site.config['SASS_OPTIONS'] - - kw = { - 'cache_folder': self.site.config['CACHE_FOLDER'], - 'themes': self.site.THEMES, - } - tasks = {} - - # Find where in the theme chain we define the Sass targets - # There can be many *.sass/*.scss in the folder, but we only - # will build the ones listed in sass/targets - if os.path.isfile(os.path.join(self.sources_folder, "targets")): - targets_path = os.path.join(self.sources_folder, "targets") - else: - targets_path = utils.get_asset_path(os.path.join(self.sources_folder, "targets"), self.site.THEMES) - try: - with codecs.open(targets_path, "rb", "utf-8") as inf: - targets = [x.strip() for x in inf.readlines()] - except Exception: - targets = [] - - for task in utils.copy_tree(self.sources_folder, os.path.join(kw['cache_folder'], self.sources_folder)): - if task['name'] in tasks: - continue - task['basename'] = 'prepare_sass_sources' - tasks[task['name']] = task - yield task - - for theme_name in kw['themes']: - src = os.path.join(utils.get_theme_path(theme_name), self.sources_folder) - for task in utils.copy_tree(src, os.path.join(kw['cache_folder'], self.sources_folder)): - if task['name'] in tasks: - continue - task['basename'] = 'prepare_sass_sources' - tasks[task['name']] = task - yield task - - # Build targets and write CSS files - base_path = utils.get_theme_path(self.site.THEMES[0]) - dst_dir = os.path.join(self.site.config['OUTPUT_FOLDER'], 'assets', 'css') - # Make everything depend on all sources, rough but enough - deps = glob.glob(os.path.join( - base_path, - self.sources_folder, - *("*{0}".format(ext) for ext in self.sources_ext))) - - def compile_target(target, dst): - utils.makedirs(dst_dir) - run_in_shell = sys.platform == 'win32' - src = os.path.join(kw['cache_folder'], self.sources_folder, target) - try: - compiled = subprocess.check_output([self.compiler_name] + self.compiler_options + [src], shell=run_in_shell) - except OSError: - utils.req_missing([self.compiler_name], - 'build Sass files (and use this theme)', - False, False) - with open(dst, "wb+") as outf: - outf.write(compiled) - - yield self.group_task() - - # We can have file conflicts. This is a way to prevent them. - # I orignally wanted to use sets and their cannot-have-duplicates - # magic, but I decided not to do this so we can show the user - # what files were problematic. - # If we didn’t do this, there would be a cryptic message from doit - # instead. - seennames = {} - for target in targets: - base = os.path.splitext(target)[0] - dst = os.path.join(dst_dir, base + ".css") - - if base in seennames: - self.logger.error( - 'Duplicate filenames for Sass compiled files: {0} and ' - '{1} (both compile to {2})'.format( - seennames[base], target, base + ".css")) - else: - seennames.update({base: target}) - - yield { - 'basename': self.name, - 'name': dst, - 'targets': [dst], - 'file_dep': deps, - 'task_dep': ['prepare_sass_sources'], - 'actions': ((compile_target, [target, dst]), ), - 'uptodate': [utils.config_changed(kw)], - 'clean': True - } diff --git a/nikola/plugins/task/bundles.py b/nikola/plugins/task/bundles.py index fcfaf42..7437a9d 100644 --- a/nikola/plugins/task/bundles.py +++ b/nikola/plugins/task/bundles.py @@ -65,8 +65,7 @@ class BuildBundles(LateTask): def build_bundle(output, inputs): out_dir = os.path.join(kw['output_folder'], os.path.dirname(output)) - inputs = [i for i in inputs if os.path.isfile( - os.path.join(out_dir, i))] + inputs = [os.path.relpath(i, out_dir) for i in inputs if os.path.isfile(i)] cache_dir = os.path.join(kw['cache_folder'], 'webassets') utils.makedirs(cache_dir) env = webassets.Environment(out_dir, os.path.dirname(output), @@ -83,20 +82,32 @@ class BuildBundles(LateTask): yield self.group_task() if (webassets is not None and self.site.config['USE_BUNDLES'] is not False): - for name, files in kw['theme_bundles'].items(): + for name, _files in kw['theme_bundles'].items(): output_path = os.path.join(kw['output_folder'], name) dname = os.path.dirname(name) - file_dep = [os.path.join(kw['output_folder'], dname, fname) + files = [] + for fname in _files: + # paths are relative to dirname + files.append(os.path.join(dname, fname)) + file_dep = [os.path.join(kw['output_folder'], fname) for fname in files if - utils.get_asset_path(fname, self.site.THEMES, self.site.config['FILES_FOLDERS'])] + utils.get_asset_path(fname, self.site.THEMES, self.site.config['FILES_FOLDERS']) + or fname == 'assets/css/code.css'] + # code.css will be generated by us if it does not exist in + # FILES_FOLDERS or theme assets. It is guaranteed that the + # generation will happen before this task. task = { 'file_dep': list(file_dep), - 'task_dep': ['copy_assets'], + 'task_dep': ['copy_assets', 'copy_files'], 'basename': str(self.name), 'name': str(output_path), - 'actions': [(build_bundle, (name, files))], + 'actions': [(build_bundle, (name, file_dep))], 'targets': [output_path], - 'uptodate': [utils.config_changed(kw)], + 'uptodate': [ + utils.config_changed({ + 1: kw, + 2: file_dep + })], 'clean': True, } yield utils.apply_filters(task, kw['filters']) diff --git a/nikola/plugins/task/copy_assets.py b/nikola/plugins/task/copy_assets.py index 93b7fb3..4801347 100644 --- a/nikola/plugins/task/copy_assets.py +++ b/nikola/plugins/task/copy_assets.py @@ -45,13 +45,21 @@ class CopyAssets(Task): kw = { "themes": self.site.THEMES, + "files_folders": self.site.config['FILES_FOLDERS'], "output_folder": self.site.config['OUTPUT_FOLDER'], "filters": self.site.config['FILTERS'], "code_color_scheme": self.site.config['CODE_COLOR_SCHEME'], + "code.css_selectors": 'pre.code', + "code.css_head": '/* code.css file generated by Nikola */\n', + "code.css_close": "\ntable.codetable { width: 100%;} td.linenos {text-align: right; width: 4em;}\n", } - has_code_css = False tasks = {} code_css_path = os.path.join(kw['output_folder'], 'assets', 'css', 'code.css') + code_css_input = utils.get_asset_path('assets/css/code.css', + themes=kw['themes'], + files_folders=kw['files_folders']) + + kw["code.css_input"] = code_css_input yield self.group_task() @@ -61,28 +69,35 @@ class CopyAssets(Task): for task in utils.copy_tree(src, dst): if task['name'] in tasks: continue - if task['targets'][0] == code_css_path: - has_code_css = True tasks[task['name']] = task task['uptodate'] = [utils.config_changed(kw)] task['basename'] = self.name + if code_css_input: + task['file_dep'] = [code_css_input] yield utils.apply_filters(task, kw['filters']) - if not has_code_css: # Generate it - + # Check whether or not there is a code.css file around. + if not code_css_input: def create_code_css(): from pygments.formatters import get_formatter_by_name formatter = get_formatter_by_name('html', style=kw["code_color_scheme"]) utils.makedirs(os.path.dirname(code_css_path)) with codecs.open(code_css_path, 'wb+', 'utf8') as outf: - outf.write(formatter.get_style_defs(['pre.code', 'div.code pre'])) - outf.write("\ntable.codetable { width: 100%;} td.linenos {text-align: right; width: 4em;}\n") + outf.write(kw["code.css_head"]) + outf.write(formatter.get_style_defs(kw["code.css_selectors"])) + outf.write(kw["code.css_close"]) + + if os.path.exists(code_css_path): + with codecs.open(code_css_path, 'r', 'utf-8') as fh: + testcontents = fh.read(len(kw["code.css_head"])) == kw["code.css_head"] + else: + testcontents = False task = { 'basename': self.name, 'name': code_css_path, 'targets': [code_css_path], - 'uptodate': [utils.config_changed(kw)], + 'uptodate': [utils.config_changed(kw), testcontents], 'actions': [(create_code_css, [])], 'clean': True, } diff --git a/nikola/plugins/task/galleries.py b/nikola/plugins/task/galleries.py index 880d47c..366374b 100644 --- a/nikola/plugins/task/galleries.py +++ b/nikola/plugins/task/galleries.py @@ -36,6 +36,7 @@ try: except ImportError: from urllib.parse import urljoin # NOQA +import natsort Image = None try: from PIL import Image, ExifTags # NOQA @@ -46,6 +47,7 @@ except ImportError: Image = _Image except ImportError: pass + import PyRSS2Gen as rss from nikola.plugin_categories import Task @@ -97,9 +99,15 @@ class Galleries(Task): 'filters': self.site.config['FILTERS'], 'translations': self.site.config['TRANSLATIONS'], 'global_context': self.site.GLOBAL_CONTEXT, - "feed_length": self.site.config['FEED_LENGTH'], + 'feed_length': self.site.config['FEED_LENGTH'], + 'tzinfo': self.site.tzinfo, + 'comments_in_galleries': self.site.config['COMMENTS_IN_GALLERIES'], + 'generate_rss': self.site.config['GENERATE_RSS'], } + for k, v in self.site.GLOBAL_CONTEXT['template_hooks'].items(): + self.kw['||template_hooks|{0}||'.format(k)] = v._items + yield self.group_task() template_name = "gallery.tmpl" @@ -152,6 +160,9 @@ class Galleries(Task): os.path.relpath(gallery, self.kw['gallery_path']), lang)) dst = os.path.normpath(dst) + for k in self.site._GLOBAL_CONTEXT_TRANSLATABLE: + self.kw[k] = self.site.GLOBAL_CONTEXT[k](lang) + context = {} context["lang"] = lang if post: @@ -165,12 +176,8 @@ class Galleries(Task): if self.kw['use_filename_as_title']: img_titles = [] for fn in image_name_list: - name_without_ext = os.path.splitext(fn)[0] - img_titles.append( - 'id="{0}" alt="{1}" title="{2}"'.format( - name_without_ext, - name_without_ext, - utils.unslugify(name_without_ext))) + name_without_ext = os.path.splitext(os.path.basename(fn))[0] + img_titles.append(utils.unslugify(name_without_ext)) else: img_titles = [''] * len(image_name_list) @@ -189,27 +196,30 @@ class Galleries(Task): ft = folder folders.append((folder, ft)) - ## TODO: in v7 remove images from context, use photo_array - context["images"] = list(zip(image_name_list, thumbs, img_titles)) - context["folders"] = folders + context["folders"] = natsort.natsorted(folders) context["crumbs"] = crumbs context["permalink"] = self.site.link( "gallery", os.path.basename( os.path.relpath(gallery, self.kw['gallery_path'])), lang) - # FIXME: use kw - context["enable_comments"] = ( - self.site.config["COMMENTS_IN_GALLERIES"]) + context["enable_comments"] = self.kw['comments_in_galleries'] context["thumbnail_size"] = self.kw["thumbnail_size"] - # FIXME: render post in a task if post: - post.compile(lang) - context['text'] = post.text(lang) + yield { + 'basename': self.name, + 'name': post.translated_base_path(lang), + 'targets': [post.translated_base_path(lang)], + 'file_dep': post.fragment_deps(lang), + 'actions': [(post.compile, [lang])], + 'uptodate': [utils.config_changed(self.kw)] + } + context['post'] = post else: - context['text'] = '' - + context['post'] = None file_dep = self.site.template_system.template_deps( template_name) + image_list + thumbs + if post: + file_dep += [post.translated_base_path(l) for l in self.kw['translations']] yield utils.apply_filters({ 'basename': self.name, @@ -222,6 +232,7 @@ class Galleries(Task): dst, context, dest_img_list, + img_titles, thumbs, file_dep))], 'clean': True, @@ -233,39 +244,40 @@ class Galleries(Task): }, self.kw['filters']) # RSS for the gallery - rss_dst = os.path.join( - self.kw['output_folder'], - self.site.path( - "gallery_rss", - os.path.relpath(gallery, self.kw['gallery_path']), lang)) - rss_dst = os.path.normpath(rss_dst) - - yield utils.apply_filters({ - 'basename': self.name, - 'name': rss_dst, - 'file_dep': file_dep, - 'targets': [rss_dst], - 'actions': [ - (self.gallery_rss, ( - image_list, - img_titles, - lang, - self.site.link( - "gallery_rss", os.path.basename(gallery), lang), - rss_dst, - context['title'] - ))], - 'clean': True, - 'uptodate': [utils.config_changed({ - 1: self.kw, - })], - }, self.kw['filters']) + if self.kw["generate_rss"]: + rss_dst = os.path.join( + self.kw['output_folder'], + self.site.path( + "gallery_rss", + os.path.relpath(gallery, self.kw['gallery_path']), lang)) + rss_dst = os.path.normpath(rss_dst) + + yield utils.apply_filters({ + 'basename': self.name, + 'name': rss_dst, + 'file_dep': file_dep, + 'targets': [rss_dst], + 'actions': [ + (self.gallery_rss, ( + image_list, + img_titles, + lang, + self.site.link( + "gallery_rss", os.path.basename(gallery), lang), + rss_dst, + context['title'] + ))], + 'clean': True, + 'uptodate': [utils.config_changed({ + 1: self.kw, + })], + }, self.kw['filters']) def find_galleries(self): """Find all galleries to be processed according to conf.py""" self.gallery_list = [] - for root, dirs, files in os.walk(self.kw['gallery_path']): + for root, dirs, files in os.walk(self.kw['gallery_path'], followlinks=True): self.gallery_list.append(root) def create_galleries(self): @@ -433,6 +445,7 @@ class Galleries(Task): output_name, context, img_list, + img_titles, thumbs, file_dep): """Build the gallery index.""" @@ -446,12 +459,9 @@ class Galleries(Task): return url photo_array = [] - for img, thumb in zip(img_list, thumbs): + for img, thumb, title in zip(img_list, thumbs, img_titles): im = Image.open(thumb) w, h = im.size - title = '' - if self.kw['use_filename_as_title']: - title = utils.unslugify(os.path.splitext(img)[0]) # Thumbs are files in output, we need URLs photo_array.append({ 'url': url_from_path(img), @@ -462,9 +472,8 @@ class Galleries(Task): 'h': h }, }) - context['photo_array_json'] = json.dumps(photo_array) context['photo_array'] = photo_array - + context['photo_array_json'] = json.dumps(photo_array) self.site.render_template(template_name, output_name, context) def gallery_rss(self, img_list, img_titles, lang, permalink, output_path, title): @@ -478,12 +487,12 @@ class Galleries(Task): return urljoin(self.site.config['BASE_URL'], url) items = [] - for img, full_title in list(zip(img_list, img_titles))[:self.kw["feed_length"]]: + for img, title in list(zip(img_list, img_titles))[:self.kw["feed_length"]]: img_size = os.stat( os.path.join( self.site.config['OUTPUT_FOLDER'], img)).st_size args = { - 'title': full_title.split('"')[-2] if full_title else '', + 'title': title, 'link': make_url(img), 'guid': rss.Guid(img, False), 'pubDate': self.image_date(img), @@ -494,17 +503,16 @@ class Galleries(Task): ), } items.append(rss.RSSItem(**args)) - rss_obj = utils.ExtendedRSS2( + rss_obj = rss.RSS2( title=title, link=make_url(permalink), description='', lastBuildDate=datetime.datetime.now(), items=items, - generator='Nikola <http://getnikola.com/>', + generator='http://getnikola.com/', language=lang ) - rss_obj.self_url = make_url(permalink) - rss_obj.rss_attrs["xmlns:atom"] = "http://www.w3.org/2005/Atom" + rss_obj.rss_attrs["xmlns:dc"] = "http://purl.org/dc/elements/1.1/" dst_dir = os.path.dirname(output_path) utils.makedirs(dst_dir) with codecs.open(output_path, "wb+", "utf-8") as rss_file: @@ -564,7 +572,7 @@ class Galleries(Task): if exif is not None: for tag, value in list(exif.items()): decoded = ExifTags.TAGS.get(tag, tag) - if decoded == 'DateTimeOriginal': + if decoded in ('DateTimeOriginal', 'DateTimeDigitized'): try: self.dates[src] = datetime.datetime.strptime( value, r'%Y:%m:%d %H:%M:%S') diff --git a/nikola/plugins/task/indexes.py b/nikola/plugins/task/indexes.py index 3f45161..386cc18 100644 --- a/nikola/plugins/task/indexes.py +++ b/nikola/plugins/task/indexes.py @@ -25,8 +25,7 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from __future__ import unicode_literals -import glob -import itertools +from collections import defaultdict import os from nikola.plugin_categories import Task @@ -54,22 +53,23 @@ class Indexes(Task): "index_teasers": self.site.config['INDEX_TEASERS'], "output_folder": self.site.config['OUTPUT_FOLDER'], "filters": self.site.config['FILTERS'], - "hide_untranslated_posts": self.site.config['HIDE_UNTRANSLATED_POSTS'], + "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], "indexes_title": self.site.config['INDEXES_TITLE'], "indexes_pages": self.site.config['INDEXES_PAGES'], "indexes_pages_main": self.site.config['INDEXES_PAGES_MAIN'], "blog_title": self.site.config["BLOG_TITLE"], + "rss_read_more_link": self.site.config["RSS_READ_MORE_LINK"], } template_name = "index.tmpl" - posts = [x for x in self.site.timeline if x.use_in_feeds] + posts = self.site.posts for lang in kw["translations"]: # Split in smaller lists lists = [] - if kw["hide_untranslated_posts"]: - filtered_posts = [x for x in posts if x.is_translation_available(lang)] - else: + if kw["show_untranslated_posts"]: filtered_posts = posts + else: + filtered_posts = [x for x in posts if x.is_translation_available(lang)] lists.append(filtered_posts[:kw["index_display_post_count"]]) filtered_posts = filtered_posts[kw["index_display_post_count"]:] while filtered_posts: @@ -78,7 +78,7 @@ class Indexes(Task): num_pages = len(lists) for i, post_list in enumerate(lists): context = {} - indexes_title = kw['indexes_title'] or kw['blog_title'] + indexes_title = kw['indexes_title'] or kw['blog_title'](lang) if kw["indexes_pages_main"]: ipages_i = i + 1 ipages_msg = "page %d" @@ -134,33 +134,33 @@ class Indexes(Task): "post_pages": self.site.config["post_pages"], "output_folder": self.site.config['OUTPUT_FOLDER'], "filters": self.site.config['FILTERS'], + "index_file": self.site.config['INDEX_FILE'], } template_name = "list.tmpl" for lang in kw["translations"]: # Need to group by folder to avoid duplicated tasks (Issue #758) - for dirname, wildcards in itertools.groupby((w for w, d, x, i in kw["post_pages"] if not i), os.path.dirname): - context = {} - # vim/pyflakes thinks it's unused - # src_dir = os.path.dirname(wildcard) - files = [] - for wildcard in wildcards: - files += glob.glob(wildcard) - post_list = [self.site.global_data[p] for p in files] - output_name = os.path.join(kw["output_folder"], - self.site.path("post_path", - wildcard, - lang)).encode('utf8') - context["items"] = [(post.title(lang), post.permalink(lang)) - for post in post_list] - task = self.site.generic_post_list_renderer(lang, post_list, - output_name, - template_name, - kw['filters'], - context) - task_cfg = {1: task['uptodate'][0].config, 2: kw} - task['uptodate'] = [config_changed(task_cfg)] - task['basename'] = self.name - yield task + # Group all pages by path prefix + groups = defaultdict(list) + for p in self.site.timeline: + if not p.is_post: + dirname = os.path.dirname(p.destination_path(lang)) + groups[dirname].append(p) + for dirname, post_list in groups.items(): + context = {} + context["items"] = [ + (post.title(lang), post.permalink(lang)) + for post in post_list + ] + output_name = os.path.join(kw['output_folder'], dirname, kw['index_file']) + task = self.site.generic_post_list_renderer(lang, post_list, + output_name, + template_name, + kw['filters'], + context) + task_cfg = {1: task['uptodate'][0].config, 2: kw} + task['uptodate'] = [config_changed(task_cfg)] + task['basename'] = self.name + yield task def index_path(self, name, lang): if name not in [None, 0]: diff --git a/nikola/plugins/task/listings.py b/nikola/plugins/task/listings.py index 86be6c4..a0fe974 100644 --- a/nikola/plugins/task/listings.py +++ b/nikola/plugins/task/listings.py @@ -31,11 +31,17 @@ import os from pygments import highlight from pygments.lexers import get_lexer_for_filename, TextLexer from pygments.formatters import HtmlFormatter +import natsort +import re from nikola.plugin_categories import Task from nikola import utils +# FIXME: (almost) duplicated with mdx_nikola.py +CODERE = re.compile('<div class="code"><pre>(.*?)</pre></div>', flags=re.MULTILINE | re.DOTALL) + + class Listings(Task): """Render pretty listings.""" @@ -69,6 +75,9 @@ class Listings(Task): linenos="table", nowrap=False, lineanchors=utils.slugify(in_name), anchorlinenos=True)) + # the pygments highlighter uses <div class="codehilite"><pre> + # for code. We switch it to reST's <pre class="code">. + code = CODERE.sub('<pre class="code literal-block">\\1</pre>', code) title = os.path.basename(in_name) else: code = '' @@ -76,14 +85,27 @@ class Listings(Task): crumbs = utils.get_crumbs(os.path.relpath(out_name, kw['output_folder']), is_file=True) + permalink = self.site.link( + 'listing', + os.path.relpath( + out_name, + os.path.join( + kw['output_folder'], + kw['listings_folder']))) + if self.site.config['COPY_SOURCES']: + source_link = permalink[:-5] + else: + source_link = None context = { 'code': code, 'title': title, 'crumbs': crumbs, + 'permalink': permalink, 'lang': kw['default_lang'], - 'folders': folders, - 'files': files, + 'folders': natsort.natsorted(folders), + 'files': natsort.natsorted(files), 'description': title, + 'source_link': source_link, } self.site.render_template('listing.tmpl', out_name, context) @@ -91,7 +113,21 @@ class Listings(Task): yield self.group_task() template_deps = self.site.template_system.template_deps('listing.tmpl') - for root, dirs, files in os.walk(kw['listings_folder']): + for root, dirs, files in os.walk(kw['listings_folder'], followlinks=True): + files = [f for f in files if os.path.splitext(f)[-1] not in ignored_extensions] + + uptodate = {'c': self.site.GLOBAL_CONTEXT} + + for k, v in self.site.GLOBAL_CONTEXT['template_hooks'].items(): + uptodate['||template_hooks|{0}||'.format(k)] = v._items + + for k in self.site._GLOBAL_CONTEXT_TRANSLATABLE: + uptodate[k] = self.site.GLOBAL_CONTEXT[k](kw['default_lang']) + + uptodate2 = uptodate.copy() + uptodate2['f'] = files + uptodate2['d'] = dirs + # Render all files out_name = os.path.join( kw['output_folder'], @@ -105,8 +141,7 @@ class Listings(Task): 'actions': [(render_listing, [None, out_name, dirs, files])], # This is necessary to reflect changes in blog title, # sidebar links, etc. - 'uptodate': [utils.config_changed( - self.site.GLOBAL_CONTEXT)], + 'uptodate': [utils.config_changed(uptodate2)], 'clean': True, } for f in files: @@ -126,11 +161,25 @@ class Listings(Task): 'actions': [(render_listing, [in_name, out_name])], # This is necessary to reflect changes in blog title, # sidebar links, etc. - 'uptodate': [utils.config_changed( - self.site.GLOBAL_CONTEXT)], + 'uptodate': [utils.config_changed(uptodate)], 'clean': True, } + if self.site.config['COPY_SOURCES']: + out_name = os.path.join( + kw['output_folder'], + root, + f) + yield { + 'basename': self.name, + 'name': out_name, + 'file_dep': [in_name], + 'targets': [out_name], + 'actions': [(utils.copy_file, [in_name, out_name])], + 'clean': True, + } def listing_path(self, name, lang): - return [_f for _f in [self.site.config['LISTINGS_FOLDER'], name + - '.html'] if _f] + if not name.endswith('.html'): + name += '.html' + path_parts = [self.site.config['LISTINGS_FOLDER']] + list(os.path.split(name)) + return [_f for _f in path_parts if _f] diff --git a/nikola/plugins/task/localsearch.plugin b/nikola/plugins/task/localsearch.plugin deleted file mode 100644 index 86accb6..0000000 --- a/nikola/plugins/task/localsearch.plugin +++ /dev/null @@ -1,10 +0,0 @@ -[Core] -Name = local_search -Module = localsearch - -[Documentation] -Author = Roberto Alsina -Version = 0.1 -Website = http://getnikola.com -Description = Create data files for local search via Tipue - diff --git a/nikola/plugins/task/localsearch/MIT-LICENSE.txt b/nikola/plugins/task/localsearch/MIT-LICENSE.txt deleted file mode 100644 index f131068..0000000 --- a/nikola/plugins/task/localsearch/MIT-LICENSE.txt +++ /dev/null @@ -1,20 +0,0 @@ -Tipue Search Copyright (c) 2012 Tipue - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/nikola/plugins/task/localsearch/__init__.py b/nikola/plugins/task/localsearch/__init__.py deleted file mode 100644 index c501d80..0000000 --- a/nikola/plugins/task/localsearch/__init__.py +++ /dev/null @@ -1,106 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2012-2014 Roberto Alsina and others. - -# Permission is hereby granted, free of charge, to any -# person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the -# Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the -# Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice -# shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from __future__ import unicode_literals -import codecs -import json -import os - -from doit.tools import result_dep - -from nikola.plugin_categories import LateTask -from nikola.utils import config_changed, copy_tree, makedirs - -# This is what we need to produce: -#var tipuesearch = {"pages": [ - #{"title": "Tipue Search, a jQuery site search engine", "text": "Tipue - #Search is a site search engine jQuery plugin. It's free for both commercial and - #non-commercial use and released under the MIT License. Tipue Search includes - #features such as word stemming and word replacement.", "tags": "JavaScript", - #"loc": "http://www.tipue.com/search"}, - #{"title": "Tipue Search demo", "text": "Tipue Search demo. Tipue Search is - #a site search engine jQuery plugin.", "tags": "JavaScript", "loc": - #"http://www.tipue.com/search/demo"}, - #{"title": "About Tipue", "text": "Tipue is a small web development/design - #studio based in North London. We've been around for over a decade.", "tags": "", - #"loc": "http://www.tipue.com/about"} -#]}; - - -class Tipue(LateTask): - """Render the blog posts as JSON data.""" - - name = "local_search" - - def gen_tasks(self): - self.site.scan_posts() - - kw = { - "translations": self.site.config['TRANSLATIONS'], - "output_folder": self.site.config['OUTPUT_FOLDER'], - } - - posts = self.site.timeline[:] - dst_path = os.path.join(kw["output_folder"], "assets", "js", - "tipuesearch_content.json") - - def save_data(): - pages = [] - for lang in kw["translations"]: - for post in posts: - # Don't index drafts (Issue #387) - if post.is_draft or post.is_retired or post.publish_later: - continue - text = post.text(lang, strip_html=True) - text = text.replace('^', '') - - data = {} - data["title"] = post.title(lang) - data["text"] = text - data["tags"] = ",".join(post.tags) - data["loc"] = post.permalink(lang) - pages.append(data) - output = json.dumps({"pages": pages}, indent=2) - makedirs(os.path.dirname(dst_path)) - with codecs.open(dst_path, "wb+", "utf8") as fd: - fd.write(output) - - yield { - "basename": str(self.name), - "name": dst_path, - "targets": [dst_path], - "actions": [(save_data, [])], - 'uptodate': [config_changed(kw), result_dep('sitemap')] - } - # Note: The task should run everytime a new file is added or a - # file is changed. We cheat, and depend on the sitemap task, - # to run everytime a new file is added. - - # Copy all the assets to the right places - asset_folder = os.path.join(os.path.dirname(__file__), "files") - for task in copy_tree(asset_folder, kw["output_folder"]): - task["basename"] = str(self.name) - yield task diff --git a/nikola/plugins/task/localsearch/files/assets/css/img/loader.gif b/nikola/plugins/task/localsearch/files/assets/css/img/loader.gif Binary files differdeleted file mode 100644 index 9c97738..0000000 --- a/nikola/plugins/task/localsearch/files/assets/css/img/loader.gif +++ /dev/null diff --git a/nikola/plugins/task/localsearch/files/assets/css/img/search.png b/nikola/plugins/task/localsearch/files/assets/css/img/search.png Binary files differdeleted file mode 100644 index 9ab0f2c..0000000 --- a/nikola/plugins/task/localsearch/files/assets/css/img/search.png +++ /dev/null diff --git a/nikola/plugins/task/localsearch/files/assets/css/tipuesearch.css b/nikola/plugins/task/localsearch/files/assets/css/tipuesearch.css deleted file mode 100644 index 2230193..0000000 --- a/nikola/plugins/task/localsearch/files/assets/css/tipuesearch.css +++ /dev/null @@ -1,159 +0,0 @@ - -/* -Tipue Search 3.0.1 -Copyright (c) 2013 Tipue -Tipue Search is released under the MIT License -http://www.tipue.com/search -*/ - - -#tipue_search_input -{ - font: 12px/1.7 'open sans', sans-serif; - color: #333; - padding: 7px; - width: 150px; - border: 1px solid #e2e2e2; - border-radius: 0; - -moz-appearance: none; - -webkit-appearance: none; - box-shadow: none; - outline: 0; - margin: 0; -} -#tipue_search_input:focus -{ - border: 1px solid #ccc; -} -#tipue_search_button -{ - width: 70px; - height: 36px; - border: 0; - border-radius: 1px; - background: #5193fb url('img/search.png') no-repeat center; - outline: none; -} -#tipue_search_button:hover -{ - background-color: #4589fb; -} - -#tipue_search_content -{ - clear: left; - max-width: 650px; - padding: 25px 0 13px 0; - margin: 0; -} -#tipue_search_loading -{ - padding-top: 60px; - background: #fff url('img/loader.gif') no-repeat left; -} - -#tipue_search_warning_head -{ - font: 300 16px/1.6 'open sans', sans-serif; - color: #333; -} -#tipue_search_warning -{ - font: 12px/1.6 'open sans', sans-serif; - color: #333; - margin: 7px 0; -} -#tipue_search_warning a -{ - color: #3f72d8; - text-decoration: none; -} -#tipue_search_warning a:hover -{ - padding-bottom: 1px; - border-bottom: 1px solid #ccc; -} -#tipue_search_results_count -{ - font: 13px/1.6 'open sans', sans-serif; - color: #333; -} -.tipue_search_content_title -{ - font: 300 23px/1.6 'open sans', sans-serif; - margin-top: 31px; -} -.tipue_search_content_title a -{ - color: #3f72d8; - text-decoration: none; -} -.tipue_search_content_title a:hover -{ - padding-bottom: 1px; - border-bottom: 1px solid #ccc; -} -.tipue_search_content_text -{ - font: 12px/1.7 'open sans', sans-serif; - color: #333; - padding: 13px 0; -} -.tipue_search_content_loc -{ - font: 300 13px/1.7 'open sans', sans-serif; - overflow: auto; -} -.tipue_search_content_loc a -{ - color: #555; - text-decoration: none; -} -.tipue_search_content_loc a:hover -{ - padding-bottom: 1px; - border-bottom: 1px solid #ccc; -} -#tipue_search_foot -{ - margin: 51px 0 21px 0; -} -#tipue_search_foot_boxes -{ - padding: 0; - margin: 0; - font: 12px/1 'open sans', sans-serif; -} -#tipue_search_foot_boxes li -{ - list-style: none; - margin: 0; - padding: 0; - display: inline; -} -#tipue_search_foot_boxes li a -{ - padding: 7px 13px 8px 13px; - background-color: #f1f1f1; - border: 1px solid #dcdcdc; - border-radius: 1px; - color: #333; - margin-right: 7px; - text-decoration: none; - text-align: center; -} -#tipue_search_foot_boxes li.current -{ - padding: 7px 13px 8px 13px; - background: #fff; - border: 1px solid #dcdcdc; - border-radius: 1px; - color: #333; - margin-right: 7px; - text-align: center; -} -#tipue_search_foot_boxes li a:hover -{ - border: 1px solid #ccc; - background-color: #f3f3f3; -} diff --git a/nikola/plugins/task/localsearch/files/assets/js/tipuesearch.js b/nikola/plugins/task/localsearch/files/assets/js/tipuesearch.js deleted file mode 100644 index a9982cd..0000000 --- a/nikola/plugins/task/localsearch/files/assets/js/tipuesearch.js +++ /dev/null @@ -1,384 +0,0 @@ - -/* -Tipue Search 3.0.1 -Copyright (c) 2013 Tipue -Tipue Search is released under the MIT License -http://www.tipue.com/search -*/ - - -(function($) { - - $.fn.tipuesearch = function(options) { - - var set = $.extend( { - - 'show' : 7, - 'newWindow' : false, - 'showURL' : true, - 'minimumLength' : 3, - 'descriptiveWords' : 25, - 'highlightTerms' : true, - 'highlightEveryTerm' : false, - 'mode' : 'static', - 'liveDescription' : '*', - 'liveContent' : '*', - 'contentLocation' : 'tipuesearch/tipuesearch_content.json' - - }, options); - - return this.each(function() { - - var tipuesearch_in = { - pages: [] - }; - $.ajaxSetup({ - async: false - }); - - if (set.mode == 'live') - { - for (var i = 0; i < tipuesearch_pages.length; i++) - { - $.get(tipuesearch_pages[i], '', - function (html) - { - var cont = $(set.liveContent, html).text(); - cont = cont.replace(/\s+/g, ' '); - var desc = $(set.liveDescription, html).text(); - desc = desc.replace(/\s+/g, ' '); - - var t_1 = html.toLowerCase().indexOf('<title>'); - var t_2 = html.toLowerCase().indexOf('</title>', t_1 + 7); - if (t_1 != -1 && t_2 != -1) - { - var tit = html.slice(t_1 + 7, t_2); - } - else - { - var tit = 'No title'; - } - - tipuesearch_in.pages.push({ - "title": tit, - "text": desc, - "tags": cont, - "loc": tipuesearch_pages[i] - }); - } - ); - } - } - - if (set.mode == 'json') - { - $.getJSON(set.contentLocation, - function(json) - { - tipuesearch_in = $.extend({}, json); - } - ); - } - - if (set.mode == 'static') - { - tipuesearch_in = $.extend({}, tipuesearch); - } - - var tipue_search_w = ''; - if (set.newWindow) - { - tipue_search_w = ' target="_blank"'; - } - - function getURLP(name) - { - return decodeURIComponent((new RegExp('[?|&]' + name + '=' + '([^&;]+?)(&|#|;|$)').exec(location.search)||[,""])[1].replace(/\+/g, '%20')) || null; - } - if (getURLP('q')) - { - $('#tipue_search_input').val(getURLP('q')); - getTipueSearch(0, true); - } - - $('#tipue_search_button').click(function() - { - getTipueSearch(0, true); - }); - $(this).keyup(function(event) - { - if(event.keyCode == '13') - { - getTipueSearch(0, true); - } - }); - - function getTipueSearch(start, replace) - { - $('#tipue_search_content').hide(); - var out = ''; - var results = ''; - var show_replace = false; - var show_stop = false; - - var d = $('#tipue_search_input').val().toLowerCase(); - d = $.trim(d); - var d_w = d.split(' '); - d = ''; - for (var i = 0; i < d_w.length; i++) - { - var a_w = true; - for (var f = 0; f < tipuesearch_stop_words.length; f++) - { - if (d_w[i] == tipuesearch_stop_words[f]) - { - a_w = false; - show_stop = true; - } - } - if (a_w) - { - d = d + ' ' + d_w[i]; - } - } - d = $.trim(d); - d_w = d.split(' '); - - if (d.length >= set.minimumLength) - { - if (replace) - { - var d_r = d; - for (var i = 0; i < d_w.length; i++) - { - for (var f = 0; f < tipuesearch_replace.words.length; f++) - { - if (d_w[i] == tipuesearch_replace.words[f].word) - { - d = d.replace(d_w[i], tipuesearch_replace.words[f].replace_with); - show_replace = true; - } - } - } - d_w = d.split(' '); - } - - var d_t = d; - for (var i = 0; i < d_w.length; i++) - { - for (var f = 0; f < tipuesearch_stem.words.length; f++) - { - if (d_w[i] == tipuesearch_stem.words[f].word) - { - d_t = d_t + ' ' + tipuesearch_stem.words[f].stem; - } - } - } - d_w = d_t.split(' '); - - var c = 0; - found = new Array(); - for (var i = 0; i < tipuesearch_in.pages.length; i++) - { - var score = 1000000000; - var s_t = tipuesearch_in.pages[i].text; - for (var f = 0; f < d_w.length; f++) - { - var pat = new RegExp(d_w[f], 'i'); - if (tipuesearch_in.pages[i].title.search(pat) != -1) - { - score -= (200000 - i); - } - if (tipuesearch_in.pages[i].text.search(pat) != -1) - { - score -= (150000 - i); - } - - if (set.highlightTerms) - { - if (set.highlightEveryTerm) - { - var patr = new RegExp('(' + d_w[f] + ')', 'gi'); - } - else - { - var patr = new RegExp('(' + d_w[f] + ')', 'i'); - } - s_t = s_t.replace(patr, "<b>$1</b>"); - } - if (tipuesearch_in.pages[i].tags.search(pat) != -1) - { - score -= (100000 - i); - } - - } - if (score < 1000000000) - { - found[c++] = score + '^' + tipuesearch_in.pages[i].title + '^' + s_t + '^' + tipuesearch_in.pages[i].loc; - } - } - - if (c != 0) - { - if (show_replace == 1) - { - out += '<div id="tipue_search_warning_head">Showing results for ' + d + '</div>'; - out += '<div id="tipue_search_warning">Search for <a href="javascript:void(0)" id="tipue_search_replaced">' + d_r + '</a></div>'; - } - if (c == 1) - { - out += '<div id="tipue_search_results_count">1 result</div>'; - } - else - { - c_c = c.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); - out += '<div id="tipue_search_results_count">' + c_c + ' results</div>'; - } - - found.sort(); - var l_o = 0; - for (var i = 0; i < found.length; i++) - { - var fo = found[i].split('^'); - if (l_o >= start && l_o < set.show + start) - { - out += '<div class="tipue_search_content_title"><a href="' + fo[3] + '"' + tipue_search_w + '>' + fo[1] + '</a></div>'; - - var t = fo[2]; - var t_d = ''; - var t_w = t.split(' '); - if (t_w.length < set.descriptiveWords) - { - t_d = t; - } - else - { - for (var f = 0; f < set.descriptiveWords; f++) - { - t_d += t_w[f] + ' '; - } - } - t_d = $.trim(t_d); - if (t_d.charAt(t_d.length - 1) != '.') - { - t_d += ' ...'; - } - out += '<div class="tipue_search_content_text">' + t_d + '</div>'; - - if (set.showURL) - { - t_url = fo[3]; - if (t_url.length > 45) - { - t_url = fo[3].substr(0, 45) + ' ...'; - } - out += '<div class="tipue_search_content_loc"><a href="' + fo[3] + '"' + tipue_search_w + '>' + t_url + '</a></div>'; - } - } - l_o++; - } - - if (c > set.show) - { - var pages = Math.ceil(c / set.show); - var page = (start / set.show); - out += '<div id="tipue_search_foot"><ul id="tipue_search_foot_boxes">'; - - if (start > 0) - { - out += '<li><a href="javascript:void(0)" class="tipue_search_foot_box" id="' + (start - set.show) + '_' + replace + '">Prev</a></li>'; - } - - if (page <= 2) - { - var p_b = pages; - if (pages > 3) - { - p_b = 3; - } - for (var f = 0; f < p_b; f++) - { - if (f == page) - { - out += '<li class="current">' + (f + 1) + '</li>'; - } - else - { - out += '<li><a href="javascript:void(0)" class="tipue_search_foot_box" id="' + (f * set.show) + '_' + replace + '">' + (f + 1) + '</a></li>'; - } - } - } - else - { - var p_b = page + 3; - if (p_b > pages) - { - p_b = pages; - } - for (var f = page; f < p_b; f++) - { - if (f == page) - { - out += '<li class="current">' + (f + 1) + '</li>'; - } - else - { - out += '<li><a href="javascript:void(0)" class="tipue_search_foot_box" id="' + (f * set.show) + '_' + replace + '">' + (f + 1) + '</a></li>'; - } - } - } - - if (page + 1 != pages) - { - out += '<li><a href="javascript:void(0)" class="tipue_search_foot_box" id="' + (start + set.show) + '_' + replace + '">Next</a></li>'; - } - - out += '</ul></div>'; - } - } - else - { - out += '<div id="tipue_search_warning_head">Nothing found</div>'; - } - } - else - { - if (show_stop) - { - out += '<div id="tipue_search_warning_head">Nothing found</div><div id="tipue_search_warning">Common words are largely ignored</div>'; - } - else - { - out += '<div id="tipue_search_warning_head">Search too short</div>'; - if (set.minimumLength == 1) - { - out += '<div id="tipue_search_warning">Should be one character or more</div>'; - } - else - { - out += '<div id="tipue_search_warning">Should be ' + set.minimumLength + ' characters or more</div>'; - } - } - } - - $('#tipue_search_content').html(out); - $('#tipue_search_content').slideDown(200); - - $('#tipue_search_replaced').click(function() - { - getTipueSearch(0, false); - }); - - $('.tipue_search_foot_box').click(function() - { - var id_v = $(this).attr('id'); - var id_a = id_v.split('_'); - - getTipueSearch(parseInt(id_a[0]), id_a[1]); - }); - } - - }); - }; - -})(jQuery); diff --git a/nikola/plugins/task/localsearch/files/assets/js/tipuesearch_set.js b/nikola/plugins/task/localsearch/files/assets/js/tipuesearch_set.js deleted file mode 100644 index 8493ec1..0000000 --- a/nikola/plugins/task/localsearch/files/assets/js/tipuesearch_set.js +++ /dev/null @@ -1,21 +0,0 @@ - -/* -Tipue Search 3.0.1 -Copyright (c) 2013 Tipue -Tipue Search is released under the MIT License -http://www.tipue.com/search -*/ - - -var tipuesearch_stop_words = ["and", "be", "by", "do", "for", "he", "how", "if", "is", "it", "my", "not", "of", "or", "the", "to", "up", "what", "when"]; - -var tipuesearch_replace = {"words": [ - {"word": "tipua", replace_with: "tipue"}, - {"word": "javscript", replace_with: "javascript"} -]}; - -var tipuesearch_stem = {"words": [ - {"word": "e-mail", stem: "email"}, - {"word": "javascript", stem: "script"}, - {"word": "javascript", stem: "js"} -]}; diff --git a/nikola/plugins/task/localsearch/files/tipue_search.html b/nikola/plugins/task/localsearch/files/tipue_search.html deleted file mode 100644 index 789fbe5..0000000 --- a/nikola/plugins/task/localsearch/files/tipue_search.html +++ /dev/null @@ -1,31 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-
-<html>
-<head>
-<title>Tipue Search</title>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-
-<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
-
-<link rel="stylesheet" type="text/css" href="assets/css/tipuesearch.css">
-<script type="text/javascript" src="assets/js/tipuesearch_set.js"></script>
-<script type="text/javascript" src="assets/js/tipuesearch.js"></script>
-
-</head>
-<body>
-
-<div style="float: left;"><input type="text" id="tipue_search_input"></div>
-<div style="float: left; margin-left: 13px;"><input type="button" id="tipue_search_button"></div>
-<div id="tipue_search_content"><div id="tipue_search_loading"></div></div>
-</div>
-
-<script type="text/javascript">
-$(document).ready(function() {
- $('#tipue_search_input').tipuesearch({
- 'mode': 'json',
- 'contentLocation': 'assets/js/tipuesearch_content.json'
- });
-});
-</script>
-</body>
-</html>
diff --git a/nikola/plugins/task/mustache.plugin b/nikola/plugins/task/mustache.plugin deleted file mode 100644 index d6b487a..0000000 --- a/nikola/plugins/task/mustache.plugin +++ /dev/null @@ -1,10 +0,0 @@ -[Core] -Name = render_mustache -Module = mustache - -[Documentation] -Author = Roberto Alsina -Version = 0.1 -Website = http://getnikola.com -Description = Generates the blog's index pages in json. - diff --git a/nikola/plugins/task/mustache/__init__.py b/nikola/plugins/task/mustache/__init__.py deleted file mode 100644 index 5be98f0..0000000 --- a/nikola/plugins/task/mustache/__init__.py +++ /dev/null @@ -1,184 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2012-2014 Roberto Alsina and others. - -# Permission is hereby granted, free of charge, to any -# person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the -# Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the -# Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice -# shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from __future__ import unicode_literals - -import codecs -import json -import os - -from nikola.plugin_categories import Task -from nikola.utils import ( - config_changed, copy_file, LocaleBorg, makedirs, unicode_str, -) - - -class Mustache(Task): - """Render the blog posts as JSON data.""" - - name = "render_mustache" - - def gen_tasks(self): - self.site.scan_posts() - - kw = { - "translations": self.site.config['TRANSLATIONS'], - "index_display_post_count": - self.site.config['INDEX_DISPLAY_POST_COUNT'], - "messages": self.site.MESSAGES, - "index_teasers": self.site.config['INDEX_TEASERS'], - "output_folder": self.site.config['OUTPUT_FOLDER'], - "filters": self.site.config['FILTERS'], - "blog_title": self.site.config['BLOG_TITLE'], - "content_footer": self.site.config['CONTENT_FOOTER'], - } - - # TODO: timeline is global, get rid of it - posts = [x for x in self.site.timeline if x.use_in_feeds] - if not posts: - yield { - 'basename': 'render_mustache', - 'actions': [], - } - return - - def write_file(path, post, lang): - - # Prev/Next links - prev_link = False - if post.prev_post: - prev_link = post.prev_post.permalink(lang).replace(".html", - ".json") - next_link = False - if post.next_post: - next_link = post.next_post.permalink(lang).replace(".html", - ".json") - data = {} - - # Configuration - for k, v in self.site.config.items(): - if isinstance(v, (str, unicode_str)): # NOQA - data[k] = v - - # Tag data - tags = [] - for tag in post.tags: - tags.append({'name': tag, 'link': self.site.link("tag", tag, - lang)}) - data.update({ - "tags": tags, - "tags?": True if tags else False, - }) - - # Template strings - for k, v in kw["messages"][lang].items(): - data["message_" + k] = v - - # Post data - data.update({ - "title": post.title(lang), - "text": post.text(lang), - "prev": prev_link, - "next": next_link, - "date": - post.date.strftime(self.site.GLOBAL_CONTEXT['date_format']), - }) - - # Comments - context = dict(post=post, lang=LocaleBorg().current_lang) - context.update(self.site.GLOBAL_CONTEXT) - data["comment_html"] = self.site.template_system.render_template( - 'mustache-comment-form.tmpl', None, context).strip() - - # Post translations - translations = [] - for langname in kw["translations"]: - if langname == lang: - continue - translations.append({'name': - kw["messages"][langname]["Read in English"], - 'link': "javascript:load_data('%s');" - % post.permalink(langname).replace( - ".html", ".json")}) - data["translations"] = translations - - makedirs(os.path.dirname(path)) - with codecs.open(path, 'wb+', 'utf8') as fd: - fd.write(json.dumps(data)) - - for lang in kw["translations"]: - for i, post in enumerate(posts): - out_path = post.destination_path(lang, ".json") - out_file = os.path.join(kw['output_folder'], out_path) - task = { - 'basename': 'render_mustache', - 'name': out_file, - 'file_dep': post.fragment_deps(lang), - 'targets': [out_file], - 'actions': [(write_file, (out_file, post, lang))], - 'task_dep': ['render_posts'], - 'uptodate': [config_changed({ - 1: post.text(lang), - 2: post.prev_post, - 3: post.next_post, - 4: post.title(lang), - })] - } - yield task - - if posts: - first_post_data = posts[0].permalink( - self.site.config["DEFAULT_LANG"]).replace(".html", ".json") - - # Copy mustache template - src = os.path.join(os.path.dirname(__file__), 'mustache-template.html') - dst = os.path.join(kw['output_folder'], 'mustache-template.html') - yield { - 'basename': 'render_mustache', - 'name': dst, - 'targets': [dst], - 'file_dep': [src], - 'actions': [(copy_file, (src, dst))], - } - - # Copy mustache.html with the right starting file in it - src = os.path.join(os.path.dirname(__file__), 'mustache.html') - dst = os.path.join(kw['output_folder'], 'mustache.html') - - def copy_mustache(): - with codecs.open(src, 'rb', 'utf8') as in_file: - with codecs.open(dst, 'wb+', 'utf8') as out_file: - data = in_file.read().replace('{{first_post_data}}', - first_post_data) - out_file.write(data) - yield { - 'basename': 'render_mustache', - 'name': dst, - 'targets': [dst], - 'file_dep': [src], - 'uptodate': [config_changed({1: first_post_data})], - 'actions': [(copy_mustache, [])], - } diff --git a/nikola/plugins/task/mustache/mustache-template.html b/nikola/plugins/task/mustache/mustache-template.html deleted file mode 100644 index e9a0213..0000000 --- a/nikola/plugins/task/mustache/mustache-template.html +++ /dev/null @@ -1,29 +0,0 @@ -<script id="view" type="text/html"> -<div class="container" id="container"> - <div class="postbox"> - <h1>{{BLOG_TITLE}}</h1> - <hr> - <h2>{{title}}</h2> - Posted on: {{date}}</br> - {{#tags?}} More posts about: - {{#tags}}<a class="tag" href={{link}}><span class="badge badge-info">{{name}}</span></a>{{/tags}} - </br> - {{/tags?}} - {{#translations}}<a href={{link}}>{{name}}</a>{{/translations}} </br> - <hr> - {{{text}}} - <ul class="pager"> - {{#prev}} - <li class="previous"><a href="javascript:load_data('{{prev}}')">{{message_Previous post}}</a></li> - {{/prev}} - {{#next}} - <li class="next"><a href="javascript:load_data('{{next}}')">{{message_Next post}}</a></li> - {{/next}} - </ul> - {{{comment_html}}} - </div> - <div class="footerbox"> - {{{CONTENT_FOOTER}}} - </div> -</div> -</script> diff --git a/nikola/plugins/task/mustache/mustache.html b/nikola/plugins/task/mustache/mustache.html deleted file mode 100644 index 7ff6312..0000000 --- a/nikola/plugins/task/mustache/mustache.html +++ /dev/null @@ -1,34 +0,0 @@ -<head> - <link href="/assets/css/bootstrap.css" rel="stylesheet" type="text/css"> - <link href="/assets/css/bootstrap-responsive.css" rel="stylesheet" type="text/css"> - <link href="/assets/css/rst.css" rel="stylesheet" type="text/css"> - <link href="/assets/css/code.css" rel="stylesheet" type="text/css"> - <link href="/assets/css/colorbox.css" rel="stylesheet" type="text/css"/> - <link href="/assets/css/theme.css" rel="stylesheet" type="text/css"/> - <link href="/assets/css/custom.css" rel="stylesheet" type="text/css"> - <script src="/assets/js/jquery-1.10.2.min.js" type="text/javascript"></script> - <script src="//cdn.jsdelivr.net/jquery.mustache/0.2.7/jquery.mustache.js"></script> - <script src="//cdn.jsdelivr.net/mustache.js/0.7.2/mustache.js"></script> - <script src="/assets/js/jquery.colorbox-min.js" type="text/javascript"></script> - <script type="text/javascript"> -function load_data(dataurl) { - jQuery.getJSON(dataurl, function(data) { - $('body').mustache('view', data, { method: 'html' }); - window.location.hash = '#' + dataurl; - }) -}; -$(document).ready(function() { -$.Mustache.load('/mustache-template.html') - .done(function () { - if (window.location.hash != '') { - load_data(window.location.hash.slice(1)); - } - else { - load_data('{{first_post_data}}'); - }; - }) -}); -</script> -</head> -<body style="padding-top: 0;"> -</body> diff --git a/nikola/plugins/task/pages.py b/nikola/plugins/task/pages.py index f4c0469..aefc5a1 100644 --- a/nikola/plugins/task/pages.py +++ b/nikola/plugins/task/pages.py @@ -40,13 +40,14 @@ class RenderPages(Task): "post_pages": self.site.config["post_pages"], "translations": self.site.config["TRANSLATIONS"], "filters": self.site.config["FILTERS"], - "hide_untranslated_posts": self.site.config['HIDE_UNTRANSLATED_POSTS'], + "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], + "demote_headers": self.site.config['DEMOTE_HEADERS'], } self.site.scan_posts() yield self.group_task() for lang in kw["translations"]: for post in self.site.timeline: - if kw["hide_untranslated_posts"] and not post.is_translation_available(lang): + if not kw["show_untranslated_posts"] and not post.is_translation_available(lang): continue for task in self.site.generic_page_renderer(lang, post, kw["filters"]): diff --git a/nikola/plugins/task/posts.py b/nikola/plugins/task/posts.py index a502b81..8e03122 100644 --- a/nikola/plugins/task/posts.py +++ b/nikola/plugins/task/posts.py @@ -25,12 +25,20 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from copy import copy -import nikola.post from nikola.plugin_categories import Task from nikola import utils +def rest_deps(post, task): + """Add extra_deps from ReST into task. + + The .dep file is created by ReST so not available before the task starts + to execute. + """ + task.file_dep.update(post.extra_deps()) + + class RenderPosts(Task): """Build HTML fragments from metadata and text.""" @@ -43,10 +51,10 @@ class RenderPosts(Task): "translations": self.site.config["TRANSLATIONS"], "timeline": self.site.timeline, "default_lang": self.site.config["DEFAULT_LANG"], - "hide_untranslated_posts": self.site.config['HIDE_UNTRANSLATED_POSTS'], + "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], + "demote_headers": self.site.config['DEMOTE_HEADERS'], } - nikola.post.READ_MORE_LINK = self.site.config['READ_MORE_LINK'] yield self.group_task() for lang in kw["translations"]: @@ -59,7 +67,9 @@ class RenderPosts(Task): 'name': dest, 'file_dep': post.fragment_deps(lang), 'targets': [dest], - 'actions': [(post.compile, (lang, ))], + 'actions': [(post.compile, (lang, )), + (rest_deps, (post,)), + ], 'clean': True, 'uptodate': [utils.config_changed(deps_dict)], } diff --git a/nikola/plugins/task/redirect.py b/nikola/plugins/task/redirect.py index 6fafd13..eccc0ab 100644 --- a/nikola/plugins/task/redirect.py +++ b/nikola/plugins/task/redirect.py @@ -63,4 +63,4 @@ def create_redirect(src, dst): with codecs.open(src, "wb+", "utf8") as fd: fd.write('<!DOCTYPE html><head><title>Redirecting...</title>' '<meta http-equiv="refresh" content="0; ' - 'url={0}"></head>'.format(dst)) + 'url={0}"></head><body><p>Page moved <a href="{0}">here</a></p></body>'.format(dst)) diff --git a/nikola/plugins/task/robots.plugin b/nikola/plugins/task/robots.plugin new file mode 100644 index 0000000..60b50fb --- /dev/null +++ b/nikola/plugins/task/robots.plugin @@ -0,0 +1,10 @@ +[Core] +Name = robots +Module = robots + +[Documentation] +Author = Daniel Aleksandersen +Version = 0.1 +Website = http://getnikola.com +Description = Generate /robots.txt exclusion file and promote sitemap. + diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py new file mode 100644 index 0000000..9944c0d --- /dev/null +++ b/nikola/plugins/task/robots.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2014 Roberto Alsina and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function, absolute_import, unicode_literals +import codecs +import os +try: + from urlparse import urljoin, urlparse +except ImportError: + from urllib.parse import urljoin, urlparse # NOQA + +from nikola.plugin_categories import LateTask +from nikola import utils + + +class RobotsFile(LateTask): + """Generate a robots.txt.""" + + name = "robots_file" + + def gen_tasks(self): + """Generate a robots.txt.""" + kw = { + "base_url": self.site.config["BASE_URL"], + "site_url": self.site.config["SITE_URL"], + "output_folder": self.site.config["OUTPUT_FOLDER"], + "files_folders": self.site.config['FILES_FOLDERS'], + "robots_exclusions": self.site.config["ROBOTS_EXCLUSIONS"] + } + + if kw["site_url"] != urljoin(kw["site_url"], "/"): + utils.LOGGER.warn('robots.txt not ending up in server root, will be useless') + + sitemapindex_url = urljoin(kw["base_url"], "sitemapindex.xml") + robots_path = os.path.join(kw['output_folder'], "robots.txt") + + def write_robots(): + with codecs.open(robots_path, 'wb+', 'utf8') as outf: + outf.write("Sitemap: {0}\n\n".format(sitemapindex_url)) + if kw["robots_exclusions"]: + outf.write("User-Agent: *\n") + for loc in kw["robots_exclusions"]: + outf.write("Disallow: {0}\n".format(loc)) + + yield self.group_task() + + if not utils.get_asset_path("robots.txt", [], files_folders=kw["files_folders"]): + yield { + "basename": self.name, + "name": robots_path, + "targets": [robots_path], + "actions": [(write_robots)], + "uptodate": [utils.config_changed(kw)], + "clean": True, + "task_dep": ["sitemap"] + } + elif kw["robots_exclusions"]: + utils.LOGGER.warn('Did not generate robots.txt as one already exists in FILES_FOLDERS. ROBOTS_EXCLUSIONS will not have any affect on the copied fie.') + else: + utils.LOGGER.debug('Did not generate robots.txt as one already exists in FILES_FOLDERS.') diff --git a/nikola/plugins/task/rss.py b/nikola/plugins/task/rss.py index 9e4204c..b16ed48 100644 --- a/nikola/plugins/task/rss.py +++ b/nikola/plugins/task/rss.py @@ -54,8 +54,11 @@ class GenerateRSS(Task): "blog_description": self.site.config["BLOG_DESCRIPTION"], "output_folder": self.site.config["OUTPUT_FOLDER"], "rss_teasers": self.site.config["RSS_TEASERS"], - "hide_untranslated_posts": self.site.config['HIDE_UNTRANSLATED_POSTS'], + "rss_plain": self.site.config["RSS_PLAIN"], + "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], "feed_length": self.site.config['FEED_LENGTH'], + "tzinfo": self.site.tzinfo, + "rss_read_more_link": self.site.config["RSS_READ_MORE_LINK"], } self.site.scan_posts() # Check for any changes in the state of use_in_feeds for any post. @@ -68,24 +71,25 @@ class GenerateRSS(Task): output_name = os.path.join(kw['output_folder'], self.site.path("rss", None, lang)) deps = [] - if kw["hide_untranslated_posts"]: - posts = [x for x in self.site.timeline if x.use_in_feeds - and x.is_translation_available(lang)][:10] + if kw["show_untranslated_posts"]: + posts = self.site.posts[:10] else: - posts = [x for x in self.site.timeline if x.use_in_feeds][:10] + posts = [x for x in self.site.posts if x.is_translation_available(lang)][:10] for post in posts: deps += post.deps(lang) feed_url = urljoin(self.site.config['BASE_URL'], self.site.link("rss", None, lang).lstrip('/')) + yield { 'basename': 'generate_rss', 'name': os.path.normpath(output_name), 'file_dep': deps, 'targets': [output_name], 'actions': [(utils.generic_rss_renderer, - (lang, kw["blog_title"], kw["site_url"], - kw["blog_description"], posts, output_name, - kw["rss_teasers"], kw['feed_length'], feed_url))], + (lang, kw["blog_title"](lang), kw["site_url"], + kw["blog_description"](lang), posts, output_name, + kw["rss_teasers"], kw["rss_plain"], kw['feed_length'], feed_url))], + 'task_dep': ['render_posts'], 'clean': True, 'uptodate': [utils.config_changed(kw)], diff --git a/nikola/plugins/task/sitemap/__init__.py b/nikola/plugins/task/sitemap/__init__.py index 147bd50..beac6cb 100644 --- a/nikola/plugins/task/sitemap/__init__.py +++ b/nikola/plugins/task/sitemap/__init__.py @@ -30,14 +30,16 @@ import datetime import os try: from urlparse import urljoin, urlparse + import robotparser as robotparser except ImportError: from urllib.parse import urljoin, urlparse # NOQA + import urllib.robotparser as robotparser # NOQA from nikola.plugin_categories import LateTask from nikola.utils import config_changed -header = """<?xml version="1.0" encoding="UTF-8"?> +urlset_header = """<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" @@ -45,13 +47,29 @@ header = """<?xml version="1.0" encoding="UTF-8"?> http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"> """ -url_format = """ <url> +loc_format = """ <url> <loc>{0}</loc> <lastmod>{1}</lastmod> </url> """ -get_lastmod = lambda p: datetime.datetime.fromtimestamp(os.stat(p).st_mtime).isoformat().split('T')[0] +urlset_footer = "</urlset>" + +sitemapindex_header = """<?xml version="1.0" encoding="UTF-8"?> +<sitemapindex + xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 + http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"> +""" + +sitemap_format = """ <sitemap> + <loc>{0}</loc> + <lastmod>{1}</lastmod> + </sitemap> +""" + +sitemapindex_footer = "</sitemapindex>" def get_base_path(base): @@ -80,12 +98,12 @@ def get_base_path(base): class Sitemap(LateTask): - """Generate google sitemap.""" + """Generate a sitemap.""" name = "sitemap" def gen_tasks(self): - """Generate Google sitemap.""" + """Generate a sitemap.""" kw = { "base_url": self.site.config["BASE_URL"], "site_url": self.site.config["SITE_URL"], @@ -93,28 +111,32 @@ class Sitemap(LateTask): "strip_indexes": self.site.config["STRIP_INDEXES"], "index_file": self.site.config["INDEX_FILE"], "sitemap_include_fileless_dirs": self.site.config["SITEMAP_INCLUDE_FILELESS_DIRS"], - "mapped_extensions": self.site.config.get('MAPPED_EXTENSIONS', ['.html', '.htm', '.xml']) + "mapped_extensions": self.site.config.get('MAPPED_EXTENSIONS', ['.html', '.htm', '.xml', '.rss']), + "robots_exclusions": self.site.config["ROBOTS_EXCLUSIONS"] } - output_path = kw['output_folder'] - sitemap_path = os.path.join(output_path, "sitemap.xml") - base_path = get_base_path(kw['base_url']) - locs = {} output = kw['output_folder'] base_url = kw['base_url'] mapped_exts = kw['mapped_extensions'] + output_path = kw['output_folder'] + sitemapindex_path = os.path.join(output_path, "sitemapindex.xml") + sitemap_path = os.path.join(output_path, "sitemap.xml") + base_path = get_base_path(kw['base_url']) + sitemapindex = {} + urlset = {} + def scan_locs(): - for root, dirs, files in os.walk(output): + for root, dirs, files in os.walk(output, followlinks=True): if not dirs and not files and not kw['sitemap_include_fileless_dirs']: continue # Totally empty, not on sitemap path = os.path.relpath(root, output) # ignore the current directory. path = (path.replace(os.sep, '/') + '/').replace('./', '') - lastmod = get_lastmod(root) + lastmod = self.get_lastmod(root) loc = urljoin(base_url, base_path + path) if kw['index_file'] in files and kw['strip_indexes']: # ignore folders when not stripping urls - locs[loc] = url_format.format(loc, lastmod) + urlset[loc] = loc_format.format(loc, lastmod) for fname in files: if kw['strip_indexes'] and fname == kw['index_file']: continue # We already mapped the folder @@ -124,38 +146,68 @@ class Sitemap(LateTask): if path.endswith(kw['index_file']) and kw['strip_indexes']: # ignore index files when stripping urls continue + if not robot_fetch(path): + continue if path.endswith('.html') or path.endswith('.htm'): - if not u'<!doctype html' in codecs.open(real_path, 'r', 'utf8').read(1024).lower(): - # ignores "html" files without doctype - # alexa-verify, google-site-verification, etc. + try: + if u'<!doctype html' not in codecs.open(real_path, 'r', 'utf8').read(1024).lower(): + # ignores "html" files without doctype + # alexa-verify, google-site-verification, etc. + continue + except UnicodeDecodeError: + # ignore ancient files + # most non-utf8 files are worthless anyways continue - if path.endswith('.xml'): - if not u'<rss' in codecs.open(real_path, 'r', 'utf8').read(512): - # ignores all XML files except those presumed to be RSS + """ put RSS in sitemapindex[] instead of in urlset[], sitemap_path is included after it is generated """ + if path.endswith('.xml') or path.endswith('.rss'): + if u'<rss' in codecs.open(real_path, 'r', 'utf8').read(512) or u'<urlset'and path != sitemap_path: + path = path.replace(os.sep, '/') + lastmod = self.get_lastmod(real_path) + loc = urljoin(base_url, base_path + path) + sitemapindex[loc] = sitemap_format.format(loc, lastmod) continue + else: + continue # ignores all XML files except those presumed to be RSS post = self.site.post_per_file.get(path) - if post and (post.is_draft or post.is_retired or post.publish_later): + if post and (post.is_draft or post.is_private or post.publish_later): continue path = path.replace(os.sep, '/') - lastmod = get_lastmod(real_path) + lastmod = self.get_lastmod(real_path) loc = urljoin(base_url, base_path + path) - locs[loc] = url_format.format(loc, lastmod) + urlset[loc] = loc_format.format(loc, lastmod) + + def robot_fetch(path): + for rule in kw["robots_exclusions"]: + robot = robotparser.RobotFileParser() + robot.parse(["User-Agent: *", "Disallow: {0}".format(rule)]) + if not robot.can_fetch("*", '/' + path): + return False # not robot food + return True def write_sitemap(): # Have to rescan, because files may have been added between # task dep scanning and task execution with codecs.open(sitemap_path, 'wb+', 'utf8') as outf: - outf.write(header) - for k in sorted(locs.keys()): - outf.write(locs[k]) - outf.write("</urlset>") + outf.write(urlset_header) + for k in sorted(urlset.keys()): + outf.write(urlset[k]) + outf.write(urlset_footer) + sitemap_url = urljoin(base_url, base_path + "sitemap.xml") + sitemapindex[sitemap_url] = sitemap_format.format(sitemap_url, self.get_lastmod(sitemap_path)) + + def write_sitemapindex(): + with codecs.open(sitemapindex_path, 'wb+', 'utf8') as outf: + outf.write(sitemapindex_header) + for k in sorted(sitemapindex.keys()): + outf.write(sitemapindex[k]) + outf.write(sitemapindex_footer) # Yield a task to calculate the dependencies of the sitemap # Other tasks can depend on this output, instead of having # to scan locations. def scan_locs_task(): scan_locs() - return {'locations': list(locs.keys())} + return {'locations': list(urlset.keys()) + list(sitemapindex.keys())} yield { "basename": "_scan_locs", @@ -164,7 +216,7 @@ class Sitemap(LateTask): } yield self.group_task() - task = { + yield { "basename": "sitemap", "name": sitemap_path, "targets": [sitemap_path], @@ -174,7 +226,21 @@ class Sitemap(LateTask): "task_dep": ["render_site"], "calc_dep": ["_scan_locs:sitemap"], } - yield task + yield { + "basename": "sitemap", + "name": sitemapindex_path, + "targets": [sitemapindex_path], + "actions": [(write_sitemapindex,)], + "uptodate": [config_changed(kw)], + "clean": True, + "file_dep": [sitemap_path] + } + + def get_lastmod(self, p): + if self.site.invariant: + return '2014-01-01' + else: + return datetime.datetime.fromtimestamp(os.stat(p).st_mtime).isoformat().split('T')[0] if __name__ == '__main__': import doctest diff --git a/nikola/plugins/task/tags.py b/nikola/plugins/task/tags.py index f6b8234..f7f3579 100644 --- a/nikola/plugins/task/tags.py +++ b/nikola/plugins/task/tags.py @@ -61,12 +61,14 @@ class RenderTags(Task): "output_folder": self.site.config['OUTPUT_FOLDER'], "filters": self.site.config['FILTERS'], "tag_pages_are_indexes": self.site.config['TAG_PAGES_ARE_INDEXES'], - "index_display_post_count": - self.site.config['INDEX_DISPLAY_POST_COUNT'], + "index_display_post_count": self.site.config['INDEX_DISPLAY_POST_COUNT'], "index_teasers": self.site.config['INDEX_TEASERS'], + "generate_rss": self.site.config['GENERATE_RSS'], "rss_teasers": self.site.config["RSS_TEASERS"], - "hide_untranslated_posts": self.site.config['HIDE_UNTRANSLATED_POSTS'], + "rss_plain": self.site.config["RSS_PLAIN"], + "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], "feed_length": self.site.config['FEED_LENGTH'], + "tzinfo": self.site.tzinfo, } self.site.scan_posts() @@ -81,16 +83,15 @@ class RenderTags(Task): cat_list = list(self.site.posts_per_category.items()) def render_lists(tag, posts, is_category=True): - post_list = [self.site.global_data[post] for post in posts] - post_list.sort(key=lambda a: a.date) + post_list = sorted(posts, key=lambda a: a.date) post_list.reverse() for lang in kw["translations"]: - if kw["hide_untranslated_posts"]: - filtered_posts = [x for x in post_list if x.is_translation_available(lang)] - else: + if kw["show_untranslated_posts"]: filtered_posts = post_list - rss_post_list = [p.source_path for p in filtered_posts] - yield self.tag_rss(tag, lang, rss_post_list, kw, is_category) + else: + filtered_posts = [x for x in post_list if x.is_translation_available(lang)] + if kw["generate_rss"]: + yield self.tag_rss(tag, lang, filtered_posts, kw, is_category) # Render HTML if kw['tag_pages_are_indexes']: yield self.tag_page_as_index(tag, lang, filtered_posts, kw, is_category) @@ -205,12 +206,13 @@ class RenderTags(Task): num_pages = len(lists) for i, post_list in enumerate(lists): context = {} - # On a tag page, the feeds include the tag's feeds - rss_link = ("""<link rel="alternate" type="application/rss+xml" """ - """type="application/rss+xml" title="RSS for tag """ - """{0} ({1})" href="{2}">""".format( - tag, lang, self.site.link(kind + "_rss", tag, lang))) - context['rss_link'] = rss_link + if kw["generate_rss"]: + # On a tag page, the feeds include the tag's feeds + rss_link = ("""<link rel="alternate" type="application/rss+xml" """ + """type="application/rss+xml" title="RSS for tag """ + """{0} ({1})" href="{2}">""".format( + tag, lang, self.site.link(kind + "_rss", tag, lang))) + context['rss_link'] = rss_link output_name = os.path.join(kw['output_folder'], page_name(tag, i, lang)) context["title"] = kw["messages"][lang][ @@ -274,15 +276,13 @@ class RenderTags(Task): def tag_rss(self, tag, lang, posts, kw, is_category): """RSS for a single tag / language""" kind = "category" if is_category else "tag" - #Render RSS + # Render RSS output_name = os.path.normpath( os.path.join(kw['output_folder'], self.site.path(kind + "_rss", tag, lang))) feed_url = urljoin(self.site.config['BASE_URL'], self.site.link(kind + "_rss", tag, lang).lstrip('/')) deps = [] - post_list = [self.site.global_data[post] for post in posts if - self.site.global_data[post].use_in_feeds] - post_list.sort(key=lambda a: a.date) + post_list = sorted(posts, key=lambda a: a.date) post_list.reverse() for post in post_list: deps += post.deps(lang) @@ -292,9 +292,10 @@ class RenderTags(Task): 'file_dep': deps, 'targets': [output_name], 'actions': [(utils.generic_rss_renderer, - (lang, "{0} ({1})".format(kw["blog_title"], tag), + (lang, "{0} ({1})".format(kw["blog_title"](lang), tag), kw["site_url"], None, post_list, - output_name, kw["rss_teasers"], kw['feed_length'], feed_url))], + output_name, kw["rss_teasers"], kw["rss_plain"], kw['feed_length'], + feed_url))], 'clean': True, 'uptodate': [utils.config_changed(kw)], 'task_dep': ['render_posts'], |
