Imported Upstream version 7.6.0upstream/7.6.0

1 files changed, 404 insertions, 73 deletions

diff --git a/nikola/utils.py b/nikola/utils.py
index 87826ff..3708775 100644
--- a/nikola/utils.py
+++ b/nikola/utils.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright © 2012-2014 Roberto Alsina and others.
+# Copyright © 2012-2015 Roberto Alsina and others.
 
 # Permission is hereby granted, free of charge, to any
 # person obtaining a copy of this software and associated
@@ -27,32 +27,66 @@
 """Utility functions."""
 
 from __future__ import print_function, unicode_literals, absolute_import
-from collections import defaultdict, Callable
 import calendar
 import datetime
 import dateutil.tz
 import hashlib
+import io
 import locale
 import logging
+import natsort
 import os
 import re
 import json
 import shutil
 import subprocess
 import sys
-from zipfile import ZipFile as zipf
-try:
-    from imp import reload
-except ImportError:
-    pass
-
 import dateutil.parser
 import dateutil.tz
 import logbook
+import warnings
+import PyRSS2Gen as rss
+from collections import defaultdict, Callable
 from logbook.more import ExceptionHandler, ColorizedStderrHandler
+from pygments.formatters import HtmlFormatter
+from zipfile import ZipFile as zipf
+from doit import tools
+from unidecode import unidecode
+from pkg_resources import resource_filename
+from doit.cmdparse import CmdParse
 
 from nikola import DEBUG
 
+__all__ = ['CustomEncoder', 'get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree',
+           'copy_file', 'slugify', 'unslugify', 'to_datetime', 'apply_filters',
+           'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path',
+           '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary',
+           'TranslatableSetting', 'TemplateHookRegistry', 'LocaleBorg',
+           'sys_encode', 'sys_decode', 'makedirs', 'get_parent_theme_name',
+           'demote_headers', 'get_translation_candidate', 'write_metadata',
+           'ask', 'ask_yesno', 'options2docstring', 'os_path_split',
+           'get_displayed_page_number', 'adjust_name_for_index_path_list',
+           'adjust_name_for_index_path', 'adjust_name_for_index_link',
+           'NikolaPygmentsHTML', 'create_redirect', 'TreeNode',
+           'flatten_tree_structure', 'parse_escaped_hierarchical_category_name',
+           'join_hierarchical_category_path', 'indent']
+
+# Are you looking for 'generic_rss_renderer'?
+# It's defined in nikola.nikola.Nikola (the site object).
+
+if sys.version_info[0] == 3:
+    # Python 3
+    bytes_str = bytes
+    unicode_str = str
+    unichr = chr
+    raw_input = input
+    from imp import reload as _reload
+else:
+    bytes_str = str
+    unicode_str = unicode  # NOQA
+    _reload = reload  # NOQA
+    unichr = unichr
+
 
 class ApplicationWarning(Exception):
     pass
@@ -72,9 +106,9 @@ def get_logger(name, handlers):
     l = logbook.Logger(name)
     for h in handlers:
         if isinstance(h, list):
-            l.handlers = h
+            l.handlers += h
         else:
-            l.handlers = [h]
+            l.handlers.append(h)
     return l
 
 
@@ -97,9 +131,6 @@ else:
     logging.basicConfig(level=logging.INFO)
 
 
-import warnings
-
-
 def showwarning(message, category, filename, lineno, file=None, line=None):
     """Show a warning (from the warnings subsystem) to the user."""
     try:
@@ -156,39 +187,8 @@ def req_missing(names, purpose, python=True, optional=False):
 
     return msg
 
-if sys.version_info[0] == 3:
-    # Python 3
-    bytes_str = bytes
-    unicode_str = str
-    unichr = chr
-    raw_input = input
-    from imp import reload as _reload
-else:
-    bytes_str = str
-    unicode_str = unicode  # NOQA
-    _reload = reload  # NOQA
-    unichr = unichr
-
-from doit import tools
-from unidecode import unidecode
-from pkg_resources import resource_filename
-from nikola import filters as task_filters
-
-import PyRSS2Gen as rss
-
-__all__ = ['get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree',
-           'copy_file', 'slugify', 'unslugify', 'to_datetime', 'apply_filters',
-           'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path',
-           '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary',
-           'TranslatableSetting', 'TemplateHookRegistry', 'LocaleBorg',
-           'sys_encode', 'sys_decode', 'makedirs', 'get_parent_theme_name',
-           'demote_headers', 'get_translation_candidate', 'write_metadata',
-           'ask', 'ask_yesno']
-
-# Are you looking for 'generic_rss_renderer'?
-# It's defined in nikola.nikola.Nikola (the site object).
-
 
+from nikola import filters as task_filters  # NOQA
 ENCODING = sys.getfilesystemencoding() or sys.stdin.encoding
 
 
@@ -208,11 +208,20 @@ def sys_decode(thing):
 
 def makedirs(path):
     """Create a folder."""
-    if not path or os.path.isdir(path):
+    if not path:
         return
     if os.path.exists(path):
-        raise OSError('Path {0} already exists and is not a folder.')
-    os.makedirs(path)
+        if not os.path.isdir(path):
+            raise OSError('Path {0} already exists and is not a folder.'.format(path))
+        else:
+            return
+    try:
+        os.makedirs(path)
+        return
+    except Exception:
+        if os.path.isdir(path):
+            return
+        raise
 
 
 class Functionary(defaultdict):
@@ -368,8 +377,10 @@ class TranslatableSetting(object):
                 for a in f[0] + tuple(f[1].values()):
                     if isinstance(a, dict):
                         langkeys += list(a)
+
             # Now that we know all this, we go through all the languages we have.
             allvalues = set(keys + langkeys + list(self.values))
+            self.values['__orig__'] = self.values[self.default_lang]
             for l in allvalues:
                 if l in keys:
                     oargs, okwargs = formats[l]
@@ -383,19 +394,22 @@ class TranslatableSetting(object):
                     # We create temporary TranslatableSettings and replace the
                     # values with them.
                     if isinstance(a, dict):
-                        a = TranslatableSetting('NULL', a)
+                        a = TranslatableSetting('NULL', a, self.translations)
                         args.append(a(l))
                     else:
                         args.append(a)
 
                 for k, v in okwargs.items():
                     if isinstance(v, dict):
-                        v = TranslatableSetting('NULL', v)
+                        v = TranslatableSetting('NULL', v, self.translations)
                         kwargs.update({k: v(l)})
                     else:
                         kwargs.update({k: v})
 
-                self.values[l] = self.values[l].format(*args, **kwargs)
+                if l in self.values:
+                    self.values[l] = self.values[l].format(*args, **kwargs)
+                else:
+                    self.values[l] = self.values['__orig__'].format(*args, **kwargs)
                 self.values.default_factory = lambda: self.values[self.default_lang]
 
         return self
@@ -472,7 +486,7 @@ class TemplateHookRegistry(object):
         self._items.append((c, inp, wants_site_and_context, args, kwargs))
 
     def __hash__(self):
-        return config_changed({self.name: self._items})
+        return hash(config_changed({self.name: self._items})._calc_digest())
 
     def __str__(self):
         return '<TemplateHookRegistry: {0}>'.format(self._items)
@@ -490,6 +504,12 @@ class CustomEncoder(json.JSONEncoder):
 class config_changed(tools.config_changed):
     """ A copy of doit's but using pickle instead of serializing manually."""
 
+    def __init__(self, config, identifier=None):
+        super(config_changed, self).__init__(config)
+        self.identifier = '_config_changed'
+        if identifier is not None:
+            self.identifier += ':' + identifier
+
     def _calc_digest(self):
         if isinstance(self.config, str):
             return self.config
@@ -507,6 +527,16 @@ class config_changed(tools.config_changed):
                             '{0}, must be string or dict'.format(type(
                                 self.config)))
 
+    def configure_task(self, task):
+        task.value_savers.append(lambda: {self.identifier: self._calc_digest()})
+
+    def __call__(self, task, values):
+        """Return True if config values are unchanged."""
+        last_success = values.get(self.identifier)
+        if last_success is None:
+            return False
+        return (last_success == self._calc_digest())
+
     def __repr__(self):
         return "Change with config: {0}".format(json.dumps(self.config,
                                                            cls=CustomEncoder))
@@ -576,7 +606,7 @@ def load_messages(themes, translations, default_lang):
     and "younger" themes have priority.
     """
     messages = Functionary(dict, default_lang)
-    oldpath = sys.path[:]
+    oldpath = list(sys.path)
     for theme_name in themes[::-1]:
         msg_folder = os.path.join(get_theme_path(theme_name), 'messages')
         default_folder = os.path.join(get_theme_path('base'), 'messages')
@@ -587,7 +617,7 @@ def load_messages(themes, translations, default_lang):
             try:
                 translation = __import__('messages_' + lang)
                 # If we don't do the reload, the module is cached
-                reload(translation)
+                _reload(translation)
                 if sorted(translation.MESSAGES.keys()) !=\
                         sorted(english.MESSAGES.keys()) and \
                         lang not in warned:
@@ -877,7 +907,10 @@ def get_crumbs(path, is_file=False, index_folder=None):
         for i, crumb in enumerate(crumbs[::-1]):
             if folder[-1] == os.sep:
                 folder = folder[:-1]
-            index_post = index_folder.parse_index(folder)
+            # We don't care about the created Post() object except for its title;
+            # hence, the input_folder and output_folder given to
+            # index_folder.parse_index() don't matter
+            index_post = index_folder.parse_index(folder, '', '')
             folder = folder.replace(crumb, '')
             if index_post:
                 crumb = index_post.title() or crumb
@@ -1038,6 +1071,25 @@ class LocaleBorg(object):
         return s
 
 
+class ExtendedRSS2(rss.RSS2):
+    xsl_stylesheet_href = None
+
+    def publish(self, handler):
+        if self.xsl_stylesheet_href:
+            handler.processingInstruction("xml-stylesheet", 'type="text/xsl" href="{0}" media="all"'.format(self.xsl_stylesheet_href))
+        # old-style class in py2
+        rss.RSS2.publish(self, handler)
+
+    def publish_extensions(self, handler):
+        if self.self_url:
+            handler.startElement("atom:link", {
+                'href': self.self_url,
+                'rel': "self",
+                'type': "application/rss+xml"
+            })
+            handler.endElement("atom:link")
+
+
 class ExtendedItem(rss.RSSItem):
 
     def __init__(self, **kw):
@@ -1096,8 +1148,13 @@ def get_root_dir():
     """Find root directory of nikola installation by looking for conf.py"""
     root = os.getcwd()
 
+    if sys.version_info[0] == 2:
+        confname = b'conf.py'
+    else:
+        confname = 'conf.py'
+
     while True:
-        if os.path.exists(os.path.join(root, 'conf.py')):
+        if os.path.exists(os.path.join(root, confname)):
             return root
         else:
             basedir = os.path.split(root)[0]
@@ -1174,7 +1231,7 @@ def get_translation_candidate(config, path, lang):
 
 def write_metadata(data):
     """Write metadata."""
-    order = ('title', 'slug', 'date', 'tags', 'link', 'description', 'type')
+    order = ('title', 'slug', 'date', 'tags', 'category', 'link', 'description', 'type')
     f = '.. {0}: {1}'
     meta = []
     for k in order:
@@ -1184,8 +1241,8 @@ def write_metadata(data):
             pass
 
     # Leftover metadata (user-specified/non-default).
-    for k, v in data.items():
-        meta.append(f.format(k, v))
+    for k in natsort.natsorted(list(data.keys()), alg=natsort.ns.F | natsort.ns.IC):
+        meta.append(f.format(k, data[k]))
 
     meta.append('')
 
@@ -1198,7 +1255,10 @@ def ask(query, default=None):
         default_q = ' [{0}]'.format(default)
     else:
         default_q = ''
-    inp = raw_input("{query}{default_q}: ".format(query=query, default_q=default_q)).strip()
+    if sys.version_info[0] == 3:
+        inp = raw_input("{query}{default_q}: ".format(query=query, default_q=default_q)).strip()
+    else:
+        inp = raw_input("{query}{default_q}: ".format(query=query, default_q=default_q).encode('utf-8')).strip()
     if inp or default is None:
         return inp
     else:
@@ -1213,7 +1273,10 @@ def ask_yesno(query, default=None):
         default_q = ' [Y/n]'
     elif default is False:
         default_q = ' [y/N]'
-    inp = raw_input("{query}{default_q} ".format(query=query, default_q=default_q)).strip()
+    if sys.version_info[0] == 3:
+        inp = raw_input("{query}{default_q} ".format(query=query, default_q=default_q)).strip()
+    else:
+        inp = raw_input("{query}{default_q} ".format(query=query, default_q=default_q).encode('utf-8')).strip()
     if inp:
         return inp.lower().startswith('y')
     elif default is not None:
@@ -1223,10 +1286,6 @@ def ask_yesno(query, default=None):
         return ask_yesno(query, default)
 
 
-from nikola.plugin_categories import Command
-from doit.cmdparse import CmdParse
-
-
 class CommandWrapper(object):
     """Converts commands into functions."""
 
@@ -1253,32 +1312,58 @@ class Commands(object):
     >>> commands.check(list=True)                # doctest: +SKIP
     """
 
-    def __init__(self, main):
+    def __init__(self, main, config, doitargs):
         """Takes a main instance, works as wrapper for commands."""
         self._cmdnames = []
-        for k, v in main.get_commands().items():
-            self._cmdnames.append(k)
+        self._main = main
+        self._config = config
+        self._doitargs = doitargs
+        try:
+            cmdict = self._doitargs['cmds'].to_dict()
+        except AttributeError:  # not a doit PluginDict
+            cmdict = self._doitargs['cmds']
+        for k, v in cmdict.items():
+            # cleanup: run is doit-only, init is useless in an existing site
             if k in ['run', 'init']:
                 continue
             if sys.version_info[0] == 2:
                 k2 = bytes(k)
             else:
                 k2 = k
+
+            self._cmdnames.append(k)
+
+            try:
+                # nikola command: already instantiated (singleton)
+                opt = v.get_options()
+            except TypeError:
+                # doit command: needs some help
+                opt = v(config=self._config, **self._doitargs).get_options()
             nc = type(
                 k2,
                 (CommandWrapper,),
                 {
-                    '__doc__': options2docstring(k, main.sub_cmds[k].options)
+                    '__doc__': options2docstring(k, opt)
                 })
             setattr(self, k, nc(k, self))
-        self.main = main
 
     def _run(self, cmd_args):
-        self.main.run(cmd_args)
+        self._main.run(cmd_args)
 
     def _run_with_kw(self, cmd, *a, **kw):
-        cmd = self.main.sub_cmds[cmd]
-        options, _ = CmdParse(cmd.options).parse([])
+        # cyclic import hack
+        from nikola.plugin_categories import Command
+        try:
+            cmd = self._doitargs['cmds'].get_plugin(cmd)
+        except AttributeError:  # not a doit PluginDict
+            cmd = self._doitargs['cmds'][cmd]
+        try:
+            opt = cmd.get_options()
+        except TypeError:
+            cmd = cmd(config=self._config, **self._doitargs)
+            opt = cmd.get_options()
+
+        options, _ = CmdParse(opt).parse([])
         options.update(kw)
         if isinstance(cmd, Command):
             cmd.execute(options=options, args=a)
@@ -1305,3 +1390,249 @@ def options2docstring(name, options):
     for opt in options:
         result.append('{0} type {1} default {2}'.format(opt.name, opt.type.__name__, opt.default))
     return '\n'.join(result)
+
+
+class NikolaPygmentsHTML(HtmlFormatter):
+    """A Nikola-specific modification of Pygments’ HtmlFormatter."""
+    def __init__(self, anchor_ref, classes=None, linenos='table', linenostart=1):
+        if classes is None:
+            classes = ['code', 'literal-block']
+        self.nclasses = classes
+        super(NikolaPygmentsHTML, self).__init__(
+            cssclass='code', linenos=linenos, linenostart=linenostart, nowrap=False,
+            lineanchors=slugify(anchor_ref, force=True), anchorlinenos=True)
+
+    def wrap(self, source, outfile):
+        """
+        Wrap the ``source``, which is a generator yielding
+        individual lines, in custom generators.
+        """
+
+        style = []
+        if self.prestyles:
+            style.append(self.prestyles)
+        if self.noclasses:
+            style.append('line-height: 125%')
+        style = '; '.join(style)
+        classes = ' '.join(self.nclasses)
+
+        yield 0, ('<pre class="{0}"'.format(classes) + (style and ' style="{0}"'.format(style)) + '>')
+        for tup in source:
+            yield tup
+        yield 0, '</pre>'
+
+
+def get_displayed_page_number(i, num_pages, site):
+    if not i:
+        i = 0
+    if site.config["INDEXES_STATIC"]:
+        return i if i > 0 else num_pages
+    else:
+        return i + 1 if site.config["INDEXES_PAGES_MAIN"] else i
+
+
+def adjust_name_for_index_path_list(path_list, i, displayed_i, lang, site, force_addition=False, extension=None):
+    index_file = site.config["INDEX_FILE"]
+    if i or force_addition:
+        path_list = list(path_list)
+        if force_addition and not i:
+            i = 0
+        if not extension:
+            _, extension = os.path.splitext(index_file)
+        if len(path_list) > 0 and path_list[-1] == '':
+            path_list[-1] = index_file
+        elif len(path_list) == 0 or not path_list[-1].endswith(extension):
+            path_list.append(index_file)
+        if site.config["PRETTY_URLS"] and site.config["INDEXES_PRETTY_PAGE_URL"](lang) and path_list[-1] == index_file:
+            path_schema = site.config["INDEXES_PRETTY_PAGE_URL"](lang)
+            if isinstance(path_schema, (bytes_str, unicode_str)):
+                path_schema = [path_schema]
+        else:
+            path_schema = None
+        if path_schema is not None:
+            del path_list[-1]
+            for entry in path_schema:
+                path_list.append(entry.format(number=displayed_i, old_number=i, index_file=index_file))
+        else:
+            path_list[-1] = '{0}-{1}{2}'.format(os.path.splitext(path_list[-1])[0], i, extension)
+    return path_list
+
+
+def os_path_split(path):
+    result = []
+    while True:
+        previous_path = path
+        path, tail = os.path.split(path)
+        if path == previous_path and tail == '':
+            result.insert(0, path)
+            break
+        result.insert(0, tail)
+        if len(path) == 0:
+            break
+    return result
+
+
+def adjust_name_for_index_path(name, i, displayed_i, lang, site, force_addition=False, extension=None):
+    return os.path.join(*adjust_name_for_index_path_list(os_path_split(name), i, displayed_i, lang, site, force_addition, extension))
+
+
+def adjust_name_for_index_link(name, i, displayed_i, lang, site, force_addition=False, extension=None):
+    link = adjust_name_for_index_path_list(name.split('/'), i, displayed_i, lang, site, force_addition, extension)
+    if not extension == ".atom":
+        if len(link) > 0 and link[-1] == site.config["INDEX_FILE"] and site.config["STRIP_INDEXES"]:
+            link[-1] = ''
+    return '/'.join(link)
+
+
+def create_redirect(src, dst):
+    makedirs(os.path.dirname(src))
+    with io.open(src, "w+", encoding="utf8") as fd:
+        fd.write('<!DOCTYPE html>\n<head>\n<meta charset="utf-8">\n'
+                 '<title>Redirecting...</title>\n<meta name="robots" '
+                 'content="noindex">\n<meta http-equiv="refresh" content="0; '
+                 'url={0}">\n</head>\n<body>\n<p>Page moved '
+                 '<a href="{0}">here</a>.</p>\n</body>'.format(dst))
+
+
+class TreeNode(object):
+    indent_levels = None  # use for formatting comments as tree
+    indent_change_before = 0  # use for formatting comments as tree
+    indent_change_after = 0  # use for formatting comments as tree
+
+    # The indent levels and changes allow to render a tree structure
+    # without keeping track of all that information during rendering.
+    #
+    # The indent_change_before is the different between the current
+    # comment's level and the previous comment's level; if the number
+    # is positive, the current level is indented further in, and if it
+    # is negative, it is indented further out. Positive values can be
+    # used to open HTML tags for each opened level.
+    #
+    # The indent_change_after is the difference between the next
+    # comment's level and the current comment's level. Negative values
+    # can be used to close HTML tags for each closed level.
+    #
+    # The indent_levels list contains one entry (index, count) per
+    # level, informing about the index of the current comment on that
+    # level and the count of comments on that level (before a comment
+    # of a higher level comes). This information can be used to render
+    # tree indicators, for example to generate a tree such as:
+    #
+    # +--- [(0,3)]
+    # +-+- [(1,3)]
+    # | +--- [(1,3), (0,2)]
+    # | +-+- [(1,3), (1,2)]
+    # |   +--- [(1,3), (1,2), (0, 1)]
+    # +-+- [(2,3)]
+    #   +- [(2,3), (0,1)]
+    #
+    # (The lists used as labels represent the content of the
+    # indent_levels property for that node.)
+
+    def __init__(self, name, parent=None):
+        self.name = name
+        self.parent = parent
+        self.children = []
+
+    def get_path(self):
+        path = []
+        curr = self
+        while curr is not None:
+            path.append(curr)
+            curr = curr.parent
+        return reversed(path)
+
+    def get_children(self):
+        return self.children
+
+
+def flatten_tree_structure(root_list):
+    elements = []
+
+    def generate(input_list, indent_levels_so_far):
+        for index, element in enumerate(input_list):
+            # add to destination
+            elements.append(element)
+            # compute and set indent levels
+            indent_levels = indent_levels_so_far + [(index, len(input_list))]
+            element.indent_levels = indent_levels
+            # add children
+            children = element.get_children()
+            element.children_count = len(children)
+            generate(children, indent_levels)
+
+    generate(root_list, [])
+    # Add indent change counters
+    level = 0
+    last_element = None
+    for element in elements:
+        new_level = len(element.indent_levels)
+        # Compute level change before this element
+        change = new_level - level
+        if last_element is not None:
+            last_element.indent_change_after = change
+        element.indent_change_before = change
+        # Update variables
+        level = new_level
+        last_element = element
+    # Set level change after last element
+    if last_element is not None:
+        last_element.indent_change_after = -level
+    return elements
+
+
+def parse_escaped_hierarchical_category_name(category_name):
+    result = []
+    current = None
+    index = 0
+    next_backslash = category_name.find('\\', index)
+    next_slash = category_name.find('/', index)
+    while index < len(category_name):
+        if next_backslash == -1 and next_slash == -1:
+            current = (current if current else "") + category_name[index:]
+            index = len(category_name)
+        elif next_slash >= 0 and (next_backslash == -1 or next_backslash > next_slash):
+            result.append((current if current else "") + category_name[index:next_slash])
+            current = ''
+            index = next_slash + 1
+            next_slash = category_name.find('/', index)
+        else:
+            if len(category_name) == next_backslash + 1:
+                raise Exception("Unexpected '\\' in '{0}' at last position!".format(category_name))
+            esc_ch = category_name[next_backslash + 1]
+            if esc_ch not in {'/', '\\'}:
+                raise Exception("Unknown escape sequence '\\{0}' in '{1}'!".format(esc_ch, category_name))
+            current = (current if current else "") + category_name[index:next_backslash] + esc_ch
+            index = next_backslash + 2
+            next_backslash = category_name.find('\\', index)
+            if esc_ch == '/':
+                next_slash = category_name.find('/', index)
+    if current is not None:
+        result.append(current)
+    return result
+
+
+def join_hierarchical_category_path(category_path):
+    def escape(s):
+        return s.replace('\\', '\\\\').replace('/', '\\/')
+
+    return '/'.join([escape(p) for p in category_path])
+
+
+# Stolen from textwrap in Python 3.4.3.
+def indent(text, prefix, predicate=None):
+    """Adds 'prefix' to the beginning of selected lines in 'text'.
+
+    If 'predicate' is provided, 'prefix' will only be added to the lines
+    where 'predicate(line)' is True. If 'predicate' is not provided,
+    it will default to adding 'prefix' to all non-empty lines that do not
+    consist solely of whitespace characters.
+    """
+    if predicate is None:
+        def predicate(line):
+            return line.strip()
+
+    def prefixed_lines():
+        for line in text.splitlines(True):
+            yield (prefix + line if predicate(line) else line)
+    return ''.join(prefixed_lines())