Merge tag 'upstream/7.8.1'

Upstream version 7.8.1

# gpg: Firmado el mar 15 nov 2016 14:18:48 ART
# gpg:                usando RSA clave A6C7B88B9583046A11C5403E0B00FB6CEBE2D002
# gpg: Firma correcta de "Ulises Vitulli <dererk@debian.org>" [absoluta]
# gpg:                 alias "Dererk <dererk@torproject.org>" [absoluta]
# gpg:                 alias "Ulises Vitulli <uvitulli@fi.uba.ar>" [absoluta]
# gpg:                 alias "Ulises Vitulli <dererk@satellogic.com>" [absoluta]

1 files changed, 343 insertions, 93 deletions

diff --git a/nikola/utils.py b/nikola/utils.py
index 3a268ff..068cb3a 100644
--- a/nikola/utils.py
+++ b/nikola/utils.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright © 2012-2015 Roberto Alsina and others.
+# Copyright © 2012-2016 Roberto Alsina and others.
 
 # Permission is hereby granted, free of charge, to any
 # person obtaining a copy of this software and associated
@@ -39,26 +39,49 @@ import os
 import re
 import json
 import shutil
+import socket
 import subprocess
 import sys
 import dateutil.parser
 import dateutil.tz
 import logbook
+try:
+    from urllib import quote as urlquote
+    from urllib import unquote as urlunquote
+    from urlparse import urlparse, urlunparse
+except ImportError:
+    from urllib.parse import quote as urlquote  # NOQA
+    from urllib.parse import unquote as urlunquote  # NOQA
+    from urllib.parse import urlparse, urlunparse  # NOQA
 import warnings
 import PyRSS2Gen as rss
-from collections import defaultdict, Callable
+try:
+    import pytoml as toml
+except ImportError:
+    toml = None
+try:
+    import yaml
+except ImportError:
+    yaml = None
+try:
+    import husl
+except ImportError:
+    husl = None
+
+from collections import defaultdict, Callable, OrderedDict
 from logbook.compat import redirect_logging
 from logbook.more import ExceptionHandler, ColorizedStderrHandler
 from pygments.formatters import HtmlFormatter
 from zipfile import ZipFile as zipf
 from doit import tools
 from unidecode import unidecode
+from unicodedata import normalize as unicodenormalize
 from pkg_resources import resource_filename
 from doit.cmdparse import CmdParse
 
 from nikola import DEBUG
 
-__all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree',
+__all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_path_real', 'get_theme_chain', 'load_messages', 'copy_tree',
            'copy_file', 'slugify', 'unslugify', 'to_datetime', 'apply_filters',
            'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path',
            '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary',
@@ -70,7 +93,8 @@ __all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_chain', 'load_messages'
            'adjust_name_for_index_path', 'adjust_name_for_index_link',
            'NikolaPygmentsHTML', 'create_redirect', 'TreeNode',
            'flatten_tree_structure', 'parse_escaped_hierarchical_category_name',
-           'join_hierarchical_category_path', 'indent')
+           'join_hierarchical_category_path', 'clean_before_deployment', 'indent',
+           'load_data')
 
 # Are you looking for 'generic_rss_renderer'?
 # It's defined in nikola.nikola.Nikola (the site object).
@@ -94,7 +118,6 @@ class ApplicationWarning(Exception):
 
 
 class ColorfulStderrHandler(ColorizedStderrHandler):
-
     """Stream handler with colors."""
 
     _colorful = False
@@ -228,7 +251,6 @@ def makedirs(path):
 
 
 class Functionary(defaultdict):
-
     """Class that looks like a function, but is a defaultdict."""
 
     def __init__(self, default, default_lang):
@@ -244,7 +266,6 @@ class Functionary(defaultdict):
 
 
 class TranslatableSetting(object):
-
     """A setting that can be translated.
 
     You can access it via: SETTING(lang).  You can omit lang, in which
@@ -294,7 +315,7 @@ class TranslatableSetting(object):
         self.overriden_default = False
         self.values = defaultdict()
 
-        if isinstance(inp, dict):
+        if isinstance(inp, dict) and inp:
             self.translated = True
             self.values.update(inp)
             if self.default_lang not in self.values.keys():
@@ -423,15 +444,20 @@ class TranslatableSetting(object):
 
     def __eq__(self, other):
         """Test whether two TranslatableSettings are equal."""
-        return self.values == other.values
+        try:
+            return self.values == other.values
+        except AttributeError:
+            return self(self.default_lang) == other
 
     def __ne__(self, other):
         """Test whether two TranslatableSettings are inequal."""
-        return self.values != other.values
+        try:
+            return self.values != other.values
+        except AttributeError:
+            return self(self.default_lang) != other
 
 
 class TemplateHookRegistry(object):
-
     r"""A registry for template hooks.
 
     Usage:
@@ -497,7 +523,6 @@ class TemplateHookRegistry(object):
 
 
 class CustomEncoder(json.JSONEncoder):
-
     """Custom JSON encoder."""
 
     def default(self, obj):
@@ -513,7 +538,6 @@ class CustomEncoder(json.JSONEncoder):
 
 
 class config_changed(tools.config_changed):
-
     """A copy of doit's config_changed, using pickle instead of serializing manually."""
 
     def __init__(self, config, identifier=None):
@@ -535,6 +559,8 @@ class config_changed(tools.config_changed):
                 byte_data = data
             digest = hashlib.md5(byte_data).hexdigest()
             # LOGGER.debug('{{"{0}": {1}}}'.format(digest, byte_data))
+            # Humanized format:
+            # LOGGER.debug('[Digest {0} for {2}]\n{1}\n[Digest {0} for {2}]'.format(digest, byte_data, self.identifier))
             return digest
         else:
             raise Exception('Invalid type of config_changed parameter -- got '
@@ -559,24 +585,30 @@ class config_changed(tools.config_changed):
                                                            sort_keys=True))
 
 
-def get_theme_path(theme, _themes_dir='themes'):
+def get_theme_path_real(theme, themes_dirs):
     """Return the path where the given theme's files are located.
 
     Looks in ./themes and in the place where themes go when installed.
     """
-    dir_name = os.path.join(_themes_dir, theme)
-    if os.path.isdir(dir_name):
-        return dir_name
+    for themes_dir in themes_dirs:
+        dir_name = os.path.join(themes_dir, theme)
+        if os.path.isdir(dir_name):
+            return dir_name
     dir_name = resource_filename('nikola', os.path.join('data', 'themes', theme))
     if os.path.isdir(dir_name):
         return dir_name
     raise Exception("Can't find theme '{0}'".format(theme))
 
 
-def get_template_engine(themes, _themes_dir='themes'):
+def get_theme_path(theme):
+    """Return the theme's path, which equals the theme's name."""
+    return theme
+
+
+def get_template_engine(themes):
     """Get template engine used by a given theme."""
     for theme_name in themes:
-        engine_path = os.path.join(get_theme_path(theme_name, _themes_dir), 'engine')
+        engine_path = os.path.join(theme_name, 'engine')
         if os.path.isfile(engine_path):
             with open(engine_path) as fd:
                 return fd.readlines()[0].strip()
@@ -584,21 +616,24 @@ def get_template_engine(themes, _themes_dir='themes'):
     return 'mako'
 
 
-def get_parent_theme_name(theme_name, _themes_dir='themes'):
+def get_parent_theme_name(theme_name, themes_dirs=None):
     """Get name of parent theme."""
-    parent_path = os.path.join(get_theme_path(theme_name, _themes_dir), 'parent')
+    parent_path = os.path.join(theme_name, 'parent')
     if os.path.isfile(parent_path):
         with open(parent_path) as fd:
-            return fd.readlines()[0].strip()
+            parent = fd.readlines()[0].strip()
+        if themes_dirs:
+            return get_theme_path_real(parent, themes_dirs)
+        return parent
     return None
 
 
-def get_theme_chain(theme, _themes_dir='themes'):
-    """Create the full theme inheritance chain."""
-    themes = [theme]
+def get_theme_chain(theme, themes_dirs):
+    """Create the full theme inheritance chain including paths."""
+    themes = [get_theme_path_real(theme, themes_dirs)]
 
     while True:
-        parent = get_parent_theme_name(themes[-1], _themes_dir)
+        parent = get_parent_theme_name(themes[-1], themes_dirs=themes_dirs)
         # Avoid silly loops
         if parent is None or parent in themes:
             break
@@ -610,7 +645,6 @@ language_incomplete_warned = []
 
 
 class LanguageNotFoundError(Exception):
-
     """An exception thrown if language is not found."""
 
     def __init__(self, lang, orig):
@@ -623,7 +657,7 @@ class LanguageNotFoundError(Exception):
         return 'cannot find language {0}'.format(self.lang)
 
 
-def load_messages(themes, translations, default_lang):
+def load_messages(themes, translations, default_lang, themes_dirs):
     """Load theme's messages into context.
 
     All the messages from parent themes are loaded,
@@ -633,10 +667,12 @@ def load_messages(themes, translations, default_lang):
     oldpath = list(sys.path)
     for theme_name in themes[::-1]:
         msg_folder = os.path.join(get_theme_path(theme_name), 'messages')
-        default_folder = os.path.join(get_theme_path('base'), 'messages')
+        default_folder = os.path.join(get_theme_path_real('base', themes_dirs), 'messages')
         sys.path.insert(0, default_folder)
         sys.path.insert(0, msg_folder)
         english = __import__('messages_en')
+        # If we don't do the reload, the module is cached
+        _reload(english)
         for lang in list(translations.keys()):
             try:
                 translation = __import__('messages_' + lang)
@@ -655,6 +691,7 @@ def load_messages(themes, translations, default_lang):
                 del(translation)
             except ImportError as orig:
                 raise LanguageNotFoundError(lang, orig)
+        del(english)
     sys.path = oldpath
     return messages
 
@@ -724,27 +761,29 @@ def remove_file(source):
     elif os.path.isfile(source) or os.path.islink(source):
         os.remove(source)
 
-# slugify is copied from
+# slugify is adopted from
 # http://code.activestate.com/recipes/
 # 577257-slugify-make-a-string-usable-in-a-url-or-filename/
 _slugify_strip_re = re.compile(r'[^+\w\s-]')
 _slugify_hyphenate_re = re.compile(r'[-\s]+')
 
 
-def slugify(value, force=False):
+def slugify(value, lang=None, force=False):
     u"""Normalize string, convert to lowercase, remove non-alpha characters, convert spaces to hyphens.
 
     From Django's "django/template/defaultfilters.py".
 
-    >>> print(slugify('áéí.óú'))
+    >>> print(slugify('áéí.óú', lang='en'))
     aeiou
 
-    >>> print(slugify('foo/bar'))
+    >>> print(slugify('foo/bar', lang='en'))
     foobar
 
-    >>> print(slugify('foo bar'))
+    >>> print(slugify('foo bar', lang='en'))
     foo-bar
     """
+    if lang is None:  # TODO: remove in v8
+        LOGGER.warn("slugify() called without language!")
     if not isinstance(value, unicode_str):
         raise ValueError("Not a unicode object: {0}".format(value))
     if USE_SLUGIFY or force:
@@ -769,12 +808,14 @@ def slugify(value, force=False):
         return value
 
 
-def unslugify(value, discard_numbers=True):
+def unslugify(value, lang=None, discard_numbers=True):
     """Given a slug string (as a filename), return a human readable string.
 
     If discard_numbers is True, numbers right at the beginning of input
     will be removed.
     """
+    if lang is None:  # TODO: remove in v8
+        LOGGER.warn("unslugify() called without language!")
     if discard_numbers:
         value = re.sub('^[0-9]+', '', value)
     value = re.sub('([_\-\.])', ' ', value)
@@ -782,11 +823,23 @@ def unslugify(value, discard_numbers=True):
     return value
 
 
+def encodelink(iri):
+    """Given an encoded or unencoded link string, return an encoded string suitable for use as a link in HTML and XML."""
+    iri = unicodenormalize('NFC', iri)
+    link = OrderedDict(urlparse(iri)._asdict())
+    link['path'] = urlquote(urlunquote(link['path']).encode('utf-8'), safe="/~")
+    try:
+        link['netloc'] = link['netloc'].encode('utf-8').decode('idna').encode('idna').decode('utf-8')
+    except UnicodeDecodeError:
+        link['netloc'] = link['netloc'].encode('idna').decode('utf-8')
+    encoded_link = urlunparse(link.values())
+    return encoded_link
+
 # A very slightly safer version of zip.extractall that works on
 # python < 2.6
 
-class UnsafeZipException(Exception):
 
+class UnsafeZipException(Exception):
     """Exception for unsafe zip files."""
 
     pass
@@ -887,7 +940,7 @@ def apply_filters(task, filters, skip_ext=None):
     return task
 
 
-def get_crumbs(path, is_file=False, index_folder=None):
+def get_crumbs(path, is_file=False, index_folder=None, lang=None):
     """Create proper links for a crumb bar.
 
     index_folder is used if you want to use title from index file
@@ -923,8 +976,10 @@ def get_crumbs(path, is_file=False, index_folder=None):
         for i, crumb in enumerate(crumbs[-3::-1]):  # Up to parent folder only
             _path = '/'.join(['..'] * (i + 1))
             _crumbs.append([_path, crumb])
-        _crumbs.insert(0, ['.', crumbs[-2]])  # file's folder
-        _crumbs.insert(0, ['#', crumbs[-1]])  # file itself
+        if len(crumbs) >= 2:
+            _crumbs.insert(0, ['.', crumbs[-2]])  # file's folder
+        if len(crumbs) >= 1:
+            _crumbs.insert(0, ['#', crumbs[-1]])  # file itself
     else:
         for i, crumb in enumerate(crumbs[::-1]):
             _path = '/'.join(['..'] * i) or '#'
@@ -940,40 +995,49 @@ def get_crumbs(path, is_file=False, index_folder=None):
             index_post = index_folder.parse_index(folder, '', '')
             folder = folder.replace(crumb, '')
             if index_post:
-                crumb = index_post.title() or crumb
+                crumb = index_post.title(lang) or crumb
             _crumbs[i][1] = crumb
     return list(reversed(_crumbs))
 
 
-def get_asset_path(path, themes, files_folders={'files': ''}, _themes_dir='themes'):
+def get_asset_path(path, themes, files_folders={'files': ''}, output_dir='output'):
     """Return the "real", absolute path to the asset.
 
     By default, it checks which theme provides the asset.
     If the asset is not provided by a theme, then it will be checked for
     in the FILES_FOLDERS.
+    If it's not provided by either, it will be chacked in output, where
+    it may have been created by another plugin.
 
-    >>> print(get_asset_path('assets/css/rst.css', ['bootstrap3', 'base']))
+    >>> print(get_asset_path('assets/css/rst.css', get_theme_chain('bootstrap3', ['themes'])))
     /.../nikola/data/themes/base/assets/css/rst.css
 
-    >>> print(get_asset_path('assets/css/theme.css', ['bootstrap3', 'base']))
+    >>> print(get_asset_path('assets/css/theme.css', get_theme_chain('bootstrap3', ['themes'])))
     /.../nikola/data/themes/bootstrap3/assets/css/theme.css
 
-    >>> print(get_asset_path('nikola.py', ['bootstrap3', 'base'], {'nikola': ''}))
+    >>> print(get_asset_path('nikola.py', get_theme_chain('bootstrap3', ['themes']), {'nikola': ''}))
     /.../nikola/nikola.py
 
-    >>> print(get_asset_path('nikola/nikola.py', ['bootstrap3', 'base'], {'nikola':'nikola'}))
+    >>> print(get_asset_path('nikola.py', get_theme_chain('bootstrap3', ['themes']), {'nikola': 'nikola'}))
     None
 
+    >>> print(get_asset_path('nikola/nikola.py', get_theme_chain('bootstrap3', ['themes']), {'nikola': 'nikola'}))
+    /.../nikola/nikola.py
+
     """
     for theme_name in themes:
-        candidate = os.path.join(
-            get_theme_path(theme_name, _themes_dir),
-            path
-        )
+        candidate = os.path.join(get_theme_path(theme_name), path)
         if os.path.isfile(candidate):
             return candidate
     for src, rel_dst in files_folders.items():
-        candidate = os.path.abspath(os.path.join(src, path))
+        relpath = os.path.normpath(os.path.relpath(path, rel_dst))
+        if not relpath.startswith('..' + os.path.sep):
+            candidate = os.path.abspath(os.path.join(src, relpath))
+            if os.path.isfile(candidate):
+                return candidate
+
+    if output_dir:
+        candidate = os.path.join(output_dir, path)
         if os.path.isfile(candidate):
             return candidate
 
@@ -982,7 +1046,6 @@ def get_asset_path(path, themes, files_folders={'files': ''}, _themes_dir='theme
 
 
 class LocaleBorgUninitializedException(Exception):
-
     """Exception for unitialized LocaleBorg."""
 
     def __init__(self):
@@ -991,7 +1054,6 @@ class LocaleBorgUninitializedException(Exception):
 
 
 class LocaleBorg(object):
-
     """Provide locale related services and autoritative current_lang.
 
     current_lang is the last lang for which the locale was set
@@ -1022,7 +1084,6 @@ class LocaleBorg(object):
     NOTE: never use locale.getlocale() , it can return values that
     locale.setlocale will not accept in Windows XP, 7 and pythons 2.6, 2.7, 3.3
     Examples: "Spanish", "French" can't do the full circle set / get / set
-    That used to break calendar, but now seems is not the case, with month at least
     """
 
     initialized = False
@@ -1040,6 +1101,8 @@ class LocaleBorg(object):
         assert initial_lang is not None and initial_lang in locales
         cls.reset()
         cls.locales = locales
+        cls.month_name_handlers = []
+        cls.formatted_date_handlers = []
 
         # needed to decode some localized output in py2x
         encodings = {}
@@ -1049,25 +1112,71 @@ class LocaleBorg(object):
             encodings[lang] = encoding
 
         cls.encodings = encodings
-        cls.__shared_state['current_lang'] = initial_lang
+        cls.__initial_lang = initial_lang
         cls.initialized = True
 
+    def __get_shared_state(self):
+        if not self.initialized:
+            raise LocaleBorgUninitializedException()
+        shared_state = getattr(self.__thread_local, 'shared_state', None)
+        if shared_state is None:
+            shared_state = {'current_lang': self.__initial_lang}
+            self.__thread_local.shared_state = shared_state
+        return shared_state
+
     @classmethod
     def reset(cls):
         """Reset LocaleBorg.
 
         Used in testing to prevent leaking state between tests.
         """
+        import threading
+        cls.__thread_local = threading.local()
+        cls.__thread_lock = threading.Lock()
+
         cls.locales = {}
         cls.encodings = {}
-        cls.__shared_state = {'current_lang': None}
         cls.initialized = False
+        cls.month_name_handlers = []
+        cls.formatted_date_handlers = []
+        cls.thread_local = None
+        cls.thread_lock = None
+
+    @classmethod
+    def add_handler(cls, month_name_handler=None, formatted_date_handler=None):
+        """Allow to add month name and formatted date handlers.
+
+        If month_name_handler is not None, it is expected to be a callable
+        which accepts (month_no, lang) and returns either a string or None.
+
+        If formatted_date_handler is not None, it is expected to be a callable
+        which accepts (date_format, date, lang) and returns either a string or
+        None.
+
+        A handler is expected to either return the correct result for the given
+        language and data, or return None to indicate it is not able to do the
+        job. In that case, the next handler is asked, and finally the default
+        implementation is used.
+        """
+        if month_name_handler is not None:
+            cls.month_name_handlers.append(month_name_handler)
+        if formatted_date_handler is not None:
+            cls.formatted_date_handlers.append(formatted_date_handler)
 
     def __init__(self):
         """Initialize."""
         if not self.initialized:
             raise LocaleBorgUninitializedException()
-        self.__dict__ = self.__shared_state
+
+    @property
+    def current_lang(self):
+        """Return the current language."""
+        return self.__get_shared_state()['current_lang']
+
+    def __set_locale(self, lang):
+        """Set the locale for language lang without updating current_lang."""
+        locale_n = self.locales[lang]
+        locale.setlocale(locale.LC_ALL, locale_n)
 
     def set_locale(self, lang):
         """Set the locale for language lang, returns an empty string.
@@ -1076,36 +1185,66 @@ class LocaleBorg(object):
         in windows that cannot be guaranted.
         In either case, the locale encoding is available in cls.encodings[lang]
         """
-        # intentional non try-except: templates must ask locales with a lang,
-        # let the code explode here and not hide the point of failure
-        # Also, not guarded with an if lang==current_lang because calendar may
-        # put that out of sync
-        locale_n = self.locales[lang]
-        self.__shared_state['current_lang'] = lang
-        locale.setlocale(locale.LC_ALL, locale_n)
-        return ''
+        with self.__thread_lock:
+            # intentional non try-except: templates must ask locales with a lang,
+            # let the code explode here and not hide the point of failure
+            # Also, not guarded with an if lang==current_lang because calendar may
+            # put that out of sync
+            self.__set_locale(lang)
+            self.__get_shared_state()['current_lang'] = lang
+            return ''
 
     def get_month_name(self, month_no, lang):
         """Return localized month name in an unicode string."""
-        if sys.version_info[0] == 3:  # Python 3
-            with calendar.different_locale(self.locales[lang]):
-                s = calendar.month_name[month_no]
-            # for py3 s is unicode
-        else:  # Python 2
-            with calendar.TimeEncoding(self.locales[lang]):
-                s = calendar.month_name[month_no]
-            enc = self.encodings[lang]
-            if not enc:
-                enc = 'UTF-8'
-
-            s = s.decode(enc)
-        # paranoid about calendar ending in the wrong locale (windows)
-        self.set_locale(self.current_lang)
-        return s
+        # For thread-safety
+        with self.__thread_lock:
+            for handler in self.month_name_handlers:
+                res = handler(month_no, lang)
+                if res is not None:
+                    return res
+            old_lang = self.current_lang
+            self.__set_locale(lang)
+            s = calendar.month_name[month_no]
+            self.__set_locale(old_lang)
+            if sys.version_info[0] == 2:
+                enc = self.encodings[lang]
+                if not enc:
+                    enc = 'UTF-8'
 
+                s = s.decode(enc)
+            return s
+
+    def formatted_date(self, date_format, date):
+        """Return the formatted date as unicode."""
+        with self.__thread_lock:
+            current_lang = self.current_lang
+            # For thread-safety
+            self.__set_locale(current_lang)
+            fmt_date = None
+            # Get a string out of a TranslatableSetting
+            if isinstance(date_format, TranslatableSetting):
+                date_format = date_format(current_lang)
+            # First check handlers
+            for handler in self.formatted_date_handlers:
+                fmt_date = handler(date_format, date, current_lang)
+                if fmt_date is not None:
+                    break
+            # If no handler was able to format the date, ask Python
+            if fmt_date is None:
+                if date_format == 'webiso':
+                    # Formatted after RFC 3339 (web ISO 8501 profile) with Zulu
+                    # zone desgignator for times in UTC and no microsecond precision.
+                    fmt_date = date.replace(microsecond=0).isoformat().replace('+00:00', 'Z')
+                else:
+                    fmt_date = date.strftime(date_format)
+
+            # Issue #383, this changes from py2 to py3
+            if isinstance(fmt_date, bytes_str):
+                fmt_date = fmt_date.decode('utf8')
+            return fmt_date
 
-class ExtendedRSS2(rss.RSS2):
 
+class ExtendedRSS2(rss.RSS2):
     """Extended RSS class."""
 
     xsl_stylesheet_href = None
@@ -1129,7 +1268,6 @@ class ExtendedRSS2(rss.RSS2):
 
 
 class ExtendedItem(rss.RSSItem):
-
     """Extended RSS item."""
 
     def __init__(self, **kw):
@@ -1181,9 +1319,9 @@ def demote_headers(doc, level=1):
         r = range(1 + level, 7)
     for i in reversed(r):
         # html headers go to 6, so we can’t “lower” beneath five
-            elements = doc.xpath('//h' + str(i))
-            for e in elements:
-                e.tag = 'h' + str(i + level)
+        elements = doc.xpath('//h' + str(i))
+        for e in elements:
+            e.tag = 'h' + str(i + level)
 
 
 def get_root_dir():
@@ -1224,10 +1362,10 @@ def get_translation_candidate(config, path, lang):
     cache/posts/fancy.post.html
     >>> print(get_translation_candidate(config, 'cache/posts/fancy.post.html', 'es'))
     cache/posts/fancy.post.es.html
-    >>> print(get_translation_candidate(config, 'cache/stories/charts.html', 'es'))
-    cache/stories/charts.es.html
-    >>> print(get_translation_candidate(config, 'cache/stories/charts.html', 'en'))
-    cache/stories/charts.html
+    >>> print(get_translation_candidate(config, 'cache/pages/charts.html', 'es'))
+    cache/pages/charts.es.html
+    >>> print(get_translation_candidate(config, 'cache/pages/charts.html', 'en'))
+    cache/pages/charts.html
 
     >>> config = {'TRANSLATIONS_PATTERN': '{path}.{ext}.{lang}', 'DEFAULT_LANG': 'en', 'TRANSLATIONS': {'es':'1', 'en': 1}}
     >>> print(get_translation_candidate(config, '*.rst', 'es'))
@@ -1326,7 +1464,6 @@ def ask_yesno(query, default=None):
 
 
 class CommandWrapper(object):
-
     """Converts commands into functions."""
 
     def __init__(self, cmd, commands_object):
@@ -1342,7 +1479,6 @@ class CommandWrapper(object):
 
 
 class Commands(object):
-
     """Nikola Commands.
 
     Sample usage:
@@ -1433,7 +1569,6 @@ def options2docstring(name, options):
 
 
 class NikolaPygmentsHTML(HtmlFormatter):
-
     """A Nikola-specific modification of Pygments' HtmlFormatter."""
 
     def __init__(self, anchor_ref, classes=None, linenos='table', linenostart=1):
@@ -1443,7 +1578,7 @@ class NikolaPygmentsHTML(HtmlFormatter):
         self.nclasses = classes
         super(NikolaPygmentsHTML, self).__init__(
             cssclass='code', linenos=linenos, linenostart=linenostart, nowrap=False,
-            lineanchors=slugify(anchor_ref, force=True), anchorlinenos=True)
+            lineanchors=slugify(anchor_ref, lang=LocaleBorg().current_lang, force=True), anchorlinenos=True)
 
     def wrap(self, source, outfile):
         """Wrap the ``source``, which is a generator yielding individual lines, in custom generators."""
@@ -1540,7 +1675,6 @@ def create_redirect(src, dst):
 
 
 class TreeNode(object):
-
     """A tree node."""
 
     indent_levels = None  # use for formatting comments as tree
@@ -1673,6 +1807,100 @@ def join_hierarchical_category_path(category_path):
     return '/'.join([escape(p) for p in category_path])
 
 
+def colorize_str_from_base_color(string, base_color):
+    """Find a perceptual similar color from a base color based on the hash of a string.
+
+    Make up to 16 attempts (number of bytes returned by hashing) at picking a
+    hue for our color at least 27 deg removed from the base color, leaving
+    lightness and saturation untouched using HUSL colorspace.
+    """
+    def hash_str(string, pos):
+        x = hashlib.md5(string.encode('utf-8')).digest()[pos]
+        try:
+            # Python 2: a string
+            # TODO: remove in v8
+            return ord(x)
+        except TypeError:
+            # Python 3: already an integer
+            return x
+
+    def degreediff(dega, degb):
+        return min(abs(dega - degb), abs((degb - dega) + 360))
+
+    if husl is None:
+        req_missing(['husl'], 'Use color mixing (section colors)',
+                    optional=True)
+        return base_color
+    h, s, l = husl.hex_to_husl(base_color)
+    old_h = h
+    idx = 0
+    while degreediff(old_h, h) < 27 and idx < 16:
+        h = 360.0 * (float(hash_str(string, idx)) / 255)
+        idx += 1
+    return husl.husl_to_hex(h, s, l)
+
+
+def color_hsl_adjust_hex(hexstr, adjust_h=None, adjust_s=None, adjust_l=None):
+    """Adjust a hex color using HSL arguments, adjustments in percentages 1.0 to -1.0. Returns a hex color."""
+    h, s, l = husl.hex_to_husl(hexstr)
+
+    if adjust_h:
+        h = h + (adjust_h * 360.0)
+
+    if adjust_s:
+        s = s + (adjust_s * 100.0)
+
+    if adjust_l:
+        l = l + (adjust_l * 100.0)
+
+    return husl.husl_to_hex(h, s, l)
+
+
+def dns_sd(port, inet6):
+    """Optimistically publish a HTTP service to the local network over DNS-SD.
+
+    Works only on Linux/FreeBSD.  Requires the `avahi` and `dbus` modules (symlinks in virtualenvs)
+    """
+    try:
+        import avahi
+        import dbus
+        inet = avahi.PROTO_INET6 if inet6 else avahi.PROTO_INET
+        name = "{0}'s Nikola Server on {1}".format(os.getlogin(), socket.gethostname())
+        bus = dbus.SystemBus()
+        bus_server = dbus.Interface(bus.get_object(avahi.DBUS_NAME,
+                                                   avahi.DBUS_PATH_SERVER),
+                                    avahi.DBUS_INTERFACE_SERVER)
+        bus_group = dbus.Interface(bus.get_object(avahi.DBUS_NAME,
+                                                  bus_server.EntryGroupNew()),
+                                   avahi.DBUS_INTERFACE_ENTRY_GROUP)
+        bus_group.AddService(avahi.IF_UNSPEC, inet, dbus.UInt32(0),
+                             name, '_http._tcp', '', '',
+                             dbus.UInt16(port), '')
+        bus_group.Commit()
+        return bus_group  # remember to bus_group.Reset() to unpublish
+    except Exception:
+        return None
+
+
+def clean_before_deployment(site):
+    """Clean drafts and future posts before deployment."""
+    undeployed_posts = []
+    deploy_drafts = site.config.get('DEPLOY_DRAFTS', True)
+    deploy_future = site.config.get('DEPLOY_FUTURE', False)
+    if not (deploy_drafts and deploy_future):  # == !drafts || !future
+        # Remove drafts and future posts
+        out_dir = site.config['OUTPUT_FOLDER']
+        site.scan_posts()
+        for post in site.timeline:
+            if (not deploy_drafts and post.is_draft) or (not deploy_future and post.publish_later):
+                for lang in post.translated_to:
+                    remove_file(os.path.join(out_dir, post.destination_path(lang)))
+                    source_path = post.destination_path(lang, post.source_ext(True))
+                    remove_file(os.path.join(out_dir, source_path))
+                undeployed_posts.append(post)
+    return undeployed_posts
+
+
 # Stolen from textwrap in Python 3.4.3.
 def indent(text, prefix, predicate=None):
     """Add 'prefix' to the beginning of selected lines in 'text'.
@@ -1690,3 +1918,25 @@ def indent(text, prefix, predicate=None):
         for line in text.splitlines(True):
             yield (prefix + line if predicate(line) else line)
     return ''.join(prefixed_lines())
+
+
+def load_data(path):
+    """Given path to a file, load data from it."""
+    ext = os.path.splitext(path)[-1]
+    loader = None
+    if ext in {'.yml', '.yaml'}:
+        loader = yaml
+        if yaml is None:
+            req_missing(['yaml'], 'use YAML data files')
+            return {}
+    elif ext in {'.json', '.js'}:
+        loader = json
+    elif ext in {'.toml', '.tml'}:
+        if toml is None:
+            req_missing(['toml'], 'use TOML data files')
+            return {}
+        loader = toml
+    if loader is None:
+        return
+    with io.open(path, 'r', encoding='utf8') as inf:
+        return loader.load(inf)