New upstream version 7.8.1upstream/7.8.1

1 files changed, 431 insertions, 150 deletions

diff --git a/nikola/nikola.py b/nikola/nikola.py
index 9e9b849..0a62360 100644
--- a/nikola/nikola.py
+++ b/nikola/nikola.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright © 2012-2015 Roberto Alsina and others.
+# Copyright © 2012-2016 Roberto Alsina and others.
 
 # Permission is hereby granted, free of charge, to any
 # person obtaining a copy of this software and associated
@@ -57,7 +57,8 @@ from yapsy.PluginManager import PluginManager
 from blinker import signal
 
 from .post import Post  # NOQA
-from . import DEBUG, utils
+from .state import Persistor
+from . import DEBUG, utils, shortcodes
 from .plugin_categories import (
     Command,
     LateTask,
@@ -65,6 +66,7 @@ from .plugin_categories import (
     CompilerExtension,
     MarkdownExtension,
     RestExtension,
+    ShortcodePlugin,
     Task,
     TaskMultiplier,
     TemplateSystem,
@@ -119,6 +121,8 @@ LEGAL_VALUES = {
         'fa': 'Persian',
         'fi': 'Finnish',
         'fr': 'French',
+        'gl': 'Galician',
+        'he': 'Hebrew',
         'hi': 'Hindi',
         'hr': 'Croatian',
         'hu': 'Hungarian',
@@ -126,7 +130,8 @@ LEGAL_VALUES = {
         'it': 'Italian',
         ('ja', '!jp'): 'Japanese',
         'ko': 'Korean',
-        'nb': 'Norwegian Bokmål',
+        'lt': 'Lithuanian',
+        'nb': 'Norwegian (Bokmål)',
         'nl': 'Dutch',
         'pa': 'Punjabi',
         'pl': 'Polish',
@@ -135,46 +140,75 @@ LEGAL_VALUES = {
         'ru': 'Russian',
         'sk': 'Slovak',
         'sl': 'Slovene',
+        'sq': 'Albanian',
         'sr': 'Serbian (Cyrillic)',
         'sr_latin': 'Serbian (Latin)',
         'sv': 'Swedish',
+        'te': 'Telugu',
         ('tr', '!tr_TR'): 'Turkish',
         'ur': 'Urdu',
         'uk': 'Ukrainian',
         'zh_cn': 'Chinese (Simplified)',
+        'zh_tw': 'Chinese (Traditional)'
     },
     '_WINDOWS_LOCALE_GUESSES': {
         # TODO incomplete
-        # some languages may need that the appropiate Microsoft Language Pack be instaled.
+        # some languages may need that the appropriate Microsoft Language Pack be installed.
+        "ar": "Arabic",
+        "az": "Azeri (Latin)",
         "bg": "Bulgarian",
+        "bs": "Bosnian",
         "ca": "Catalan",
+        "cs": "Czech",
+        "da": "Danish",
         "de": "German",
         "el": "Greek",
         "en": "English",
-        "eo": "Esperanto",
+        # "eo": "Esperanto", # Not available
         "es": "Spanish",
-        "fa": "Farsi",  # Persian
+        "et": "Estonian",
+        "eu": "Basque",
+        "fa": "Persian",  # Persian
+        "fi": "Finnish",
         "fr": "French",
+        "gl": "Galician",
+        "he": "Hebrew",
+        "hi": "Hindi",
         "hr": "Croatian",
         "hu": "Hungarian",
+        "id": "Indonesian",
         "it": "Italian",
-        "jp": "Japanese",
+        "ja": "Japanese",
+        "ko": "Korean",
+        "nb": "Norwegian",  # Not Bokmål, as Windows doesn't find it for unknown reasons.
         "nl": "Dutch",
+        "pa": "Punjabi",
         "pl": "Polish",
+        "pt": "Portuguese_Portugal",
         "pt_br": "Portuguese_Brazil",
         "ru": "Russian",
-        "sl_si": "Slovenian",
-        "tr_tr": "Turkish",
+        "sk": "Slovak",
+        "sl": "Slovenian",
+        "sq": "Albanian",
+        "sr": "Serbian",
+        "sr_latin": "Serbian (Latin)",
+        "sv": "Swedish",
+        "te": "Telugu",
+        "tr": "Turkish",
+        "uk": "Ukrainian",
+        "ur": "Urdu",
         "zh_cn": "Chinese_China",  # Chinese (Simplified)
+        "zh_tw": "Chinese_Taiwan",  # Chinese (Traditional)
     },
     '_TRANSLATIONS_WITH_COUNTRY_SPECIFIERS': {
         # This dict is used in `init` in case of locales that exist with a
         # country specifier.  If there is no other locale that has the same
         # language with a different country, ``nikola init`` (but nobody else!)
         # will accept it, warning the user about it.
-        'zh': 'zh_cn',
+
+        # This dict is currently empty.
     },
-    'RTL_LANGUAGES': ('ar', 'fa', 'ur'),
+    'RTL_LANGUAGES': ('ar', 'fa', 'he', 'ur'),
     'COLORBOX_LOCALES': defaultdict(
         str,
         ar='ar',
@@ -190,12 +224,14 @@ LEGAL_VALUES = {
         fa='fa',
         fi='fi',
         fr='fr',
+        he='he',
         hr='hr',
         hu='hu',
         id='id',
         it='it',
         ja='ja',
         ko='kr',  # kr is South Korea, ko is the Korean language
+        lt='lt',
         nb='no',
         nl='nl',
         pl='pl',
@@ -209,7 +245,8 @@ LEGAL_VALUES = {
         sv='sv',
         tr='tr',
         uk='uk',
-        zh_cn='zh-CN'
+        zh_cn='zh-CN',
+        zh_tw='zh-TW'
     ),
     'MOMENTJS_LOCALES': defaultdict(
         str,
@@ -232,13 +269,16 @@ LEGAL_VALUES = {
         fa='fa',
         fi='fi',
         fr='fr',
+        gl='gl',
         hi='hi',
+        he='he',
         hr='hr',
         hu='hu',
         id='id',
         it='it',
         ja='ja',
         ko='ko',
+        lt='lt',
         nb='nb',
         nl='nl',
         pl='pl',
@@ -247,12 +287,14 @@ LEGAL_VALUES = {
         ru='ru',
         sk='sk',
         sl='sl',
+        sq='sq',
         sr='sr-cyrl',
         sr_latin='sr',
         sv='sv',
         tr='tr',
         uk='uk',
-        zh_cn='zh-cn'
+        zh_cn='zh-cn',
+        zh_tw='zh-tw'
     ),
     'PYPHEN_LOCALES': {
         'bg': 'bg',
@@ -262,13 +304,14 @@ LEGAL_VALUES = {
         'da': 'da',
         'de': 'de',
         'el': 'el',
-        'en': 'en',
+        'en': 'en_US',
         'es': 'es',
         'et': 'et',
         'fr': 'fr',
         'hr': 'hr',
         'hu': 'hu',
         'it': 'it',
+        'lt': 'lt',
         'nb': 'nb',
         'nl': 'nl',
         'pl': 'pl',
@@ -279,8 +322,32 @@ LEGAL_VALUES = {
         'sl': 'sl',
         'sr': 'sr',
         'sv': 'sv',
+        'te': 'te',
         'uk': 'uk',
     },
+    'DOCUTILS_LOCALES': {
+        'ca': 'ca',
+        'da': 'da',
+        'de': 'de',
+        'en': 'en',
+        'eo': 'eo',
+        'es': 'es',
+        'fi': 'fi',
+        'fr': 'fr',
+        'gl': 'gl',
+        'he': 'he',
+        'it': 'it',
+        'ja': 'ja',
+        'lt': 'lt',
+        'pl': 'pl',
+        'pt': 'pt_br',  # hope nobody will mind
+        'pt_br': 'pt_br',
+        'ru': 'ru',
+        'sk': 'sk',
+        'sv': 'sv',
+        'zh_cn': 'zh_cn',
+        'zh_tw': 'zh_tw'
+    }
 }
 
 
@@ -288,7 +355,13 @@ def _enclosure(post, lang):
     """Add an enclosure to RSS."""
     enclosure = post.meta('enclosure', lang)
     if enclosure:
-        length = 0
+        try:
+            length = int(post.meta('enclosure_length', lang) or 0)
+        except KeyError:
+            length = 0
+        except ValueError:
+            utils.LOGGER.warn("Invalid enclosure length for post {0}".format(post.source_path))
+            length = 0
         url = enclosure
         mime = mimetypes.guess_type(url)[0]
         return url, length, mime
@@ -324,6 +397,7 @@ class Nikola(object):
         self._scanned = False
         self._template_system = None
         self._THEMES = None
+        self._MESSAGES = None
         self.debug = DEBUG
         self.loghandlers = utils.STDERR_HANDLER  # TODO remove on v8
         self.colorful = config.pop('__colorful__', False)
@@ -334,6 +408,7 @@ class Nikola(object):
         self.configuration_filename = config.pop('__configuration_filename__', False)
         self.configured = bool(config)
         self.injected_deps = defaultdict(list)
+        self.shortcode_registry = {}
 
         self.rst_transforms = []
         self.template_hooks = {
@@ -373,7 +448,7 @@ class Nikola(object):
             'CODE_COLOR_SCHEME': 'default',
             'COMMENT_SYSTEM': 'disqus',
             'COMMENTS_IN_GALLERIES': False,
-            'COMMENTS_IN_STORIES': False,
+            'COMMENTS_IN_PAGES': False,
             'COMPILERS': {
                 "rest": ('.txt', '.rst'),
                 "markdown": ('.md', '.mdown', '.markdown'),
@@ -398,8 +473,10 @@ class Nikola(object):
             'DEPLOY_COMMANDS': {'default': []},
             'DISABLED_PLUGINS': [],
             'EXTRA_PLUGINS_DIRS': [],
+            'EXTRA_THEMES_DIRS': [],
             'COMMENT_SYSTEM_ID': 'nikolademo',
             'ENABLE_AUTHOR_PAGES': True,
+            'EXIF_WHITELIST': {},
             'EXTRA_HEAD_DATA': '',
             'FAVICONS': (),
             'FEED_LENGTH': 10,
@@ -408,6 +485,7 @@ class Nikola(object):
             'FILES_FOLDERS': {'files': ''},
             'FILTERS': {},
             'FORCE_ISO8601': False,
+            'FRONT_INDEX_HEADER': '',
             'GALLERY_FOLDERS': {'galleries': 'galleries'},
             'GALLERY_SORT_BY_DATE': True,
             'GLOBAL_CONTEXT_FILLER': [],
@@ -430,6 +508,7 @@ class Nikola(object):
             'INDEXES_STATIC': True,
             'INDEX_PATH': '',
             'IPYNB_CONFIG': {},
+            'KATEX_AUTO_RENDER': '',
             'LESS_COMPILER': 'lessc',
             'LESS_OPTIONS': [],
             'LICENSE': '',
@@ -444,11 +523,14 @@ class Nikola(object):
             'OUTPUT_FOLDER': 'output',
             'POSTS': (("posts/*.txt", "posts", "post.tmpl"),),
             'POSTS_SECTIONS': True,
+            'POSTS_SECTION_COLORS': {},
             'POSTS_SECTION_ARE_INDEXES': True,
             'POSTS_SECTION_DESCRIPTIONS': "",
             'POSTS_SECTION_FROM_META': False,
             'POSTS_SECTION_NAME': "",
             'POSTS_SECTION_TITLE': "{name}",
+            'PRESERVE_EXIF_DATA': False,
+            # TODO: change in v8
             'PAGES': (("stories/*.txt", "stories", "story.tmpl"),),
             'PANDOC_OPTIONS': [],
             'PRETTY_URLS': False,
@@ -475,7 +557,7 @@ class Nikola(object):
             'SLUG_TAG_PATH': True,
             'SOCIAL_BUTTONS_CODE': '',
             'SITE_URL': 'https://example.com/',
-            'STORY_INDEX': False,
+            'PAGE_INDEX': False,
             'STRIP_INDEXES': False,
             'SITEMAP_INCLUDE_FILELESS_DIRS': True,
             'TAG_PATH': 'categories',
@@ -492,7 +574,7 @@ class Nikola(object):
             'THUMBNAIL_SIZE': 180,
             'UNSLUGIFY_TITLES': False,  # WARNING: conf.py.in overrides this with True for backwards compatibility
             'URL_TYPE': 'rel_path',
-            'USE_BASE_TAG': True,
+            'USE_BASE_TAG': False,
             'USE_BUNDLES': True,
             'USE_CDN': False,
             'USE_CDN_WARNING': True,
@@ -511,6 +593,7 @@ class Nikola(object):
             'GITHUB_SOURCE_BRANCH': 'master',
             'GITHUB_DEPLOY_BRANCH': 'gh-pages',
             'GITHUB_REMOTE_NAME': 'origin',
+            'GITHUB_COMMIT_SOURCE': False,  # WARNING: conf.py.in overrides this with True for backwards compatibility
         }
 
         # set global_context for template rendering
@@ -548,6 +631,7 @@ class Nikola(object):
                                       'BODY_END',
                                       'EXTRA_HEAD_DATA',
                                       'NAVIGATION_LINKS',
+                                      'FRONT_INDEX_HEADER',
                                       'INDEX_READ_MORE_LINK',
                                       'FEED_READ_MORE_LINK',
                                       'INDEXES_TITLE',
@@ -576,7 +660,13 @@ class Nikola(object):
                                              'body_end',
                                              'extra_head_data',
                                              'date_format',
-                                             'js_date_format',)
+                                             'js_date_format',
+                                             'posts_section_colors',
+                                             'posts_section_descriptions',
+                                             'posts_section_name',
+                                             'posts_section_title',
+                                             'front_index_header',
+                                             )
         # WARNING: navigation_links SHOULD NOT be added to the list above.
         #          Themes ask for [lang] there and we should provide it.
 
@@ -594,6 +684,15 @@ class Nikola(object):
             except KeyError:
                 pass
 
+        # A EXIF_WHITELIST implies you want to keep EXIF data
+        if self.config['EXIF_WHITELIST'] and not self.config['PRESERVE_EXIF_DATA']:
+            utils.LOGGER.warn('Setting EXIF_WHITELIST implies PRESERVE_EXIF_DATA is set to True')
+            self.config['PRESERVE_EXIF_DATA'] = True
+
+        # Setting PRESERVE_EXIF_DATA with an empty EXIF_WHITELIST implies 'keep everything'
+        if self.config['PRESERVE_EXIF_DATA'] and not self.config['EXIF_WHITELIST']:
+            utils.LOGGER.warn('You are setting PRESERVE_EXIF_DATA and not EXIF_WHITELIST so EXIF data is not really kept.')
+
         # Handle CONTENT_FOOTER properly.
         # We provide the arguments to format in CONTENT_FOOTER_FORMATS.
         self.config['CONTENT_FOOTER'].langformat(self.config['CONTENT_FOOTER_FORMATS'])
@@ -635,7 +734,7 @@ class Nikola(object):
         # TODO: remove on v8
         if 'RSS_LINKS_APPEND_QUERY' in config:
             utils.LOGGER.warn('The RSS_LINKS_APPEND_QUERY option is deprecated, use FEED_LINKS_APPEND_QUERY instead.')
-            if 'FEED_TEASERS' in config:
+            if 'FEED_LINKS_APPEND_QUERY' in config:
                 utils.LOGGER.warn('FEED_LINKS_APPEND_QUERY conflicts with RSS_LINKS_APPEND_QUERY, ignoring RSS_LINKS_APPEND_QUERY.')
             self.config['FEED_LINKS_APPEND_QUERY'] = config['RSS_LINKS_APPEND_QUERY']
 
@@ -772,6 +871,15 @@ class Nikola(object):
             utils.LOGGER.warn("WRITE_TAG_CLOUD is not set in your config.  Defaulting to True (== writing tag_cloud_data.json).")
             utils.LOGGER.warn("Please explicitly add the setting to your conf.py with the desired value, as the setting will default to False in the future.")
 
+        # Rename stories to pages (#1891, #2518)
+        # TODO: remove in v8
+        if 'COMMENTS_IN_STORIES' in config:
+            utils.LOGGER.warn('The COMMENTS_IN_STORIES option is deprecated, use COMMENTS_IN_PAGES instead.')
+            self.config['COMMENTS_IN_PAGES'] = config['COMMENTS_IN_STORIES']
+        if 'STORY_INDEX' in config:
+            utils.LOGGER.warn('The STORY_INDEX option is deprecated, use PAGE_INDEX instead.')
+            self.config['PAGE_INDEX'] = config['STORY_INDEX']
+
         # We use one global tzinfo object all over Nikola.
         try:
             self.tzinfo = dateutil.tz.gettz(self.config['TIMEZONE'])
@@ -794,6 +902,9 @@ class Nikola(object):
                     candidate = utils.get_translation_candidate(self.config, "f" + ext, lang)
                     compilers[compiler].add(candidate)
 
+        # Get search path for themes
+        self.themes_dirs = ['themes'] + self.config['EXTRA_THEMES_DIRS']
+
         # Avoid redundant compilers
         # Remove compilers that match nothing in POSTS/PAGES
         # And put them in "bad compilers"
@@ -807,9 +918,21 @@ class Nikola(object):
             else:
                 self.bad_compilers.add(k)
 
-        self._set_global_context()
+        self._set_global_context_from_config()
+        self._set_global_context_from_data()
 
-    def init_plugins(self, commands_only=False):
+        # Set persistent state facility
+        self.state = Persistor('state_data.json')
+
+        # Set cache facility
+        self.cache = Persistor(os.path.join(self.config['CACHE_FOLDER'], 'cache_data.json'))
+
+        # Create directories for persistors only if a site exists (Issue #2334)
+        if self.configured:
+            self.state._set_site(self)
+            self.cache._set_site(self)
+
+    def init_plugins(self, commands_only=False, load_all=False):
         """Load plugins as needed."""
         self.plugin_manager = PluginManager(categories_filter={
             "Command": Command,
@@ -821,6 +944,7 @@ class Nikola(object):
             "CompilerExtension": CompilerExtension,
             "MarkdownExtension": MarkdownExtension,
             "RestExtension": RestExtension,
+            "ShortcodePlugin": ShortcodePlugin,
             "SignalHandler": SignalHandler,
             "ConfigPlugin": ConfigPlugin,
             "PostScanner": PostScanner,
@@ -828,47 +952,71 @@ class Nikola(object):
         self.plugin_manager.getPluginLocator().setPluginInfoExtension('plugin')
         extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
         if sys.version_info[0] == 3:
-            places = [
+            self._plugin_places = [
                 resource_filename('nikola', 'plugins'),
-                os.path.join(os.getcwd(), 'plugins'),
                 os.path.expanduser('~/.nikola/plugins'),
+                os.path.join(os.getcwd(), 'plugins'),
             ] + [path for path in extra_plugins_dirs if path]
         else:
-            places = [
+            self._plugin_places = [
                 resource_filename('nikola', utils.sys_encode('plugins')),
                 os.path.join(os.getcwd(), utils.sys_encode('plugins')),
                 os.path.expanduser('~/.nikola/plugins'),
             ] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
 
-        self.plugin_manager.getPluginLocator().setPluginPlaces(places)
+        self.plugin_manager.getPluginLocator().setPluginPlaces(self._plugin_places)
         self.plugin_manager.locatePlugins()
         bad_candidates = set([])
-        for p in self.plugin_manager._candidates:
-            if commands_only:
-                if p[-1].details.has_option('Nikola', 'plugincategory'):
-                    # FIXME TemplateSystem should not be needed
-                    if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}:
+        if not load_all:
+            for p in self.plugin_manager._candidates:
+                if commands_only:
+                    if p[-1].details.has_option('Nikola', 'plugincategory'):
+                        # FIXME TemplateSystem should not be needed
+                        if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}:
+                            bad_candidates.add(p)
+                    else:
+                        bad_candidates.add(p)
+                elif self.configured:  # Not commands-only, and configured
+                    # Remove compilers we don't use
+                    if p[-1].name in self.bad_compilers:
+                        bad_candidates.add(p)
+                        self.disabled_compilers[p[-1].name] = p
+                        utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
+                    if p[-1].name not in self.config['COMPILERS'] and \
+                            p[-1].details.has_option('Nikola', 'plugincategory') and p[-1].details.get('Nikola', 'PluginCategory') == 'Compiler':
                         bad_candidates.add(p)
-            else:  # Not commands-only
-                # Remove compilers we don't use
-                if p[-1].name in self.bad_compilers:
-                    bad_candidates.add(p)
-                    self.disabled_compilers[p[-1].name] = p
-                    utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
-                if p[-1].name not in self.config['COMPILERS'] and \
-                        p[-1].details.has_option('Nikola', 'plugincategory') and p[-1].details.get('Nikola', 'PluginCategory') == 'Compiler':
-                    bad_candidates.add(p)
-                    self.disabled_compilers[p[-1].name] = p
-                    utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
-                # Remove blacklisted plugins
-                if p[-1].name in self.config['DISABLED_PLUGINS']:
-                    bad_candidates.add(p)
-                    utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name)
-                # Remove compiler extensions we don't need
-                if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers:
-                    bad_candidates.add(p)
-                    utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name)
-        self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates)
+                        self.disabled_compilers[p[-1].name] = p
+                        utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
+                    # Remove blacklisted plugins
+                    if p[-1].name in self.config['DISABLED_PLUGINS']:
+                        bad_candidates.add(p)
+                        utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name)
+                    # Remove compiler extensions we don't need
+                    if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers:
+                        bad_candidates.add(p)
+                        utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name)
+            self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates)
+
+        # Find repeated plugins and discard the less local copy
+        def plugin_position_in_places(plugin):
+            # plugin here is a tuple:
+            # (path to the .plugin file, path to plugin module w/o .py, plugin metadata)
+            for i, place in enumerate(self._plugin_places):
+                if plugin[0].startswith(place):
+                    return i
+
+        plugin_dict = defaultdict(list)
+        for data in self.plugin_manager._candidates:
+            plugin_dict[data[2].name].append(data)
+        self.plugin_manager._candidates = []
+        for name, plugins in plugin_dict.items():
+            if len(plugins) > 1:
+                # Sort by locality
+                plugins.sort(key=plugin_position_in_places)
+                utils.LOGGER.debug("Plugin {} exists in multiple places, using {}".format(
+                    plugins[-1][2].name, plugins[-1][0]))
+            self.plugin_manager._candidates.append(plugins[-1])
+
         self.plugin_manager.loadPlugins()
 
         self._activate_plugins_of_category("SignalHandler")
@@ -895,6 +1043,9 @@ class Nikola(object):
                 self.plugin_manager.activatePluginByName(plugin_info.name)
                 plugin_info.plugin_object.set_site(self)
 
+        # Activate shortcode plugins
+        self._activate_plugins_of_category("ShortcodePlugin")
+
         # Load compiler plugins
         self.compilers = {}
         self.inverse_compilers = {}
@@ -905,11 +1056,15 @@ class Nikola(object):
                 plugin_info.plugin_object
 
         self._activate_plugins_of_category("ConfigPlugin")
-
+        self._register_templated_shortcodes()
         signal('configured').send(self)
 
-    def _set_global_context(self):
-        """Create global context from configuration."""
+    def _set_global_context_from_config(self):
+        """Create global context from configuration.
+
+        These are options that are used by templates, so they always need to be
+        available.
+        """
         self._GLOBAL_CONTEXT['url_type'] = self.config['URL_TYPE']
         self._GLOBAL_CONTEXT['timezone'] = self.tzinfo
         self._GLOBAL_CONTEXT['_link'] = self.link
@@ -937,6 +1092,7 @@ class Nikola(object):
         self._GLOBAL_CONTEXT['show_blog_title'] = self.config.get('SHOW_BLOG_TITLE')
         self._GLOBAL_CONTEXT['logo_url'] = self.config.get('LOGO_URL')
         self._GLOBAL_CONTEXT['blog_description'] = self.config.get('BLOG_DESCRIPTION')
+        self._GLOBAL_CONTEXT['front_index_header'] = self.config.get('FRONT_INDEX_HEADER')
         self._GLOBAL_CONTEXT['color_hsl_adjust_hex'] = utils.color_hsl_adjust_hex
         self._GLOBAL_CONTEXT['colorize_str_from_base_color'] = utils.colorize_str_from_base_color
 
@@ -956,6 +1112,7 @@ class Nikola(object):
         self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
             'MATHJAX_CONFIG')
         self._GLOBAL_CONTEXT['use_katex'] = self.config.get('USE_KATEX')
+        self._GLOBAL_CONTEXT['katex_auto_render'] = self.config.get('KATEX_AUTO_RENDER')
         self._GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONFIG_SUBTHEME')
         self._GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONFIG_TRANSITION')
         self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
@@ -999,6 +1156,16 @@ class Nikola(object):
 
         self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
 
+    def _set_global_context_from_data(self):
+        """Load files from data/ and put them in the global context."""
+        self._GLOBAL_CONTEXT['data'] = {}
+        for root, dirs, files in os.walk('data', followlinks=True):
+            for fname in files:
+                fname = os.path.join(root, fname)
+                data = utils.load_data(fname)
+                key = os.path.splitext(fname.split(os.sep, 1)[1])[0]
+                self._GLOBAL_CONTEXT['data'][key] = data
+
     def _activate_plugins_of_category(self, category):
         """Activate all the plugins of a given category and return them."""
         # this code duplicated in tests/base.py
@@ -1012,11 +1179,13 @@ class Nikola(object):
     def _get_themes(self):
         if self._THEMES is None:
             try:
-                self._THEMES = utils.get_theme_chain(self.config['THEME'])
+                self._THEMES = utils.get_theme_chain(self.config['THEME'], self.themes_dirs)
             except Exception:
-                utils.LOGGER.warn('''Cannot load theme "{0}", using 'bootstrap3' instead.'''.format(self.config['THEME']))
-                self.config['THEME'] = 'bootstrap3'
-                return self._get_themes()
+                if self.config['THEME'] != 'bootstrap3':
+                    utils.LOGGER.warn('''Cannot load theme "{0}", using 'bootstrap3' instead.'''.format(self.config['THEME']))
+                    self.config['THEME'] = 'bootstrap3'
+                    return self._get_themes()
+                raise
             # Check consistency of USE_CDN and the current THEME (Issue #386)
             if self.config['USE_CDN'] and self.config['USE_CDN_WARNING']:
                 bootstrap_path = utils.get_asset_path(os.path.join(
@@ -1030,9 +1199,12 @@ class Nikola(object):
 
     def _get_messages(self):
         try:
-            return utils.load_messages(self.THEMES,
-                                       self.translations,
-                                       self.default_lang)
+            if self._MESSAGES is None:
+                self._MESSAGES = utils.load_messages(self.THEMES,
+                                                     self.translations,
+                                                     self.default_lang,
+                                                     themes_dirs=self.themes_dirs)
+            return self._MESSAGES
         except utils.LanguageNotFoundError as e:
             utils.LOGGER.error('''Cannot load language "{0}".  Please make sure it is supported by Nikola itself, or that you have the appropriate messages files in your themes.'''.format(e.lang))
             sys.exit(1)
@@ -1115,7 +1287,7 @@ class Nikola(object):
 
         return compile_html
 
-    def render_template(self, template_name, output_name, context):
+    def render_template(self, template_name, output_name, context, url_type=None):
         """Render a template with the global context.
 
         If ``output_name`` is None, will return a string and all URL
@@ -1123,6 +1295,9 @@ class Nikola(object):
         If ``output_name`` is a string, URLs will be normalized and
         the resultant HTML will be saved to the named file (path must
         start with OUTPUT_FOLDER).
+
+        The argument ``url_type`` allows to override the ``URL_TYPE``
+        configuration.
         """
         local_context = {}
         local_context["template_name"] = template_name
@@ -1159,22 +1334,22 @@ class Nikola(object):
         utils.makedirs(os.path.dirname(output_name))
         parser = lxml.html.HTMLParser(remove_blank_text=True)
         doc = lxml.html.document_fromstring(data, parser)
-        self.rewrite_links(doc, src, context['lang'])
+        self.rewrite_links(doc, src, context['lang'], url_type)
         data = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True)
         with open(output_name, "wb+") as post_file:
             post_file.write(data)
 
-    def rewrite_links(self, doc, src, lang):
+    def rewrite_links(self, doc, src, lang, url_type=None):
         """Replace links in document to point to the right places."""
         # First let lxml replace most of them
-        doc.rewrite_links(lambda dst: self.url_replacer(src, dst, lang), resolve_base_href=False)
+        doc.rewrite_links(lambda dst: self.url_replacer(src, dst, lang, url_type), resolve_base_href=False)
 
         # lxml ignores srcset in img and source elements, so do that by hand
         objs = list(doc.xpath('(*//img|*//source)'))
         for obj in objs:
             if 'srcset' in obj.attrib:
                 urls = [u.strip() for u in obj.attrib['srcset'].split(',')]
-                urls = [self.url_replacer(src, dst, lang) for dst in urls]
+                urls = [self.url_replacer(src, dst, lang, url_type) for dst in urls]
                 obj.set('srcset', ', '.join(urls))
 
     def url_replacer(self, src, dst, lang=None, url_type=None):
@@ -1277,7 +1452,7 @@ class Nikola(object):
         # Now i is the longest common prefix
         result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
 
-        if not result:
+        if not result and not parsed_dst.fragment:
             result = "."
 
         # Don't forget the query part of the link
@@ -1292,6 +1467,83 @@ class Nikola(object):
 
         return result
 
+    def _make_renderfunc(self, t_data, fname=None):
+        """Return a function that can be registered as a template shortcode.
+
+        The returned function has access to the passed template data and
+        accepts any number of positional and keyword arguments. Positional
+        arguments values are added as a tuple under the key ``_args`` to the
+        keyword argument dict and then the latter provides the template
+        context.
+
+        """
+        def render_shortcode(*args, **kw):
+            context = self.GLOBAL_CONTEXT.copy()
+            context.update(kw)
+            context['_args'] = args
+            context['lang'] = utils.LocaleBorg().current_lang
+            for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
+                context[k] = context[k](context['lang'])
+            output = self.template_system.render_template_to_string(t_data, context)
+            if fname is not None:
+                dependencies = [fname] + self.template_system.get_deps(fname)
+            else:
+                dependencies = []
+            return output, dependencies
+        return render_shortcode
+
+    def _register_templated_shortcodes(self):
+        """Register shortcodes based on templates.
+
+        This will register a shortcode for any template found in shortcodes/
+        folders and a generic "template" shortcode which will consider the
+        content in the shortcode as a template in itself.
+        """
+        self.register_shortcode('template', self._template_shortcode_handler)
+
+        builtin_sc_dir = resource_filename(
+            'nikola',
+            os.path.join('data', 'shortcodes', utils.get_template_engine(self.THEMES)))
+
+        for sc_dir in [builtin_sc_dir, 'shortcodes']:
+            if not os.path.isdir(sc_dir):
+                continue
+
+            for fname in os.listdir(sc_dir):
+                name, ext = os.path.splitext(fname)
+
+                if ext != '.tmpl':
+                    continue
+                with open(os.path.join(sc_dir, fname)) as fd:
+                    self.register_shortcode(name, self._make_renderfunc(
+                        fd.read(), os.path.join(sc_dir, fname)))
+
+    def _template_shortcode_handler(self, *args, **kw):
+        t_data = kw.pop('data', '')
+        context = self.GLOBAL_CONTEXT.copy()
+        context.update(kw)
+        context['_args'] = args
+        context['lang'] = utils.LocaleBorg().current_lang
+        for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
+            context[k] = context[k](context['lang'])
+        output = self.template_system.render_template_to_string(t_data, context)
+        dependencies = self.template_system.get_string_deps(t_data)
+        return output, dependencies
+
+    def register_shortcode(self, name, f):
+        """Register function f to handle shortcode "name"."""
+        if name in self.shortcode_registry:
+            utils.LOGGER.warn('Shortcode name conflict: {}', name)
+            return
+        self.shortcode_registry[name] = f
+
+    # XXX in v8, get rid of with_dependencies
+    def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False, extra_context={}):
+        """Apply shortcodes from the registry on data."""
+        if lang is None:
+            lang = utils.LocaleBorg().current_lang
+        return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies, extra_context=extra_context)
+
     def generic_rss_renderer(self, lang, title, link, description, timeline, output_path,
                              rss_teasers, rss_plain, feed_length=10, feed_url=None,
                              enclosure=_enclosure, rss_links_append_query=None):
@@ -1398,8 +1650,8 @@ class Nikola(object):
         * gallery (name is the gallery name)
         * listing (name is the source code file name)
         * post_path (name is 1st element in a POSTS/PAGES tuple)
-        * slug (name is the slug of a post or story)
-        * filename (name is the source filename of a post/story, in DEFAULT_LANG, relative to conf.py)
+        * slug (name is the slug of a post or page)
+        * filename (name is the source filename of a post/page, in DEFAULT_LANG, relative to conf.py)
 
         The returned value is always a path relative to output, like
         "categories/whatever.html"
@@ -1461,7 +1713,7 @@ class Nikola(object):
 
         Example:
 
-        links://slug/yellow-camaro => /posts/cars/awful/yellow-camaro/index.html
+        link://slug/yellow-camaro => /posts/cars/awful/yellow-camaro/index.html
         """
         results = [p for p in self.timeline if p.meta('slug') == name]
         if not results:
@@ -1472,7 +1724,7 @@ class Nikola(object):
             return [_f for _f in results[0].permalink(lang).split('/') if _f]
 
     def filename_path(self, name, lang):
-        """Link to post or story by source filename.
+        """Link to post or page by source filename.
 
         Example:
 
@@ -1660,6 +1912,7 @@ class Nikola(object):
         self.tags_per_language = defaultdict(list)
         self.category_hierarchy = {}
         self.post_per_file = {}
+        self.post_per_input_file = {}
         self.timeline = []
         self.pages = []
 
@@ -1670,27 +1923,28 @@ class Nikola(object):
 
         quit = False
         # Classify posts per year/tag/month/whatever
-        slugged_tags = set([])
+        slugged_tags = defaultdict(set)
         for post in self.timeline:
             if post.use_in_feeds:
                 self.posts.append(post)
                 self.posts_per_year[str(post.date.year)].append(post)
                 self.posts_per_month[
                     '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
-                for tag in post.alltags:
-                    _tag_slugified = utils.slugify(tag)
-                    if _tag_slugified in slugged_tags:
-                        if tag not in self.posts_per_tag:
-                            # Tags that differ only in case
-                            other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0]
-                            utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
-                            utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
-                            utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]])))
-                            quit = True
-                    else:
-                        slugged_tags.add(utils.slugify(tag))
-                    self.posts_per_tag[tag].append(post)
                 for lang in self.config['TRANSLATIONS'].keys():
+                    for tag in post.tags_for_language(lang):
+                        _tag_slugified = utils.slugify(tag, lang)
+                        if _tag_slugified in slugged_tags[lang]:
+                            if tag not in self.posts_per_tag:
+                                # Tags that differ only in case
+                                other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing, lang) == _tag_slugified][0]
+                                utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag))
+                                utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
+                                utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]])))
+                                quit = True
+                        else:
+                            slugged_tags[lang].add(_tag_slugified)
+                        if post not in self.posts_per_tag[tag]:
+                            self.posts_per_tag[tag].append(post)
                     self.tags_per_language[lang].extend(post.tags_for_language(lang))
                 self._add_post_to_category(post, post.meta('category'))
 
@@ -1703,6 +1957,7 @@ class Nikola(object):
             for lang in self.config['TRANSLATIONS'].keys():
                 dest = post.destination_path(lang=lang)
                 src_dest = post.destination_path(lang=lang, extension=post.source_ext())
+                src_file = post.translated_source_path(lang=lang)
                 if dest in self.post_per_file:
                     utils.LOGGER.error('Two posts are trying to generate {0}: {1} and {2}'.format(
                         dest,
@@ -1717,13 +1972,16 @@ class Nikola(object):
                     quit = True
                 self.post_per_file[dest] = post
                 self.post_per_file[src_dest] = post
+                self.post_per_input_file[src_file] = post
                 # deduplicate tags_per_language
                 self.tags_per_language[lang] = list(set(self.tags_per_language[lang]))
 
         # Sort everything.
 
         for thing in self.timeline, self.posts, self.all_posts, self.pages:
-            thing.sort(key=lambda p: (p.date, p.source_path))
+            thing.sort(key=lambda p:
+                       (int(p.meta('priority')) if p.meta('priority') else 0,
+                        p.date, p.source_path))
             thing.reverse()
         self._sort_category_hierarchy()
 
@@ -1738,38 +1996,38 @@ class Nikola(object):
             sys.exit(1)
         signal('scanned').send(self)
 
-    def generic_page_renderer(self, lang, post, filters, context=None):
-        """Render post fragments to final HTML pages."""
+    def generic_renderer(self, lang, output_name, template_name, filters, file_deps=None, uptodate_deps=None, context=None, context_deps_remove=None, post_deps_dict=None, url_type=None):
+        """Helper function for rendering pages and post lists and other related pages.
+
+        lang is the current language.
+        output_name is the destination file name.
+        template_name is the template to be used.
+        filters is the list of filters (usually site.config['FILTERS']) which will be used to post-process the result.
+        file_deps (optional) is a list of additional file dependencies (next to template and its dependencies).
+        uptodate_deps (optional) is a list of additional entries added to the task's uptodate list.
+        context (optional) a dict used as a basis for the template context. The lang parameter will always be added.
+        context_deps_remove (optional) is a list of keys to remove from the context after using it as an uptodate dependency. This should name all keys containing non-trivial Python objects; they can be replaced by adding JSON-style dicts in post_deps_dict.
+        post_deps_dict (optional) is a dict merged into the copy of context which is used as an uptodate dependency.
+        url_type (optional) allows to override the ``URL_TYPE`` configuration
+        """
         utils.LocaleBorg().set_locale(lang)
-        context = context.copy() if context else {}
-        deps = post.deps(lang) + \
-            self.template_system.template_deps(post.template_name)
-        deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine'))
-        deps = list(filter(None, deps))
-        context['post'] = post
-        context['lang'] = lang
-        context['title'] = post.title(lang)
-        context['description'] = post.description(lang)
-        context['permalink'] = post.permalink(lang)
-        if 'pagekind' not in context:
-            context['pagekind'] = ['generic_page']
-        if post.use_in_feeds:
-            context['enable_comments'] = True
-        else:
-            context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
-        extension = self.get_compiler(post.source_path).extension()
-        output_name = os.path.join(self.config['OUTPUT_FOLDER'],
-                                   post.destination_path(lang, extension))
+
+        file_deps = copy(file_deps) if file_deps else []
+        file_deps += self.template_system.template_deps(template_name)
+        file_deps = sorted(list(filter(None, file_deps)))
+
+        context = copy(context) if context else {}
+        context["lang"] = lang
+
         deps_dict = copy(context)
-        deps_dict.pop('post')
-        if post.prev_post:
-            deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
-        if post.next_post:
-            deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
+        if context_deps_remove:
+            for key in context_deps_remove:
+                deps_dict.pop(key)
         deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
         deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
         deps_dict['global'] = self.GLOBAL_CONTEXT
-        deps_dict['comments'] = context['enable_comments']
+        if post_deps_dict:
+            deps_dict.update(post_deps_dict)
 
         for k, v in self.GLOBAL_CONTEXT['template_hooks'].items():
             deps_dict['||template_hooks|{0}||'.format(k)] = v._items
@@ -1779,62 +2037,81 @@ class Nikola(object):
 
         deps_dict['navigation_links'] = deps_dict['global']['navigation_links'](lang)
 
-        if post:
-            deps_dict['post_translations'] = post.translated_to
-
         task = {
             'name': os.path.normpath(output_name),
-            'file_dep': sorted(deps),
             'targets': [output_name],
-            'actions': [(self.render_template, [post.template_name,
-                                                output_name, context])],
+            'file_dep': file_deps,
+            'actions': [(self.render_template, [template_name, output_name,
+                                                context, url_type])],
             'clean': True,
-            'uptodate': [config_changed(deps_dict, 'nikola.nikola.Nikola.generic_page_renderer')] + post.deps_uptodate(lang),
+            'uptodate': [config_changed(deps_dict, 'nikola.nikola.Nikola.generic_renderer')] + ([] if uptodate_deps is None else uptodate_deps)
         }
 
-        yield utils.apply_filters(task, filters)
+        return utils.apply_filters(task, filters)
 
-    def generic_post_list_renderer(self, lang, posts, output_name,
-                                   template_name, filters, extra_context):
+    def generic_page_renderer(self, lang, post, filters, context=None):
+        """Render post fragments to final HTML pages."""
+        extension = self.get_compiler(post.source_path).extension()
+        output_name = os.path.join(self.config['OUTPUT_FOLDER'],
+                                   post.destination_path(lang, extension))
+
+        deps = post.deps(lang)
+        uptodate_deps = post.deps_uptodate(lang)
+        deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine'))
+
+        context = copy(context) if context else {}
+        context['post'] = post
+        context['title'] = post.title(lang)
+        context['description'] = post.description(lang)
+        context['permalink'] = post.permalink(lang)
+        if 'pagekind' not in context:
+            context['pagekind'] = ['generic_page']
+        if post.use_in_feeds:
+            context['enable_comments'] = True
+        else:
+            context['enable_comments'] = self.config['COMMENTS_IN_PAGES']
+
+        deps_dict = {}
+        if post.prev_post:
+            deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
+        if post.next_post:
+            deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
+        deps_dict['comments'] = context['enable_comments']
+        if post:
+            deps_dict['post_translations'] = post.translated_to
+
+        yield self.generic_renderer(lang, output_name, post.template_name, filters,
+                                    file_deps=deps,
+                                    uptodate_deps=uptodate_deps,
+                                    context=context,
+                                    context_deps_remove=['post'],
+                                    post_deps_dict=deps_dict)
+
+    def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
         """Render pages with lists of posts."""
         deps = []
-        deps += self.template_system.template_deps(template_name)
         uptodate_deps = []
         for post in posts:
             deps += post.deps(lang)
             uptodate_deps += post.deps_uptodate(lang)
+
         context = {}
         context["posts"] = posts
         context["title"] = self.config['BLOG_TITLE'](lang)
         context["description"] = self.config['BLOG_DESCRIPTION'](lang)
-        context["lang"] = lang
         context["prevlink"] = None
         context["nextlink"] = None
-        context.update(extra_context)
-        deps_context = copy(context)
-        deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
-                                 posts]
-        deps_context["global"] = self.GLOBAL_CONTEXT
-
-        for k, v in self.GLOBAL_CONTEXT['template_hooks'].items():
-            deps_context['||template_hooks|{0}||'.format(k)] = v._items
-
-        for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
-            deps_context[k] = deps_context['global'][k](lang)
-
-        deps_context['navigation_links'] = deps_context['global']['navigation_links'](lang)
+        if extra_context:
+            context.update(extra_context)
 
-        task = {
-            'name': os.path.normpath(output_name),
-            'targets': [output_name],
-            'file_dep': sorted(deps),
-            'actions': [(self.render_template, [template_name, output_name,
-                                                context])],
-            'clean': True,
-            'uptodate': [config_changed(deps_context, 'nikola.nikola.Nikola.generic_post_list_renderer')] + uptodate_deps
-        }
+        post_deps_dict = {}
+        post_deps_dict["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in posts]
 
-        return utils.apply_filters(task, filters)
+        return self.generic_renderer(lang, output_name, template_name, filters,
+                                     file_deps=deps,
+                                     uptodate_deps=uptodate_deps,
+                                     context=context,
+                                     post_deps_dict=post_deps_dict)
 
     def atom_feed_renderer(self, lang, posts, output_path, filters,
                            extra_context):
@@ -1991,7 +2268,7 @@ class Nikola(object):
                 entry_content.text = content
             for category in post.tags_for_language(lang):
                 entry_category = lxml.etree.SubElement(entry_root, "category")
-                entry_category.set("term", utils.slugify(category))
+                entry_category.set("term", utils.slugify(category, lang))
                 entry_category.set("label", category)
 
         dst_dir = os.path.dirname(output_path)
@@ -2137,6 +2414,7 @@ class Nikola(object):
                     "basename": basename,
                     "name": atom_output_name,
                     "file_dep": sorted([_.base_path for _ in post_list]),
+                    "task_dep": ['render_posts'],
                     "targets": [atom_output_name],
                     "actions": [(self.atom_feed_renderer,
                                 (lang,
@@ -2164,7 +2442,7 @@ class Nikola(object):
 
     def __repr__(self):
         """Representation of a Nikola site."""
-        return '<Nikola Site: {0!r}>'.format(self.config['BLOG_TITLE']())
+        return '<Nikola Site: {0!r}>'.format(self.config['BLOG_TITLE'](self.config['DEFAULT_LANG']))
 
 
 def sanitized_locales(locale_fallback, locale_default, locales, translations):
@@ -2239,6 +2517,9 @@ def sanitized_locales(locale_fallback, locale_default, locales, translations):
                 locale_n = locale_fallback
                 msg = "Could not guess locale for language {0}, using locale {1}"
                 utils.LOGGER.warn(msg.format(lang, locale_n))
+                utils.LOGGER.warn("Please fix your OS locale configuration or use the LOCALES option in conf.py to specify your preferred locale.")
+                if sys.platform != 'win32':
+                    utils.LOGGER.warn("Make sure to use an UTF-8 locale to ensure Unicode support.")
             locales[lang] = locale_n
 
     return locale_fallback, locale_default, locales