diff options
Diffstat (limited to 'nikola/nikola.py')
| -rw-r--r-- | nikola/nikola.py | 581 |
1 files changed, 431 insertions, 150 deletions
diff --git a/nikola/nikola.py b/nikola/nikola.py index 9e9b849..0a62360 100644 --- a/nikola/nikola.py +++ b/nikola/nikola.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -57,7 +57,8 @@ from yapsy.PluginManager import PluginManager from blinker import signal from .post import Post # NOQA -from . import DEBUG, utils +from .state import Persistor +from . import DEBUG, utils, shortcodes from .plugin_categories import ( Command, LateTask, @@ -65,6 +66,7 @@ from .plugin_categories import ( CompilerExtension, MarkdownExtension, RestExtension, + ShortcodePlugin, Task, TaskMultiplier, TemplateSystem, @@ -119,6 +121,8 @@ LEGAL_VALUES = { 'fa': 'Persian', 'fi': 'Finnish', 'fr': 'French', + 'gl': 'Galician', + 'he': 'Hebrew', 'hi': 'Hindi', 'hr': 'Croatian', 'hu': 'Hungarian', @@ -126,7 +130,8 @@ LEGAL_VALUES = { 'it': 'Italian', ('ja', '!jp'): 'Japanese', 'ko': 'Korean', - 'nb': 'Norwegian Bokmål', + 'lt': 'Lithuanian', + 'nb': 'Norwegian (Bokmål)', 'nl': 'Dutch', 'pa': 'Punjabi', 'pl': 'Polish', @@ -135,46 +140,75 @@ LEGAL_VALUES = { 'ru': 'Russian', 'sk': 'Slovak', 'sl': 'Slovene', + 'sq': 'Albanian', 'sr': 'Serbian (Cyrillic)', 'sr_latin': 'Serbian (Latin)', 'sv': 'Swedish', + 'te': 'Telugu', ('tr', '!tr_TR'): 'Turkish', 'ur': 'Urdu', 'uk': 'Ukrainian', 'zh_cn': 'Chinese (Simplified)', + 'zh_tw': 'Chinese (Traditional)' }, '_WINDOWS_LOCALE_GUESSES': { # TODO incomplete - # some languages may need that the appropiate Microsoft Language Pack be instaled. + # some languages may need that the appropriate Microsoft Language Pack be installed. + "ar": "Arabic", + "az": "Azeri (Latin)", "bg": "Bulgarian", + "bs": "Bosnian", "ca": "Catalan", + "cs": "Czech", + "da": "Danish", "de": "German", "el": "Greek", "en": "English", - "eo": "Esperanto", + # "eo": "Esperanto", # Not available "es": "Spanish", - "fa": "Farsi", # Persian + "et": "Estonian", + "eu": "Basque", + "fa": "Persian", # Persian + "fi": "Finnish", "fr": "French", + "gl": "Galician", + "he": "Hebrew", + "hi": "Hindi", "hr": "Croatian", "hu": "Hungarian", + "id": "Indonesian", "it": "Italian", - "jp": "Japanese", + "ja": "Japanese", + "ko": "Korean", + "nb": "Norwegian", # Not Bokmål, as Windows doesn't find it for unknown reasons. "nl": "Dutch", + "pa": "Punjabi", "pl": "Polish", + "pt": "Portuguese_Portugal", "pt_br": "Portuguese_Brazil", "ru": "Russian", - "sl_si": "Slovenian", - "tr_tr": "Turkish", + "sk": "Slovak", + "sl": "Slovenian", + "sq": "Albanian", + "sr": "Serbian", + "sr_latin": "Serbian (Latin)", + "sv": "Swedish", + "te": "Telugu", + "tr": "Turkish", + "uk": "Ukrainian", + "ur": "Urdu", "zh_cn": "Chinese_China", # Chinese (Simplified) + "zh_tw": "Chinese_Taiwan", # Chinese (Traditional) }, '_TRANSLATIONS_WITH_COUNTRY_SPECIFIERS': { # This dict is used in `init` in case of locales that exist with a # country specifier. If there is no other locale that has the same # language with a different country, ``nikola init`` (but nobody else!) # will accept it, warning the user about it. - 'zh': 'zh_cn', + + # This dict is currently empty. }, - 'RTL_LANGUAGES': ('ar', 'fa', 'ur'), + 'RTL_LANGUAGES': ('ar', 'fa', 'he', 'ur'), 'COLORBOX_LOCALES': defaultdict( str, ar='ar', @@ -190,12 +224,14 @@ LEGAL_VALUES = { fa='fa', fi='fi', fr='fr', + he='he', hr='hr', hu='hu', id='id', it='it', ja='ja', ko='kr', # kr is South Korea, ko is the Korean language + lt='lt', nb='no', nl='nl', pl='pl', @@ -209,7 +245,8 @@ LEGAL_VALUES = { sv='sv', tr='tr', uk='uk', - zh_cn='zh-CN' + zh_cn='zh-CN', + zh_tw='zh-TW' ), 'MOMENTJS_LOCALES': defaultdict( str, @@ -232,13 +269,16 @@ LEGAL_VALUES = { fa='fa', fi='fi', fr='fr', + gl='gl', hi='hi', + he='he', hr='hr', hu='hu', id='id', it='it', ja='ja', ko='ko', + lt='lt', nb='nb', nl='nl', pl='pl', @@ -247,12 +287,14 @@ LEGAL_VALUES = { ru='ru', sk='sk', sl='sl', + sq='sq', sr='sr-cyrl', sr_latin='sr', sv='sv', tr='tr', uk='uk', - zh_cn='zh-cn' + zh_cn='zh-cn', + zh_tw='zh-tw' ), 'PYPHEN_LOCALES': { 'bg': 'bg', @@ -262,13 +304,14 @@ LEGAL_VALUES = { 'da': 'da', 'de': 'de', 'el': 'el', - 'en': 'en', + 'en': 'en_US', 'es': 'es', 'et': 'et', 'fr': 'fr', 'hr': 'hr', 'hu': 'hu', 'it': 'it', + 'lt': 'lt', 'nb': 'nb', 'nl': 'nl', 'pl': 'pl', @@ -279,8 +322,32 @@ LEGAL_VALUES = { 'sl': 'sl', 'sr': 'sr', 'sv': 'sv', + 'te': 'te', 'uk': 'uk', }, + 'DOCUTILS_LOCALES': { + 'ca': 'ca', + 'da': 'da', + 'de': 'de', + 'en': 'en', + 'eo': 'eo', + 'es': 'es', + 'fi': 'fi', + 'fr': 'fr', + 'gl': 'gl', + 'he': 'he', + 'it': 'it', + 'ja': 'ja', + 'lt': 'lt', + 'pl': 'pl', + 'pt': 'pt_br', # hope nobody will mind + 'pt_br': 'pt_br', + 'ru': 'ru', + 'sk': 'sk', + 'sv': 'sv', + 'zh_cn': 'zh_cn', + 'zh_tw': 'zh_tw' + } } @@ -288,7 +355,13 @@ def _enclosure(post, lang): """Add an enclosure to RSS.""" enclosure = post.meta('enclosure', lang) if enclosure: - length = 0 + try: + length = int(post.meta('enclosure_length', lang) or 0) + except KeyError: + length = 0 + except ValueError: + utils.LOGGER.warn("Invalid enclosure length for post {0}".format(post.source_path)) + length = 0 url = enclosure mime = mimetypes.guess_type(url)[0] return url, length, mime @@ -324,6 +397,7 @@ class Nikola(object): self._scanned = False self._template_system = None self._THEMES = None + self._MESSAGES = None self.debug = DEBUG self.loghandlers = utils.STDERR_HANDLER # TODO remove on v8 self.colorful = config.pop('__colorful__', False) @@ -334,6 +408,7 @@ class Nikola(object): self.configuration_filename = config.pop('__configuration_filename__', False) self.configured = bool(config) self.injected_deps = defaultdict(list) + self.shortcode_registry = {} self.rst_transforms = [] self.template_hooks = { @@ -373,7 +448,7 @@ class Nikola(object): 'CODE_COLOR_SCHEME': 'default', 'COMMENT_SYSTEM': 'disqus', 'COMMENTS_IN_GALLERIES': False, - 'COMMENTS_IN_STORIES': False, + 'COMMENTS_IN_PAGES': False, 'COMPILERS': { "rest": ('.txt', '.rst'), "markdown": ('.md', '.mdown', '.markdown'), @@ -398,8 +473,10 @@ class Nikola(object): 'DEPLOY_COMMANDS': {'default': []}, 'DISABLED_PLUGINS': [], 'EXTRA_PLUGINS_DIRS': [], + 'EXTRA_THEMES_DIRS': [], 'COMMENT_SYSTEM_ID': 'nikolademo', 'ENABLE_AUTHOR_PAGES': True, + 'EXIF_WHITELIST': {}, 'EXTRA_HEAD_DATA': '', 'FAVICONS': (), 'FEED_LENGTH': 10, @@ -408,6 +485,7 @@ class Nikola(object): 'FILES_FOLDERS': {'files': ''}, 'FILTERS': {}, 'FORCE_ISO8601': False, + 'FRONT_INDEX_HEADER': '', 'GALLERY_FOLDERS': {'galleries': 'galleries'}, 'GALLERY_SORT_BY_DATE': True, 'GLOBAL_CONTEXT_FILLER': [], @@ -430,6 +508,7 @@ class Nikola(object): 'INDEXES_STATIC': True, 'INDEX_PATH': '', 'IPYNB_CONFIG': {}, + 'KATEX_AUTO_RENDER': '', 'LESS_COMPILER': 'lessc', 'LESS_OPTIONS': [], 'LICENSE': '', @@ -444,11 +523,14 @@ class Nikola(object): 'OUTPUT_FOLDER': 'output', 'POSTS': (("posts/*.txt", "posts", "post.tmpl"),), 'POSTS_SECTIONS': True, + 'POSTS_SECTION_COLORS': {}, 'POSTS_SECTION_ARE_INDEXES': True, 'POSTS_SECTION_DESCRIPTIONS': "", 'POSTS_SECTION_FROM_META': False, 'POSTS_SECTION_NAME': "", 'POSTS_SECTION_TITLE': "{name}", + 'PRESERVE_EXIF_DATA': False, + # TODO: change in v8 'PAGES': (("stories/*.txt", "stories", "story.tmpl"),), 'PANDOC_OPTIONS': [], 'PRETTY_URLS': False, @@ -475,7 +557,7 @@ class Nikola(object): 'SLUG_TAG_PATH': True, 'SOCIAL_BUTTONS_CODE': '', 'SITE_URL': 'https://example.com/', - 'STORY_INDEX': False, + 'PAGE_INDEX': False, 'STRIP_INDEXES': False, 'SITEMAP_INCLUDE_FILELESS_DIRS': True, 'TAG_PATH': 'categories', @@ -492,7 +574,7 @@ class Nikola(object): 'THUMBNAIL_SIZE': 180, 'UNSLUGIFY_TITLES': False, # WARNING: conf.py.in overrides this with True for backwards compatibility 'URL_TYPE': 'rel_path', - 'USE_BASE_TAG': True, + 'USE_BASE_TAG': False, 'USE_BUNDLES': True, 'USE_CDN': False, 'USE_CDN_WARNING': True, @@ -511,6 +593,7 @@ class Nikola(object): 'GITHUB_SOURCE_BRANCH': 'master', 'GITHUB_DEPLOY_BRANCH': 'gh-pages', 'GITHUB_REMOTE_NAME': 'origin', + 'GITHUB_COMMIT_SOURCE': False, # WARNING: conf.py.in overrides this with True for backwards compatibility } # set global_context for template rendering @@ -548,6 +631,7 @@ class Nikola(object): 'BODY_END', 'EXTRA_HEAD_DATA', 'NAVIGATION_LINKS', + 'FRONT_INDEX_HEADER', 'INDEX_READ_MORE_LINK', 'FEED_READ_MORE_LINK', 'INDEXES_TITLE', @@ -576,7 +660,13 @@ class Nikola(object): 'body_end', 'extra_head_data', 'date_format', - 'js_date_format',) + 'js_date_format', + 'posts_section_colors', + 'posts_section_descriptions', + 'posts_section_name', + 'posts_section_title', + 'front_index_header', + ) # WARNING: navigation_links SHOULD NOT be added to the list above. # Themes ask for [lang] there and we should provide it. @@ -594,6 +684,15 @@ class Nikola(object): except KeyError: pass + # A EXIF_WHITELIST implies you want to keep EXIF data + if self.config['EXIF_WHITELIST'] and not self.config['PRESERVE_EXIF_DATA']: + utils.LOGGER.warn('Setting EXIF_WHITELIST implies PRESERVE_EXIF_DATA is set to True') + self.config['PRESERVE_EXIF_DATA'] = True + + # Setting PRESERVE_EXIF_DATA with an empty EXIF_WHITELIST implies 'keep everything' + if self.config['PRESERVE_EXIF_DATA'] and not self.config['EXIF_WHITELIST']: + utils.LOGGER.warn('You are setting PRESERVE_EXIF_DATA and not EXIF_WHITELIST so EXIF data is not really kept.') + # Handle CONTENT_FOOTER properly. # We provide the arguments to format in CONTENT_FOOTER_FORMATS. self.config['CONTENT_FOOTER'].langformat(self.config['CONTENT_FOOTER_FORMATS']) @@ -635,7 +734,7 @@ class Nikola(object): # TODO: remove on v8 if 'RSS_LINKS_APPEND_QUERY' in config: utils.LOGGER.warn('The RSS_LINKS_APPEND_QUERY option is deprecated, use FEED_LINKS_APPEND_QUERY instead.') - if 'FEED_TEASERS' in config: + if 'FEED_LINKS_APPEND_QUERY' in config: utils.LOGGER.warn('FEED_LINKS_APPEND_QUERY conflicts with RSS_LINKS_APPEND_QUERY, ignoring RSS_LINKS_APPEND_QUERY.') self.config['FEED_LINKS_APPEND_QUERY'] = config['RSS_LINKS_APPEND_QUERY'] @@ -772,6 +871,15 @@ class Nikola(object): utils.LOGGER.warn("WRITE_TAG_CLOUD is not set in your config. Defaulting to True (== writing tag_cloud_data.json).") utils.LOGGER.warn("Please explicitly add the setting to your conf.py with the desired value, as the setting will default to False in the future.") + # Rename stories to pages (#1891, #2518) + # TODO: remove in v8 + if 'COMMENTS_IN_STORIES' in config: + utils.LOGGER.warn('The COMMENTS_IN_STORIES option is deprecated, use COMMENTS_IN_PAGES instead.') + self.config['COMMENTS_IN_PAGES'] = config['COMMENTS_IN_STORIES'] + if 'STORY_INDEX' in config: + utils.LOGGER.warn('The STORY_INDEX option is deprecated, use PAGE_INDEX instead.') + self.config['PAGE_INDEX'] = config['STORY_INDEX'] + # We use one global tzinfo object all over Nikola. try: self.tzinfo = dateutil.tz.gettz(self.config['TIMEZONE']) @@ -794,6 +902,9 @@ class Nikola(object): candidate = utils.get_translation_candidate(self.config, "f" + ext, lang) compilers[compiler].add(candidate) + # Get search path for themes + self.themes_dirs = ['themes'] + self.config['EXTRA_THEMES_DIRS'] + # Avoid redundant compilers # Remove compilers that match nothing in POSTS/PAGES # And put them in "bad compilers" @@ -807,9 +918,21 @@ class Nikola(object): else: self.bad_compilers.add(k) - self._set_global_context() + self._set_global_context_from_config() + self._set_global_context_from_data() - def init_plugins(self, commands_only=False): + # Set persistent state facility + self.state = Persistor('state_data.json') + + # Set cache facility + self.cache = Persistor(os.path.join(self.config['CACHE_FOLDER'], 'cache_data.json')) + + # Create directories for persistors only if a site exists (Issue #2334) + if self.configured: + self.state._set_site(self) + self.cache._set_site(self) + + def init_plugins(self, commands_only=False, load_all=False): """Load plugins as needed.""" self.plugin_manager = PluginManager(categories_filter={ "Command": Command, @@ -821,6 +944,7 @@ class Nikola(object): "CompilerExtension": CompilerExtension, "MarkdownExtension": MarkdownExtension, "RestExtension": RestExtension, + "ShortcodePlugin": ShortcodePlugin, "SignalHandler": SignalHandler, "ConfigPlugin": ConfigPlugin, "PostScanner": PostScanner, @@ -828,47 +952,71 @@ class Nikola(object): self.plugin_manager.getPluginLocator().setPluginInfoExtension('plugin') extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS'] if sys.version_info[0] == 3: - places = [ + self._plugin_places = [ resource_filename('nikola', 'plugins'), - os.path.join(os.getcwd(), 'plugins'), os.path.expanduser('~/.nikola/plugins'), + os.path.join(os.getcwd(), 'plugins'), ] + [path for path in extra_plugins_dirs if path] else: - places = [ + self._plugin_places = [ resource_filename('nikola', utils.sys_encode('plugins')), os.path.join(os.getcwd(), utils.sys_encode('plugins')), os.path.expanduser('~/.nikola/plugins'), ] + [utils.sys_encode(path) for path in extra_plugins_dirs if path] - self.plugin_manager.getPluginLocator().setPluginPlaces(places) + self.plugin_manager.getPluginLocator().setPluginPlaces(self._plugin_places) self.plugin_manager.locatePlugins() bad_candidates = set([]) - for p in self.plugin_manager._candidates: - if commands_only: - if p[-1].details.has_option('Nikola', 'plugincategory'): - # FIXME TemplateSystem should not be needed - if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}: + if not load_all: + for p in self.plugin_manager._candidates: + if commands_only: + if p[-1].details.has_option('Nikola', 'plugincategory'): + # FIXME TemplateSystem should not be needed + if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}: + bad_candidates.add(p) + else: + bad_candidates.add(p) + elif self.configured: # Not commands-only, and configured + # Remove compilers we don't use + if p[-1].name in self.bad_compilers: + bad_candidates.add(p) + self.disabled_compilers[p[-1].name] = p + utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) + if p[-1].name not in self.config['COMPILERS'] and \ + p[-1].details.has_option('Nikola', 'plugincategory') and p[-1].details.get('Nikola', 'PluginCategory') == 'Compiler': bad_candidates.add(p) - else: # Not commands-only - # Remove compilers we don't use - if p[-1].name in self.bad_compilers: - bad_candidates.add(p) - self.disabled_compilers[p[-1].name] = p - utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) - if p[-1].name not in self.config['COMPILERS'] and \ - p[-1].details.has_option('Nikola', 'plugincategory') and p[-1].details.get('Nikola', 'PluginCategory') == 'Compiler': - bad_candidates.add(p) - self.disabled_compilers[p[-1].name] = p - utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) - # Remove blacklisted plugins - if p[-1].name in self.config['DISABLED_PLUGINS']: - bad_candidates.add(p) - utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name) - # Remove compiler extensions we don't need - if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers: - bad_candidates.add(p) - utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name) - self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates) + self.disabled_compilers[p[-1].name] = p + utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) + # Remove blacklisted plugins + if p[-1].name in self.config['DISABLED_PLUGINS']: + bad_candidates.add(p) + utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name) + # Remove compiler extensions we don't need + if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers: + bad_candidates.add(p) + utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name) + self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates) + + # Find repeated plugins and discard the less local copy + def plugin_position_in_places(plugin): + # plugin here is a tuple: + # (path to the .plugin file, path to plugin module w/o .py, plugin metadata) + for i, place in enumerate(self._plugin_places): + if plugin[0].startswith(place): + return i + + plugin_dict = defaultdict(list) + for data in self.plugin_manager._candidates: + plugin_dict[data[2].name].append(data) + self.plugin_manager._candidates = [] + for name, plugins in plugin_dict.items(): + if len(plugins) > 1: + # Sort by locality + plugins.sort(key=plugin_position_in_places) + utils.LOGGER.debug("Plugin {} exists in multiple places, using {}".format( + plugins[-1][2].name, plugins[-1][0])) + self.plugin_manager._candidates.append(plugins[-1]) + self.plugin_manager.loadPlugins() self._activate_plugins_of_category("SignalHandler") @@ -895,6 +1043,9 @@ class Nikola(object): self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) + # Activate shortcode plugins + self._activate_plugins_of_category("ShortcodePlugin") + # Load compiler plugins self.compilers = {} self.inverse_compilers = {} @@ -905,11 +1056,15 @@ class Nikola(object): plugin_info.plugin_object self._activate_plugins_of_category("ConfigPlugin") - + self._register_templated_shortcodes() signal('configured').send(self) - def _set_global_context(self): - """Create global context from configuration.""" + def _set_global_context_from_config(self): + """Create global context from configuration. + + These are options that are used by templates, so they always need to be + available. + """ self._GLOBAL_CONTEXT['url_type'] = self.config['URL_TYPE'] self._GLOBAL_CONTEXT['timezone'] = self.tzinfo self._GLOBAL_CONTEXT['_link'] = self.link @@ -937,6 +1092,7 @@ class Nikola(object): self._GLOBAL_CONTEXT['show_blog_title'] = self.config.get('SHOW_BLOG_TITLE') self._GLOBAL_CONTEXT['logo_url'] = self.config.get('LOGO_URL') self._GLOBAL_CONTEXT['blog_description'] = self.config.get('BLOG_DESCRIPTION') + self._GLOBAL_CONTEXT['front_index_header'] = self.config.get('FRONT_INDEX_HEADER') self._GLOBAL_CONTEXT['color_hsl_adjust_hex'] = utils.color_hsl_adjust_hex self._GLOBAL_CONTEXT['colorize_str_from_base_color'] = utils.colorize_str_from_base_color @@ -956,6 +1112,7 @@ class Nikola(object): self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get( 'MATHJAX_CONFIG') self._GLOBAL_CONTEXT['use_katex'] = self.config.get('USE_KATEX') + self._GLOBAL_CONTEXT['katex_auto_render'] = self.config.get('KATEX_AUTO_RENDER') self._GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONFIG_SUBTHEME') self._GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONFIG_TRANSITION') self._GLOBAL_CONTEXT['content_footer'] = self.config.get( @@ -999,6 +1156,16 @@ class Nikola(object): self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {})) + def _set_global_context_from_data(self): + """Load files from data/ and put them in the global context.""" + self._GLOBAL_CONTEXT['data'] = {} + for root, dirs, files in os.walk('data', followlinks=True): + for fname in files: + fname = os.path.join(root, fname) + data = utils.load_data(fname) + key = os.path.splitext(fname.split(os.sep, 1)[1])[0] + self._GLOBAL_CONTEXT['data'][key] = data + def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in tests/base.py @@ -1012,11 +1179,13 @@ class Nikola(object): def _get_themes(self): if self._THEMES is None: try: - self._THEMES = utils.get_theme_chain(self.config['THEME']) + self._THEMES = utils.get_theme_chain(self.config['THEME'], self.themes_dirs) except Exception: - utils.LOGGER.warn('''Cannot load theme "{0}", using 'bootstrap3' instead.'''.format(self.config['THEME'])) - self.config['THEME'] = 'bootstrap3' - return self._get_themes() + if self.config['THEME'] != 'bootstrap3': + utils.LOGGER.warn('''Cannot load theme "{0}", using 'bootstrap3' instead.'''.format(self.config['THEME'])) + self.config['THEME'] = 'bootstrap3' + return self._get_themes() + raise # Check consistency of USE_CDN and the current THEME (Issue #386) if self.config['USE_CDN'] and self.config['USE_CDN_WARNING']: bootstrap_path = utils.get_asset_path(os.path.join( @@ -1030,9 +1199,12 @@ class Nikola(object): def _get_messages(self): try: - return utils.load_messages(self.THEMES, - self.translations, - self.default_lang) + if self._MESSAGES is None: + self._MESSAGES = utils.load_messages(self.THEMES, + self.translations, + self.default_lang, + themes_dirs=self.themes_dirs) + return self._MESSAGES except utils.LanguageNotFoundError as e: utils.LOGGER.error('''Cannot load language "{0}". Please make sure it is supported by Nikola itself, or that you have the appropriate messages files in your themes.'''.format(e.lang)) sys.exit(1) @@ -1115,7 +1287,7 @@ class Nikola(object): return compile_html - def render_template(self, template_name, output_name, context): + def render_template(self, template_name, output_name, context, url_type=None): """Render a template with the global context. If ``output_name`` is None, will return a string and all URL @@ -1123,6 +1295,9 @@ class Nikola(object): If ``output_name`` is a string, URLs will be normalized and the resultant HTML will be saved to the named file (path must start with OUTPUT_FOLDER). + + The argument ``url_type`` allows to override the ``URL_TYPE`` + configuration. """ local_context = {} local_context["template_name"] = template_name @@ -1159,22 +1334,22 @@ class Nikola(object): utils.makedirs(os.path.dirname(output_name)) parser = lxml.html.HTMLParser(remove_blank_text=True) doc = lxml.html.document_fromstring(data, parser) - self.rewrite_links(doc, src, context['lang']) + self.rewrite_links(doc, src, context['lang'], url_type) data = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True) with open(output_name, "wb+") as post_file: post_file.write(data) - def rewrite_links(self, doc, src, lang): + def rewrite_links(self, doc, src, lang, url_type=None): """Replace links in document to point to the right places.""" # First let lxml replace most of them - doc.rewrite_links(lambda dst: self.url_replacer(src, dst, lang), resolve_base_href=False) + doc.rewrite_links(lambda dst: self.url_replacer(src, dst, lang, url_type), resolve_base_href=False) # lxml ignores srcset in img and source elements, so do that by hand objs = list(doc.xpath('(*//img|*//source)')) for obj in objs: if 'srcset' in obj.attrib: urls = [u.strip() for u in obj.attrib['srcset'].split(',')] - urls = [self.url_replacer(src, dst, lang) for dst in urls] + urls = [self.url_replacer(src, dst, lang, url_type) for dst in urls] obj.set('srcset', ', '.join(urls)) def url_replacer(self, src, dst, lang=None, url_type=None): @@ -1277,7 +1452,7 @@ class Nikola(object): # Now i is the longest common prefix result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) - if not result: + if not result and not parsed_dst.fragment: result = "." # Don't forget the query part of the link @@ -1292,6 +1467,83 @@ class Nikola(object): return result + def _make_renderfunc(self, t_data, fname=None): + """Return a function that can be registered as a template shortcode. + + The returned function has access to the passed template data and + accepts any number of positional and keyword arguments. Positional + arguments values are added as a tuple under the key ``_args`` to the + keyword argument dict and then the latter provides the template + context. + + """ + def render_shortcode(*args, **kw): + context = self.GLOBAL_CONTEXT.copy() + context.update(kw) + context['_args'] = args + context['lang'] = utils.LocaleBorg().current_lang + for k in self._GLOBAL_CONTEXT_TRANSLATABLE: + context[k] = context[k](context['lang']) + output = self.template_system.render_template_to_string(t_data, context) + if fname is not None: + dependencies = [fname] + self.template_system.get_deps(fname) + else: + dependencies = [] + return output, dependencies + return render_shortcode + + def _register_templated_shortcodes(self): + """Register shortcodes based on templates. + + This will register a shortcode for any template found in shortcodes/ + folders and a generic "template" shortcode which will consider the + content in the shortcode as a template in itself. + """ + self.register_shortcode('template', self._template_shortcode_handler) + + builtin_sc_dir = resource_filename( + 'nikola', + os.path.join('data', 'shortcodes', utils.get_template_engine(self.THEMES))) + + for sc_dir in [builtin_sc_dir, 'shortcodes']: + if not os.path.isdir(sc_dir): + continue + + for fname in os.listdir(sc_dir): + name, ext = os.path.splitext(fname) + + if ext != '.tmpl': + continue + with open(os.path.join(sc_dir, fname)) as fd: + self.register_shortcode(name, self._make_renderfunc( + fd.read(), os.path.join(sc_dir, fname))) + + def _template_shortcode_handler(self, *args, **kw): + t_data = kw.pop('data', '') + context = self.GLOBAL_CONTEXT.copy() + context.update(kw) + context['_args'] = args + context['lang'] = utils.LocaleBorg().current_lang + for k in self._GLOBAL_CONTEXT_TRANSLATABLE: + context[k] = context[k](context['lang']) + output = self.template_system.render_template_to_string(t_data, context) + dependencies = self.template_system.get_string_deps(t_data) + return output, dependencies + + def register_shortcode(self, name, f): + """Register function f to handle shortcode "name".""" + if name in self.shortcode_registry: + utils.LOGGER.warn('Shortcode name conflict: {}', name) + return + self.shortcode_registry[name] = f + + # XXX in v8, get rid of with_dependencies + def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False, extra_context={}): + """Apply shortcodes from the registry on data.""" + if lang is None: + lang = utils.LocaleBorg().current_lang + return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies, extra_context=extra_context) + def generic_rss_renderer(self, lang, title, link, description, timeline, output_path, rss_teasers, rss_plain, feed_length=10, feed_url=None, enclosure=_enclosure, rss_links_append_query=None): @@ -1398,8 +1650,8 @@ class Nikola(object): * gallery (name is the gallery name) * listing (name is the source code file name) * post_path (name is 1st element in a POSTS/PAGES tuple) - * slug (name is the slug of a post or story) - * filename (name is the source filename of a post/story, in DEFAULT_LANG, relative to conf.py) + * slug (name is the slug of a post or page) + * filename (name is the source filename of a post/page, in DEFAULT_LANG, relative to conf.py) The returned value is always a path relative to output, like "categories/whatever.html" @@ -1461,7 +1713,7 @@ class Nikola(object): Example: - links://slug/yellow-camaro => /posts/cars/awful/yellow-camaro/index.html + link://slug/yellow-camaro => /posts/cars/awful/yellow-camaro/index.html """ results = [p for p in self.timeline if p.meta('slug') == name] if not results: @@ -1472,7 +1724,7 @@ class Nikola(object): return [_f for _f in results[0].permalink(lang).split('/') if _f] def filename_path(self, name, lang): - """Link to post or story by source filename. + """Link to post or page by source filename. Example: @@ -1660,6 +1912,7 @@ class Nikola(object): self.tags_per_language = defaultdict(list) self.category_hierarchy = {} self.post_per_file = {} + self.post_per_input_file = {} self.timeline = [] self.pages = [] @@ -1670,27 +1923,28 @@ class Nikola(object): quit = False # Classify posts per year/tag/month/whatever - slugged_tags = set([]) + slugged_tags = defaultdict(set) for post in self.timeline: if post.use_in_feeds: self.posts.append(post) self.posts_per_year[str(post.date.year)].append(post) self.posts_per_month[ '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post) - for tag in post.alltags: - _tag_slugified = utils.slugify(tag) - if _tag_slugified in slugged_tags: - if tag not in self.posts_per_tag: - # Tags that differ only in case - other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0] - utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag)) - utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path)) - utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]]))) - quit = True - else: - slugged_tags.add(utils.slugify(tag)) - self.posts_per_tag[tag].append(post) for lang in self.config['TRANSLATIONS'].keys(): + for tag in post.tags_for_language(lang): + _tag_slugified = utils.slugify(tag, lang) + if _tag_slugified in slugged_tags[lang]: + if tag not in self.posts_per_tag: + # Tags that differ only in case + other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing, lang) == _tag_slugified][0] + utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag)) + utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path)) + utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]]))) + quit = True + else: + slugged_tags[lang].add(_tag_slugified) + if post not in self.posts_per_tag[tag]: + self.posts_per_tag[tag].append(post) self.tags_per_language[lang].extend(post.tags_for_language(lang)) self._add_post_to_category(post, post.meta('category')) @@ -1703,6 +1957,7 @@ class Nikola(object): for lang in self.config['TRANSLATIONS'].keys(): dest = post.destination_path(lang=lang) src_dest = post.destination_path(lang=lang, extension=post.source_ext()) + src_file = post.translated_source_path(lang=lang) if dest in self.post_per_file: utils.LOGGER.error('Two posts are trying to generate {0}: {1} and {2}'.format( dest, @@ -1717,13 +1972,16 @@ class Nikola(object): quit = True self.post_per_file[dest] = post self.post_per_file[src_dest] = post + self.post_per_input_file[src_file] = post # deduplicate tags_per_language self.tags_per_language[lang] = list(set(self.tags_per_language[lang])) # Sort everything. for thing in self.timeline, self.posts, self.all_posts, self.pages: - thing.sort(key=lambda p: (p.date, p.source_path)) + thing.sort(key=lambda p: + (int(p.meta('priority')) if p.meta('priority') else 0, + p.date, p.source_path)) thing.reverse() self._sort_category_hierarchy() @@ -1738,38 +1996,38 @@ class Nikola(object): sys.exit(1) signal('scanned').send(self) - def generic_page_renderer(self, lang, post, filters, context=None): - """Render post fragments to final HTML pages.""" + def generic_renderer(self, lang, output_name, template_name, filters, file_deps=None, uptodate_deps=None, context=None, context_deps_remove=None, post_deps_dict=None, url_type=None): + """Helper function for rendering pages and post lists and other related pages. + + lang is the current language. + output_name is the destination file name. + template_name is the template to be used. + filters is the list of filters (usually site.config['FILTERS']) which will be used to post-process the result. + file_deps (optional) is a list of additional file dependencies (next to template and its dependencies). + uptodate_deps (optional) is a list of additional entries added to the task's uptodate list. + context (optional) a dict used as a basis for the template context. The lang parameter will always be added. + context_deps_remove (optional) is a list of keys to remove from the context after using it as an uptodate dependency. This should name all keys containing non-trivial Python objects; they can be replaced by adding JSON-style dicts in post_deps_dict. + post_deps_dict (optional) is a dict merged into the copy of context which is used as an uptodate dependency. + url_type (optional) allows to override the ``URL_TYPE`` configuration + """ utils.LocaleBorg().set_locale(lang) - context = context.copy() if context else {} - deps = post.deps(lang) + \ - self.template_system.template_deps(post.template_name) - deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine')) - deps = list(filter(None, deps)) - context['post'] = post - context['lang'] = lang - context['title'] = post.title(lang) - context['description'] = post.description(lang) - context['permalink'] = post.permalink(lang) - if 'pagekind' not in context: - context['pagekind'] = ['generic_page'] - if post.use_in_feeds: - context['enable_comments'] = True - else: - context['enable_comments'] = self.config['COMMENTS_IN_STORIES'] - extension = self.get_compiler(post.source_path).extension() - output_name = os.path.join(self.config['OUTPUT_FOLDER'], - post.destination_path(lang, extension)) + + file_deps = copy(file_deps) if file_deps else [] + file_deps += self.template_system.template_deps(template_name) + file_deps = sorted(list(filter(None, file_deps))) + + context = copy(context) if context else {} + context["lang"] = lang + deps_dict = copy(context) - deps_dict.pop('post') - if post.prev_post: - deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)] - if post.next_post: - deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)] + if context_deps_remove: + for key in context_deps_remove: + deps_dict.pop(key) deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER'] deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS'] deps_dict['global'] = self.GLOBAL_CONTEXT - deps_dict['comments'] = context['enable_comments'] + if post_deps_dict: + deps_dict.update(post_deps_dict) for k, v in self.GLOBAL_CONTEXT['template_hooks'].items(): deps_dict['||template_hooks|{0}||'.format(k)] = v._items @@ -1779,62 +2037,81 @@ class Nikola(object): deps_dict['navigation_links'] = deps_dict['global']['navigation_links'](lang) - if post: - deps_dict['post_translations'] = post.translated_to - task = { 'name': os.path.normpath(output_name), - 'file_dep': sorted(deps), 'targets': [output_name], - 'actions': [(self.render_template, [post.template_name, - output_name, context])], + 'file_dep': file_deps, + 'actions': [(self.render_template, [template_name, output_name, + context, url_type])], 'clean': True, - 'uptodate': [config_changed(deps_dict, 'nikola.nikola.Nikola.generic_page_renderer')] + post.deps_uptodate(lang), + 'uptodate': [config_changed(deps_dict, 'nikola.nikola.Nikola.generic_renderer')] + ([] if uptodate_deps is None else uptodate_deps) } - yield utils.apply_filters(task, filters) + return utils.apply_filters(task, filters) - def generic_post_list_renderer(self, lang, posts, output_name, - template_name, filters, extra_context): + def generic_page_renderer(self, lang, post, filters, context=None): + """Render post fragments to final HTML pages.""" + extension = self.get_compiler(post.source_path).extension() + output_name = os.path.join(self.config['OUTPUT_FOLDER'], + post.destination_path(lang, extension)) + + deps = post.deps(lang) + uptodate_deps = post.deps_uptodate(lang) + deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine')) + + context = copy(context) if context else {} + context['post'] = post + context['title'] = post.title(lang) + context['description'] = post.description(lang) + context['permalink'] = post.permalink(lang) + if 'pagekind' not in context: + context['pagekind'] = ['generic_page'] + if post.use_in_feeds: + context['enable_comments'] = True + else: + context['enable_comments'] = self.config['COMMENTS_IN_PAGES'] + + deps_dict = {} + if post.prev_post: + deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)] + if post.next_post: + deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)] + deps_dict['comments'] = context['enable_comments'] + if post: + deps_dict['post_translations'] = post.translated_to + + yield self.generic_renderer(lang, output_name, post.template_name, filters, + file_deps=deps, + uptodate_deps=uptodate_deps, + context=context, + context_deps_remove=['post'], + post_deps_dict=deps_dict) + + def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Render pages with lists of posts.""" deps = [] - deps += self.template_system.template_deps(template_name) uptodate_deps = [] for post in posts: deps += post.deps(lang) uptodate_deps += post.deps_uptodate(lang) + context = {} context["posts"] = posts context["title"] = self.config['BLOG_TITLE'](lang) context["description"] = self.config['BLOG_DESCRIPTION'](lang) - context["lang"] = lang context["prevlink"] = None context["nextlink"] = None - context.update(extra_context) - deps_context = copy(context) - deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in - posts] - deps_context["global"] = self.GLOBAL_CONTEXT - - for k, v in self.GLOBAL_CONTEXT['template_hooks'].items(): - deps_context['||template_hooks|{0}||'.format(k)] = v._items - - for k in self._GLOBAL_CONTEXT_TRANSLATABLE: - deps_context[k] = deps_context['global'][k](lang) - - deps_context['navigation_links'] = deps_context['global']['navigation_links'](lang) + if extra_context: + context.update(extra_context) - task = { - 'name': os.path.normpath(output_name), - 'targets': [output_name], - 'file_dep': sorted(deps), - 'actions': [(self.render_template, [template_name, output_name, - context])], - 'clean': True, - 'uptodate': [config_changed(deps_context, 'nikola.nikola.Nikola.generic_post_list_renderer')] + uptodate_deps - } + post_deps_dict = {} + post_deps_dict["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in posts] - return utils.apply_filters(task, filters) + return self.generic_renderer(lang, output_name, template_name, filters, + file_deps=deps, + uptodate_deps=uptodate_deps, + context=context, + post_deps_dict=post_deps_dict) def atom_feed_renderer(self, lang, posts, output_path, filters, extra_context): @@ -1991,7 +2268,7 @@ class Nikola(object): entry_content.text = content for category in post.tags_for_language(lang): entry_category = lxml.etree.SubElement(entry_root, "category") - entry_category.set("term", utils.slugify(category)) + entry_category.set("term", utils.slugify(category, lang)) entry_category.set("label", category) dst_dir = os.path.dirname(output_path) @@ -2137,6 +2414,7 @@ class Nikola(object): "basename": basename, "name": atom_output_name, "file_dep": sorted([_.base_path for _ in post_list]), + "task_dep": ['render_posts'], "targets": [atom_output_name], "actions": [(self.atom_feed_renderer, (lang, @@ -2164,7 +2442,7 @@ class Nikola(object): def __repr__(self): """Representation of a Nikola site.""" - return '<Nikola Site: {0!r}>'.format(self.config['BLOG_TITLE']()) + return '<Nikola Site: {0!r}>'.format(self.config['BLOG_TITLE'](self.config['DEFAULT_LANG'])) def sanitized_locales(locale_fallback, locale_default, locales, translations): @@ -2239,6 +2517,9 @@ def sanitized_locales(locale_fallback, locale_default, locales, translations): locale_n = locale_fallback msg = "Could not guess locale for language {0}, using locale {1}" utils.LOGGER.warn(msg.format(lang, locale_n)) + utils.LOGGER.warn("Please fix your OS locale configuration or use the LOCALES option in conf.py to specify your preferred locale.") + if sys.platform != 'win32': + utils.LOGGER.warn("Make sure to use an UTF-8 locale to ensure Unicode support.") locales[lang] = locale_n return locale_fallback, locale_default, locales |