diff options
Diffstat (limited to 'nikola/nikola.py')
| -rw-r--r-- | nikola/nikola.py | 838 |
1 files changed, 637 insertions, 201 deletions
diff --git a/nikola/nikola.py b/nikola/nikola.py index e0af7ad..0a62360 100644 --- a/nikola/nikola.py +++ b/nikola/nikola.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -57,7 +57,8 @@ from yapsy.PluginManager import PluginManager from blinker import signal from .post import Post # NOQA -from . import DEBUG, utils +from .state import Persistor +from . import DEBUG, utils, shortcodes from .plugin_categories import ( Command, LateTask, @@ -65,6 +66,7 @@ from .plugin_categories import ( CompilerExtension, MarkdownExtension, RestExtension, + ShortcodePlugin, Task, TaskMultiplier, TemplateSystem, @@ -80,7 +82,7 @@ else: # Default "Read more..." link DEFAULT_INDEX_READ_MORE_LINK = '<p class="more"><a href="{link}">{read_more}…</a></p>' -DEFAULT_RSS_READ_MORE_LINK = '<p><a href="{link}">{read_more}…</a> ({min_remaining_read})</p>' +DEFAULT_FEED_READ_MORE_LINK = '<p><a href="{link}">{read_more}…</a> ({min_remaining_read})</p>' # Default pattern for translation files' names DEFAULT_TRANSLATIONS_PATTERN = '{path}.{lang}.{ext}' @@ -105,6 +107,7 @@ LEGAL_VALUES = { 'ar': 'Arabic', 'az': 'Azerbaijani', 'bg': 'Bulgarian', + 'bs': 'Bosnian', 'ca': 'Catalan', ('cs', 'cz'): 'Czech', 'da': 'Danish', @@ -118,59 +121,94 @@ LEGAL_VALUES = { 'fa': 'Persian', 'fi': 'Finnish', 'fr': 'French', + 'gl': 'Galician', + 'he': 'Hebrew', 'hi': 'Hindi', 'hr': 'Croatian', + 'hu': 'Hungarian', 'id': 'Indonesian', 'it': 'Italian', ('ja', '!jp'): 'Japanese', 'ko': 'Korean', - 'nb': 'Norwegian Bokmål', + 'lt': 'Lithuanian', + 'nb': 'Norwegian (Bokmål)', 'nl': 'Dutch', 'pa': 'Punjabi', 'pl': 'Polish', - 'pt_br': 'Portuguese (Brasil)', + 'pt': 'Portuguese', + 'pt_br': 'Portuguese (Brazil)', 'ru': 'Russian', 'sk': 'Slovak', 'sl': 'Slovene', + 'sq': 'Albanian', 'sr': 'Serbian (Cyrillic)', + 'sr_latin': 'Serbian (Latin)', 'sv': 'Swedish', + 'te': 'Telugu', ('tr', '!tr_TR'): 'Turkish', 'ur': 'Urdu', 'uk': 'Ukrainian', 'zh_cn': 'Chinese (Simplified)', + 'zh_tw': 'Chinese (Traditional)' }, '_WINDOWS_LOCALE_GUESSES': { # TODO incomplete - # some languages may need that the appropiate Microsoft Language Pack be instaled. + # some languages may need that the appropriate Microsoft Language Pack be installed. + "ar": "Arabic", + "az": "Azeri (Latin)", "bg": "Bulgarian", + "bs": "Bosnian", "ca": "Catalan", + "cs": "Czech", + "da": "Danish", "de": "German", "el": "Greek", "en": "English", - "eo": "Esperanto", + # "eo": "Esperanto", # Not available "es": "Spanish", - "fa": "Farsi", # Persian + "et": "Estonian", + "eu": "Basque", + "fa": "Persian", # Persian + "fi": "Finnish", "fr": "French", + "gl": "Galician", + "he": "Hebrew", + "hi": "Hindi", "hr": "Croatian", + "hu": "Hungarian", + "id": "Indonesian", "it": "Italian", - "jp": "Japanese", + "ja": "Japanese", + "ko": "Korean", + "nb": "Norwegian", # Not Bokmål, as Windows doesn't find it for unknown reasons. "nl": "Dutch", + "pa": "Punjabi", "pl": "Polish", + "pt": "Portuguese_Portugal", "pt_br": "Portuguese_Brazil", "ru": "Russian", - "sl_si": "Slovenian", - "tr_tr": "Turkish", + "sk": "Slovak", + "sl": "Slovenian", + "sq": "Albanian", + "sr": "Serbian", + "sr_latin": "Serbian (Latin)", + "sv": "Swedish", + "te": "Telugu", + "tr": "Turkish", + "uk": "Ukrainian", + "ur": "Urdu", "zh_cn": "Chinese_China", # Chinese (Simplified) + "zh_tw": "Chinese_Taiwan", # Chinese (Traditional) }, '_TRANSLATIONS_WITH_COUNTRY_SPECIFIERS': { # This dict is used in `init` in case of locales that exist with a # country specifier. If there is no other locale that has the same # language with a different country, ``nikola init`` (but nobody else!) # will accept it, warning the user about it. - 'pt': 'pt_br', - 'zh': 'zh_cn', + + # This dict is currently empty. }, - 'RTL_LANGUAGES': ('ar', 'fa', 'ur'), + 'RTL_LANGUAGES': ('ar', 'fa', 'he', 'ur'), 'COLORBOX_LOCALES': defaultdict( str, ar='ar', @@ -186,56 +224,77 @@ LEGAL_VALUES = { fa='fa', fi='fi', fr='fr', + he='he', hr='hr', + hu='hu', id='id', it='it', ja='ja', ko='kr', # kr is South Korea, ko is the Korean language + lt='lt', nb='no', nl='nl', pl='pl', + pt='pt-BR', # hope nobody will mind pt_br='pt-BR', ru='ru', sk='sk', sl='si', # country code is si, language code is sl, colorbox is wrong sr='sr', # warning: this is serbian in Latin alphabet + sr_latin='sr', sv='sv', tr='tr', uk='uk', - zh_cn='zh-CN' + zh_cn='zh-CN', + zh_tw='zh-TW' ), 'MOMENTJS_LOCALES': defaultdict( str, ar='ar', + az='az', bg='bg', bn='bn', + bs='bs', ca='ca', cs='cs', cz='cs', da='da', de='de', - en='', + el='el', + en='en', + eo='eo', es='es', et='et', + eu='eu', fa='fa', fi='fi', fr='fr', + gl='gl', + hi='hi', + he='he', hr='hr', + hu='hu', id='id', it='it', ja='ja', ko='ko', + lt='lt', nb='nb', nl='nl', pl='pl', + pt='pt', pt_br='pt-br', ru='ru', sk='sk', sl='sl', + sq='sq', sr='sr-cyrl', + sr_latin='sr', sv='sv', tr='tr', - zh_cn='zh-cn' + uk='uk', + zh_cn='zh-cn', + zh_tw='zh-tw' ), 'PYPHEN_LOCALES': { 'bg': 'bg', @@ -245,22 +304,50 @@ LEGAL_VALUES = { 'da': 'da', 'de': 'de', 'el': 'el', - 'en': 'en', + 'en': 'en_US', 'es': 'es', 'et': 'et', 'fr': 'fr', 'hr': 'hr', + 'hu': 'hu', 'it': 'it', + 'lt': 'lt', 'nb': 'nb', 'nl': 'nl', 'pl': 'pl', + 'pt': 'pt', 'pt_br': 'pt_BR', 'ru': 'ru', 'sk': 'sk', 'sl': 'sl', 'sr': 'sr', 'sv': 'sv', + 'te': 'te', + 'uk': 'uk', }, + 'DOCUTILS_LOCALES': { + 'ca': 'ca', + 'da': 'da', + 'de': 'de', + 'en': 'en', + 'eo': 'eo', + 'es': 'es', + 'fi': 'fi', + 'fr': 'fr', + 'gl': 'gl', + 'he': 'he', + 'it': 'it', + 'ja': 'ja', + 'lt': 'lt', + 'pl': 'pl', + 'pt': 'pt_br', # hope nobody will mind + 'pt_br': 'pt_br', + 'ru': 'ru', + 'sk': 'sk', + 'sv': 'sv', + 'zh_cn': 'zh_cn', + 'zh_tw': 'zh_tw' + } } @@ -268,14 +355,19 @@ def _enclosure(post, lang): """Add an enclosure to RSS.""" enclosure = post.meta('enclosure', lang) if enclosure: - length = 0 + try: + length = int(post.meta('enclosure_length', lang) or 0) + except KeyError: + length = 0 + except ValueError: + utils.LOGGER.warn("Invalid enclosure length for post {0}".format(post.source_path)) + length = 0 url = enclosure mime = mimetypes.guess_type(url)[0] return url, length, mime class Nikola(object): - """Class that handles site generation. Takes a site config as argument on creation. @@ -305,6 +397,7 @@ class Nikola(object): self._scanned = False self._template_system = None self._THEMES = None + self._MESSAGES = None self.debug = DEBUG self.loghandlers = utils.STDERR_HANDLER # TODO remove on v8 self.colorful = config.pop('__colorful__', False) @@ -315,6 +408,7 @@ class Nikola(object): self.configuration_filename = config.pop('__configuration_filename__', False) self.configured = bool(config) self.injected_deps = defaultdict(list) + self.shortcode_registry = {} self.rst_transforms = [] self.template_hooks = { @@ -335,6 +429,10 @@ class Nikola(object): 'ARCHIVE_PATH': "", 'ARCHIVE_FILENAME': "archive.html", 'ARCHIVES_ARE_INDEXES': False, + 'AUTHOR_PATH': 'authors', + 'AUTHOR_PAGES_ARE_INDEXES': False, + 'AUTHOR_PAGES_DESCRIPTIONS': {}, + 'AUTHORLIST_MINIMUM_POSTS': 1, 'BLOG_AUTHOR': 'Default Author', 'BLOG_TITLE': 'Default Title', 'BLOG_DESCRIPTION': 'Default Description', @@ -343,13 +441,14 @@ class Nikola(object): 'CATEGORY_PATH': None, # None means: same as TAG_PATH 'CATEGORY_PAGES_ARE_INDEXES': None, # None means: same as TAG_PAGES_ARE_INDEXES 'CATEGORY_PAGES_DESCRIPTIONS': {}, + 'CATEGORY_PAGES_TITLES': {}, 'CATEGORY_PREFIX': 'cat_', 'CATEGORY_ALLOW_HIERARCHIES': False, 'CATEGORY_OUTPUT_FLAT_HIERARCHY': False, 'CODE_COLOR_SCHEME': 'default', 'COMMENT_SYSTEM': 'disqus', 'COMMENTS_IN_GALLERIES': False, - 'COMMENTS_IN_STORIES': False, + 'COMMENTS_IN_PAGES': False, 'COMPILERS': { "rest": ('.txt', '.rst'), "markdown": ('.md', '.mdown', '.markdown'), @@ -374,7 +473,10 @@ class Nikola(object): 'DEPLOY_COMMANDS': {'default': []}, 'DISABLED_PLUGINS': [], 'EXTRA_PLUGINS_DIRS': [], + 'EXTRA_THEMES_DIRS': [], 'COMMENT_SYSTEM_ID': 'nikolademo', + 'ENABLE_AUTHOR_PAGES': True, + 'EXIF_WHITELIST': {}, 'EXTRA_HEAD_DATA': '', 'FAVICONS': (), 'FEED_LENGTH': 10, @@ -383,12 +485,14 @@ class Nikola(object): 'FILES_FOLDERS': {'files': ''}, 'FILTERS': {}, 'FORCE_ISO8601': False, + 'FRONT_INDEX_HEADER': '', 'GALLERY_FOLDERS': {'galleries': 'galleries'}, 'GALLERY_SORT_BY_DATE': True, 'GLOBAL_CONTEXT_FILLER': [], 'GZIP_COMMAND': None, 'GZIP_FILES': False, 'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json', '.xml'), + 'HIDDEN_AUTHORS': [], 'HIDDEN_TAGS': [], 'HIDDEN_CATEGORIES': [], 'HYPHENATE': False, @@ -404,6 +508,7 @@ class Nikola(object): 'INDEXES_STATIC': True, 'INDEX_PATH': '', 'IPYNB_CONFIG': {}, + 'KATEX_AUTO_RENDER': '', 'LESS_COMPILER': 'lessc', 'LESS_OPTIONS': [], 'LICENSE': '', @@ -417,48 +522,64 @@ class Nikola(object): 'OLD_THEME_SUPPORT': True, 'OUTPUT_FOLDER': 'output', 'POSTS': (("posts/*.txt", "posts", "post.tmpl"),), + 'POSTS_SECTIONS': True, + 'POSTS_SECTION_COLORS': {}, + 'POSTS_SECTION_ARE_INDEXES': True, + 'POSTS_SECTION_DESCRIPTIONS': "", + 'POSTS_SECTION_FROM_META': False, + 'POSTS_SECTION_NAME': "", + 'POSTS_SECTION_TITLE': "{name}", + 'PRESERVE_EXIF_DATA': False, + # TODO: change in v8 'PAGES': (("stories/*.txt", "stories", "story.tmpl"),), 'PANDOC_OPTIONS': [], 'PRETTY_URLS': False, 'FUTURE_IS_NOW': False, 'INDEX_READ_MORE_LINK': DEFAULT_INDEX_READ_MORE_LINK, - 'RSS_READ_MORE_LINK': DEFAULT_RSS_READ_MORE_LINK, - 'RSS_LINKS_APPEND_QUERY': False, 'REDIRECTIONS': [], 'ROBOTS_EXCLUSIONS': [], 'GENERATE_ATOM': False, + 'FEED_TEASERS': True, + 'FEED_PLAIN': False, + 'FEED_PREVIEWIMAGE': True, + 'FEED_READ_MORE_LINK': DEFAULT_FEED_READ_MORE_LINK, + 'FEED_LINKS_APPEND_QUERY': False, 'GENERATE_RSS': True, 'RSS_LINK': None, 'RSS_PATH': '', - 'RSS_PLAIN': False, - 'RSS_TEASERS': True, 'SASS_COMPILER': 'sass', 'SASS_OPTIONS': [], 'SEARCH_FORM': '', 'SHOW_BLOG_TITLE': True, 'SHOW_SOURCELINK': True, 'SHOW_UNTRANSLATED_POSTS': True, + 'SLUG_AUTHOR_PATH': True, 'SLUG_TAG_PATH': True, 'SOCIAL_BUTTONS_CODE': '', 'SITE_URL': 'https://example.com/', - 'STORY_INDEX': False, + 'PAGE_INDEX': False, 'STRIP_INDEXES': False, 'SITEMAP_INCLUDE_FILELESS_DIRS': True, 'TAG_PATH': 'categories', 'TAG_PAGES_ARE_INDEXES': False, 'TAG_PAGES_DESCRIPTIONS': {}, + 'TAG_PAGES_TITLES': {}, + 'TAGS_INDEX_PATH': '', 'TAGLIST_MINIMUM_POSTS': 1, 'TEMPLATE_FILTERS': {}, 'THEME': 'bootstrap3', + 'THEME_COLOR': '#5670d4', # light "corporate blue" 'THEME_REVEAL_CONFIG_SUBTHEME': 'sky', 'THEME_REVEAL_CONFIG_TRANSITION': 'cube', 'THUMBNAIL_SIZE': 180, 'UNSLUGIFY_TITLES': False, # WARNING: conf.py.in overrides this with True for backwards compatibility 'URL_TYPE': 'rel_path', + 'USE_BASE_TAG': False, 'USE_BUNDLES': True, 'USE_CDN': False, 'USE_CDN_WARNING': True, 'USE_FILENAME_AS_TITLE': True, + 'USE_KATEX': False, 'USE_OPEN_GRAPH': True, 'USE_SLUGIFY': True, 'TIMEZONE': 'UTC', @@ -472,6 +593,7 @@ class Nikola(object): 'GITHUB_SOURCE_BRANCH': 'master', 'GITHUB_DEPLOY_BRANCH': 'gh-pages', 'GITHUB_REMOTE_NAME': 'origin', + 'GITHUB_COMMIT_SOURCE': False, # WARNING: conf.py.in overrides this with True for backwards compatibility } # set global_context for template rendering @@ -509,11 +631,23 @@ class Nikola(object): 'BODY_END', 'EXTRA_HEAD_DATA', 'NAVIGATION_LINKS', + 'FRONT_INDEX_HEADER', 'INDEX_READ_MORE_LINK', - 'RSS_READ_MORE_LINK', + 'FEED_READ_MORE_LINK', 'INDEXES_TITLE', + 'POSTS_SECTION_COLORS', + 'POSTS_SECTION_DESCRIPTIONS', + 'POSTS_SECTION_NAME', + 'POSTS_SECTION_TITLE', 'INDEXES_PAGES', - 'INDEXES_PRETTY_PAGE_URL',) + 'INDEXES_PRETTY_PAGE_URL', + # PATH options (Issue #1914) + 'TAG_PATH', + 'TAGS_INDEX_PATH', + 'CATEGORY_PATH', + 'DATE_FORMAT', + 'JS_DATE_FORMAT', + ) self._GLOBAL_CONTEXT_TRANSLATABLE = ('blog_author', 'blog_title', @@ -524,16 +658,41 @@ class Nikola(object): 'social_buttons_code', 'search_form', 'body_end', - 'extra_head_data',) + 'extra_head_data', + 'date_format', + 'js_date_format', + 'posts_section_colors', + 'posts_section_descriptions', + 'posts_section_name', + 'posts_section_title', + 'front_index_header', + ) # WARNING: navigation_links SHOULD NOT be added to the list above. # Themes ask for [lang] there and we should provide it. + # We first have to massage JS_DATE_FORMAT, otherwise we run into trouble + if 'JS_DATE_FORMAT' in self.config: + if isinstance(self.config['JS_DATE_FORMAT'], dict): + for k in self.config['JS_DATE_FORMAT']: + self.config['JS_DATE_FORMAT'][k] = json.dumps(self.config['JS_DATE_FORMAT'][k]) + else: + self.config['JS_DATE_FORMAT'] = json.dumps(self.config['JS_DATE_FORMAT']) + for i in self.TRANSLATABLE_SETTINGS: try: self.config[i] = utils.TranslatableSetting(i, self.config[i], self.config['TRANSLATIONS']) except KeyError: pass + # A EXIF_WHITELIST implies you want to keep EXIF data + if self.config['EXIF_WHITELIST'] and not self.config['PRESERVE_EXIF_DATA']: + utils.LOGGER.warn('Setting EXIF_WHITELIST implies PRESERVE_EXIF_DATA is set to True') + self.config['PRESERVE_EXIF_DATA'] = True + + # Setting PRESERVE_EXIF_DATA with an empty EXIF_WHITELIST implies 'keep everything' + if self.config['PRESERVE_EXIF_DATA'] and not self.config['EXIF_WHITELIST']: + utils.LOGGER.warn('You are setting PRESERVE_EXIF_DATA and not EXIF_WHITELIST so EXIF data is not really kept.') + # Handle CONTENT_FOOTER properly. # We provide the arguments to format in CONTENT_FOOTER_FORMATS. self.config['CONTENT_FOOTER'].langformat(self.config['CONTENT_FOOTER_FORMATS']) @@ -555,6 +714,38 @@ class Nikola(object): for i1, i2, i3 in self.config['PAGES']: self.config['post_pages'].append([i1, i2, i3, False]) + # RSS_TEASERS has been replaced with FEED_TEASERS + # TODO: remove on v8 + if 'RSS_TEASERS' in config: + utils.LOGGER.warn('The RSS_TEASERS option is deprecated, use FEED_TEASERS instead.') + if 'FEED_TEASERS' in config: + utils.LOGGER.warn('FEED_TEASERS conflicts with RSS_TEASERS, ignoring RSS_TEASERS.') + self.config['FEED_TEASERS'] = config['RSS_TEASERS'] + + # RSS_PLAIN has been replaced with FEED_PLAIN + # TODO: remove on v8 + if 'RSS_PLAIN' in config: + utils.LOGGER.warn('The RSS_PLAIN option is deprecated, use FEED_PLAIN instead.') + if 'FEED_PLAIN' in config: + utils.LOGGER.warn('FEED_PLIN conflicts with RSS_PLAIN, ignoring RSS_PLAIN.') + self.config['FEED_PLAIN'] = config['RSS_PLAIN'] + + # RSS_LINKS_APPEND_QUERY has been replaced with FEED_LINKS_APPEND_QUERY + # TODO: remove on v8 + if 'RSS_LINKS_APPEND_QUERY' in config: + utils.LOGGER.warn('The RSS_LINKS_APPEND_QUERY option is deprecated, use FEED_LINKS_APPEND_QUERY instead.') + if 'FEED_LINKS_APPEND_QUERY' in config: + utils.LOGGER.warn('FEED_LINKS_APPEND_QUERY conflicts with RSS_LINKS_APPEND_QUERY, ignoring RSS_LINKS_APPEND_QUERY.') + self.config['FEED_LINKS_APPEND_QUERY'] = config['RSS_LINKS_APPEND_QUERY'] + + # RSS_READ_MORE_LINK has been replaced with FEED_READ_MORE_LINK + # TODO: remove on v8 + if 'RSS_READ_MORE_LINK' in config: + utils.LOGGER.warn('The RSS_READ_MORE_LINK option is deprecated, use FEED_READ_MORE_LINK instead.') + if 'FEED_READ_MORE_LINK' in config: + utils.LOGGER.warn('FEED_READ_MORE_LINK conflicts with RSS_READ_MORE_LINK, ignoring RSS_READ_MORE_LINK') + self.config['FEED_READ_MORE_LINK'] = utils.TranslatableSetting('FEED_READ_MORE_LINK', config['RSS_READ_MORE_LINK'], self.config['TRANSLATIONS']) + # DEFAULT_TRANSLATIONS_PATTERN was changed from "p.e.l" to "p.l.e" # TODO: remove on v8 if 'TRANSLATIONS_PATTERN' not in self.config: @@ -634,7 +825,7 @@ class Nikola(object): if not self.config.get('COPY_SOURCES'): self.config['SHOW_SOURCELINK'] = False - if self.config['CATEGORY_PATH'] is None: + if self.config['CATEGORY_PATH']._inp is None: self.config['CATEGORY_PATH'] = self.config['TAG_PATH'] if self.config['CATEGORY_PAGES_ARE_INDEXES'] is None: self.config['CATEGORY_PAGES_ARE_INDEXES'] = self.config['TAG_PAGES_ARE_INDEXES'] @@ -680,6 +871,15 @@ class Nikola(object): utils.LOGGER.warn("WRITE_TAG_CLOUD is not set in your config. Defaulting to True (== writing tag_cloud_data.json).") utils.LOGGER.warn("Please explicitly add the setting to your conf.py with the desired value, as the setting will default to False in the future.") + # Rename stories to pages (#1891, #2518) + # TODO: remove in v8 + if 'COMMENTS_IN_STORIES' in config: + utils.LOGGER.warn('The COMMENTS_IN_STORIES option is deprecated, use COMMENTS_IN_PAGES instead.') + self.config['COMMENTS_IN_PAGES'] = config['COMMENTS_IN_STORIES'] + if 'STORY_INDEX' in config: + utils.LOGGER.warn('The STORY_INDEX option is deprecated, use PAGE_INDEX instead.') + self.config['PAGE_INDEX'] = config['STORY_INDEX'] + # We use one global tzinfo object all over Nikola. try: self.tzinfo = dateutil.tz.gettz(self.config['TIMEZONE']) @@ -702,6 +902,9 @@ class Nikola(object): candidate = utils.get_translation_candidate(self.config, "f" + ext, lang) compilers[compiler].add(candidate) + # Get search path for themes + self.themes_dirs = ['themes'] + self.config['EXTRA_THEMES_DIRS'] + # Avoid redundant compilers # Remove compilers that match nothing in POSTS/PAGES # And put them in "bad compilers" @@ -715,9 +918,21 @@ class Nikola(object): else: self.bad_compilers.add(k) - self._set_global_context() + self._set_global_context_from_config() + self._set_global_context_from_data() + + # Set persistent state facility + self.state = Persistor('state_data.json') + + # Set cache facility + self.cache = Persistor(os.path.join(self.config['CACHE_FOLDER'], 'cache_data.json')) + + # Create directories for persistors only if a site exists (Issue #2334) + if self.configured: + self.state._set_site(self) + self.cache._set_site(self) - def init_plugins(self, commands_only=False): + def init_plugins(self, commands_only=False, load_all=False): """Load plugins as needed.""" self.plugin_manager = PluginManager(categories_filter={ "Command": Command, @@ -729,6 +944,7 @@ class Nikola(object): "CompilerExtension": CompilerExtension, "MarkdownExtension": MarkdownExtension, "RestExtension": RestExtension, + "ShortcodePlugin": ShortcodePlugin, "SignalHandler": SignalHandler, "ConfigPlugin": ConfigPlugin, "PostScanner": PostScanner, @@ -736,47 +952,71 @@ class Nikola(object): self.plugin_manager.getPluginLocator().setPluginInfoExtension('plugin') extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS'] if sys.version_info[0] == 3: - places = [ + self._plugin_places = [ resource_filename('nikola', 'plugins'), - os.path.join(os.getcwd(), 'plugins'), os.path.expanduser('~/.nikola/plugins'), + os.path.join(os.getcwd(), 'plugins'), ] + [path for path in extra_plugins_dirs if path] else: - places = [ + self._plugin_places = [ resource_filename('nikola', utils.sys_encode('plugins')), os.path.join(os.getcwd(), utils.sys_encode('plugins')), os.path.expanduser('~/.nikola/plugins'), ] + [utils.sys_encode(path) for path in extra_plugins_dirs if path] - self.plugin_manager.getPluginLocator().setPluginPlaces(places) + self.plugin_manager.getPluginLocator().setPluginPlaces(self._plugin_places) self.plugin_manager.locatePlugins() bad_candidates = set([]) - for p in self.plugin_manager._candidates: - if commands_only: - if p[-1].details.has_option('Nikola', 'plugincategory'): - # FIXME TemplateSystem should not be needed - if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}: + if not load_all: + for p in self.plugin_manager._candidates: + if commands_only: + if p[-1].details.has_option('Nikola', 'plugincategory'): + # FIXME TemplateSystem should not be needed + if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}: + bad_candidates.add(p) + else: + bad_candidates.add(p) + elif self.configured: # Not commands-only, and configured + # Remove compilers we don't use + if p[-1].name in self.bad_compilers: + bad_candidates.add(p) + self.disabled_compilers[p[-1].name] = p + utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) + if p[-1].name not in self.config['COMPILERS'] and \ + p[-1].details.has_option('Nikola', 'plugincategory') and p[-1].details.get('Nikola', 'PluginCategory') == 'Compiler': bad_candidates.add(p) - else: # Not commands-only - # Remove compilers we don't use - if p[-1].name in self.bad_compilers: - bad_candidates.add(p) - self.disabled_compilers[p[-1].name] = p - utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) - if p[-1].name not in self.config['COMPILERS'] and \ - p[-1].details.has_option('Nikola', 'plugincategory') and p[-1].details.get('Nikola', 'PluginCategory') == 'Compiler': - bad_candidates.add(p) - self.disabled_compilers[p[-1].name] = p - utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) - # Remove blacklisted plugins - if p[-1].name in self.config['DISABLED_PLUGINS']: - bad_candidates.add(p) - utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name) - # Remove compiler extensions we don't need - if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers: - bad_candidates.add(p) - utils.LOGGER.debug('Not loading comopiler extension {}', p[-1].name) - self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates) + self.disabled_compilers[p[-1].name] = p + utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name) + # Remove blacklisted plugins + if p[-1].name in self.config['DISABLED_PLUGINS']: + bad_candidates.add(p) + utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name) + # Remove compiler extensions we don't need + if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers: + bad_candidates.add(p) + utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name) + self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates) + + # Find repeated plugins and discard the less local copy + def plugin_position_in_places(plugin): + # plugin here is a tuple: + # (path to the .plugin file, path to plugin module w/o .py, plugin metadata) + for i, place in enumerate(self._plugin_places): + if plugin[0].startswith(place): + return i + + plugin_dict = defaultdict(list) + for data in self.plugin_manager._candidates: + plugin_dict[data[2].name].append(data) + self.plugin_manager._candidates = [] + for name, plugins in plugin_dict.items(): + if len(plugins) > 1: + # Sort by locality + plugins.sort(key=plugin_position_in_places) + utils.LOGGER.debug("Plugin {} exists in multiple places, using {}".format( + plugins[-1][2].name, plugins[-1][0])) + self.plugin_manager._candidates.append(plugins[-1]) + self.plugin_manager.loadPlugins() self._activate_plugins_of_category("SignalHandler") @@ -803,6 +1043,9 @@ class Nikola(object): self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) + # Activate shortcode plugins + self._activate_plugins_of_category("ShortcodePlugin") + # Load compiler plugins self.compilers = {} self.inverse_compilers = {} @@ -813,11 +1056,15 @@ class Nikola(object): plugin_info.plugin_object self._activate_plugins_of_category("ConfigPlugin") - + self._register_templated_shortcodes() signal('configured').send(self) - def _set_global_context(self): - """Create global context from configuration.""" + def _set_global_context_from_config(self): + """Create global context from configuration. + + These are options that are used by templates, so they always need to be + available. + """ self._GLOBAL_CONTEXT['url_type'] = self.config['URL_TYPE'] self._GLOBAL_CONTEXT['timezone'] = self.tzinfo self._GLOBAL_CONTEXT['_link'] = self.link @@ -828,13 +1075,16 @@ class Nikola(object): self._GLOBAL_CONTEXT['rel_link'] = self.rel_link self._GLOBAL_CONTEXT['abs_link'] = self.abs_link self._GLOBAL_CONTEXT['exists'] = self.file_exists + self._GLOBAL_CONTEXT['SLUG_AUTHOR_PATH'] = self.config['SLUG_AUTHOR_PATH'] self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH'] self._GLOBAL_CONTEXT['annotations'] = self.config['ANNOTATIONS'] self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[ 'INDEX_DISPLAY_POST_COUNT'] self._GLOBAL_CONTEXT['index_file'] = self.config['INDEX_FILE'] + self._GLOBAL_CONTEXT['use_base_tag'] = self.config['USE_BASE_TAG'] self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES'] self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN") + self._GLOBAL_CONTEXT['theme_color'] = self.config.get("THEME_COLOR") self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS'] self._GLOBAL_CONTEXT['date_format'] = self.config.get('DATE_FORMAT') self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR') @@ -842,6 +1092,9 @@ class Nikola(object): self._GLOBAL_CONTEXT['show_blog_title'] = self.config.get('SHOW_BLOG_TITLE') self._GLOBAL_CONTEXT['logo_url'] = self.config.get('LOGO_URL') self._GLOBAL_CONTEXT['blog_description'] = self.config.get('BLOG_DESCRIPTION') + self._GLOBAL_CONTEXT['front_index_header'] = self.config.get('FRONT_INDEX_HEADER') + self._GLOBAL_CONTEXT['color_hsl_adjust_hex'] = utils.color_hsl_adjust_hex + self._GLOBAL_CONTEXT['colorize_str_from_base_color'] = utils.colorize_str_from_base_color # TODO: remove in v8 self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION') @@ -858,6 +1111,8 @@ class Nikola(object): self._GLOBAL_CONTEXT['site_has_comments'] = bool(self.config.get('COMMENT_SYSTEM')) self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get( 'MATHJAX_CONFIG') + self._GLOBAL_CONTEXT['use_katex'] = self.config.get('USE_KATEX') + self._GLOBAL_CONTEXT['katex_auto_render'] = self.config.get('KATEX_AUTO_RENDER') self._GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONFIG_SUBTHEME') self._GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONFIG_TRANSITION') self._GLOBAL_CONTEXT['content_footer'] = self.config.get( @@ -879,12 +1134,20 @@ class Nikola(object): 'SHOW_SOURCELINK') self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA') self._GLOBAL_CONTEXT['date_fanciness'] = self.config.get('DATE_FANCINESS') - self._GLOBAL_CONTEXT['js_date_format'] = json.dumps(self.config.get('JS_DATE_FORMAT')) + self._GLOBAL_CONTEXT['js_date_format'] = self.config.get('JS_DATE_FORMAT') self._GLOBAL_CONTEXT['colorbox_locales'] = LEGAL_VALUES['COLORBOX_LOCALES'] self._GLOBAL_CONTEXT['momentjs_locales'] = LEGAL_VALUES['MOMENTJS_LOCALES'] self._GLOBAL_CONTEXT['hidden_tags'] = self.config.get('HIDDEN_TAGS') self._GLOBAL_CONTEXT['hidden_categories'] = self.config.get('HIDDEN_CATEGORIES') + self._GLOBAL_CONTEXT['hidden_authors'] = self.config.get('HIDDEN_AUTHORS') self._GLOBAL_CONTEXT['url_replacer'] = self.url_replacer + self._GLOBAL_CONTEXT['posts_sections'] = self.config.get('POSTS_SECTIONS') + self._GLOBAL_CONTEXT['posts_section_are_indexes'] = self.config.get('POSTS_SECTION_ARE_INDEXES') + self._GLOBAL_CONTEXT['posts_section_colors'] = self.config.get('POSTS_SECTION_COLORS') + self._GLOBAL_CONTEXT['posts_section_descriptions'] = self.config.get('POSTS_SECTION_DESCRIPTIONS') + self._GLOBAL_CONTEXT['posts_section_from_meta'] = self.config.get('POSTS_SECTION_FROM_META') + self._GLOBAL_CONTEXT['posts_section_name'] = self.config.get('POSTS_SECTION_NAME') + self._GLOBAL_CONTEXT['posts_section_title'] = self.config.get('POSTS_SECTION_TITLE') # IPython theme configuration. If a website has ipynb enabled in post_pages # we should enable the IPython CSS (leaving that up to the theme itself). @@ -893,6 +1156,16 @@ class Nikola(object): self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {})) + def _set_global_context_from_data(self): + """Load files from data/ and put them in the global context.""" + self._GLOBAL_CONTEXT['data'] = {} + for root, dirs, files in os.walk('data', followlinks=True): + for fname in files: + fname = os.path.join(root, fname) + data = utils.load_data(fname) + key = os.path.splitext(fname.split(os.sep, 1)[1])[0] + self._GLOBAL_CONTEXT['data'][key] = data + def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in tests/base.py @@ -906,11 +1179,13 @@ class Nikola(object): def _get_themes(self): if self._THEMES is None: try: - self._THEMES = utils.get_theme_chain(self.config['THEME']) + self._THEMES = utils.get_theme_chain(self.config['THEME'], self.themes_dirs) except Exception: - utils.LOGGER.warn('''Cannot load theme "{0}", using 'bootstrap3' instead.'''.format(self.config['THEME'])) - self.config['THEME'] = 'bootstrap3' - return self._get_themes() + if self.config['THEME'] != 'bootstrap3': + utils.LOGGER.warn('''Cannot load theme "{0}", using 'bootstrap3' instead.'''.format(self.config['THEME'])) + self.config['THEME'] = 'bootstrap3' + return self._get_themes() + raise # Check consistency of USE_CDN and the current THEME (Issue #386) if self.config['USE_CDN'] and self.config['USE_CDN_WARNING']: bootstrap_path = utils.get_asset_path(os.path.join( @@ -924,9 +1199,12 @@ class Nikola(object): def _get_messages(self): try: - return utils.load_messages(self.THEMES, - self.translations, - self.default_lang) + if self._MESSAGES is None: + self._MESSAGES = utils.load_messages(self.THEMES, + self.translations, + self.default_lang, + themes_dirs=self.themes_dirs) + return self._MESSAGES except utils.LanguageNotFoundError as e: utils.LOGGER.error('''Cannot load language "{0}". Please make sure it is supported by Nikola itself, or that you have the appropriate messages files in your themes.'''.format(e.lang)) sys.exit(1) @@ -1009,7 +1287,7 @@ class Nikola(object): return compile_html - def render_template(self, template_name, output_name, context): + def render_template(self, template_name, output_name, context, url_type=None): """Render a template with the global context. If ``output_name`` is None, will return a string and all URL @@ -1017,6 +1295,9 @@ class Nikola(object): If ``output_name`` is a string, URLs will be normalized and the resultant HTML will be saved to the named file (path must start with OUTPUT_FOLDER). + + The argument ``url_type`` allows to override the ``URL_TYPE`` + configuration. """ local_context = {} local_context["template_name"] = template_name @@ -1053,22 +1334,22 @@ class Nikola(object): utils.makedirs(os.path.dirname(output_name)) parser = lxml.html.HTMLParser(remove_blank_text=True) doc = lxml.html.document_fromstring(data, parser) - self.rewrite_links(doc, src, context['lang']) + self.rewrite_links(doc, src, context['lang'], url_type) data = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True) with open(output_name, "wb+") as post_file: post_file.write(data) - def rewrite_links(self, doc, src, lang): + def rewrite_links(self, doc, src, lang, url_type=None): """Replace links in document to point to the right places.""" # First let lxml replace most of them - doc.rewrite_links(lambda dst: self.url_replacer(src, dst, lang), resolve_base_href=False) + doc.rewrite_links(lambda dst: self.url_replacer(src, dst, lang, url_type), resolve_base_href=False) # lxml ignores srcset in img and source elements, so do that by hand - objs = list(doc.findall('*//img')) + list(doc.findall('*//source')) + objs = list(doc.xpath('(*//img|*//source)')) for obj in objs: if 'srcset' in obj.attrib: urls = [u.strip() for u in obj.attrib['srcset'].split(',')] - urls = [self.url_replacer(src, dst, lang) for dst in urls] + urls = [self.url_replacer(src, dst, lang, url_type) for dst in urls] obj.set('srcset', ', '.join(urls)) def url_replacer(self, src, dst, lang=None, url_type=None): @@ -1171,7 +1452,7 @@ class Nikola(object): # Now i is the longest common prefix result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) - if not result: + if not result and not parsed_dst.fragment: result = "." # Don't forget the query part of the link @@ -1186,13 +1467,90 @@ class Nikola(object): return result + def _make_renderfunc(self, t_data, fname=None): + """Return a function that can be registered as a template shortcode. + + The returned function has access to the passed template data and + accepts any number of positional and keyword arguments. Positional + arguments values are added as a tuple under the key ``_args`` to the + keyword argument dict and then the latter provides the template + context. + + """ + def render_shortcode(*args, **kw): + context = self.GLOBAL_CONTEXT.copy() + context.update(kw) + context['_args'] = args + context['lang'] = utils.LocaleBorg().current_lang + for k in self._GLOBAL_CONTEXT_TRANSLATABLE: + context[k] = context[k](context['lang']) + output = self.template_system.render_template_to_string(t_data, context) + if fname is not None: + dependencies = [fname] + self.template_system.get_deps(fname) + else: + dependencies = [] + return output, dependencies + return render_shortcode + + def _register_templated_shortcodes(self): + """Register shortcodes based on templates. + + This will register a shortcode for any template found in shortcodes/ + folders and a generic "template" shortcode which will consider the + content in the shortcode as a template in itself. + """ + self.register_shortcode('template', self._template_shortcode_handler) + + builtin_sc_dir = resource_filename( + 'nikola', + os.path.join('data', 'shortcodes', utils.get_template_engine(self.THEMES))) + + for sc_dir in [builtin_sc_dir, 'shortcodes']: + if not os.path.isdir(sc_dir): + continue + + for fname in os.listdir(sc_dir): + name, ext = os.path.splitext(fname) + + if ext != '.tmpl': + continue + with open(os.path.join(sc_dir, fname)) as fd: + self.register_shortcode(name, self._make_renderfunc( + fd.read(), os.path.join(sc_dir, fname))) + + def _template_shortcode_handler(self, *args, **kw): + t_data = kw.pop('data', '') + context = self.GLOBAL_CONTEXT.copy() + context.update(kw) + context['_args'] = args + context['lang'] = utils.LocaleBorg().current_lang + for k in self._GLOBAL_CONTEXT_TRANSLATABLE: + context[k] = context[k](context['lang']) + output = self.template_system.render_template_to_string(t_data, context) + dependencies = self.template_system.get_string_deps(t_data) + return output, dependencies + + def register_shortcode(self, name, f): + """Register function f to handle shortcode "name".""" + if name in self.shortcode_registry: + utils.LOGGER.warn('Shortcode name conflict: {}', name) + return + self.shortcode_registry[name] = f + + # XXX in v8, get rid of with_dependencies + def apply_shortcodes(self, data, filename=None, lang=None, with_dependencies=False, extra_context={}): + """Apply shortcodes from the registry on data.""" + if lang is None: + lang = utils.LocaleBorg().current_lang + return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, with_dependencies=with_dependencies, extra_context=extra_context) + def generic_rss_renderer(self, lang, title, link, description, timeline, output_path, rss_teasers, rss_plain, feed_length=10, feed_url=None, enclosure=_enclosure, rss_links_append_query=None): """Take all necessary data, and render a RSS feed in output_path.""" rss_obj = utils.ExtendedRSS2( title=title, - link=link, + link=utils.encodelink(link), description=description, lastBuildDate=datetime.datetime.utcnow(), generator='https://getnikola.com/', @@ -1213,10 +1571,12 @@ class Nikola(object): for post in timeline[:feed_length]: data = post.text(lang, teaser_only=rss_teasers, strip_html=rss_plain, - rss_read_more_link=True, rss_links_append_query=feed_append_query) + feed_read_more_link=True, feed_links_append_query=feed_append_query) if feed_url is not None and data: # Massage the post's HTML (unless plain) if not rss_plain: + if self.config["FEED_PREVIEWIMAGE"] and 'previewimage' in post.meta[lang] and post.meta[lang]['previewimage'] not in data: + data = "<figure><img src=\"{}\"></figure> {}".format(post.meta[lang]['previewimage'], data) # FIXME: this is duplicated with code in Post.text() try: doc = lxml.html.document_fromstring(data) @@ -1290,8 +1650,8 @@ class Nikola(object): * gallery (name is the gallery name) * listing (name is the source code file name) * post_path (name is 1st element in a POSTS/PAGES tuple) - * slug (name is the slug of a post or story) - * filename (name is the source filename of a post/story, in DEFAULT_LANG, relative to conf.py) + * slug (name is the slug of a post or page) + * filename (name is the source filename of a post/page, in DEFAULT_LANG, relative to conf.py) The returned value is always a path relative to output, like "categories/whatever.html" @@ -1308,7 +1668,6 @@ class Nikola(object): try: path = self.path_handlers[kind](name, lang) path = [os.path.normpath(p) for p in path if p != '.'] # Fix Issue #1028 - if is_link: link = '/' + ('/'.join(path)) index_len = len(self.config['INDEX_FILE']) @@ -1324,13 +1683,25 @@ class Nikola(object): return "" def post_path(self, name, lang): - """Handle post_path paths.""" + """Link to the destination of an element in the POSTS/PAGES settings. + + Example: + + link://post_path/posts => /blog + """ return [_f for _f in [self.config['TRANSLATIONS'][lang], os.path.dirname(name), self.config['INDEX_FILE']] if _f] def root_path(self, name, lang): - """Handle root_path paths.""" + """Link to the current language's root. + + Example: + + link://root_path => / + + link://root_path => /translations/spanish/ + """ d = self.config['TRANSLATIONS'][lang] if d: return [d, ''] @@ -1338,7 +1709,12 @@ class Nikola(object): return [] def slug_path(self, name, lang): - """Handle slug paths.""" + """A link to a post with given slug, if not ambiguous. + + Example: + + link://slug/yellow-camaro => /posts/cars/awful/yellow-camaro/index.html + """ results = [p for p in self.timeline if p.meta('slug') == name] if not results: utils.LOGGER.warning("Cannot resolve path request for slug: {0}".format(name)) @@ -1348,7 +1724,12 @@ class Nikola(object): return [_f for _f in results[0].permalink(lang).split('/') if _f] def filename_path(self, name, lang): - """Handle filename paths.""" + """Link to post or page by source filename. + + Example: + + link://filename/manual.txt => /docs/handbook.html + """ results = [p for p in self.timeline if p.source_path == name] if not results: utils.LOGGER.warning("Cannot resolve path request for filename: {0}".format(name)) @@ -1366,7 +1747,9 @@ class Nikola(object): def link(self, *args): """Create a link.""" - return self.path(*args, is_link=True) + url = self.path(*args, is_link=True) + url = utils.encodelink(url) + return url def abs_link(self, dst, protocol_relative=False): """Get an absolute link.""" @@ -1378,6 +1761,7 @@ class Nikola(object): url = urlparse(dst).geturl() if protocol_relative: url = url.split(":", 1)[1] + url = utils.encodelink(url) return url def rel_link(self, src, dst): @@ -1392,7 +1776,7 @@ class Nikola(object): parsed_src = urlsplit(src) parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: - return dst + return utils.encodelink(dst) # Now both paths are on the same site and absolute src_elems = parsed_src.path.split('/')[1:] dst_elems = parsed_dst.path.split('/')[1:] @@ -1403,7 +1787,9 @@ class Nikola(object): else: i += 1 # Now i is the longest common prefix - return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) + url = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) + url = utils.encodelink(url) + return url def file_exists(self, path, not_empty=False): """Check if the file exists. If not_empty is True, it also must not be empty.""" @@ -1526,6 +1912,7 @@ class Nikola(object): self.tags_per_language = defaultdict(list) self.category_hierarchy = {} self.post_per_file = {} + self.post_per_input_file = {} self.timeline = [] self.pages = [] @@ -1536,27 +1923,28 @@ class Nikola(object): quit = False # Classify posts per year/tag/month/whatever - slugged_tags = set([]) + slugged_tags = defaultdict(set) for post in self.timeline: if post.use_in_feeds: self.posts.append(post) self.posts_per_year[str(post.date.year)].append(post) self.posts_per_month[ '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post) - for tag in post.alltags: - _tag_slugified = utils.slugify(tag) - if _tag_slugified in slugged_tags: - if tag not in self.posts_per_tag: - # Tags that differ only in case - other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing) == _tag_slugified][0] - utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag)) - utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path)) - utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]]))) - quit = True - else: - slugged_tags.add(utils.slugify(tag, force=True)) - self.posts_per_tag[tag].append(post) for lang in self.config['TRANSLATIONS'].keys(): + for tag in post.tags_for_language(lang): + _tag_slugified = utils.slugify(tag, lang) + if _tag_slugified in slugged_tags[lang]: + if tag not in self.posts_per_tag: + # Tags that differ only in case + other_tag = [existing for existing in self.posts_per_tag.keys() if utils.slugify(existing, lang) == _tag_slugified][0] + utils.LOGGER.error('You have tags that are too similar: {0} and {1}'.format(tag, other_tag)) + utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path)) + utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join([p.source_path for p in self.posts_per_tag[other_tag]]))) + quit = True + else: + slugged_tags[lang].add(_tag_slugified) + if post not in self.posts_per_tag[tag]: + self.posts_per_tag[tag].append(post) self.tags_per_language[lang].extend(post.tags_for_language(lang)) self._add_post_to_category(post, post.meta('category')) @@ -1569,6 +1957,7 @@ class Nikola(object): for lang in self.config['TRANSLATIONS'].keys(): dest = post.destination_path(lang=lang) src_dest = post.destination_path(lang=lang, extension=post.source_ext()) + src_file = post.translated_source_path(lang=lang) if dest in self.post_per_file: utils.LOGGER.error('Two posts are trying to generate {0}: {1} and {2}'.format( dest, @@ -1583,13 +1972,16 @@ class Nikola(object): quit = True self.post_per_file[dest] = post self.post_per_file[src_dest] = post + self.post_per_input_file[src_file] = post # deduplicate tags_per_language self.tags_per_language[lang] = list(set(self.tags_per_language[lang])) # Sort everything. for thing in self.timeline, self.posts, self.all_posts, self.pages: - thing.sort(key=lambda p: (p.date, p.source_path)) + thing.sort(key=lambda p: + (int(p.meta('priority')) if p.meta('priority') else 0, + p.date, p.source_path)) thing.reverse() self._sort_category_hierarchy() @@ -1604,15 +1996,71 @@ class Nikola(object): sys.exit(1) signal('scanned').send(self) + def generic_renderer(self, lang, output_name, template_name, filters, file_deps=None, uptodate_deps=None, context=None, context_deps_remove=None, post_deps_dict=None, url_type=None): + """Helper function for rendering pages and post lists and other related pages. + + lang is the current language. + output_name is the destination file name. + template_name is the template to be used. + filters is the list of filters (usually site.config['FILTERS']) which will be used to post-process the result. + file_deps (optional) is a list of additional file dependencies (next to template and its dependencies). + uptodate_deps (optional) is a list of additional entries added to the task's uptodate list. + context (optional) a dict used as a basis for the template context. The lang parameter will always be added. + context_deps_remove (optional) is a list of keys to remove from the context after using it as an uptodate dependency. This should name all keys containing non-trivial Python objects; they can be replaced by adding JSON-style dicts in post_deps_dict. + post_deps_dict (optional) is a dict merged into the copy of context which is used as an uptodate dependency. + url_type (optional) allows to override the ``URL_TYPE`` configuration + """ + utils.LocaleBorg().set_locale(lang) + + file_deps = copy(file_deps) if file_deps else [] + file_deps += self.template_system.template_deps(template_name) + file_deps = sorted(list(filter(None, file_deps))) + + context = copy(context) if context else {} + context["lang"] = lang + + deps_dict = copy(context) + if context_deps_remove: + for key in context_deps_remove: + deps_dict.pop(key) + deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER'] + deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS'] + deps_dict['global'] = self.GLOBAL_CONTEXT + if post_deps_dict: + deps_dict.update(post_deps_dict) + + for k, v in self.GLOBAL_CONTEXT['template_hooks'].items(): + deps_dict['||template_hooks|{0}||'.format(k)] = v._items + + for k in self._GLOBAL_CONTEXT_TRANSLATABLE: + deps_dict[k] = deps_dict['global'][k](lang) + + deps_dict['navigation_links'] = deps_dict['global']['navigation_links'](lang) + + task = { + 'name': os.path.normpath(output_name), + 'targets': [output_name], + 'file_dep': file_deps, + 'actions': [(self.render_template, [template_name, output_name, + context, url_type])], + 'clean': True, + 'uptodate': [config_changed(deps_dict, 'nikola.nikola.Nikola.generic_renderer')] + ([] if uptodate_deps is None else uptodate_deps) + } + + return utils.apply_filters(task, filters) + def generic_page_renderer(self, lang, post, filters, context=None): """Render post fragments to final HTML pages.""" - context = context.copy() if context else {} - deps = post.deps(lang) + \ - self.template_system.template_deps(post.template_name) + extension = self.get_compiler(post.source_path).extension() + output_name = os.path.join(self.config['OUTPUT_FOLDER'], + post.destination_path(lang, extension)) + + deps = post.deps(lang) + uptodate_deps = post.deps_uptodate(lang) deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine')) - deps = list(filter(None, deps)) + + context = copy(context) if context else {} context['post'] = post - context['lang'] = lang context['title'] = post.title(lang) context['description'] = post.description(lang) context['permalink'] = post.permalink(lang) @@ -1621,85 +2069,49 @@ class Nikola(object): if post.use_in_feeds: context['enable_comments'] = True else: - context['enable_comments'] = self.config['COMMENTS_IN_STORIES'] - extension = self.get_compiler(post.source_path).extension() - output_name = os.path.join(self.config['OUTPUT_FOLDER'], - post.destination_path(lang, extension)) - deps_dict = copy(context) - deps_dict.pop('post') + context['enable_comments'] = self.config['COMMENTS_IN_PAGES'] + + deps_dict = {} if post.prev_post: deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)] if post.next_post: deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)] - deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER'] - deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS'] - deps_dict['global'] = self.GLOBAL_CONTEXT deps_dict['comments'] = context['enable_comments'] - - for k, v in self.GLOBAL_CONTEXT['template_hooks'].items(): - deps_dict['||template_hooks|{0}||'.format(k)] = v._items - - for k in self._GLOBAL_CONTEXT_TRANSLATABLE: - deps_dict[k] = deps_dict['global'][k](lang) - - deps_dict['navigation_links'] = deps_dict['global']['navigation_links'](lang) - if post: deps_dict['post_translations'] = post.translated_to - task = { - 'name': os.path.normpath(output_name), - 'file_dep': sorted(deps), - 'targets': [output_name], - 'actions': [(self.render_template, [post.template_name, - output_name, context])], - 'clean': True, - 'uptodate': [config_changed(deps_dict, 'nikola.nikola.Nikola.generic_page_renderer')] + post.deps_uptodate(lang), - } - - yield utils.apply_filters(task, filters) + yield self.generic_renderer(lang, output_name, post.template_name, filters, + file_deps=deps, + uptodate_deps=uptodate_deps, + context=context, + context_deps_remove=['post'], + post_deps_dict=deps_dict) - def generic_post_list_renderer(self, lang, posts, output_name, - template_name, filters, extra_context): + def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Render pages with lists of posts.""" deps = [] - deps += self.template_system.template_deps(template_name) uptodate_deps = [] for post in posts: deps += post.deps(lang) uptodate_deps += post.deps_uptodate(lang) + context = {} context["posts"] = posts context["title"] = self.config['BLOG_TITLE'](lang) context["description"] = self.config['BLOG_DESCRIPTION'](lang) - context["lang"] = lang context["prevlink"] = None context["nextlink"] = None - context.update(extra_context) - deps_context = copy(context) - deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in - posts] - deps_context["global"] = self.GLOBAL_CONTEXT - - for k, v in self.GLOBAL_CONTEXT['template_hooks'].items(): - deps_context['||template_hooks|{0}||'.format(k)] = v._items - - for k in self._GLOBAL_CONTEXT_TRANSLATABLE: - deps_context[k] = deps_context['global'][k](lang) - - deps_context['navigation_links'] = deps_context['global']['navigation_links'](lang) + if extra_context: + context.update(extra_context) - task = { - 'name': os.path.normpath(output_name), - 'targets': [output_name], - 'file_dep': sorted(deps), - 'actions': [(self.render_template, [template_name, output_name, - context])], - 'clean': True, - 'uptodate': [config_changed(deps_context, 'nikola.nikola.Nikola.generic_post_list_renderer')] + uptodate_deps - } + post_deps_dict = {} + post_deps_dict["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in posts] - return utils.apply_filters(task, filters) + return self.generic_renderer(lang, output_name, template_name, filters, + file_deps=deps, + uptodate_deps=uptodate_deps, + context=context, + post_deps_dict=post_deps_dict) def atom_feed_renderer(self, lang, posts, output_path, filters, extra_context): @@ -1711,9 +2123,10 @@ class Nikola(object): link = lxml.etree.Element("link") link.set("rel", link_rel) link.set("type", link_type) - link.set("href", link_href) + link.set("href", utils.encodelink(link_href)) return link + utils.LocaleBorg().set_locale(lang) deps = [] uptodate_deps = [] for post in posts: @@ -1739,15 +2152,13 @@ class Nikola(object): deps_context['navigation_links'] = deps_context['global']['navigation_links'](lang) nslist = {} - if context["is_feed_stale"] or (not context["feedpagenum"] == context["feedpagecount"] - 1 and not context["feedpagenum"] == 0): + if context["is_feed_stale"] or "feedpagenum" in context and (not context["feedpagenum"] == context["feedpagecount"] - 1 and not context["feedpagenum"] == 0): nslist["fh"] = "http://purl.org/syndication/history/1.0" - if not self.config["RSS_TEASERS"]: - nslist["xh"] = "http://www.w3.org/1999/xhtml" feed_xsl_link = self.abs_link("/assets/xml/atom.xsl") feed_root = lxml.etree.Element("feed", nsmap=nslist) feed_root.addprevious(lxml.etree.ProcessingInstruction( "xml-stylesheet", - 'href="' + feed_xsl_link + '" type="text/xsl media="all"')) + 'href="' + utils.encodelink(feed_xsl_link) + '" type="text/xsl media="all"')) feed_root.set("{http://www.w3.org/XML/1998/namespace}lang", lang) feed_root.set("xmlns", "http://www.w3.org/2005/Atom") feed_title = lxml.etree.SubElement(feed_root, "title") @@ -1755,7 +2166,7 @@ class Nikola(object): feed_id = lxml.etree.SubElement(feed_root, "id") feed_id.text = self.abs_link(context["feedlink"]) feed_updated = lxml.etree.SubElement(feed_root, "updated") - feed_updated.text = post.formatted_date('webiso', datetime.datetime.now(tz=dateutil.tz.tzutc())) + feed_updated.text = utils.LocaleBorg().formatted_date('webiso', datetime.datetime.now(tz=dateutil.tz.tzutc())) feed_author = lxml.etree.SubElement(feed_root, "author") feed_author_name = lxml.etree.SubElement(feed_author, "name") feed_author_name.text = self.config["BLOG_AUTHOR"](lang) @@ -1768,50 +2179,64 @@ class Nikola(object): if "prevfeedlink" in context: feed_root.append(atom_link("previous", "application/atom+xml", self.abs_link(context["prevfeedlink"]))) - if context["is_feed_stale"] or not context["feedpagenum"] == 0: + if context["is_feed_stale"] or "feedpagenum" in context and not context["feedpagenum"] == 0: feed_root.append(atom_link("current", "application/atom+xml", self.abs_link(context["currentfeedlink"]))) # Older is "prev-archive" and newer is "next-archive" in archived feeds (opposite of paginated) - if "prevfeedlink" in context and (context["is_feed_stale"] or not context["feedpagenum"] == context["feedpagecount"] - 1): + if "prevfeedlink" in context and (context["is_feed_stale"] or "feedpagenum" in context and not context["feedpagenum"] == context["feedpagecount"] - 1): feed_root.append(atom_link("next-archive", "application/atom+xml", self.abs_link(context["prevfeedlink"]))) if "nextfeedlink" in context: feed_root.append(atom_link("prev-archive", "application/atom+xml", self.abs_link(context["nextfeedlink"]))) - if context["is_feed_stale"] or not context["feedpagenum"] == context["feedpagecount"] - 1: + if context["is_feed_stale"] or "feedpagenum" and not context["feedpagenum"] == context["feedpagecount"] - 1: lxml.etree.SubElement(feed_root, "{http://purl.org/syndication/history/1.0}archive") feed_root.append(atom_link("alternate", "text/html", self.abs_link(context["permalink"]))) feed_generator = lxml.etree.SubElement(feed_root, "generator") - feed_generator.set("uri", "http://getnikola.com/") + feed_generator.set("uri", "https://getnikola.com/") feed_generator.text = "Nikola" feed_append_query = None - if self.config["RSS_LINKS_APPEND_QUERY"]: - feed_append_query = self.config["RSS_LINKS_APPEND_QUERY"].format( + if self.config["FEED_LINKS_APPEND_QUERY"]: + feed_append_query = self.config["FEED_LINKS_APPEND_QUERY"].format( feedRelUri=context["feedlink"], feedFormat="atom") - for post in posts: - data = post.text(lang, teaser_only=self.config["RSS_TEASERS"], strip_html=self.config["RSS_TEASERS"], - rss_read_more_link=True, rss_links_append_query=feed_append_query) - if not self.config["RSS_TEASERS"]: + def atom_post_text(post, text): + if not self.config["FEED_PLAIN"]: + if self.config["FEED_PREVIEWIMAGE"] and 'previewimage' in post.meta[lang] and post.meta[lang]['previewimage'] not in text: + text = "<figure><img src=\"{}\"></figure> {}".format(post.meta[lang]['previewimage'], text) + # FIXME: this is duplicated with code in Post.text() and generic_rss_renderer try: - doc = lxml.html.document_fromstring(data) - doc.rewrite_links(lambda dst: self.url_replacer(post.permalink(), dst, lang, 'absolute')) + doc = lxml.html.document_fromstring(text) + doc.rewrite_links(lambda dst: self.url_replacer(post.permalink(lang), dst, lang, 'absolute')) try: body = doc.body - data = (body.text or '') + ''.join( + text = (body.text or '') + ''.join( [lxml.html.tostring(child, encoding='unicode') for child in body.iterchildren()]) except IndexError: # No body there, it happens sometimes - data = '' + text = '' except lxml.etree.ParserError as e: if str(e) == "Document is empty": - data = "" + text = "" else: # let other errors raise raise(e) + return text.strip() + + for post in posts: + summary = atom_post_text(post, post.text(lang, teaser_only=True, + strip_html=self.config["FEED_PLAIN"], + feed_read_more_link=True, + feed_links_append_query=feed_append_query)) + content = None + if not self.config["FEED_TEASERS"]: + content = atom_post_text(post, post.text(lang, teaser_only=self.config["FEED_TEASERS"], + strip_html=self.config["FEED_PLAIN"], + feed_read_more_link=True, + feed_links_append_query=feed_append_query)) entry_root = lxml.etree.SubElement(feed_root, "entry") entry_title = lxml.etree.SubElement(entry_root, "title") @@ -1828,17 +2253,22 @@ class Nikola(object): entry_root.append(atom_link("alternate", "text/html", post.permalink(lang, absolute=True, query=feed_append_query))) - if self.config["RSS_TEASERS"]: - entry_summary = lxml.etree.SubElement(entry_root, "summary") - entry_summary.text = data + entry_summary = lxml.etree.SubElement(entry_root, "summary") + if not self.config["FEED_PLAIN"]: + entry_summary.set("type", "html") else: + entry_summary.set("type", "text") + entry_summary.text = summary + if content: entry_content = lxml.etree.SubElement(entry_root, "content") - entry_content.set("type", "xhtml") - entry_content_nsdiv = lxml.etree.SubElement(entry_content, "{http://www.w3.org/1999/xhtml}div") - entry_content_nsdiv.text = data - for category in post.tags: + if not self.config["FEED_PLAIN"]: + entry_content.set("type", "html") + else: + entry_content.set("type", "text") + entry_content.text = content + for category in post.tags_for_language(lang): entry_category = lxml.etree.SubElement(entry_root, "category") - entry_category.set("term", utils.slugify(category)) + entry_category.set("term", utils.slugify(category, lang)) entry_category.set("label", category) dst_dir = os.path.dirname(output_path) @@ -1886,8 +2316,7 @@ class Nikola(object): kw['indexes_prety_page_url'] = self.config["INDEXES_PRETTY_PAGE_URL"] kw['demote_headers'] = self.config['DEMOTE_HEADERS'] kw['generate_atom'] = self.config["GENERATE_ATOM"] - kw['feed_link_append_query'] = self.config["RSS_LINKS_APPEND_QUERY"] - kw['feed_teasers'] = self.config["RSS_TEASERS"] + kw['feed_link_append_query'] = self.config["FEED_LINKS_APPEND_QUERY"] kw['currentfeed'] = None # Split in smaller lists @@ -1978,10 +2407,14 @@ class Nikola(object): context["currentfeedlink"] = kw["currentfeed"] context["feedpagenum"] = i context["feedpagecount"] = num_pages + kw['feed_teasers'] = self.config['FEED_TEASERS'] + kw['feed_plain'] = self.config['FEED_PLAIN'] + kw['feed_previewimage'] = self.config['FEED_PREVIEWIMAGE'] atom_task = { "basename": basename, "name": atom_output_name, "file_dep": sorted([_.base_path for _ in post_list]), + "task_dep": ['render_posts'], "targets": [atom_output_name], "actions": [(self.atom_feed_renderer, (lang, @@ -2009,7 +2442,7 @@ class Nikola(object): def __repr__(self): """Representation of a Nikola site.""" - return '<Nikola Site: {0!r}>'.format(self.config['BLOG_TITLE']()) + return '<Nikola Site: {0!r}>'.format(self.config['BLOG_TITLE'](self.config['DEFAULT_LANG'])) def sanitized_locales(locale_fallback, locale_default, locales, translations): @@ -2084,6 +2517,9 @@ def sanitized_locales(locale_fallback, locale_default, locales, translations): locale_n = locale_fallback msg = "Could not guess locale for language {0}, using locale {1}" utils.LOGGER.warn(msg.format(lang, locale_n)) + utils.LOGGER.warn("Please fix your OS locale configuration or use the LOCALES option in conf.py to specify your preferred locale.") + if sys.platform != 'win32': + utils.LOGGER.warn("Make sure to use an UTF-8 locale to ensure Unicode support.") locales[lang] = locale_n return locale_fallback, locale_default, locales @@ -2105,7 +2541,7 @@ def valid_locale_fallback(desired_locale=None): """ # Whenever fallbacks change, adjust test TestHarcodedFallbacksWork candidates_windows = [str('English'), str('C')] - candidates_posix = [str('en_US.utf8'), str('C')] + candidates_posix = [str('en_US.UTF-8'), str('C')] candidates = candidates_windows if sys.platform == 'win32' else candidates_posix if desired_locale: candidates = list(candidates) @@ -2139,7 +2575,7 @@ def guess_locale_from_lang_posix(lang): locale_n = str(lang) else: # this works in Travis when locale support set by Travis suggestion - locale_n = str((locale.normalize(lang).split('.')[0]) + '.utf8') + locale_n = str((locale.normalize(lang).split('.')[0]) + '.UTF-8') if not is_valid_locale(locale_n): # http://thread.gmane.org/gmane.comp.web.nikola/337/focus=343 locale_n = str((locale.normalize(lang).split('.')[0])) |