diff options
Diffstat (limited to 'nikola/utils.py')
| -rw-r--r-- | nikola/utils.py | 214 |
1 files changed, 144 insertions, 70 deletions
diff --git a/nikola/utils.py b/nikola/utils.py index 3359514..068cb3a 100644 --- a/nikola/utils.py +++ b/nikola/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -31,7 +31,6 @@ import calendar import datetime import dateutil.tz import hashlib -import husl import io import locale import logging @@ -56,6 +55,19 @@ except ImportError: from urllib.parse import urlparse, urlunparse # NOQA import warnings import PyRSS2Gen as rss +try: + import pytoml as toml +except ImportError: + toml = None +try: + import yaml +except ImportError: + yaml = None +try: + import husl +except ImportError: + husl = None + from collections import defaultdict, Callable, OrderedDict from logbook.compat import redirect_logging from logbook.more import ExceptionHandler, ColorizedStderrHandler @@ -69,7 +81,7 @@ from doit.cmdparse import CmdParse from nikola import DEBUG -__all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree', +__all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_path_real', 'get_theme_chain', 'load_messages', 'copy_tree', 'copy_file', 'slugify', 'unslugify', 'to_datetime', 'apply_filters', 'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path', '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary', @@ -81,7 +93,8 @@ __all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_chain', 'load_messages' 'adjust_name_for_index_path', 'adjust_name_for_index_link', 'NikolaPygmentsHTML', 'create_redirect', 'TreeNode', 'flatten_tree_structure', 'parse_escaped_hierarchical_category_name', - 'join_hierarchical_category_path', 'indent') + 'join_hierarchical_category_path', 'clean_before_deployment', 'indent', + 'load_data') # Are you looking for 'generic_rss_renderer'? # It's defined in nikola.nikola.Nikola (the site object). @@ -126,7 +139,7 @@ def get_logger(name, handlers): STDERR_HANDLER = [ColorfulStderrHandler( - level=logbook.NOTICE if not DEBUG else logbook.DEBUG, + level=logbook.INFO if not DEBUG else logbook.DEBUG, format_string=u'[{record.time:%Y-%m-%dT%H:%M:%SZ}] {record.level_name}: {record.channel}: {record.message}' )] @@ -302,7 +315,7 @@ class TranslatableSetting(object): self.overriden_default = False self.values = defaultdict() - if isinstance(inp, dict): + if isinstance(inp, dict) and inp: self.translated = True self.values.update(inp) if self.default_lang not in self.values.keys(): @@ -546,6 +559,8 @@ class config_changed(tools.config_changed): byte_data = data digest = hashlib.md5(byte_data).hexdigest() # LOGGER.debug('{{"{0}": {1}}}'.format(digest, byte_data)) + # Humanized format: + # LOGGER.debug('[Digest {0} for {2}]\n{1}\n[Digest {0} for {2}]'.format(digest, byte_data, self.identifier)) return digest else: raise Exception('Invalid type of config_changed parameter -- got ' @@ -570,24 +585,30 @@ class config_changed(tools.config_changed): sort_keys=True)) -def get_theme_path(theme, _themes_dir='themes'): +def get_theme_path_real(theme, themes_dirs): """Return the path where the given theme's files are located. Looks in ./themes and in the place where themes go when installed. """ - dir_name = os.path.join(_themes_dir, theme) - if os.path.isdir(dir_name): - return dir_name + for themes_dir in themes_dirs: + dir_name = os.path.join(themes_dir, theme) + if os.path.isdir(dir_name): + return dir_name dir_name = resource_filename('nikola', os.path.join('data', 'themes', theme)) if os.path.isdir(dir_name): return dir_name raise Exception("Can't find theme '{0}'".format(theme)) -def get_template_engine(themes, _themes_dir='themes'): +def get_theme_path(theme): + """Return the theme's path, which equals the theme's name.""" + return theme + + +def get_template_engine(themes): """Get template engine used by a given theme.""" for theme_name in themes: - engine_path = os.path.join(get_theme_path(theme_name, _themes_dir), 'engine') + engine_path = os.path.join(theme_name, 'engine') if os.path.isfile(engine_path): with open(engine_path) as fd: return fd.readlines()[0].strip() @@ -595,21 +616,24 @@ def get_template_engine(themes, _themes_dir='themes'): return 'mako' -def get_parent_theme_name(theme_name, _themes_dir='themes'): +def get_parent_theme_name(theme_name, themes_dirs=None): """Get name of parent theme.""" - parent_path = os.path.join(get_theme_path(theme_name, _themes_dir), 'parent') + parent_path = os.path.join(theme_name, 'parent') if os.path.isfile(parent_path): with open(parent_path) as fd: - return fd.readlines()[0].strip() + parent = fd.readlines()[0].strip() + if themes_dirs: + return get_theme_path_real(parent, themes_dirs) + return parent return None -def get_theme_chain(theme, _themes_dir='themes'): - """Create the full theme inheritance chain.""" - themes = [theme] +def get_theme_chain(theme, themes_dirs): + """Create the full theme inheritance chain including paths.""" + themes = [get_theme_path_real(theme, themes_dirs)] while True: - parent = get_parent_theme_name(themes[-1], _themes_dir) + parent = get_parent_theme_name(themes[-1], themes_dirs=themes_dirs) # Avoid silly loops if parent is None or parent in themes: break @@ -633,7 +657,7 @@ class LanguageNotFoundError(Exception): return 'cannot find language {0}'.format(self.lang) -def load_messages(themes, translations, default_lang): +def load_messages(themes, translations, default_lang, themes_dirs): """Load theme's messages into context. All the messages from parent themes are loaded, @@ -643,10 +667,12 @@ def load_messages(themes, translations, default_lang): oldpath = list(sys.path) for theme_name in themes[::-1]: msg_folder = os.path.join(get_theme_path(theme_name), 'messages') - default_folder = os.path.join(get_theme_path('base'), 'messages') + default_folder = os.path.join(get_theme_path_real('base', themes_dirs), 'messages') sys.path.insert(0, default_folder) sys.path.insert(0, msg_folder) english = __import__('messages_en') + # If we don't do the reload, the module is cached + _reload(english) for lang in list(translations.keys()): try: translation = __import__('messages_' + lang) @@ -665,6 +691,7 @@ def load_messages(themes, translations, default_lang): del(translation) except ImportError as orig: raise LanguageNotFoundError(lang, orig) + del(english) sys.path = oldpath return messages @@ -741,20 +768,22 @@ _slugify_strip_re = re.compile(r'[^+\w\s-]') _slugify_hyphenate_re = re.compile(r'[-\s]+') -def slugify(value, force=False): +def slugify(value, lang=None, force=False): u"""Normalize string, convert to lowercase, remove non-alpha characters, convert spaces to hyphens. From Django's "django/template/defaultfilters.py". - >>> print(slugify('áéí.óú')) + >>> print(slugify('áéí.óú', lang='en')) aeiou - >>> print(slugify('foo/bar')) + >>> print(slugify('foo/bar', lang='en')) foobar - >>> print(slugify('foo bar')) + >>> print(slugify('foo bar', lang='en')) foo-bar """ + if lang is None: # TODO: remove in v8 + LOGGER.warn("slugify() called without language!") if not isinstance(value, unicode_str): raise ValueError("Not a unicode object: {0}".format(value)) if USE_SLUGIFY or force: @@ -779,12 +808,14 @@ def slugify(value, force=False): return value -def unslugify(value, discard_numbers=True): +def unslugify(value, lang=None, discard_numbers=True): """Given a slug string (as a filename), return a human readable string. If discard_numbers is True, numbers right at the beginning of input will be removed. """ + if lang is None: # TODO: remove in v8 + LOGGER.warn("unslugify() called without language!") if discard_numbers: value = re.sub('^[0-9]+', '', value) value = re.sub('([_\-\.])', ' ', value) @@ -796,7 +827,7 @@ def encodelink(iri): """Given an encoded or unencoded link string, return an encoded string suitable for use as a link in HTML and XML.""" iri = unicodenormalize('NFC', iri) link = OrderedDict(urlparse(iri)._asdict()) - link['path'] = urlquote(urlunquote(link['path']).encode('utf-8')) + link['path'] = urlquote(urlunquote(link['path']).encode('utf-8'), safe="/~") try: link['netloc'] = link['netloc'].encode('utf-8').decode('idna').encode('idna').decode('utf-8') except UnicodeDecodeError: @@ -909,7 +940,7 @@ def apply_filters(task, filters, skip_ext=None): return task -def get_crumbs(path, is_file=False, index_folder=None): +def get_crumbs(path, is_file=False, index_folder=None, lang=None): """Create proper links for a crumb bar. index_folder is used if you want to use title from index file @@ -945,8 +976,10 @@ def get_crumbs(path, is_file=False, index_folder=None): for i, crumb in enumerate(crumbs[-3::-1]): # Up to parent folder only _path = '/'.join(['..'] * (i + 1)) _crumbs.append([_path, crumb]) - _crumbs.insert(0, ['.', crumbs[-2]]) # file's folder - _crumbs.insert(0, ['#', crumbs[-1]]) # file itself + if len(crumbs) >= 2: + _crumbs.insert(0, ['.', crumbs[-2]]) # file's folder + if len(crumbs) >= 1: + _crumbs.insert(0, ['#', crumbs[-1]]) # file itself else: for i, crumb in enumerate(crumbs[::-1]): _path = '/'.join(['..'] * i) or '#' @@ -962,12 +995,12 @@ def get_crumbs(path, is_file=False, index_folder=None): index_post = index_folder.parse_index(folder, '', '') folder = folder.replace(crumb, '') if index_post: - crumb = index_post.title() or crumb + crumb = index_post.title(lang) or crumb _crumbs[i][1] = crumb return list(reversed(_crumbs)) -def get_asset_path(path, themes, files_folders={'files': ''}, _themes_dir='themes', output_dir='output'): +def get_asset_path(path, themes, files_folders={'files': ''}, output_dir='output'): """Return the "real", absolute path to the asset. By default, it checks which theme provides the asset. @@ -976,27 +1009,24 @@ def get_asset_path(path, themes, files_folders={'files': ''}, _themes_dir='theme If it's not provided by either, it will be chacked in output, where it may have been created by another plugin. - >>> print(get_asset_path('assets/css/rst.css', ['bootstrap3', 'base'])) + >>> print(get_asset_path('assets/css/rst.css', get_theme_chain('bootstrap3', ['themes']))) /.../nikola/data/themes/base/assets/css/rst.css - >>> print(get_asset_path('assets/css/theme.css', ['bootstrap3', 'base'])) + >>> print(get_asset_path('assets/css/theme.css', get_theme_chain('bootstrap3', ['themes']))) /.../nikola/data/themes/bootstrap3/assets/css/theme.css - >>> print(get_asset_path('nikola.py', ['bootstrap3', 'base'], {'nikola': ''})) + >>> print(get_asset_path('nikola.py', get_theme_chain('bootstrap3', ['themes']), {'nikola': ''})) /.../nikola/nikola.py - >>> print(get_asset_path('nikola.py', ['bootstrap3', 'base'], {'nikola': 'nikola'})) + >>> print(get_asset_path('nikola.py', get_theme_chain('bootstrap3', ['themes']), {'nikola': 'nikola'})) None - >>> print(get_asset_path('nikola/nikola.py', ['bootstrap3', 'base'], {'nikola': 'nikola'})) + >>> print(get_asset_path('nikola/nikola.py', get_theme_chain('bootstrap3', ['themes']), {'nikola': 'nikola'})) /.../nikola/nikola.py """ for theme_name in themes: - candidate = os.path.join( - get_theme_path(theme_name, _themes_dir), - path - ) + candidate = os.path.join(get_theme_path(theme_name), path) if os.path.isfile(candidate): return candidate for src, rel_dst in files_folders.items(): @@ -1054,7 +1084,6 @@ class LocaleBorg(object): NOTE: never use locale.getlocale() , it can return values that locale.setlocale will not accept in Windows XP, 7 and pythons 2.6, 2.7, 3.3 Examples: "Spanish", "French" can't do the full circle set / get / set - That used to break calendar, but now seems is not the case, with month at least """ initialized = False @@ -1173,20 +1202,16 @@ class LocaleBorg(object): res = handler(month_no, lang) if res is not None: return res - if sys.version_info[0] == 3: # Python 3 - with calendar.different_locale(self.locales[lang]): - s = calendar.month_name[month_no] - # for py3 s is unicode - else: # Python 2 - with calendar.TimeEncoding(self.locales[lang]): - s = calendar.month_name[month_no] + old_lang = self.current_lang + self.__set_locale(lang) + s = calendar.month_name[month_no] + self.__set_locale(old_lang) + if sys.version_info[0] == 2: enc = self.encodings[lang] if not enc: enc = 'UTF-8' s = s.decode(enc) - # paranoid about calendar ending in the wrong locale (windows) - self.__set_locale(self.current_lang) return s def formatted_date(self, date_format, date): @@ -1294,9 +1319,9 @@ def demote_headers(doc, level=1): r = range(1 + level, 7) for i in reversed(r): # html headers go to 6, so we can’t “lower” beneath five - elements = doc.xpath('//h' + str(i)) - for e in elements: - e.tag = 'h' + str(i + level) + elements = doc.xpath('//h' + str(i)) + for e in elements: + e.tag = 'h' + str(i + level) def get_root_dir(): @@ -1337,10 +1362,10 @@ def get_translation_candidate(config, path, lang): cache/posts/fancy.post.html >>> print(get_translation_candidate(config, 'cache/posts/fancy.post.html', 'es')) cache/posts/fancy.post.es.html - >>> print(get_translation_candidate(config, 'cache/stories/charts.html', 'es')) - cache/stories/charts.es.html - >>> print(get_translation_candidate(config, 'cache/stories/charts.html', 'en')) - cache/stories/charts.html + >>> print(get_translation_candidate(config, 'cache/pages/charts.html', 'es')) + cache/pages/charts.es.html + >>> print(get_translation_candidate(config, 'cache/pages/charts.html', 'en')) + cache/pages/charts.html >>> config = {'TRANSLATIONS_PATTERN': '{path}.{ext}.{lang}', 'DEFAULT_LANG': 'en', 'TRANSLATIONS': {'es':'1', 'en': 1}} >>> print(get_translation_candidate(config, '*.rst', 'es')) @@ -1553,7 +1578,7 @@ class NikolaPygmentsHTML(HtmlFormatter): self.nclasses = classes super(NikolaPygmentsHTML, self).__init__( cssclass='code', linenos=linenos, linenostart=linenostart, nowrap=False, - lineanchors=slugify(anchor_ref, force=True), anchorlinenos=True) + lineanchors=slugify(anchor_ref, lang=LocaleBorg().current_lang, force=True), anchorlinenos=True) def wrap(self, source, outfile): """Wrap the ``source``, which is a generator yielding individual lines, in custom generators.""" @@ -1790,21 +1815,29 @@ def colorize_str_from_base_color(string, base_color): lightness and saturation untouched using HUSL colorspace. """ def hash_str(string, pos): - return hashlib.md5(string.encode('utf-8')).digest()[pos] + x = hashlib.md5(string.encode('utf-8')).digest()[pos] + try: + # Python 2: a string + # TODO: remove in v8 + return ord(x) + except TypeError: + # Python 3: already an integer + return x def degreediff(dega, degb): return min(abs(dega - degb), abs((degb - dega) + 360)) - def husl_similar_from_base(string, base_color): - h, s, l = husl.hex_to_husl(base_color) - old_h = h - idx = 0 - while degreediff(old_h, h) < 27 and idx < 16: - h = 360.0 * (float(hash_str(string, idx)) / 255) - idx += 1 - return husl.husl_to_hex(h, s, l) - - return husl_similar_from_base(string, base_color) + if husl is None: + req_missing(['husl'], 'Use color mixing (section colors)', + optional=True) + return base_color + h, s, l = husl.hex_to_husl(base_color) + old_h = h + idx = 0 + while degreediff(old_h, h) < 27 and idx < 16: + h = 360.0 * (float(hash_str(string, idx)) / 255) + idx += 1 + return husl.husl_to_hex(h, s, l) def color_hsl_adjust_hex(hexstr, adjust_h=None, adjust_s=None, adjust_l=None): @@ -1849,6 +1882,25 @@ def dns_sd(port, inet6): return None +def clean_before_deployment(site): + """Clean drafts and future posts before deployment.""" + undeployed_posts = [] + deploy_drafts = site.config.get('DEPLOY_DRAFTS', True) + deploy_future = site.config.get('DEPLOY_FUTURE', False) + if not (deploy_drafts and deploy_future): # == !drafts || !future + # Remove drafts and future posts + out_dir = site.config['OUTPUT_FOLDER'] + site.scan_posts() + for post in site.timeline: + if (not deploy_drafts and post.is_draft) or (not deploy_future and post.publish_later): + for lang in post.translated_to: + remove_file(os.path.join(out_dir, post.destination_path(lang))) + source_path = post.destination_path(lang, post.source_ext(True)) + remove_file(os.path.join(out_dir, source_path)) + undeployed_posts.append(post) + return undeployed_posts + + # Stolen from textwrap in Python 3.4.3. def indent(text, prefix, predicate=None): """Add 'prefix' to the beginning of selected lines in 'text'. @@ -1866,3 +1918,25 @@ def indent(text, prefix, predicate=None): for line in text.splitlines(True): yield (prefix + line if predicate(line) else line) return ''.join(prefixed_lines()) + + +def load_data(path): + """Given path to a file, load data from it.""" + ext = os.path.splitext(path)[-1] + loader = None + if ext in {'.yml', '.yaml'}: + loader = yaml + if yaml is None: + req_missing(['yaml'], 'use YAML data files') + return {} + elif ext in {'.json', '.js'}: + loader = json + elif ext in {'.toml', '.tml'}: + if toml is None: + req_missing(['toml'], 'use TOML data files') + return {} + loader = toml + if loader is None: + return + with io.open(path, 'r', encoding='utf8') as inf: + return loader.load(inf) |