diff options
Diffstat (limited to 'nikola/utils.py')
| -rw-r--r-- | nikola/utils.py | 477 |
1 files changed, 404 insertions, 73 deletions
diff --git a/nikola/utils.py b/nikola/utils.py index 87826ff..3708775 100644 --- a/nikola/utils.py +++ b/nikola/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,32 +27,66 @@ """Utility functions.""" from __future__ import print_function, unicode_literals, absolute_import -from collections import defaultdict, Callable import calendar import datetime import dateutil.tz import hashlib +import io import locale import logging +import natsort import os import re import json import shutil import subprocess import sys -from zipfile import ZipFile as zipf -try: - from imp import reload -except ImportError: - pass - import dateutil.parser import dateutil.tz import logbook +import warnings +import PyRSS2Gen as rss +from collections import defaultdict, Callable from logbook.more import ExceptionHandler, ColorizedStderrHandler +from pygments.formatters import HtmlFormatter +from zipfile import ZipFile as zipf +from doit import tools +from unidecode import unidecode +from pkg_resources import resource_filename +from doit.cmdparse import CmdParse from nikola import DEBUG +__all__ = ['CustomEncoder', 'get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree', + 'copy_file', 'slugify', 'unslugify', 'to_datetime', 'apply_filters', + 'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path', + '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary', + 'TranslatableSetting', 'TemplateHookRegistry', 'LocaleBorg', + 'sys_encode', 'sys_decode', 'makedirs', 'get_parent_theme_name', + 'demote_headers', 'get_translation_candidate', 'write_metadata', + 'ask', 'ask_yesno', 'options2docstring', 'os_path_split', + 'get_displayed_page_number', 'adjust_name_for_index_path_list', + 'adjust_name_for_index_path', 'adjust_name_for_index_link', + 'NikolaPygmentsHTML', 'create_redirect', 'TreeNode', + 'flatten_tree_structure', 'parse_escaped_hierarchical_category_name', + 'join_hierarchical_category_path', 'indent'] + +# Are you looking for 'generic_rss_renderer'? +# It's defined in nikola.nikola.Nikola (the site object). + +if sys.version_info[0] == 3: + # Python 3 + bytes_str = bytes + unicode_str = str + unichr = chr + raw_input = input + from imp import reload as _reload +else: + bytes_str = str + unicode_str = unicode # NOQA + _reload = reload # NOQA + unichr = unichr + class ApplicationWarning(Exception): pass @@ -72,9 +106,9 @@ def get_logger(name, handlers): l = logbook.Logger(name) for h in handlers: if isinstance(h, list): - l.handlers = h + l.handlers += h else: - l.handlers = [h] + l.handlers.append(h) return l @@ -97,9 +131,6 @@ else: logging.basicConfig(level=logging.INFO) -import warnings - - def showwarning(message, category, filename, lineno, file=None, line=None): """Show a warning (from the warnings subsystem) to the user.""" try: @@ -156,39 +187,8 @@ def req_missing(names, purpose, python=True, optional=False): return msg -if sys.version_info[0] == 3: - # Python 3 - bytes_str = bytes - unicode_str = str - unichr = chr - raw_input = input - from imp import reload as _reload -else: - bytes_str = str - unicode_str = unicode # NOQA - _reload = reload # NOQA - unichr = unichr - -from doit import tools -from unidecode import unidecode -from pkg_resources import resource_filename -from nikola import filters as task_filters - -import PyRSS2Gen as rss - -__all__ = ['get_theme_path', 'get_theme_chain', 'load_messages', 'copy_tree', - 'copy_file', 'slugify', 'unslugify', 'to_datetime', 'apply_filters', - 'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path', - '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary', - 'TranslatableSetting', 'TemplateHookRegistry', 'LocaleBorg', - 'sys_encode', 'sys_decode', 'makedirs', 'get_parent_theme_name', - 'demote_headers', 'get_translation_candidate', 'write_metadata', - 'ask', 'ask_yesno'] - -# Are you looking for 'generic_rss_renderer'? -# It's defined in nikola.nikola.Nikola (the site object). - +from nikola import filters as task_filters # NOQA ENCODING = sys.getfilesystemencoding() or sys.stdin.encoding @@ -208,11 +208,20 @@ def sys_decode(thing): def makedirs(path): """Create a folder.""" - if not path or os.path.isdir(path): + if not path: return if os.path.exists(path): - raise OSError('Path {0} already exists and is not a folder.') - os.makedirs(path) + if not os.path.isdir(path): + raise OSError('Path {0} already exists and is not a folder.'.format(path)) + else: + return + try: + os.makedirs(path) + return + except Exception: + if os.path.isdir(path): + return + raise class Functionary(defaultdict): @@ -368,8 +377,10 @@ class TranslatableSetting(object): for a in f[0] + tuple(f[1].values()): if isinstance(a, dict): langkeys += list(a) + # Now that we know all this, we go through all the languages we have. allvalues = set(keys + langkeys + list(self.values)) + self.values['__orig__'] = self.values[self.default_lang] for l in allvalues: if l in keys: oargs, okwargs = formats[l] @@ -383,19 +394,22 @@ class TranslatableSetting(object): # We create temporary TranslatableSettings and replace the # values with them. if isinstance(a, dict): - a = TranslatableSetting('NULL', a) + a = TranslatableSetting('NULL', a, self.translations) args.append(a(l)) else: args.append(a) for k, v in okwargs.items(): if isinstance(v, dict): - v = TranslatableSetting('NULL', v) + v = TranslatableSetting('NULL', v, self.translations) kwargs.update({k: v(l)}) else: kwargs.update({k: v}) - self.values[l] = self.values[l].format(*args, **kwargs) + if l in self.values: + self.values[l] = self.values[l].format(*args, **kwargs) + else: + self.values[l] = self.values['__orig__'].format(*args, **kwargs) self.values.default_factory = lambda: self.values[self.default_lang] return self @@ -472,7 +486,7 @@ class TemplateHookRegistry(object): self._items.append((c, inp, wants_site_and_context, args, kwargs)) def __hash__(self): - return config_changed({self.name: self._items}) + return hash(config_changed({self.name: self._items})._calc_digest()) def __str__(self): return '<TemplateHookRegistry: {0}>'.format(self._items) @@ -490,6 +504,12 @@ class CustomEncoder(json.JSONEncoder): class config_changed(tools.config_changed): """ A copy of doit's but using pickle instead of serializing manually.""" + def __init__(self, config, identifier=None): + super(config_changed, self).__init__(config) + self.identifier = '_config_changed' + if identifier is not None: + self.identifier += ':' + identifier + def _calc_digest(self): if isinstance(self.config, str): return self.config @@ -507,6 +527,16 @@ class config_changed(tools.config_changed): '{0}, must be string or dict'.format(type( self.config))) + def configure_task(self, task): + task.value_savers.append(lambda: {self.identifier: self._calc_digest()}) + + def __call__(self, task, values): + """Return True if config values are unchanged.""" + last_success = values.get(self.identifier) + if last_success is None: + return False + return (last_success == self._calc_digest()) + def __repr__(self): return "Change with config: {0}".format(json.dumps(self.config, cls=CustomEncoder)) @@ -576,7 +606,7 @@ def load_messages(themes, translations, default_lang): and "younger" themes have priority. """ messages = Functionary(dict, default_lang) - oldpath = sys.path[:] + oldpath = list(sys.path) for theme_name in themes[::-1]: msg_folder = os.path.join(get_theme_path(theme_name), 'messages') default_folder = os.path.join(get_theme_path('base'), 'messages') @@ -587,7 +617,7 @@ def load_messages(themes, translations, default_lang): try: translation = __import__('messages_' + lang) # If we don't do the reload, the module is cached - reload(translation) + _reload(translation) if sorted(translation.MESSAGES.keys()) !=\ sorted(english.MESSAGES.keys()) and \ lang not in warned: @@ -877,7 +907,10 @@ def get_crumbs(path, is_file=False, index_folder=None): for i, crumb in enumerate(crumbs[::-1]): if folder[-1] == os.sep: folder = folder[:-1] - index_post = index_folder.parse_index(folder) + # We don't care about the created Post() object except for its title; + # hence, the input_folder and output_folder given to + # index_folder.parse_index() don't matter + index_post = index_folder.parse_index(folder, '', '') folder = folder.replace(crumb, '') if index_post: crumb = index_post.title() or crumb @@ -1038,6 +1071,25 @@ class LocaleBorg(object): return s +class ExtendedRSS2(rss.RSS2): + xsl_stylesheet_href = None + + def publish(self, handler): + if self.xsl_stylesheet_href: + handler.processingInstruction("xml-stylesheet", 'type="text/xsl" href="{0}" media="all"'.format(self.xsl_stylesheet_href)) + # old-style class in py2 + rss.RSS2.publish(self, handler) + + def publish_extensions(self, handler): + if self.self_url: + handler.startElement("atom:link", { + 'href': self.self_url, + 'rel': "self", + 'type': "application/rss+xml" + }) + handler.endElement("atom:link") + + class ExtendedItem(rss.RSSItem): def __init__(self, **kw): @@ -1096,8 +1148,13 @@ def get_root_dir(): """Find root directory of nikola installation by looking for conf.py""" root = os.getcwd() + if sys.version_info[0] == 2: + confname = b'conf.py' + else: + confname = 'conf.py' + while True: - if os.path.exists(os.path.join(root, 'conf.py')): + if os.path.exists(os.path.join(root, confname)): return root else: basedir = os.path.split(root)[0] @@ -1174,7 +1231,7 @@ def get_translation_candidate(config, path, lang): def write_metadata(data): """Write metadata.""" - order = ('title', 'slug', 'date', 'tags', 'link', 'description', 'type') + order = ('title', 'slug', 'date', 'tags', 'category', 'link', 'description', 'type') f = '.. {0}: {1}' meta = [] for k in order: @@ -1184,8 +1241,8 @@ def write_metadata(data): pass # Leftover metadata (user-specified/non-default). - for k, v in data.items(): - meta.append(f.format(k, v)) + for k in natsort.natsorted(list(data.keys()), alg=natsort.ns.F | natsort.ns.IC): + meta.append(f.format(k, data[k])) meta.append('') @@ -1198,7 +1255,10 @@ def ask(query, default=None): default_q = ' [{0}]'.format(default) else: default_q = '' - inp = raw_input("{query}{default_q}: ".format(query=query, default_q=default_q)).strip() + if sys.version_info[0] == 3: + inp = raw_input("{query}{default_q}: ".format(query=query, default_q=default_q)).strip() + else: + inp = raw_input("{query}{default_q}: ".format(query=query, default_q=default_q).encode('utf-8')).strip() if inp or default is None: return inp else: @@ -1213,7 +1273,10 @@ def ask_yesno(query, default=None): default_q = ' [Y/n]' elif default is False: default_q = ' [y/N]' - inp = raw_input("{query}{default_q} ".format(query=query, default_q=default_q)).strip() + if sys.version_info[0] == 3: + inp = raw_input("{query}{default_q} ".format(query=query, default_q=default_q)).strip() + else: + inp = raw_input("{query}{default_q} ".format(query=query, default_q=default_q).encode('utf-8')).strip() if inp: return inp.lower().startswith('y') elif default is not None: @@ -1223,10 +1286,6 @@ def ask_yesno(query, default=None): return ask_yesno(query, default) -from nikola.plugin_categories import Command -from doit.cmdparse import CmdParse - - class CommandWrapper(object): """Converts commands into functions.""" @@ -1253,32 +1312,58 @@ class Commands(object): >>> commands.check(list=True) # doctest: +SKIP """ - def __init__(self, main): + def __init__(self, main, config, doitargs): """Takes a main instance, works as wrapper for commands.""" self._cmdnames = [] - for k, v in main.get_commands().items(): - self._cmdnames.append(k) + self._main = main + self._config = config + self._doitargs = doitargs + try: + cmdict = self._doitargs['cmds'].to_dict() + except AttributeError: # not a doit PluginDict + cmdict = self._doitargs['cmds'] + for k, v in cmdict.items(): + # cleanup: run is doit-only, init is useless in an existing site if k in ['run', 'init']: continue if sys.version_info[0] == 2: k2 = bytes(k) else: k2 = k + + self._cmdnames.append(k) + + try: + # nikola command: already instantiated (singleton) + opt = v.get_options() + except TypeError: + # doit command: needs some help + opt = v(config=self._config, **self._doitargs).get_options() nc = type( k2, (CommandWrapper,), { - '__doc__': options2docstring(k, main.sub_cmds[k].options) + '__doc__': options2docstring(k, opt) }) setattr(self, k, nc(k, self)) - self.main = main def _run(self, cmd_args): - self.main.run(cmd_args) + self._main.run(cmd_args) def _run_with_kw(self, cmd, *a, **kw): - cmd = self.main.sub_cmds[cmd] - options, _ = CmdParse(cmd.options).parse([]) + # cyclic import hack + from nikola.plugin_categories import Command + try: + cmd = self._doitargs['cmds'].get_plugin(cmd) + except AttributeError: # not a doit PluginDict + cmd = self._doitargs['cmds'][cmd] + try: + opt = cmd.get_options() + except TypeError: + cmd = cmd(config=self._config, **self._doitargs) + opt = cmd.get_options() + + options, _ = CmdParse(opt).parse([]) options.update(kw) if isinstance(cmd, Command): cmd.execute(options=options, args=a) @@ -1305,3 +1390,249 @@ def options2docstring(name, options): for opt in options: result.append('{0} type {1} default {2}'.format(opt.name, opt.type.__name__, opt.default)) return '\n'.join(result) + + +class NikolaPygmentsHTML(HtmlFormatter): + """A Nikola-specific modification of Pygments’ HtmlFormatter.""" + def __init__(self, anchor_ref, classes=None, linenos='table', linenostart=1): + if classes is None: + classes = ['code', 'literal-block'] + self.nclasses = classes + super(NikolaPygmentsHTML, self).__init__( + cssclass='code', linenos=linenos, linenostart=linenostart, nowrap=False, + lineanchors=slugify(anchor_ref, force=True), anchorlinenos=True) + + def wrap(self, source, outfile): + """ + Wrap the ``source``, which is a generator yielding + individual lines, in custom generators. + """ + + style = [] + if self.prestyles: + style.append(self.prestyles) + if self.noclasses: + style.append('line-height: 125%') + style = '; '.join(style) + classes = ' '.join(self.nclasses) + + yield 0, ('<pre class="{0}"'.format(classes) + (style and ' style="{0}"'.format(style)) + '>') + for tup in source: + yield tup + yield 0, '</pre>' + + +def get_displayed_page_number(i, num_pages, site): + if not i: + i = 0 + if site.config["INDEXES_STATIC"]: + return i if i > 0 else num_pages + else: + return i + 1 if site.config["INDEXES_PAGES_MAIN"] else i + + +def adjust_name_for_index_path_list(path_list, i, displayed_i, lang, site, force_addition=False, extension=None): + index_file = site.config["INDEX_FILE"] + if i or force_addition: + path_list = list(path_list) + if force_addition and not i: + i = 0 + if not extension: + _, extension = os.path.splitext(index_file) + if len(path_list) > 0 and path_list[-1] == '': + path_list[-1] = index_file + elif len(path_list) == 0 or not path_list[-1].endswith(extension): + path_list.append(index_file) + if site.config["PRETTY_URLS"] and site.config["INDEXES_PRETTY_PAGE_URL"](lang) and path_list[-1] == index_file: + path_schema = site.config["INDEXES_PRETTY_PAGE_URL"](lang) + if isinstance(path_schema, (bytes_str, unicode_str)): + path_schema = [path_schema] + else: + path_schema = None + if path_schema is not None: + del path_list[-1] + for entry in path_schema: + path_list.append(entry.format(number=displayed_i, old_number=i, index_file=index_file)) + else: + path_list[-1] = '{0}-{1}{2}'.format(os.path.splitext(path_list[-1])[0], i, extension) + return path_list + + +def os_path_split(path): + result = [] + while True: + previous_path = path + path, tail = os.path.split(path) + if path == previous_path and tail == '': + result.insert(0, path) + break + result.insert(0, tail) + if len(path) == 0: + break + return result + + +def adjust_name_for_index_path(name, i, displayed_i, lang, site, force_addition=False, extension=None): + return os.path.join(*adjust_name_for_index_path_list(os_path_split(name), i, displayed_i, lang, site, force_addition, extension)) + + +def adjust_name_for_index_link(name, i, displayed_i, lang, site, force_addition=False, extension=None): + link = adjust_name_for_index_path_list(name.split('/'), i, displayed_i, lang, site, force_addition, extension) + if not extension == ".atom": + if len(link) > 0 and link[-1] == site.config["INDEX_FILE"] and site.config["STRIP_INDEXES"]: + link[-1] = '' + return '/'.join(link) + + +def create_redirect(src, dst): + makedirs(os.path.dirname(src)) + with io.open(src, "w+", encoding="utf8") as fd: + fd.write('<!DOCTYPE html>\n<head>\n<meta charset="utf-8">\n' + '<title>Redirecting...</title>\n<meta name="robots" ' + 'content="noindex">\n<meta http-equiv="refresh" content="0; ' + 'url={0}">\n</head>\n<body>\n<p>Page moved ' + '<a href="{0}">here</a>.</p>\n</body>'.format(dst)) + + +class TreeNode(object): + indent_levels = None # use for formatting comments as tree + indent_change_before = 0 # use for formatting comments as tree + indent_change_after = 0 # use for formatting comments as tree + + # The indent levels and changes allow to render a tree structure + # without keeping track of all that information during rendering. + # + # The indent_change_before is the different between the current + # comment's level and the previous comment's level; if the number + # is positive, the current level is indented further in, and if it + # is negative, it is indented further out. Positive values can be + # used to open HTML tags for each opened level. + # + # The indent_change_after is the difference between the next + # comment's level and the current comment's level. Negative values + # can be used to close HTML tags for each closed level. + # + # The indent_levels list contains one entry (index, count) per + # level, informing about the index of the current comment on that + # level and the count of comments on that level (before a comment + # of a higher level comes). This information can be used to render + # tree indicators, for example to generate a tree such as: + # + # +--- [(0,3)] + # +-+- [(1,3)] + # | +--- [(1,3), (0,2)] + # | +-+- [(1,3), (1,2)] + # | +--- [(1,3), (1,2), (0, 1)] + # +-+- [(2,3)] + # +- [(2,3), (0,1)] + # + # (The lists used as labels represent the content of the + # indent_levels property for that node.) + + def __init__(self, name, parent=None): + self.name = name + self.parent = parent + self.children = [] + + def get_path(self): + path = [] + curr = self + while curr is not None: + path.append(curr) + curr = curr.parent + return reversed(path) + + def get_children(self): + return self.children + + +def flatten_tree_structure(root_list): + elements = [] + + def generate(input_list, indent_levels_so_far): + for index, element in enumerate(input_list): + # add to destination + elements.append(element) + # compute and set indent levels + indent_levels = indent_levels_so_far + [(index, len(input_list))] + element.indent_levels = indent_levels + # add children + children = element.get_children() + element.children_count = len(children) + generate(children, indent_levels) + + generate(root_list, []) + # Add indent change counters + level = 0 + last_element = None + for element in elements: + new_level = len(element.indent_levels) + # Compute level change before this element + change = new_level - level + if last_element is not None: + last_element.indent_change_after = change + element.indent_change_before = change + # Update variables + level = new_level + last_element = element + # Set level change after last element + if last_element is not None: + last_element.indent_change_after = -level + return elements + + +def parse_escaped_hierarchical_category_name(category_name): + result = [] + current = None + index = 0 + next_backslash = category_name.find('\\', index) + next_slash = category_name.find('/', index) + while index < len(category_name): + if next_backslash == -1 and next_slash == -1: + current = (current if current else "") + category_name[index:] + index = len(category_name) + elif next_slash >= 0 and (next_backslash == -1 or next_backslash > next_slash): + result.append((current if current else "") + category_name[index:next_slash]) + current = '' + index = next_slash + 1 + next_slash = category_name.find('/', index) + else: + if len(category_name) == next_backslash + 1: + raise Exception("Unexpected '\\' in '{0}' at last position!".format(category_name)) + esc_ch = category_name[next_backslash + 1] + if esc_ch not in {'/', '\\'}: + raise Exception("Unknown escape sequence '\\{0}' in '{1}'!".format(esc_ch, category_name)) + current = (current if current else "") + category_name[index:next_backslash] + esc_ch + index = next_backslash + 2 + next_backslash = category_name.find('\\', index) + if esc_ch == '/': + next_slash = category_name.find('/', index) + if current is not None: + result.append(current) + return result + + +def join_hierarchical_category_path(category_path): + def escape(s): + return s.replace('\\', '\\\\').replace('/', '\\/') + + return '/'.join([escape(p) for p in category_path]) + + +# Stolen from textwrap in Python 3.4.3. +def indent(text, prefix, predicate=None): + """Adds 'prefix' to the beginning of selected lines in 'text'. + + If 'predicate' is provided, 'prefix' will only be added to the lines + where 'predicate(line)' is True. If 'predicate' is not provided, + it will default to adding 'prefix' to all non-empty lines that do not + consist solely of whitespace characters. + """ + if predicate is None: + def predicate(line): + return line.strip() + + def prefixed_lines(): + for line in text.splitlines(True): + yield (prefix + line if predicate(line) else line) + return ''.join(prefixed_lines()) |