diff options
Diffstat (limited to 'nikola/plugins')
124 files changed, 2807 insertions, 1085 deletions
diff --git a/nikola/plugins/basic_import.py b/nikola/plugins/basic_import.py index 073a539..cf98ebc 100644 --- a/nikola/plugins/basic_import.py +++ b/nikola/plugins/basic_import.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -48,7 +48,6 @@ links = {} class ImportMixin(object): - """Mixin with common used methods.""" name = "import_mixin" @@ -77,8 +76,11 @@ class ImportMixin(object): return channel @staticmethod - def configure_redirections(url_map): + def configure_redirections(url_map, base_dir=''): """Configure redirections from an url_map.""" + index = base_dir + 'index.html' + if index.startswith('/'): + index = index[1:] redirections = [] for k, v in url_map.items(): if not k[-1] == '/': @@ -87,11 +89,10 @@ class ImportMixin(object): # remove the initial "/" because src is a relative file path src = (urlparse(k).path + 'index.html')[1:] dst = (urlparse(v).path) - if src == 'index.html': + if src == index: utils.LOGGER.warn("Can't do a redirect for: {0!r}".format(k)) else: redirections.append((src, dst)) - return redirections def generate_base_site(self): @@ -126,9 +127,12 @@ class ImportMixin(object): def write_content(cls, filename, content, rewrite_html=True): """Write content to file.""" if rewrite_html: - doc = html.document_fromstring(content) - doc.rewrite_links(replacer) - content = html.tostring(doc, encoding='utf8') + try: + doc = html.document_fromstring(content) + doc.rewrite_links(replacer) + content = html.tostring(doc, encoding='utf8') + except etree.ParserError: + content = content.encode('utf-8') else: content = content.encode('utf-8') @@ -136,6 +140,24 @@ class ImportMixin(object): with open(filename, "wb+") as fd: fd.write(content) + @classmethod + def write_post(cls, filename, content, headers, compiler, rewrite_html=True): + """Ask the specified compiler to write the post to disk.""" + if rewrite_html: + try: + doc = html.document_fromstring(content) + doc.rewrite_links(replacer) + content = html.tostring(doc, encoding='utf8') + except etree.ParserError: + pass + if isinstance(content, utils.bytes_str): + content = content.decode('utf-8') + compiler.create_post( + filename, + content=content, + onefile=True, + **headers) + @staticmethod def write_metadata(filename, title, slug, post_date, description, tags, **kwargs): """Write metadata to meta file.""" diff --git a/nikola/plugins/command/__init__.py b/nikola/plugins/command/__init__.py index 2aa5267..62d7086 100644 --- a/nikola/plugins/command/__init__.py +++ b/nikola/plugins/command/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/auto.plugin b/nikola/plugins/command/auto.plugin index 3e2b17d..1081c78 100644 --- a/nikola/plugins/command/auto.plugin +++ b/nikola/plugins/command/auto.plugin @@ -5,7 +5,7 @@ module = auto [Documentation] author = Roberto Alsina version = 2.1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Automatically detect site changes, rebuild and optionally refresh a browser. [Nikola] diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py index 71f9624..a82dc3e 100644 --- a/nikola/plugins/command/auto/__init__.py +++ b/nikola/plugins/command/auto/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -43,6 +43,7 @@ except ImportError: import webbrowser from wsgiref.simple_server import make_server import wsgiref.util +import pkg_resources from blinker import signal try: @@ -61,9 +62,8 @@ except ImportError: FileSystemEventHandler = object PatternMatchingEventHandler = object - from nikola.plugin_categories import Command -from nikola.utils import req_missing, get_logger, get_theme_path, STDERR_HANDLER +from nikola.utils import dns_sd, req_missing, get_logger, get_theme_path, STDERR_HANDLER LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js') error_signal = signal('error') refresh_signal = signal('refresh') @@ -79,13 +79,14 @@ ERROR {} class CommandAuto(Command): - """Automatic rebuilds for Nikola.""" name = "auto" logger = None has_server = True doc_purpose = "builds and serves a site; automatically detects site changes, rebuilds, and optionally refreshes a browser" + dns_sd = None + cmd_options = [ { 'name': 'port', @@ -101,7 +102,7 @@ class CommandAuto(Command): 'long': 'address', 'type': str, 'default': '127.0.0.1', - 'help': 'Address to bind (default: 127.0.0.1 – localhost)', + 'help': 'Address to bind (default: 127.0.0.1 -- localhost)', }, { 'name': 'browser', @@ -142,7 +143,7 @@ class CommandAuto(Command): self.cmd_arguments = ['nikola', 'build'] if self.site.configuration_filename != 'conf.py': - self.cmd_arguments = ['--conf=' + self.site.configuration_filename] + self.cmd_arguments + self.cmd_arguments.append('--conf=' + self.site.configuration_filename) # Run an initial build so we are up-to-date subprocess.call(self.cmd_arguments) @@ -156,7 +157,7 @@ class CommandAuto(Command): # Do not duplicate entries -- otherwise, multiple rebuilds are triggered watched = set([ - 'templates/', + 'templates/' ] + [get_theme_path(name) for name in self.site.THEMES]) for item in self.site.config['post_pages']: watched.add(os.path.dirname(item[0])) @@ -166,6 +167,10 @@ class CommandAuto(Command): watched.add(item) for item in self.site.config['LISTINGS_FOLDERS']: watched.add(item) + for item in self.site._plugin_places: + watched.add(item) + # Nikola itself (useful for developers) + watched.add(pkg_resources.resource_filename('nikola', '')) out_folder = self.site.config['OUTPUT_FOLDER'] if options and options.get('browser'): @@ -208,7 +213,6 @@ class CommandAuto(Command): parent = self class Mixed(WebSocketWSGIApplication): - """A class that supports WS and HTTP protocols on the same port.""" def __call__(self, environ, start_response): @@ -235,9 +239,12 @@ class CommandAuto(Command): webbrowser.open('http://{0}:{1}'.format(host, port)) try: + self.dns_sd = dns_sd(port, (options['ipv6'] or '::' in host)) ws.serve_forever() except KeyboardInterrupt: self.logger.info("Server is shutting down.") + if self.dns_sd: + self.dns_sd.Reset() # This is a hack, but something is locking up in a futex # and exit() doesn't work. os.kill(os.getpid(), 15) @@ -262,6 +269,8 @@ class CommandAuto(Command): fname = os.path.basename(event_path) if (fname.endswith('~') or fname.startswith('.') or + '__pycache__' in event_path or + event_path.endswith(('.pyc', '.pyo', '.pyd')) or os.path.isdir(event_path)): # Skip on folders, these are usually duplicates return self.logger.info('REBUILDING SITE (from {0})'.format(event_path)) @@ -293,18 +302,21 @@ class CommandAuto(Command): mimetype = 'text/html' if uri.endswith('/') else mimetypes.guess_type(uri)[0] or 'application/octet-stream' if os.path.isdir(f_path): - if not f_path.endswith('/'): # Redirect to avoid breakage - start_response('301 Redirect', [('Location', p_uri.path + '/')]) + if not p_uri.path.endswith('/'): # Redirect to avoid breakage + start_response('301 Moved Permanently', [('Location', p_uri.path + '/')]) return [] f_path = os.path.join(f_path, self.site.config['INDEX_FILE']) mimetype = 'text/html' if p_uri.path == '/robots.txt': - start_response('200 OK', [('Content-type', 'text/plain')]) + start_response('200 OK', [('Content-type', 'text/plain; charset=UTF-8')]) return ['User-Agent: *\nDisallow: /\n'.encode('utf-8')] elif os.path.isfile(f_path): with open(f_path, 'rb') as fd: - start_response('200 OK', [('Content-type', mimetype)]) + if mimetype.startswith('text/') or mimetype.endswith('+xml'): + start_response('200 OK', [('Content-type', "{0}; charset=UTF-8".format(mimetype))]) + else: + start_response('200 OK', [('Content-type', mimetype)]) return [self.file_filter(mimetype, fd.read())] elif p_uri.path == '/livereload.js': with open(LRJS_PATH, 'rb') as fd: @@ -337,7 +349,6 @@ pending = [] class LRSocket(WebSocket): - """Speak Livereload protocol.""" def __init__(self, *a, **kw): @@ -410,7 +421,6 @@ class LRSocket(WebSocket): class OurWatchHandler(FileSystemEventHandler): - """A Nikola-specific handler for Watchdog.""" def __init__(self, function): @@ -424,7 +434,6 @@ class OurWatchHandler(FileSystemEventHandler): class ConfigWatchHandler(FileSystemEventHandler): - """A Nikola-specific handler for Watchdog that handles the config file (as a workaround).""" def __init__(self, configuration_filename, function): diff --git a/nikola/plugins/command/bootswatch_theme.plugin b/nikola/plugins/command/bootswatch_theme.plugin index fc25045..51e6718 100644 --- a/nikola/plugins/command/bootswatch_theme.plugin +++ b/nikola/plugins/command/bootswatch_theme.plugin @@ -5,7 +5,7 @@ module = bootswatch_theme [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Given a swatch name and a parent theme, creates a custom theme. [Nikola] diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py index b5644a1..4808fdb 100644 --- a/nikola/plugins/command/bootswatch_theme.py +++ b/nikola/plugins/command/bootswatch_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,8 +36,14 @@ from nikola import utils LOGGER = utils.get_logger('bootswatch_theme', utils.STDERR_HANDLER) -class CommandBootswatchTheme(Command): +def _check_for_theme(theme, themes): + for t in themes: + if t.endswith(os.sep + theme): + return True + return False + +class CommandBootswatchTheme(Command): """Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" name = "bootswatch_theme" @@ -80,23 +86,27 @@ class CommandBootswatchTheme(Command): version = '' # See if we need bootswatch for bootstrap v2 or v3 - themes = utils.get_theme_chain(parent) - if 'bootstrap3' not in themes and 'bootstrap3-jinja' not in themes: + themes = utils.get_theme_chain(parent, self.site.themes_dirs) + if not _check_for_theme('bootstrap3', themes) and not _check_for_theme('bootstrap3-jinja', themes): version = '2' - elif 'bootstrap' not in themes and 'bootstrap-jinja' not in themes: + elif not _check_for_theme('bootstrap', themes) and not _check_for_theme('bootstrap-jinja', themes): LOGGER.warn('"bootswatch_theme" only makes sense for themes that use bootstrap') - elif 'bootstrap3-gradients' in themes or 'bootstrap3-gradients-jinja' in themes: + elif _check_for_theme('bootstrap3-gradients', themes) or _check_for_theme('bootstrap3-gradients-jinja', themes): LOGGER.warn('"bootswatch_theme" doesn\'t work well with the bootstrap3-gradients family') LOGGER.info("Creating '{0}' theme from '{1}' and '{2}'".format(name, swatch, parent)) utils.makedirs(os.path.join('themes', name, 'assets', 'css')) for fname in ('bootstrap.min.css', 'bootstrap.css'): - url = 'http://bootswatch.com' + url = 'https://bootswatch.com' if version: url += '/' + version url = '/'.join((url, swatch, fname)) LOGGER.info("Downloading: " + url) - data = requests.get(url).text + r = requests.get(url) + if r.status_code > 299: + LOGGER.error('Error {} getting {}', r.status_code, url) + exit(1) + data = r.text with open(os.path.join('themes', name, 'assets', 'css', fname), 'wb+') as output: output.write(data.encode('utf-8')) diff --git a/nikola/plugins/command/check.plugin b/nikola/plugins/command/check.plugin index e380e64..6d2df82 100644 --- a/nikola/plugins/command/check.plugin +++ b/nikola/plugins/command/check.plugin @@ -5,7 +5,7 @@ module = check [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Check the generated site [Nikola] diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py index abf183e..0141a6b 100644 --- a/nikola/plugins/command/check.py +++ b/nikola/plugins/command/check.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,6 +32,7 @@ import os import re import sys import time +import logbook try: from urllib import unquote from urlparse import urlparse, urljoin, urldefrag @@ -46,7 +47,10 @@ from nikola.plugin_categories import Command from nikola.utils import get_logger, STDERR_HANDLER -def _call_nikola_list(site): +def _call_nikola_list(site, cache=None): + if cache is not None: + if 'files' in cache and 'deps' in cache: + return cache['files'], cache['deps'] files = [] deps = defaultdict(list) for task in generate_tasks('render_site', site.gen_tasks('render_site', "Task", '')): @@ -57,16 +61,19 @@ def _call_nikola_list(site): files.extend(task.targets) for target in task.targets: deps[target].extend(task.file_dep) + if cache is not None: + cache['files'] = files + cache['deps'] = deps return files, deps -def real_scan_files(site): +def real_scan_files(site, cache=None): """Scan for files.""" task_fnames = set([]) real_fnames = set([]) output_folder = site.config['OUTPUT_FOLDER'] # First check that all targets are generated in the right places - for fname in _call_nikola_list(site)[0]: + for fname in _call_nikola_list(site, cache)[0]: fname = fname.strip() if fname.startswith(output_folder): task_fnames.add(fname) @@ -94,7 +101,6 @@ def fs_relpath_from_url_path(url_path): class CommandCheck(Command): - """Check the generated site.""" name = "check" @@ -159,25 +165,28 @@ class CommandCheck(Command): print(self.help()) return False if options['verbose']: - self.logger.level = 1 + self.logger.level = logbook.DEBUG else: - self.logger.level = 4 + self.logger.level = logbook.NOTICE + failure = False if options['links']: - failure = self.scan_links(options['find_sources'], options['remote']) + failure |= self.scan_links(options['find_sources'], options['remote']) if options['files']: - failure = self.scan_files() + failure |= self.scan_files() if options['clean']: - failure = self.clean_files() + failure |= self.clean_files() if failure: return 1 existing_targets = set([]) checked_remote_targets = {} + cache = {} def analyze(self, fname, find_sources=False, check_remote=False): """Analyze links on a page.""" rv = False self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']] + self.internal_redirects = [urljoin('/', _[0]) for _ in self.site.config['REDIRECTIONS']] base_url = urlparse(self.site.config['BASE_URL']) self.existing_targets.add(self.site.config['SITE_URL']) self.existing_targets.add(self.site.config['BASE_URL']) @@ -185,7 +194,7 @@ class CommandCheck(Command): deps = {} if find_sources: - deps = _call_nikola_list(self.site)[1] + deps = _call_nikola_list(self.site, self.cache)[1] if url_type in ('absolute', 'full_path'): url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path @@ -203,31 +212,70 @@ class CommandCheck(Command): # Quietly ignore files that don’t exist; use `nikola check -f` instead (Issue #1831) return False - d = lxml.html.fromstring(open(filename, 'rb').read()) - for l in d.iterlinks(): + if '.html' == fname[-5:]: + d = lxml.html.fromstring(open(filename, 'rb').read()) + extra_objs = lxml.html.fromstring('<html/>') + + # Turn elements with a srcset attribute into individual img elements with src attributes + for obj in list(d.xpath('(*//img|*//source)')): + if 'srcset' in obj.attrib: + for srcset_item in obj.attrib['srcset'].split(','): + extra_objs.append(lxml.etree.Element('img', src=srcset_item.strip().split(' ')[0])) + link_elements = list(d.iterlinks()) + list(extra_objs.iterlinks()) + # Extract links from XML formats to minimal HTML, allowing those to go through the link checks + elif '.atom' == filename[-5:]: + d = lxml.etree.parse(filename) + link_elements = lxml.html.fromstring('<html/>') + for elm in d.findall('*//{http://www.w3.org/2005/Atom}link'): + feed_link = elm.attrib['href'].split('?')[0].strip() # strip FEED_LINKS_APPEND_QUERY + link_elements.append(lxml.etree.Element('a', href=feed_link)) + link_elements = list(link_elements.iterlinks()) + elif filename.endswith('sitemap.xml') or filename.endswith('sitemapindex.xml'): + d = lxml.etree.parse(filename) + link_elements = lxml.html.fromstring('<html/>') + for elm in d.getroot().findall("*//{http://www.sitemaps.org/schemas/sitemap/0.9}loc"): + link_elements.append(lxml.etree.Element('a', href=elm.text.strip())) + link_elements = list(link_elements.iterlinks()) + else: # unsupported file type + return False + + for l in link_elements: target = l[2] if target == "#": continue - target, _ = urldefrag(target) + target = urldefrag(target)[0] + + if any([urlparse(target).netloc.endswith(_) for _ in ['example.com', 'example.net', 'example.org']]): + self.logger.debug("Not testing example address \"{0}\".".format(target)) + continue + + # absolute URL to root-relative + if target.startswith(base_url.geturl()): + target = target.replace(base_url.geturl(), '/') + parsed = urlparse(target) # Warn about links from https to http (mixed-security) if base_url.netloc == parsed.netloc and base_url.scheme == "https" and parsed.scheme == "http": self.logger.warn("Mixed-content security for link in {0}: {1}".format(filename, target)) + # Link to an internal REDIRECTIONS page + if target in self.internal_redirects: + redir_status_code = 301 + redir_target = [_dest for _target, _dest in self.site.config['REDIRECTIONS'] if urljoin('/', _target) == target][0] + self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: 301]".format(redir_target, filename, target)) + # Absolute links to other domains, skip # Absolute links when using only paths, skip. if ((parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc) or \ ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')): if not check_remote or parsed.scheme not in ["http", "https"]: continue - if parsed.netloc == base_url.netloc: # absolute URL to self.site - continue if target in self.checked_remote_targets: # already checked this exact target - if self.checked_remote_targets[target] in [301, 307]: + if self.checked_remote_targets[target] in [301, 308]: self.logger.warn("Remote link PERMANENTLY redirected in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) - elif self.checked_remote_targets[target] in [302, 308]: - self.logger.info("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target])) + elif self.checked_remote_targets[target] in [302, 307]: + self.logger.debug("Remote link temporarily redirected in {0}: {1} [HTTP: {2}]".format(filename, target, self.checked_remote_targets[target])) elif self.checked_remote_targets[target] > 399: self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) continue @@ -255,7 +303,7 @@ class CommandCheck(Command): if redir_status_code in [301, 308]: self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) if redir_status_code in [302, 307]: - self.logger.info("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + self.logger.debug("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) self.checked_remote_targets[resp.url] = resp.status_code self.checked_remote_targets[target] = redir_status_code else: @@ -275,8 +323,9 @@ class CommandCheck(Command): target_filename = os.path.abspath( os.path.join(self.site.config['OUTPUT_FOLDER'], unquote(target.lstrip('/')))) else: # Relative path + unquoted_target = unquote(target).encode('utf-8') if sys.version_info.major >= 3 else unquote(target).decode('utf-8') target_filename = os.path.abspath( - os.path.join(os.path.dirname(filename), unquote(target))) + os.path.join(os.path.dirname(filename).encode('utf-8'), unquoted_target)) elif url_type in ('full_path', 'absolute'): if url_type == 'absolute': @@ -292,9 +341,10 @@ class CommandCheck(Command): if any(re.search(x, target_filename) for x in self.whitelist): continue + elif target_filename not in self.existing_targets: if os.path.exists(target_filename): - self.logger.notice("Good link {0} => {1}".format(target, target_filename)) + self.logger.info("Good link {0} => {1}".format(target, target_filename)) self.existing_targets.add(target_filename) else: rv = True @@ -304,31 +354,42 @@ class CommandCheck(Command): self.logger.warn("\n".join(deps[filename])) self.logger.warn("===============================\n") except Exception as exc: - self.logger.error("Error with: {0} {1}".format(filename, exc)) + self.logger.error(u"Error with: {0} {1}".format(filename, exc)) return rv def scan_links(self, find_sources=False, check_remote=False): """Check links on the site.""" - self.logger.info("Checking Links:") - self.logger.info("===============\n") - self.logger.notice("{0} mode".format(self.site.config['URL_TYPE'])) + self.logger.debug("Checking Links:") + self.logger.debug("===============\n") + self.logger.debug("{0} mode".format(self.site.config['URL_TYPE'])) failure = False # Maybe we should just examine all HTML files output_folder = self.site.config['OUTPUT_FOLDER'] - for fname in _call_nikola_list(self.site)[0]: - if fname.startswith(output_folder) and '.html' == fname[-5:]: - if self.analyze(fname, find_sources, check_remote): - failure = True + + if urlparse(self.site.config['BASE_URL']).netloc == 'example.com': + self.logger.error("You've not changed the SITE_URL (or BASE_URL) setting from \"example.com\"!") + + for fname in _call_nikola_list(self.site, self.cache)[0]: + if fname.startswith(output_folder): + if '.html' == fname[-5:]: + if self.analyze(fname, find_sources, check_remote): + failure = True + if '.atom' == fname[-5:]: + if self.analyze(fname, find_sources, False): + failure = True + if fname.endswith('sitemap.xml') or fname.endswith('sitemapindex.xml'): + if self.analyze(fname, find_sources, False): + failure = True if not failure: - self.logger.info("All links checked.") + self.logger.debug("All links checked.") return failure def scan_files(self): """Check files in the site, find missing and orphaned files.""" failure = False - self.logger.info("Checking Files:") - self.logger.info("===============\n") - only_on_output, only_on_input = real_scan_files(self.site) + self.logger.debug("Checking Files:") + self.logger.debug("===============\n") + only_on_output, only_on_input = real_scan_files(self.site, self.cache) # Ignore folders only_on_output = [p for p in only_on_output if not os.path.isdir(p)] @@ -346,16 +407,18 @@ class CommandCheck(Command): for f in only_on_input: self.logger.warn(f) if not failure: - self.logger.info("All files checked.") + self.logger.debug("All files checked.") return failure def clean_files(self): """Remove orphaned files.""" - only_on_output, _ = real_scan_files(self.site) + only_on_output, _ = real_scan_files(self.site, self.cache) for f in only_on_output: - self.logger.info('removed: {0}'.format(f)) + self.logger.debug('removed: {0}'.format(f)) os.unlink(f) + warn_flag = bool(only_on_output) + # Find empty directories and remove them output_folder = self.site.config['OUTPUT_FOLDER'] all_dirs = [] @@ -365,7 +428,12 @@ class CommandCheck(Command): for d in all_dirs: try: os.rmdir(d) - self.logger.info('removed: {0}/'.format(d)) + self.logger.debug('removed: {0}/'.format(d)) + warn_flag = True except OSError: pass + + if warn_flag: + self.logger.warn('Some files or directories have been removed, your site may need rebuilding') + return True diff --git a/nikola/plugins/command/console.plugin b/nikola/plugins/command/console.plugin index 333762c..9bcc909 100644 --- a/nikola/plugins/command/console.plugin +++ b/nikola/plugins/command/console.plugin @@ -5,7 +5,7 @@ module = console [Documentation] author = Chris Warrick, Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Start a debugging python console [Nikola] diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py index 539fa08..c6a8376 100644 --- a/nikola/plugins/command/console.py +++ b/nikola/plugins/command/console.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Chris Warrick, Roberto Alsina and others. +# Copyright © 2012-2016 Chris Warrick, Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,7 +38,6 @@ LOGGER = get_logger('console', STDERR_HANDLER) class CommandConsole(Command): - """Start debugging console.""" name = "console" @@ -76,7 +75,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f ] def ipython(self, willful=True): - """IPython shell.""" + """Run an IPython shell.""" try: import IPython except ImportError as e: @@ -85,12 +84,13 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f raise e # That’s how _execute knows whether to try something else. else: site = self.context['site'] # NOQA + nikola_site = self.context['site'] # NOQA conf = self.context['conf'] # NOQA commands = self.context['commands'] # NOQA IPython.embed(header=self.header.format('IPython')) def bpython(self, willful=True): - """bpython shell.""" + """Run a bpython shell.""" try: import bpython except ImportError as e: @@ -101,7 +101,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f bpython.embed(banner=self.header.format('bpython'), locals_=self.context) def plain(self, willful=True): - """Plain Python shell.""" + """Run a plain Python shell.""" import code try: import readline @@ -131,6 +131,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f self.context = { 'conf': self.site.config, 'site': self.site, + 'nikola_site': self.site, 'commands': self.site.commands, } if options['bpython']: diff --git a/nikola/plugins/command/deploy.plugin b/nikola/plugins/command/deploy.plugin index 4743ca2..8bdc0e2 100644 --- a/nikola/plugins/command/deploy.plugin +++ b/nikola/plugins/command/deploy.plugin @@ -5,7 +5,7 @@ module = deploy [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Deploy the site [Nikola] diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py index 821ea11..c2289e8 100644 --- a/nikola/plugins/command/deploy.py +++ b/nikola/plugins/command/deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -30,6 +30,7 @@ from __future__ import print_function import io from datetime import datetime from dateutil.tz import gettz +import dateutil import os import subprocess import time @@ -37,16 +38,15 @@ import time from blinker import signal from nikola.plugin_categories import Command -from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER +from nikola.utils import get_logger, clean_before_deployment, STDERR_HANDLER class CommandDeploy(Command): - """Deploy site.""" name = "deploy" - doc_usage = "[[preset [preset...]]" + doc_usage = "[preset [preset...]]" doc_purpose = "deploy the site" doc_description = "Deploy the site by executing deploy commands from the presets listed on the command line. If no presets are specified, `default` is executed." logger = None @@ -56,27 +56,42 @@ class CommandDeploy(Command): self.logger = get_logger('deploy', STDERR_HANDLER) # Get last successful deploy date timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy') + + # Get last-deploy from persistent state + last_deploy = self.site.state.get('last_deploy') + if last_deploy is None: + # If there is a last-deploy saved, move it to the new state persistence thing + # FIXME: remove in Nikola 8 + if os.path.isfile(timestamp_path): + try: + with io.open(timestamp_path, 'r', encoding='utf8') as inf: + last_deploy = dateutil.parser.parse(inf.read()) + clean = False + except (IOError, Exception) as e: + self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) + last_deploy = datetime(1970, 1, 1) + clean = True + os.unlink(timestamp_path) # Remove because from now on it's in state + else: # Just a default + last_deploy = datetime(1970, 1, 1) + clean = True + else: + last_deploy = dateutil.parser.parse(last_deploy) + clean = False + if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo': self.logger.warn("\nWARNING WARNING WARNING WARNING\n" "You are deploying using the nikolademo Disqus account.\n" "That means you will not be able to moderate the comments in your own site.\n" "And is probably not what you want to do.\n" - "Think about it for 5 seconds, I'll wait :-)\n\n") + "Think about it for 5 seconds, I'll wait :-)\n" + "(press Ctrl+C to abort)\n") time.sleep(5) - deploy_drafts = self.site.config.get('DEPLOY_DRAFTS', True) - deploy_future = self.site.config.get('DEPLOY_FUTURE', False) - undeployed_posts = [] - if not (deploy_drafts and deploy_future): - # Remove drafts and future posts - out_dir = self.site.config['OUTPUT_FOLDER'] - self.site.scan_posts() - for post in self.site.timeline: - if (not deploy_drafts and post.is_draft) or \ - (not deploy_future and post.publish_later): - remove_file(os.path.join(out_dir, post.destination_path())) - remove_file(os.path.join(out_dir, post.source_path)) - undeployed_posts.append(post) + # Remove drafts and future posts if requested + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) if args: presets = args @@ -98,27 +113,22 @@ class CommandDeploy(Command): try: subprocess.check_call(command, shell=True) except subprocess.CalledProcessError as e: - self.logger.error('Failed deployment — command {0} ' + self.logger.error('Failed deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode)) return e.returncode self.logger.info("Successful deployment") - try: - with io.open(timestamp_path, 'r', encoding='utf8') as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - clean = False - except (IOError, Exception) as e: - self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) - last_deploy = datetime(1970, 1, 1) - clean = True new_deploy = datetime.utcnow() self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts) - makedirs(self.site.config['CACHE_FOLDER']) # Store timestamp of successful deployment - with io.open(timestamp_path, 'w+', encoding='utf8') as outf: - outf.write(unicode_str(new_deploy.isoformat())) + self.site.state.set('last_deploy', new_deploy.isoformat()) + if clean: + self.logger.info( + 'Looks like this is the first time you deployed this site. ' + 'Let us know you are using Nikola ' + 'at <https://users.getnikola.com/add/> if you want!') def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None): """Emit events for all timeline entries newer than last deploy. diff --git a/nikola/plugins/command/github_deploy.plugin b/nikola/plugins/command/github_deploy.plugin index e793548..21e246c 100644 --- a/nikola/plugins/command/github_deploy.plugin +++ b/nikola/plugins/command/github_deploy.plugin @@ -5,7 +5,7 @@ module = github_deploy [Documentation] author = Puneeth Chaganti version = 1,0 -website = http://getnikola.com +website = https://getnikola.com/ description = Deploy the site to GitHub pages. [Nikola] diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py index 0ab9332..b5ad322 100644 --- a/nikola/plugins/command/github_deploy.py +++ b/nikola/plugins/command/github_deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Puneeth Chaganti and others. +# Copyright © 2014-2016 Puneeth Chaganti and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,15 +27,13 @@ """Deploy site to GitHub Pages.""" from __future__ import print_function -from datetime import datetime -import io import os import subprocess from textwrap import dedent from nikola.plugin_categories import Command from nikola.plugins.command.check import real_scan_files -from nikola.utils import get_logger, req_missing, makedirs, unicode_str, STDERR_HANDLER +from nikola.utils import get_logger, req_missing, clean_before_deployment, STDERR_HANDLER from nikola.__main__ import main from nikola import __version__ @@ -53,16 +51,15 @@ def check_ghp_import_installed(): except OSError: # req_missing defaults to `python=True` — and it’s meant to be like this. # `ghp-import` is installed via pip, but the only way to use it is by executing the script it installs. - req_missing(['ghp-import'], 'deploy the site to GitHub Pages') + req_missing(['ghp-import2'], 'deploy the site to GitHub Pages') class CommandGitHubDeploy(Command): - """Deploy site to GitHub Pages.""" name = 'github_deploy' - doc_usage = '' + doc_usage = '[-m COMMIT_MESSAGE]' doc_purpose = 'deploy the site to GitHub Pages' doc_description = dedent( """\ @@ -72,10 +69,19 @@ class CommandGitHubDeploy(Command): """ ) - + cmd_options = [ + { + 'name': 'commit_message', + 'short': 'm', + 'long': 'message', + 'default': 'Nikola auto commit.', + 'type': str, + 'help': 'Commit message (default: Nikola auto commit.)', + }, + ] logger = None - def _execute(self, command, args): + def _execute(self, options, args): """Run the deployment.""" self.logger = get_logger(CommandGitHubDeploy.name, STDERR_HANDLER) @@ -93,41 +99,69 @@ class CommandGitHubDeploy(Command): for f in only_on_output: os.unlink(f) + # Remove drafts and future posts if requested (Issue #2406) + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) + # Commit and push - self._commit_and_push() + self._commit_and_push(options['commit_message']) return - def _commit_and_push(self): - """Commit all the files and push.""" - source = self.site.config['GITHUB_SOURCE_BRANCH'] - deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] - remote = self.site.config['GITHUB_REMOTE_NAME'] - source_commit = uni_check_output(['git', 'rev-parse', source]) - commit_message = ( - 'Nikola auto commit.\n\n' - 'Source commit: %s' - 'Nikola version: %s' % (source_commit, __version__) - ) - output_folder = self.site.config['OUTPUT_FOLDER'] - - command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] - + def _run_command(self, command, xfail=False): + """Run a command that may or may not fail.""" self.logger.info("==> {0}".format(command)) try: subprocess.check_call(command) + return 0 except subprocess.CalledProcessError as e: + if xfail: + return e.returncode self.logger.error( - 'Failed GitHub deployment — command {0} ' + 'Failed GitHub deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode) ) - return e.returncode + raise SystemError(e.returncode) - self.logger.info("Successful deployment") + def _commit_and_push(self, commit_first_line): + """Commit all the files and push.""" + source = self.site.config['GITHUB_SOURCE_BRANCH'] + deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] + remote = self.site.config['GITHUB_REMOTE_NAME'] + autocommit = self.site.config['GITHUB_COMMIT_SOURCE'] + try: + if autocommit: + commit_message = ( + '{0}\n\n' + 'Nikola version: {1}'.format(commit_first_line, __version__) + ) + e = self._run_command(['git', 'checkout', source], True) + if e != 0: + self._run_command(['git', 'checkout', '-b', source]) + self._run_command(['git', 'add', '.']) + # Figure out if there is anything to commit + e = self._run_command(['git', 'diff-index', '--quiet', 'HEAD'], True) + if e != 0: + self._run_command(['git', 'commit', '-am', commit_message]) + else: + self.logger.notice('Nothing to commit to source branch.') + + source_commit = uni_check_output(['git', 'rev-parse', source]) + commit_message = ( + '{0}\n\n' + 'Source commit: {1}' + 'Nikola version: {2}'.format(commit_first_line, source_commit, __version__) + ) + output_folder = self.site.config['OUTPUT_FOLDER'] + + command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] + + self._run_command(command) - # Store timestamp of successful deployment - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - new_deploy = datetime.utcnow() - makedirs(self.site.config["CACHE_FOLDER"]) - with io.open(timestamp_path, "w+", encoding="utf8") as outf: - outf.write(unicode_str(new_deploy.isoformat())) + if autocommit: + self._run_command(['git', 'push', '-u', remote, source]) + except SystemError as e: + return e.args[0] + + self.logger.info("Successful deployment") diff --git a/nikola/plugins/command/import_wordpress.plugin b/nikola/plugins/command/import_wordpress.plugin index 6c4384e..eab9d17 100644 --- a/nikola/plugins/command/import_wordpress.plugin +++ b/nikola/plugins/command/import_wordpress.plugin @@ -5,7 +5,7 @@ module = import_wordpress [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Import a wordpress site from a XML dump (requires markdown). [Nikola] diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index a652ec8..0b48583 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,6 +38,11 @@ from lxml import etree from collections import defaultdict try: + import html2text +except: + html2text = None + +try: from urlparse import urlparse from urllib import unquote except ImportError: @@ -50,7 +55,7 @@ except ImportError: from nikola.plugin_categories import Command from nikola import utils -from nikola.utils import req_missing +from nikola.utils import req_missing, unicode_str from nikola.plugins.basic_import import ImportMixin, links from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN from nikola.plugins.command.init import SAMPLE_CONF, prepare_config, format_default_translations_config @@ -88,7 +93,6 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False) class CommandImportWordpress(Command, ImportMixin): - """Import a WordPress dump.""" name = "import_wordpress" @@ -171,6 +175,20 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Export comments as .wpcomment files", }, { + 'name': 'html2text', + 'long': 'html2text', + 'default': False, + 'type': bool, + 'help': "Uses html2text (needs to be installed with pip) to transform WordPress posts to MarkDown during import", + }, + { + 'name': 'transform_to_markdown', + 'long': 'transform-to-markdown', + 'default': False, + 'type': bool, + 'help': "Uses WordPress page compiler to transform WordPress posts to HTML and then use html2text to transform them to MarkDown during import", + }, + { 'name': 'transform_to_html', 'long': 'transform-to-html', 'default': False, @@ -191,9 +209,36 @@ class CommandImportWordpress(Command, ImportMixin): 'type': bool, 'help': "Automatically installs the WordPress page compiler (either locally or in the new site) if required by other options.\nWarning: the compiler is GPL software!", }, + { + 'name': 'tag_sanitizing_strategy', + 'long': 'tag-sanitizing-strategy', + 'default': 'first', + 'help': 'lower: Convert all tag and category names to lower case\nfirst: Keep first spelling of tag or category name', + }, + { + 'name': 'one_file', + 'long': 'one-file', + 'default': False, + 'type': bool, + 'help': "Save imported posts in the more modern one-file format.", + }, ] all_tags = set([]) + def _get_compiler(self): + """Return whatever compiler we will use.""" + self._find_wordpress_compiler() + if self.wordpress_page_compiler is not None: + return self.wordpress_page_compiler + plugin_info = self.site.plugin_manager.getPluginByName('markdown', 'PageCompiler') + if plugin_info is not None: + if not plugin_info.is_activated: + self.site.plugin_manager.activatePluginByName(plugin_info.name) + plugin_info.plugin_object.set_site(self.site) + return plugin_info.plugin_object + else: + LOGGER.error("Can't find markdown post compiler.") + def _find_wordpress_compiler(self): """Find WordPress compiler plugin.""" if self.wordpress_page_compiler is not None: @@ -218,6 +263,8 @@ class CommandImportWordpress(Command, ImportMixin): 'putting these arguments before the filename if you ' 'are running into problems.'.format(args)) + self.onefile = options.get('one_file', False) + self.import_into_existing_site = False self.url_map = {} self.timezone = None @@ -234,11 +281,16 @@ class CommandImportWordpress(Command, ImportMixin): self.export_categories_as_categories = options.get('export_categories_as_categories', False) self.export_comments = options.get('export_comments', False) + self.html2text = options.get('html2text', False) + self.transform_to_markdown = options.get('transform_to_markdown', False) + self.transform_to_html = options.get('transform_to_html', False) self.use_wordpress_compiler = options.get('use_wordpress_compiler', False) self.install_wordpress_compiler = options.get('install_wordpress_compiler', False) self.wordpress_page_compiler = None + self.tag_saniziting_strategy = options.get('tag_saniziting_strategy', 'first') + self.auth = None if options.get('download_auth') is not None: username_password = options.get('download_auth') @@ -250,10 +302,18 @@ class CommandImportWordpress(Command, ImportMixin): self.separate_qtranslate_content = options.get('separate_qtranslate_content') self.translations_pattern = options.get('translations_pattern') - if self.transform_to_html and self.use_wordpress_compiler: - LOGGER.warn("It does not make sense to combine --transform-to-html with --use-wordpress-compiler, as the first converts all posts to HTML and the latter option affects zero posts.") + count = (1 if self.html2text else 0) + (1 if self.transform_to_html else 0) + (1 if self.transform_to_markdown else 0) + if count > 1: + LOGGER.error("You can use at most one of the options --html2text, --transform-to-html and --transform-to-markdown.") + return False + if (self.html2text or self.transform_to_html or self.transform_to_markdown) and self.use_wordpress_compiler: + LOGGER.warn("It does not make sense to combine --use-wordpress-compiler with any of --html2text, --transform-to-html and --transform-to-markdown, as the latter convert all posts to HTML and the first option then affects zero posts.") + + if (self.html2text or self.transform_to_markdown) and not html2text: + LOGGER.error("You need to install html2text via 'pip install html2text' before you can use the --html2text and --transform-to-markdown options.") + return False - if self.transform_to_html: + if self.transform_to_html or self.transform_to_markdown: self._find_wordpress_compiler() if not self.wordpress_page_compiler and self.install_wordpress_compiler: if not install_plugin(self.site, 'wordpress_compiler', output_dir='plugins'): # local install @@ -327,7 +387,7 @@ class CommandImportWordpress(Command, ImportMixin): self.context['TRANSLATIONS'] = format_default_translations_config( self.extra_languages) self.context['REDIRECTIONS'] = self.configure_redirections( - self.url_map) + self.url_map, self.base_dir) if self.timezone: self.context['TIMEZONE'] = self.timezone if self.export_categories_as_categories: @@ -337,10 +397,13 @@ class CommandImportWordpress(Command, ImportMixin): # Add tag redirects for tag in self.all_tags: try: - tag_str = tag.decode('utf8') + if isinstance(tag, utils.bytes_str): + tag_str = tag.decode('utf8', 'replace') + else: + tag_str = tag except AttributeError: tag_str = tag - tag = utils.slugify(tag_str) + tag = utils.slugify(tag_str, self.lang) src_url = '{}tag/{}'.format(self.context['SITE_URL'], tag) dst_url = self.site.link('tag', tag) if src_url != dst_url: @@ -372,7 +435,7 @@ class CommandImportWordpress(Command, ImportMixin): if b'<atom:link rel=' in line: continue xml.append(line) - return b'\n'.join(xml) + return b''.join(xml) @classmethod def get_channel_from_file(cls, filename): @@ -386,7 +449,8 @@ class CommandImportWordpress(Command, ImportMixin): wordpress_namespace = channel.nsmap['wp'] context = SAMPLE_CONF.copy() - context['DEFAULT_LANG'] = get_text_tag(channel, 'language', 'en')[:2] + self.lang = get_text_tag(channel, 'language', 'en')[:2] + context['DEFAULT_LANG'] = self.lang context['TRANSLATIONS_PATTERN'] = DEFAULT_TRANSLATIONS_PATTERN context['BLOG_TITLE'] = get_text_tag(channel, 'title', 'PUT TITLE HERE') @@ -418,7 +482,7 @@ class CommandImportWordpress(Command, ImportMixin): PAGES = '(\n' for extension in extensions: POSTS += ' ("posts/*.{0}", "posts", "post.tmpl"),\n'.format(extension) - PAGES += ' ("stories/*.{0}", "stories", "story.tmpl"),\n'.format(extension) + PAGES += ' ("pages/*.{0}", "pages", "story.tmpl"),\n'.format(extension) POSTS += ')\n' PAGES += ')\n' context['POSTS'] = POSTS @@ -436,9 +500,6 @@ class CommandImportWordpress(Command, ImportMixin): def download_url_content_to_file(self, url, dst_path): """Download some content (attachments) to a file.""" - if self.no_downloads: - return - try: request = requests.get(url, auth=self.auth) if request.status_code >= 400: @@ -458,10 +519,13 @@ class CommandImportWordpress(Command, ImportMixin): 'foo') path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[link] = '/' + dst_url links[url] = '/' + dst_url @@ -507,6 +571,8 @@ class CommandImportWordpress(Command, ImportMixin): if meta_key in metadata: image_meta = metadata[meta_key] + if not image_meta: + continue dst_meta = {} def add(our_key, wp_key, is_int=False, ignore_zero=False, is_float=False): @@ -552,15 +618,18 @@ class CommandImportWordpress(Command, ImportMixin): meta = {} meta['size'] = size.decode('utf-8') if width_key in metadata[size_key][size] and height_key in metadata[size_key][size]: - meta['width'] = metadata[size_key][size][width_key] - meta['height'] = metadata[size_key][size][height_key] + meta['width'] = int(metadata[size_key][size][width_key]) + meta['height'] = int(metadata[size_key][size][height_key]) path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[url] = '/' + dst_url @@ -604,7 +673,7 @@ class CommandImportWordpress(Command, ImportMixin): def transform_code(self, content): """Transform code blocks.""" - # http://en.support.wordpress.com/code/posting-source-code/. There are + # https://en.support.wordpress.com/code/posting-source-code/. There are # a ton of things not supported here. We only do a basic [code # lang="x"] -> ```x translation, and remove quoted html entities (<, # >, &, and "). @@ -628,10 +697,10 @@ class CommandImportWordpress(Command, ImportMixin): return content @staticmethod - def transform_caption(content): + def transform_caption(content, use_html=False): """Transform captions.""" - new_caption = re.sub(r'\[/caption\]', '', content) - new_caption = re.sub(r'\[caption.*\]', '', new_caption) + new_caption = re.sub(r'\[/caption\]', '</h1>' if use_html else '', content) + new_caption = re.sub(r'\[caption.*\]', '<h1>' if use_html else '', new_caption) return new_caption @@ -654,6 +723,26 @@ class CommandImportWordpress(Command, ImportMixin): except TypeError: # old versions of the plugin don't support the additional argument content = self.wordpress_page_compiler.compile_to_string(content) return content, 'html', True + elif self.transform_to_markdown: + # First convert to HTML with WordPress plugin + additional_data = {} + if attachments is not None: + additional_data['attachments'] = attachments + try: + content = self.wordpress_page_compiler.compile_to_string(content, additional_data=additional_data) + except TypeError: # old versions of the plugin don't support the additional argument + content = self.wordpress_page_compiler.compile_to_string(content) + # Now convert to MarkDown with html2text + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False + elif self.html2text: + # TODO: what to do with [code] blocks? + # content = self.transform_code(content) + content = self.transform_caption(content, use_html=True) + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False elif self.use_wordpress_compiler: return content, 'wp', False else: @@ -686,7 +775,7 @@ class CommandImportWordpress(Command, ImportMixin): elif approved == 'spam' or approved == 'trash': pass else: - LOGGER.warn("Unknown comment approved status: " + str(approved)) + LOGGER.warn("Unknown comment approved status: {0}".format(approved)) parent = int(get_text_tag(comment, "{{{0}}}comment_parent".format(wordpress_namespace), 0)) if parent == 0: parent = None @@ -707,7 +796,7 @@ class CommandImportWordpress(Command, ImportMixin): """Write comment header line.""" if header_content is None: return - header_content = str(header_content).replace('\n', ' ') + header_content = unicode_str(header_content).replace('\n', ' ') line = '.. ' + header_field + ': ' + header_content + '\n' fd.write(line.encode('utf8')) @@ -747,12 +836,36 @@ class CommandImportWordpress(Command, ImportMixin): tags_cats = tags + categories return tags_cats, other_meta + _tag_sanitize_map = {True: {}, False: {}} + + def _sanitize(self, tag, is_category): + if self.tag_saniziting_strategy == 'lower': + return tag.lower() + if tag.lower() not in self._tag_sanitize_map[is_category]: + self._tag_sanitize_map[is_category][tag.lower()] = [tag] + return tag + previous = self._tag_sanitize_map[is_category][tag.lower()] + if self.tag_saniziting_strategy == 'first': + if tag != previous[0]: + LOGGER.warn("Changing spelling of {0} name '{1}' to {2}.".format('category' if is_category else 'tag', tag, previous[0])) + return previous[0] + else: + LOGGER.error("Unknown tag sanitizing strategy '{0}'!".format(self.tag_saniziting_strategy)) + sys.exit(1) + return tag + def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None): """Take an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') + + # titles can have line breaks in them, particularly when they are + # created by third-party tools that post to Wordpress. + # Handle windows-style and unix-style line endings. + title = title.replace('\r\n', ' ').replace('\n', ' ') + # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) @@ -760,7 +873,10 @@ class CommandImportWordpress(Command, ImportMixin): path = unquote(parsed.path.strip('/')) try: - path = path.decode('utf8') + if isinstance(path, utils.bytes_str): + path = path.decode('utf8', 'replace') + else: + path = path except AttributeError: pass @@ -782,7 +898,7 @@ class CommandImportWordpress(Command, ImportMixin): else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) - slug = utils.slugify(pathlist[-1]) + slug = utils.slugify(pathlist[-1], self.lang) description = get_text_tag(item, 'description', '') post_date = get_text_tag( @@ -831,15 +947,24 @@ class CommandImportWordpress(Command, ImportMixin): type = tag.attrib['domain'] if text == 'Uncategorized' and type == 'category': continue - self.all_tags.add(text) if type == 'category': - categories.append(type) + categories.append(text) else: tags.append(text) if '$latex' in content: tags.append('mathjax') + for i, cat in enumerate(categories[:]): + cat = self._sanitize(cat, True) + categories[i] = cat + self.all_tags.add(cat) + + for i, tag in enumerate(tags[:]): + tag = self._sanitize(tag, False) + tags[i] = tag + self.all_tags.add(tag) + # Find post format if it's there post_format = 'wp' format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format'] @@ -888,14 +1013,32 @@ class CommandImportWordpress(Command, ImportMixin): meta_slug = slug tags, other_meta = self._create_metadata(status, excerpt, tags, categories, post_name=os.path.join(out_folder, slug)) - self.write_metadata(os.path.join(self.output_folder, out_folder, - out_meta_filename), - title, meta_slug, post_date, description, tags, **other_meta) - self.write_content( - os.path.join(self.output_folder, - out_folder, out_content_filename), - content, - rewrite_html) + + meta = { + "title": title, + "slug": meta_slug, + "date": post_date, + "description": description, + "tags": ','.join(tags), + } + meta.update(other_meta) + if self.onefile: + self.write_post( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + meta, + self._get_compiler(), + rewrite_html) + else: + self.write_metadata(os.path.join(self.output_folder, out_folder, + out_meta_filename), + title, meta_slug, post_date, description, tags, **other_meta) + self.write_content( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + rewrite_html) if self.export_comments: comments = [] @@ -905,7 +1048,7 @@ class CommandImportWordpress(Command, ImportMixin): comments.append(comment) for comment in comments: - comment_filename = slug + "." + str(comment['id']) + ".wpcomment" + comment_filename = "{0}.{1}.wpcomment".format(slug, comment['id']) self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment) return (out_folder, slug) @@ -955,7 +1098,7 @@ class CommandImportWordpress(Command, ImportMixin): if post_type == 'post': out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'posts', attachments) else: - out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'stories', attachments) + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'pages', attachments) # Process attachment data if attachments is not None: # If post was exported, store data diff --git a/nikola/plugins/command/init.plugin b/nikola/plugins/command/init.plugin index a5404c4..a8b1523 100644 --- a/nikola/plugins/command/init.plugin +++ b/nikola/plugins/command/init.plugin @@ -5,7 +5,7 @@ module = init [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create a new site. [Nikola] diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py index 91ccdb4..3d6669c 100644 --- a/nikola/plugins/command/init.py +++ b/nikola/plugins/command/init.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -41,7 +41,7 @@ from pkg_resources import resource_filename import tarfile import nikola -from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_RSS_READ_MORE_LINK, LEGAL_VALUES, urlsplit, urlunsplit +from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_FEED_READ_MORE_LINK, LEGAL_VALUES, urlsplit, urlunsplit from nikola.plugin_categories import Command from nikola.utils import ask, ask_yesno, get_logger, makedirs, STDERR_HANDLER, load_messages from nikola.packages.tzlocal import get_localzone @@ -71,14 +71,16 @@ SAMPLE_CONF = { 'CATEGORY_OUTPUT_FLAT_HIERARCHY': False, 'TRANSLATIONS_PATTERN': DEFAULT_TRANSLATIONS_PATTERN, 'INDEX_READ_MORE_LINK': DEFAULT_INDEX_READ_MORE_LINK, - 'RSS_READ_MORE_LINK': DEFAULT_RSS_READ_MORE_LINK, + 'FEED_READ_MORE_LINK': DEFAULT_FEED_READ_MORE_LINK, 'POSTS': """( ("posts/*.rst", "posts", "post.tmpl"), ("posts/*.txt", "posts", "post.tmpl"), + ("posts/*.html", "posts", "post.tmpl"), )""", 'PAGES': """( - ("stories/*.rst", "stories", "story.tmpl"), - ("stories/*.txt", "stories", "story.tmpl"), + ("pages/*.rst", "pages", "story.tmpl"), + ("pages/*.txt", "pages", "story.tmpl"), + ("pages/*.html", "pages", "story.tmpl"), )""", 'COMPILERS': """{ "rest": ('.rst', '.txt'), @@ -210,17 +212,28 @@ def prepare_config(config): """Parse sample config with JSON.""" p = config.copy() p.update({k: json.dumps(v, ensure_ascii=False) for k, v in p.items() - if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK')}) + if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'FEED_READ_MORE_LINK')}) # READ_MORE_LINKs require some special treatment. p['INDEX_READ_MORE_LINK'] = "'" + p['INDEX_READ_MORE_LINK'].replace("'", "\\'") + "'" - p['RSS_READ_MORE_LINK'] = "'" + p['RSS_READ_MORE_LINK'].replace("'", "\\'") + "'" + p['FEED_READ_MORE_LINK'] = "'" + p['FEED_READ_MORE_LINK'].replace("'", "\\'") + "'" # fix booleans and None p.update({k: str(v) for k, v in config.items() if isinstance(v, bool) or v is None}) return p -class CommandInit(Command): +def test_destination(destination, demo=False): + """Check if the destination already exists, which can break demo site creation.""" + # Issue #2214 + if demo and os.path.exists(destination): + LOGGER.warning("The directory {0} already exists, and a new demo site cannot be initialized in an existing directory.".format(destination)) + LOGGER.warning("Please remove the directory and try again, or use another directory.") + LOGGER.info("Hint: If you want to initialize a git repository in this directory, run `git init` in the directory after creating a Nikola site.") + return False + else: + return True + +class CommandInit(Command): """Create a new site.""" name = "init" @@ -272,11 +285,11 @@ class CommandInit(Command): @classmethod def create_empty_site(cls, target): """Create an empty site with directories only.""" - for folder in ('files', 'galleries', 'listings', 'posts', 'stories'): + for folder in ('files', 'galleries', 'listings', 'posts', 'pages'): makedirs(os.path.join(target, folder)) @staticmethod - def ask_questions(target): + def ask_questions(target, demo=False): """Ask some questions about Nikola.""" def urlhandler(default, toconf): answer = ask('Site URL', 'https://example.com/') @@ -347,7 +360,7 @@ class CommandInit(Command): # Assuming that base contains all the locales, and that base does # not inherit from anywhere. try: - messages = load_messages(['base'], tr, default) + messages = load_messages(['base'], tr, default, themes_dirs=['themes']) SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages, SAMPLE_CONF['STRIP_INDEXES']) except nikola.utils.LanguageNotFoundError as e: print(" ERROR: the language '{0}' is not supported.".format(e.lang)) @@ -358,7 +371,7 @@ class CommandInit(Command): def tzhandler(default, toconf): print("\nPlease choose the correct time zone for your blog. Nikola uses the tz database.") print("You can find your time zone here:") - print("http://en.wikipedia.org/wiki/List_of_tz_database_time_zones") + print("https://en.wikipedia.org/wiki/List_of_tz_database_time_zones") print("") answered = False while not answered: @@ -441,7 +454,7 @@ class CommandInit(Command): print("If you do not want to answer and want to go with the defaults instead, simply restart with the `-q` parameter.") for query, default, toconf, destination in questions: - if target and destination == '!target': + if target and destination == '!target' and test_destination(target, demo): # Skip the destination question if we know it already pass else: @@ -458,8 +471,9 @@ class CommandInit(Command): if toconf: SAMPLE_CONF[destination] = answer if destination == '!target': - while not answer: - print(' ERROR: you need to specify a target directory.\n') + while not answer or not test_destination(answer, demo): + if not answer: + print(' ERROR: you need to specify a target directory.\n') answer = ask(query, default) STORAGE['target'] = answer @@ -475,7 +489,7 @@ class CommandInit(Command): except IndexError: target = None if not options.get('quiet'): - st = self.ask_questions(target=target) + st = self.ask_questions(target=target, demo=options.get('demo')) try: if not target: target = st['target'] @@ -488,11 +502,13 @@ class CommandInit(Command): Options: -q, --quiet Do not ask questions about config. -d, --demo Create a site filled with example data.""") - return False + return 1 if not options.get('demo'): self.create_empty_site(target) LOGGER.info('Created empty site at {0}.'.format(target)) else: + if not test_destination(target, True): + return 2 self.copy_sample_site(target) LOGGER.info("A new site with example data has been created at " "{0}.".format(target)) diff --git a/nikola/plugins/command/install_theme.plugin b/nikola/plugins/command/install_theme.plugin index 8434f2e..aa68773 100644 --- a/nikola/plugins/command/install_theme.plugin +++ b/nikola/plugins/command/install_theme.plugin @@ -5,7 +5,7 @@ module = install_theme [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Install a theme into the current site. [Nikola] diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py index f02252e..28f7aa3 100644 --- a/nikola/plugins/command/install_theme.py +++ b/nikola/plugins/command/install_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,23 +27,13 @@ """Install a theme.""" from __future__ import print_function -import os -import io -import time -import requests -import pygments -from pygments.lexers import PythonLexer -from pygments.formatters import TerminalFormatter - -from nikola.plugin_categories import Command from nikola import utils - +from nikola.plugin_categories import Command LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER) class CommandInstallTheme(Command): - """Install a theme.""" name = "install_theme" @@ -80,6 +70,7 @@ class CommandInstallTheme(Command): def _execute(self, options, args): """Install theme into current site.""" + p = self.site.plugin_manager.getPluginByName('theme', 'Command').plugin_object listing = options['list'] url = options['url'] if args: @@ -88,85 +79,13 @@ class CommandInstallTheme(Command): name = None if options['getpath'] and name: - path = utils.get_theme_path(name) - if path: - print(path) - else: - print('not installed') - return 0 + return p.get_path(name) if name is None and not listing: LOGGER.error("This command needs either a theme name or the -l option.") return False - try: - data = requests.get(url).json() - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - data = requests.get(url).json() - if listing: - print("Themes:") - print("-------") - for theme in sorted(data.keys()): - print(theme) - return True - else: - # `name` may be modified by the while loop. - origname = name - installstatus = self.do_install(name, data) - # See if the theme's parent is available. If not, install it - while True: - parent_name = utils.get_parent_theme_name(name) - if parent_name is None: - break - try: - utils.get_theme_path(parent_name) - break - except: # Not available - self.do_install(parent_name, data) - name = parent_name - if installstatus: - LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) - - def do_install(self, name, data): - """Download and install a theme.""" - if name in data: - utils.makedirs(self.output_dir) - url = data[name] - LOGGER.info("Downloading '{0}'".format(url)) - try: - zip_data = requests.get(url).content - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - zip_data = requests.get(url).content - zip_file = io.BytesIO() - zip_file.write(zip_data) - LOGGER.info("Extracting '{0}' into themes/".format(name)) - utils.extract_all(zip_file) - dest_path = os.path.join(self.output_dir, name) + if listing: + p.list_available(url) else: - dest_path = os.path.join(self.output_dir, name) - try: - theme_path = utils.get_theme_path(name) - LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) - except Exception: - LOGGER.error("Can't find theme {0}".format(name)) - - return False - - confpypath = os.path.join(dest_path, 'conf.py.sample') - if os.path.exists(confpypath): - LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') - print('Contents of the conf.py.sample file:\n') - with io.open(confpypath, 'r', encoding='utf-8') as fh: - if self.site.colorful: - print(utils.indent(pygments.highlight( - fh.read(), PythonLexer(), TerminalFormatter()), - 4 * ' ')) - else: - print(utils.indent(fh.read(), 4 * ' ')) - return True + p.do_install_deps(url, name) diff --git a/nikola/plugins/command/new_page.plugin b/nikola/plugins/command/new_page.plugin index 145a419..3eaecb4 100644 --- a/nikola/plugins/command/new_page.plugin +++ b/nikola/plugins/command/new_page.plugin @@ -5,7 +5,7 @@ module = new_page [Documentation] author = Roberto Alsina, Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create a new page. [Nikola] diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index 811e28b..c09b4be 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,7 +32,6 @@ from nikola.plugin_categories import Command class CommandNewPage(Command): - """Create a new page.""" name = "new_page" diff --git a/nikola/plugins/command/new_post.plugin b/nikola/plugins/command/new_post.plugin index d88469f..e9c3af5 100644 --- a/nikola/plugins/command/new_post.plugin +++ b/nikola/plugins/command/new_post.plugin @@ -5,7 +5,7 @@ module = new_post [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create a new post. [Nikola] diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index f9fe3ff..36cc04f 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,10 +29,11 @@ from __future__ import unicode_literals, print_function import io import datetime +import operator import os -import sys +import shutil import subprocess -import operator +import sys from blinker import signal import dateutil.tz @@ -114,7 +115,6 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): class CommandNewPost(Command): - """Create a new post.""" name = "new_post" @@ -294,14 +294,14 @@ class CommandNewPost(Command): title = title.strip() if not path: - slug = utils.slugify(title) + slug = utils.slugify(title, lang=self.site.default_lang) else: if isinstance(path, utils.bytes_str): try: path = path.decode(sys.stdin.encoding) except (AttributeError, TypeError): # for tests path = path.decode('utf-8') - slug = utils.slugify(os.path.splitext(os.path.basename(path))[0]) + slug = utils.slugify(os.path.splitext(os.path.basename(path))[0], lang=self.site.default_lang) if isinstance(author, utils.bytes_str): try: @@ -325,14 +325,17 @@ class CommandNewPost(Command): 'description': '', 'type': 'text', } - output_path = os.path.dirname(entry[0]) - meta_path = os.path.join(output_path, slug + ".meta") - pattern = os.path.basename(entry[0]) - suffix = pattern[1:] + if not path: + pattern = os.path.basename(entry[0]) + suffix = pattern[1:] + output_path = os.path.dirname(entry[0]) + txt_path = os.path.join(output_path, slug + suffix) + meta_path = os.path.join(output_path, slug + ".meta") else: txt_path = os.path.join(self.site.original_cwd, path) + meta_path = os.path.splitext(txt_path)[0] + ".meta" if (not onefile and os.path.isfile(meta_path)) or \ os.path.isfile(txt_path): @@ -344,6 +347,9 @@ class CommandNewPost(Command): signal('existing_' + content_type).send(self, **event) LOGGER.error("The title already exists!") + LOGGER.info("Existing {0}'s text is at: {1}".format(content_type, txt_path)) + if not onefile: + LOGGER.info("Existing {0}'s metadata is at: {1}".format(content_type, meta_path)) return 8 d_name = os.path.dirname(txt_path) @@ -364,17 +370,22 @@ class CommandNewPost(Command): onefile = False LOGGER.warn('This compiler does not support one-file posts.') - if import_file: + if onefile and import_file: with io.open(import_file, 'r', encoding='utf-8') as fh: content = fh.read() - else: + elif not import_file: if is_page: content = self.site.MESSAGES[self.site.default_lang]["Write your page here."] else: content = self.site.MESSAGES[self.site.default_lang]["Write your post here."] - compiler_plugin.create_post( - txt_path, content=content, onefile=onefile, title=title, - slug=slug, date=date, tags=tags, is_page=is_page, **metadata) + + if (not onefile) and import_file: + # Two-file posts are copied on import (Issue #2380) + shutil.copy(import_file, txt_path) + else: + compiler_plugin.create_post( + txt_path, content=content, onefile=onefile, title=title, + slug=slug, date=date, tags=tags, is_page=is_page, **metadata) event = dict(path=txt_path) diff --git a/nikola/plugins/command/orphans.plugin b/nikola/plugins/command/orphans.plugin index 669429d..d20c539 100644 --- a/nikola/plugins/command/orphans.plugin +++ b/nikola/plugins/command/orphans.plugin @@ -5,7 +5,7 @@ module = orphans [Documentation] author = Roberto Alsina, Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = List all orphans [Nikola] diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py index b12cc67..5e2574d 100644 --- a/nikola/plugins/command/orphans.py +++ b/nikola/plugins/command/orphans.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -34,7 +34,6 @@ from nikola.plugins.command.check import real_scan_files class CommandOrphans(Command): - """List all orphans.""" name = "orphans" diff --git a/nikola/plugins/command/plugin.plugin b/nikola/plugins/command/plugin.plugin index d44dcf3..016bcaa 100644 --- a/nikola/plugins/command/plugin.plugin +++ b/nikola/plugins/command/plugin.plugin @@ -5,7 +5,7 @@ module = plugin [Documentation] author = Roberto Alsina and Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Manage Nikola plugins [Nikola] diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index f892ee9..364f343 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,6 +29,7 @@ from __future__ import print_function import io import os +import sys import shutil import subprocess import time @@ -45,12 +46,11 @@ LOGGER = utils.get_logger('plugin', utils.STDERR_HANDLER) class CommandPlugin(Command): - """Manage plugins.""" json = None name = "plugin" - doc_usage = "[[-u][--user] --install name] | [[-u] [-l |--upgrade|--list-installed] | [--uninstall name]]" + doc_usage = "[-u url] [--user] [-i name] [-r name] [--upgrade] [-l] [--list-installed]" doc_purpose = "manage plugins" output_dir = None needs_config = False @@ -177,8 +177,11 @@ class CommandPlugin(Command): plugins.append([plugin.name, p]) plugins.sort() + print('Installed Plugins:') + print('------------------') for name, path in plugins: print('{0} at {1}'.format(name, path)) + print('\n\nAlso, you have disabled these plugins: {}'.format(self.site.config['DISABLED_PLUGINS'])) return 0 def do_upgrade(self, url): @@ -252,7 +255,7 @@ class CommandPlugin(Command): LOGGER.notice('This plugin has Python dependencies.') LOGGER.info('Installing dependencies with pip...') try: - subprocess.check_call(('pip', 'install', '-r', reqpath)) + subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath)) except subprocess.CalledProcessError: LOGGER.error('Could not install the dependencies.') print('Contents of the requirements.txt file:\n') @@ -293,12 +296,15 @@ class CommandPlugin(Command): def do_uninstall(self, name): """Uninstall a plugin.""" for plugin in self.site.plugin_manager.getAllPlugins(): # FIXME: this is repeated thrice - p = plugin.path - if os.path.isdir(p): - p = p + os.sep - else: - p = os.path.dirname(p) if name == plugin.name: # Uninstall this one + p = plugin.path + if os.path.isdir(p): + # Plugins that have a package in them need to delete parent + # Issue #2356 + p = p + os.sep + p = os.path.abspath(os.path.join(p, os.pardir)) + else: + p = os.path.dirname(p) LOGGER.warning('About to uninstall plugin: {0}'.format(name)) LOGGER.warning('This will delete {0}'.format(p)) sure = utils.ask_yesno('Are you sure?') diff --git a/nikola/plugins/command/rst2html.plugin b/nikola/plugins/command/rst2html.plugin index 02c9276..a095705 100644 --- a/nikola/plugins/command/rst2html.plugin +++ b/nikola/plugins/command/rst2html.plugin @@ -5,7 +5,7 @@ module = rst2html [Documentation] author = Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Compile reStructuredText to HTML using the Nikola architecture [Nikola] diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py index 06afffd..c877f63 100644 --- a/nikola/plugins/command/rst2html/__init__.py +++ b/nikola/plugins/command/rst2html/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2015 Chris Warrick and others. +# Copyright © 2015-2016 Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,7 +36,6 @@ from nikola.plugin_categories import Command class CommandRst2Html(Command): - """Compile reStructuredText to HTML, using Nikola architecture.""" name = "rst2html" @@ -65,7 +64,7 @@ class CommandRst2Html(Command): parser = lxml.html.HTMLParser(remove_blank_text=True) doc = lxml.html.document_fromstring(template_output, parser) html = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True) - print(html) + print(html.decode('utf-8')) if error_level < 3: return 0 else: diff --git a/nikola/plugins/command/serve.plugin b/nikola/plugins/command/serve.plugin index aca71ec..a4a726f 100644 --- a/nikola/plugins/command/serve.plugin +++ b/nikola/plugins/command/serve.plugin @@ -5,7 +5,7 @@ module = serve [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Start test server. [Nikola] diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py index 0441c93..c9702d5 100644 --- a/nikola/plugins/command/serve.py +++ b/nikola/plugins/command/serve.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -45,24 +45,23 @@ except ImportError: from nikola.plugin_categories import Command -from nikola.utils import get_logger, STDERR_HANDLER +from nikola.utils import dns_sd, get_logger, STDERR_HANDLER class IPv6Server(HTTPServer): - """An IPv6 HTTPServer.""" address_family = socket.AF_INET6 class CommandServe(Command): - """Start test server.""" name = "serve" doc_usage = "[options]" doc_purpose = "start the test webserver" logger = None + dns_sd = None cmd_options = ( { @@ -79,7 +78,7 @@ class CommandServe(Command): 'long': 'address', 'type': str, 'default': '', - 'help': 'Address to bind (default: 0.0.0.0 – all local IPv4 interfaces)', + 'help': 'Address to bind (default: 0.0.0.0 -- all local IPv4 interfaces)', }, { 'name': 'detach', @@ -152,14 +151,16 @@ class CommandServe(Command): raise e else: try: + self.dns_sd = dns_sd(options['port'], (options['ipv6'] or '::' in options['address'])) httpd.serve_forever() except KeyboardInterrupt: self.logger.info("Server is shutting down.") + if self.dns_sd: + self.dns_sd.Reset() return 130 class OurHTTPRequestHandler(SimpleHTTPRequestHandler): - """A request handler, modified for Nikola.""" extensions_map = dict(SimpleHTTPRequestHandler.extensions_map) @@ -242,7 +243,10 @@ class OurHTTPRequestHandler(SimpleHTTPRequestHandler): f.seek(0) self.send_response(200) - self.send_header("Content-type", ctype) + if ctype.startswith('text/') or ctype.endswith('+xml'): + self.send_header("Content-Type", "{0}; charset=UTF-8".format(ctype)) + else: + self.send_header("Content-Type", ctype) if os.path.splitext(path)[1] == '.svgz': # Special handling for svgz to make it work nice with browsers. self.send_header("Content-Encoding", 'gzip') diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py index 55e7f95..b3ffbb4 100644 --- a/nikola/plugins/command/status.py +++ b/nikola/plugins/command/status.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,7 +27,6 @@ """Display site status.""" from __future__ import print_function -import io import os from datetime import datetime from dateutil.tz import gettz, tzlocal @@ -36,14 +35,13 @@ from nikola.plugin_categories import Command class CommandStatus(Command): - """Display site status.""" name = "status" doc_purpose = "display site status" doc_description = "Show information about the posts and site deployment." - doc_usage = '[-l|--list-drafts] [-m|--list-modified] [-s|--list-scheduled]' + doc_usage = '[-d|--list-drafts] [-m|--list-modified] [-p|--list-private] [-P|--list-published] [-s|--list-scheduled]' logger = None cmd_options = [ { @@ -63,6 +61,22 @@ class CommandStatus(Command): 'help': 'List all modified files since last deployment', }, { + 'name': 'list_private', + 'short': 'p', + 'long': 'list-private', + 'type': bool, + 'default': False, + 'help': 'List all private posts', + }, + { + 'name': 'list_published', + 'short': 'P', + 'long': 'list-published', + 'type': bool, + 'default': False, + 'help': 'List all published posts', + }, + { 'name': 'list_scheduled', 'short': 's', 'long': 'list-scheduled', @@ -76,16 +90,12 @@ class CommandStatus(Command): """Display site status.""" self.site.scan_posts() - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - - last_deploy = None - - try: - with io.open(timestamp_path, "r", encoding="utf8") as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - last_deploy_offset = datetime.utcnow() - last_deploy - except (IOError, Exception): - print("It does not seem like you’ve ever deployed the site (or cache missing).") + last_deploy = self.site.state.get('last_deploy') + if last_deploy is not None: + last_deploy = datetime.strptime(last_deploy, "%Y-%m-%dT%H:%M:%S.%f") + last_deploy_offset = datetime.utcnow() - last_deploy + else: + print("It does not seem like you've ever deployed the site (or cache missing).") if last_deploy: @@ -111,12 +121,23 @@ class CommandStatus(Command): posts_count = len(self.site.all_posts) + # find all published posts + posts_published = [post for post in self.site.all_posts if post.use_in_feeds] + posts_published = sorted(posts_published, key=lambda post: post.source_path) + + # find all private posts + posts_private = [post for post in self.site.all_posts if post.is_private] + posts_private = sorted(posts_private, key=lambda post: post.source_path) + # find all drafts posts_drafts = [post for post in self.site.all_posts if post.is_draft] posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path) # find all scheduled posts with offset from now until publishing time - posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later] + posts_scheduled = [ + (post.date - now, post) for post in self.site.all_posts + if post.publish_later and not (post.is_draft or post.is_private) + ] posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path)) if len(posts_scheduled) > 0: @@ -129,7 +150,13 @@ class CommandStatus(Command): if options['list_drafts']: for post in posts_drafts: print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) - print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts))) + if options['list_private']: + for post in posts_private: + print("Private: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + if options['list_published']: + for post in posts_published: + print("Published: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + print("{0} posts in total, {1} scheduled, {2} drafts, {3} private and {4} published.".format(posts_count, len(posts_scheduled), len(posts_drafts), len(posts_private), len(posts_published))) def human_time(self, dt): """Translate time into a human-friendly representation.""" diff --git a/nikola/plugins/command/theme.plugin b/nikola/plugins/command/theme.plugin new file mode 100644 index 0000000..b0c1886 --- /dev/null +++ b/nikola/plugins/command/theme.plugin @@ -0,0 +1,13 @@ +[Core] +name = theme +module = theme + +[Documentation] +author = Roberto Alsina and Chris Warrick +version = 1.0 +website = https://getnikola.com/ +description = Manage Nikola themes + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/theme.py b/nikola/plugins/command/theme.py new file mode 100644 index 0000000..7513491 --- /dev/null +++ b/nikola/plugins/command/theme.py @@ -0,0 +1,365 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Manage themes.""" + +from __future__ import print_function +import os +import io +import shutil +import time +import requests + +import pygments +from pygments.lexers import PythonLexer +from pygments.formatters import TerminalFormatter +from pkg_resources import resource_filename + +from nikola.plugin_categories import Command +from nikola import utils + +LOGGER = utils.get_logger('theme', utils.STDERR_HANDLER) + + +class CommandTheme(Command): + """Manage themes.""" + + json = None + name = "theme" + doc_usage = "[-u url] [-i theme_name] [-r theme_name] [-l] [--list-installed] [-g] [-n theme_name] [-c template_name]" + doc_purpose = "manage themes" + output_dir = 'themes' + cmd_options = [ + { + 'name': 'install', + 'short': 'i', + 'long': 'install', + 'type': str, + 'default': '', + 'help': 'Install a theme.' + }, + { + 'name': 'uninstall', + 'long': 'uninstall', + 'short': 'r', + 'type': str, + 'default': '', + 'help': 'Uninstall a theme.' + }, + { + 'name': 'list', + 'short': 'l', + 'long': 'list', + 'type': bool, + 'default': False, + 'help': 'Show list of available themes.' + }, + { + 'name': 'list_installed', + 'long': 'list-installed', + 'type': bool, + 'help': "List the installed themes with their location.", + 'default': False + }, + { + 'name': 'url', + 'short': 'u', + 'long': 'url', + 'type': str, + 'help': "URL for the theme repository (default: " + "https://themes.getnikola.com/v7/themes.json)", + 'default': 'https://themes.getnikola.com/v7/themes.json' + }, + { + 'name': 'getpath', + 'short': 'g', + 'long': 'get-path', + 'type': str, + 'default': '', + 'help': "Print the path for installed theme", + }, + { + 'name': 'copy-template', + 'short': 'c', + 'long': 'copy-template', + 'type': str, + 'default': '', + 'help': 'Copy a built-in template into templates/ or your theme', + }, + { + 'name': 'new', + 'short': 'n', + 'long': 'new', + 'type': str, + 'default': '', + 'help': 'Create a new theme', + }, + { + 'name': 'new_engine', + 'long': 'engine', + 'type': str, + 'default': 'mako', + 'help': 'Engine to use for new theme (mako or jinja -- default: mako)', + }, + { + 'name': 'new_parent', + 'long': 'parent', + 'type': str, + 'default': 'base', + 'help': 'Parent to use for new theme (default: base)', + }, + ] + + def _execute(self, options, args): + """Install theme into current site.""" + url = options['url'] + + # See the "mode" we need to operate in + install = options.get('install') + uninstall = options.get('uninstall') + list_available = options.get('list') + list_installed = options.get('list_installed') + get_path = options.get('getpath') + copy_template = options.get('copy-template') + new = options.get('new') + new_engine = options.get('new_engine') + new_parent = options.get('new_parent') + command_count = [bool(x) for x in ( + install, + uninstall, + list_available, + list_installed, + get_path, + copy_template, + new)].count(True) + if command_count > 1 or command_count == 0: + print(self.help()) + return 2 + + if list_available: + return self.list_available(url) + elif list_installed: + return self.list_installed() + elif install: + return self.do_install_deps(url, install) + elif uninstall: + return self.do_uninstall(uninstall) + elif get_path: + return self.get_path(get_path) + elif copy_template: + return self.copy_template(copy_template) + elif new: + return self.new_theme(new, new_engine, new_parent) + + def do_install_deps(self, url, name): + """Install themes and their dependencies.""" + data = self.get_json(url) + # `name` may be modified by the while loop. + origname = name + installstatus = self.do_install(name, data) + # See if the theme's parent is available. If not, install it + while True: + parent_name = utils.get_parent_theme_name(utils.get_theme_path_real(name, self.site.themes_dirs)) + if parent_name is None: + break + try: + utils.get_theme_path_real(parent_name, self.site.themes_dirs) + break + except: # Not available + self.do_install(parent_name, data) + name = parent_name + if installstatus: + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) + + def do_install(self, name, data): + """Download and install a theme.""" + if name in data: + utils.makedirs(self.output_dir) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + + zip_file = io.BytesIO() + zip_file.write(zip_data) + LOGGER.info("Extracting '{0}' into themes/".format(name)) + utils.extract_all(zip_file) + dest_path = os.path.join(self.output_dir, name) + else: + dest_path = os.path.join(self.output_dir, name) + try: + theme_path = utils.get_theme_path_real(name, self.site.themes_dirs) + LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) + except Exception: + LOGGER.error("Can't find theme {0}".format(name)) + + return False + + confpypath = os.path.join(dest_path, 'conf.py.sample') + if os.path.exists(confpypath): + LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') + print('Contents of the conf.py.sample file:\n') + with io.open(confpypath, 'r', encoding='utf-8') as fh: + if self.site.colorful: + print(utils.indent(pygments.highlight( + fh.read(), PythonLexer(), TerminalFormatter()), + 4 * ' ')) + else: + print(utils.indent(fh.read(), 4 * ' ')) + return True + + def do_uninstall(self, name): + """Uninstall a theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + except Exception: + LOGGER.error('Unknown theme: {0}'.format(name)) + return 1 + # Don't uninstall builtin themes (Issue #2510) + blocked = os.path.dirname(utils.__file__) + if path.startswith(blocked): + LOGGER.error("Can't delete builtin theme: {0}".format(name)) + return 1 + LOGGER.warning('About to uninstall theme: {0}'.format(name)) + LOGGER.warning('This will delete {0}'.format(path)) + sure = utils.ask_yesno('Are you sure?') + if sure: + LOGGER.warning('Removing {0}'.format(path)) + shutil.rmtree(path) + return 0 + return 1 + + def get_path(self, name): + """Get path for an installed theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + print(path) + except Exception: + print("not installed") + return 0 + + def list_available(self, url): + """List all available themes.""" + data = self.get_json(url) + print("Available Themes:") + print("-----------------") + for theme in sorted(data.keys()): + print(theme) + return 0 + + def list_installed(self): + """List all installed themes.""" + print("Installed Themes:") + print("-----------------") + themes = [] + themes_dirs = self.site.themes_dirs + [resource_filename('nikola', os.path.join('data', 'themes'))] + for tdir in themes_dirs: + themes += [(i, os.path.join(tdir, i)) for i in os.listdir(tdir)] + for tname, tpath in sorted(set(themes)): + if os.path.isdir(tpath): + print("{0} at {1}".format(tname, tpath)) + + def copy_template(self, template): + """Copy the named template file from the parent to a local theme or to templates/.""" + # Find template + t = self.site.template_system.get_template_path(template) + if t is None: + LOGGER.error("Cannot find template {0} in the lookup.".format(template)) + return 2 + + # Figure out where to put it. + # Check if a local theme exists. + theme_path = utils.get_theme_path(self.site.THEMES[0]) + if theme_path.startswith('themes' + os.sep): + # Theme in local themes/ directory + base = os.path.join(theme_path, 'templates') + else: + # Put it in templates/ + base = 'templates' + + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + try: + out = shutil.copy(t, base) + LOGGER.info("Copied template from {0} to {1}".format(t, out)) + except shutil.SameFileError: + LOGGER.error("This file already exists in your templates directory ({0}).".format(base)) + return 3 + + def new_theme(self, name, engine, parent): + """Create a new theme.""" + base = 'themes' + themedir = os.path.join(base, name) + LOGGER.info("Creating theme {0} with parent {1} and engine {2} in {3}".format(name, parent, engine, themedir)) + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + # Check if engine and parent match + engine_file = utils.get_asset_path('engine', utils.get_theme_chain(parent, self.site.themes_dirs)) + with io.open(engine_file, 'r', encoding='utf-8') as fh: + parent_engine = fh.read().strip() + + if parent_engine != engine: + LOGGER.error("Cannot use engine {0} because parent theme '{1}' uses {2}".format(engine, parent, parent_engine)) + return 2 + + # Create theme + if not os.path.exists(themedir): + os.mkdir(themedir) + LOGGER.info("Created directory {0}".format(themedir)) + else: + LOGGER.error("Theme already exists") + return 2 + + with io.open(os.path.join(themedir, 'parent'), 'w', encoding='utf-8') as fh: + fh.write(parent + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'parent'))) + with io.open(os.path.join(themedir, 'engine'), 'w', encoding='utf-8') as fh: + fh.write(engine + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'engine'))) + + LOGGER.info("Theme {0} created successfully.".format(themedir)) + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(name)) + + def get_json(self, url): + """Download the JSON file with all plugins.""" + if self.json is None: + try: + self.json = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + self.json = requests.get(url).json() + return self.json diff --git a/nikola/plugins/command/version.plugin b/nikola/plugins/command/version.plugin index 4708bdb..d78b79b 100644 --- a/nikola/plugins/command/version.plugin +++ b/nikola/plugins/command/version.plugin @@ -5,7 +5,7 @@ module = version [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Show nikola version [Nikola] diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py index ad08f64..267837e 100644 --- a/nikola/plugins/command/version.py +++ b/nikola/plugins/command/version.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,7 +38,6 @@ URL = 'https://pypi.python.org/pypi?:action=doap&name=Nikola' class CommandVersion(Command): - """Print Nikola version.""" name = "version" diff --git a/nikola/plugins/compile/__init__.py b/nikola/plugins/compile/__init__.py index 60f1919..ff7e9a2 100644 --- a/nikola/plugins/compile/__init__.py +++ b/nikola/plugins/compile/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/compile/html.plugin b/nikola/plugins/compile/html.plugin index 53ade61..f95bdd5 100644 --- a/nikola/plugins/compile/html.plugin +++ b/nikola/plugins/compile/html.plugin @@ -5,7 +5,7 @@ module = html [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Compile HTML into HTML (just copy) [Nikola] diff --git a/nikola/plugins/compile/html.py b/nikola/plugins/compile/html.py index 5f8b244..942d6da 100644 --- a/nikola/plugins/compile/html.py +++ b/nikola/plugins/compile/html.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,7 +36,6 @@ from nikola.utils import makedirs, write_metadata class CompileHtml(PageCompiler): - """Compile HTML into HTML.""" name = "html" @@ -45,12 +44,24 @@ class CompileHtml(PageCompiler): def compile_html(self, source, dest, is_two_file=True): """Compile source file into HTML and save as dest.""" makedirs(os.path.dirname(dest)) + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(dest, "w+", encoding="utf8") as out_file: with io.open(source, "r", encoding="utf8") as in_file: data = in_file.read() if not is_two_file: _, data = self.split_metadata(data) + data, shortcode_deps = self.site.apply_shortcodes(data, with_dependencies=True, extra_context=dict(post=post)) out_file.write(data) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps return True def create_post(self, path, **kw): diff --git a/nikola/plugins/compile/ipynb.py b/nikola/plugins/compile/ipynb.py index a9dedde..f3fdeea 100644 --- a/nikola/plugins/compile/ipynb.py +++ b/nikola/plugins/compile/ipynb.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2013-2015 Damián Avila, Chris Warrick and others. +# Copyright © 2013-2016 Damián Avila, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,28 +32,39 @@ import os import sys try: - import IPython - from IPython.nbconvert.exporters import HTMLExporter - if IPython.version_info[0] >= 3: # API changed with 3.0.0 - from IPython import nbformat - current_nbformat = nbformat.current_nbformat - from IPython.kernel import kernelspec - else: - import IPython.nbformat.current as nbformat - current_nbformat = 'json' - kernelspec = None - - from IPython.config import Config + from nbconvert.exporters import HTMLExporter + import nbformat + current_nbformat = nbformat.current_nbformat + from jupyter_client import kernelspec + from traitlets.config import Config flag = True + ipy_modern = True except ImportError: - flag = None + try: + import IPython + from IPython.nbconvert.exporters import HTMLExporter + if IPython.version_info[0] >= 3: # API changed with 3.0.0 + from IPython import nbformat + current_nbformat = nbformat.current_nbformat + from IPython.kernel import kernelspec + ipy_modern = True + else: + import IPython.nbformat.current as nbformat + current_nbformat = 'json' + kernelspec = None + ipy_modern = False + + from IPython.config import Config + flag = True + except ImportError: + flag = None + ipy_modern = None from nikola.plugin_categories import PageCompiler from nikola.utils import makedirs, req_missing, get_logger, STDERR_HANDLER class CompileIPynb(PageCompiler): - """Compile IPynb into HTML.""" name = "ipynb" @@ -70,7 +81,6 @@ class CompileIPynb(PageCompiler): """Export notebooks as HTML strings.""" if flag is None: req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)') - HTMLExporter.default_template = 'basic' c = Config(self.site.config['IPYNB_CONFIG']) exportHtml = HTMLExporter(config=c) with io.open(source, "r", encoding="utf8") as in_file: @@ -81,8 +91,21 @@ class CompileIPynb(PageCompiler): def compile_html(self, source, dest, is_two_file=True): """Compile source file into HTML and save as dest.""" makedirs(os.path.dirname(dest)) + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(dest, "w+", encoding="utf8") as out_file: - out_file.write(self.compile_html_string(source, is_two_file)) + output = self.compile_html_string(source, is_two_file) + output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post)) + out_file.write(output) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps def read_metadata(self, post, file_metadata_regexp=None, unslugify_titles=False, lang=None): """Read metadata directly from ipynb file. @@ -119,7 +142,7 @@ class CompileIPynb(PageCompiler): # imported .ipynb file, guaranteed to start with "{" because it’s JSON. nb = nbformat.reads(content, current_nbformat) else: - if IPython.version_info[0] >= 3: + if ipy_modern: nb = nbformat.v4.new_notebook() nb["cells"] = [nbformat.v4.new_markdown_cell(content)] else: @@ -152,7 +175,7 @@ class CompileIPynb(PageCompiler): nb["metadata"]["nikola"] = metadata with io.open(path, "w+", encoding="utf8") as fd: - if IPython.version_info[0] >= 3: + if ipy_modern: nbformat.write(nb, fd, 4) else: nbformat.write(nb, fd, 'ipynb') diff --git a/nikola/plugins/compile/markdown.plugin b/nikola/plugins/compile/markdown.plugin index f7d11b1..2607413 100644 --- a/nikola/plugins/compile/markdown.plugin +++ b/nikola/plugins/compile/markdown.plugin @@ -5,7 +5,7 @@ module = markdown [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Compile Markdown into HTML [Nikola] diff --git a/nikola/plugins/compile/markdown/__init__.py b/nikola/plugins/compile/markdown/__init__.py index c1425a1..2e4234c 100644 --- a/nikola/plugins/compile/markdown/__init__.py +++ b/nikola/plugins/compile/markdown/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -44,7 +44,6 @@ from nikola.utils import makedirs, req_missing, write_metadata class CompileMarkdown(PageCompiler): - """Compile Markdown into HTML.""" name = "markdown" @@ -70,13 +69,25 @@ class CompileMarkdown(PageCompiler): req_missing(['markdown'], 'build this site (compile Markdown)') makedirs(os.path.dirname(dest)) self.extensions += self.site.config.get("MARKDOWN_EXTENSIONS") + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(dest, "w+", encoding="utf8") as out_file: with io.open(source, "r", encoding="utf8") as in_file: data = in_file.read() if not is_two_file: _, data = self.split_metadata(data) - output = markdown(data, self.extensions) + output = markdown(data, self.extensions, output_format="html5") + output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post)) out_file.write(output) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps def create_post(self, path, **kw): """Create a new post.""" diff --git a/nikola/plugins/compile/markdown/mdx_gist.plugin b/nikola/plugins/compile/markdown/mdx_gist.plugin index 7fe676c..85b5450 100644 --- a/nikola/plugins/compile/markdown/mdx_gist.plugin +++ b/nikola/plugins/compile/markdown/mdx_gist.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Extension for embedding gists diff --git a/nikola/plugins/compile/markdown/mdx_gist.py b/nikola/plugins/compile/markdown/mdx_gist.py index f439fa2..25c071f 100644 --- a/nikola/plugins/compile/markdown/mdx_gist.py +++ b/nikola/plugins/compile/markdown/mdx_gist.py @@ -22,7 +22,7 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # Warning: URL formats of "raw" gists are undocummented and subject to change. -# See also: http://developer.github.com/v3/gists/ +# See also: https://developer.github.com/v3/gists/ # # Inspired by "[Python] reStructuredText GitHub Gist directive" # (https://gist.github.com/brianhsu/1407759), public domain by Brian Hsu @@ -31,161 +31,48 @@ Extension to Python Markdown for Embedded Gists (gist.github.com). Basic Example: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 4747847] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js"></script> - <noscript> - <pre>import this</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: 4747847] Example with filename: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 4747847 zen.py] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js?file=zen.py"></script> - <noscript> - <pre>import this</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: 4747847 zen.py] Basic Example with hexidecimal id: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: c4a43d6fdce612284ac0] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: c4a43d6fdce612284ac0] Example with hexidecimal id filename: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: c4a43d6fdce612284ac0 cow.txt] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js?file=cow.txt"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: c4a43d6fdce612284ac0 cow.txt] Example using reStructuredText syntax: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... .. gist:: 4747847 zen.py - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js?file=zen.py"></script> - <noscript> - <pre>import this</pre> - </noscript> - </div> - </p> + Text of the gist: + .. gist:: 4747847 zen.py Example using hexidecimal ID with reStructuredText syntax: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... .. gist:: c4a43d6fdce612284ac0 - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + .. gist:: c4a43d6fdce612284ac0 Example using hexidecimal ID and filename with reStructuredText syntax: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... .. gist:: c4a43d6fdce612284ac0 cow.txt - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js?file=cow.txt"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + .. gist:: c4a43d6fdce612284ac0 cow.txt Error Case: non-existent Gist ID: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 0] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/0.js"></script> - <noscript><!-- WARNING: Received a 404 response from Gist URL: https://gist.githubusercontent.com/raw/0 --></noscript> - </div> - </p> - -Error Case: non-existent file: - - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 4747847 doesntexist.py] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js?file=doesntexist.py"></script> - <noscript><!-- WARNING: Received a 404 response from Gist URL: https://gist.githubusercontent.com/raw/4747847/doesntexist.py --></noscript> - </div> - </p> + Text of the gist: + [:gist: 0] + +Error Case: non-existent file: + + Text of the gist: + [:gist: 4747847 doesntexist.py] """ from __future__ import unicode_literals, print_function @@ -217,7 +104,6 @@ GIST_RST_RE = r'(?m)^\.\.\s*gist::\s*(?P<gist_id>[^\]\s]+)(?:\s*(?P<filename>.+? class GistFetchException(Exception): - """Raised when attempt to fetch content of a Gist from github.com fails.""" def __init__(self, url, status_code): @@ -228,7 +114,6 @@ class GistFetchException(Exception): class GistPattern(Pattern): - """InlinePattern for footnote markers in a document's body text.""" def __init__(self, pattern, configs): @@ -290,7 +175,6 @@ class GistPattern(Pattern): class GistExtension(MarkdownExtension, Extension): - """Gist extension for Markdown.""" def __init__(self, configs={}): diff --git a/nikola/plugins/compile/markdown/mdx_nikola.plugin b/nikola/plugins/compile/markdown/mdx_nikola.plugin index 12e4fb6..3c5c638 100644 --- a/nikola/plugins/compile/markdown/mdx_nikola.plugin +++ b/nikola/plugins/compile/markdown/mdx_nikola.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Nikola-specific Markdown extensions diff --git a/nikola/plugins/compile/markdown/mdx_nikola.py b/nikola/plugins/compile/markdown/mdx_nikola.py index 54cc18c..59a5d5b 100644 --- a/nikola/plugins/compile/markdown/mdx_nikola.py +++ b/nikola/plugins/compile/markdown/mdx_nikola.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -24,25 +24,31 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Markdown Extension for Nikola-specific post-processing.""" +"""Markdown Extension for Nikola. + +- Specific post-processing. +- Strikethrough inline patterns. +""" from __future__ import unicode_literals import re try: from markdown.postprocessors import Postprocessor + from markdown.inlinepatterns import SimpleTagPattern from markdown.extensions import Extension except ImportError: # No need to catch this, if you try to use this without Markdown, # the markdown compiler will fail first - Postprocessor = Extension = object + Postprocessor = SimpleTagPattern = Extension = object from nikola.plugin_categories import MarkdownExtension + CODERE = re.compile('<div class="codehilite"><pre>(.*?)</pre></div>', flags=re.MULTILINE | re.DOTALL) +STRIKE_RE = r"(~{2})(.+?)(~{2})" # ~~strike~~ class NikolaPostProcessor(Postprocessor): - """Nikola-specific post-processing for Markdown.""" def run(self, text): @@ -57,13 +63,22 @@ class NikolaPostProcessor(Postprocessor): class NikolaExtension(MarkdownExtension, Extension): + """Nikola Markdown extensions.""" - """Extension for injecting the postprocessor.""" - - def extendMarkdown(self, md, md_globals): + def _add_nikola_post_processor(self, md): """Extend Markdown with the postprocessor.""" pp = NikolaPostProcessor() md.postprocessors.add('nikola_post_processor', pp, '_end') + + def _add_strikethrough_inline_pattern(self, md): + """Support PHP-Markdown style strikethrough, for example: ``~~strike~~``.""" + pattern = SimpleTagPattern(STRIKE_RE, 'del') + md.inlinePatterns.add('strikethrough', pattern, '_end') + + def extendMarkdown(self, md, md_globals): + """Extend markdown to Nikola flavours.""" + self._add_nikola_post_processor(md) + self._add_strikethrough_inline_pattern(md) md.registerExtension(self) diff --git a/nikola/plugins/compile/markdown/mdx_podcast.plugin b/nikola/plugins/compile/markdown/mdx_podcast.plugin index c92a8a0..c4ee7e9 100644 --- a/nikola/plugins/compile/markdown/mdx_podcast.plugin +++ b/nikola/plugins/compile/markdown/mdx_podcast.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Markdown extensions for embedding podcasts and other audio files diff --git a/nikola/plugins/compile/markdown/mdx_podcast.py b/nikola/plugins/compile/markdown/mdx_podcast.py index 61afdbf..96a70ed 100644 --- a/nikola/plugins/compile/markdown/mdx_podcast.py +++ b/nikola/plugins/compile/markdown/mdx_podcast.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright © 2013-2015 Michael Rabbitt, Roberto Alsina and others. +# Copyright © 2013-2016 Michael Rabbitt, Roberto Alsina and others. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the @@ -30,10 +30,10 @@ Extension to Python Markdown for Embedded Audio. Basic Example: >>> import markdown ->>> text = "[podcast]http://archive.org/download/Rebeldes_Stereotipos/rs20120609_1.mp3[/podcast]" +>>> text = "[podcast]https://archive.org/download/Rebeldes_Stereotipos/rs20120609_1.mp3[/podcast]" >>> html = markdown.markdown(text, [PodcastExtension()]) >>> print(html) -<p><audio controls=""><source src="http://archive.org/download/Rebeldes_Stereotipos/rs20120609_1.mp3" type="audio/mpeg"></source></audio></p> +<p><audio controls=""><source src="https://archive.org/download/Rebeldes_Stereotipos/rs20120609_1.mp3" type="audio/mpeg"></source></audio></p> """ from __future__ import print_function, unicode_literals @@ -51,7 +51,6 @@ PODCAST_RE = r'\[podcast\](?P<url>.+)\[/podcast\]' class PodcastPattern(Pattern): - """InlinePattern for footnote markers in a document's body text.""" def __init__(self, pattern, configs): @@ -70,7 +69,6 @@ class PodcastPattern(Pattern): class PodcastExtension(MarkdownExtension, Extension): - """"Podcast extension for Markdown.""" def __init__(self, configs={}): diff --git a/nikola/plugins/compile/pandoc.plugin b/nikola/plugins/compile/pandoc.plugin index 3ff3668..2a69095 100644 --- a/nikola/plugins/compile/pandoc.plugin +++ b/nikola/plugins/compile/pandoc.plugin @@ -5,7 +5,7 @@ module = pandoc [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Compile markups into HTML using pandoc [Nikola] diff --git a/nikola/plugins/compile/pandoc.py b/nikola/plugins/compile/pandoc.py index 3030626..2368ae9 100644 --- a/nikola/plugins/compile/pandoc.py +++ b/nikola/plugins/compile/pandoc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -40,7 +40,6 @@ from nikola.utils import req_missing, makedirs, write_metadata class CompilePandoc(PageCompiler): - """Compile markups into HTML using pandoc.""" name = "pandoc" @@ -55,7 +54,22 @@ class CompilePandoc(PageCompiler): """Compile source file into HTML and save as dest.""" makedirs(os.path.dirname(dest)) try: + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None subprocess.check_call(['pandoc', '-o', dest, source] + self.site.config['PANDOC_OPTIONS']) + with open(dest, 'r', encoding='utf-8') as inf: + output, shortcode_deps = self.site.apply_shortcodes(inf.read(), with_dependencies=True) + with open(dest, 'w', encoding='utf-8') as outf: + outf.write(output) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps except OSError as e: if e.strreror == 'No such file or directory': req_missing(['pandoc'], 'build this site (compile with pandoc)', python=False) diff --git a/nikola/plugins/compile/php.plugin b/nikola/plugins/compile/php.plugin index 151c022..f4fb0c1 100644 --- a/nikola/plugins/compile/php.plugin +++ b/nikola/plugins/compile/php.plugin @@ -5,7 +5,7 @@ module = php [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Compile PHP into HTML (just copy and name the file .php) [Nikola] diff --git a/nikola/plugins/compile/php.py b/nikola/plugins/compile/php.py index 28f4923..d2559fd 100644 --- a/nikola/plugins/compile/php.py +++ b/nikola/plugins/compile/php.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -37,7 +37,6 @@ from hashlib import md5 class CompilePhp(PageCompiler): - """Compile PHP into PHP.""" name = "php" diff --git a/nikola/plugins/compile/rest.plugin b/nikola/plugins/compile/rest.plugin index cf842c7..4d9041a 100644 --- a/nikola/plugins/compile/rest.plugin +++ b/nikola/plugins/compile/rest.plugin @@ -5,7 +5,7 @@ module = rest [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Compile reSt into HTML [Nikola] diff --git a/nikola/plugins/compile/rest/__init__.py b/nikola/plugins/compile/rest/__init__.py index b99e872..b75849f 100644 --- a/nikola/plugins/compile/rest/__init__.py +++ b/nikola/plugins/compile/rest/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,13 +36,22 @@ import docutils.utils import docutils.io import docutils.readers.standalone import docutils.writers.html4css1 +import docutils.parsers.rst.directives +from docutils.parsers.rst import roles +from nikola.nikola import LEGAL_VALUES from nikola.plugin_categories import PageCompiler -from nikola.utils import unicode_str, get_logger, makedirs, write_metadata, STDERR_HANDLER +from nikola.utils import ( + unicode_str, + get_logger, + makedirs, + write_metadata, + STDERR_HANDLER, + LocaleBorg +) class CompileRest(PageCompiler): - """Compile reStructuredText into HTML.""" name = "rest" @@ -50,19 +59,6 @@ class CompileRest(PageCompiler): demote_headers = True logger = None - def _read_extra_deps(self, post): - """Read contents of .dep file and returns them as a list.""" - dep_path = post.base_path + '.dep' - if os.path.isfile(dep_path): - with io.open(dep_path, 'r+', encoding='utf8') as depf: - deps = [l.strip() for l in depf.readlines()] - return deps - return [] - - def register_extra_dependencies(self, post): - """Add dependency to post object to check .dep file.""" - post.add_dependency(lambda: self._read_extra_deps(post), 'fragment') - def compile_html_string(self, data, source_path=None, is_two_file=True): """Compile reST into HTML strings.""" # If errors occur, this will be added to the line number reported by @@ -74,16 +70,20 @@ class CompileRest(PageCompiler): add_ln = len(m_data.splitlines()) + 1 default_template_path = os.path.join(os.path.dirname(__file__), 'template.txt') + settings_overrides = { + 'initial_header_level': 1, + 'record_dependencies': True, + 'stylesheet_path': None, + 'link_stylesheet': True, + 'syntax_highlight': 'short', + 'math_output': 'mathjax', + 'template': default_template_path, + 'language_code': LEGAL_VALUES['DOCUTILS_LOCALES'].get(LocaleBorg().current_lang, 'en') + } + output, error_level, deps = rst2html( - data, settings_overrides={ - 'initial_header_level': 1, - 'record_dependencies': True, - 'stylesheet_path': None, - 'link_stylesheet': True, - 'syntax_highlight': 'short', - 'math_output': 'mathjax', - 'template': default_template_path, - }, logger=self.logger, source_path=source_path, l_add_ln=add_ln, transforms=self.site.rst_transforms) + data, settings_overrides=settings_overrides, logger=self.logger, source_path=source_path, l_add_ln=add_ln, transforms=self.site.rst_transforms, + no_title_transform=self.site.config.get('NO_DOCUTILS_TITLE_TRANSFORM', False)) if not isinstance(output, unicode_str): # To prevent some weird bugs here or there. # Original issue: empty files. `output` became a bytestring. @@ -95,18 +95,23 @@ class CompileRest(PageCompiler): makedirs(os.path.dirname(dest)) error_level = 100 with io.open(dest, "w+", encoding="utf8") as out_file: + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(source, "r", encoding="utf8") as in_file: data = in_file.read() output, error_level, deps = self.compile_html_string(data, source, is_two_file) + output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post)) out_file.write(output) - deps_path = dest + '.dep' - if deps.list: - deps.list = [p for p in deps.list if p != dest] # Don't depend on yourself (#1671) - with io.open(deps_path, "w+", encoding="utf8") as deps_file: - deps_file.write('\n'.join(deps.list)) + if post is None: + if deps.list: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) else: - if os.path.isfile(deps_path): - os.unlink(deps_path) + post._depfile[dest] += deps.list + post._depfile[dest] += shortcode_deps if error_level < 3: return True else: @@ -172,17 +177,20 @@ def get_observer(settings): class NikolaReader(docutils.readers.standalone.Reader): - """Nikola-specific docutils reader.""" def __init__(self, *args, **kwargs): """Initialize the reader.""" self.transforms = kwargs.pop('transforms', []) + self.no_title_transform = kwargs.pop('no_title_transform', False) docutils.readers.standalone.Reader.__init__(self, *args, **kwargs) def get_transforms(self): """Get docutils transforms.""" - return docutils.readers.standalone.Reader(self).get_transforms() + self.transforms + transforms = docutils.readers.standalone.Reader(self).get_transforms() + self.transforms + if self.no_title_transform: + transforms = [t for t in transforms if str(t) != "<class 'docutils.transforms.frontmatter.DocTitle'>"] + return transforms def new_document(self): """Create and return a new empty document tree (root node).""" @@ -192,11 +200,21 @@ class NikolaReader(docutils.readers.standalone.Reader): return document +def shortcode_role(name, rawtext, text, lineno, inliner, + options={}, content=[]): + """A shortcode role that passes through raw inline HTML.""" + return [docutils.nodes.raw('', text, format='html')], [] + +roles.register_canonical_role('raw-html', shortcode_role) +roles.register_canonical_role('html', shortcode_role) +roles.register_canonical_role('sc', shortcode_role) + + def add_node(node, visit_function=None, depart_function=None): """Register a Docutils node class. This function is completely optional. It is a same concept as - `Sphinx add_node function <http://sphinx-doc.org/ext/appapi.html#sphinx.application.Sphinx.add_node>`_. + `Sphinx add_node function <http://sphinx-doc.org/extdev/appapi.html#sphinx.application.Sphinx.add_node>`_. For example:: @@ -237,7 +255,8 @@ def rst2html(source, source_path=None, source_class=docutils.io.StringInput, parser=None, parser_name='restructuredtext', writer=None, writer_name='html', settings=None, settings_spec=None, settings_overrides=None, config_section=None, - enable_exit_status=None, logger=None, l_add_ln=0, transforms=None): + enable_exit_status=None, logger=None, l_add_ln=0, transforms=None, + no_title_transform=False): """Set up & run a ``Publisher``, and return a dictionary of document parts. Dictionary keys are the names of parts, and values are Unicode strings; @@ -255,7 +274,7 @@ def rst2html(source, source_path=None, source_class=docutils.io.StringInput, reStructuredText syntax errors. """ if reader is None: - reader = NikolaReader(transforms=transforms) + reader = NikolaReader(transforms=transforms, no_title_transform=no_title_transform) # For our custom logging, we have special needs and special settings we # specify here. # logger a logger from Nikola @@ -276,3 +295,10 @@ def rst2html(source, source_path=None, source_class=docutils.io.StringInput, pub.publish(enable_exit_status=enable_exit_status) return pub.writer.parts['docinfo'] + pub.writer.parts['fragment'], pub.document.reporter.max_level, pub.settings.record_dependencies + +# Alignment helpers for extensions +_align_options_base = ('left', 'center', 'right') + + +def _align_choice(argument): + return docutils.parsers.rst.directives.choice(argument, _align_options_base + ("none", "")) diff --git a/nikola/plugins/compile/rest/chart.plugin b/nikola/plugins/compile/rest/chart.plugin index 438abe4..0a7896f 100644 --- a/nikola/plugins/compile/rest/chart.plugin +++ b/nikola/plugins/compile/rest/chart.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Chart directive based in PyGal diff --git a/nikola/plugins/compile/rest/chart.py b/nikola/plugins/compile/rest/chart.py index 88fdff3..24f459b 100644 --- a/nikola/plugins/compile/rest/chart.py +++ b/nikola/plugins/compile/rest/chart.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -43,7 +43,6 @@ _site = None class Plugin(RestExtension): - """Plugin for chart role.""" name = "rest_chart" @@ -53,11 +52,11 @@ class Plugin(RestExtension): global _site _site = self.site = site directives.register_directive('chart', Chart) + self.site.register_shortcode('chart', _gen_chart) return super(Plugin, self).set_site(site) class Chart(Directive): - """reStructuredText extension for inserting charts as SVG. Usage: @@ -74,52 +73,68 @@ class Chart(Directive): has_content = True required_arguments = 1 option_spec = { - "copy": directives.unchanged, + "box_mode": directives.unchanged, + "classes": directives.unchanged, "css": directives.unchanged, + "defs": directives.unchanged, "disable_xml_declaration": directives.unchanged, "dots_size": directives.unchanged, + "dynamic_print_values": directives.unchanged, "explicit_size": directives.unchanged, "fill": directives.unchanged, - "font_sizes": directives.unchanged, + "force_uri_protocol": directives.unchanged, + "half_pie": directives.unchanged, "height": directives.unchanged, "human_readable": directives.unchanged, "include_x_axis": directives.unchanged, + "inner_radius": directives.unchanged, "interpolate": directives.unchanged, "interpolation_parameters": directives.unchanged, "interpolation_precision": directives.unchanged, + "inverse_y_axis": directives.unchanged, "js": directives.unchanged, - "label_font_size": directives.unchanged, "legend_at_bottom": directives.unchanged, + "legend_at_bottom_columns": directives.unchanged, "legend_box_size": directives.unchanged, - "legend_font_size": directives.unchanged, "logarithmic": directives.unchanged, - "major_label_font_size": directives.unchanged, "margin": directives.unchanged, - "no_data_font_size": directives.unchanged, + "margin_bottom": directives.unchanged, + "margin_left": directives.unchanged, + "margin_right": directives.unchanged, + "margin_top": directives.unchanged, + "max_scale": directives.unchanged, + "min_scale": directives.unchanged, + "missing_value_fill_truncation": directives.unchanged, "no_data_text": directives.unchanged, "no_prefix": directives.unchanged, "order_min": directives.unchanged, "pretty_print": directives.unchanged, + "print_labels": directives.unchanged, "print_values": directives.unchanged, + "print_values_position": directives.unchanged, "print_zeroes": directives.unchanged, "range": directives.unchanged, "rounded_bars": directives.unchanged, + "secondary_range": directives.unchanged, "show_dots": directives.unchanged, "show_legend": directives.unchanged, "show_minor_x_labels": directives.unchanged, + "show_minor_y_labels": directives.unchanged, + "show_only_major_dots": directives.unchanged, + "show_x_guides": directives.unchanged, + "show_x_labels": directives.unchanged, + "show_y_guides": directives.unchanged, "show_y_labels": directives.unchanged, "spacing": directives.unchanged, + "stack_from_top": directives.unchanged, "strict": directives.unchanged, "stroke": directives.unchanged, + "stroke_style": directives.unchanged, "style": directives.unchanged, "title": directives.unchanged, - "title_font_size": directives.unchanged, - "to_dict": directives.unchanged, "tooltip_border_radius": directives.unchanged, - "tooltip_font_size": directives.unchanged, "truncate_label": directives.unchanged, "truncate_legend": directives.unchanged, - "value_font_size": directives.unchanged, "value_formatter": directives.unchanged, "width": directives.unchanged, "x_label_rotation": directives.unchanged, @@ -128,37 +143,55 @@ class Chart(Directive): "x_labels_major_count": directives.unchanged, "x_labels_major_every": directives.unchanged, "x_title": directives.unchanged, + "x_value_formatter": directives.unchanged, + "xrange": directives.unchanged, "y_label_rotation": directives.unchanged, "y_labels": directives.unchanged, + "y_labels_major": directives.unchanged, + "y_labels_major_count": directives.unchanged, + "y_labels_major_every": directives.unchanged, "y_title": directives.unchanged, "zero": directives.unchanged, } def run(self): """Run the directive.""" - if pygal is None: - msg = req_missing(['pygal'], 'use the Chart directive', optional=True) - return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')] - options = {} - if 'style' in self.options: - style_name = self.options.pop('style') - else: - style_name = 'BlueStyle' - if '(' in style_name: # Parametric style - style = eval('pygal.style.' + style_name) - else: - style = getattr(pygal.style, style_name) - for k, v in self.options.items(): + self.options['site'] = None + html = _gen_chart(self.arguments[0], data='\n'.join(self.content), **self.options) + return [nodes.raw('', html, format='html')] + + +def _gen_chart(chart_type, **_options): + if pygal is None: + msg = req_missing(['pygal'], 'use the Chart directive', optional=True) + return '<div class="text-error">{0}</div>'.format(msg) + options = {} + data = _options.pop('data') + _options.pop('post', None) + _options.pop('site') + if 'style' in _options: + style_name = _options.pop('style') + else: + style_name = 'BlueStyle' + if '(' in style_name: # Parametric style + style = eval('pygal.style.' + style_name) + else: + style = getattr(pygal.style, style_name) + for k, v in _options.items(): + try: options[k] = literal_eval(v) - - chart = getattr(pygal, self.arguments[0])(style=style) - chart.config(**options) - for line in self.content: + except: + options[k] = v + chart = pygal + for o in chart_type.split('.'): + chart = getattr(chart, o) + chart = chart(style=style) + if _site and _site.invariant: + chart.no_prefix = True + chart.config(**options) + for line in data.splitlines(): + line = line.strip() + if line: label, series = literal_eval('({0})'.format(line)) chart.add(label, series) - data = chart.render().decode('utf8') - if _site and _site.invariant: - import re - data = re.sub('id="chart-[a-f0-9\-]+"', 'id="chart-foobar"', data) - data = re.sub('#chart-[a-f0-9\-]+', '#chart-foobar', data) - return [nodes.raw('', data, format='html')] + return chart.render().decode('utf8') diff --git a/nikola/plugins/compile/rest/doc.plugin b/nikola/plugins/compile/rest/doc.plugin index facdd03..e447eb2 100644 --- a/nikola/plugins/compile/rest/doc.plugin +++ b/nikola/plugins/compile/rest/doc.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Manuel Kaufmann version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Role to link another page / post from the blog diff --git a/nikola/plugins/compile/rest/doc.py b/nikola/plugins/compile/rest/doc.py index 99cce81..55f576d 100644 --- a/nikola/plugins/compile/rest/doc.py +++ b/nikola/plugins/compile/rest/doc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,12 +29,11 @@ from docutils import nodes from docutils.parsers.rst import roles -from nikola.utils import split_explicit_title +from nikola.utils import split_explicit_title, LOGGER from nikola.plugin_categories import RestExtension class Plugin(RestExtension): - """Plugin for doc role.""" name = 'rest_doc' @@ -43,12 +42,12 @@ class Plugin(RestExtension): """Set Nikola site.""" self.site = site roles.register_canonical_role('doc', doc_role) + self.site.register_shortcode('doc', doc_shortcode) doc_role.site = site return super(Plugin, self).set_site(site) -def doc_role(name, rawtext, text, lineno, inliner, - options={}, content=[]): +def _doc_link(rawtext, text, options={}, content=[]): """Handle the doc role.""" # split link's text and post's slug in role content has_explicit_title, title, slug = split_explicit_title(text) @@ -67,22 +66,48 @@ def doc_role(name, rawtext, text, lineno, inliner, if post is None: raise ValueError except ValueError: + return False, False, None, None, slug + + if not has_explicit_title: + # use post's title as link's text + title = post.title() + permalink = post.permalink() + + return True, twin_slugs, title, permalink, slug + + +def doc_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + """Handle the doc role.""" + success, twin_slugs, title, permalink, slug = _doc_link(rawtext, text, options, content) + if success: + if twin_slugs: + inliner.reporter.warning( + 'More than one post with the same slug. Using "{0}"'.format(permalink)) + LOGGER.warn( + 'More than one post with the same slug. Using "{0}" for doc role'.format(permalink)) + node = make_link_node(rawtext, title, permalink, options) + return [node], [] + else: msg = inliner.reporter.error( '"{0}" slug doesn\'t exist.'.format(slug), line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] - if not has_explicit_title: - # use post's title as link's text - title = post.title() - permalink = post.permalink() - if twin_slugs: - msg = inliner.reporter.warning( - 'More than one post with the same slug. Using "{0}"'.format(permalink)) - node = make_link_node(rawtext, title, permalink, options) - return [node], [] +def doc_shortcode(*args, **kwargs): + """Implement the doc shortcode.""" + text = kwargs['data'] + success, twin_slugs, title, permalink, slug = _doc_link(text, text, LOGGER) + if success: + if twin_slugs: + LOGGER.warn( + 'More than one post with the same slug. Using "{0}" for doc shortcode'.format(permalink)) + return '<a href="{0}">{1}</a>'.format(permalink, title) + else: + LOGGER.error( + '"{0}" slug doesn\'t exist.'.format(slug)) + return '<span class="error text-error" style="color: red;">Invalid link: {0}</span>'.format(text) def make_link_node(rawtext, text, url, options): diff --git a/nikola/plugins/compile/rest/gist.plugin b/nikola/plugins/compile/rest/gist.plugin index 9fa2e82..763c1d2 100644 --- a/nikola/plugins/compile/rest/gist.plugin +++ b/nikola/plugins/compile/rest/gist.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Gist directive diff --git a/nikola/plugins/compile/rest/gist.py b/nikola/plugins/compile/rest/gist.py index 736ee37..e40c3b2 100644 --- a/nikola/plugins/compile/rest/gist.py +++ b/nikola/plugins/compile/rest/gist.py @@ -11,7 +11,6 @@ from nikola.plugin_categories import RestExtension class Plugin(RestExtension): - """Plugin for gist directive.""" name = "rest_gist" @@ -24,7 +23,6 @@ class Plugin(RestExtension): class GitHubGist(Directive): - """Embed GitHub Gist. Usage: diff --git a/nikola/plugins/compile/rest/listing.plugin b/nikola/plugins/compile/rest/listing.plugin index 85c780f..3ebb296 100644 --- a/nikola/plugins/compile/rest/listing.plugin +++ b/nikola/plugins/compile/rest/listing.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Extension for source listings diff --git a/nikola/plugins/compile/rest/listing.py b/nikola/plugins/compile/rest/listing.py index 4871bf3..4dfbedc 100644 --- a/nikola/plugins/compile/rest/listing.py +++ b/nikola/plugins/compile/rest/listing.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -55,7 +55,6 @@ from nikola.plugin_categories import RestExtension # A sanitized version of docutils.parsers.rst.directives.body.CodeBlock. class CodeBlock(Directive): - """Parse and mark up content of a code block.""" optional_arguments = 1 @@ -126,7 +125,6 @@ docutils.parsers.rst.directives.misc.CodeBlock = CodeBlock class Plugin(RestExtension): - """Plugin for listing directive.""" name = "rest_listing" @@ -138,6 +136,7 @@ class Plugin(RestExtension): # leaving these to make the code directive work with # docutils < 0.9 CodeBlock.site = site + Listing.site = site directives.register_directive('code', CodeBlock) directives.register_directive('code-block', CodeBlock) directives.register_directive('sourcecode', CodeBlock) @@ -152,7 +151,6 @@ listing_spec['linenos'] = directives.unchanged class Listing(Include): - """Create a highlighted block of code from a file in listings/. Usage: @@ -171,7 +169,12 @@ class Listing(Include): """Run listing directive.""" _fname = self.arguments.pop(0) fname = _fname.replace('/', os.sep) - lang = self.arguments.pop(0) + try: + lang = self.arguments.pop(0) + self.options['code'] = lang + except IndexError: + self.options['literal'] = True + if len(self.folders) == 1: listings_folder = next(iter(self.folders.keys())) if fname.startswith(listings_folder): @@ -181,15 +184,17 @@ class Listing(Include): else: fpath = os.path.join(fname) # must be new syntax: specify folder name self.arguments.insert(0, fpath) - self.options['code'] = lang if 'linenos' in self.options: self.options['number-lines'] = self.options['linenos'] with io.open(fpath, 'r+', encoding='utf8') as fileobject: self.content = fileobject.read().splitlines() self.state.document.settings.record_dependencies.add(fpath) target = urlunsplit(("link", 'listing', fpath.replace('\\', '/'), '', '')) + src_target = urlunsplit(("link", 'listing_source', fpath.replace('\\', '/'), '', '')) + src_label = self.site.MESSAGES('Source') generated_nodes = ( - [core.publish_doctree('`{0} <{1}>`_'.format(_fname, target))[0]]) + [core.publish_doctree('`{0} <{1}>`_ `({2}) <{3}>`_' .format( + _fname, target, src_label, src_target))[0]]) generated_nodes += self.get_code_from_file(fileobject) return generated_nodes diff --git a/nikola/plugins/compile/rest/media.plugin b/nikola/plugins/compile/rest/media.plugin index 9803c8f..8dfb19c 100644 --- a/nikola/plugins/compile/rest/media.plugin +++ b/nikola/plugins/compile/rest/media.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Directive to support oembed via micawber diff --git a/nikola/plugins/compile/rest/media.py b/nikola/plugins/compile/rest/media.py index 345e331..8a69586 100644 --- a/nikola/plugins/compile/rest/media.py +++ b/nikola/plugins/compile/rest/media.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -40,7 +40,6 @@ from nikola.utils import req_missing class Plugin(RestExtension): - """Plugin for reST media directive.""" name = "rest_media" @@ -49,11 +48,11 @@ class Plugin(RestExtension): """Set Nikola site.""" self.site = site directives.register_directive('media', Media) + self.site.register_shortcode('media', _gen_media_embed) return super(Plugin, self).set_site(site) class Media(Directive): - """reST extension for inserting any sort of media using micawber.""" has_content = False @@ -62,9 +61,13 @@ class Media(Directive): def run(self): """Run media directive.""" - if micawber is None: - msg = req_missing(['micawber'], 'use the media directive', optional=True) - return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')] + html = _gen_media_embed(" ".join(self.arguments)) + return [nodes.raw('', html, format='html')] + - providers = micawber.bootstrap_basic() - return [nodes.raw('', micawber.parse_text(" ".join(self.arguments), providers), format='html')] +def _gen_media_embed(url, *q, **kw): + if micawber is None: + msg = req_missing(['micawber'], 'use the media directive', optional=True) + return '<div class="text-error">{0}</div>'.format(msg) + providers = micawber.bootstrap_basic() + return micawber.parse_text(url, providers) diff --git a/nikola/plugins/compile/rest/post_list.plugin b/nikola/plugins/compile/rest/post_list.plugin index 48969bf..1802f2b 100644 --- a/nikola/plugins/compile/rest/post_list.plugin +++ b/nikola/plugins/compile/rest/post_list.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Udo Spallek version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Includes a list of posts with tag and slide based filters. diff --git a/nikola/plugins/compile/rest/post_list.py b/nikola/plugins/compile/rest/post_list.py index a22ee85..8cfd5bf 100644 --- a/nikola/plugins/compile/rest/post_list.py +++ b/nikola/plugins/compile/rest/post_list.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2013-2015 Udo Spallek, Roberto Alsina and others. +# Copyright © 2013-2016 Udo Spallek, Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -37,13 +37,13 @@ from docutils.parsers.rst import Directive, directives from nikola import utils from nikola.plugin_categories import RestExtension +from nikola.packages.datecond import date_in_range # WARNING: the directive name is post-list # (with a DASH instead of an UNDERSCORE) class Plugin(RestExtension): - """Plugin for reST post-list directive.""" name = "rest_post_list" @@ -51,19 +51,19 @@ class Plugin(RestExtension): def set_site(self, site): """Set Nikola site.""" self.site = site + self.site.register_shortcode('post-list', _do_post_list) directives.register_directive('post-list', PostList) PostList.site = site return super(Plugin, self).set_site(site) class PostList(Directive): - """Provide a reStructuredText directive to create a list of posts. Post List ========= :Directive Arguments: None. - :Directive Options: lang, start, stop, reverse, sort, tags, categories, slugs, all, template, id + :Directive Options: lang, start, stop, reverse, sort, date, tags, categories, sections, slugs, post_type, all, template, id :Directive Content: None. The posts appearing in the list can be filtered by options. @@ -87,10 +87,19 @@ class PostList(Directive): Reverse the order of the post-list. Defaults is to not reverse the order of posts. - ``sort``: string + ``sort`` : string Sort post list by one of each post's attributes, usually ``title`` or a custom ``priority``. Defaults to None (chronological sorting). + ``date`` : string + Show posts that match date range specified by this option. Format: + + * comma-separated clauses (AND) + * clause: attribute comparison_operator value (spaces optional) + * attribute: year, month, day, hour, month, second, weekday, isoweekday; or empty for full datetime + * comparison_operator: == != <= >= < > + * value: integer or dateutil-compatible date input + ``tags`` : string [, string...] Filter posts to show only posts having at least one of the ``tags``. Defaults to None. @@ -99,13 +108,21 @@ class PostList(Directive): Filter posts to show only posts having one of the ``categories``. Defaults to None. + ``sections`` : string [, string...] + Filter posts to show only posts having one of the ``sections``. + Defaults to None. + ``slugs`` : string [, string...] Filter posts to show only posts having at least one of the ``slugs``. Defaults to None. + ``post_type`` (or ``type``) : string + Show only ``posts``, ``pages`` or ``all``. + Replaces ``all``. Defaults to ``posts``. + ``all`` : flag - Shows all posts and pages in the post list. - Defaults to show only posts with set *use_in_feeds*. + (deprecated, use ``post_type`` instead) + Shows all posts and pages in the post list. Defaults to show only posts. ``lang`` : string The language of post *titles* and *links*. @@ -127,11 +144,15 @@ class PostList(Directive): 'sort': directives.unchanged, 'tags': directives.unchanged, 'categories': directives.unchanged, + 'sections': directives.unchanged, 'slugs': directives.unchanged, + 'post_type': directives.unchanged, + 'type': directives.unchanged, 'all': directives.flag, 'lang': directives.unchanged, 'template': directives.path, 'id': directives.unchanged, + 'date': directives.unchanged, } def run(self): @@ -140,73 +161,151 @@ class PostList(Directive): stop = self.options.get('stop') reverse = self.options.get('reverse', False) tags = self.options.get('tags') - tags = [t.strip().lower() for t in tags.split(',')] if tags else [] categories = self.options.get('categories') - categories = [c.strip().lower() for c in categories.split(',')] if categories else [] + sections = self.options.get('sections') slugs = self.options.get('slugs') - slugs = [s.strip() for s in slugs.split(',')] if slugs else [] - show_all = self.options.get('all', False) + post_type = self.options.get('post_type') + type = self.options.get('type', False) + all = self.options.get('all', False) lang = self.options.get('lang', utils.LocaleBorg().current_lang) template = self.options.get('template', 'post_list_directive.tmpl') sort = self.options.get('sort') - if self.site.invariant: # for testing purposes - post_list_id = self.options.get('id', 'post_list_' + 'fixedvaluethatisnotauuid') - else: - post_list_id = self.options.get('id', 'post_list_' + uuid.uuid4().hex) + date = self.options.get('date') - filtered_timeline = [] - posts = [] - step = -1 if reverse is None else None - if show_all is None: - timeline = [p for p in self.site.timeline] + output, deps = _do_post_list(start, stop, reverse, tags, categories, sections, slugs, post_type, type, + all, lang, template, sort, state=self.state, site=self.site, date=date) + self.state.document.settings.record_dependencies.add("####MAGIC####TIMELINE") + for d in deps: + self.state.document.settings.record_dependencies.add(d) + if output: + return [nodes.raw('', output, format='html')] else: - timeline = [p for p in self.site.timeline if p.use_in_feeds] - - if categories: - timeline = [p for p in timeline if p.meta('category', lang=lang).lower() in categories] - - for post in timeline: - if tags: - cont = True - tags_lower = [t.lower() for t in post.tags] - for tag in tags: - if tag in tags_lower: - cont = False - - if cont: - continue - - filtered_timeline.append(post) - - if sort: - filtered_timeline = natsort.natsorted(filtered_timeline, key=lambda post: post.meta[lang][sort], alg=natsort.ns.F | natsort.ns.IC) - - for post in filtered_timeline[start:stop:step]: - if slugs: - cont = True - for slug in slugs: - if slug == post.meta('slug'): - cont = False - - if cont: - continue - - bp = post.translated_base_path(lang) - if os.path.exists(bp): - self.state.document.settings.record_dependencies.add(bp) + return [] - posts += [post] - if not posts: - return [] - self.state.document.settings.record_dependencies.add("####MAGIC####TIMELINE") +def _do_post_list(start=None, stop=None, reverse=False, tags=None, categories=None, + sections=None, slugs=None, post_type='post', type=False, all=False, + lang=None, template='post_list_directive.tmpl', sort=None, + id=None, data=None, state=None, site=None, date=None, filename=None, post=None): + if lang is None: + lang = utils.LocaleBorg().current_lang + if site.invariant: # for testing purposes + post_list_id = id or 'post_list_' + 'fixedvaluethatisnotauuid' + else: + post_list_id = id or 'post_list_' + uuid.uuid4().hex + + # Get post from filename if available + if filename: + self_post = site.post_per_input_file.get(filename) + else: + self_post = None + + if self_post: + self_post.register_depfile("####MAGIC####TIMELINE", lang=lang) + + # If we get strings for start/stop, make them integers + if start is not None: + start = int(start) + if stop is not None: + stop = int(stop) + + # Parse tags/categories/sections/slugs (input is strings) + tags = [t.strip().lower() for t in tags.split(',')] if tags else [] + categories = [c.strip().lower() for c in categories.split(',')] if categories else [] + sections = [s.strip().lower() for s in sections.split(',')] if sections else [] + slugs = [s.strip() for s in slugs.split(',')] if slugs else [] + + filtered_timeline = [] + posts = [] + step = -1 if reverse is None else None + + if type is not False: + post_type = type + + # TODO: remove in v8 + if all is not False: + timeline = [p for p in site.timeline] + elif post_type == 'page' or post_type == 'pages': + timeline = [p for p in site.timeline if not p.use_in_feeds] + elif post_type == 'all': + timeline = [p for p in site.timeline] + else: # post + timeline = [p for p in site.timeline if p.use_in_feeds] + + # TODO: replaces all, uncomment in v8 + # if post_type == 'page' or post_type == 'pages': + # timeline = [p for p in site.timeline if not p.use_in_feeds] + # elif post_type == 'all': + # timeline = [p for p in site.timeline] + # else: # post + # timeline = [p for p in site.timeline if p.use_in_feeds] + + if categories: + timeline = [p for p in timeline if p.meta('category', lang=lang).lower() in categories] + + if sections: + timeline = [p for p in timeline if p.section_name(lang).lower() in sections] + + for post in timeline: + if tags: + cont = True + tags_lower = [t.lower() for t in post.tags] + for tag in tags: + if tag in tags_lower: + cont = False + + if cont: + continue + + filtered_timeline.append(post) + + if sort: + filtered_timeline = natsort.natsorted(filtered_timeline, key=lambda post: post.meta[lang][sort], alg=natsort.ns.F | natsort.ns.IC) + + if date: + filtered_timeline = [p for p in filtered_timeline if date_in_range(date, p.date)] + + for post in filtered_timeline[start:stop:step]: + if slugs: + cont = True + for slug in slugs: + if slug == post.meta('slug'): + cont = False + + if cont: + continue + + bp = post.translated_base_path(lang) + if os.path.exists(bp) and state: + state.document.settings.record_dependencies.add(bp) + elif os.path.exists(bp) and self_post: + self_post.register_depfile(bp, lang=lang) + + posts += [post] + + if not posts: + return '', [] + + template_deps = site.template_system.template_deps(template) + if state: + # Register template as a dependency (Issue #2391) + for d in template_deps: + state.document.settings.record_dependencies.add(d) + elif self_post: + for d in template_deps: + self_post.register_depfile(d, lang=lang) + + template_data = { + 'lang': lang, + 'posts': posts, + # Need to provide str, not TranslatableSetting (Issue #2104) + 'date_format': site.GLOBAL_CONTEXT.get('date_format')[lang], + 'post_list_id': post_list_id, + 'messages': site.MESSAGES, + } + output = site.template_system.render_template( + template, None, template_data) + return output, template_deps - template_data = { - 'lang': lang, - 'posts': posts, - 'date_format': self.site.GLOBAL_CONTEXT.get('date_format'), - 'post_list_id': post_list_id, - } - output = self.site.template_system.render_template( - template, None, template_data) - return [nodes.raw('', output, format='html')] +# Request file name from shortcode (Issue #2412) +_do_post_list.nikola_shortcode_pass_filename = True diff --git a/nikola/plugins/compile/rest/slides.plugin b/nikola/plugins/compile/rest/slides.plugin index 5c05b89..389da39 100644 --- a/nikola/plugins/compile/rest/slides.plugin +++ b/nikola/plugins/compile/rest/slides.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Slides directive diff --git a/nikola/plugins/compile/rest/slides.py b/nikola/plugins/compile/rest/slides.py index 2522e55..7c5b34b 100644 --- a/nikola/plugins/compile/rest/slides.py +++ b/nikola/plugins/compile/rest/slides.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -37,7 +37,6 @@ from nikola.plugin_categories import RestExtension class Plugin(RestExtension): - """Plugin for reST slides directive.""" name = "rest_slides" @@ -51,7 +50,6 @@ class Plugin(RestExtension): class Slides(Directive): - """reST extension for inserting slideshows.""" has_content = True diff --git a/nikola/plugins/compile/rest/soundcloud.plugin b/nikola/plugins/compile/rest/soundcloud.plugin index 75469e4..4e36ea4 100644 --- a/nikola/plugins/compile/rest/soundcloud.plugin +++ b/nikola/plugins/compile/rest/soundcloud.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Roberto Alsina version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = Soundcloud directive diff --git a/nikola/plugins/compile/rest/soundcloud.py b/nikola/plugins/compile/rest/soundcloud.py index 30134a9..9fabe70 100644 --- a/nikola/plugins/compile/rest/soundcloud.py +++ b/nikola/plugins/compile/rest/soundcloud.py @@ -4,13 +4,12 @@ from docutils import nodes from docutils.parsers.rst import Directive, directives - +from nikola.plugins.compile.rest import _align_choice, _align_options_base from nikola.plugin_categories import RestExtension class Plugin(RestExtension): - """Plugin for soundclound directive.""" name = "rest_soundcloud" @@ -23,15 +22,16 @@ class Plugin(RestExtension): return super(Plugin, self).set_site(site) -CODE = ("""<iframe width="{width}" height="{height}" +CODE = """\ +<div class="soundcloud-player{align}"> +<iframe width="{width}" height="{height}" scrolling="no" frameborder="no" -src="https://w.soundcloud.com/player/?url=http://api.soundcloud.com/{preslug}/""" - """{sid}"> -</iframe>""") +src="https://w.soundcloud.com/player/?url=http://api.soundcloud.com/{preslug}/{sid}"> +</iframe> +</div>""" class SoundCloud(Directive): - """reST extension for inserting SoundCloud embedded music. Usage: @@ -46,6 +46,7 @@ class SoundCloud(Directive): option_spec = { 'width': directives.positive_int, 'height': directives.positive_int, + "align": _align_choice } preslug = "tracks" @@ -59,6 +60,10 @@ class SoundCloud(Directive): 'preslug': self.preslug, } options.update(self.options) + if self.options.get('align') in _align_options_base: + options['align'] = ' align-' + self.options['align'] + else: + options['align'] = '' return [nodes.raw('', CODE.format(**options), format='html')] def check_content(self): @@ -70,7 +75,6 @@ class SoundCloud(Directive): class SoundCloudPlaylist(SoundCloud): - """reST directive for SoundCloud playlists.""" preslug = "playlists" diff --git a/nikola/plugins/compile/rest/thumbnail.plugin b/nikola/plugins/compile/rest/thumbnail.plugin index 0084310..3324c31 100644 --- a/nikola/plugins/compile/rest/thumbnail.plugin +++ b/nikola/plugins/compile/rest/thumbnail.plugin @@ -9,6 +9,6 @@ plugincategory = CompilerExtension [Documentation] author = Pelle Nilsson version = 0.1 -website = http://getnikola.com +website = https://getnikola.com/ description = reST directive to facilitate enlargeable images with thumbnails diff --git a/nikola/plugins/compile/rest/thumbnail.py b/nikola/plugins/compile/rest/thumbnail.py index 1fae06c..37e0973 100644 --- a/nikola/plugins/compile/rest/thumbnail.py +++ b/nikola/plugins/compile/rest/thumbnail.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Pelle Nilsson and others. +# Copyright © 2014-2016 Pelle Nilsson and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -35,7 +35,6 @@ from nikola.plugin_categories import RestExtension class Plugin(RestExtension): - """Plugin for thumbnail directive.""" name = "rest_thumbnail" @@ -48,7 +47,6 @@ class Plugin(RestExtension): class Thumbnail(Figure): - """Thumbnail directive for reST.""" def align(argument): @@ -70,8 +68,12 @@ class Thumbnail(Figure): def run(self): """Run the thumbnail directive.""" uri = directives.uri(self.arguments[0]) + if uri.endswith('.svg'): + # the ? at the end makes docutil output an <img> instead of an object for the svg, which colorbox requires + self.arguments[0] = '.thumbnail'.join(os.path.splitext(uri)) + '?' + else: + self.arguments[0] = '.thumbnail'.join(os.path.splitext(uri)) self.options['target'] = uri - self.arguments[0] = '.thumbnail'.join(os.path.splitext(uri)) if self.content: (node,) = Figure.run(self) else: diff --git a/nikola/plugins/compile/rest/vimeo.py b/nikola/plugins/compile/rest/vimeo.py index c694a87..f1ac6c3 100644 --- a/nikola/plugins/compile/rest/vimeo.py +++ b/nikola/plugins/compile/rest/vimeo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -28,6 +28,7 @@ from docutils import nodes from docutils.parsers.rst import Directive, directives +from nikola.plugins.compile.rest import _align_choice, _align_options_base import requests import json @@ -37,7 +38,6 @@ from nikola.plugin_categories import RestExtension class Plugin(RestExtension): - """Plugin for vimeo reST directive.""" name = "rest_vimeo" @@ -49,10 +49,12 @@ class Plugin(RestExtension): return super(Plugin, self).set_site(site) -CODE = """<iframe src="//player.vimeo.com/video/{vimeo_id}" +CODE = """<div class="vimeo-video{align}"> +<iframe src="https://player.vimeo.com/video/{vimeo_id}" width="{width}" height="{height}" frameborder="0" webkitAllowFullScreen="webkitAllowFullScreen" mozallowfullscreen="mozallowfullscreen" allowFullScreen="allowFullScreen"> </iframe> +</div> """ VIDEO_DEFAULT_HEIGHT = 500 @@ -60,7 +62,6 @@ VIDEO_DEFAULT_WIDTH = 281 class Vimeo(Directive): - """reST extension for inserting vimeo embedded videos. Usage: @@ -75,6 +76,7 @@ class Vimeo(Directive): option_spec = { "width": directives.positive_int, "height": directives.positive_int, + "align": _align_choice } # set to False for not querying the vimeo api for size @@ -94,6 +96,10 @@ class Vimeo(Directive): return err self.set_video_size() options.update(self.options) + if self.options.get('align') in _align_options_base: + options['align'] = ' align-' + self.options['align'] + else: + options['align'] = '' return [nodes.raw('', CODE.format(**options), format='html')] def check_modules(self): @@ -109,7 +115,7 @@ class Vimeo(Directive): if json: # we can attempt to retrieve video attributes from vimeo try: - url = ('//vimeo.com/api/v2/video/{0}' + url = ('https://vimeo.com/api/v2/video/{0}' '.json'.format(self.arguments[0])) data = requests.get(url).text video_attributes = json.loads(data)[0] diff --git a/nikola/plugins/compile/rest/youtube.py b/nikola/plugins/compile/rest/youtube.py index 6c5c211..b3dde62 100644 --- a/nikola/plugins/compile/rest/youtube.py +++ b/nikola/plugins/compile/rest/youtube.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -28,13 +28,12 @@ from docutils import nodes from docutils.parsers.rst import Directive, directives - +from nikola.plugins.compile.rest import _align_choice, _align_options_base from nikola.plugin_categories import RestExtension class Plugin(RestExtension): - """Plugin for the youtube directive.""" name = "rest_youtube" @@ -47,14 +46,14 @@ class Plugin(RestExtension): CODE = """\ -<iframe width="{width}" -height="{height}" -src="//www.youtube.com/embed/{yid}?rel=0&hd=1&wmode=transparent" -></iframe>""" +<div class="youtube-video{align}"> +<iframe width="{width}" height="{height}" +src="https://www.youtube.com/embed/{yid}?rel=0&hd=1&wmode=transparent" +></iframe> +</div>""" class Youtube(Directive): - """reST extension for inserting youtube embedded videos. Usage: @@ -69,6 +68,7 @@ class Youtube(Directive): option_spec = { "width": directives.positive_int, "height": directives.positive_int, + "align": _align_choice } def run(self): @@ -80,6 +80,10 @@ class Youtube(Directive): 'height': 344, } options.update(self.options) + if self.options.get('align') in _align_options_base: + options['align'] = ' align-' + self.options['align'] + else: + options['align'] = '' return [nodes.raw('', CODE.format(**options), format='html')] def check_content(self): diff --git a/nikola/plugins/misc/__init__.py b/nikola/plugins/misc/__init__.py index c0d8961..518fac1 100644 --- a/nikola/plugins/misc/__init__.py +++ b/nikola/plugins/misc/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/misc/scan_posts.plugin b/nikola/plugins/misc/scan_posts.plugin index 6d2351f..f4af811 100644 --- a/nikola/plugins/misc/scan_posts.plugin +++ b/nikola/plugins/misc/scan_posts.plugin @@ -5,6 +5,6 @@ Module = scan_posts [Documentation] Author = Roberto Alsina Version = 1.0 -Website = http://getnikola.com +Website = https://getnikola.com/ Description = Scan posts and create timeline diff --git a/nikola/plugins/misc/scan_posts.py b/nikola/plugins/misc/scan_posts.py index 1f4f995..f584a05 100644 --- a/nikola/plugins/misc/scan_posts.py +++ b/nikola/plugins/misc/scan_posts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -35,9 +35,10 @@ from nikola.plugin_categories import PostScanner from nikola import utils from nikola.post import Post +LOGGER = utils.get_logger('scan_posts', utils.STDERR_HANDLER) -class ScanPosts(PostScanner): +class ScanPosts(PostScanner): """Scan posts in the site.""" name = "scan_posts" @@ -88,15 +89,19 @@ class ScanPosts(PostScanner): continue else: seen.add(base_path) - post = Post( - base_path, - self.site.config, - dest_dir, - use_in_feeds, - self.site.MESSAGES, - template_name, - self.site.get_compiler(base_path) - ) - timeline.append(post) + try: + post = Post( + base_path, + self.site.config, + dest_dir, + use_in_feeds, + self.site.MESSAGES, + template_name, + self.site.get_compiler(base_path) + ) + timeline.append(post) + except Exception as err: + LOGGER.error('Error reading post {}'.format(base_path)) + raise err return timeline diff --git a/nikola/plugins/shortcode/gist.plugin b/nikola/plugins/shortcode/gist.plugin new file mode 100644 index 0000000..cd19a72 --- /dev/null +++ b/nikola/plugins/shortcode/gist.plugin @@ -0,0 +1,13 @@ +[Core] +name = gist +module = gist + +[Nikola] +plugincategory = Shortcode + +[Documentation] +author = Roberto Alsina +version = 0.1 +website = https://getnikola.com/ +description = Gist shortcode + diff --git a/nikola/plugins/shortcode/gist.py b/nikola/plugins/shortcode/gist.py new file mode 100644 index 0000000..64fd0d9 --- /dev/null +++ b/nikola/plugins/shortcode/gist.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# This file is public domain according to its author, Brian Hsu + +"""Gist directive for reStructuredText.""" + +import requests + +from nikola.plugin_categories import ShortcodePlugin + + +class Plugin(ShortcodePlugin): + """Plugin for gist directive.""" + + name = "gist" + + def set_site(self, site): + """Set Nikola site.""" + self.site = site + site.register_shortcode('gist', self.handler) + return super(Plugin, self).set_site(site) + + def get_raw_gist_with_filename(self, gistID, filename): + """Get raw gist text for a filename.""" + url = '/'.join(("https://gist.github.com/raw", gistID, filename)) + return requests.get(url).text + + def get_raw_gist(self, gistID): + """Get raw gist text.""" + url = "https://gist.github.com/raw/{0}".format(gistID) + try: + return requests.get(url).text + except requests.exceptions.RequestException: + raise self.error('Cannot get gist for url={0}'.format(url)) + + def handler(self, gistID, filename=None, site=None, data=None, lang=None, post=None): + """Create HTML for gist.""" + if 'https://' in gistID: + gistID = gistID.split('/')[-1].strip() + else: + gistID = gistID.strip() + embedHTML = "" + rawGist = "" + + if filename is not None: + rawGist = (self.get_raw_gist_with_filename(gistID, filename)) + embedHTML = ('<script src="https://gist.github.com/{0}.js' + '?file={1}"></script>').format(gistID, filename) + else: + rawGist = (self.get_raw_gist(gistID)) + embedHTML = ('<script src="https://gist.github.com/{0}.js">' + '</script>').format(gistID) + + output = '''{} + <noscript><pre>{}</pre></noscript>'''.format(embedHTML, rawGist) + + return output, [] diff --git a/nikola/plugins/task/__init__.py b/nikola/plugins/task/__init__.py index fd9a48f..4eeae62 100644 --- a/nikola/plugins/task/__init__.py +++ b/nikola/plugins/task/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/archive.plugin b/nikola/plugins/task/archive.plugin index 25f1195..eb079da 100644 --- a/nikola/plugins/task/archive.plugin +++ b/nikola/plugins/task/archive.plugin @@ -5,7 +5,7 @@ module = archive [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Generates the blog's archive pages. [Nikola] diff --git a/nikola/plugins/task/archive.py b/nikola/plugins/task/archive.py index 126aed4..303d349 100644 --- a/nikola/plugins/task/archive.py +++ b/nikola/plugins/task/archive.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -37,7 +37,6 @@ from nikola.utils import config_changed, adjust_name_for_index_path, adjust_name class Archive(Task): - """Render the post archives.""" name = "render_archive" @@ -53,7 +52,7 @@ class Archive(Task): """Prepare an archive task.""" # name: used to build permalink and destination # posts, items: posts or items; only one of them should be used, - # the other be None + # the other should be None # template_name: name of the template to use # title: the (translated) title for the generated page # deps_translatable: dependencies (None if not added) @@ -175,10 +174,10 @@ class Archive(Task): if not kw["create_monthly_archive"] or kw["create_full_archives"]: yield self._generate_posts_task(kw, year, lang, posts, title, deps_translatable) else: - months = set([(m.split('/')[1], self.site.link("archive", m, lang)) for m in self.site.posts_per_month.keys() if m.startswith(str(year))]) + months = set([(m.split('/')[1], self.site.link("archive", m, lang), len(self.site.posts_per_month[m])) for m in self.site.posts_per_month.keys() if m.startswith(str(year))]) months = sorted(list(months)) months.reverse() - items = [[nikola.utils.LocaleBorg().get_month_name(int(month), lang), link] for month, link in months] + items = [[nikola.utils.LocaleBorg().get_month_name(int(month), lang), link, count] for month, link, count in months] yield self._prepare_task(kw, year, lang, None, items, "list.tmpl", title, deps_translatable) if not kw["create_monthly_archive"] and not kw["create_full_archives"] and not kw["create_daily_archive"]: @@ -219,11 +218,16 @@ class Archive(Task): years.sort(reverse=True) kw['years'] = years for lang in kw["translations"]: - items = [(y, self.site.link("archive", y, lang)) for y in years] + items = [(y, self.site.link("archive", y, lang), len(self.site.posts_per_year[y])) for y in years] yield self._prepare_task(kw, None, lang, None, items, "list.tmpl", kw["messages"][lang]["Archive"]) def archive_path(self, name, lang, is_feed=False): - """Return archive paths.""" + """Link to archive path, name is the year. + + Example: + + link://archive/2013 => /archives/2013/index.html + """ if is_feed: extension = ".atom" archive_file = os.path.splitext(self.site.config['ARCHIVE_FILENAME'])[0] + extension @@ -241,5 +245,10 @@ class Archive(Task): archive_file] if _f] def archive_atom_path(self, name, lang): - """Return Atom archive paths.""" + """Link to atom archive path, name is the year. + + Example: + + link://archive_atom/2013 => /archives/2013/index.atom + """ return self.archive_path(name, lang, is_feed=True) diff --git a/nikola/plugins/task/authors.plugin b/nikola/plugins/task/authors.plugin new file mode 100644 index 0000000..3fc4ef2 --- /dev/null +++ b/nikola/plugins/task/authors.plugin @@ -0,0 +1,10 @@ +[Core] +Name = render_authors +Module = authors + +[Documentation] +Author = Juanjo Conti +Version = 0.1 +Website = http://getnikola.com +Description = Render the author pages and feeds. + diff --git a/nikola/plugins/task/authors.py b/nikola/plugins/task/authors.py new file mode 100644 index 0000000..ec61800 --- /dev/null +++ b/nikola/plugins/task/authors.py @@ -0,0 +1,326 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2015-2016 Juanjo Conti and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Render the author pages and feeds.""" + +from __future__ import unicode_literals +import os +import natsort +try: + from urlparse import urljoin +except ImportError: + from urllib.parse import urljoin # NOQA +from collections import defaultdict + +from blinker import signal + +from nikola.plugin_categories import Task +from nikola import utils + + +class RenderAuthors(Task): + """Render the author pages and feeds.""" + + name = "render_authors" + posts_per_author = None + + def set_site(self, site): + """Set Nikola site.""" + self.generate_author_pages = False + if site.config["ENABLE_AUTHOR_PAGES"]: + site.register_path_handler('author_index', self.author_index_path) + site.register_path_handler('author', self.author_path) + site.register_path_handler('author_atom', self.author_atom_path) + site.register_path_handler('author_rss', self.author_rss_path) + signal('scanned').connect(self.posts_scanned) + return super(RenderAuthors, self).set_site(site) + + def posts_scanned(self, event): + """Called after posts are scanned via signal.""" + self.generate_author_pages = self.site.config["ENABLE_AUTHOR_PAGES"] and len(self._posts_per_author()) > 1 + self.site.GLOBAL_CONTEXT["author_pages_generated"] = self.generate_author_pages + + def gen_tasks(self): + """Render the author pages and feeds.""" + kw = { + "translations": self.site.config["TRANSLATIONS"], + "blog_title": self.site.config["BLOG_TITLE"], + "site_url": self.site.config["SITE_URL"], + "base_url": self.site.config["BASE_URL"], + "messages": self.site.MESSAGES, + "output_folder": self.site.config['OUTPUT_FOLDER'], + "filters": self.site.config['FILTERS'], + 'author_path': self.site.config['AUTHOR_PATH'], + "author_pages_are_indexes": self.site.config['AUTHOR_PAGES_ARE_INDEXES'], + "generate_rss": self.site.config['GENERATE_RSS'], + "feed_teasers": self.site.config["FEED_TEASERS"], + "feed_plain": self.site.config["FEED_PLAIN"], + "feed_link_append_query": self.site.config["FEED_LINKS_APPEND_QUERY"], + "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], + "feed_length": self.site.config['FEED_LENGTH'], + "tzinfo": self.site.tzinfo, + "pretty_urls": self.site.config['PRETTY_URLS'], + "strip_indexes": self.site.config['STRIP_INDEXES'], + "index_file": self.site.config['INDEX_FILE'], + } + + self.site.scan_posts() + yield self.group_task() + + if self.generate_author_pages: + yield self.list_authors_page(kw) + + if not self._posts_per_author(): # this may be self.site.posts_per_author + return + + author_list = list(self._posts_per_author().items()) + + def render_lists(author, posts): + """Render author pages as RSS files and lists/indexes.""" + post_list = sorted(posts, key=lambda a: a.date) + post_list.reverse() + for lang in kw["translations"]: + if kw["show_untranslated_posts"]: + filtered_posts = post_list + else: + filtered_posts = [x for x in post_list if x.is_translation_available(lang)] + if kw["generate_rss"]: + yield self.author_rss(author, lang, filtered_posts, kw) + # Render HTML + if kw['author_pages_are_indexes']: + yield self.author_page_as_index(author, lang, filtered_posts, kw) + else: + yield self.author_page_as_list(author, lang, filtered_posts, kw) + + for author, posts in author_list: + for task in render_lists(author, posts): + yield task + + def _create_authors_page(self, kw): + """Create a global "all authors" page for each language.""" + template_name = "authors.tmpl" + kw = kw.copy() + for lang in kw["translations"]: + authors = natsort.natsorted([author for author in self._posts_per_author().keys()], + alg=natsort.ns.F | natsort.ns.IC) + has_authors = (authors != []) + kw['authors'] = authors + output_name = os.path.join( + kw['output_folder'], self.site.path('author_index', None, lang)) + context = {} + if has_authors: + context["title"] = kw["messages"][lang]["Authors"] + context["items"] = [(author, self.site.link("author", author, lang)) for author + in authors] + context["description"] = context["title"] + else: + context["items"] = None + context["permalink"] = self.site.link("author_index", None, lang) + context["pagekind"] = ["list", "authors_page"] + task = self.site.generic_post_list_renderer( + lang, + [], + output_name, + template_name, + kw['filters'], + context, + ) + task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.authors:page')] + task['basename'] = str(self.name) + yield task + + def list_authors_page(self, kw): + """Create a global "all authors" page for each language.""" + yield self._create_authors_page(kw) + + def _get_title(self, author): + return author + + def _get_description(self, author, lang): + descriptions = self.site.config['AUTHOR_PAGES_DESCRIPTIONS'] + return descriptions[lang][author] if lang in descriptions and author in descriptions[lang] else None + + def author_page_as_index(self, author, lang, post_list, kw): + """Render a sort of index page collection using only this author's posts.""" + kind = "author" + + def page_link(i, displayed_i, num_pages, force_addition, extension=None): + feed = "_atom" if extension == ".atom" else "" + return utils.adjust_name_for_index_link(self.site.link(kind + feed, author, lang), i, displayed_i, lang, self.site, force_addition, extension) + + def page_path(i, displayed_i, num_pages, force_addition, extension=None): + feed = "_atom" if extension == ".atom" else "" + return utils.adjust_name_for_index_path(self.site.path(kind + feed, author, lang), i, displayed_i, lang, self.site, force_addition, extension) + + context_source = {} + title = self._get_title(author) + if kw["generate_rss"]: + # On a author page, the feeds include the author's feeds + rss_link = ("""<link rel="alternate" type="application/rss+xml" """ + """title="RSS for author """ + """{0} ({1})" href="{2}">""".format( + title, lang, self.site.link(kind + "_rss", author, lang))) + context_source['rss_link'] = rss_link + context_source["author"] = title + indexes_title = kw["messages"][lang]["Posts by %s"] % title + context_source["description"] = self._get_description(author, lang) + context_source["pagekind"] = ["index", "author_page"] + template_name = "authorindex.tmpl" + + yield self.site.generic_index_renderer(lang, post_list, indexes_title, template_name, context_source, kw, str(self.name), page_link, page_path) + + def author_page_as_list(self, author, lang, post_list, kw): + """Render a single flat link list with this author's posts.""" + kind = "author" + template_name = "author.tmpl" + output_name = os.path.join(kw['output_folder'], self.site.path( + kind, author, lang)) + context = {} + context["lang"] = lang + title = self._get_title(author) + context["author"] = title + context["title"] = kw["messages"][lang]["Posts by %s"] % title + context["posts"] = post_list + context["permalink"] = self.site.link(kind, author, lang) + context["kind"] = kind + context["description"] = self._get_description(author, lang) + context["pagekind"] = ["list", "author_page"] + task = self.site.generic_post_list_renderer( + lang, + post_list, + output_name, + template_name, + kw['filters'], + context, + ) + task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.authors:list')] + task['basename'] = str(self.name) + yield task + + def author_rss(self, author, lang, posts, kw): + """Create a RSS feed for a single author in a given language.""" + kind = "author" + # Render RSS + output_name = os.path.normpath( + os.path.join(kw['output_folder'], + self.site.path(kind + "_rss", author, lang))) + feed_url = urljoin(self.site.config['BASE_URL'], self.site.link(kind + "_rss", author, lang).lstrip('/')) + deps = [] + deps_uptodate = [] + post_list = sorted(posts, key=lambda a: a.date) + post_list.reverse() + for post in post_list: + deps += post.deps(lang) + deps_uptodate += post.deps_uptodate(lang) + task = { + 'basename': str(self.name), + 'name': output_name, + 'file_dep': deps, + 'targets': [output_name], + 'actions': [(utils.generic_rss_renderer, + (lang, "{0} ({1})".format(kw["blog_title"](lang), self._get_title(author)), + kw["site_url"], None, post_list, + output_name, kw["feed_teasers"], kw["feed_plain"], kw['feed_length'], + feed_url, None, kw["feed_link_append_query"]))], + 'clean': True, + 'uptodate': [utils.config_changed(kw, 'nikola.plugins.task.authors:rss')] + deps_uptodate, + 'task_dep': ['render_posts'], + } + return utils.apply_filters(task, kw['filters']) + + def slugify_author_name(self, name, lang=None): + """Slugify an author name.""" + if lang is None: # TODO: remove in v8 + utils.LOGGER.warn("RenderAuthors.slugify_author_name() called without language!") + lang = '' + if self.site.config['SLUG_AUTHOR_PATH']: + name = utils.slugify(name, lang) + return name + + def author_index_path(self, name, lang): + """Link to the author's index. + + Example: + + link://authors/ => /authors/index.html + """ + return [_f for _f in [self.site.config['TRANSLATIONS'][lang], + self.site.config['AUTHOR_PATH'], + self.site.config['INDEX_FILE']] if _f] + + def author_path(self, name, lang): + """Link to an author's page. + + Example: + + link://author/joe => /authors/joe.html + """ + if self.site.config['PRETTY_URLS']: + return [_f for _f in [ + self.site.config['TRANSLATIONS'][lang], + self.site.config['AUTHOR_PATH'], + self.slugify_author_name(name, lang), + self.site.config['INDEX_FILE']] if _f] + else: + return [_f for _f in [ + self.site.config['TRANSLATIONS'][lang], + self.site.config['AUTHOR_PATH'], + self.slugify_author_name(name, lang) + ".html"] if _f] + + def author_atom_path(self, name, lang): + """Link to an author's Atom feed. + + Example: + + link://author_atom/joe => /authors/joe.atom + """ + return [_f for _f in [self.site.config['TRANSLATIONS'][lang], + self.site.config['AUTHOR_PATH'], self.slugify_author_name(name, lang) + ".atom"] if + _f] + + def author_rss_path(self, name, lang): + """Link to an author's RSS feed. + + Example: + + link://author_rss/joe => /authors/joe.rss + """ + return [_f for _f in [self.site.config['TRANSLATIONS'][lang], + self.site.config['AUTHOR_PATH'], self.slugify_author_name(name, lang) + ".xml"] if + _f] + + def _add_extension(self, path, extension): + path[-1] += extension + return path + + def _posts_per_author(self): + """Return a dict of posts per author.""" + if self.posts_per_author is None: + self.posts_per_author = defaultdict(list) + for post in self.site.timeline: + if post.is_post: + self.posts_per_author[post.author()].append(post) + return self.posts_per_author diff --git a/nikola/plugins/task/bundles.plugin b/nikola/plugins/task/bundles.plugin index ca997d0..b5bf6e4 100644 --- a/nikola/plugins/task/bundles.plugin +++ b/nikola/plugins/task/bundles.plugin @@ -5,7 +5,7 @@ module = bundles [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Theme bundles using WebAssets [Nikola] diff --git a/nikola/plugins/task/bundles.py b/nikola/plugins/task/bundles.py index b9c57b9..b33d8e0 100644 --- a/nikola/plugins/task/bundles.py +++ b/nikola/plugins/task/bundles.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -40,7 +40,6 @@ from nikola import utils class BuildBundles(LateTask): - """Bundle assets using WebAssets.""" name = "create_bundles" @@ -52,6 +51,7 @@ class BuildBundles(LateTask): utils.req_missing(['webassets'], 'USE_BUNDLES', optional=True) self.logger.warn('Setting USE_BUNDLES to False.') site.config['USE_BUNDLES'] = False + site._GLOBAL_CONTEXT['use_bundles'] = False super(BuildBundles, self).set_site(site) def gen_tasks(self): @@ -100,7 +100,11 @@ class BuildBundles(LateTask): files.append(os.path.join(dname, fname)) file_dep = [os.path.join(kw['output_folder'], fname) for fname in files if - utils.get_asset_path(fname, self.site.THEMES, self.site.config['FILES_FOLDERS']) or fname == os.path.join('assets', 'css', 'code.css')] + utils.get_asset_path( + fname, + self.site.THEMES, + self.site.config['FILES_FOLDERS'], + output_dir=kw['output_folder']) or fname == os.path.join('assets', 'css', 'code.css')] # code.css will be generated by us if it does not exist in # FILES_FOLDERS or theme assets. It is guaranteed that the # generation will happen before this task. diff --git a/nikola/plugins/task/copy_assets.plugin b/nikola/plugins/task/copy_assets.plugin index c182150..ddd38df 100644 --- a/nikola/plugins/task/copy_assets.plugin +++ b/nikola/plugins/task/copy_assets.plugin @@ -5,7 +5,7 @@ module = copy_assets [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Copy theme assets into output. [Nikola] diff --git a/nikola/plugins/task/copy_assets.py b/nikola/plugins/task/copy_assets.py index 58521d4..4ed7414 100644 --- a/nikola/plugins/task/copy_assets.py +++ b/nikola/plugins/task/copy_assets.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,7 +36,6 @@ from nikola import utils class CopyAssets(Task): - """Copy theme assets into output.""" name = "copy_assets" @@ -61,10 +60,7 @@ class CopyAssets(Task): code_css_path = os.path.join(kw['output_folder'], 'assets', 'css', 'code.css') code_css_input = utils.get_asset_path('assets/css/code.css', themes=kw['themes'], - files_folders=kw['files_folders']) - - kw["code.css_input"] = code_css_input - + files_folders=kw['files_folders'], output_dir=None) yield self.group_task() for theme_name in kw['themes']: @@ -77,7 +73,9 @@ class CopyAssets(Task): task['uptodate'] = [utils.config_changed(kw, 'nikola.plugins.task.copy_assets')] task['basename'] = self.name if code_css_input: - task['file_dep'] = [code_css_input] + if 'file_dep' not in task: + task['file_dep'] = [] + task['file_dep'].append(code_css_input) yield utils.apply_filters(task, kw['filters']) # Check whether or not there is a code.css file around. diff --git a/nikola/plugins/task/copy_files.plugin b/nikola/plugins/task/copy_files.plugin index ce8f5d0..e4bb1cf 100644 --- a/nikola/plugins/task/copy_files.plugin +++ b/nikola/plugins/task/copy_files.plugin @@ -5,7 +5,7 @@ module = copy_files [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Copy static files into the output. [Nikola] diff --git a/nikola/plugins/task/copy_files.py b/nikola/plugins/task/copy_files.py index 1232248..6f6cfb8 100644 --- a/nikola/plugins/task/copy_files.py +++ b/nikola/plugins/task/copy_files.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -33,7 +33,6 @@ from nikola import utils class CopyFiles(Task): - """Copy static files into the output folder.""" name = "copy_files" diff --git a/nikola/plugins/task/galleries.plugin b/nikola/plugins/task/galleries.plugin index 9d3fa28..2064e68 100644 --- a/nikola/plugins/task/galleries.plugin +++ b/nikola/plugins/task/galleries.plugin @@ -5,7 +5,7 @@ module = galleries [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create image galleries automatically. [Nikola] diff --git a/nikola/plugins/task/galleries.py b/nikola/plugins/task/galleries.py index c0df4a4..edfd33d 100644 --- a/nikola/plugins/task/galleries.py +++ b/nikola/plugins/task/galleries.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -33,7 +33,6 @@ import io import json import mimetypes import os -import sys try: from urlparse import urljoin except ImportError: @@ -57,7 +56,6 @@ _image_size_cache = {} class Galleries(Task, ImageProcessor): - """Render image galleries.""" name = 'render_galleries' @@ -87,6 +85,8 @@ class Galleries(Task, ImageProcessor): 'tzinfo': site.tzinfo, 'comments_in_galleries': site.config['COMMENTS_IN_GALLERIES'], 'generate_rss': site.config['GENERATE_RSS'], + 'preserve_exif_data': site.config['PRESERVE_EXIF_DATA'], + 'exif_whitelist': site.config['EXIF_WHITELIST'], } # Verify that no folder in GALLERY_FOLDERS appears twice @@ -94,8 +94,8 @@ class Galleries(Task, ImageProcessor): for source, dest in self.kw['gallery_folders'].items(): if source in appearing_paths or dest in appearing_paths: problem = source if source in appearing_paths else dest - utils.LOGGER.error("The gallery input or output folder '{0}' appears in more than one entry in GALLERY_FOLDERS, exiting.".format(problem)) - sys.exit(1) + utils.LOGGER.error("The gallery input or output folder '{0}' appears in more than one entry in GALLERY_FOLDERS, ignoring.".format(problem)) + continue appearing_paths.add(source) appearing_paths.add(dest) @@ -116,26 +116,52 @@ class Galleries(Task, ImageProcessor): if len(candidates) == 1: return candidates[0] self.logger.error("Gallery name '{0}' is not unique! Possible output paths: {1}".format(name, candidates)) + raise RuntimeError("Gallery name '{0}' is not unique! Possible output paths: {1}".format(name, candidates)) else: self.logger.error("Unknown gallery '{0}'!".format(name)) self.logger.info("Known galleries: " + str(list(self.proper_gallery_links.keys()))) - sys.exit(1) + raise RuntimeError("Unknown gallery '{0}'!".format(name)) def gallery_path(self, name, lang): - """Return a gallery path.""" + """Link to an image gallery's path. + + It will try to find a gallery with that name if it's not ambiguous + or with that path. For example: + + link://gallery/london => /galleries/trips/london/index.html + + link://gallery/trips/london => /galleries/trips/london/index.html + """ gallery_path = self._find_gallery_path(name) return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] + gallery_path.split(os.sep) + [self.site.config['INDEX_FILE']] if _f] def gallery_global_path(self, name, lang): - """Return the global gallery path, which contains images.""" + """Link to the global gallery path, which contains all the images in galleries. + + There is only one copy of an image on multilingual blogs, in the site root. + + link://gallery_global/london => /galleries/trips/london/index.html + + link://gallery_global/trips/london => /galleries/trips/london/index.html + + (a ``gallery`` link could lead to eg. /en/galleries/trips/london/index.html) + """ gallery_path = self._find_gallery_path(name) return [_f for _f in gallery_path.split(os.sep) + [self.site.config['INDEX_FILE']] if _f] def gallery_rss_path(self, name, lang): - """Return path to the RSS file for a gallery.""" + """Link to an image gallery's RSS feed. + + It will try to find a gallery with that name if it's not ambiguous + or with that path. For example: + + link://gallery_rss/london => /galleries/trips/london/rss.xml + + link://gallery_rss/trips/london => /galleries/trips/london/rss.xml + """ gallery_path = self._find_gallery_path(name) return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] + gallery_path.split(os.sep) + @@ -149,6 +175,7 @@ class Galleries(Task, ImageProcessor): for k, v in self.site.GLOBAL_CONTEXT['template_hooks'].items(): self.kw['||template_hooks|{0}||'.format(k)] = v._items + self.site.scan_posts() yield self.group_task() template_name = "gallery.tmpl" @@ -170,13 +197,6 @@ class Galleries(Task, ImageProcessor): # Create image list, filter exclusions image_list = self.get_image_list(gallery) - # Sort as needed - # Sort by date - if self.kw['sort_by_date']: - image_list.sort(key=lambda a: self.image_date(a)) - else: # Sort by name - image_list.sort() - # Create thumbnails and large images in destination for image in image_list: for task in self.create_target_images(image, input_folder): @@ -187,8 +207,6 @@ class Galleries(Task, ImageProcessor): for task in self.remove_excluded_image(image, input_folder): yield task - crumbs = utils.get_crumbs(gallery, index_folder=self) - for lang in self.kw['translations']: # save navigation links as dependencies self.kw['navigation_links|{0}'.format(lang)] = self.kw['global_context']['navigation_links'](lang) @@ -218,7 +236,7 @@ class Galleries(Task, ImageProcessor): img_titles = [] for fn in image_name_list: name_without_ext = os.path.splitext(os.path.basename(fn))[0] - img_titles.append(utils.unslugify(name_without_ext)) + img_titles.append(utils.unslugify(name_without_ext, lang)) else: img_titles = [''] * len(image_name_list) @@ -242,7 +260,7 @@ class Galleries(Task, ImageProcessor): context["folders"] = natsort.natsorted( folders, alg=natsort.ns.F | natsort.ns.IC) - context["crumbs"] = crumbs + context["crumbs"] = utils.get_crumbs(gallery, index_folder=self, lang=lang) context["permalink"] = self.site.link("gallery", gallery, lang) context["enable_comments"] = self.kw['comments_in_galleries'] context["thumbnail_size"] = self.kw["thumbnail_size"] @@ -399,6 +417,8 @@ class Galleries(Task, ImageProcessor): # may break) if post.title == 'index': post.title = os.path.split(gallery)[1] + # Register the post (via #2417) + self.site.post_per_input_file[index_path] = post else: post = None return post @@ -458,7 +478,8 @@ class Galleries(Task, ImageProcessor): 'targets': [thumb_path], 'actions': [ (self.resize_image, - (img, thumb_path, self.kw['thumbnail_size'])) + (img, thumb_path, self.kw['thumbnail_size'], False, self.kw['preserve_exif_data'], + self.kw['exif_whitelist'])) ], 'clean': True, 'uptodate': [utils.config_changed({ @@ -473,7 +494,8 @@ class Galleries(Task, ImageProcessor): 'targets': [orig_dest_path], 'actions': [ (self.resize_image, - (img, orig_dest_path, self.kw['max_image_size'])) + (img, orig_dest_path, self.kw['max_image_size'], False, self.kw['preserve_exif_data'], + self.kw['exif_whitelist'])) ], 'clean': True, 'uptodate': [utils.config_changed({ @@ -534,13 +556,28 @@ class Galleries(Task, ImageProcessor): url = '/'.join(os.path.relpath(p, os.path.dirname(output_name) + os.sep).split(os.sep)) return url + all_data = list(zip(img_list, thumbs, img_titles)) + + if self.kw['sort_by_date']: + all_data.sort(key=lambda a: self.image_date(a[0])) + else: # Sort by name + all_data.sort(key=lambda a: a[0]) + + if all_data: + img_list, thumbs, img_titles = zip(*all_data) + else: + img_list, thumbs, img_titles = [], [], [] + photo_array = [] for img, thumb, title in zip(img_list, thumbs, img_titles): w, h = _image_size_cache.get(thumb, (None, None)) if w is None: - im = Image.open(thumb) - w, h = im.size - _image_size_cache[thumb] = w, h + if os.path.splitext(thumb)[1] in ['.svg', '.svgz']: + w, h = 200, 200 + else: + im = Image.open(thumb) + w, h = im.size + _image_size_cache[thumb] = w, h # Thumbs are files in output, we need URLs photo_array.append({ 'url': url_from_path(img), @@ -564,6 +601,18 @@ class Galleries(Task, ImageProcessor): def make_url(url): return urljoin(self.site.config['BASE_URL'], url.lstrip('/')) + all_data = list(zip(img_list, dest_img_list, img_titles)) + + if self.kw['sort_by_date']: + all_data.sort(key=lambda a: self.image_date(a[0])) + else: # Sort by name + all_data.sort(key=lambda a: a[0]) + + if all_data: + img_list, dest_img_list, img_titles = zip(*all_data) + else: + img_list, dest_img_list, img_titles = [], [], [] + items = [] for img, srcimg, title in list(zip(dest_img_list, img_list, img_titles))[:self.kw["feed_length"]]: img_size = os.stat( @@ -587,7 +636,7 @@ class Galleries(Task, ImageProcessor): description='', lastBuildDate=datetime.datetime.utcnow(), items=items, - generator='http://getnikola.com/', + generator='https://getnikola.com/', language=lang ) diff --git a/nikola/plugins/task/gzip.plugin b/nikola/plugins/task/gzip.plugin index 7834d22..d3a34ee 100644 --- a/nikola/plugins/task/gzip.plugin +++ b/nikola/plugins/task/gzip.plugin @@ -5,7 +5,7 @@ module = gzip [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create gzipped copies of files [Nikola] diff --git a/nikola/plugins/task/gzip.py b/nikola/plugins/task/gzip.py index cf16f63..79a11dc 100644 --- a/nikola/plugins/task/gzip.py +++ b/nikola/plugins/task/gzip.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -35,7 +35,6 @@ from nikola.plugin_categories import TaskMultiplier class GzipFiles(TaskMultiplier): - """If appropiate, create tasks to create gzipped versions of files.""" name = "gzip" diff --git a/nikola/plugins/task/indexes.plugin b/nikola/plugins/task/indexes.plugin index d9b0e5f..553b5ad 100644 --- a/nikola/plugins/task/indexes.plugin +++ b/nikola/plugins/task/indexes.plugin @@ -5,7 +5,7 @@ module = indexes [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Generates the blog's index pages. [Nikola] diff --git a/nikola/plugins/task/indexes.py b/nikola/plugins/task/indexes.py index c02818e..8ecd1de 100644 --- a/nikola/plugins/task/indexes.py +++ b/nikola/plugins/task/indexes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,23 +29,47 @@ from __future__ import unicode_literals from collections import defaultdict import os +try: + from urlparse import urljoin +except ImportError: + from urllib.parse import urljoin # NOQA from nikola.plugin_categories import Task from nikola import utils +from nikola.nikola import _enclosure class Indexes(Task): - """Render the blog indexes.""" name = "render_indexes" def set_site(self, site): """Set Nikola site.""" + self.number_of_pages = dict() + self.number_of_pages_section = {lang: dict() for lang in site.config['TRANSLATIONS']} site.register_path_handler('index', self.index_path) site.register_path_handler('index_atom', self.index_atom_path) + site.register_path_handler('section_index', self.index_section_path) + site.register_path_handler('section_index_atom', self.index_section_atom_path) + site.register_path_handler('section_index_rss', self.index_section_rss_path) return super(Indexes, self).set_site(site) + def _get_filtered_posts(self, lang, show_untranslated_posts): + """Return a filtered list of all posts for the given language. + + If show_untranslated_posts is True, will only include posts which + are translated to the given language. Otherwise, returns all posts. + """ + if show_untranslated_posts: + return self.site.posts + else: + return [x for x in self.site.posts if x.is_translation_available(lang)] + + def _compute_number_of_pages(self, filtered_posts, posts_count): + """Given a list of posts and the maximal number of posts per page, computes the number of pages needed.""" + return min(1, (len(filtered_posts) + posts_count - 1) // posts_count) + def gen_tasks(self): """Render the blog indexes.""" self.site.scan_posts() @@ -55,17 +79,22 @@ class Indexes(Task): "translations": self.site.config['TRANSLATIONS'], "messages": self.site.MESSAGES, "output_folder": self.site.config['OUTPUT_FOLDER'], + "feed_length": self.site.config['FEED_LENGTH'], + "feed_links_append_query": self.site.config["FEED_LINKS_APPEND_QUERY"], + "feed_teasers": self.site.config["FEED_TEASERS"], + "feed_plain": self.site.config["FEED_PLAIN"], "filters": self.site.config['FILTERS'], + "index_file": self.site.config['INDEX_FILE'], "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], "index_display_post_count": self.site.config['INDEX_DISPLAY_POST_COUNT'], "indexes_title": self.site.config['INDEXES_TITLE'], + "strip_indexes": self.site.config['STRIP_INDEXES'], "blog_title": self.site.config["BLOG_TITLE"], "generate_atom": self.site.config["GENERATE_ATOM"], + "site_url": self.site.config["SITE_URL"], } template_name = "index.tmpl" - posts = self.site.posts - self.number_of_pages = dict() for lang in kw["translations"]: def page_link(i, displayed_i, num_pages, force_addition, extension=None): feed = "_atom" if extension == ".atom" else "" @@ -77,20 +106,104 @@ class Indexes(Task): return utils.adjust_name_for_index_path(self.site.path("index" + feed, None, lang), i, displayed_i, lang, self.site, force_addition, extension) - if kw["show_untranslated_posts"]: - filtered_posts = posts - else: - filtered_posts = [x for x in posts if x.is_translation_available(lang)] + filtered_posts = self._get_filtered_posts(lang, kw["show_untranslated_posts"]) indexes_title = kw['indexes_title'](lang) or kw['blog_title'](lang) - self.number_of_pages[lang] = (len(filtered_posts) + kw['index_display_post_count'] - 1) // kw['index_display_post_count'] + self.number_of_pages[lang] = self._compute_number_of_pages(filtered_posts, kw['index_display_post_count']) context = {} - context["pagekind"] = ["index"] + context["pagekind"] = ["main_index", "index"] yield self.site.generic_index_renderer(lang, filtered_posts, indexes_title, template_name, context, kw, 'render_indexes', page_link, page_path) - if not self.site.config["STORY_INDEX"]: + if self.site.config['POSTS_SECTIONS']: + index_len = len(kw['index_file']) + + groups = defaultdict(list) + for p in filtered_posts: + groups[p.section_slug(lang)].append(p) + + # don't build sections when there is only one, aka. default setups + if not len(groups.items()) > 1: + continue + + for section_slug, post_list in groups.items(): + self.number_of_pages_section[lang][section_slug] = self._compute_number_of_pages(post_list, kw['index_display_post_count']) + + def cat_link(i, displayed_i, num_pages, force_addition, extension=None): + feed = "_atom" if extension == ".atom" else "" + return utils.adjust_name_for_index_link(self.site.link("section_index" + feed, section_slug, lang), i, displayed_i, + lang, self.site, force_addition, extension) + + def cat_path(i, displayed_i, num_pages, force_addition, extension=None): + feed = "_atom" if extension == ".atom" else "" + return utils.adjust_name_for_index_path(self.site.path("section_index" + feed, section_slug, lang), i, displayed_i, + lang, self.site, force_addition, extension) + + context = {} + + short_destination = os.path.join(section_slug, kw['index_file']) + link = short_destination.replace('\\', '/') + if kw['strip_indexes'] and link[-(1 + index_len):] == '/' + kw['index_file']: + link = link[:-index_len] + context["permalink"] = link + context["pagekind"] = ["section_page"] + context["description"] = self.site.config['POSTS_SECTION_DESCRIPTIONS'](lang)[section_slug] if section_slug in self.site.config['POSTS_SECTION_DESCRIPTIONS'](lang) else "" + + if self.site.config["POSTS_SECTION_ARE_INDEXES"]: + context["pagekind"].append("index") + posts_section_title = self.site.config['POSTS_SECTION_TITLE'](lang) + + section_title = None + if type(posts_section_title) is dict: + if section_slug in posts_section_title: + section_title = posts_section_title[section_slug] + elif type(posts_section_title) is str: + section_title = posts_section_title + if not section_title: + section_title = post_list[0].section_name(lang) + section_title = section_title.format(name=post_list[0].section_name(lang)) + + task = self.site.generic_index_renderer(lang, post_list, section_title, "sectionindex.tmpl", context, kw, self.name, cat_link, cat_path) + else: + context["pagekind"].append("list") + output_name = os.path.join(kw['output_folder'], section_slug, kw['index_file']) + task = self.site.generic_post_list_renderer(lang, post_list, output_name, "list.tmpl", kw['filters'], context) + task['uptodate'] = [utils.config_changed(kw, 'nikola.plugins.task.indexes')] + task['basename'] = self.name + yield task + + # RSS feed for section + deps = [] + deps_uptodate = [] + if kw["show_untranslated_posts"]: + posts = post_list[:kw['feed_length']] + else: + posts = [x for x in post_list if x.is_translation_available(lang)][:kw['feed_length']] + for post in posts: + deps += post.deps(lang) + deps_uptodate += post.deps_uptodate(lang) + + feed_url = urljoin(self.site.config['BASE_URL'], self.site.link('section_index_rss', section_slug, lang).lstrip('/')) + output_name = os.path.join(kw['output_folder'], self.site.path('section_index_rss', section_slug, lang).lstrip(os.sep)) + task = { + 'basename': self.name, + 'name': os.path.normpath(output_name), + 'file_dep': deps, + 'targets': [output_name], + 'actions': [(utils.generic_rss_renderer, + (lang, kw["blog_title"](lang), kw["site_url"], + context["description"], posts, output_name, + kw["feed_teasers"], kw["feed_plain"], kw['feed_length'], feed_url, + _enclosure, kw["feed_links_append_query"]))], + + 'task_dep': ['render_posts'], + 'clean': True, + 'uptodate': [utils.config_changed(kw, 'nikola.plugins.indexes')] + deps_uptodate, + } + yield task + + if not self.site.config["PAGE_INDEX"]: return kw = { "translations": self.site.config['TRANSLATIONS'], @@ -129,12 +242,13 @@ class Indexes(Task): for post in post_list: # If there is an index.html pending to be created from - # a story, do not generate the STORY_INDEX + # a page, do not generate the PAGE_INDEX if post.destination_path(lang) == short_destination: should_render = False else: context["items"].append((post.title(lang), - post.permalink(lang))) + post.permalink(lang), + None)) if should_render: task = self.site.generic_post_list_renderer(lang, post_list, @@ -147,22 +261,86 @@ class Indexes(Task): yield task def index_path(self, name, lang, is_feed=False): - """Return path to an index.""" + """Link to a numbered index. + + Example: + + link://index/3 => /index-3.html + """ extension = None if is_feed: extension = ".atom" index_file = os.path.splitext(self.site.config['INDEX_FILE'])[0] + extension else: index_file = self.site.config['INDEX_FILE'] + if lang in self.number_of_pages: + number_of_pages = self.number_of_pages[lang] + else: + number_of_pages = self._compute_number_of_pages(self._get_filtered_posts(lang, self.site.config['SHOW_UNTRANSLATED_POSTS']), self.site.config['INDEX_DISPLAY_POST_COUNT']) + self.number_of_pages[lang] = number_of_pages return utils.adjust_name_for_index_path_list([_f for _f in [self.site.config['TRANSLATIONS'][lang], self.site.config['INDEX_PATH'], index_file] if _f], name, - utils.get_displayed_page_number(name, self.number_of_pages[lang], self.site), + utils.get_displayed_page_number(name, number_of_pages, self.site), + lang, + self.site, + extension=extension) + + def index_section_path(self, name, lang, is_feed=False, is_rss=False): + """Link to the index for a section. + + Example: + + link://section_index/cars => /cars/index.html + """ + extension = None + + if is_feed: + extension = ".atom" + index_file = os.path.splitext(self.site.config['INDEX_FILE'])[0] + extension + elif is_rss: + index_file = 'rss.xml' + else: + index_file = self.site.config['INDEX_FILE'] + if name in self.number_of_pages_section[lang]: + number_of_pages = self.number_of_pages_section[lang][name] + else: + posts = [post for post in self._get_filtered_posts(lang, self.site.config['SHOW_UNTRANSLATED_POSTS']) if post.section_slug(lang) == name] + number_of_pages = self._compute_number_of_pages(posts, self.site.config['INDEX_DISPLAY_POST_COUNT']) + self.number_of_pages_section[lang][name] = number_of_pages + return utils.adjust_name_for_index_path_list([_f for _f in [self.site.config['TRANSLATIONS'][lang], + name, + index_file] if _f], + None, + utils.get_displayed_page_number(None, number_of_pages, self.site), lang, self.site, extension=extension) def index_atom_path(self, name, lang): - """Return path to an Atom index.""" + """Link to a numbered Atom index. + + Example: + + link://index_atom/3 => /index-3.atom + """ return self.index_path(name, lang, is_feed=True) + + def index_section_atom_path(self, name, lang): + """Link to the Atom index for a section. + + Example: + + link://section_index_atom/cars => /cars/index.atom + """ + return self.index_section_path(name, lang, is_feed=True) + + def index_section_rss_path(self, name, lang): + """Link to the RSS feed for a section. + + Example: + + link://section_index_rss/cars => /cars/rss.xml + """ + return self.index_section_path(name, lang, is_rss=True) diff --git a/nikola/plugins/task/listings.plugin b/nikola/plugins/task/listings.plugin index 435234b..8fc2e2d 100644 --- a/nikola/plugins/task/listings.plugin +++ b/nikola/plugins/task/listings.plugin @@ -5,7 +5,7 @@ module = listings [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Render code listings into output [Nikola] diff --git a/nikola/plugins/task/listings.py b/nikola/plugins/task/listings.py index 5f79724..e694aa5 100644 --- a/nikola/plugins/task/listings.py +++ b/nikola/plugins/task/listings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -28,12 +28,12 @@ from __future__ import unicode_literals, print_function -import sys +from collections import defaultdict import os import lxml.html from pygments import highlight -from pygments.lexers import get_lexer_for_filename, TextLexer +from pygments.lexers import get_lexer_for_filename, guess_lexer, TextLexer import natsort from nikola.plugin_categories import Task @@ -41,22 +41,20 @@ from nikola import utils class Listings(Task): - """Render code listings.""" name = "render_listings" def register_output_name(self, input_folder, rel_name, rel_output_name): """Register proper and improper file mappings.""" - if rel_name not in self.improper_input_file_mapping: - self.improper_input_file_mapping[rel_name] = [] - self.improper_input_file_mapping[rel_name].append(rel_output_name) + self.improper_input_file_mapping[rel_name].add(rel_output_name) self.proper_input_file_mapping[os.path.join(input_folder, rel_name)] = rel_output_name self.proper_input_file_mapping[rel_output_name] = rel_output_name def set_site(self, site): """Set Nikola site.""" site.register_path_handler('listing', self.listing_path) + site.register_path_handler('listing_source', self.listing_source_path) # We need to prepare some things for the listings path handler to work. @@ -75,7 +73,7 @@ class Listings(Task): if source in appearing_paths or dest in appearing_paths: problem = source if source in appearing_paths else dest utils.LOGGER.error("The listings input or output folder '{0}' appears in more than one entry in LISTINGS_FOLDERS, exiting.".format(problem)) - sys.exit(1) + continue appearing_paths.add(source) appearing_paths.add(dest) @@ -85,7 +83,7 @@ class Listings(Task): # a list is needed. This is needed for compatibility to previous Nikola # versions, where there was no need to specify the input directory name # when asking for a link via site.link('listing', ...). - self.improper_input_file_mapping = {} + self.improper_input_file_mapping = defaultdict(set) # proper_input_file_mapping maps relative input file (relative to CWD) # to a generated output file. Since we don't allow an input directory @@ -129,7 +127,11 @@ class Listings(Task): try: lexer = get_lexer_for_filename(in_name) except: - lexer = TextLexer() + try: + lexer = guess_lexer(fd.read()) + except: + lexer = TextLexer() + fd.seek(0) code = highlight(fd.read(), lexer, utils.NikolaPygmentsHTML(in_name)) title = os.path.basename(in_name) else: @@ -147,7 +149,7 @@ class Listings(Task): os.path.join( self.kw['output_folder'], output_folder)))) - if self.site.config['COPY_SOURCES'] and in_name: + if in_name: source_link = permalink[:-5] # remove '.html' else: source_link = None @@ -240,22 +242,47 @@ class Listings(Task): 'uptodate': [utils.config_changed(uptodate, 'nikola.plugins.task.listings:source')], 'clean': True, }, self.kw["filters"]) - if self.site.config['COPY_SOURCES']: - rel_name = os.path.join(rel_path, f) - rel_output_name = os.path.join(output_folder, rel_path, f) - self.register_output_name(input_folder, rel_name, rel_output_name) - out_name = os.path.join(self.kw['output_folder'], rel_output_name) - yield utils.apply_filters({ - 'basename': self.name, - 'name': out_name, - 'file_dep': [in_name], - 'targets': [out_name], - 'actions': [(utils.copy_file, [in_name, out_name])], - 'clean': True, - }, self.kw["filters"]) + + rel_name = os.path.join(rel_path, f) + rel_output_name = os.path.join(output_folder, rel_path, f) + self.register_output_name(input_folder, rel_name, rel_output_name) + out_name = os.path.join(self.kw['output_folder'], rel_output_name) + yield utils.apply_filters({ + 'basename': self.name, + 'name': out_name, + 'file_dep': [in_name], + 'targets': [out_name], + 'actions': [(utils.copy_file, [in_name, out_name])], + 'clean': True, + }, self.kw["filters"]) + + def listing_source_path(self, name, lang): + """A link to the source code for a listing. + + It will try to use the file name if it's not ambiguous, or the file path. + + Example: + + link://listing_source/hello.py => /listings/tutorial/hello.py + + link://listing_source/tutorial/hello.py => /listings/tutorial/hello.py + """ + result = self.listing_path(name, lang) + if result[-1].endswith('.html'): + result[-1] = result[-1][:-5] + return result def listing_path(self, namep, lang): - """Return path to a listing.""" + """A link to a listing. + + It will try to use the file name if it's not ambiguous, or the file path. + + Example: + + link://listing/hello.py => /listings/tutorial/hello.py.html + + link://listing/tutorial/hello.py => /listings/tutorial/hello.py.html + """ namep = namep.replace('/', os.sep) nameh = namep + '.html' for name in (namep, nameh): @@ -268,14 +295,14 @@ class Listings(Task): # ambiguities. if len(self.improper_input_file_mapping[name]) > 1: utils.LOGGER.error("Using non-unique listing name '{0}', which maps to more than one listing name ({1})!".format(name, str(self.improper_input_file_mapping[name]))) - sys.exit(1) + return ["ERROR"] if len(self.site.config['LISTINGS_FOLDERS']) > 1: utils.LOGGER.notice("Using listings names in site.link() without input directory prefix while configuration's LISTINGS_FOLDERS has more than one entry.") - name = self.improper_input_file_mapping[name][0] + name = list(self.improper_input_file_mapping[name])[0] break else: utils.LOGGER.error("Unknown listing name {0}!".format(namep)) - sys.exit(1) + return ["ERROR"] if not name.endswith(os.sep + self.site.config["INDEX_FILE"]): name += '.html' path_parts = name.split(os.sep) diff --git a/nikola/plugins/task/pages.plugin b/nikola/plugins/task/pages.plugin index 023d41b..1bdc7f4 100644 --- a/nikola/plugins/task/pages.plugin +++ b/nikola/plugins/task/pages.plugin @@ -5,7 +5,7 @@ module = pages [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create pages in the output. [Nikola] diff --git a/nikola/plugins/task/pages.py b/nikola/plugins/task/pages.py index e6a8a82..7d8287b 100644 --- a/nikola/plugins/task/pages.py +++ b/nikola/plugins/task/pages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,7 +32,6 @@ from nikola.utils import config_changed class RenderPages(Task): - """Render pages into output.""" name = "render_pages" @@ -55,7 +54,7 @@ class RenderPages(Task): if post.is_post: context = {'pagekind': ['post_page']} else: - context = {'pagekind': ['story_page']} + context = {'pagekind': ['story_page', 'page_page']} for task in self.site.generic_page_renderer(lang, post, kw["filters"], context): task['uptodate'] = task['uptodate'] + [config_changed(kw, 'nikola.plugins.task.pages')] task['basename'] = self.name diff --git a/nikola/plugins/task/posts.plugin b/nikola/plugins/task/posts.plugin index 79b7c51..c9578bc 100644 --- a/nikola/plugins/task/posts.plugin +++ b/nikola/plugins/task/posts.plugin @@ -5,7 +5,7 @@ module = posts [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create HTML fragments out of posts. [Nikola] diff --git a/nikola/plugins/task/posts.py b/nikola/plugins/task/posts.py index a3a8375..fe10c5f 100644 --- a/nikola/plugins/task/posts.py +++ b/nikola/plugins/task/posts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -44,7 +44,6 @@ def update_deps(post, lang, task): class RenderPosts(Task): - """Build HTML fragments from metadata and text.""" name = "render_posts" @@ -77,6 +76,8 @@ class RenderPosts(Task): deps_dict = copy(kw) deps_dict.pop('timeline') for post in kw['timeline']: + if not post.is_translation_available(lang) and not self.site.config['SHOW_UNTRANSLATED_POSTS']: + continue # Extra config dependencies picked from config for p in post.fragment_deps(lang): if p.startswith('####MAGIC####CONFIG:'): @@ -114,7 +115,7 @@ class RenderPosts(Task): pass else: flist.append(f) - yield utils.apply_filters(task, {os.path.splitext(dest): flist}) + yield utils.apply_filters(task, {os.path.splitext(dest)[-1]: flist}) def dependence_on_timeline(self, post, lang): """Check if a post depends on the timeline.""" diff --git a/nikola/plugins/task/py3_switch.plugin b/nikola/plugins/task/py3_switch.plugin new file mode 100644 index 0000000..b0014e1 --- /dev/null +++ b/nikola/plugins/task/py3_switch.plugin @@ -0,0 +1,13 @@ +[Core] +name = py3_switch +module = py3_switch + +[Documentation] +author = Roberto Alsina +version = 1.0 +website = https://getnikola.com/ +description = Beg the user to switch to Python 3 + +[Nikola] +plugincategory = Task + diff --git a/nikola/plugins/task/py3_switch.py b/nikola/plugins/task/py3_switch.py new file mode 100644 index 0000000..2ff4e2d --- /dev/null +++ b/nikola/plugins/task/py3_switch.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2016 Roberto Alsina and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Beg the user to switch to python 3.""" + +import datetime +import os +import random +import sys + +import doit.tools + +from nikola.utils import get_logger, STDERR_HANDLER +from nikola.plugin_categories import LateTask + +PY2_AND_NO_PY3_WARNING = """Nikola is going to deprecate Python 2 support in 2016. Your current +version will continue to work, but please consider upgrading to Python 3. + +Please check http://bit.ly/1FKEsiX for details. +""" +PY2_WARNING = """Nikola is going to deprecate Python 2 support in 2016. You already have Python 3 +available in your system. Why not switch? + +Please check http://bit.ly/1FKEsiX for details. +""" +PY2_BARBS = [ + "Python 2 has been deprecated for years. Stop clinging to your long gone youth and switch to Python3.", + "Python 2 is the safety blanket of languages. Be a big kid and switch to Python 3", + "Python 2 is old and busted. Python 3 is the new hotness.", + "Nice unicode you have there, would be a shame something happened to it.. switch to python 3!.", + "Don't get in the way of progress! Upgrade to Python 3 and save a developer's mind today!", + "Winners don't use Python 2 -- Signed: The FBI", + "Python 2? What year is it?", + "I just wanna tell you how I'm feeling\n" + "Gotta make you understand\n" + "Never gonna give you up [But Python 2 has to go]", + "The year 2009 called, and they want their Python 2.7 back.", +] + + +LOGGER = get_logger('Nikola', STDERR_HANDLER) + + +def has_python_3(): + """Check if python 3 is available.""" + if 'win' in sys.platform: + py_bin = 'py.exe' + else: + py_bin = 'python3' + for path in os.environ["PATH"].split(os.pathsep): + if os.access(os.path.join(path, py_bin), os.X_OK): + return True + return False + + +class Py3Switch(LateTask): + """Beg the user to switch to python 3.""" + + name = "_switch to py3" + + def gen_tasks(self): + """Beg the user to switch to python 3.""" + def give_warning(): + if sys.version_info[0] == 3: + return + if has_python_3(): + LOGGER.warn(random.choice(PY2_BARBS)) + LOGGER.warn(PY2_WARNING) + else: + LOGGER.warn(PY2_AND_NO_PY3_WARNING) + + task = { + 'basename': self.name, + 'name': 'please!', + 'actions': [give_warning], + 'clean': True, + 'uptodate': [doit.tools.timeout(datetime.timedelta(days=3))] + } + + return task diff --git a/nikola/plugins/task/redirect.plugin b/nikola/plugins/task/redirect.plugin index c3137b9..c5a3042 100644 --- a/nikola/plugins/task/redirect.plugin +++ b/nikola/plugins/task/redirect.plugin @@ -5,7 +5,7 @@ module = redirect [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create redirect pages. [Nikola] diff --git a/nikola/plugins/task/redirect.py b/nikola/plugins/task/redirect.py index 8530f5e..b170b81 100644 --- a/nikola/plugins/task/redirect.py +++ b/nikola/plugins/task/redirect.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -35,7 +35,6 @@ from nikola import utils class Redirect(Task): - """Generate redirections.""" name = "redirect" @@ -51,7 +50,7 @@ class Redirect(Task): yield self.group_task() if kw['redirections']: for src, dst in kw["redirections"]: - src_path = os.path.join(kw["output_folder"], src) + src_path = os.path.join(kw["output_folder"], src.lstrip('/')) yield utils.apply_filters({ 'basename': self.name, 'name': src_path, diff --git a/nikola/plugins/task/robots.plugin b/nikola/plugins/task/robots.plugin index 72ce31f..7ae56c6 100644 --- a/nikola/plugins/task/robots.plugin +++ b/nikola/plugins/task/robots.plugin @@ -5,7 +5,7 @@ module = robots [Documentation] author = Daniel Aleksandersen version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Generate /robots.txt exclusion file and promote sitemap. [Nikola] diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py index 65254b6..8537fc8 100644 --- a/nikola/plugins/task/robots.py +++ b/nikola/plugins/task/robots.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -39,7 +39,6 @@ from nikola import utils class RobotsFile(LateTask): - """Generate a robots.txt file.""" name = "robots_file" @@ -64,14 +63,15 @@ class RobotsFile(LateTask): with io.open(robots_path, 'w+', encoding='utf8') as outf: outf.write("Sitemap: {0}\n\n".format(sitemapindex_url)) + outf.write("User-Agent: *\n") if kw["robots_exclusions"]: - outf.write("User-Agent: *\n") for loc in kw["robots_exclusions"]: outf.write("Disallow: {0}\n".format(loc)) + outf.write("Host: {0}\n".format(urlparse(kw["base_url"]).netloc)) yield self.group_task() - if not utils.get_asset_path("robots.txt", [], files_folders=kw["files_folders"]): + if not utils.get_asset_path("robots.txt", [], files_folders=kw["files_folders"], output_dir=False): yield utils.apply_filters({ "basename": self.name, "name": robots_path, @@ -82,6 +82,6 @@ class RobotsFile(LateTask): "task_dep": ["sitemap"] }, kw["filters"]) elif kw["robots_exclusions"]: - utils.LOGGER.warn('Did not generate robots.txt as one already exists in FILES_FOLDERS. ROBOTS_EXCLUSIONS will not have any affect on the copied fie.') + utils.LOGGER.warn('Did not generate robots.txt as one already exists in FILES_FOLDERS. ROBOTS_EXCLUSIONS will not have any affect on the copied file.') else: utils.LOGGER.debug('Did not generate robots.txt as one already exists in FILES_FOLDERS.') diff --git a/nikola/plugins/task/rss.plugin b/nikola/plugins/task/rss.plugin index cf9b7a7..4dd8aba 100644 --- a/nikola/plugins/task/rss.plugin +++ b/nikola/plugins/task/rss.plugin @@ -5,7 +5,7 @@ module = rss [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Generate RSS feeds. [Nikola] diff --git a/nikola/plugins/task/rss.py b/nikola/plugins/task/rss.py index 9020a06..780559b 100644 --- a/nikola/plugins/task/rss.py +++ b/nikola/plugins/task/rss.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -34,11 +34,11 @@ except ImportError: from urllib.parse import urljoin # NOQA from nikola import utils +from nikola.nikola import _enclosure from nikola.plugin_categories import Task class GenerateRSS(Task): - """Generate RSS feeds.""" name = "generate_rss" @@ -58,13 +58,14 @@ class GenerateRSS(Task): "base_url": self.site.config["BASE_URL"], "blog_description": self.site.config["BLOG_DESCRIPTION"], "output_folder": self.site.config["OUTPUT_FOLDER"], - "rss_teasers": self.site.config["RSS_TEASERS"], - "rss_plain": self.site.config["RSS_PLAIN"], + "feed_teasers": self.site.config["FEED_TEASERS"], + "feed_plain": self.site.config["FEED_PLAIN"], "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], "feed_length": self.site.config['FEED_LENGTH'], + "feed_previewimage": self.site.config["FEED_PREVIEWIMAGE"], "tzinfo": self.site.tzinfo, - "rss_read_more_link": self.site.config["RSS_READ_MORE_LINK"], - "rss_links_append_query": self.site.config["RSS_LINKS_APPEND_QUERY"], + "feed_read_more_link": self.site.config["FEED_READ_MORE_LINK"], + "feed_links_append_query": self.site.config["FEED_LINKS_APPEND_QUERY"], } self.site.scan_posts() # Check for any changes in the state of use_in_feeds for any post. @@ -96,8 +97,8 @@ class GenerateRSS(Task): 'actions': [(utils.generic_rss_renderer, (lang, kw["blog_title"](lang), kw["site_url"], kw["blog_description"](lang), posts, output_name, - kw["rss_teasers"], kw["rss_plain"], kw['feed_length'], feed_url, - None, kw["rss_links_append_query"]))], + kw["feed_teasers"], kw["feed_plain"], kw['feed_length'], feed_url, + _enclosure, kw["feed_links_append_query"]))], 'task_dep': ['render_posts'], 'clean': True, @@ -106,6 +107,11 @@ class GenerateRSS(Task): yield utils.apply_filters(task, kw['filters']) def rss_path(self, name, lang): - """Return RSS path.""" + """A link to the RSS feed path. + + Example: + + link://rss => /blog/rss.xml + """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], self.site.config['RSS_PATH'], 'rss.xml'] if _f] diff --git a/nikola/plugins/task/scale_images.plugin b/nikola/plugins/task/scale_images.plugin index d906b8c..3edd0c6 100644 --- a/nikola/plugins/task/scale_images.plugin +++ b/nikola/plugins/task/scale_images.plugin @@ -5,7 +5,7 @@ module = scale_images [Documentation] author = Pelle Nilsson version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create down-scaled images and thumbnails. [Nikola] diff --git a/nikola/plugins/task/scale_images.py b/nikola/plugins/task/scale_images.py index 22ed2ab..2b483ae 100644 --- a/nikola/plugins/task/scale_images.py +++ b/nikola/plugins/task/scale_images.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Pelle Nilsson and others. +# Copyright © 2014-2016 Pelle Nilsson and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -34,7 +34,6 @@ from nikola import utils class ScaleImage(Task, ImageProcessor): - """Resize images and create thumbnails for them.""" name = "scale_images" @@ -72,8 +71,8 @@ class ScaleImage(Task, ImageProcessor): def process_image(self, src, dst, thumb): """Resize an image.""" - self.resize_image(src, dst, self.kw['max_image_size'], False) - self.resize_image(src, thumb, self.kw['image_thumbnail_size'], False) + self.resize_image(src, dst, self.kw['max_image_size'], False, preserve_exif_data=self.kw['preserve_exif_data'], exif_whitelist=self.kw['exif_whitelist']) + self.resize_image(src, thumb, self.kw['image_thumbnail_size'], False, preserve_exif_data=self.kw['preserve_exif_data'], exif_whitelist=self.kw['exif_whitelist']) def gen_tasks(self): """Copy static files into the output folder.""" @@ -83,6 +82,8 @@ class ScaleImage(Task, ImageProcessor): 'image_folders': self.site.config['IMAGE_FOLDERS'], 'output_folder': self.site.config['OUTPUT_FOLDER'], 'filters': self.site.config['FILTERS'], + 'preserve_exif_data': self.site.config['PRESERVE_EXIF_DATA'], + 'exif_whitelist': self.site.config['EXIF_WHITELIST'], } self.image_ext_list = self.image_ext_list_builtin diff --git a/nikola/plugins/task/sitemap.plugin b/nikola/plugins/task/sitemap.plugin index e3c991f..83e72c4 100644 --- a/nikola/plugins/task/sitemap.plugin +++ b/nikola/plugins/task/sitemap.plugin @@ -5,7 +5,7 @@ module = sitemap [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Generate google sitemap. [Nikola] diff --git a/nikola/plugins/task/sitemap/__init__.py b/nikola/plugins/task/sitemap/__init__.py index fd781d6..64fcb45 100644 --- a/nikola/plugins/task/sitemap/__init__.py +++ b/nikola/plugins/task/sitemap/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -31,6 +31,7 @@ import io import datetime import dateutil.tz import os +import sys try: from urlparse import urljoin, urlparse import robotparser as robotparser @@ -39,7 +40,7 @@ except ImportError: import urllib.robotparser as robotparser # NOQA from nikola.plugin_categories import LateTask -from nikola.utils import config_changed, apply_filters +from nikola.utils import apply_filters, config_changed, encodelink urlset_header = """<?xml version="1.0" encoding="UTF-8"?> @@ -106,7 +107,6 @@ def get_base_path(base): class Sitemap(LateTask): - """Generate a sitemap.""" name = "sitemap" @@ -146,7 +146,10 @@ class Sitemap(LateTask): continue # Totally empty, not on sitemap path = os.path.relpath(root, output) # ignore the current directory. - path = (path.replace(os.sep, '/') + '/').replace('./', '') + if path == '.': + path = '' + else: + path = path.replace(os.sep, '/') + '/' lastmod = self.get_lastmod(root) loc = urljoin(base_url, base_path + path) if kw['index_file'] in files and kw['strip_indexes']: # ignore folders when not stripping urls @@ -155,12 +158,12 @@ class Sitemap(LateTask): continue alternates = [] if post: - for lang in kw['translations']: + for lang in post.translated_to: alt_url = post.permalink(lang=lang, absolute=True) - if loc == alt_url: + if encodelink(loc) == alt_url: continue alternates.append(alternates_format.format(lang, alt_url)) - urlset[loc] = loc_format.format(loc, lastmod, ''.join(alternates)) + urlset[loc] = loc_format.format(encodelink(loc), lastmod, ''.join(alternates)) for fname in files: if kw['strip_indexes'] and fname == kw['index_file']: continue # We already mapped the folder @@ -200,7 +203,7 @@ class Sitemap(LateTask): path = path.replace(os.sep, '/') lastmod = self.get_lastmod(real_path) loc = urljoin(base_url, base_path + path) - sitemapindex[loc] = sitemap_format.format(loc, lastmod) + sitemapindex[loc] = sitemap_format.format(encodelink(loc), lastmod) continue else: continue # ignores all XML files except those presumed to be RSS @@ -212,20 +215,24 @@ class Sitemap(LateTask): loc = urljoin(base_url, base_path + path) alternates = [] if post: - for lang in kw['translations']: + for lang in post.translated_to: alt_url = post.permalink(lang=lang, absolute=True) - if loc == alt_url: + if encodelink(loc) == alt_url: continue alternates.append(alternates_format.format(lang, alt_url)) - urlset[loc] = loc_format.format(loc, lastmod, '\n'.join(alternates)) + urlset[loc] = loc_format.format(encodelink(loc), lastmod, '\n'.join(alternates)) def robot_fetch(path): """Check if robots can fetch a file.""" for rule in kw["robots_exclusions"]: robot = robotparser.RobotFileParser() robot.parse(["User-Agent: *", "Disallow: {0}".format(rule)]) - if not robot.can_fetch("*", '/' + path): - return False # not robot food + if sys.version_info[0] == 3: + if not robot.can_fetch("*", '/' + path): + return False # not robot food + else: + if not robot.can_fetch("*", ('/' + path).encode('utf-8')): + return False # not robot food return True def write_sitemap(): diff --git a/nikola/plugins/task/sources.plugin b/nikola/plugins/task/sources.plugin index d232c2b..66856f1 100644 --- a/nikola/plugins/task/sources.plugin +++ b/nikola/plugins/task/sources.plugin @@ -5,7 +5,7 @@ module = sources [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Copy page sources into the output. [Nikola] diff --git a/nikola/plugins/task/sources.py b/nikola/plugins/task/sources.py index 87b4ae7..0d77aba 100644 --- a/nikola/plugins/task/sources.py +++ b/nikola/plugins/task/sources.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -33,7 +33,6 @@ from nikola import utils class Sources(Task): - """Copy page sources into the output.""" name = "render_sources" diff --git a/nikola/plugins/task/tags.plugin b/nikola/plugins/task/tags.plugin index 283a16a..c3a5be3 100644 --- a/nikola/plugins/task/tags.plugin +++ b/nikola/plugins/task/tags.plugin @@ -5,7 +5,7 @@ module = tags [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Render the tag pages and feeds. [Nikola] diff --git a/nikola/plugins/task/tags.py b/nikola/plugins/task/tags.py index 3186636..8b4683e 100644 --- a/nikola/plugins/task/tags.py +++ b/nikola/plugins/task/tags.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,7 +29,6 @@ from __future__ import unicode_literals import json import os -import sys import natsort try: from urlparse import urljoin @@ -38,10 +37,10 @@ except ImportError: from nikola.plugin_categories import Task from nikola import utils +from nikola.nikola import _enclosure class RenderTags(Task): - """Render the tag/category pages and feeds.""" name = "render_tags" @@ -74,9 +73,9 @@ class RenderTags(Task): 'category_prefix': self.site.config['CATEGORY_PREFIX'], "category_pages_are_indexes": self.site.config['CATEGORY_PAGES_ARE_INDEXES'], "generate_rss": self.site.config['GENERATE_RSS'], - "rss_teasers": self.site.config["RSS_TEASERS"], - "rss_plain": self.site.config["RSS_PLAIN"], - "rss_link_append_query": self.site.config["RSS_LINKS_APPEND_QUERY"], + "feed_teasers": self.site.config["FEED_TEASERS"], + "feed_plain": self.site.config["FEED_PLAIN"], + "feed_link_append_query": self.site.config["FEED_LINKS_APPEND_QUERY"], "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], "feed_length": self.site.config['FEED_LENGTH'], "taglist_minimum_post_count": self.site.config['TAGLIST_MINIMUM_POSTS'], @@ -84,6 +83,10 @@ class RenderTags(Task): "pretty_urls": self.site.config['PRETTY_URLS'], "strip_indexes": self.site.config['STRIP_INDEXES'], "index_file": self.site.config['INDEX_FILE'], + "category_pages_descriptions": self.site.config['CATEGORY_PAGES_DESCRIPTIONS'], + "category_pages_titles": self.site.config['CATEGORY_PAGES_TITLES'], + "tag_pages_descriptions": self.site.config['TAG_PAGES_DESCRIPTIONS'], + "tag_pages_titles": self.site.config['TAG_PAGES_TITLES'], } self.site.scan_posts() @@ -94,31 +97,31 @@ class RenderTags(Task): if not self.site.posts_per_tag and not self.site.posts_per_category: return - if kw['category_path'] == kw['tag_path']: - tags = {self.slugify_tag_name(tag): tag for tag in self.site.posts_per_tag.keys()} - cats = {tuple(self.slugify_category_name(category)): category for category in self.site.posts_per_category.keys()} - categories = {k[0]: v for k, v in cats.items() if len(k) == 1} - intersect = set(tags.keys()) & set(categories.keys()) - if len(intersect) > 0: - for slug in intersect: - utils.LOGGER.error("Category '{0}' and tag '{1}' both have the same slug '{2}'!".format('/'.join(categories[slug]), tags[slug], slug)) - sys.exit(1) - - # Test for category slug clashes - categories = {} - for category in self.site.posts_per_category.keys(): - slug = tuple(self.slugify_category_name(category)) - for part in slug: - if len(part) == 0: - utils.LOGGER.error("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) - sys.exit(1) - if slug in categories: - other_category = categories[slug] - utils.LOGGER.error('You have categories that are too similar: {0} and {1}'.format(category, other_category)) - utils.LOGGER.error('Category {0} is used in: {1}'.format(category, ', '.join([p.source_path for p in self.site.posts_per_category[category]]))) - utils.LOGGER.error('Category {0} is used in: {1}'.format(other_category, ', '.join([p.source_path for p in self.site.posts_per_category[other_category]]))) - sys.exit(1) - categories[slug] = category + for lang in kw["translations"]: + if kw['category_path'][lang] == kw['tag_path'][lang]: + tags = {self.slugify_tag_name(tag, lang): tag for tag in self.site.tags_per_language[lang]} + cats = {tuple(self.slugify_category_name(category, lang)): category for category in self.site.posts_per_category.keys()} + categories = {k[0]: v for k, v in cats.items() if len(k) == 1} + intersect = set(tags.keys()) & set(categories.keys()) + if len(intersect) > 0: + for slug in intersect: + utils.LOGGER.error("Category '{0}' and tag '{1}' both have the same slug '{2}' for language {3}!".format('/'.join(categories[slug]), tags[slug], slug, lang)) + + # Test for category slug clashes + categories = {} + for category in self.site.posts_per_category.keys(): + slug = tuple(self.slugify_category_name(category, lang)) + for part in slug: + if len(part) == 0: + utils.LOGGER.error("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) + raise RuntimeError("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) + if slug in categories: + other_category = categories[slug] + utils.LOGGER.error('You have categories that are too similar: {0} and {1} (language {2})'.format(category, other_category, lang)) + utils.LOGGER.error('Category {0} is used in: {1}'.format(category, ', '.join([p.source_path for p in self.site.posts_per_category[category]]))) + utils.LOGGER.error('Category {0} is used in: {1}'.format(other_category, ', '.join([p.source_path for p in self.site.posts_per_category[other_category]]))) + raise RuntimeError("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) + categories[slug] = category tag_list = list(self.site.posts_per_tag.items()) cat_list = list(self.site.posts_per_category.items()) @@ -168,7 +171,7 @@ class RenderTags(Task): """Write tag data into JSON file, for use in tag clouds.""" utils.makedirs(os.path.dirname(output_name)) with open(output_name, 'w+') as fd: - json.dump(data, fd) + json.dump(data, fd, sort_keys=True) if self.site.config['WRITE_TAG_CLOUD']: task = { @@ -182,7 +185,7 @@ class RenderTags(Task): task['clean'] = True yield utils.apply_filters(task, kw['filters']) - def _create_tags_page(self, kw, include_tags=True, include_categories=True): + def _create_tags_page(self, kw, lang, include_tags=True, include_categories=True): """Create a global "all your tags/categories" page for each language.""" categories = [cat.category_name for cat in self.site.category_hierarchy] has_categories = (categories != []) and include_categories @@ -190,60 +193,59 @@ class RenderTags(Task): kw = kw.copy() if include_categories: kw['categories'] = categories - for lang in kw["translations"]: - tags = natsort.natsorted([tag for tag in self.site.tags_per_language[lang] - if len(self.site.posts_per_tag[tag]) >= kw["taglist_minimum_post_count"]], - alg=natsort.ns.F | natsort.ns.IC) - has_tags = (tags != []) and include_tags - if include_tags: - kw['tags'] = tags - output_name = os.path.join( - kw['output_folder'], self.site.path('tag_index' if has_tags else 'category_index', None, lang)) - output_name = output_name - context = {} - if has_categories and has_tags: - context["title"] = kw["messages"][lang]["Tags and Categories"] - elif has_categories: - context["title"] = kw["messages"][lang]["Categories"] - else: - context["title"] = kw["messages"][lang]["Tags"] - if has_tags: - context["items"] = [(tag, self.site.link("tag", tag, lang)) for tag - in tags] - else: - context["items"] = None - if has_categories: - context["cat_items"] = [(tag, self.site.link("category", tag, lang)) for tag - in categories] - context['cat_hierarchy'] = [(node.name, node.category_name, node.category_path, - self.site.link("category", node.category_name), - node.indent_levels, node.indent_change_before, - node.indent_change_after) - for node in self.site.category_hierarchy] - else: - context["cat_items"] = None - context["permalink"] = self.site.link("tag_index" if has_tags else "category_index", None, lang) - context["description"] = context["title"] - context["pagekind"] = ["list", "tags_page"] - task = self.site.generic_post_list_renderer( - lang, - [], - output_name, - template_name, - kw['filters'], - context, - ) - task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.tags:page')] - task['basename'] = str(self.name) - yield task + tags = natsort.natsorted([tag for tag in self.site.tags_per_language[lang] + if len(self.site.posts_per_tag[tag]) >= kw["taglist_minimum_post_count"]], + alg=natsort.ns.F | natsort.ns.IC) + has_tags = (tags != []) and include_tags + if include_tags: + kw['tags'] = tags + output_name = os.path.join( + kw['output_folder'], self.site.path('tag_index' if has_tags else 'category_index', None, lang)) + context = {} + if has_categories and has_tags: + context["title"] = kw["messages"][lang]["Tags and Categories"] + elif has_categories: + context["title"] = kw["messages"][lang]["Categories"] + else: + context["title"] = kw["messages"][lang]["Tags"] + if has_tags: + context["items"] = [(tag, self.site.link("tag", tag, lang)) for tag + in tags] + else: + context["items"] = None + if has_categories: + context["cat_items"] = [(tag, self.site.link("category", tag, lang)) for tag + in categories] + context['cat_hierarchy'] = [(node.name, node.category_name, node.category_path, + self.site.link("category", node.category_name), + node.indent_levels, node.indent_change_before, + node.indent_change_after) + for node in self.site.category_hierarchy] + else: + context["cat_items"] = None + context["permalink"] = self.site.link("tag_index" if has_tags else "category_index", None, lang) + context["description"] = context["title"] + context["pagekind"] = ["list", "tags_page"] + task = self.site.generic_post_list_renderer( + lang, + [], + output_name, + template_name, + kw['filters'], + context, + ) + task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.tags:page')] + task['basename'] = str(self.name) + yield task def list_tags_page(self, kw): """Create a global "all your tags/categories" page for each language.""" - if self.site.config['TAG_PATH'] == self.site.config['CATEGORY_PATH']: - yield self._create_tags_page(kw, True, True) - else: - yield self._create_tags_page(kw, False, True) - yield self._create_tags_page(kw, True, False) + for lang in kw["translations"]: + if self.site.config['TAG_PATH'][lang] == self.site.config['CATEGORY_PATH'][lang]: + yield self._create_tags_page(kw, lang, True, True) + else: + yield self._create_tags_page(kw, lang, False, True) + yield self._create_tags_page(kw, lang, True, False) def _get_title(self, tag, is_category): if is_category: @@ -251,6 +253,10 @@ class RenderTags(Task): else: return tag + def _get_indexes_title(self, tag, nice_tag, is_category, lang, messages): + titles = self.site.config['CATEGORY_PAGES_TITLES'] if is_category else self.site.config['TAG_PAGES_TITLES'] + return titles[lang][tag] if lang in titles and tag in titles[lang] else messages[lang]["Posts about %s"] % nice_tag + def _get_description(self, tag, is_category, lang): descriptions = self.site.config['CATEGORY_PAGES_DESCRIPTIONS'] if is_category else self.site.config['TAG_PAGES_DESCRIPTIONS'] return descriptions[lang][tag] if lang in descriptions and tag in descriptions[lang] else None @@ -276,7 +282,7 @@ class RenderTags(Task): if kw["generate_rss"]: # On a tag page, the feeds include the tag's feeds rss_link = ("""<link rel="alternate" type="application/rss+xml" """ - """type="application/rss+xml" title="RSS for tag """ + """title="RSS for tag """ """{0} ({1})" href="{2}">""".format( title, lang, self.site.link(kind + "_rss", tag, lang))) context_source['rss_link'] = rss_link @@ -284,7 +290,7 @@ class RenderTags(Task): context_source["category"] = tag context_source["category_path"] = self.site.parse_category_name(tag) context_source["tag"] = title - indexes_title = kw["messages"][lang]["Posts about %s"] % title + indexes_title = self._get_indexes_title(tag, title, is_category, lang, kw["messages"]) context_source["description"] = self._get_description(tag, is_category, lang) if is_category: context_source["subcategories"] = self._get_subcategories(tag) @@ -306,7 +312,7 @@ class RenderTags(Task): context["category"] = tag context["category_path"] = self.site.parse_category_name(tag) context["tag"] = title - context["title"] = kw["messages"][lang]["Posts about %s"] % title + context["title"] = self._get_indexes_title(tag, title, is_category, lang, kw["messages"]) context["posts"] = post_list context["permalink"] = self.site.link(kind, tag, lang) context["kind"] = kind @@ -326,6 +332,29 @@ class RenderTags(Task): task['basename'] = str(self.name) yield task + if self.site.config['GENERATE_ATOM']: + yield self.atom_feed_list(kind, tag, lang, post_list, context, kw) + + def atom_feed_list(self, kind, tag, lang, post_list, context, kw): + """Generate atom feeds for tag lists.""" + if kind == 'tag': + context['feedlink'] = self.site.abs_link(self.site.path('tag_atom', tag, lang)) + feed_path = os.path.join(kw['output_folder'], self.site.path('tag_atom', tag, lang)) + elif kind == 'category': + context['feedlink'] = self.site.abs_link(self.site.path('category_atom', tag, lang)) + feed_path = os.path.join(kw['output_folder'], self.site.path('category_atom', tag, lang)) + + task = { + 'basename': str(self.name), + 'name': feed_path, + 'targets': [feed_path], + 'actions': [(self.site.atom_feed_renderer, (lang, post_list, feed_path, kw['filters'], context))], + 'clean': True, + 'uptodate': [utils.config_changed(kw, 'nikola.plugins.task.tags:atom')], + 'task_dep': ['render_posts'], + } + return task + def tag_rss(self, tag, lang, posts, kw, is_category): """Create a RSS feed for a single tag in a given language.""" kind = "category" if is_category else "tag" @@ -349,64 +378,100 @@ class RenderTags(Task): 'actions': [(utils.generic_rss_renderer, (lang, "{0} ({1})".format(kw["blog_title"](lang), self._get_title(tag, is_category)), kw["site_url"], None, post_list, - output_name, kw["rss_teasers"], kw["rss_plain"], kw['feed_length'], - feed_url, None, kw["rss_link_append_query"]))], + output_name, kw["feed_teasers"], kw["feed_plain"], kw['feed_length'], + feed_url, _enclosure, kw["feed_link_append_query"]))], 'clean': True, 'uptodate': [utils.config_changed(kw, 'nikola.plugins.task.tags:rss')] + deps_uptodate, 'task_dep': ['render_posts'], } return utils.apply_filters(task, kw['filters']) - def slugify_tag_name(self, name): + def slugify_tag_name(self, name, lang): """Slugify a tag name.""" + if lang is None: # TODO: remove in v8 + utils.LOGGER.warn("RenderTags.slugify_tag_name() called without language!") + lang = '' if self.site.config['SLUG_TAG_PATH']: - name = utils.slugify(name) + name = utils.slugify(name, lang) return name def tag_index_path(self, name, lang): - """Return path to the tag index.""" - return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['TAG_PATH'], - self.site.config['INDEX_FILE']] if _f] + """A link to the tag index. + + Example: + + link://tag_index => /tags/index.html + """ + if self.site.config['TAGS_INDEX_PATH'][lang]: + paths = [_f for _f in [self.site.config['TRANSLATIONS'][lang], + self.site.config['TAGS_INDEX_PATH'][lang]] if _f] + else: + paths = [_f for _f in [self.site.config['TRANSLATIONS'][lang], + self.site.config['TAG_PATH'][lang], + self.site.config['INDEX_FILE']] if _f] + return paths def category_index_path(self, name, lang): - """Return path to the category index.""" + """A link to the category index. + + Example: + + link://category_index => /categories/index.html + """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['CATEGORY_PATH'], + self.site.config['CATEGORY_PATH'][lang], self.site.config['INDEX_FILE']] if _f] def tag_path(self, name, lang): - """Return path to a tag.""" + """A link to a tag's page. + + Example: + + link://tag/cats => /tags/cats.html + """ if self.site.config['PRETTY_URLS']: return [_f for _f in [ self.site.config['TRANSLATIONS'][lang], - self.site.config['TAG_PATH'], - self.slugify_tag_name(name), + self.site.config['TAG_PATH'][lang], + self.slugify_tag_name(name, lang), self.site.config['INDEX_FILE']] if _f] else: return [_f for _f in [ self.site.config['TRANSLATIONS'][lang], - self.site.config['TAG_PATH'], - self.slugify_tag_name(name) + ".html"] if _f] + self.site.config['TAG_PATH'][lang], + self.slugify_tag_name(name, lang) + ".html"] if _f] def tag_atom_path(self, name, lang): - """Return path to a tag Atom feed.""" + """A link to a tag's Atom feed. + + Example: + + link://tag_atom/cats => /tags/cats.atom + """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['TAG_PATH'], self.slugify_tag_name(name) + ".atom"] if + self.site.config['TAG_PATH'][lang], self.slugify_tag_name(name, lang) + ".atom"] if _f] def tag_rss_path(self, name, lang): - """Return path to a tag RSS feed.""" + """A link to a tag's RSS feed. + + Example: + + link://tag_rss/cats => /tags/cats.xml + """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['TAG_PATH'], self.slugify_tag_name(name) + ".xml"] if + self.site.config['TAG_PATH'][lang], self.slugify_tag_name(name, lang) + ".xml"] if _f] - def slugify_category_name(self, name): + def slugify_category_name(self, name, lang): """Slugify a category name.""" + if lang is None: # TODO: remove in v8 + utils.LOGGER.warn("RenderTags.slugify_category_name() called without language!") + lang = '' path = self.site.parse_category_name(name) if self.site.config['CATEGORY_OUTPUT_FLAT_HIERARCHY']: path = path[-1:] # only the leaf - result = [self.slugify_tag_name(part) for part in path] + result = [self.slugify_tag_name(part, lang) for part in path] result[0] = self.site.config['CATEGORY_PREFIX'] + result[0] if not self.site.config['PRETTY_URLS']: result = ['-'.join(result)] @@ -417,24 +482,39 @@ class RenderTags(Task): return path def category_path(self, name, lang): - """Return path to a category.""" + """A link to a category. + + Example: + + link://category/dogs => /categories/dogs.html + """ if self.site.config['PRETTY_URLS']: return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['CATEGORY_PATH']] if - _f] + self.slugify_category_name(name) + [self.site.config['INDEX_FILE']] + self.site.config['CATEGORY_PATH'][lang]] if + _f] + self.slugify_category_name(name, lang) + [self.site.config['INDEX_FILE']] else: return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['CATEGORY_PATH']] if - _f] + self._add_extension(self.slugify_category_name(name), ".html") + self.site.config['CATEGORY_PATH'][lang]] if + _f] + self._add_extension(self.slugify_category_name(name, lang), ".html") def category_atom_path(self, name, lang): - """Return path to a category Atom feed.""" + """A link to a category's Atom feed. + + Example: + + link://category_atom/dogs => /categories/dogs.atom + """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['CATEGORY_PATH']] if - _f] + self._add_extension(self.slugify_category_name(name), ".atom") + self.site.config['CATEGORY_PATH'][lang]] if + _f] + self._add_extension(self.slugify_category_name(name, lang), ".atom") def category_rss_path(self, name, lang): - """Return path to a category RSS feed.""" + """A link to a category's RSS feed. + + Example: + + link://category_rss/dogs => /categories/dogs.xml + """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['CATEGORY_PATH']] if - _f] + self._add_extension(self.slugify_category_name(name), ".xml") + self.site.config['CATEGORY_PATH'][lang]] if + _f] + self._add_extension(self.slugify_category_name(name, lang), ".xml") diff --git a/nikola/plugins/template/__init__.py b/nikola/plugins/template/__init__.py index d416ad7..d5efd61 100644 --- a/nikola/plugins/template/__init__.py +++ b/nikola/plugins/template/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/template/jinja.plugin b/nikola/plugins/template/jinja.plugin index cfe9fa8..78fd41b 100644 --- a/nikola/plugins/template/jinja.plugin +++ b/nikola/plugins/template/jinja.plugin @@ -5,7 +5,7 @@ module = jinja [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Support for Jinja2 templates. [Nikola] diff --git a/nikola/plugins/template/jinja.py b/nikola/plugins/template/jinja.py index b02d75c..5a2135f 100644 --- a/nikola/plugins/template/jinja.py +++ b/nikola/plugins/template/jinja.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,8 +29,8 @@ from __future__ import unicode_literals import os +import io import json -from collections import deque try: import jinja2 from jinja2 import meta @@ -42,29 +42,32 @@ from nikola.utils import makedirs, req_missing class JinjaTemplates(TemplateSystem): - """Support for Jinja2 templates.""" name = "jinja" lookup = None dependency_cache = {} + per_file_cache = {} def __init__(self): """Initialize Jinja2 environment with extended set of filters.""" if jinja2 is None: return - self.lookup = jinja2.Environment() + + def set_directories(self, directories, cache_folder): + """Create a new template lookup with set directories.""" + if jinja2 is None: + req_missing(['jinja2'], 'use this theme') + cache_folder = os.path.join(cache_folder, 'jinja') + makedirs(cache_folder) + cache = jinja2.FileSystemBytecodeCache(cache_folder) + self.lookup = jinja2.Environment(bytecode_cache=cache) self.lookup.trim_blocks = True self.lookup.lstrip_blocks = True self.lookup.filters['tojson'] = json.dumps self.lookup.globals['enumerate'] = enumerate self.lookup.globals['isinstance'] = isinstance self.lookup.globals['tuple'] = tuple - - def set_directories(self, directories, cache_folder): - """Create a new template lookup with set directories.""" - if jinja2 is None: - req_missing(['jinja2'], 'use this theme') self.directories = directories self.create_lookup() @@ -89,36 +92,46 @@ class JinjaTemplates(TemplateSystem): if jinja2 is None: req_missing(['jinja2'], 'use this theme') template = self.lookup.get_template(template_name) - output = template.render(**context) + data = template.render(**context) if output_name is not None: makedirs(os.path.dirname(output_name)) - with open(output_name, 'w+') as output: - output.write(output.encode('utf8')) - return output + with io.open(output_name, 'w', encoding='utf-8') as output: + output.write(data) + return data def render_template_to_string(self, template, context): """Render template to a string using context.""" return self.lookup.from_string(template).render(**context) + def get_string_deps(self, text): + """Find dependencies for a template string.""" + deps = set([]) + ast = self.lookup.parse(text) + dep_names = meta.find_referenced_templates(ast) + for dep_name in dep_names: + filename = self.lookup.loader.get_source(self.lookup, dep_name)[1] + sub_deps = [filename] + self.get_deps(filename) + self.dependency_cache[dep_name] = sub_deps + deps |= set(sub_deps) + return list(deps) + + def get_deps(self, filename): + """Return paths to dependencies for the template loaded from filename.""" + with io.open(filename, 'r', encoding='utf-8') as fd: + text = fd.read() + return self.get_string_deps(text) + def template_deps(self, template_name): """Generate list of dependencies for a template.""" - # Cache the lists of dependencies for each template name. if self.dependency_cache.get(template_name) is None: - # Use a breadth-first search to find all templates this one - # depends on. - queue = deque([template_name]) - visited_templates = set([template_name]) - deps = [] - while len(queue) > 0: - curr = queue.popleft() - source, filename = self.lookup.loader.get_source(self.lookup, - curr)[:2] - deps.append(filename) - ast = self.lookup.parse(source) - dep_names = meta.find_referenced_templates(ast) - for dep_name in dep_names: - if (dep_name not in visited_templates and dep_name is not None): - visited_templates.add(dep_name) - queue.append(dep_name) - self.dependency_cache[template_name] = deps + filename = self.lookup.loader.get_source(self.lookup, template_name)[1] + self.dependency_cache[template_name] = [filename] + self.get_deps(filename) return self.dependency_cache[template_name] + + def get_template_path(self, template_name): + """Get the path to a template or return None.""" + try: + t = self.lookup.get_template(template_name) + return t.filename + except jinja2.TemplateNotFound: + return None diff --git a/nikola/plugins/template/mako.plugin b/nikola/plugins/template/mako.plugin index d256faf..308d291 100644 --- a/nikola/plugins/template/mako.plugin +++ b/nikola/plugins/template/mako.plugin @@ -5,7 +5,7 @@ module = mako [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Support for Mako templates. [Nikola] diff --git a/nikola/plugins/template/mako.py b/nikola/plugins/template/mako.py index aed6596..0c9bb64 100644 --- a/nikola/plugins/template/mako.py +++ b/nikola/plugins/template/mako.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,12 +27,13 @@ """Mako template handler.""" from __future__ import unicode_literals, print_function, absolute_import +import io import os import shutil import sys import tempfile -from mako import util, lexer, parsetree +from mako import exceptions, util, lexer, parsetree from mako.lookup import TemplateLookup from mako.template import Template from markupsafe import Markup # It's ok, Mako requires it @@ -44,7 +45,6 @@ LOGGER = get_logger('mako', STDERR_HANDLER) class MakoTemplates(TemplateSystem): - """Support for Mako templates.""" name = "mako" @@ -55,9 +55,8 @@ class MakoTemplates(TemplateSystem): directories = [] cache_dir = None - def get_deps(self, filename): - """Get dependencies for a template (internal function).""" - text = util.read_file(filename) + def get_string_deps(self, text, filename=None): + """Find dependencies for a template string.""" lex = lexer.Lexer(text=text, filename=filename) lex.parse() @@ -66,8 +65,17 @@ class MakoTemplates(TemplateSystem): keyword = getattr(n, 'keyword', None) if keyword in ["inherit", "namespace"] or isinstance(n, parsetree.IncludeTag): deps.append(n.attributes['file']) + # Some templates will include "foo.tmpl" and we need paths, so normalize them + # using the template lookup + for i, d in enumerate(deps): + deps[i] = self.get_template_path(d) return deps + def get_deps(self, filename): + """Get paths to dependencies for a template.""" + text = util.read_file(filename) + return self.get_string_deps(text, filename) + def set_directories(self, directories, cache_folder): """Create a new template lookup with set directories.""" cache_dir = os.path.join(cache_folder, '.mako.tmp') @@ -109,14 +117,14 @@ class MakoTemplates(TemplateSystem): data = template.render_unicode(**context) if output_name is not None: makedirs(os.path.dirname(output_name)) - with open(output_name, 'w+') as output: + with io.open(output_name, 'w', encoding='utf-8') as output: output.write(data) return data def render_template_to_string(self, template, context): """Render template to a string using context.""" context.update(self.filters) - return Template(template).render(**context) + return Template(template, lookup=self.lookup).render(**context) def template_deps(self, template_name): """Generate list of dependencies for a template.""" @@ -127,10 +135,18 @@ class MakoTemplates(TemplateSystem): dep_filenames = self.get_deps(template.filename) deps = [template.filename] for fname in dep_filenames: - deps += self.template_deps(fname) - self.cache[template_name] = tuple(deps) + deps += [fname] + self.get_deps(fname) + self.cache[template_name] = deps return list(self.cache[template_name]) + def get_template_path(self, template_name): + """Get the path to a template or return None.""" + try: + t = self.lookup.get_template(template_name) + return t.filename + except exceptions.TopLevelLookupException: + return None + def striphtml(text): """Strip HTML tags from text.""" |