diff options
Diffstat (limited to 'nikola/plugins')
68 files changed, 1781 insertions, 845 deletions
diff --git a/nikola/plugins/basic_import.py b/nikola/plugins/basic_import.py index 04f1091..cf98ebc 100644 --- a/nikola/plugins/basic_import.py +++ b/nikola/plugins/basic_import.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -76,8 +76,11 @@ class ImportMixin(object): return channel @staticmethod - def configure_redirections(url_map): + def configure_redirections(url_map, base_dir=''): """Configure redirections from an url_map.""" + index = base_dir + 'index.html' + if index.startswith('/'): + index = index[1:] redirections = [] for k, v in url_map.items(): if not k[-1] == '/': @@ -86,11 +89,10 @@ class ImportMixin(object): # remove the initial "/" because src is a relative file path src = (urlparse(k).path + 'index.html')[1:] dst = (urlparse(v).path) - if src == 'index.html': + if src == index: utils.LOGGER.warn("Can't do a redirect for: {0!r}".format(k)) else: redirections.append((src, dst)) - return redirections def generate_base_site(self): @@ -125,9 +127,12 @@ class ImportMixin(object): def write_content(cls, filename, content, rewrite_html=True): """Write content to file.""" if rewrite_html: - doc = html.document_fromstring(content) - doc.rewrite_links(replacer) - content = html.tostring(doc, encoding='utf8') + try: + doc = html.document_fromstring(content) + doc.rewrite_links(replacer) + content = html.tostring(doc, encoding='utf8') + except etree.ParserError: + content = content.encode('utf-8') else: content = content.encode('utf-8') @@ -135,6 +140,24 @@ class ImportMixin(object): with open(filename, "wb+") as fd: fd.write(content) + @classmethod + def write_post(cls, filename, content, headers, compiler, rewrite_html=True): + """Ask the specified compiler to write the post to disk.""" + if rewrite_html: + try: + doc = html.document_fromstring(content) + doc.rewrite_links(replacer) + content = html.tostring(doc, encoding='utf8') + except etree.ParserError: + pass + if isinstance(content, utils.bytes_str): + content = content.decode('utf-8') + compiler.create_post( + filename, + content=content, + onefile=True, + **headers) + @staticmethod def write_metadata(filename, title, slug, post_date, description, tags, **kwargs): """Write metadata to meta file.""" diff --git a/nikola/plugins/command/__init__.py b/nikola/plugins/command/__init__.py index 2aa5267..62d7086 100644 --- a/nikola/plugins/command/__init__.py +++ b/nikola/plugins/command/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py index e339c06..a82dc3e 100644 --- a/nikola/plugins/command/auto/__init__.py +++ b/nikola/plugins/command/auto/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -43,6 +43,7 @@ except ImportError: import webbrowser from wsgiref.simple_server import make_server import wsgiref.util +import pkg_resources from blinker import signal try: @@ -61,7 +62,6 @@ except ImportError: FileSystemEventHandler = object PatternMatchingEventHandler = object - from nikola.plugin_categories import Command from nikola.utils import dns_sd, req_missing, get_logger, get_theme_path, STDERR_HANDLER LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js') @@ -102,7 +102,7 @@ class CommandAuto(Command): 'long': 'address', 'type': str, 'default': '127.0.0.1', - 'help': 'Address to bind (default: 127.0.0.1 – localhost)', + 'help': 'Address to bind (default: 127.0.0.1 -- localhost)', }, { 'name': 'browser', @@ -143,7 +143,7 @@ class CommandAuto(Command): self.cmd_arguments = ['nikola', 'build'] if self.site.configuration_filename != 'conf.py': - self.cmd_arguments = ['--conf=' + self.site.configuration_filename] + self.cmd_arguments + self.cmd_arguments.append('--conf=' + self.site.configuration_filename) # Run an initial build so we are up-to-date subprocess.call(self.cmd_arguments) @@ -157,7 +157,7 @@ class CommandAuto(Command): # Do not duplicate entries -- otherwise, multiple rebuilds are triggered watched = set([ - 'templates/', 'plugins/', + 'templates/' ] + [get_theme_path(name) for name in self.site.THEMES]) for item in self.site.config['post_pages']: watched.add(os.path.dirname(item[0])) @@ -167,6 +167,10 @@ class CommandAuto(Command): watched.add(item) for item in self.site.config['LISTINGS_FOLDERS']: watched.add(item) + for item in self.site._plugin_places: + watched.add(item) + # Nikola itself (useful for developers) + watched.add(pkg_resources.resource_filename('nikola', '')) out_folder = self.site.config['OUTPUT_FOLDER'] if options and options.get('browser'): @@ -298,8 +302,8 @@ class CommandAuto(Command): mimetype = 'text/html' if uri.endswith('/') else mimetypes.guess_type(uri)[0] or 'application/octet-stream' if os.path.isdir(f_path): - if not f_path.endswith('/'): # Redirect to avoid breakage - start_response('301 Redirect', [('Location', p_uri.path + '/')]) + if not p_uri.path.endswith('/'): # Redirect to avoid breakage + start_response('301 Moved Permanently', [('Location', p_uri.path + '/')]) return [] f_path = os.path.join(f_path, self.site.config['INDEX_FILE']) mimetype = 'text/html' diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py index afd15af..4808fdb 100644 --- a/nikola/plugins/command/bootswatch_theme.py +++ b/nikola/plugins/command/bootswatch_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,6 +36,13 @@ from nikola import utils LOGGER = utils.get_logger('bootswatch_theme', utils.STDERR_HANDLER) +def _check_for_theme(theme, themes): + for t in themes: + if t.endswith(os.sep + theme): + return True + return False + + class CommandBootswatchTheme(Command): """Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" @@ -79,12 +86,12 @@ class CommandBootswatchTheme(Command): version = '' # See if we need bootswatch for bootstrap v2 or v3 - themes = utils.get_theme_chain(parent) - if 'bootstrap3' not in themes and 'bootstrap3-jinja' not in themes: + themes = utils.get_theme_chain(parent, self.site.themes_dirs) + if not _check_for_theme('bootstrap3', themes) and not _check_for_theme('bootstrap3-jinja', themes): version = '2' - elif 'bootstrap' not in themes and 'bootstrap-jinja' not in themes: + elif not _check_for_theme('bootstrap', themes) and not _check_for_theme('bootstrap-jinja', themes): LOGGER.warn('"bootswatch_theme" only makes sense for themes that use bootstrap') - elif 'bootstrap3-gradients' in themes or 'bootstrap3-gradients-jinja' in themes: + elif _check_for_theme('bootstrap3-gradients', themes) or _check_for_theme('bootstrap3-gradients-jinja', themes): LOGGER.warn('"bootswatch_theme" doesn\'t work well with the bootstrap3-gradients family') LOGGER.info("Creating '{0}' theme from '{1}' and '{2}'".format(name, swatch, parent)) diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py index bfc6ee2..0141a6b 100644 --- a/nikola/plugins/command/check.py +++ b/nikola/plugins/command/check.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,6 +32,7 @@ import os import re import sys import time +import logbook try: from urllib import unquote from urlparse import urlparse, urljoin, urldefrag @@ -164,9 +165,9 @@ class CommandCheck(Command): print(self.help()) return False if options['verbose']: - self.logger.level = 1 + self.logger.level = logbook.DEBUG else: - self.logger.level = 4 + self.logger.level = logbook.NOTICE failure = False if options['links']: failure |= self.scan_links(options['find_sources'], options['remote']) @@ -245,7 +246,7 @@ class CommandCheck(Command): target = urldefrag(target)[0] if any([urlparse(target).netloc.endswith(_) for _ in ['example.com', 'example.net', 'example.org']]): - self.logger.info("Not testing example address \"{0}\".".format(target)) + self.logger.debug("Not testing example address \"{0}\".".format(target)) continue # absolute URL to root-relative @@ -274,7 +275,7 @@ class CommandCheck(Command): if self.checked_remote_targets[target] in [301, 308]: self.logger.warn("Remote link PERMANENTLY redirected in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) elif self.checked_remote_targets[target] in [302, 307]: - self.logger.notice("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target])) + self.logger.debug("Remote link temporarily redirected in {0}: {1} [HTTP: {2}]".format(filename, target, self.checked_remote_targets[target])) elif self.checked_remote_targets[target] > 399: self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) continue @@ -302,7 +303,7 @@ class CommandCheck(Command): if redir_status_code in [301, 308]: self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) if redir_status_code in [302, 307]: - self.logger.notice("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + self.logger.debug("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) self.checked_remote_targets[resp.url] = resp.status_code self.checked_remote_targets[target] = redir_status_code else: @@ -343,7 +344,7 @@ class CommandCheck(Command): elif target_filename not in self.existing_targets: if os.path.exists(target_filename): - self.logger.info(u"Good link {0} => {1}".format(target, target_filename)) + self.logger.info("Good link {0} => {1}".format(target, target_filename)) self.existing_targets.add(target_filename) else: rv = True @@ -358,9 +359,9 @@ class CommandCheck(Command): def scan_links(self, find_sources=False, check_remote=False): """Check links on the site.""" - self.logger.info("Checking Links:") - self.logger.info("===============\n") - self.logger.notice("{0} mode".format(self.site.config['URL_TYPE'])) + self.logger.debug("Checking Links:") + self.logger.debug("===============\n") + self.logger.debug("{0} mode".format(self.site.config['URL_TYPE'])) failure = False # Maybe we should just examine all HTML files output_folder = self.site.config['OUTPUT_FOLDER'] @@ -380,14 +381,14 @@ class CommandCheck(Command): if self.analyze(fname, find_sources, False): failure = True if not failure: - self.logger.info("All links checked.") + self.logger.debug("All links checked.") return failure def scan_files(self): """Check files in the site, find missing and orphaned files.""" failure = False - self.logger.info("Checking Files:") - self.logger.info("===============\n") + self.logger.debug("Checking Files:") + self.logger.debug("===============\n") only_on_output, only_on_input = real_scan_files(self.site, self.cache) # Ignore folders @@ -406,16 +407,18 @@ class CommandCheck(Command): for f in only_on_input: self.logger.warn(f) if not failure: - self.logger.info("All files checked.") + self.logger.debug("All files checked.") return failure def clean_files(self): """Remove orphaned files.""" only_on_output, _ = real_scan_files(self.site, self.cache) for f in only_on_output: - self.logger.info('removed: {0}'.format(f)) + self.logger.debug('removed: {0}'.format(f)) os.unlink(f) + warn_flag = bool(only_on_output) + # Find empty directories and remove them output_folder = self.site.config['OUTPUT_FOLDER'] all_dirs = [] @@ -425,7 +428,12 @@ class CommandCheck(Command): for d in all_dirs: try: os.rmdir(d) - self.logger.info('removed: {0}/'.format(d)) + self.logger.debug('removed: {0}/'.format(d)) + warn_flag = True except OSError: pass + + if warn_flag: + self.logger.warn('Some files or directories have been removed, your site may need rebuilding') + return True diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py index 3d3daab..c6a8376 100644 --- a/nikola/plugins/command/console.py +++ b/nikola/plugins/command/console.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Chris Warrick, Roberto Alsina and others. +# Copyright © 2012-2016 Chris Warrick, Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -75,7 +75,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f ] def ipython(self, willful=True): - """IPython shell.""" + """Run an IPython shell.""" try: import IPython except ImportError as e: @@ -84,12 +84,13 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f raise e # That’s how _execute knows whether to try something else. else: site = self.context['site'] # NOQA + nikola_site = self.context['site'] # NOQA conf = self.context['conf'] # NOQA commands = self.context['commands'] # NOQA IPython.embed(header=self.header.format('IPython')) def bpython(self, willful=True): - """bpython shell.""" + """Run a bpython shell.""" try: import bpython except ImportError as e: @@ -100,7 +101,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f bpython.embed(banner=self.header.format('bpython'), locals_=self.context) def plain(self, willful=True): - """Plain Python shell.""" + """Run a plain Python shell.""" import code try: import readline @@ -130,6 +131,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f self.context = { 'conf': self.site.config, 'site': self.site, + 'nikola_site': self.site, 'commands': self.site.commands, } if options['bpython']: diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py index 757c0d2..c2289e8 100644 --- a/nikola/plugins/command/deploy.py +++ b/nikola/plugins/command/deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -30,6 +30,7 @@ from __future__ import print_function import io from datetime import datetime from dateutil.tz import gettz +import dateutil import os import subprocess import time @@ -37,7 +38,7 @@ import time from blinker import signal from nikola.plugin_categories import Command -from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER +from nikola.utils import get_logger, clean_before_deployment, STDERR_HANDLER class CommandDeploy(Command): @@ -45,7 +46,7 @@ class CommandDeploy(Command): name = "deploy" - doc_usage = "[[preset [preset...]]" + doc_usage = "[preset [preset...]]" doc_purpose = "deploy the site" doc_description = "Deploy the site by executing deploy commands from the presets listed on the command line. If no presets are specified, `default` is executed." logger = None @@ -55,27 +56,42 @@ class CommandDeploy(Command): self.logger = get_logger('deploy', STDERR_HANDLER) # Get last successful deploy date timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy') + + # Get last-deploy from persistent state + last_deploy = self.site.state.get('last_deploy') + if last_deploy is None: + # If there is a last-deploy saved, move it to the new state persistence thing + # FIXME: remove in Nikola 8 + if os.path.isfile(timestamp_path): + try: + with io.open(timestamp_path, 'r', encoding='utf8') as inf: + last_deploy = dateutil.parser.parse(inf.read()) + clean = False + except (IOError, Exception) as e: + self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) + last_deploy = datetime(1970, 1, 1) + clean = True + os.unlink(timestamp_path) # Remove because from now on it's in state + else: # Just a default + last_deploy = datetime(1970, 1, 1) + clean = True + else: + last_deploy = dateutil.parser.parse(last_deploy) + clean = False + if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo': self.logger.warn("\nWARNING WARNING WARNING WARNING\n" "You are deploying using the nikolademo Disqus account.\n" "That means you will not be able to moderate the comments in your own site.\n" "And is probably not what you want to do.\n" - "Think about it for 5 seconds, I'll wait :-)\n\n") + "Think about it for 5 seconds, I'll wait :-)\n" + "(press Ctrl+C to abort)\n") time.sleep(5) - deploy_drafts = self.site.config.get('DEPLOY_DRAFTS', True) - deploy_future = self.site.config.get('DEPLOY_FUTURE', False) - undeployed_posts = [] - if not (deploy_drafts and deploy_future): - # Remove drafts and future posts - out_dir = self.site.config['OUTPUT_FOLDER'] - self.site.scan_posts() - for post in self.site.timeline: - if (not deploy_drafts and post.is_draft) or \ - (not deploy_future and post.publish_later): - remove_file(os.path.join(out_dir, post.destination_path())) - remove_file(os.path.join(out_dir, post.source_path)) - undeployed_posts.append(post) + # Remove drafts and future posts if requested + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) if args: presets = args @@ -97,27 +113,22 @@ class CommandDeploy(Command): try: subprocess.check_call(command, shell=True) except subprocess.CalledProcessError as e: - self.logger.error('Failed deployment — command {0} ' + self.logger.error('Failed deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode)) return e.returncode self.logger.info("Successful deployment") - try: - with io.open(timestamp_path, 'r', encoding='utf8') as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - clean = False - except (IOError, Exception) as e: - self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) - last_deploy = datetime(1970, 1, 1) - clean = True new_deploy = datetime.utcnow() self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts) - makedirs(self.site.config['CACHE_FOLDER']) # Store timestamp of successful deployment - with io.open(timestamp_path, 'w+', encoding='utf8') as outf: - outf.write(unicode_str(new_deploy.isoformat())) + self.site.state.set('last_deploy', new_deploy.isoformat()) + if clean: + self.logger.info( + 'Looks like this is the first time you deployed this site. ' + 'Let us know you are using Nikola ' + 'at <https://users.getnikola.com/add/> if you want!') def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None): """Emit events for all timeline entries newer than last deploy. diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py index 2fe0d4e..b5ad322 100644 --- a/nikola/plugins/command/github_deploy.py +++ b/nikola/plugins/command/github_deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Puneeth Chaganti and others. +# Copyright © 2014-2016 Puneeth Chaganti and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,15 +27,13 @@ """Deploy site to GitHub Pages.""" from __future__ import print_function -from datetime import datetime -import io import os import subprocess from textwrap import dedent from nikola.plugin_categories import Command from nikola.plugins.command.check import real_scan_files -from nikola.utils import get_logger, req_missing, makedirs, unicode_str, STDERR_HANDLER +from nikola.utils import get_logger, req_missing, clean_before_deployment, STDERR_HANDLER from nikola.__main__ import main from nikola import __version__ @@ -53,7 +51,7 @@ def check_ghp_import_installed(): except OSError: # req_missing defaults to `python=True` — and it’s meant to be like this. # `ghp-import` is installed via pip, but the only way to use it is by executing the script it installs. - req_missing(['ghp-import'], 'deploy the site to GitHub Pages') + req_missing(['ghp-import2'], 'deploy the site to GitHub Pages') class CommandGitHubDeploy(Command): @@ -61,7 +59,7 @@ class CommandGitHubDeploy(Command): name = 'github_deploy' - doc_usage = '' + doc_usage = '[-m COMMIT_MESSAGE]' doc_purpose = 'deploy the site to GitHub Pages' doc_description = dedent( """\ @@ -71,10 +69,19 @@ class CommandGitHubDeploy(Command): """ ) - + cmd_options = [ + { + 'name': 'commit_message', + 'short': 'm', + 'long': 'message', + 'default': 'Nikola auto commit.', + 'type': str, + 'help': 'Commit message (default: Nikola auto commit.)', + }, + ] logger = None - def _execute(self, command, args): + def _execute(self, options, args): """Run the deployment.""" self.logger = get_logger(CommandGitHubDeploy.name, STDERR_HANDLER) @@ -92,41 +99,69 @@ class CommandGitHubDeploy(Command): for f in only_on_output: os.unlink(f) + # Remove drafts and future posts if requested (Issue #2406) + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) + # Commit and push - self._commit_and_push() + self._commit_and_push(options['commit_message']) return - def _commit_and_push(self): - """Commit all the files and push.""" - source = self.site.config['GITHUB_SOURCE_BRANCH'] - deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] - remote = self.site.config['GITHUB_REMOTE_NAME'] - source_commit = uni_check_output(['git', 'rev-parse', source]) - commit_message = ( - 'Nikola auto commit.\n\n' - 'Source commit: %s' - 'Nikola version: %s' % (source_commit, __version__) - ) - output_folder = self.site.config['OUTPUT_FOLDER'] - - command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] - + def _run_command(self, command, xfail=False): + """Run a command that may or may not fail.""" self.logger.info("==> {0}".format(command)) try: subprocess.check_call(command) + return 0 except subprocess.CalledProcessError as e: + if xfail: + return e.returncode self.logger.error( - 'Failed GitHub deployment — command {0} ' + 'Failed GitHub deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode) ) - return e.returncode + raise SystemError(e.returncode) - self.logger.info("Successful deployment") + def _commit_and_push(self, commit_first_line): + """Commit all the files and push.""" + source = self.site.config['GITHUB_SOURCE_BRANCH'] + deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] + remote = self.site.config['GITHUB_REMOTE_NAME'] + autocommit = self.site.config['GITHUB_COMMIT_SOURCE'] + try: + if autocommit: + commit_message = ( + '{0}\n\n' + 'Nikola version: {1}'.format(commit_first_line, __version__) + ) + e = self._run_command(['git', 'checkout', source], True) + if e != 0: + self._run_command(['git', 'checkout', '-b', source]) + self._run_command(['git', 'add', '.']) + # Figure out if there is anything to commit + e = self._run_command(['git', 'diff-index', '--quiet', 'HEAD'], True) + if e != 0: + self._run_command(['git', 'commit', '-am', commit_message]) + else: + self.logger.notice('Nothing to commit to source branch.') + + source_commit = uni_check_output(['git', 'rev-parse', source]) + commit_message = ( + '{0}\n\n' + 'Source commit: {1}' + 'Nikola version: {2}'.format(commit_first_line, source_commit, __version__) + ) + output_folder = self.site.config['OUTPUT_FOLDER'] - # Store timestamp of successful deployment - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - new_deploy = datetime.utcnow() - makedirs(self.site.config["CACHE_FOLDER"]) - with io.open(timestamp_path, "w+", encoding="utf8") as outf: - outf.write(unicode_str(new_deploy.isoformat())) + command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] + + self._run_command(command) + + if autocommit: + self._run_command(['git', 'push', '-u', remote, source]) + except SystemError as e: + return e.args[0] + + self.logger.info("Successful deployment") diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index 69ef144..0b48583 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,6 +38,11 @@ from lxml import etree from collections import defaultdict try: + import html2text +except: + html2text = None + +try: from urlparse import urlparse from urllib import unquote except ImportError: @@ -170,6 +175,20 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Export comments as .wpcomment files", }, { + 'name': 'html2text', + 'long': 'html2text', + 'default': False, + 'type': bool, + 'help': "Uses html2text (needs to be installed with pip) to transform WordPress posts to MarkDown during import", + }, + { + 'name': 'transform_to_markdown', + 'long': 'transform-to-markdown', + 'default': False, + 'type': bool, + 'help': "Uses WordPress page compiler to transform WordPress posts to HTML and then use html2text to transform them to MarkDown during import", + }, + { 'name': 'transform_to_html', 'long': 'transform-to-html', 'default': False, @@ -191,14 +210,35 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Automatically installs the WordPress page compiler (either locally or in the new site) if required by other options.\nWarning: the compiler is GPL software!", }, { - 'name': 'tag_saniziting_strategy', - 'long': 'tag-saniziting-strategy', + 'name': 'tag_sanitizing_strategy', + 'long': 'tag-sanitizing-strategy', 'default': 'first', 'help': 'lower: Convert all tag and category names to lower case\nfirst: Keep first spelling of tag or category name', }, + { + 'name': 'one_file', + 'long': 'one-file', + 'default': False, + 'type': bool, + 'help': "Save imported posts in the more modern one-file format.", + }, ] all_tags = set([]) + def _get_compiler(self): + """Return whatever compiler we will use.""" + self._find_wordpress_compiler() + if self.wordpress_page_compiler is not None: + return self.wordpress_page_compiler + plugin_info = self.site.plugin_manager.getPluginByName('markdown', 'PageCompiler') + if plugin_info is not None: + if not plugin_info.is_activated: + self.site.plugin_manager.activatePluginByName(plugin_info.name) + plugin_info.plugin_object.set_site(self.site) + return plugin_info.plugin_object + else: + LOGGER.error("Can't find markdown post compiler.") + def _find_wordpress_compiler(self): """Find WordPress compiler plugin.""" if self.wordpress_page_compiler is not None: @@ -223,6 +263,8 @@ class CommandImportWordpress(Command, ImportMixin): 'putting these arguments before the filename if you ' 'are running into problems.'.format(args)) + self.onefile = options.get('one_file', False) + self.import_into_existing_site = False self.url_map = {} self.timezone = None @@ -239,6 +281,9 @@ class CommandImportWordpress(Command, ImportMixin): self.export_categories_as_categories = options.get('export_categories_as_categories', False) self.export_comments = options.get('export_comments', False) + self.html2text = options.get('html2text', False) + self.transform_to_markdown = options.get('transform_to_markdown', False) + self.transform_to_html = options.get('transform_to_html', False) self.use_wordpress_compiler = options.get('use_wordpress_compiler', False) self.install_wordpress_compiler = options.get('install_wordpress_compiler', False) @@ -257,10 +302,18 @@ class CommandImportWordpress(Command, ImportMixin): self.separate_qtranslate_content = options.get('separate_qtranslate_content') self.translations_pattern = options.get('translations_pattern') - if self.transform_to_html and self.use_wordpress_compiler: - LOGGER.warn("It does not make sense to combine --transform-to-html with --use-wordpress-compiler, as the first converts all posts to HTML and the latter option affects zero posts.") + count = (1 if self.html2text else 0) + (1 if self.transform_to_html else 0) + (1 if self.transform_to_markdown else 0) + if count > 1: + LOGGER.error("You can use at most one of the options --html2text, --transform-to-html and --transform-to-markdown.") + return False + if (self.html2text or self.transform_to_html or self.transform_to_markdown) and self.use_wordpress_compiler: + LOGGER.warn("It does not make sense to combine --use-wordpress-compiler with any of --html2text, --transform-to-html and --transform-to-markdown, as the latter convert all posts to HTML and the first option then affects zero posts.") + + if (self.html2text or self.transform_to_markdown) and not html2text: + LOGGER.error("You need to install html2text via 'pip install html2text' before you can use the --html2text and --transform-to-markdown options.") + return False - if self.transform_to_html: + if self.transform_to_html or self.transform_to_markdown: self._find_wordpress_compiler() if not self.wordpress_page_compiler and self.install_wordpress_compiler: if not install_plugin(self.site, 'wordpress_compiler', output_dir='plugins'): # local install @@ -334,7 +387,7 @@ class CommandImportWordpress(Command, ImportMixin): self.context['TRANSLATIONS'] = format_default_translations_config( self.extra_languages) self.context['REDIRECTIONS'] = self.configure_redirections( - self.url_map) + self.url_map, self.base_dir) if self.timezone: self.context['TIMEZONE'] = self.timezone if self.export_categories_as_categories: @@ -350,7 +403,7 @@ class CommandImportWordpress(Command, ImportMixin): tag_str = tag except AttributeError: tag_str = tag - tag = utils.slugify(tag_str) + tag = utils.slugify(tag_str, self.lang) src_url = '{}tag/{}'.format(self.context['SITE_URL'], tag) dst_url = self.site.link('tag', tag) if src_url != dst_url: @@ -382,7 +435,7 @@ class CommandImportWordpress(Command, ImportMixin): if b'<atom:link rel=' in line: continue xml.append(line) - return b'\n'.join(xml) + return b''.join(xml) @classmethod def get_channel_from_file(cls, filename): @@ -396,7 +449,8 @@ class CommandImportWordpress(Command, ImportMixin): wordpress_namespace = channel.nsmap['wp'] context = SAMPLE_CONF.copy() - context['DEFAULT_LANG'] = get_text_tag(channel, 'language', 'en')[:2] + self.lang = get_text_tag(channel, 'language', 'en')[:2] + context['DEFAULT_LANG'] = self.lang context['TRANSLATIONS_PATTERN'] = DEFAULT_TRANSLATIONS_PATTERN context['BLOG_TITLE'] = get_text_tag(channel, 'title', 'PUT TITLE HERE') @@ -428,7 +482,7 @@ class CommandImportWordpress(Command, ImportMixin): PAGES = '(\n' for extension in extensions: POSTS += ' ("posts/*.{0}", "posts", "post.tmpl"),\n'.format(extension) - PAGES += ' ("stories/*.{0}", "stories", "story.tmpl"),\n'.format(extension) + PAGES += ' ("pages/*.{0}", "pages", "story.tmpl"),\n'.format(extension) POSTS += ')\n' PAGES += ')\n' context['POSTS'] = POSTS @@ -446,9 +500,6 @@ class CommandImportWordpress(Command, ImportMixin): def download_url_content_to_file(self, url, dst_path): """Download some content (attachments) to a file.""" - if self.no_downloads: - return - try: request = requests.get(url, auth=self.auth) if request.status_code >= 400: @@ -468,10 +519,13 @@ class CommandImportWordpress(Command, ImportMixin): 'foo') path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[link] = '/' + dst_url links[url] = '/' + dst_url @@ -517,6 +571,8 @@ class CommandImportWordpress(Command, ImportMixin): if meta_key in metadata: image_meta = metadata[meta_key] + if not image_meta: + continue dst_meta = {} def add(our_key, wp_key, is_int=False, ignore_zero=False, is_float=False): @@ -562,15 +618,18 @@ class CommandImportWordpress(Command, ImportMixin): meta = {} meta['size'] = size.decode('utf-8') if width_key in metadata[size_key][size] and height_key in metadata[size_key][size]: - meta['width'] = metadata[size_key][size][width_key] - meta['height'] = metadata[size_key][size][height_key] + meta['width'] = int(metadata[size_key][size][width_key]) + meta['height'] = int(metadata[size_key][size][height_key]) path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[url] = '/' + dst_url @@ -638,10 +697,10 @@ class CommandImportWordpress(Command, ImportMixin): return content @staticmethod - def transform_caption(content): + def transform_caption(content, use_html=False): """Transform captions.""" - new_caption = re.sub(r'\[/caption\]', '', content) - new_caption = re.sub(r'\[caption.*\]', '', new_caption) + new_caption = re.sub(r'\[/caption\]', '</h1>' if use_html else '', content) + new_caption = re.sub(r'\[caption.*\]', '<h1>' if use_html else '', new_caption) return new_caption @@ -664,6 +723,26 @@ class CommandImportWordpress(Command, ImportMixin): except TypeError: # old versions of the plugin don't support the additional argument content = self.wordpress_page_compiler.compile_to_string(content) return content, 'html', True + elif self.transform_to_markdown: + # First convert to HTML with WordPress plugin + additional_data = {} + if attachments is not None: + additional_data['attachments'] = attachments + try: + content = self.wordpress_page_compiler.compile_to_string(content, additional_data=additional_data) + except TypeError: # old versions of the plugin don't support the additional argument + content = self.wordpress_page_compiler.compile_to_string(content) + # Now convert to MarkDown with html2text + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False + elif self.html2text: + # TODO: what to do with [code] blocks? + # content = self.transform_code(content) + content = self.transform_caption(content, use_html=True) + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False elif self.use_wordpress_compiler: return content, 'wp', False else: @@ -781,6 +860,12 @@ class CommandImportWordpress(Command, ImportMixin): out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') + + # titles can have line breaks in them, particularly when they are + # created by third-party tools that post to Wordpress. + # Handle windows-style and unix-style line endings. + title = title.replace('\r\n', ' ').replace('\n', ' ') + # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) @@ -813,7 +898,7 @@ class CommandImportWordpress(Command, ImportMixin): else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) - slug = utils.slugify(pathlist[-1]) + slug = utils.slugify(pathlist[-1], self.lang) description = get_text_tag(item, 'description', '') post_date = get_text_tag( @@ -928,14 +1013,32 @@ class CommandImportWordpress(Command, ImportMixin): meta_slug = slug tags, other_meta = self._create_metadata(status, excerpt, tags, categories, post_name=os.path.join(out_folder, slug)) - self.write_metadata(os.path.join(self.output_folder, out_folder, - out_meta_filename), - title, meta_slug, post_date, description, tags, **other_meta) - self.write_content( - os.path.join(self.output_folder, - out_folder, out_content_filename), - content, - rewrite_html) + + meta = { + "title": title, + "slug": meta_slug, + "date": post_date, + "description": description, + "tags": ','.join(tags), + } + meta.update(other_meta) + if self.onefile: + self.write_post( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + meta, + self._get_compiler(), + rewrite_html) + else: + self.write_metadata(os.path.join(self.output_folder, out_folder, + out_meta_filename), + title, meta_slug, post_date, description, tags, **other_meta) + self.write_content( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + rewrite_html) if self.export_comments: comments = [] @@ -995,7 +1098,7 @@ class CommandImportWordpress(Command, ImportMixin): if post_type == 'post': out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'posts', attachments) else: - out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'stories', attachments) + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'pages', attachments) # Process attachment data if attachments is not None: # If post was exported, store data diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py index 2dbee43..3d6669c 100644 --- a/nikola/plugins/command/init.py +++ b/nikola/plugins/command/init.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -75,10 +75,12 @@ SAMPLE_CONF = { 'POSTS': """( ("posts/*.rst", "posts", "post.tmpl"), ("posts/*.txt", "posts", "post.tmpl"), + ("posts/*.html", "posts", "post.tmpl"), )""", 'PAGES': """( - ("stories/*.rst", "stories", "story.tmpl"), - ("stories/*.txt", "stories", "story.tmpl"), + ("pages/*.rst", "pages", "story.tmpl"), + ("pages/*.txt", "pages", "story.tmpl"), + ("pages/*.html", "pages", "story.tmpl"), )""", 'COMPILERS': """{ "rest": ('.rst', '.txt'), @@ -219,6 +221,18 @@ def prepare_config(config): return p +def test_destination(destination, demo=False): + """Check if the destination already exists, which can break demo site creation.""" + # Issue #2214 + if demo and os.path.exists(destination): + LOGGER.warning("The directory {0} already exists, and a new demo site cannot be initialized in an existing directory.".format(destination)) + LOGGER.warning("Please remove the directory and try again, or use another directory.") + LOGGER.info("Hint: If you want to initialize a git repository in this directory, run `git init` in the directory after creating a Nikola site.") + return False + else: + return True + + class CommandInit(Command): """Create a new site.""" @@ -271,11 +285,11 @@ class CommandInit(Command): @classmethod def create_empty_site(cls, target): """Create an empty site with directories only.""" - for folder in ('files', 'galleries', 'listings', 'posts', 'stories'): + for folder in ('files', 'galleries', 'listings', 'posts', 'pages'): makedirs(os.path.join(target, folder)) @staticmethod - def ask_questions(target): + def ask_questions(target, demo=False): """Ask some questions about Nikola.""" def urlhandler(default, toconf): answer = ask('Site URL', 'https://example.com/') @@ -346,7 +360,7 @@ class CommandInit(Command): # Assuming that base contains all the locales, and that base does # not inherit from anywhere. try: - messages = load_messages(['base'], tr, default) + messages = load_messages(['base'], tr, default, themes_dirs=['themes']) SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages, SAMPLE_CONF['STRIP_INDEXES']) except nikola.utils.LanguageNotFoundError as e: print(" ERROR: the language '{0}' is not supported.".format(e.lang)) @@ -440,7 +454,7 @@ class CommandInit(Command): print("If you do not want to answer and want to go with the defaults instead, simply restart with the `-q` parameter.") for query, default, toconf, destination in questions: - if target and destination == '!target': + if target and destination == '!target' and test_destination(target, demo): # Skip the destination question if we know it already pass else: @@ -457,8 +471,9 @@ class CommandInit(Command): if toconf: SAMPLE_CONF[destination] = answer if destination == '!target': - while not answer: - print(' ERROR: you need to specify a target directory.\n') + while not answer or not test_destination(answer, demo): + if not answer: + print(' ERROR: you need to specify a target directory.\n') answer = ask(query, default) STORAGE['target'] = answer @@ -474,7 +489,7 @@ class CommandInit(Command): except IndexError: target = None if not options.get('quiet'): - st = self.ask_questions(target=target) + st = self.ask_questions(target=target, demo=options.get('demo')) try: if not target: target = st['target'] @@ -487,11 +502,13 @@ class CommandInit(Command): Options: -q, --quiet Do not ask questions about config. -d, --demo Create a site filled with example data.""") - return False + return 1 if not options.get('demo'): self.create_empty_site(target) LOGGER.info('Created empty site at {0}.'.format(target)) else: + if not test_destination(target, True): + return 2 self.copy_sample_site(target) LOGGER.info("A new site with example data has been created at " "{0}.".format(target)) diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py index bad335c..28f7aa3 100644 --- a/nikola/plugins/command/install_theme.py +++ b/nikola/plugins/command/install_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,18 +27,9 @@ """Install a theme.""" from __future__ import print_function -import os -import io -import time -import requests -import pygments -from pygments.lexers import PythonLexer -from pygments.formatters import TerminalFormatter - -from nikola.plugin_categories import Command from nikola import utils - +from nikola.plugin_categories import Command LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER) @@ -79,6 +70,7 @@ class CommandInstallTheme(Command): def _execute(self, options, args): """Install theme into current site.""" + p = self.site.plugin_manager.getPluginByName('theme', 'Command').plugin_object listing = options['list'] url = options['url'] if args: @@ -87,85 +79,13 @@ class CommandInstallTheme(Command): name = None if options['getpath'] and name: - path = utils.get_theme_path(name) - if path: - print(path) - else: - print('not installed') - return 0 + return p.get_path(name) if name is None and not listing: LOGGER.error("This command needs either a theme name or the -l option.") return False - try: - data = requests.get(url).json() - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - data = requests.get(url).json() - if listing: - print("Themes:") - print("-------") - for theme in sorted(data.keys()): - print(theme) - return True - else: - # `name` may be modified by the while loop. - origname = name - installstatus = self.do_install(name, data) - # See if the theme's parent is available. If not, install it - while True: - parent_name = utils.get_parent_theme_name(name) - if parent_name is None: - break - try: - utils.get_theme_path(parent_name) - break - except: # Not available - self.do_install(parent_name, data) - name = parent_name - if installstatus: - LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) - def do_install(self, name, data): - """Download and install a theme.""" - if name in data: - utils.makedirs(self.output_dir) - url = data[name] - LOGGER.info("Downloading '{0}'".format(url)) - try: - zip_data = requests.get(url).content - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - zip_data = requests.get(url).content - - zip_file = io.BytesIO() - zip_file.write(zip_data) - LOGGER.info("Extracting '{0}' into themes/".format(name)) - utils.extract_all(zip_file) - dest_path = os.path.join(self.output_dir, name) + if listing: + p.list_available(url) else: - dest_path = os.path.join(self.output_dir, name) - try: - theme_path = utils.get_theme_path(name) - LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) - except Exception: - LOGGER.error("Can't find theme {0}".format(name)) - - return False - - confpypath = os.path.join(dest_path, 'conf.py.sample') - if os.path.exists(confpypath): - LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') - print('Contents of the conf.py.sample file:\n') - with io.open(confpypath, 'r', encoding='utf-8') as fh: - if self.site.colorful: - print(utils.indent(pygments.highlight( - fh.read(), PythonLexer(), TerminalFormatter()), - 4 * ' ')) - else: - print(utils.indent(fh.read(), 4 * ' ')) - return True + p.do_install_deps(url, name) diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index 8843421..c09b4be 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index 30d009d..36cc04f 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,10 +29,11 @@ from __future__ import unicode_literals, print_function import io import datetime +import operator import os -import sys +import shutil import subprocess -import operator +import sys from blinker import signal import dateutil.tz @@ -293,14 +294,14 @@ class CommandNewPost(Command): title = title.strip() if not path: - slug = utils.slugify(title) + slug = utils.slugify(title, lang=self.site.default_lang) else: if isinstance(path, utils.bytes_str): try: path = path.decode(sys.stdin.encoding) except (AttributeError, TypeError): # for tests path = path.decode('utf-8') - slug = utils.slugify(os.path.splitext(os.path.basename(path))[0]) + slug = utils.slugify(os.path.splitext(os.path.basename(path))[0], lang=self.site.default_lang) if isinstance(author, utils.bytes_str): try: @@ -324,14 +325,17 @@ class CommandNewPost(Command): 'description': '', 'type': 'text', } - output_path = os.path.dirname(entry[0]) - meta_path = os.path.join(output_path, slug + ".meta") - pattern = os.path.basename(entry[0]) - suffix = pattern[1:] + if not path: + pattern = os.path.basename(entry[0]) + suffix = pattern[1:] + output_path = os.path.dirname(entry[0]) + txt_path = os.path.join(output_path, slug + suffix) + meta_path = os.path.join(output_path, slug + ".meta") else: txt_path = os.path.join(self.site.original_cwd, path) + meta_path = os.path.splitext(txt_path)[0] + ".meta" if (not onefile and os.path.isfile(meta_path)) or \ os.path.isfile(txt_path): @@ -343,6 +347,9 @@ class CommandNewPost(Command): signal('existing_' + content_type).send(self, **event) LOGGER.error("The title already exists!") + LOGGER.info("Existing {0}'s text is at: {1}".format(content_type, txt_path)) + if not onefile: + LOGGER.info("Existing {0}'s metadata is at: {1}".format(content_type, meta_path)) return 8 d_name = os.path.dirname(txt_path) @@ -363,17 +370,22 @@ class CommandNewPost(Command): onefile = False LOGGER.warn('This compiler does not support one-file posts.') - if import_file: + if onefile and import_file: with io.open(import_file, 'r', encoding='utf-8') as fh: content = fh.read() - else: + elif not import_file: if is_page: content = self.site.MESSAGES[self.site.default_lang]["Write your page here."] else: content = self.site.MESSAGES[self.site.default_lang]["Write your post here."] - compiler_plugin.create_post( - txt_path, content=content, onefile=onefile, title=title, - slug=slug, date=date, tags=tags, is_page=is_page, **metadata) + + if (not onefile) and import_file: + # Two-file posts are copied on import (Issue #2380) + shutil.copy(import_file, txt_path) + else: + compiler_plugin.create_post( + txt_path, content=content, onefile=onefile, title=title, + slug=slug, date=date, tags=tags, is_page=is_page, **metadata) event = dict(path=txt_path) diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py index f061d39..5e2574d 100644 --- a/nikola/plugins/command/orphans.py +++ b/nikola/plugins/command/orphans.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index 1df7b71..364f343 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,6 +29,7 @@ from __future__ import print_function import io import os +import sys import shutil import subprocess import time @@ -49,7 +50,7 @@ class CommandPlugin(Command): json = None name = "plugin" - doc_usage = "[[-u][--user] --install name] | [[-u] [-l |--upgrade|--list-installed] | [--uninstall name]]" + doc_usage = "[-u url] [--user] [-i name] [-r name] [--upgrade] [-l] [--list-installed]" doc_purpose = "manage plugins" output_dir = None needs_config = False @@ -176,8 +177,11 @@ class CommandPlugin(Command): plugins.append([plugin.name, p]) plugins.sort() + print('Installed Plugins:') + print('------------------') for name, path in plugins: print('{0} at {1}'.format(name, path)) + print('\n\nAlso, you have disabled these plugins: {}'.format(self.site.config['DISABLED_PLUGINS'])) return 0 def do_upgrade(self, url): @@ -251,7 +255,7 @@ class CommandPlugin(Command): LOGGER.notice('This plugin has Python dependencies.') LOGGER.info('Installing dependencies with pip...') try: - subprocess.check_call(('pip', 'install', '-r', reqpath)) + subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath)) except subprocess.CalledProcessError: LOGGER.error('Could not install the dependencies.') print('Contents of the requirements.txt file:\n') @@ -292,12 +296,15 @@ class CommandPlugin(Command): def do_uninstall(self, name): """Uninstall a plugin.""" for plugin in self.site.plugin_manager.getAllPlugins(): # FIXME: this is repeated thrice - p = plugin.path - if os.path.isdir(p): - p = p + os.sep - else: - p = os.path.dirname(p) if name == plugin.name: # Uninstall this one + p = plugin.path + if os.path.isdir(p): + # Plugins that have a package in them need to delete parent + # Issue #2356 + p = p + os.sep + p = os.path.abspath(os.path.join(p, os.pardir)) + else: + p = os.path.dirname(p) LOGGER.warning('About to uninstall plugin: {0}'.format(name)) LOGGER.warning('This will delete {0}'.format(p)) sure = utils.ask_yesno('Are you sure?') diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py index 68ea13d..c877f63 100644 --- a/nikola/plugins/command/rst2html/__init__.py +++ b/nikola/plugins/command/rst2html/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2015 Chris Warrick and others. +# Copyright © 2015-2016 Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py index b647bb7..c9702d5 100644 --- a/nikola/plugins/command/serve.py +++ b/nikola/plugins/command/serve.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -78,7 +78,7 @@ class CommandServe(Command): 'long': 'address', 'type': str, 'default': '', - 'help': 'Address to bind (default: 0.0.0.0 – all local IPv4 interfaces)', + 'help': 'Address to bind (default: 0.0.0.0 -- all local IPv4 interfaces)', }, { 'name': 'detach', diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py index 40f4f77..b3ffbb4 100644 --- a/nikola/plugins/command/status.py +++ b/nikola/plugins/command/status.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,7 +27,6 @@ """Display site status.""" from __future__ import print_function -import io import os from datetime import datetime from dateutil.tz import gettz, tzlocal @@ -42,7 +41,7 @@ class CommandStatus(Command): doc_purpose = "display site status" doc_description = "Show information about the posts and site deployment." - doc_usage = '[-l|--list-drafts] [-m|--list-modified] [-s|--list-scheduled]' + doc_usage = '[-d|--list-drafts] [-m|--list-modified] [-p|--list-private] [-P|--list-published] [-s|--list-scheduled]' logger = None cmd_options = [ { @@ -62,6 +61,22 @@ class CommandStatus(Command): 'help': 'List all modified files since last deployment', }, { + 'name': 'list_private', + 'short': 'p', + 'long': 'list-private', + 'type': bool, + 'default': False, + 'help': 'List all private posts', + }, + { + 'name': 'list_published', + 'short': 'P', + 'long': 'list-published', + 'type': bool, + 'default': False, + 'help': 'List all published posts', + }, + { 'name': 'list_scheduled', 'short': 's', 'long': 'list-scheduled', @@ -75,16 +90,12 @@ class CommandStatus(Command): """Display site status.""" self.site.scan_posts() - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - - last_deploy = None - - try: - with io.open(timestamp_path, "r", encoding="utf8") as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - last_deploy_offset = datetime.utcnow() - last_deploy - except (IOError, Exception): - print("It does not seem like you’ve ever deployed the site (or cache missing).") + last_deploy = self.site.state.get('last_deploy') + if last_deploy is not None: + last_deploy = datetime.strptime(last_deploy, "%Y-%m-%dT%H:%M:%S.%f") + last_deploy_offset = datetime.utcnow() - last_deploy + else: + print("It does not seem like you've ever deployed the site (or cache missing).") if last_deploy: @@ -110,12 +121,23 @@ class CommandStatus(Command): posts_count = len(self.site.all_posts) + # find all published posts + posts_published = [post for post in self.site.all_posts if post.use_in_feeds] + posts_published = sorted(posts_published, key=lambda post: post.source_path) + + # find all private posts + posts_private = [post for post in self.site.all_posts if post.is_private] + posts_private = sorted(posts_private, key=lambda post: post.source_path) + # find all drafts posts_drafts = [post for post in self.site.all_posts if post.is_draft] posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path) # find all scheduled posts with offset from now until publishing time - posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later] + posts_scheduled = [ + (post.date - now, post) for post in self.site.all_posts + if post.publish_later and not (post.is_draft or post.is_private) + ] posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path)) if len(posts_scheduled) > 0: @@ -128,7 +150,13 @@ class CommandStatus(Command): if options['list_drafts']: for post in posts_drafts: print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) - print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts))) + if options['list_private']: + for post in posts_private: + print("Private: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + if options['list_published']: + for post in posts_published: + print("Published: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + print("{0} posts in total, {1} scheduled, {2} drafts, {3} private and {4} published.".format(posts_count, len(posts_scheduled), len(posts_drafts), len(posts_private), len(posts_published))) def human_time(self, dt): """Translate time into a human-friendly representation.""" diff --git a/nikola/plugins/command/theme.plugin b/nikola/plugins/command/theme.plugin new file mode 100644 index 0000000..b0c1886 --- /dev/null +++ b/nikola/plugins/command/theme.plugin @@ -0,0 +1,13 @@ +[Core] +name = theme +module = theme + +[Documentation] +author = Roberto Alsina and Chris Warrick +version = 1.0 +website = https://getnikola.com/ +description = Manage Nikola themes + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/theme.py b/nikola/plugins/command/theme.py new file mode 100644 index 0000000..7513491 --- /dev/null +++ b/nikola/plugins/command/theme.py @@ -0,0 +1,365 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Manage themes.""" + +from __future__ import print_function +import os +import io +import shutil +import time +import requests + +import pygments +from pygments.lexers import PythonLexer +from pygments.formatters import TerminalFormatter +from pkg_resources import resource_filename + +from nikola.plugin_categories import Command +from nikola import utils + +LOGGER = utils.get_logger('theme', utils.STDERR_HANDLER) + + +class CommandTheme(Command): + """Manage themes.""" + + json = None + name = "theme" + doc_usage = "[-u url] [-i theme_name] [-r theme_name] [-l] [--list-installed] [-g] [-n theme_name] [-c template_name]" + doc_purpose = "manage themes" + output_dir = 'themes' + cmd_options = [ + { + 'name': 'install', + 'short': 'i', + 'long': 'install', + 'type': str, + 'default': '', + 'help': 'Install a theme.' + }, + { + 'name': 'uninstall', + 'long': 'uninstall', + 'short': 'r', + 'type': str, + 'default': '', + 'help': 'Uninstall a theme.' + }, + { + 'name': 'list', + 'short': 'l', + 'long': 'list', + 'type': bool, + 'default': False, + 'help': 'Show list of available themes.' + }, + { + 'name': 'list_installed', + 'long': 'list-installed', + 'type': bool, + 'help': "List the installed themes with their location.", + 'default': False + }, + { + 'name': 'url', + 'short': 'u', + 'long': 'url', + 'type': str, + 'help': "URL for the theme repository (default: " + "https://themes.getnikola.com/v7/themes.json)", + 'default': 'https://themes.getnikola.com/v7/themes.json' + }, + { + 'name': 'getpath', + 'short': 'g', + 'long': 'get-path', + 'type': str, + 'default': '', + 'help': "Print the path for installed theme", + }, + { + 'name': 'copy-template', + 'short': 'c', + 'long': 'copy-template', + 'type': str, + 'default': '', + 'help': 'Copy a built-in template into templates/ or your theme', + }, + { + 'name': 'new', + 'short': 'n', + 'long': 'new', + 'type': str, + 'default': '', + 'help': 'Create a new theme', + }, + { + 'name': 'new_engine', + 'long': 'engine', + 'type': str, + 'default': 'mako', + 'help': 'Engine to use for new theme (mako or jinja -- default: mako)', + }, + { + 'name': 'new_parent', + 'long': 'parent', + 'type': str, + 'default': 'base', + 'help': 'Parent to use for new theme (default: base)', + }, + ] + + def _execute(self, options, args): + """Install theme into current site.""" + url = options['url'] + + # See the "mode" we need to operate in + install = options.get('install') + uninstall = options.get('uninstall') + list_available = options.get('list') + list_installed = options.get('list_installed') + get_path = options.get('getpath') + copy_template = options.get('copy-template') + new = options.get('new') + new_engine = options.get('new_engine') + new_parent = options.get('new_parent') + command_count = [bool(x) for x in ( + install, + uninstall, + list_available, + list_installed, + get_path, + copy_template, + new)].count(True) + if command_count > 1 or command_count == 0: + print(self.help()) + return 2 + + if list_available: + return self.list_available(url) + elif list_installed: + return self.list_installed() + elif install: + return self.do_install_deps(url, install) + elif uninstall: + return self.do_uninstall(uninstall) + elif get_path: + return self.get_path(get_path) + elif copy_template: + return self.copy_template(copy_template) + elif new: + return self.new_theme(new, new_engine, new_parent) + + def do_install_deps(self, url, name): + """Install themes and their dependencies.""" + data = self.get_json(url) + # `name` may be modified by the while loop. + origname = name + installstatus = self.do_install(name, data) + # See if the theme's parent is available. If not, install it + while True: + parent_name = utils.get_parent_theme_name(utils.get_theme_path_real(name, self.site.themes_dirs)) + if parent_name is None: + break + try: + utils.get_theme_path_real(parent_name, self.site.themes_dirs) + break + except: # Not available + self.do_install(parent_name, data) + name = parent_name + if installstatus: + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) + + def do_install(self, name, data): + """Download and install a theme.""" + if name in data: + utils.makedirs(self.output_dir) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + + zip_file = io.BytesIO() + zip_file.write(zip_data) + LOGGER.info("Extracting '{0}' into themes/".format(name)) + utils.extract_all(zip_file) + dest_path = os.path.join(self.output_dir, name) + else: + dest_path = os.path.join(self.output_dir, name) + try: + theme_path = utils.get_theme_path_real(name, self.site.themes_dirs) + LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) + except Exception: + LOGGER.error("Can't find theme {0}".format(name)) + + return False + + confpypath = os.path.join(dest_path, 'conf.py.sample') + if os.path.exists(confpypath): + LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') + print('Contents of the conf.py.sample file:\n') + with io.open(confpypath, 'r', encoding='utf-8') as fh: + if self.site.colorful: + print(utils.indent(pygments.highlight( + fh.read(), PythonLexer(), TerminalFormatter()), + 4 * ' ')) + else: + print(utils.indent(fh.read(), 4 * ' ')) + return True + + def do_uninstall(self, name): + """Uninstall a theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + except Exception: + LOGGER.error('Unknown theme: {0}'.format(name)) + return 1 + # Don't uninstall builtin themes (Issue #2510) + blocked = os.path.dirname(utils.__file__) + if path.startswith(blocked): + LOGGER.error("Can't delete builtin theme: {0}".format(name)) + return 1 + LOGGER.warning('About to uninstall theme: {0}'.format(name)) + LOGGER.warning('This will delete {0}'.format(path)) + sure = utils.ask_yesno('Are you sure?') + if sure: + LOGGER.warning('Removing {0}'.format(path)) + shutil.rmtree(path) + return 0 + return 1 + + def get_path(self, name): + """Get path for an installed theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + print(path) + except Exception: + print("not installed") + return 0 + + def list_available(self, url): + """List all available themes.""" + data = self.get_json(url) + print("Available Themes:") + print("-----------------") + for theme in sorted(data.keys()): + print(theme) + return 0 + + def list_installed(self): + """List all installed themes.""" + print("Installed Themes:") + print("-----------------") + themes = [] + themes_dirs = self.site.themes_dirs + [resource_filename('nikola', os.path.join('data', 'themes'))] + for tdir in themes_dirs: + themes += [(i, os.path.join(tdir, i)) for i in os.listdir(tdir)] + for tname, tpath in sorted(set(themes)): + if os.path.isdir(tpath): + print("{0} at {1}".format(tname, tpath)) + + def copy_template(self, template): + """Copy the named template file from the parent to a local theme or to templates/.""" + # Find template + t = self.site.template_system.get_template_path(template) + if t is None: + LOGGER.error("Cannot find template {0} in the lookup.".format(template)) + return 2 + + # Figure out where to put it. + # Check if a local theme exists. + theme_path = utils.get_theme_path(self.site.THEMES[0]) + if theme_path.startswith('themes' + os.sep): + # Theme in local themes/ directory + base = os.path.join(theme_path, 'templates') + else: + # Put it in templates/ + base = 'templates' + + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + try: + out = shutil.copy(t, base) + LOGGER.info("Copied template from {0} to {1}".format(t, out)) + except shutil.SameFileError: + LOGGER.error("This file already exists in your templates directory ({0}).".format(base)) + return 3 + + def new_theme(self, name, engine, parent): + """Create a new theme.""" + base = 'themes' + themedir = os.path.join(base, name) + LOGGER.info("Creating theme {0} with parent {1} and engine {2} in {3}".format(name, parent, engine, themedir)) + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + # Check if engine and parent match + engine_file = utils.get_asset_path('engine', utils.get_theme_chain(parent, self.site.themes_dirs)) + with io.open(engine_file, 'r', encoding='utf-8') as fh: + parent_engine = fh.read().strip() + + if parent_engine != engine: + LOGGER.error("Cannot use engine {0} because parent theme '{1}' uses {2}".format(engine, parent, parent_engine)) + return 2 + + # Create theme + if not os.path.exists(themedir): + os.mkdir(themedir) + LOGGER.info("Created directory {0}".format(themedir)) + else: + LOGGER.error("Theme already exists") + return 2 + + with io.open(os.path.join(themedir, 'parent'), 'w', encoding='utf-8') as fh: + fh.write(parent + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'parent'))) + with io.open(os.path.join(themedir, 'engine'), 'w', encoding='utf-8') as fh: + fh.write(engine + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'engine'))) + + LOGGER.info("Theme {0} created successfully.".format(themedir)) + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(name)) + + def get_json(self, url): + """Download the JSON file with all plugins.""" + if self.json is None: + try: + self.json = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + self.json = requests.get(url).json() + return self.json diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py index b83d622..267837e 100644 --- a/nikola/plugins/command/version.py +++ b/nikola/plugins/command/version.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/compile/__init__.py b/nikola/plugins/compile/__init__.py index 60f1919..ff7e9a2 100644 --- a/nikola/plugins/compile/__init__.py +++ b/nikola/plugins/compile/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/compile/html.py b/nikola/plugins/compile/html.py index 6ff5de8..942d6da 100644 --- a/nikola/plugins/compile/html.py +++ b/nikola/plugins/compile/html.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -44,12 +44,24 @@ class CompileHtml(PageCompiler): def compile_html(self, source, dest, is_two_file=True): """Compile source file into HTML and save as dest.""" makedirs(os.path.dirname(dest)) + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(dest, "w+", encoding="utf8") as out_file: with io.open(source, "r", encoding="utf8") as in_file: data = in_file.read() if not is_two_file: _, data = self.split_metadata(data) + data, shortcode_deps = self.site.apply_shortcodes(data, with_dependencies=True, extra_context=dict(post=post)) out_file.write(data) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps return True def create_post(self, path, **kw): diff --git a/nikola/plugins/compile/ipynb.py b/nikola/plugins/compile/ipynb.py index 1023b31..f3fdeea 100644 --- a/nikola/plugins/compile/ipynb.py +++ b/nikola/plugins/compile/ipynb.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2013-2015 Damián Avila, Chris Warrick and others. +# Copyright © 2013-2016 Damián Avila, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,21 +32,33 @@ import os import sys try: - import IPython - from IPython.nbconvert.exporters import HTMLExporter - if IPython.version_info[0] >= 3: # API changed with 3.0.0 - from IPython import nbformat - current_nbformat = nbformat.current_nbformat - from IPython.kernel import kernelspec - else: - import IPython.nbformat.current as nbformat - current_nbformat = 'json' - kernelspec = None - - from IPython.config import Config + from nbconvert.exporters import HTMLExporter + import nbformat + current_nbformat = nbformat.current_nbformat + from jupyter_client import kernelspec + from traitlets.config import Config flag = True + ipy_modern = True except ImportError: - flag = None + try: + import IPython + from IPython.nbconvert.exporters import HTMLExporter + if IPython.version_info[0] >= 3: # API changed with 3.0.0 + from IPython import nbformat + current_nbformat = nbformat.current_nbformat + from IPython.kernel import kernelspec + ipy_modern = True + else: + import IPython.nbformat.current as nbformat + current_nbformat = 'json' + kernelspec = None + ipy_modern = False + + from IPython.config import Config + flag = True + except ImportError: + flag = None + ipy_modern = None from nikola.plugin_categories import PageCompiler from nikola.utils import makedirs, req_missing, get_logger, STDERR_HANDLER @@ -69,7 +81,6 @@ class CompileIPynb(PageCompiler): """Export notebooks as HTML strings.""" if flag is None: req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)') - HTMLExporter.default_template = 'basic' c = Config(self.site.config['IPYNB_CONFIG']) exportHtml = HTMLExporter(config=c) with io.open(source, "r", encoding="utf8") as in_file: @@ -80,8 +91,21 @@ class CompileIPynb(PageCompiler): def compile_html(self, source, dest, is_two_file=True): """Compile source file into HTML and save as dest.""" makedirs(os.path.dirname(dest)) + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(dest, "w+", encoding="utf8") as out_file: - out_file.write(self.compile_html_string(source, is_two_file)) + output = self.compile_html_string(source, is_two_file) + output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post)) + out_file.write(output) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps def read_metadata(self, post, file_metadata_regexp=None, unslugify_titles=False, lang=None): """Read metadata directly from ipynb file. @@ -118,7 +142,7 @@ class CompileIPynb(PageCompiler): # imported .ipynb file, guaranteed to start with "{" because it’s JSON. nb = nbformat.reads(content, current_nbformat) else: - if IPython.version_info[0] >= 3: + if ipy_modern: nb = nbformat.v4.new_notebook() nb["cells"] = [nbformat.v4.new_markdown_cell(content)] else: @@ -151,7 +175,7 @@ class CompileIPynb(PageCompiler): nb["metadata"]["nikola"] = metadata with io.open(path, "w+", encoding="utf8") as fd: - if IPython.version_info[0] >= 3: + if ipy_modern: nbformat.write(nb, fd, 4) else: nbformat.write(nb, fd, 'ipynb') diff --git a/nikola/plugins/compile/markdown/__init__.py b/nikola/plugins/compile/markdown/__init__.py index 93438a3..2e4234c 100644 --- a/nikola/plugins/compile/markdown/__init__.py +++ b/nikola/plugins/compile/markdown/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -69,13 +69,25 @@ class CompileMarkdown(PageCompiler): req_missing(['markdown'], 'build this site (compile Markdown)') makedirs(os.path.dirname(dest)) self.extensions += self.site.config.get("MARKDOWN_EXTENSIONS") + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(dest, "w+", encoding="utf8") as out_file: with io.open(source, "r", encoding="utf8") as in_file: data = in_file.read() if not is_two_file: _, data = self.split_metadata(data) - output = markdown(data, self.extensions) + output = markdown(data, self.extensions, output_format="html5") + output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post)) out_file.write(output) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps def create_post(self, path, **kw): """Create a new post.""" diff --git a/nikola/plugins/compile/markdown/mdx_gist.py b/nikola/plugins/compile/markdown/mdx_gist.py index a930de5..25c071f 100644 --- a/nikola/plugins/compile/markdown/mdx_gist.py +++ b/nikola/plugins/compile/markdown/mdx_gist.py @@ -31,161 +31,48 @@ Extension to Python Markdown for Embedded Gists (gist.github.com). Basic Example: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 4747847] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js"></script> - <noscript> - <pre>import this</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: 4747847] Example with filename: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 4747847 zen.py] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js?file=zen.py"></script> - <noscript> - <pre>import this</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: 4747847 zen.py] Basic Example with hexidecimal id: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: c4a43d6fdce612284ac0] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: c4a43d6fdce612284ac0] Example with hexidecimal id filename: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: c4a43d6fdce612284ac0 cow.txt] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js?file=cow.txt"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + [:gist: c4a43d6fdce612284ac0 cow.txt] Example using reStructuredText syntax: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... .. gist:: 4747847 zen.py - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js?file=zen.py"></script> - <noscript> - <pre>import this</pre> - </noscript> - </div> - </p> + Text of the gist: + .. gist:: 4747847 zen.py Example using hexidecimal ID with reStructuredText syntax: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... .. gist:: c4a43d6fdce612284ac0 - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + .. gist:: c4a43d6fdce612284ac0 Example using hexidecimal ID and filename with reStructuredText syntax: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... .. gist:: c4a43d6fdce612284ac0 cow.txt - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/c4a43d6fdce612284ac0.js?file=cow.txt"></script> - <noscript> - <pre>Moo</pre> - </noscript> - </div> - </p> + Text of the gist: + .. gist:: c4a43d6fdce612284ac0 cow.txt Error Case: non-existent Gist ID: - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 0] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/0.js"></script> - <noscript><!-- WARNING: Received a 404 response from Gist URL: https://gist.githubusercontent.com/raw/0 --></noscript> - </div> - </p> - -Error Case: non-existent file: - - >>> import markdown - >>> text = ''' - ... Text of the gist: - ... [:gist: 4747847 doesntexist.py] - ... ''' - >>> html = markdown.markdown(text, [GistExtension()]) - >>> print(html) - <p>Text of the gist: - <div class="gist"> - <script src="https://gist.github.com/4747847.js?file=doesntexist.py"></script> - <noscript><!-- WARNING: Received a 404 response from Gist URL: https://gist.githubusercontent.com/raw/4747847/doesntexist.py --></noscript> - </div> - </p> + Text of the gist: + [:gist: 0] + +Error Case: non-existent file: + + Text of the gist: + [:gist: 4747847 doesntexist.py] """ from __future__ import unicode_literals, print_function diff --git a/nikola/plugins/compile/markdown/mdx_nikola.py b/nikola/plugins/compile/markdown/mdx_nikola.py index 7984121..59a5d5b 100644 --- a/nikola/plugins/compile/markdown/mdx_nikola.py +++ b/nikola/plugins/compile/markdown/mdx_nikola.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/compile/markdown/mdx_podcast.py b/nikola/plugins/compile/markdown/mdx_podcast.py index 0f68e40..96a70ed 100644 --- a/nikola/plugins/compile/markdown/mdx_podcast.py +++ b/nikola/plugins/compile/markdown/mdx_podcast.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright © 2013-2015 Michael Rabbitt, Roberto Alsina and others. +# Copyright © 2013-2016 Michael Rabbitt, Roberto Alsina and others. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the diff --git a/nikola/plugins/compile/pandoc.py b/nikola/plugins/compile/pandoc.py index 85e84fc..2368ae9 100644 --- a/nikola/plugins/compile/pandoc.py +++ b/nikola/plugins/compile/pandoc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -54,7 +54,22 @@ class CompilePandoc(PageCompiler): """Compile source file into HTML and save as dest.""" makedirs(os.path.dirname(dest)) try: + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None subprocess.check_call(['pandoc', '-o', dest, source] + self.site.config['PANDOC_OPTIONS']) + with open(dest, 'r', encoding='utf-8') as inf: + output, shortcode_deps = self.site.apply_shortcodes(inf.read(), with_dependencies=True) + with open(dest, 'w', encoding='utf-8') as outf: + outf.write(output) + if post is None: + if shortcode_deps: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) + else: + post._depfile[dest] += shortcode_deps except OSError as e: if e.strreror == 'No such file or directory': req_missing(['pandoc'], 'build this site (compile with pandoc)', python=False) diff --git a/nikola/plugins/compile/php.py b/nikola/plugins/compile/php.py index a60e31a..d2559fd 100644 --- a/nikola/plugins/compile/php.py +++ b/nikola/plugins/compile/php.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/compile/rest/__init__.py b/nikola/plugins/compile/rest/__init__.py index 4b04958..b75849f 100644 --- a/nikola/plugins/compile/rest/__init__.py +++ b/nikola/plugins/compile/rest/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,9 +36,19 @@ import docutils.utils import docutils.io import docutils.readers.standalone import docutils.writers.html4css1 +import docutils.parsers.rst.directives +from docutils.parsers.rst import roles +from nikola.nikola import LEGAL_VALUES from nikola.plugin_categories import PageCompiler -from nikola.utils import unicode_str, get_logger, makedirs, write_metadata, STDERR_HANDLER +from nikola.utils import ( + unicode_str, + get_logger, + makedirs, + write_metadata, + STDERR_HANDLER, + LocaleBorg +) class CompileRest(PageCompiler): @@ -49,19 +59,6 @@ class CompileRest(PageCompiler): demote_headers = True logger = None - def _read_extra_deps(self, post): - """Read contents of .dep file and returns them as a list.""" - dep_path = post.base_path + '.dep' - if os.path.isfile(dep_path): - with io.open(dep_path, 'r+', encoding='utf8') as depf: - deps = [l.strip() for l in depf.readlines()] - return deps - return [] - - def register_extra_dependencies(self, post): - """Add dependency to post object to check .dep file.""" - post.add_dependency(lambda: self._read_extra_deps(post), 'fragment') - def compile_html_string(self, data, source_path=None, is_two_file=True): """Compile reST into HTML strings.""" # If errors occur, this will be added to the line number reported by @@ -73,16 +70,20 @@ class CompileRest(PageCompiler): add_ln = len(m_data.splitlines()) + 1 default_template_path = os.path.join(os.path.dirname(__file__), 'template.txt') + settings_overrides = { + 'initial_header_level': 1, + 'record_dependencies': True, + 'stylesheet_path': None, + 'link_stylesheet': True, + 'syntax_highlight': 'short', + 'math_output': 'mathjax', + 'template': default_template_path, + 'language_code': LEGAL_VALUES['DOCUTILS_LOCALES'].get(LocaleBorg().current_lang, 'en') + } + output, error_level, deps = rst2html( - data, settings_overrides={ - 'initial_header_level': 1, - 'record_dependencies': True, - 'stylesheet_path': None, - 'link_stylesheet': True, - 'syntax_highlight': 'short', - 'math_output': 'mathjax', - 'template': default_template_path, - }, logger=self.logger, source_path=source_path, l_add_ln=add_ln, transforms=self.site.rst_transforms) + data, settings_overrides=settings_overrides, logger=self.logger, source_path=source_path, l_add_ln=add_ln, transforms=self.site.rst_transforms, + no_title_transform=self.site.config.get('NO_DOCUTILS_TITLE_TRANSFORM', False)) if not isinstance(output, unicode_str): # To prevent some weird bugs here or there. # Original issue: empty files. `output` became a bytestring. @@ -94,18 +95,23 @@ class CompileRest(PageCompiler): makedirs(os.path.dirname(dest)) error_level = 100 with io.open(dest, "w+", encoding="utf8") as out_file: + try: + post = self.site.post_per_input_file[source] + except KeyError: + post = None with io.open(source, "r", encoding="utf8") as in_file: data = in_file.read() output, error_level, deps = self.compile_html_string(data, source, is_two_file) + output, shortcode_deps = self.site.apply_shortcodes(output, filename=source, with_dependencies=True, extra_context=dict(post=post)) out_file.write(output) - deps_path = dest + '.dep' - if deps.list: - deps.list = [p for p in deps.list if p != dest] # Don't depend on yourself (#1671) - with io.open(deps_path, "w+", encoding="utf8") as deps_file: - deps_file.write('\n'.join(deps.list)) + if post is None: + if deps.list: + self.logger.error( + "Cannot save dependencies for post {0} due to unregistered source file name", + source) else: - if os.path.isfile(deps_path): - os.unlink(deps_path) + post._depfile[dest] += deps.list + post._depfile[dest] += shortcode_deps if error_level < 3: return True else: @@ -176,11 +182,15 @@ class NikolaReader(docutils.readers.standalone.Reader): def __init__(self, *args, **kwargs): """Initialize the reader.""" self.transforms = kwargs.pop('transforms', []) + self.no_title_transform = kwargs.pop('no_title_transform', False) docutils.readers.standalone.Reader.__init__(self, *args, **kwargs) def get_transforms(self): """Get docutils transforms.""" - return docutils.readers.standalone.Reader(self).get_transforms() + self.transforms + transforms = docutils.readers.standalone.Reader(self).get_transforms() + self.transforms + if self.no_title_transform: + transforms = [t for t in transforms if str(t) != "<class 'docutils.transforms.frontmatter.DocTitle'>"] + return transforms def new_document(self): """Create and return a new empty document tree (root node).""" @@ -190,6 +200,16 @@ class NikolaReader(docutils.readers.standalone.Reader): return document +def shortcode_role(name, rawtext, text, lineno, inliner, + options={}, content=[]): + """A shortcode role that passes through raw inline HTML.""" + return [docutils.nodes.raw('', text, format='html')], [] + +roles.register_canonical_role('raw-html', shortcode_role) +roles.register_canonical_role('html', shortcode_role) +roles.register_canonical_role('sc', shortcode_role) + + def add_node(node, visit_function=None, depart_function=None): """Register a Docutils node class. @@ -235,7 +255,8 @@ def rst2html(source, source_path=None, source_class=docutils.io.StringInput, parser=None, parser_name='restructuredtext', writer=None, writer_name='html', settings=None, settings_spec=None, settings_overrides=None, config_section=None, - enable_exit_status=None, logger=None, l_add_ln=0, transforms=None): + enable_exit_status=None, logger=None, l_add_ln=0, transforms=None, + no_title_transform=False): """Set up & run a ``Publisher``, and return a dictionary of document parts. Dictionary keys are the names of parts, and values are Unicode strings; @@ -253,7 +274,7 @@ def rst2html(source, source_path=None, source_class=docutils.io.StringInput, reStructuredText syntax errors. """ if reader is None: - reader = NikolaReader(transforms=transforms) + reader = NikolaReader(transforms=transforms, no_title_transform=no_title_transform) # For our custom logging, we have special needs and special settings we # specify here. # logger a logger from Nikola @@ -274,3 +295,10 @@ def rst2html(source, source_path=None, source_class=docutils.io.StringInput, pub.publish(enable_exit_status=enable_exit_status) return pub.writer.parts['docinfo'] + pub.writer.parts['fragment'], pub.document.reporter.max_level, pub.settings.record_dependencies + +# Alignment helpers for extensions +_align_options_base = ('left', 'center', 'right') + + +def _align_choice(argument): + return docutils.parsers.rst.directives.choice(argument, _align_options_base + ("none", "")) diff --git a/nikola/plugins/compile/rest/chart.py b/nikola/plugins/compile/rest/chart.py index ed8a250..24f459b 100644 --- a/nikola/plugins/compile/rest/chart.py +++ b/nikola/plugins/compile/rest/chart.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -52,6 +52,7 @@ class Plugin(RestExtension): global _site _site = self.site = site directives.register_directive('chart', Chart) + self.site.register_shortcode('chart', _gen_chart) return super(Plugin, self).set_site(site) @@ -72,52 +73,68 @@ class Chart(Directive): has_content = True required_arguments = 1 option_spec = { - "copy": directives.unchanged, + "box_mode": directives.unchanged, + "classes": directives.unchanged, "css": directives.unchanged, + "defs": directives.unchanged, "disable_xml_declaration": directives.unchanged, "dots_size": directives.unchanged, + "dynamic_print_values": directives.unchanged, "explicit_size": directives.unchanged, "fill": directives.unchanged, - "font_sizes": directives.unchanged, + "force_uri_protocol": directives.unchanged, + "half_pie": directives.unchanged, "height": directives.unchanged, "human_readable": directives.unchanged, "include_x_axis": directives.unchanged, + "inner_radius": directives.unchanged, "interpolate": directives.unchanged, "interpolation_parameters": directives.unchanged, "interpolation_precision": directives.unchanged, + "inverse_y_axis": directives.unchanged, "js": directives.unchanged, - "label_font_size": directives.unchanged, "legend_at_bottom": directives.unchanged, + "legend_at_bottom_columns": directives.unchanged, "legend_box_size": directives.unchanged, - "legend_font_size": directives.unchanged, "logarithmic": directives.unchanged, - "major_label_font_size": directives.unchanged, "margin": directives.unchanged, - "no_data_font_size": directives.unchanged, + "margin_bottom": directives.unchanged, + "margin_left": directives.unchanged, + "margin_right": directives.unchanged, + "margin_top": directives.unchanged, + "max_scale": directives.unchanged, + "min_scale": directives.unchanged, + "missing_value_fill_truncation": directives.unchanged, "no_data_text": directives.unchanged, "no_prefix": directives.unchanged, "order_min": directives.unchanged, "pretty_print": directives.unchanged, + "print_labels": directives.unchanged, "print_values": directives.unchanged, + "print_values_position": directives.unchanged, "print_zeroes": directives.unchanged, "range": directives.unchanged, "rounded_bars": directives.unchanged, + "secondary_range": directives.unchanged, "show_dots": directives.unchanged, "show_legend": directives.unchanged, "show_minor_x_labels": directives.unchanged, + "show_minor_y_labels": directives.unchanged, + "show_only_major_dots": directives.unchanged, + "show_x_guides": directives.unchanged, + "show_x_labels": directives.unchanged, + "show_y_guides": directives.unchanged, "show_y_labels": directives.unchanged, "spacing": directives.unchanged, + "stack_from_top": directives.unchanged, "strict": directives.unchanged, "stroke": directives.unchanged, + "stroke_style": directives.unchanged, "style": directives.unchanged, "title": directives.unchanged, - "title_font_size": directives.unchanged, - "to_dict": directives.unchanged, "tooltip_border_radius": directives.unchanged, - "tooltip_font_size": directives.unchanged, "truncate_label": directives.unchanged, "truncate_legend": directives.unchanged, - "value_font_size": directives.unchanged, "value_formatter": directives.unchanged, "width": directives.unchanged, "x_label_rotation": directives.unchanged, @@ -126,37 +143,55 @@ class Chart(Directive): "x_labels_major_count": directives.unchanged, "x_labels_major_every": directives.unchanged, "x_title": directives.unchanged, + "x_value_formatter": directives.unchanged, + "xrange": directives.unchanged, "y_label_rotation": directives.unchanged, "y_labels": directives.unchanged, + "y_labels_major": directives.unchanged, + "y_labels_major_count": directives.unchanged, + "y_labels_major_every": directives.unchanged, "y_title": directives.unchanged, "zero": directives.unchanged, } def run(self): """Run the directive.""" - if pygal is None: - msg = req_missing(['pygal'], 'use the Chart directive', optional=True) - return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')] - options = {} - if 'style' in self.options: - style_name = self.options.pop('style') - else: - style_name = 'BlueStyle' - if '(' in style_name: # Parametric style - style = eval('pygal.style.' + style_name) - else: - style = getattr(pygal.style, style_name) - for k, v in self.options.items(): + self.options['site'] = None + html = _gen_chart(self.arguments[0], data='\n'.join(self.content), **self.options) + return [nodes.raw('', html, format='html')] + + +def _gen_chart(chart_type, **_options): + if pygal is None: + msg = req_missing(['pygal'], 'use the Chart directive', optional=True) + return '<div class="text-error">{0}</div>'.format(msg) + options = {} + data = _options.pop('data') + _options.pop('post', None) + _options.pop('site') + if 'style' in _options: + style_name = _options.pop('style') + else: + style_name = 'BlueStyle' + if '(' in style_name: # Parametric style + style = eval('pygal.style.' + style_name) + else: + style = getattr(pygal.style, style_name) + for k, v in _options.items(): + try: options[k] = literal_eval(v) - chart = pygal - for o in self.arguments[0].split('.'): - chart = getattr(chart, o) - chart = chart(style=style) - if _site and _site.invariant: - chart.no_prefix = True - chart.config(**options) - for line in self.content: + except: + options[k] = v + chart = pygal + for o in chart_type.split('.'): + chart = getattr(chart, o) + chart = chart(style=style) + if _site and _site.invariant: + chart.no_prefix = True + chart.config(**options) + for line in data.splitlines(): + line = line.strip() + if line: label, series = literal_eval('({0})'.format(line)) chart.add(label, series) - data = chart.render().decode('utf8') - return [nodes.raw('', data, format='html')] + return chart.render().decode('utf8') diff --git a/nikola/plugins/compile/rest/doc.py b/nikola/plugins/compile/rest/doc.py index 578f012..55f576d 100644 --- a/nikola/plugins/compile/rest/doc.py +++ b/nikola/plugins/compile/rest/doc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,7 +29,7 @@ from docutils import nodes from docutils.parsers.rst import roles -from nikola.utils import split_explicit_title +from nikola.utils import split_explicit_title, LOGGER from nikola.plugin_categories import RestExtension @@ -42,12 +42,12 @@ class Plugin(RestExtension): """Set Nikola site.""" self.site = site roles.register_canonical_role('doc', doc_role) + self.site.register_shortcode('doc', doc_shortcode) doc_role.site = site return super(Plugin, self).set_site(site) -def doc_role(name, rawtext, text, lineno, inliner, - options={}, content=[]): +def _doc_link(rawtext, text, options={}, content=[]): """Handle the doc role.""" # split link's text and post's slug in role content has_explicit_title, title, slug = split_explicit_title(text) @@ -66,22 +66,48 @@ def doc_role(name, rawtext, text, lineno, inliner, if post is None: raise ValueError except ValueError: + return False, False, None, None, slug + + if not has_explicit_title: + # use post's title as link's text + title = post.title() + permalink = post.permalink() + + return True, twin_slugs, title, permalink, slug + + +def doc_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + """Handle the doc role.""" + success, twin_slugs, title, permalink, slug = _doc_link(rawtext, text, options, content) + if success: + if twin_slugs: + inliner.reporter.warning( + 'More than one post with the same slug. Using "{0}"'.format(permalink)) + LOGGER.warn( + 'More than one post with the same slug. Using "{0}" for doc role'.format(permalink)) + node = make_link_node(rawtext, title, permalink, options) + return [node], [] + else: msg = inliner.reporter.error( '"{0}" slug doesn\'t exist.'.format(slug), line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] - if not has_explicit_title: - # use post's title as link's text - title = post.title() - permalink = post.permalink() - if twin_slugs: - msg = inliner.reporter.warning( - 'More than one post with the same slug. Using "{0}"'.format(permalink)) - node = make_link_node(rawtext, title, permalink, options) - return [node], [] +def doc_shortcode(*args, **kwargs): + """Implement the doc shortcode.""" + text = kwargs['data'] + success, twin_slugs, title, permalink, slug = _doc_link(text, text, LOGGER) + if success: + if twin_slugs: + LOGGER.warn( + 'More than one post with the same slug. Using "{0}" for doc shortcode'.format(permalink)) + return '<a href="{0}">{1}</a>'.format(permalink, title) + else: + LOGGER.error( + '"{0}" slug doesn\'t exist.'.format(slug)) + return '<span class="error text-error" style="color: red;">Invalid link: {0}</span>'.format(text) def make_link_node(rawtext, text, url, options): diff --git a/nikola/plugins/compile/rest/listing.py b/nikola/plugins/compile/rest/listing.py index 07f1686..4dfbedc 100644 --- a/nikola/plugins/compile/rest/listing.py +++ b/nikola/plugins/compile/rest/listing.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -136,6 +136,7 @@ class Plugin(RestExtension): # leaving these to make the code directive work with # docutils < 0.9 CodeBlock.site = site + Listing.site = site directives.register_directive('code', CodeBlock) directives.register_directive('code-block', CodeBlock) directives.register_directive('sourcecode', CodeBlock) @@ -189,8 +190,11 @@ class Listing(Include): self.content = fileobject.read().splitlines() self.state.document.settings.record_dependencies.add(fpath) target = urlunsplit(("link", 'listing', fpath.replace('\\', '/'), '', '')) + src_target = urlunsplit(("link", 'listing_source', fpath.replace('\\', '/'), '', '')) + src_label = self.site.MESSAGES('Source') generated_nodes = ( - [core.publish_doctree('`{0} <{1}>`_'.format(_fname, target))[0]]) + [core.publish_doctree('`{0} <{1}>`_ `({2}) <{3}>`_' .format( + _fname, target, src_label, src_target))[0]]) generated_nodes += self.get_code_from_file(fileobject) return generated_nodes diff --git a/nikola/plugins/compile/rest/media.py b/nikola/plugins/compile/rest/media.py index d075e44..8a69586 100644 --- a/nikola/plugins/compile/rest/media.py +++ b/nikola/plugins/compile/rest/media.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -48,6 +48,7 @@ class Plugin(RestExtension): """Set Nikola site.""" self.site = site directives.register_directive('media', Media) + self.site.register_shortcode('media', _gen_media_embed) return super(Plugin, self).set_site(site) @@ -60,9 +61,13 @@ class Media(Directive): def run(self): """Run media directive.""" - if micawber is None: - msg = req_missing(['micawber'], 'use the media directive', optional=True) - return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')] + html = _gen_media_embed(" ".join(self.arguments)) + return [nodes.raw('', html, format='html')] - providers = micawber.bootstrap_basic() - return [nodes.raw('', micawber.parse_text(" ".join(self.arguments), providers), format='html')] + +def _gen_media_embed(url, *q, **kw): + if micawber is None: + msg = req_missing(['micawber'], 'use the media directive', optional=True) + return '<div class="text-error">{0}</div>'.format(msg) + providers = micawber.bootstrap_basic() + return micawber.parse_text(url, providers) diff --git a/nikola/plugins/compile/rest/post_list.py b/nikola/plugins/compile/rest/post_list.py index df9376b..8cfd5bf 100644 --- a/nikola/plugins/compile/rest/post_list.py +++ b/nikola/plugins/compile/rest/post_list.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2013-2015 Udo Spallek, Roberto Alsina and others. +# Copyright © 2013-2016 Udo Spallek, Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -37,6 +37,7 @@ from docutils.parsers.rst import Directive, directives from nikola import utils from nikola.plugin_categories import RestExtension +from nikola.packages.datecond import date_in_range # WARNING: the directive name is post-list # (with a DASH instead of an UNDERSCORE) @@ -50,6 +51,7 @@ class Plugin(RestExtension): def set_site(self, site): """Set Nikola site.""" self.site = site + self.site.register_shortcode('post-list', _do_post_list) directives.register_directive('post-list', PostList) PostList.site = site return super(Plugin, self).set_site(site) @@ -61,7 +63,7 @@ class PostList(Directive): Post List ========= :Directive Arguments: None. - :Directive Options: lang, start, stop, reverse, sort, tags, categories, slugs, all, template, id + :Directive Options: lang, start, stop, reverse, sort, date, tags, categories, sections, slugs, post_type, all, template, id :Directive Content: None. The posts appearing in the list can be filtered by options. @@ -85,10 +87,19 @@ class PostList(Directive): Reverse the order of the post-list. Defaults is to not reverse the order of posts. - ``sort``: string + ``sort`` : string Sort post list by one of each post's attributes, usually ``title`` or a custom ``priority``. Defaults to None (chronological sorting). + ``date`` : string + Show posts that match date range specified by this option. Format: + + * comma-separated clauses (AND) + * clause: attribute comparison_operator value (spaces optional) + * attribute: year, month, day, hour, month, second, weekday, isoweekday; or empty for full datetime + * comparison_operator: == != <= >= < > + * value: integer or dateutil-compatible date input + ``tags`` : string [, string...] Filter posts to show only posts having at least one of the ``tags``. Defaults to None. @@ -97,13 +108,21 @@ class PostList(Directive): Filter posts to show only posts having one of the ``categories``. Defaults to None. + ``sections`` : string [, string...] + Filter posts to show only posts having one of the ``sections``. + Defaults to None. + ``slugs`` : string [, string...] Filter posts to show only posts having at least one of the ``slugs``. Defaults to None. + ``post_type`` (or ``type``) : string + Show only ``posts``, ``pages`` or ``all``. + Replaces ``all``. Defaults to ``posts``. + ``all`` : flag - Shows all posts and pages in the post list. - Defaults to show only posts with set *use_in_feeds*. + (deprecated, use ``post_type`` instead) + Shows all posts and pages in the post list. Defaults to show only posts. ``lang`` : string The language of post *titles* and *links*. @@ -125,11 +144,15 @@ class PostList(Directive): 'sort': directives.unchanged, 'tags': directives.unchanged, 'categories': directives.unchanged, + 'sections': directives.unchanged, 'slugs': directives.unchanged, + 'post_type': directives.unchanged, + 'type': directives.unchanged, 'all': directives.flag, 'lang': directives.unchanged, 'template': directives.path, 'id': directives.unchanged, + 'date': directives.unchanged, } def run(self): @@ -138,75 +161,151 @@ class PostList(Directive): stop = self.options.get('stop') reverse = self.options.get('reverse', False) tags = self.options.get('tags') - tags = [t.strip().lower() for t in tags.split(',')] if tags else [] categories = self.options.get('categories') - categories = [c.strip().lower() for c in categories.split(',')] if categories else [] + sections = self.options.get('sections') slugs = self.options.get('slugs') - slugs = [s.strip() for s in slugs.split(',')] if slugs else [] - show_all = self.options.get('all', False) + post_type = self.options.get('post_type') + type = self.options.get('type', False) + all = self.options.get('all', False) lang = self.options.get('lang', utils.LocaleBorg().current_lang) template = self.options.get('template', 'post_list_directive.tmpl') sort = self.options.get('sort') - if self.site.invariant: # for testing purposes - post_list_id = self.options.get('id', 'post_list_' + 'fixedvaluethatisnotauuid') - else: - post_list_id = self.options.get('id', 'post_list_' + uuid.uuid4().hex) + date = self.options.get('date') - filtered_timeline = [] - posts = [] - step = -1 if reverse is None else None - if show_all is None: - timeline = [p for p in self.site.timeline] + output, deps = _do_post_list(start, stop, reverse, tags, categories, sections, slugs, post_type, type, + all, lang, template, sort, state=self.state, site=self.site, date=date) + self.state.document.settings.record_dependencies.add("####MAGIC####TIMELINE") + for d in deps: + self.state.document.settings.record_dependencies.add(d) + if output: + return [nodes.raw('', output, format='html')] else: - timeline = [p for p in self.site.timeline if p.use_in_feeds] - - if categories: - timeline = [p for p in timeline if p.meta('category', lang=lang).lower() in categories] - - for post in timeline: - if tags: - cont = True - tags_lower = [t.lower() for t in post.tags] - for tag in tags: - if tag in tags_lower: - cont = False - - if cont: - continue - - filtered_timeline.append(post) - - if sort: - filtered_timeline = natsort.natsorted(filtered_timeline, key=lambda post: post.meta[lang][sort], alg=natsort.ns.F | natsort.ns.IC) - - for post in filtered_timeline[start:stop:step]: - if slugs: - cont = True - for slug in slugs: - if slug == post.meta('slug'): - cont = False - - if cont: - continue - - bp = post.translated_base_path(lang) - if os.path.exists(bp): - self.state.document.settings.record_dependencies.add(bp) + return [] - posts += [post] - if not posts: - return [] - self.state.document.settings.record_dependencies.add("####MAGIC####TIMELINE") +def _do_post_list(start=None, stop=None, reverse=False, tags=None, categories=None, + sections=None, slugs=None, post_type='post', type=False, all=False, + lang=None, template='post_list_directive.tmpl', sort=None, + id=None, data=None, state=None, site=None, date=None, filename=None, post=None): + if lang is None: + lang = utils.LocaleBorg().current_lang + if site.invariant: # for testing purposes + post_list_id = id or 'post_list_' + 'fixedvaluethatisnotauuid' + else: + post_list_id = id or 'post_list_' + uuid.uuid4().hex + + # Get post from filename if available + if filename: + self_post = site.post_per_input_file.get(filename) + else: + self_post = None + + if self_post: + self_post.register_depfile("####MAGIC####TIMELINE", lang=lang) + + # If we get strings for start/stop, make them integers + if start is not None: + start = int(start) + if stop is not None: + stop = int(stop) + + # Parse tags/categories/sections/slugs (input is strings) + tags = [t.strip().lower() for t in tags.split(',')] if tags else [] + categories = [c.strip().lower() for c in categories.split(',')] if categories else [] + sections = [s.strip().lower() for s in sections.split(',')] if sections else [] + slugs = [s.strip() for s in slugs.split(',')] if slugs else [] + + filtered_timeline = [] + posts = [] + step = -1 if reverse is None else None + + if type is not False: + post_type = type + + # TODO: remove in v8 + if all is not False: + timeline = [p for p in site.timeline] + elif post_type == 'page' or post_type == 'pages': + timeline = [p for p in site.timeline if not p.use_in_feeds] + elif post_type == 'all': + timeline = [p for p in site.timeline] + else: # post + timeline = [p for p in site.timeline if p.use_in_feeds] + + # TODO: replaces all, uncomment in v8 + # if post_type == 'page' or post_type == 'pages': + # timeline = [p for p in site.timeline if not p.use_in_feeds] + # elif post_type == 'all': + # timeline = [p for p in site.timeline] + # else: # post + # timeline = [p for p in site.timeline if p.use_in_feeds] + + if categories: + timeline = [p for p in timeline if p.meta('category', lang=lang).lower() in categories] + + if sections: + timeline = [p for p in timeline if p.section_name(lang).lower() in sections] + + for post in timeline: + if tags: + cont = True + tags_lower = [t.lower() for t in post.tags] + for tag in tags: + if tag in tags_lower: + cont = False + + if cont: + continue + + filtered_timeline.append(post) + + if sort: + filtered_timeline = natsort.natsorted(filtered_timeline, key=lambda post: post.meta[lang][sort], alg=natsort.ns.F | natsort.ns.IC) + + if date: + filtered_timeline = [p for p in filtered_timeline if date_in_range(date, p.date)] + + for post in filtered_timeline[start:stop:step]: + if slugs: + cont = True + for slug in slugs: + if slug == post.meta('slug'): + cont = False + + if cont: + continue + + bp = post.translated_base_path(lang) + if os.path.exists(bp) and state: + state.document.settings.record_dependencies.add(bp) + elif os.path.exists(bp) and self_post: + self_post.register_depfile(bp, lang=lang) + + posts += [post] + + if not posts: + return '', [] + + template_deps = site.template_system.template_deps(template) + if state: + # Register template as a dependency (Issue #2391) + for d in template_deps: + state.document.settings.record_dependencies.add(d) + elif self_post: + for d in template_deps: + self_post.register_depfile(d, lang=lang) + + template_data = { + 'lang': lang, + 'posts': posts, + # Need to provide str, not TranslatableSetting (Issue #2104) + 'date_format': site.GLOBAL_CONTEXT.get('date_format')[lang], + 'post_list_id': post_list_id, + 'messages': site.MESSAGES, + } + output = site.template_system.render_template( + template, None, template_data) + return output, template_deps - template_data = { - 'lang': lang, - 'posts': posts, - # Need to provide str, not TranslatableSetting (Issue #2104) - 'date_format': self.site.GLOBAL_CONTEXT.get('date_format')[lang], - 'post_list_id': post_list_id, - 'messages': self.site.MESSAGES, - } - output = self.site.template_system.render_template( - template, None, template_data) - return [nodes.raw('', output, format='html')] +# Request file name from shortcode (Issue #2412) +_do_post_list.nikola_shortcode_pass_filename = True diff --git a/nikola/plugins/compile/rest/slides.py b/nikola/plugins/compile/rest/slides.py index eb2cc97..7c5b34b 100644 --- a/nikola/plugins/compile/rest/slides.py +++ b/nikola/plugins/compile/rest/slides.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/compile/rest/soundcloud.py b/nikola/plugins/compile/rest/soundcloud.py index 2577ff1..9fabe70 100644 --- a/nikola/plugins/compile/rest/soundcloud.py +++ b/nikola/plugins/compile/rest/soundcloud.py @@ -4,7 +4,7 @@ from docutils import nodes from docutils.parsers.rst import Directive, directives - +from nikola.plugins.compile.rest import _align_choice, _align_options_base from nikola.plugin_categories import RestExtension @@ -22,11 +22,13 @@ class Plugin(RestExtension): return super(Plugin, self).set_site(site) -CODE = ("""<iframe width="{width}" height="{height}" +CODE = """\ +<div class="soundcloud-player{align}"> +<iframe width="{width}" height="{height}" scrolling="no" frameborder="no" -src="https://w.soundcloud.com/player/?url=http://api.soundcloud.com/{preslug}/""" - """{sid}"> -</iframe>""") +src="https://w.soundcloud.com/player/?url=http://api.soundcloud.com/{preslug}/{sid}"> +</iframe> +</div>""" class SoundCloud(Directive): @@ -44,6 +46,7 @@ class SoundCloud(Directive): option_spec = { 'width': directives.positive_int, 'height': directives.positive_int, + "align": _align_choice } preslug = "tracks" @@ -57,6 +60,10 @@ class SoundCloud(Directive): 'preslug': self.preslug, } options.update(self.options) + if self.options.get('align') in _align_options_base: + options['align'] = ' align-' + self.options['align'] + else: + options['align'] = '' return [nodes.raw('', CODE.format(**options), format='html')] def check_content(self): diff --git a/nikola/plugins/compile/rest/thumbnail.py b/nikola/plugins/compile/rest/thumbnail.py index c24134a..37e0973 100644 --- a/nikola/plugins/compile/rest/thumbnail.py +++ b/nikola/plugins/compile/rest/thumbnail.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Pelle Nilsson and others. +# Copyright © 2014-2016 Pelle Nilsson and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/compile/rest/vimeo.py b/nikola/plugins/compile/rest/vimeo.py index 29ce5c1..f1ac6c3 100644 --- a/nikola/plugins/compile/rest/vimeo.py +++ b/nikola/plugins/compile/rest/vimeo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -28,6 +28,7 @@ from docutils import nodes from docutils.parsers.rst import Directive, directives +from nikola.plugins.compile.rest import _align_choice, _align_options_base import requests import json @@ -48,10 +49,12 @@ class Plugin(RestExtension): return super(Plugin, self).set_site(site) -CODE = """<iframe src="//player.vimeo.com/video/{vimeo_id}" +CODE = """<div class="vimeo-video{align}"> +<iframe src="https://player.vimeo.com/video/{vimeo_id}" width="{width}" height="{height}" frameborder="0" webkitAllowFullScreen="webkitAllowFullScreen" mozallowfullscreen="mozallowfullscreen" allowFullScreen="allowFullScreen"> </iframe> +</div> """ VIDEO_DEFAULT_HEIGHT = 500 @@ -73,6 +76,7 @@ class Vimeo(Directive): option_spec = { "width": directives.positive_int, "height": directives.positive_int, + "align": _align_choice } # set to False for not querying the vimeo api for size @@ -92,6 +96,10 @@ class Vimeo(Directive): return err self.set_video_size() options.update(self.options) + if self.options.get('align') in _align_options_base: + options['align'] = ' align-' + self.options['align'] + else: + options['align'] = '' return [nodes.raw('', CODE.format(**options), format='html')] def check_modules(self): @@ -107,7 +115,7 @@ class Vimeo(Directive): if json: # we can attempt to retrieve video attributes from vimeo try: - url = ('//vimeo.com/api/v2/video/{0}' + url = ('https://vimeo.com/api/v2/video/{0}' '.json'.format(self.arguments[0])) data = requests.get(url).text video_attributes = json.loads(data)[0] diff --git a/nikola/plugins/compile/rest/youtube.py b/nikola/plugins/compile/rest/youtube.py index b3b84b0..b3dde62 100644 --- a/nikola/plugins/compile/rest/youtube.py +++ b/nikola/plugins/compile/rest/youtube.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -28,7 +28,7 @@ from docutils import nodes from docutils.parsers.rst import Directive, directives - +from nikola.plugins.compile.rest import _align_choice, _align_options_base from nikola.plugin_categories import RestExtension @@ -46,10 +46,11 @@ class Plugin(RestExtension): CODE = """\ -<iframe width="{width}" -height="{height}" -src="//www.youtube.com/embed/{yid}?rel=0&hd=1&wmode=transparent" -></iframe>""" +<div class="youtube-video{align}"> +<iframe width="{width}" height="{height}" +src="https://www.youtube.com/embed/{yid}?rel=0&hd=1&wmode=transparent" +></iframe> +</div>""" class Youtube(Directive): @@ -67,6 +68,7 @@ class Youtube(Directive): option_spec = { "width": directives.positive_int, "height": directives.positive_int, + "align": _align_choice } def run(self): @@ -78,6 +80,10 @@ class Youtube(Directive): 'height': 344, } options.update(self.options) + if self.options.get('align') in _align_options_base: + options['align'] = ' align-' + self.options['align'] + else: + options['align'] = '' return [nodes.raw('', CODE.format(**options), format='html')] def check_content(self): diff --git a/nikola/plugins/misc/__init__.py b/nikola/plugins/misc/__init__.py index c0d8961..518fac1 100644 --- a/nikola/plugins/misc/__init__.py +++ b/nikola/plugins/misc/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/misc/scan_posts.py b/nikola/plugins/misc/scan_posts.py index 9db4533..f584a05 100644 --- a/nikola/plugins/misc/scan_posts.py +++ b/nikola/plugins/misc/scan_posts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -35,6 +35,8 @@ from nikola.plugin_categories import PostScanner from nikola import utils from nikola.post import Post +LOGGER = utils.get_logger('scan_posts', utils.STDERR_HANDLER) + class ScanPosts(PostScanner): """Scan posts in the site.""" @@ -87,15 +89,19 @@ class ScanPosts(PostScanner): continue else: seen.add(base_path) - post = Post( - base_path, - self.site.config, - dest_dir, - use_in_feeds, - self.site.MESSAGES, - template_name, - self.site.get_compiler(base_path) - ) - timeline.append(post) + try: + post = Post( + base_path, + self.site.config, + dest_dir, + use_in_feeds, + self.site.MESSAGES, + template_name, + self.site.get_compiler(base_path) + ) + timeline.append(post) + except Exception as err: + LOGGER.error('Error reading post {}'.format(base_path)) + raise err return timeline diff --git a/nikola/plugins/shortcode/gist.plugin b/nikola/plugins/shortcode/gist.plugin new file mode 100644 index 0000000..cd19a72 --- /dev/null +++ b/nikola/plugins/shortcode/gist.plugin @@ -0,0 +1,13 @@ +[Core] +name = gist +module = gist + +[Nikola] +plugincategory = Shortcode + +[Documentation] +author = Roberto Alsina +version = 0.1 +website = https://getnikola.com/ +description = Gist shortcode + diff --git a/nikola/plugins/shortcode/gist.py b/nikola/plugins/shortcode/gist.py new file mode 100644 index 0000000..64fd0d9 --- /dev/null +++ b/nikola/plugins/shortcode/gist.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# This file is public domain according to its author, Brian Hsu + +"""Gist directive for reStructuredText.""" + +import requests + +from nikola.plugin_categories import ShortcodePlugin + + +class Plugin(ShortcodePlugin): + """Plugin for gist directive.""" + + name = "gist" + + def set_site(self, site): + """Set Nikola site.""" + self.site = site + site.register_shortcode('gist', self.handler) + return super(Plugin, self).set_site(site) + + def get_raw_gist_with_filename(self, gistID, filename): + """Get raw gist text for a filename.""" + url = '/'.join(("https://gist.github.com/raw", gistID, filename)) + return requests.get(url).text + + def get_raw_gist(self, gistID): + """Get raw gist text.""" + url = "https://gist.github.com/raw/{0}".format(gistID) + try: + return requests.get(url).text + except requests.exceptions.RequestException: + raise self.error('Cannot get gist for url={0}'.format(url)) + + def handler(self, gistID, filename=None, site=None, data=None, lang=None, post=None): + """Create HTML for gist.""" + if 'https://' in gistID: + gistID = gistID.split('/')[-1].strip() + else: + gistID = gistID.strip() + embedHTML = "" + rawGist = "" + + if filename is not None: + rawGist = (self.get_raw_gist_with_filename(gistID, filename)) + embedHTML = ('<script src="https://gist.github.com/{0}.js' + '?file={1}"></script>').format(gistID, filename) + else: + rawGist = (self.get_raw_gist(gistID)) + embedHTML = ('<script src="https://gist.github.com/{0}.js">' + '</script>').format(gistID) + + output = '''{} + <noscript><pre>{}</pre></noscript>'''.format(embedHTML, rawGist) + + return output, [] diff --git a/nikola/plugins/task/__init__.py b/nikola/plugins/task/__init__.py index fd9a48f..4eeae62 100644 --- a/nikola/plugins/task/__init__.py +++ b/nikola/plugins/task/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/archive.py b/nikola/plugins/task/archive.py index 3cdd33b..303d349 100644 --- a/nikola/plugins/task/archive.py +++ b/nikola/plugins/task/archive.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/authors.py b/nikola/plugins/task/authors.py index 081d21d..ec61800 100644 --- a/nikola/plugins/task/authors.py +++ b/nikola/plugins/task/authors.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2015 Juanjo Conti. +# Copyright © 2015-2016 Juanjo Conti and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -35,6 +35,8 @@ except ImportError: from urllib.parse import urljoin # NOQA from collections import defaultdict +from blinker import signal + from nikola.plugin_categories import Task from nikola import utils @@ -47,13 +49,20 @@ class RenderAuthors(Task): def set_site(self, site): """Set Nikola site.""" + self.generate_author_pages = False if site.config["ENABLE_AUTHOR_PAGES"]: site.register_path_handler('author_index', self.author_index_path) site.register_path_handler('author', self.author_path) site.register_path_handler('author_atom', self.author_atom_path) site.register_path_handler('author_rss', self.author_rss_path) + signal('scanned').connect(self.posts_scanned) return super(RenderAuthors, self).set_site(site) + def posts_scanned(self, event): + """Called after posts are scanned via signal.""" + self.generate_author_pages = self.site.config["ENABLE_AUTHOR_PAGES"] and len(self._posts_per_author()) > 1 + self.site.GLOBAL_CONTEXT["author_pages_generated"] = self.generate_author_pages + def gen_tasks(self): """Render the author pages and feeds.""" kw = { @@ -78,12 +87,10 @@ class RenderAuthors(Task): "index_file": self.site.config['INDEX_FILE'], } - yield self.group_task() self.site.scan_posts() + yield self.group_task() - generate_author_pages = self.site.config["ENABLE_AUTHOR_PAGES"] and len(self._posts_per_author()) > 1 - self.site.GLOBAL_CONTEXT["author_pages_generated"] = generate_author_pages - if generate_author_pages: + if self.generate_author_pages: yield self.list_authors_page(kw) if not self._posts_per_author(): # this may be self.site.posts_per_author @@ -244,10 +251,13 @@ class RenderAuthors(Task): } return utils.apply_filters(task, kw['filters']) - def slugify_author_name(self, name): + def slugify_author_name(self, name, lang=None): """Slugify an author name.""" + if lang is None: # TODO: remove in v8 + utils.LOGGER.warn("RenderAuthors.slugify_author_name() called without language!") + lang = '' if self.site.config['SLUG_AUTHOR_PATH']: - name = utils.slugify(name) + name = utils.slugify(name, lang) return name def author_index_path(self, name, lang): @@ -272,13 +282,13 @@ class RenderAuthors(Task): return [_f for _f in [ self.site.config['TRANSLATIONS'][lang], self.site.config['AUTHOR_PATH'], - self.slugify_author_name(name), + self.slugify_author_name(name, lang), self.site.config['INDEX_FILE']] if _f] else: return [_f for _f in [ self.site.config['TRANSLATIONS'][lang], self.site.config['AUTHOR_PATH'], - self.slugify_author_name(name) + ".html"] if _f] + self.slugify_author_name(name, lang) + ".html"] if _f] def author_atom_path(self, name, lang): """Link to an author's Atom feed. @@ -288,7 +298,7 @@ class RenderAuthors(Task): link://author_atom/joe => /authors/joe.atom """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['AUTHOR_PATH'], self.slugify_author_name(name) + ".atom"] if + self.site.config['AUTHOR_PATH'], self.slugify_author_name(name, lang) + ".atom"] if _f] def author_rss_path(self, name, lang): @@ -299,7 +309,7 @@ class RenderAuthors(Task): link://author_rss/joe => /authors/joe.rss """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['AUTHOR_PATH'], self.slugify_author_name(name) + ".xml"] if + self.site.config['AUTHOR_PATH'], self.slugify_author_name(name, lang) + ".xml"] if _f] def _add_extension(self, path, extension): diff --git a/nikola/plugins/task/bundles.py b/nikola/plugins/task/bundles.py index e709133..b33d8e0 100644 --- a/nikola/plugins/task/bundles.py +++ b/nikola/plugins/task/bundles.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/copy_assets.py b/nikola/plugins/task/copy_assets.py index 2cab71a..4ed7414 100644 --- a/nikola/plugins/task/copy_assets.py +++ b/nikola/plugins/task/copy_assets.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/copy_files.py b/nikola/plugins/task/copy_files.py index 0488011..6f6cfb8 100644 --- a/nikola/plugins/task/copy_files.py +++ b/nikola/plugins/task/copy_files.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/galleries.py b/nikola/plugins/task/galleries.py index d3f1db7..edfd33d 100644 --- a/nikola/plugins/task/galleries.py +++ b/nikola/plugins/task/galleries.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -33,7 +33,6 @@ import io import json import mimetypes import os -import sys try: from urlparse import urljoin except ImportError: @@ -86,6 +85,8 @@ class Galleries(Task, ImageProcessor): 'tzinfo': site.tzinfo, 'comments_in_galleries': site.config['COMMENTS_IN_GALLERIES'], 'generate_rss': site.config['GENERATE_RSS'], + 'preserve_exif_data': site.config['PRESERVE_EXIF_DATA'], + 'exif_whitelist': site.config['EXIF_WHITELIST'], } # Verify that no folder in GALLERY_FOLDERS appears twice @@ -93,8 +94,8 @@ class Galleries(Task, ImageProcessor): for source, dest in self.kw['gallery_folders'].items(): if source in appearing_paths or dest in appearing_paths: problem = source if source in appearing_paths else dest - utils.LOGGER.error("The gallery input or output folder '{0}' appears in more than one entry in GALLERY_FOLDERS, exiting.".format(problem)) - sys.exit(1) + utils.LOGGER.error("The gallery input or output folder '{0}' appears in more than one entry in GALLERY_FOLDERS, ignoring.".format(problem)) + continue appearing_paths.add(source) appearing_paths.add(dest) @@ -115,10 +116,11 @@ class Galleries(Task, ImageProcessor): if len(candidates) == 1: return candidates[0] self.logger.error("Gallery name '{0}' is not unique! Possible output paths: {1}".format(name, candidates)) + raise RuntimeError("Gallery name '{0}' is not unique! Possible output paths: {1}".format(name, candidates)) else: self.logger.error("Unknown gallery '{0}'!".format(name)) self.logger.info("Known galleries: " + str(list(self.proper_gallery_links.keys()))) - sys.exit(1) + raise RuntimeError("Unknown gallery '{0}'!".format(name)) def gallery_path(self, name, lang): """Link to an image gallery's path. @@ -173,6 +175,7 @@ class Galleries(Task, ImageProcessor): for k, v in self.site.GLOBAL_CONTEXT['template_hooks'].items(): self.kw['||template_hooks|{0}||'.format(k)] = v._items + self.site.scan_posts() yield self.group_task() template_name = "gallery.tmpl" @@ -194,13 +197,6 @@ class Galleries(Task, ImageProcessor): # Create image list, filter exclusions image_list = self.get_image_list(gallery) - # Sort as needed - # Sort by date - if self.kw['sort_by_date']: - image_list.sort(key=lambda a: self.image_date(a)) - else: # Sort by name - image_list.sort() - # Create thumbnails and large images in destination for image in image_list: for task in self.create_target_images(image, input_folder): @@ -211,8 +207,6 @@ class Galleries(Task, ImageProcessor): for task in self.remove_excluded_image(image, input_folder): yield task - crumbs = utils.get_crumbs(gallery, index_folder=self) - for lang in self.kw['translations']: # save navigation links as dependencies self.kw['navigation_links|{0}'.format(lang)] = self.kw['global_context']['navigation_links'](lang) @@ -242,7 +236,7 @@ class Galleries(Task, ImageProcessor): img_titles = [] for fn in image_name_list: name_without_ext = os.path.splitext(os.path.basename(fn))[0] - img_titles.append(utils.unslugify(name_without_ext)) + img_titles.append(utils.unslugify(name_without_ext, lang)) else: img_titles = [''] * len(image_name_list) @@ -266,7 +260,7 @@ class Galleries(Task, ImageProcessor): context["folders"] = natsort.natsorted( folders, alg=natsort.ns.F | natsort.ns.IC) - context["crumbs"] = crumbs + context["crumbs"] = utils.get_crumbs(gallery, index_folder=self, lang=lang) context["permalink"] = self.site.link("gallery", gallery, lang) context["enable_comments"] = self.kw['comments_in_galleries'] context["thumbnail_size"] = self.kw["thumbnail_size"] @@ -423,6 +417,8 @@ class Galleries(Task, ImageProcessor): # may break) if post.title == 'index': post.title = os.path.split(gallery)[1] + # Register the post (via #2417) + self.site.post_per_input_file[index_path] = post else: post = None return post @@ -482,7 +478,8 @@ class Galleries(Task, ImageProcessor): 'targets': [thumb_path], 'actions': [ (self.resize_image, - (img, thumb_path, self.kw['thumbnail_size'])) + (img, thumb_path, self.kw['thumbnail_size'], False, self.kw['preserve_exif_data'], + self.kw['exif_whitelist'])) ], 'clean': True, 'uptodate': [utils.config_changed({ @@ -497,7 +494,8 @@ class Galleries(Task, ImageProcessor): 'targets': [orig_dest_path], 'actions': [ (self.resize_image, - (img, orig_dest_path, self.kw['max_image_size'])) + (img, orig_dest_path, self.kw['max_image_size'], False, self.kw['preserve_exif_data'], + self.kw['exif_whitelist'])) ], 'clean': True, 'uptodate': [utils.config_changed({ @@ -558,6 +556,18 @@ class Galleries(Task, ImageProcessor): url = '/'.join(os.path.relpath(p, os.path.dirname(output_name) + os.sep).split(os.sep)) return url + all_data = list(zip(img_list, thumbs, img_titles)) + + if self.kw['sort_by_date']: + all_data.sort(key=lambda a: self.image_date(a[0])) + else: # Sort by name + all_data.sort(key=lambda a: a[0]) + + if all_data: + img_list, thumbs, img_titles = zip(*all_data) + else: + img_list, thumbs, img_titles = [], [], [] + photo_array = [] for img, thumb, title in zip(img_list, thumbs, img_titles): w, h = _image_size_cache.get(thumb, (None, None)) @@ -591,6 +601,18 @@ class Galleries(Task, ImageProcessor): def make_url(url): return urljoin(self.site.config['BASE_URL'], url.lstrip('/')) + all_data = list(zip(img_list, dest_img_list, img_titles)) + + if self.kw['sort_by_date']: + all_data.sort(key=lambda a: self.image_date(a[0])) + else: # Sort by name + all_data.sort(key=lambda a: a[0]) + + if all_data: + img_list, dest_img_list, img_titles = zip(*all_data) + else: + img_list, dest_img_list, img_titles = [], [], [] + items = [] for img, srcimg, title in list(zip(dest_img_list, img_list, img_titles))[:self.kw["feed_length"]]: img_size = os.stat( diff --git a/nikola/plugins/task/gzip.py b/nikola/plugins/task/gzip.py index aaa213d..79a11dc 100644 --- a/nikola/plugins/task/gzip.py +++ b/nikola/plugins/task/gzip.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/indexes.py b/nikola/plugins/task/indexes.py index 2ab97fa..8ecd1de 100644 --- a/nikola/plugins/task/indexes.py +++ b/nikola/plugins/task/indexes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,6 +36,7 @@ except ImportError: from nikola.plugin_categories import Task from nikola import utils +from nikola.nikola import _enclosure class Indexes(Task): @@ -51,6 +52,7 @@ class Indexes(Task): site.register_path_handler('index_atom', self.index_atom_path) site.register_path_handler('section_index', self.index_section_path) site.register_path_handler('section_index_atom', self.index_section_atom_path) + site.register_path_handler('section_index_rss', self.index_section_rss_path) return super(Indexes, self).set_site(site) def _get_filtered_posts(self, lang, show_untranslated_posts): @@ -77,6 +79,10 @@ class Indexes(Task): "translations": self.site.config['TRANSLATIONS'], "messages": self.site.MESSAGES, "output_folder": self.site.config['OUTPUT_FOLDER'], + "feed_length": self.site.config['FEED_LENGTH'], + "feed_links_append_query": self.site.config["FEED_LINKS_APPEND_QUERY"], + "feed_teasers": self.site.config["FEED_TEASERS"], + "feed_plain": self.site.config["FEED_PLAIN"], "filters": self.site.config['FILTERS'], "index_file": self.site.config['INDEX_FILE'], "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], @@ -85,6 +91,7 @@ class Indexes(Task): "strip_indexes": self.site.config['STRIP_INDEXES'], "blog_title": self.site.config["BLOG_TITLE"], "generate_atom": self.site.config["GENERATE_ATOM"], + "site_url": self.site.config["SITE_URL"], } template_name = "index.tmpl" @@ -110,8 +117,6 @@ class Indexes(Task): yield self.site.generic_index_renderer(lang, filtered_posts, indexes_title, template_name, context, kw, 'render_indexes', page_link, page_path) if self.site.config['POSTS_SECTIONS']: - - kw["posts_section_are_indexes"] = self.site.config['POSTS_SECTION_ARE_INDEXES'] index_len = len(kw['index_file']) groups = defaultdict(list) @@ -145,16 +150,16 @@ class Indexes(Task): context["pagekind"] = ["section_page"] context["description"] = self.site.config['POSTS_SECTION_DESCRIPTIONS'](lang)[section_slug] if section_slug in self.site.config['POSTS_SECTION_DESCRIPTIONS'](lang) else "" - if kw["posts_section_are_indexes"]: + if self.site.config["POSTS_SECTION_ARE_INDEXES"]: context["pagekind"].append("index") - kw["posts_section_title"] = self.site.config['POSTS_SECTION_TITLE'](lang) + posts_section_title = self.site.config['POSTS_SECTION_TITLE'](lang) section_title = None - if type(kw["posts_section_title"]) is dict: - if section_slug in kw["posts_section_title"]: - section_title = kw["posts_section_title"][section_slug] - elif type(kw["posts_section_title"]) is str: - section_title = kw["posts_section_title"] + if type(posts_section_title) is dict: + if section_slug in posts_section_title: + section_title = posts_section_title[section_slug] + elif type(posts_section_title) is str: + section_title = posts_section_title if not section_title: section_title = post_list[0].section_name(lang) section_title = section_title.format(name=post_list[0].section_name(lang)) @@ -168,7 +173,37 @@ class Indexes(Task): task['basename'] = self.name yield task - if not self.site.config["STORY_INDEX"]: + # RSS feed for section + deps = [] + deps_uptodate = [] + if kw["show_untranslated_posts"]: + posts = post_list[:kw['feed_length']] + else: + posts = [x for x in post_list if x.is_translation_available(lang)][:kw['feed_length']] + for post in posts: + deps += post.deps(lang) + deps_uptodate += post.deps_uptodate(lang) + + feed_url = urljoin(self.site.config['BASE_URL'], self.site.link('section_index_rss', section_slug, lang).lstrip('/')) + output_name = os.path.join(kw['output_folder'], self.site.path('section_index_rss', section_slug, lang).lstrip(os.sep)) + task = { + 'basename': self.name, + 'name': os.path.normpath(output_name), + 'file_dep': deps, + 'targets': [output_name], + 'actions': [(utils.generic_rss_renderer, + (lang, kw["blog_title"](lang), kw["site_url"], + context["description"], posts, output_name, + kw["feed_teasers"], kw["feed_plain"], kw['feed_length'], feed_url, + _enclosure, kw["feed_links_append_query"]))], + + 'task_dep': ['render_posts'], + 'clean': True, + 'uptodate': [utils.config_changed(kw, 'nikola.plugins.indexes')] + deps_uptodate, + } + yield task + + if not self.site.config["PAGE_INDEX"]: return kw = { "translations": self.site.config['TRANSLATIONS'], @@ -207,7 +242,7 @@ class Indexes(Task): for post in post_list: # If there is an index.html pending to be created from - # a story, do not generate the STORY_INDEX + # a page, do not generate the PAGE_INDEX if post.destination_path(lang) == short_destination: should_render = False else: @@ -252,7 +287,7 @@ class Indexes(Task): self.site, extension=extension) - def index_section_path(self, name, lang, is_feed=False): + def index_section_path(self, name, lang, is_feed=False, is_rss=False): """Link to the index for a section. Example: @@ -264,6 +299,8 @@ class Indexes(Task): if is_feed: extension = ".atom" index_file = os.path.splitext(self.site.config['INDEX_FILE'])[0] + extension + elif is_rss: + index_file = 'rss.xml' else: index_file = self.site.config['INDEX_FILE'] if name in self.number_of_pages_section[lang]: @@ -298,3 +335,12 @@ class Indexes(Task): link://section_index_atom/cars => /cars/index.atom """ return self.index_section_path(name, lang, is_feed=True) + + def index_section_rss_path(self, name, lang): + """Link to the RSS feed for a section. + + Example: + + link://section_index_rss/cars => /cars/rss.xml + """ + return self.index_section_path(name, lang, is_rss=True) diff --git a/nikola/plugins/task/listings.py b/nikola/plugins/task/listings.py index 891f361..e694aa5 100644 --- a/nikola/plugins/task/listings.py +++ b/nikola/plugins/task/listings.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,12 +29,11 @@ from __future__ import unicode_literals, print_function from collections import defaultdict -import sys import os import lxml.html from pygments import highlight -from pygments.lexers import get_lexer_for_filename, TextLexer +from pygments.lexers import get_lexer_for_filename, guess_lexer, TextLexer import natsort from nikola.plugin_categories import Task @@ -55,6 +54,7 @@ class Listings(Task): def set_site(self, site): """Set Nikola site.""" site.register_path_handler('listing', self.listing_path) + site.register_path_handler('listing_source', self.listing_source_path) # We need to prepare some things for the listings path handler to work. @@ -73,7 +73,7 @@ class Listings(Task): if source in appearing_paths or dest in appearing_paths: problem = source if source in appearing_paths else dest utils.LOGGER.error("The listings input or output folder '{0}' appears in more than one entry in LISTINGS_FOLDERS, exiting.".format(problem)) - sys.exit(1) + continue appearing_paths.add(source) appearing_paths.add(dest) @@ -127,7 +127,11 @@ class Listings(Task): try: lexer = get_lexer_for_filename(in_name) except: - lexer = TextLexer() + try: + lexer = guess_lexer(fd.read()) + except: + lexer = TextLexer() + fd.seek(0) code = highlight(fd.read(), lexer, utils.NikolaPygmentsHTML(in_name)) title = os.path.basename(in_name) else: @@ -145,7 +149,7 @@ class Listings(Task): os.path.join( self.kw['output_folder'], output_folder)))) - if self.site.config['COPY_SOURCES'] and in_name: + if in_name: source_link = permalink[:-5] # remove '.html' else: source_link = None @@ -238,19 +242,35 @@ class Listings(Task): 'uptodate': [utils.config_changed(uptodate, 'nikola.plugins.task.listings:source')], 'clean': True, }, self.kw["filters"]) - if self.site.config['COPY_SOURCES']: - rel_name = os.path.join(rel_path, f) - rel_output_name = os.path.join(output_folder, rel_path, f) - self.register_output_name(input_folder, rel_name, rel_output_name) - out_name = os.path.join(self.kw['output_folder'], rel_output_name) - yield utils.apply_filters({ - 'basename': self.name, - 'name': out_name, - 'file_dep': [in_name], - 'targets': [out_name], - 'actions': [(utils.copy_file, [in_name, out_name])], - 'clean': True, - }, self.kw["filters"]) + + rel_name = os.path.join(rel_path, f) + rel_output_name = os.path.join(output_folder, rel_path, f) + self.register_output_name(input_folder, rel_name, rel_output_name) + out_name = os.path.join(self.kw['output_folder'], rel_output_name) + yield utils.apply_filters({ + 'basename': self.name, + 'name': out_name, + 'file_dep': [in_name], + 'targets': [out_name], + 'actions': [(utils.copy_file, [in_name, out_name])], + 'clean': True, + }, self.kw["filters"]) + + def listing_source_path(self, name, lang): + """A link to the source code for a listing. + + It will try to use the file name if it's not ambiguous, or the file path. + + Example: + + link://listing_source/hello.py => /listings/tutorial/hello.py + + link://listing_source/tutorial/hello.py => /listings/tutorial/hello.py + """ + result = self.listing_path(name, lang) + if result[-1].endswith('.html'): + result[-1] = result[-1][:-5] + return result def listing_path(self, namep, lang): """A link to a listing. @@ -275,14 +295,14 @@ class Listings(Task): # ambiguities. if len(self.improper_input_file_mapping[name]) > 1: utils.LOGGER.error("Using non-unique listing name '{0}', which maps to more than one listing name ({1})!".format(name, str(self.improper_input_file_mapping[name]))) - sys.exit(1) + return ["ERROR"] if len(self.site.config['LISTINGS_FOLDERS']) > 1: utils.LOGGER.notice("Using listings names in site.link() without input directory prefix while configuration's LISTINGS_FOLDERS has more than one entry.") name = list(self.improper_input_file_mapping[name])[0] break else: utils.LOGGER.error("Unknown listing name {0}!".format(namep)) - sys.exit(1) + return ["ERROR"] if not name.endswith(os.sep + self.site.config["INDEX_FILE"]): name += '.html' path_parts = name.split(os.sep) diff --git a/nikola/plugins/task/pages.py b/nikola/plugins/task/pages.py index 8d41035..7d8287b 100644 --- a/nikola/plugins/task/pages.py +++ b/nikola/plugins/task/pages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -54,7 +54,7 @@ class RenderPages(Task): if post.is_post: context = {'pagekind': ['post_page']} else: - context = {'pagekind': ['story_page']} + context = {'pagekind': ['story_page', 'page_page']} for task in self.site.generic_page_renderer(lang, post, kw["filters"], context): task['uptodate'] = task['uptodate'] + [config_changed(kw, 'nikola.plugins.task.pages')] task['basename'] = self.name diff --git a/nikola/plugins/task/posts.py b/nikola/plugins/task/posts.py index 8735beb..fe10c5f 100644 --- a/nikola/plugins/task/posts.py +++ b/nikola/plugins/task/posts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/py3_switch.py b/nikola/plugins/task/py3_switch.py index 930c593..2ff4e2d 100644 --- a/nikola/plugins/task/py3_switch.py +++ b/nikola/plugins/task/py3_switch.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -51,7 +51,7 @@ PY2_BARBS = [ "Python 2 is the safety blanket of languages. Be a big kid and switch to Python 3", "Python 2 is old and busted. Python 3 is the new hotness.", "Nice unicode you have there, would be a shame something happened to it.. switch to python 3!.", - "Don’t get in the way of progress! Upgrade to Python 3 and save a developer’s mind today!", + "Don't get in the way of progress! Upgrade to Python 3 and save a developer's mind today!", "Winners don't use Python 2 -- Signed: The FBI", "Python 2? What year is it?", "I just wanna tell you how I'm feeling\n" diff --git a/nikola/plugins/task/redirect.py b/nikola/plugins/task/redirect.py index 2d4eba4..b170b81 100644 --- a/nikola/plugins/task/redirect.py +++ b/nikola/plugins/task/redirect.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -50,7 +50,7 @@ class Redirect(Task): yield self.group_task() if kw['redirections']: for src, dst in kw["redirections"]: - src_path = os.path.join(kw["output_folder"], src) + src_path = os.path.join(kw["output_folder"], src.lstrip('/')) yield utils.apply_filters({ 'basename': self.name, 'name': src_path, diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py index 7c7f5df..8537fc8 100644 --- a/nikola/plugins/task/robots.py +++ b/nikola/plugins/task/robots.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/rss.py b/nikola/plugins/task/rss.py index be57f5c..780559b 100644 --- a/nikola/plugins/task/rss.py +++ b/nikola/plugins/task/rss.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -34,6 +34,7 @@ except ImportError: from urllib.parse import urljoin # NOQA from nikola import utils +from nikola.nikola import _enclosure from nikola.plugin_categories import Task @@ -97,7 +98,7 @@ class GenerateRSS(Task): (lang, kw["blog_title"](lang), kw["site_url"], kw["blog_description"](lang), posts, output_name, kw["feed_teasers"], kw["feed_plain"], kw['feed_length'], feed_url, - None, kw["feed_links_append_query"]))], + _enclosure, kw["feed_links_append_query"]))], 'task_dep': ['render_posts'], 'clean': True, diff --git a/nikola/plugins/task/scale_images.py b/nikola/plugins/task/scale_images.py index e55dc6c..2b483ae 100644 --- a/nikola/plugins/task/scale_images.py +++ b/nikola/plugins/task/scale_images.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Pelle Nilsson and others. +# Copyright © 2014-2016 Pelle Nilsson and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -71,8 +71,8 @@ class ScaleImage(Task, ImageProcessor): def process_image(self, src, dst, thumb): """Resize an image.""" - self.resize_image(src, dst, self.kw['max_image_size'], False) - self.resize_image(src, thumb, self.kw['image_thumbnail_size'], False) + self.resize_image(src, dst, self.kw['max_image_size'], False, preserve_exif_data=self.kw['preserve_exif_data'], exif_whitelist=self.kw['exif_whitelist']) + self.resize_image(src, thumb, self.kw['image_thumbnail_size'], False, preserve_exif_data=self.kw['preserve_exif_data'], exif_whitelist=self.kw['exif_whitelist']) def gen_tasks(self): """Copy static files into the output folder.""" @@ -82,6 +82,8 @@ class ScaleImage(Task, ImageProcessor): 'image_folders': self.site.config['IMAGE_FOLDERS'], 'output_folder': self.site.config['OUTPUT_FOLDER'], 'filters': self.site.config['FILTERS'], + 'preserve_exif_data': self.site.config['PRESERVE_EXIF_DATA'], + 'exif_whitelist': self.site.config['EXIF_WHITELIST'], } self.image_ext_list = self.image_ext_list_builtin diff --git a/nikola/plugins/task/sitemap/__init__.py b/nikola/plugins/task/sitemap/__init__.py index 90acdd3..64fcb45 100644 --- a/nikola/plugins/task/sitemap/__init__.py +++ b/nikola/plugins/task/sitemap/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -158,7 +158,7 @@ class Sitemap(LateTask): continue alternates = [] if post: - for lang in kw['translations']: + for lang in post.translated_to: alt_url = post.permalink(lang=lang, absolute=True) if encodelink(loc) == alt_url: continue @@ -215,7 +215,7 @@ class Sitemap(LateTask): loc = urljoin(base_url, base_path + path) alternates = [] if post: - for lang in kw['translations']: + for lang in post.translated_to: alt_url = post.permalink(lang=lang, absolute=True) if encodelink(loc) == alt_url: continue diff --git a/nikola/plugins/task/sources.py b/nikola/plugins/task/sources.py index f782ad4..0d77aba 100644 --- a/nikola/plugins/task/sources.py +++ b/nikola/plugins/task/sources.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/task/tags.py b/nikola/plugins/task/tags.py index 6d9d495..8b4683e 100644 --- a/nikola/plugins/task/tags.py +++ b/nikola/plugins/task/tags.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,7 +29,6 @@ from __future__ import unicode_literals import json import os -import sys import natsort try: from urlparse import urljoin @@ -38,6 +37,7 @@ except ImportError: from nikola.plugin_categories import Task from nikola import utils +from nikola.nikola import _enclosure class RenderTags(Task): @@ -97,31 +97,31 @@ class RenderTags(Task): if not self.site.posts_per_tag and not self.site.posts_per_category: return - if kw['category_path'] == kw['tag_path']: - tags = {self.slugify_tag_name(tag): tag for tag in self.site.posts_per_tag.keys()} - cats = {tuple(self.slugify_category_name(category)): category for category in self.site.posts_per_category.keys()} - categories = {k[0]: v for k, v in cats.items() if len(k) == 1} - intersect = set(tags.keys()) & set(categories.keys()) - if len(intersect) > 0: - for slug in intersect: - utils.LOGGER.error("Category '{0}' and tag '{1}' both have the same slug '{2}'!".format('/'.join(categories[slug]), tags[slug], slug)) - sys.exit(1) - - # Test for category slug clashes - categories = {} - for category in self.site.posts_per_category.keys(): - slug = tuple(self.slugify_category_name(category)) - for part in slug: - if len(part) == 0: - utils.LOGGER.error("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) - sys.exit(1) - if slug in categories: - other_category = categories[slug] - utils.LOGGER.error('You have categories that are too similar: {0} and {1}'.format(category, other_category)) - utils.LOGGER.error('Category {0} is used in: {1}'.format(category, ', '.join([p.source_path for p in self.site.posts_per_category[category]]))) - utils.LOGGER.error('Category {0} is used in: {1}'.format(other_category, ', '.join([p.source_path for p in self.site.posts_per_category[other_category]]))) - sys.exit(1) - categories[slug] = category + for lang in kw["translations"]: + if kw['category_path'][lang] == kw['tag_path'][lang]: + tags = {self.slugify_tag_name(tag, lang): tag for tag in self.site.tags_per_language[lang]} + cats = {tuple(self.slugify_category_name(category, lang)): category for category in self.site.posts_per_category.keys()} + categories = {k[0]: v for k, v in cats.items() if len(k) == 1} + intersect = set(tags.keys()) & set(categories.keys()) + if len(intersect) > 0: + for slug in intersect: + utils.LOGGER.error("Category '{0}' and tag '{1}' both have the same slug '{2}' for language {3}!".format('/'.join(categories[slug]), tags[slug], slug, lang)) + + # Test for category slug clashes + categories = {} + for category in self.site.posts_per_category.keys(): + slug = tuple(self.slugify_category_name(category, lang)) + for part in slug: + if len(part) == 0: + utils.LOGGER.error("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) + raise RuntimeError("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) + if slug in categories: + other_category = categories[slug] + utils.LOGGER.error('You have categories that are too similar: {0} and {1} (language {2})'.format(category, other_category, lang)) + utils.LOGGER.error('Category {0} is used in: {1}'.format(category, ', '.join([p.source_path for p in self.site.posts_per_category[category]]))) + utils.LOGGER.error('Category {0} is used in: {1}'.format(other_category, ', '.join([p.source_path for p in self.site.posts_per_category[other_category]]))) + raise RuntimeError("Category '{0}' yields invalid slug '{1}'!".format(category, '/'.join(slug))) + categories[slug] = category tag_list = list(self.site.posts_per_tag.items()) cat_list = list(self.site.posts_per_category.items()) @@ -185,7 +185,7 @@ class RenderTags(Task): task['clean'] = True yield utils.apply_filters(task, kw['filters']) - def _create_tags_page(self, kw, include_tags=True, include_categories=True): + def _create_tags_page(self, kw, lang, include_tags=True, include_categories=True): """Create a global "all your tags/categories" page for each language.""" categories = [cat.category_name for cat in self.site.category_hierarchy] has_categories = (categories != []) and include_categories @@ -193,59 +193,59 @@ class RenderTags(Task): kw = kw.copy() if include_categories: kw['categories'] = categories - for lang in kw["translations"]: - tags = natsort.natsorted([tag for tag in self.site.tags_per_language[lang] - if len(self.site.posts_per_tag[tag]) >= kw["taglist_minimum_post_count"]], - alg=natsort.ns.F | natsort.ns.IC) - has_tags = (tags != []) and include_tags - if include_tags: - kw['tags'] = tags - output_name = os.path.join( - kw['output_folder'], self.site.path('tag_index' if has_tags else 'category_index', None, lang)) - context = {} - if has_categories and has_tags: - context["title"] = kw["messages"][lang]["Tags and Categories"] - elif has_categories: - context["title"] = kw["messages"][lang]["Categories"] - else: - context["title"] = kw["messages"][lang]["Tags"] - if has_tags: - context["items"] = [(tag, self.site.link("tag", tag, lang)) for tag - in tags] - else: - context["items"] = None - if has_categories: - context["cat_items"] = [(tag, self.site.link("category", tag, lang)) for tag - in categories] - context['cat_hierarchy'] = [(node.name, node.category_name, node.category_path, - self.site.link("category", node.category_name), - node.indent_levels, node.indent_change_before, - node.indent_change_after) - for node in self.site.category_hierarchy] - else: - context["cat_items"] = None - context["permalink"] = self.site.link("tag_index" if has_tags else "category_index", None, lang) - context["description"] = context["title"] - context["pagekind"] = ["list", "tags_page"] - task = self.site.generic_post_list_renderer( - lang, - [], - output_name, - template_name, - kw['filters'], - context, - ) - task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.tags:page')] - task['basename'] = str(self.name) - yield task + tags = natsort.natsorted([tag for tag in self.site.tags_per_language[lang] + if len(self.site.posts_per_tag[tag]) >= kw["taglist_minimum_post_count"]], + alg=natsort.ns.F | natsort.ns.IC) + has_tags = (tags != []) and include_tags + if include_tags: + kw['tags'] = tags + output_name = os.path.join( + kw['output_folder'], self.site.path('tag_index' if has_tags else 'category_index', None, lang)) + context = {} + if has_categories and has_tags: + context["title"] = kw["messages"][lang]["Tags and Categories"] + elif has_categories: + context["title"] = kw["messages"][lang]["Categories"] + else: + context["title"] = kw["messages"][lang]["Tags"] + if has_tags: + context["items"] = [(tag, self.site.link("tag", tag, lang)) for tag + in tags] + else: + context["items"] = None + if has_categories: + context["cat_items"] = [(tag, self.site.link("category", tag, lang)) for tag + in categories] + context['cat_hierarchy'] = [(node.name, node.category_name, node.category_path, + self.site.link("category", node.category_name), + node.indent_levels, node.indent_change_before, + node.indent_change_after) + for node in self.site.category_hierarchy] + else: + context["cat_items"] = None + context["permalink"] = self.site.link("tag_index" if has_tags else "category_index", None, lang) + context["description"] = context["title"] + context["pagekind"] = ["list", "tags_page"] + task = self.site.generic_post_list_renderer( + lang, + [], + output_name, + template_name, + kw['filters'], + context, + ) + task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.tags:page')] + task['basename'] = str(self.name) + yield task def list_tags_page(self, kw): """Create a global "all your tags/categories" page for each language.""" - if self.site.config['TAG_PATH'] == self.site.config['CATEGORY_PATH']: - yield self._create_tags_page(kw, True, True) - else: - yield self._create_tags_page(kw, False, True) - yield self._create_tags_page(kw, True, False) + for lang in kw["translations"]: + if self.site.config['TAG_PATH'][lang] == self.site.config['CATEGORY_PATH'][lang]: + yield self._create_tags_page(kw, lang, True, True) + else: + yield self._create_tags_page(kw, lang, False, True) + yield self._create_tags_page(kw, lang, True, False) def _get_title(self, tag, is_category): if is_category: @@ -253,9 +253,9 @@ class RenderTags(Task): else: return tag - def _get_indexes_title(self, tag, is_category, lang, messages): + def _get_indexes_title(self, tag, nice_tag, is_category, lang, messages): titles = self.site.config['CATEGORY_PAGES_TITLES'] if is_category else self.site.config['TAG_PAGES_TITLES'] - return titles[lang][tag] if lang in titles and tag in titles[lang] else messages[lang]["Posts about %s"] % tag + return titles[lang][tag] if lang in titles and tag in titles[lang] else messages[lang]["Posts about %s"] % nice_tag def _get_description(self, tag, is_category, lang): descriptions = self.site.config['CATEGORY_PAGES_DESCRIPTIONS'] if is_category else self.site.config['TAG_PAGES_DESCRIPTIONS'] @@ -290,7 +290,7 @@ class RenderTags(Task): context_source["category"] = tag context_source["category_path"] = self.site.parse_category_name(tag) context_source["tag"] = title - indexes_title = self._get_indexes_title(title, is_category, lang, kw["messages"]) + indexes_title = self._get_indexes_title(tag, title, is_category, lang, kw["messages"]) context_source["description"] = self._get_description(tag, is_category, lang) if is_category: context_source["subcategories"] = self._get_subcategories(tag) @@ -312,7 +312,7 @@ class RenderTags(Task): context["category"] = tag context["category_path"] = self.site.parse_category_name(tag) context["tag"] = title - context["title"] = self._get_indexes_title(title, is_category, lang, kw["messages"]) + context["title"] = self._get_indexes_title(tag, title, is_category, lang, kw["messages"]) context["posts"] = post_list context["permalink"] = self.site.link(kind, tag, lang) context["kind"] = kind @@ -379,17 +379,20 @@ class RenderTags(Task): (lang, "{0} ({1})".format(kw["blog_title"](lang), self._get_title(tag, is_category)), kw["site_url"], None, post_list, output_name, kw["feed_teasers"], kw["feed_plain"], kw['feed_length'], - feed_url, None, kw["feed_link_append_query"]))], + feed_url, _enclosure, kw["feed_link_append_query"]))], 'clean': True, 'uptodate': [utils.config_changed(kw, 'nikola.plugins.task.tags:rss')] + deps_uptodate, 'task_dep': ['render_posts'], } return utils.apply_filters(task, kw['filters']) - def slugify_tag_name(self, name): + def slugify_tag_name(self, name, lang): """Slugify a tag name.""" + if lang is None: # TODO: remove in v8 + utils.LOGGER.warn("RenderTags.slugify_tag_name() called without language!") + lang = '' if self.site.config['SLUG_TAG_PATH']: - name = utils.slugify(name) + name = utils.slugify(name, lang) return name def tag_index_path(self, name, lang): @@ -430,13 +433,13 @@ class RenderTags(Task): return [_f for _f in [ self.site.config['TRANSLATIONS'][lang], self.site.config['TAG_PATH'][lang], - self.slugify_tag_name(name), + self.slugify_tag_name(name, lang), self.site.config['INDEX_FILE']] if _f] else: return [_f for _f in [ self.site.config['TRANSLATIONS'][lang], self.site.config['TAG_PATH'][lang], - self.slugify_tag_name(name) + ".html"] if _f] + self.slugify_tag_name(name, lang) + ".html"] if _f] def tag_atom_path(self, name, lang): """A link to a tag's Atom feed. @@ -446,7 +449,7 @@ class RenderTags(Task): link://tag_atom/cats => /tags/cats.atom """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['TAG_PATH'][lang], self.slugify_tag_name(name) + ".atom"] if + self.site.config['TAG_PATH'][lang], self.slugify_tag_name(name, lang) + ".atom"] if _f] def tag_rss_path(self, name, lang): @@ -457,15 +460,18 @@ class RenderTags(Task): link://tag_rss/cats => /tags/cats.xml """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], - self.site.config['TAG_PATH'][lang], self.slugify_tag_name(name) + ".xml"] if + self.site.config['TAG_PATH'][lang], self.slugify_tag_name(name, lang) + ".xml"] if _f] - def slugify_category_name(self, name): + def slugify_category_name(self, name, lang): """Slugify a category name.""" + if lang is None: # TODO: remove in v8 + utils.LOGGER.warn("RenderTags.slugify_category_name() called without language!") + lang = '' path = self.site.parse_category_name(name) if self.site.config['CATEGORY_OUTPUT_FLAT_HIERARCHY']: path = path[-1:] # only the leaf - result = [self.slugify_tag_name(part) for part in path] + result = [self.slugify_tag_name(part, lang) for part in path] result[0] = self.site.config['CATEGORY_PREFIX'] + result[0] if not self.site.config['PRETTY_URLS']: result = ['-'.join(result)] @@ -485,11 +491,11 @@ class RenderTags(Task): if self.site.config['PRETTY_URLS']: return [_f for _f in [self.site.config['TRANSLATIONS'][lang], self.site.config['CATEGORY_PATH'][lang]] if - _f] + self.slugify_category_name(name) + [self.site.config['INDEX_FILE']] + _f] + self.slugify_category_name(name, lang) + [self.site.config['INDEX_FILE']] else: return [_f for _f in [self.site.config['TRANSLATIONS'][lang], self.site.config['CATEGORY_PATH'][lang]] if - _f] + self._add_extension(self.slugify_category_name(name), ".html") + _f] + self._add_extension(self.slugify_category_name(name, lang), ".html") def category_atom_path(self, name, lang): """A link to a category's Atom feed. @@ -500,7 +506,7 @@ class RenderTags(Task): """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], self.site.config['CATEGORY_PATH'][lang]] if - _f] + self._add_extension(self.slugify_category_name(name), ".atom") + _f] + self._add_extension(self.slugify_category_name(name, lang), ".atom") def category_rss_path(self, name, lang): """A link to a category's RSS feed. @@ -511,4 +517,4 @@ class RenderTags(Task): """ return [_f for _f in [self.site.config['TRANSLATIONS'][lang], self.site.config['CATEGORY_PATH'][lang]] if - _f] + self._add_extension(self.slugify_category_name(name), ".xml") + _f] + self._add_extension(self.slugify_category_name(name, lang), ".xml") diff --git a/nikola/plugins/template/__init__.py b/nikola/plugins/template/__init__.py index d416ad7..d5efd61 100644 --- a/nikola/plugins/template/__init__.py +++ b/nikola/plugins/template/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/template/jinja.py b/nikola/plugins/template/jinja.py index e7df102..5a2135f 100644 --- a/nikola/plugins/template/jinja.py +++ b/nikola/plugins/template/jinja.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,8 +29,8 @@ from __future__ import unicode_literals import os +import io import json -from collections import deque try: import jinja2 from jinja2 import meta @@ -47,23 +47,27 @@ class JinjaTemplates(TemplateSystem): name = "jinja" lookup = None dependency_cache = {} + per_file_cache = {} def __init__(self): """Initialize Jinja2 environment with extended set of filters.""" if jinja2 is None: return - self.lookup = jinja2.Environment() + + def set_directories(self, directories, cache_folder): + """Create a new template lookup with set directories.""" + if jinja2 is None: + req_missing(['jinja2'], 'use this theme') + cache_folder = os.path.join(cache_folder, 'jinja') + makedirs(cache_folder) + cache = jinja2.FileSystemBytecodeCache(cache_folder) + self.lookup = jinja2.Environment(bytecode_cache=cache) self.lookup.trim_blocks = True self.lookup.lstrip_blocks = True self.lookup.filters['tojson'] = json.dumps self.lookup.globals['enumerate'] = enumerate self.lookup.globals['isinstance'] = isinstance self.lookup.globals['tuple'] = tuple - - def set_directories(self, directories, cache_folder): - """Create a new template lookup with set directories.""" - if jinja2 is None: - req_missing(['jinja2'], 'use this theme') self.directories = directories self.create_lookup() @@ -88,36 +92,46 @@ class JinjaTemplates(TemplateSystem): if jinja2 is None: req_missing(['jinja2'], 'use this theme') template = self.lookup.get_template(template_name) - output = template.render(**context) + data = template.render(**context) if output_name is not None: makedirs(os.path.dirname(output_name)) - with open(output_name, 'w+') as output: - output.write(output.encode('utf8')) - return output + with io.open(output_name, 'w', encoding='utf-8') as output: + output.write(data) + return data def render_template_to_string(self, template, context): """Render template to a string using context.""" return self.lookup.from_string(template).render(**context) + def get_string_deps(self, text): + """Find dependencies for a template string.""" + deps = set([]) + ast = self.lookup.parse(text) + dep_names = meta.find_referenced_templates(ast) + for dep_name in dep_names: + filename = self.lookup.loader.get_source(self.lookup, dep_name)[1] + sub_deps = [filename] + self.get_deps(filename) + self.dependency_cache[dep_name] = sub_deps + deps |= set(sub_deps) + return list(deps) + + def get_deps(self, filename): + """Return paths to dependencies for the template loaded from filename.""" + with io.open(filename, 'r', encoding='utf-8') as fd: + text = fd.read() + return self.get_string_deps(text) + def template_deps(self, template_name): """Generate list of dependencies for a template.""" - # Cache the lists of dependencies for each template name. if self.dependency_cache.get(template_name) is None: - # Use a breadth-first search to find all templates this one - # depends on. - queue = deque([template_name]) - visited_templates = set([template_name]) - deps = [] - while len(queue) > 0: - curr = queue.popleft() - source, filename = self.lookup.loader.get_source(self.lookup, - curr)[:2] - deps.append(filename) - ast = self.lookup.parse(source) - dep_names = meta.find_referenced_templates(ast) - for dep_name in dep_names: - if (dep_name not in visited_templates and dep_name is not None): - visited_templates.add(dep_name) - queue.append(dep_name) - self.dependency_cache[template_name] = deps + filename = self.lookup.loader.get_source(self.lookup, template_name)[1] + self.dependency_cache[template_name] = [filename] + self.get_deps(filename) return self.dependency_cache[template_name] + + def get_template_path(self, template_name): + """Get the path to a template or return None.""" + try: + t = self.lookup.get_template(template_name) + return t.filename + except jinja2.TemplateNotFound: + return None diff --git a/nikola/plugins/template/mako.py b/nikola/plugins/template/mako.py index 6da21db..0c9bb64 100644 --- a/nikola/plugins/template/mako.py +++ b/nikola/plugins/template/mako.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,12 +27,13 @@ """Mako template handler.""" from __future__ import unicode_literals, print_function, absolute_import +import io import os import shutil import sys import tempfile -from mako import util, lexer, parsetree +from mako import exceptions, util, lexer, parsetree from mako.lookup import TemplateLookup from mako.template import Template from markupsafe import Markup # It's ok, Mako requires it @@ -54,9 +55,8 @@ class MakoTemplates(TemplateSystem): directories = [] cache_dir = None - def get_deps(self, filename): - """Get dependencies for a template (internal function).""" - text = util.read_file(filename) + def get_string_deps(self, text, filename=None): + """Find dependencies for a template string.""" lex = lexer.Lexer(text=text, filename=filename) lex.parse() @@ -65,8 +65,17 @@ class MakoTemplates(TemplateSystem): keyword = getattr(n, 'keyword', None) if keyword in ["inherit", "namespace"] or isinstance(n, parsetree.IncludeTag): deps.append(n.attributes['file']) + # Some templates will include "foo.tmpl" and we need paths, so normalize them + # using the template lookup + for i, d in enumerate(deps): + deps[i] = self.get_template_path(d) return deps + def get_deps(self, filename): + """Get paths to dependencies for a template.""" + text = util.read_file(filename) + return self.get_string_deps(text, filename) + def set_directories(self, directories, cache_folder): """Create a new template lookup with set directories.""" cache_dir = os.path.join(cache_folder, '.mako.tmp') @@ -108,14 +117,14 @@ class MakoTemplates(TemplateSystem): data = template.render_unicode(**context) if output_name is not None: makedirs(os.path.dirname(output_name)) - with open(output_name, 'w+') as output: + with io.open(output_name, 'w', encoding='utf-8') as output: output.write(data) return data def render_template_to_string(self, template, context): """Render template to a string using context.""" context.update(self.filters) - return Template(template).render(**context) + return Template(template, lookup=self.lookup).render(**context) def template_deps(self, template_name): """Generate list of dependencies for a template.""" @@ -126,10 +135,18 @@ class MakoTemplates(TemplateSystem): dep_filenames = self.get_deps(template.filename) deps = [template.filename] for fname in dep_filenames: - deps += self.template_deps(fname) - self.cache[template_name] = tuple(deps) + deps += [fname] + self.get_deps(fname) + self.cache[template_name] = deps return list(self.cache[template_name]) + def get_template_path(self, template_name): + """Get the path to a template or return None.""" + try: + t = self.lookup.get_template(template_name) + return t.filename + except exceptions.TopLevelLookupException: + return None + def striphtml(text): """Strip HTML tags from text.""" |