From 8041475aab2b8efad7d2857027331bd626d26312 Mon Sep 17 00:00:00 2001 From: Agustin Henze Date: Wed, 26 Aug 2015 07:57:04 -0300 Subject: Imported Upstream version 7.6.4 --- nikola/plugins/__init__.py | 3 + nikola/plugins/basic_import.py | 14 + nikola/plugins/command/__init__.py | 2 + nikola/plugins/command/auto.plugin | 16 +- nikola/plugins/command/auto/__init__.py | 196 ++++-- nikola/plugins/command/bootswatch_theme.plugin | 15 +- nikola/plugins/command/bootswatch_theme.py | 3 + nikola/plugins/command/check.plugin | 15 +- nikola/plugins/command/check.py | 70 ++- nikola/plugins/command/console.plugin | 16 +- nikola/plugins/command/console.py | 4 + nikola/plugins/command/deploy.plugin | 16 +- nikola/plugins/command/deploy.py | 12 +- nikola/plugins/command/github_deploy.plugin | 16 +- nikola/plugins/command/github_deploy.py | 19 +- nikola/plugins/command/import_wordpress.plugin | 15 +- nikola/plugins/command/import_wordpress.py | 666 +++++++++++++++++---- nikola/plugins/command/init.plugin | 16 +- nikola/plugins/command/init.py | 43 +- nikola/plugins/command/install_theme.plugin | 15 +- nikola/plugins/command/install_theme.py | 28 +- nikola/plugins/command/new_page.plugin | 16 +- nikola/plugins/command/new_page.py | 3 + nikola/plugins/command/new_post.plugin | 15 +- nikola/plugins/command/new_post.py | 236 ++++---- nikola/plugins/command/orphans.plugin | 15 +- nikola/plugins/command/orphans.py | 6 + nikola/plugins/command/plugin.plugin | 15 +- nikola/plugins/command/plugin.py | 88 ++- nikola/plugins/command/rst2html.plugin | 16 +- nikola/plugins/command/rst2html/__init__.py | 3 + nikola/plugins/command/serve.plugin | 15 +- nikola/plugins/command/serve.py | 83 ++- nikola/plugins/command/status.plugin | 16 +- nikola/plugins/command/status.py | 11 +- nikola/plugins/command/version.plugin | 16 +- nikola/plugins/command/version.py | 5 +- nikola/plugins/compile/__init__.py | 2 + nikola/plugins/compile/html.plugin | 15 +- nikola/plugins/compile/html.py | 4 + nikola/plugins/compile/ipynb.plugin | 15 +- nikola/plugins/compile/ipynb.py | 26 +- nikola/plugins/compile/markdown.plugin | 15 +- nikola/plugins/compile/markdown/__init__.py | 13 +- nikola/plugins/compile/markdown/mdx_gist.plugin | 17 +- nikola/plugins/compile/markdown/mdx_gist.py | 68 ++- nikola/plugins/compile/markdown/mdx_nikola.plugin | 17 +- nikola/plugins/compile/markdown/mdx_nikola.py | 14 +- nikola/plugins/compile/markdown/mdx_podcast.plugin | 17 +- nikola/plugins/compile/markdown/mdx_podcast.py | 27 +- nikola/plugins/compile/pandoc.plugin | 15 +- nikola/plugins/compile/pandoc.py | 5 +- nikola/plugins/compile/php.plugin | 15 +- nikola/plugins/compile/php.py | 4 + nikola/plugins/compile/rest.plugin | 15 +- nikola/plugins/compile/rest/__init__.py | 40 +- nikola/plugins/compile/rest/chart.plugin | 16 +- nikola/plugins/compile/rest/chart.py | 25 +- nikola/plugins/compile/rest/doc.plugin | 16 +- nikola/plugins/compile/rest/doc.py | 7 +- nikola/plugins/compile/rest/gist.plugin | 16 +- nikola/plugins/compile/rest/gist.py | 19 +- nikola/plugins/compile/rest/listing.plugin | 16 +- nikola/plugins/compile/rest/listing.py | 17 +- nikola/plugins/compile/rest/media.plugin | 16 +- nikola/plugins/compile/rest/media.py | 9 +- nikola/plugins/compile/rest/post_list.plugin | 17 +- nikola/plugins/compile/rest/post_list.py | 26 +- nikola/plugins/compile/rest/slides.plugin | 16 +- nikola/plugins/compile/rest/slides.py | 12 +- nikola/plugins/compile/rest/soundcloud.plugin | 16 +- nikola/plugins/compile/rest/soundcloud.py | 17 +- nikola/plugins/compile/rest/thumbnail.plugin | 17 +- nikola/plugins/compile/rest/thumbnail.py | 10 + nikola/plugins/compile/rest/vimeo.plugin | 10 +- nikola/plugins/compile/rest/vimeo.py | 20 +- nikola/plugins/compile/rest/youtube.plugin | 12 +- nikola/plugins/compile/rest/youtube.py | 13 +- nikola/plugins/loghandler/__init__.py | 25 - nikola/plugins/loghandler/smtp.plugin | 9 - nikola/plugins/loghandler/smtp.py | 54 -- nikola/plugins/loghandler/stderr.plugin | 9 - nikola/plugins/loghandler/stderr.py | 56 -- nikola/plugins/misc/__init__.py | 27 + nikola/plugins/misc/scan_posts.py | 6 +- nikola/plugins/task/__init__.py | 2 + nikola/plugins/task/archive.plugin | 15 +- nikola/plugins/task/archive.py | 26 +- nikola/plugins/task/bundles.plugin | 15 +- nikola/plugins/task/bundles.py | 7 +- nikola/plugins/task/copy_assets.plugin | 15 +- nikola/plugins/task/copy_assets.py | 4 +- nikola/plugins/task/copy_files.plugin | 15 +- nikola/plugins/task/copy_files.py | 4 +- nikola/plugins/task/galleries.plugin | 15 +- nikola/plugins/task/galleries.py | 52 +- nikola/plugins/task/gzip.plugin | 15 +- nikola/plugins/task/gzip.py | 3 + nikola/plugins/task/indexes.plugin | 15 +- nikola/plugins/task/indexes.py | 22 +- nikola/plugins/task/listings.plugin | 15 +- nikola/plugins/task/listings.py | 26 +- nikola/plugins/task/pages.plugin | 15 +- nikola/plugins/task/pages.py | 10 +- nikola/plugins/task/posts.plugin | 15 +- nikola/plugins/task/posts.py | 12 +- nikola/plugins/task/redirect.plugin | 15 +- nikola/plugins/task/redirect.py | 6 +- nikola/plugins/task/robots.plugin | 15 +- nikola/plugins/task/robots.py | 7 +- nikola/plugins/task/rss.plugin | 15 +- nikola/plugins/task/rss.py | 5 + nikola/plugins/task/scale_images.plugin | 16 +- nikola/plugins/task/scale_images.py | 13 +- nikola/plugins/task/sitemap.plugin | 15 +- nikola/plugins/task/sitemap/__init__.py | 44 +- nikola/plugins/task/sources.plugin | 15 +- nikola/plugins/task/sources.py | 13 +- nikola/plugins/task/tags.plugin | 15 +- nikola/plugins/task/tags.py | 44 +- nikola/plugins/template/__init__.py | 2 + nikola/plugins/template/jinja.plugin | 16 +- nikola/plugins/template/jinja.py | 19 +- nikola/plugins/template/mako.plugin | 16 +- nikola/plugins/template/mako.py | 28 +- 125 files changed, 2203 insertions(+), 1066 deletions(-) delete mode 100644 nikola/plugins/loghandler/__init__.py delete mode 100644 nikola/plugins/loghandler/smtp.plugin delete mode 100644 nikola/plugins/loghandler/smtp.py delete mode 100644 nikola/plugins/loghandler/stderr.plugin delete mode 100644 nikola/plugins/loghandler/stderr.py create mode 100644 nikola/plugins/misc/__init__.py (limited to 'nikola/plugins') diff --git a/nikola/plugins/__init__.py b/nikola/plugins/__init__.py index 139759b..b83f43f 100644 --- a/nikola/plugins/__init__.py +++ b/nikola/plugins/__init__.py @@ -1,2 +1,5 @@ # -*- coding: utf-8 -*- + +"""Plugins for Nikola.""" + from __future__ import absolute_import diff --git a/nikola/plugins/basic_import.py b/nikola/plugins/basic_import.py index f8a3a3c..073a539 100644 --- a/nikola/plugins/basic_import.py +++ b/nikola/plugins/basic_import.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Mixin for importer plugins.""" + from __future__ import unicode_literals, print_function import io import csv @@ -46,6 +48,7 @@ links = {} class ImportMixin(object): + """Mixin with common used methods.""" name = "import_mixin" @@ -68,12 +71,14 @@ class ImportMixin(object): @classmethod def get_channel_from_file(cls, filename): + """Get channel from XML file.""" tree = etree.fromstring(cls.read_xml_file(filename)) channel = tree.find('channel') return channel @staticmethod def configure_redirections(url_map): + """Configure redirections from an url_map.""" redirections = [] for k, v in url_map.items(): if not k[-1] == '/': @@ -90,6 +95,7 @@ class ImportMixin(object): return redirections def generate_base_site(self): + """Generate a base Nikola site.""" if not os.path.exists(self.output_folder): os.system('nikola init -q ' + self.output_folder) else: @@ -108,14 +114,17 @@ class ImportMixin(object): @staticmethod def populate_context(channel): + """Populate context with settings.""" raise NotImplementedError("Must be implemented by a subclass.") @classmethod def transform_content(cls, content): + """Transform content to a Nikola-friendly format.""" return content @classmethod def write_content(cls, filename, content, rewrite_html=True): + """Write content to file.""" if rewrite_html: doc = html.document_fromstring(content) doc.rewrite_links(replacer) @@ -129,6 +138,7 @@ class ImportMixin(object): @staticmethod def write_metadata(filename, title, slug, post_date, description, tags, **kwargs): + """Write metadata to meta file.""" if not description: description = "" @@ -140,6 +150,7 @@ class ImportMixin(object): @staticmethod def write_urlmap_csv(output_file, url_map): + """Write urlmap to csv file.""" utils.makedirs(os.path.dirname(output_file)) fmode = 'wb+' if sys.version_info[0] == 2 else 'w+' with io.open(output_file, fmode) as fd: @@ -148,6 +159,7 @@ class ImportMixin(object): csv_writer.writerow(item) def get_configuration_output_path(self): + """Get path for the output configuration file.""" if not self.import_into_existing_site: filename = 'conf.py' else: @@ -161,10 +173,12 @@ class ImportMixin(object): @staticmethod def write_configuration(filename, rendered_template): + """Write the configuration file.""" utils.makedirs(os.path.dirname(filename)) with io.open(filename, 'w+', encoding='utf8') as fd: fd.write(rendered_template) def replacer(dst): + """Replace links.""" return links.get(dst, dst) diff --git a/nikola/plugins/command/__init__.py b/nikola/plugins/command/__init__.py index a1d17a6..2aa5267 100644 --- a/nikola/plugins/command/__init__.py +++ b/nikola/plugins/command/__init__.py @@ -23,3 +23,5 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Commands for Nikola.""" diff --git a/nikola/plugins/command/auto.plugin b/nikola/plugins/command/auto.plugin index a1c6820..3e2b17d 100644 --- a/nikola/plugins/command/auto.plugin +++ b/nikola/plugins/command/auto.plugin @@ -1,9 +1,13 @@ [Core] -Name = auto -Module = auto +name = auto +module = auto [Documentation] -Author = Roberto Alsina -Version = 2.1.0 -Website = http://getnikola.com -Description = Automatically detect site changes, rebuild and optionally refresh a browser. +author = Roberto Alsina +version = 2.1.0 +website = http://getnikola.com +description = Automatically detect site changes, rebuild and optionally refresh a browser. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py index c25ef8a..71f9624 100644 --- a/nikola/plugins/command/auto/__init__.py +++ b/nikola/plugins/command/auto/__init__.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Automatic rebuilds for Nikola.""" + from __future__ import print_function import json @@ -31,10 +33,13 @@ import mimetypes import os import re import subprocess +import sys +import time try: from urlparse import urlparse + from urllib2 import unquote except ImportError: - from urllib.parse import urlparse # NOQA + from urllib.parse import urlparse, unquote # NOQA import webbrowser from wsgiref.simple_server import make_server import wsgiref.util @@ -42,7 +47,7 @@ import wsgiref.util from blinker import signal try: from ws4py.websocket import WebSocket - from ws4py.server.wsgirefserver import WSGIServer, WebSocketWSGIRequestHandler + from ws4py.server.wsgirefserver import WSGIServer, WebSocketWSGIRequestHandler, WebSocketWSGIHandler from ws4py.server.wsgiutils import WebSocketWSGIApplication from ws4py.messaging import TextMessage except ImportError: @@ -58,7 +63,7 @@ except ImportError: from nikola.plugin_categories import Command -from nikola.utils import req_missing, get_logger, get_theme_path +from nikola.utils import req_missing, get_logger, get_theme_path, STDERR_HANDLER LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js') error_signal = signal('error') refresh_signal = signal('refresh') @@ -74,9 +79,12 @@ ERROR {} class CommandAuto(Command): - """Start debugging console.""" + + """Automatic rebuilds for Nikola.""" + name = "auto" logger = None + has_server = True doc_purpose = "builds and serves a site; automatically detects site changes, rebuilds, and optionally refreshes a browser" cmd_options = [ { @@ -100,7 +108,7 @@ class CommandAuto(Command): 'short': 'b', 'long': 'browser', 'type': bool, - 'help': 'Start a web browser.', + 'help': 'Start a web browser', 'default': False, }, { @@ -111,12 +119,18 @@ class CommandAuto(Command): 'type': bool, 'help': 'Use IPv6', }, + { + 'name': 'no-server', + 'long': 'no-server', + 'default': False, + 'type': bool, + 'help': 'Disable the server, automate rebuilds only' + }, ] def _execute(self, options, args): """Start the watcher.""" - - self.logger = get_logger('auto', self.site.loghandlers) + self.logger = get_logger('auto', STDERR_HANDLER) LRSocket.logger = self.logger if WebSocket is object and watchdog is None: @@ -166,10 +180,14 @@ class CommandAuto(Command): host = options['address'].strip('[').strip(']') or dhost + # Server can be disabled (Issue #1883) + self.has_server = not options['no-server'] + # Instantiate global observer observer = Observer() - # Watch output folders and trigger reloads - observer.schedule(OurWatchHandler(self.do_refresh), out_folder, recursive=True) + if self.has_server: + # Watch output folders and trigger reloads + observer.schedule(OurWatchHandler(self.do_refresh), out_folder, recursive=True) # Watch input folders and trigger rebuilds for p in watched: @@ -181,101 +199,155 @@ class CommandAuto(Command): _conf_dn = os.path.dirname(_conf_fn) observer.schedule(ConfigWatchHandler(_conf_fn, self.do_rebuild), _conf_dn, recursive=False) - observer.start() + try: + self.logger.info("Watching files for changes...") + observer.start() + except KeyboardInterrupt: + pass parent = self class Mixed(WebSocketWSGIApplication): - """A class that supports WS and HTTP protocols in the same port.""" + + """A class that supports WS and HTTP protocols on the same port.""" + def __call__(self, environ, start_response): if environ.get('HTTP_UPGRADE') is None: return parent.serve_static(environ, start_response) return super(Mixed, self).__call__(environ, start_response) - ws = make_server( - host, port, server_class=WSGIServer, - handler_class=WebSocketWSGIRequestHandler, - app=Mixed(handler_cls=LRSocket) - ) - ws.initialize_websockets_manager() - self.logger.info("Serving HTTP on {0} port {1}...".format(host, port)) - if browser: - if options['ipv6'] or '::' in host: - server_url = "http://[{0}]:{1}/".format(host, port) - else: - server_url = "http://{0}:{1}/".format(host, port) - - self.logger.info("Opening {0} in the default web browser...".format(server_url)) - # Yes, this is racy - webbrowser.open('http://{0}:{1}'.format(host, port)) - - try: - ws.serve_forever() - except KeyboardInterrupt: - self.logger.info("Server is shutting down.") - observer.stop() - observer.join() + if self.has_server: + ws = make_server( + host, port, server_class=WSGIServer, + handler_class=WebSocketWSGIRequestHandler, + app=Mixed(handler_cls=LRSocket) + ) + ws.initialize_websockets_manager() + self.logger.info("Serving HTTP on {0} port {1}...".format(host, port)) + if browser: + if options['ipv6'] or '::' in host: + server_url = "http://[{0}]:{1}/".format(host, port) + else: + server_url = "http://{0}:{1}/".format(host, port) + + self.logger.info("Opening {0} in the default web browser...".format(server_url)) + # Yes, this is racy + webbrowser.open('http://{0}:{1}'.format(host, port)) + + try: + ws.serve_forever() + except KeyboardInterrupt: + self.logger.info("Server is shutting down.") + # This is a hack, but something is locking up in a futex + # and exit() doesn't work. + os.kill(os.getpid(), 15) + else: + # Workaround: can’t have nothing running (instant exit) + # but also can’t join threads (no way to exit) + # The joys of threading. + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + self.logger.info("Shutting down.") + # This is a hack, but something is locking up in a futex + # and exit() doesn't work. + os.kill(os.getpid(), 15) def do_rebuild(self, event): - self.logger.info('REBUILDING SITE (from {0})'.format(event.src_path)) + """Rebuild the site.""" + # Move events have a dest_path, some editors like gedit use a + # move on larger save operations for write protection + event_path = event.dest_path if hasattr(event, 'dest_path') else event.src_path + fname = os.path.basename(event_path) + if (fname.endswith('~') or + fname.startswith('.') or + os.path.isdir(event_path)): # Skip on folders, these are usually duplicates + return + self.logger.info('REBUILDING SITE (from {0})'.format(event_path)) p = subprocess.Popen(self.cmd_arguments, stderr=subprocess.PIPE) + error = p.stderr.read() + errord = error.decode('utf-8') if p.wait() != 0: - error = p.stderr.read() - self.logger.error(error) - error_signal.send(error=error) + self.logger.error(errord) + error_signal.send(error=errord) else: - error = p.stderr.read() - print(error) + print(errord) def do_refresh(self, event): - self.logger.info('REFRESHING: {0}'.format(event.src_path)) - p = os.path.relpath(event.src_path, os.path.abspath(self.site.config['OUTPUT_FOLDER'])) + """Refresh the page.""" + # Move events have a dest_path, some editors like gedit use a + # move on larger save operations for write protection + event_path = event.dest_path if hasattr(event, 'dest_path') else event.src_path + self.logger.info('REFRESHING: {0}'.format(event_path)) + p = os.path.relpath(event_path, os.path.abspath(self.site.config['OUTPUT_FOLDER'])) refresh_signal.send(path=p) def serve_static(self, environ, start_response): """Trivial static file server.""" uri = wsgiref.util.request_uri(environ) p_uri = urlparse(uri) - f_path = os.path.join(self.site.config['OUTPUT_FOLDER'], *p_uri.path.split('/')) - mimetype = mimetypes.guess_type(uri)[0] or 'text/html' + f_path = os.path.join(self.site.config['OUTPUT_FOLDER'], *[unquote(x) for x in p_uri.path.split('/')]) + + # ‘Pretty’ URIs and root are assumed to be HTML + mimetype = 'text/html' if uri.endswith('/') else mimetypes.guess_type(uri)[0] or 'application/octet-stream' if os.path.isdir(f_path): + if not f_path.endswith('/'): # Redirect to avoid breakage + start_response('301 Redirect', [('Location', p_uri.path + '/')]) + return [] f_path = os.path.join(f_path, self.site.config['INDEX_FILE']) + mimetype = 'text/html' if p_uri.path == '/robots.txt': start_response('200 OK', [('Content-type', 'text/plain')]) - return ['User-Agent: *\nDisallow: /\n'] + return ['User-Agent: *\nDisallow: /\n'.encode('utf-8')] elif os.path.isfile(f_path): with open(f_path, 'rb') as fd: start_response('200 OK', [('Content-type', mimetype)]) - return [self.inject_js(mimetype, fd.read())] + return [self.file_filter(mimetype, fd.read())] elif p_uri.path == '/livereload.js': with open(LRJS_PATH, 'rb') as fd: start_response('200 OK', [('Content-type', mimetype)]) - return [self.inject_js(mimetype, fd.read())] + return [self.file_filter(mimetype, fd.read())] start_response('404 ERR', []) - return [self.inject_js('text/html', ERROR_N.format(404).format(uri))] + return [self.file_filter('text/html', ERROR_N.format(404).format(uri).encode('utf-8'))] - def inject_js(self, mimetype, data): - """Inject livereload.js in HTML files.""" + def file_filter(self, mimetype, data): + """Apply necessary changes to document before serving.""" if mimetype == 'text/html': - data = re.sub('', self.snippet, data.decode('utf8'), 1, re.IGNORECASE) + data = data.decode('utf8') + data = self.remove_base_tag(data) + data = self.inject_js(data) data = data.encode('utf8') return data + def inject_js(self, data): + """Inject livereload.js.""" + data = re.sub('', self.snippet, data, 1, re.IGNORECASE) + return data + + def remove_base_tag(self, data): + """Comment out any to allow local resolution of relative URLs.""" + data = re.sub(r']*)>', '', data, re.IGNORECASE) + return data + pending = [] class LRSocket(WebSocket): + """Speak Livereload protocol.""" def __init__(self, *a, **kw): + """Initialize protocol handler.""" refresh_signal.connect(self.notify) error_signal.connect(self.send_error) super(LRSocket, self).__init__(*a, **kw) def received_message(self, message): + """Handle received message.""" message = json.loads(message.data.decode('utf8')) self.logger.info('<--- {0}'.format(message)) response = None @@ -364,3 +436,25 @@ class ConfigWatchHandler(FileSystemEventHandler): """Call the provided function on any event.""" if event._src_path == self.configuration_filename: self.function(event) + + +try: + # Monkeypatch to hide Broken Pipe Errors + f = WebSocketWSGIHandler.finish_response + + if sys.version_info[0] == 3: + EX = BrokenPipeError # NOQA + else: + EX = IOError + + def finish_response(self): + """Monkeypatched finish_response that ignores broken pipes.""" + try: + f(self) + except EX: # Client closed the connection, not a real error + pass + + WebSocketWSGIHandler.finish_response = finish_response +except NameError: + # In case there is no WebSocketWSGIHandler because of a failed import. + pass diff --git a/nikola/plugins/command/bootswatch_theme.plugin b/nikola/plugins/command/bootswatch_theme.plugin index b428da3..fc25045 100644 --- a/nikola/plugins/command/bootswatch_theme.plugin +++ b/nikola/plugins/command/bootswatch_theme.plugin @@ -1,10 +1,13 @@ [Core] -Name = bootswatch_theme -Module = bootswatch_theme +name = bootswatch_theme +module = bootswatch_theme [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Given a swatch name and a parent theme, creates a custom theme. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Given a swatch name and a parent theme, creates a custom theme. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py index e19c937..b5644a1 100644 --- a/nikola/plugins/command/bootswatch_theme.py +++ b/nikola/plugins/command/bootswatch_theme.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" + from __future__ import print_function import os import requests @@ -35,6 +37,7 @@ LOGGER = utils.get_logger('bootswatch_theme', utils.STDERR_HANDLER) class CommandBootswatchTheme(Command): + """Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" name = "bootswatch_theme" diff --git a/nikola/plugins/command/check.plugin b/nikola/plugins/command/check.plugin index dd0980e..e380e64 100644 --- a/nikola/plugins/command/check.plugin +++ b/nikola/plugins/command/check.plugin @@ -1,10 +1,13 @@ [Core] -Name = check -Module = check +name = check +module = check [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Check the generated site +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Check the generated site + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py index a9bc44a..abf183e 100644 --- a/nikola/plugins/command/check.py +++ b/nikola/plugins/command/check.py @@ -24,11 +24,14 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Check the generated site.""" + from __future__ import print_function from collections import defaultdict import os import re import sys +import time try: from urllib import unquote from urlparse import urlparse, urljoin, urldefrag @@ -40,7 +43,7 @@ import lxml.html import requests from nikola.plugin_categories import Command -from nikola.utils import get_logger +from nikola.utils import get_logger, STDERR_HANDLER def _call_nikola_list(site): @@ -58,6 +61,7 @@ def _call_nikola_list(site): def real_scan_files(site): + """Scan for files.""" task_fnames = set([]) real_fnames = set([]) output_folder = site.config['OUTPUT_FOLDER'] @@ -80,7 +84,8 @@ def real_scan_files(site): def fs_relpath_from_url_path(url_path): - """Expects as input an urlparse(s).path""" + """Create a filesystem relative path from an URL path.""" + # Expects as input an urlparse(s).path url_path = unquote(url_path) # in windows relative paths don't begin with os.sep if sys.platform == 'win32' and len(url_path): @@ -89,6 +94,7 @@ def fs_relpath_from_url_path(url_path): class CommandCheck(Command): + """Check the generated site.""" name = "check" @@ -147,7 +153,7 @@ class CommandCheck(Command): def _execute(self, options, args): """Check the generated site.""" - self.logger = get_logger('check', self.site.loghandlers) + self.logger = get_logger('check', STDERR_HANDLER) if not options['links'] and not options['files'] and not options['clean']: print(self.help()) @@ -169,6 +175,7 @@ class CommandCheck(Command): checked_remote_targets = {} def analyze(self, fname, find_sources=False, check_remote=False): + """Analyze links on a page.""" rv = False self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']] base_url = urlparse(self.site.config['BASE_URL']) @@ -217,15 +224,45 @@ class CommandCheck(Command): if parsed.netloc == base_url.netloc: # absolute URL to self.site continue if target in self.checked_remote_targets: # already checked this exact target - if self.checked_remote_targets[target] > 399: - self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) + if self.checked_remote_targets[target] in [301, 307]: + self.logger.warn("Remote link PERMANENTLY redirected in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) + elif self.checked_remote_targets[target] in [302, 308]: + self.logger.info("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target])) + elif self.checked_remote_targets[target] > 399: + self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) continue + + # Skip whitelisted targets + if any(re.search(_, target) for _ in self.whitelist): + continue + # Check the remote link works req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'} # I’m a real boy! - resp = requests.head(target, headers=req_headers) - self.checked_remote_targets[target] = resp.status_code + resp = requests.head(target, headers=req_headers, allow_redirects=False) + + # Retry client errors (4xx) as GET requests because many servers are broken + if resp.status_code >= 400 and resp.status_code <= 499: + time.sleep(0.5) + resp = requests.get(target, headers=req_headers, allow_redirects=False) + + # Follow redirects and see where they lead, redirects to errors will be reported twice + if resp.status_code in [301, 302, 307, 308]: + redir_status_code = resp.status_code + time.sleep(0.5) + # Known redirects are retested using GET because IIS servers otherwise get HEADaches + resp = requests.get(target, headers=req_headers, allow_redirects=True) + # Permanent redirects should be updated + if redir_status_code in [301, 308]: + self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + if redir_status_code in [302, 307]: + self.logger.info("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + self.checked_remote_targets[resp.url] = resp.status_code + self.checked_remote_targets[target] = redir_status_code + else: + self.checked_remote_targets[target] = resp.status_code + if resp.status_code > 399: # Error - self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, resp.status_code)) + self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, resp.status_code)) continue elif resp.status_code <= 399: # The address leads *somewhere* that is not an error self.logger.debug("Successfully checked remote link in {0}: {1} [HTTP: {2}]".format(filename, target, resp.status_code)) @@ -271,6 +308,7 @@ class CommandCheck(Command): return rv def scan_links(self, find_sources=False, check_remote=False): + """Check links on the site.""" self.logger.info("Checking Links:") self.logger.info("===============\n") self.logger.notice("{0} mode".format(self.site.config['URL_TYPE'])) @@ -286,6 +324,7 @@ class CommandCheck(Command): return failure def scan_files(self): + """Check files in the site, find missing and orphaned files.""" failure = False self.logger.info("Checking Files:") self.logger.info("===============\n") @@ -311,7 +350,22 @@ class CommandCheck(Command): return failure def clean_files(self): + """Remove orphaned files.""" only_on_output, _ = real_scan_files(self.site) for f in only_on_output: + self.logger.info('removed: {0}'.format(f)) os.unlink(f) + + # Find empty directories and remove them + output_folder = self.site.config['OUTPUT_FOLDER'] + all_dirs = [] + for root, dirs, files in os.walk(output_folder, followlinks=True): + all_dirs.append(root) + all_dirs.sort(key=len, reverse=True) + for d in all_dirs: + try: + os.rmdir(d) + self.logger.info('removed: {0}/'.format(d)) + except OSError: + pass return True diff --git a/nikola/plugins/command/console.plugin b/nikola/plugins/command/console.plugin index 3aef2e7..333762c 100644 --- a/nikola/plugins/command/console.plugin +++ b/nikola/plugins/command/console.plugin @@ -1,9 +1,13 @@ [Core] -Name = console -Module = console +name = console +module = console [Documentation] -Author = Chris Warrick, Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Start a debugging python console +author = Chris Warrick, Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Start a debugging python console + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py index b8e7825..539fa08 100644 --- a/nikola/plugins/command/console.py +++ b/nikola/plugins/command/console.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Start debugging console.""" + from __future__ import print_function, unicode_literals import os @@ -36,7 +38,9 @@ LOGGER = get_logger('console', STDERR_HANDLER) class CommandConsole(Command): + """Start debugging console.""" + name = "console" shells = ['ipython', 'bpython', 'plain'] doc_purpose = "start an interactive Python console with access to your site" diff --git a/nikola/plugins/command/deploy.plugin b/nikola/plugins/command/deploy.plugin index 14fd53f..4743ca2 100644 --- a/nikola/plugins/command/deploy.plugin +++ b/nikola/plugins/command/deploy.plugin @@ -1,9 +1,13 @@ [Core] -Name = deploy -Module = deploy +name = deploy +module = deploy [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Deploy the site +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Deploy the site + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py index 2c44e87..821ea11 100644 --- a/nikola/plugins/command/deploy.py +++ b/nikola/plugins/command/deploy.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Deploy site.""" + from __future__ import print_function import io from datetime import datetime @@ -35,11 +37,13 @@ import time from blinker import signal from nikola.plugin_categories import Command -from nikola.utils import get_logger, remove_file, unicode_str, makedirs +from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER class CommandDeploy(Command): + """Deploy site.""" + name = "deploy" doc_usage = "[[preset [preset...]]" @@ -48,7 +52,8 @@ class CommandDeploy(Command): logger = None def _execute(self, command, args): - self.logger = get_logger('deploy', self.site.loghandlers) + """Execute the deploy command.""" + self.logger = get_logger('deploy', STDERR_HANDLER) # Get last successful deploy date timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy') if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo': @@ -116,7 +121,7 @@ class CommandDeploy(Command): outf.write(unicode_str(new_deploy.isoformat())) def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None): - """ Emit events for all timeline entries newer than last deploy. + """Emit events for all timeline entries newer than last deploy. last_deploy: datetime Time stamp of the last successful deployment. @@ -128,7 +133,6 @@ class CommandDeploy(Command): True when it appears like deploy is being run after a clean. """ - event = { 'last_deploy': last_deploy, 'new_deploy': new_deploy, diff --git a/nikola/plugins/command/github_deploy.plugin b/nikola/plugins/command/github_deploy.plugin index 74e7902..e793548 100644 --- a/nikola/plugins/command/github_deploy.plugin +++ b/nikola/plugins/command/github_deploy.plugin @@ -1,9 +1,13 @@ [Core] -Name = github_deploy -Module = github_deploy +name = github_deploy +module = github_deploy [Documentation] -Author = Puneeth Chaganti -Version = 1,0 -Website = http://getnikola.com -Description = Deploy the site to GitHub pages. +author = Puneeth Chaganti +version = 1,0 +website = http://getnikola.com +description = Deploy the site to GitHub pages. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py index 888a4f9..0ab9332 100644 --- a/nikola/plugins/command/github_deploy.py +++ b/nikola/plugins/command/github_deploy.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Deploy site to GitHub Pages.""" + from __future__ import print_function from datetime import datetime import io @@ -33,17 +35,19 @@ from textwrap import dedent from nikola.plugin_categories import Command from nikola.plugins.command.check import real_scan_files -from nikola.utils import get_logger, req_missing, makedirs, unicode_str +from nikola.utils import get_logger, req_missing, makedirs, unicode_str, STDERR_HANDLER from nikola.__main__ import main from nikola import __version__ def uni_check_output(*args, **kwargs): + """Run command and return output as Unicode (UTf-8).""" o = subprocess.check_output(*args, **kwargs) return o.decode('utf-8') def check_ghp_import_installed(): + """Check if ghp-import is installed.""" try: subprocess.check_output(['ghp-import', '-h']) except OSError: @@ -53,7 +57,9 @@ def check_ghp_import_installed(): class CommandGitHubDeploy(Command): - """ Deploy site to GitHub Pages. """ + + """Deploy site to GitHub Pages.""" + name = 'github_deploy' doc_usage = '' @@ -70,10 +76,8 @@ class CommandGitHubDeploy(Command): logger = None def _execute(self, command, args): - - self.logger = get_logger( - CommandGitHubDeploy.name, self.site.loghandlers - ) + """Run the deployment.""" + self.logger = get_logger(CommandGitHubDeploy.name, STDERR_HANDLER) # Check if ghp-import is installed check_ghp_import_installed() @@ -95,8 +99,7 @@ class CommandGitHubDeploy(Command): return def _commit_and_push(self): - """ Commit all the files and push. """ - + """Commit all the files and push.""" source = self.site.config['GITHUB_SOURCE_BRANCH'] deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] remote = self.site.config['GITHUB_REMOTE_NAME'] diff --git a/nikola/plugins/command/import_wordpress.plugin b/nikola/plugins/command/import_wordpress.plugin index e072224..6c4384e 100644 --- a/nikola/plugins/command/import_wordpress.plugin +++ b/nikola/plugins/command/import_wordpress.plugin @@ -1,10 +1,13 @@ [Core] -Name = import_wordpress -Module = import_wordpress +name = import_wordpress +module = import_wordpress [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Import a wordpress site from a XML dump (requires markdown). +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Import a wordpress site from a XML dump (requires markdown). + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index 674fc2a..a652ec8 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -24,13 +24,18 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Import a WordPress dump.""" + from __future__ import unicode_literals, print_function import os import re import sys import datetime +import io +import json import requests from lxml import etree +from collections import defaultdict try: from urlparse import urlparse @@ -53,7 +58,37 @@ from nikola.plugins.command.init import SAMPLE_CONF, prepare_config, format_defa LOGGER = utils.get_logger('import_wordpress', utils.STDERR_HANDLER) +def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False): + """Install a Nikola plugin.""" + LOGGER.notice("Installing plugin '{0}'".format(plugin_name)) + # Get hold of the 'plugin' plugin + plugin_installer_info = site.plugin_manager.getPluginByName('plugin', 'Command') + if plugin_installer_info is None: + LOGGER.error('Internal error: cannot find the "plugin" plugin which is supposed to come with Nikola!') + return False + if not plugin_installer_info.is_activated: + # Someone might have disabled the plugin in the `conf.py` used + site.plugin_manager.activatePluginByName(plugin_installer_info.name) + plugin_installer_info.plugin_object.set_site(site) + plugin_installer = plugin_installer_info.plugin_object + # Try to install the requested plugin + options = {} + for option in plugin_installer.cmd_options: + options[option['name']] = option['default'] + options['install'] = plugin_name + options['output_dir'] = output_dir + options['show_install_notes'] = show_install_notes + if plugin_installer.execute(options=options) > 0: + return False + # Let the plugin manager find newly installed plugins + site.plugin_manager.collectPlugins() + # Re-scan for compiler extensions + site.compiler_extensions = site._activate_plugins_of_category("CompilerExtension") + return True + + class CommandImportWordpress(Command, ImportMixin): + """Import a WordPress dump.""" name = "import_wordpress" @@ -69,6 +104,20 @@ class CommandImportWordpress(Command, ImportMixin): 'type': bool, 'help': "Don't import drafts", }, + { + 'name': 'exclude_privates', + 'long': 'exclude-privates', + 'default': False, + 'type': bool, + 'help': "Don't import private posts", + }, + { + 'name': 'include_empty_items', + 'long': 'include-empty-items', + 'default': False, + 'type': bool, + 'help': "Include empty posts and pages", + }, { 'name': 'squash_newlines', 'long': 'squash-newlines', @@ -107,15 +156,57 @@ class CommandImportWordpress(Command, ImportMixin): 'type': str, 'help': "The pattern for translation files names", }, + { + 'name': 'export_categories_as_categories', + 'long': 'export-categories-as-categories', + 'default': False, + 'type': bool, + 'help': "Export categories as categories, instead of treating them as tags", + }, + { + 'name': 'export_comments', + 'long': 'export-comments', + 'default': False, + 'type': bool, + 'help': "Export comments as .wpcomment files", + }, + { + 'name': 'transform_to_html', + 'long': 'transform-to-html', + 'default': False, + 'type': bool, + 'help': "Uses WordPress page compiler to transform WordPress posts directly to HTML during import", + }, + { + 'name': 'use_wordpress_compiler', + 'long': 'use-wordpress-compiler', + 'default': False, + 'type': bool, + 'help': "Instead of converting posts to markdown, leave them as is and use the WordPress page compiler", + }, + { + 'name': 'install_wordpress_compiler', + 'long': 'install-wordpress-compiler', + 'default': False, + 'type': bool, + 'help': "Automatically installs the WordPress page compiler (either locally or in the new site) if required by other options.\nWarning: the compiler is GPL software!", + }, ] all_tags = set([]) - def _execute(self, options={}, args=[]): - """Import a WordPress blog from an export file into a Nikola site.""" - if not args: - print(self.help()) + def _find_wordpress_compiler(self): + """Find WordPress compiler plugin.""" + if self.wordpress_page_compiler is not None: return - + plugin_info = self.site.plugin_manager.getPluginByName('wordpress', 'PageCompiler') + if plugin_info is not None: + if not plugin_info.is_activated: + self.site.plugin_manager.activatePluginByName(plugin_info.name) + plugin_info.plugin_object.set_site(self.site) + self.wordpress_page_compiler = plugin_info.plugin_object + + def _read_options(self, options, args): + """Read command-line options.""" options['filename'] = args.pop(0) if args and ('output_folder' not in args or @@ -136,19 +227,76 @@ class CommandImportWordpress(Command, ImportMixin): self.output_folder = options.get('output_folder', 'new_site') self.exclude_drafts = options.get('exclude_drafts', False) + self.exclude_privates = options.get('exclude_privates', False) self.no_downloads = options.get('no_downloads', False) + self.import_empty_items = options.get('include_empty_items', False) + + self.export_categories_as_categories = options.get('export_categories_as_categories', False) + self.export_comments = options.get('export_comments', False) + + self.transform_to_html = options.get('transform_to_html', False) + self.use_wordpress_compiler = options.get('use_wordpress_compiler', False) + self.install_wordpress_compiler = options.get('install_wordpress_compiler', False) + self.wordpress_page_compiler = None self.auth = None if options.get('download_auth') is not None: username_password = options.get('download_auth') self.auth = tuple(username_password.split(':', 1)) if len(self.auth) < 2: - print("Please specify HTTP authentication credentials in the form username:password.") + LOGGER.error("Please specify HTTP authentication credentials in the form username:password.") return False self.separate_qtranslate_content = options.get('separate_qtranslate_content') self.translations_pattern = options.get('translations_pattern') + if self.transform_to_html and self.use_wordpress_compiler: + LOGGER.warn("It does not make sense to combine --transform-to-html with --use-wordpress-compiler, as the first converts all posts to HTML and the latter option affects zero posts.") + + if self.transform_to_html: + self._find_wordpress_compiler() + if not self.wordpress_page_compiler and self.install_wordpress_compiler: + if not install_plugin(self.site, 'wordpress_compiler', output_dir='plugins'): # local install + return False + self._find_wordpress_compiler() + if not self.wordpress_page_compiler: + LOGGER.error("To compile WordPress posts to HTML, the WordPress post compiler is needed. You can install it via:") + LOGGER.error(" nikola plugin -i wordpress_compiler") + LOGGER.error("Please note that the WordPress post compiler is licensed under the GPL v2.") + return False + + return True + + def _prepare(self, channel): + """Prepare context and category hierarchy.""" + self.context = self.populate_context(channel) + self.base_dir = urlparse(self.context['BASE_URL']).path + + if self.export_categories_as_categories: + wordpress_namespace = channel.nsmap['wp'] + cat_map = dict() + for cat in channel.findall('{{{0}}}category'.format(wordpress_namespace)): + # cat_id = get_text_tag(cat, '{{{0}}}term_id'.format(wordpress_namespace), None) + cat_slug = get_text_tag(cat, '{{{0}}}category_nicename'.format(wordpress_namespace), None) + cat_parent_slug = get_text_tag(cat, '{{{0}}}category_parent'.format(wordpress_namespace), None) + cat_name = get_text_tag(cat, '{{{0}}}cat_name'.format(wordpress_namespace), None) + cat_path = [cat_name] + if cat_parent_slug in cat_map: + cat_path = cat_map[cat_parent_slug] + cat_path + cat_map[cat_slug] = cat_path + self._category_paths = dict() + for cat, path in cat_map.items(): + self._category_paths[cat] = utils.join_hierarchical_category_path(path) + + def _execute(self, options={}, args=[]): + """Import a WordPress blog from an export file into a Nikola site.""" + if not args: + print(self.help()) + return False + + if not self._read_options(options, args): + return False + # A place holder where extra language (if detected) will be stored self.extra_languages = set() @@ -166,8 +314,7 @@ class CommandImportWordpress(Command, ImportMixin): req_missing(['phpserialize'], 'import WordPress dumps without --no-downloads') channel = self.get_channel_from_file(self.wordpress_export_file) - self.context = self.populate_context(channel) - self.base_dir = urlparse(self.context['BASE_URL']).path + self._prepare(channel) conf_template = self.generate_base_site() # If user has specified a custom pattern for translation files we @@ -181,6 +328,11 @@ class CommandImportWordpress(Command, ImportMixin): self.extra_languages) self.context['REDIRECTIONS'] = self.configure_redirections( self.url_map) + if self.timezone: + self.context['TIMEZONE'] = self.timezone + if self.export_categories_as_categories: + self.context['CATEGORY_ALLOW_HIERARCHIES'] = True + self.context['CATEGORY_OUTPUT_FLAT_HIERARCHY'] = True # Add tag redirects for tag in self.all_tags: @@ -197,18 +349,21 @@ class CommandImportWordpress(Command, ImportMixin): self.write_urlmap_csv( os.path.join(self.output_folder, 'url_map.csv'), self.url_map) rendered_template = conf_template.render(**prepare_config(self.context)) - rendered_template = re.sub('# REDIRECTIONS = ', 'REDIRECTIONS = ', - rendered_template) - - if self.timezone: - rendered_template = re.sub('# TIMEZONE = \'UTC\'', - 'TIMEZONE = \'' + self.timezone + '\'', - rendered_template) self.write_configuration(self.get_configuration_output_path(), rendered_template) + if self.use_wordpress_compiler: + if self.install_wordpress_compiler: + if not install_plugin(self.site, 'wordpress_compiler', output_dir=os.path.join(self.output_folder, 'plugins')): + return False + else: + LOGGER.warn("Make sure to install the WordPress page compiler via") + LOGGER.warn(" nikola plugin -i wordpress_compiler") + LOGGER.warn("in your imported blog's folder ({0}), if you haven't installed it system-wide or user-wide. Otherwise, your newly imported blog won't compile.".format(self.output_folder)) + @classmethod def read_xml_file(cls, filename): + """Read XML file into memory.""" xml = [] with open(filename, 'rb') as fd: @@ -221,12 +376,13 @@ class CommandImportWordpress(Command, ImportMixin): @classmethod def get_channel_from_file(cls, filename): + """Get channel from XML file.""" tree = etree.fromstring(cls.read_xml_file(filename)) channel = tree.find('channel') return channel - @staticmethod - def populate_context(channel): + def populate_context(self, channel): + """Populate context with config for the site.""" wordpress_namespace = channel.nsmap['wp'] context = SAMPLE_CONF.copy() @@ -255,28 +411,31 @@ class CommandImportWordpress(Command, ImportMixin): author, '{{{0}}}author_display_name'.format(wordpress_namespace), "Joe Example") - context['POSTS'] = '''( - ("posts/*.rst", "posts", "post.tmpl"), - ("posts/*.txt", "posts", "post.tmpl"), - ("posts/*.md", "posts", "post.tmpl"), - ("posts/*.wp", "posts", "post.tmpl"), - )''' - context['PAGES'] = '''( - ("stories/*.rst", "stories", "story.tmpl"), - ("stories/*.txt", "stories", "story.tmpl"), - ("stories/*.md", "stories", "story.tmpl"), - ("stories/*.wp", "stories", "story.tmpl"), - )''' - context['COMPILERS'] = '''{ - "rest": ('.txt', '.rst'), - "markdown": ('.md', '.mdown', '.markdown', '.wp'), - "html": ('.html', '.htm') - } - ''' + extensions = ['rst', 'txt', 'md', 'html'] + if self.use_wordpress_compiler: + extensions.append('wp') + POSTS = '(\n' + PAGES = '(\n' + for extension in extensions: + POSTS += ' ("posts/*.{0}", "posts", "post.tmpl"),\n'.format(extension) + PAGES += ' ("stories/*.{0}", "stories", "story.tmpl"),\n'.format(extension) + POSTS += ')\n' + PAGES += ')\n' + context['POSTS'] = POSTS + context['PAGES'] = PAGES + COMPILERS = '{\n' + COMPILERS += ''' "rest": ('.txt', '.rst'),''' + '\n' + COMPILERS += ''' "markdown": ('.md', '.mdown', '.markdown'),''' + '\n' + COMPILERS += ''' "html": ('.html', '.htm'),''' + '\n' + if self.use_wordpress_compiler: + COMPILERS += ''' "wordpress": ('.wp'),''' + '\n' + COMPILERS += '}' + context['COMPILERS'] = COMPILERS return context def download_url_content_to_file(self, url, dst_path): + """Download some content (attachments) to a file.""" if self.no_downloads: return @@ -291,6 +450,8 @@ class CommandImportWordpress(Command, ImportMixin): LOGGER.warn("Downloading {0} to {1} failed: {2}".format(url, dst_path, err)) def import_attachment(self, item, wordpress_namespace): + """Import an attachment to the site.""" + # Download main image url = get_text_tag( item, '{{{0}}}attachment_url'.format(wordpress_namespace), 'foo') link = get_text_tag(item, '{{{0}}}link'.format(wordpress_namespace), @@ -305,59 +466,136 @@ class CommandImportWordpress(Command, ImportMixin): links[link] = '/' + dst_url links[url] = '/' + dst_url - self.download_additional_image_sizes( - item, - wordpress_namespace, - os.path.dirname(url) - ) - - def download_additional_image_sizes(self, item, wordpress_namespace, source_path): - if phpserialize is None: - return + files = [path] + files_meta = [{}] additional_metadata = item.findall('{{{0}}}postmeta'.format(wordpress_namespace)) - if additional_metadata is None: - return - - for element in additional_metadata: - meta_key = element.find('{{{0}}}meta_key'.format(wordpress_namespace)) - if meta_key is not None and meta_key.text == '_wp_attachment_metadata': - meta_value = element.find('{{{0}}}meta_value'.format(wordpress_namespace)) - - if meta_value is None: - continue - - # Someone from Wordpress thought it was a good idea - # serialize PHP objects into that metadata field. Given - # that the export should give you the power to insert - # your blogging into another site or system its not. - # Why don't they just use JSON? - if sys.version_info[0] == 2: - try: - metadata = phpserialize.loads(utils.sys_encode(meta_value.text)) - except ValueError: - # local encoding might be wrong sometimes + if phpserialize and additional_metadata: + source_path = os.path.dirname(url) + for element in additional_metadata: + meta_key = element.find('{{{0}}}meta_key'.format(wordpress_namespace)) + if meta_key is not None and meta_key.text == '_wp_attachment_metadata': + meta_value = element.find('{{{0}}}meta_value'.format(wordpress_namespace)) + + if meta_value is None: + continue + + # Someone from Wordpress thought it was a good idea + # serialize PHP objects into that metadata field. Given + # that the export should give you the power to insert + # your blogging into another site or system its not. + # Why don't they just use JSON? + if sys.version_info[0] == 2: + try: + metadata = phpserialize.loads(utils.sys_encode(meta_value.text)) + except ValueError: + # local encoding might be wrong sometimes + metadata = phpserialize.loads(meta_value.text.encode('utf-8')) + else: metadata = phpserialize.loads(meta_value.text.encode('utf-8')) - else: - metadata = phpserialize.loads(meta_value.text.encode('utf-8')) - size_key = b'sizes' - file_key = b'file' - - if size_key not in metadata: - continue - - for filename in [metadata[size_key][size][file_key] for size in metadata[size_key]]: - url = '/'.join([source_path, filename.decode('utf-8')]) - path = urlparse(url).path - dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) - dst_url = '/'.join(dst_path.split(os.sep)[2:]) - links[url] = '/' + dst_url - links[url] = '/' + dst_url + meta_key = b'image_meta' + size_key = b'sizes' + file_key = b'file' + width_key = b'width' + height_key = b'height' + + # Extract metadata + if width_key in metadata and height_key in metadata: + files_meta[0]['width'] = int(metadata[width_key]) + files_meta[0]['height'] = int(metadata[height_key]) + + if meta_key in metadata: + image_meta = metadata[meta_key] + dst_meta = {} + + def add(our_key, wp_key, is_int=False, ignore_zero=False, is_float=False): + if wp_key in image_meta: + value = image_meta[wp_key] + if is_int: + value = int(value) + if ignore_zero and value == 0: + return + elif is_float: + value = float(value) + if ignore_zero and value == 0: + return + else: + value = value.decode('utf-8') # assume UTF-8 + if value == '': # skip empty values + return + dst_meta[our_key] = value + + add('aperture', b'aperture', is_float=True, ignore_zero=True) + add('credit', b'credit') + add('camera', b'camera') + add('caption', b'caption') + add('created_timestamp', b'created_timestamp', is_float=True, ignore_zero=True) + add('copyright', b'copyright') + add('focal_length', b'focal_length', is_float=True, ignore_zero=True) + add('iso', b'iso', is_float=True, ignore_zero=True) + add('shutter_speed', b'shutter_speed', ignore_zero=True, is_float=True) + add('title', b'title') + + if len(dst_meta) > 0: + files_meta[0]['meta'] = dst_meta + + # Find other sizes of image + if size_key not in metadata: + continue + + for size in metadata[size_key]: + filename = metadata[size_key][size][file_key] + url = '/'.join([source_path, filename.decode('utf-8')]) + + # Construct metadata + meta = {} + meta['size'] = size.decode('utf-8') + if width_key in metadata[size_key][size] and height_key in metadata[size_key][size]: + meta['width'] = metadata[size_key][size][width_key] + meta['height'] = metadata[size_key][size][height_key] + + path = urlparse(url).path + dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) + dst_url = '/'.join(dst_path.split(os.sep)[2:]) + links[url] = '/' + dst_url + + files.append(path) + files_meta.append(meta) + + # Prepare result + result = {} + result['files'] = files + result['files_meta'] = files_meta + + # Prepare extraction of more information + dc_namespace = item.nsmap['dc'] + content_namespace = item.nsmap['content'] + excerpt_namespace = item.nsmap['excerpt'] + + def add(result_key, key, namespace=None, filter=None, store_empty=False): + if namespace is not None: + value = get_text_tag(item, '{{{0}}}{1}'.format(namespace, key), None) + else: + value = get_text_tag(item, key, None) + if value is not None: + if filter: + value = filter(value) + if value or store_empty: + result[result_key] = value + + add('title', 'title') + add('date_utc', 'post_date_gmt', namespace=wordpress_namespace) + add('wordpress_user_name', 'creator', namespace=dc_namespace) + add('content', 'encoded', namespace=content_namespace) + add('excerpt', 'encoded', namespace=excerpt_namespace) + add('description', 'description') + + return result code_re1 = re.compile(r'\[code.* lang.*?="(.*?)?".*\](.*?)\[/code\]', re.DOTALL | re.MULTILINE) code_re2 = re.compile(r'\[sourcecode.* lang.*?="(.*?)?".*\](.*?)\[/sourcecode\]', re.DOTALL | re.MULTILINE) @@ -365,6 +603,7 @@ class CommandImportWordpress(Command, ImportMixin): code_re4 = re.compile(r'\[sourcecode.*?\](.*?)\[/sourcecode\]', re.DOTALL | re.MULTILINE) def transform_code(self, content): + """Transform code blocks.""" # http://en.support.wordpress.com/code/posting-source-code/. There are # a ton of things not supported here. We only do a basic [code # lang="x"] -> ```x translation, and remove quoted html entities (<, @@ -390,26 +629,126 @@ class CommandImportWordpress(Command, ImportMixin): @staticmethod def transform_caption(content): + """Transform captions.""" new_caption = re.sub(r'\[/caption\]', '', content) new_caption = re.sub(r'\[caption.*\]', '', new_caption) return new_caption def transform_multiple_newlines(self, content): - """Replaces multiple newlines with only two.""" + """Replace multiple newlines with only two.""" if self.squash_newlines: return re.sub(r'\n{3,}', r'\n\n', content) else: return content - def transform_content(self, content): - content = self.transform_code(content) - content = self.transform_caption(content) - content = self.transform_multiple_newlines(content) - return content + def transform_content(self, content, post_format, attachments): + """Transform content into appropriate format.""" + if post_format == 'wp': + if self.transform_to_html: + additional_data = {} + if attachments is not None: + additional_data['attachments'] = attachments + try: + content = self.wordpress_page_compiler.compile_to_string(content, additional_data=additional_data) + except TypeError: # old versions of the plugin don't support the additional argument + content = self.wordpress_page_compiler.compile_to_string(content) + return content, 'html', True + elif self.use_wordpress_compiler: + return content, 'wp', False + else: + content = self.transform_code(content) + content = self.transform_caption(content) + content = self.transform_multiple_newlines(content) + return content, 'md', True + elif post_format == 'markdown': + return content, 'md', True + elif post_format == 'none': + return content, 'html', True + else: + return None + + def _extract_comment(self, comment, wordpress_namespace): + """Extract comment from dump.""" + id = int(get_text_tag(comment, "{{{0}}}comment_id".format(wordpress_namespace), None)) + author = get_text_tag(comment, "{{{0}}}comment_author".format(wordpress_namespace), None) + author_email = get_text_tag(comment, "{{{0}}}comment_author_email".format(wordpress_namespace), None) + author_url = get_text_tag(comment, "{{{0}}}comment_author_url".format(wordpress_namespace), None) + author_IP = get_text_tag(comment, "{{{0}}}comment_author_IP".format(wordpress_namespace), None) + # date = get_text_tag(comment, "{{{0}}}comment_date".format(wordpress_namespace), None) + date_gmt = get_text_tag(comment, "{{{0}}}comment_date_gmt".format(wordpress_namespace), None) + content = get_text_tag(comment, "{{{0}}}comment_content".format(wordpress_namespace), None) + approved = get_text_tag(comment, "{{{0}}}comment_approved".format(wordpress_namespace), '0') + if approved == '0': + approved = 'hold' + elif approved == '1': + approved = 'approved' + elif approved == 'spam' or approved == 'trash': + pass + else: + LOGGER.warn("Unknown comment approved status: " + str(approved)) + parent = int(get_text_tag(comment, "{{{0}}}comment_parent".format(wordpress_namespace), 0)) + if parent == 0: + parent = None + user_id = int(get_text_tag(comment, "{{{0}}}comment_user_id".format(wordpress_namespace), 0)) + if user_id == 0: + user_id = None + + if approved == 'trash' or approved == 'spam': + return None + + return {"id": id, "status": str(approved), "approved": approved == "approved", + "author": author, "email": author_email, "url": author_url, "ip": author_IP, + "date": date_gmt, "content": content, "parent": parent, "user_id": user_id} + + def _write_comment(self, filename, comment): + """Write comment to file.""" + def write_header_line(fd, header_field, header_content): + """Write comment header line.""" + if header_content is None: + return + header_content = str(header_content).replace('\n', ' ') + line = '.. ' + header_field + ': ' + header_content + '\n' + fd.write(line.encode('utf8')) + + with open(filename, "wb+") as fd: + write_header_line(fd, "id", comment["id"]) + write_header_line(fd, "status", comment["status"]) + write_header_line(fd, "approved", comment["approved"]) + write_header_line(fd, "author", comment["author"]) + write_header_line(fd, "author_email", comment["email"]) + write_header_line(fd, "author_url", comment["url"]) + write_header_line(fd, "author_IP", comment["ip"]) + write_header_line(fd, "date_utc", comment["date"]) + write_header_line(fd, "parent_id", comment["parent"]) + write_header_line(fd, "wordpress_user_id", comment["user_id"]) + fd.write(('\n' + comment['content']).encode('utf8')) + + def _create_metadata(self, status, excerpt, tags, categories, post_name=None): + """Create post metadata.""" + other_meta = {'wp-status': status} + if excerpt is not None: + other_meta['excerpt'] = excerpt + if self.export_categories_as_categories: + cats = [] + for text in categories: + if text in self._category_paths: + cats.append(self._category_paths[text]) + else: + cats.append(utils.join_hierarchical_category_path([text])) + other_meta['categories'] = ','.join(cats) + if len(cats) > 0: + other_meta['category'] = cats[0] + if len(cats) > 1: + LOGGER.warn(('Post "{0}" has more than one category! ' + + 'Will only use the first one.').format(post_name)) + tags_cats = tags + else: + tags_cats = tags + categories + return tags_cats, other_meta - def import_item(self, item, wordpress_namespace, out_folder=None): - """Takes an item from the feed and creates a post file.""" + def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None): + """Take an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' @@ -439,7 +778,7 @@ class CommandImportWordpress(Command, ImportMixin): item, '{{{0}}}post_id'.format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) - return + return False else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) @@ -461,23 +800,42 @@ class CommandImportWordpress(Command, ImportMixin): item, '{{{0}}}status'.format(wordpress_namespace), 'publish') content = get_text_tag( item, '{http://purl.org/rss/1.0/modules/content/}encoded', '') + excerpt = get_text_tag( + item, '{http://wordpress.org/export/1.2/excerpt/}encoded', None) + + if excerpt is not None: + if len(excerpt) == 0: + excerpt = None tags = [] + categories = [] if status == 'trash': LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title)) - return + return False + elif status == 'private': + tags.append('private') + is_draft = False + is_private = True elif status != 'publish': tags.append('draft') is_draft = True + is_private = False else: is_draft = False + is_private = False for tag in item.findall('category'): text = tag.text - if text == 'Uncategorized': + type = 'category' + if 'domain' in tag.attrib: + type = tag.attrib['domain'] + if text == 'Uncategorized' and type == 'category': continue - tags.append(text) self.all_tags.add(text) + if type == 'category': + categories.append(type) + else: + tags.append(text) if '$latex' in content: tags.append('mathjax') @@ -487,11 +845,16 @@ class CommandImportWordpress(Command, ImportMixin): format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format'] if format_tag: post_format = format_tag[0].getparent().find('{%s}meta_value' % wordpress_namespace).text + if post_format == 'wpautop': + post_format = 'wp' if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) - - elif content.strip(): + return False + elif is_private and self.exclude_privates: + LOGGER.notice('Private post "{0}" will not be imported.'.format(title)) + return False + elif content.strip() or self.import_empty_items: # If no content is found, no files are written. self.url_map[link] = (self.context['SITE_URL'] + out_folder.rstrip('/') + '/' + slug + @@ -503,53 +866,121 @@ class CommandImportWordpress(Command, ImportMixin): content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): + try: + content, extension, rewrite_html = self.transform_content(content, post_format, attachments) + except: + LOGGER.error(('Cannot interpret post "{0}" (language {1}) with post ' + + 'format {2}!').format(os.path.join(out_folder, slug), lang, post_format)) + return False if lang: out_meta_filename = slug + '.meta' if lang == default_language: - out_content_filename = slug + '.wp' + out_content_filename = slug + '.' + extension else: out_content_filename \ = utils.get_translation_candidate(self.context, - slug + ".wp", lang) + slug + "." + extension, lang) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + '.meta' - out_content_filename = slug + '.wp' + out_content_filename = slug + '.' + extension meta_slug = slug - if post_format == 'wp': - content = self.transform_content(content) + tags, other_meta = self._create_metadata(status, excerpt, tags, categories, + post_name=os.path.join(out_folder, slug)) self.write_metadata(os.path.join(self.output_folder, out_folder, out_meta_filename), - title, meta_slug, post_date, description, tags) + title, meta_slug, post_date, description, tags, **other_meta) self.write_content( os.path.join(self.output_folder, out_folder, out_content_filename), - content) + content, + rewrite_html) + + if self.export_comments: + comments = [] + for tag in item.findall('{{{0}}}comment'.format(wordpress_namespace)): + comment = self._extract_comment(tag, wordpress_namespace) + if comment is not None: + comments.append(comment) + + for comment in comments: + comment_filename = slug + "." + str(comment['id']) + ".wpcomment" + self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment) + + return (out_folder, slug) else: - LOGGER.warn('Not going to import "{0}" because it seems to contain' - ' no content.'.format(title)) + LOGGER.warn(('Not going to import "{0}" because it seems to contain' + ' no content.').format(title)) + return False - def process_item(self, item): + def _extract_item_info(self, item): + """Extract information about an item.""" # The namespace usually is something like: # http://wordpress.org/export/1.2/ wordpress_namespace = item.nsmap['wp'] post_type = get_text_tag( item, '{{{0}}}post_type'.format(wordpress_namespace), 'post') + post_id = int(get_text_tag( + item, '{{{0}}}post_id'.format(wordpress_namespace), "0")) + parent_id = get_text_tag( + item, '{{{0}}}post_parent'.format(wordpress_namespace), None) + return wordpress_namespace, post_type, post_id, parent_id + + def process_item_if_attachment(self, item): + """Process attachments.""" + wordpress_namespace, post_type, post_id, parent_id = self._extract_item_info(item) if post_type == 'attachment': - self.import_attachment(item, wordpress_namespace) - elif post_type == 'post': - self.import_item(item, wordpress_namespace, 'posts') - else: - self.import_item(item, wordpress_namespace, 'stories') + data = self.import_attachment(item, wordpress_namespace) + # If parent was found, store relation with imported files + if parent_id is not None and int(parent_id) != 0: + self.attachments[int(parent_id)][post_id] = data + else: + LOGGER.warn("Attachment #{0} ({1}) has no parent!".format(post_id, data['files'])) + + def write_attachments_info(self, path, attachments): + """Write attachments info file.""" + with io.open(path, "wb") as file: + file.write(json.dumps(attachments).encode('utf-8')) + + def process_item_if_post_or_page(self, item): + """Process posts and pages.""" + wordpress_namespace, post_type, post_id, parent_id = self._extract_item_info(item) + + if post_type != 'attachment': + # Get attachments for post + attachments = self.attachments.pop(post_id, None) + # Import item + if post_type == 'post': + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'posts', attachments) + else: + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'stories', attachments) + # Process attachment data + if attachments is not None: + # If post was exported, store data + if out_folder_slug: + destination = os.path.join(self.output_folder, out_folder_slug[0], + out_folder_slug[1] + ".attachments.json") + self.write_attachments_info(destination, attachments) def import_posts(self, channel): + """Import posts into the site.""" + self.attachments = defaultdict(dict) + # First process attachments + for item in channel.findall('item'): + self.process_item_if_attachment(item) + # Next process posts for item in channel.findall('item'): - self.process_item(item) + self.process_item_if_post_or_page(item) + # Assign attachments to posts + for post_id in self.attachments: + LOGGER.warn(("Found attachments for post or page #{0}, but didn't find post or page. " + + "(Attachments: {1})").format(post_id, [e['files'][0] for e in self.attachments[post_id].values()])) def get_text_tag(tag, name, default): + """Get the text of an XML tag.""" if tag is None: return default t = tag.find(name) @@ -560,9 +991,10 @@ def get_text_tag(tag, name, default): def separate_qtranslate_content(text): - """Parse the content of a wordpress post or page and separate - the various language specific contents when they are delimited - with qtranslate tags: blabla""" + """Parse the content of a wordpress post or page and separate qtranslate languages. + + qtranslate tags: blabla + """ # TODO: uniformize qtranslate tags => qt_start = "" diff --git a/nikola/plugins/command/init.plugin b/nikola/plugins/command/init.plugin index 850dba9..a5404c4 100644 --- a/nikola/plugins/command/init.plugin +++ b/nikola/plugins/command/init.plugin @@ -1,9 +1,13 @@ [Core] -Name = init -Module = init +name = init +module = init [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Create a new site. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Create a new site. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py index 7a36894..91ccdb4 100644 --- a/nikola/plugins/command/init.py +++ b/nikola/plugins/command/init.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Create a new site.""" + from __future__ import print_function, unicode_literals import os import shutil @@ -54,6 +56,7 @@ SAMPLE_CONF = { 'BLOG_EMAIL': "joe@demo.site", 'BLOG_DESCRIPTION': "This is a demo site for Nikola.", 'PRETTY_URLS': False, + 'STRIP_INDEXES': False, 'DEFAULT_LANG': "en", 'TRANSLATIONS': """{ DEFAULT_LANG: "", @@ -64,6 +67,8 @@ SAMPLE_CONF = { 'TIMEZONE': 'UTC', 'COMMENT_SYSTEM': 'disqus', 'COMMENT_SYSTEM_ID': 'nikolademo', + 'CATEGORY_ALLOW_HIERARCHIES': False, + 'CATEGORY_OUTPUT_FLAT_HIERARCHY': False, 'TRANSLATIONS_PATTERN': DEFAULT_TRANSLATIONS_PATTERN, 'INDEX_READ_MORE_LINK': DEFAULT_INDEX_READ_MORE_LINK, 'RSS_READ_MORE_LINK': DEFAULT_RSS_READ_MORE_LINK, @@ -103,6 +108,7 @@ SAMPLE_CONF = { 'REDIRECTIONS': [], } + # Generate a list of supported languages here. # Ugly code follows. _suplang = {} @@ -154,8 +160,7 @@ SAMPLE_CONF['_SUPPORTED_COMMENT_SYSTEMS'] = '\n'.join(textwrap.wrap( def format_default_translations_config(additional_languages): - """Return the string to configure the TRANSLATIONS config variable to - make each additional language visible on the generated site.""" + """Adapt TRANSLATIONS setting for all additional languages.""" if not additional_languages: return SAMPLE_CONF["TRANSLATIONS"] lang_paths = [' DEFAULT_LANG: "",'] @@ -164,12 +169,12 @@ def format_default_translations_config(additional_languages): return "{{\n{0}\n}}".format("\n".join(lang_paths)) -def format_navigation_links(additional_languages, default_lang, messages): +def format_navigation_links(additional_languages, default_lang, messages, strip_indexes=False): """Return the string to configure NAVIGATION_LINKS.""" f = u"""\ {0}: ( ("{1}/archive.html", "{2[Archive]}"), - ("{1}/categories/index.html", "{2[Tags]}"), + ("{1}/categories/{3}", "{2[Tags]}"), ("{1}/rss.xml", "{2[RSS feed]}"), ),""" @@ -185,27 +190,32 @@ def format_navigation_links(additional_languages, default_lang, messages): fmsg[i] = i return fmsg + if strip_indexes: + index_html = '' + else: + index_html = 'index.html' + # handle the default language - pairs.append(f.format('DEFAULT_LANG', '', get_msg(default_lang))) + pairs.append(f.format('DEFAULT_LANG', '', get_msg(default_lang), index_html)) for l in additional_languages: - pairs.append(f.format(json.dumps(l, ensure_ascii=False), '/' + l, get_msg(l))) + pairs.append(f.format(json.dumps(l, ensure_ascii=False), '/' + l, get_msg(l), index_html)) return u'{{\n{0}\n}}'.format('\n\n'.join(pairs)) -# In order to ensure proper escaping, all variables but the three -# pre-formatted ones are handled by json.dumps(). +# In order to ensure proper escaping, all variables but the pre-formatted ones +# are handled by json.dumps(). def prepare_config(config): """Parse sample config with JSON.""" p = config.copy() - p.update(dict((k, json.dumps(v, ensure_ascii=False)) for k, v in p.items() - if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK', 'PRETTY_URLS'))) + p.update({k: json.dumps(v, ensure_ascii=False) for k, v in p.items() + if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK')}) # READ_MORE_LINKs require some special treatment. p['INDEX_READ_MORE_LINK'] = "'" + p['INDEX_READ_MORE_LINK'].replace("'", "\\'") + "'" p['RSS_READ_MORE_LINK'] = "'" + p['RSS_READ_MORE_LINK'].replace("'", "\\'") + "'" - # json would make that `true` instead of `True` - p['PRETTY_URLS'] = str(p['PRETTY_URLS']) + # fix booleans and None + p.update({k: str(v) for k, v in config.items() if isinstance(v, bool) or v is None}) return p @@ -239,11 +249,13 @@ class CommandInit(Command): @classmethod def copy_sample_site(cls, target): + """Copy sample site data to target directory.""" src = resource_filename('nikola', os.path.join('data', 'samplesite')) shutil.copytree(src, target) @staticmethod def create_configuration(target): + """Create configuration file.""" template_path = resource_filename('nikola', 'conf.py.in') conf_template = Template(filename=template_path) conf_path = os.path.join(target, 'conf.py') @@ -252,12 +264,14 @@ class CommandInit(Command): @staticmethod def create_configuration_to_string(): + """Return configuration file as a string.""" template_path = resource_filename('nikola', 'conf.py.in') conf_template = Template(filename=template_path) return conf_template.render(**prepare_config(SAMPLE_CONF)) @classmethod def create_empty_site(cls, target): + """Create an empty site with directories only.""" for folder in ('files', 'galleries', 'listings', 'posts', 'stories'): makedirs(os.path.join(target, folder)) @@ -295,7 +309,8 @@ class CommandInit(Command): SAMPLE_CONF['SITE_URL'] = answer def prettyhandler(default, toconf): - SAMPLE_CONF['PRETTY_URLS'] = ask_yesno('Enable pretty URLs (/page/ instead of /page.html) that don’t need web server configuration?', default=True) + SAMPLE_CONF['PRETTY_URLS'] = ask_yesno('Enable pretty URLs (/page/ instead of /page.html) that don\'t need web server configuration?', default=True) + SAMPLE_CONF['STRIP_INDEXES'] = SAMPLE_CONF['PRETTY_URLS'] def lhandler(default, toconf, show_header=True): if show_header: @@ -333,7 +348,7 @@ class CommandInit(Command): # not inherit from anywhere. try: messages = load_messages(['base'], tr, default) - SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages) + SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages, SAMPLE_CONF['STRIP_INDEXES']) except nikola.utils.LanguageNotFoundError as e: print(" ERROR: the language '{0}' is not supported.".format(e.lang)) print(" Are you sure you spelled the name correctly? Names are case-sensitive and need to be reproduced as-is (complete with the country specifier, if any).") diff --git a/nikola/plugins/command/install_theme.plugin b/nikola/plugins/command/install_theme.plugin index 54a91ff..8434f2e 100644 --- a/nikola/plugins/command/install_theme.plugin +++ b/nikola/plugins/command/install_theme.plugin @@ -1,10 +1,13 @@ [Core] -Name = install_theme -Module = install_theme +name = install_theme +module = install_theme [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Install a theme into the current site. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Install a theme into the current site. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py index 4937509..f02252e 100644 --- a/nikola/plugins/command/install_theme.py +++ b/nikola/plugins/command/install_theme.py @@ -24,10 +24,12 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Install a theme.""" + from __future__ import print_function import os import io -import json +import time import requests import pygments @@ -41,6 +43,7 @@ LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER) class CommandInstallTheme(Command): + """Install a theme.""" name = "install_theme" @@ -95,8 +98,13 @@ class CommandInstallTheme(Command): if name is None and not listing: LOGGER.error("This command needs either a theme name or the -l option.") return False - data = requests.get(url).text - data = json.loads(data) + try: + data = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + data = requests.get(url).json() if listing: print("Themes:") print("-------") @@ -122,11 +130,21 @@ class CommandInstallTheme(Command): LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) def do_install(self, name, data): + """Download and install a theme.""" if name in data: utils.makedirs(self.output_dir) - LOGGER.info("Downloading '{0}'".format(data[name])) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + zip_file = io.BytesIO() - zip_file.write(requests.get(data[name]).content) + zip_file.write(zip_data) LOGGER.info("Extracting '{0}' into themes/".format(name)) utils.extract_all(zip_file) dest_path = os.path.join(self.output_dir, name) diff --git a/nikola/plugins/command/new_page.plugin b/nikola/plugins/command/new_page.plugin index f078dd6..145a419 100644 --- a/nikola/plugins/command/new_page.plugin +++ b/nikola/plugins/command/new_page.plugin @@ -1,9 +1,13 @@ [Core] -Name = new_page -Module = new_page +name = new_page +module = new_page [Documentation] -Author = Roberto Alsina, Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = Create a new page. +author = Roberto Alsina, Chris Warrick +version = 1.0 +website = http://getnikola.com +description = Create a new page. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index 39a85bd..811e28b 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -24,12 +24,15 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Create a new page.""" + from __future__ import unicode_literals, print_function from nikola.plugin_categories import Command class CommandNewPage(Command): + """Create a new page.""" name = "new_page" diff --git a/nikola/plugins/command/new_post.plugin b/nikola/plugins/command/new_post.plugin index fec4b1d..d88469f 100644 --- a/nikola/plugins/command/new_post.plugin +++ b/nikola/plugins/command/new_post.plugin @@ -1,10 +1,13 @@ [Core] -Name = new_post -Module = new_post +name = new_post +module = new_post [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Create a new post. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Create a new post. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index 5141c7e..f9fe3ff 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Create a new post.""" + from __future__ import unicode_literals, print_function import io import datetime @@ -44,107 +46,8 @@ PAGELOGGER = utils.get_logger('new_page', utils.STDERR_HANDLER) LOGGER = POSTLOGGER -def filter_post_pages(compiler, is_post, compilers, post_pages, compiler_objs, compilers_raw): - """Given a compiler ("markdown", "rest"), and whether it's meant for - a post or a page, and compilers, return the correct entry from - post_pages.""" - - # First throw away all the post_pages with the wrong is_post - filtered = [entry for entry in post_pages if entry[3] == is_post] - - # These are the extensions supported by the required format - extensions = compilers.get(compiler) - if extensions is None: - if compiler in compiler_objs: - LOGGER.error("There is a {0} compiler available, but it's not set in your COMPILERS option.".format(compiler)) - LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) - else: - LOGGER.error('Unknown format {0}'.format(compiler)) - print_compilers(compilers_raw, post_pages, compiler_objs) - return False - - # Throw away the post_pages with the wrong extensions - filtered = [entry for entry in filtered if any([ext in entry[0] for ext in - extensions])] - - if not filtered: - type_name = "post" if is_post else "page" - LOGGER.error("Can't find a way, using your configuration, to create " - "a {0} in format {1}. You may want to tweak " - "COMPILERS or {2}S in conf.py".format( - type_name, compiler, type_name.upper())) - LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) - - return False - return filtered[0] - - -def print_compilers(compilers_raw, post_pages, compiler_objs): - """ - List all available compilers in a human-friendly format. - - :param compilers_raw: The compilers dict, mapping compiler names to tuples of extensions - :param post_pages: The post_pages structure - :param compilers_objs: Compiler objects - """ - - # We use compilers_raw, because the normal dict can contain - # garbage coming from the translation candidate implementation. - # Entries are in format: (name, extensions, used_in_post_pages) - parsed_compilers = {'used': [], 'unused': [], 'disabled': []} - - for compiler_name, compiler_obj in compiler_objs.items(): - fname = compiler_obj.friendly_name or compiler_name - if compiler_name not in compilers_raw: - parsed_compilers['disabled'].append((compiler_name, fname, (), False)) - else: - # stolen from filter_post_pages - extensions = compilers_raw[compiler_name] - filtered = [entry for entry in post_pages if any( - [ext in entry[0] for ext in extensions])] - if filtered: - parsed_compilers['used'].append((compiler_name, fname, extensions, True)) - else: - parsed_compilers['unused'].append((compiler_name, fname, extensions, False)) - - # Sort compilers alphabetically by name, just so it’s prettier (and - # deterministic) - parsed_compilers['used'].sort(key=operator.itemgetter(0)) - parsed_compilers['unused'].sort(key=operator.itemgetter(0)) - parsed_compilers['disabled'].sort(key=operator.itemgetter(0)) - - # We also group the compilers by status for readability. - parsed_list = parsed_compilers['used'] + parsed_compilers['unused'] + parsed_compilers['disabled'] - - print("Available input formats:\n") - - name_width = max([len(i[0]) for i in parsed_list] + [4]) # 4 == len('NAME') - fname_width = max([len(i[1]) for i in parsed_list] + [11]) # 11 == len('DESCRIPTION') - - print((' {0:<' + str(name_width) + '} {1:<' + str(fname_width) + '} EXTENSIONS\n').format('NAME', 'DESCRIPTION')) - - for name, fname, extensions, used in parsed_list: - flag = ' ' if used else '!' - flag = flag if extensions else '~' - - extensions = ', '.join(extensions) if extensions else '(disabled: not in COMPILERS)' - - print(('{flag}{name:<' + str(name_width) + '} {fname:<' + str(fname_width) + '} {extensions}').format(flag=flag, name=name, fname=fname, extensions=extensions)) - - print(""" -More compilers are available in the Plugins Index. - -Compilers marked with ! and ~ require additional configuration: - ! not in the PAGES/POSTS tuples (unused) - ~ not in the COMPILERS dict (disabled) -Read more: {0}""".format(COMPILERS_DOC_LINK)) - - def get_default_compiler(is_post, compilers, post_pages): - """Given compilers and post_pages, return a reasonable - default compiler for this kind of post/page. - """ - + """Given compilers and post_pages, return a reasonable default compiler for this kind of post/page.""" # First throw away all the post_pages with the wrong is_post filtered = [entry for entry in post_pages if entry[3] == is_post] @@ -159,7 +62,7 @@ def get_default_compiler(is_post, compilers, post_pages): def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): - """Returns a date stamp, given a recurrence rule. + """Return a date stamp, given a recurrence rule. schedule - bool: whether to use the recurrence rule or not @@ -177,7 +80,6 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): whether to force ISO 8601 dates (instead of locale-specific ones) """ - if tz is None: tz = dateutil.tz.tzlocal() date = now = datetime.datetime.now(tz) @@ -212,6 +114,7 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): class CommandNewPost(Command): + """Create a new post.""" name = "new_post" @@ -333,7 +236,7 @@ class CommandNewPost(Command): wants_available = options['available-formats'] if wants_available: - print_compilers(self.site.config['_COMPILERS_RAW'], self.site.config['post_pages'], self.site.compilers) + self.print_compilers() return if is_page: @@ -360,17 +263,13 @@ class CommandNewPost(Command): if content_format not in compiler_names: LOGGER.error("Unknown {0} format {1}, maybe you need to install a plugin?".format(content_type, content_format)) - print_compilers(self.site.config['_COMPILERS_RAW'], self.site.config['post_pages'], self.site.compilers) + self.print_compilers() return compiler_plugin = self.site.plugin_manager.getPluginByName( content_format, "PageCompiler").plugin_object # Guess where we should put this - entry = filter_post_pages(content_format, is_post, - self.site.config['COMPILERS'], - self.site.config['post_pages'], - self.site.compilers, - self.site.config['_COMPILERS_RAW']) + entry = self.filter_post_pages(content_format, is_post) if entry is False: return 1 @@ -497,3 +396,122 @@ class CommandNewPost(Command): subprocess.call(to_run) else: LOGGER.error('$EDITOR not set, cannot edit the post. Please do it manually.') + + def filter_post_pages(self, compiler, is_post): + """Return the correct entry from post_pages. + + Information based on: + * selected compilers + * available compilers + * post/page status + """ + compilers = self.site.config['COMPILERS'] + post_pages = self.site.config['post_pages'] + compiler_objs = self.site.compilers + + # First throw away all the post_pages with the wrong is_post + filtered = [entry for entry in post_pages if entry[3] == is_post] + + # These are the extensions supported by the required format + extensions = compilers.get(compiler) + if extensions is None: + if compiler in compiler_objs: + LOGGER.error("There is a {0} compiler available, but it's not set in your COMPILERS option.".format(compiler)) + LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) + else: + LOGGER.error('Unknown format {0}'.format(compiler)) + self.print_compilers() + return False + + # Throw away the post_pages with the wrong extensions + filtered = [entry for entry in filtered if any([ext in entry[0] for ext in + extensions])] + + if not filtered: + type_name = "post" if is_post else "page" + LOGGER.error("Can't find a way, using your configuration, to create " + "a {0} in format {1}. You may want to tweak " + "COMPILERS or {2}S in conf.py".format( + type_name, compiler, type_name.upper())) + LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) + + return False + return filtered[0] + + def print_compilers(self): + """List all available compilers in a human-friendly format.""" + # We use compilers_raw, because the normal dict can contain + # garbage coming from the translation candidate implementation. + # Entries are in format: (name, extensions, used_in_post_pages) + + compilers_raw = self.site.config['_COMPILERS_RAW'] + + used_compilers = [] + unused_compilers = [] + disabled_compilers = [] + + for name, plugin in self.site.compilers.items(): + if name in compilers_raw: + used_compilers.append([ + name, + plugin.friendly_name or name, + compilers_raw[name], + True + ]) + else: + disabled_compilers.append([ + name, + plugin.friendly_name or name, + (), + False + ]) + + for name, (_, _, pi) in self.site.disabled_compilers.items(): + if pi.details.has_option('Nikola', 'Friendlyname'): + f_name = pi.details.get('Nikola', 'Friendlyname') + else: + f_name = name + if name in compilers_raw: + unused_compilers.append([ + name, + f_name, + compilers_raw[name], + False + ]) + else: + disabled_compilers.append([ + name, + f_name, + (), + False + ]) + + used_compilers.sort(key=operator.itemgetter(0)) + unused_compilers.sort(key=operator.itemgetter(0)) + disabled_compilers.sort(key=operator.itemgetter(0)) + + # We also group the compilers by status for readability. + parsed_list = used_compilers + unused_compilers + disabled_compilers + + print("Available input formats:\n") + + name_width = max([len(i[0]) for i in parsed_list] + [4]) # 4 == len('NAME') + fname_width = max([len(i[1]) for i in parsed_list] + [11]) # 11 == len('DESCRIPTION') + + print((' {0:<' + str(name_width) + '} {1:<' + str(fname_width) + '} EXTENSIONS\n').format('NAME', 'DESCRIPTION')) + + for name, fname, extensions, used in parsed_list: + flag = ' ' if used else '!' + flag = flag if extensions else '~' + + extensions = ', '.join(extensions) if extensions else '(disabled: not in COMPILERS)' + + print(('{flag}{name:<' + str(name_width) + '} {fname:<' + str(fname_width) + '} {extensions}').format(flag=flag, name=name, fname=fname, extensions=extensions)) + + print(""" + More compilers are available in the Plugins Index. + + Compilers marked with ! and ~ require additional configuration: + ! not in the PAGES/POSTS tuples (unused) + ~ not in the COMPILERS dict (disabled) + Read more: {0}""".format(COMPILERS_DOC_LINK)) diff --git a/nikola/plugins/command/orphans.plugin b/nikola/plugins/command/orphans.plugin index f491eaf..669429d 100644 --- a/nikola/plugins/command/orphans.plugin +++ b/nikola/plugins/command/orphans.plugin @@ -1,10 +1,13 @@ [Core] -Name = orphans -Module = orphans +name = orphans +module = orphans [Documentation] -Author = Roberto Alsina, Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = List all orphans +author = Roberto Alsina, Chris Warrick +version = 1.0 +website = http://getnikola.com +description = List all orphans + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py index f550e17..b12cc67 100644 --- a/nikola/plugins/command/orphans.py +++ b/nikola/plugins/command/orphans.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""List all orphans.""" + from __future__ import print_function import os @@ -32,6 +34,9 @@ from nikola.plugins.command.check import real_scan_files class CommandOrphans(Command): + + """List all orphans.""" + name = "orphans" doc_purpose = "list all orphans" doc_description = """\ @@ -41,5 +46,6 @@ but are not generated by Nikola. Output contains filenames only (it is passable to `xargs rm` or the like).""" def _execute(self, options, args): + """Run the orphans command.""" orphans = real_scan_files(self.site)[0] print('\n'.join([p for p in orphans if not os.path.isdir(p)])) diff --git a/nikola/plugins/command/plugin.plugin b/nikola/plugins/command/plugin.plugin index 2815caa..d44dcf3 100644 --- a/nikola/plugins/command/plugin.plugin +++ b/nikola/plugins/command/plugin.plugin @@ -1,10 +1,13 @@ [Core] -Name = plugin -Module = plugin +name = plugin +module = plugin [Documentation] -Author = Roberto Alsina and Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = Manage Nikola plugins +author = Roberto Alsina and Chris Warrick +version = 1.0 +website = http://getnikola.com +description = Manage Nikola plugins + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index 56eb1d7..f892ee9 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -24,12 +24,14 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Manage plugins.""" + from __future__ import print_function import io import os import shutil import subprocess -import sys +import time import requests import pygments @@ -43,6 +45,7 @@ LOGGER = utils.get_logger('plugin', utils.STDERR_HANDLER) class CommandPlugin(Command): + """Manage plugins.""" json = None @@ -119,6 +122,7 @@ class CommandPlugin(Command): upgrade = options.get('upgrade') list_available = options.get('list') list_installed = options.get('list_installed') + show_install_notes = options.get('show_install_notes', True) command_count = [bool(x) for x in ( install, uninstall, @@ -127,37 +131,42 @@ class CommandPlugin(Command): list_installed)].count(True) if command_count > 1 or command_count == 0: print(self.help()) - return + return 2 - if not self.site.configured and not user_mode and install: - LOGGER.notice('No site found, assuming --user') - user_mode = True - - if user_mode: - self.output_dir = os.path.expanduser('~/.nikola/plugins') + if options.get('output_dir') is not None: + self.output_dir = options.get('output_dir') else: - self.output_dir = 'plugins' + if not self.site.configured and not user_mode and install: + LOGGER.notice('No site found, assuming --user') + user_mode = True + + if user_mode: + self.output_dir = os.path.expanduser('~/.nikola/plugins') + else: + self.output_dir = 'plugins' if list_available: - self.list_available(url) + return self.list_available(url) elif list_installed: - self.list_installed() + return self.list_installed() elif upgrade: - self.do_upgrade(url) + return self.do_upgrade(url) elif uninstall: - self.do_uninstall(uninstall) + return self.do_uninstall(uninstall) elif install: - self.do_install(url, install) + return self.do_install(url, install, show_install_notes) def list_available(self, url): + """List all available plugins.""" data = self.get_json(url) print("Available Plugins:") print("------------------") for plugin in sorted(data.keys()): print(plugin) - return True + return 0 def list_installed(self): + """List installed plugins.""" plugins = [] for plugin in self.site.plugin_manager.getAllPlugins(): p = plugin.path @@ -170,8 +179,10 @@ class CommandPlugin(Command): plugins.sort() for name, path in plugins: print('{0} at {1}'.format(name, path)) + return 0 def do_upgrade(self, url): + """Upgrade all installed plugins.""" LOGGER.warning('This is not very smart, it just reinstalls some plugins and hopes for the best') data = self.get_json(url) plugins = [] @@ -194,18 +205,29 @@ class CommandPlugin(Command): break elif tail == '': LOGGER.error("Can't find the plugins folder for path: {0}".format(p)) - return False + return 1 else: path = tail self.do_install(url, name) + return 0 - def do_install(self, url, name): + def do_install(self, url, name, show_install_notes=True): + """Download and install a plugin.""" data = self.get_json(url) if name in data: utils.makedirs(self.output_dir) - LOGGER.info('Downloading: ' + data[name]) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + zip_file = io.BytesIO() - zip_file.write(requests.get(data[name]).content) + zip_file.write(zip_data) LOGGER.info('Extracting: {0} into {1}/'.format(name, self.output_dir)) utils.extract_all(zip_file, self.output_dir) dest_path = os.path.join(self.output_dir, name) @@ -214,13 +236,13 @@ class CommandPlugin(Command): plugin_path = utils.get_plugin_path(name) except: LOGGER.error("Can't find plugin " + name) - return False + return 1 utils.makedirs(self.output_dir) dest_path = os.path.join(self.output_dir, name) if os.path.exists(dest_path): LOGGER.error("{0} is already installed".format(name)) - return False + return 1 LOGGER.info('Copying {0} into plugins'.format(plugin_path)) shutil.copytree(plugin_path, dest_path) @@ -256,7 +278,7 @@ class CommandPlugin(Command): print('You have to install those yourself or through a package ' 'manager.') confpypath = os.path.join(dest_path, 'conf.py.sample') - if os.path.exists(confpypath): + if os.path.exists(confpypath) and show_install_notes: LOGGER.notice('This plugin has a sample config file. Integrate it with yours in order to make this plugin work!') print('Contents of the conf.py.sample file:\n') with io.open(confpypath, 'r', encoding='utf-8') as fh: @@ -266,9 +288,10 @@ class CommandPlugin(Command): 4 * ' ')) else: print(utils.indent(fh.read(), 4 * ' ')) - return True + return 0 def do_uninstall(self, name): + """Uninstall a plugin.""" for plugin in self.site.plugin_manager.getAllPlugins(): # FIXME: this is repeated thrice p = plugin.path if os.path.isdir(p): @@ -278,16 +301,23 @@ class CommandPlugin(Command): if name == plugin.name: # Uninstall this one LOGGER.warning('About to uninstall plugin: {0}'.format(name)) LOGGER.warning('This will delete {0}'.format(p)) - inpf = raw_input if sys.version_info[0] == 2 else input - sure = inpf('Are you sure? [y/n] ') - if sure.lower().startswith('y'): + sure = utils.ask_yesno('Are you sure?') + if sure: LOGGER.warning('Removing {0}'.format(p)) shutil.rmtree(p) - return True + return 0 + return 1 LOGGER.error('Unknown plugin: {0}'.format(name)) - return False + return 1 def get_json(self, url): + """Download the JSON file with all plugins.""" if self.json is None: - self.json = requests.get(url).json() + try: + self.json = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + self.json = requests.get(url).json() return self.json diff --git a/nikola/plugins/command/rst2html.plugin b/nikola/plugins/command/rst2html.plugin index 0d0d3b0..02c9276 100644 --- a/nikola/plugins/command/rst2html.plugin +++ b/nikola/plugins/command/rst2html.plugin @@ -1,9 +1,13 @@ [Core] -Name = rst2html -Module = rst2html +name = rst2html +module = rst2html [Documentation] -Author = Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = Compile reStructuredText to HTML using the Nikola architecture +author = Chris Warrick +version = 1.0 +website = http://getnikola.com +description = Compile reStructuredText to HTML using the Nikola architecture + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py index 342aaeb..06afffd 100644 --- a/nikola/plugins/command/rst2html/__init__.py +++ b/nikola/plugins/command/rst2html/__init__.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Compile reStructuredText to HTML, using Nikola architecture.""" + from __future__ import unicode_literals, print_function import io @@ -34,6 +36,7 @@ from nikola.plugin_categories import Command class CommandRst2Html(Command): + """Compile reStructuredText to HTML, using Nikola architecture.""" name = "rst2html" diff --git a/nikola/plugins/command/serve.plugin b/nikola/plugins/command/serve.plugin index 0c1176d..aca71ec 100644 --- a/nikola/plugins/command/serve.plugin +++ b/nikola/plugins/command/serve.plugin @@ -1,10 +1,13 @@ [Core] -Name = serve -Module = serve +name = serve +module = serve [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Start test server. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Start test server. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py index 0e4d01f..0441c93 100644 --- a/nikola/plugins/command/serve.py +++ b/nikola/plugins/command/serve.py @@ -24,8 +24,11 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Start test server.""" + from __future__ import print_function import os +import re import socket import webbrowser try: @@ -35,16 +38,25 @@ except ImportError: from http.server import HTTPServer # NOQA from http.server import SimpleHTTPRequestHandler # NOQA +try: + from StringIO import StringIO +except ImportError: + from io import BytesIO as StringIO # NOQA + + from nikola.plugin_categories import Command -from nikola.utils import get_logger +from nikola.utils import get_logger, STDERR_HANDLER class IPv6Server(HTTPServer): + """An IPv6 HTTPServer.""" + address_family = socket.AF_INET6 class CommandServe(Command): + """Start test server.""" name = "serve" @@ -69,6 +81,14 @@ class CommandServe(Command): 'default': '', 'help': 'Address to bind (default: 0.0.0.0 – all local IPv4 interfaces)', }, + { + 'name': 'detach', + 'short': 'd', + 'long': 'detach', + 'type': bool, + 'default': False, + 'help': 'Detach from TTY (work in the background)', + }, { 'name': 'browser', 'short': 'b', @@ -89,7 +109,7 @@ class CommandServe(Command): def _execute(self, options, args): """Start test server.""" - self.logger = get_logger('serve', self.site.loghandlers) + self.logger = get_logger('serve', STDERR_HANDLER) out_dir = self.site.config['OUTPUT_FOLDER'] if not os.path.isdir(out_dir): self.logger.error("Missing '{0}' folder?".format(out_dir)) @@ -117,16 +137,42 @@ class CommandServe(Command): server_url = "http://{0}:{1}/".format(*sa) self.logger.info("Opening {0} in the default web browser...".format(server_url)) webbrowser.open(server_url) - try: - httpd.serve_forever() - except KeyboardInterrupt: - self.logger.info("Server is shutting down.") - return 130 + if options['detach']: + OurHTTPRequestHandler.quiet = True + try: + pid = os.fork() + if pid == 0: + httpd.serve_forever() + else: + self.logger.info("Detached with PID {0}. Run `kill {0}` to stop the server.".format(pid)) + except AttributeError as e: + if os.name == 'nt': + self.logger.warning("Detaching is not available on Windows, server is running in the foreground.") + else: + raise e + else: + try: + httpd.serve_forever() + except KeyboardInterrupt: + self.logger.info("Server is shutting down.") + return 130 class OurHTTPRequestHandler(SimpleHTTPRequestHandler): + + """A request handler, modified for Nikola.""" + extensions_map = dict(SimpleHTTPRequestHandler.extensions_map) extensions_map[""] = "text/plain" + quiet = False + + def log_message(self, *args): + """Log messages. Or not, depending on a setting.""" + if self.quiet: + return + else: + # Old-style class in Python 2.7, cannot use super() + return SimpleHTTPRequestHandler.log_message(self, *args) # NOTICE: this is a patched version of send_head() to disable all sorts of # caching. `nikola serve` is a development server, hence caching should @@ -182,14 +228,31 @@ class OurHTTPRequestHandler(SimpleHTTPRequestHandler): except IOError: self.send_error(404, "File not found") return None + + filtered_bytes = None + if ctype == 'text/html': + # Comment out any to allow local resolution of relative URLs. + data = f.read().decode('utf8') + f.close() + data = re.sub(r']*)>', '', data, re.IGNORECASE) + data = data.encode('utf8') + f = StringIO() + f.write(data) + filtered_bytes = len(data) + f.seek(0) + self.send_response(200) self.send_header("Content-type", ctype) if os.path.splitext(path)[1] == '.svgz': # Special handling for svgz to make it work nice with browsers. self.send_header("Content-Encoding", 'gzip') - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + + if filtered_bytes is None: + fs = os.fstat(f.fileno()) + self.send_header('Content-Length', str(fs[6])) + else: + self.send_header('Content-Length', filtered_bytes) + # begin no-cache patch # For standard requests. self.send_header("Cache-Control", "no-cache, no-store, " diff --git a/nikola/plugins/command/status.plugin b/nikola/plugins/command/status.plugin index e02da8b..91390d2 100644 --- a/nikola/plugins/command/status.plugin +++ b/nikola/plugins/command/status.plugin @@ -1,9 +1,13 @@ [Core] -Name = status -Module = status +name = status +module = status [Documentation] -Author = Daniel Aleksandersen -Version = 1.0 -Website = https://getnikola.com -Description = Site status +author = Daniel Aleksandersen +version = 1.0 +website = https://getnikola.com +description = Site status + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py index b8a6a60..55e7f95 100644 --- a/nikola/plugins/command/status.py +++ b/nikola/plugins/command/status.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Display site status.""" + from __future__ import print_function import io import os @@ -33,8 +35,10 @@ from dateutil.tz import gettz, tzlocal from nikola.plugin_categories import Command -class CommandDeploy(Command): - """ Site status. """ +class CommandStatus(Command): + + """Display site status.""" + name = "status" doc_purpose = "display site status" @@ -69,7 +73,7 @@ class CommandDeploy(Command): ] def _execute(self, options, args): - + """Display site status.""" self.site.scan_posts() timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") @@ -128,6 +132,7 @@ class CommandDeploy(Command): print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts))) def human_time(self, dt): + """Translate time into a human-friendly representation.""" days = dt.days hours = dt.seconds / 60 // 60 minutes = dt.seconds / 60 - (hours * 60) diff --git a/nikola/plugins/command/version.plugin b/nikola/plugins/command/version.plugin index a3f58e8..4708bdb 100644 --- a/nikola/plugins/command/version.plugin +++ b/nikola/plugins/command/version.plugin @@ -1,9 +1,13 @@ [Core] -Name = version -Module = version +name = version +module = version [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Show nikola version +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Show nikola version + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py index b6520d7..ad08f64 100644 --- a/nikola/plugins/command/version.py +++ b/nikola/plugins/command/version.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Print Nikola version.""" + from __future__ import print_function import lxml @@ -36,7 +38,8 @@ URL = 'https://pypi.python.org/pypi?:action=doap&name=Nikola' class CommandVersion(Command): - """Print the version.""" + + """Print Nikola version.""" name = "version" diff --git a/nikola/plugins/compile/__init__.py b/nikola/plugins/compile/__init__.py index a1d17a6..60f1919 100644 --- a/nikola/plugins/compile/__init__.py +++ b/nikola/plugins/compile/__init__.py @@ -23,3 +23,5 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Compilers for Nikola.""" diff --git a/nikola/plugins/compile/html.plugin b/nikola/plugins/compile/html.plugin index 66623b2..53ade61 100644 --- a/nikola/plugins/compile/html.plugin +++ b/nikola/plugins/compile/html.plugin @@ -1,10 +1,13 @@ [Core] -Name = html -Module = html +name = html +module = html [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Compile HTML into HTML (just copy) +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Compile HTML into HTML (just copy) +[Nikola] +plugincategory = Compiler +friendlyname = HTML diff --git a/nikola/plugins/compile/html.py b/nikola/plugins/compile/html.py index ab0c2f6..5f8b244 100644 --- a/nikola/plugins/compile/html.py +++ b/nikola/plugins/compile/html.py @@ -36,11 +36,14 @@ from nikola.utils import makedirs, write_metadata class CompileHtml(PageCompiler): + """Compile HTML into HTML.""" + name = "html" friendly_name = "HTML" def compile_html(self, source, dest, is_two_file=True): + """Compile source file into HTML and save as dest.""" makedirs(os.path.dirname(dest)) with io.open(dest, "w+", encoding="utf8") as out_file: with io.open(source, "r", encoding="utf8") as in_file: @@ -51,6 +54,7 @@ class CompileHtml(PageCompiler): return True def create_post(self, path, **kw): + """Create a new post.""" content = kw.pop('content', None) onefile = kw.pop('onefile', False) # is_page is not used by create_post as of now. diff --git a/nikola/plugins/compile/ipynb.plugin b/nikola/plugins/compile/ipynb.plugin index efe6702..c369ab2 100644 --- a/nikola/plugins/compile/ipynb.plugin +++ b/nikola/plugins/compile/ipynb.plugin @@ -1,10 +1,13 @@ [Core] -Name = ipynb -Module = ipynb +name = ipynb +module = ipynb [Documentation] -Author = Damian Avila, Chris Warrick and others -Version = 2.0.0 -Website = http://www.damian.oquanta.info/ -Description = Compile IPython notebooks into Nikola posts +author = Damian Avila, Chris Warrick and others +version = 2.0.0 +website = http://www.damian.oquanta.info/ +description = Compile IPython notebooks into Nikola posts +[Nikola] +plugincategory = Compiler +friendlyname = Jupyter/IPython Notebook diff --git a/nikola/plugins/compile/ipynb.py b/nikola/plugins/compile/ipynb.py index 82b76c8..a9dedde 100644 --- a/nikola/plugins/compile/ipynb.py +++ b/nikola/plugins/compile/ipynb.py @@ -49,10 +49,11 @@ except ImportError: flag = None from nikola.plugin_categories import PageCompiler -from nikola.utils import makedirs, req_missing, get_logger +from nikola.utils import makedirs, req_missing, get_logger, STDERR_HANDLER class CompileIPynb(PageCompiler): + """Compile IPynb into HTML.""" name = "ipynb" @@ -61,24 +62,30 @@ class CompileIPynb(PageCompiler): default_kernel = 'python2' if sys.version_info[0] == 2 else 'python3' def set_site(self, site): - self.logger = get_logger('compile_ipynb', site.loghandlers) + """Set Nikola site.""" + self.logger = get_logger('compile_ipynb', STDERR_HANDLER) super(CompileIPynb, self).set_site(site) - def compile_html(self, source, dest, is_two_file=True): + def compile_html_string(self, source, is_two_file=True): + """Export notebooks as HTML strings.""" if flag is None: req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)') - makedirs(os.path.dirname(dest)) HTMLExporter.default_template = 'basic' c = Config(self.site.config['IPYNB_CONFIG']) exportHtml = HTMLExporter(config=c) + with io.open(source, "r", encoding="utf8") as in_file: + nb_json = nbformat.read(in_file, current_nbformat) + (body, resources) = exportHtml.from_notebook_node(nb_json) + return body + + def compile_html(self, source, dest, is_two_file=True): + """Compile source file into HTML and save as dest.""" + makedirs(os.path.dirname(dest)) with io.open(dest, "w+", encoding="utf8") as out_file: - with io.open(source, "r", encoding="utf8") as in_file: - nb_json = nbformat.read(in_file, current_nbformat) - (body, resources) = exportHtml.from_notebook_node(nb_json) - out_file.write(body) + out_file.write(self.compile_html_string(source, is_two_file)) def read_metadata(self, post, file_metadata_regexp=None, unslugify_titles=False, lang=None): - """read metadata directly from ipynb file. + """Read metadata directly from ipynb file. As ipynb file support arbitrary metadata as json, the metadata used by Nikola will be assume to be in the 'nikola' subfield. @@ -93,6 +100,7 @@ class CompileIPynb(PageCompiler): return nb_json.get('metadata', {}).get('nikola', {}) def create_post(self, path, **kw): + """Create a new post.""" if flag is None: req_missing(['ipython[notebook]>=2.0.0'], 'build this site (compile ipynb)') content = kw.pop('content', None) diff --git a/nikola/plugins/compile/markdown.plugin b/nikola/plugins/compile/markdown.plugin index a44b798..f7d11b1 100644 --- a/nikola/plugins/compile/markdown.plugin +++ b/nikola/plugins/compile/markdown.plugin @@ -1,10 +1,13 @@ [Core] -Name = markdown -Module = markdown +name = markdown +module = markdown [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Compile Markdown into HTML +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Compile Markdown into HTML +[Nikola] +plugincategory = Compiler +friendlyname = Markdown diff --git a/nikola/plugins/compile/markdown/__init__.py b/nikola/plugins/compile/markdown/__init__.py index fbe049d..c1425a1 100644 --- a/nikola/plugins/compile/markdown/__init__.py +++ b/nikola/plugins/compile/markdown/__init__.py @@ -44,6 +44,7 @@ from nikola.utils import makedirs, req_missing, write_metadata class CompileMarkdown(PageCompiler): + """Compile Markdown into HTML.""" name = "markdown" @@ -53,21 +54,18 @@ class CompileMarkdown(PageCompiler): site = None def set_site(self, site): + """Set Nikola site.""" + super(CompileMarkdown, self).set_site(site) self.config_dependencies = [] - for plugin_info in site.plugin_manager.getPluginsOfCategory("MarkdownExtension"): - if plugin_info.name in site.config['DISABLED_PLUGINS']: - site.plugin_manager.removePluginFromCategory(plugin_info, "MarkdownExtension") - continue + for plugin_info in self.get_compiler_extensions(): self.config_dependencies.append(plugin_info.name) - site.plugin_manager.activatePluginByName(plugin_info.name) - plugin_info.plugin_object.set_site(site) self.extensions.append(plugin_info.plugin_object) plugin_info.plugin_object.short_help = plugin_info.description self.config_dependencies.append(str(sorted(site.config.get("MARKDOWN_EXTENSIONS")))) - return super(CompileMarkdown, self).set_site(site) def compile_html(self, source, dest, is_two_file=True): + """Compile source file into HTML and save as dest.""" if markdown is None: req_missing(['markdown'], 'build this site (compile Markdown)') makedirs(os.path.dirname(dest)) @@ -81,6 +79,7 @@ class CompileMarkdown(PageCompiler): out_file.write(output) def create_post(self, path, **kw): + """Create a new post.""" content = kw.pop('content', None) onefile = kw.pop('onefile', False) # is_page is not used by create_post as of now. diff --git a/nikola/plugins/compile/markdown/mdx_gist.plugin b/nikola/plugins/compile/markdown/mdx_gist.plugin index 0e5c578..7fe676c 100644 --- a/nikola/plugins/compile/markdown/mdx_gist.plugin +++ b/nikola/plugins/compile/markdown/mdx_gist.plugin @@ -1,9 +1,14 @@ [Core] -Name = mdx_gist -Module = mdx_gist +name = mdx_gist +module = mdx_gist + +[Nikola] +compiler = markdown +plugincategory = CompilerExtension [Documentation] -Author = Roberto Alsina -Version = 0.1 -Website = http://getnikola.com -Description = Extension for embedding gists +author = Roberto Alsina +version = 0.1 +website = http://getnikola.com +description = Extension for embedding gists + diff --git a/nikola/plugins/compile/markdown/mdx_gist.py b/nikola/plugins/compile/markdown/mdx_gist.py index 70e7394..f439fa2 100644 --- a/nikola/plugins/compile/markdown/mdx_gist.py +++ b/nikola/plugins/compile/markdown/mdx_gist.py @@ -26,16 +26,16 @@ # # Inspired by "[Python] reStructuredText GitHub Gist directive" # (https://gist.github.com/brianhsu/1407759), public domain by Brian Hsu -''' -Extension to Python Markdown for Embedded Gists (gist.github.com) +""" +Extension to Python Markdown for Embedded Gists (gist.github.com). Basic Example: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... [:gist: 4747847] - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist: @@ -50,10 +50,10 @@ Basic Example: Example with filename: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... [:gist: 4747847 zen.py] - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist: @@ -68,10 +68,10 @@ Example with filename: Basic Example with hexidecimal id: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... [:gist: c4a43d6fdce612284ac0] - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist: @@ -86,10 +86,10 @@ Basic Example with hexidecimal id: Example with hexidecimal id filename: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... [:gist: c4a43d6fdce612284ac0 cow.txt] - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist: @@ -104,10 +104,10 @@ Example with hexidecimal id filename: Example using reStructuredText syntax: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... .. gist:: 4747847 zen.py - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist: @@ -122,10 +122,10 @@ Example using reStructuredText syntax: Example using hexidecimal ID with reStructuredText syntax: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... .. gist:: c4a43d6fdce612284ac0 - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist: @@ -140,10 +140,10 @@ Example using hexidecimal ID with reStructuredText syntax: Example using hexidecimal ID and filename with reStructuredText syntax: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... .. gist:: c4a43d6fdce612284ac0 cow.txt - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist: @@ -158,38 +158,36 @@ Example using hexidecimal ID and filename with reStructuredText syntax: Error Case: non-existent Gist ID: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... [:gist: 0] - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist:

- +

Error Case: non-existent file: >>> import markdown - >>> text = """ + >>> text = ''' ... Text of the gist: ... [:gist: 4747847 doesntexist.py] - ... """ + ... ''' >>> html = markdown.markdown(text, [GistExtension()]) >>> print(html)

Text of the gist:

- +

+""" -''' from __future__ import unicode_literals, print_function try: @@ -219,20 +217,26 @@ GIST_RST_RE = r'(?m)^\.\.\s*gist::\s*(?P[^\]\s]+)(?:\s*(?P.+? class GistFetchException(Exception): - '''Raised when attempt to fetch content of a Gist from github.com fails.''' + + """Raised when attempt to fetch content of a Gist from github.com fails.""" + def __init__(self, url, status_code): + """Initialize the exception.""" Exception.__init__(self) self.message = 'Received a {0} response from Gist URL: {1}'.format( status_code, url) class GistPattern(Pattern): - """ InlinePattern for footnote markers in a document's body text. """ + + """InlinePattern for footnote markers in a document's body text.""" def __init__(self, pattern, configs): + """Initialize the pattern.""" Pattern.__init__(self, pattern) def get_raw_gist_with_filename(self, gist_id, filename): + """Get raw gist text for a filename.""" url = GIST_FILE_RAW_URL.format(gist_id, filename) resp = requests.get(url) @@ -242,6 +246,7 @@ class GistPattern(Pattern): return resp.text def get_raw_gist(self, gist_id): + """Get raw gist text.""" url = GIST_RAW_URL.format(gist_id) resp = requests.get(url) @@ -251,6 +256,7 @@ class GistPattern(Pattern): return resp.text def handleMatch(self, m): + """Handle pattern match.""" gist_id = m.group('gist_id') gist_file = m.group('filename') @@ -284,7 +290,11 @@ class GistPattern(Pattern): class GistExtension(MarkdownExtension, Extension): + + """Gist extension for Markdown.""" + def __init__(self, configs={}): + """Initialize the extension.""" # set extension defaults self.config = {} @@ -293,6 +303,7 @@ class GistExtension(MarkdownExtension, Extension): self.setConfig(key, value) def extendMarkdown(self, md, md_globals): + """Extend Markdown.""" gist_md_pattern = GistPattern(GIST_MD_RE, self.getConfigs()) gist_md_pattern.md = md md.inlinePatterns.add('gist', gist_md_pattern, "
(.*?)
', flags=re.M class NikolaPostProcessor(Postprocessor): + + """Nikola-specific post-processing for Markdown.""" + def run(self, text): + """Run the postprocessor.""" output = text # python-markdown's highlighter uses 
@@ -52,11 +57,16 @@ class NikolaPostProcessor(Postprocessor):
 
 
 class NikolaExtension(MarkdownExtension, Extension):
+
+    """Extension for injecting the postprocessor."""
+
     def extendMarkdown(self, md, md_globals):
+        """Extend Markdown with the postprocessor."""
         pp = NikolaPostProcessor()
         md.postprocessors.add('nikola_post_processor', pp, '_end')
         md.registerExtension(self)
 
 
-def makeExtension(configs=None):
+def makeExtension(configs=None):  # pragma: no cover
+    """Make extension."""
     return NikolaExtension(configs)
diff --git a/nikola/plugins/compile/markdown/mdx_podcast.plugin b/nikola/plugins/compile/markdown/mdx_podcast.plugin
index dc16044..c92a8a0 100644
--- a/nikola/plugins/compile/markdown/mdx_podcast.plugin
+++ b/nikola/plugins/compile/markdown/mdx_podcast.plugin
@@ -1,9 +1,14 @@
 [Core]
-Name = mdx_podcast
-Module = mdx_podcast
+name = mdx_podcast
+module = mdx_podcast
+
+[Nikola]
+compiler = markdown
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Roberto Alsina
-Version = 0.1
-Website = http://getnikola.com
-Description = Markdown extensions for embedding podcasts and other audio files
+author = Roberto Alsina
+version = 0.1
+website = http://getnikola.com
+description = Markdown extensions for embedding podcasts and other audio files
+
diff --git a/nikola/plugins/compile/markdown/mdx_podcast.py b/nikola/plugins/compile/markdown/mdx_podcast.py
index 670973a..61afdbf 100644
--- a/nikola/plugins/compile/markdown/mdx_podcast.py
+++ b/nikola/plugins/compile/markdown/mdx_podcast.py
@@ -24,21 +24,19 @@
 # Inspired by "[Python] reStructuredText GitHub Podcast directive"
 # (https://gist.github.com/brianhsu/1407759), public domain by Brian Hsu
 
-from __future__ import print_function, unicode_literals
-
-
-'''
-Extension to Python Markdown for Embedded Audio
+"""
+Extension to Python Markdown for Embedded Audio.
 
 Basic Example:
 
 >>> import markdown
->>> text = """[podcast]http://archive.org/download/Rebeldes_Stereotipos/rs20120609_1.mp3[/podcast]"""
+>>> text = "[podcast]http://archive.org/download/Rebeldes_Stereotipos/rs20120609_1.mp3[/podcast]"
 >>> html = markdown.markdown(text, [PodcastExtension()])
 >>> print(html)
-

-'''
+

+"""
 
+from __future__ import print_function, unicode_literals
 from nikola.plugin_categories import MarkdownExtension
 try:
     from markdown.extensions import Extension
@@ -53,12 +51,15 @@ PODCAST_RE = r'\[podcast\](?P.+)\[/podcast\]'
 
 
 class PodcastPattern(Pattern):
-    """ InlinePattern for footnote markers in a document's body text. """
+
+    """InlinePattern for footnote markers in a document's body text."""
 
     def __init__(self, pattern, configs):
+        """Initialize pattern."""
         Pattern.__init__(self, pattern)
 
     def handleMatch(self, m):
+        """Handle pattern matches."""
         url = m.group('url').strip()
         audio_elem = etree.Element('audio')
         audio_elem.set('controls', '')
@@ -69,7 +70,11 @@ class PodcastPattern(Pattern):
 
 
 class PodcastExtension(MarkdownExtension, Extension):
+
+    """"Podcast extension for Markdown."""
+
     def __init__(self, configs={}):
+        """Initialize extension."""
         # set extension defaults
         self.config = {}
 
@@ -78,13 +83,15 @@ class PodcastExtension(MarkdownExtension, Extension):
             self.setConfig(key, value)
 
     def extendMarkdown(self, md, md_globals):
+        """Extend Markdown."""
         podcast_md_pattern = PodcastPattern(PODCAST_RE, self.getConfigs())
         podcast_md_pattern.md = md
         md.inlinePatterns.add('podcast', podcast_md_pattern, "`_.
 
@@ -236,8 +238,8 @@ def rst2html(source, source_path=None, source_class=docutils.io.StringInput,
              writer_name='html', settings=None, settings_spec=None,
              settings_overrides=None, config_section=None,
              enable_exit_status=None, logger=None, l_add_ln=0, transforms=None):
-    """
-    Set up & run a `Publisher`, and return a dictionary of document parts.
+    """Set up & run a ``Publisher``, and return a dictionary of document parts.
+
     Dictionary keys are the names of parts, and values are Unicode strings;
     encoding is up to the client.  For programmatic use with string I/O.
 
diff --git a/nikola/plugins/compile/rest/chart.plugin b/nikola/plugins/compile/rest/chart.plugin
index 3e27a25..438abe4 100644
--- a/nikola/plugins/compile/rest/chart.plugin
+++ b/nikola/plugins/compile/rest/chart.plugin
@@ -1,10 +1,14 @@
 [Core]
-Name = rest_chart
-Module = chart
+name = rest_chart
+module = chart
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Roberto Alsina
-Version = 0.1
-Website = http://getnikola.com
-Description = Chart directive based in PyGal
+author = Roberto Alsina
+version = 0.1
+website = http://getnikola.com
+description = Chart directive based in PyGal
 
diff --git a/nikola/plugins/compile/rest/chart.py b/nikola/plugins/compile/rest/chart.py
index 59b9dc7..88fdff3 100644
--- a/nikola/plugins/compile/rest/chart.py
+++ b/nikola/plugins/compile/rest/chart.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Chart directive for reSTructuredText."""
+
 from ast import literal_eval
 
 from docutils import nodes
@@ -42,9 +44,12 @@ _site = None
 
 class Plugin(RestExtension):
 
+    """Plugin for chart role."""
+
     name = "rest_chart"
 
     def set_site(self, site):
+        """Set Nikola site."""
         global _site
         _site = self.site = site
         directives.register_directive('chart', Chart)
@@ -52,17 +57,18 @@ class Plugin(RestExtension):
 
 
 class Chart(Directive):
-    """ Restructured text extension for inserting charts as SVG
 
-        Usage:
-            .. chart:: Bar
-               :title: 'Browser usage evolution (in %)'
-               :x_labels: ["2002", "2003", "2004", "2005", "2006", "2007"]
+    """reStructuredText extension for inserting charts as SVG.
+
+    Usage:
+        .. chart:: Bar
+           :title: 'Browser usage evolution (in %)'
+           :x_labels: ["2002", "2003", "2004", "2005", "2006", "2007"]
 
-               'Firefox', [None, None, 0, 16.6, 25, 31]
-               'Chrome',  [None, None, None, None, None, None]
-               'IE',      [85.8, 84.6, 84.7, 74.5, 66, 58.6]
-               'Others',  [14.2, 15.4, 15.3, 8.9, 9, 10.4]
+           'Firefox', [None, None, 0, 16.6, 25, 31]
+           'Chrome',  [None, None, None, None, None, None]
+           'IE',      [85.8, 84.6, 84.7, 74.5, 66, 58.6]
+           'Others',  [14.2, 15.4, 15.3, 8.9, 9, 10.4]
     """
 
     has_content = True
@@ -129,6 +135,7 @@ class Chart(Directive):
     }
 
     def run(self):
+        """Run the directive."""
         if pygal is None:
             msg = req_missing(['pygal'], 'use the Chart directive', optional=True)
             return [nodes.raw('', '
{0}
'.format(msg), format='html')]
diff --git a/nikola/plugins/compile/rest/doc.plugin b/nikola/plugins/compile/rest/doc.plugin
index 1984f52..facdd03 100644
--- a/nikola/plugins/compile/rest/doc.plugin
+++ b/nikola/plugins/compile/rest/doc.plugin
@@ -1,10 +1,14 @@
 [Core]
-Name = rest_doc
-Module = doc
+name = rest_doc
+module = doc
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Manuel Kaufmann
-Version = 0.1
-Website = http://getnikola.com
-Description = Role to link another page / post from the blog
+author = Manuel Kaufmann
+version = 0.1
+website = http://getnikola.com
+description = Role to link another page / post from the blog
 
diff --git a/nikola/plugins/compile/rest/doc.py b/nikola/plugins/compile/rest/doc.py
index 703c234..99cce81 100644
--- a/nikola/plugins/compile/rest/doc.py
+++ b/nikola/plugins/compile/rest/doc.py
@@ -24,6 +24,7 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""reST role for linking to other documents."""
 
 from docutils import nodes
 from docutils.parsers.rst import roles
@@ -34,9 +35,12 @@ from nikola.plugin_categories import RestExtension
 
 class Plugin(RestExtension):
 
+    """Plugin for doc role."""
+
     name = 'rest_doc'
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         roles.register_canonical_role('doc', doc_role)
         doc_role.site = site
@@ -45,7 +49,7 @@ class Plugin(RestExtension):
 
 def doc_role(name, rawtext, text, lineno, inliner,
              options={}, content=[]):
-
+    """Handle the doc role."""
     # split link's text and post's slug in role content
     has_explicit_title, title, slug = split_explicit_title(text)
     # check if the slug given is part of our blog posts/pages
@@ -82,5 +86,6 @@ def doc_role(name, rawtext, text, lineno, inliner,
 
 
 def make_link_node(rawtext, text, url, options):
+    """Make a reST link node."""
     node = nodes.reference(rawtext, text, refuri=url, *options)
     return node
diff --git a/nikola/plugins/compile/rest/gist.plugin b/nikola/plugins/compile/rest/gist.plugin
index 8f498ec..9fa2e82 100644
--- a/nikola/plugins/compile/rest/gist.plugin
+++ b/nikola/plugins/compile/rest/gist.plugin
@@ -1,10 +1,14 @@
 [Core]
-Name = rest_gist
-Module = gist
+name = rest_gist
+module = gist
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Roberto Alsina
-Version = 0.1
-Website = http://getnikola.com
-Description = Gist directive
+author = Roberto Alsina
+version = 0.1
+website = http://getnikola.com
+description = Gist directive
 
diff --git a/nikola/plugins/compile/rest/gist.py b/nikola/plugins/compile/rest/gist.py
index ab4d56d..736ee37 100644
--- a/nikola/plugins/compile/rest/gist.py
+++ b/nikola/plugins/compile/rest/gist.py
@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # This file is public domain according to its author, Brian Hsu
 
+"""Gist directive for reStructuredText."""
+
 import requests
 from docutils.parsers.rst import Directive, directives
 from docutils import nodes
@@ -10,26 +12,28 @@ from nikola.plugin_categories import RestExtension
 
 class Plugin(RestExtension):
 
+    """Plugin for gist directive."""
+
     name = "rest_gist"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('gist', GitHubGist)
         return super(Plugin, self).set_site(site)
 
 
 class GitHubGist(Directive):
-    """ Embed GitHub Gist.
-
-        Usage:
 
-          .. gist:: GIST_ID
+    """Embed GitHub Gist.
 
-        or
+    Usage:
 
-          .. gist:: GIST_URL
+      .. gist:: GIST_ID
 
+    or
 
+      .. gist:: GIST_URL
     """
 
     required_arguments = 1
@@ -39,10 +43,12 @@ class GitHubGist(Directive):
     has_content = False
 
     def get_raw_gist_with_filename(self, gistID, filename):
+        """Get raw gist text for a filename."""
         url = '/'.join(("https://gist.github.com/raw", gistID, filename))
         return requests.get(url).text
 
     def get_raw_gist(self, gistID):
+        """Get raw gist text."""
         url = "https://gist.github.com/raw/{0}".format(gistID)
         try:
             return requests.get(url).text
@@ -50,6 +56,7 @@ class GitHubGist(Directive):
             raise self.error('Cannot get gist for url={0}'.format(url))
 
     def run(self):
+        """Run the gist directive."""
         if 'https://' in self.arguments[0]:
             gistID = self.arguments[0].split('/')[-1].strip()
         else:
diff --git a/nikola/plugins/compile/rest/listing.plugin b/nikola/plugins/compile/rest/listing.plugin
index 4c9883e..85c780f 100644
--- a/nikola/plugins/compile/rest/listing.plugin
+++ b/nikola/plugins/compile/rest/listing.plugin
@@ -1,10 +1,14 @@
 [Core]
-Name = rest_listing
-Module = listing
+name = rest_listing
+module = listing
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Roberto Alsina
-Version = 0.1
-Website = http://getnikola.com
-Description = Extension for source listings
+author = Roberto Alsina
+version = 0.1
+website = http://getnikola.com
+description = Extension for source listings
 
diff --git a/nikola/plugins/compile/rest/listing.py b/nikola/plugins/compile/rest/listing.py
index b8340cf..4871bf3 100644
--- a/nikola/plugins/compile/rest/listing.py
+++ b/nikola/plugins/compile/rest/listing.py
@@ -25,7 +25,7 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
-""" Define and register a listing directive using the existing CodeBlock """
+"""Define and register a listing directive using the existing CodeBlock."""
 
 
 from __future__ import unicode_literals
@@ -55,7 +55,9 @@ from nikola.plugin_categories import RestExtension
 
 # A sanitized version of docutils.parsers.rst.directives.body.CodeBlock.
 class CodeBlock(Directive):
+
     """Parse and mark up content of a code block."""
+
     optional_arguments = 1
     option_spec = {'class': directives.class_option,
                    'name': directives.unchanged,
@@ -65,6 +67,7 @@ class CodeBlock(Directive):
     has_content = True
 
     def run(self):
+        """Run code block directive."""
         self.assert_has_content()
 
         if 'linenos' in self.options:
@@ -124,9 +127,12 @@ docutils.parsers.rst.directives.misc.CodeBlock = CodeBlock
 
 class Plugin(RestExtension):
 
+    """Plugin for listing directive."""
+
     name = "rest_listing"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         # Even though listings don't use CodeBlock anymore, I am
         # leaving these to make the code directive work with
@@ -146,7 +152,8 @@ listing_spec['linenos'] = directives.unchanged
 
 
 class Listing(Include):
-    """ listing directive: create a highlighted block of code from a file in listings/
+
+    """Create a highlighted block of code from a file in listings/.
 
     Usage:
 
@@ -154,12 +161,14 @@ class Listing(Include):
            :number-lines:
 
     """
+
     has_content = False
     required_arguments = 1
     optional_arguments = 1
     option_spec = listing_spec
 
     def run(self):
+        """Run listing directive."""
         _fname = self.arguments.pop(0)
         fname = _fname.replace('/', os.sep)
         lang = self.arguments.pop(0)
@@ -185,9 +194,9 @@ class Listing(Include):
         return generated_nodes
 
     def get_code_from_file(self, data):
-        """ Create CodeBlock nodes from file object content """
+        """Create CodeBlock nodes from file object content."""
         return super(Listing, self).run()
 
     def assert_has_content(self):
-        """ Listing has no content, override check from superclass """
+        """Listing has no content, override check from superclass."""
         pass
diff --git a/nikola/plugins/compile/rest/media.plugin b/nikola/plugins/compile/rest/media.plugin
index 5f5276b..9803c8f 100644
--- a/nikola/plugins/compile/rest/media.plugin
+++ b/nikola/plugins/compile/rest/media.plugin
@@ -1,10 +1,14 @@
 [Core]
-Name = rest_media
-Module = media
+name = rest_media
+module = media
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Roberto Alsina
-Version = 0.1
-Website = http://getnikola.com
-Description = Directive to support oembed via micawber
+author = Roberto Alsina
+version = 0.1
+website = http://getnikola.com
+description = Directive to support oembed via micawber
 
diff --git a/nikola/plugins/compile/rest/media.py b/nikola/plugins/compile/rest/media.py
index 0363d28..345e331 100644
--- a/nikola/plugins/compile/rest/media.py
+++ b/nikola/plugins/compile/rest/media.py
@@ -24,6 +24,7 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Media directive for reStructuredText."""
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
@@ -40,21 +41,27 @@ from nikola.utils import req_missing
 
 class Plugin(RestExtension):
 
+    """Plugin for reST media directive."""
+
     name = "rest_media"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('media', Media)
         return super(Plugin, self).set_site(site)
 
 
 class Media(Directive):
-    """ Restructured text extension for inserting any sort of media using micawber."""
+
+    """reST extension for inserting any sort of media using micawber."""
+
     has_content = False
     required_arguments = 1
     optional_arguments = 999
 
     def run(self):
+        """Run media directive."""
         if micawber is None:
             msg = req_missing(['micawber'], 'use the media directive', optional=True)
             return [nodes.raw('', '
{0}
'.format(msg), format='html')]
diff --git a/nikola/plugins/compile/rest/post_list.plugin b/nikola/plugins/compile/rest/post_list.plugin
index 82450a0..48969bf 100644
--- a/nikola/plugins/compile/rest/post_list.plugin
+++ b/nikola/plugins/compile/rest/post_list.plugin
@@ -1,9 +1,14 @@
 [Core]
-Name = rest_post_list
-Module = post_list
+name = rest_post_list
+module = post_list
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Udo Spallek
-Version = 0.1
-Website = http://getnikola.com
-Description = Includes a list of posts with tag and slide based filters.
+author = Udo Spallek
+version = 0.1
+website = http://getnikola.com
+description = Includes a list of posts with tag and slide based filters.
+
diff --git a/nikola/plugins/compile/rest/post_list.py b/nikola/plugins/compile/rest/post_list.py
index ddbd82d..a22ee85 100644
--- a/nikola/plugins/compile/rest/post_list.py
+++ b/nikola/plugins/compile/rest/post_list.py
@@ -23,6 +23,9 @@
 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Post list directive for reStructuredText."""
+
 from __future__ import unicode_literals
 
 import os
@@ -40,9 +43,13 @@ from nikola.plugin_categories import RestExtension
 
 
 class Plugin(RestExtension):
+
+    """Plugin for reST post-list directive."""
+
     name = "rest_post_list"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('post-list', PostList)
         PostList.site = site
@@ -50,14 +57,15 @@ class Plugin(RestExtension):
 
 
 class PostList(Directive):
-    """
+
+    """Provide a reStructuredText directive to create a list of posts.
+
     Post List
     =========
     :Directive Arguments: None.
-    :Directive Options: lang, start, stop, reverse, sort, tags, template, id
+    :Directive Options: lang, start, stop, reverse, sort, tags, categories, slugs, all, template, id
     :Directive Content: None.
 
-    Provides a reStructuredText directive to create a list of posts.
     The posts appearing in the list can be filtered by options.
     *List slicing* is provided with the *start*, *stop* and *reverse* options.
 
@@ -87,6 +95,10 @@ class PostList(Directive):
         Filter posts to show only posts having at least one of the ``tags``.
         Defaults to None.
 
+    ``categories`` : string [, string...]
+        Filter posts to show only posts having one of the ``categories``.
+        Defaults to None.
+
     ``slugs`` : string [, string...]
         Filter posts to show only posts having at least one of the ``slugs``.
         Defaults to None.
@@ -107,12 +119,14 @@ class PostList(Directive):
         A manual id for the post list.
         Defaults to a random name composed by 'post_list_' + uuid.uuid4().hex.
     """
+
     option_spec = {
         'start': int,
         'stop': int,
         'reverse': directives.flag,
         'sort': directives.unchanged,
         'tags': directives.unchanged,
+        'categories': directives.unchanged,
         'slugs': directives.unchanged,
         'all': directives.flag,
         'lang': directives.unchanged,
@@ -121,11 +135,14 @@ class PostList(Directive):
     }
 
     def run(self):
+        """Run post-list directive."""
         start = self.options.get('start')
         stop = self.options.get('stop')
         reverse = self.options.get('reverse', False)
         tags = self.options.get('tags')
         tags = [t.strip().lower() for t in tags.split(',')] if tags else []
+        categories = self.options.get('categories')
+        categories = [c.strip().lower() for c in categories.split(',')] if categories else []
         slugs = self.options.get('slugs')
         slugs = [s.strip() for s in slugs.split(',')] if slugs else []
         show_all = self.options.get('all', False)
@@ -145,6 +162,9 @@ class PostList(Directive):
         else:
             timeline = [p for p in self.site.timeline if p.use_in_feeds]
 
+        if categories:
+            timeline = [p for p in timeline if p.meta('category', lang=lang).lower() in categories]
+
         for post in timeline:
             if tags:
                 cont = True
diff --git a/nikola/plugins/compile/rest/slides.plugin b/nikola/plugins/compile/rest/slides.plugin
index cee4b06..5c05b89 100644
--- a/nikola/plugins/compile/rest/slides.plugin
+++ b/nikola/plugins/compile/rest/slides.plugin
@@ -1,10 +1,14 @@
 [Core]
-Name = rest_slides
-Module = slides
+name = rest_slides
+module = slides
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Roberto Alsina
-Version = 0.1
-Website = http://getnikola.com
-Description = Slides directive
+author = Roberto Alsina
+version = 0.1
+website = http://getnikola.com
+description = Slides directive
 
diff --git a/nikola/plugins/compile/rest/slides.py b/nikola/plugins/compile/rest/slides.py
index 7826f6a..2522e55 100644
--- a/nikola/plugins/compile/rest/slides.py
+++ b/nikola/plugins/compile/rest/slides.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Slides directive for reStructuredText."""
+
 from __future__ import unicode_literals
 
 import uuid
@@ -36,9 +38,12 @@ from nikola.plugin_categories import RestExtension
 
 class Plugin(RestExtension):
 
+    """Plugin for reST slides directive."""
+
     name = "rest_slides"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('slides', Slides)
         Slides.site = site
@@ -46,11 +51,14 @@ class Plugin(RestExtension):
 
 
 class Slides(Directive):
-    """ Restructured text extension for inserting slideshows."""
+
+    """reST extension for inserting slideshows."""
+
     has_content = True
 
     def run(self):
-        if len(self.content) == 0:
+        """Run the slides directive."""
+        if len(self.content) == 0:  # pragma: no cover
             return
 
         if self.site.invariant:  # for testing purposes
diff --git a/nikola/plugins/compile/rest/soundcloud.plugin b/nikola/plugins/compile/rest/soundcloud.plugin
index 1d31a8f..75469e4 100644
--- a/nikola/plugins/compile/rest/soundcloud.plugin
+++ b/nikola/plugins/compile/rest/soundcloud.plugin
@@ -1,10 +1,14 @@
 [Core]
-Name = rest_soundcloud
-Module = soundcloud
+name = rest_soundcloud
+module = soundcloud
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Roberto Alsina
-Version = 0.1
-Website = http://getnikola.com
-Description = Soundcloud directive
+author = Roberto Alsina
+version = 0.1
+website = http://getnikola.com
+description = Soundcloud directive
 
diff --git a/nikola/plugins/compile/rest/soundcloud.py b/nikola/plugins/compile/rest/soundcloud.py
index a26806c..30134a9 100644
--- a/nikola/plugins/compile/rest/soundcloud.py
+++ b/nikola/plugins/compile/rest/soundcloud.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+"""SoundCloud directive for reStructuredText."""
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
@@ -10,9 +11,12 @@ from nikola.plugin_categories import RestExtension
 
 class Plugin(RestExtension):
 
+    """Plugin for soundclound directive."""
+
     name = "rest_soundcloud"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('soundcloud', SoundCloud)
         directives.register_directive('soundcloud_playlist', SoundCloudPlaylist)
@@ -27,7 +31,8 @@ src="https://w.soundcloud.com/player/?url=http://api.soundcloud.com/{preslug}/""
 
 
 class SoundCloud(Directive):
-    """ Restructured text extension for inserting SoundCloud embedded music
+
+    """reST extension for inserting SoundCloud embedded music.
 
     Usage:
         .. soundcloud:: 
@@ -35,6 +40,7 @@ class SoundCloud(Directive):
            :width: 600
 
     """
+
     has_content = True
     required_arguments = 1
     option_spec = {
@@ -44,7 +50,7 @@ class SoundCloud(Directive):
     preslug = "tracks"
 
     def run(self):
-        """ Required by the Directive interface. Create docutils nodes """
+        """Run the soundcloud directive."""
         self.check_content()
         options = {
             'sid': self.arguments[0],
@@ -56,12 +62,15 @@ class SoundCloud(Directive):
         return [nodes.raw('', CODE.format(**options), format='html')]
 
     def check_content(self):
-        """ Emit a deprecation warning if there is content """
-        if self.content:
+        """Emit a deprecation warning if there is content."""
+        if self.content:  # pragma: no cover
             raise self.warning("This directive does not accept content. The "
                                "'key=value' format for options is deprecated, "
                                "use ':key: value' instead")
 
 
 class SoundCloudPlaylist(SoundCloud):
+
+    """reST directive for SoundCloud playlists."""
+
     preslug = "playlists"
diff --git a/nikola/plugins/compile/rest/thumbnail.plugin b/nikola/plugins/compile/rest/thumbnail.plugin
index 3b73340..0084310 100644
--- a/nikola/plugins/compile/rest/thumbnail.plugin
+++ b/nikola/plugins/compile/rest/thumbnail.plugin
@@ -1,9 +1,14 @@
 [Core]
-Name = rest_thumbnail
-Module = thumbnail
+name = rest_thumbnail
+module = thumbnail
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Author = Pelle Nilsson
-Version = 0.1
-Website = http://getnikola.com
-Description = reST directive to facilitate enlargeable images with thumbnails
+author = Pelle Nilsson
+version = 0.1
+website = http://getnikola.com
+description = reST directive to facilitate enlargeable images with thumbnails
+
diff --git a/nikola/plugins/compile/rest/thumbnail.py b/nikola/plugins/compile/rest/thumbnail.py
index 5388d8d..1fae06c 100644
--- a/nikola/plugins/compile/rest/thumbnail.py
+++ b/nikola/plugins/compile/rest/thumbnail.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Thumbnail directive for reStructuredText."""
+
 import os
 
 from docutils.parsers.rst import directives
@@ -34,9 +36,12 @@ from nikola.plugin_categories import RestExtension
 
 class Plugin(RestExtension):
 
+    """Plugin for thumbnail directive."""
+
     name = "rest_thumbnail"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('thumbnail', Thumbnail)
         return super(Plugin, self).set_site(site)
@@ -44,10 +49,14 @@ class Plugin(RestExtension):
 
 class Thumbnail(Figure):
 
+    """Thumbnail directive for reST."""
+
     def align(argument):
+        """Return thumbnail alignment."""
         return directives.choice(argument, Image.align_values)
 
     def figwidth_value(argument):
+        """Return figure width."""
         if argument.lower() == 'image':
             return 'image'
         else:
@@ -59,6 +68,7 @@ class Thumbnail(Figure):
     has_content = True
 
     def run(self):
+        """Run the thumbnail directive."""
         uri = directives.uri(self.arguments[0])
         self.options['target'] = uri
         self.arguments[0] = '.thumbnail'.join(os.path.splitext(uri))
diff --git a/nikola/plugins/compile/rest/vimeo.plugin b/nikola/plugins/compile/rest/vimeo.plugin
index e0ff3f1..688f981 100644
--- a/nikola/plugins/compile/rest/vimeo.plugin
+++ b/nikola/plugins/compile/rest/vimeo.plugin
@@ -1,7 +1,11 @@
 [Core]
-Name = rest_vimeo
-Module = vimeo
+name = rest_vimeo
+module = vimeo
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Description = Vimeo directive
+description = Vimeo directive
 
diff --git a/nikola/plugins/compile/rest/vimeo.py b/nikola/plugins/compile/rest/vimeo.py
index bc44b0e..c694a87 100644
--- a/nikola/plugins/compile/rest/vimeo.py
+++ b/nikola/plugins/compile/rest/vimeo.py
@@ -24,6 +24,7 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Vimeo directive for reStructuredText."""
 
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
@@ -37,9 +38,12 @@ from nikola.plugin_categories import RestExtension
 
 class Plugin(RestExtension):
 
+    """Plugin for vimeo reST directive."""
+
     name = "rest_vimeo"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('vimeo', Vimeo)
         return super(Plugin, self).set_site(site)
@@ -56,14 +60,16 @@ VIDEO_DEFAULT_WIDTH = 281
 
 
 class Vimeo(Directive):
-    """ Restructured text extension for inserting vimeo embedded videos
 
-        Usage:
-            .. vimeo:: 20241459
-               :height: 400
-               :width: 600
+    """reST extension for inserting vimeo embedded videos.
+
+    Usage:
+        .. vimeo:: 20241459
+           :height: 400
+           :width: 600
 
     """
+
     has_content = True
     required_arguments = 1
     option_spec = {
@@ -75,6 +81,7 @@ class Vimeo(Directive):
     request_size = True
 
     def run(self):
+        """Run the vimeo directive."""
         self.check_content()
         options = {
             'vimeo_id': self.arguments[0],
@@ -90,9 +97,11 @@ class Vimeo(Directive):
         return [nodes.raw('', CODE.format(**options), format='html')]
 
     def check_modules(self):
+        """Check modules."""
         return None
 
     def set_video_size(self):
+        """Set video size."""
         # Only need to make a connection if width and height aren't provided
         if 'height' not in self.options or 'width' not in self.options:
             self.options['height'] = VIDEO_DEFAULT_HEIGHT
@@ -111,6 +120,7 @@ class Vimeo(Directive):
                     pass
 
     def check_content(self):
+        """Check if content exists."""
         if self.content:
             raise self.warning("This directive does not accept content. The "
                                "'key=value' format for options is deprecated, "
diff --git a/nikola/plugins/compile/rest/youtube.plugin b/nikola/plugins/compile/rest/youtube.plugin
index 01275be..5fbd67b 100644
--- a/nikola/plugins/compile/rest/youtube.plugin
+++ b/nikola/plugins/compile/rest/youtube.plugin
@@ -1,8 +1,12 @@
 [Core]
-Name = rest_youtube
-Module = youtube
+name = rest_youtube
+module = youtube
+
+[Nikola]
+compiler = rest
+plugincategory = CompilerExtension
 
 [Documentation]
-Version = 0.1
-Description = Youtube directive
+version = 0.1
+description = Youtube directive
 
diff --git a/nikola/plugins/compile/rest/youtube.py b/nikola/plugins/compile/rest/youtube.py
index 7c6bba1..6c5c211 100644
--- a/nikola/plugins/compile/rest/youtube.py
+++ b/nikola/plugins/compile/rest/youtube.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""YouTube directive for reStructuredText."""
+
 from docutils import nodes
 from docutils.parsers.rst import Directive, directives
 
@@ -33,9 +35,12 @@ from nikola.plugin_categories import RestExtension
 
 class Plugin(RestExtension):
 
+    """Plugin for the youtube directive."""
+
     name = "rest_youtube"
 
     def set_site(self, site):
+        """Set Nikola site."""
         self.site = site
         directives.register_directive('youtube', Youtube)
         return super(Plugin, self).set_site(site)
@@ -49,7 +54,8 @@ src="//www.youtube.com/embed/{yid}?rel=0&hd=1&wmode=transparent"
 
 
 class Youtube(Directive):
-    """ Restructured text extension for inserting youtube embedded videos
+
+    """reST extension for inserting youtube embedded videos.
 
     Usage:
         .. youtube:: lyViVmaBQDg
@@ -57,6 +63,7 @@ class Youtube(Directive):
            :width: 600
 
     """
+
     has_content = True
     required_arguments = 1
     option_spec = {
@@ -65,6 +72,7 @@ class Youtube(Directive):
     }
 
     def run(self):
+        """Run the youtube directive."""
         self.check_content()
         options = {
             'yid': self.arguments[0],
@@ -75,7 +83,8 @@ class Youtube(Directive):
         return [nodes.raw('', CODE.format(**options), format='html')]
 
     def check_content(self):
-        if self.content:
+        """Check if content exists."""
+        if self.content:  # pragma: no cover
             raise self.warning("This directive does not accept content. The "
                                "'key=value' format for options is deprecated, "
                                "use ':key: value' instead")
diff --git a/nikola/plugins/loghandler/__init__.py b/nikola/plugins/loghandler/__init__.py
deleted file mode 100644
index a1d17a6..0000000
--- a/nikola/plugins/loghandler/__init__.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright © 2012-2015 Roberto Alsina and others.
-
-# Permission is hereby granted, free of charge, to any
-# person obtaining a copy of this software and associated
-# documentation files (the "Software"), to deal in the
-# Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the
-# Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice
-# shall be included in all copies or substantial portions of
-# the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
-# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
-# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/nikola/plugins/loghandler/smtp.plugin b/nikola/plugins/loghandler/smtp.plugin
deleted file mode 100644
index 38c1d96..0000000
--- a/nikola/plugins/loghandler/smtp.plugin
+++ /dev/null
@@ -1,9 +0,0 @@
-[Core]
-Name = smtp
-Module = smtp
-
-[Documentation]
-Author = Daniel Devine
-Version = 1.0
-Website = http://getnikola.com
-Description = Log over smtp (email).
diff --git a/nikola/plugins/loghandler/smtp.py b/nikola/plugins/loghandler/smtp.py
deleted file mode 100644
index 146a658..0000000
--- a/nikola/plugins/loghandler/smtp.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright © 2012-2015 Daniel Devine and others.
-
-# Permission is hereby granted, free of charge, to any
-# person obtaining a copy of this software and associated
-# documentation files (the "Software"), to deal in the
-# Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the
-# Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice
-# shall be included in all copies or substantial portions of
-# the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
-# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
-# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-from nikola.plugin_categories import SignalHandler
-from blinker import signal
-import logbook
-
-
-class SmtpHandler(SignalHandler):
-    name = 'smtp'
-
-    def attach_handler(self, sender):
-        """Add the handler to a list of handlers that are attached when get_logger() is called.."""
-        smtpconf = self.site.config.get('LOGGING_HANDLERS').get('smtp')
-        if smtpconf:
-            smtpconf['format_string'] = '''\
-Subject: {record.level_name}: {record.channel}
-
-{record.message}
-'''
-            self.site.loghandlers.append(logbook.MailHandler(
-                smtpconf.pop('from_addr'),
-                smtpconf.pop('recipients'),
-                **smtpconf
-            ))
-
-    def set_site(self, site):
-        self.site = site
-
-        ready = signal('sighandlers_loaded')
-        ready.connect(self.attach_handler)
diff --git a/nikola/plugins/loghandler/stderr.plugin b/nikola/plugins/loghandler/stderr.plugin
deleted file mode 100644
index 6c20ea1..0000000
--- a/nikola/plugins/loghandler/stderr.plugin
+++ /dev/null
@@ -1,9 +0,0 @@
-[Core]
-Name = stderr
-Module = stderr
-
-[Documentation]
-Author = Daniel Devine
-Version = 1.0
-Website = http://getnikola.com
-Description = Log to stderr, the default logger.
diff --git a/nikola/plugins/loghandler/stderr.py b/nikola/plugins/loghandler/stderr.py
deleted file mode 100644
index 79ace68..0000000
--- a/nikola/plugins/loghandler/stderr.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright © 2012-2015 Daniel Devine and others.
-
-# Permission is hereby granted, free of charge, to any
-# person obtaining a copy of this software and associated
-# documentation files (the "Software"), to deal in the
-# Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the
-# Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice
-# shall be included in all copies or substantial portions of
-# the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
-# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
-# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-from nikola.plugin_categories import SignalHandler
-from blinker import signal
-import os
-
-from nikola import DEBUG
-from nikola.utils import ColorfulStderrHandler
-
-
-class StderrHandler(SignalHandler):
-    """Logs messages to stderr."""
-    name = 'stderr'
-
-    def attach_handler(self, sender):
-        """Attach the handler to the logger."""
-        conf = self.site.config.get('LOGGING_HANDLERS').get('stderr')
-        if conf or os.getenv('NIKOLA_DEBUG'):
-            self.site.loghandlers.append(ColorfulStderrHandler(
-                # We do not allow the level to be something else than 'DEBUG'
-                # or 'INFO'  Any other level can have bad effects on the user
-                # experience and is discouraged.
-                # (oh, and it was incorrectly set to WARNING before)
-                level='DEBUG' if DEBUG or (conf.get('loglevel', 'INFO').upper() == 'DEBUG') else 'INFO',
-                format_string=u'[{record.time:%Y-%m-%dT%H:%M:%SZ}] {record.level_name}: {record.channel}: {record.message}'
-            ))
-
-    def set_site(self, site):
-        self.site = site
-
-        ready = signal('sighandlers_loaded')
-        ready.connect(self.attach_handler)
diff --git a/nikola/plugins/misc/__init__.py b/nikola/plugins/misc/__init__.py
new file mode 100644
index 0000000..c0d8961
--- /dev/null
+++ b/nikola/plugins/misc/__init__.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2015 Roberto Alsina and others.
+
+# Permission is hereby granted, free of charge, to any
+# person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the
+# Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the
+# Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice
+# shall be included in all copies or substantial portions of
+# the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Miscellaneous Nikola plugins."""
diff --git a/nikola/plugins/misc/scan_posts.py b/nikola/plugins/misc/scan_posts.py
index a6f04e6..1f4f995 100644
--- a/nikola/plugins/misc/scan_posts.py
+++ b/nikola/plugins/misc/scan_posts.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""The default post scanner."""
+
 from __future__ import unicode_literals, print_function
 import glob
 import os
@@ -35,13 +37,13 @@ from nikola.post import Post
 
 
 class ScanPosts(PostScanner):
-    """Render pages into output."""
+
+    """Scan posts in the site."""
 
     name = "scan_posts"
 
     def scan(self):
         """Create list of posts from POSTS and PAGES options."""
-
         seen = set([])
         if not self.site.quiet:
             print("Scanning posts", end='', file=sys.stderr)
diff --git a/nikola/plugins/task/__init__.py b/nikola/plugins/task/__init__.py
index a1d17a6..fd9a48f 100644
--- a/nikola/plugins/task/__init__.py
+++ b/nikola/plugins/task/__init__.py
@@ -23,3 +23,5 @@
 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Tasks for Nikola."""
diff --git a/nikola/plugins/task/archive.plugin b/nikola/plugins/task/archive.plugin
index 6687209..25f1195 100644
--- a/nikola/plugins/task/archive.plugin
+++ b/nikola/plugins/task/archive.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = render_archive
-Module = archive
+name = render_archive
+module = archive
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Generates the blog's archive pages.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Generates the blog's archive pages.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/archive.py b/nikola/plugins/task/archive.py
index 533be69..126aed4 100644
--- a/nikola/plugins/task/archive.py
+++ b/nikola/plugins/task/archive.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Render the post archives."""
+
 import copy
 import os
 
@@ -35,17 +37,20 @@ from nikola.utils import config_changed, adjust_name_for_index_path, adjust_name
 
 
 class Archive(Task):
+
     """Render the post archives."""
 
     name = "render_archive"
 
     def set_site(self, site):
+        """Set Nikola site."""
         site.register_path_handler('archive', self.archive_path)
         site.register_path_handler('archive_atom', self.archive_atom_path)
         return super(Archive, self).set_site(site)
 
     def _prepare_task(self, kw, name, lang, posts, items, template_name,
                       title, deps_translatable=None):
+        """Prepare an archive task."""
         # name: used to build permalink and destination
         # posts, items: posts or items; only one of them should be used,
         #               the other be None
@@ -53,17 +58,20 @@ class Archive(Task):
         # title: the (translated) title for the generated page
         # deps_translatable: dependencies (None if not added)
         assert posts is not None or items is not None
-
+        task_cfg = [copy.copy(kw)]
         context = {}
         context["lang"] = lang
         context["title"] = title
         context["permalink"] = self.site.link("archive", name, lang)
+        context["pagekind"] = ["list", "archive_page"]
         if posts is not None:
             context["posts"] = posts
-            n = len(posts)
+            # Depend on all post metadata because it can be used in templates (Issue #1931)
+            task_cfg.append([repr(p) for p in posts])
         else:
+            # Depend on the content of items, to rebuild if links change (Issue #1931)
             context["items"] = items
-            n = len(items)
+            task_cfg.append(items)
         task = self.site.generic_post_list_renderer(
             lang,
             [],
@@ -73,7 +81,7 @@ class Archive(Task):
             context,
         )
 
-        task_cfg = {1: copy.copy(kw), 2: n}
+        task_cfg = {i: x for i, x in enumerate(task_cfg)}
         if deps_translatable is not None:
             task_cfg[3] = deps_translatable
         task['uptodate'] = task['uptodate'] + [config_changed(task_cfg, 'nikola.plugins.task.archive')]
@@ -81,6 +89,7 @@ class Archive(Task):
         return task
 
     def _generate_posts_task(self, kw, name, lang, posts, title, deps_translatable=None):
+        """Genereate a task for an archive with posts."""
         posts = sorted(posts, key=lambda a: a.date)
         posts.reverse()
         if kw['archives_are_indexes']:
@@ -97,13 +106,15 @@ class Archive(Task):
             uptodate = []
             if deps_translatable is not None:
                 uptodate += [config_changed(deps_translatable, 'nikola.plugins.task.archive')]
+            context = {"archive_name": name,
+                       "is_feed_stale": kw["is_feed_stale"],
+                       "pagekind": ["index", "archive_page"]}
             yield self.site.generic_index_renderer(
                 lang,
                 posts,
                 title,
                 "archiveindex.tmpl",
-                {"archive_name": name,
-                 "is_feed_stale": kw["is_feed_stale"]},
+                context,
                 kw,
                 str(self.name),
                 page_link,
@@ -113,6 +124,7 @@ class Archive(Task):
             yield self._prepare_task(kw, name, lang, posts, None, "list_post.tmpl", title, deps_translatable)
 
     def gen_tasks(self):
+        """Generate archive tasks."""
         kw = {
             "messages": self.site.MESSAGES,
             "translations": self.site.config['TRANSLATIONS'],
@@ -211,6 +223,7 @@ class Archive(Task):
                 yield self._prepare_task(kw, None, lang, None, items, "list.tmpl", kw["messages"][lang]["Archive"])
 
     def archive_path(self, name, lang, is_feed=False):
+        """Return archive paths."""
         if is_feed:
             extension = ".atom"
             archive_file = os.path.splitext(self.site.config['ARCHIVE_FILENAME'])[0] + extension
@@ -228,4 +241,5 @@ class Archive(Task):
                                   archive_file] if _f]
 
     def archive_atom_path(self, name, lang):
+        """Return Atom archive paths."""
         return self.archive_path(name, lang, is_feed=True)
diff --git a/nikola/plugins/task/bundles.plugin b/nikola/plugins/task/bundles.plugin
index 3fe049b..ca997d0 100644
--- a/nikola/plugins/task/bundles.plugin
+++ b/nikola/plugins/task/bundles.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = create_bundles
-Module = bundles
+name = create_bundles
+module = bundles
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Theme bundles using WebAssets
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Theme bundles using WebAssets
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/bundles.py b/nikola/plugins/task/bundles.py
index 6f88d0c..b9c57b9 100644
--- a/nikola/plugins/task/bundles.py
+++ b/nikola/plugins/task/bundles.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Bundle assets using WebAssets."""
+
 from __future__ import unicode_literals
 
 import os
@@ -38,12 +40,14 @@ from nikola import utils
 
 
 class BuildBundles(LateTask):
+
     """Bundle assets using WebAssets."""
 
     name = "create_bundles"
 
     def set_site(self, site):
-        self.logger = utils.get_logger('bundles', site.loghandlers)
+        """Set Nikola site."""
+        self.logger = utils.get_logger('bundles', utils.STDERR_HANDLER)
         if webassets is None and site.config['USE_BUNDLES']:
             utils.req_missing(['webassets'], 'USE_BUNDLES', optional=True)
             self.logger.warn('Setting USE_BUNDLES to False.')
@@ -52,7 +56,6 @@ class BuildBundles(LateTask):
 
     def gen_tasks(self):
         """Bundle assets using WebAssets."""
-
         kw = {
             'filters': self.site.config['FILTERS'],
             'output_folder': self.site.config['OUTPUT_FOLDER'],
diff --git a/nikola/plugins/task/copy_assets.plugin b/nikola/plugins/task/copy_assets.plugin
index 0530ebf..c182150 100644
--- a/nikola/plugins/task/copy_assets.plugin
+++ b/nikola/plugins/task/copy_assets.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = copy_assets
-Module = copy_assets
+name = copy_assets
+module = copy_assets
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Copy theme assets into output.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Copy theme assets into output.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/copy_assets.py b/nikola/plugins/task/copy_assets.py
index a72bfdf..58521d4 100644
--- a/nikola/plugins/task/copy_assets.py
+++ b/nikola/plugins/task/copy_assets.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Copy theme assets into output."""
+
 from __future__ import unicode_literals
 
 import io
@@ -34,6 +36,7 @@ from nikola import utils
 
 
 class CopyAssets(Task):
+
     """Copy theme assets into output."""
 
     name = "copy_assets"
@@ -44,7 +47,6 @@ class CopyAssets(Task):
         If a file is present on two themes, use the version
         from the "youngest" theme.
         """
-
         kw = {
             "themes": self.site.THEMES,
             "files_folders": self.site.config['FILES_FOLDERS'],
diff --git a/nikola/plugins/task/copy_files.plugin b/nikola/plugins/task/copy_files.plugin
index 073676b..ce8f5d0 100644
--- a/nikola/plugins/task/copy_files.plugin
+++ b/nikola/plugins/task/copy_files.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = copy_files
-Module = copy_files
+name = copy_files
+module = copy_files
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Copy static files into the output.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Copy static files into the output.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/copy_files.py b/nikola/plugins/task/copy_files.py
index 9a039f1..1232248 100644
--- a/nikola/plugins/task/copy_files.py
+++ b/nikola/plugins/task/copy_files.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Copy static files into the output folder."""
+
 import os
 
 from nikola.plugin_categories import Task
@@ -31,13 +33,13 @@ from nikola import utils
 
 
 class CopyFiles(Task):
+
     """Copy static files into the output folder."""
 
     name = "copy_files"
 
     def gen_tasks(self):
         """Copy static files into the output folder."""
-
         kw = {
             'files_folders': self.site.config['FILES_FOLDERS'],
             'output_folder': self.site.config['OUTPUT_FOLDER'],
diff --git a/nikola/plugins/task/galleries.plugin b/nikola/plugins/task/galleries.plugin
index 73085cd..9d3fa28 100644
--- a/nikola/plugins/task/galleries.plugin
+++ b/nikola/plugins/task/galleries.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = render_galleries
-Module = galleries
+name = render_galleries
+module = galleries
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Create image galleries automatically.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Create image galleries automatically.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/galleries.py b/nikola/plugins/task/galleries.py
index e887f18..c0df4a4 100644
--- a/nikola/plugins/task/galleries.py
+++ b/nikola/plugins/task/galleries.py
@@ -24,10 +24,12 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Render image galleries."""
+
 from __future__ import unicode_literals
-import io
 import datetime
 import glob
+import io
 import json
 import mimetypes
 import os
@@ -55,17 +57,19 @@ _image_size_cache = {}
 
 
 class Galleries(Task, ImageProcessor):
+
     """Render image galleries."""
 
     name = 'render_galleries'
     dates = {}
 
     def set_site(self, site):
+        """Set Nikola site."""
         site.register_path_handler('gallery', self.gallery_path)
         site.register_path_handler('gallery_global', self.gallery_global_path)
         site.register_path_handler('gallery_rss', self.gallery_rss_path)
 
-        self.logger = utils.get_logger('render_galleries', site.loghandlers)
+        self.logger = utils.get_logger('render_galleries', utils.STDERR_HANDLER)
 
         self.kw = {
             'thumbnail_size': site.config['THUMBNAIL_SIZE'],
@@ -118,17 +122,20 @@ class Galleries(Task, ImageProcessor):
         sys.exit(1)
 
     def gallery_path(self, name, lang):
+        """Return a gallery path."""
         gallery_path = self._find_gallery_path(name)
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] +
                 gallery_path.split(os.sep) +
                 [self.site.config['INDEX_FILE']] if _f]
 
     def gallery_global_path(self, name, lang):
+        """Return the global gallery path, which contains images."""
         gallery_path = self._find_gallery_path(name)
         return [_f for _f in gallery_path.split(os.sep) +
                 [self.site.config['INDEX_FILE']] if _f]
 
     def gallery_rss_path(self, name, lang):
+        """Return path to the RSS file for a gallery."""
         gallery_path = self._find_gallery_path(name)
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] +
                 gallery_path.split(os.sep) +
@@ -136,7 +143,6 @@ class Galleries(Task, ImageProcessor):
 
     def gen_tasks(self):
         """Render image galleries."""
-
         self.image_ext_list = self.image_ext_list_builtin
         self.image_ext_list.extend(self.site.config.get('EXTRA_IMAGE_EXTENSIONS', []))
 
@@ -183,11 +189,13 @@ class Galleries(Task, ImageProcessor):
 
             crumbs = utils.get_crumbs(gallery, index_folder=self)
 
-            # Create index.html for each language
             for lang in self.kw['translations']:
                 # save navigation links as dependencies
                 self.kw['navigation_links|{0}'.format(lang)] = self.kw['global_context']['navigation_links'](lang)
 
+            # Create index.html for each language
+            for lang in self.kw['translations']:
+
                 dst = os.path.join(
                     self.kw['output_folder'],
                     self.site.path("gallery", gallery, lang))
@@ -238,6 +246,7 @@ class Galleries(Task, ImageProcessor):
                 context["permalink"] = self.site.link("gallery", gallery, lang)
                 context["enable_comments"] = self.kw['comments_in_galleries']
                 context["thumbnail_size"] = self.kw["thumbnail_size"]
+                context["pagekind"] = ["gallery_front"]
 
                 if post:
                     yield {
@@ -246,7 +255,7 @@ class Galleries(Task, ImageProcessor):
                         'targets': [post.translated_base_path(lang)],
                         'file_dep': post.fragment_deps(lang),
                         'actions': [(post.compile, [lang])],
-                        'uptodate': [utils.config_changed(self.kw, 'nikola.plugins.task.galleries:post')] + post.fragment_deps_uptodate(lang)
+                        'uptodate': [utils.config_changed(self.kw.copy(), 'nikola.plugins.task.galleries:post')] + post.fragment_deps_uptodate(lang)
                     }
                     context['post'] = post
                 else:
@@ -259,6 +268,8 @@ class Galleries(Task, ImageProcessor):
                     file_dep += [post.translated_base_path(l) for l in self.kw['translations']]
                     file_dep_dest += [post.translated_base_path(l) for l in self.kw['translations']]
 
+                context["pagekind"] = ["gallery_page"]
+
                 yield utils.apply_filters({
                     'basename': self.name,
                     'name': dst,
@@ -268,14 +279,14 @@ class Galleries(Task, ImageProcessor):
                         (self.render_gallery_index, (
                             template_name,
                             dst,
-                            context,
+                            context.copy(),
                             dest_img_list,
                             img_titles,
                             thumbs,
                             file_dep))],
                     'clean': True,
                     'uptodate': [utils.config_changed({
-                        1: self.kw,
+                        1: self.kw.copy(),
                         2: self.site.config["COMMENTS_IN_GALLERIES"],
                         3: context.copy(),
                     }, 'nikola.plugins.task.galleries:gallery')],
@@ -305,21 +316,19 @@ class Galleries(Task, ImageProcessor):
                             ))],
                         'clean': True,
                         'uptodate': [utils.config_changed({
-                            1: self.kw,
+                            1: self.kw.copy(),
                         }, 'nikola.plugins.task.galleries:rss')],
                     }, self.kw['filters'])
 
     def find_galleries(self):
-        """Find all galleries to be processed according to conf.py"""
-
+        """Find all galleries to be processed according to conf.py."""
         self.gallery_list = []
         for input_folder, output_folder in self.kw['gallery_folders'].items():
             for root, dirs, files in os.walk(input_folder, followlinks=True):
                 self.gallery_list.append((root, input_folder, output_folder))
 
     def create_galleries_paths(self):
-        """Given a list of galleries, puts their paths into self.gallery_links."""
-
+        """Given a list of galleries, put their paths into self.gallery_links."""
         # gallery_path is "gallery/foo/name"
         self.proper_gallery_links = dict()
         self.improper_gallery_links = dict()
@@ -350,7 +359,6 @@ class Galleries(Task, ImageProcessor):
 
     def create_galleries(self):
         """Given a list of galleries, create the output folders."""
-
         # gallery_path is "gallery/foo/name"
         for gallery_path, input_folder, _ in self.gallery_list:
             # have to use dirname because site.path returns .../index.html
@@ -366,12 +374,11 @@ class Galleries(Task, ImageProcessor):
                 'actions': [(utils.makedirs, (output_gallery,))],
                 'targets': [output_gallery],
                 'clean': True,
-                'uptodate': [utils.config_changed(self.kw, 'nikola.plugins.task.galleries:mkdir')],
+                'uptodate': [utils.config_changed(self.kw.copy(), 'nikola.plugins.task.galleries:mkdir')],
             }
 
     def parse_index(self, gallery, input_folder, output_folder):
-        """Returns a Post object if there is an index.txt."""
-
+        """Return a Post object if there is an index.txt."""
         index_path = os.path.join(gallery, "index.txt")
         destination = os.path.join(
             self.kw["output_folder"], output_folder,
@@ -397,6 +404,7 @@ class Galleries(Task, ImageProcessor):
         return post
 
     def get_excluded_images(self, gallery_path):
+        """Get list of excluded images."""
         exclude_path = os.path.join(gallery_path, "exclude.meta")
 
         try:
@@ -409,7 +417,7 @@ class Galleries(Task, ImageProcessor):
         return excluded_image_list
 
     def get_image_list(self, gallery_path):
-
+        """Get list of included images."""
         # Gather image_list contains "gallery/name/image_name.jpg"
         image_list = []
 
@@ -424,6 +432,7 @@ class Galleries(Task, ImageProcessor):
         return image_list
 
     def create_target_images(self, img, input_path):
+        """Copy images to output."""
         gallery_name = os.path.dirname(img)
         output_gallery = os.path.dirname(
             os.path.join(
@@ -473,6 +482,7 @@ class Galleries(Task, ImageProcessor):
         }, self.kw['filters'])
 
     def remove_excluded_image(self, img, input_folder):
+        """Remove excluded images."""
         # Remove excluded images
         # img is something like input_folder/demo/tesla2_lg.jpg so it's the *source* path
         # and we should remove both the large and thumbnail *destination* paths
@@ -493,7 +503,7 @@ class Galleries(Task, ImageProcessor):
                 (utils.remove_file, (thumb_path,))
             ],
             'clean': True,
-            'uptodate': [utils.config_changed(self.kw, 'nikola.plugins.task.galleries:clean_thumb')],
+            'uptodate': [utils.config_changed(self.kw.copy(), 'nikola.plugins.task.galleries:clean_thumb')],
         }, self.kw['filters'])
 
         yield utils.apply_filters({
@@ -503,7 +513,7 @@ class Galleries(Task, ImageProcessor):
                 (utils.remove_file, (img_path,))
             ],
             'clean': True,
-            'uptodate': [utils.config_changed(self.kw, 'nikola.plugins.task.galleries:clean_file')],
+            'uptodate': [utils.config_changed(self.kw.copy(), 'nikola.plugins.task.galleries:clean_file')],
         }, self.kw['filters'])
 
     def render_gallery_index(
@@ -516,7 +526,6 @@ class Galleries(Task, ImageProcessor):
             thumbs,
             file_dep):
         """Build the gallery index."""
-
         # The photo array needs to be created here, because
         # it relies on thumbnails already being created on
         # output
@@ -543,7 +552,7 @@ class Galleries(Task, ImageProcessor):
                 },
             })
         context['photo_array'] = photo_array
-        context['photo_array_json'] = json.dumps(photo_array)
+        context['photo_array_json'] = json.dumps(photo_array, sort_keys=True)
         self.site.render_template(template_name, output_name, context)
 
     def gallery_rss(self, img_list, dest_img_list, img_titles, lang, permalink, output_path, title):
@@ -552,7 +561,6 @@ class Galleries(Task, ImageProcessor):
         This doesn't use generic_rss_renderer because it
         doesn't involve Post objects.
         """
-
         def make_url(url):
             return urljoin(self.site.config['BASE_URL'], url.lstrip('/'))
 
diff --git a/nikola/plugins/task/gzip.plugin b/nikola/plugins/task/gzip.plugin
index 4867fd6..7834d22 100644
--- a/nikola/plugins/task/gzip.plugin
+++ b/nikola/plugins/task/gzip.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = gzip
-Module = gzip
+name = gzip
+module = gzip
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Create gzipped copies of files
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Create gzipped copies of files
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/gzip.py b/nikola/plugins/task/gzip.py
index 5799839..cf16f63 100644
--- a/nikola/plugins/task/gzip.py
+++ b/nikola/plugins/task/gzip.py
@@ -35,12 +35,14 @@ from nikola.plugin_categories import TaskMultiplier
 
 
 class GzipFiles(TaskMultiplier):
+
     """If appropiate, create tasks to create gzipped versions of files."""
 
     name = "gzip"
     is_default = True
 
     def process(self, task, prefix):
+        """Process tasks."""
         if not self.site.config['GZIP_FILES']:
             return []
         if task.get('name') is None:
@@ -70,6 +72,7 @@ class GzipFiles(TaskMultiplier):
 
 
 def create_gzipped_copy(in_path, out_path, command=None):
+    """Create gzipped copy of in_path and save it as out_path."""
     if command:
         subprocess.check_call(shlex.split(command.format(filename=in_path)))
     else:
diff --git a/nikola/plugins/task/indexes.plugin b/nikola/plugins/task/indexes.plugin
index 5d2bf5a..d9b0e5f 100644
--- a/nikola/plugins/task/indexes.plugin
+++ b/nikola/plugins/task/indexes.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = render_indexes
-Module = indexes
+name = render_indexes
+module = indexes
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Generates the blog's index pages.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Generates the blog's index pages.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/indexes.py b/nikola/plugins/task/indexes.py
index 03d36b1..c02818e 100644
--- a/nikola/plugins/task/indexes.py
+++ b/nikola/plugins/task/indexes.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Render the blog indexes."""
+
 from __future__ import unicode_literals
 from collections import defaultdict
 import os
@@ -33,16 +35,19 @@ from nikola import utils
 
 
 class Indexes(Task):
+
     """Render the blog indexes."""
 
     name = "render_indexes"
 
     def set_site(self, site):
+        """Set Nikola site."""
         site.register_path_handler('index', self.index_path)
         site.register_path_handler('index_atom', self.index_atom_path)
         return super(Indexes, self).set_site(site)
 
     def gen_tasks(self):
+        """Render the blog indexes."""
         self.site.scan_posts()
         yield self.group_task()
 
@@ -80,7 +85,10 @@ class Indexes(Task):
             indexes_title = kw['indexes_title'](lang) or kw['blog_title'](lang)
             self.number_of_pages[lang] = (len(filtered_posts) + kw['index_display_post_count'] - 1) // kw['index_display_post_count']
 
-            yield self.site.generic_index_renderer(lang, filtered_posts, indexes_title, template_name, {}, kw, 'render_indexes', page_link, page_path)
+            context = {}
+            context["pagekind"] = ["index"]
+
+            yield self.site.generic_index_renderer(lang, filtered_posts, indexes_title, template_name, context, kw, 'render_indexes', page_link, page_path)
 
         if not self.site.config["STORY_INDEX"]:
             return
@@ -93,13 +101,17 @@ class Indexes(Task):
             "strip_indexes": self.site.config['STRIP_INDEXES'],
         }
         template_name = "list.tmpl"
+        index_len = len(kw['index_file'])
         for lang in kw["translations"]:
             # Need to group by folder to avoid duplicated tasks (Issue #758)
                 # Group all pages by path prefix
                 groups = defaultdict(list)
                 for p in self.site.timeline:
                     if not p.is_post:
-                        dirname = os.path.dirname(p.destination_path(lang))
+                        destpath = p.destination_path(lang)
+                        if destpath[-(1 + index_len):] == '/' + kw['index_file']:
+                            destpath = destpath[:-(1 + index_len)]
+                        dirname = os.path.dirname(destpath)
                         groups[dirname].append(p)
                 for dirname, post_list in groups.items():
                     context = {}
@@ -108,10 +120,12 @@ class Indexes(Task):
                     output_name = os.path.join(kw['output_folder'], dirname, kw['index_file'])
                     short_destination = os.path.join(dirname, kw['index_file'])
                     link = short_destination.replace('\\', '/')
-                    index_len = len(kw['index_file'])
                     if kw['strip_indexes'] and link[-(1 + index_len):] == '/' + kw['index_file']:
                         link = link[:-index_len]
                     context["permalink"] = link
+                    context["pagekind"] = ["list"]
+                    if dirname == "/":
+                        context["pagekind"].append("front_page")
 
                     for post in post_list:
                         # If there is an index.html pending to be created from
@@ -133,6 +147,7 @@ class Indexes(Task):
                         yield task
 
     def index_path(self, name, lang, is_feed=False):
+        """Return path to an index."""
         extension = None
         if is_feed:
             extension = ".atom"
@@ -149,4 +164,5 @@ class Indexes(Task):
                                                      extension=extension)
 
     def index_atom_path(self, name, lang):
+        """Return path to an Atom index."""
         return self.index_path(name, lang, is_feed=True)
diff --git a/nikola/plugins/task/listings.plugin b/nikola/plugins/task/listings.plugin
index a5ba77a..435234b 100644
--- a/nikola/plugins/task/listings.plugin
+++ b/nikola/plugins/task/listings.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = render_listings
-Module = listings
+name = render_listings
+module = listings
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Render code listings into output
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Render code listings into output
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/listings.py b/nikola/plugins/task/listings.py
index b913330..5f79724 100644
--- a/nikola/plugins/task/listings.py
+++ b/nikola/plugins/task/listings.py
@@ -24,10 +24,13 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Render code listings."""
+
 from __future__ import unicode_literals, print_function
 
 import sys
 import os
+import lxml.html
 
 from pygments import highlight
 from pygments.lexers import get_lexer_for_filename, TextLexer
@@ -38,7 +41,8 @@ from nikola import utils
 
 
 class Listings(Task):
-    """Render pretty listings."""
+
+    """Render code listings."""
 
     name = "render_listings"
 
@@ -51,6 +55,7 @@ class Listings(Task):
         self.proper_input_file_mapping[rel_output_name] = rel_output_name
 
     def set_site(self, site):
+        """Set Nikola site."""
         site.register_path_handler('listing', self.listing_path)
 
         # We need to prepare some things for the listings path handler to work.
@@ -105,12 +110,21 @@ class Listings(Task):
 
     def gen_tasks(self):
         """Render pretty code listings."""
-
         # Things to ignore in listings
         ignored_extensions = (".pyc", ".pyo")
 
         def render_listing(in_name, out_name, input_folder, output_folder, folders=[], files=[]):
-            if in_name:
+            needs_ipython_css = False
+            if in_name and in_name.endswith('.ipynb'):
+                # Special handling: render ipynbs in listings (Issue #1900)
+                ipynb_compiler = self.site.plugin_manager.getPluginByName("ipynb", "PageCompiler").plugin_object
+                ipynb_raw = ipynb_compiler.compile_html_string(in_name, True)
+                ipynb_html = lxml.html.fromstring(ipynb_raw)
+                # The raw HTML contains garbage (scripts and styles), we can’t leave it in
+                code = lxml.html.tostring(ipynb_html.xpath('//*[@id="notebook"]')[0], encoding='unicode')
+                title = os.path.basename(in_name)
+                needs_ipython_css = True
+            elif in_name:
                 with open(in_name, 'r') as fd:
                     try:
                         lexer = get_lexer_for_filename(in_name)
@@ -149,7 +163,12 @@ class Listings(Task):
                     files, alg=natsort.ns.F | natsort.ns.IC),
                 'description': title,
                 'source_link': source_link,
+                'pagekind': ['listing'],
             }
+            if needs_ipython_css:
+                # If someone does not have ipynb posts and only listings, we
+                # need to enable ipynb CSS for ipynb listings.
+                context['needs_ipython_css'] = True
             self.site.render_template('listing.tmpl', out_name, context)
 
         yield self.group_task()
@@ -236,6 +255,7 @@ class Listings(Task):
                         }, self.kw["filters"])
 
     def listing_path(self, namep, lang):
+        """Return path to a listing."""
         namep = namep.replace('/', os.sep)
         nameh = namep + '.html'
         for name in (namep, nameh):
diff --git a/nikola/plugins/task/pages.plugin b/nikola/plugins/task/pages.plugin
index 4cad7b7..023d41b 100644
--- a/nikola/plugins/task/pages.plugin
+++ b/nikola/plugins/task/pages.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = render_pages
-Module = pages
+name = render_pages
+module = pages
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Create pages in the output.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Create pages in the output.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/pages.py b/nikola/plugins/task/pages.py
index d0edb56..e6a8a82 100644
--- a/nikola/plugins/task/pages.py
+++ b/nikola/plugins/task/pages.py
@@ -24,12 +24,15 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Render pages into output."""
+
 from __future__ import unicode_literals
 from nikola.plugin_categories import Task
 from nikola.utils import config_changed
 
 
 class RenderPages(Task):
+
     """Render pages into output."""
 
     name = "render_pages"
@@ -49,8 +52,11 @@ class RenderPages(Task):
             for post in self.site.timeline:
                 if not kw["show_untranslated_posts"] and not post.is_translation_available(lang):
                     continue
-                for task in self.site.generic_page_renderer(lang, post,
-                                                            kw["filters"]):
+                if post.is_post:
+                    context = {'pagekind': ['post_page']}
+                else:
+                    context = {'pagekind': ['story_page']}
+                for task in self.site.generic_page_renderer(lang, post, kw["filters"], context):
                     task['uptodate'] = task['uptodate'] + [config_changed(kw, 'nikola.plugins.task.pages')]
                     task['basename'] = self.name
                     task['task_dep'] = ['render_posts']
diff --git a/nikola/plugins/task/posts.plugin b/nikola/plugins/task/posts.plugin
index 707b3c2..79b7c51 100644
--- a/nikola/plugins/task/posts.plugin
+++ b/nikola/plugins/task/posts.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = render_posts
-Module = posts
+name = render_posts
+module = posts
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Create HTML fragments out of posts.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Create HTML fragments out of posts.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/posts.py b/nikola/plugins/task/posts.py
index d3f17fd..a3a8375 100644
--- a/nikola/plugins/task/posts.py
+++ b/nikola/plugins/task/posts.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Build HTML fragments from metadata and text."""
+
 from copy import copy
 import os
 
@@ -32,7 +34,7 @@ from nikola import filters, utils
 
 
 def update_deps(post, lang, task):
-    """Updates file dependencies as they might have been updated during compilation.
+    """Update file dependencies as they might have been updated during compilation.
 
     This is done for example by the ReST page compiler, which writes its
     dependencies into a .dep file. This file is read and incorporated when calling
@@ -42,6 +44,7 @@ def update_deps(post, lang, task):
 
 
 class RenderPosts(Task):
+
     """Build HTML fragments from metadata and text."""
 
     name = "render_posts"
@@ -74,7 +77,11 @@ class RenderPosts(Task):
             deps_dict = copy(kw)
             deps_dict.pop('timeline')
             for post in kw['timeline']:
-
+                # Extra config dependencies picked from config
+                for p in post.fragment_deps(lang):
+                    if p.startswith('####MAGIC####CONFIG:'):
+                        k = p.split('####MAGIC####CONFIG:', 1)[-1]
+                        deps_dict[k] = self.site.config.get(k)
                 dest = post.translated_base_path(lang)
                 file_dep = [p for p in post.fragment_deps(lang) if not p.startswith("####MAGIC####")]
                 task = {
@@ -110,6 +117,7 @@ class RenderPosts(Task):
                 yield utils.apply_filters(task, {os.path.splitext(dest): flist})
 
     def dependence_on_timeline(self, post, lang):
+        """Check if a post depends on the timeline."""
         if "####MAGIC####TIMELINE" not in post.fragment_deps(lang):
             return True  # No dependency on timeline
         elif self.tl_changed:
diff --git a/nikola/plugins/task/redirect.plugin b/nikola/plugins/task/redirect.plugin
index 0228c70..c3137b9 100644
--- a/nikola/plugins/task/redirect.plugin
+++ b/nikola/plugins/task/redirect.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = redirect
-Module = redirect
+name = redirect
+module = redirect
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Create redirect pages.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Create redirect pages.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/redirect.py b/nikola/plugins/task/redirect.py
index 428dd5a..8530f5e 100644
--- a/nikola/plugins/task/redirect.py
+++ b/nikola/plugins/task/redirect.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Generate redirections."""
+
 from __future__ import unicode_literals
 
 import os
@@ -33,13 +35,13 @@ from nikola import utils
 
 
 class Redirect(Task):
-    """Generate redirections"""
+
+    """Generate redirections."""
 
     name = "redirect"
 
     def gen_tasks(self):
         """Generate redirections tasks."""
-
         kw = {
             'redirections': self.site.config['REDIRECTIONS'],
             'output_folder': self.site.config['OUTPUT_FOLDER'],
diff --git a/nikola/plugins/task/robots.plugin b/nikola/plugins/task/robots.plugin
index b4b43a3..72ce31f 100644
--- a/nikola/plugins/task/robots.plugin
+++ b/nikola/plugins/task/robots.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = robots
-Module = robots
+name = robots
+module = robots
 
 [Documentation]
-Author = Daniel Aleksandersen
-Version = 1.0
-Website = http://getnikola.com
-Description = Generate /robots.txt exclusion file and promote sitemap.
+author = Daniel Aleksandersen
+version = 1.0
+website = http://getnikola.com
+description = Generate /robots.txt exclusion file and promote sitemap.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py
index 2f25a21..65254b6 100644
--- a/nikola/plugins/task/robots.py
+++ b/nikola/plugins/task/robots.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Generate a robots.txt file."""
+
 from __future__ import print_function, absolute_import, unicode_literals
 import io
 import os
@@ -37,12 +39,13 @@ from nikola import utils
 
 
 class RobotsFile(LateTask):
-    """Generate a robots.txt."""
+
+    """Generate a robots.txt file."""
 
     name = "robots_file"
 
     def gen_tasks(self):
-        """Generate a robots.txt."""
+        """Generate a robots.txt file."""
         kw = {
             "base_url": self.site.config["BASE_URL"],
             "site_url": self.site.config["SITE_URL"],
diff --git a/nikola/plugins/task/rss.plugin b/nikola/plugins/task/rss.plugin
index 56f0bf4..cf9b7a7 100644
--- a/nikola/plugins/task/rss.plugin
+++ b/nikola/plugins/task/rss.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = generate_rss
-Module = rss
+name = generate_rss
+module = rss
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Generate RSS feeds.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Generate RSS feeds.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/rss.py b/nikola/plugins/task/rss.py
index 26a4da1..9020a06 100644
--- a/nikola/plugins/task/rss.py
+++ b/nikola/plugins/task/rss.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Generate RSS feeds."""
+
 from __future__ import unicode_literals, print_function
 import os
 try:
@@ -36,11 +38,13 @@ from nikola.plugin_categories import Task
 
 
 class GenerateRSS(Task):
+
     """Generate RSS feeds."""
 
     name = "generate_rss"
 
     def set_site(self, site):
+        """Set Nikola site."""
         site.register_path_handler('rss', self.rss_path)
         return super(GenerateRSS, self).set_site(site)
 
@@ -102,5 +106,6 @@ class GenerateRSS(Task):
             yield utils.apply_filters(task, kw['filters'])
 
     def rss_path(self, name, lang):
+        """Return RSS path."""
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                               self.site.config['RSS_PATH'], 'rss.xml'] if _f]
diff --git a/nikola/plugins/task/scale_images.plugin b/nikola/plugins/task/scale_images.plugin
index c0f0f28..d906b8c 100644
--- a/nikola/plugins/task/scale_images.plugin
+++ b/nikola/plugins/task/scale_images.plugin
@@ -1,9 +1,13 @@
 [Core]
-Name = scale_images
-Module = scale_images
+name = scale_images
+module = scale_images
 
 [Documentation]
-Author = Pelle Nilsson
-Version = 1.0
-Website = http://getnikola.com
-Description = Create down-scaled images and thumbnails.
+author = Pelle Nilsson
+version = 1.0
+website = http://getnikola.com
+description = Create down-scaled images and thumbnails.
+
+[Nikola]
+plugincategory = Task
+
diff --git a/nikola/plugins/task/scale_images.py b/nikola/plugins/task/scale_images.py
index f97027e..22ed2ab 100644
--- a/nikola/plugins/task/scale_images.py
+++ b/nikola/plugins/task/scale_images.py
@@ -24,6 +24,8 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Resize images and create thumbnails for them."""
+
 import os
 
 from nikola.plugin_categories import Task
@@ -32,17 +34,18 @@ from nikola import utils
 
 
 class ScaleImage(Task, ImageProcessor):
-    """Copy static files into the output folder."""
+
+    """Resize images and create thumbnails for them."""
 
     name = "scale_images"
 
     def set_site(self, site):
-        self.logger = utils.get_logger('scale_images', site.loghandlers)
+        """Set Nikola site."""
+        self.logger = utils.get_logger('scale_images', utils.STDERR_HANDLER)
         return super(ScaleImage, self).set_site(site)
 
     def process_tree(self, src, dst):
-        """Processes all images in a src tree and put the (possibly) rescaled
-        images in the dst folder."""
+        """Process all images in a src tree and put the (possibly) rescaled images in the dst folder."""
         ignore = set(['.svn'])
         base_len = len(src.split(os.sep))
         for root, dirs, files in os.walk(src, followlinks=True):
@@ -68,12 +71,12 @@ class ScaleImage(Task, ImageProcessor):
                 }
 
     def process_image(self, src, dst, thumb):
+        """Resize an image."""
         self.resize_image(src, dst, self.kw['max_image_size'], False)
         self.resize_image(src, thumb, self.kw['image_thumbnail_size'], False)
 
     def gen_tasks(self):
         """Copy static files into the output folder."""
-
         self.kw = {
             'image_thumbnail_size': self.site.config['IMAGE_THUMBNAIL_SIZE'],
             'max_image_size': self.site.config['MAX_IMAGE_SIZE'],
diff --git a/nikola/plugins/task/sitemap.plugin b/nikola/plugins/task/sitemap.plugin
index 0b992b8..e3c991f 100644
--- a/nikola/plugins/task/sitemap.plugin
+++ b/nikola/plugins/task/sitemap.plugin
@@ -1,10 +1,13 @@
 [Core]
-Name = sitemap
-Module = sitemap
+name = sitemap
+module = sitemap
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Generate google sitemap.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Generate google sitemap.
+
+[Nikola]
+plugincategory = Task
 
diff --git a/nikola/plugins/task/sitemap/__init__.py b/nikola/plugins/task/sitemap/__init__.py
index 92d557d..fd781d6 100644
--- a/nikola/plugins/task/sitemap/__init__.py
+++ b/nikola/plugins/task/sitemap/__init__.py
@@ -24,9 +24,12 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+"""Generate a sitemap."""
+
 from __future__ import print_function, absolute_import, unicode_literals
 import io
 import datetime
+import dateutil.tz
 import os
 try:
     from urlparse import urljoin, urlparse
@@ -42,6 +45,7 @@ from nikola.utils import config_changed, apply_filters
 urlset_header = """
 
@@ -58,6 +62,7 @@ urlset_footer = ""
 sitemapindex_header = """
 
@@ -76,7 +81,7 @@ sitemapindex_footer = ""
 
 
 def get_base_path(base):
-    """returns the path of a base URL if it contains one.
+    """Return the path of a base URL if it contains one.
 
     >>> get_base_path('http://some.site') == '/'
     True
@@ -101,6 +106,7 @@ def get_base_path(base):
 
 
 class Sitemap(LateTask):
+
     """Generate a sitemap."""
 
     name = "sitemap"
@@ -114,10 +120,12 @@ class Sitemap(LateTask):
             "strip_indexes": self.site.config["STRIP_INDEXES"],
             "index_file": self.site.config["INDEX_FILE"],
             "sitemap_include_fileless_dirs": self.site.config["SITEMAP_INCLUDE_FILELESS_DIRS"],
-            "mapped_extensions": self.site.config.get('MAPPED_EXTENSIONS', ['.atom', '.html', '.htm', '.xml', '.rss']),
+            "mapped_extensions": self.site.config.get('MAPPED_EXTENSIONS', ['.atom', '.html', '.htm', '.php', '.xml', '.rss']),
             "robots_exclusions": self.site.config["ROBOTS_EXCLUSIONS"],
             "filters": self.site.config["FILTERS"],
             "translations": self.site.config["TRANSLATIONS"],
+            "tzinfo": self.site.config['__tzinfo__'],
+            "sitemap_plugin_revision": 1,
         }
 
         output = kw['output_folder']
@@ -132,6 +140,7 @@ class Sitemap(LateTask):
         urlset = {}
 
         def scan_locs():
+            """Scan site locations."""
             for root, dirs, files in os.walk(output, followlinks=True):
                 if not dirs and not files and not kw['sitemap_include_fileless_dirs']:
                     continue  # Totally empty, not on sitemap
@@ -169,17 +178,18 @@ class Sitemap(LateTask):
                         filehead = fh.read(1024)
                         fh.close()
 
-                        if path.endswith('.html') or path.endswith('.htm'):
+                        if path.endswith('.html') or path.endswith('.htm') or path.endswith('.php'):
                             """ ignores "html" files without doctype """
                             if b'= kw["taglist_minimum_post_count"]],
-                                 alg=natsort.ns.F | natsort.ns.IC)
+        """Create a global "all your tags/categories" page for each language."""
         categories = [cat.category_name for cat in self.site.category_hierarchy]
-        has_tags = (tags != []) and include_tags
         has_categories = (categories != []) and include_categories
         template_name = "tags.tmpl"
         kw = kw.copy()
-        if include_tags:
-            kw['tags'] = tags
         if include_categories:
             kw['categories'] = categories
         for lang in kw["translations"]:
+            tags = natsort.natsorted([tag for tag in self.site.tags_per_language[lang]
+                                      if len(self.site.posts_per_tag[tag]) >= kw["taglist_minimum_post_count"]],
+                                     alg=natsort.ns.F | natsort.ns.IC)
+            has_tags = (tags != []) and include_tags
+            if include_tags:
+                kw['tags'] = tags
             output_name = os.path.join(
                 kw['output_folder'], self.site.path('tag_index' if has_tags else 'category_index', None, lang))
             output_name = output_name
@@ -219,6 +224,7 @@ class RenderTags(Task):
                 context["cat_items"] = None
             context["permalink"] = self.site.link("tag_index" if has_tags else "category_index", None, lang)
             context["description"] = context["title"]
+            context["pagekind"] = ["list", "tags_page"]
             task = self.site.generic_post_list_renderer(
                 lang,
                 [],
@@ -232,7 +238,7 @@ class RenderTags(Task):
             yield task
 
     def list_tags_page(self, kw):
-        """a global "all your tags/categories" page for each language"""
+        """Create a global "all your tags/categories" page for each language."""
         if self.site.config['TAG_PATH'] == self.site.config['CATEGORY_PATH']:
             yield self._create_tags_page(kw, True, True)
         else:
@@ -254,9 +260,7 @@ class RenderTags(Task):
         return [(child.name, self.site.link("category", child.category_name)) for child in node.children]
 
     def tag_page_as_index(self, tag, lang, post_list, kw, is_category):
-        """render a sort of index page collection using only this
-        tag's posts."""
-
+        """Render a sort of index page collection using only this tag's posts."""
         kind = "category" if is_category else "tag"
 
         def page_link(i, displayed_i, num_pages, force_addition, extension=None):
@@ -284,12 +288,13 @@ class RenderTags(Task):
         context_source["description"] = self._get_description(tag, is_category, lang)
         if is_category:
             context_source["subcategories"] = self._get_subcategories(tag)
+        context_source["pagekind"] = ["index", "tag_page"]
         template_name = "tagindex.tmpl"
 
         yield self.site.generic_index_renderer(lang, post_list, indexes_title, template_name, context_source, kw, str(self.name), page_link, page_path)
 
     def tag_page_as_list(self, tag, lang, post_list, kw, is_category):
-        """We render a single flat link list with this tag's posts"""
+        """Render a single flat link list with this tag's posts."""
         kind = "category" if is_category else "tag"
         template_name = "tag.tmpl"
         output_name = os.path.join(kw['output_folder'], self.site.path(
@@ -308,6 +313,7 @@ class RenderTags(Task):
         context["description"] = self._get_description(tag, is_category, lang)
         if is_category:
             context["subcategories"] = self._get_subcategories(tag)
+        context["pagekind"] = ["list", "tag_page"]
         task = self.site.generic_post_list_renderer(
             lang,
             post_list,
@@ -321,7 +327,7 @@ class RenderTags(Task):
         yield task
 
     def tag_rss(self, tag, lang, posts, kw, is_category):
-        """RSS for a single tag / language"""
+        """Create a RSS feed for a single tag in a given language."""
         kind = "category" if is_category else "tag"
         # Render RSS
         output_name = os.path.normpath(
@@ -352,21 +358,25 @@ class RenderTags(Task):
         return utils.apply_filters(task, kw['filters'])
 
     def slugify_tag_name(self, name):
+        """Slugify a tag name."""
         if self.site.config['SLUG_TAG_PATH']:
             name = utils.slugify(name)
         return name
 
     def tag_index_path(self, name, lang):
+        """Return path to the tag index."""
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                               self.site.config['TAG_PATH'],
                               self.site.config['INDEX_FILE']] if _f]
 
     def category_index_path(self, name, lang):
+        """Return path to the category index."""
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                               self.site.config['CATEGORY_PATH'],
                               self.site.config['INDEX_FILE']] if _f]
 
     def tag_path(self, name, lang):
+        """Return path to a tag."""
         if self.site.config['PRETTY_URLS']:
             return [_f for _f in [
                 self.site.config['TRANSLATIONS'][lang],
@@ -380,16 +390,19 @@ class RenderTags(Task):
                 self.slugify_tag_name(name) + ".html"] if _f]
 
     def tag_atom_path(self, name, lang):
+        """Return path to a tag Atom feed."""
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                               self.site.config['TAG_PATH'], self.slugify_tag_name(name) + ".atom"] if
                 _f]
 
     def tag_rss_path(self, name, lang):
+        """Return path to a tag RSS feed."""
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                               self.site.config['TAG_PATH'], self.slugify_tag_name(name) + ".xml"] if
                 _f]
 
     def slugify_category_name(self, name):
+        """Slugify a category name."""
         path = self.site.parse_category_name(name)
         if self.site.config['CATEGORY_OUTPUT_FLAT_HIERARCHY']:
             path = path[-1:]  # only the leaf
@@ -404,6 +417,7 @@ class RenderTags(Task):
         return path
 
     def category_path(self, name, lang):
+        """Return path to a category."""
         if self.site.config['PRETTY_URLS']:
             return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                                   self.site.config['CATEGORY_PATH']] if
@@ -414,11 +428,13 @@ class RenderTags(Task):
                     _f] + self._add_extension(self.slugify_category_name(name), ".html")
 
     def category_atom_path(self, name, lang):
+        """Return path to a category Atom feed."""
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                               self.site.config['CATEGORY_PATH']] if
                 _f] + self._add_extension(self.slugify_category_name(name), ".atom")
 
     def category_rss_path(self, name, lang):
+        """Return path to a category RSS feed."""
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                               self.site.config['CATEGORY_PATH']] if
                 _f] + self._add_extension(self.slugify_category_name(name), ".xml")
diff --git a/nikola/plugins/template/__init__.py b/nikola/plugins/template/__init__.py
index a1d17a6..d416ad7 100644
--- a/nikola/plugins/template/__init__.py
+++ b/nikola/plugins/template/__init__.py
@@ -23,3 +23,5 @@
 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Default template engines for Nikola."""
diff --git a/nikola/plugins/template/jinja.plugin b/nikola/plugins/template/jinja.plugin
index 0bdcb94..cfe9fa8 100644
--- a/nikola/plugins/template/jinja.plugin
+++ b/nikola/plugins/template/jinja.plugin
@@ -1,9 +1,13 @@
 [Core]
-Name = jinja
-Module = jinja
+name = jinja
+module = jinja
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Support for Jinja2 templates.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Support for Jinja2 templates.
+
+[Nikola]
+plugincategory = Template
+
diff --git a/nikola/plugins/template/jinja.py b/nikola/plugins/template/jinja.py
index 82e8397..b02d75c 100644
--- a/nikola/plugins/template/jinja.py
+++ b/nikola/plugins/template/jinja.py
@@ -24,8 +24,10 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-"""Jinja template handlers"""
 
+"""Jinja template handler."""
+
+from __future__ import unicode_literals
 import os
 import json
 from collections import deque
@@ -40,14 +42,15 @@ from nikola.utils import makedirs, req_missing
 
 
 class JinjaTemplates(TemplateSystem):
-    """Wrapper for Jinja2 templates."""
+
+    """Support for Jinja2 templates."""
 
     name = "jinja"
     lookup = None
     dependency_cache = {}
 
     def __init__(self):
-        """ initialize Jinja2 wrapper with extended set of filters"""
+        """Initialize Jinja2 environment with extended set of filters."""
         if jinja2 is None:
             return
         self.lookup = jinja2.Environment()
@@ -59,26 +62,25 @@ class JinjaTemplates(TemplateSystem):
         self.lookup.globals['tuple'] = tuple
 
     def set_directories(self, directories, cache_folder):
-        """Create a template lookup."""
+        """Create a new template lookup with set directories."""
         if jinja2 is None:
             req_missing(['jinja2'], 'use this theme')
         self.directories = directories
         self.create_lookup()
 
     def inject_directory(self, directory):
-        """if it's not there, add the directory to the lookup with lowest priority, and
-        recreate the lookup."""
+        """Add a directory to the lookup and recreate it if it's not there yet."""
         if directory not in self.directories:
             self.directories.append(directory)
             self.create_lookup()
 
     def create_lookup(self):
-        """Create a template lookup object."""
+        """Create a template lookup."""
         self.lookup.loader = jinja2.FileSystemLoader(self.directories,
                                                      encoding='utf-8')
 
     def set_site(self, site):
-        """Sets the site."""
+        """Set the Nikola site."""
         self.site = site
         self.lookup.filters.update(self.site.config['TEMPLATE_FILTERS'])
 
@@ -99,6 +101,7 @@ class JinjaTemplates(TemplateSystem):
         return self.lookup.from_string(template).render(**context)
 
     def template_deps(self, template_name):
+        """Generate list of dependencies for a template."""
         # Cache the lists of dependencies for each template name.
         if self.dependency_cache.get(template_name) is None:
             # Use a breadth-first search to find all templates this one
diff --git a/nikola/plugins/template/mako.plugin b/nikola/plugins/template/mako.plugin
index 2fe6d98..d256faf 100644
--- a/nikola/plugins/template/mako.plugin
+++ b/nikola/plugins/template/mako.plugin
@@ -1,9 +1,13 @@
 [Core]
-Name = mako
-Module = mako
+name = mako
+module = mako
 
 [Documentation]
-Author = Roberto Alsina
-Version = 1.0
-Website = http://getnikola.com
-Description = Support for Mako templates.
+author = Roberto Alsina
+version = 1.0
+website = http://getnikola.com
+description = Support for Mako templates.
+
+[Nikola]
+plugincategory = Template
+
diff --git a/nikola/plugins/template/mako.py b/nikola/plugins/template/mako.py
index e5545f6..aed6596 100644
--- a/nikola/plugins/template/mako.py
+++ b/nikola/plugins/template/mako.py
@@ -24,14 +24,15 @@
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-"""Mako template handlers"""
+"""Mako template handler."""
+
 from __future__ import unicode_literals, print_function, absolute_import
 import os
 import shutil
 import sys
 import tempfile
 
-from mako import util, lexer
+from mako import util, lexer, parsetree
 from mako.lookup import TemplateLookup
 from mako.template import Template
 from markupsafe import Markup  # It's ok, Mako requires it
@@ -43,7 +44,8 @@ LOGGER = get_logger('mako', STDERR_HANDLER)
 
 
 class MakoTemplates(TemplateSystem):
-    """Wrapper for Mako templates."""
+
+    """Support for Mako templates."""
 
     name = "mako"
 
@@ -54,6 +56,7 @@ class MakoTemplates(TemplateSystem):
     cache_dir = None
 
     def get_deps(self, filename):
+        """Get dependencies for a template (internal function)."""
         text = util.read_file(filename)
         lex = lexer.Lexer(text=text, filename=filename)
         lex.parse()
@@ -61,13 +64,12 @@ class MakoTemplates(TemplateSystem):
         deps = []
         for n in lex.template.nodes:
             keyword = getattr(n, 'keyword', None)
-            if keyword in ["inherit", "namespace"]:
+            if keyword in ["inherit", "namespace"] or isinstance(n, parsetree.IncludeTag):
                 deps.append(n.attributes['file'])
-            # TODO: include tags are not handled
         return deps
 
     def set_directories(self, directories, cache_folder):
-        """Set directories and create a template lookup."""
+        """Create a new template lookup with set directories."""
         cache_dir = os.path.join(cache_folder, '.mako.tmp')
         # Workaround for a Mako bug, Issue #825
         if sys.version_info[0] == 2:
@@ -83,21 +85,20 @@ class MakoTemplates(TemplateSystem):
         self.create_lookup()
 
     def inject_directory(self, directory):
-        """if it's not there, add the directory to the lookup with lowest priority, and
-        recreate the lookup."""
+        """Add a directory to the lookup and recreate it if it's not there yet."""
         if directory not in self.directories:
             self.directories.append(directory)
             self.create_lookup()
 
     def create_lookup(self):
-        """Create a template lookup object."""
+        """Create a template lookup."""
         self.lookup = TemplateLookup(
             directories=self.directories,
             module_directory=self.cache_dir,
             output_encoding='utf-8')
 
     def set_site(self, site):
-        """Sets the site."""
+        """Set the Nikola site."""
         self.site = site
         self.filters.update(self.site.config['TEMPLATE_FILTERS'])
 
@@ -113,14 +114,12 @@ class MakoTemplates(TemplateSystem):
         return data
 
     def render_template_to_string(self, template, context):
-        """ Render template to a string using context. """
-
+        """Render template to a string using context."""
         context.update(self.filters)
-
         return Template(template).render(**context)
 
     def template_deps(self, template_name):
-        """Returns filenames which are dependencies for a template."""
+        """Generate list of dependencies for a template."""
         # We can cache here because dependencies should
         # not change between runs
         if self.cache.get(template_name, None) is None:
@@ -134,4 +133,5 @@ class MakoTemplates(TemplateSystem):
 
 
 def striphtml(text):
+    """Strip HTML tags from text."""
     return Markup(text).striptags()
-- 
cgit v1.2.3