diff options
Diffstat (limited to 'nikola/plugins/command')
36 files changed, 985 insertions, 353 deletions
diff --git a/nikola/plugins/command/__init__.py b/nikola/plugins/command/__init__.py index 2aa5267..62d7086 100644 --- a/nikola/plugins/command/__init__.py +++ b/nikola/plugins/command/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/auto.plugin b/nikola/plugins/command/auto.plugin index 3e2b17d..1081c78 100644 --- a/nikola/plugins/command/auto.plugin +++ b/nikola/plugins/command/auto.plugin @@ -5,7 +5,7 @@ module = auto [Documentation] author = Roberto Alsina version = 2.1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Automatically detect site changes, rebuild and optionally refresh a browser. [Nikola] diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py index 71f9624..a82dc3e 100644 --- a/nikola/plugins/command/auto/__init__.py +++ b/nikola/plugins/command/auto/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -43,6 +43,7 @@ except ImportError: import webbrowser from wsgiref.simple_server import make_server import wsgiref.util +import pkg_resources from blinker import signal try: @@ -61,9 +62,8 @@ except ImportError: FileSystemEventHandler = object PatternMatchingEventHandler = object - from nikola.plugin_categories import Command -from nikola.utils import req_missing, get_logger, get_theme_path, STDERR_HANDLER +from nikola.utils import dns_sd, req_missing, get_logger, get_theme_path, STDERR_HANDLER LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js') error_signal = signal('error') refresh_signal = signal('refresh') @@ -79,13 +79,14 @@ ERROR {} class CommandAuto(Command): - """Automatic rebuilds for Nikola.""" name = "auto" logger = None has_server = True doc_purpose = "builds and serves a site; automatically detects site changes, rebuilds, and optionally refreshes a browser" + dns_sd = None + cmd_options = [ { 'name': 'port', @@ -101,7 +102,7 @@ class CommandAuto(Command): 'long': 'address', 'type': str, 'default': '127.0.0.1', - 'help': 'Address to bind (default: 127.0.0.1 – localhost)', + 'help': 'Address to bind (default: 127.0.0.1 -- localhost)', }, { 'name': 'browser', @@ -142,7 +143,7 @@ class CommandAuto(Command): self.cmd_arguments = ['nikola', 'build'] if self.site.configuration_filename != 'conf.py': - self.cmd_arguments = ['--conf=' + self.site.configuration_filename] + self.cmd_arguments + self.cmd_arguments.append('--conf=' + self.site.configuration_filename) # Run an initial build so we are up-to-date subprocess.call(self.cmd_arguments) @@ -156,7 +157,7 @@ class CommandAuto(Command): # Do not duplicate entries -- otherwise, multiple rebuilds are triggered watched = set([ - 'templates/', + 'templates/' ] + [get_theme_path(name) for name in self.site.THEMES]) for item in self.site.config['post_pages']: watched.add(os.path.dirname(item[0])) @@ -166,6 +167,10 @@ class CommandAuto(Command): watched.add(item) for item in self.site.config['LISTINGS_FOLDERS']: watched.add(item) + for item in self.site._plugin_places: + watched.add(item) + # Nikola itself (useful for developers) + watched.add(pkg_resources.resource_filename('nikola', '')) out_folder = self.site.config['OUTPUT_FOLDER'] if options and options.get('browser'): @@ -208,7 +213,6 @@ class CommandAuto(Command): parent = self class Mixed(WebSocketWSGIApplication): - """A class that supports WS and HTTP protocols on the same port.""" def __call__(self, environ, start_response): @@ -235,9 +239,12 @@ class CommandAuto(Command): webbrowser.open('http://{0}:{1}'.format(host, port)) try: + self.dns_sd = dns_sd(port, (options['ipv6'] or '::' in host)) ws.serve_forever() except KeyboardInterrupt: self.logger.info("Server is shutting down.") + if self.dns_sd: + self.dns_sd.Reset() # This is a hack, but something is locking up in a futex # and exit() doesn't work. os.kill(os.getpid(), 15) @@ -262,6 +269,8 @@ class CommandAuto(Command): fname = os.path.basename(event_path) if (fname.endswith('~') or fname.startswith('.') or + '__pycache__' in event_path or + event_path.endswith(('.pyc', '.pyo', '.pyd')) or os.path.isdir(event_path)): # Skip on folders, these are usually duplicates return self.logger.info('REBUILDING SITE (from {0})'.format(event_path)) @@ -293,18 +302,21 @@ class CommandAuto(Command): mimetype = 'text/html' if uri.endswith('/') else mimetypes.guess_type(uri)[0] or 'application/octet-stream' if os.path.isdir(f_path): - if not f_path.endswith('/'): # Redirect to avoid breakage - start_response('301 Redirect', [('Location', p_uri.path + '/')]) + if not p_uri.path.endswith('/'): # Redirect to avoid breakage + start_response('301 Moved Permanently', [('Location', p_uri.path + '/')]) return [] f_path = os.path.join(f_path, self.site.config['INDEX_FILE']) mimetype = 'text/html' if p_uri.path == '/robots.txt': - start_response('200 OK', [('Content-type', 'text/plain')]) + start_response('200 OK', [('Content-type', 'text/plain; charset=UTF-8')]) return ['User-Agent: *\nDisallow: /\n'.encode('utf-8')] elif os.path.isfile(f_path): with open(f_path, 'rb') as fd: - start_response('200 OK', [('Content-type', mimetype)]) + if mimetype.startswith('text/') or mimetype.endswith('+xml'): + start_response('200 OK', [('Content-type', "{0}; charset=UTF-8".format(mimetype))]) + else: + start_response('200 OK', [('Content-type', mimetype)]) return [self.file_filter(mimetype, fd.read())] elif p_uri.path == '/livereload.js': with open(LRJS_PATH, 'rb') as fd: @@ -337,7 +349,6 @@ pending = [] class LRSocket(WebSocket): - """Speak Livereload protocol.""" def __init__(self, *a, **kw): @@ -410,7 +421,6 @@ class LRSocket(WebSocket): class OurWatchHandler(FileSystemEventHandler): - """A Nikola-specific handler for Watchdog.""" def __init__(self, function): @@ -424,7 +434,6 @@ class OurWatchHandler(FileSystemEventHandler): class ConfigWatchHandler(FileSystemEventHandler): - """A Nikola-specific handler for Watchdog that handles the config file (as a workaround).""" def __init__(self, configuration_filename, function): diff --git a/nikola/plugins/command/bootswatch_theme.plugin b/nikola/plugins/command/bootswatch_theme.plugin index fc25045..51e6718 100644 --- a/nikola/plugins/command/bootswatch_theme.plugin +++ b/nikola/plugins/command/bootswatch_theme.plugin @@ -5,7 +5,7 @@ module = bootswatch_theme [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Given a swatch name and a parent theme, creates a custom theme. [Nikola] diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py index b5644a1..4808fdb 100644 --- a/nikola/plugins/command/bootswatch_theme.py +++ b/nikola/plugins/command/bootswatch_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,8 +36,14 @@ from nikola import utils LOGGER = utils.get_logger('bootswatch_theme', utils.STDERR_HANDLER) -class CommandBootswatchTheme(Command): +def _check_for_theme(theme, themes): + for t in themes: + if t.endswith(os.sep + theme): + return True + return False + +class CommandBootswatchTheme(Command): """Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" name = "bootswatch_theme" @@ -80,23 +86,27 @@ class CommandBootswatchTheme(Command): version = '' # See if we need bootswatch for bootstrap v2 or v3 - themes = utils.get_theme_chain(parent) - if 'bootstrap3' not in themes and 'bootstrap3-jinja' not in themes: + themes = utils.get_theme_chain(parent, self.site.themes_dirs) + if not _check_for_theme('bootstrap3', themes) and not _check_for_theme('bootstrap3-jinja', themes): version = '2' - elif 'bootstrap' not in themes and 'bootstrap-jinja' not in themes: + elif not _check_for_theme('bootstrap', themes) and not _check_for_theme('bootstrap-jinja', themes): LOGGER.warn('"bootswatch_theme" only makes sense for themes that use bootstrap') - elif 'bootstrap3-gradients' in themes or 'bootstrap3-gradients-jinja' in themes: + elif _check_for_theme('bootstrap3-gradients', themes) or _check_for_theme('bootstrap3-gradients-jinja', themes): LOGGER.warn('"bootswatch_theme" doesn\'t work well with the bootstrap3-gradients family') LOGGER.info("Creating '{0}' theme from '{1}' and '{2}'".format(name, swatch, parent)) utils.makedirs(os.path.join('themes', name, 'assets', 'css')) for fname in ('bootstrap.min.css', 'bootstrap.css'): - url = 'http://bootswatch.com' + url = 'https://bootswatch.com' if version: url += '/' + version url = '/'.join((url, swatch, fname)) LOGGER.info("Downloading: " + url) - data = requests.get(url).text + r = requests.get(url) + if r.status_code > 299: + LOGGER.error('Error {} getting {}', r.status_code, url) + exit(1) + data = r.text with open(os.path.join('themes', name, 'assets', 'css', fname), 'wb+') as output: output.write(data.encode('utf-8')) diff --git a/nikola/plugins/command/check.plugin b/nikola/plugins/command/check.plugin index e380e64..6d2df82 100644 --- a/nikola/plugins/command/check.plugin +++ b/nikola/plugins/command/check.plugin @@ -5,7 +5,7 @@ module = check [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Check the generated site [Nikola] diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py index abf183e..0141a6b 100644 --- a/nikola/plugins/command/check.py +++ b/nikola/plugins/command/check.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,6 +32,7 @@ import os import re import sys import time +import logbook try: from urllib import unquote from urlparse import urlparse, urljoin, urldefrag @@ -46,7 +47,10 @@ from nikola.plugin_categories import Command from nikola.utils import get_logger, STDERR_HANDLER -def _call_nikola_list(site): +def _call_nikola_list(site, cache=None): + if cache is not None: + if 'files' in cache and 'deps' in cache: + return cache['files'], cache['deps'] files = [] deps = defaultdict(list) for task in generate_tasks('render_site', site.gen_tasks('render_site', "Task", '')): @@ -57,16 +61,19 @@ def _call_nikola_list(site): files.extend(task.targets) for target in task.targets: deps[target].extend(task.file_dep) + if cache is not None: + cache['files'] = files + cache['deps'] = deps return files, deps -def real_scan_files(site): +def real_scan_files(site, cache=None): """Scan for files.""" task_fnames = set([]) real_fnames = set([]) output_folder = site.config['OUTPUT_FOLDER'] # First check that all targets are generated in the right places - for fname in _call_nikola_list(site)[0]: + for fname in _call_nikola_list(site, cache)[0]: fname = fname.strip() if fname.startswith(output_folder): task_fnames.add(fname) @@ -94,7 +101,6 @@ def fs_relpath_from_url_path(url_path): class CommandCheck(Command): - """Check the generated site.""" name = "check" @@ -159,25 +165,28 @@ class CommandCheck(Command): print(self.help()) return False if options['verbose']: - self.logger.level = 1 + self.logger.level = logbook.DEBUG else: - self.logger.level = 4 + self.logger.level = logbook.NOTICE + failure = False if options['links']: - failure = self.scan_links(options['find_sources'], options['remote']) + failure |= self.scan_links(options['find_sources'], options['remote']) if options['files']: - failure = self.scan_files() + failure |= self.scan_files() if options['clean']: - failure = self.clean_files() + failure |= self.clean_files() if failure: return 1 existing_targets = set([]) checked_remote_targets = {} + cache = {} def analyze(self, fname, find_sources=False, check_remote=False): """Analyze links on a page.""" rv = False self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']] + self.internal_redirects = [urljoin('/', _[0]) for _ in self.site.config['REDIRECTIONS']] base_url = urlparse(self.site.config['BASE_URL']) self.existing_targets.add(self.site.config['SITE_URL']) self.existing_targets.add(self.site.config['BASE_URL']) @@ -185,7 +194,7 @@ class CommandCheck(Command): deps = {} if find_sources: - deps = _call_nikola_list(self.site)[1] + deps = _call_nikola_list(self.site, self.cache)[1] if url_type in ('absolute', 'full_path'): url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path @@ -203,31 +212,70 @@ class CommandCheck(Command): # Quietly ignore files that don’t exist; use `nikola check -f` instead (Issue #1831) return False - d = lxml.html.fromstring(open(filename, 'rb').read()) - for l in d.iterlinks(): + if '.html' == fname[-5:]: + d = lxml.html.fromstring(open(filename, 'rb').read()) + extra_objs = lxml.html.fromstring('<html/>') + + # Turn elements with a srcset attribute into individual img elements with src attributes + for obj in list(d.xpath('(*//img|*//source)')): + if 'srcset' in obj.attrib: + for srcset_item in obj.attrib['srcset'].split(','): + extra_objs.append(lxml.etree.Element('img', src=srcset_item.strip().split(' ')[0])) + link_elements = list(d.iterlinks()) + list(extra_objs.iterlinks()) + # Extract links from XML formats to minimal HTML, allowing those to go through the link checks + elif '.atom' == filename[-5:]: + d = lxml.etree.parse(filename) + link_elements = lxml.html.fromstring('<html/>') + for elm in d.findall('*//{http://www.w3.org/2005/Atom}link'): + feed_link = elm.attrib['href'].split('?')[0].strip() # strip FEED_LINKS_APPEND_QUERY + link_elements.append(lxml.etree.Element('a', href=feed_link)) + link_elements = list(link_elements.iterlinks()) + elif filename.endswith('sitemap.xml') or filename.endswith('sitemapindex.xml'): + d = lxml.etree.parse(filename) + link_elements = lxml.html.fromstring('<html/>') + for elm in d.getroot().findall("*//{http://www.sitemaps.org/schemas/sitemap/0.9}loc"): + link_elements.append(lxml.etree.Element('a', href=elm.text.strip())) + link_elements = list(link_elements.iterlinks()) + else: # unsupported file type + return False + + for l in link_elements: target = l[2] if target == "#": continue - target, _ = urldefrag(target) + target = urldefrag(target)[0] + + if any([urlparse(target).netloc.endswith(_) for _ in ['example.com', 'example.net', 'example.org']]): + self.logger.debug("Not testing example address \"{0}\".".format(target)) + continue + + # absolute URL to root-relative + if target.startswith(base_url.geturl()): + target = target.replace(base_url.geturl(), '/') + parsed = urlparse(target) # Warn about links from https to http (mixed-security) if base_url.netloc == parsed.netloc and base_url.scheme == "https" and parsed.scheme == "http": self.logger.warn("Mixed-content security for link in {0}: {1}".format(filename, target)) + # Link to an internal REDIRECTIONS page + if target in self.internal_redirects: + redir_status_code = 301 + redir_target = [_dest for _target, _dest in self.site.config['REDIRECTIONS'] if urljoin('/', _target) == target][0] + self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: 301]".format(redir_target, filename, target)) + # Absolute links to other domains, skip # Absolute links when using only paths, skip. if ((parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc) or \ ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')): if not check_remote or parsed.scheme not in ["http", "https"]: continue - if parsed.netloc == base_url.netloc: # absolute URL to self.site - continue if target in self.checked_remote_targets: # already checked this exact target - if self.checked_remote_targets[target] in [301, 307]: + if self.checked_remote_targets[target] in [301, 308]: self.logger.warn("Remote link PERMANENTLY redirected in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) - elif self.checked_remote_targets[target] in [302, 308]: - self.logger.info("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target])) + elif self.checked_remote_targets[target] in [302, 307]: + self.logger.debug("Remote link temporarily redirected in {0}: {1} [HTTP: {2}]".format(filename, target, self.checked_remote_targets[target])) elif self.checked_remote_targets[target] > 399: self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) continue @@ -255,7 +303,7 @@ class CommandCheck(Command): if redir_status_code in [301, 308]: self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) if redir_status_code in [302, 307]: - self.logger.info("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + self.logger.debug("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) self.checked_remote_targets[resp.url] = resp.status_code self.checked_remote_targets[target] = redir_status_code else: @@ -275,8 +323,9 @@ class CommandCheck(Command): target_filename = os.path.abspath( os.path.join(self.site.config['OUTPUT_FOLDER'], unquote(target.lstrip('/')))) else: # Relative path + unquoted_target = unquote(target).encode('utf-8') if sys.version_info.major >= 3 else unquote(target).decode('utf-8') target_filename = os.path.abspath( - os.path.join(os.path.dirname(filename), unquote(target))) + os.path.join(os.path.dirname(filename).encode('utf-8'), unquoted_target)) elif url_type in ('full_path', 'absolute'): if url_type == 'absolute': @@ -292,9 +341,10 @@ class CommandCheck(Command): if any(re.search(x, target_filename) for x in self.whitelist): continue + elif target_filename not in self.existing_targets: if os.path.exists(target_filename): - self.logger.notice("Good link {0} => {1}".format(target, target_filename)) + self.logger.info("Good link {0} => {1}".format(target, target_filename)) self.existing_targets.add(target_filename) else: rv = True @@ -304,31 +354,42 @@ class CommandCheck(Command): self.logger.warn("\n".join(deps[filename])) self.logger.warn("===============================\n") except Exception as exc: - self.logger.error("Error with: {0} {1}".format(filename, exc)) + self.logger.error(u"Error with: {0} {1}".format(filename, exc)) return rv def scan_links(self, find_sources=False, check_remote=False): """Check links on the site.""" - self.logger.info("Checking Links:") - self.logger.info("===============\n") - self.logger.notice("{0} mode".format(self.site.config['URL_TYPE'])) + self.logger.debug("Checking Links:") + self.logger.debug("===============\n") + self.logger.debug("{0} mode".format(self.site.config['URL_TYPE'])) failure = False # Maybe we should just examine all HTML files output_folder = self.site.config['OUTPUT_FOLDER'] - for fname in _call_nikola_list(self.site)[0]: - if fname.startswith(output_folder) and '.html' == fname[-5:]: - if self.analyze(fname, find_sources, check_remote): - failure = True + + if urlparse(self.site.config['BASE_URL']).netloc == 'example.com': + self.logger.error("You've not changed the SITE_URL (or BASE_URL) setting from \"example.com\"!") + + for fname in _call_nikola_list(self.site, self.cache)[0]: + if fname.startswith(output_folder): + if '.html' == fname[-5:]: + if self.analyze(fname, find_sources, check_remote): + failure = True + if '.atom' == fname[-5:]: + if self.analyze(fname, find_sources, False): + failure = True + if fname.endswith('sitemap.xml') or fname.endswith('sitemapindex.xml'): + if self.analyze(fname, find_sources, False): + failure = True if not failure: - self.logger.info("All links checked.") + self.logger.debug("All links checked.") return failure def scan_files(self): """Check files in the site, find missing and orphaned files.""" failure = False - self.logger.info("Checking Files:") - self.logger.info("===============\n") - only_on_output, only_on_input = real_scan_files(self.site) + self.logger.debug("Checking Files:") + self.logger.debug("===============\n") + only_on_output, only_on_input = real_scan_files(self.site, self.cache) # Ignore folders only_on_output = [p for p in only_on_output if not os.path.isdir(p)] @@ -346,16 +407,18 @@ class CommandCheck(Command): for f in only_on_input: self.logger.warn(f) if not failure: - self.logger.info("All files checked.") + self.logger.debug("All files checked.") return failure def clean_files(self): """Remove orphaned files.""" - only_on_output, _ = real_scan_files(self.site) + only_on_output, _ = real_scan_files(self.site, self.cache) for f in only_on_output: - self.logger.info('removed: {0}'.format(f)) + self.logger.debug('removed: {0}'.format(f)) os.unlink(f) + warn_flag = bool(only_on_output) + # Find empty directories and remove them output_folder = self.site.config['OUTPUT_FOLDER'] all_dirs = [] @@ -365,7 +428,12 @@ class CommandCheck(Command): for d in all_dirs: try: os.rmdir(d) - self.logger.info('removed: {0}/'.format(d)) + self.logger.debug('removed: {0}/'.format(d)) + warn_flag = True except OSError: pass + + if warn_flag: + self.logger.warn('Some files or directories have been removed, your site may need rebuilding') + return True diff --git a/nikola/plugins/command/console.plugin b/nikola/plugins/command/console.plugin index 333762c..9bcc909 100644 --- a/nikola/plugins/command/console.plugin +++ b/nikola/plugins/command/console.plugin @@ -5,7 +5,7 @@ module = console [Documentation] author = Chris Warrick, Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Start a debugging python console [Nikola] diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py index 539fa08..c6a8376 100644 --- a/nikola/plugins/command/console.py +++ b/nikola/plugins/command/console.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Chris Warrick, Roberto Alsina and others. +# Copyright © 2012-2016 Chris Warrick, Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,7 +38,6 @@ LOGGER = get_logger('console', STDERR_HANDLER) class CommandConsole(Command): - """Start debugging console.""" name = "console" @@ -76,7 +75,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f ] def ipython(self, willful=True): - """IPython shell.""" + """Run an IPython shell.""" try: import IPython except ImportError as e: @@ -85,12 +84,13 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f raise e # That’s how _execute knows whether to try something else. else: site = self.context['site'] # NOQA + nikola_site = self.context['site'] # NOQA conf = self.context['conf'] # NOQA commands = self.context['commands'] # NOQA IPython.embed(header=self.header.format('IPython')) def bpython(self, willful=True): - """bpython shell.""" + """Run a bpython shell.""" try: import bpython except ImportError as e: @@ -101,7 +101,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f bpython.embed(banner=self.header.format('bpython'), locals_=self.context) def plain(self, willful=True): - """Plain Python shell.""" + """Run a plain Python shell.""" import code try: import readline @@ -131,6 +131,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f self.context = { 'conf': self.site.config, 'site': self.site, + 'nikola_site': self.site, 'commands': self.site.commands, } if options['bpython']: diff --git a/nikola/plugins/command/deploy.plugin b/nikola/plugins/command/deploy.plugin index 4743ca2..8bdc0e2 100644 --- a/nikola/plugins/command/deploy.plugin +++ b/nikola/plugins/command/deploy.plugin @@ -5,7 +5,7 @@ module = deploy [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Deploy the site [Nikola] diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py index 821ea11..c2289e8 100644 --- a/nikola/plugins/command/deploy.py +++ b/nikola/plugins/command/deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -30,6 +30,7 @@ from __future__ import print_function import io from datetime import datetime from dateutil.tz import gettz +import dateutil import os import subprocess import time @@ -37,16 +38,15 @@ import time from blinker import signal from nikola.plugin_categories import Command -from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER +from nikola.utils import get_logger, clean_before_deployment, STDERR_HANDLER class CommandDeploy(Command): - """Deploy site.""" name = "deploy" - doc_usage = "[[preset [preset...]]" + doc_usage = "[preset [preset...]]" doc_purpose = "deploy the site" doc_description = "Deploy the site by executing deploy commands from the presets listed on the command line. If no presets are specified, `default` is executed." logger = None @@ -56,27 +56,42 @@ class CommandDeploy(Command): self.logger = get_logger('deploy', STDERR_HANDLER) # Get last successful deploy date timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy') + + # Get last-deploy from persistent state + last_deploy = self.site.state.get('last_deploy') + if last_deploy is None: + # If there is a last-deploy saved, move it to the new state persistence thing + # FIXME: remove in Nikola 8 + if os.path.isfile(timestamp_path): + try: + with io.open(timestamp_path, 'r', encoding='utf8') as inf: + last_deploy = dateutil.parser.parse(inf.read()) + clean = False + except (IOError, Exception) as e: + self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) + last_deploy = datetime(1970, 1, 1) + clean = True + os.unlink(timestamp_path) # Remove because from now on it's in state + else: # Just a default + last_deploy = datetime(1970, 1, 1) + clean = True + else: + last_deploy = dateutil.parser.parse(last_deploy) + clean = False + if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo': self.logger.warn("\nWARNING WARNING WARNING WARNING\n" "You are deploying using the nikolademo Disqus account.\n" "That means you will not be able to moderate the comments in your own site.\n" "And is probably not what you want to do.\n" - "Think about it for 5 seconds, I'll wait :-)\n\n") + "Think about it for 5 seconds, I'll wait :-)\n" + "(press Ctrl+C to abort)\n") time.sleep(5) - deploy_drafts = self.site.config.get('DEPLOY_DRAFTS', True) - deploy_future = self.site.config.get('DEPLOY_FUTURE', False) - undeployed_posts = [] - if not (deploy_drafts and deploy_future): - # Remove drafts and future posts - out_dir = self.site.config['OUTPUT_FOLDER'] - self.site.scan_posts() - for post in self.site.timeline: - if (not deploy_drafts and post.is_draft) or \ - (not deploy_future and post.publish_later): - remove_file(os.path.join(out_dir, post.destination_path())) - remove_file(os.path.join(out_dir, post.source_path)) - undeployed_posts.append(post) + # Remove drafts and future posts if requested + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) if args: presets = args @@ -98,27 +113,22 @@ class CommandDeploy(Command): try: subprocess.check_call(command, shell=True) except subprocess.CalledProcessError as e: - self.logger.error('Failed deployment — command {0} ' + self.logger.error('Failed deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode)) return e.returncode self.logger.info("Successful deployment") - try: - with io.open(timestamp_path, 'r', encoding='utf8') as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - clean = False - except (IOError, Exception) as e: - self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) - last_deploy = datetime(1970, 1, 1) - clean = True new_deploy = datetime.utcnow() self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts) - makedirs(self.site.config['CACHE_FOLDER']) # Store timestamp of successful deployment - with io.open(timestamp_path, 'w+', encoding='utf8') as outf: - outf.write(unicode_str(new_deploy.isoformat())) + self.site.state.set('last_deploy', new_deploy.isoformat()) + if clean: + self.logger.info( + 'Looks like this is the first time you deployed this site. ' + 'Let us know you are using Nikola ' + 'at <https://users.getnikola.com/add/> if you want!') def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None): """Emit events for all timeline entries newer than last deploy. diff --git a/nikola/plugins/command/github_deploy.plugin b/nikola/plugins/command/github_deploy.plugin index e793548..21e246c 100644 --- a/nikola/plugins/command/github_deploy.plugin +++ b/nikola/plugins/command/github_deploy.plugin @@ -5,7 +5,7 @@ module = github_deploy [Documentation] author = Puneeth Chaganti version = 1,0 -website = http://getnikola.com +website = https://getnikola.com/ description = Deploy the site to GitHub pages. [Nikola] diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py index 0ab9332..b5ad322 100644 --- a/nikola/plugins/command/github_deploy.py +++ b/nikola/plugins/command/github_deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Puneeth Chaganti and others. +# Copyright © 2014-2016 Puneeth Chaganti and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,15 +27,13 @@ """Deploy site to GitHub Pages.""" from __future__ import print_function -from datetime import datetime -import io import os import subprocess from textwrap import dedent from nikola.plugin_categories import Command from nikola.plugins.command.check import real_scan_files -from nikola.utils import get_logger, req_missing, makedirs, unicode_str, STDERR_HANDLER +from nikola.utils import get_logger, req_missing, clean_before_deployment, STDERR_HANDLER from nikola.__main__ import main from nikola import __version__ @@ -53,16 +51,15 @@ def check_ghp_import_installed(): except OSError: # req_missing defaults to `python=True` — and it’s meant to be like this. # `ghp-import` is installed via pip, but the only way to use it is by executing the script it installs. - req_missing(['ghp-import'], 'deploy the site to GitHub Pages') + req_missing(['ghp-import2'], 'deploy the site to GitHub Pages') class CommandGitHubDeploy(Command): - """Deploy site to GitHub Pages.""" name = 'github_deploy' - doc_usage = '' + doc_usage = '[-m COMMIT_MESSAGE]' doc_purpose = 'deploy the site to GitHub Pages' doc_description = dedent( """\ @@ -72,10 +69,19 @@ class CommandGitHubDeploy(Command): """ ) - + cmd_options = [ + { + 'name': 'commit_message', + 'short': 'm', + 'long': 'message', + 'default': 'Nikola auto commit.', + 'type': str, + 'help': 'Commit message (default: Nikola auto commit.)', + }, + ] logger = None - def _execute(self, command, args): + def _execute(self, options, args): """Run the deployment.""" self.logger = get_logger(CommandGitHubDeploy.name, STDERR_HANDLER) @@ -93,41 +99,69 @@ class CommandGitHubDeploy(Command): for f in only_on_output: os.unlink(f) + # Remove drafts and future posts if requested (Issue #2406) + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) + # Commit and push - self._commit_and_push() + self._commit_and_push(options['commit_message']) return - def _commit_and_push(self): - """Commit all the files and push.""" - source = self.site.config['GITHUB_SOURCE_BRANCH'] - deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] - remote = self.site.config['GITHUB_REMOTE_NAME'] - source_commit = uni_check_output(['git', 'rev-parse', source]) - commit_message = ( - 'Nikola auto commit.\n\n' - 'Source commit: %s' - 'Nikola version: %s' % (source_commit, __version__) - ) - output_folder = self.site.config['OUTPUT_FOLDER'] - - command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] - + def _run_command(self, command, xfail=False): + """Run a command that may or may not fail.""" self.logger.info("==> {0}".format(command)) try: subprocess.check_call(command) + return 0 except subprocess.CalledProcessError as e: + if xfail: + return e.returncode self.logger.error( - 'Failed GitHub deployment — command {0} ' + 'Failed GitHub deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode) ) - return e.returncode + raise SystemError(e.returncode) - self.logger.info("Successful deployment") + def _commit_and_push(self, commit_first_line): + """Commit all the files and push.""" + source = self.site.config['GITHUB_SOURCE_BRANCH'] + deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] + remote = self.site.config['GITHUB_REMOTE_NAME'] + autocommit = self.site.config['GITHUB_COMMIT_SOURCE'] + try: + if autocommit: + commit_message = ( + '{0}\n\n' + 'Nikola version: {1}'.format(commit_first_line, __version__) + ) + e = self._run_command(['git', 'checkout', source], True) + if e != 0: + self._run_command(['git', 'checkout', '-b', source]) + self._run_command(['git', 'add', '.']) + # Figure out if there is anything to commit + e = self._run_command(['git', 'diff-index', '--quiet', 'HEAD'], True) + if e != 0: + self._run_command(['git', 'commit', '-am', commit_message]) + else: + self.logger.notice('Nothing to commit to source branch.') + + source_commit = uni_check_output(['git', 'rev-parse', source]) + commit_message = ( + '{0}\n\n' + 'Source commit: {1}' + 'Nikola version: {2}'.format(commit_first_line, source_commit, __version__) + ) + output_folder = self.site.config['OUTPUT_FOLDER'] + + command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] + + self._run_command(command) - # Store timestamp of successful deployment - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - new_deploy = datetime.utcnow() - makedirs(self.site.config["CACHE_FOLDER"]) - with io.open(timestamp_path, "w+", encoding="utf8") as outf: - outf.write(unicode_str(new_deploy.isoformat())) + if autocommit: + self._run_command(['git', 'push', '-u', remote, source]) + except SystemError as e: + return e.args[0] + + self.logger.info("Successful deployment") diff --git a/nikola/plugins/command/import_wordpress.plugin b/nikola/plugins/command/import_wordpress.plugin index 6c4384e..eab9d17 100644 --- a/nikola/plugins/command/import_wordpress.plugin +++ b/nikola/plugins/command/import_wordpress.plugin @@ -5,7 +5,7 @@ module = import_wordpress [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Import a wordpress site from a XML dump (requires markdown). [Nikola] diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index a652ec8..0b48583 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,6 +38,11 @@ from lxml import etree from collections import defaultdict try: + import html2text +except: + html2text = None + +try: from urlparse import urlparse from urllib import unquote except ImportError: @@ -50,7 +55,7 @@ except ImportError: from nikola.plugin_categories import Command from nikola import utils -from nikola.utils import req_missing +from nikola.utils import req_missing, unicode_str from nikola.plugins.basic_import import ImportMixin, links from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN from nikola.plugins.command.init import SAMPLE_CONF, prepare_config, format_default_translations_config @@ -88,7 +93,6 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False) class CommandImportWordpress(Command, ImportMixin): - """Import a WordPress dump.""" name = "import_wordpress" @@ -171,6 +175,20 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Export comments as .wpcomment files", }, { + 'name': 'html2text', + 'long': 'html2text', + 'default': False, + 'type': bool, + 'help': "Uses html2text (needs to be installed with pip) to transform WordPress posts to MarkDown during import", + }, + { + 'name': 'transform_to_markdown', + 'long': 'transform-to-markdown', + 'default': False, + 'type': bool, + 'help': "Uses WordPress page compiler to transform WordPress posts to HTML and then use html2text to transform them to MarkDown during import", + }, + { 'name': 'transform_to_html', 'long': 'transform-to-html', 'default': False, @@ -191,9 +209,36 @@ class CommandImportWordpress(Command, ImportMixin): 'type': bool, 'help': "Automatically installs the WordPress page compiler (either locally or in the new site) if required by other options.\nWarning: the compiler is GPL software!", }, + { + 'name': 'tag_sanitizing_strategy', + 'long': 'tag-sanitizing-strategy', + 'default': 'first', + 'help': 'lower: Convert all tag and category names to lower case\nfirst: Keep first spelling of tag or category name', + }, + { + 'name': 'one_file', + 'long': 'one-file', + 'default': False, + 'type': bool, + 'help': "Save imported posts in the more modern one-file format.", + }, ] all_tags = set([]) + def _get_compiler(self): + """Return whatever compiler we will use.""" + self._find_wordpress_compiler() + if self.wordpress_page_compiler is not None: + return self.wordpress_page_compiler + plugin_info = self.site.plugin_manager.getPluginByName('markdown', 'PageCompiler') + if plugin_info is not None: + if not plugin_info.is_activated: + self.site.plugin_manager.activatePluginByName(plugin_info.name) + plugin_info.plugin_object.set_site(self.site) + return plugin_info.plugin_object + else: + LOGGER.error("Can't find markdown post compiler.") + def _find_wordpress_compiler(self): """Find WordPress compiler plugin.""" if self.wordpress_page_compiler is not None: @@ -218,6 +263,8 @@ class CommandImportWordpress(Command, ImportMixin): 'putting these arguments before the filename if you ' 'are running into problems.'.format(args)) + self.onefile = options.get('one_file', False) + self.import_into_existing_site = False self.url_map = {} self.timezone = None @@ -234,11 +281,16 @@ class CommandImportWordpress(Command, ImportMixin): self.export_categories_as_categories = options.get('export_categories_as_categories', False) self.export_comments = options.get('export_comments', False) + self.html2text = options.get('html2text', False) + self.transform_to_markdown = options.get('transform_to_markdown', False) + self.transform_to_html = options.get('transform_to_html', False) self.use_wordpress_compiler = options.get('use_wordpress_compiler', False) self.install_wordpress_compiler = options.get('install_wordpress_compiler', False) self.wordpress_page_compiler = None + self.tag_saniziting_strategy = options.get('tag_saniziting_strategy', 'first') + self.auth = None if options.get('download_auth') is not None: username_password = options.get('download_auth') @@ -250,10 +302,18 @@ class CommandImportWordpress(Command, ImportMixin): self.separate_qtranslate_content = options.get('separate_qtranslate_content') self.translations_pattern = options.get('translations_pattern') - if self.transform_to_html and self.use_wordpress_compiler: - LOGGER.warn("It does not make sense to combine --transform-to-html with --use-wordpress-compiler, as the first converts all posts to HTML and the latter option affects zero posts.") + count = (1 if self.html2text else 0) + (1 if self.transform_to_html else 0) + (1 if self.transform_to_markdown else 0) + if count > 1: + LOGGER.error("You can use at most one of the options --html2text, --transform-to-html and --transform-to-markdown.") + return False + if (self.html2text or self.transform_to_html or self.transform_to_markdown) and self.use_wordpress_compiler: + LOGGER.warn("It does not make sense to combine --use-wordpress-compiler with any of --html2text, --transform-to-html and --transform-to-markdown, as the latter convert all posts to HTML and the first option then affects zero posts.") + + if (self.html2text or self.transform_to_markdown) and not html2text: + LOGGER.error("You need to install html2text via 'pip install html2text' before you can use the --html2text and --transform-to-markdown options.") + return False - if self.transform_to_html: + if self.transform_to_html or self.transform_to_markdown: self._find_wordpress_compiler() if not self.wordpress_page_compiler and self.install_wordpress_compiler: if not install_plugin(self.site, 'wordpress_compiler', output_dir='plugins'): # local install @@ -327,7 +387,7 @@ class CommandImportWordpress(Command, ImportMixin): self.context['TRANSLATIONS'] = format_default_translations_config( self.extra_languages) self.context['REDIRECTIONS'] = self.configure_redirections( - self.url_map) + self.url_map, self.base_dir) if self.timezone: self.context['TIMEZONE'] = self.timezone if self.export_categories_as_categories: @@ -337,10 +397,13 @@ class CommandImportWordpress(Command, ImportMixin): # Add tag redirects for tag in self.all_tags: try: - tag_str = tag.decode('utf8') + if isinstance(tag, utils.bytes_str): + tag_str = tag.decode('utf8', 'replace') + else: + tag_str = tag except AttributeError: tag_str = tag - tag = utils.slugify(tag_str) + tag = utils.slugify(tag_str, self.lang) src_url = '{}tag/{}'.format(self.context['SITE_URL'], tag) dst_url = self.site.link('tag', tag) if src_url != dst_url: @@ -372,7 +435,7 @@ class CommandImportWordpress(Command, ImportMixin): if b'<atom:link rel=' in line: continue xml.append(line) - return b'\n'.join(xml) + return b''.join(xml) @classmethod def get_channel_from_file(cls, filename): @@ -386,7 +449,8 @@ class CommandImportWordpress(Command, ImportMixin): wordpress_namespace = channel.nsmap['wp'] context = SAMPLE_CONF.copy() - context['DEFAULT_LANG'] = get_text_tag(channel, 'language', 'en')[:2] + self.lang = get_text_tag(channel, 'language', 'en')[:2] + context['DEFAULT_LANG'] = self.lang context['TRANSLATIONS_PATTERN'] = DEFAULT_TRANSLATIONS_PATTERN context['BLOG_TITLE'] = get_text_tag(channel, 'title', 'PUT TITLE HERE') @@ -418,7 +482,7 @@ class CommandImportWordpress(Command, ImportMixin): PAGES = '(\n' for extension in extensions: POSTS += ' ("posts/*.{0}", "posts", "post.tmpl"),\n'.format(extension) - PAGES += ' ("stories/*.{0}", "stories", "story.tmpl"),\n'.format(extension) + PAGES += ' ("pages/*.{0}", "pages", "story.tmpl"),\n'.format(extension) POSTS += ')\n' PAGES += ')\n' context['POSTS'] = POSTS @@ -436,9 +500,6 @@ class CommandImportWordpress(Command, ImportMixin): def download_url_content_to_file(self, url, dst_path): """Download some content (attachments) to a file.""" - if self.no_downloads: - return - try: request = requests.get(url, auth=self.auth) if request.status_code >= 400: @@ -458,10 +519,13 @@ class CommandImportWordpress(Command, ImportMixin): 'foo') path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[link] = '/' + dst_url links[url] = '/' + dst_url @@ -507,6 +571,8 @@ class CommandImportWordpress(Command, ImportMixin): if meta_key in metadata: image_meta = metadata[meta_key] + if not image_meta: + continue dst_meta = {} def add(our_key, wp_key, is_int=False, ignore_zero=False, is_float=False): @@ -552,15 +618,18 @@ class CommandImportWordpress(Command, ImportMixin): meta = {} meta['size'] = size.decode('utf-8') if width_key in metadata[size_key][size] and height_key in metadata[size_key][size]: - meta['width'] = metadata[size_key][size][width_key] - meta['height'] = metadata[size_key][size][height_key] + meta['width'] = int(metadata[size_key][size][width_key]) + meta['height'] = int(metadata[size_key][size][height_key]) path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[url] = '/' + dst_url @@ -604,7 +673,7 @@ class CommandImportWordpress(Command, ImportMixin): def transform_code(self, content): """Transform code blocks.""" - # http://en.support.wordpress.com/code/posting-source-code/. There are + # https://en.support.wordpress.com/code/posting-source-code/. There are # a ton of things not supported here. We only do a basic [code # lang="x"] -> ```x translation, and remove quoted html entities (<, # >, &, and "). @@ -628,10 +697,10 @@ class CommandImportWordpress(Command, ImportMixin): return content @staticmethod - def transform_caption(content): + def transform_caption(content, use_html=False): """Transform captions.""" - new_caption = re.sub(r'\[/caption\]', '', content) - new_caption = re.sub(r'\[caption.*\]', '', new_caption) + new_caption = re.sub(r'\[/caption\]', '</h1>' if use_html else '', content) + new_caption = re.sub(r'\[caption.*\]', '<h1>' if use_html else '', new_caption) return new_caption @@ -654,6 +723,26 @@ class CommandImportWordpress(Command, ImportMixin): except TypeError: # old versions of the plugin don't support the additional argument content = self.wordpress_page_compiler.compile_to_string(content) return content, 'html', True + elif self.transform_to_markdown: + # First convert to HTML with WordPress plugin + additional_data = {} + if attachments is not None: + additional_data['attachments'] = attachments + try: + content = self.wordpress_page_compiler.compile_to_string(content, additional_data=additional_data) + except TypeError: # old versions of the plugin don't support the additional argument + content = self.wordpress_page_compiler.compile_to_string(content) + # Now convert to MarkDown with html2text + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False + elif self.html2text: + # TODO: what to do with [code] blocks? + # content = self.transform_code(content) + content = self.transform_caption(content, use_html=True) + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False elif self.use_wordpress_compiler: return content, 'wp', False else: @@ -686,7 +775,7 @@ class CommandImportWordpress(Command, ImportMixin): elif approved == 'spam' or approved == 'trash': pass else: - LOGGER.warn("Unknown comment approved status: " + str(approved)) + LOGGER.warn("Unknown comment approved status: {0}".format(approved)) parent = int(get_text_tag(comment, "{{{0}}}comment_parent".format(wordpress_namespace), 0)) if parent == 0: parent = None @@ -707,7 +796,7 @@ class CommandImportWordpress(Command, ImportMixin): """Write comment header line.""" if header_content is None: return - header_content = str(header_content).replace('\n', ' ') + header_content = unicode_str(header_content).replace('\n', ' ') line = '.. ' + header_field + ': ' + header_content + '\n' fd.write(line.encode('utf8')) @@ -747,12 +836,36 @@ class CommandImportWordpress(Command, ImportMixin): tags_cats = tags + categories return tags_cats, other_meta + _tag_sanitize_map = {True: {}, False: {}} + + def _sanitize(self, tag, is_category): + if self.tag_saniziting_strategy == 'lower': + return tag.lower() + if tag.lower() not in self._tag_sanitize_map[is_category]: + self._tag_sanitize_map[is_category][tag.lower()] = [tag] + return tag + previous = self._tag_sanitize_map[is_category][tag.lower()] + if self.tag_saniziting_strategy == 'first': + if tag != previous[0]: + LOGGER.warn("Changing spelling of {0} name '{1}' to {2}.".format('category' if is_category else 'tag', tag, previous[0])) + return previous[0] + else: + LOGGER.error("Unknown tag sanitizing strategy '{0}'!".format(self.tag_saniziting_strategy)) + sys.exit(1) + return tag + def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None): """Take an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') + + # titles can have line breaks in them, particularly when they are + # created by third-party tools that post to Wordpress. + # Handle windows-style and unix-style line endings. + title = title.replace('\r\n', ' ').replace('\n', ' ') + # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) @@ -760,7 +873,10 @@ class CommandImportWordpress(Command, ImportMixin): path = unquote(parsed.path.strip('/')) try: - path = path.decode('utf8') + if isinstance(path, utils.bytes_str): + path = path.decode('utf8', 'replace') + else: + path = path except AttributeError: pass @@ -782,7 +898,7 @@ class CommandImportWordpress(Command, ImportMixin): else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) - slug = utils.slugify(pathlist[-1]) + slug = utils.slugify(pathlist[-1], self.lang) description = get_text_tag(item, 'description', '') post_date = get_text_tag( @@ -831,15 +947,24 @@ class CommandImportWordpress(Command, ImportMixin): type = tag.attrib['domain'] if text == 'Uncategorized' and type == 'category': continue - self.all_tags.add(text) if type == 'category': - categories.append(type) + categories.append(text) else: tags.append(text) if '$latex' in content: tags.append('mathjax') + for i, cat in enumerate(categories[:]): + cat = self._sanitize(cat, True) + categories[i] = cat + self.all_tags.add(cat) + + for i, tag in enumerate(tags[:]): + tag = self._sanitize(tag, False) + tags[i] = tag + self.all_tags.add(tag) + # Find post format if it's there post_format = 'wp' format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format'] @@ -888,14 +1013,32 @@ class CommandImportWordpress(Command, ImportMixin): meta_slug = slug tags, other_meta = self._create_metadata(status, excerpt, tags, categories, post_name=os.path.join(out_folder, slug)) - self.write_metadata(os.path.join(self.output_folder, out_folder, - out_meta_filename), - title, meta_slug, post_date, description, tags, **other_meta) - self.write_content( - os.path.join(self.output_folder, - out_folder, out_content_filename), - content, - rewrite_html) + + meta = { + "title": title, + "slug": meta_slug, + "date": post_date, + "description": description, + "tags": ','.join(tags), + } + meta.update(other_meta) + if self.onefile: + self.write_post( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + meta, + self._get_compiler(), + rewrite_html) + else: + self.write_metadata(os.path.join(self.output_folder, out_folder, + out_meta_filename), + title, meta_slug, post_date, description, tags, **other_meta) + self.write_content( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + rewrite_html) if self.export_comments: comments = [] @@ -905,7 +1048,7 @@ class CommandImportWordpress(Command, ImportMixin): comments.append(comment) for comment in comments: - comment_filename = slug + "." + str(comment['id']) + ".wpcomment" + comment_filename = "{0}.{1}.wpcomment".format(slug, comment['id']) self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment) return (out_folder, slug) @@ -955,7 +1098,7 @@ class CommandImportWordpress(Command, ImportMixin): if post_type == 'post': out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'posts', attachments) else: - out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'stories', attachments) + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'pages', attachments) # Process attachment data if attachments is not None: # If post was exported, store data diff --git a/nikola/plugins/command/init.plugin b/nikola/plugins/command/init.plugin index a5404c4..a8b1523 100644 --- a/nikola/plugins/command/init.plugin +++ b/nikola/plugins/command/init.plugin @@ -5,7 +5,7 @@ module = init [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create a new site. [Nikola] diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py index 91ccdb4..3d6669c 100644 --- a/nikola/plugins/command/init.py +++ b/nikola/plugins/command/init.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -41,7 +41,7 @@ from pkg_resources import resource_filename import tarfile import nikola -from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_RSS_READ_MORE_LINK, LEGAL_VALUES, urlsplit, urlunsplit +from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_FEED_READ_MORE_LINK, LEGAL_VALUES, urlsplit, urlunsplit from nikola.plugin_categories import Command from nikola.utils import ask, ask_yesno, get_logger, makedirs, STDERR_HANDLER, load_messages from nikola.packages.tzlocal import get_localzone @@ -71,14 +71,16 @@ SAMPLE_CONF = { 'CATEGORY_OUTPUT_FLAT_HIERARCHY': False, 'TRANSLATIONS_PATTERN': DEFAULT_TRANSLATIONS_PATTERN, 'INDEX_READ_MORE_LINK': DEFAULT_INDEX_READ_MORE_LINK, - 'RSS_READ_MORE_LINK': DEFAULT_RSS_READ_MORE_LINK, + 'FEED_READ_MORE_LINK': DEFAULT_FEED_READ_MORE_LINK, 'POSTS': """( ("posts/*.rst", "posts", "post.tmpl"), ("posts/*.txt", "posts", "post.tmpl"), + ("posts/*.html", "posts", "post.tmpl"), )""", 'PAGES': """( - ("stories/*.rst", "stories", "story.tmpl"), - ("stories/*.txt", "stories", "story.tmpl"), + ("pages/*.rst", "pages", "story.tmpl"), + ("pages/*.txt", "pages", "story.tmpl"), + ("pages/*.html", "pages", "story.tmpl"), )""", 'COMPILERS': """{ "rest": ('.rst', '.txt'), @@ -210,17 +212,28 @@ def prepare_config(config): """Parse sample config with JSON.""" p = config.copy() p.update({k: json.dumps(v, ensure_ascii=False) for k, v in p.items() - if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK')}) + if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'FEED_READ_MORE_LINK')}) # READ_MORE_LINKs require some special treatment. p['INDEX_READ_MORE_LINK'] = "'" + p['INDEX_READ_MORE_LINK'].replace("'", "\\'") + "'" - p['RSS_READ_MORE_LINK'] = "'" + p['RSS_READ_MORE_LINK'].replace("'", "\\'") + "'" + p['FEED_READ_MORE_LINK'] = "'" + p['FEED_READ_MORE_LINK'].replace("'", "\\'") + "'" # fix booleans and None p.update({k: str(v) for k, v in config.items() if isinstance(v, bool) or v is None}) return p -class CommandInit(Command): +def test_destination(destination, demo=False): + """Check if the destination already exists, which can break demo site creation.""" + # Issue #2214 + if demo and os.path.exists(destination): + LOGGER.warning("The directory {0} already exists, and a new demo site cannot be initialized in an existing directory.".format(destination)) + LOGGER.warning("Please remove the directory and try again, or use another directory.") + LOGGER.info("Hint: If you want to initialize a git repository in this directory, run `git init` in the directory after creating a Nikola site.") + return False + else: + return True + +class CommandInit(Command): """Create a new site.""" name = "init" @@ -272,11 +285,11 @@ class CommandInit(Command): @classmethod def create_empty_site(cls, target): """Create an empty site with directories only.""" - for folder in ('files', 'galleries', 'listings', 'posts', 'stories'): + for folder in ('files', 'galleries', 'listings', 'posts', 'pages'): makedirs(os.path.join(target, folder)) @staticmethod - def ask_questions(target): + def ask_questions(target, demo=False): """Ask some questions about Nikola.""" def urlhandler(default, toconf): answer = ask('Site URL', 'https://example.com/') @@ -347,7 +360,7 @@ class CommandInit(Command): # Assuming that base contains all the locales, and that base does # not inherit from anywhere. try: - messages = load_messages(['base'], tr, default) + messages = load_messages(['base'], tr, default, themes_dirs=['themes']) SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages, SAMPLE_CONF['STRIP_INDEXES']) except nikola.utils.LanguageNotFoundError as e: print(" ERROR: the language '{0}' is not supported.".format(e.lang)) @@ -358,7 +371,7 @@ class CommandInit(Command): def tzhandler(default, toconf): print("\nPlease choose the correct time zone for your blog. Nikola uses the tz database.") print("You can find your time zone here:") - print("http://en.wikipedia.org/wiki/List_of_tz_database_time_zones") + print("https://en.wikipedia.org/wiki/List_of_tz_database_time_zones") print("") answered = False while not answered: @@ -441,7 +454,7 @@ class CommandInit(Command): print("If you do not want to answer and want to go with the defaults instead, simply restart with the `-q` parameter.") for query, default, toconf, destination in questions: - if target and destination == '!target': + if target and destination == '!target' and test_destination(target, demo): # Skip the destination question if we know it already pass else: @@ -458,8 +471,9 @@ class CommandInit(Command): if toconf: SAMPLE_CONF[destination] = answer if destination == '!target': - while not answer: - print(' ERROR: you need to specify a target directory.\n') + while not answer or not test_destination(answer, demo): + if not answer: + print(' ERROR: you need to specify a target directory.\n') answer = ask(query, default) STORAGE['target'] = answer @@ -475,7 +489,7 @@ class CommandInit(Command): except IndexError: target = None if not options.get('quiet'): - st = self.ask_questions(target=target) + st = self.ask_questions(target=target, demo=options.get('demo')) try: if not target: target = st['target'] @@ -488,11 +502,13 @@ class CommandInit(Command): Options: -q, --quiet Do not ask questions about config. -d, --demo Create a site filled with example data.""") - return False + return 1 if not options.get('demo'): self.create_empty_site(target) LOGGER.info('Created empty site at {0}.'.format(target)) else: + if not test_destination(target, True): + return 2 self.copy_sample_site(target) LOGGER.info("A new site with example data has been created at " "{0}.".format(target)) diff --git a/nikola/plugins/command/install_theme.plugin b/nikola/plugins/command/install_theme.plugin index 8434f2e..aa68773 100644 --- a/nikola/plugins/command/install_theme.plugin +++ b/nikola/plugins/command/install_theme.plugin @@ -5,7 +5,7 @@ module = install_theme [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Install a theme into the current site. [Nikola] diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py index f02252e..28f7aa3 100644 --- a/nikola/plugins/command/install_theme.py +++ b/nikola/plugins/command/install_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,23 +27,13 @@ """Install a theme.""" from __future__ import print_function -import os -import io -import time -import requests -import pygments -from pygments.lexers import PythonLexer -from pygments.formatters import TerminalFormatter - -from nikola.plugin_categories import Command from nikola import utils - +from nikola.plugin_categories import Command LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER) class CommandInstallTheme(Command): - """Install a theme.""" name = "install_theme" @@ -80,6 +70,7 @@ class CommandInstallTheme(Command): def _execute(self, options, args): """Install theme into current site.""" + p = self.site.plugin_manager.getPluginByName('theme', 'Command').plugin_object listing = options['list'] url = options['url'] if args: @@ -88,85 +79,13 @@ class CommandInstallTheme(Command): name = None if options['getpath'] and name: - path = utils.get_theme_path(name) - if path: - print(path) - else: - print('not installed') - return 0 + return p.get_path(name) if name is None and not listing: LOGGER.error("This command needs either a theme name or the -l option.") return False - try: - data = requests.get(url).json() - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - data = requests.get(url).json() - if listing: - print("Themes:") - print("-------") - for theme in sorted(data.keys()): - print(theme) - return True - else: - # `name` may be modified by the while loop. - origname = name - installstatus = self.do_install(name, data) - # See if the theme's parent is available. If not, install it - while True: - parent_name = utils.get_parent_theme_name(name) - if parent_name is None: - break - try: - utils.get_theme_path(parent_name) - break - except: # Not available - self.do_install(parent_name, data) - name = parent_name - if installstatus: - LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) - - def do_install(self, name, data): - """Download and install a theme.""" - if name in data: - utils.makedirs(self.output_dir) - url = data[name] - LOGGER.info("Downloading '{0}'".format(url)) - try: - zip_data = requests.get(url).content - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - zip_data = requests.get(url).content - zip_file = io.BytesIO() - zip_file.write(zip_data) - LOGGER.info("Extracting '{0}' into themes/".format(name)) - utils.extract_all(zip_file) - dest_path = os.path.join(self.output_dir, name) + if listing: + p.list_available(url) else: - dest_path = os.path.join(self.output_dir, name) - try: - theme_path = utils.get_theme_path(name) - LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) - except Exception: - LOGGER.error("Can't find theme {0}".format(name)) - - return False - - confpypath = os.path.join(dest_path, 'conf.py.sample') - if os.path.exists(confpypath): - LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') - print('Contents of the conf.py.sample file:\n') - with io.open(confpypath, 'r', encoding='utf-8') as fh: - if self.site.colorful: - print(utils.indent(pygments.highlight( - fh.read(), PythonLexer(), TerminalFormatter()), - 4 * ' ')) - else: - print(utils.indent(fh.read(), 4 * ' ')) - return True + p.do_install_deps(url, name) diff --git a/nikola/plugins/command/new_page.plugin b/nikola/plugins/command/new_page.plugin index 145a419..3eaecb4 100644 --- a/nikola/plugins/command/new_page.plugin +++ b/nikola/plugins/command/new_page.plugin @@ -5,7 +5,7 @@ module = new_page [Documentation] author = Roberto Alsina, Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create a new page. [Nikola] diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index 811e28b..c09b4be 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,7 +32,6 @@ from nikola.plugin_categories import Command class CommandNewPage(Command): - """Create a new page.""" name = "new_page" diff --git a/nikola/plugins/command/new_post.plugin b/nikola/plugins/command/new_post.plugin index d88469f..e9c3af5 100644 --- a/nikola/plugins/command/new_post.plugin +++ b/nikola/plugins/command/new_post.plugin @@ -5,7 +5,7 @@ module = new_post [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Create a new post. [Nikola] diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index f9fe3ff..36cc04f 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,10 +29,11 @@ from __future__ import unicode_literals, print_function import io import datetime +import operator import os -import sys +import shutil import subprocess -import operator +import sys from blinker import signal import dateutil.tz @@ -114,7 +115,6 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): class CommandNewPost(Command): - """Create a new post.""" name = "new_post" @@ -294,14 +294,14 @@ class CommandNewPost(Command): title = title.strip() if not path: - slug = utils.slugify(title) + slug = utils.slugify(title, lang=self.site.default_lang) else: if isinstance(path, utils.bytes_str): try: path = path.decode(sys.stdin.encoding) except (AttributeError, TypeError): # for tests path = path.decode('utf-8') - slug = utils.slugify(os.path.splitext(os.path.basename(path))[0]) + slug = utils.slugify(os.path.splitext(os.path.basename(path))[0], lang=self.site.default_lang) if isinstance(author, utils.bytes_str): try: @@ -325,14 +325,17 @@ class CommandNewPost(Command): 'description': '', 'type': 'text', } - output_path = os.path.dirname(entry[0]) - meta_path = os.path.join(output_path, slug + ".meta") - pattern = os.path.basename(entry[0]) - suffix = pattern[1:] + if not path: + pattern = os.path.basename(entry[0]) + suffix = pattern[1:] + output_path = os.path.dirname(entry[0]) + txt_path = os.path.join(output_path, slug + suffix) + meta_path = os.path.join(output_path, slug + ".meta") else: txt_path = os.path.join(self.site.original_cwd, path) + meta_path = os.path.splitext(txt_path)[0] + ".meta" if (not onefile and os.path.isfile(meta_path)) or \ os.path.isfile(txt_path): @@ -344,6 +347,9 @@ class CommandNewPost(Command): signal('existing_' + content_type).send(self, **event) LOGGER.error("The title already exists!") + LOGGER.info("Existing {0}'s text is at: {1}".format(content_type, txt_path)) + if not onefile: + LOGGER.info("Existing {0}'s metadata is at: {1}".format(content_type, meta_path)) return 8 d_name = os.path.dirname(txt_path) @@ -364,17 +370,22 @@ class CommandNewPost(Command): onefile = False LOGGER.warn('This compiler does not support one-file posts.') - if import_file: + if onefile and import_file: with io.open(import_file, 'r', encoding='utf-8') as fh: content = fh.read() - else: + elif not import_file: if is_page: content = self.site.MESSAGES[self.site.default_lang]["Write your page here."] else: content = self.site.MESSAGES[self.site.default_lang]["Write your post here."] - compiler_plugin.create_post( - txt_path, content=content, onefile=onefile, title=title, - slug=slug, date=date, tags=tags, is_page=is_page, **metadata) + + if (not onefile) and import_file: + # Two-file posts are copied on import (Issue #2380) + shutil.copy(import_file, txt_path) + else: + compiler_plugin.create_post( + txt_path, content=content, onefile=onefile, title=title, + slug=slug, date=date, tags=tags, is_page=is_page, **metadata) event = dict(path=txt_path) diff --git a/nikola/plugins/command/orphans.plugin b/nikola/plugins/command/orphans.plugin index 669429d..d20c539 100644 --- a/nikola/plugins/command/orphans.plugin +++ b/nikola/plugins/command/orphans.plugin @@ -5,7 +5,7 @@ module = orphans [Documentation] author = Roberto Alsina, Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = List all orphans [Nikola] diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py index b12cc67..5e2574d 100644 --- a/nikola/plugins/command/orphans.py +++ b/nikola/plugins/command/orphans.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -34,7 +34,6 @@ from nikola.plugins.command.check import real_scan_files class CommandOrphans(Command): - """List all orphans.""" name = "orphans" diff --git a/nikola/plugins/command/plugin.plugin b/nikola/plugins/command/plugin.plugin index d44dcf3..016bcaa 100644 --- a/nikola/plugins/command/plugin.plugin +++ b/nikola/plugins/command/plugin.plugin @@ -5,7 +5,7 @@ module = plugin [Documentation] author = Roberto Alsina and Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Manage Nikola plugins [Nikola] diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index f892ee9..364f343 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,6 +29,7 @@ from __future__ import print_function import io import os +import sys import shutil import subprocess import time @@ -45,12 +46,11 @@ LOGGER = utils.get_logger('plugin', utils.STDERR_HANDLER) class CommandPlugin(Command): - """Manage plugins.""" json = None name = "plugin" - doc_usage = "[[-u][--user] --install name] | [[-u] [-l |--upgrade|--list-installed] | [--uninstall name]]" + doc_usage = "[-u url] [--user] [-i name] [-r name] [--upgrade] [-l] [--list-installed]" doc_purpose = "manage plugins" output_dir = None needs_config = False @@ -177,8 +177,11 @@ class CommandPlugin(Command): plugins.append([plugin.name, p]) plugins.sort() + print('Installed Plugins:') + print('------------------') for name, path in plugins: print('{0} at {1}'.format(name, path)) + print('\n\nAlso, you have disabled these plugins: {}'.format(self.site.config['DISABLED_PLUGINS'])) return 0 def do_upgrade(self, url): @@ -252,7 +255,7 @@ class CommandPlugin(Command): LOGGER.notice('This plugin has Python dependencies.') LOGGER.info('Installing dependencies with pip...') try: - subprocess.check_call(('pip', 'install', '-r', reqpath)) + subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath)) except subprocess.CalledProcessError: LOGGER.error('Could not install the dependencies.') print('Contents of the requirements.txt file:\n') @@ -293,12 +296,15 @@ class CommandPlugin(Command): def do_uninstall(self, name): """Uninstall a plugin.""" for plugin in self.site.plugin_manager.getAllPlugins(): # FIXME: this is repeated thrice - p = plugin.path - if os.path.isdir(p): - p = p + os.sep - else: - p = os.path.dirname(p) if name == plugin.name: # Uninstall this one + p = plugin.path + if os.path.isdir(p): + # Plugins that have a package in them need to delete parent + # Issue #2356 + p = p + os.sep + p = os.path.abspath(os.path.join(p, os.pardir)) + else: + p = os.path.dirname(p) LOGGER.warning('About to uninstall plugin: {0}'.format(name)) LOGGER.warning('This will delete {0}'.format(p)) sure = utils.ask_yesno('Are you sure?') diff --git a/nikola/plugins/command/rst2html.plugin b/nikola/plugins/command/rst2html.plugin index 02c9276..a095705 100644 --- a/nikola/plugins/command/rst2html.plugin +++ b/nikola/plugins/command/rst2html.plugin @@ -5,7 +5,7 @@ module = rst2html [Documentation] author = Chris Warrick version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Compile reStructuredText to HTML using the Nikola architecture [Nikola] diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py index 06afffd..c877f63 100644 --- a/nikola/plugins/command/rst2html/__init__.py +++ b/nikola/plugins/command/rst2html/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2015 Chris Warrick and others. +# Copyright © 2015-2016 Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,7 +36,6 @@ from nikola.plugin_categories import Command class CommandRst2Html(Command): - """Compile reStructuredText to HTML, using Nikola architecture.""" name = "rst2html" @@ -65,7 +64,7 @@ class CommandRst2Html(Command): parser = lxml.html.HTMLParser(remove_blank_text=True) doc = lxml.html.document_fromstring(template_output, parser) html = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True) - print(html) + print(html.decode('utf-8')) if error_level < 3: return 0 else: diff --git a/nikola/plugins/command/serve.plugin b/nikola/plugins/command/serve.plugin index aca71ec..a4a726f 100644 --- a/nikola/plugins/command/serve.plugin +++ b/nikola/plugins/command/serve.plugin @@ -5,7 +5,7 @@ module = serve [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Start test server. [Nikola] diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py index 0441c93..c9702d5 100644 --- a/nikola/plugins/command/serve.py +++ b/nikola/plugins/command/serve.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -45,24 +45,23 @@ except ImportError: from nikola.plugin_categories import Command -from nikola.utils import get_logger, STDERR_HANDLER +from nikola.utils import dns_sd, get_logger, STDERR_HANDLER class IPv6Server(HTTPServer): - """An IPv6 HTTPServer.""" address_family = socket.AF_INET6 class CommandServe(Command): - """Start test server.""" name = "serve" doc_usage = "[options]" doc_purpose = "start the test webserver" logger = None + dns_sd = None cmd_options = ( { @@ -79,7 +78,7 @@ class CommandServe(Command): 'long': 'address', 'type': str, 'default': '', - 'help': 'Address to bind (default: 0.0.0.0 – all local IPv4 interfaces)', + 'help': 'Address to bind (default: 0.0.0.0 -- all local IPv4 interfaces)', }, { 'name': 'detach', @@ -152,14 +151,16 @@ class CommandServe(Command): raise e else: try: + self.dns_sd = dns_sd(options['port'], (options['ipv6'] or '::' in options['address'])) httpd.serve_forever() except KeyboardInterrupt: self.logger.info("Server is shutting down.") + if self.dns_sd: + self.dns_sd.Reset() return 130 class OurHTTPRequestHandler(SimpleHTTPRequestHandler): - """A request handler, modified for Nikola.""" extensions_map = dict(SimpleHTTPRequestHandler.extensions_map) @@ -242,7 +243,10 @@ class OurHTTPRequestHandler(SimpleHTTPRequestHandler): f.seek(0) self.send_response(200) - self.send_header("Content-type", ctype) + if ctype.startswith('text/') or ctype.endswith('+xml'): + self.send_header("Content-Type", "{0}; charset=UTF-8".format(ctype)) + else: + self.send_header("Content-Type", ctype) if os.path.splitext(path)[1] == '.svgz': # Special handling for svgz to make it work nice with browsers. self.send_header("Content-Encoding", 'gzip') diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py index 55e7f95..b3ffbb4 100644 --- a/nikola/plugins/command/status.py +++ b/nikola/plugins/command/status.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,7 +27,6 @@ """Display site status.""" from __future__ import print_function -import io import os from datetime import datetime from dateutil.tz import gettz, tzlocal @@ -36,14 +35,13 @@ from nikola.plugin_categories import Command class CommandStatus(Command): - """Display site status.""" name = "status" doc_purpose = "display site status" doc_description = "Show information about the posts and site deployment." - doc_usage = '[-l|--list-drafts] [-m|--list-modified] [-s|--list-scheduled]' + doc_usage = '[-d|--list-drafts] [-m|--list-modified] [-p|--list-private] [-P|--list-published] [-s|--list-scheduled]' logger = None cmd_options = [ { @@ -63,6 +61,22 @@ class CommandStatus(Command): 'help': 'List all modified files since last deployment', }, { + 'name': 'list_private', + 'short': 'p', + 'long': 'list-private', + 'type': bool, + 'default': False, + 'help': 'List all private posts', + }, + { + 'name': 'list_published', + 'short': 'P', + 'long': 'list-published', + 'type': bool, + 'default': False, + 'help': 'List all published posts', + }, + { 'name': 'list_scheduled', 'short': 's', 'long': 'list-scheduled', @@ -76,16 +90,12 @@ class CommandStatus(Command): """Display site status.""" self.site.scan_posts() - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - - last_deploy = None - - try: - with io.open(timestamp_path, "r", encoding="utf8") as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - last_deploy_offset = datetime.utcnow() - last_deploy - except (IOError, Exception): - print("It does not seem like you’ve ever deployed the site (or cache missing).") + last_deploy = self.site.state.get('last_deploy') + if last_deploy is not None: + last_deploy = datetime.strptime(last_deploy, "%Y-%m-%dT%H:%M:%S.%f") + last_deploy_offset = datetime.utcnow() - last_deploy + else: + print("It does not seem like you've ever deployed the site (or cache missing).") if last_deploy: @@ -111,12 +121,23 @@ class CommandStatus(Command): posts_count = len(self.site.all_posts) + # find all published posts + posts_published = [post for post in self.site.all_posts if post.use_in_feeds] + posts_published = sorted(posts_published, key=lambda post: post.source_path) + + # find all private posts + posts_private = [post for post in self.site.all_posts if post.is_private] + posts_private = sorted(posts_private, key=lambda post: post.source_path) + # find all drafts posts_drafts = [post for post in self.site.all_posts if post.is_draft] posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path) # find all scheduled posts with offset from now until publishing time - posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later] + posts_scheduled = [ + (post.date - now, post) for post in self.site.all_posts + if post.publish_later and not (post.is_draft or post.is_private) + ] posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path)) if len(posts_scheduled) > 0: @@ -129,7 +150,13 @@ class CommandStatus(Command): if options['list_drafts']: for post in posts_drafts: print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) - print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts))) + if options['list_private']: + for post in posts_private: + print("Private: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + if options['list_published']: + for post in posts_published: + print("Published: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + print("{0} posts in total, {1} scheduled, {2} drafts, {3} private and {4} published.".format(posts_count, len(posts_scheduled), len(posts_drafts), len(posts_private), len(posts_published))) def human_time(self, dt): """Translate time into a human-friendly representation.""" diff --git a/nikola/plugins/command/theme.plugin b/nikola/plugins/command/theme.plugin new file mode 100644 index 0000000..b0c1886 --- /dev/null +++ b/nikola/plugins/command/theme.plugin @@ -0,0 +1,13 @@ +[Core] +name = theme +module = theme + +[Documentation] +author = Roberto Alsina and Chris Warrick +version = 1.0 +website = https://getnikola.com/ +description = Manage Nikola themes + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/theme.py b/nikola/plugins/command/theme.py new file mode 100644 index 0000000..7513491 --- /dev/null +++ b/nikola/plugins/command/theme.py @@ -0,0 +1,365 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Manage themes.""" + +from __future__ import print_function +import os +import io +import shutil +import time +import requests + +import pygments +from pygments.lexers import PythonLexer +from pygments.formatters import TerminalFormatter +from pkg_resources import resource_filename + +from nikola.plugin_categories import Command +from nikola import utils + +LOGGER = utils.get_logger('theme', utils.STDERR_HANDLER) + + +class CommandTheme(Command): + """Manage themes.""" + + json = None + name = "theme" + doc_usage = "[-u url] [-i theme_name] [-r theme_name] [-l] [--list-installed] [-g] [-n theme_name] [-c template_name]" + doc_purpose = "manage themes" + output_dir = 'themes' + cmd_options = [ + { + 'name': 'install', + 'short': 'i', + 'long': 'install', + 'type': str, + 'default': '', + 'help': 'Install a theme.' + }, + { + 'name': 'uninstall', + 'long': 'uninstall', + 'short': 'r', + 'type': str, + 'default': '', + 'help': 'Uninstall a theme.' + }, + { + 'name': 'list', + 'short': 'l', + 'long': 'list', + 'type': bool, + 'default': False, + 'help': 'Show list of available themes.' + }, + { + 'name': 'list_installed', + 'long': 'list-installed', + 'type': bool, + 'help': "List the installed themes with their location.", + 'default': False + }, + { + 'name': 'url', + 'short': 'u', + 'long': 'url', + 'type': str, + 'help': "URL for the theme repository (default: " + "https://themes.getnikola.com/v7/themes.json)", + 'default': 'https://themes.getnikola.com/v7/themes.json' + }, + { + 'name': 'getpath', + 'short': 'g', + 'long': 'get-path', + 'type': str, + 'default': '', + 'help': "Print the path for installed theme", + }, + { + 'name': 'copy-template', + 'short': 'c', + 'long': 'copy-template', + 'type': str, + 'default': '', + 'help': 'Copy a built-in template into templates/ or your theme', + }, + { + 'name': 'new', + 'short': 'n', + 'long': 'new', + 'type': str, + 'default': '', + 'help': 'Create a new theme', + }, + { + 'name': 'new_engine', + 'long': 'engine', + 'type': str, + 'default': 'mako', + 'help': 'Engine to use for new theme (mako or jinja -- default: mako)', + }, + { + 'name': 'new_parent', + 'long': 'parent', + 'type': str, + 'default': 'base', + 'help': 'Parent to use for new theme (default: base)', + }, + ] + + def _execute(self, options, args): + """Install theme into current site.""" + url = options['url'] + + # See the "mode" we need to operate in + install = options.get('install') + uninstall = options.get('uninstall') + list_available = options.get('list') + list_installed = options.get('list_installed') + get_path = options.get('getpath') + copy_template = options.get('copy-template') + new = options.get('new') + new_engine = options.get('new_engine') + new_parent = options.get('new_parent') + command_count = [bool(x) for x in ( + install, + uninstall, + list_available, + list_installed, + get_path, + copy_template, + new)].count(True) + if command_count > 1 or command_count == 0: + print(self.help()) + return 2 + + if list_available: + return self.list_available(url) + elif list_installed: + return self.list_installed() + elif install: + return self.do_install_deps(url, install) + elif uninstall: + return self.do_uninstall(uninstall) + elif get_path: + return self.get_path(get_path) + elif copy_template: + return self.copy_template(copy_template) + elif new: + return self.new_theme(new, new_engine, new_parent) + + def do_install_deps(self, url, name): + """Install themes and their dependencies.""" + data = self.get_json(url) + # `name` may be modified by the while loop. + origname = name + installstatus = self.do_install(name, data) + # See if the theme's parent is available. If not, install it + while True: + parent_name = utils.get_parent_theme_name(utils.get_theme_path_real(name, self.site.themes_dirs)) + if parent_name is None: + break + try: + utils.get_theme_path_real(parent_name, self.site.themes_dirs) + break + except: # Not available + self.do_install(parent_name, data) + name = parent_name + if installstatus: + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) + + def do_install(self, name, data): + """Download and install a theme.""" + if name in data: + utils.makedirs(self.output_dir) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + + zip_file = io.BytesIO() + zip_file.write(zip_data) + LOGGER.info("Extracting '{0}' into themes/".format(name)) + utils.extract_all(zip_file) + dest_path = os.path.join(self.output_dir, name) + else: + dest_path = os.path.join(self.output_dir, name) + try: + theme_path = utils.get_theme_path_real(name, self.site.themes_dirs) + LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) + except Exception: + LOGGER.error("Can't find theme {0}".format(name)) + + return False + + confpypath = os.path.join(dest_path, 'conf.py.sample') + if os.path.exists(confpypath): + LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') + print('Contents of the conf.py.sample file:\n') + with io.open(confpypath, 'r', encoding='utf-8') as fh: + if self.site.colorful: + print(utils.indent(pygments.highlight( + fh.read(), PythonLexer(), TerminalFormatter()), + 4 * ' ')) + else: + print(utils.indent(fh.read(), 4 * ' ')) + return True + + def do_uninstall(self, name): + """Uninstall a theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + except Exception: + LOGGER.error('Unknown theme: {0}'.format(name)) + return 1 + # Don't uninstall builtin themes (Issue #2510) + blocked = os.path.dirname(utils.__file__) + if path.startswith(blocked): + LOGGER.error("Can't delete builtin theme: {0}".format(name)) + return 1 + LOGGER.warning('About to uninstall theme: {0}'.format(name)) + LOGGER.warning('This will delete {0}'.format(path)) + sure = utils.ask_yesno('Are you sure?') + if sure: + LOGGER.warning('Removing {0}'.format(path)) + shutil.rmtree(path) + return 0 + return 1 + + def get_path(self, name): + """Get path for an installed theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + print(path) + except Exception: + print("not installed") + return 0 + + def list_available(self, url): + """List all available themes.""" + data = self.get_json(url) + print("Available Themes:") + print("-----------------") + for theme in sorted(data.keys()): + print(theme) + return 0 + + def list_installed(self): + """List all installed themes.""" + print("Installed Themes:") + print("-----------------") + themes = [] + themes_dirs = self.site.themes_dirs + [resource_filename('nikola', os.path.join('data', 'themes'))] + for tdir in themes_dirs: + themes += [(i, os.path.join(tdir, i)) for i in os.listdir(tdir)] + for tname, tpath in sorted(set(themes)): + if os.path.isdir(tpath): + print("{0} at {1}".format(tname, tpath)) + + def copy_template(self, template): + """Copy the named template file from the parent to a local theme or to templates/.""" + # Find template + t = self.site.template_system.get_template_path(template) + if t is None: + LOGGER.error("Cannot find template {0} in the lookup.".format(template)) + return 2 + + # Figure out where to put it. + # Check if a local theme exists. + theme_path = utils.get_theme_path(self.site.THEMES[0]) + if theme_path.startswith('themes' + os.sep): + # Theme in local themes/ directory + base = os.path.join(theme_path, 'templates') + else: + # Put it in templates/ + base = 'templates' + + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + try: + out = shutil.copy(t, base) + LOGGER.info("Copied template from {0} to {1}".format(t, out)) + except shutil.SameFileError: + LOGGER.error("This file already exists in your templates directory ({0}).".format(base)) + return 3 + + def new_theme(self, name, engine, parent): + """Create a new theme.""" + base = 'themes' + themedir = os.path.join(base, name) + LOGGER.info("Creating theme {0} with parent {1} and engine {2} in {3}".format(name, parent, engine, themedir)) + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + # Check if engine and parent match + engine_file = utils.get_asset_path('engine', utils.get_theme_chain(parent, self.site.themes_dirs)) + with io.open(engine_file, 'r', encoding='utf-8') as fh: + parent_engine = fh.read().strip() + + if parent_engine != engine: + LOGGER.error("Cannot use engine {0} because parent theme '{1}' uses {2}".format(engine, parent, parent_engine)) + return 2 + + # Create theme + if not os.path.exists(themedir): + os.mkdir(themedir) + LOGGER.info("Created directory {0}".format(themedir)) + else: + LOGGER.error("Theme already exists") + return 2 + + with io.open(os.path.join(themedir, 'parent'), 'w', encoding='utf-8') as fh: + fh.write(parent + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'parent'))) + with io.open(os.path.join(themedir, 'engine'), 'w', encoding='utf-8') as fh: + fh.write(engine + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'engine'))) + + LOGGER.info("Theme {0} created successfully.".format(themedir)) + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(name)) + + def get_json(self, url): + """Download the JSON file with all plugins.""" + if self.json is None: + try: + self.json = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + self.json = requests.get(url).json() + return self.json diff --git a/nikola/plugins/command/version.plugin b/nikola/plugins/command/version.plugin index 4708bdb..d78b79b 100644 --- a/nikola/plugins/command/version.plugin +++ b/nikola/plugins/command/version.plugin @@ -5,7 +5,7 @@ module = version [Documentation] author = Roberto Alsina version = 1.0 -website = http://getnikola.com +website = https://getnikola.com/ description = Show nikola version [Nikola] diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py index ad08f64..267837e 100644 --- a/nikola/plugins/command/version.py +++ b/nikola/plugins/command/version.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,7 +38,6 @@ URL = 'https://pypi.python.org/pypi?:action=doap&name=Nikola' class CommandVersion(Command): - """Print Nikola version.""" name = "version" |