diff options
| author | 2016-11-15 14:18:46 -0300 | |
|---|---|---|
| committer | 2016-11-15 14:18:46 -0300 | |
| commit | ffb671c61a24a9086343b54bad080e145ff33fc5 (patch) | |
| tree | 2c5291f7a34edf4afdc8e07887a148291bfa3fa1 /nikola/plugins/command | |
| parent | 4e3224c012df9f74f010eb92203520515e8537b9 (diff) | |
New upstream version 7.8.1upstream/7.8.1
Diffstat (limited to 'nikola/plugins/command')
| -rw-r--r-- | nikola/plugins/command/__init__.py | 2 | ||||
| -rw-r--r-- | nikola/plugins/command/auto/__init__.py | 18 | ||||
| -rw-r--r-- | nikola/plugins/command/bootswatch_theme.py | 17 | ||||
| -rw-r--r-- | nikola/plugins/command/check.py | 40 | ||||
| -rw-r--r-- | nikola/plugins/command/console.py | 10 | ||||
| -rw-r--r-- | nikola/plugins/command/deploy.py | 69 | ||||
| -rw-r--r-- | nikola/plugins/command/github_deploy.py | 101 | ||||
| -rw-r--r-- | nikola/plugins/command/import_wordpress.py | 177 | ||||
| -rw-r--r-- | nikola/plugins/command/init.py | 39 | ||||
| -rw-r--r-- | nikola/plugins/command/install_theme.py | 94 | ||||
| -rw-r--r-- | nikola/plugins/command/new_page.py | 2 | ||||
| -rw-r--r-- | nikola/plugins/command/new_post.py | 40 | ||||
| -rw-r--r-- | nikola/plugins/command/orphans.py | 2 | ||||
| -rw-r--r-- | nikola/plugins/command/plugin.py | 23 | ||||
| -rw-r--r-- | nikola/plugins/command/rst2html/__init__.py | 2 | ||||
| -rw-r--r-- | nikola/plugins/command/serve.py | 4 | ||||
| -rw-r--r-- | nikola/plugins/command/status.py | 58 | ||||
| -rw-r--r-- | nikola/plugins/command/theme.plugin | 13 | ||||
| -rw-r--r-- | nikola/plugins/command/theme.py | 365 | ||||
| -rw-r--r-- | nikola/plugins/command/version.py | 2 |
20 files changed, 805 insertions, 273 deletions
diff --git a/nikola/plugins/command/__init__.py b/nikola/plugins/command/__init__.py index 2aa5267..62d7086 100644 --- a/nikola/plugins/command/__init__.py +++ b/nikola/plugins/command/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py index e339c06..a82dc3e 100644 --- a/nikola/plugins/command/auto/__init__.py +++ b/nikola/plugins/command/auto/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -43,6 +43,7 @@ except ImportError: import webbrowser from wsgiref.simple_server import make_server import wsgiref.util +import pkg_resources from blinker import signal try: @@ -61,7 +62,6 @@ except ImportError: FileSystemEventHandler = object PatternMatchingEventHandler = object - from nikola.plugin_categories import Command from nikola.utils import dns_sd, req_missing, get_logger, get_theme_path, STDERR_HANDLER LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js') @@ -102,7 +102,7 @@ class CommandAuto(Command): 'long': 'address', 'type': str, 'default': '127.0.0.1', - 'help': 'Address to bind (default: 127.0.0.1 – localhost)', + 'help': 'Address to bind (default: 127.0.0.1 -- localhost)', }, { 'name': 'browser', @@ -143,7 +143,7 @@ class CommandAuto(Command): self.cmd_arguments = ['nikola', 'build'] if self.site.configuration_filename != 'conf.py': - self.cmd_arguments = ['--conf=' + self.site.configuration_filename] + self.cmd_arguments + self.cmd_arguments.append('--conf=' + self.site.configuration_filename) # Run an initial build so we are up-to-date subprocess.call(self.cmd_arguments) @@ -157,7 +157,7 @@ class CommandAuto(Command): # Do not duplicate entries -- otherwise, multiple rebuilds are triggered watched = set([ - 'templates/', 'plugins/', + 'templates/' ] + [get_theme_path(name) for name in self.site.THEMES]) for item in self.site.config['post_pages']: watched.add(os.path.dirname(item[0])) @@ -167,6 +167,10 @@ class CommandAuto(Command): watched.add(item) for item in self.site.config['LISTINGS_FOLDERS']: watched.add(item) + for item in self.site._plugin_places: + watched.add(item) + # Nikola itself (useful for developers) + watched.add(pkg_resources.resource_filename('nikola', '')) out_folder = self.site.config['OUTPUT_FOLDER'] if options and options.get('browser'): @@ -298,8 +302,8 @@ class CommandAuto(Command): mimetype = 'text/html' if uri.endswith('/') else mimetypes.guess_type(uri)[0] or 'application/octet-stream' if os.path.isdir(f_path): - if not f_path.endswith('/'): # Redirect to avoid breakage - start_response('301 Redirect', [('Location', p_uri.path + '/')]) + if not p_uri.path.endswith('/'): # Redirect to avoid breakage + start_response('301 Moved Permanently', [('Location', p_uri.path + '/')]) return [] f_path = os.path.join(f_path, self.site.config['INDEX_FILE']) mimetype = 'text/html' diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py index afd15af..4808fdb 100644 --- a/nikola/plugins/command/bootswatch_theme.py +++ b/nikola/plugins/command/bootswatch_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -36,6 +36,13 @@ from nikola import utils LOGGER = utils.get_logger('bootswatch_theme', utils.STDERR_HANDLER) +def _check_for_theme(theme, themes): + for t in themes: + if t.endswith(os.sep + theme): + return True + return False + + class CommandBootswatchTheme(Command): """Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" @@ -79,12 +86,12 @@ class CommandBootswatchTheme(Command): version = '' # See if we need bootswatch for bootstrap v2 or v3 - themes = utils.get_theme_chain(parent) - if 'bootstrap3' not in themes and 'bootstrap3-jinja' not in themes: + themes = utils.get_theme_chain(parent, self.site.themes_dirs) + if not _check_for_theme('bootstrap3', themes) and not _check_for_theme('bootstrap3-jinja', themes): version = '2' - elif 'bootstrap' not in themes and 'bootstrap-jinja' not in themes: + elif not _check_for_theme('bootstrap', themes) and not _check_for_theme('bootstrap-jinja', themes): LOGGER.warn('"bootswatch_theme" only makes sense for themes that use bootstrap') - elif 'bootstrap3-gradients' in themes or 'bootstrap3-gradients-jinja' in themes: + elif _check_for_theme('bootstrap3-gradients', themes) or _check_for_theme('bootstrap3-gradients-jinja', themes): LOGGER.warn('"bootswatch_theme" doesn\'t work well with the bootstrap3-gradients family') LOGGER.info("Creating '{0}' theme from '{1}' and '{2}'".format(name, swatch, parent)) diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py index bfc6ee2..0141a6b 100644 --- a/nikola/plugins/command/check.py +++ b/nikola/plugins/command/check.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -32,6 +32,7 @@ import os import re import sys import time +import logbook try: from urllib import unquote from urlparse import urlparse, urljoin, urldefrag @@ -164,9 +165,9 @@ class CommandCheck(Command): print(self.help()) return False if options['verbose']: - self.logger.level = 1 + self.logger.level = logbook.DEBUG else: - self.logger.level = 4 + self.logger.level = logbook.NOTICE failure = False if options['links']: failure |= self.scan_links(options['find_sources'], options['remote']) @@ -245,7 +246,7 @@ class CommandCheck(Command): target = urldefrag(target)[0] if any([urlparse(target).netloc.endswith(_) for _ in ['example.com', 'example.net', 'example.org']]): - self.logger.info("Not testing example address \"{0}\".".format(target)) + self.logger.debug("Not testing example address \"{0}\".".format(target)) continue # absolute URL to root-relative @@ -274,7 +275,7 @@ class CommandCheck(Command): if self.checked_remote_targets[target] in [301, 308]: self.logger.warn("Remote link PERMANENTLY redirected in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) elif self.checked_remote_targets[target] in [302, 307]: - self.logger.notice("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target])) + self.logger.debug("Remote link temporarily redirected in {0}: {1} [HTTP: {2}]".format(filename, target, self.checked_remote_targets[target])) elif self.checked_remote_targets[target] > 399: self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) continue @@ -302,7 +303,7 @@ class CommandCheck(Command): if redir_status_code in [301, 308]: self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) if redir_status_code in [302, 307]: - self.logger.notice("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + self.logger.debug("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) self.checked_remote_targets[resp.url] = resp.status_code self.checked_remote_targets[target] = redir_status_code else: @@ -343,7 +344,7 @@ class CommandCheck(Command): elif target_filename not in self.existing_targets: if os.path.exists(target_filename): - self.logger.info(u"Good link {0} => {1}".format(target, target_filename)) + self.logger.info("Good link {0} => {1}".format(target, target_filename)) self.existing_targets.add(target_filename) else: rv = True @@ -358,9 +359,9 @@ class CommandCheck(Command): def scan_links(self, find_sources=False, check_remote=False): """Check links on the site.""" - self.logger.info("Checking Links:") - self.logger.info("===============\n") - self.logger.notice("{0} mode".format(self.site.config['URL_TYPE'])) + self.logger.debug("Checking Links:") + self.logger.debug("===============\n") + self.logger.debug("{0} mode".format(self.site.config['URL_TYPE'])) failure = False # Maybe we should just examine all HTML files output_folder = self.site.config['OUTPUT_FOLDER'] @@ -380,14 +381,14 @@ class CommandCheck(Command): if self.analyze(fname, find_sources, False): failure = True if not failure: - self.logger.info("All links checked.") + self.logger.debug("All links checked.") return failure def scan_files(self): """Check files in the site, find missing and orphaned files.""" failure = False - self.logger.info("Checking Files:") - self.logger.info("===============\n") + self.logger.debug("Checking Files:") + self.logger.debug("===============\n") only_on_output, only_on_input = real_scan_files(self.site, self.cache) # Ignore folders @@ -406,16 +407,18 @@ class CommandCheck(Command): for f in only_on_input: self.logger.warn(f) if not failure: - self.logger.info("All files checked.") + self.logger.debug("All files checked.") return failure def clean_files(self): """Remove orphaned files.""" only_on_output, _ = real_scan_files(self.site, self.cache) for f in only_on_output: - self.logger.info('removed: {0}'.format(f)) + self.logger.debug('removed: {0}'.format(f)) os.unlink(f) + warn_flag = bool(only_on_output) + # Find empty directories and remove them output_folder = self.site.config['OUTPUT_FOLDER'] all_dirs = [] @@ -425,7 +428,12 @@ class CommandCheck(Command): for d in all_dirs: try: os.rmdir(d) - self.logger.info('removed: {0}/'.format(d)) + self.logger.debug('removed: {0}/'.format(d)) + warn_flag = True except OSError: pass + + if warn_flag: + self.logger.warn('Some files or directories have been removed, your site may need rebuilding') + return True diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py index 3d3daab..c6a8376 100644 --- a/nikola/plugins/command/console.py +++ b/nikola/plugins/command/console.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Chris Warrick, Roberto Alsina and others. +# Copyright © 2012-2016 Chris Warrick, Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -75,7 +75,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f ] def ipython(self, willful=True): - """IPython shell.""" + """Run an IPython shell.""" try: import IPython except ImportError as e: @@ -84,12 +84,13 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f raise e # That’s how _execute knows whether to try something else. else: site = self.context['site'] # NOQA + nikola_site = self.context['site'] # NOQA conf = self.context['conf'] # NOQA commands = self.context['commands'] # NOQA IPython.embed(header=self.header.format('IPython')) def bpython(self, willful=True): - """bpython shell.""" + """Run a bpython shell.""" try: import bpython except ImportError as e: @@ -100,7 +101,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f bpython.embed(banner=self.header.format('bpython'), locals_=self.context) def plain(self, willful=True): - """Plain Python shell.""" + """Run a plain Python shell.""" import code try: import readline @@ -130,6 +131,7 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f self.context = { 'conf': self.site.config, 'site': self.site, + 'nikola_site': self.site, 'commands': self.site.commands, } if options['bpython']: diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py index 757c0d2..c2289e8 100644 --- a/nikola/plugins/command/deploy.py +++ b/nikola/plugins/command/deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -30,6 +30,7 @@ from __future__ import print_function import io from datetime import datetime from dateutil.tz import gettz +import dateutil import os import subprocess import time @@ -37,7 +38,7 @@ import time from blinker import signal from nikola.plugin_categories import Command -from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER +from nikola.utils import get_logger, clean_before_deployment, STDERR_HANDLER class CommandDeploy(Command): @@ -45,7 +46,7 @@ class CommandDeploy(Command): name = "deploy" - doc_usage = "[[preset [preset...]]" + doc_usage = "[preset [preset...]]" doc_purpose = "deploy the site" doc_description = "Deploy the site by executing deploy commands from the presets listed on the command line. If no presets are specified, `default` is executed." logger = None @@ -55,27 +56,42 @@ class CommandDeploy(Command): self.logger = get_logger('deploy', STDERR_HANDLER) # Get last successful deploy date timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy') + + # Get last-deploy from persistent state + last_deploy = self.site.state.get('last_deploy') + if last_deploy is None: + # If there is a last-deploy saved, move it to the new state persistence thing + # FIXME: remove in Nikola 8 + if os.path.isfile(timestamp_path): + try: + with io.open(timestamp_path, 'r', encoding='utf8') as inf: + last_deploy = dateutil.parser.parse(inf.read()) + clean = False + except (IOError, Exception) as e: + self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) + last_deploy = datetime(1970, 1, 1) + clean = True + os.unlink(timestamp_path) # Remove because from now on it's in state + else: # Just a default + last_deploy = datetime(1970, 1, 1) + clean = True + else: + last_deploy = dateutil.parser.parse(last_deploy) + clean = False + if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo': self.logger.warn("\nWARNING WARNING WARNING WARNING\n" "You are deploying using the nikolademo Disqus account.\n" "That means you will not be able to moderate the comments in your own site.\n" "And is probably not what you want to do.\n" - "Think about it for 5 seconds, I'll wait :-)\n\n") + "Think about it for 5 seconds, I'll wait :-)\n" + "(press Ctrl+C to abort)\n") time.sleep(5) - deploy_drafts = self.site.config.get('DEPLOY_DRAFTS', True) - deploy_future = self.site.config.get('DEPLOY_FUTURE', False) - undeployed_posts = [] - if not (deploy_drafts and deploy_future): - # Remove drafts and future posts - out_dir = self.site.config['OUTPUT_FOLDER'] - self.site.scan_posts() - for post in self.site.timeline: - if (not deploy_drafts and post.is_draft) or \ - (not deploy_future and post.publish_later): - remove_file(os.path.join(out_dir, post.destination_path())) - remove_file(os.path.join(out_dir, post.source_path)) - undeployed_posts.append(post) + # Remove drafts and future posts if requested + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) if args: presets = args @@ -97,27 +113,22 @@ class CommandDeploy(Command): try: subprocess.check_call(command, shell=True) except subprocess.CalledProcessError as e: - self.logger.error('Failed deployment — command {0} ' + self.logger.error('Failed deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode)) return e.returncode self.logger.info("Successful deployment") - try: - with io.open(timestamp_path, 'r', encoding='utf8') as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - clean = False - except (IOError, Exception) as e: - self.logger.debug("Problem when reading `{0}`: {1}".format(timestamp_path, e)) - last_deploy = datetime(1970, 1, 1) - clean = True new_deploy = datetime.utcnow() self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts) - makedirs(self.site.config['CACHE_FOLDER']) # Store timestamp of successful deployment - with io.open(timestamp_path, 'w+', encoding='utf8') as outf: - outf.write(unicode_str(new_deploy.isoformat())) + self.site.state.set('last_deploy', new_deploy.isoformat()) + if clean: + self.logger.info( + 'Looks like this is the first time you deployed this site. ' + 'Let us know you are using Nikola ' + 'at <https://users.getnikola.com/add/> if you want!') def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None): """Emit events for all timeline entries newer than last deploy. diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py index 2fe0d4e..b5ad322 100644 --- a/nikola/plugins/command/github_deploy.py +++ b/nikola/plugins/command/github_deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014-2015 Puneeth Chaganti and others. +# Copyright © 2014-2016 Puneeth Chaganti and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,15 +27,13 @@ """Deploy site to GitHub Pages.""" from __future__ import print_function -from datetime import datetime -import io import os import subprocess from textwrap import dedent from nikola.plugin_categories import Command from nikola.plugins.command.check import real_scan_files -from nikola.utils import get_logger, req_missing, makedirs, unicode_str, STDERR_HANDLER +from nikola.utils import get_logger, req_missing, clean_before_deployment, STDERR_HANDLER from nikola.__main__ import main from nikola import __version__ @@ -53,7 +51,7 @@ def check_ghp_import_installed(): except OSError: # req_missing defaults to `python=True` — and it’s meant to be like this. # `ghp-import` is installed via pip, but the only way to use it is by executing the script it installs. - req_missing(['ghp-import'], 'deploy the site to GitHub Pages') + req_missing(['ghp-import2'], 'deploy the site to GitHub Pages') class CommandGitHubDeploy(Command): @@ -61,7 +59,7 @@ class CommandGitHubDeploy(Command): name = 'github_deploy' - doc_usage = '' + doc_usage = '[-m COMMIT_MESSAGE]' doc_purpose = 'deploy the site to GitHub Pages' doc_description = dedent( """\ @@ -71,10 +69,19 @@ class CommandGitHubDeploy(Command): """ ) - + cmd_options = [ + { + 'name': 'commit_message', + 'short': 'm', + 'long': 'message', + 'default': 'Nikola auto commit.', + 'type': str, + 'help': 'Commit message (default: Nikola auto commit.)', + }, + ] logger = None - def _execute(self, command, args): + def _execute(self, options, args): """Run the deployment.""" self.logger = get_logger(CommandGitHubDeploy.name, STDERR_HANDLER) @@ -92,41 +99,69 @@ class CommandGitHubDeploy(Command): for f in only_on_output: os.unlink(f) + # Remove drafts and future posts if requested (Issue #2406) + undeployed_posts = clean_before_deployment(self.site) + if undeployed_posts: + self.logger.notice("Deleted {0} posts due to DEPLOY_* settings".format(len(undeployed_posts))) + # Commit and push - self._commit_and_push() + self._commit_and_push(options['commit_message']) return - def _commit_and_push(self): - """Commit all the files and push.""" - source = self.site.config['GITHUB_SOURCE_BRANCH'] - deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] - remote = self.site.config['GITHUB_REMOTE_NAME'] - source_commit = uni_check_output(['git', 'rev-parse', source]) - commit_message = ( - 'Nikola auto commit.\n\n' - 'Source commit: %s' - 'Nikola version: %s' % (source_commit, __version__) - ) - output_folder = self.site.config['OUTPUT_FOLDER'] - - command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] - + def _run_command(self, command, xfail=False): + """Run a command that may or may not fail.""" self.logger.info("==> {0}".format(command)) try: subprocess.check_call(command) + return 0 except subprocess.CalledProcessError as e: + if xfail: + return e.returncode self.logger.error( - 'Failed GitHub deployment — command {0} ' + 'Failed GitHub deployment -- command {0} ' 'returned {1}'.format(e.cmd, e.returncode) ) - return e.returncode + raise SystemError(e.returncode) - self.logger.info("Successful deployment") + def _commit_and_push(self, commit_first_line): + """Commit all the files and push.""" + source = self.site.config['GITHUB_SOURCE_BRANCH'] + deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] + remote = self.site.config['GITHUB_REMOTE_NAME'] + autocommit = self.site.config['GITHUB_COMMIT_SOURCE'] + try: + if autocommit: + commit_message = ( + '{0}\n\n' + 'Nikola version: {1}'.format(commit_first_line, __version__) + ) + e = self._run_command(['git', 'checkout', source], True) + if e != 0: + self._run_command(['git', 'checkout', '-b', source]) + self._run_command(['git', 'add', '.']) + # Figure out if there is anything to commit + e = self._run_command(['git', 'diff-index', '--quiet', 'HEAD'], True) + if e != 0: + self._run_command(['git', 'commit', '-am', commit_message]) + else: + self.logger.notice('Nothing to commit to source branch.') + + source_commit = uni_check_output(['git', 'rev-parse', source]) + commit_message = ( + '{0}\n\n' + 'Source commit: {1}' + 'Nikola version: {2}'.format(commit_first_line, source_commit, __version__) + ) + output_folder = self.site.config['OUTPUT_FOLDER'] - # Store timestamp of successful deployment - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - new_deploy = datetime.utcnow() - makedirs(self.site.config["CACHE_FOLDER"]) - with io.open(timestamp_path, "w+", encoding="utf8") as outf: - outf.write(unicode_str(new_deploy.isoformat())) + command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] + + self._run_command(command) + + if autocommit: + self._run_command(['git', 'push', '-u', remote, source]) + except SystemError as e: + return e.args[0] + + self.logger.info("Successful deployment") diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index 69ef144..0b48583 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -38,6 +38,11 @@ from lxml import etree from collections import defaultdict try: + import html2text +except: + html2text = None + +try: from urlparse import urlparse from urllib import unquote except ImportError: @@ -170,6 +175,20 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Export comments as .wpcomment files", }, { + 'name': 'html2text', + 'long': 'html2text', + 'default': False, + 'type': bool, + 'help': "Uses html2text (needs to be installed with pip) to transform WordPress posts to MarkDown during import", + }, + { + 'name': 'transform_to_markdown', + 'long': 'transform-to-markdown', + 'default': False, + 'type': bool, + 'help': "Uses WordPress page compiler to transform WordPress posts to HTML and then use html2text to transform them to MarkDown during import", + }, + { 'name': 'transform_to_html', 'long': 'transform-to-html', 'default': False, @@ -191,14 +210,35 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Automatically installs the WordPress page compiler (either locally or in the new site) if required by other options.\nWarning: the compiler is GPL software!", }, { - 'name': 'tag_saniziting_strategy', - 'long': 'tag-saniziting-strategy', + 'name': 'tag_sanitizing_strategy', + 'long': 'tag-sanitizing-strategy', 'default': 'first', 'help': 'lower: Convert all tag and category names to lower case\nfirst: Keep first spelling of tag or category name', }, + { + 'name': 'one_file', + 'long': 'one-file', + 'default': False, + 'type': bool, + 'help': "Save imported posts in the more modern one-file format.", + }, ] all_tags = set([]) + def _get_compiler(self): + """Return whatever compiler we will use.""" + self._find_wordpress_compiler() + if self.wordpress_page_compiler is not None: + return self.wordpress_page_compiler + plugin_info = self.site.plugin_manager.getPluginByName('markdown', 'PageCompiler') + if plugin_info is not None: + if not plugin_info.is_activated: + self.site.plugin_manager.activatePluginByName(plugin_info.name) + plugin_info.plugin_object.set_site(self.site) + return plugin_info.plugin_object + else: + LOGGER.error("Can't find markdown post compiler.") + def _find_wordpress_compiler(self): """Find WordPress compiler plugin.""" if self.wordpress_page_compiler is not None: @@ -223,6 +263,8 @@ class CommandImportWordpress(Command, ImportMixin): 'putting these arguments before the filename if you ' 'are running into problems.'.format(args)) + self.onefile = options.get('one_file', False) + self.import_into_existing_site = False self.url_map = {} self.timezone = None @@ -239,6 +281,9 @@ class CommandImportWordpress(Command, ImportMixin): self.export_categories_as_categories = options.get('export_categories_as_categories', False) self.export_comments = options.get('export_comments', False) + self.html2text = options.get('html2text', False) + self.transform_to_markdown = options.get('transform_to_markdown', False) + self.transform_to_html = options.get('transform_to_html', False) self.use_wordpress_compiler = options.get('use_wordpress_compiler', False) self.install_wordpress_compiler = options.get('install_wordpress_compiler', False) @@ -257,10 +302,18 @@ class CommandImportWordpress(Command, ImportMixin): self.separate_qtranslate_content = options.get('separate_qtranslate_content') self.translations_pattern = options.get('translations_pattern') - if self.transform_to_html and self.use_wordpress_compiler: - LOGGER.warn("It does not make sense to combine --transform-to-html with --use-wordpress-compiler, as the first converts all posts to HTML and the latter option affects zero posts.") + count = (1 if self.html2text else 0) + (1 if self.transform_to_html else 0) + (1 if self.transform_to_markdown else 0) + if count > 1: + LOGGER.error("You can use at most one of the options --html2text, --transform-to-html and --transform-to-markdown.") + return False + if (self.html2text or self.transform_to_html or self.transform_to_markdown) and self.use_wordpress_compiler: + LOGGER.warn("It does not make sense to combine --use-wordpress-compiler with any of --html2text, --transform-to-html and --transform-to-markdown, as the latter convert all posts to HTML and the first option then affects zero posts.") + + if (self.html2text or self.transform_to_markdown) and not html2text: + LOGGER.error("You need to install html2text via 'pip install html2text' before you can use the --html2text and --transform-to-markdown options.") + return False - if self.transform_to_html: + if self.transform_to_html or self.transform_to_markdown: self._find_wordpress_compiler() if not self.wordpress_page_compiler and self.install_wordpress_compiler: if not install_plugin(self.site, 'wordpress_compiler', output_dir='plugins'): # local install @@ -334,7 +387,7 @@ class CommandImportWordpress(Command, ImportMixin): self.context['TRANSLATIONS'] = format_default_translations_config( self.extra_languages) self.context['REDIRECTIONS'] = self.configure_redirections( - self.url_map) + self.url_map, self.base_dir) if self.timezone: self.context['TIMEZONE'] = self.timezone if self.export_categories_as_categories: @@ -350,7 +403,7 @@ class CommandImportWordpress(Command, ImportMixin): tag_str = tag except AttributeError: tag_str = tag - tag = utils.slugify(tag_str) + tag = utils.slugify(tag_str, self.lang) src_url = '{}tag/{}'.format(self.context['SITE_URL'], tag) dst_url = self.site.link('tag', tag) if src_url != dst_url: @@ -382,7 +435,7 @@ class CommandImportWordpress(Command, ImportMixin): if b'<atom:link rel=' in line: continue xml.append(line) - return b'\n'.join(xml) + return b''.join(xml) @classmethod def get_channel_from_file(cls, filename): @@ -396,7 +449,8 @@ class CommandImportWordpress(Command, ImportMixin): wordpress_namespace = channel.nsmap['wp'] context = SAMPLE_CONF.copy() - context['DEFAULT_LANG'] = get_text_tag(channel, 'language', 'en')[:2] + self.lang = get_text_tag(channel, 'language', 'en')[:2] + context['DEFAULT_LANG'] = self.lang context['TRANSLATIONS_PATTERN'] = DEFAULT_TRANSLATIONS_PATTERN context['BLOG_TITLE'] = get_text_tag(channel, 'title', 'PUT TITLE HERE') @@ -428,7 +482,7 @@ class CommandImportWordpress(Command, ImportMixin): PAGES = '(\n' for extension in extensions: POSTS += ' ("posts/*.{0}", "posts", "post.tmpl"),\n'.format(extension) - PAGES += ' ("stories/*.{0}", "stories", "story.tmpl"),\n'.format(extension) + PAGES += ' ("pages/*.{0}", "pages", "story.tmpl"),\n'.format(extension) POSTS += ')\n' PAGES += ')\n' context['POSTS'] = POSTS @@ -446,9 +500,6 @@ class CommandImportWordpress(Command, ImportMixin): def download_url_content_to_file(self, url, dst_path): """Download some content (attachments) to a file.""" - if self.no_downloads: - return - try: request = requests.get(url, auth=self.auth) if request.status_code >= 400: @@ -468,10 +519,13 @@ class CommandImportWordpress(Command, ImportMixin): 'foo') path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[link] = '/' + dst_url links[url] = '/' + dst_url @@ -517,6 +571,8 @@ class CommandImportWordpress(Command, ImportMixin): if meta_key in metadata: image_meta = metadata[meta_key] + if not image_meta: + continue dst_meta = {} def add(our_key, wp_key, is_int=False, ignore_zero=False, is_float=False): @@ -562,15 +618,18 @@ class CommandImportWordpress(Command, ImportMixin): meta = {} meta['size'] = size.decode('utf-8') if width_key in metadata[size_key][size] and height_key in metadata[size_key][size]: - meta['width'] = metadata[size_key][size][width_key] - meta['height'] = metadata[size_key][size][height_key] + meta['width'] = int(metadata[size_key][size][width_key]) + meta['height'] = int(metadata[size_key][size][height_key]) path = urlparse(url).path dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) + if self.no_downloads: + LOGGER.info("Skipping downloading {0} => {1}".format(url, dst_path)) + else: + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) dst_url = '/'.join(dst_path.split(os.sep)[2:]) links[url] = '/' + dst_url @@ -638,10 +697,10 @@ class CommandImportWordpress(Command, ImportMixin): return content @staticmethod - def transform_caption(content): + def transform_caption(content, use_html=False): """Transform captions.""" - new_caption = re.sub(r'\[/caption\]', '', content) - new_caption = re.sub(r'\[caption.*\]', '', new_caption) + new_caption = re.sub(r'\[/caption\]', '</h1>' if use_html else '', content) + new_caption = re.sub(r'\[caption.*\]', '<h1>' if use_html else '', new_caption) return new_caption @@ -664,6 +723,26 @@ class CommandImportWordpress(Command, ImportMixin): except TypeError: # old versions of the plugin don't support the additional argument content = self.wordpress_page_compiler.compile_to_string(content) return content, 'html', True + elif self.transform_to_markdown: + # First convert to HTML with WordPress plugin + additional_data = {} + if attachments is not None: + additional_data['attachments'] = attachments + try: + content = self.wordpress_page_compiler.compile_to_string(content, additional_data=additional_data) + except TypeError: # old versions of the plugin don't support the additional argument + content = self.wordpress_page_compiler.compile_to_string(content) + # Now convert to MarkDown with html2text + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False + elif self.html2text: + # TODO: what to do with [code] blocks? + # content = self.transform_code(content) + content = self.transform_caption(content, use_html=True) + h = html2text.HTML2Text() + content = h.handle(content) + return content, 'md', False elif self.use_wordpress_compiler: return content, 'wp', False else: @@ -781,6 +860,12 @@ class CommandImportWordpress(Command, ImportMixin): out_folder = 'posts' title = get_text_tag(item, 'title', 'NO TITLE') + + # titles can have line breaks in them, particularly when they are + # created by third-party tools that post to Wordpress. + # Handle windows-style and unix-style line endings. + title = title.replace('\r\n', ' ').replace('\n', ' ') + # link is something like http://foo.com/2012/09/01/hello-world/ # So, take the path, utils.slugify it, and that's our slug link = get_text_tag(item, 'link', None) @@ -813,7 +898,7 @@ class CommandImportWordpress(Command, ImportMixin): else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) - slug = utils.slugify(pathlist[-1]) + slug = utils.slugify(pathlist[-1], self.lang) description = get_text_tag(item, 'description', '') post_date = get_text_tag( @@ -928,14 +1013,32 @@ class CommandImportWordpress(Command, ImportMixin): meta_slug = slug tags, other_meta = self._create_metadata(status, excerpt, tags, categories, post_name=os.path.join(out_folder, slug)) - self.write_metadata(os.path.join(self.output_folder, out_folder, - out_meta_filename), - title, meta_slug, post_date, description, tags, **other_meta) - self.write_content( - os.path.join(self.output_folder, - out_folder, out_content_filename), - content, - rewrite_html) + + meta = { + "title": title, + "slug": meta_slug, + "date": post_date, + "description": description, + "tags": ','.join(tags), + } + meta.update(other_meta) + if self.onefile: + self.write_post( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + meta, + self._get_compiler(), + rewrite_html) + else: + self.write_metadata(os.path.join(self.output_folder, out_folder, + out_meta_filename), + title, meta_slug, post_date, description, tags, **other_meta) + self.write_content( + os.path.join(self.output_folder, + out_folder, out_content_filename), + content, + rewrite_html) if self.export_comments: comments = [] @@ -995,7 +1098,7 @@ class CommandImportWordpress(Command, ImportMixin): if post_type == 'post': out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'posts', attachments) else: - out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'stories', attachments) + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'pages', attachments) # Process attachment data if attachments is not None: # If post was exported, store data diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py index 2dbee43..3d6669c 100644 --- a/nikola/plugins/command/init.py +++ b/nikola/plugins/command/init.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -75,10 +75,12 @@ SAMPLE_CONF = { 'POSTS': """( ("posts/*.rst", "posts", "post.tmpl"), ("posts/*.txt", "posts", "post.tmpl"), + ("posts/*.html", "posts", "post.tmpl"), )""", 'PAGES': """( - ("stories/*.rst", "stories", "story.tmpl"), - ("stories/*.txt", "stories", "story.tmpl"), + ("pages/*.rst", "pages", "story.tmpl"), + ("pages/*.txt", "pages", "story.tmpl"), + ("pages/*.html", "pages", "story.tmpl"), )""", 'COMPILERS': """{ "rest": ('.rst', '.txt'), @@ -219,6 +221,18 @@ def prepare_config(config): return p +def test_destination(destination, demo=False): + """Check if the destination already exists, which can break demo site creation.""" + # Issue #2214 + if demo and os.path.exists(destination): + LOGGER.warning("The directory {0} already exists, and a new demo site cannot be initialized in an existing directory.".format(destination)) + LOGGER.warning("Please remove the directory and try again, or use another directory.") + LOGGER.info("Hint: If you want to initialize a git repository in this directory, run `git init` in the directory after creating a Nikola site.") + return False + else: + return True + + class CommandInit(Command): """Create a new site.""" @@ -271,11 +285,11 @@ class CommandInit(Command): @classmethod def create_empty_site(cls, target): """Create an empty site with directories only.""" - for folder in ('files', 'galleries', 'listings', 'posts', 'stories'): + for folder in ('files', 'galleries', 'listings', 'posts', 'pages'): makedirs(os.path.join(target, folder)) @staticmethod - def ask_questions(target): + def ask_questions(target, demo=False): """Ask some questions about Nikola.""" def urlhandler(default, toconf): answer = ask('Site URL', 'https://example.com/') @@ -346,7 +360,7 @@ class CommandInit(Command): # Assuming that base contains all the locales, and that base does # not inherit from anywhere. try: - messages = load_messages(['base'], tr, default) + messages = load_messages(['base'], tr, default, themes_dirs=['themes']) SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages, SAMPLE_CONF['STRIP_INDEXES']) except nikola.utils.LanguageNotFoundError as e: print(" ERROR: the language '{0}' is not supported.".format(e.lang)) @@ -440,7 +454,7 @@ class CommandInit(Command): print("If you do not want to answer and want to go with the defaults instead, simply restart with the `-q` parameter.") for query, default, toconf, destination in questions: - if target and destination == '!target': + if target and destination == '!target' and test_destination(target, demo): # Skip the destination question if we know it already pass else: @@ -457,8 +471,9 @@ class CommandInit(Command): if toconf: SAMPLE_CONF[destination] = answer if destination == '!target': - while not answer: - print(' ERROR: you need to specify a target directory.\n') + while not answer or not test_destination(answer, demo): + if not answer: + print(' ERROR: you need to specify a target directory.\n') answer = ask(query, default) STORAGE['target'] = answer @@ -474,7 +489,7 @@ class CommandInit(Command): except IndexError: target = None if not options.get('quiet'): - st = self.ask_questions(target=target) + st = self.ask_questions(target=target, demo=options.get('demo')) try: if not target: target = st['target'] @@ -487,11 +502,13 @@ class CommandInit(Command): Options: -q, --quiet Do not ask questions about config. -d, --demo Create a site filled with example data.""") - return False + return 1 if not options.get('demo'): self.create_empty_site(target) LOGGER.info('Created empty site at {0}.'.format(target)) else: + if not test_destination(target, True): + return 2 self.copy_sample_site(target) LOGGER.info("A new site with example data has been created at " "{0}.".format(target)) diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py index bad335c..28f7aa3 100644 --- a/nikola/plugins/command/install_theme.py +++ b/nikola/plugins/command/install_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,18 +27,9 @@ """Install a theme.""" from __future__ import print_function -import os -import io -import time -import requests -import pygments -from pygments.lexers import PythonLexer -from pygments.formatters import TerminalFormatter - -from nikola.plugin_categories import Command from nikola import utils - +from nikola.plugin_categories import Command LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER) @@ -79,6 +70,7 @@ class CommandInstallTheme(Command): def _execute(self, options, args): """Install theme into current site.""" + p = self.site.plugin_manager.getPluginByName('theme', 'Command').plugin_object listing = options['list'] url = options['url'] if args: @@ -87,85 +79,13 @@ class CommandInstallTheme(Command): name = None if options['getpath'] and name: - path = utils.get_theme_path(name) - if path: - print(path) - else: - print('not installed') - return 0 + return p.get_path(name) if name is None and not listing: LOGGER.error("This command needs either a theme name or the -l option.") return False - try: - data = requests.get(url).json() - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - data = requests.get(url).json() - if listing: - print("Themes:") - print("-------") - for theme in sorted(data.keys()): - print(theme) - return True - else: - # `name` may be modified by the while loop. - origname = name - installstatus = self.do_install(name, data) - # See if the theme's parent is available. If not, install it - while True: - parent_name = utils.get_parent_theme_name(name) - if parent_name is None: - break - try: - utils.get_theme_path(parent_name) - break - except: # Not available - self.do_install(parent_name, data) - name = parent_name - if installstatus: - LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) - def do_install(self, name, data): - """Download and install a theme.""" - if name in data: - utils.makedirs(self.output_dir) - url = data[name] - LOGGER.info("Downloading '{0}'".format(url)) - try: - zip_data = requests.get(url).content - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - zip_data = requests.get(url).content - - zip_file = io.BytesIO() - zip_file.write(zip_data) - LOGGER.info("Extracting '{0}' into themes/".format(name)) - utils.extract_all(zip_file) - dest_path = os.path.join(self.output_dir, name) + if listing: + p.list_available(url) else: - dest_path = os.path.join(self.output_dir, name) - try: - theme_path = utils.get_theme_path(name) - LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) - except Exception: - LOGGER.error("Can't find theme {0}".format(name)) - - return False - - confpypath = os.path.join(dest_path, 'conf.py.sample') - if os.path.exists(confpypath): - LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') - print('Contents of the conf.py.sample file:\n') - with io.open(confpypath, 'r', encoding='utf-8') as fh: - if self.site.colorful: - print(utils.indent(pygments.highlight( - fh.read(), PythonLexer(), TerminalFormatter()), - 4 * ' ')) - else: - print(utils.indent(fh.read(), 4 * ' ')) - return True + p.do_install_deps(url, name) diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index 8843421..c09b4be 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index 30d009d..36cc04f 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,10 +29,11 @@ from __future__ import unicode_literals, print_function import io import datetime +import operator import os -import sys +import shutil import subprocess -import operator +import sys from blinker import signal import dateutil.tz @@ -293,14 +294,14 @@ class CommandNewPost(Command): title = title.strip() if not path: - slug = utils.slugify(title) + slug = utils.slugify(title, lang=self.site.default_lang) else: if isinstance(path, utils.bytes_str): try: path = path.decode(sys.stdin.encoding) except (AttributeError, TypeError): # for tests path = path.decode('utf-8') - slug = utils.slugify(os.path.splitext(os.path.basename(path))[0]) + slug = utils.slugify(os.path.splitext(os.path.basename(path))[0], lang=self.site.default_lang) if isinstance(author, utils.bytes_str): try: @@ -324,14 +325,17 @@ class CommandNewPost(Command): 'description': '', 'type': 'text', } - output_path = os.path.dirname(entry[0]) - meta_path = os.path.join(output_path, slug + ".meta") - pattern = os.path.basename(entry[0]) - suffix = pattern[1:] + if not path: + pattern = os.path.basename(entry[0]) + suffix = pattern[1:] + output_path = os.path.dirname(entry[0]) + txt_path = os.path.join(output_path, slug + suffix) + meta_path = os.path.join(output_path, slug + ".meta") else: txt_path = os.path.join(self.site.original_cwd, path) + meta_path = os.path.splitext(txt_path)[0] + ".meta" if (not onefile and os.path.isfile(meta_path)) or \ os.path.isfile(txt_path): @@ -343,6 +347,9 @@ class CommandNewPost(Command): signal('existing_' + content_type).send(self, **event) LOGGER.error("The title already exists!") + LOGGER.info("Existing {0}'s text is at: {1}".format(content_type, txt_path)) + if not onefile: + LOGGER.info("Existing {0}'s metadata is at: {1}".format(content_type, meta_path)) return 8 d_name = os.path.dirname(txt_path) @@ -363,17 +370,22 @@ class CommandNewPost(Command): onefile = False LOGGER.warn('This compiler does not support one-file posts.') - if import_file: + if onefile and import_file: with io.open(import_file, 'r', encoding='utf-8') as fh: content = fh.read() - else: + elif not import_file: if is_page: content = self.site.MESSAGES[self.site.default_lang]["Write your page here."] else: content = self.site.MESSAGES[self.site.default_lang]["Write your post here."] - compiler_plugin.create_post( - txt_path, content=content, onefile=onefile, title=title, - slug=slug, date=date, tags=tags, is_page=is_page, **metadata) + + if (not onefile) and import_file: + # Two-file posts are copied on import (Issue #2380) + shutil.copy(import_file, txt_path) + else: + compiler_plugin.create_post( + txt_path, content=content, onefile=onefile, title=title, + slug=slug, date=date, tags=tags, is_page=is_page, **metadata) event = dict(path=txt_path) diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py index f061d39..5e2574d 100644 --- a/nikola/plugins/command/orphans.py +++ b/nikola/plugins/command/orphans.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index 1df7b71..364f343 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,6 +29,7 @@ from __future__ import print_function import io import os +import sys import shutil import subprocess import time @@ -49,7 +50,7 @@ class CommandPlugin(Command): json = None name = "plugin" - doc_usage = "[[-u][--user] --install name] | [[-u] [-l |--upgrade|--list-installed] | [--uninstall name]]" + doc_usage = "[-u url] [--user] [-i name] [-r name] [--upgrade] [-l] [--list-installed]" doc_purpose = "manage plugins" output_dir = None needs_config = False @@ -176,8 +177,11 @@ class CommandPlugin(Command): plugins.append([plugin.name, p]) plugins.sort() + print('Installed Plugins:') + print('------------------') for name, path in plugins: print('{0} at {1}'.format(name, path)) + print('\n\nAlso, you have disabled these plugins: {}'.format(self.site.config['DISABLED_PLUGINS'])) return 0 def do_upgrade(self, url): @@ -251,7 +255,7 @@ class CommandPlugin(Command): LOGGER.notice('This plugin has Python dependencies.') LOGGER.info('Installing dependencies with pip...') try: - subprocess.check_call(('pip', 'install', '-r', reqpath)) + subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath)) except subprocess.CalledProcessError: LOGGER.error('Could not install the dependencies.') print('Contents of the requirements.txt file:\n') @@ -292,12 +296,15 @@ class CommandPlugin(Command): def do_uninstall(self, name): """Uninstall a plugin.""" for plugin in self.site.plugin_manager.getAllPlugins(): # FIXME: this is repeated thrice - p = plugin.path - if os.path.isdir(p): - p = p + os.sep - else: - p = os.path.dirname(p) if name == plugin.name: # Uninstall this one + p = plugin.path + if os.path.isdir(p): + # Plugins that have a package in them need to delete parent + # Issue #2356 + p = p + os.sep + p = os.path.abspath(os.path.join(p, os.pardir)) + else: + p = os.path.dirname(p) LOGGER.warning('About to uninstall plugin: {0}'.format(name)) LOGGER.warning('This will delete {0}'.format(p)) sure = utils.ask_yesno('Are you sure?') diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py index 68ea13d..c877f63 100644 --- a/nikola/plugins/command/rst2html/__init__.py +++ b/nikola/plugins/command/rst2html/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2015 Chris Warrick and others. +# Copyright © 2015-2016 Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py index b647bb7..c9702d5 100644 --- a/nikola/plugins/command/serve.py +++ b/nikola/plugins/command/serve.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -78,7 +78,7 @@ class CommandServe(Command): 'long': 'address', 'type': str, 'default': '', - 'help': 'Address to bind (default: 0.0.0.0 – all local IPv4 interfaces)', + 'help': 'Address to bind (default: 0.0.0.0 -- all local IPv4 interfaces)', }, { 'name': 'detach', diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py index 40f4f77..b3ffbb4 100644 --- a/nikola/plugins/command/status.py +++ b/nikola/plugins/command/status.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -27,7 +27,6 @@ """Display site status.""" from __future__ import print_function -import io import os from datetime import datetime from dateutil.tz import gettz, tzlocal @@ -42,7 +41,7 @@ class CommandStatus(Command): doc_purpose = "display site status" doc_description = "Show information about the posts and site deployment." - doc_usage = '[-l|--list-drafts] [-m|--list-modified] [-s|--list-scheduled]' + doc_usage = '[-d|--list-drafts] [-m|--list-modified] [-p|--list-private] [-P|--list-published] [-s|--list-scheduled]' logger = None cmd_options = [ { @@ -62,6 +61,22 @@ class CommandStatus(Command): 'help': 'List all modified files since last deployment', }, { + 'name': 'list_private', + 'short': 'p', + 'long': 'list-private', + 'type': bool, + 'default': False, + 'help': 'List all private posts', + }, + { + 'name': 'list_published', + 'short': 'P', + 'long': 'list-published', + 'type': bool, + 'default': False, + 'help': 'List all published posts', + }, + { 'name': 'list_scheduled', 'short': 's', 'long': 'list-scheduled', @@ -75,16 +90,12 @@ class CommandStatus(Command): """Display site status.""" self.site.scan_posts() - timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") - - last_deploy = None - - try: - with io.open(timestamp_path, "r", encoding="utf8") as inf: - last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") - last_deploy_offset = datetime.utcnow() - last_deploy - except (IOError, Exception): - print("It does not seem like you’ve ever deployed the site (or cache missing).") + last_deploy = self.site.state.get('last_deploy') + if last_deploy is not None: + last_deploy = datetime.strptime(last_deploy, "%Y-%m-%dT%H:%M:%S.%f") + last_deploy_offset = datetime.utcnow() - last_deploy + else: + print("It does not seem like you've ever deployed the site (or cache missing).") if last_deploy: @@ -110,12 +121,23 @@ class CommandStatus(Command): posts_count = len(self.site.all_posts) + # find all published posts + posts_published = [post for post in self.site.all_posts if post.use_in_feeds] + posts_published = sorted(posts_published, key=lambda post: post.source_path) + + # find all private posts + posts_private = [post for post in self.site.all_posts if post.is_private] + posts_private = sorted(posts_private, key=lambda post: post.source_path) + # find all drafts posts_drafts = [post for post in self.site.all_posts if post.is_draft] posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path) # find all scheduled posts with offset from now until publishing time - posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later] + posts_scheduled = [ + (post.date - now, post) for post in self.site.all_posts + if post.publish_later and not (post.is_draft or post.is_private) + ] posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path)) if len(posts_scheduled) > 0: @@ -128,7 +150,13 @@ class CommandStatus(Command): if options['list_drafts']: for post in posts_drafts: print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) - print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts))) + if options['list_private']: + for post in posts_private: + print("Private: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + if options['list_published']: + for post in posts_published: + print("Published: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + print("{0} posts in total, {1} scheduled, {2} drafts, {3} private and {4} published.".format(posts_count, len(posts_scheduled), len(posts_drafts), len(posts_private), len(posts_published))) def human_time(self, dt): """Translate time into a human-friendly representation.""" diff --git a/nikola/plugins/command/theme.plugin b/nikola/plugins/command/theme.plugin new file mode 100644 index 0000000..b0c1886 --- /dev/null +++ b/nikola/plugins/command/theme.plugin @@ -0,0 +1,13 @@ +[Core] +name = theme +module = theme + +[Documentation] +author = Roberto Alsina and Chris Warrick +version = 1.0 +website = https://getnikola.com/ +description = Manage Nikola themes + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/theme.py b/nikola/plugins/command/theme.py new file mode 100644 index 0000000..7513491 --- /dev/null +++ b/nikola/plugins/command/theme.py @@ -0,0 +1,365 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2016 Roberto Alsina, Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Manage themes.""" + +from __future__ import print_function +import os +import io +import shutil +import time +import requests + +import pygments +from pygments.lexers import PythonLexer +from pygments.formatters import TerminalFormatter +from pkg_resources import resource_filename + +from nikola.plugin_categories import Command +from nikola import utils + +LOGGER = utils.get_logger('theme', utils.STDERR_HANDLER) + + +class CommandTheme(Command): + """Manage themes.""" + + json = None + name = "theme" + doc_usage = "[-u url] [-i theme_name] [-r theme_name] [-l] [--list-installed] [-g] [-n theme_name] [-c template_name]" + doc_purpose = "manage themes" + output_dir = 'themes' + cmd_options = [ + { + 'name': 'install', + 'short': 'i', + 'long': 'install', + 'type': str, + 'default': '', + 'help': 'Install a theme.' + }, + { + 'name': 'uninstall', + 'long': 'uninstall', + 'short': 'r', + 'type': str, + 'default': '', + 'help': 'Uninstall a theme.' + }, + { + 'name': 'list', + 'short': 'l', + 'long': 'list', + 'type': bool, + 'default': False, + 'help': 'Show list of available themes.' + }, + { + 'name': 'list_installed', + 'long': 'list-installed', + 'type': bool, + 'help': "List the installed themes with their location.", + 'default': False + }, + { + 'name': 'url', + 'short': 'u', + 'long': 'url', + 'type': str, + 'help': "URL for the theme repository (default: " + "https://themes.getnikola.com/v7/themes.json)", + 'default': 'https://themes.getnikola.com/v7/themes.json' + }, + { + 'name': 'getpath', + 'short': 'g', + 'long': 'get-path', + 'type': str, + 'default': '', + 'help': "Print the path for installed theme", + }, + { + 'name': 'copy-template', + 'short': 'c', + 'long': 'copy-template', + 'type': str, + 'default': '', + 'help': 'Copy a built-in template into templates/ or your theme', + }, + { + 'name': 'new', + 'short': 'n', + 'long': 'new', + 'type': str, + 'default': '', + 'help': 'Create a new theme', + }, + { + 'name': 'new_engine', + 'long': 'engine', + 'type': str, + 'default': 'mako', + 'help': 'Engine to use for new theme (mako or jinja -- default: mako)', + }, + { + 'name': 'new_parent', + 'long': 'parent', + 'type': str, + 'default': 'base', + 'help': 'Parent to use for new theme (default: base)', + }, + ] + + def _execute(self, options, args): + """Install theme into current site.""" + url = options['url'] + + # See the "mode" we need to operate in + install = options.get('install') + uninstall = options.get('uninstall') + list_available = options.get('list') + list_installed = options.get('list_installed') + get_path = options.get('getpath') + copy_template = options.get('copy-template') + new = options.get('new') + new_engine = options.get('new_engine') + new_parent = options.get('new_parent') + command_count = [bool(x) for x in ( + install, + uninstall, + list_available, + list_installed, + get_path, + copy_template, + new)].count(True) + if command_count > 1 or command_count == 0: + print(self.help()) + return 2 + + if list_available: + return self.list_available(url) + elif list_installed: + return self.list_installed() + elif install: + return self.do_install_deps(url, install) + elif uninstall: + return self.do_uninstall(uninstall) + elif get_path: + return self.get_path(get_path) + elif copy_template: + return self.copy_template(copy_template) + elif new: + return self.new_theme(new, new_engine, new_parent) + + def do_install_deps(self, url, name): + """Install themes and their dependencies.""" + data = self.get_json(url) + # `name` may be modified by the while loop. + origname = name + installstatus = self.do_install(name, data) + # See if the theme's parent is available. If not, install it + while True: + parent_name = utils.get_parent_theme_name(utils.get_theme_path_real(name, self.site.themes_dirs)) + if parent_name is None: + break + try: + utils.get_theme_path_real(parent_name, self.site.themes_dirs) + break + except: # Not available + self.do_install(parent_name, data) + name = parent_name + if installstatus: + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) + + def do_install(self, name, data): + """Download and install a theme.""" + if name in data: + utils.makedirs(self.output_dir) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + + zip_file = io.BytesIO() + zip_file.write(zip_data) + LOGGER.info("Extracting '{0}' into themes/".format(name)) + utils.extract_all(zip_file) + dest_path = os.path.join(self.output_dir, name) + else: + dest_path = os.path.join(self.output_dir, name) + try: + theme_path = utils.get_theme_path_real(name, self.site.themes_dirs) + LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) + except Exception: + LOGGER.error("Can't find theme {0}".format(name)) + + return False + + confpypath = os.path.join(dest_path, 'conf.py.sample') + if os.path.exists(confpypath): + LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') + print('Contents of the conf.py.sample file:\n') + with io.open(confpypath, 'r', encoding='utf-8') as fh: + if self.site.colorful: + print(utils.indent(pygments.highlight( + fh.read(), PythonLexer(), TerminalFormatter()), + 4 * ' ')) + else: + print(utils.indent(fh.read(), 4 * ' ')) + return True + + def do_uninstall(self, name): + """Uninstall a theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + except Exception: + LOGGER.error('Unknown theme: {0}'.format(name)) + return 1 + # Don't uninstall builtin themes (Issue #2510) + blocked = os.path.dirname(utils.__file__) + if path.startswith(blocked): + LOGGER.error("Can't delete builtin theme: {0}".format(name)) + return 1 + LOGGER.warning('About to uninstall theme: {0}'.format(name)) + LOGGER.warning('This will delete {0}'.format(path)) + sure = utils.ask_yesno('Are you sure?') + if sure: + LOGGER.warning('Removing {0}'.format(path)) + shutil.rmtree(path) + return 0 + return 1 + + def get_path(self, name): + """Get path for an installed theme.""" + try: + path = utils.get_theme_path_real(name, self.site.themes_dirs) + print(path) + except Exception: + print("not installed") + return 0 + + def list_available(self, url): + """List all available themes.""" + data = self.get_json(url) + print("Available Themes:") + print("-----------------") + for theme in sorted(data.keys()): + print(theme) + return 0 + + def list_installed(self): + """List all installed themes.""" + print("Installed Themes:") + print("-----------------") + themes = [] + themes_dirs = self.site.themes_dirs + [resource_filename('nikola', os.path.join('data', 'themes'))] + for tdir in themes_dirs: + themes += [(i, os.path.join(tdir, i)) for i in os.listdir(tdir)] + for tname, tpath in sorted(set(themes)): + if os.path.isdir(tpath): + print("{0} at {1}".format(tname, tpath)) + + def copy_template(self, template): + """Copy the named template file from the parent to a local theme or to templates/.""" + # Find template + t = self.site.template_system.get_template_path(template) + if t is None: + LOGGER.error("Cannot find template {0} in the lookup.".format(template)) + return 2 + + # Figure out where to put it. + # Check if a local theme exists. + theme_path = utils.get_theme_path(self.site.THEMES[0]) + if theme_path.startswith('themes' + os.sep): + # Theme in local themes/ directory + base = os.path.join(theme_path, 'templates') + else: + # Put it in templates/ + base = 'templates' + + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + try: + out = shutil.copy(t, base) + LOGGER.info("Copied template from {0} to {1}".format(t, out)) + except shutil.SameFileError: + LOGGER.error("This file already exists in your templates directory ({0}).".format(base)) + return 3 + + def new_theme(self, name, engine, parent): + """Create a new theme.""" + base = 'themes' + themedir = os.path.join(base, name) + LOGGER.info("Creating theme {0} with parent {1} and engine {2} in {3}".format(name, parent, engine, themedir)) + if not os.path.exists(base): + os.mkdir(base) + LOGGER.info("Created directory {0}".format(base)) + + # Check if engine and parent match + engine_file = utils.get_asset_path('engine', utils.get_theme_chain(parent, self.site.themes_dirs)) + with io.open(engine_file, 'r', encoding='utf-8') as fh: + parent_engine = fh.read().strip() + + if parent_engine != engine: + LOGGER.error("Cannot use engine {0} because parent theme '{1}' uses {2}".format(engine, parent, parent_engine)) + return 2 + + # Create theme + if not os.path.exists(themedir): + os.mkdir(themedir) + LOGGER.info("Created directory {0}".format(themedir)) + else: + LOGGER.error("Theme already exists") + return 2 + + with io.open(os.path.join(themedir, 'parent'), 'w', encoding='utf-8') as fh: + fh.write(parent + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'parent'))) + with io.open(os.path.join(themedir, 'engine'), 'w', encoding='utf-8') as fh: + fh.write(engine + '\n') + LOGGER.info("Created file {0}".format(os.path.join(themedir, 'engine'))) + + LOGGER.info("Theme {0} created successfully.".format(themedir)) + LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(name)) + + def get_json(self, url): + """Download the JSON file with all plugins.""" + if self.json is None: + try: + self.json = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + self.json = requests.get(url).json() + return self.json diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py index b83d622..267837e 100644 --- a/nikola/plugins/command/version.py +++ b/nikola/plugins/command/version.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated |
