diff options
Diffstat (limited to 'nikola/plugins/command')
38 files changed, 1271 insertions, 528 deletions
diff --git a/nikola/plugins/command/__init__.py b/nikola/plugins/command/__init__.py index 6ad8bac..a1d17a6 100644 --- a/nikola/plugins/command/__init__.py +++ b/nikola/plugins/command/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/auto.plugin b/nikola/plugins/command/auto.plugin index 87939b2..a1c6820 100644 --- a/nikola/plugins/command/auto.plugin +++ b/nikola/plugins/command/auto.plugin @@ -4,6 +4,6 @@ Module = auto [Documentation] Author = Roberto Alsina -Version = 0.2 +Version = 2.1.0 Website = http://getnikola.com Description = Automatically detect site changes, rebuild and optionally refresh a browser. diff --git a/nikola/plugins/command/auto.py b/nikola/plugins/command/auto.py deleted file mode 100644 index 7f3f66f..0000000 --- a/nikola/plugins/command/auto.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2012-2014 Roberto Alsina and others. - -# Permission is hereby granted, free of charge, to any -# person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the -# Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the -# Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice -# shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -from __future__ import print_function, unicode_literals - -import os -import subprocess - -from nikola.plugin_categories import Command -from nikola.utils import req_missing - - -class CommandAuto(Command): - """Start debugging console.""" - name = "auto" - doc_purpose = "automatically detect site changes, rebuild and optionally refresh a browser" - cmd_options = [ - { - 'name': 'browser', - 'short': 'b', - 'type': bool, - 'help': 'Start a web browser.', - 'default': False, - }, - { - 'name': 'port', - 'short': 'p', - 'long': 'port', - 'default': 8000, - 'type': int, - 'help': 'Port nummber (default: 8000)', - }, - ] - - def _execute(self, options, args): - """Start the watcher.""" - try: - from livereload import Server - except ImportError: - req_missing(['livereload'], 'use the "auto" command') - return - - # Run an initial build so we are up-to-date - subprocess.call(("nikola", "build")) - - port = options and options.get('port') - - server = Server() - server.watch('conf.py', 'nikola build') - server.watch('themes/', 'nikola build') - server.watch('templates/', 'nikola build') - server.watch(self.site.config['GALLERY_PATH'], 'nikola build') - for item in self.site.config['post_pages']: - server.watch(os.path.dirname(item[0]), 'nikola build') - for item in self.site.config['FILES_FOLDERS']: - server.watch(item, 'nikola build') - - out_folder = self.site.config['OUTPUT_FOLDER'] - if options and options.get('browser'): - browser = True - else: - browser = False - - server.serve(port, None, out_folder, True, browser) diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py new file mode 100644 index 0000000..c25ef8a --- /dev/null +++ b/nikola/plugins/command/auto/__init__.py @@ -0,0 +1,366 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2015 Roberto Alsina and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function + +import json +import mimetypes +import os +import re +import subprocess +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # NOQA +import webbrowser +from wsgiref.simple_server import make_server +import wsgiref.util + +from blinker import signal +try: + from ws4py.websocket import WebSocket + from ws4py.server.wsgirefserver import WSGIServer, WebSocketWSGIRequestHandler + from ws4py.server.wsgiutils import WebSocketWSGIApplication + from ws4py.messaging import TextMessage +except ImportError: + WebSocket = object +try: + import watchdog + from watchdog.observers import Observer + from watchdog.events import FileSystemEventHandler, PatternMatchingEventHandler +except ImportError: + watchdog = None + FileSystemEventHandler = object + PatternMatchingEventHandler = object + + +from nikola.plugin_categories import Command +from nikola.utils import req_missing, get_logger, get_theme_path +LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js') +error_signal = signal('error') +refresh_signal = signal('refresh') + +ERROR_N = '''<html> +<head> +</head> +<boody> +ERROR {} +</body> +</html> +''' + + +class CommandAuto(Command): + """Start debugging console.""" + name = "auto" + logger = None + doc_purpose = "builds and serves a site; automatically detects site changes, rebuilds, and optionally refreshes a browser" + cmd_options = [ + { + 'name': 'port', + 'short': 'p', + 'long': 'port', + 'default': 8000, + 'type': int, + 'help': 'Port nummber (default: 8000)', + }, + { + 'name': 'address', + 'short': 'a', + 'long': 'address', + 'type': str, + 'default': '127.0.0.1', + 'help': 'Address to bind (default: 127.0.0.1 – localhost)', + }, + { + 'name': 'browser', + 'short': 'b', + 'long': 'browser', + 'type': bool, + 'help': 'Start a web browser.', + 'default': False, + }, + { + 'name': 'ipv6', + 'short': '6', + 'long': 'ipv6', + 'default': False, + 'type': bool, + 'help': 'Use IPv6', + }, + ] + + def _execute(self, options, args): + """Start the watcher.""" + + self.logger = get_logger('auto', self.site.loghandlers) + LRSocket.logger = self.logger + + if WebSocket is object and watchdog is None: + req_missing(['ws4py', 'watchdog'], 'use the "auto" command') + elif WebSocket is object: + req_missing(['ws4py'], 'use the "auto" command') + elif watchdog is None: + req_missing(['watchdog'], 'use the "auto" command') + + self.cmd_arguments = ['nikola', 'build'] + if self.site.configuration_filename != 'conf.py': + self.cmd_arguments = ['--conf=' + self.site.configuration_filename] + self.cmd_arguments + + # Run an initial build so we are up-to-date + subprocess.call(self.cmd_arguments) + + port = options and options.get('port') + self.snippet = '''<script>document.write('<script src="http://' + + (location.host || 'localhost').split(':')[0] + + ':{0}/livereload.js?snipver=1"></' + + 'script>')</script> + </head>'''.format(port) + + # Do not duplicate entries -- otherwise, multiple rebuilds are triggered + watched = set([ + 'templates/', + ] + [get_theme_path(name) for name in self.site.THEMES]) + for item in self.site.config['post_pages']: + watched.add(os.path.dirname(item[0])) + for item in self.site.config['FILES_FOLDERS']: + watched.add(item) + for item in self.site.config['GALLERY_FOLDERS']: + watched.add(item) + for item in self.site.config['LISTINGS_FOLDERS']: + watched.add(item) + + out_folder = self.site.config['OUTPUT_FOLDER'] + if options and options.get('browser'): + browser = True + else: + browser = False + + if options['ipv6']: + dhost = '::' + else: + dhost = None + + host = options['address'].strip('[').strip(']') or dhost + + # Instantiate global observer + observer = Observer() + # Watch output folders and trigger reloads + observer.schedule(OurWatchHandler(self.do_refresh), out_folder, recursive=True) + + # Watch input folders and trigger rebuilds + for p in watched: + if os.path.exists(p): + observer.schedule(OurWatchHandler(self.do_rebuild), p, recursive=True) + + # Watch config file (a bit of a hack, but we need a directory) + _conf_fn = os.path.abspath(self.site.configuration_filename or 'conf.py') + _conf_dn = os.path.dirname(_conf_fn) + observer.schedule(ConfigWatchHandler(_conf_fn, self.do_rebuild), _conf_dn, recursive=False) + + observer.start() + + parent = self + + class Mixed(WebSocketWSGIApplication): + """A class that supports WS and HTTP protocols in the same port.""" + def __call__(self, environ, start_response): + if environ.get('HTTP_UPGRADE') is None: + return parent.serve_static(environ, start_response) + return super(Mixed, self).__call__(environ, start_response) + + ws = make_server( + host, port, server_class=WSGIServer, + handler_class=WebSocketWSGIRequestHandler, + app=Mixed(handler_cls=LRSocket) + ) + ws.initialize_websockets_manager() + self.logger.info("Serving HTTP on {0} port {1}...".format(host, port)) + if browser: + if options['ipv6'] or '::' in host: + server_url = "http://[{0}]:{1}/".format(host, port) + else: + server_url = "http://{0}:{1}/".format(host, port) + + self.logger.info("Opening {0} in the default web browser...".format(server_url)) + # Yes, this is racy + webbrowser.open('http://{0}:{1}'.format(host, port)) + + try: + ws.serve_forever() + except KeyboardInterrupt: + self.logger.info("Server is shutting down.") + observer.stop() + observer.join() + + def do_rebuild(self, event): + self.logger.info('REBUILDING SITE (from {0})'.format(event.src_path)) + p = subprocess.Popen(self.cmd_arguments, stderr=subprocess.PIPE) + if p.wait() != 0: + error = p.stderr.read() + self.logger.error(error) + error_signal.send(error=error) + else: + error = p.stderr.read() + print(error) + + def do_refresh(self, event): + self.logger.info('REFRESHING: {0}'.format(event.src_path)) + p = os.path.relpath(event.src_path, os.path.abspath(self.site.config['OUTPUT_FOLDER'])) + refresh_signal.send(path=p) + + def serve_static(self, environ, start_response): + """Trivial static file server.""" + uri = wsgiref.util.request_uri(environ) + p_uri = urlparse(uri) + f_path = os.path.join(self.site.config['OUTPUT_FOLDER'], *p_uri.path.split('/')) + mimetype = mimetypes.guess_type(uri)[0] or 'text/html' + + if os.path.isdir(f_path): + f_path = os.path.join(f_path, self.site.config['INDEX_FILE']) + + if p_uri.path == '/robots.txt': + start_response('200 OK', [('Content-type', 'text/plain')]) + return ['User-Agent: *\nDisallow: /\n'] + elif os.path.isfile(f_path): + with open(f_path, 'rb') as fd: + start_response('200 OK', [('Content-type', mimetype)]) + return [self.inject_js(mimetype, fd.read())] + elif p_uri.path == '/livereload.js': + with open(LRJS_PATH, 'rb') as fd: + start_response('200 OK', [('Content-type', mimetype)]) + return [self.inject_js(mimetype, fd.read())] + start_response('404 ERR', []) + return [self.inject_js('text/html', ERROR_N.format(404).format(uri))] + + def inject_js(self, mimetype, data): + """Inject livereload.js in HTML files.""" + if mimetype == 'text/html': + data = re.sub('</head>', self.snippet, data.decode('utf8'), 1, re.IGNORECASE) + data = data.encode('utf8') + return data + + +pending = [] + + +class LRSocket(WebSocket): + """Speak Livereload protocol.""" + + def __init__(self, *a, **kw): + refresh_signal.connect(self.notify) + error_signal.connect(self.send_error) + super(LRSocket, self).__init__(*a, **kw) + + def received_message(self, message): + message = json.loads(message.data.decode('utf8')) + self.logger.info('<--- {0}'.format(message)) + response = None + if message['command'] == 'hello': # Handshake + response = { + 'command': 'hello', + 'protocols': [ + 'http://livereload.com/protocols/official-7', + ], + 'serverName': 'nikola-livereload', + } + elif message['command'] == 'info': # Someone connected + self.logger.info('****** Browser connected: {0}'.format(message.get('url'))) + self.logger.info('****** sending {0} pending messages'.format(len(pending))) + while pending: + msg = pending.pop() + self.logger.info('---> {0}'.format(msg.data)) + self.send(msg, msg.is_binary) + else: + response = { + 'command': 'alert', + 'message': 'HEY', + } + if response is not None: + response = json.dumps(response) + self.logger.info('---> {0}'.format(response)) + response = TextMessage(response) + self.send(response, response.is_binary) + + def notify(self, sender, path): + """Send reload requests to the client.""" + p = os.path.join('/', path) + message = { + 'command': 'reload', + 'liveCSS': True, + 'path': p, + } + response = json.dumps(message) + self.logger.info('---> {0}'.format(p)) + response = TextMessage(response) + if self.stream is None: # No client connected or whatever + pending.append(response) + else: + self.send(response, response.is_binary) + + def send_error(self, sender, error=None): + """Send reload requests to the client.""" + if self.stream is None: # No client connected or whatever + return + message = { + 'command': 'alert', + 'message': error, + } + response = json.dumps(message) + response = TextMessage(response) + if self.stream is None: # No client connected or whatever + pending.append(response) + else: + self.send(response, response.is_binary) + + +class OurWatchHandler(FileSystemEventHandler): + + """A Nikola-specific handler for Watchdog.""" + + def __init__(self, function): + """Initialize the handler.""" + self.function = function + super(OurWatchHandler, self).__init__() + + def on_any_event(self, event): + """Call the provided function on any event.""" + self.function(event) + + +class ConfigWatchHandler(FileSystemEventHandler): + + """A Nikola-specific handler for Watchdog that handles the config file (as a workaround).""" + + def __init__(self, configuration_filename, function): + """Initialize the handler.""" + self.configuration_filename = configuration_filename + self.function = function + + def on_any_event(self, event): + """Call the provided function on any event.""" + if event._src_path == self.configuration_filename: + self.function(event) diff --git a/nikola/plugins/command/auto/livereload.js b/nikola/plugins/command/auto/livereload.js new file mode 120000 index 0000000..b4cafb3 --- /dev/null +++ b/nikola/plugins/command/auto/livereload.js @@ -0,0 +1 @@ +../../../../bower_components/livereload-js/dist/livereload.js
\ No newline at end of file diff --git a/nikola/plugins/command/bootswatch_theme.plugin b/nikola/plugins/command/bootswatch_theme.plugin index 7091310..b428da3 100644 --- a/nikola/plugins/command/bootswatch_theme.plugin +++ b/nikola/plugins/command/bootswatch_theme.plugin @@ -4,7 +4,7 @@ Module = bootswatch_theme [Documentation] Author = Roberto Alsina -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Given a swatch name and a parent theme, creates a custom theme. diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py index e65413b..e19c937 100644 --- a/nikola/plugins/command/bootswatch_theme.py +++ b/nikola/plugins/command/bootswatch_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -26,11 +26,7 @@ from __future__ import print_function import os - -try: - import requests -except ImportError: - requests = None # NOQA +import requests from nikola.plugin_categories import Command from nikola import utils @@ -57,7 +53,7 @@ class CommandBootswatchTheme(Command): { 'name': 'swatch', 'short': 's', - 'default': 'slate', + 'default': '', 'type': str, 'help': 'Name of the swatch from bootswatch.com.' }, @@ -72,19 +68,19 @@ class CommandBootswatchTheme(Command): def _execute(self, options, args): """Given a swatch name and a parent theme, creates a custom theme.""" - if requests is None: - utils.req_missing(['requests'], 'install Bootswatch themes') - name = options['name'] swatch = options['swatch'] + if not swatch: + LOGGER.error('The -s option is mandatory') + return 1 parent = options['parent'] version = '' # See if we need bootswatch for bootstrap v2 or v3 themes = utils.get_theme_chain(parent) - if 'bootstrap3' not in themes or 'bootstrap3-jinja' not in themes: + if 'bootstrap3' not in themes and 'bootstrap3-jinja' not in themes: version = '2' - elif 'bootstrap' not in themes or 'bootstrap-jinja' not in themes: + elif 'bootstrap' not in themes and 'bootstrap-jinja' not in themes: LOGGER.warn('"bootswatch_theme" only makes sense for themes that use bootstrap') elif 'bootstrap3-gradients' in themes or 'bootstrap3-gradients-jinja' in themes: LOGGER.warn('"bootswatch_theme" doesn\'t work well with the bootstrap3-gradients family') diff --git a/nikola/plugins/command/check.plugin b/nikola/plugins/command/check.plugin index 8ceda5f..dd0980e 100644 --- a/nikola/plugins/command/check.plugin +++ b/nikola/plugins/command/check.plugin @@ -4,7 +4,7 @@ Module = check [Documentation] Author = Roberto Alsina -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Check the generated site diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py index bd254f4..a9bc44a 100644 --- a/nikola/plugins/command/check.py +++ b/nikola/plugins/command/check.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -25,6 +25,7 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from __future__ import print_function +from collections import defaultdict import os import re import sys @@ -34,21 +35,36 @@ try: except ImportError: from urllib.parse import unquote, urlparse, urljoin, urldefrag # NOQA +from doit.loader import generate_tasks import lxml.html +import requests from nikola.plugin_categories import Command from nikola.utils import get_logger +def _call_nikola_list(site): + files = [] + deps = defaultdict(list) + for task in generate_tasks('render_site', site.gen_tasks('render_site', "Task", '')): + files.extend(task.targets) + for target in task.targets: + deps[target].extend(task.file_dep) + for task in generate_tasks('post_render', site.gen_tasks('render_site', "LateTask", '')): + files.extend(task.targets) + for target in task.targets: + deps[target].extend(task.file_dep) + return files, deps + + def real_scan_files(site): task_fnames = set([]) real_fnames = set([]) output_folder = site.config['OUTPUT_FOLDER'] # First check that all targets are generated in the right places - for task in os.popen('nikola list --all', 'r').readlines(): - task = task.strip() - if output_folder in task and ':' in task: - fname = task.split(':', 1)[-1] + for fname in _call_nikola_list(site)[0]: + fname = fname.strip() + if fname.startswith(output_folder): task_fnames.add(fname) # And now check that there are no non-target files for root, dirs, files in os.walk(output_folder, followlinks=True): @@ -68,7 +84,7 @@ def fs_relpath_from_url_path(url_path): url_path = unquote(url_path) # in windows relative paths don't begin with os.sep if sys.platform == 'win32' and len(url_path): - url_path = url_path[1:].replace('/', '\\') + url_path = url_path.replace('/', '\\') return url_path @@ -78,7 +94,7 @@ class CommandCheck(Command): name = "check" logger = None - doc_usage = "-l [--find-sources] | -f" + doc_usage = "[-v] (-l [--find-sources] [-r] | -f [--clean-files])" doc_purpose = "check links and files in the generated site" cmd_options = [ { @@ -119,11 +135,18 @@ class CommandCheck(Command): 'default': False, 'help': 'Be more verbose.', }, + { + 'name': 'remote', + 'long': 'remote', + 'short': 'r', + 'type': bool, + 'default': False, + 'help': 'Check that remote links work.', + }, ] def _execute(self, options, args): """Check the generated site.""" - self.logger = get_logger('check', self.site.loghandlers) if not options['links'] and not options['files'] and not options['clean']: @@ -134,59 +157,103 @@ class CommandCheck(Command): else: self.logger.level = 4 if options['links']: - failure = self.scan_links(options['find_sources']) + failure = self.scan_links(options['find_sources'], options['remote']) if options['files']: failure = self.scan_files() if options['clean']: failure = self.clean_files() if failure: - sys.exit(1) + return 1 existing_targets = set([]) + checked_remote_targets = {} - def analyze(self, task, find_sources=False): + def analyze(self, fname, find_sources=False, check_remote=False): rv = False self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']] base_url = urlparse(self.site.config['BASE_URL']) self.existing_targets.add(self.site.config['SITE_URL']) self.existing_targets.add(self.site.config['BASE_URL']) url_type = self.site.config['URL_TYPE'] - if url_type == 'absolute': - url_netloc_to_root = urlparse(self.site.config['SITE_URL']).path + + deps = {} + if find_sources: + deps = _call_nikola_list(self.site)[1] + + if url_type in ('absolute', 'full_path'): + url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path try: - filename = task.split(":")[-1] - d = lxml.html.fromstring(open(filename).read()) + filename = fname + + if filename.startswith(self.site.config['CACHE_FOLDER']): + # Do not look at links in the cache, which are not parsed by + # anyone and may result in false positives. Problems arise + # with galleries, for example. Full rationale: (Issue #1447) + self.logger.notice("Ignoring {0} (in cache, links may be incorrect)".format(filename)) + return False + + if not os.path.exists(fname): + # Quietly ignore files that don’t exist; use `nikola check -f` instead (Issue #1831) + return False + + d = lxml.html.fromstring(open(filename, 'rb').read()) for l in d.iterlinks(): - target = l[0].attrib[l[1]] + target = l[2] if target == "#": continue target, _ = urldefrag(target) parsed = urlparse(target) - # Absolute links when using only paths, skip. - if (parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path'): - continue + # Warn about links from https to http (mixed-security) + if base_url.netloc == parsed.netloc and base_url.scheme == "https" and parsed.scheme == "http": + self.logger.warn("Mixed-content security for link in {0}: {1}".format(filename, target)) # Absolute links to other domains, skip - if (parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc: + # Absolute links when using only paths, skip. + if ((parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc) or \ + ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')): + if not check_remote or parsed.scheme not in ["http", "https"]: + continue + if parsed.netloc == base_url.netloc: # absolute URL to self.site + continue + if target in self.checked_remote_targets: # already checked this exact target + if self.checked_remote_targets[target] > 399: + self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) + continue + # Check the remote link works + req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'} # I’m a real boy! + resp = requests.head(target, headers=req_headers) + self.checked_remote_targets[target] = resp.status_code + if resp.status_code > 399: # Error + self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, resp.status_code)) + continue + elif resp.status_code <= 399: # The address leads *somewhere* that is not an error + self.logger.debug("Successfully checked remote link in {0}: {1} [HTTP: {2}]".format(filename, target, resp.status_code)) + continue + self.logger.warn("Could not check remote link in {0}: {1} [Unknown problem]".format(filename, target)) continue if url_type == 'rel_path': - target_filename = os.path.abspath( - os.path.join(os.path.dirname(filename), unquote(target))) + if target.startswith('/'): + target_filename = os.path.abspath( + os.path.join(self.site.config['OUTPUT_FOLDER'], unquote(target.lstrip('/')))) + else: # Relative path + target_filename = os.path.abspath( + os.path.join(os.path.dirname(filename), unquote(target))) elif url_type in ('full_path', 'absolute'): if url_type == 'absolute': # convert to 'full_path' case, ie url relative to root - url_rel_path = target.path[len(url_netloc_to_root):] + url_rel_path = parsed.path[len(url_netloc_to_root):] else: - url_rel_path = target.path + # convert to relative to base path + url_rel_path = target[len(url_netloc_to_root):] if url_rel_path == '' or url_rel_path.endswith('/'): url_rel_path = urljoin(url_rel_path, self.site.config['INDEX_FILE']) fs_rel_path = fs_relpath_from_url_path(url_rel_path) target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], fs_rel_path) - if any(re.match(x, target_filename) for x in self.whitelist): + if any(re.search(x, target_filename) for x in self.whitelist): continue elif target_filename not in self.existing_targets: if os.path.exists(target_filename): @@ -197,25 +264,22 @@ class CommandCheck(Command): self.logger.warn("Broken link in {0}: {1}".format(filename, target)) if find_sources: self.logger.warn("Possible sources:") - self.logger.warn(os.popen('nikola list --deps ' + task, 'r').read()) + self.logger.warn("\n".join(deps[filename])) self.logger.warn("===============================\n") except Exception as exc: self.logger.error("Error with: {0} {1}".format(filename, exc)) return rv - def scan_links(self, find_sources=False): + def scan_links(self, find_sources=False, check_remote=False): self.logger.info("Checking Links:") self.logger.info("===============\n") self.logger.notice("{0} mode".format(self.site.config['URL_TYPE'])) failure = False - for task in os.popen('nikola list --all', 'r').readlines(): - task = task.strip() - if task.split(':')[0] in ( - 'render_tags', 'render_archive', - 'render_galleries', 'render_indexes', - 'render_pages' - 'render_site') and '.html' in task: - if self.analyze(task, find_sources): + # Maybe we should just examine all HTML files + output_folder = self.site.config['OUTPUT_FOLDER'] + for fname in _call_nikola_list(self.site)[0]: + if fname.startswith(output_folder) and '.html' == fname[-5:]: + if self.analyze(fname, find_sources, check_remote): failure = True if not failure: self.logger.info("All links checked.") diff --git a/nikola/plugins/command/console.plugin b/nikola/plugins/command/console.plugin index 2eeedae..3aef2e7 100644 --- a/nikola/plugins/command/console.plugin +++ b/nikola/plugins/command/console.plugin @@ -4,6 +4,6 @@ Module = console [Documentation] Author = Chris Warrick, Roberto Alsina -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Start a debugging python console diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py index 9dfc975..b8e7825 100644 --- a/nikola/plugins/command/console.py +++ b/nikola/plugins/command/console.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Chris Warrick, Roberto Alsina and others. +# Copyright © 2012-2015 Chris Warrick, Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -30,7 +30,7 @@ import os from nikola import __version__ from nikola.plugin_categories import Command -from nikola.utils import get_logger, STDERR_HANDLER, req_missing +from nikola.utils import get_logger, STDERR_HANDLER, req_missing, Commands LOGGER = get_logger('console', STDERR_HANDLER) @@ -122,6 +122,8 @@ If there is no console to use specified (as -b, -i, -p) it tries IPython, then f self.site.scan_posts() # Create nice object with all commands: + self.site.commands = Commands(self.site.doit, self.config, self._doitargs) + self.context = { 'conf': self.site.config, 'site': self.site, diff --git a/nikola/plugins/command/deploy.plugin b/nikola/plugins/command/deploy.plugin index 10cc796..14fd53f 100644 --- a/nikola/plugins/command/deploy.plugin +++ b/nikola/plugins/command/deploy.plugin @@ -4,6 +4,6 @@ Module = deploy [Documentation] Author = Roberto Alsina -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Deploy the site diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py index fde43fa..2c44e87 100644 --- a/nikola/plugins/command/deploy.py +++ b/nikola/plugins/command/deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -29,23 +29,22 @@ import io from datetime import datetime from dateutil.tz import gettz import os -import sys import subprocess import time from blinker import signal from nikola.plugin_categories import Command -from nikola.utils import get_logger, remove_file, unicode_str +from nikola.utils import get_logger, remove_file, unicode_str, makedirs class CommandDeploy(Command): """Deploy site.""" name = "deploy" - doc_usage = "" + doc_usage = "[[preset [preset...]]" doc_purpose = "deploy the site" - + doc_description = "Deploy the site by executing deploy commands from the presets listed on the command line. If no presets are specified, `default` is executed." logger = None def _execute(self, command, args): @@ -74,14 +73,29 @@ class CommandDeploy(Command): remove_file(os.path.join(out_dir, post.source_path)) undeployed_posts.append(post) - for command in self.site.config['DEPLOY_COMMANDS']: - self.logger.info("==> {0}".format(command)) + if args: + presets = args + else: + presets = ['default'] + + # test for preset existence + for preset in presets: try: - subprocess.check_call(command, shell=True) - except subprocess.CalledProcessError as e: - self.logger.error('Failed deployment — command {0} ' - 'returned {1}'.format(e.cmd, e.returncode)) - sys.exit(e.returncode) + self.site.config['DEPLOY_COMMANDS'][preset] + except: + self.logger.error('No such preset: {0}'.format(preset)) + return 255 + + for preset in presets: + self.logger.info("=> preset '{0}'".format(preset)) + for command in self.site.config['DEPLOY_COMMANDS'][preset]: + self.logger.info("==> {0}".format(command)) + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError as e: + self.logger.error('Failed deployment — command {0} ' + 'returned {1}'.format(e.cmd, e.returncode)) + return e.returncode self.logger.info("Successful deployment") try: @@ -96,6 +110,7 @@ class CommandDeploy(Command): new_deploy = datetime.utcnow() self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts) + makedirs(self.site.config['CACHE_FOLDER']) # Store timestamp of successful deployment with io.open(timestamp_path, 'w+', encoding='utf8') as outf: outf.write(unicode_str(new_deploy.isoformat())) diff --git a/nikola/plugins/command/github_deploy.plugin b/nikola/plugins/command/github_deploy.plugin index 4cbc422..74e7902 100644 --- a/nikola/plugins/command/github_deploy.plugin +++ b/nikola/plugins/command/github_deploy.plugin @@ -4,6 +4,6 @@ Module = github_deploy [Documentation] Author = Puneeth Chaganti -Version = 0.1 +Version = 1,0 Website = http://getnikola.com Description = Deploy the site to GitHub pages. diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py index 13da48c..888a4f9 100644 --- a/nikola/plugins/command/github_deploy.py +++ b/nikola/plugins/command/github_deploy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2014 Puneeth Chaganti and others. +# Copyright © 2014-2015 Puneeth Chaganti and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -25,15 +25,15 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from __future__ import print_function +from datetime import datetime +import io import os -import shutil import subprocess -import sys from textwrap import dedent from nikola.plugin_categories import Command from nikola.plugins.command.check import real_scan_files -from nikola.utils import ask_yesno, get_logger +from nikola.utils import get_logger, req_missing, makedirs, unicode_str from nikola.__main__ import main from nikola import __version__ @@ -43,79 +43,53 @@ def uni_check_output(*args, **kwargs): return o.decode('utf-8') +def check_ghp_import_installed(): + try: + subprocess.check_output(['ghp-import', '-h']) + except OSError: + # req_missing defaults to `python=True` — and it’s meant to be like this. + # `ghp-import` is installed via pip, but the only way to use it is by executing the script it installs. + req_missing(['ghp-import'], 'deploy the site to GitHub Pages') + + class CommandGitHubDeploy(Command): - """ Deploy site to GitHub pages. """ + """ Deploy site to GitHub Pages. """ name = 'github_deploy' doc_usage = '' - doc_purpose = 'deploy the site to GitHub pages' + doc_purpose = 'deploy the site to GitHub Pages' doc_description = dedent( """\ - This command can be used to deploy your site to GitHub pages. - It performs the following actions: + This command can be used to deploy your site to GitHub Pages. - 1. Ensure that your site is a git repository, and git is on the PATH. - 2. Ensure that the output directory is not committed on the - source branch. - 3. Check for changes, and prompt the user to continue, if required. - 4. Build the site - 5. Clean any files that are "unknown" to Nikola. - 6. Create a deploy branch, if one doesn't exist. - 7. Commit the output to this branch. (NOTE: Any untracked source - files, may get committed at this stage, on the wrong branch!) - 8. Push and deploy! + It uses ghp-import to do this task. - NOTE: This command needs your site to be a git repository, with a - master branch (or a different branch, configured using - GITHUB_SOURCE_BRANCH if you are pushing to user.github - .io/organization.github.io pages) containing the sources of your - site. You also, obviously, need to have `git` on your PATH, - and should be able to push to the repository specified as the remote - (origin, by default). """ ) logger = None - _deploy_branch = '' - _source_branch = '' - _remote_name = '' - def _execute(self, command, args): self.logger = get_logger( CommandGitHubDeploy.name, self.site.loghandlers ) - self._source_branch = self.site.config.get( - 'GITHUB_SOURCE_BRANCH', 'master' - ) - self._deploy_branch = self.site.config.get( - 'GITHUB_DEPLOY_BRANCH', 'gh-pages' - ) - self._remote_name = self.site.config.get( - 'GITHUB_REMOTE_NAME', 'origin' - ) - - self._ensure_git_repo() - - self._exit_if_output_committed() - if not self._prompt_continue(): - return + # Check if ghp-import is installed + check_ghp_import_installed() + # Build before deploying build = main(['build']) if build != 0: self.logger.error('Build failed, not deploying to GitHub') - sys.exit(build) + return build + # Clean non-target files only_on_output, _ = real_scan_files(self.site) for f in only_on_output: os.unlink(f) - self._checkout_deploy_branch() - - self._copy_output() - + # Commit and push self._commit_and_push() return @@ -123,150 +97,34 @@ class CommandGitHubDeploy(Command): def _commit_and_push(self): """ Commit all the files and push. """ - deploy = self._deploy_branch - source = self._source_branch - remote = self._remote_name - + source = self.site.config['GITHUB_SOURCE_BRANCH'] + deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] + remote = self.site.config['GITHUB_REMOTE_NAME'] source_commit = uni_check_output(['git', 'rev-parse', source]) commit_message = ( 'Nikola auto commit.\n\n' 'Source commit: %s' 'Nikola version: %s' % (source_commit, __version__) ) - - commands = [ - ['git', 'pull', remote, '%s:%s' % (deploy, deploy)], - ['git', 'add', '-A'], - ['git', 'commit', '-m', commit_message], - ['git', 'push', remote, '%s:%s' % (deploy, deploy)], - ['git', 'checkout', source], - ] - - for command in commands: - self.logger.info("==> {0}".format(command)) - try: - subprocess.check_call(command) - except subprocess.CalledProcessError as e: - self.logger.error( - 'Failed GitHub deployment — command {0} ' - 'returned {1}'.format(e.cmd, e.returncode) - ) - sys.exit(e.returncode) - - def _copy_output(self): - """ Copy all output to the top level directory. """ output_folder = self.site.config['OUTPUT_FOLDER'] - for each in os.listdir(output_folder): - if os.path.exists(each): - if os.path.isdir(each): - shutil.rmtree(each) - - else: - os.unlink(each) - - shutil.move(os.path.join(output_folder, each), '.') - - def _checkout_deploy_branch(self): - """ Check out the deploy branch - - Creates an orphan branch if not present. - - """ - deploy = self._deploy_branch + command = ['ghp-import', '-n', '-m', commit_message, '-p', '-r', remote, '-b', deploy, output_folder] + self.logger.info("==> {0}".format(command)) try: - subprocess.check_call( - [ - 'git', 'show-ref', '--verify', '--quiet', - 'refs/heads/%s' % deploy - ] - ) - except subprocess.CalledProcessError: - self._create_orphan_deploy_branch() - else: - subprocess.check_call(['git', 'checkout', deploy]) - - def _create_orphan_deploy_branch(self): - """ Create an orphan deploy branch """ - - result = subprocess.check_call( - ['git', 'checkout', '--orphan', self._deploy_branch] - ) - if result != 0: - self.logger.error('Failed to create a deploy branch') - sys.exit(1) - - result = subprocess.check_call(['git', 'rm', '-rf', '.']) - if result != 0: - self.logger.error('Failed to create a deploy branch') - sys.exit(1) - - with open('.gitignore', 'w') as f: - f.write('%s\n' % self.site.config['OUTPUT_FOLDER']) - f.write('%s\n' % self.site.config['CACHE_FOLDER']) - f.write('*.pyc\n') - f.write('*.db\n') - - subprocess.check_call(['git', 'add', '.gitignore']) - subprocess.check_call(['git', 'commit', '-m', 'Add .gitignore']) - - def _ensure_git_repo(self): - """ Ensure that the site is a git-repo. - - Also make sure that a remote with the specified name exists. - - """ - - try: - remotes = uni_check_output(['git', 'remote']) + subprocess.check_call(command) except subprocess.CalledProcessError as e: - self.logger.notice('github_deploy needs a git repository!') - sys.exit(e.returncode) - except OSError as e: - import errno - self.logger.error('Running git failed with {0}'.format(e)) - if e.errno == errno.ENOENT: - self.logger.notice('Is git on the PATH?') - sys.exit(1) - else: - if self._remote_name not in remotes: - self.logger.error( - 'Need a remote called "%s" configured' % self._remote_name - ) - sys.exit(1) - - def _exit_if_output_committed(self): - """ Exit if the output folder is committed on the source branch. """ - - source = self._source_branch - subprocess.check_call(['git', 'checkout', source]) - - output_folder = self.site.config['OUTPUT_FOLDER'] - output_log = uni_check_output( - ['git', 'ls-files', '--', output_folder] - ) - - if len(output_log.strip()) > 0: self.logger.error( - 'Output folder is committed on the source branch. ' - 'Cannot proceed until it is removed.' + 'Failed GitHub deployment — command {0} ' + 'returned {1}'.format(e.cmd, e.returncode) ) - sys.exit(1) - - def _prompt_continue(self): - """ Show uncommitted changes, and ask if user wants to continue. """ + return e.returncode - changes = uni_check_output(['git', 'status', '--porcelain']) - if changes.strip(): - changes = uni_check_output(['git', 'status']).strip() - message = ( - "You have the following changes:\n%s\n\n" - "Anything not committed, and unknown to Nikola may be lost, " - "or committed onto the wrong branch. Do you wish to continue?" - ) % changes - proceed = ask_yesno(message, False) - else: - proceed = True + self.logger.info("Successful deployment") - return proceed + # Store timestamp of successful deployment + timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") + new_deploy = datetime.utcnow() + makedirs(self.site.config["CACHE_FOLDER"]) + with io.open(timestamp_path, "w+", encoding="utf8") as outf: + outf.write(unicode_str(new_deploy.isoformat())) diff --git a/nikola/plugins/command/import_wordpress.plugin b/nikola/plugins/command/import_wordpress.plugin index fadc759..e072224 100644 --- a/nikola/plugins/command/import_wordpress.plugin +++ b/nikola/plugins/command/import_wordpress.plugin @@ -4,7 +4,7 @@ Module = import_wordpress [Documentation] Author = Roberto Alsina -Version = 0.2 +Version = 1.0 Website = http://getnikola.com Description = Import a wordpress site from a XML dump (requires markdown). diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index 1af4083..674fc2a 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -28,6 +28,8 @@ from __future__ import unicode_literals, print_function import os import re import sys +import datetime +import requests from lxml import etree try: @@ -37,11 +39,6 @@ except ImportError: from urllib.parse import urlparse, unquote # NOQA try: - import requests -except ImportError: - requests = None # NOQA - -try: import phpserialize except ImportError: phpserialize = None # NOQA @@ -87,6 +84,13 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Do not try to download files for the import", }, { + 'name': 'download_auth', + 'long': 'download-auth', + 'default': None, + 'type': str, + 'help': "Specify username and password for HTTP authentication (separated by ':')", + }, + { 'name': 'separate_qtranslate_content', 'long': 'qtranslate', 'default': False, @@ -104,6 +108,7 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "The pattern for translation files names", }, ] + all_tags = set([]) def _execute(self, options={}, args=[]): """Import a WordPress blog from an export file into a Nikola site.""" @@ -133,6 +138,14 @@ class CommandImportWordpress(Command, ImportMixin): self.exclude_drafts = options.get('exclude_drafts', False) self.no_downloads = options.get('no_downloads', False) + self.auth = None + if options.get('download_auth') is not None: + username_password = options.get('download_auth') + self.auth = tuple(username_password.split(':', 1)) + if len(self.auth) < 2: + print("Please specify HTTP authentication credentials in the form username:password.") + return False + self.separate_qtranslate_content = options.get('separate_qtranslate_content') self.translations_pattern = options.get('translations_pattern') @@ -149,11 +162,7 @@ class CommandImportWordpress(Command, ImportMixin): package=modulename) ) - if requests is None and phpserialize is None: - req_missing(['requests', 'phpserialize'], 'import WordPress dumps without --no-downloads') - elif requests is None: - req_missing(['requests'], 'import WordPress dumps without --no-downloads') - elif phpserialize is None: + if phpserialize is None: req_missing(['phpserialize'], 'import WordPress dumps without --no-downloads') channel = self.get_channel_from_file(self.wordpress_export_file) @@ -172,6 +181,19 @@ class CommandImportWordpress(Command, ImportMixin): self.extra_languages) self.context['REDIRECTIONS'] = self.configure_redirections( self.url_map) + + # Add tag redirects + for tag in self.all_tags: + try: + tag_str = tag.decode('utf8') + except AttributeError: + tag_str = tag + tag = utils.slugify(tag_str) + src_url = '{}tag/{}'.format(self.context['SITE_URL'], tag) + dst_url = self.site.link('tag', tag) + if src_url != dst_url: + self.url_map[src_url] = dst_url + self.write_urlmap_csv( os.path.join(self.output_folder, 'url_map.csv'), self.url_map) rendered_template = conf_template.render(**prepare_config(self.context)) @@ -186,26 +208,6 @@ class CommandImportWordpress(Command, ImportMixin): rendered_template) @classmethod - def _glue_xml_lines(cls, xml): - new_xml = xml[0] - previous_line_ended_in_newline = new_xml.endswith(b'\n') - previous_line_was_indentet = False - for line in xml[1:]: - if (re.match(b'^[ \t]+', line) and previous_line_ended_in_newline): - new_xml = b''.join((new_xml, line)) - previous_line_was_indentet = True - elif previous_line_was_indentet: - new_xml = b''.join((new_xml, line)) - previous_line_was_indentet = False - else: - new_xml = b'\n'.join((new_xml, line)) - previous_line_was_indentet = False - - previous_line_ended_in_newline = line.endswith(b'\n') - - return new_xml - - @classmethod def read_xml_file(cls, filename): xml = [] @@ -215,8 +217,7 @@ class CommandImportWordpress(Command, ImportMixin): if b'<atom:link rel=' in line: continue xml.append(line) - - return cls._glue_xml_lines(xml) + return b'\n'.join(xml) @classmethod def get_channel_from_file(cls, filename): @@ -255,9 +256,15 @@ class CommandImportWordpress(Command, ImportMixin): '{{{0}}}author_display_name'.format(wordpress_namespace), "Joe Example") context['POSTS'] = '''( + ("posts/*.rst", "posts", "post.tmpl"), + ("posts/*.txt", "posts", "post.tmpl"), + ("posts/*.md", "posts", "post.tmpl"), ("posts/*.wp", "posts", "post.tmpl"), )''' context['PAGES'] = '''( + ("stories/*.rst", "stories", "story.tmpl"), + ("stories/*.txt", "stories", "story.tmpl"), + ("stories/*.md", "stories", "story.tmpl"), ("stories/*.wp", "stories", "story.tmpl"), )''' context['COMPILERS'] = '''{ @@ -274,8 +281,12 @@ class CommandImportWordpress(Command, ImportMixin): return try: + request = requests.get(url, auth=self.auth) + if request.status_code >= 400: + LOGGER.warn("Downloading {0} to {1} failed with HTTP status code {2}".format(url, dst_path, request.status_code)) + return with open(dst_path, 'wb+') as fd: - fd.write(requests.get(url).content) + fd.write(request.content) except requests.exceptions.ConnectionError as err: LOGGER.warn("Downloading {0} to {1} failed: {2}".format(url, dst_path, err)) @@ -285,8 +296,7 @@ class CommandImportWordpress(Command, ImportMixin): link = get_text_tag(item, '{{{0}}}link'.format(wordpress_namespace), 'foo') path = urlparse(url).path - dst_path = os.path.join(*([self.output_folder, 'files'] - + list(path.split('/')))) + dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) dst_dir = os.path.dirname(dst_path) utils.makedirs(dst_dir) LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) @@ -306,7 +316,6 @@ class CommandImportWordpress(Command, ImportMixin): return additional_metadata = item.findall('{{{0}}}postmeta'.format(wordpress_namespace)) - if additional_metadata is None: return @@ -341,8 +350,7 @@ class CommandImportWordpress(Command, ImportMixin): url = '/'.join([source_path, filename.decode('utf-8')]) path = urlparse(url).path - dst_path = os.path.join(*([self.output_folder, 'files'] - + list(path.split('/')))) + dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) dst_dir = os.path.dirname(dst_path) utils.makedirs(dst_dir) LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) @@ -351,13 +359,34 @@ class CommandImportWordpress(Command, ImportMixin): links[url] = '/' + dst_url links[url] = '/' + dst_url - @staticmethod - def transform_sourcecode(content): - new_content = re.sub('\[sourcecode language="([^"]+)"\]', - "\n~~~~~~~~~~~~{.\\1}\n", content) - new_content = new_content.replace('[/sourcecode]', - "\n~~~~~~~~~~~~\n") - return new_content + code_re1 = re.compile(r'\[code.* lang.*?="(.*?)?".*\](.*?)\[/code\]', re.DOTALL | re.MULTILINE) + code_re2 = re.compile(r'\[sourcecode.* lang.*?="(.*?)?".*\](.*?)\[/sourcecode\]', re.DOTALL | re.MULTILINE) + code_re3 = re.compile(r'\[code.*?\](.*?)\[/code\]', re.DOTALL | re.MULTILINE) + code_re4 = re.compile(r'\[sourcecode.*?\](.*?)\[/sourcecode\]', re.DOTALL | re.MULTILINE) + + def transform_code(self, content): + # http://en.support.wordpress.com/code/posting-source-code/. There are + # a ton of things not supported here. We only do a basic [code + # lang="x"] -> ```x translation, and remove quoted html entities (<, + # >, &, and "). + def replacement(m, c=content): + if len(m.groups()) == 1: + language = '' + code = m.group(0) + else: + language = m.group(1) or '' + code = m.group(2) + code = code.replace('&', '&') + code = code.replace('>', '>') + code = code.replace('<', '<') + code = code.replace('"', '"') + return '```{language}\n{code}\n```'.format(language=language, code=code) + + content = self.code_re1.sub(replacement, content) + content = self.code_re2.sub(replacement, content) + content = self.code_re3.sub(replacement, content) + content = self.code_re4.sub(replacement, content) + return content @staticmethod def transform_caption(content): @@ -374,10 +403,10 @@ class CommandImportWordpress(Command, ImportMixin): return content def transform_content(self, content): - new_content = self.transform_sourcecode(content) - new_content = self.transform_caption(new_content) - new_content = self.transform_multiple_newlines(new_content) - return new_content + content = self.transform_code(content) + content = self.transform_caption(content) + content = self.transform_multiple_newlines(content) + return content def import_item(self, item, wordpress_namespace, out_folder=None): """Takes an item from the feed and creates a post file.""" @@ -391,11 +420,10 @@ class CommandImportWordpress(Command, ImportMixin): parsed = urlparse(link) path = unquote(parsed.path.strip('/')) - # In python 2, path is a str. slug requires a unicode - # object. According to wikipedia, unquoted strings will - # usually be UTF8 - if isinstance(path, utils.bytes_str): + try: path = path.decode('utf8') + except AttributeError: + pass # Cut out the base directory. if path.startswith(self.base_dir.strip('/')): @@ -420,7 +448,13 @@ class CommandImportWordpress(Command, ImportMixin): description = get_text_tag(item, 'description', '') post_date = get_text_tag( item, '{{{0}}}post_date'.format(wordpress_namespace), None) - dt = utils.to_datetime(post_date) + try: + dt = utils.to_datetime(post_date) + except ValueError: + dt = datetime.datetime(1970, 1, 1, 0, 0, 0) + LOGGER.error('Malformed date "{0}" in "{1}" [{2}], assuming 1970-01-01 00:00:00 instead.'.format(post_date, title, slug)) + post_date = dt.strftime('%Y-%m-%d %H:%M:%S') + if dt.tzinfo and self.timezone is None: self.timezone = utils.get_tzname(dt) status = get_text_tag( @@ -443,12 +477,20 @@ class CommandImportWordpress(Command, ImportMixin): if text == 'Uncategorized': continue tags.append(text) + self.all_tags.add(text) if '$latex' in content: tags.append('mathjax') + # Find post format if it's there + post_format = 'wp' + format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format'] + if format_tag: + post_format = format_tag[0].getparent().find('{%s}meta_value' % wordpress_namespace).text + if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) + elif content.strip(): # If no content is found, no files are written. self.url_map[link] = (self.context['SITE_URL'] + @@ -475,7 +517,8 @@ class CommandImportWordpress(Command, ImportMixin): out_meta_filename = slug + '.meta' out_content_filename = slug + '.wp' meta_slug = slug - content = self.transform_content(content) + if post_format == 'wp': + content = self.transform_content(content) self.write_metadata(os.path.join(self.output_folder, out_folder, out_meta_filename), title, meta_slug, post_date, description, tags) @@ -510,7 +553,7 @@ def get_text_tag(tag, name, default): if tag is None: return default t = tag.find(name) - if t is not None: + if t is not None and t.text is not None: return t.text else: return default diff --git a/nikola/plugins/command/init.plugin b/nikola/plugins/command/init.plugin index a539f51..850dba9 100644 --- a/nikola/plugins/command/init.plugin +++ b/nikola/plugins/command/init.plugin @@ -4,6 +4,6 @@ Module = init [Documentation] Author = Roberto Alsina -Version = 0.2 +Version = 1.0 Website = http://getnikola.com Description = Create a new site. diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py index a8b60db..7a36894 100644 --- a/nikola/plugins/command/init.py +++ b/nikola/plugins/command/init.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -33,11 +33,13 @@ import textwrap import datetime import unidecode import dateutil.tz +import dateutil.zoneinfo from mako.template import Template from pkg_resources import resource_filename +import tarfile import nikola -from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_RSS_READ_MORE_LINK, LEGAL_VALUES +from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_RSS_READ_MORE_LINK, LEGAL_VALUES, urlsplit, urlunsplit from nikola.plugin_categories import Command from nikola.utils import ask, ask_yesno, get_logger, makedirs, STDERR_HANDLER, load_messages from nikola.packages.tzlocal import get_localzone @@ -48,9 +50,10 @@ LOGGER = get_logger('init', STDERR_HANDLER) SAMPLE_CONF = { 'BLOG_AUTHOR': "Your Name", 'BLOG_TITLE': "Demo Site", - 'SITE_URL': "http://getnikola.com/", + 'SITE_URL': "https://example.com/", 'BLOG_EMAIL': "joe@demo.site", 'BLOG_DESCRIPTION': "This is a demo site for Nikola.", + 'PRETTY_URLS': False, 'DEFAULT_LANG': "en", 'TRANSLATIONS': """{ DEFAULT_LANG: "", @@ -186,7 +189,7 @@ def format_navigation_links(additional_languages, default_lang, messages): pairs.append(f.format('DEFAULT_LANG', '', get_msg(default_lang))) for l in additional_languages: - pairs.append(f.format(json.dumps(l), '/' + l, get_msg(l))) + pairs.append(f.format(json.dumps(l, ensure_ascii=False), '/' + l, get_msg(l))) return u'{{\n{0}\n}}'.format('\n\n'.join(pairs)) @@ -196,11 +199,13 @@ def format_navigation_links(additional_languages, default_lang, messages): def prepare_config(config): """Parse sample config with JSON.""" p = config.copy() - p.update(dict((k, json.dumps(v)) for k, v in p.items() - if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK'))) + p.update(dict((k, json.dumps(v, ensure_ascii=False)) for k, v in p.items() + if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK', 'PRETTY_URLS'))) # READ_MORE_LINKs require some special treatment. p['INDEX_READ_MORE_LINK'] = "'" + p['INDEX_READ_MORE_LINK'].replace("'", "\\'") + "'" p['RSS_READ_MORE_LINK'] = "'" + p['RSS_READ_MORE_LINK'].replace("'", "\\'") + "'" + # json would make that `true` instead of `True` + p['PRETTY_URLS'] = str(p['PRETTY_URLS']) return p @@ -237,14 +242,20 @@ class CommandInit(Command): src = resource_filename('nikola', os.path.join('data', 'samplesite')) shutil.copytree(src, target) - @classmethod - def create_configuration(cls, target): + @staticmethod + def create_configuration(target): template_path = resource_filename('nikola', 'conf.py.in') conf_template = Template(filename=template_path) conf_path = os.path.join(target, 'conf.py') with io.open(conf_path, 'w+', encoding='utf8') as fd: fd.write(conf_template.render(**prepare_config(SAMPLE_CONF))) + @staticmethod + def create_configuration_to_string(): + template_path = resource_filename('nikola', 'conf.py.in') + conf_template = Template(filename=template_path) + return conf_template.render(**prepare_config(SAMPLE_CONF)) + @classmethod def create_empty_site(cls, target): for folder in ('files', 'galleries', 'listings', 'posts', 'stories'): @@ -253,6 +264,39 @@ class CommandInit(Command): @staticmethod def ask_questions(target): """Ask some questions about Nikola.""" + def urlhandler(default, toconf): + answer = ask('Site URL', 'https://example.com/') + try: + answer = answer.decode('utf-8') + except (AttributeError, UnicodeDecodeError): + pass + if not answer.startswith(u'http'): + print(" ERROR: You must specify a protocol (http or https).") + urlhandler(default, toconf) + return + if not answer.endswith('/'): + print(" The URL does not end in '/' -- adding it.") + answer += '/' + + dst_url = urlsplit(answer) + try: + dst_url.netloc.encode('ascii') + except (UnicodeEncodeError, UnicodeDecodeError): + # The IDN contains characters beyond ASCII. We must convert it + # to Punycode. (Issue #1644) + nl = dst_url.netloc.encode('idna') + answer = urlunsplit((dst_url.scheme, + nl, + dst_url.path, + dst_url.query, + dst_url.fragment)) + print(" Converting to Punycode:", answer) + + SAMPLE_CONF['SITE_URL'] = answer + + def prettyhandler(default, toconf): + SAMPLE_CONF['PRETTY_URLS'] = ask_yesno('Enable pretty URLs (/page/ instead of /page.html) that don’t need web server configuration?', default=True) + def lhandler(default, toconf, show_header=True): if show_header: print("We will now ask you to provide the list of languages you want to use.") @@ -297,7 +341,7 @@ class CommandInit(Command): lhandler(default, toconf, show_header=False) def tzhandler(default, toconf): - print("\nPlease choose the correct time zone for your blog. Nikola uses the tz database.") + print("\nPlease choose the correct time zone for your blog. Nikola uses the tz database.") print("You can find your time zone here:") print("http://en.wikipedia.org/wiki/List_of_tz_database_time_zones") print("") @@ -309,12 +353,26 @@ class CommandInit(Command): lz = None answer = ask('Time zone', lz if lz else "UTC") tz = dateutil.tz.gettz(answer) + + if tz is None: + print(" WARNING: Time zone not found. Searching list of time zones for a match.") + zonesfile = tarfile.open(fileobj=dateutil.zoneinfo.getzoneinfofile_stream()) + zonenames = [zone for zone in zonesfile.getnames() if answer.lower() in zone.lower()] + if len(zonenames) == 1: + tz = dateutil.tz.gettz(zonenames[0]) + answer = zonenames[0] + print(" Picking '{0}'.".format(answer)) + elif len(zonenames) > 1: + print(" The following time zones match your query:") + print(' ' + '\n '.join(zonenames)) + continue + if tz is not None: time = datetime.datetime.now(tz).strftime('%H:%M:%S') print(" Current time in {0}: {1}".format(answer, time)) answered = ask_yesno("Use this time zone?", True) else: - print(" ERROR: Time zone not found. Please try again. Time zones are case-sensitive.") + print(" ERROR: No matches found. Please try again.") SAMPLE_CONF['TIMEZONE'] = answer @@ -353,7 +411,8 @@ class CommandInit(Command): ('Site author', 'Nikola Tesla', True, 'BLOG_AUTHOR'), ('Site author\'s e-mail', 'n.tesla@example.com', True, 'BLOG_EMAIL'), ('Site description', 'This is a demo site for Nikola.', True, 'BLOG_DESCRIPTION'), - ('Site URL', 'http://getnikola.com/', True, 'SITE_URL'), + (urlhandler, None, True, True), + (prettyhandler, None, True, True), ('Questions about languages and locales', None, None, None), (lhandler, None, True, True), (tzhandler, None, True, True), @@ -377,6 +436,10 @@ class CommandInit(Command): query(default, toconf) else: answer = ask(query, default) + try: + answer = answer.decode('utf-8') + except (AttributeError, UnicodeDecodeError): + pass if toconf: SAMPLE_CONF[destination] = answer if destination == '!target': @@ -386,7 +449,7 @@ class CommandInit(Command): STORAGE['target'] = answer print("\nThat's it, Nikola is now configured. Make sure to edit conf.py to your liking.") - print("If you are looking for themes and addons, check out http://themes.getnikola.com/ and http://plugins.getnikola.com/.") + print("If you are looking for themes and addons, check out https://themes.getnikola.com/ and https://plugins.getnikola.com/.") print("Have fun!") return STORAGE diff --git a/nikola/plugins/command/install_theme.plugin b/nikola/plugins/command/install_theme.plugin index 84b2623..54a91ff 100644 --- a/nikola/plugins/command/install_theme.plugin +++ b/nikola/plugins/command/install_theme.plugin @@ -4,7 +4,7 @@ Module = install_theme [Documentation] Author = Roberto Alsina -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Install a theme into the current site. diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py index 5397772..4937509 100644 --- a/nikola/plugins/command/install_theme.py +++ b/nikola/plugins/command/install_theme.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -28,42 +28,18 @@ from __future__ import print_function import os import io import json -import shutil +import requests import pygments from pygments.lexers import PythonLexer from pygments.formatters import TerminalFormatter -try: - import requests -except ImportError: - requests = None # NOQA - from nikola.plugin_categories import Command from nikola import utils LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER) -# Stolen from textwrap in Python 3.3.2. -def indent(text, prefix, predicate=None): # NOQA - """Adds 'prefix' to the beginning of selected lines in 'text'. - - If 'predicate' is provided, 'prefix' will only be added to the lines - where 'predicate(line)' is True. If 'predicate' is not provided, - it will default to adding 'prefix' to all non-empty lines that do not - consist solely of whitespace characters. - """ - if predicate is None: - def predicate(line): - return line.strip() - - def prefixed_lines(): - for line in text.splitlines(True): - yield (prefix + line if predicate(line) else line) - return ''.join(prefixed_lines()) - - class CommandInstallTheme(Command): """Install a theme.""" @@ -86,16 +62,21 @@ class CommandInstallTheme(Command): 'long': 'url', 'type': str, 'help': "URL for the theme repository (default: " - "http://themes.getnikola.com/v7/themes.json)", - 'default': 'http://themes.getnikola.com/v7/themes.json' + "https://themes.getnikola.com/v7/themes.json)", + 'default': 'https://themes.getnikola.com/v7/themes.json' + }, + { + 'name': 'getpath', + 'short': 'g', + 'long': 'get-path', + 'type': bool, + 'default': False, + 'help': "Print the path for installed theme", }, ] def _execute(self, options, args): """Install theme into current site.""" - if requests is None: - utils.req_missing(['requests'], 'install themes') - listing = options['list'] url = options['url'] if args: @@ -103,6 +84,14 @@ class CommandInstallTheme(Command): else: name = None + if options['getpath'] and name: + path = utils.get_theme_path(name) + if path: + print(path) + else: + print('not installed') + return 0 + if name is None and not listing: LOGGER.error("This command needs either a theme name or the -l option.") return False @@ -135,36 +124,31 @@ class CommandInstallTheme(Command): def do_install(self, name, data): if name in data: utils.makedirs(self.output_dir) - LOGGER.info('Downloading: ' + data[name]) + LOGGER.info("Downloading '{0}'".format(data[name])) zip_file = io.BytesIO() zip_file.write(requests.get(data[name]).content) - LOGGER.info('Extracting: {0} into themes'.format(name)) + LOGGER.info("Extracting '{0}' into themes/".format(name)) utils.extract_all(zip_file) - dest_path = os.path.join('themes', name) + dest_path = os.path.join(self.output_dir, name) else: + dest_path = os.path.join(self.output_dir, name) try: theme_path = utils.get_theme_path(name) - except: - LOGGER.error("Can't find theme " + name) - return False + LOGGER.error("Theme '{0}' is already installed in {1}".format(name, theme_path)) + except Exception: + LOGGER.error("Can't find theme {0}".format(name)) - utils.makedirs(self.output_dir) - dest_path = os.path.join(self.output_dir, name) - if os.path.exists(dest_path): - LOGGER.error("{0} is already installed".format(name)) - return False + return False - LOGGER.info('Copying {0} into themes'.format(theme_path)) - shutil.copytree(theme_path, dest_path) confpypath = os.path.join(dest_path, 'conf.py.sample') if os.path.exists(confpypath): LOGGER.notice('This theme has a sample config file. Integrate it with yours in order to make this theme work!') print('Contents of the conf.py.sample file:\n') with io.open(confpypath, 'r', encoding='utf-8') as fh: if self.site.colorful: - print(indent(pygments.highlight( + print(utils.indent(pygments.highlight( fh.read(), PythonLexer(), TerminalFormatter()), 4 * ' ')) else: - print(indent(fh.read(), 4 * ' ')) + print(utils.indent(fh.read(), 4 * ' ')) return True diff --git a/nikola/plugins/command/new_page.plugin b/nikola/plugins/command/new_page.plugin index 1f1c84c..f078dd6 100644 --- a/nikola/plugins/command/new_page.plugin +++ b/nikola/plugins/command/new_page.plugin @@ -4,6 +4,6 @@ Module = new_page [Documentation] Author = Roberto Alsina, Chris Warrick -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Create a new page. diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index f07ba39..39a85bd 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -45,6 +45,14 @@ class CommandNewPage(Command): 'help': 'Title for the page.' }, { + 'name': 'author', + 'short': 'a', + 'long': 'author', + 'type': str, + 'default': '', + 'help': 'Author of the post.' + }, + { 'name': 'onefile', 'short': '1', 'type': bool, @@ -71,13 +79,29 @@ class CommandNewPage(Command): 'long': 'format', 'type': str, 'default': '', - 'help': 'Markup format for the page, one of rest, markdown, wiki, ' - 'bbcode, html, textile, txt2tags', + 'help': 'Markup format for the page (use --available-formats for list)', + }, + { + 'name': 'available-formats', + 'short': 'F', + 'long': 'available-formats', + 'type': bool, + 'default': False, + 'help': 'List all available input formats' + }, + { + 'name': 'import', + 'short': 'i', + 'long': 'import', + 'type': str, + 'default': '', + 'help': 'Import an existing file instead of creating a placeholder' }, ] def _execute(self, options, args): """Create a new page.""" + # Defaults for some values that don’t apply to pages and the is_page option (duh!) options['tags'] = '' options['schedule'] = False options['is_page'] = True diff --git a/nikola/plugins/command/new_post.plugin b/nikola/plugins/command/new_post.plugin index ec35c35..fec4b1d 100644 --- a/nikola/plugins/command/new_post.plugin +++ b/nikola/plugins/command/new_post.plugin @@ -4,7 +4,7 @@ Module = new_post [Documentation] Author = Roberto Alsina -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Create a new post. diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index 24c09d0..5141c7e 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -30,6 +30,7 @@ import datetime import os import sys import subprocess +import operator from blinker import signal import dateutil.tz @@ -37,12 +38,13 @@ import dateutil.tz from nikola.plugin_categories import Command from nikola import utils +COMPILERS_DOC_LINK = 'https://getnikola.com/handbook.html#configuring-other-input-formats' POSTLOGGER = utils.get_logger('new_post', utils.STDERR_HANDLER) PAGELOGGER = utils.get_logger('new_page', utils.STDERR_HANDLER) LOGGER = POSTLOGGER -def filter_post_pages(compiler, is_post, compilers, post_pages): +def filter_post_pages(compiler, is_post, compilers, post_pages, compiler_objs, compilers_raw): """Given a compiler ("markdown", "rest"), and whether it's meant for a post or a page, and compilers, return the correct entry from post_pages.""" @@ -51,7 +53,15 @@ def filter_post_pages(compiler, is_post, compilers, post_pages): filtered = [entry for entry in post_pages if entry[3] == is_post] # These are the extensions supported by the required format - extensions = compilers[compiler] + extensions = compilers.get(compiler) + if extensions is None: + if compiler in compiler_objs: + LOGGER.error("There is a {0} compiler available, but it's not set in your COMPILERS option.".format(compiler)) + LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) + else: + LOGGER.error('Unknown format {0}'.format(compiler)) + print_compilers(compilers_raw, post_pages, compiler_objs) + return False # Throw away the post_pages with the wrong extensions filtered = [entry for entry in filtered if any([ext in entry[0] for ext in @@ -59,13 +69,77 @@ def filter_post_pages(compiler, is_post, compilers, post_pages): if not filtered: type_name = "post" if is_post else "page" - raise Exception("Can't find a way, using your configuration, to create " - "a {0} in format {1}. You may want to tweak " - "COMPILERS or {2}S in conf.py".format( - type_name, compiler, type_name.upper())) + LOGGER.error("Can't find a way, using your configuration, to create " + "a {0} in format {1}. You may want to tweak " + "COMPILERS or {2}S in conf.py".format( + type_name, compiler, type_name.upper())) + LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) + + return False return filtered[0] +def print_compilers(compilers_raw, post_pages, compiler_objs): + """ + List all available compilers in a human-friendly format. + + :param compilers_raw: The compilers dict, mapping compiler names to tuples of extensions + :param post_pages: The post_pages structure + :param compilers_objs: Compiler objects + """ + + # We use compilers_raw, because the normal dict can contain + # garbage coming from the translation candidate implementation. + # Entries are in format: (name, extensions, used_in_post_pages) + parsed_compilers = {'used': [], 'unused': [], 'disabled': []} + + for compiler_name, compiler_obj in compiler_objs.items(): + fname = compiler_obj.friendly_name or compiler_name + if compiler_name not in compilers_raw: + parsed_compilers['disabled'].append((compiler_name, fname, (), False)) + else: + # stolen from filter_post_pages + extensions = compilers_raw[compiler_name] + filtered = [entry for entry in post_pages if any( + [ext in entry[0] for ext in extensions])] + if filtered: + parsed_compilers['used'].append((compiler_name, fname, extensions, True)) + else: + parsed_compilers['unused'].append((compiler_name, fname, extensions, False)) + + # Sort compilers alphabetically by name, just so it’s prettier (and + # deterministic) + parsed_compilers['used'].sort(key=operator.itemgetter(0)) + parsed_compilers['unused'].sort(key=operator.itemgetter(0)) + parsed_compilers['disabled'].sort(key=operator.itemgetter(0)) + + # We also group the compilers by status for readability. + parsed_list = parsed_compilers['used'] + parsed_compilers['unused'] + parsed_compilers['disabled'] + + print("Available input formats:\n") + + name_width = max([len(i[0]) for i in parsed_list] + [4]) # 4 == len('NAME') + fname_width = max([len(i[1]) for i in parsed_list] + [11]) # 11 == len('DESCRIPTION') + + print((' {0:<' + str(name_width) + '} {1:<' + str(fname_width) + '} EXTENSIONS\n').format('NAME', 'DESCRIPTION')) + + for name, fname, extensions, used in parsed_list: + flag = ' ' if used else '!' + flag = flag if extensions else '~' + + extensions = ', '.join(extensions) if extensions else '(disabled: not in COMPILERS)' + + print(('{flag}{name:<' + str(name_width) + '} {fname:<' + str(fname_width) + '} {extensions}').format(flag=flag, name=name, fname=fname, extensions=extensions)) + + print(""" +More compilers are available in the Plugins Index. + +Compilers marked with ! and ~ require additional configuration: + ! not in the PAGES/POSTS tuples (unused) + ~ not in the COMPILERS dict (disabled) +Read more: {0}""".format(COMPILERS_DOC_LINK)) + + def get_default_compiler(is_post, compilers, post_pages): """Given compilers and post_pages, return a reasonable default compiler for this kind of post/page. @@ -116,7 +190,7 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): rrule = None # NOQA if schedule and rrule and rule: try: - rule_ = rrule.rrulestr(rule, dtstart=last_date) + rule_ = rrule.rrulestr(rule, dtstart=last_date or date) except Exception: LOGGER.error('Unable to parse rule string, using current time.') else: @@ -161,6 +235,14 @@ class CommandNewPost(Command): 'help': 'Title for the post.' }, { + 'name': 'author', + 'short': 'a', + 'long': 'author', + 'type': str, + 'default': '', + 'help': 'Author of the post.' + }, + { 'name': 'tags', 'long': 'tags', 'type': str, @@ -194,8 +276,15 @@ class CommandNewPost(Command): 'long': 'format', 'type': str, 'default': '', - 'help': 'Markup format for the post, one of rest, markdown, wiki, ' - 'bbcode, html, textile, txt2tags', + 'help': 'Markup format for the post (use --available-formats for list)', + }, + { + 'name': 'available-formats', + 'short': 'F', + 'long': 'available-formats', + 'type': bool, + 'default': False, + 'help': 'List all available input formats' }, { 'name': 'schedule', @@ -204,6 +293,14 @@ class CommandNewPost(Command): 'default': False, 'help': 'Schedule the post based on recurrence rule' }, + { + 'name': 'import', + 'short': 'i', + 'long': 'import', + 'type': str, + 'default': '', + 'help': 'Import an existing file instead of creating a placeholder' + }, ] @@ -228,9 +325,16 @@ class CommandNewPost(Command): is_post = not is_page content_type = 'page' if is_page else 'post' title = options['title'] or None + author = options['author'] or '' tags = options['tags'] onefile = options['onefile'] twofile = options['twofile'] + import_file = options['import'] + wants_available = options['available-formats'] + + if wants_available: + print_compilers(self.site.config['_COMPILERS_RAW'], self.site.config['post_pages'], self.site.compilers) + return if is_page: LOGGER = PAGELOGGER @@ -243,6 +347,10 @@ class CommandNewPost(Command): onefile = self.site.config.get('ONE_FILE_POSTS', True) content_format = options['content_format'] + content_subformat = None + + if "@" in content_format: + content_format, content_subformat = content_format.split("@") if not content_format: # Issue #400 content_format = get_default_compiler( @@ -251,7 +359,8 @@ class CommandNewPost(Command): self.site.config['post_pages']) if content_format not in compiler_names: - LOGGER.error("Unknown {0} format {1}".format(content_type, content_format)) + LOGGER.error("Unknown {0} format {1}, maybe you need to install a plugin?".format(content_type, content_format)) + print_compilers(self.site.config['_COMPILERS_RAW'], self.site.config['post_pages'], self.site.compilers) return compiler_plugin = self.site.plugin_manager.getPluginByName( content_format, "PageCompiler").plugin_object @@ -259,10 +368,19 @@ class CommandNewPost(Command): # Guess where we should put this entry = filter_post_pages(content_format, is_post, self.site.config['COMPILERS'], - self.site.config['post_pages']) + self.site.config['post_pages'], + self.site.compilers, + self.site.config['_COMPILERS_RAW']) + + if entry is False: + return 1 - print("Creating New {0}".format(content_type.title())) - print("-----------------\n") + if import_file: + print("Importing Existing {xx}".format(xx=content_type.title())) + print("-----------------------\n") + else: + print("Creating New {xx}".format(xx=content_type.title())) + print("-----------------\n") if title is not None: print("Title:", title) else: @@ -272,7 +390,7 @@ class CommandNewPost(Command): if isinstance(title, utils.bytes_str): try: title = title.decode(sys.stdin.encoding) - except AttributeError: # for tests + except (AttributeError, TypeError): # for tests title = title.decode('utf-8') title = title.strip() @@ -282,9 +400,16 @@ class CommandNewPost(Command): if isinstance(path, utils.bytes_str): try: path = path.decode(sys.stdin.encoding) - except AttributeError: # for tests + except (AttributeError, TypeError): # for tests path = path.decode('utf-8') slug = utils.slugify(os.path.splitext(os.path.basename(path))[0]) + + if isinstance(author, utils.bytes_str): + try: + author = author.decode(sys.stdin.encoding) + except (AttributeError, TypeError): # for tests + author = author.decode('utf-8') + # Calculate the date to use for the content schedule = options['schedule'] or self.site.config['SCHEDULE_ALL'] rule = self.site.config['SCHEDULE_RULE'] @@ -308,23 +433,46 @@ class CommandNewPost(Command): if not path: txt_path = os.path.join(output_path, slug + suffix) else: - txt_path = path + txt_path = os.path.join(self.site.original_cwd, path) if (not onefile and os.path.isfile(meta_path)) or \ os.path.isfile(txt_path): + + # Emit an event when a post exists + event = dict(path=txt_path) + if not onefile: # write metadata file + event['meta_path'] = meta_path + signal('existing_' + content_type).send(self, **event) + LOGGER.error("The title already exists!") - exit() + return 8 d_name = os.path.dirname(txt_path) utils.makedirs(d_name) - metadata = self.site.config['ADDITIONAL_METADATA'] + metadata = {} + if author: + metadata['author'] = author + metadata.update(self.site.config['ADDITIONAL_METADATA']) + data.update(metadata) + + # ipynb plugin needs the ipython kernel info. We get the kernel name + # from the content_subformat and pass it to the compiler in the metadata + if content_format == "ipynb" and content_subformat is not None: + metadata["ipython_kernel"] = content_subformat # Override onefile if not really supported. if not compiler_plugin.supports_onefile and onefile: onefile = False LOGGER.warn('This compiler does not support one-file posts.') - content = "Write your {0} here.".format('page' if is_page else 'post') + if import_file: + with io.open(import_file, 'r', encoding='utf-8') as fh: + content = fh.read() + else: + if is_page: + content = self.site.MESSAGES[self.site.default_lang]["Write your page here."] + else: + content = self.site.MESSAGES[self.site.default_lang]["Write your post here."] compiler_plugin.create_post( txt_path, content=content, onefile=onefile, title=title, slug=slug, date=date, tags=tags, is_page=is_page, **metadata) diff --git a/nikola/plugins/command/orphans.plugin b/nikola/plugins/command/orphans.plugin index 408578b..f491eaf 100644 --- a/nikola/plugins/command/orphans.plugin +++ b/nikola/plugins/command/orphans.plugin @@ -4,7 +4,7 @@ Module = orphans [Documentation] Author = Roberto Alsina, Chris Warrick -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = List all orphans diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py index ff114b4..f550e17 100644 --- a/nikola/plugins/command/orphans.py +++ b/nikola/plugins/command/orphans.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina, Chris Warrick and others. +# Copyright © 2012-2015 Roberto Alsina, Chris Warrick and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated diff --git a/nikola/plugins/command/plugin.plugin b/nikola/plugins/command/plugin.plugin index d2bca92..2815caa 100644 --- a/nikola/plugins/command/plugin.plugin +++ b/nikola/plugins/command/plugin.plugin @@ -4,7 +4,7 @@ Module = plugin [Documentation] Author = Roberto Alsina and Chris Warrick -Version = 0.2 +Version = 1.0 Website = http://getnikola.com Description = Manage Nikola plugins diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index 71901b8..56eb1d7 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -30,41 +30,18 @@ import os import shutil import subprocess import sys +import requests import pygments from pygments.lexers import PythonLexer from pygments.formatters import TerminalFormatter -try: - import requests -except ImportError: - requests = None # NOQA - from nikola.plugin_categories import Command from nikola import utils LOGGER = utils.get_logger('plugin', utils.STDERR_HANDLER) -# Stolen from textwrap in Python 3.3.2. -def indent(text, prefix, predicate=None): # NOQA - """Adds 'prefix' to the beginning of selected lines in 'text'. - - If 'predicate' is provided, 'prefix' will only be added to the lines - where 'predicate(line)' is True. If 'predicate' is not provided, - it will default to adding 'prefix' to all non-empty lines that do not - consist solely of whitespace characters. - """ - if predicate is None: - def predicate(line): - return line.strip() - - def prefixed_lines(): - for line in text.splitlines(True): - yield (prefix + line if predicate(line) else line) - return ''.join(prefixed_lines()) - - class CommandPlugin(Command): """Manage plugins.""" @@ -105,8 +82,8 @@ class CommandPlugin(Command): 'long': 'url', 'type': str, 'help': "URL for the plugin repository (default: " - "http://plugins.getnikola.com/v7/plugins.json)", - 'default': 'http://plugins.getnikola.com/v7/plugins.json' + "https://plugins.getnikola.com/v7/plugins.json)", + 'default': 'https://plugins.getnikola.com/v7/plugins.json' }, { 'name': 'user', @@ -258,7 +235,7 @@ class CommandPlugin(Command): LOGGER.error('Could not install the dependencies.') print('Contents of the requirements.txt file:\n') with io.open(reqpath, 'r', encoding='utf-8') as fh: - print(indent(fh.read(), 4 * ' ')) + print(utils.indent(fh.read(), 4 * ' ')) print('You have to install those yourself or through a ' 'package manager.') else: @@ -272,8 +249,8 @@ class CommandPlugin(Command): with io.open(reqnpypath, 'r', encoding='utf-8') as fh: for l in fh.readlines(): i, j = l.split('::') - print(indent(i.strip(), 4 * ' ')) - print(indent(j.strip(), 8 * ' ')) + print(utils.indent(i.strip(), 4 * ' ')) + print(utils.indent(j.strip(), 8 * ' ')) print() print('You have to install those yourself or through a package ' @@ -284,11 +261,11 @@ class CommandPlugin(Command): print('Contents of the conf.py.sample file:\n') with io.open(confpypath, 'r', encoding='utf-8') as fh: if self.site.colorful: - print(indent(pygments.highlight( + print(utils.indent(pygments.highlight( fh.read(), PythonLexer(), TerminalFormatter()), 4 * ' ')) else: - print(indent(fh.read(), 4 * ' ')) + print(utils.indent(fh.read(), 4 * ' ')) return True def do_uninstall(self, name): @@ -311,8 +288,6 @@ class CommandPlugin(Command): return False def get_json(self, url): - if requests is None: - utils.req_missing(['requests'], 'install or list available plugins', python=True, optional=False) if self.json is None: self.json = requests.get(url).json() return self.json diff --git a/nikola/plugins/command/rst2html.plugin b/nikola/plugins/command/rst2html.plugin new file mode 100644 index 0000000..0d0d3b0 --- /dev/null +++ b/nikola/plugins/command/rst2html.plugin @@ -0,0 +1,9 @@ +[Core] +Name = rst2html +Module = rst2html + +[Documentation] +Author = Chris Warrick +Version = 1.0 +Website = http://getnikola.com +Description = Compile reStructuredText to HTML using the Nikola architecture diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py new file mode 100644 index 0000000..342aaeb --- /dev/null +++ b/nikola/plugins/command/rst2html/__init__.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2015 Chris Warrick and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import unicode_literals, print_function + +import io +import lxml.html +from pkg_resources import resource_filename +from mako.template import Template +from nikola.plugin_categories import Command + + +class CommandRst2Html(Command): + """Compile reStructuredText to HTML, using Nikola architecture.""" + + name = "rst2html" + doc_usage = "infile" + doc_purpose = "compile reStructuredText to HTML files" + needs_config = False + + def _execute(self, options, args): + """Compile reStructuredText to standalone HTML files.""" + compiler = self.site.plugin_manager.getPluginByName('rest', 'PageCompiler').plugin_object + if len(args) != 1: + print("This command takes only one argument (input file name).") + return 2 + source = args[0] + with io.open(source, "r", encoding="utf8") as in_file: + data = in_file.read() + output, error_level, deps = compiler.compile_html_string(data, source, True) + + rstcss_path = resource_filename('nikola', 'data/themes/base/assets/css/rst.css') + with io.open(rstcss_path, "r", encoding="utf8") as fh: + rstcss = fh.read() + + template_path = resource_filename('nikola', 'plugins/command/rst2html/rst2html.tmpl') + template = Template(filename=template_path) + template_output = template.render(rstcss=rstcss, output=output) + parser = lxml.html.HTMLParser(remove_blank_text=True) + doc = lxml.html.document_fromstring(template_output, parser) + html = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True) + print(html) + if error_level < 3: + return 0 + else: + return 1 diff --git a/nikola/plugins/command/rst2html/rst2html.tmpl b/nikola/plugins/command/rst2html/rst2html.tmpl new file mode 100644 index 0000000..5a892ea --- /dev/null +++ b/nikola/plugins/command/rst2html/rst2html.tmpl @@ -0,0 +1,13 @@ +<!DOCTYPE html> +<html> +<head> +<meta charset="utf-8"> +<style class="text/css"> +${rstcss} +</style> +</head> + +<body> +${output} +</body> +</html> diff --git a/nikola/plugins/command/serve.plugin b/nikola/plugins/command/serve.plugin index e663cc6..0c1176d 100644 --- a/nikola/plugins/command/serve.plugin +++ b/nikola/plugins/command/serve.plugin @@ -4,7 +4,7 @@ Module = serve [Documentation] Author = Roberto Alsina -Version = 0.1 +Version = 1.0 Website = http://getnikola.com Description = Start test server. diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py index de4f6e2..0e4d01f 100644 --- a/nikola/plugins/command/serve.py +++ b/nikola/plugins/command/serve.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -26,6 +26,7 @@ from __future__ import print_function import os +import socket import webbrowser try: from BaseHTTPServer import HTTPServer @@ -38,6 +39,11 @@ from nikola.plugin_categories import Command from nikola.utils import get_logger +class IPv6Server(HTTPServer): + """An IPv6 HTTPServer.""" + address_family = socket.AF_INET6 + + class CommandServe(Command): """Start test server.""" @@ -53,7 +59,7 @@ class CommandServe(Command): 'long': 'port', 'default': 8000, 'type': int, - 'help': 'Port nummber (default: 8000)', + 'help': 'Port number (default: 8000)', }, { 'name': 'address', @@ -61,7 +67,7 @@ class CommandServe(Command): 'long': 'address', 'type': str, 'default': '', - 'help': 'Address to bind (default: 0.0.0.0 – all local interfaces)', + 'help': 'Address to bind (default: 0.0.0.0 – all local IPv4 interfaces)', }, { 'name': 'browser', @@ -70,7 +76,15 @@ class CommandServe(Command): 'type': bool, 'default': False, 'help': 'Open the test server in a web browser', - } + }, + { + 'name': 'ipv6', + 'short': '6', + 'long': 'ipv6', + 'type': bool, + 'default': False, + 'help': 'Use IPv6', + }, ) def _execute(self, options, args): @@ -81,19 +95,33 @@ class CommandServe(Command): self.logger.error("Missing '{0}' folder?".format(out_dir)) else: os.chdir(out_dir) - httpd = HTTPServer((options['address'], options['port']), - OurHTTPRequestHandler) + if '[' in options['address']: + options['address'] = options['address'].strip('[').strip(']') + ipv6 = True + OurHTTP = IPv6Server + elif options['ipv6']: + ipv6 = True + OurHTTP = IPv6Server + else: + ipv6 = False + OurHTTP = HTTPServer + + httpd = OurHTTP((options['address'], options['port']), + OurHTTPRequestHandler) sa = httpd.socket.getsockname() self.logger.info("Serving HTTP on {0} port {1}...".format(*sa)) if options['browser']: - server_url = "http://{0}:{1}/".format(*sa) + if ipv6: + server_url = "http://[{0}]:{1}/".format(*sa) + else: + server_url = "http://{0}:{1}/".format(*sa) self.logger.info("Opening {0} in the default web browser...".format(server_url)) webbrowser.open(server_url) try: httpd.serve_forever() except KeyboardInterrupt: self.logger.info("Server is shutting down.") - exit(130) + return 130 class OurHTTPRequestHandler(SimpleHTTPRequestHandler): diff --git a/nikola/plugins/command/status.plugin b/nikola/plugins/command/status.plugin new file mode 100644 index 0000000..e02da8b --- /dev/null +++ b/nikola/plugins/command/status.plugin @@ -0,0 +1,9 @@ +[Core] +Name = status +Module = status + +[Documentation] +Author = Daniel Aleksandersen +Version = 1.0 +Website = https://getnikola.com +Description = Site status diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py new file mode 100644 index 0000000..b8a6a60 --- /dev/null +++ b/nikola/plugins/command/status.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2012-2015 Roberto Alsina and others. + +# Permission is hereby granted, free of charge, to any +# person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the +# Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the +# Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice +# shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function +import io +import os +from datetime import datetime +from dateutil.tz import gettz, tzlocal + +from nikola.plugin_categories import Command + + +class CommandDeploy(Command): + """ Site status. """ + name = "status" + + doc_purpose = "display site status" + doc_description = "Show information about the posts and site deployment." + doc_usage = '[-l|--list-drafts] [-m|--list-modified] [-s|--list-scheduled]' + logger = None + cmd_options = [ + { + 'name': 'list_drafts', + 'short': 'd', + 'long': 'list-drafts', + 'type': bool, + 'default': False, + 'help': 'List all drafts', + }, + { + 'name': 'list_modified', + 'short': 'm', + 'long': 'list-modified', + 'type': bool, + 'default': False, + 'help': 'List all modified files since last deployment', + }, + { + 'name': 'list_scheduled', + 'short': 's', + 'long': 'list-scheduled', + 'type': bool, + 'default': False, + 'help': 'List all scheduled posts', + }, + ] + + def _execute(self, options, args): + + self.site.scan_posts() + + timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") + + last_deploy = None + + try: + with io.open(timestamp_path, "r", encoding="utf8") as inf: + last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f") + last_deploy_offset = datetime.utcnow() - last_deploy + except (IOError, Exception): + print("It does not seem like you’ve ever deployed the site (or cache missing).") + + if last_deploy: + + fmod_since_deployment = [] + for root, dirs, files in os.walk(self.site.config["OUTPUT_FOLDER"], followlinks=True): + if not dirs and not files: + continue + for fname in files: + fpath = os.path.join(root, fname) + fmodtime = datetime.fromtimestamp(os.stat(fpath).st_mtime) + if fmodtime.replace(tzinfo=tzlocal()) > last_deploy.replace(tzinfo=gettz("UTC")).astimezone(tz=tzlocal()): + fmod_since_deployment.append(fpath) + + if len(fmod_since_deployment) > 0: + print("{0} output files modified since last deployment {1} ago.".format(str(len(fmod_since_deployment)), self.human_time(last_deploy_offset))) + if options['list_modified']: + for fpath in fmod_since_deployment: + print("Modified: '{0}'".format(fpath)) + else: + print("Last deployment {0} ago.".format(self.human_time(last_deploy_offset))) + + now = datetime.utcnow().replace(tzinfo=gettz("UTC")) + + posts_count = len(self.site.all_posts) + + # find all drafts + posts_drafts = [post for post in self.site.all_posts if post.is_draft] + posts_drafts = sorted(posts_drafts, key=lambda post: post.source_path) + + # find all scheduled posts with offset from now until publishing time + posts_scheduled = [(post.date - now, post) for post in self.site.all_posts if post.publish_later] + posts_scheduled = sorted(posts_scheduled, key=lambda offset_post: (offset_post[0], offset_post[1].source_path)) + + if len(posts_scheduled) > 0: + if options['list_scheduled']: + for offset, post in posts_scheduled: + print("Scheduled: '{1}' ({2}; source: {3}) in {0}".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path)) + else: + offset, post = posts_scheduled[0] + print("{0} to next scheduled post ('{1}'; {2}; source: {3}).".format(self.human_time(offset), post.meta('title'), post.permalink(), post.source_path)) + if options['list_drafts']: + for post in posts_drafts: + print("Draft: '{0}' ({1}; source: {2})".format(post.meta('title'), post.permalink(), post.source_path)) + print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts))) + + def human_time(self, dt): + days = dt.days + hours = dt.seconds / 60 // 60 + minutes = dt.seconds / 60 - (hours * 60) + if days > 0: + return "{0:.0f} days and {1:.0f} hours".format(days, hours) + elif hours > 0: + return "{0:.0f} hours and {1:.0f} minutes".format(hours, minutes) + elif minutes: + return "{0:.0f} minutes".format(minutes) + return False diff --git a/nikola/plugins/command/version.plugin b/nikola/plugins/command/version.plugin index 3c1ae95..a3f58e8 100644 --- a/nikola/plugins/command/version.plugin +++ b/nikola/plugins/command/version.plugin @@ -4,6 +4,6 @@ Module = version [Documentation] Author = Roberto Alsina -Version = 0.2 +Version = 1.0 Website = http://getnikola.com Description = Show nikola version diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py index 9b42423..b6520d7 100644 --- a/nikola/plugins/command/version.py +++ b/nikola/plugins/command/version.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2014 Roberto Alsina and others. +# Copyright © 2012-2015 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -26,19 +26,42 @@ from __future__ import print_function +import lxml +import requests + from nikola.plugin_categories import Command from nikola import __version__ +URL = 'https://pypi.python.org/pypi?:action=doap&name=Nikola' + class CommandVersion(Command): """Print the version.""" name = "version" - doc_usage = "" + doc_usage = "[--check]" needs_config = False doc_purpose = "print the Nikola version number" + cmd_options = [ + { + 'name': 'check', + 'long': 'check', + 'short': '', + 'default': False, + 'type': bool, + 'help': "Check for new versions.", + } + ] def _execute(self, options={}, args=None): """Print the version number.""" print("Nikola v" + __version__) + if options.get('check'): + data = requests.get(URL).text + doc = lxml.etree.fromstring(data.encode('utf8')) + revision = doc.findall('*//{http://usefulinc.com/ns/doap#}revision')[0].text + if revision == __version__: + print("Nikola is up-to-date") + else: + print("The latest version of Nikola is v{0} -- please upgrade using `pip install --upgrade Nikola=={0}` or your system package manager".format(revision)) |