diff options
Diffstat (limited to 'nikola/plugins/command')
35 files changed, 1281 insertions, 461 deletions
diff --git a/nikola/plugins/command/__init__.py b/nikola/plugins/command/__init__.py index a1d17a6..2aa5267 100644 --- a/nikola/plugins/command/__init__.py +++ b/nikola/plugins/command/__init__.py @@ -23,3 +23,5 @@ # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Commands for Nikola.""" diff --git a/nikola/plugins/command/auto.plugin b/nikola/plugins/command/auto.plugin index a1c6820..3e2b17d 100644 --- a/nikola/plugins/command/auto.plugin +++ b/nikola/plugins/command/auto.plugin @@ -1,9 +1,13 @@ [Core] -Name = auto -Module = auto +name = auto +module = auto [Documentation] -Author = Roberto Alsina -Version = 2.1.0 -Website = http://getnikola.com -Description = Automatically detect site changes, rebuild and optionally refresh a browser. +author = Roberto Alsina +version = 2.1.0 +website = http://getnikola.com +description = Automatically detect site changes, rebuild and optionally refresh a browser. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py index c25ef8a..71f9624 100644 --- a/nikola/plugins/command/auto/__init__.py +++ b/nikola/plugins/command/auto/__init__.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Automatic rebuilds for Nikola.""" + from __future__ import print_function import json @@ -31,10 +33,13 @@ import mimetypes import os import re import subprocess +import sys +import time try: from urlparse import urlparse + from urllib2 import unquote except ImportError: - from urllib.parse import urlparse # NOQA + from urllib.parse import urlparse, unquote # NOQA import webbrowser from wsgiref.simple_server import make_server import wsgiref.util @@ -42,7 +47,7 @@ import wsgiref.util from blinker import signal try: from ws4py.websocket import WebSocket - from ws4py.server.wsgirefserver import WSGIServer, WebSocketWSGIRequestHandler + from ws4py.server.wsgirefserver import WSGIServer, WebSocketWSGIRequestHandler, WebSocketWSGIHandler from ws4py.server.wsgiutils import WebSocketWSGIApplication from ws4py.messaging import TextMessage except ImportError: @@ -58,7 +63,7 @@ except ImportError: from nikola.plugin_categories import Command -from nikola.utils import req_missing, get_logger, get_theme_path +from nikola.utils import req_missing, get_logger, get_theme_path, STDERR_HANDLER LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js') error_signal = signal('error') refresh_signal = signal('refresh') @@ -74,9 +79,12 @@ ERROR {} class CommandAuto(Command): - """Start debugging console.""" + + """Automatic rebuilds for Nikola.""" + name = "auto" logger = None + has_server = True doc_purpose = "builds and serves a site; automatically detects site changes, rebuilds, and optionally refreshes a browser" cmd_options = [ { @@ -100,7 +108,7 @@ class CommandAuto(Command): 'short': 'b', 'long': 'browser', 'type': bool, - 'help': 'Start a web browser.', + 'help': 'Start a web browser', 'default': False, }, { @@ -111,12 +119,18 @@ class CommandAuto(Command): 'type': bool, 'help': 'Use IPv6', }, + { + 'name': 'no-server', + 'long': 'no-server', + 'default': False, + 'type': bool, + 'help': 'Disable the server, automate rebuilds only' + }, ] def _execute(self, options, args): """Start the watcher.""" - - self.logger = get_logger('auto', self.site.loghandlers) + self.logger = get_logger('auto', STDERR_HANDLER) LRSocket.logger = self.logger if WebSocket is object and watchdog is None: @@ -166,10 +180,14 @@ class CommandAuto(Command): host = options['address'].strip('[').strip(']') or dhost + # Server can be disabled (Issue #1883) + self.has_server = not options['no-server'] + # Instantiate global observer observer = Observer() - # Watch output folders and trigger reloads - observer.schedule(OurWatchHandler(self.do_refresh), out_folder, recursive=True) + if self.has_server: + # Watch output folders and trigger reloads + observer.schedule(OurWatchHandler(self.do_refresh), out_folder, recursive=True) # Watch input folders and trigger rebuilds for p in watched: @@ -181,101 +199,155 @@ class CommandAuto(Command): _conf_dn = os.path.dirname(_conf_fn) observer.schedule(ConfigWatchHandler(_conf_fn, self.do_rebuild), _conf_dn, recursive=False) - observer.start() + try: + self.logger.info("Watching files for changes...") + observer.start() + except KeyboardInterrupt: + pass parent = self class Mixed(WebSocketWSGIApplication): - """A class that supports WS and HTTP protocols in the same port.""" + + """A class that supports WS and HTTP protocols on the same port.""" + def __call__(self, environ, start_response): if environ.get('HTTP_UPGRADE') is None: return parent.serve_static(environ, start_response) return super(Mixed, self).__call__(environ, start_response) - ws = make_server( - host, port, server_class=WSGIServer, - handler_class=WebSocketWSGIRequestHandler, - app=Mixed(handler_cls=LRSocket) - ) - ws.initialize_websockets_manager() - self.logger.info("Serving HTTP on {0} port {1}...".format(host, port)) - if browser: - if options['ipv6'] or '::' in host: - server_url = "http://[{0}]:{1}/".format(host, port) - else: - server_url = "http://{0}:{1}/".format(host, port) - - self.logger.info("Opening {0} in the default web browser...".format(server_url)) - # Yes, this is racy - webbrowser.open('http://{0}:{1}'.format(host, port)) - - try: - ws.serve_forever() - except KeyboardInterrupt: - self.logger.info("Server is shutting down.") - observer.stop() - observer.join() + if self.has_server: + ws = make_server( + host, port, server_class=WSGIServer, + handler_class=WebSocketWSGIRequestHandler, + app=Mixed(handler_cls=LRSocket) + ) + ws.initialize_websockets_manager() + self.logger.info("Serving HTTP on {0} port {1}...".format(host, port)) + if browser: + if options['ipv6'] or '::' in host: + server_url = "http://[{0}]:{1}/".format(host, port) + else: + server_url = "http://{0}:{1}/".format(host, port) + + self.logger.info("Opening {0} in the default web browser...".format(server_url)) + # Yes, this is racy + webbrowser.open('http://{0}:{1}'.format(host, port)) + + try: + ws.serve_forever() + except KeyboardInterrupt: + self.logger.info("Server is shutting down.") + # This is a hack, but something is locking up in a futex + # and exit() doesn't work. + os.kill(os.getpid(), 15) + else: + # Workaround: can’t have nothing running (instant exit) + # but also can’t join threads (no way to exit) + # The joys of threading. + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + self.logger.info("Shutting down.") + # This is a hack, but something is locking up in a futex + # and exit() doesn't work. + os.kill(os.getpid(), 15) def do_rebuild(self, event): - self.logger.info('REBUILDING SITE (from {0})'.format(event.src_path)) + """Rebuild the site.""" + # Move events have a dest_path, some editors like gedit use a + # move on larger save operations for write protection + event_path = event.dest_path if hasattr(event, 'dest_path') else event.src_path + fname = os.path.basename(event_path) + if (fname.endswith('~') or + fname.startswith('.') or + os.path.isdir(event_path)): # Skip on folders, these are usually duplicates + return + self.logger.info('REBUILDING SITE (from {0})'.format(event_path)) p = subprocess.Popen(self.cmd_arguments, stderr=subprocess.PIPE) + error = p.stderr.read() + errord = error.decode('utf-8') if p.wait() != 0: - error = p.stderr.read() - self.logger.error(error) - error_signal.send(error=error) + self.logger.error(errord) + error_signal.send(error=errord) else: - error = p.stderr.read() - print(error) + print(errord) def do_refresh(self, event): - self.logger.info('REFRESHING: {0}'.format(event.src_path)) - p = os.path.relpath(event.src_path, os.path.abspath(self.site.config['OUTPUT_FOLDER'])) + """Refresh the page.""" + # Move events have a dest_path, some editors like gedit use a + # move on larger save operations for write protection + event_path = event.dest_path if hasattr(event, 'dest_path') else event.src_path + self.logger.info('REFRESHING: {0}'.format(event_path)) + p = os.path.relpath(event_path, os.path.abspath(self.site.config['OUTPUT_FOLDER'])) refresh_signal.send(path=p) def serve_static(self, environ, start_response): """Trivial static file server.""" uri = wsgiref.util.request_uri(environ) p_uri = urlparse(uri) - f_path = os.path.join(self.site.config['OUTPUT_FOLDER'], *p_uri.path.split('/')) - mimetype = mimetypes.guess_type(uri)[0] or 'text/html' + f_path = os.path.join(self.site.config['OUTPUT_FOLDER'], *[unquote(x) for x in p_uri.path.split('/')]) + + # ‘Pretty’ URIs and root are assumed to be HTML + mimetype = 'text/html' if uri.endswith('/') else mimetypes.guess_type(uri)[0] or 'application/octet-stream' if os.path.isdir(f_path): + if not f_path.endswith('/'): # Redirect to avoid breakage + start_response('301 Redirect', [('Location', p_uri.path + '/')]) + return [] f_path = os.path.join(f_path, self.site.config['INDEX_FILE']) + mimetype = 'text/html' if p_uri.path == '/robots.txt': start_response('200 OK', [('Content-type', 'text/plain')]) - return ['User-Agent: *\nDisallow: /\n'] + return ['User-Agent: *\nDisallow: /\n'.encode('utf-8')] elif os.path.isfile(f_path): with open(f_path, 'rb') as fd: start_response('200 OK', [('Content-type', mimetype)]) - return [self.inject_js(mimetype, fd.read())] + return [self.file_filter(mimetype, fd.read())] elif p_uri.path == '/livereload.js': with open(LRJS_PATH, 'rb') as fd: start_response('200 OK', [('Content-type', mimetype)]) - return [self.inject_js(mimetype, fd.read())] + return [self.file_filter(mimetype, fd.read())] start_response('404 ERR', []) - return [self.inject_js('text/html', ERROR_N.format(404).format(uri))] + return [self.file_filter('text/html', ERROR_N.format(404).format(uri).encode('utf-8'))] - def inject_js(self, mimetype, data): - """Inject livereload.js in HTML files.""" + def file_filter(self, mimetype, data): + """Apply necessary changes to document before serving.""" if mimetype == 'text/html': - data = re.sub('</head>', self.snippet, data.decode('utf8'), 1, re.IGNORECASE) + data = data.decode('utf8') + data = self.remove_base_tag(data) + data = self.inject_js(data) data = data.encode('utf8') return data + def inject_js(self, data): + """Inject livereload.js.""" + data = re.sub('</head>', self.snippet, data, 1, re.IGNORECASE) + return data + + def remove_base_tag(self, data): + """Comment out any <base> to allow local resolution of relative URLs.""" + data = re.sub(r'<base\s([^>]*)>', '<!--base \g<1>-->', data, re.IGNORECASE) + return data + pending = [] class LRSocket(WebSocket): + """Speak Livereload protocol.""" def __init__(self, *a, **kw): + """Initialize protocol handler.""" refresh_signal.connect(self.notify) error_signal.connect(self.send_error) super(LRSocket, self).__init__(*a, **kw) def received_message(self, message): + """Handle received message.""" message = json.loads(message.data.decode('utf8')) self.logger.info('<--- {0}'.format(message)) response = None @@ -364,3 +436,25 @@ class ConfigWatchHandler(FileSystemEventHandler): """Call the provided function on any event.""" if event._src_path == self.configuration_filename: self.function(event) + + +try: + # Monkeypatch to hide Broken Pipe Errors + f = WebSocketWSGIHandler.finish_response + + if sys.version_info[0] == 3: + EX = BrokenPipeError # NOQA + else: + EX = IOError + + def finish_response(self): + """Monkeypatched finish_response that ignores broken pipes.""" + try: + f(self) + except EX: # Client closed the connection, not a real error + pass + + WebSocketWSGIHandler.finish_response = finish_response +except NameError: + # In case there is no WebSocketWSGIHandler because of a failed import. + pass diff --git a/nikola/plugins/command/bootswatch_theme.plugin b/nikola/plugins/command/bootswatch_theme.plugin index b428da3..fc25045 100644 --- a/nikola/plugins/command/bootswatch_theme.plugin +++ b/nikola/plugins/command/bootswatch_theme.plugin @@ -1,10 +1,13 @@ [Core] -Name = bootswatch_theme -Module = bootswatch_theme +name = bootswatch_theme +module = bootswatch_theme [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Given a swatch name and a parent theme, creates a custom theme. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Given a swatch name and a parent theme, creates a custom theme. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py index e19c937..b5644a1 100644 --- a/nikola/plugins/command/bootswatch_theme.py +++ b/nikola/plugins/command/bootswatch_theme.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" + from __future__ import print_function import os import requests @@ -35,6 +37,7 @@ LOGGER = utils.get_logger('bootswatch_theme', utils.STDERR_HANDLER) class CommandBootswatchTheme(Command): + """Given a swatch name from bootswatch.com and a parent theme, creates a custom theme.""" name = "bootswatch_theme" diff --git a/nikola/plugins/command/check.plugin b/nikola/plugins/command/check.plugin index dd0980e..e380e64 100644 --- a/nikola/plugins/command/check.plugin +++ b/nikola/plugins/command/check.plugin @@ -1,10 +1,13 @@ [Core] -Name = check -Module = check +name = check +module = check [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Check the generated site +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Check the generated site + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py index a9bc44a..abf183e 100644 --- a/nikola/plugins/command/check.py +++ b/nikola/plugins/command/check.py @@ -24,11 +24,14 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Check the generated site.""" + from __future__ import print_function from collections import defaultdict import os import re import sys +import time try: from urllib import unquote from urlparse import urlparse, urljoin, urldefrag @@ -40,7 +43,7 @@ import lxml.html import requests from nikola.plugin_categories import Command -from nikola.utils import get_logger +from nikola.utils import get_logger, STDERR_HANDLER def _call_nikola_list(site): @@ -58,6 +61,7 @@ def _call_nikola_list(site): def real_scan_files(site): + """Scan for files.""" task_fnames = set([]) real_fnames = set([]) output_folder = site.config['OUTPUT_FOLDER'] @@ -80,7 +84,8 @@ def real_scan_files(site): def fs_relpath_from_url_path(url_path): - """Expects as input an urlparse(s).path""" + """Create a filesystem relative path from an URL path.""" + # Expects as input an urlparse(s).path url_path = unquote(url_path) # in windows relative paths don't begin with os.sep if sys.platform == 'win32' and len(url_path): @@ -89,6 +94,7 @@ def fs_relpath_from_url_path(url_path): class CommandCheck(Command): + """Check the generated site.""" name = "check" @@ -147,7 +153,7 @@ class CommandCheck(Command): def _execute(self, options, args): """Check the generated site.""" - self.logger = get_logger('check', self.site.loghandlers) + self.logger = get_logger('check', STDERR_HANDLER) if not options['links'] and not options['files'] and not options['clean']: print(self.help()) @@ -169,6 +175,7 @@ class CommandCheck(Command): checked_remote_targets = {} def analyze(self, fname, find_sources=False, check_remote=False): + """Analyze links on a page.""" rv = False self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']] base_url = urlparse(self.site.config['BASE_URL']) @@ -217,15 +224,45 @@ class CommandCheck(Command): if parsed.netloc == base_url.netloc: # absolute URL to self.site continue if target in self.checked_remote_targets: # already checked this exact target - if self.checked_remote_targets[target] > 399: - self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) + if self.checked_remote_targets[target] in [301, 307]: + self.logger.warn("Remote link PERMANENTLY redirected in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) + elif self.checked_remote_targets[target] in [302, 308]: + self.logger.info("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target])) + elif self.checked_remote_targets[target] > 399: + self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target])) continue + + # Skip whitelisted targets + if any(re.search(_, target) for _ in self.whitelist): + continue + # Check the remote link works req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'} # I’m a real boy! - resp = requests.head(target, headers=req_headers) - self.checked_remote_targets[target] = resp.status_code + resp = requests.head(target, headers=req_headers, allow_redirects=False) + + # Retry client errors (4xx) as GET requests because many servers are broken + if resp.status_code >= 400 and resp.status_code <= 499: + time.sleep(0.5) + resp = requests.get(target, headers=req_headers, allow_redirects=False) + + # Follow redirects and see where they lead, redirects to errors will be reported twice + if resp.status_code in [301, 302, 307, 308]: + redir_status_code = resp.status_code + time.sleep(0.5) + # Known redirects are retested using GET because IIS servers otherwise get HEADaches + resp = requests.get(target, headers=req_headers, allow_redirects=True) + # Permanent redirects should be updated + if redir_status_code in [301, 308]: + self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + if redir_status_code in [302, 307]: + self.logger.info("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code)) + self.checked_remote_targets[resp.url] = resp.status_code + self.checked_remote_targets[target] = redir_status_code + else: + self.checked_remote_targets[target] = resp.status_code + if resp.status_code > 399: # Error - self.logger.warn("Broken link in {0}: {1} [Error {2}]".format(filename, target, resp.status_code)) + self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, resp.status_code)) continue elif resp.status_code <= 399: # The address leads *somewhere* that is not an error self.logger.debug("Successfully checked remote link in {0}: {1} [HTTP: {2}]".format(filename, target, resp.status_code)) @@ -271,6 +308,7 @@ class CommandCheck(Command): return rv def scan_links(self, find_sources=False, check_remote=False): + """Check links on the site.""" self.logger.info("Checking Links:") self.logger.info("===============\n") self.logger.notice("{0} mode".format(self.site.config['URL_TYPE'])) @@ -286,6 +324,7 @@ class CommandCheck(Command): return failure def scan_files(self): + """Check files in the site, find missing and orphaned files.""" failure = False self.logger.info("Checking Files:") self.logger.info("===============\n") @@ -311,7 +350,22 @@ class CommandCheck(Command): return failure def clean_files(self): + """Remove orphaned files.""" only_on_output, _ = real_scan_files(self.site) for f in only_on_output: + self.logger.info('removed: {0}'.format(f)) os.unlink(f) + + # Find empty directories and remove them + output_folder = self.site.config['OUTPUT_FOLDER'] + all_dirs = [] + for root, dirs, files in os.walk(output_folder, followlinks=True): + all_dirs.append(root) + all_dirs.sort(key=len, reverse=True) + for d in all_dirs: + try: + os.rmdir(d) + self.logger.info('removed: {0}/'.format(d)) + except OSError: + pass return True diff --git a/nikola/plugins/command/console.plugin b/nikola/plugins/command/console.plugin index 3aef2e7..333762c 100644 --- a/nikola/plugins/command/console.plugin +++ b/nikola/plugins/command/console.plugin @@ -1,9 +1,13 @@ [Core] -Name = console -Module = console +name = console +module = console [Documentation] -Author = Chris Warrick, Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Start a debugging python console +author = Chris Warrick, Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Start a debugging python console + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py index b8e7825..539fa08 100644 --- a/nikola/plugins/command/console.py +++ b/nikola/plugins/command/console.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Start debugging console.""" + from __future__ import print_function, unicode_literals import os @@ -36,7 +38,9 @@ LOGGER = get_logger('console', STDERR_HANDLER) class CommandConsole(Command): + """Start debugging console.""" + name = "console" shells = ['ipython', 'bpython', 'plain'] doc_purpose = "start an interactive Python console with access to your site" diff --git a/nikola/plugins/command/deploy.plugin b/nikola/plugins/command/deploy.plugin index 14fd53f..4743ca2 100644 --- a/nikola/plugins/command/deploy.plugin +++ b/nikola/plugins/command/deploy.plugin @@ -1,9 +1,13 @@ [Core] -Name = deploy -Module = deploy +name = deploy +module = deploy [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Deploy the site +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Deploy the site + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py index 2c44e87..821ea11 100644 --- a/nikola/plugins/command/deploy.py +++ b/nikola/plugins/command/deploy.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Deploy site.""" + from __future__ import print_function import io from datetime import datetime @@ -35,11 +37,13 @@ import time from blinker import signal from nikola.plugin_categories import Command -from nikola.utils import get_logger, remove_file, unicode_str, makedirs +from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_HANDLER class CommandDeploy(Command): + """Deploy site.""" + name = "deploy" doc_usage = "[[preset [preset...]]" @@ -48,7 +52,8 @@ class CommandDeploy(Command): logger = None def _execute(self, command, args): - self.logger = get_logger('deploy', self.site.loghandlers) + """Execute the deploy command.""" + self.logger = get_logger('deploy', STDERR_HANDLER) # Get last successful deploy date timestamp_path = os.path.join(self.site.config['CACHE_FOLDER'], 'lastdeploy') if self.site.config['COMMENT_SYSTEM_ID'] == 'nikolademo': @@ -116,7 +121,7 @@ class CommandDeploy(Command): outf.write(unicode_str(new_deploy.isoformat())) def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None): - """ Emit events for all timeline entries newer than last deploy. + """Emit events for all timeline entries newer than last deploy. last_deploy: datetime Time stamp of the last successful deployment. @@ -128,7 +133,6 @@ class CommandDeploy(Command): True when it appears like deploy is being run after a clean. """ - event = { 'last_deploy': last_deploy, 'new_deploy': new_deploy, diff --git a/nikola/plugins/command/github_deploy.plugin b/nikola/plugins/command/github_deploy.plugin index 74e7902..e793548 100644 --- a/nikola/plugins/command/github_deploy.plugin +++ b/nikola/plugins/command/github_deploy.plugin @@ -1,9 +1,13 @@ [Core] -Name = github_deploy -Module = github_deploy +name = github_deploy +module = github_deploy [Documentation] -Author = Puneeth Chaganti -Version = 1,0 -Website = http://getnikola.com -Description = Deploy the site to GitHub pages. +author = Puneeth Chaganti +version = 1,0 +website = http://getnikola.com +description = Deploy the site to GitHub pages. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py index 888a4f9..0ab9332 100644 --- a/nikola/plugins/command/github_deploy.py +++ b/nikola/plugins/command/github_deploy.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Deploy site to GitHub Pages.""" + from __future__ import print_function from datetime import datetime import io @@ -33,17 +35,19 @@ from textwrap import dedent from nikola.plugin_categories import Command from nikola.plugins.command.check import real_scan_files -from nikola.utils import get_logger, req_missing, makedirs, unicode_str +from nikola.utils import get_logger, req_missing, makedirs, unicode_str, STDERR_HANDLER from nikola.__main__ import main from nikola import __version__ def uni_check_output(*args, **kwargs): + """Run command and return output as Unicode (UTf-8).""" o = subprocess.check_output(*args, **kwargs) return o.decode('utf-8') def check_ghp_import_installed(): + """Check if ghp-import is installed.""" try: subprocess.check_output(['ghp-import', '-h']) except OSError: @@ -53,7 +57,9 @@ def check_ghp_import_installed(): class CommandGitHubDeploy(Command): - """ Deploy site to GitHub Pages. """ + + """Deploy site to GitHub Pages.""" + name = 'github_deploy' doc_usage = '' @@ -70,10 +76,8 @@ class CommandGitHubDeploy(Command): logger = None def _execute(self, command, args): - - self.logger = get_logger( - CommandGitHubDeploy.name, self.site.loghandlers - ) + """Run the deployment.""" + self.logger = get_logger(CommandGitHubDeploy.name, STDERR_HANDLER) # Check if ghp-import is installed check_ghp_import_installed() @@ -95,8 +99,7 @@ class CommandGitHubDeploy(Command): return def _commit_and_push(self): - """ Commit all the files and push. """ - + """Commit all the files and push.""" source = self.site.config['GITHUB_SOURCE_BRANCH'] deploy = self.site.config['GITHUB_DEPLOY_BRANCH'] remote = self.site.config['GITHUB_REMOTE_NAME'] diff --git a/nikola/plugins/command/import_wordpress.plugin b/nikola/plugins/command/import_wordpress.plugin index e072224..6c4384e 100644 --- a/nikola/plugins/command/import_wordpress.plugin +++ b/nikola/plugins/command/import_wordpress.plugin @@ -1,10 +1,13 @@ [Core] -Name = import_wordpress -Module = import_wordpress +name = import_wordpress +module = import_wordpress [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Import a wordpress site from a XML dump (requires markdown). +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Import a wordpress site from a XML dump (requires markdown). + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index 674fc2a..a652ec8 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -24,13 +24,18 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Import a WordPress dump.""" + from __future__ import unicode_literals, print_function import os import re import sys import datetime +import io +import json import requests from lxml import etree +from collections import defaultdict try: from urlparse import urlparse @@ -53,7 +58,37 @@ from nikola.plugins.command.init import SAMPLE_CONF, prepare_config, format_defa LOGGER = utils.get_logger('import_wordpress', utils.STDERR_HANDLER) +def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False): + """Install a Nikola plugin.""" + LOGGER.notice("Installing plugin '{0}'".format(plugin_name)) + # Get hold of the 'plugin' plugin + plugin_installer_info = site.plugin_manager.getPluginByName('plugin', 'Command') + if plugin_installer_info is None: + LOGGER.error('Internal error: cannot find the "plugin" plugin which is supposed to come with Nikola!') + return False + if not plugin_installer_info.is_activated: + # Someone might have disabled the plugin in the `conf.py` used + site.plugin_manager.activatePluginByName(plugin_installer_info.name) + plugin_installer_info.plugin_object.set_site(site) + plugin_installer = plugin_installer_info.plugin_object + # Try to install the requested plugin + options = {} + for option in plugin_installer.cmd_options: + options[option['name']] = option['default'] + options['install'] = plugin_name + options['output_dir'] = output_dir + options['show_install_notes'] = show_install_notes + if plugin_installer.execute(options=options) > 0: + return False + # Let the plugin manager find newly installed plugins + site.plugin_manager.collectPlugins() + # Re-scan for compiler extensions + site.compiler_extensions = site._activate_plugins_of_category("CompilerExtension") + return True + + class CommandImportWordpress(Command, ImportMixin): + """Import a WordPress dump.""" name = "import_wordpress" @@ -70,6 +105,20 @@ class CommandImportWordpress(Command, ImportMixin): 'help': "Don't import drafts", }, { + 'name': 'exclude_privates', + 'long': 'exclude-privates', + 'default': False, + 'type': bool, + 'help': "Don't import private posts", + }, + { + 'name': 'include_empty_items', + 'long': 'include-empty-items', + 'default': False, + 'type': bool, + 'help': "Include empty posts and pages", + }, + { 'name': 'squash_newlines', 'long': 'squash-newlines', 'default': False, @@ -107,15 +156,57 @@ class CommandImportWordpress(Command, ImportMixin): 'type': str, 'help': "The pattern for translation files names", }, + { + 'name': 'export_categories_as_categories', + 'long': 'export-categories-as-categories', + 'default': False, + 'type': bool, + 'help': "Export categories as categories, instead of treating them as tags", + }, + { + 'name': 'export_comments', + 'long': 'export-comments', + 'default': False, + 'type': bool, + 'help': "Export comments as .wpcomment files", + }, + { + 'name': 'transform_to_html', + 'long': 'transform-to-html', + 'default': False, + 'type': bool, + 'help': "Uses WordPress page compiler to transform WordPress posts directly to HTML during import", + }, + { + 'name': 'use_wordpress_compiler', + 'long': 'use-wordpress-compiler', + 'default': False, + 'type': bool, + 'help': "Instead of converting posts to markdown, leave them as is and use the WordPress page compiler", + }, + { + 'name': 'install_wordpress_compiler', + 'long': 'install-wordpress-compiler', + 'default': False, + 'type': bool, + 'help': "Automatically installs the WordPress page compiler (either locally or in the new site) if required by other options.\nWarning: the compiler is GPL software!", + }, ] all_tags = set([]) - def _execute(self, options={}, args=[]): - """Import a WordPress blog from an export file into a Nikola site.""" - if not args: - print(self.help()) + def _find_wordpress_compiler(self): + """Find WordPress compiler plugin.""" + if self.wordpress_page_compiler is not None: return - + plugin_info = self.site.plugin_manager.getPluginByName('wordpress', 'PageCompiler') + if plugin_info is not None: + if not plugin_info.is_activated: + self.site.plugin_manager.activatePluginByName(plugin_info.name) + plugin_info.plugin_object.set_site(self.site) + self.wordpress_page_compiler = plugin_info.plugin_object + + def _read_options(self, options, args): + """Read command-line options.""" options['filename'] = args.pop(0) if args and ('output_folder' not in args or @@ -136,19 +227,76 @@ class CommandImportWordpress(Command, ImportMixin): self.output_folder = options.get('output_folder', 'new_site') self.exclude_drafts = options.get('exclude_drafts', False) + self.exclude_privates = options.get('exclude_privates', False) self.no_downloads = options.get('no_downloads', False) + self.import_empty_items = options.get('include_empty_items', False) + + self.export_categories_as_categories = options.get('export_categories_as_categories', False) + self.export_comments = options.get('export_comments', False) + + self.transform_to_html = options.get('transform_to_html', False) + self.use_wordpress_compiler = options.get('use_wordpress_compiler', False) + self.install_wordpress_compiler = options.get('install_wordpress_compiler', False) + self.wordpress_page_compiler = None self.auth = None if options.get('download_auth') is not None: username_password = options.get('download_auth') self.auth = tuple(username_password.split(':', 1)) if len(self.auth) < 2: - print("Please specify HTTP authentication credentials in the form username:password.") + LOGGER.error("Please specify HTTP authentication credentials in the form username:password.") return False self.separate_qtranslate_content = options.get('separate_qtranslate_content') self.translations_pattern = options.get('translations_pattern') + if self.transform_to_html and self.use_wordpress_compiler: + LOGGER.warn("It does not make sense to combine --transform-to-html with --use-wordpress-compiler, as the first converts all posts to HTML and the latter option affects zero posts.") + + if self.transform_to_html: + self._find_wordpress_compiler() + if not self.wordpress_page_compiler and self.install_wordpress_compiler: + if not install_plugin(self.site, 'wordpress_compiler', output_dir='plugins'): # local install + return False + self._find_wordpress_compiler() + if not self.wordpress_page_compiler: + LOGGER.error("To compile WordPress posts to HTML, the WordPress post compiler is needed. You can install it via:") + LOGGER.error(" nikola plugin -i wordpress_compiler") + LOGGER.error("Please note that the WordPress post compiler is licensed under the GPL v2.") + return False + + return True + + def _prepare(self, channel): + """Prepare context and category hierarchy.""" + self.context = self.populate_context(channel) + self.base_dir = urlparse(self.context['BASE_URL']).path + + if self.export_categories_as_categories: + wordpress_namespace = channel.nsmap['wp'] + cat_map = dict() + for cat in channel.findall('{{{0}}}category'.format(wordpress_namespace)): + # cat_id = get_text_tag(cat, '{{{0}}}term_id'.format(wordpress_namespace), None) + cat_slug = get_text_tag(cat, '{{{0}}}category_nicename'.format(wordpress_namespace), None) + cat_parent_slug = get_text_tag(cat, '{{{0}}}category_parent'.format(wordpress_namespace), None) + cat_name = get_text_tag(cat, '{{{0}}}cat_name'.format(wordpress_namespace), None) + cat_path = [cat_name] + if cat_parent_slug in cat_map: + cat_path = cat_map[cat_parent_slug] + cat_path + cat_map[cat_slug] = cat_path + self._category_paths = dict() + for cat, path in cat_map.items(): + self._category_paths[cat] = utils.join_hierarchical_category_path(path) + + def _execute(self, options={}, args=[]): + """Import a WordPress blog from an export file into a Nikola site.""" + if not args: + print(self.help()) + return False + + if not self._read_options(options, args): + return False + # A place holder where extra language (if detected) will be stored self.extra_languages = set() @@ -166,8 +314,7 @@ class CommandImportWordpress(Command, ImportMixin): req_missing(['phpserialize'], 'import WordPress dumps without --no-downloads') channel = self.get_channel_from_file(self.wordpress_export_file) - self.context = self.populate_context(channel) - self.base_dir = urlparse(self.context['BASE_URL']).path + self._prepare(channel) conf_template = self.generate_base_site() # If user has specified a custom pattern for translation files we @@ -181,6 +328,11 @@ class CommandImportWordpress(Command, ImportMixin): self.extra_languages) self.context['REDIRECTIONS'] = self.configure_redirections( self.url_map) + if self.timezone: + self.context['TIMEZONE'] = self.timezone + if self.export_categories_as_categories: + self.context['CATEGORY_ALLOW_HIERARCHIES'] = True + self.context['CATEGORY_OUTPUT_FLAT_HIERARCHY'] = True # Add tag redirects for tag in self.all_tags: @@ -197,18 +349,21 @@ class CommandImportWordpress(Command, ImportMixin): self.write_urlmap_csv( os.path.join(self.output_folder, 'url_map.csv'), self.url_map) rendered_template = conf_template.render(**prepare_config(self.context)) - rendered_template = re.sub('# REDIRECTIONS = ', 'REDIRECTIONS = ', - rendered_template) - - if self.timezone: - rendered_template = re.sub('# TIMEZONE = \'UTC\'', - 'TIMEZONE = \'' + self.timezone + '\'', - rendered_template) self.write_configuration(self.get_configuration_output_path(), rendered_template) + if self.use_wordpress_compiler: + if self.install_wordpress_compiler: + if not install_plugin(self.site, 'wordpress_compiler', output_dir=os.path.join(self.output_folder, 'plugins')): + return False + else: + LOGGER.warn("Make sure to install the WordPress page compiler via") + LOGGER.warn(" nikola plugin -i wordpress_compiler") + LOGGER.warn("in your imported blog's folder ({0}), if you haven't installed it system-wide or user-wide. Otherwise, your newly imported blog won't compile.".format(self.output_folder)) + @classmethod def read_xml_file(cls, filename): + """Read XML file into memory.""" xml = [] with open(filename, 'rb') as fd: @@ -221,12 +376,13 @@ class CommandImportWordpress(Command, ImportMixin): @classmethod def get_channel_from_file(cls, filename): + """Get channel from XML file.""" tree = etree.fromstring(cls.read_xml_file(filename)) channel = tree.find('channel') return channel - @staticmethod - def populate_context(channel): + def populate_context(self, channel): + """Populate context with config for the site.""" wordpress_namespace = channel.nsmap['wp'] context = SAMPLE_CONF.copy() @@ -255,28 +411,31 @@ class CommandImportWordpress(Command, ImportMixin): author, '{{{0}}}author_display_name'.format(wordpress_namespace), "Joe Example") - context['POSTS'] = '''( - ("posts/*.rst", "posts", "post.tmpl"), - ("posts/*.txt", "posts", "post.tmpl"), - ("posts/*.md", "posts", "post.tmpl"), - ("posts/*.wp", "posts", "post.tmpl"), - )''' - context['PAGES'] = '''( - ("stories/*.rst", "stories", "story.tmpl"), - ("stories/*.txt", "stories", "story.tmpl"), - ("stories/*.md", "stories", "story.tmpl"), - ("stories/*.wp", "stories", "story.tmpl"), - )''' - context['COMPILERS'] = '''{ - "rest": ('.txt', '.rst'), - "markdown": ('.md', '.mdown', '.markdown', '.wp'), - "html": ('.html', '.htm') - } - ''' + extensions = ['rst', 'txt', 'md', 'html'] + if self.use_wordpress_compiler: + extensions.append('wp') + POSTS = '(\n' + PAGES = '(\n' + for extension in extensions: + POSTS += ' ("posts/*.{0}", "posts", "post.tmpl"),\n'.format(extension) + PAGES += ' ("stories/*.{0}", "stories", "story.tmpl"),\n'.format(extension) + POSTS += ')\n' + PAGES += ')\n' + context['POSTS'] = POSTS + context['PAGES'] = PAGES + COMPILERS = '{\n' + COMPILERS += ''' "rest": ('.txt', '.rst'),''' + '\n' + COMPILERS += ''' "markdown": ('.md', '.mdown', '.markdown'),''' + '\n' + COMPILERS += ''' "html": ('.html', '.htm'),''' + '\n' + if self.use_wordpress_compiler: + COMPILERS += ''' "wordpress": ('.wp'),''' + '\n' + COMPILERS += '}' + context['COMPILERS'] = COMPILERS return context def download_url_content_to_file(self, url, dst_path): + """Download some content (attachments) to a file.""" if self.no_downloads: return @@ -291,6 +450,8 @@ class CommandImportWordpress(Command, ImportMixin): LOGGER.warn("Downloading {0} to {1} failed: {2}".format(url, dst_path, err)) def import_attachment(self, item, wordpress_namespace): + """Import an attachment to the site.""" + # Download main image url = get_text_tag( item, '{{{0}}}attachment_url'.format(wordpress_namespace), 'foo') link = get_text_tag(item, '{{{0}}}link'.format(wordpress_namespace), @@ -305,59 +466,136 @@ class CommandImportWordpress(Command, ImportMixin): links[link] = '/' + dst_url links[url] = '/' + dst_url - self.download_additional_image_sizes( - item, - wordpress_namespace, - os.path.dirname(url) - ) - - def download_additional_image_sizes(self, item, wordpress_namespace, source_path): - if phpserialize is None: - return + files = [path] + files_meta = [{}] additional_metadata = item.findall('{{{0}}}postmeta'.format(wordpress_namespace)) - if additional_metadata is None: - return - - for element in additional_metadata: - meta_key = element.find('{{{0}}}meta_key'.format(wordpress_namespace)) - if meta_key is not None and meta_key.text == '_wp_attachment_metadata': - meta_value = element.find('{{{0}}}meta_value'.format(wordpress_namespace)) - - if meta_value is None: - continue - - # Someone from Wordpress thought it was a good idea - # serialize PHP objects into that metadata field. Given - # that the export should give you the power to insert - # your blogging into another site or system its not. - # Why don't they just use JSON? - if sys.version_info[0] == 2: - try: - metadata = phpserialize.loads(utils.sys_encode(meta_value.text)) - except ValueError: - # local encoding might be wrong sometimes + if phpserialize and additional_metadata: + source_path = os.path.dirname(url) + for element in additional_metadata: + meta_key = element.find('{{{0}}}meta_key'.format(wordpress_namespace)) + if meta_key is not None and meta_key.text == '_wp_attachment_metadata': + meta_value = element.find('{{{0}}}meta_value'.format(wordpress_namespace)) + + if meta_value is None: + continue + + # Someone from Wordpress thought it was a good idea + # serialize PHP objects into that metadata field. Given + # that the export should give you the power to insert + # your blogging into another site or system its not. + # Why don't they just use JSON? + if sys.version_info[0] == 2: + try: + metadata = phpserialize.loads(utils.sys_encode(meta_value.text)) + except ValueError: + # local encoding might be wrong sometimes + metadata = phpserialize.loads(meta_value.text.encode('utf-8')) + else: metadata = phpserialize.loads(meta_value.text.encode('utf-8')) - else: - metadata = phpserialize.loads(meta_value.text.encode('utf-8')) - size_key = b'sizes' - file_key = b'file' - - if size_key not in metadata: - continue - - for filename in [metadata[size_key][size][file_key] for size in metadata[size_key]]: - url = '/'.join([source_path, filename.decode('utf-8')]) - path = urlparse(url).path - dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) - dst_dir = os.path.dirname(dst_path) - utils.makedirs(dst_dir) - LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) - self.download_url_content_to_file(url, dst_path) - dst_url = '/'.join(dst_path.split(os.sep)[2:]) - links[url] = '/' + dst_url - links[url] = '/' + dst_url + meta_key = b'image_meta' + size_key = b'sizes' + file_key = b'file' + width_key = b'width' + height_key = b'height' + + # Extract metadata + if width_key in metadata and height_key in metadata: + files_meta[0]['width'] = int(metadata[width_key]) + files_meta[0]['height'] = int(metadata[height_key]) + + if meta_key in metadata: + image_meta = metadata[meta_key] + dst_meta = {} + + def add(our_key, wp_key, is_int=False, ignore_zero=False, is_float=False): + if wp_key in image_meta: + value = image_meta[wp_key] + if is_int: + value = int(value) + if ignore_zero and value == 0: + return + elif is_float: + value = float(value) + if ignore_zero and value == 0: + return + else: + value = value.decode('utf-8') # assume UTF-8 + if value == '': # skip empty values + return + dst_meta[our_key] = value + + add('aperture', b'aperture', is_float=True, ignore_zero=True) + add('credit', b'credit') + add('camera', b'camera') + add('caption', b'caption') + add('created_timestamp', b'created_timestamp', is_float=True, ignore_zero=True) + add('copyright', b'copyright') + add('focal_length', b'focal_length', is_float=True, ignore_zero=True) + add('iso', b'iso', is_float=True, ignore_zero=True) + add('shutter_speed', b'shutter_speed', ignore_zero=True, is_float=True) + add('title', b'title') + + if len(dst_meta) > 0: + files_meta[0]['meta'] = dst_meta + + # Find other sizes of image + if size_key not in metadata: + continue + + for size in metadata[size_key]: + filename = metadata[size_key][size][file_key] + url = '/'.join([source_path, filename.decode('utf-8')]) + + # Construct metadata + meta = {} + meta['size'] = size.decode('utf-8') + if width_key in metadata[size_key][size] and height_key in metadata[size_key][size]: + meta['width'] = metadata[size_key][size][width_key] + meta['height'] = metadata[size_key][size][height_key] + + path = urlparse(url).path + dst_path = os.path.join(*([self.output_folder, 'files'] + list(path.split('/')))) + dst_dir = os.path.dirname(dst_path) + utils.makedirs(dst_dir) + LOGGER.info("Downloading {0} => {1}".format(url, dst_path)) + self.download_url_content_to_file(url, dst_path) + dst_url = '/'.join(dst_path.split(os.sep)[2:]) + links[url] = '/' + dst_url + + files.append(path) + files_meta.append(meta) + + # Prepare result + result = {} + result['files'] = files + result['files_meta'] = files_meta + + # Prepare extraction of more information + dc_namespace = item.nsmap['dc'] + content_namespace = item.nsmap['content'] + excerpt_namespace = item.nsmap['excerpt'] + + def add(result_key, key, namespace=None, filter=None, store_empty=False): + if namespace is not None: + value = get_text_tag(item, '{{{0}}}{1}'.format(namespace, key), None) + else: + value = get_text_tag(item, key, None) + if value is not None: + if filter: + value = filter(value) + if value or store_empty: + result[result_key] = value + + add('title', 'title') + add('date_utc', 'post_date_gmt', namespace=wordpress_namespace) + add('wordpress_user_name', 'creator', namespace=dc_namespace) + add('content', 'encoded', namespace=content_namespace) + add('excerpt', 'encoded', namespace=excerpt_namespace) + add('description', 'description') + + return result code_re1 = re.compile(r'\[code.* lang.*?="(.*?)?".*\](.*?)\[/code\]', re.DOTALL | re.MULTILINE) code_re2 = re.compile(r'\[sourcecode.* lang.*?="(.*?)?".*\](.*?)\[/sourcecode\]', re.DOTALL | re.MULTILINE) @@ -365,6 +603,7 @@ class CommandImportWordpress(Command, ImportMixin): code_re4 = re.compile(r'\[sourcecode.*?\](.*?)\[/sourcecode\]', re.DOTALL | re.MULTILINE) def transform_code(self, content): + """Transform code blocks.""" # http://en.support.wordpress.com/code/posting-source-code/. There are # a ton of things not supported here. We only do a basic [code # lang="x"] -> ```x translation, and remove quoted html entities (<, @@ -390,26 +629,126 @@ class CommandImportWordpress(Command, ImportMixin): @staticmethod def transform_caption(content): + """Transform captions.""" new_caption = re.sub(r'\[/caption\]', '', content) new_caption = re.sub(r'\[caption.*\]', '', new_caption) return new_caption def transform_multiple_newlines(self, content): - """Replaces multiple newlines with only two.""" + """Replace multiple newlines with only two.""" if self.squash_newlines: return re.sub(r'\n{3,}', r'\n\n', content) else: return content - def transform_content(self, content): - content = self.transform_code(content) - content = self.transform_caption(content) - content = self.transform_multiple_newlines(content) - return content + def transform_content(self, content, post_format, attachments): + """Transform content into appropriate format.""" + if post_format == 'wp': + if self.transform_to_html: + additional_data = {} + if attachments is not None: + additional_data['attachments'] = attachments + try: + content = self.wordpress_page_compiler.compile_to_string(content, additional_data=additional_data) + except TypeError: # old versions of the plugin don't support the additional argument + content = self.wordpress_page_compiler.compile_to_string(content) + return content, 'html', True + elif self.use_wordpress_compiler: + return content, 'wp', False + else: + content = self.transform_code(content) + content = self.transform_caption(content) + content = self.transform_multiple_newlines(content) + return content, 'md', True + elif post_format == 'markdown': + return content, 'md', True + elif post_format == 'none': + return content, 'html', True + else: + return None + + def _extract_comment(self, comment, wordpress_namespace): + """Extract comment from dump.""" + id = int(get_text_tag(comment, "{{{0}}}comment_id".format(wordpress_namespace), None)) + author = get_text_tag(comment, "{{{0}}}comment_author".format(wordpress_namespace), None) + author_email = get_text_tag(comment, "{{{0}}}comment_author_email".format(wordpress_namespace), None) + author_url = get_text_tag(comment, "{{{0}}}comment_author_url".format(wordpress_namespace), None) + author_IP = get_text_tag(comment, "{{{0}}}comment_author_IP".format(wordpress_namespace), None) + # date = get_text_tag(comment, "{{{0}}}comment_date".format(wordpress_namespace), None) + date_gmt = get_text_tag(comment, "{{{0}}}comment_date_gmt".format(wordpress_namespace), None) + content = get_text_tag(comment, "{{{0}}}comment_content".format(wordpress_namespace), None) + approved = get_text_tag(comment, "{{{0}}}comment_approved".format(wordpress_namespace), '0') + if approved == '0': + approved = 'hold' + elif approved == '1': + approved = 'approved' + elif approved == 'spam' or approved == 'trash': + pass + else: + LOGGER.warn("Unknown comment approved status: " + str(approved)) + parent = int(get_text_tag(comment, "{{{0}}}comment_parent".format(wordpress_namespace), 0)) + if parent == 0: + parent = None + user_id = int(get_text_tag(comment, "{{{0}}}comment_user_id".format(wordpress_namespace), 0)) + if user_id == 0: + user_id = None + + if approved == 'trash' or approved == 'spam': + return None + + return {"id": id, "status": str(approved), "approved": approved == "approved", + "author": author, "email": author_email, "url": author_url, "ip": author_IP, + "date": date_gmt, "content": content, "parent": parent, "user_id": user_id} + + def _write_comment(self, filename, comment): + """Write comment to file.""" + def write_header_line(fd, header_field, header_content): + """Write comment header line.""" + if header_content is None: + return + header_content = str(header_content).replace('\n', ' ') + line = '.. ' + header_field + ': ' + header_content + '\n' + fd.write(line.encode('utf8')) + + with open(filename, "wb+") as fd: + write_header_line(fd, "id", comment["id"]) + write_header_line(fd, "status", comment["status"]) + write_header_line(fd, "approved", comment["approved"]) + write_header_line(fd, "author", comment["author"]) + write_header_line(fd, "author_email", comment["email"]) + write_header_line(fd, "author_url", comment["url"]) + write_header_line(fd, "author_IP", comment["ip"]) + write_header_line(fd, "date_utc", comment["date"]) + write_header_line(fd, "parent_id", comment["parent"]) + write_header_line(fd, "wordpress_user_id", comment["user_id"]) + fd.write(('\n' + comment['content']).encode('utf8')) + + def _create_metadata(self, status, excerpt, tags, categories, post_name=None): + """Create post metadata.""" + other_meta = {'wp-status': status} + if excerpt is not None: + other_meta['excerpt'] = excerpt + if self.export_categories_as_categories: + cats = [] + for text in categories: + if text in self._category_paths: + cats.append(self._category_paths[text]) + else: + cats.append(utils.join_hierarchical_category_path([text])) + other_meta['categories'] = ','.join(cats) + if len(cats) > 0: + other_meta['category'] = cats[0] + if len(cats) > 1: + LOGGER.warn(('Post "{0}" has more than one category! ' + + 'Will only use the first one.').format(post_name)) + tags_cats = tags + else: + tags_cats = tags + categories + return tags_cats, other_meta - def import_item(self, item, wordpress_namespace, out_folder=None): - """Takes an item from the feed and creates a post file.""" + def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None): + """Take an item from the feed and creates a post file.""" if out_folder is None: out_folder = 'posts' @@ -439,7 +778,7 @@ class CommandImportWordpress(Command, ImportMixin): item, '{{{0}}}post_id'.format(wordpress_namespace), None) if not slug: # should never happen LOGGER.error("Error converting post:", title) - return + return False else: if len(pathlist) > 1: out_folder = os.path.join(*([out_folder] + pathlist[:-1])) @@ -461,23 +800,42 @@ class CommandImportWordpress(Command, ImportMixin): item, '{{{0}}}status'.format(wordpress_namespace), 'publish') content = get_text_tag( item, '{http://purl.org/rss/1.0/modules/content/}encoded', '') + excerpt = get_text_tag( + item, '{http://wordpress.org/export/1.2/excerpt/}encoded', None) + + if excerpt is not None: + if len(excerpt) == 0: + excerpt = None tags = [] + categories = [] if status == 'trash': LOGGER.warn('Trashed post "{0}" will not be imported.'.format(title)) - return + return False + elif status == 'private': + tags.append('private') + is_draft = False + is_private = True elif status != 'publish': tags.append('draft') is_draft = True + is_private = False else: is_draft = False + is_private = False for tag in item.findall('category'): text = tag.text - if text == 'Uncategorized': + type = 'category' + if 'domain' in tag.attrib: + type = tag.attrib['domain'] + if text == 'Uncategorized' and type == 'category': continue - tags.append(text) self.all_tags.add(text) + if type == 'category': + categories.append(type) + else: + tags.append(text) if '$latex' in content: tags.append('mathjax') @@ -487,11 +845,16 @@ class CommandImportWordpress(Command, ImportMixin): format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format'] if format_tag: post_format = format_tag[0].getparent().find('{%s}meta_value' % wordpress_namespace).text + if post_format == 'wpautop': + post_format = 'wp' if is_draft and self.exclude_drafts: LOGGER.notice('Draft "{0}" will not be imported.'.format(title)) - - elif content.strip(): + return False + elif is_private and self.exclude_privates: + LOGGER.notice('Private post "{0}" will not be imported.'.format(title)) + return False + elif content.strip() or self.import_empty_items: # If no content is found, no files are written. self.url_map[link] = (self.context['SITE_URL'] + out_folder.rstrip('/') + '/' + slug + @@ -503,53 +866,121 @@ class CommandImportWordpress(Command, ImportMixin): content_translations = {"": content} default_language = self.context["DEFAULT_LANG"] for lang, content in content_translations.items(): + try: + content, extension, rewrite_html = self.transform_content(content, post_format, attachments) + except: + LOGGER.error(('Cannot interpret post "{0}" (language {1}) with post ' + + 'format {2}!').format(os.path.join(out_folder, slug), lang, post_format)) + return False if lang: out_meta_filename = slug + '.meta' if lang == default_language: - out_content_filename = slug + '.wp' + out_content_filename = slug + '.' + extension else: out_content_filename \ = utils.get_translation_candidate(self.context, - slug + ".wp", lang) + slug + "." + extension, lang) self.extra_languages.add(lang) meta_slug = slug else: out_meta_filename = slug + '.meta' - out_content_filename = slug + '.wp' + out_content_filename = slug + '.' + extension meta_slug = slug - if post_format == 'wp': - content = self.transform_content(content) + tags, other_meta = self._create_metadata(status, excerpt, tags, categories, + post_name=os.path.join(out_folder, slug)) self.write_metadata(os.path.join(self.output_folder, out_folder, out_meta_filename), - title, meta_slug, post_date, description, tags) + title, meta_slug, post_date, description, tags, **other_meta) self.write_content( os.path.join(self.output_folder, out_folder, out_content_filename), - content) + content, + rewrite_html) + + if self.export_comments: + comments = [] + for tag in item.findall('{{{0}}}comment'.format(wordpress_namespace)): + comment = self._extract_comment(tag, wordpress_namespace) + if comment is not None: + comments.append(comment) + + for comment in comments: + comment_filename = slug + "." + str(comment['id']) + ".wpcomment" + self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment) + + return (out_folder, slug) else: - LOGGER.warn('Not going to import "{0}" because it seems to contain' - ' no content.'.format(title)) + LOGGER.warn(('Not going to import "{0}" because it seems to contain' + ' no content.').format(title)) + return False - def process_item(self, item): + def _extract_item_info(self, item): + """Extract information about an item.""" # The namespace usually is something like: # http://wordpress.org/export/1.2/ wordpress_namespace = item.nsmap['wp'] post_type = get_text_tag( item, '{{{0}}}post_type'.format(wordpress_namespace), 'post') + post_id = int(get_text_tag( + item, '{{{0}}}post_id'.format(wordpress_namespace), "0")) + parent_id = get_text_tag( + item, '{{{0}}}post_parent'.format(wordpress_namespace), None) + return wordpress_namespace, post_type, post_id, parent_id + + def process_item_if_attachment(self, item): + """Process attachments.""" + wordpress_namespace, post_type, post_id, parent_id = self._extract_item_info(item) if post_type == 'attachment': - self.import_attachment(item, wordpress_namespace) - elif post_type == 'post': - self.import_item(item, wordpress_namespace, 'posts') - else: - self.import_item(item, wordpress_namespace, 'stories') + data = self.import_attachment(item, wordpress_namespace) + # If parent was found, store relation with imported files + if parent_id is not None and int(parent_id) != 0: + self.attachments[int(parent_id)][post_id] = data + else: + LOGGER.warn("Attachment #{0} ({1}) has no parent!".format(post_id, data['files'])) + + def write_attachments_info(self, path, attachments): + """Write attachments info file.""" + with io.open(path, "wb") as file: + file.write(json.dumps(attachments).encode('utf-8')) + + def process_item_if_post_or_page(self, item): + """Process posts and pages.""" + wordpress_namespace, post_type, post_id, parent_id = self._extract_item_info(item) + + if post_type != 'attachment': + # Get attachments for post + attachments = self.attachments.pop(post_id, None) + # Import item + if post_type == 'post': + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'posts', attachments) + else: + out_folder_slug = self.import_postpage_item(item, wordpress_namespace, 'stories', attachments) + # Process attachment data + if attachments is not None: + # If post was exported, store data + if out_folder_slug: + destination = os.path.join(self.output_folder, out_folder_slug[0], + out_folder_slug[1] + ".attachments.json") + self.write_attachments_info(destination, attachments) def import_posts(self, channel): + """Import posts into the site.""" + self.attachments = defaultdict(dict) + # First process attachments + for item in channel.findall('item'): + self.process_item_if_attachment(item) + # Next process posts for item in channel.findall('item'): - self.process_item(item) + self.process_item_if_post_or_page(item) + # Assign attachments to posts + for post_id in self.attachments: + LOGGER.warn(("Found attachments for post or page #{0}, but didn't find post or page. " + + "(Attachments: {1})").format(post_id, [e['files'][0] for e in self.attachments[post_id].values()])) def get_text_tag(tag, name, default): + """Get the text of an XML tag.""" if tag is None: return default t = tag.find(name) @@ -560,9 +991,10 @@ def get_text_tag(tag, name, default): def separate_qtranslate_content(text): - """Parse the content of a wordpress post or page and separate - the various language specific contents when they are delimited - with qtranslate tags: <!--:LL-->blabla<!--:-->""" + """Parse the content of a wordpress post or page and separate qtranslate languages. + + qtranslate tags: <!--:LL-->blabla<!--:--> + """ # TODO: uniformize qtranslate tags <!--/en--> => <!--:--> qt_start = "<!--:" qt_end = "-->" diff --git a/nikola/plugins/command/init.plugin b/nikola/plugins/command/init.plugin index 850dba9..a5404c4 100644 --- a/nikola/plugins/command/init.plugin +++ b/nikola/plugins/command/init.plugin @@ -1,9 +1,13 @@ [Core] -Name = init -Module = init +name = init +module = init [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Create a new site. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Create a new site. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py index 7a36894..91ccdb4 100644 --- a/nikola/plugins/command/init.py +++ b/nikola/plugins/command/init.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Create a new site.""" + from __future__ import print_function, unicode_literals import os import shutil @@ -54,6 +56,7 @@ SAMPLE_CONF = { 'BLOG_EMAIL': "joe@demo.site", 'BLOG_DESCRIPTION': "This is a demo site for Nikola.", 'PRETTY_URLS': False, + 'STRIP_INDEXES': False, 'DEFAULT_LANG': "en", 'TRANSLATIONS': """{ DEFAULT_LANG: "", @@ -64,6 +67,8 @@ SAMPLE_CONF = { 'TIMEZONE': 'UTC', 'COMMENT_SYSTEM': 'disqus', 'COMMENT_SYSTEM_ID': 'nikolademo', + 'CATEGORY_ALLOW_HIERARCHIES': False, + 'CATEGORY_OUTPUT_FLAT_HIERARCHY': False, 'TRANSLATIONS_PATTERN': DEFAULT_TRANSLATIONS_PATTERN, 'INDEX_READ_MORE_LINK': DEFAULT_INDEX_READ_MORE_LINK, 'RSS_READ_MORE_LINK': DEFAULT_RSS_READ_MORE_LINK, @@ -103,6 +108,7 @@ SAMPLE_CONF = { 'REDIRECTIONS': [], } + # Generate a list of supported languages here. # Ugly code follows. _suplang = {} @@ -154,8 +160,7 @@ SAMPLE_CONF['_SUPPORTED_COMMENT_SYSTEMS'] = '\n'.join(textwrap.wrap( def format_default_translations_config(additional_languages): - """Return the string to configure the TRANSLATIONS config variable to - make each additional language visible on the generated site.""" + """Adapt TRANSLATIONS setting for all additional languages.""" if not additional_languages: return SAMPLE_CONF["TRANSLATIONS"] lang_paths = [' DEFAULT_LANG: "",'] @@ -164,12 +169,12 @@ def format_default_translations_config(additional_languages): return "{{\n{0}\n}}".format("\n".join(lang_paths)) -def format_navigation_links(additional_languages, default_lang, messages): +def format_navigation_links(additional_languages, default_lang, messages, strip_indexes=False): """Return the string to configure NAVIGATION_LINKS.""" f = u"""\ {0}: ( ("{1}/archive.html", "{2[Archive]}"), - ("{1}/categories/index.html", "{2[Tags]}"), + ("{1}/categories/{3}", "{2[Tags]}"), ("{1}/rss.xml", "{2[RSS feed]}"), ),""" @@ -185,27 +190,32 @@ def format_navigation_links(additional_languages, default_lang, messages): fmsg[i] = i return fmsg + if strip_indexes: + index_html = '' + else: + index_html = 'index.html' + # handle the default language - pairs.append(f.format('DEFAULT_LANG', '', get_msg(default_lang))) + pairs.append(f.format('DEFAULT_LANG', '', get_msg(default_lang), index_html)) for l in additional_languages: - pairs.append(f.format(json.dumps(l, ensure_ascii=False), '/' + l, get_msg(l))) + pairs.append(f.format(json.dumps(l, ensure_ascii=False), '/' + l, get_msg(l), index_html)) return u'{{\n{0}\n}}'.format('\n\n'.join(pairs)) -# In order to ensure proper escaping, all variables but the three -# pre-formatted ones are handled by json.dumps(). +# In order to ensure proper escaping, all variables but the pre-formatted ones +# are handled by json.dumps(). def prepare_config(config): """Parse sample config with JSON.""" p = config.copy() - p.update(dict((k, json.dumps(v, ensure_ascii=False)) for k, v in p.items() - if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK', 'PRETTY_URLS'))) + p.update({k: json.dumps(v, ensure_ascii=False) for k, v in p.items() + if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK')}) # READ_MORE_LINKs require some special treatment. p['INDEX_READ_MORE_LINK'] = "'" + p['INDEX_READ_MORE_LINK'].replace("'", "\\'") + "'" p['RSS_READ_MORE_LINK'] = "'" + p['RSS_READ_MORE_LINK'].replace("'", "\\'") + "'" - # json would make that `true` instead of `True` - p['PRETTY_URLS'] = str(p['PRETTY_URLS']) + # fix booleans and None + p.update({k: str(v) for k, v in config.items() if isinstance(v, bool) or v is None}) return p @@ -239,11 +249,13 @@ class CommandInit(Command): @classmethod def copy_sample_site(cls, target): + """Copy sample site data to target directory.""" src = resource_filename('nikola', os.path.join('data', 'samplesite')) shutil.copytree(src, target) @staticmethod def create_configuration(target): + """Create configuration file.""" template_path = resource_filename('nikola', 'conf.py.in') conf_template = Template(filename=template_path) conf_path = os.path.join(target, 'conf.py') @@ -252,12 +264,14 @@ class CommandInit(Command): @staticmethod def create_configuration_to_string(): + """Return configuration file as a string.""" template_path = resource_filename('nikola', 'conf.py.in') conf_template = Template(filename=template_path) return conf_template.render(**prepare_config(SAMPLE_CONF)) @classmethod def create_empty_site(cls, target): + """Create an empty site with directories only.""" for folder in ('files', 'galleries', 'listings', 'posts', 'stories'): makedirs(os.path.join(target, folder)) @@ -295,7 +309,8 @@ class CommandInit(Command): SAMPLE_CONF['SITE_URL'] = answer def prettyhandler(default, toconf): - SAMPLE_CONF['PRETTY_URLS'] = ask_yesno('Enable pretty URLs (/page/ instead of /page.html) that don’t need web server configuration?', default=True) + SAMPLE_CONF['PRETTY_URLS'] = ask_yesno('Enable pretty URLs (/page/ instead of /page.html) that don\'t need web server configuration?', default=True) + SAMPLE_CONF['STRIP_INDEXES'] = SAMPLE_CONF['PRETTY_URLS'] def lhandler(default, toconf, show_header=True): if show_header: @@ -333,7 +348,7 @@ class CommandInit(Command): # not inherit from anywhere. try: messages = load_messages(['base'], tr, default) - SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages) + SAMPLE_CONF['NAVIGATION_LINKS'] = format_navigation_links(langs, default, messages, SAMPLE_CONF['STRIP_INDEXES']) except nikola.utils.LanguageNotFoundError as e: print(" ERROR: the language '{0}' is not supported.".format(e.lang)) print(" Are you sure you spelled the name correctly? Names are case-sensitive and need to be reproduced as-is (complete with the country specifier, if any).") diff --git a/nikola/plugins/command/install_theme.plugin b/nikola/plugins/command/install_theme.plugin index 54a91ff..8434f2e 100644 --- a/nikola/plugins/command/install_theme.plugin +++ b/nikola/plugins/command/install_theme.plugin @@ -1,10 +1,13 @@ [Core] -Name = install_theme -Module = install_theme +name = install_theme +module = install_theme [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Install a theme into the current site. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Install a theme into the current site. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py index 4937509..f02252e 100644 --- a/nikola/plugins/command/install_theme.py +++ b/nikola/plugins/command/install_theme.py @@ -24,10 +24,12 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Install a theme.""" + from __future__ import print_function import os import io -import json +import time import requests import pygments @@ -41,6 +43,7 @@ LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER) class CommandInstallTheme(Command): + """Install a theme.""" name = "install_theme" @@ -95,8 +98,13 @@ class CommandInstallTheme(Command): if name is None and not listing: LOGGER.error("This command needs either a theme name or the -l option.") return False - data = requests.get(url).text - data = json.loads(data) + try: + data = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + data = requests.get(url).json() if listing: print("Themes:") print("-------") @@ -122,11 +130,21 @@ class CommandInstallTheme(Command): LOGGER.notice('Remember to set THEME="{0}" in conf.py to use this theme.'.format(origname)) def do_install(self, name, data): + """Download and install a theme.""" if name in data: utils.makedirs(self.output_dir) - LOGGER.info("Downloading '{0}'".format(data[name])) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + zip_file = io.BytesIO() - zip_file.write(requests.get(data[name]).content) + zip_file.write(zip_data) LOGGER.info("Extracting '{0}' into themes/".format(name)) utils.extract_all(zip_file) dest_path = os.path.join(self.output_dir, name) diff --git a/nikola/plugins/command/new_page.plugin b/nikola/plugins/command/new_page.plugin index f078dd6..145a419 100644 --- a/nikola/plugins/command/new_page.plugin +++ b/nikola/plugins/command/new_page.plugin @@ -1,9 +1,13 @@ [Core] -Name = new_page -Module = new_page +name = new_page +module = new_page [Documentation] -Author = Roberto Alsina, Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = Create a new page. +author = Roberto Alsina, Chris Warrick +version = 1.0 +website = http://getnikola.com +description = Create a new page. + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py index 39a85bd..811e28b 100644 --- a/nikola/plugins/command/new_page.py +++ b/nikola/plugins/command/new_page.py @@ -24,12 +24,15 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Create a new page.""" + from __future__ import unicode_literals, print_function from nikola.plugin_categories import Command class CommandNewPage(Command): + """Create a new page.""" name = "new_page" diff --git a/nikola/plugins/command/new_post.plugin b/nikola/plugins/command/new_post.plugin index fec4b1d..d88469f 100644 --- a/nikola/plugins/command/new_post.plugin +++ b/nikola/plugins/command/new_post.plugin @@ -1,10 +1,13 @@ [Core] -Name = new_post -Module = new_post +name = new_post +module = new_post [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Create a new post. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Create a new post. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py index 5141c7e..f9fe3ff 100644 --- a/nikola/plugins/command/new_post.py +++ b/nikola/plugins/command/new_post.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Create a new post.""" + from __future__ import unicode_literals, print_function import io import datetime @@ -44,107 +46,8 @@ PAGELOGGER = utils.get_logger('new_page', utils.STDERR_HANDLER) LOGGER = POSTLOGGER -def filter_post_pages(compiler, is_post, compilers, post_pages, compiler_objs, compilers_raw): - """Given a compiler ("markdown", "rest"), and whether it's meant for - a post or a page, and compilers, return the correct entry from - post_pages.""" - - # First throw away all the post_pages with the wrong is_post - filtered = [entry for entry in post_pages if entry[3] == is_post] - - # These are the extensions supported by the required format - extensions = compilers.get(compiler) - if extensions is None: - if compiler in compiler_objs: - LOGGER.error("There is a {0} compiler available, but it's not set in your COMPILERS option.".format(compiler)) - LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) - else: - LOGGER.error('Unknown format {0}'.format(compiler)) - print_compilers(compilers_raw, post_pages, compiler_objs) - return False - - # Throw away the post_pages with the wrong extensions - filtered = [entry for entry in filtered if any([ext in entry[0] for ext in - extensions])] - - if not filtered: - type_name = "post" if is_post else "page" - LOGGER.error("Can't find a way, using your configuration, to create " - "a {0} in format {1}. You may want to tweak " - "COMPILERS or {2}S in conf.py".format( - type_name, compiler, type_name.upper())) - LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) - - return False - return filtered[0] - - -def print_compilers(compilers_raw, post_pages, compiler_objs): - """ - List all available compilers in a human-friendly format. - - :param compilers_raw: The compilers dict, mapping compiler names to tuples of extensions - :param post_pages: The post_pages structure - :param compilers_objs: Compiler objects - """ - - # We use compilers_raw, because the normal dict can contain - # garbage coming from the translation candidate implementation. - # Entries are in format: (name, extensions, used_in_post_pages) - parsed_compilers = {'used': [], 'unused': [], 'disabled': []} - - for compiler_name, compiler_obj in compiler_objs.items(): - fname = compiler_obj.friendly_name or compiler_name - if compiler_name not in compilers_raw: - parsed_compilers['disabled'].append((compiler_name, fname, (), False)) - else: - # stolen from filter_post_pages - extensions = compilers_raw[compiler_name] - filtered = [entry for entry in post_pages if any( - [ext in entry[0] for ext in extensions])] - if filtered: - parsed_compilers['used'].append((compiler_name, fname, extensions, True)) - else: - parsed_compilers['unused'].append((compiler_name, fname, extensions, False)) - - # Sort compilers alphabetically by name, just so it’s prettier (and - # deterministic) - parsed_compilers['used'].sort(key=operator.itemgetter(0)) - parsed_compilers['unused'].sort(key=operator.itemgetter(0)) - parsed_compilers['disabled'].sort(key=operator.itemgetter(0)) - - # We also group the compilers by status for readability. - parsed_list = parsed_compilers['used'] + parsed_compilers['unused'] + parsed_compilers['disabled'] - - print("Available input formats:\n") - - name_width = max([len(i[0]) for i in parsed_list] + [4]) # 4 == len('NAME') - fname_width = max([len(i[1]) for i in parsed_list] + [11]) # 11 == len('DESCRIPTION') - - print((' {0:<' + str(name_width) + '} {1:<' + str(fname_width) + '} EXTENSIONS\n').format('NAME', 'DESCRIPTION')) - - for name, fname, extensions, used in parsed_list: - flag = ' ' if used else '!' - flag = flag if extensions else '~' - - extensions = ', '.join(extensions) if extensions else '(disabled: not in COMPILERS)' - - print(('{flag}{name:<' + str(name_width) + '} {fname:<' + str(fname_width) + '} {extensions}').format(flag=flag, name=name, fname=fname, extensions=extensions)) - - print(""" -More compilers are available in the Plugins Index. - -Compilers marked with ! and ~ require additional configuration: - ! not in the PAGES/POSTS tuples (unused) - ~ not in the COMPILERS dict (disabled) -Read more: {0}""".format(COMPILERS_DOC_LINK)) - - def get_default_compiler(is_post, compilers, post_pages): - """Given compilers and post_pages, return a reasonable - default compiler for this kind of post/page. - """ - + """Given compilers and post_pages, return a reasonable default compiler for this kind of post/page.""" # First throw away all the post_pages with the wrong is_post filtered = [entry for entry in post_pages if entry[3] == is_post] @@ -159,7 +62,7 @@ def get_default_compiler(is_post, compilers, post_pages): def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): - """Returns a date stamp, given a recurrence rule. + """Return a date stamp, given a recurrence rule. schedule - bool: whether to use the recurrence rule or not @@ -177,7 +80,6 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): whether to force ISO 8601 dates (instead of locale-specific ones) """ - if tz is None: tz = dateutil.tz.tzlocal() date = now = datetime.datetime.now(tz) @@ -212,6 +114,7 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False): class CommandNewPost(Command): + """Create a new post.""" name = "new_post" @@ -333,7 +236,7 @@ class CommandNewPost(Command): wants_available = options['available-formats'] if wants_available: - print_compilers(self.site.config['_COMPILERS_RAW'], self.site.config['post_pages'], self.site.compilers) + self.print_compilers() return if is_page: @@ -360,17 +263,13 @@ class CommandNewPost(Command): if content_format not in compiler_names: LOGGER.error("Unknown {0} format {1}, maybe you need to install a plugin?".format(content_type, content_format)) - print_compilers(self.site.config['_COMPILERS_RAW'], self.site.config['post_pages'], self.site.compilers) + self.print_compilers() return compiler_plugin = self.site.plugin_manager.getPluginByName( content_format, "PageCompiler").plugin_object # Guess where we should put this - entry = filter_post_pages(content_format, is_post, - self.site.config['COMPILERS'], - self.site.config['post_pages'], - self.site.compilers, - self.site.config['_COMPILERS_RAW']) + entry = self.filter_post_pages(content_format, is_post) if entry is False: return 1 @@ -497,3 +396,122 @@ class CommandNewPost(Command): subprocess.call(to_run) else: LOGGER.error('$EDITOR not set, cannot edit the post. Please do it manually.') + + def filter_post_pages(self, compiler, is_post): + """Return the correct entry from post_pages. + + Information based on: + * selected compilers + * available compilers + * post/page status + """ + compilers = self.site.config['COMPILERS'] + post_pages = self.site.config['post_pages'] + compiler_objs = self.site.compilers + + # First throw away all the post_pages with the wrong is_post + filtered = [entry for entry in post_pages if entry[3] == is_post] + + # These are the extensions supported by the required format + extensions = compilers.get(compiler) + if extensions is None: + if compiler in compiler_objs: + LOGGER.error("There is a {0} compiler available, but it's not set in your COMPILERS option.".format(compiler)) + LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) + else: + LOGGER.error('Unknown format {0}'.format(compiler)) + self.print_compilers() + return False + + # Throw away the post_pages with the wrong extensions + filtered = [entry for entry in filtered if any([ext in entry[0] for ext in + extensions])] + + if not filtered: + type_name = "post" if is_post else "page" + LOGGER.error("Can't find a way, using your configuration, to create " + "a {0} in format {1}. You may want to tweak " + "COMPILERS or {2}S in conf.py".format( + type_name, compiler, type_name.upper())) + LOGGER.info("Read more: {0}".format(COMPILERS_DOC_LINK)) + + return False + return filtered[0] + + def print_compilers(self): + """List all available compilers in a human-friendly format.""" + # We use compilers_raw, because the normal dict can contain + # garbage coming from the translation candidate implementation. + # Entries are in format: (name, extensions, used_in_post_pages) + + compilers_raw = self.site.config['_COMPILERS_RAW'] + + used_compilers = [] + unused_compilers = [] + disabled_compilers = [] + + for name, plugin in self.site.compilers.items(): + if name in compilers_raw: + used_compilers.append([ + name, + plugin.friendly_name or name, + compilers_raw[name], + True + ]) + else: + disabled_compilers.append([ + name, + plugin.friendly_name or name, + (), + False + ]) + + for name, (_, _, pi) in self.site.disabled_compilers.items(): + if pi.details.has_option('Nikola', 'Friendlyname'): + f_name = pi.details.get('Nikola', 'Friendlyname') + else: + f_name = name + if name in compilers_raw: + unused_compilers.append([ + name, + f_name, + compilers_raw[name], + False + ]) + else: + disabled_compilers.append([ + name, + f_name, + (), + False + ]) + + used_compilers.sort(key=operator.itemgetter(0)) + unused_compilers.sort(key=operator.itemgetter(0)) + disabled_compilers.sort(key=operator.itemgetter(0)) + + # We also group the compilers by status for readability. + parsed_list = used_compilers + unused_compilers + disabled_compilers + + print("Available input formats:\n") + + name_width = max([len(i[0]) for i in parsed_list] + [4]) # 4 == len('NAME') + fname_width = max([len(i[1]) for i in parsed_list] + [11]) # 11 == len('DESCRIPTION') + + print((' {0:<' + str(name_width) + '} {1:<' + str(fname_width) + '} EXTENSIONS\n').format('NAME', 'DESCRIPTION')) + + for name, fname, extensions, used in parsed_list: + flag = ' ' if used else '!' + flag = flag if extensions else '~' + + extensions = ', '.join(extensions) if extensions else '(disabled: not in COMPILERS)' + + print(('{flag}{name:<' + str(name_width) + '} {fname:<' + str(fname_width) + '} {extensions}').format(flag=flag, name=name, fname=fname, extensions=extensions)) + + print(""" + More compilers are available in the Plugins Index. + + Compilers marked with ! and ~ require additional configuration: + ! not in the PAGES/POSTS tuples (unused) + ~ not in the COMPILERS dict (disabled) + Read more: {0}""".format(COMPILERS_DOC_LINK)) diff --git a/nikola/plugins/command/orphans.plugin b/nikola/plugins/command/orphans.plugin index f491eaf..669429d 100644 --- a/nikola/plugins/command/orphans.plugin +++ b/nikola/plugins/command/orphans.plugin @@ -1,10 +1,13 @@ [Core] -Name = orphans -Module = orphans +name = orphans +module = orphans [Documentation] -Author = Roberto Alsina, Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = List all orphans +author = Roberto Alsina, Chris Warrick +version = 1.0 +website = http://getnikola.com +description = List all orphans + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py index f550e17..b12cc67 100644 --- a/nikola/plugins/command/orphans.py +++ b/nikola/plugins/command/orphans.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""List all orphans.""" + from __future__ import print_function import os @@ -32,6 +34,9 @@ from nikola.plugins.command.check import real_scan_files class CommandOrphans(Command): + + """List all orphans.""" + name = "orphans" doc_purpose = "list all orphans" doc_description = """\ @@ -41,5 +46,6 @@ but are not generated by Nikola. Output contains filenames only (it is passable to `xargs rm` or the like).""" def _execute(self, options, args): + """Run the orphans command.""" orphans = real_scan_files(self.site)[0] print('\n'.join([p for p in orphans if not os.path.isdir(p)])) diff --git a/nikola/plugins/command/plugin.plugin b/nikola/plugins/command/plugin.plugin index 2815caa..d44dcf3 100644 --- a/nikola/plugins/command/plugin.plugin +++ b/nikola/plugins/command/plugin.plugin @@ -1,10 +1,13 @@ [Core] -Name = plugin -Module = plugin +name = plugin +module = plugin [Documentation] -Author = Roberto Alsina and Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = Manage Nikola plugins +author = Roberto Alsina and Chris Warrick +version = 1.0 +website = http://getnikola.com +description = Manage Nikola plugins + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index 56eb1d7..f892ee9 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -24,12 +24,14 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Manage plugins.""" + from __future__ import print_function import io import os import shutil import subprocess -import sys +import time import requests import pygments @@ -43,6 +45,7 @@ LOGGER = utils.get_logger('plugin', utils.STDERR_HANDLER) class CommandPlugin(Command): + """Manage plugins.""" json = None @@ -119,6 +122,7 @@ class CommandPlugin(Command): upgrade = options.get('upgrade') list_available = options.get('list') list_installed = options.get('list_installed') + show_install_notes = options.get('show_install_notes', True) command_count = [bool(x) for x in ( install, uninstall, @@ -127,37 +131,42 @@ class CommandPlugin(Command): list_installed)].count(True) if command_count > 1 or command_count == 0: print(self.help()) - return + return 2 - if not self.site.configured and not user_mode and install: - LOGGER.notice('No site found, assuming --user') - user_mode = True - - if user_mode: - self.output_dir = os.path.expanduser('~/.nikola/plugins') + if options.get('output_dir') is not None: + self.output_dir = options.get('output_dir') else: - self.output_dir = 'plugins' + if not self.site.configured and not user_mode and install: + LOGGER.notice('No site found, assuming --user') + user_mode = True + + if user_mode: + self.output_dir = os.path.expanduser('~/.nikola/plugins') + else: + self.output_dir = 'plugins' if list_available: - self.list_available(url) + return self.list_available(url) elif list_installed: - self.list_installed() + return self.list_installed() elif upgrade: - self.do_upgrade(url) + return self.do_upgrade(url) elif uninstall: - self.do_uninstall(uninstall) + return self.do_uninstall(uninstall) elif install: - self.do_install(url, install) + return self.do_install(url, install, show_install_notes) def list_available(self, url): + """List all available plugins.""" data = self.get_json(url) print("Available Plugins:") print("------------------") for plugin in sorted(data.keys()): print(plugin) - return True + return 0 def list_installed(self): + """List installed plugins.""" plugins = [] for plugin in self.site.plugin_manager.getAllPlugins(): p = plugin.path @@ -170,8 +179,10 @@ class CommandPlugin(Command): plugins.sort() for name, path in plugins: print('{0} at {1}'.format(name, path)) + return 0 def do_upgrade(self, url): + """Upgrade all installed plugins.""" LOGGER.warning('This is not very smart, it just reinstalls some plugins and hopes for the best') data = self.get_json(url) plugins = [] @@ -194,18 +205,29 @@ class CommandPlugin(Command): break elif tail == '': LOGGER.error("Can't find the plugins folder for path: {0}".format(p)) - return False + return 1 else: path = tail self.do_install(url, name) + return 0 - def do_install(self, url, name): + def do_install(self, url, name, show_install_notes=True): + """Download and install a plugin.""" data = self.get_json(url) if name in data: utils.makedirs(self.output_dir) - LOGGER.info('Downloading: ' + data[name]) + url = data[name] + LOGGER.info("Downloading '{0}'".format(url)) + try: + zip_data = requests.get(url).content + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + zip_data = requests.get(url).content + zip_file = io.BytesIO() - zip_file.write(requests.get(data[name]).content) + zip_file.write(zip_data) LOGGER.info('Extracting: {0} into {1}/'.format(name, self.output_dir)) utils.extract_all(zip_file, self.output_dir) dest_path = os.path.join(self.output_dir, name) @@ -214,13 +236,13 @@ class CommandPlugin(Command): plugin_path = utils.get_plugin_path(name) except: LOGGER.error("Can't find plugin " + name) - return False + return 1 utils.makedirs(self.output_dir) dest_path = os.path.join(self.output_dir, name) if os.path.exists(dest_path): LOGGER.error("{0} is already installed".format(name)) - return False + return 1 LOGGER.info('Copying {0} into plugins'.format(plugin_path)) shutil.copytree(plugin_path, dest_path) @@ -256,7 +278,7 @@ class CommandPlugin(Command): print('You have to install those yourself or through a package ' 'manager.') confpypath = os.path.join(dest_path, 'conf.py.sample') - if os.path.exists(confpypath): + if os.path.exists(confpypath) and show_install_notes: LOGGER.notice('This plugin has a sample config file. Integrate it with yours in order to make this plugin work!') print('Contents of the conf.py.sample file:\n') with io.open(confpypath, 'r', encoding='utf-8') as fh: @@ -266,9 +288,10 @@ class CommandPlugin(Command): 4 * ' ')) else: print(utils.indent(fh.read(), 4 * ' ')) - return True + return 0 def do_uninstall(self, name): + """Uninstall a plugin.""" for plugin in self.site.plugin_manager.getAllPlugins(): # FIXME: this is repeated thrice p = plugin.path if os.path.isdir(p): @@ -278,16 +301,23 @@ class CommandPlugin(Command): if name == plugin.name: # Uninstall this one LOGGER.warning('About to uninstall plugin: {0}'.format(name)) LOGGER.warning('This will delete {0}'.format(p)) - inpf = raw_input if sys.version_info[0] == 2 else input - sure = inpf('Are you sure? [y/n] ') - if sure.lower().startswith('y'): + sure = utils.ask_yesno('Are you sure?') + if sure: LOGGER.warning('Removing {0}'.format(p)) shutil.rmtree(p) - return True + return 0 + return 1 LOGGER.error('Unknown plugin: {0}'.format(name)) - return False + return 1 def get_json(self, url): + """Download the JSON file with all plugins.""" if self.json is None: - self.json = requests.get(url).json() + try: + self.json = requests.get(url).json() + except requests.exceptions.SSLError: + LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") + time.sleep(1) + url = url.replace('https', 'http', 1) + self.json = requests.get(url).json() return self.json diff --git a/nikola/plugins/command/rst2html.plugin b/nikola/plugins/command/rst2html.plugin index 0d0d3b0..02c9276 100644 --- a/nikola/plugins/command/rst2html.plugin +++ b/nikola/plugins/command/rst2html.plugin @@ -1,9 +1,13 @@ [Core] -Name = rst2html -Module = rst2html +name = rst2html +module = rst2html [Documentation] -Author = Chris Warrick -Version = 1.0 -Website = http://getnikola.com -Description = Compile reStructuredText to HTML using the Nikola architecture +author = Chris Warrick +version = 1.0 +website = http://getnikola.com +description = Compile reStructuredText to HTML using the Nikola architecture + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py index 342aaeb..06afffd 100644 --- a/nikola/plugins/command/rst2html/__init__.py +++ b/nikola/plugins/command/rst2html/__init__.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Compile reStructuredText to HTML, using Nikola architecture.""" + from __future__ import unicode_literals, print_function import io @@ -34,6 +36,7 @@ from nikola.plugin_categories import Command class CommandRst2Html(Command): + """Compile reStructuredText to HTML, using Nikola architecture.""" name = "rst2html" diff --git a/nikola/plugins/command/serve.plugin b/nikola/plugins/command/serve.plugin index 0c1176d..aca71ec 100644 --- a/nikola/plugins/command/serve.plugin +++ b/nikola/plugins/command/serve.plugin @@ -1,10 +1,13 @@ [Core] -Name = serve -Module = serve +name = serve +module = serve [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Start test server. +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Start test server. + +[Nikola] +plugincategory = Command diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py index 0e4d01f..0441c93 100644 --- a/nikola/plugins/command/serve.py +++ b/nikola/plugins/command/serve.py @@ -24,8 +24,11 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Start test server.""" + from __future__ import print_function import os +import re import socket import webbrowser try: @@ -35,16 +38,25 @@ except ImportError: from http.server import HTTPServer # NOQA from http.server import SimpleHTTPRequestHandler # NOQA +try: + from StringIO import StringIO +except ImportError: + from io import BytesIO as StringIO # NOQA + + from nikola.plugin_categories import Command -from nikola.utils import get_logger +from nikola.utils import get_logger, STDERR_HANDLER class IPv6Server(HTTPServer): + """An IPv6 HTTPServer.""" + address_family = socket.AF_INET6 class CommandServe(Command): + """Start test server.""" name = "serve" @@ -70,6 +82,14 @@ class CommandServe(Command): 'help': 'Address to bind (default: 0.0.0.0 – all local IPv4 interfaces)', }, { + 'name': 'detach', + 'short': 'd', + 'long': 'detach', + 'type': bool, + 'default': False, + 'help': 'Detach from TTY (work in the background)', + }, + { 'name': 'browser', 'short': 'b', 'long': 'browser', @@ -89,7 +109,7 @@ class CommandServe(Command): def _execute(self, options, args): """Start test server.""" - self.logger = get_logger('serve', self.site.loghandlers) + self.logger = get_logger('serve', STDERR_HANDLER) out_dir = self.site.config['OUTPUT_FOLDER'] if not os.path.isdir(out_dir): self.logger.error("Missing '{0}' folder?".format(out_dir)) @@ -117,16 +137,42 @@ class CommandServe(Command): server_url = "http://{0}:{1}/".format(*sa) self.logger.info("Opening {0} in the default web browser...".format(server_url)) webbrowser.open(server_url) - try: - httpd.serve_forever() - except KeyboardInterrupt: - self.logger.info("Server is shutting down.") - return 130 + if options['detach']: + OurHTTPRequestHandler.quiet = True + try: + pid = os.fork() + if pid == 0: + httpd.serve_forever() + else: + self.logger.info("Detached with PID {0}. Run `kill {0}` to stop the server.".format(pid)) + except AttributeError as e: + if os.name == 'nt': + self.logger.warning("Detaching is not available on Windows, server is running in the foreground.") + else: + raise e + else: + try: + httpd.serve_forever() + except KeyboardInterrupt: + self.logger.info("Server is shutting down.") + return 130 class OurHTTPRequestHandler(SimpleHTTPRequestHandler): + + """A request handler, modified for Nikola.""" + extensions_map = dict(SimpleHTTPRequestHandler.extensions_map) extensions_map[""] = "text/plain" + quiet = False + + def log_message(self, *args): + """Log messages. Or not, depending on a setting.""" + if self.quiet: + return + else: + # Old-style class in Python 2.7, cannot use super() + return SimpleHTTPRequestHandler.log_message(self, *args) # NOTICE: this is a patched version of send_head() to disable all sorts of # caching. `nikola serve` is a development server, hence caching should @@ -182,14 +228,31 @@ class OurHTTPRequestHandler(SimpleHTTPRequestHandler): except IOError: self.send_error(404, "File not found") return None + + filtered_bytes = None + if ctype == 'text/html': + # Comment out any <base> to allow local resolution of relative URLs. + data = f.read().decode('utf8') + f.close() + data = re.sub(r'<base\s([^>]*)>', '<!--base \g<1>-->', data, re.IGNORECASE) + data = data.encode('utf8') + f = StringIO() + f.write(data) + filtered_bytes = len(data) + f.seek(0) + self.send_response(200) self.send_header("Content-type", ctype) if os.path.splitext(path)[1] == '.svgz': # Special handling for svgz to make it work nice with browsers. self.send_header("Content-Encoding", 'gzip') - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + + if filtered_bytes is None: + fs = os.fstat(f.fileno()) + self.send_header('Content-Length', str(fs[6])) + else: + self.send_header('Content-Length', filtered_bytes) + # begin no-cache patch # For standard requests. self.send_header("Cache-Control", "no-cache, no-store, " diff --git a/nikola/plugins/command/status.plugin b/nikola/plugins/command/status.plugin index e02da8b..91390d2 100644 --- a/nikola/plugins/command/status.plugin +++ b/nikola/plugins/command/status.plugin @@ -1,9 +1,13 @@ [Core] -Name = status -Module = status +name = status +module = status [Documentation] -Author = Daniel Aleksandersen -Version = 1.0 -Website = https://getnikola.com -Description = Site status +author = Daniel Aleksandersen +version = 1.0 +website = https://getnikola.com +description = Site status + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py index b8a6a60..55e7f95 100644 --- a/nikola/plugins/command/status.py +++ b/nikola/plugins/command/status.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Display site status.""" + from __future__ import print_function import io import os @@ -33,8 +35,10 @@ from dateutil.tz import gettz, tzlocal from nikola.plugin_categories import Command -class CommandDeploy(Command): - """ Site status. """ +class CommandStatus(Command): + + """Display site status.""" + name = "status" doc_purpose = "display site status" @@ -69,7 +73,7 @@ class CommandDeploy(Command): ] def _execute(self, options, args): - + """Display site status.""" self.site.scan_posts() timestamp_path = os.path.join(self.site.config["CACHE_FOLDER"], "lastdeploy") @@ -128,6 +132,7 @@ class CommandDeploy(Command): print("{0} posts in total, {1} scheduled, and {2} drafts.".format(posts_count, len(posts_scheduled), len(posts_drafts))) def human_time(self, dt): + """Translate time into a human-friendly representation.""" days = dt.days hours = dt.seconds / 60 // 60 minutes = dt.seconds / 60 - (hours * 60) diff --git a/nikola/plugins/command/version.plugin b/nikola/plugins/command/version.plugin index a3f58e8..4708bdb 100644 --- a/nikola/plugins/command/version.plugin +++ b/nikola/plugins/command/version.plugin @@ -1,9 +1,13 @@ [Core] -Name = version -Module = version +name = version +module = version [Documentation] -Author = Roberto Alsina -Version = 1.0 -Website = http://getnikola.com -Description = Show nikola version +author = Roberto Alsina +version = 1.0 +website = http://getnikola.com +description = Show nikola version + +[Nikola] +plugincategory = Command + diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py index b6520d7..ad08f64 100644 --- a/nikola/plugins/command/version.py +++ b/nikola/plugins/command/version.py @@ -24,6 +24,8 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Print Nikola version.""" + from __future__ import print_function import lxml @@ -36,7 +38,8 @@ URL = 'https://pypi.python.org/pypi?:action=doap&name=Nikola' class CommandVersion(Command): - """Print the version.""" + + """Print Nikola version.""" name = "version" |