Merge tag 'upstream/7.1.0'

Upstream version 7.1.0

11 files changed, 95 insertions, 68 deletions

diff --git a/nikola/plugins/command/auto.py b/nikola/plugins/command/auto.py
index c46e0a3..7f3f66f 100644
--- a/nikola/plugins/command/auto.py
+++ b/nikola/plugins/command/auto.py
@@ -28,7 +28,6 @@ from __future__ import print_function, unicode_literals
 
 import os
 import subprocess
-import webbrowser
 
 from nikola.plugin_categories import Command
 from nikola.utils import req_missing
@@ -61,7 +60,7 @@ class CommandAuto(Command):
         try:
             from livereload import Server
         except ImportError:
-            req_missing(['livereload==2.1.0'], 'use the "auto" command')
+            req_missing(['livereload'], 'use the "auto" command')
             return
 
         # Run an initial build so we are up-to-date
@@ -81,6 +80,8 @@ class CommandAuto(Command):
 
         out_folder = self.site.config['OUTPUT_FOLDER']
         if options and options.get('browser'):
-            webbrowser.open('http://localhost:{0}'.format(port))
+            browser = True
+        else:
+            browser = False
 
-        server.serve(port, None, out_folder)
+            server.serve(port, None, out_folder, True, browser)
diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py
index 871a5ce..e65413b 100644
--- a/nikola/plugins/command/bootswatch_theme.py
+++ b/nikola/plugins/command/bootswatch_theme.py
@@ -82,9 +82,9 @@ class CommandBootswatchTheme(Command):
 
         # See if we need bootswatch for bootstrap v2 or v3
         themes = utils.get_theme_chain(parent)
-        if 'bootstrap3' not in themes:
+        if 'bootstrap3' not in themes or 'bootstrap3-jinja' not in themes:
             version = '2'
-        elif 'bootstrap' not in themes:
+        elif 'bootstrap' not in themes or 'bootstrap-jinja' not in themes:
             LOGGER.warn('"bootswatch_theme" only makes sense for themes that use bootstrap')
         elif 'bootstrap3-gradients' in themes or 'bootstrap3-gradients-jinja' in themes:
             LOGGER.warn('"bootswatch_theme" doesn\'t work well with the bootstrap3-gradients family')
diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py
index 76571a0..bd254f4 100644
--- a/nikola/plugins/command/check.py
+++ b/nikola/plugins/command/check.py
@@ -30,9 +30,9 @@ import re
 import sys
 try:
     from urllib import unquote
-    from urlparse import urlparse
+    from urlparse import urlparse, urljoin, urldefrag
 except ImportError:
-    from urllib.parse import unquote, urlparse  # NOQA
+    from urllib.parse import unquote, urlparse, urljoin, urldefrag  # NOQA
 
 import lxml.html
 
@@ -63,6 +63,15 @@ def real_scan_files(site):
     return (only_on_output, only_on_input)
 
 
+def fs_relpath_from_url_path(url_path):
+    """Expects as input an urlparse(s).path"""
+    url_path = unquote(url_path)
+    # in windows relative paths don't begin with os.sep
+    if sys.platform == 'win32' and len(url_path):
+        url_path = url_path[1:].replace('/', '\\')
+    return url_path
+
+
 class CommandCheck(Command):
     """Check the generated site."""
 
@@ -142,6 +151,8 @@ class CommandCheck(Command):
         self.existing_targets.add(self.site.config['SITE_URL'])
         self.existing_targets.add(self.site.config['BASE_URL'])
         url_type = self.site.config['URL_TYPE']
+        if url_type == 'absolute':
+            url_netloc_to_root = urlparse(self.site.config['SITE_URL']).path
         try:
             filename = task.split(":")[-1]
             d = lxml.html.fromstring(open(filename).read())
@@ -149,6 +160,7 @@ class CommandCheck(Command):
                 target = l[0].attrib[l[1]]
                 if target == "#":
                     continue
+                target, _ = urldefrag(target)
                 parsed = urlparse(target)
 
                 # Absolute links when using only paths, skip.
@@ -159,24 +171,20 @@ class CommandCheck(Command):
                 if (parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc:
                     continue
 
-                if parsed.fragment:
-                    target = target.split('#')[0]
                 if url_type == 'rel_path':
                     target_filename = os.path.abspath(
                         os.path.join(os.path.dirname(filename), unquote(target)))
 
                 elif url_type in ('full_path', 'absolute'):
-                    target_filename = os.path.abspath(
-                        os.path.join(os.path.dirname(filename), parsed.path))
-                    if parsed.path in ['', '/']:
-                        target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], self.site.config['INDEX_FILE'])
-                    elif parsed.path.endswith('/'):  # abspath removes trailing slashes
-                        target_filename += '/{0}'.format(self.site.config['INDEX_FILE'])
-                    if target_filename.startswith(base_url.path):
-                        target_filename = target_filename[len(base_url.path):]
-                    target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], target_filename)
-                    if parsed.path in ['', '/']:
-                        target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], self.site.config['INDEX_FILE'])
+                    if url_type == 'absolute':
+                        # convert to 'full_path' case, ie url relative to root
+                        url_rel_path = target.path[len(url_netloc_to_root):]
+                    else:
+                        url_rel_path = target.path
+                    if url_rel_path == '' or url_rel_path.endswith('/'):
+                        url_rel_path = urljoin(url_rel_path, self.site.config['INDEX_FILE'])
+                    fs_rel_path = fs_relpath_from_url_path(url_rel_path)
+                    target_filename = os.path.join(self.site.config['OUTPUT_FOLDER'], fs_rel_path)
 
                 if any(re.match(x, target_filename) for x in self.whitelist):
                     continue
diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py
index 1bec1d3..fde43fa 100644
--- a/nikola/plugins/command/deploy.py
+++ b/nikola/plugins/command/deploy.py
@@ -25,8 +25,9 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from __future__ import print_function
-import codecs
+import io
 from datetime import datetime
+from dateutil.tz import gettz
 import os
 import sys
 import subprocess
@@ -35,7 +36,7 @@ import time
 from blinker import signal
 
 from nikola.plugin_categories import Command
-from nikola.utils import remove_file, get_logger
+from nikola.utils import get_logger, remove_file, unicode_str
 
 
 class CommandDeploy(Command):
@@ -84,7 +85,7 @@ class CommandDeploy(Command):
 
         self.logger.info("Successful deployment")
         try:
-            with codecs.open(timestamp_path, 'rb', 'utf8') as inf:
+            with io.open(timestamp_path, 'r', encoding='utf8') as inf:
                 last_deploy = datetime.strptime(inf.read().strip(), "%Y-%m-%dT%H:%M:%S.%f")
                 clean = False
         except (IOError, Exception) as e:
@@ -96,8 +97,8 @@ class CommandDeploy(Command):
         self._emit_deploy_event(last_deploy, new_deploy, clean, undeployed_posts)
 
         # Store timestamp of successful deployment
-        with codecs.open(timestamp_path, 'wb+', 'utf8') as outf:
-            outf.write(new_deploy.isoformat())
+        with io.open(timestamp_path, 'w+', encoding='utf8') as outf:
+            outf.write(unicode_str(new_deploy.isoformat()))
 
     def _emit_deploy_event(self, last_deploy, new_deploy, clean=False, undeployed=None):
         """ Emit events for all timeline entries newer than last deploy.
@@ -120,9 +121,12 @@ class CommandDeploy(Command):
             'undeployed': undeployed
         }
 
+        if last_deploy.tzinfo is None:
+            last_deploy = last_deploy.replace(tzinfo=gettz('UTC'))
+
         deployed = [
             entry for entry in self.site.timeline
-            if entry.date > last_deploy.replace(tzinfo=self.site.tzinfo) and entry not in undeployed
+            if entry.date > last_deploy and entry not in undeployed
         ]
 
         event['deployed'] = deployed
diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py
index d4dd8c5..13da48c 100644
--- a/nikola/plugins/command/github_deploy.py
+++ b/nikola/plugins/command/github_deploy.py
@@ -135,9 +135,10 @@ class CommandGitHubDeploy(Command):
         )
 
         commands = [
+            ['git', 'pull', remote, '%s:%s' % (deploy, deploy)],
             ['git', 'add', '-A'],
             ['git', 'commit', '-m', commit_message],
-            ['git', 'push', '-f', remote, '%s:%s' % (deploy, deploy)],
+            ['git', 'push', remote, '%s:%s' % (deploy, deploy)],
             ['git', 'checkout', source],
         ]
 
diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py
index 8ddc8c7..1af4083 100644
--- a/nikola/plugins/command/import_wordpress.py
+++ b/nikola/plugins/command/import_wordpress.py
@@ -158,6 +158,7 @@ class CommandImportWordpress(Command, ImportMixin):
 
         channel = self.get_channel_from_file(self.wordpress_export_file)
         self.context = self.populate_context(channel)
+        self.base_dir = urlparse(self.context['BASE_URL']).path
         conf_template = self.generate_base_site()
 
         # If user  has specified a custom pattern for translation files we
@@ -323,13 +324,15 @@ class CommandImportWordpress(Command, ImportMixin):
                 # your blogging into another site or system its not.
                 # Why don't they just use JSON?
                 if sys.version_info[0] == 2:
-                    metadata = phpserialize.loads(utils.sys_encode(meta_value.text))
-                    size_key = 'sizes'
-                    file_key = 'file'
+                    try:
+                        metadata = phpserialize.loads(utils.sys_encode(meta_value.text))
+                    except ValueError:
+                        # local encoding might be wrong sometimes
+                        metadata = phpserialize.loads(meta_value.text.encode('utf-8'))
                 else:
-                    metadata = phpserialize.loads(meta_value.text.encode('UTF-8'))
-                    size_key = b'sizes'
-                    file_key = b'file'
+                    metadata = phpserialize.loads(meta_value.text.encode('utf-8'))
+                size_key = b'sizes'
+                file_key = b'file'
 
                 if size_key not in metadata:
                     continue
@@ -385,26 +388,34 @@ class CommandImportWordpress(Command, ImportMixin):
         # link is something like http://foo.com/2012/09/01/hello-world/
         # So, take the path, utils.slugify it, and that's our slug
         link = get_text_tag(item, 'link', None)
-        path = unquote(urlparse(link).path.strip('/'))
+        parsed = urlparse(link)
+        path = unquote(parsed.path.strip('/'))
 
         # In python 2, path is a str. slug requires a unicode
         # object. According to wikipedia, unquoted strings will
         # usually be UTF8
         if isinstance(path, utils.bytes_str):
             path = path.decode('utf8')
+
+        # Cut out the base directory.
+        if path.startswith(self.base_dir.strip('/')):
+            path = path.replace(self.base_dir.strip('/'), '', 1)
+
         pathlist = path.split('/')
-        if len(pathlist) > 1:
-            out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
-        slug = utils.slugify(pathlist[-1])
-        if not slug:  # it happens if the post has no "nice" URL
+        if parsed.query:  # if there are no nice URLs and query strings are used
+            out_folder = os.path.join(*([out_folder] + pathlist))
             slug = get_text_tag(
                 item, '{{{0}}}post_name'.format(wordpress_namespace), None)
-        if not slug:  # it *may* happen
-            slug = get_text_tag(
-                item, '{{{0}}}post_id'.format(wordpress_namespace), None)
-        if not slug:  # should never happen
-            LOGGER.error("Error converting post:", title)
-            return
+            if not slug:  # it *may* happen
+                slug = get_text_tag(
+                    item, '{{{0}}}post_id'.format(wordpress_namespace), None)
+            if not slug:  # should never happen
+                LOGGER.error("Error converting post:", title)
+                return
+        else:
+            if len(pathlist) > 1:
+                out_folder = os.path.join(*([out_folder] + pathlist[:-1]))
+            slug = utils.slugify(pathlist[-1])
 
         description = get_text_tag(item, 'description', '')
         post_date = get_text_tag(
@@ -440,8 +451,9 @@ class CommandImportWordpress(Command, ImportMixin):
             LOGGER.notice('Draft "{0}" will not be imported.'.format(title))
         elif content.strip():
             # If no content is found, no files are written.
-            self.url_map[link] = (self.context['SITE_URL'] + out_folder + '/'
-                                  + slug + '.html')
+            self.url_map[link] = (self.context['SITE_URL'] +
+                                  out_folder.rstrip('/') + '/' + slug +
+                                  '.html').replace(os.sep, '/')
             if hasattr(self, "separate_qtranslate_content") \
                and self.separate_qtranslate_content:
                 content_translations = separate_qtranslate_content(content)
diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py
index 8fb15e0..a8b60db 100644
--- a/nikola/plugins/command/init.py
+++ b/nikola/plugins/command/init.py
@@ -27,7 +27,7 @@
 from __future__ import print_function, unicode_literals
 import os
 import shutil
-import codecs
+import io
 import json
 import textwrap
 import datetime
@@ -242,7 +242,7 @@ class CommandInit(Command):
         template_path = resource_filename('nikola', 'conf.py.in')
         conf_template = Template(filename=template_path)
         conf_path = os.path.join(target, 'conf.py')
-        with codecs.open(conf_path, 'w+', 'utf8') as fd:
+        with io.open(conf_path, 'w+', encoding='utf8') as fd:
             fd.write(conf_template.render(**prepare_config(SAMPLE_CONF)))
 
     @classmethod
diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py
index 859bd56..5397772 100644
--- a/nikola/plugins/command/install_theme.py
+++ b/nikola/plugins/command/install_theme.py
@@ -26,10 +26,9 @@
 
 from __future__ import print_function
 import os
-import codecs
+import io
 import json
 import shutil
-from io import BytesIO
 
 import pygments
 from pygments.lexers import PythonLexer
@@ -137,7 +136,7 @@ class CommandInstallTheme(Command):
         if name in data:
             utils.makedirs(self.output_dir)
             LOGGER.info('Downloading: ' + data[name])
-            zip_file = BytesIO()
+            zip_file = io.BytesIO()
             zip_file.write(requests.get(data[name]).content)
             LOGGER.info('Extracting: {0} into themes'.format(name))
             utils.extract_all(zip_file)
@@ -161,7 +160,7 @@ class CommandInstallTheme(Command):
         if os.path.exists(confpypath):
             LOGGER.notice('This theme has a sample config file.  Integrate it with yours in order to make this theme work!')
             print('Contents of the conf.py.sample file:\n')
-            with codecs.open(confpypath, 'rb', 'utf-8') as fh:
+            with io.open(confpypath, 'r', encoding='utf-8') as fh:
                 if self.site.colorful:
                     print(indent(pygments.highlight(
                         fh.read(), PythonLexer(), TerminalFormatter()),
diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py
index 42f77cc..24c09d0 100644
--- a/nikola/plugins/command/new_post.py
+++ b/nikola/plugins/command/new_post.py
@@ -25,7 +25,7 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from __future__ import unicode_literals, print_function
-import codecs
+import io
 import datetime
 import os
 import sys
@@ -332,7 +332,7 @@ class CommandNewPost(Command):
         event = dict(path=txt_path)
 
         if not onefile:  # write metadata file
-            with codecs.open(meta_path, "wb+", "utf8") as fd:
+            with io.open(meta_path, "w+", encoding="utf8") as fd:
                 fd.write(utils.write_metadata(data))
             LOGGER.info("Your {0}'s metadata is at: {1}".format(content_type, meta_path))
             event['meta_path'] = meta_path
@@ -341,8 +341,8 @@ class CommandNewPost(Command):
         signal('new_' + content_type).send(self, **event)
 
         if options['edit']:
-            editor = os.getenv('EDITOR')
-            to_run = [editor, txt_path]
+            editor = os.getenv('EDITOR', '').split()
+            to_run = editor + [txt_path]
             if not onefile:
                 to_run.append(meta_path)
             if editor:
diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py
index df0e7a4..71901b8 100644
--- a/nikola/plugins/command/plugin.py
+++ b/nikola/plugins/command/plugin.py
@@ -25,8 +25,7 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from __future__ import print_function
-import codecs
-from io import BytesIO
+import io
 import os
 import shutil
 import subprocess
@@ -228,7 +227,7 @@ class CommandPlugin(Command):
         if name in data:
             utils.makedirs(self.output_dir)
             LOGGER.info('Downloading: ' + data[name])
-            zip_file = BytesIO()
+            zip_file = io.BytesIO()
             zip_file.write(requests.get(data[name]).content)
             LOGGER.info('Extracting: {0} into {1}/'.format(name, self.output_dir))
             utils.extract_all(zip_file, self.output_dir)
@@ -258,7 +257,7 @@ class CommandPlugin(Command):
             except subprocess.CalledProcessError:
                 LOGGER.error('Could not install the dependencies.')
                 print('Contents of the requirements.txt file:\n')
-                with codecs.open(reqpath, 'rb', 'utf-8') as fh:
+                with io.open(reqpath, 'r', encoding='utf-8') as fh:
                     print(indent(fh.read(), 4 * ' '))
                 print('You have to install those yourself or through a '
                       'package manager.')
@@ -270,7 +269,7 @@ class CommandPlugin(Command):
                           'dependencies you need to install '
                           'manually.')
             print('Contents of the requirements-nonpy.txt file:\n')
-            with codecs.open(reqnpypath, 'rb', 'utf-8') as fh:
+            with io.open(reqnpypath, 'r', encoding='utf-8') as fh:
                 for l in fh.readlines():
                     i, j = l.split('::')
                     print(indent(i.strip(), 4 * ' '))
@@ -283,7 +282,7 @@ class CommandPlugin(Command):
         if os.path.exists(confpypath):
             LOGGER.notice('This plugin has a sample config file.  Integrate it with yours in order to make this plugin work!')
             print('Contents of the conf.py.sample file:\n')
-            with codecs.open(confpypath, 'rb', 'utf-8') as fh:
+            with io.open(confpypath, 'r', encoding='utf-8') as fh:
                 if self.site.colorful:
                     print(indent(pygments.highlight(
                         fh.read(), PythonLexer(), TerminalFormatter()),
diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py
index 623e2db..de4f6e2 100644
--- a/nikola/plugins/command/serve.py
+++ b/nikola/plugins/command/serve.py
@@ -60,8 +60,8 @@ class CommandServe(Command):
             'short': 'a',
             'long': 'address',
             'type': str,
-            'default': '127.0.0.1',
-            'help': 'Address to bind (default: 127.0.0.1)',
+            'default': '',
+            'help': 'Address to bind (default: 0.0.0.0 – all local interfaces)',
         },
         {
             'name': 'browser',
@@ -84,10 +84,10 @@ class CommandServe(Command):
             httpd = HTTPServer((options['address'], options['port']),
                                OurHTTPRequestHandler)
             sa = httpd.socket.getsockname()
-            self.logger.info("Serving HTTP on {0} port {1} ...".format(*sa))
+            self.logger.info("Serving HTTP on {0} port {1}...".format(*sa))
             if options['browser']:
-                server_url = "http://{0}:{1}/".format(options['address'], options['port'])
-                self.logger.info("Opening {0} in the default web browser ...".format(server_url))
+                server_url = "http://{0}:{1}/".format(*sa)
+                self.logger.info("Opening {0} in the default web browser...".format(server_url))
                 webbrowser.open(server_url)
             try:
                 httpd.serve_forever()
@@ -156,6 +156,9 @@ class OurHTTPRequestHandler(SimpleHTTPRequestHandler):
             return None
         self.send_response(200)
         self.send_header("Content-type", ctype)
+        if os.path.splitext(path)[1] == '.svgz':
+            # Special handling for svgz to make it work nice with browsers.
+            self.send_header("Content-Encoding", 'gzip')
         fs = os.fstat(f.fileno())
         self.send_header("Content-Length", str(fs[6]))
         self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))