Imported Upstream version 7.7.3upstream/7.7.3

33 files changed, 187 insertions, 87 deletions

diff --git a/nikola/plugins/command/auto.plugin b/nikola/plugins/command/auto.plugin
index 3e2b17d..1081c78 100644
--- a/nikola/plugins/command/auto.plugin
+++ b/nikola/plugins/command/auto.plugin
@@ -5,7 +5,7 @@ module = auto
 [Documentation]
 author = Roberto Alsina
 version = 2.1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Automatically detect site changes, rebuild and optionally refresh a browser.
 
 [Nikola]
diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py
index 71f9624..e339c06 100644
--- a/nikola/plugins/command/auto/__init__.py
+++ b/nikola/plugins/command/auto/__init__.py
@@ -63,7 +63,7 @@ except ImportError:
 
 
 from nikola.plugin_categories import Command
-from nikola.utils import req_missing, get_logger, get_theme_path, STDERR_HANDLER
+from nikola.utils import dns_sd, req_missing, get_logger, get_theme_path, STDERR_HANDLER
 LRJS_PATH = os.path.join(os.path.dirname(__file__), 'livereload.js')
 error_signal = signal('error')
 refresh_signal = signal('refresh')
@@ -79,13 +79,14 @@ ERROR {}
 
 
 class CommandAuto(Command):
-
     """Automatic rebuilds for Nikola."""
 
     name = "auto"
     logger = None
     has_server = True
     doc_purpose = "builds and serves a site; automatically detects site changes, rebuilds, and optionally refreshes a browser"
+    dns_sd = None
+
     cmd_options = [
         {
             'name': 'port',
@@ -156,7 +157,7 @@ class CommandAuto(Command):
 
         # Do not duplicate entries -- otherwise, multiple rebuilds are triggered
         watched = set([
-            'templates/',
+            'templates/', 'plugins/',
         ] + [get_theme_path(name) for name in self.site.THEMES])
         for item in self.site.config['post_pages']:
             watched.add(os.path.dirname(item[0]))
@@ -208,7 +209,6 @@ class CommandAuto(Command):
         parent = self
 
         class Mixed(WebSocketWSGIApplication):
-
             """A class that supports WS and HTTP protocols on the same port."""
 
             def __call__(self, environ, start_response):
@@ -235,9 +235,12 @@ class CommandAuto(Command):
                 webbrowser.open('http://{0}:{1}'.format(host, port))
 
             try:
+                self.dns_sd = dns_sd(port, (options['ipv6'] or '::' in host))
                 ws.serve_forever()
             except KeyboardInterrupt:
                 self.logger.info("Server is shutting down.")
+                if self.dns_sd:
+                    self.dns_sd.Reset()
                 # This is a hack, but something is locking up in a futex
                 # and exit() doesn't work.
                 os.kill(os.getpid(), 15)
@@ -262,6 +265,8 @@ class CommandAuto(Command):
         fname = os.path.basename(event_path)
         if (fname.endswith('~') or
                 fname.startswith('.') or
+                '__pycache__' in event_path or
+                event_path.endswith(('.pyc', '.pyo', '.pyd')) or
                 os.path.isdir(event_path)):  # Skip on folders, these are usually duplicates
             return
         self.logger.info('REBUILDING SITE (from {0})'.format(event_path))
@@ -300,11 +305,14 @@ class CommandAuto(Command):
             mimetype = 'text/html'
 
         if p_uri.path == '/robots.txt':
-            start_response('200 OK', [('Content-type', 'text/plain')])
+            start_response('200 OK', [('Content-type', 'text/plain; charset=UTF-8')])
             return ['User-Agent: *\nDisallow: /\n'.encode('utf-8')]
         elif os.path.isfile(f_path):
             with open(f_path, 'rb') as fd:
-                start_response('200 OK', [('Content-type', mimetype)])
+                if mimetype.startswith('text/') or mimetype.endswith('+xml'):
+                    start_response('200 OK', [('Content-type', "{0}; charset=UTF-8".format(mimetype))])
+                else:
+                    start_response('200 OK', [('Content-type', mimetype)])
                 return [self.file_filter(mimetype, fd.read())]
         elif p_uri.path == '/livereload.js':
             with open(LRJS_PATH, 'rb') as fd:
@@ -337,7 +345,6 @@ pending = []
 
 
 class LRSocket(WebSocket):
-
     """Speak Livereload protocol."""
 
     def __init__(self, *a, **kw):
@@ -410,7 +417,6 @@ class LRSocket(WebSocket):
 
 
 class OurWatchHandler(FileSystemEventHandler):
-
     """A Nikola-specific handler for Watchdog."""
 
     def __init__(self, function):
@@ -424,7 +430,6 @@ class OurWatchHandler(FileSystemEventHandler):
 
 
 class ConfigWatchHandler(FileSystemEventHandler):
-
     """A Nikola-specific handler for Watchdog that handles the config file (as a workaround)."""
 
     def __init__(self, configuration_filename, function):
diff --git a/nikola/plugins/command/bootswatch_theme.plugin b/nikola/plugins/command/bootswatch_theme.plugin
index fc25045..51e6718 100644
--- a/nikola/plugins/command/bootswatch_theme.plugin
+++ b/nikola/plugins/command/bootswatch_theme.plugin
@@ -5,7 +5,7 @@ module = bootswatch_theme
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Given a swatch name and a parent theme, creates a custom theme.
 
 [Nikola]
diff --git a/nikola/plugins/command/bootswatch_theme.py b/nikola/plugins/command/bootswatch_theme.py
index b5644a1..afd15af 100644
--- a/nikola/plugins/command/bootswatch_theme.py
+++ b/nikola/plugins/command/bootswatch_theme.py
@@ -37,7 +37,6 @@ LOGGER = utils.get_logger('bootswatch_theme', utils.STDERR_HANDLER)
 
 
 class CommandBootswatchTheme(Command):
-
     """Given a swatch name from bootswatch.com and a parent theme, creates a custom theme."""
 
     name = "bootswatch_theme"
@@ -91,12 +90,16 @@ class CommandBootswatchTheme(Command):
         LOGGER.info("Creating '{0}' theme from '{1}' and '{2}'".format(name, swatch, parent))
         utils.makedirs(os.path.join('themes', name, 'assets', 'css'))
         for fname in ('bootstrap.min.css', 'bootstrap.css'):
-            url = 'http://bootswatch.com'
+            url = 'https://bootswatch.com'
             if version:
                 url += '/' + version
             url = '/'.join((url, swatch, fname))
             LOGGER.info("Downloading: " + url)
-            data = requests.get(url).text
+            r = requests.get(url)
+            if r.status_code > 299:
+                LOGGER.error('Error {} getting {}', r.status_code, url)
+                exit(1)
+            data = r.text
             with open(os.path.join('themes', name, 'assets', 'css', fname),
                       'wb+') as output:
                 output.write(data.encode('utf-8'))
diff --git a/nikola/plugins/command/check.plugin b/nikola/plugins/command/check.plugin
index e380e64..6d2df82 100644
--- a/nikola/plugins/command/check.plugin
+++ b/nikola/plugins/command/check.plugin
@@ -5,7 +5,7 @@ module = check
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Check the generated site
 
 [Nikola]
diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py
index abf183e..bfc6ee2 100644
--- a/nikola/plugins/command/check.py
+++ b/nikola/plugins/command/check.py
@@ -46,7 +46,10 @@ from nikola.plugin_categories import Command
 from nikola.utils import get_logger, STDERR_HANDLER
 
 
-def _call_nikola_list(site):
+def _call_nikola_list(site, cache=None):
+    if cache is not None:
+        if 'files' in cache and 'deps' in cache:
+            return cache['files'], cache['deps']
     files = []
     deps = defaultdict(list)
     for task in generate_tasks('render_site', site.gen_tasks('render_site', "Task", '')):
@@ -57,16 +60,19 @@ def _call_nikola_list(site):
         files.extend(task.targets)
         for target in task.targets:
             deps[target].extend(task.file_dep)
+    if cache is not None:
+        cache['files'] = files
+        cache['deps'] = deps
     return files, deps
 
 
-def real_scan_files(site):
+def real_scan_files(site, cache=None):
     """Scan for files."""
     task_fnames = set([])
     real_fnames = set([])
     output_folder = site.config['OUTPUT_FOLDER']
     # First check that all targets are generated in the right places
-    for fname in _call_nikola_list(site)[0]:
+    for fname in _call_nikola_list(site, cache)[0]:
         fname = fname.strip()
         if fname.startswith(output_folder):
             task_fnames.add(fname)
@@ -94,7 +100,6 @@ def fs_relpath_from_url_path(url_path):
 
 
 class CommandCheck(Command):
-
     """Check the generated site."""
 
     name = "check"
@@ -162,22 +167,25 @@ class CommandCheck(Command):
             self.logger.level = 1
         else:
             self.logger.level = 4
+        failure = False
         if options['links']:
-            failure = self.scan_links(options['find_sources'], options['remote'])
+            failure |= self.scan_links(options['find_sources'], options['remote'])
         if options['files']:
-            failure = self.scan_files()
+            failure |= self.scan_files()
         if options['clean']:
-            failure = self.clean_files()
+            failure |= self.clean_files()
         if failure:
             return 1
 
     existing_targets = set([])
     checked_remote_targets = {}
+    cache = {}
 
     def analyze(self, fname, find_sources=False, check_remote=False):
         """Analyze links on a page."""
         rv = False
         self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]
+        self.internal_redirects = [urljoin('/', _[0]) for _ in self.site.config['REDIRECTIONS']]
         base_url = urlparse(self.site.config['BASE_URL'])
         self.existing_targets.add(self.site.config['SITE_URL'])
         self.existing_targets.add(self.site.config['BASE_URL'])
@@ -185,7 +193,7 @@ class CommandCheck(Command):
 
         deps = {}
         if find_sources:
-            deps = _call_nikola_list(self.site)[1]
+            deps = _call_nikola_list(self.site, self.cache)[1]
 
         if url_type in ('absolute', 'full_path'):
             url_netloc_to_root = urlparse(self.site.config['BASE_URL']).path
@@ -203,31 +211,70 @@ class CommandCheck(Command):
                 # Quietly ignore files that don’t exist; use `nikola check -f` instead (Issue #1831)
                 return False
 
-            d = lxml.html.fromstring(open(filename, 'rb').read())
-            for l in d.iterlinks():
+            if '.html' == fname[-5:]:
+                d = lxml.html.fromstring(open(filename, 'rb').read())
+                extra_objs = lxml.html.fromstring('<html/>')
+
+                # Turn elements with a srcset attribute into individual img elements with src attributes
+                for obj in list(d.xpath('(*//img|*//source)')):
+                    if 'srcset' in obj.attrib:
+                        for srcset_item in obj.attrib['srcset'].split(','):
+                            extra_objs.append(lxml.etree.Element('img', src=srcset_item.strip().split(' ')[0]))
+                link_elements = list(d.iterlinks()) + list(extra_objs.iterlinks())
+            # Extract links from XML formats to minimal HTML, allowing those to go through the link checks
+            elif '.atom' == filename[-5:]:
+                d = lxml.etree.parse(filename)
+                link_elements = lxml.html.fromstring('<html/>')
+                for elm in d.findall('*//{http://www.w3.org/2005/Atom}link'):
+                    feed_link = elm.attrib['href'].split('?')[0].strip()  # strip FEED_LINKS_APPEND_QUERY
+                    link_elements.append(lxml.etree.Element('a', href=feed_link))
+                link_elements = list(link_elements.iterlinks())
+            elif filename.endswith('sitemap.xml') or filename.endswith('sitemapindex.xml'):
+                d = lxml.etree.parse(filename)
+                link_elements = lxml.html.fromstring('<html/>')
+                for elm in d.getroot().findall("*//{http://www.sitemaps.org/schemas/sitemap/0.9}loc"):
+                    link_elements.append(lxml.etree.Element('a', href=elm.text.strip()))
+                link_elements = list(link_elements.iterlinks())
+            else:  # unsupported file type
+                return False
+
+            for l in link_elements:
                 target = l[2]
                 if target == "#":
                     continue
-                target, _ = urldefrag(target)
+                target = urldefrag(target)[0]
+
+                if any([urlparse(target).netloc.endswith(_) for _ in ['example.com', 'example.net', 'example.org']]):
+                    self.logger.info("Not testing example address \"{0}\".".format(target))
+                    continue
+
+                # absolute URL to root-relative
+                if target.startswith(base_url.geturl()):
+                    target = target.replace(base_url.geturl(), '/')
+
                 parsed = urlparse(target)
 
                 # Warn about links from https to http (mixed-security)
                 if base_url.netloc == parsed.netloc and base_url.scheme == "https" and parsed.scheme == "http":
                     self.logger.warn("Mixed-content security for link in {0}: {1}".format(filename, target))
 
+                # Link to an internal REDIRECTIONS page
+                if target in self.internal_redirects:
+                    redir_status_code = 301
+                    redir_target = [_dest for _target, _dest in self.site.config['REDIRECTIONS'] if urljoin('/', _target) == target][0]
+                    self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: 301]".format(redir_target, filename, target))
+
                 # Absolute links to other domains, skip
                 # Absolute links when using only paths, skip.
                 if ((parsed.scheme or target.startswith('//')) and parsed.netloc != base_url.netloc) or \
                         ((parsed.scheme or target.startswith('//')) and url_type in ('rel_path', 'full_path')):
                     if not check_remote or parsed.scheme not in ["http", "https"]:
                         continue
-                    if parsed.netloc == base_url.netloc:  # absolute URL to self.site
-                        continue
                     if target in self.checked_remote_targets:  # already checked this exact target
-                        if self.checked_remote_targets[target] in [301, 307]:
+                        if self.checked_remote_targets[target] in [301, 308]:
                             self.logger.warn("Remote link PERMANENTLY redirected in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target]))
-                        elif self.checked_remote_targets[target] in [302, 308]:
-                            self.logger.info("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target]))
+                        elif self.checked_remote_targets[target] in [302, 307]:
+                            self.logger.notice("Remote link temporarily redirected in {1}: {2} [HTTP: {3}]".format(filename, target, self.checked_remote_targets[target]))
                         elif self.checked_remote_targets[target] > 399:
                             self.logger.error("Broken link in {0}: {1} [Error {2}]".format(filename, target, self.checked_remote_targets[target]))
                         continue
@@ -255,7 +302,7 @@ class CommandCheck(Command):
                         if redir_status_code in [301, 308]:
                             self.logger.warn("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code))
                         if redir_status_code in [302, 307]:
-                            self.logger.info("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code))
+                            self.logger.notice("Remote link temporarily redirected to \"{0}\" in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code))
                         self.checked_remote_targets[resp.url] = resp.status_code
                         self.checked_remote_targets[target] = redir_status_code
                     else:
@@ -275,8 +322,9 @@ class CommandCheck(Command):
                         target_filename = os.path.abspath(
                             os.path.join(self.site.config['OUTPUT_FOLDER'], unquote(target.lstrip('/'))))
                     else:  # Relative path
+                        unquoted_target = unquote(target).encode('utf-8') if sys.version_info.major >= 3 else unquote(target).decode('utf-8')
                         target_filename = os.path.abspath(
-                            os.path.join(os.path.dirname(filename), unquote(target)))
+                            os.path.join(os.path.dirname(filename).encode('utf-8'), unquoted_target))
 
                 elif url_type in ('full_path', 'absolute'):
                     if url_type == 'absolute':
@@ -292,9 +340,10 @@ class CommandCheck(Command):
 
                 if any(re.search(x, target_filename) for x in self.whitelist):
                     continue
+
                 elif target_filename not in self.existing_targets:
                     if os.path.exists(target_filename):
-                        self.logger.notice("Good link {0} => {1}".format(target, target_filename))
+                        self.logger.info(u"Good link {0} => {1}".format(target, target_filename))
                         self.existing_targets.add(target_filename)
                     else:
                         rv = True
@@ -304,7 +353,7 @@ class CommandCheck(Command):
                             self.logger.warn("\n".join(deps[filename]))
                             self.logger.warn("===============================\n")
         except Exception as exc:
-            self.logger.error("Error with: {0} {1}".format(filename, exc))
+            self.logger.error(u"Error with: {0} {1}".format(filename, exc))
         return rv
 
     def scan_links(self, find_sources=False, check_remote=False):
@@ -315,10 +364,21 @@ class CommandCheck(Command):
         failure = False
         # Maybe we should just examine all HTML files
         output_folder = self.site.config['OUTPUT_FOLDER']
-        for fname in _call_nikola_list(self.site)[0]:
-            if fname.startswith(output_folder) and '.html' == fname[-5:]:
-                if self.analyze(fname, find_sources, check_remote):
-                    failure = True
+
+        if urlparse(self.site.config['BASE_URL']).netloc == 'example.com':
+            self.logger.error("You've not changed the SITE_URL (or BASE_URL) setting from \"example.com\"!")
+
+        for fname in _call_nikola_list(self.site, self.cache)[0]:
+            if fname.startswith(output_folder):
+                if '.html' == fname[-5:]:
+                    if self.analyze(fname, find_sources, check_remote):
+                        failure = True
+                if '.atom' == fname[-5:]:
+                    if self.analyze(fname, find_sources, False):
+                        failure = True
+                if fname.endswith('sitemap.xml') or fname.endswith('sitemapindex.xml'):
+                    if self.analyze(fname, find_sources, False):
+                        failure = True
         if not failure:
             self.logger.info("All links checked.")
         return failure
@@ -328,7 +388,7 @@ class CommandCheck(Command):
         failure = False
         self.logger.info("Checking Files:")
         self.logger.info("===============\n")
-        only_on_output, only_on_input = real_scan_files(self.site)
+        only_on_output, only_on_input = real_scan_files(self.site, self.cache)
 
         # Ignore folders
         only_on_output = [p for p in only_on_output if not os.path.isdir(p)]
@@ -351,7 +411,7 @@ class CommandCheck(Command):
 
     def clean_files(self):
         """Remove orphaned files."""
-        only_on_output, _ = real_scan_files(self.site)
+        only_on_output, _ = real_scan_files(self.site, self.cache)
         for f in only_on_output:
             self.logger.info('removed: {0}'.format(f))
             os.unlink(f)
diff --git a/nikola/plugins/command/console.plugin b/nikola/plugins/command/console.plugin
index 333762c..9bcc909 100644
--- a/nikola/plugins/command/console.plugin
+++ b/nikola/plugins/command/console.plugin
@@ -5,7 +5,7 @@ module = console
 [Documentation]
 author = Chris Warrick, Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Start a debugging python console
 
 [Nikola]
diff --git a/nikola/plugins/command/console.py b/nikola/plugins/command/console.py
index 539fa08..3d3daab 100644
--- a/nikola/plugins/command/console.py
+++ b/nikola/plugins/command/console.py
@@ -38,7 +38,6 @@ LOGGER = get_logger('console', STDERR_HANDLER)
 
 
 class CommandConsole(Command):
-
     """Start debugging console."""
 
     name = "console"
diff --git a/nikola/plugins/command/deploy.plugin b/nikola/plugins/command/deploy.plugin
index 4743ca2..8bdc0e2 100644
--- a/nikola/plugins/command/deploy.plugin
+++ b/nikola/plugins/command/deploy.plugin
@@ -5,7 +5,7 @@ module = deploy
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Deploy the site
 
 [Nikola]
diff --git a/nikola/plugins/command/deploy.py b/nikola/plugins/command/deploy.py
index 821ea11..757c0d2 100644
--- a/nikola/plugins/command/deploy.py
+++ b/nikola/plugins/command/deploy.py
@@ -41,7 +41,6 @@ from nikola.utils import get_logger, remove_file, unicode_str, makedirs, STDERR_
 
 
 class CommandDeploy(Command):
-
     """Deploy site."""
 
     name = "deploy"
diff --git a/nikola/plugins/command/github_deploy.plugin b/nikola/plugins/command/github_deploy.plugin
index e793548..21e246c 100644
--- a/nikola/plugins/command/github_deploy.plugin
+++ b/nikola/plugins/command/github_deploy.plugin
@@ -5,7 +5,7 @@ module = github_deploy
 [Documentation]
 author = Puneeth Chaganti
 version = 1,0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Deploy the site to GitHub pages.
 
 [Nikola]
diff --git a/nikola/plugins/command/github_deploy.py b/nikola/plugins/command/github_deploy.py
index 0ab9332..2fe0d4e 100644
--- a/nikola/plugins/command/github_deploy.py
+++ b/nikola/plugins/command/github_deploy.py
@@ -57,7 +57,6 @@ def check_ghp_import_installed():
 
 
 class CommandGitHubDeploy(Command):
-
     """Deploy site to GitHub Pages."""
 
     name = 'github_deploy'
diff --git a/nikola/plugins/command/import_wordpress.plugin b/nikola/plugins/command/import_wordpress.plugin
index 6c4384e..eab9d17 100644
--- a/nikola/plugins/command/import_wordpress.plugin
+++ b/nikola/plugins/command/import_wordpress.plugin
@@ -5,7 +5,7 @@ module = import_wordpress
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Import a wordpress site from a XML dump (requires markdown).
 
 [Nikola]
diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py
index a652ec8..69ef144 100644
--- a/nikola/plugins/command/import_wordpress.py
+++ b/nikola/plugins/command/import_wordpress.py
@@ -50,7 +50,7 @@ except ImportError:
 
 from nikola.plugin_categories import Command
 from nikola import utils
-from nikola.utils import req_missing
+from nikola.utils import req_missing, unicode_str
 from nikola.plugins.basic_import import ImportMixin, links
 from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN
 from nikola.plugins.command.init import SAMPLE_CONF, prepare_config, format_default_translations_config
@@ -88,7 +88,6 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False)
 
 
 class CommandImportWordpress(Command, ImportMixin):
-
     """Import a WordPress dump."""
 
     name = "import_wordpress"
@@ -191,6 +190,12 @@ class CommandImportWordpress(Command, ImportMixin):
             'type': bool,
             'help': "Automatically installs the WordPress page compiler (either locally or in the new site) if required by other options.\nWarning: the compiler is GPL software!",
         },
+        {
+            'name': 'tag_saniziting_strategy',
+            'long': 'tag-saniziting-strategy',
+            'default': 'first',
+            'help': 'lower: Convert all tag and category names to lower case\nfirst: Keep first spelling of tag or category name',
+        },
     ]
     all_tags = set([])
 
@@ -239,6 +244,8 @@ class CommandImportWordpress(Command, ImportMixin):
         self.install_wordpress_compiler = options.get('install_wordpress_compiler', False)
         self.wordpress_page_compiler = None
 
+        self.tag_saniziting_strategy = options.get('tag_saniziting_strategy', 'first')
+
         self.auth = None
         if options.get('download_auth') is not None:
             username_password = options.get('download_auth')
@@ -337,7 +344,10 @@ class CommandImportWordpress(Command, ImportMixin):
         # Add tag redirects
         for tag in self.all_tags:
             try:
-                tag_str = tag.decode('utf8')
+                if isinstance(tag, utils.bytes_str):
+                    tag_str = tag.decode('utf8', 'replace')
+                else:
+                    tag_str = tag
             except AttributeError:
                 tag_str = tag
             tag = utils.slugify(tag_str)
@@ -604,7 +614,7 @@ class CommandImportWordpress(Command, ImportMixin):
 
     def transform_code(self, content):
         """Transform code blocks."""
-        # http://en.support.wordpress.com/code/posting-source-code/. There are
+        # https://en.support.wordpress.com/code/posting-source-code/. There are
         # a ton of things not supported here. We only do a basic [code
         # lang="x"] -> ```x translation, and remove quoted html entities (<,
         # >, &, and ").
@@ -686,7 +696,7 @@ class CommandImportWordpress(Command, ImportMixin):
         elif approved == 'spam' or approved == 'trash':
             pass
         else:
-            LOGGER.warn("Unknown comment approved status: " + str(approved))
+            LOGGER.warn("Unknown comment approved status: {0}".format(approved))
         parent = int(get_text_tag(comment, "{{{0}}}comment_parent".format(wordpress_namespace), 0))
         if parent == 0:
             parent = None
@@ -707,7 +717,7 @@ class CommandImportWordpress(Command, ImportMixin):
             """Write comment header line."""
             if header_content is None:
                 return
-            header_content = str(header_content).replace('\n', ' ')
+            header_content = unicode_str(header_content).replace('\n', ' ')
             line = '.. ' + header_field + ': ' + header_content + '\n'
             fd.write(line.encode('utf8'))
 
@@ -747,6 +757,24 @@ class CommandImportWordpress(Command, ImportMixin):
             tags_cats = tags + categories
         return tags_cats, other_meta
 
+    _tag_sanitize_map = {True: {}, False: {}}
+
+    def _sanitize(self, tag, is_category):
+        if self.tag_saniziting_strategy == 'lower':
+            return tag.lower()
+        if tag.lower() not in self._tag_sanitize_map[is_category]:
+            self._tag_sanitize_map[is_category][tag.lower()] = [tag]
+            return tag
+        previous = self._tag_sanitize_map[is_category][tag.lower()]
+        if self.tag_saniziting_strategy == 'first':
+            if tag != previous[0]:
+                LOGGER.warn("Changing spelling of {0} name '{1}' to {2}.".format('category' if is_category else 'tag', tag, previous[0]))
+            return previous[0]
+        else:
+            LOGGER.error("Unknown tag sanitizing strategy '{0}'!".format(self.tag_saniziting_strategy))
+            sys.exit(1)
+        return tag
+
     def import_postpage_item(self, item, wordpress_namespace, out_folder=None, attachments=None):
         """Take an item from the feed and creates a post file."""
         if out_folder is None:
@@ -760,7 +788,10 @@ class CommandImportWordpress(Command, ImportMixin):
         path = unquote(parsed.path.strip('/'))
 
         try:
-            path = path.decode('utf8')
+            if isinstance(path, utils.bytes_str):
+                path = path.decode('utf8', 'replace')
+            else:
+                path = path
         except AttributeError:
             pass
 
@@ -831,15 +862,24 @@ class CommandImportWordpress(Command, ImportMixin):
                 type = tag.attrib['domain']
             if text == 'Uncategorized' and type == 'category':
                 continue
-            self.all_tags.add(text)
             if type == 'category':
-                categories.append(type)
+                categories.append(text)
             else:
                 tags.append(text)
 
         if '$latex' in content:
             tags.append('mathjax')
 
+        for i, cat in enumerate(categories[:]):
+            cat = self._sanitize(cat, True)
+            categories[i] = cat
+            self.all_tags.add(cat)
+
+        for i, tag in enumerate(tags[:]):
+            tag = self._sanitize(tag, False)
+            tags[i] = tag
+            self.all_tags.add(tag)
+
         # Find post format if it's there
         post_format = 'wp'
         format_tag = [x for x in item.findall('*//{%s}meta_key' % wordpress_namespace) if x.text == '_tc_post_format']
@@ -905,7 +945,7 @@ class CommandImportWordpress(Command, ImportMixin):
                         comments.append(comment)
 
                 for comment in comments:
-                    comment_filename = slug + "." + str(comment['id']) + ".wpcomment"
+                    comment_filename = "{0}.{1}.wpcomment".format(slug, comment['id'])
                     self._write_comment(os.path.join(self.output_folder, out_folder, comment_filename), comment)
 
             return (out_folder, slug)
diff --git a/nikola/plugins/command/init.plugin b/nikola/plugins/command/init.plugin
index a5404c4..a8b1523 100644
--- a/nikola/plugins/command/init.plugin
+++ b/nikola/plugins/command/init.plugin
@@ -5,7 +5,7 @@ module = init
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Create a new site.
 
 [Nikola]
diff --git a/nikola/plugins/command/init.py b/nikola/plugins/command/init.py
index 91ccdb4..2dbee43 100644
--- a/nikola/plugins/command/init.py
+++ b/nikola/plugins/command/init.py
@@ -41,7 +41,7 @@ from pkg_resources import resource_filename
 import tarfile
 
 import nikola
-from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_RSS_READ_MORE_LINK, LEGAL_VALUES, urlsplit, urlunsplit
+from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN, DEFAULT_INDEX_READ_MORE_LINK, DEFAULT_FEED_READ_MORE_LINK, LEGAL_VALUES, urlsplit, urlunsplit
 from nikola.plugin_categories import Command
 from nikola.utils import ask, ask_yesno, get_logger, makedirs, STDERR_HANDLER, load_messages
 from nikola.packages.tzlocal import get_localzone
@@ -71,7 +71,7 @@ SAMPLE_CONF = {
     'CATEGORY_OUTPUT_FLAT_HIERARCHY': False,
     'TRANSLATIONS_PATTERN': DEFAULT_TRANSLATIONS_PATTERN,
     'INDEX_READ_MORE_LINK': DEFAULT_INDEX_READ_MORE_LINK,
-    'RSS_READ_MORE_LINK': DEFAULT_RSS_READ_MORE_LINK,
+    'FEED_READ_MORE_LINK': DEFAULT_FEED_READ_MORE_LINK,
     'POSTS': """(
     ("posts/*.rst", "posts", "post.tmpl"),
     ("posts/*.txt", "posts", "post.tmpl"),
@@ -210,17 +210,16 @@ def prepare_config(config):
     """Parse sample config with JSON."""
     p = config.copy()
     p.update({k: json.dumps(v, ensure_ascii=False) for k, v in p.items()
-             if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'RSS_READ_MORE_LINK')})
+             if k not in ('POSTS', 'PAGES', 'COMPILERS', 'TRANSLATIONS', 'NAVIGATION_LINKS', '_SUPPORTED_LANGUAGES', '_SUPPORTED_COMMENT_SYSTEMS', 'INDEX_READ_MORE_LINK', 'FEED_READ_MORE_LINK')})
     # READ_MORE_LINKs require some special treatment.
     p['INDEX_READ_MORE_LINK'] = "'" + p['INDEX_READ_MORE_LINK'].replace("'", "\\'") + "'"
-    p['RSS_READ_MORE_LINK'] = "'" + p['RSS_READ_MORE_LINK'].replace("'", "\\'") + "'"
+    p['FEED_READ_MORE_LINK'] = "'" + p['FEED_READ_MORE_LINK'].replace("'", "\\'") + "'"
     # fix booleans and None
     p.update({k: str(v) for k, v in config.items() if isinstance(v, bool) or v is None})
     return p
 
 
 class CommandInit(Command):
-
     """Create a new site."""
 
     name = "init"
@@ -358,7 +357,7 @@ class CommandInit(Command):
         def tzhandler(default, toconf):
             print("\nPlease choose the correct time zone for your blog. Nikola uses the tz database.")
             print("You can find your time zone here:")
-            print("http://en.wikipedia.org/wiki/List_of_tz_database_time_zones")
+            print("https://en.wikipedia.org/wiki/List_of_tz_database_time_zones")
             print("")
             answered = False
             while not answered:
diff --git a/nikola/plugins/command/install_theme.plugin b/nikola/plugins/command/install_theme.plugin
index 8434f2e..aa68773 100644
--- a/nikola/plugins/command/install_theme.plugin
+++ b/nikola/plugins/command/install_theme.plugin
@@ -5,7 +5,7 @@ module = install_theme
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Install a theme into the current site.
 
 [Nikola]
diff --git a/nikola/plugins/command/install_theme.py b/nikola/plugins/command/install_theme.py
index f02252e..bad335c 100644
--- a/nikola/plugins/command/install_theme.py
+++ b/nikola/plugins/command/install_theme.py
@@ -43,7 +43,6 @@ LOGGER = utils.get_logger('install_theme', utils.STDERR_HANDLER)
 
 
 class CommandInstallTheme(Command):
-
     """Install a theme."""
 
     name = "install_theme"
diff --git a/nikola/plugins/command/new_page.plugin b/nikola/plugins/command/new_page.plugin
index 145a419..3eaecb4 100644
--- a/nikola/plugins/command/new_page.plugin
+++ b/nikola/plugins/command/new_page.plugin
@@ -5,7 +5,7 @@ module = new_page
 [Documentation]
 author = Roberto Alsina, Chris Warrick
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Create a new page.
 
 [Nikola]
diff --git a/nikola/plugins/command/new_page.py b/nikola/plugins/command/new_page.py
index 811e28b..8843421 100644
--- a/nikola/plugins/command/new_page.py
+++ b/nikola/plugins/command/new_page.py
@@ -32,7 +32,6 @@ from nikola.plugin_categories import Command
 
 
 class CommandNewPage(Command):
-
     """Create a new page."""
 
     name = "new_page"
diff --git a/nikola/plugins/command/new_post.plugin b/nikola/plugins/command/new_post.plugin
index d88469f..e9c3af5 100644
--- a/nikola/plugins/command/new_post.plugin
+++ b/nikola/plugins/command/new_post.plugin
@@ -5,7 +5,7 @@ module = new_post
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Create a new post.
 
 [Nikola]
diff --git a/nikola/plugins/command/new_post.py b/nikola/plugins/command/new_post.py
index f9fe3ff..30d009d 100644
--- a/nikola/plugins/command/new_post.py
+++ b/nikola/plugins/command/new_post.py
@@ -114,7 +114,6 @@ def get_date(schedule=False, rule=None, last_date=None, tz=None, iso8601=False):
 
 
 class CommandNewPost(Command):
-
     """Create a new post."""
 
     name = "new_post"
diff --git a/nikola/plugins/command/orphans.plugin b/nikola/plugins/command/orphans.plugin
index 669429d..d20c539 100644
--- a/nikola/plugins/command/orphans.plugin
+++ b/nikola/plugins/command/orphans.plugin
@@ -5,7 +5,7 @@ module = orphans
 [Documentation]
 author = Roberto Alsina, Chris Warrick
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = List all orphans
 
 [Nikola]
diff --git a/nikola/plugins/command/orphans.py b/nikola/plugins/command/orphans.py
index b12cc67..f061d39 100644
--- a/nikola/plugins/command/orphans.py
+++ b/nikola/plugins/command/orphans.py
@@ -34,7 +34,6 @@ from nikola.plugins.command.check import real_scan_files
 
 
 class CommandOrphans(Command):
-
     """List all orphans."""
 
     name = "orphans"
diff --git a/nikola/plugins/command/plugin.plugin b/nikola/plugins/command/plugin.plugin
index d44dcf3..016bcaa 100644
--- a/nikola/plugins/command/plugin.plugin
+++ b/nikola/plugins/command/plugin.plugin
@@ -5,7 +5,7 @@ module = plugin
 [Documentation]
 author = Roberto Alsina and Chris Warrick
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Manage Nikola plugins
 
 [Nikola]
diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py
index f892ee9..1df7b71 100644
--- a/nikola/plugins/command/plugin.py
+++ b/nikola/plugins/command/plugin.py
@@ -45,7 +45,6 @@ LOGGER = utils.get_logger('plugin', utils.STDERR_HANDLER)
 
 
 class CommandPlugin(Command):
-
     """Manage plugins."""
 
     json = None
diff --git a/nikola/plugins/command/rst2html.plugin b/nikola/plugins/command/rst2html.plugin
index 02c9276..a095705 100644
--- a/nikola/plugins/command/rst2html.plugin
+++ b/nikola/plugins/command/rst2html.plugin
@@ -5,7 +5,7 @@ module = rst2html
 [Documentation]
 author = Chris Warrick
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Compile reStructuredText to HTML using the Nikola architecture
 
 [Nikola]
diff --git a/nikola/plugins/command/rst2html/__init__.py b/nikola/plugins/command/rst2html/__init__.py
index 06afffd..68ea13d 100644
--- a/nikola/plugins/command/rst2html/__init__.py
+++ b/nikola/plugins/command/rst2html/__init__.py
@@ -36,7 +36,6 @@ from nikola.plugin_categories import Command
 
 
 class CommandRst2Html(Command):
-
     """Compile reStructuredText to HTML, using Nikola architecture."""
 
     name = "rst2html"
@@ -65,7 +64,7 @@ class CommandRst2Html(Command):
         parser = lxml.html.HTMLParser(remove_blank_text=True)
         doc = lxml.html.document_fromstring(template_output, parser)
         html = b'<!DOCTYPE html>\n' + lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True)
-        print(html)
+        print(html.decode('utf-8'))
         if error_level < 3:
             return 0
         else:
diff --git a/nikola/plugins/command/serve.plugin b/nikola/plugins/command/serve.plugin
index aca71ec..a4a726f 100644
--- a/nikola/plugins/command/serve.plugin
+++ b/nikola/plugins/command/serve.plugin
@@ -5,7 +5,7 @@ module = serve
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Start test server.
 
 [Nikola]
diff --git a/nikola/plugins/command/serve.py b/nikola/plugins/command/serve.py
index 0441c93..b647bb7 100644
--- a/nikola/plugins/command/serve.py
+++ b/nikola/plugins/command/serve.py
@@ -45,24 +45,23 @@ except ImportError:
 
 
 from nikola.plugin_categories import Command
-from nikola.utils import get_logger, STDERR_HANDLER
+from nikola.utils import dns_sd, get_logger, STDERR_HANDLER
 
 
 class IPv6Server(HTTPServer):
-
     """An IPv6 HTTPServer."""
 
     address_family = socket.AF_INET6
 
 
 class CommandServe(Command):
-
     """Start test server."""
 
     name = "serve"
     doc_usage = "[options]"
     doc_purpose = "start the test webserver"
     logger = None
+    dns_sd = None
 
     cmd_options = (
         {
@@ -152,14 +151,16 @@ class CommandServe(Command):
                         raise e
             else:
                 try:
+                    self.dns_sd = dns_sd(options['port'], (options['ipv6'] or '::' in options['address']))
                     httpd.serve_forever()
                 except KeyboardInterrupt:
                     self.logger.info("Server is shutting down.")
+                    if self.dns_sd:
+                        self.dns_sd.Reset()
                     return 130
 
 
 class OurHTTPRequestHandler(SimpleHTTPRequestHandler):
-
     """A request handler, modified for Nikola."""
 
     extensions_map = dict(SimpleHTTPRequestHandler.extensions_map)
@@ -242,7 +243,10 @@ class OurHTTPRequestHandler(SimpleHTTPRequestHandler):
             f.seek(0)
 
         self.send_response(200)
-        self.send_header("Content-type", ctype)
+        if ctype.startswith('text/') or ctype.endswith('+xml'):
+            self.send_header("Content-Type", "{0}; charset=UTF-8".format(ctype))
+        else:
+            self.send_header("Content-Type", ctype)
         if os.path.splitext(path)[1] == '.svgz':
             # Special handling for svgz to make it work nice with browsers.
             self.send_header("Content-Encoding", 'gzip')
diff --git a/nikola/plugins/command/status.py b/nikola/plugins/command/status.py
index 55e7f95..40f4f77 100644
--- a/nikola/plugins/command/status.py
+++ b/nikola/plugins/command/status.py
@@ -36,7 +36,6 @@ from nikola.plugin_categories import Command
 
 
 class CommandStatus(Command):
-
     """Display site status."""
 
     name = "status"
diff --git a/nikola/plugins/command/version.plugin b/nikola/plugins/command/version.plugin
index 4708bdb..d78b79b 100644
--- a/nikola/plugins/command/version.plugin
+++ b/nikola/plugins/command/version.plugin
@@ -5,7 +5,7 @@ module = version
 [Documentation]
 author = Roberto Alsina
 version = 1.0
-website = http://getnikola.com
+website = https://getnikola.com/
 description = Show nikola version
 
 [Nikola]
diff --git a/nikola/plugins/command/version.py b/nikola/plugins/command/version.py
index ad08f64..b83d622 100644
--- a/nikola/plugins/command/version.py
+++ b/nikola/plugins/command/version.py
@@ -38,7 +38,6 @@ URL = 'https://pypi.python.org/pypi?:action=doap&name=Nikola'
 
 
 class CommandVersion(Command):
-
     """Print Nikola version."""
 
     name = "version"