New upstream version 8.1.2.upstream/8.1.2

1 files changed, 608 insertions, 422 deletions

diff --git a/nikola/post.py b/nikola/post.py
index 7badfc6..82d957d 100644
--- a/nikola/post.py
+++ b/nikola/post.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright © 2012-2015 Roberto Alsina and others.
+# Copyright © 2012-2020 Roberto Alsina and others.
 
 # Permission is hereby granted, free of charge, to any
 # person obtaining a copy of this software and associated
@@ -26,60 +26,63 @@
 
 """The Post class."""
 
-from __future__ import unicode_literals, print_function, absolute_import
-
 import io
-from collections import defaultdict
 import datetime
 import hashlib
 import json
 import os
 import re
-import string
-try:
-    from urlparse import urljoin
-except ImportError:
-    from urllib.parse import urljoin  # NOQA
-
-from . import utils
+from collections import defaultdict
+from math import ceil  # for reading time feature
+from urllib.parse import urljoin
 
 import dateutil.tz
 import lxml.html
 import natsort
-try:
-    import pyphen
-except ImportError:
-    pyphen = None
-
-from math import ceil
+from blinker import signal
 
 # for tearDown with _reload we cannot use 'from import' to get forLocaleBorg
 import nikola.utils
+from . import metadata_extractors
+from . import utils
 from .utils import (
-    bytes_str,
     current_time,
     Functionary,
     LOGGER,
     LocaleBorg,
     slugify,
     to_datetime,
-    unicode_str,
     demote_headers,
     get_translation_candidate,
-    unslugify,
+    map_metadata
 )
-from .rc4 import rc4
+
+try:
+    import pyphen
+except ImportError:
+    pyphen = None
+
 
 __all__ = ('Post',)
 
-TEASER_REGEXP = re.compile('<!--\s*TEASER_END(:(.+))?\s*-->', re.IGNORECASE)
-_UPGRADE_METADATA_ADVERTISED = False
+TEASER_REGEXP = re.compile(r'<!--\s*(TEASER_END|END_TEASER)(:(.+))?\s*-->', re.IGNORECASE)
 
 
 class Post(object):
-
     """Represent a blog post or site page."""
 
+    _prev_post = None
+    _next_post = None
+    is_draft = False
+    is_private = False
+    _is_two_file = None
+    _reading_time = None
+    _remaining_reading_time = None
+    _paragraph_count = None
+    _remaining_paragraph_count = None
+    post_status = 'published'
+    has_oldstyle_metadata_tags = False
+
     def __init__(
         self,
         source_path,
@@ -88,74 +91,249 @@ class Post(object):
         use_in_feeds,
         messages,
         template_name,
-        compiler
+        compiler,
+        destination_base=None,
+        metadata_extractors_by=None
     ):
         """Initialize post.
 
         The source path is the user created post file. From it we calculate
         the meta file, as well as any translations available, and
         the .html fragment file path.
+
+        destination_base must be None or a TranslatableSetting instance. If
+        specified, it will be prepended to the destination path.
         """
-        self.config = config
+        self._load_config(config)
+        self._set_paths(source_path)
+
         self.compiler = compiler
-        self.compile_html = self.compiler.compile_html
+        self.is_post = use_in_feeds
+        self.messages = messages
+        self._template_name = template_name
+        self.compile_html = self.compiler.compile
         self.demote_headers = self.compiler.demote_headers and self.config['DEMOTE_HEADERS']
-        tzinfo = self.config['__tzinfo__']
+        self._dependency_file_fragment = defaultdict(list)
+        self._dependency_file_page = defaultdict(list)
+        self._dependency_uptodate_fragment = defaultdict(list)
+        self._dependency_uptodate_page = defaultdict(list)
+        self._depfile = defaultdict(list)
+        if metadata_extractors_by is None:
+            self.metadata_extractors_by = {'priority': {}, 'source': {}}
+        else:
+            self.metadata_extractors_by = metadata_extractors_by
+
+        self._set_translated_to()
+        self._set_folders(destination, destination_base)
+
+        # Load default metadata
+        default_metadata, default_used_extractor = get_meta(self, lang=None)
+        self.meta = Functionary(lambda: None, self.default_lang)
+        self.used_extractor = Functionary(lambda: None, self.default_lang)
+        self.meta[self.default_lang] = default_metadata
+        self.used_extractor[self.default_lang] = default_used_extractor
+
+        self._set_date(default_metadata)
+
+        # These are the required metadata fields
+        if 'title' not in default_metadata or 'slug' not in default_metadata:
+            raise ValueError("You must set a title (found '{0}') and a slug (found '{1}')! "
+                             "[in file {2}]".format(default_metadata.get('title', None),
+                                                    default_metadata.get('slug', None),
+                                                    source_path))
+
+        if 'type' not in default_metadata:
+            default_metadata['type'] = 'text'
+
+        self._load_translated_metadata(default_metadata)
+        self._load_data()
+        self.__migrate_section_to_category()
+        self._set_tags()
+
+        self.publish_later = False if self.current_time is None else self.date >= self.current_time
+
+        # While draft comes from the tags, it's not really a tag
+        self.use_in_feeds = self.is_post and not self.is_draft and not self.is_private and not self.publish_later
+
+        # Allow overriding URL_TYPE via meta
+        # The check is done here so meta dicts won’t change inside of
+        # generic_post_renderer
+        self.url_type = self.meta('url_type') or None
+        # Register potential extra dependencies
+        self.compiler.register_extra_dependencies(self)
+
+    def _load_config(self, config):
+        """Set members to configured values."""
+        self.config = config
         if self.config['FUTURE_IS_NOW']:
             self.current_time = None
         else:
-            self.current_time = current_time(tzinfo)
-        self.translated_to = set([])
-        self._prev_post = None
-        self._next_post = None
+            self.current_time = current_time(self.config['__tzinfo__'])
         self.base_url = self.config['BASE_URL']
-        self.is_draft = False
-        self.is_private = False
-        self.is_mathjax = False
         self.strip_indexes = self.config['STRIP_INDEXES']
         self.index_file = self.config['INDEX_FILE']
         self.pretty_urls = self.config['PRETTY_URLS']
+        self.default_lang = self.config['DEFAULT_LANG']
+        self.translations = self.config['TRANSLATIONS']
+        self.skip_untranslated = not self.config['SHOW_UNTRANSLATED_POSTS']
+        self._default_preview_image = self.config['DEFAULT_PREVIEW_IMAGE']
+        self.types_to_hide_title = self.config['TYPES_TO_HIDE_TITLE']
+
+    def _set_tags(self):
+        """Set post tags."""
+        self._tags = {}
+        for lang in self.translated_to:
+            if isinstance(self.meta[lang]['tags'], (list, tuple, set)):
+                _tag_list = self.meta[lang]['tags']
+            else:
+                _tag_list = self.meta[lang]['tags'].split(',')
+            self._tags[lang] = natsort.natsorted(
+                list(set([x.strip() for x in _tag_list])),
+                alg=natsort.ns.F | natsort.ns.IC)
+            self._tags[lang] = [t for t in self._tags[lang] if t]
+
+            status = self.meta[lang].get('status')
+            if status:
+                if status == 'published':
+                    pass  # already set before, mixing published + something else should result in the other thing
+                elif status == 'featured':
+                    self.post_status = status
+                elif status == 'private':
+                    self.post_status = status
+                    self.is_private = True
+                elif status == 'draft':
+                    self.post_status = status
+                    self.is_draft = True
+                else:
+                    LOGGER.warning(('The post "{0}" has the unknown status "{1}". '
+                                    'Valid values are "published", "featured", "private" and "draft".').format(self.source_path, status))
+
+            if self.config['WARN_ABOUT_TAG_METADATA']:
+                show_warning = False
+                if 'draft' in [_.lower() for _ in self._tags[lang]]:
+                    LOGGER.warning('The post "{0}" uses the "draft" tag.'.format(self.source_path))
+                    show_warning = True
+                if 'private' in self._tags[lang]:
+                    LOGGER.warning('The post "{0}" uses the "private" tag.'.format(self.source_path))
+                    show_warning = True
+                if 'mathjax' in self._tags[lang]:
+                    LOGGER.warning('The post "{0}" uses the "mathjax" tag.'.format(self.source_path))
+                    show_warning = True
+                if show_warning:
+                    LOGGER.warning('It is suggested that you convert special tags to metadata and set '
+                                   'USE_TAG_METADATA to False. You can use the upgrade_metadata_v8 '
+                                   'command plugin for conversion (install with: nikola plugin -i '
+                                   'upgrade_metadata_v8). Change the WARN_ABOUT_TAG_METADATA '
+                                   'configuration to disable this warning.')
+            if self.config['USE_TAG_METADATA']:
+                if 'draft' in [_.lower() for _ in self._tags[lang]]:
+                    self.is_draft = True
+                    LOGGER.debug('The post "{0}" is a draft.'.format(self.source_path))
+                    self._tags[lang].remove('draft')
+                    self.post_status = 'draft'
+                    self.has_oldstyle_metadata_tags = True
+
+                if 'private' in self._tags[lang]:
+                    self.is_private = True
+                    LOGGER.debug('The post "{0}" is private.'.format(self.source_path))
+                    self._tags[lang].remove('private')
+                    self.post_status = 'private'
+                    self.has_oldstyle_metadata_tags = True
+
+                if 'mathjax' in self._tags[lang]:
+                    self.has_oldstyle_metadata_tags = True
+
+    def _set_paths(self, source_path):
+        """Set the various paths and the post_name.
+
+        TODO: WTF is all this.
+        """
         self.source_path = source_path  # posts/blah.txt
         self.post_name = os.path.splitext(source_path)[0]  # posts/blah
+        _relpath = os.path.relpath(self.post_name)
+        if _relpath != self.post_name:
+            self.post_name = _relpath.replace('..' + os.sep, '_..' + os.sep)
         # cache[\/]posts[\/]blah.html
         self.base_path = os.path.join(self.config['CACHE_FOLDER'], self.post_name + ".html")
         # cache/posts/blah.html
         self._base_path = self.base_path.replace('\\', '/')
         self.metadata_path = self.post_name + ".meta"  # posts/blah.meta
-        self.folder = destination
-        self.translations = self.config['TRANSLATIONS']
-        self.default_lang = self.config['DEFAULT_LANG']
-        self.messages = messages
-        self.skip_untranslated = not self.config['SHOW_UNTRANSLATED_POSTS']
-        self._template_name = template_name
-        self.is_two_file = True
-        self.newstylemeta = True
-        self.hyphenate = self.config['HYPHENATE']
-        self._reading_time = None
-        self._remaining_reading_time = None
-        self._paragraph_count = None
-        self._remaining_paragraph_count = None
-        self._dependency_file_fragment = defaultdict(list)
-        self._dependency_file_page = defaultdict(list)
-        self._dependency_uptodate_fragment = defaultdict(list)
-        self._dependency_uptodate_page = defaultdict(list)
-
-        default_metadata, self.newstylemeta = get_meta(self, self.config['FILE_METADATA_REGEXP'], self.config['UNSLUGIFY_TITLES'])
 
-        self.meta = Functionary(lambda: None, self.default_lang)
-        self.meta[self.default_lang] = default_metadata
-
-        # Load internationalized metadata
+    def _set_translated_to(self):
+        """Find post's translations."""
+        self.translated_to = set([])
         for lang in self.translations:
             if os.path.isfile(get_translation_candidate(self.config, self.source_path, lang)):
                 self.translated_to.add(lang)
+
+        # If we don't have anything in translated_to, the file does not exist
+        if not self.translated_to and os.path.isfile(self.source_path):
+            raise Exception(("Could not find translations for {}, check your "
+                            "TRANSLATIONS_PATTERN").format(self.source_path))
+        elif not self.translated_to:
+            raise Exception(("Cannot use {} (not a file, perhaps a broken "
+                            "symbolic link?)").format(self.source_path))
+
+    def _set_folders(self, destination, destination_base):
+        """Compose destination paths."""
+        self.folder_relative = destination
+        self.folder_base = destination_base
+
+        if self.folder_base is not None:
+            # Use translatable destination folders
+            self.folders = {}
+            for lang in self.config['TRANSLATIONS']:
+                if os.path.isabs(self.folder_base(lang)):  # Issue 2982
+                    self.folder_base[lang] = os.path.relpath(self.folder_base(lang), '/')
+                self.folders[lang] = os.path.normpath(os.path.join(self.folder_base(lang), self.folder_relative))
+        else:
+            # Old behavior (non-translatable destination path, normalized by scanner)
+            self.folders = {lang: self.folder_relative for lang in self.config['TRANSLATIONS'].keys()}
+        self.folder = self.folders[self.default_lang]
+
+    def __migrate_section_to_category(self):
+        """TODO: remove in v9."""
+        for lang, meta in self.meta.items():
+            # Migrate section to category
+            # TODO: remove in v9
+            if 'section' in meta:
+                if 'category' in meta:
+                    LOGGER.warning("Post {0} has both 'category' and 'section' metadata. Section will be ignored.".format(self.source_path))
+                else:
+                    meta['category'] = meta['section']
+                    LOGGER.info("Post {0} uses 'section' metadata, setting its value to 'category'".format(self.source_path))
+
+            # Handle CATEGORY_DESTPATH_AS_DEFAULT
+            if 'category' not in meta and self.config['CATEGORY_DESTPATH_AS_DEFAULT']:
+                self.category_from_destpath = True
+                if self.config['CATEGORY_DESTPATH_TRIM_PREFIX'] and self.folder_relative != '.':
+                    category = self.folder_relative
+                else:
+                    category = self.folders[lang]
+                category = category.replace(os.sep, '/')
+                if self.config['CATEGORY_DESTPATH_FIRST_DIRECTORY_ONLY']:
+                    category = category.split('/')[0]
+                meta['category'] = self.config['CATEGORY_DESTPATH_NAMES'](lang).get(category, category)
+            else:
+                self.category_from_destpath = False
+
+    def _load_data(self):
+        """Load data field from metadata."""
+        self.data = Functionary(lambda: None, self.default_lang)
+        for lang in self.translations:
+            if self.meta[lang].get('data') is not None:
+                self.data[lang] = utils.load_data(self.meta[lang]['data'])
+
+    def _load_translated_metadata(self, default_metadata):
+        """Load metadata from all translation sources."""
+        for lang in self.translations:
             if lang != self.default_lang:
                 meta = defaultdict(lambda: '')
                 meta.update(default_metadata)
-                _meta, _nsm = get_meta(self, self.config['FILE_METADATA_REGEXP'], self.config['UNSLUGIFY_TITLES'], lang)
-                self.newstylemeta = self.newstylemeta and _nsm
+                _meta, _extractors = get_meta(self, lang)
                 meta.update(_meta)
                 self.meta[lang] = meta
+                self.used_extractor[lang] = _extractors
 
         if not self.is_translation_available(self.default_lang):
             # Special case! (Issue #373)
@@ -163,81 +341,80 @@ class Post(object):
             for lang in sorted(self.translated_to):
                 default_metadata.update(self.meta[lang])
 
-        if 'date' not in default_metadata and not use_in_feeds:
-            # For stories we don't *really* need a date
+    def _set_date(self, default_metadata):
+        """Set post date/updated based on metadata and configuration."""
+        if 'date' not in default_metadata and not self.is_post:
+            # For pages we don't *really* need a date
             if self.config['__invariant__']:
-                default_metadata['date'] = datetime.datetime(2013, 12, 31, 23, 59, 59, tzinfo=tzinfo)
+                default_metadata['date'] = datetime.datetime(2013, 12, 31, 23, 59, 59, tzinfo=self.config['__tzinfo__'])
             else:
                 default_metadata['date'] = datetime.datetime.utcfromtimestamp(
-                    os.stat(self.source_path).st_ctime).replace(tzinfo=dateutil.tz.tzutc()).astimezone(tzinfo)
+                    os.stat(self.source_path).st_ctime).replace(tzinfo=dateutil.tz.tzutc()).astimezone(self.config['__tzinfo__'])
 
         # If time zone is set, build localized datetime.
-        self.date = to_datetime(self.meta[self.default_lang]['date'], tzinfo)
+        try:
+            self.date = to_datetime(self.meta[self.default_lang]['date'], self.config['__tzinfo__'])
+        except ValueError:
+            if not self.meta[self.default_lang]['date']:
+                msg = 'Missing date in file {}'.format(self.source_path)
+            else:
+                msg = "Invalid date '{0}' in file {1}".format(self.meta[self.default_lang]['date'], self.source_path)
+            LOGGER.error(msg)
+            raise ValueError(msg)
 
         if 'updated' not in default_metadata:
             default_metadata['updated'] = default_metadata.get('date', None)
 
-        self.updated = to_datetime(default_metadata['updated'])
-
-        if 'title' not in default_metadata or 'slug' not in default_metadata \
-                or 'date' not in default_metadata:
-            raise OSError("You must set a title (found '{0}'), a slug (found "
-                          "'{1}') and a date (found '{2}')! [in file "
-                          "{3}]".format(default_metadata.get('title', None),
-                                        default_metadata.get('slug', None),
-                                        default_metadata.get('date', None),
-                                        source_path))
-
-        if 'type' not in default_metadata:
-            # default value is 'text'
-            default_metadata['type'] = 'text'
+        self.updated = to_datetime(default_metadata['updated'], self.config['__tzinfo__'])
 
-        self.publish_later = False if self.current_time is None else self.date >= self.current_time
+    @property
+    def hyphenate(self):
+        """Post is hyphenated."""
+        return bool(self.config['HYPHENATE'] or self.meta('hyphenate'))
 
-        is_draft = False
-        is_private = False
-        self._tags = {}
-        for lang in self.translated_to:
-            self._tags[lang] = natsort.natsorted(
-                list(set([x.strip() for x in self.meta[lang]['tags'].split(',')])),
-                alg=natsort.ns.F | natsort.ns.IC)
-            self._tags[lang] = [t for t in self._tags[lang] if t]
-            if 'draft' in [_.lower() for _ in self._tags[lang]]:
-                is_draft = True
-                LOGGER.debug('The post "{0}" is a draft.'.format(self.source_path))
-                self._tags[lang].remove('draft')
-
-            # TODO: remove in v8
-            if 'retired' in self._tags[lang]:
-                is_private = True
-                LOGGER.warning('The "retired" tag in post "{0}" is now deprecated and will be removed in v8.  Use "private" instead.'.format(self.source_path))
-                self._tags[lang].remove('retired')
-            # end remove in v8
-
-            if 'private' in self._tags[lang]:
-                is_private = True
-                LOGGER.debug('The post "{0}" is private.'.format(self.source_path))
-                self._tags[lang].remove('private')
+    @property
+    def is_two_file(self):
+        """Post has a separate .meta file."""
+        if self._is_two_file is None:
+            return True
+        return self._is_two_file
 
-        # While draft comes from the tags, it's not really a tag
-        self.is_draft = is_draft
-        self.is_private = is_private
-        self.is_post = use_in_feeds
-        self.use_in_feeds = use_in_feeds and not is_draft and not is_private \
-            and not self.publish_later
+    @is_two_file.setter
+    def is_two_file(self, value):
+        """Set the is_two_file property, use with care.
 
-        # If mathjax is a tag, or it's a ipynb post, then enable mathjax rendering support
-        self.is_mathjax = ('mathjax' in self.tags) or (self.compiler.name == 'ipynb')
+        Caution: this MAY REWRITE THE POST FILE.
+        Only should happen if you effectively *change* the value.
 
-        # Register potential extra dependencies
-        self.compiler.register_extra_dependencies(self)
+        Arguments:
+            value {bool} -- Whether the post has a separate .meta file
+        """
+        # for lang in self.translated_to:
+
+        if self._is_two_file is None:
+            # Initial setting, this happens on post creation
+            self._is_two_file = value
+        elif value != self._is_two_file:
+            # Changing the value, this means you are transforming a 2-file
+            # into a 1-file or viceversa.
+            if value and not self.compiler.supports_metadata:
+                raise ValueError("Can't save metadata as 1-file using this compiler {}".format(self.compiler))
+            for lang in self.translated_to:
+                source = self.source(lang)
+                meta = self.meta(lang)
+                self._is_two_file = value
+                self.save(lang=lang, source=source, meta=meta)
+                if not value:  # Need to delete old meta file
+                    meta_path = get_translation_candidate(self.config, self.metadata_path, lang)
+                    if os.path.isfile(meta_path):
+                        os.unlink(meta_path)
 
     def __repr__(self):
         """Provide a representation of the post object."""
         # Calculate a hash that represents most data about the post
         m = hashlib.md5()
         # source_path modification date (to avoid reading it)
-        m.update(utils.unicode_str(os.stat(self.source_path).st_mtime).encode('utf-8'))
+        m.update(str(os.stat(self.source_path).st_mtime).encode('utf-8'))
         clean_meta = {}
         for k, v in self.meta.items():
             sub_meta = {}
@@ -245,16 +422,45 @@ class Post(object):
             for kk, vv in v.items():
                 if vv:
                     sub_meta[kk] = vv
-        m.update(utils.unicode_str(json.dumps(clean_meta, cls=utils.CustomEncoder, sort_keys=True)).encode('utf-8'))
+        m.update(str(json.dumps(clean_meta, cls=utils.CustomEncoder, sort_keys=True)).encode('utf-8'))
         return '<Post: {0!r} {1}>'.format(self.source_path, m.hexdigest())
 
+    def has_pretty_url(self, lang):
+        """Check if this page has a pretty URL."""
+        m = self.meta[lang].get('pretty_url', '')
+        if m:
+            # match is a non-empty string, overides anything
+            return m.lower() == 'true' or m.lower() == 'yes'
+        else:
+            # use PRETTY_URLS, unless the slug is 'index'
+            return self.pretty_urls and self.meta[lang]['slug'] != 'index'
+
     def _has_pretty_url(self, lang):
-        if self.pretty_urls and \
-                self.meta[lang].get('pretty_url', '') != 'False' and \
-                self.meta[lang]['slug'] != 'index':
+        """Check if this page has a pretty URL."""
+        return self.has_pretty_url(lang)
+
+    @property
+    def has_math(self):
+        """Return True if this post has has_math set to True or is a python notebook.
+
+        Alternatively, it will return True if it has set the mathjax tag in the
+        current language and the USE_TAG_METADATA config setting is True.
+        """
+        if self.compiler.name == 'ipynb':
             return True
-        else:
-            return False
+        lang = nikola.utils.LocaleBorg().current_lang
+        if self.is_translation_available(lang):
+            if self.meta[lang].get('has_math') in ('true', 'True', 'yes', '1', 1, True):
+                return True
+            if self.config['USE_TAG_METADATA']:
+                return 'mathjax' in self.tags_for_language(lang)
+        # If it has math in ANY other language, enable it. Better inefficient than broken.
+        for lang in self.translated_to:
+            if self.meta[lang].get('has_math') in ('true', 'True', 'yes', '1', 1, True):
+                return True
+        if self.config['USE_TAG_METADATA']:
+            return 'mathjax' in self.alltags
+        return False
 
     @property
     def alltags(self):
@@ -294,7 +500,7 @@ class Post(object):
             rv = rv._prev_post
         return rv
 
-    @prev_post.setter  # NOQA
+    @prev_post.setter
     def prev_post(self, v):
         """Set previous post."""
         self._prev_post = v
@@ -312,7 +518,7 @@ class Post(object):
             rv = rv._next_post
         return rv
 
-    @next_post.setter  # NOQA
+    @next_post.setter
     def next_post(self, v):
         """Set next post."""
         self._next_post = v
@@ -320,26 +526,15 @@ class Post(object):
     @property
     def template_name(self):
         """Return template name for this post."""
-        return self.meta('template') or self._template_name
+        lang = nikola.utils.LocaleBorg().current_lang
+        return self.meta[lang]['template'] or self._template_name
 
     def formatted_date(self, date_format, date=None):
-        """Return the formatted date as unicode."""
-        date = date if date else self.date
-
-        if date_format == 'webiso':
-            # Formatted after RFC 3339 (web ISO 8501 profile) with Zulu
-            # zone desgignator for times in UTC and no microsecond precision.
-            fmt_date = date.replace(microsecond=0).isoformat().replace('+00:00', 'Z')
-        else:
-            fmt_date = date.strftime(date_format)
-
-        # Issue #383, this changes from py2 to py3
-        if isinstance(fmt_date, bytes_str):
-            fmt_date = fmt_date.decode('utf8')
-        return fmt_date
+        """Return the formatted date as string."""
+        return utils.LocaleBorg().formatted_date(date_format, date if date else self.date)
 
     def formatted_updated(self, date_format):
-        """Return the updated date as unicode."""
+        """Return the updated date as string."""
         return self.formatted_date(date_format, self.updated)
 
     def title(self, lang=None):
@@ -352,7 +547,7 @@ class Post(object):
             lang = nikola.utils.LocaleBorg().current_lang
         return self.meta[lang]['title']
 
-    def author(self, lang=None):
+    def author(self, lang=None) -> str:
         """Return localized author or BLOG_AUTHOR if unspecified.
 
         If lang is not specified, it defaults to the current language from
@@ -367,12 +562,38 @@ class Post(object):
 
         return author
 
+    def authors(self, lang=None) -> list:
+        """Return localized authors or BLOG_AUTHOR if unspecified.
+
+        If lang is not specified, it defaults to the current language from
+        templates, as set in LocaleBorg.
+        """
+        if lang is None:
+            lang = nikola.utils.LocaleBorg().current_lang
+        if self.meta[lang]['author']:
+            author = [i.strip() for i in self.meta[lang]['author'].split(",")]
+        else:
+            author = [self.config['BLOG_AUTHOR'](lang)]
+
+        return author
+
     def description(self, lang=None):
         """Return localized description."""
         if lang is None:
             lang = nikola.utils.LocaleBorg().current_lang
         return self.meta[lang]['description']
 
+    def guid(self, lang=None):
+        """Return localized GUID."""
+        if lang is None:
+            lang = nikola.utils.LocaleBorg().current_lang
+        if self.meta[lang]['guid']:
+            guid = self.meta[lang]['guid']
+        else:
+            guid = self.permalink(lang, absolute=True)
+
+        return guid
+
     def add_dependency(self, dependency, add='both', lang=None):
         """Add a file dependency for tasks using that post.
 
@@ -425,6 +646,27 @@ class Post(object):
         if add == 'page' or add == 'both':
             self._dependency_uptodate_page[lang].append((is_callable, dependency))
 
+    def register_depfile(self, dep, dest=None, lang=None):
+        """Register a dependency in the dependency file."""
+        if not dest:
+            dest = self.translated_base_path(lang)
+        self._depfile[dest].append(dep)
+
+    @staticmethod
+    def write_depfile(dest, deps_list, post=None, lang=None):
+        """Write a depfile for a given language."""
+        if post is None or lang is None:
+            deps_path = dest + '.dep'
+        else:
+            deps_path = post.compiler.get_dep_filename(post, lang)
+        if deps_list or (post.compiler.use_dep_file if post else False):
+            deps_list = [p for p in deps_list if p != dest]  # Don't depend on yourself (#1671)
+            with io.open(deps_path, "w+", encoding="utf-8") as deps_file:
+                deps_file.write('\n'.join(deps_list))
+        else:
+            if os.path.isfile(deps_path):
+                os.unlink(deps_path)
+
     def _get_dependencies(self, deps_list):
         deps = []
         for dep in deps_list:
@@ -444,17 +686,23 @@ class Post(object):
     def deps(self, lang):
         """Return a list of file dependencies to build this post's page."""
         deps = []
-        if self.default_lang in self.translated_to:
-            deps.append(self.base_path)
-            deps.append(self.source_path)
+        deps.append(self.base_path)
+        deps.append(self.source_path)
+        if os.path.exists(self.metadata_path):
+            deps.append(self.metadata_path)
         if lang != self.default_lang:
             cand_1 = get_translation_candidate(self.config, self.source_path, lang)
             cand_2 = get_translation_candidate(self.config, self.base_path, lang)
             if os.path.exists(cand_1):
                 deps.extend([cand_1, cand_2])
+            cand_3 = get_translation_candidate(self.config, self.metadata_path, lang)
+            if os.path.exists(cand_3):
+                deps.append(cand_3)
+        if self.meta('data', lang):
+            deps.append(self.meta('data', lang))
         deps += self._get_dependencies(self._dependency_file_page[lang])
         deps += self._get_dependencies(self._dependency_file_page[None])
-        return sorted(deps)
+        return sorted(set(deps))
 
     def deps_uptodate(self, lang):
         """Return a list of uptodate dependencies to build this post's page.
@@ -470,14 +718,6 @@ class Post(object):
 
     def compile(self, lang):
         """Generate the cache/ file with the compiled post."""
-        def wrap_encrypt(path, password):
-            """Wrap a post with encryption."""
-            with io.open(path, 'r+', encoding='utf8') as inf:
-                data = inf.read() + "<!--tail-->"
-            data = CRYPT.substitute(data=rc4(password, data))
-            with io.open(path, 'w+', encoding='utf8') as outf:
-                outf.write(data)
-
         dest = self.translated_base_path(lang)
         if not self.is_translation_available(lang) and not self.config['SHOW_UNTRANSLATED_POSTS']:
             return
@@ -486,26 +726,25 @@ class Post(object):
         self.compile_html(
             self.translated_source_path(lang),
             dest,
-            self.is_two_file),
-        if self.meta('password'):
-            # TODO: get rid of this feature one day (v8?; warning added in v7.3.0.)
-            LOGGER.warn("The post {0} is using the `password` attribute, which may stop working in the future.")
-            LOGGER.warn("Please consider switching to a more secure method of encryption.")
-            LOGGER.warn("More details: https://github.com/getnikola/nikola/issues/1547")
-            wrap_encrypt(dest, self.meta('password'))
+            self.is_two_file,
+            self,
+            lang)
+        Post.write_depfile(dest, self._depfile[dest], post=self, lang=lang)
+
+        signal('compiled').send({
+            'source': self.translated_source_path(lang),
+            'dest': dest,
+            'post': self,
+            'lang': lang,
+        })
+
         if self.publish_later:
-            LOGGER.notice('{0} is scheduled to be published in the future ({1})'.format(
+            LOGGER.info('{0} is scheduled to be published in the future ({1})'.format(
                 self.source_path, self.date))
 
     def fragment_deps(self, lang):
-        """Return a list of uptodate dependencies to build this post's fragment.
-
-        These dependencies should be included in ``uptodate`` for the task
-        which generates the fragment.
-        """
-        deps = []
-        if self.default_lang in self.translated_to:
-            deps.append(self.source_path)
+        """Return a list of dependencies to build this post's fragment."""
+        deps = [self.source_path]
         if os.path.isfile(self.metadata_path):
             deps.append(self.metadata_path)
         lang_deps = []
@@ -546,25 +785,95 @@ class Post(object):
         return get_translation_candidate(self.config, self.base_path, lang)
 
     def _translated_file_path(self, lang):
-        """Return path to the translation's file, or to the original."""
+        """Get path to a post's translation.
+
+        Returns path to the translation's file, or to as good a file as it can
+        plus "real" language of the text.
+        """
         if lang in self.translated_to:
             if lang == self.default_lang:
-                return self.base_path
+                return self.base_path, lang
             else:
-                return get_translation_candidate(self.config, self.base_path, lang)
+                return get_translation_candidate(self.config, self.base_path, lang), lang
         elif lang != self.default_lang:
-            return self.base_path
+            return self.base_path, self.default_lang
         else:
-            return get_translation_candidate(self.config, self.base_path, sorted(self.translated_to)[0])
+            real_lang = sorted(self.translated_to)[0]
+            return get_translation_candidate(self.config, self.base_path, real_lang), real_lang
+
+    def write_metadata(self, lang=None):
+        """Save the post's metadata.
+
+        Keep in mind that this will save either in the
+        post file or in a .meta file, depending on self.is_two_file.
+
+        metadata obtained from filenames or document contents will
+        be superseded by this, and becomes inaccessible.
+
+        Post contents will **not** be modified.
+
+        If you write to a language not in self.translated_to
+        an exception will be raised.
+
+        Remember to scan_posts(really=True) after you update metadata if
+        you want the rest of the system to know about the change.
+        """
+        if lang is None:
+            lang = nikola.utils.LocaleBorg().current_lang
+        if lang not in self.translated_to:
+            raise ValueError("Can't save post metadata to language [{}] it's not translated to.".format(lang))
+
+        source = self.source(lang)
+        source_path = self.translated_source_path(lang)
+        metadata = self.meta[lang]
+        self.compiler.create_post(source_path, content=source, onefile=not self.is_two_file, is_page=not self.is_post, **metadata)
+
+    def save(self, lang=None, source=None, meta=None):
+        """Write post source to disk.
+
+        Use this with utmost care, it may wipe out a post.
+
+        Keyword Arguments:
+            lang str -- Language for this source. If set to None,
+                use current language.
+            source str -- The source text for the post in the
+                language. If set to None, use current source for
+                this language.
+            meta dict -- Metadata for this language, if not set,
+                use current metadata for this language.
+        """
+        if lang is None:
+            lang = nikola.utils.LocaleBorg().current_lang
+        if source is None:
+            source = self.source(lang)
+        if meta is None:
+            metadata = self.meta[lang]
+        source_path = self.translated_source_path(lang)
+        metadata = self.meta[lang]
+        self.compiler.create_post(source_path, content=source, onefile=not self.is_two_file, is_page=not self.is_post, **metadata)
+
+    def source(self, lang=None):
+        """Read the post and return its source."""
+        if lang is None:
+            lang = nikola.utils.LocaleBorg().current_lang
+
+        source = self.translated_source_path(lang)
+        with open(source, 'r', encoding='utf-8-sig') as inf:
+            data = inf.read()
+        if self.is_two_file:  # Metadata is not here
+            source_data = data
+        else:
+            source_data = self.compiler.split_metadata(data, self, lang)[1]
+        return source_data
 
     def text(self, lang=None, teaser_only=False, strip_html=False, show_read_more_link=True,
-             rss_read_more_link=False, rss_links_append_query=None):
-        """Read the post file for that language and return its contents.
+             feed_read_more_link=False, feed_links_append_query=None):
+        """Read the post file for that language and return its compiled contents.
 
         teaser_only=True breaks at the teaser marker and returns only the teaser.
         strip_html=True removes HTML tags
         show_read_more_link=False does not add the Read more... link
-        rss_read_more_link=True uses RSS_READ_MORE_LINK instead of INDEX_READ_MORE_LINK
+        feed_read_more_link=True uses FEED_READ_MORE_LINK instead of INDEX_READ_MORE_LINK
         lang=None uses the last used to set locale
 
         All links in the returned HTML will be relative.
@@ -572,7 +881,7 @@ class Post(object):
         """
         if lang is None:
             lang = nikola.utils.LocaleBorg().current_lang
-        file_name = self._translated_file_path(lang)
+        file_name, real_lang = self._translated_file_path(lang)
 
         # Yes, we compile it and screw it.
         # This may be controversial, but the user (or someone) is asking for the post text
@@ -580,7 +889,7 @@ class Post(object):
         if not os.path.isfile(file_name):
             self.compile(lang)
 
-        with io.open(file_name, "r", encoding="utf8") as post_file:
+        with io.open(file_name, "r", encoding="utf-8-sig") as post_file:
             data = post_file.read().strip()
 
         if self.compiler.extension() == '.php':
@@ -592,16 +901,16 @@ class Post(object):
             if str(e) == "Document is empty":
                 return ""
             # let other errors raise
-            raise(e)
+            raise
         base_url = self.permalink(lang=lang)
         document.make_links_absolute(base_url)
 
         if self.hyphenate:
-            hyphenate(document, lang)
+            hyphenate(document, real_lang)
 
         try:
             data = lxml.html.tostring(document.body, encoding='unicode')
-        except:
+        except Exception:
             data = lxml.html.tostring(document, encoding='unicode')
 
         if teaser_only:
@@ -613,15 +922,16 @@ class Post(object):
                         teaser_text = teaser_regexp.search(data).groups()[-1]
                     else:
                         teaser_text = self.messages[lang]["Read more"]
-                    l = self.config['RSS_READ_MORE_LINK'](lang) if rss_read_more_link else self.config['INDEX_READ_MORE_LINK'](lang)
+                    l = self.config['FEED_READ_MORE_LINK'](lang) if feed_read_more_link else self.config['INDEX_READ_MORE_LINK'](lang)
                     teaser += l.format(
-                        link=self.permalink(lang, query=rss_links_append_query),
+                        link=self.permalink(lang, query=feed_links_append_query),
                         read_more=teaser_text,
                         min_remaining_read=self.messages[lang]["%d min remaining to read"] % (self.remaining_reading_time),
                         reading_time=self.reading_time,
                         remaining_reading_time=self.remaining_reading_time,
                         paragraph_count=self.paragraph_count,
-                        remaining_paragraph_count=self.remaining_paragraph_count)
+                        remaining_paragraph_count=self.remaining_paragraph_count,
+                        post_title=self.title(lang))
                 # This closes all open tags and sanitizes the broken HTML
                 document = lxml.html.fromstring(teaser)
                 try:
@@ -634,7 +944,7 @@ class Post(object):
                 # Not all posts have a body. For example, you may have a page statically defined in the template that does not take content as input.
                 content = lxml.html.fromstring(data)
                 data = content.text_content().strip()  # No whitespace wanted.
-            except lxml.etree.ParserError:
+            except (lxml.etree.ParserError, ValueError):
                 data = ""
         elif data:
             if self.demote_headers:
@@ -650,7 +960,7 @@ class Post(object):
 
     @property
     def reading_time(self):
-        """Reading time based on length of text."""
+        """Return reading time based on length of text."""
         if self._reading_time is None:
             text = self.text(strip_html=True)
             words_per_minute = 220
@@ -679,8 +989,8 @@ class Post(object):
         if self._paragraph_count is None:
             # duplicated with Post.text()
             lang = nikola.utils.LocaleBorg().current_lang
-            file_name = self._translated_file_path(lang)
-            with io.open(file_name, "r", encoding="utf8") as post_file:
+            file_name, _ = self._translated_file_path(lang)
+            with io.open(file_name, "r", encoding="utf-8-sig") as post_file:
                 data = post_file.read().strip()
             try:
                 document = lxml.html.fragment_fromstring(data, "body")
@@ -689,7 +999,7 @@ class Post(object):
                 if str(e) == "Document is empty":
                     return ""
                 # let other errors raise
-                raise(e)
+                raise
 
             # output is a float, for no real reason at all
             self._paragraph_count = int(document.xpath('count(//p)'))
@@ -707,7 +1017,7 @@ class Post(object):
                 if str(e) == "Document is empty":
                     return ""
                 # let other errors raise
-                raise(e)
+                raise
 
             self._remaining_paragraph_count = self.paragraph_count - int(document.xpath('count(//p)'))
         return self._remaining_paragraph_count
@@ -715,10 +1025,9 @@ class Post(object):
     def source_link(self, lang=None):
         """Return absolute link to the post's source."""
         ext = self.source_ext(True)
-        return "/" + self.destination_path(
-            lang=lang,
-            extension=ext,
-            sep='/')
+        link = "/" + self.destination_path(lang=lang, extension=ext, sep='/')
+        link = utils.encodelink(link)
+        return link
 
     def destination_path(self, lang=None, extension='.html', sep=os.sep):
         """Destination path for this post, relative to output/.
@@ -728,12 +1037,13 @@ class Post(object):
         """
         if lang is None:
             lang = nikola.utils.LocaleBorg().current_lang
-        if self._has_pretty_url(lang):
+        folder = self.folders[lang]
+        if self.has_pretty_url(lang):
             path = os.path.join(self.translations[lang],
-                                self.folder, self.meta[lang]['slug'], 'index' + extension)
+                                folder, self.meta[lang]['slug'], 'index' + extension)
         else:
             path = os.path.join(self.translations[lang],
-                                self.folder, self.meta[lang]['slug'] + extension)
+                                folder, self.meta[lang]['slug'] + extension)
         if sep != os.sep:
             path = path.replace(os.sep, sep)
         if path.startswith('./'):
@@ -750,8 +1060,8 @@ class Post(object):
             extension = self.compiler.extension()
 
         pieces = self.translations[lang].split(os.sep)
-        pieces += self.folder.split(os.sep)
-        if self._has_pretty_url(lang):
+        pieces += self.folders[lang].split(os.sep)
+        if self.has_pretty_url(lang):
             pieces += [self.meta[lang]['slug'], 'index' + extension]
         else:
             pieces += [self.meta[lang]['slug'] + extension]
@@ -764,6 +1074,7 @@ class Post(object):
             link = link[:-index_len]
         if query:
             link = link + "?" + query
+        link = utils.encodelink(link)
         return link
 
     @property
@@ -773,13 +1084,14 @@ class Post(object):
             lang = nikola.utils.LocaleBorg().current_lang
 
         image_path = self.meta[lang]['previewimage']
-
         if not image_path:
-            return None
+            image_path = self._default_preview_image
 
-        # This is further parsed by the template, because we don’t have access
-        # to the URL replacer here.  (Issue #1473)
-        return image_path
+        if not image_path or image_path.startswith("/"):
+            # Paths starting with slashes are expected to be root-relative, pass them directly.
+            return image_path
+        # Other paths are relative to the permalink. The path will be made prettier by the URL replacer later.
+        return urljoin(self.permalink(lang), image_path)
 
     def source_ext(self, prefix=False):
         """Return the source file extension.
@@ -795,227 +1107,143 @@ class Post(object):
         else:
             return ext
 
-# Code that fetches metadata from different places
-
-
-def re_meta(line, match=None):
-    """Find metadata using regular expressions."""
-    if match:
-        reStr = re.compile('^\.\. {0}: (.*)'.format(re.escape(match)))
-    else:
-        reStr = re.compile('^\.\. (.*?): (.*)')
-    result = reStr.findall(line.strip())
-    if match and result:
-        return (match, result[0])
-    elif not match and result:
-        return (result[0][0], result[0][1].strip())
-    else:
-        return (None,)
+    def should_hide_title(self):
+        """Return True if this post's title should be hidden. Use in templates to manage posts without titles."""
+        return self.title().strip() in ('NO TITLE', '') or self.meta('hidetitle') or \
+            self.meta('type').strip() in self.types_to_hide_title
 
+    def should_show_title(self):
+        """Return True if this post's title should be displayed. Use in templates to manage posts without titles."""
+        return not self.should_hide_title()
 
-def _get_metadata_from_filename_by_regex(filename, metadata_regexp, unslugify_titles):
-    """Try to reed the metadata from the filename based on the given re.
 
-    This requires to use symbolic group names in the pattern.
-    The part to read the metadata from the filename based on a regular
-    expression is taken from Pelican - pelican/readers.py
-    """
-    match = re.match(metadata_regexp, filename)
-    meta = {}
-
-    if match:
-        # .items() for py3k compat.
-        for key, value in match.groupdict().items():
-            k = key.lower().strip()  # metadata must be lowercase
-            if k == 'title' and unslugify_titles:
-                meta[k] = unslugify(value, discard_numbers=False)
-            else:
-                meta[k] = value
-
-    return meta
-
-
-def get_metadata_from_file(source_path, config=None, lang=None):
+def get_metadata_from_file(source_path, post, config, lang, metadata_extractors_by):
     """Extract metadata from the file itself, by parsing contents."""
     try:
         if lang and config:
             source_path = get_translation_candidate(config, source_path, lang)
         elif lang:
             source_path += '.' + lang
-        with io.open(source_path, "r", encoding="utf8") as meta_file:
-            meta_data = [x.strip() for x in meta_file.readlines()]
-        return _get_metadata_from_file(meta_data)
+        with io.open(source_path, "r", encoding="utf-8-sig") as meta_file:
+            source_text = meta_file.read()
     except (UnicodeDecodeError, UnicodeEncodeError):
-        raise ValueError('Error reading {0}: Nikola only supports UTF-8 files'.format(source_path))
+        msg = 'Error reading {0}: Nikola only supports UTF-8 files'.format(source_path)
+        LOGGER.error(msg)
+        raise ValueError(msg)
     except Exception:  # The file may not exist, for multilingual sites
-        return {}
+        return {}, None
 
-
-def _get_metadata_from_file(meta_data):
-    """Extract metadata from a post's source file."""
     meta = {}
-    if not meta_data:
-        return meta
-
-    re_md_title = re.compile(r'^{0}([^{0}].*)'.format(re.escape('#')))
-    # Assuming rst titles are going to be at least 4 chars long
-    # otherwise this detects things like ''' wich breaks other markups.
-    re_rst_title = re.compile(r'^([{0}]{{4,}})'.format(re.escape(
-        string.punctuation)))
-
-    # Skip up to one empty line at the beginning (for txt2tags)
-    if not meta_data[0]:
-        meta_data = meta_data[1:]
-
-    # First, get metadata from the beginning of the file,
-    # up to first empty line
-
-    for i, line in enumerate(meta_data):
-        if not line:
-            break
-        match = re_meta(line)
-        if match[0]:
-            meta[match[0]] = match[1]
-
-    # If we have no title, try to get it from document
-    if 'title' not in meta:
-        piece = meta_data[:]
-        for i, line in enumerate(piece):
-            if re_rst_title.findall(line) and i > 0:
-                meta['title'] = meta_data[i - 1].strip()
-                break
-            if (re_rst_title.findall(line) and i >= 0 and
-                    re_rst_title.findall(meta_data[i + 2])):
-                meta['title'] = meta_data[i + 1].strip()
-                break
-            if re_md_title.findall(line):
-                meta['title'] = re_md_title.findall(line)[0]
+    used_extractor = None
+    for priority in metadata_extractors.MetaPriority:
+        found_in_priority = False
+        for extractor in metadata_extractors_by['priority'].get(priority, []):
+            if not metadata_extractors.check_conditions(post, source_path, extractor.conditions, config, source_text):
+                continue
+            extractor.check_requirements()
+            new_meta = extractor.extract_text(source_text)
+            if new_meta:
+                found_in_priority = True
+                used_extractor = extractor
+                # Map metadata from other platforms to names Nikola expects (Issue #2817)
+                # Map metadata values (Issue #3025)
+                map_metadata(new_meta, extractor.map_from, config)
+
+                meta.update(new_meta)
                 break
 
-    return meta
+        if found_in_priority:
+            break
+    return meta, used_extractor
 
 
-def get_metadata_from_meta_file(path, config=None, lang=None):
+def get_metadata_from_meta_file(path, post, config, lang, metadata_extractors_by=None):
     """Take a post path, and gets data from a matching .meta file."""
-    global _UPGRADE_METADATA_ADVERTISED
     meta_path = os.path.splitext(path)[0] + '.meta'
     if lang and config:
         meta_path = get_translation_candidate(config, meta_path, lang)
     elif lang:
         meta_path += '.' + lang
     if os.path.isfile(meta_path):
-        with io.open(meta_path, "r", encoding="utf8") as meta_file:
-            meta_data = meta_file.readlines()
-
-        # Detect new-style metadata.
-        newstyleregexp = re.compile(r'\.\. .*?: .*')
-        newstylemeta = False
-        for l in meta_data:
-            if l.strip():
-                if re.match(newstyleregexp, l):
-                    newstylemeta = True
-
-        if newstylemeta:
-            # New-style metadata is basically the same as reading metadata from
-            # a 1-file post.
-            return get_metadata_from_file(path, config, lang), newstylemeta
-        else:
-            if not _UPGRADE_METADATA_ADVERTISED:
-                LOGGER.warn("Some posts on your site have old-style metadata. You should upgrade them to the new format, with support for extra fields.")
-                LOGGER.warn("Install the 'upgrade_metadata' plugin (with 'nikola plugin -i upgrade_metadata') and run 'nikola upgrade_metadata'.")
-                _UPGRADE_METADATA_ADVERTISED = True
-            while len(meta_data) < 7:
-                meta_data.append("")
-            (title, slug, date, tags, link, description, _type) = [
-                x.strip() for x in meta_data][:7]
-
-            meta = {}
-
-            if title:
-                meta['title'] = title
-            if slug:
-                meta['slug'] = slug
-            if date:
-                meta['date'] = date
-            if tags:
-                meta['tags'] = tags
-            if link:
-                meta['link'] = link
-            if description:
-                meta['description'] = description
-            if _type:
-                meta['type'] = _type
-
-            return meta, newstylemeta
-
+        return get_metadata_from_file(meta_path, post, config, lang, metadata_extractors_by)
     elif lang:
         # Metadata file doesn't exist, but not default language,
         # So, if default language metadata exists, return that.
         # This makes the 2-file format detection more reliable (Issue #525)
-        return get_metadata_from_meta_file(path, config, lang=None)
-    else:
-        return {}, True
+        return get_metadata_from_meta_file(meta_path, post, config, None, metadata_extractors_by)
+    else:  # No 2-file metadata
+        return {}, None
 
 
-def get_meta(post, file_metadata_regexp=None, unslugify_titles=False, lang=None):
-    """Get post's meta from source.
-
-    If ``file_metadata_regexp`` is given it will be tried to read
-    metadata from the filename.
-    If ``unslugify_titles`` is True, the extracted title (if any) will be unslugified, as is done in galleries.
-    If any metadata is then found inside the file the metadata from the
-    file will override previous findings.
-    """
+def get_meta(post, lang):
+    """Get post meta from compiler or source file."""
     meta = defaultdict(lambda: '')
+    used_extractor = None
 
-    try:
-        config = post.config
-    except AttributeError:
-        config = None
+    config = getattr(post, 'config', None)
+    metadata_extractors_by = getattr(post, 'metadata_extractors_by')
+    if metadata_extractors_by is None:
+        metadata_extractors_by = metadata_extractors.default_metadata_extractors_by()
 
-    _, newstylemeta = get_metadata_from_meta_file(post.metadata_path, config, lang)
-    meta.update(_)
+    # If meta file exists, use it
+    metafile_meta, used_extractor = get_metadata_from_meta_file(post.metadata_path, post, config, lang, metadata_extractors_by)
 
-    if not meta:
-        post.is_two_file = False
+    is_two_file = bool(metafile_meta)
 
-    if file_metadata_regexp is not None:
-        meta.update(_get_metadata_from_filename_by_regex(post.source_path,
-                                                         file_metadata_regexp,
-                                                         unslugify_titles))
+    # Filename-based metadata extractors (priority 1).
+    if config.get('FILE_METADATA_REGEXP'):
+        extractors = metadata_extractors_by['source'].get(metadata_extractors.MetaSource.filename, [])
+        for extractor in extractors:
+            if not metadata_extractors.check_conditions(post, post.source_path, extractor.conditions, config, None):
+                continue
+            meta.update(extractor.extract_filename(post.source_path, lang))
 
+    # Fetch compiler metadata (priority 2, overrides filename-based metadata).
     compiler_meta = {}
 
-    if getattr(post, 'compiler', None):
-        compiler_meta = post.compiler.read_metadata(post, file_metadata_regexp, unslugify_titles, lang)
+    if (getattr(post, 'compiler', None) and post.compiler.supports_metadata and
+            metadata_extractors.check_conditions(post, post.source_path, post.compiler.metadata_conditions, config, None)):
+        compiler_meta = post.compiler.read_metadata(post, lang=lang)
+        used_extractor = post.compiler
         meta.update(compiler_meta)
 
-    if not post.is_two_file and not compiler_meta:
-        # Meta file has precedence over file, which can contain garbage.
-        # Moreover, we should not to talk to the file if we have compiler meta.
-        meta.update(get_metadata_from_file(post.source_path, config, lang))
+    # Meta files and inter-file metadata (priority 3, overrides compiler and filename-based metadata).
+    if not metafile_meta:
+        new_meta, used_extractor = get_metadata_from_file(post.source_path, post, config, lang, metadata_extractors_by)
+        meta.update(new_meta)
+    else:
+        meta.update(metafile_meta)
 
     if lang is None:
         # Only perform these checks for the default language
-
         if 'slug' not in meta:
             # If no slug is found in the metadata use the filename
-            meta['slug'] = slugify(unicode_str(os.path.splitext(
-                os.path.basename(post.source_path))[0]))
+            meta['slug'] = slugify(os.path.splitext(
+                os.path.basename(post.source_path))[0], post.default_lang)
 
         if 'title' not in meta:
             # If no title is found, use the filename without extension
             meta['title'] = os.path.splitext(
                 os.path.basename(post.source_path))[0]
 
-    return meta, newstylemeta
+    # Set one-file status basing on default language only (Issue #3191)
+    if is_two_file or lang is None:
+        # Direct access because setter is complicated
+        post._is_two_file = is_two_file
+
+    return meta, used_extractor
 
 
 def hyphenate(dom, _lang):
     """Hyphenate a post."""
     # circular import prevention
     from .nikola import LEGAL_VALUES
-    lang = LEGAL_VALUES['PYPHEN_LOCALES'].get(_lang, pyphen.language_fallback(_lang))
+    lang = None
+    if pyphen is not None:
+        lang = LEGAL_VALUES['PYPHEN_LOCALES'].get(_lang, pyphen.language_fallback(_lang))
+    else:
+        utils.req_missing(['pyphen'], 'hyphenate texts', optional=True)
+    hyphenator = None
     if pyphen is not None and lang is not None:
         # If pyphen does exist, we tell the user when configuring the site.
         # If it does not support a language, we ignore it quietly.
@@ -1024,13 +1252,15 @@ def hyphenate(dom, _lang):
         except KeyError:
             LOGGER.error("Cannot find hyphenation dictoniaries for {0} (from {1}).".format(lang, _lang))
             LOGGER.error("Pyphen cannot be installed to ~/.local (pip install --user).")
+    if hyphenator is not None:
         for tag in ('p', 'li', 'span'):
             for node in dom.xpath("//%s[not(parent::pre)]" % tag):
                 skip_node = False
-                skippable_nodes = ['kbd', 'code', 'samp', 'mark', 'math', 'data', 'ruby', 'svg']
+                skippable_nodes = ['kbd', 'pre', 'code', 'samp', 'mark', 'math', 'data', 'ruby', 'svg']
                 if node.getchildren():
                     for child in node.getchildren():
-                        if child.tag in skippable_nodes or (child.tag == 'span' and 'math' in child.get('class', [])):
+                        if child.tag in skippable_nodes or (child.tag == 'span' and 'math'
+                                                            in child.get('class', [])):
                             skip_node = True
                 elif 'math' in node.get('class', []):
                     skip_node = True
@@ -1049,8 +1279,14 @@ def insert_hyphens(node, hyphenator):
         text = getattr(node, attr)
         if not text:
             continue
-        new_data = ' '.join([hyphenator.inserted(w, hyphen='\u00AD')
-                             for w in text.split(' ')])
+
+        lines = text.splitlines()
+        new_data = "\n".join(
+            [
+                " ".join([hyphenator.inserted(w, hyphen="\u00AD") for w in line.split(" ")])
+                for line in lines
+            ]
+        )
         # Spaces are trimmed, we have to add them manually back
         if text[0].isspace():
             new_data = ' ' + new_data
@@ -1060,53 +1296,3 @@ def insert_hyphens(node, hyphenator):
 
     for child in node.iterchildren():
         insert_hyphens(child, hyphenator)
-
-
-CRYPT = string.Template("""\
-<script>
-function rc4(key, str) {
-    var s = [], j = 0, x, res = '';
-    for (var i = 0; i < 256; i++) {
-        s[i] = i;
-    }
-    for (i = 0; i < 256; i++) {
-        j = (j + s[i] + key.charCodeAt(i % key.length)) % 256;
-        x = s[i];
-        s[i] = s[j];
-        s[j] = x;
-    }
-    i = 0;
-    j = 0;
-    for (var y = 0; y < str.length; y++) {
-        i = (i + 1) % 256;
-        j = (j + s[i]) % 256;
-        x = s[i];
-        s[i] = s[j];
-        s[j] = x;
-        res += String.fromCharCode(str.charCodeAt(y) ^ s[(s[i] + s[j]) % 256]);
-    }
-    return res;
-}
-function decrypt() {
-    key = $$("#key").val();
-    crypt_div = $$("#encr")
-    crypted = crypt_div.html();
-    decrypted = rc4(key, window.atob(crypted));
-    if (decrypted.substr(decrypted.length - 11) == "<!--tail-->"){
-        crypt_div.html(decrypted);
-        $$("#pwform").hide();
-        crypt_div.show();
-    } else { alert("Wrong password"); };
-}
-</script>
-
-<div id="encr" style="display: none;">${data}</div>
-<div id="pwform">
-<form onsubmit="javascript:decrypt(); return false;" class="form-inline">
-<fieldset>
-<legend>This post is password-protected.</legend>
-<input type="password" id="key" placeholder="Type password here">
-<button type="submit" class="btn">Show Content</button>
-</fieldset>
-</form>
-</div>""")