New upstream version 8.1.2.upstream/8.1.2

1 files changed, 243 insertions, 81 deletions

diff --git a/nikola/plugins/task/galleries.py b/nikola/plugins/task/galleries.py
index c0df4a4..b8ac9ee 100644
--- a/nikola/plugins/task/galleries.py
+++ b/nikola/plugins/task/galleries.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright © 2012-2015 Roberto Alsina and others.
+# Copyright © 2012-2020 Roberto Alsina and others.
 
 # Permission is hereby granted, free of charge, to any
 # person obtaining a copy of this software and associated
@@ -26,38 +26,33 @@
 
 """Render image galleries."""
 
-from __future__ import unicode_literals
 import datetime
 import glob
 import io
 import json
 import mimetypes
 import os
-import sys
-try:
-    from urlparse import urljoin
-except ImportError:
-    from urllib.parse import urljoin  # NOQA
+from collections import OrderedDict
+from urllib.parse import urljoin
 
 import natsort
-try:
-    from PIL import Image  # NOQA
-except ImportError:
-    import Image as _Image
-    Image = _Image
-
 import PyRSS2Gen as rss
+from PIL import Image
 
 from nikola.plugin_categories import Task
 from nikola import utils
 from nikola.image_processing import ImageProcessor
 from nikola.post import Post
 
+try:
+    from ruamel.yaml import YAML
+except ImportError:
+    YAML = None
+
 _image_size_cache = {}
 
 
 class Galleries(Task, ImageProcessor):
-
     """Render image galleries."""
 
     name = 'render_galleries'
@@ -65,12 +60,11 @@ class Galleries(Task, ImageProcessor):
 
     def set_site(self, site):
         """Set Nikola site."""
+        super().set_site(site)
         site.register_path_handler('gallery', self.gallery_path)
         site.register_path_handler('gallery_global', self.gallery_global_path)
         site.register_path_handler('gallery_rss', self.gallery_rss_path)
 
-        self.logger = utils.get_logger('render_galleries', utils.STDERR_HANDLER)
-
         self.kw = {
             'thumbnail_size': site.config['THUMBNAIL_SIZE'],
             'max_image_size': site.config['MAX_IMAGE_SIZE'],
@@ -87,6 +81,13 @@ class Galleries(Task, ImageProcessor):
             'tzinfo': site.tzinfo,
             'comments_in_galleries': site.config['COMMENTS_IN_GALLERIES'],
             'generate_rss': site.config['GENERATE_RSS'],
+            'preserve_exif_data': site.config['PRESERVE_EXIF_DATA'],
+            'exif_whitelist': site.config['EXIF_WHITELIST'],
+            'preserve_icc_profiles': site.config['PRESERVE_ICC_PROFILES'],
+            'index_path': site.config['INDEX_PATH'],
+            'disable_indexes': site.config['DISABLE_INDEXES'],
+            'galleries_use_thumbnail': site.config['GALLERIES_USE_THUMBNAIL'],
+            'galleries_default_thumbnail': site.config['GALLERIES_DEFAULT_THUMBNAIL'],
         }
 
         # Verify that no folder in GALLERY_FOLDERS appears twice
@@ -94,8 +95,8 @@ class Galleries(Task, ImageProcessor):
         for source, dest in self.kw['gallery_folders'].items():
             if source in appearing_paths or dest in appearing_paths:
                 problem = source if source in appearing_paths else dest
-                utils.LOGGER.error("The gallery input or output folder '{0}' appears in more than one entry in GALLERY_FOLDERS, exiting.".format(problem))
-                sys.exit(1)
+                utils.LOGGER.error("The gallery input or output folder '{0}' appears in more than one entry in GALLERY_FOLDERS, ignoring.".format(problem))
+                continue
             appearing_paths.add(source)
             appearing_paths.add(dest)
 
@@ -104,8 +105,6 @@ class Galleries(Task, ImageProcessor):
         # Create self.gallery_links
         self.create_galleries_paths()
 
-        return super(Galleries, self).set_site(site)
-
     def _find_gallery_path(self, name):
         # The system using self.proper_gallery_links and self.improper_gallery_links
         # is similar as in listings.py.
@@ -116,30 +115,56 @@ class Galleries(Task, ImageProcessor):
             if len(candidates) == 1:
                 return candidates[0]
             self.logger.error("Gallery name '{0}' is not unique! Possible output paths: {1}".format(name, candidates))
+            raise RuntimeError("Gallery name '{0}' is not unique! Possible output paths: {1}".format(name, candidates))
         else:
             self.logger.error("Unknown gallery '{0}'!".format(name))
             self.logger.info("Known galleries: " + str(list(self.proper_gallery_links.keys())))
-        sys.exit(1)
+            raise RuntimeError("Unknown gallery '{0}'!".format(name))
 
     def gallery_path(self, name, lang):
-        """Return a gallery path."""
+        """Link to an image gallery's path.
+
+        It will try to find a gallery with that name if it's not ambiguous
+        or with that path. For example:
+
+        link://gallery/london => /galleries/trips/london/index.html
+
+        link://gallery/trips/london => /galleries/trips/london/index.html
+        """
         gallery_path = self._find_gallery_path(name)
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] +
                 gallery_path.split(os.sep) +
                 [self.site.config['INDEX_FILE']] if _f]
 
     def gallery_global_path(self, name, lang):
-        """Return the global gallery path, which contains images."""
+        """Link to the global gallery path, which contains all the images in galleries.
+
+        There is only one copy of an image on multilingual blogs, in the site root.
+
+        link://gallery_global/london => /galleries/trips/london/index.html
+
+        link://gallery_global/trips/london => /galleries/trips/london/index.html
+
+        (a ``gallery`` link could lead to eg. /en/galleries/trips/london/index.html)
+        """
         gallery_path = self._find_gallery_path(name)
         return [_f for _f in gallery_path.split(os.sep) +
                 [self.site.config['INDEX_FILE']] if _f]
 
     def gallery_rss_path(self, name, lang):
-        """Return path to the RSS file for a gallery."""
+        """Link to an image gallery's RSS feed.
+
+        It will try to find a gallery with that name if it's not ambiguous
+        or with that path. For example:
+
+        link://gallery_rss/london => /galleries/trips/london/rss.xml
+
+        link://gallery_rss/trips/london => /galleries/trips/london/rss.xml
+        """
         gallery_path = self._find_gallery_path(name)
         return [_f for _f in [self.site.config['TRANSLATIONS'][lang]] +
                 gallery_path.split(os.sep) +
-                ['rss.xml'] if _f]
+                [self.site.config['RSS_FILENAME_BASE'](lang) + self.site.config['RSS_EXTENSION']] if _f]
 
     def gen_tasks(self):
         """Render image galleries."""
@@ -147,8 +172,9 @@ class Galleries(Task, ImageProcessor):
         self.image_ext_list.extend(self.site.config.get('EXTRA_IMAGE_EXTENSIONS', []))
 
         for k, v in self.site.GLOBAL_CONTEXT['template_hooks'].items():
-            self.kw['||template_hooks|{0}||'.format(k)] = v._items
+            self.kw['||template_hooks|{0}||'.format(k)] = v.calculate_deps()
 
+        self.site.scan_posts()
         yield self.group_task()
 
         template_name = "gallery.tmpl"
@@ -170,13 +196,6 @@ class Galleries(Task, ImageProcessor):
             # Create image list, filter exclusions
             image_list = self.get_image_list(gallery)
 
-            # Sort as needed
-            # Sort by date
-            if self.kw['sort_by_date']:
-                image_list.sort(key=lambda a: self.image_date(a))
-            else:  # Sort by name
-                image_list.sort()
-
             # Create thumbnails and large images in destination
             for image in image_list:
                 for task in self.create_target_images(image, input_folder):
@@ -187,8 +206,6 @@ class Galleries(Task, ImageProcessor):
                 for task in self.remove_excluded_image(image, input_folder):
                     yield task
 
-            crumbs = utils.get_crumbs(gallery, index_folder=self)
-
             for lang in self.kw['translations']:
                 # save navigation links as dependencies
                 self.kw['navigation_links|{0}'.format(lang)] = self.kw['global_context']['navigation_links'](lang)
@@ -205,6 +222,12 @@ class Galleries(Task, ImageProcessor):
                     self.kw[k] = self.site.GLOBAL_CONTEXT[k](lang)
 
                 context = {}
+
+                # Do we have a metadata file?
+                meta_path, order, captions, img_metadata = self.find_metadata(gallery, lang)
+                context['meta_path'] = meta_path
+                context['order'] = order
+                context['captions'] = captions
                 context["lang"] = lang
                 if post:
                     context["title"] = post.title(lang)
@@ -214,11 +237,24 @@ class Galleries(Task, ImageProcessor):
 
                 image_name_list = [os.path.basename(p) for p in image_list]
 
-                if self.kw['use_filename_as_title']:
+                if captions:
+                    img_titles = []
+                    for fn in image_name_list:
+                        if fn in captions:
+                            img_titles.append(captions[fn])
+                        else:
+                            if self.kw['use_filename_as_title']:
+                                img_titles.append(fn)
+                            else:
+                                img_titles.append('')
+                            self.logger.debug(
+                                "Image {0} found in gallery but not listed in {1}".
+                                format(fn, context['meta_path']))
+                elif self.kw['use_filename_as_title']:
                     img_titles = []
                     for fn in image_name_list:
                         name_without_ext = os.path.splitext(os.path.basename(fn))[0]
-                        img_titles.append(utils.unslugify(name_without_ext))
+                        img_titles.append(utils.unslugify(name_without_ext, lang))
                 else:
                     img_titles = [''] * len(image_name_list)
 
@@ -230,6 +266,7 @@ class Galleries(Task, ImageProcessor):
                 folders = []
 
                 # Generate friendly gallery names
+                fpost_list = []
                 for path, folder in folder_list:
                     fpost = self.parse_index(path, input_folder, output_folder)
                     if fpost:
@@ -238,15 +275,25 @@ class Galleries(Task, ImageProcessor):
                         ft = folder
                     if not folder.endswith('/'):
                         folder += '/'
-                    folders.append((folder, ft))
 
+                    # TODO: This is to keep compatibility with user's custom gallery.tmpl
+                    # To be removed in v9 someday
+                    if self.kw['galleries_use_thumbnail']:
+                        folders.append((folder, ft, fpost))
+                        if fpost:
+                            fpost_list.append(fpost.source_path)
+                    else:
+                        folders.append((folder, ft))
+
+                context["gallery_path"] = gallery
                 context["folders"] = natsort.natsorted(
                     folders, alg=natsort.ns.F | natsort.ns.IC)
-                context["crumbs"] = crumbs
+                context["crumbs"] = utils.get_crumbs(gallery, index_folder=self, lang=lang)
                 context["permalink"] = self.site.link("gallery", gallery, lang)
                 context["enable_comments"] = self.kw['comments_in_galleries']
                 context["thumbnail_size"] = self.kw["thumbnail_size"]
                 context["pagekind"] = ["gallery_front"]
+                context["galleries_use_thumbnail"] = self.kw['galleries_use_thumbnail']
 
                 if post:
                     yield {
@@ -273,7 +320,7 @@ class Galleries(Task, ImageProcessor):
                 yield utils.apply_filters({
                     'basename': self.name,
                     'name': dst,
-                    'file_dep': file_dep,
+                    'file_dep': file_dep + dest_img_list + fpost_list,
                     'targets': [dst],
                     'actions': [
                         (self.render_gallery_index, (
@@ -283,7 +330,7 @@ class Galleries(Task, ImageProcessor):
                             dest_img_list,
                             img_titles,
                             thumbs,
-                            file_dep))],
+                            img_metadata))],
                     'clean': True,
                     'uptodate': [utils.config_changed({
                         1: self.kw.copy(),
@@ -325,7 +372,14 @@ class Galleries(Task, ImageProcessor):
         self.gallery_list = []
         for input_folder, output_folder in self.kw['gallery_folders'].items():
             for root, dirs, files in os.walk(input_folder, followlinks=True):
-                self.gallery_list.append((root, input_folder, output_folder))
+                # If output folder is empty, the top-level gallery
+                # index will collide with the main page for the site.
+                # Don't generate the top-level gallery index in that
+                # case.
+                # FIXME: also ignore pages named index
+                if (output_folder or root != input_folder and
+                        (not self.kw['disable_indexes'] and self.kw['index_path'] == '')):
+                    self.gallery_list.append((root, input_folder, output_folder))
 
     def create_galleries_paths(self):
         """Given a list of galleries, put their paths into self.gallery_links."""
@@ -377,12 +431,73 @@ class Galleries(Task, ImageProcessor):
                 'uptodate': [utils.config_changed(self.kw.copy(), 'nikola.plugins.task.galleries:mkdir')],
             }
 
+    def find_metadata(self, gallery, lang):
+        """Search for a gallery metadata file.
+
+        If there is an metadata file for the gallery, use that to determine
+        captions and the order in which images shall be displayed in the
+        gallery. You only need to list the images if a specific ordering or
+        caption is required. The metadata file is YAML-formatted, with field
+        names of
+        #
+        name:
+        caption:
+        order:
+        #
+        If a numeric order value is specified, we use that directly, otherwise
+        we depend on how the library returns the information - which may or may not
+        be in the same order as in the file itself. Non-numeric ordering is not
+        supported. If no caption is specified, then we return an empty string.
+        Returns a string (l18n'd filename), list (ordering), dict (captions),
+        dict (image metadata).
+        """
+        base_meta_path = os.path.join(gallery, "metadata.yml")
+        localized_meta_path = utils.get_translation_candidate(self.site.config,
+                                                              base_meta_path, lang)
+        order = []
+        captions = {}
+        custom_metadata = {}
+        used_path = ""
+
+        if os.path.isfile(localized_meta_path):
+            used_path = localized_meta_path
+        elif os.path.isfile(base_meta_path):
+            used_path = base_meta_path
+        else:
+            return "", [], {}, {}
+
+        self.logger.debug("Using {0} for gallery {1}".format(
+            used_path, gallery))
+        with open(used_path, "r", encoding='utf-8-sig') as meta_file:
+            if YAML is None:
+                utils.req_missing(['ruamel.yaml'], 'use metadata.yml files for galleries')
+            yaml = YAML(typ='safe')
+            meta = yaml.load_all(meta_file)
+            for img in meta:
+                # load_all and safe_load_all both return None as their
+                # final element, so skip it
+                if not img:
+                    continue
+                if 'name' in img:
+                    img_name = img.pop('name')
+                    if 'caption' in img and img['caption']:
+                        captions[img_name] = img.pop('caption')
+
+                    if 'order' in img and img['order'] is not None:
+                        order.insert(img.pop('order'), img_name)
+                    else:
+                        order.append(img_name)
+                    custom_metadata[img_name] = img
+                else:
+                    self.logger.error("no 'name:' for ({0}) in {1}".format(
+                        img, used_path))
+        return used_path, order, captions, custom_metadata
+
     def parse_index(self, gallery, input_folder, output_folder):
         """Return a Post object if there is an index.txt."""
         index_path = os.path.join(gallery, "index.txt")
-        destination = os.path.join(
-            self.kw["output_folder"], output_folder,
-            os.path.relpath(gallery, input_folder))
+        destination = os.path.join(output_folder,
+                                   os.path.relpath(gallery, input_folder))
         if os.path.isfile(index_path):
             post = Post(
                 index_path,
@@ -390,15 +505,20 @@ class Galleries(Task, ImageProcessor):
                 destination,
                 False,
                 self.site.MESSAGES,
-                'story.tmpl',
-                self.site.get_compiler(index_path)
+                'page.tmpl',
+                self.site.get_compiler(index_path),
+                None,
+                self.site.metadata_extractors_by
             )
             # If this did not exist, galleries without a title in the
             # index.txt file would be errorneously named `index`
             # (warning: galleries titled index and filenamed differently
             #  may break)
-            if post.title == 'index':
-                post.title = os.path.split(gallery)[1]
+            if post.title() == 'index':
+                for lang in post.meta.keys():
+                    post.meta[lang]['title'] = os.path.split(gallery)[1]
+            # Register the post (via #2417)
+            self.site.post_per_input_file[index_path] = post
         else:
             post = None
         return post
@@ -408,8 +528,8 @@ class Galleries(Task, ImageProcessor):
         exclude_path = os.path.join(gallery_path, "exclude.meta")
 
         try:
-            f = open(exclude_path, 'r')
-            excluded_image_name_list = f.read().split()
+            with open(exclude_path, 'r') as f:
+                excluded_image_name_list = f.read().split()
         except IOError:
             excluded_image_name_list = []
 
@@ -453,32 +573,26 @@ class Galleries(Task, ImageProcessor):
         orig_dest_path = os.path.join(output_gallery, img_name)
         yield utils.apply_filters({
             'basename': self.name,
-            'name': thumb_path,
-            'file_dep': [img],
-            'targets': [thumb_path],
-            'actions': [
-                (self.resize_image,
-                    (img, thumb_path, self.kw['thumbnail_size']))
-            ],
-            'clean': True,
-            'uptodate': [utils.config_changed({
-                1: self.kw['thumbnail_size']
-            }, 'nikola.plugins.task.galleries:resize_thumb')],
-        }, self.kw['filters'])
-
-        yield utils.apply_filters({
-            'basename': self.name,
             'name': orig_dest_path,
             'file_dep': [img],
-            'targets': [orig_dest_path],
+            'targets': [thumb_path, orig_dest_path],
             'actions': [
                 (self.resize_image,
-                    (img, orig_dest_path, self.kw['max_image_size']))
-            ],
+                    [img], {
+                        'dst_paths': [thumb_path, orig_dest_path],
+                        'max_sizes': [self.kw['thumbnail_size'], self.kw['max_image_size']],
+                        'bigger_panoramas': True,
+                        'preserve_exif_data': self.kw['preserve_exif_data'],
+                        'exif_whitelist': self.kw['exif_whitelist'],
+                        'preserve_icc_profiles': self.kw['preserve_icc_profiles']})],
             'clean': True,
             'uptodate': [utils.config_changed({
-                1: self.kw['max_image_size']
-            }, 'nikola.plugins.task.galleries:resize_max')],
+                1: self.kw['thumbnail_size'],
+                2: self.kw['max_image_size'],
+                3: self.kw['preserve_exif_data'],
+                4: self.kw['exif_whitelist'],
+                5: self.kw['preserve_icc_profiles'],
+            }, 'nikola.plugins.task.galleries:resize_thumb')],
         }, self.kw['filters'])
 
     def remove_excluded_image(self, img, input_folder):
@@ -524,7 +638,7 @@ class Galleries(Task, ImageProcessor):
             img_list,
             img_titles,
             thumbs,
-            file_dep):
+            img_metadata):
         """Build the gallery index."""
         # The photo array needs to be created here, because
         # it relies on thumbnails already being created on
@@ -534,15 +648,33 @@ class Galleries(Task, ImageProcessor):
             url = '/'.join(os.path.relpath(p, os.path.dirname(output_name) + os.sep).split(os.sep))
             return url
 
-        photo_array = []
+        all_data = list(zip(img_list, thumbs, img_titles))
+
+        if self.kw['sort_by_date']:
+            all_data.sort(key=lambda a: self.image_date(a[0]))
+        else:  # Sort by name
+            all_data.sort(key=lambda a: a[0])
+
+        if all_data:
+            img_list, thumbs, img_titles = zip(*all_data)
+        else:
+            img_list, thumbs, img_titles = [], [], []
+
+        photo_info = OrderedDict()
         for img, thumb, title in zip(img_list, thumbs, img_titles):
             w, h = _image_size_cache.get(thumb, (None, None))
             if w is None:
-                im = Image.open(thumb)
-                w, h = im.size
-                _image_size_cache[thumb] = w, h
-            # Thumbs are files in output, we need URLs
-            photo_array.append({
+                if os.path.splitext(thumb)[1] in ['.svg', '.svgz']:
+                    w, h = 200, 200
+                else:
+                    im = Image.open(thumb)
+                    w, h = im.size
+                    _image_size_cache[thumb] = w, h
+                    im.close()
+            # Use basename to avoid issues with multilingual sites (Issue #3078)
+            img_basename = os.path.basename(img)
+            photo_info[img_basename] = {
+                # Thumbs are files in output, we need URLs
                 'url': url_from_path(img),
                 'url_thumb': url_from_path(thumb),
                 'title': title,
@@ -550,9 +682,27 @@ class Galleries(Task, ImageProcessor):
                     'w': w,
                     'h': h
                 },
-            })
+                'width': w,
+                'height': h
+            }
+            if img_basename in img_metadata:
+                photo_info[img_basename].update(img_metadata[img_basename])
+        photo_array = []
+        if context['order']:
+            for entry in context['order']:
+                photo_array.append(photo_info.pop(entry))
+            # Do we have any orphan entries from metadata.yml, or
+            # are the files from the gallery not listed in metadata.yml?
+            if photo_info:
+                for entry in photo_info:
+                    photo_array.append(photo_info[entry])
+        else:
+            for entry in photo_info:
+                photo_array.append(photo_info[entry])
+
         context['photo_array'] = photo_array
         context['photo_array_json'] = json.dumps(photo_array, sort_keys=True)
+
         self.site.render_template(template_name, output_name, context)
 
     def gallery_rss(self, img_list, dest_img_list, img_titles, lang, permalink, output_path, title):
@@ -564,6 +714,18 @@ class Galleries(Task, ImageProcessor):
         def make_url(url):
             return urljoin(self.site.config['BASE_URL'], url.lstrip('/'))
 
+        all_data = list(zip(img_list, dest_img_list, img_titles))
+
+        if self.kw['sort_by_date']:
+            all_data.sort(key=lambda a: self.image_date(a[0]))
+        else:  # Sort by name
+            all_data.sort(key=lambda a: a[0])
+
+        if all_data:
+            img_list, dest_img_list, img_titles = zip(*all_data)
+        else:
+            img_list, dest_img_list, img_titles = [], [], []
+
         items = []
         for img, srcimg, title in list(zip(dest_img_list, img_list, img_titles))[:self.kw["feed_length"]]:
             img_size = os.stat(
@@ -587,7 +749,7 @@ class Galleries(Task, ImageProcessor):
             description='',
             lastBuildDate=datetime.datetime.utcnow(),
             items=items,
-            generator='http://getnikola.com/',
+            generator='https://getnikola.com/',
             language=lang
         )
 
@@ -598,6 +760,6 @@ class Galleries(Task, ImageProcessor):
         utils.makedirs(dst_dir)
         with io.open(output_path, "w+", encoding="utf-8") as rss_file:
             data = rss_obj.to_xml(encoding='utf-8')
-            if isinstance(data, utils.bytes_str):
+            if isinstance(data, bytes):
                 data = data.decode('utf-8')
             rss_file.write(data)