New upstream version 1.25.0.upstream/1.25.0

3 files changed, 73 insertions, 32 deletions

diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py
index ef211e6..c28d060 100644
--- a/gallery_dl/postprocessor/common.py
+++ b/gallery_dl/postprocessor/common.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2020 Mike Fährmann
+# Copyright 2018-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -8,13 +8,42 @@
 
 """Common classes and constants used by postprocessor modules."""
 
+from .. import util, formatter
+
 
 class PostProcessor():
     """Base class for postprocessors"""
 
     def __init__(self, job):
-        name = self.__class__.__name__[:-2].lower()
-        self.log = job.get_logger("postprocessor." + name)
+        self.name = self.__class__.__name__[:-2].lower()
+        self.log = job.get_logger("postprocessor." + self.name)
 
     def __repr__(self):
         return self.__class__.__name__
+
+    def _init_archive(self, job, options, prefix=None):
+        archive = options.get("archive")
+        if archive:
+            extr = job.extractor
+            archive = util.expand_path(archive)
+            if not prefix:
+                prefix = "_" + self.name.upper() + "_"
+            archive_format = (
+                options.get("archive-prefix", extr.category) +
+                options.get("archive-format", prefix + extr.archive_fmt))
+            try:
+                if "{" in archive:
+                    archive = formatter.parse(archive).format_map(
+                        job.pathfmt.kwdict)
+                self.archive = util.DownloadArchive(
+                    archive, archive_format,
+                    options.get("archive-pragma"),
+                    "_archive_" + self.name)
+            except Exception as exc:
+                self.log.warning(
+                    "Failed to open %s archive at '%s' ('%s: %s')",
+                    self.name, archive, exc.__class__.__name__, exc)
+            else:
+                self.log.debug("Using %s archive '%s'", self.name, archive)
+        else:
+            self.archive = None
diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py
index cc217c3..e81c6cf 100644
--- a/gallery_dl/postprocessor/exec.py
+++ b/gallery_dl/postprocessor/exec.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2021 Mike Fährmann
+# Copyright 2018-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -43,11 +43,18 @@ class ExecPP(PostProcessor):
             events = events.split(",")
         job.register_hooks({event: execute for event in events}, options)
 
+        self._init_archive(job, options)
+
     def exec_list(self, pathfmt, status=None):
         if status:
             return
 
+        archive = self.archive
         kwdict = pathfmt.kwdict
+
+        if archive and archive.check(kwdict):
+            return
+
         kwdict["_directory"] = pathfmt.realdirectory
         kwdict["_filename"] = pathfmt.filename
         kwdict["_path"] = pathfmt.realpath
@@ -55,10 +62,17 @@ class ExecPP(PostProcessor):
         args = [arg.format_map(kwdict) for arg in self.args]
         self._exec(args, False)
 
+        if archive:
+            archive.add(kwdict)
+
     def exec_string(self, pathfmt, status=None):
         if status:
             return
 
+        archive = self.archive
+        if archive and archive.check(pathfmt.kwdict):
+            return
+
         if status is None and pathfmt.realpath:
             args = self.args.replace("{}", quote(pathfmt.realpath))
         else:
@@ -66,6 +80,9 @@ class ExecPP(PostProcessor):
 
         self._exec(args, True)
 
+        if archive:
+            archive.add(pathfmt.kwdict)
+
     def _exec(self, args, shell):
         self.log.debug("Running '%s'", args)
         retcode = subprocess.Popen(args, shell=shell).wait()
diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py
index 2ee1cf8..9667a41 100644
--- a/gallery_dl/postprocessor/metadata.py
+++ b/gallery_dl/postprocessor/metadata.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2019-2022 Mike Fährmann
+# Copyright 2019-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -10,6 +10,7 @@
 
 from .common import PostProcessor
 from .. import util, formatter
+import json
 import sys
 import os
 
@@ -46,14 +47,12 @@ class MetadataPP(PostProcessor):
             ext = "txt"
         elif mode == "jsonl":
             self.write = self._write_json
-            self.indent = None
-            self.ascii = options.get("ascii", False)
+            self._json_encode = self._make_encoder(options).encode
             omode = "a"
             filename = "data.jsonl"
         else:
             self.write = self._write_json
-            self.indent = options.get("indent", 4)
-            self.ascii = options.get("ascii", False)
+            self._json_encode = self._make_encoder(options, 4).encode
             ext = "json"
 
         directory = options.get("directory")
@@ -83,28 +82,7 @@ class MetadataPP(PostProcessor):
             events = events.split(",")
         job.register_hooks({event: self.run for event in events}, options)
 
-        archive = options.get("archive")
-        if archive:
-            extr = job.extractor
-            archive = util.expand_path(archive)
-            archive_format = (
-                options.get("archive-prefix", extr.category) +
-                options.get("archive-format", "_MD_" + extr.archive_fmt))
-            try:
-                if "{" in archive:
-                    archive = formatter.parse(archive).format_map(
-                        job.pathfmt.kwdict)
-                self.archive = util.DownloadArchive(
-                    archive, archive_format, "_archive_metadata")
-            except Exception as exc:
-                self.log.warning(
-                    "Failed to open download archive at '%s' ('%s: %s')",
-                    archive, exc.__class__.__name__, exc)
-            else:
-                self.log.debug("Using download archive '%s'", archive)
-        else:
-            self.archive = None
-
+        self._init_archive(job, options, "_MD_")
         self.mtime = options.get("mtime")
         self.omode = options.get("open", omode)
         self.encoding = options.get("encoding", "utf-8")
@@ -206,13 +184,30 @@ class MetadataPP(PostProcessor):
             for taglist in taglists:
                 extend(taglist)
             tags.sort()
+        elif all(isinstance(e, dict) for e in tags):
+            taglists = tags
+            tags = []
+            extend = tags.extend
+            for tagdict in taglists:
+                extend([x for x in tagdict.values() if x is not None])
+            tags.sort()
 
         fp.write("\n".join(tags) + "\n")
 
     def _write_json(self, fp, kwdict):
         if not self.private:
             kwdict = util.filter_dict(kwdict)
-        util.dump_json(kwdict, fp, self.ascii, self.indent)
+        fp.write(self._json_encode(kwdict) + "\n")
+
+    @staticmethod
+    def _make_encoder(options, indent=None):
+        return json.JSONEncoder(
+            ensure_ascii=options.get("ascii", False),
+            sort_keys=options.get("sort", False),
+            separators=options.get("separators"),
+            indent=options.get("indent", indent),
+            check_circular=False, default=str,
+        )
 
 
 __postprocessor__ = MetadataPP