New upstream version 1.21.1.upstream/1.21.1

3 files changed, 183 insertions, 40 deletions

diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py
index e776888..5e8f3e9 100644
--- a/gallery_dl/postprocessor/metadata.py
+++ b/gallery_dl/postprocessor/metadata.py
@@ -59,9 +59,35 @@ class MetadataPP(PostProcessor):
             events = events.split(",")
         job.register_hooks({event: self.run for event in events}, options)
 
+        archive = options.get("archive")
+        if archive:
+            extr = job.extractor
+            archive = util.expand_path(archive)
+            archive_format = (
+                options.get("archive-prefix", extr.category) +
+                options.get("archive-format", "_MD_" + extr.archive_fmt))
+            try:
+                if "{" in archive:
+                    archive = formatter.parse(archive).format_map(
+                        job.pathfmt.kwdict)
+                self.archive = util.DownloadArchive(
+                    archive, archive_format, "_archive_metadata")
+            except Exception as exc:
+                self.log.warning(
+                    "Failed to open download archive at '%s' ('%s: %s')",
+                    archive, exc.__class__.__name__, exc)
+            else:
+                self.log.debug("Using download archive '%s'", archive)
+        else:
+            self.archive = None
+
         self.mtime = options.get("mtime")
 
     def run(self, pathfmt):
+        archive = self.archive
+        if archive and archive.check(pathfmt.kwdict):
+            return
+
         directory = self._directory(pathfmt)
         path = directory + self._filename(pathfmt)
 
@@ -73,6 +99,9 @@ class MetadataPP(PostProcessor):
             with open(path, "w", encoding="utf-8") as fp:
                 self.write(fp, pathfmt.kwdict)
 
+        if archive:
+            archive.add(pathfmt.kwdict)
+
         if self.mtime:
             mtime = pathfmt.kwdict.get("_mtime")
             if mtime:
diff --git a/gallery_dl/postprocessor/mtime.py b/gallery_dl/postprocessor/mtime.py
index 098984a..3f8d90a 100644
--- a/gallery_dl/postprocessor/mtime.py
+++ b/gallery_dl/postprocessor/mtime.py
@@ -9,7 +9,8 @@
 """Use metadata as file modification time"""
 
 from .common import PostProcessor
-from ..text import parse_int
+from .. import text, util
+from datetime import datetime
 
 
 class MtimePP(PostProcessor):
@@ -27,8 +28,11 @@ class MtimePP(PostProcessor):
 
     def run(self, pathfmt):
         mtime = pathfmt.kwdict.get(self.key)
-        ts = getattr(mtime, "timestamp", None)
-        pathfmt.kwdict["_mtime"] = ts() if ts else parse_int(mtime)
+        pathfmt.kwdict["_mtime"] = (
+            util.datetime_to_timestamp(mtime)
+            if isinstance(mtime, datetime) else
+            text.parse_int(mtime)
+        )
 
 
 __postprocessor__ = MtimePP
diff --git a/gallery_dl/postprocessor/ugoira.py b/gallery_dl/postprocessor/ugoira.py
index e5bdebc..c5477d2 100644
--- a/gallery_dl/postprocessor/ugoira.py
+++ b/gallery_dl/postprocessor/ugoira.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2021 Mike Fährmann
+# Copyright 2018-2022 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -10,12 +10,20 @@
 
 from .common import PostProcessor
 from .. import util
-import collections
 import subprocess
 import tempfile
 import zipfile
+import shutil
 import os
 
+try:
+    from math import gcd
+except ImportError:
+    def gcd(a, b):
+        while b:
+            a, b = b, a % b
+        return a
+
 
 class UgoiraPP(PostProcessor):
 
@@ -27,19 +35,37 @@ class UgoiraPP(PostProcessor):
         self.output = options.get("ffmpeg-output", True)
         self.delete = not options.get("keep-files", False)
         self.repeat = options.get("repeat-last-frame", True)
+        self.mtime = options.get("mtime")
 
         ffmpeg = options.get("ffmpeg-location")
         self.ffmpeg = util.expand_path(ffmpeg) if ffmpeg else "ffmpeg"
 
+        mkvmerge = options.get("mkvmerge-location")
+        self.mkvmerge = util.expand_path(mkvmerge) if mkvmerge else "mkvmerge"
+
+        demuxer = options.get("ffmpeg-demuxer")
+        if demuxer is None or demuxer == "auto":
+            if self.extension in ("webm", "mkv") and (
+                    mkvmerge or shutil.which("mkvmerge")):
+                demuxer = "mkvmerge"
+            else:
+                demuxer = "concat" if util.WINDOWS else "image2"
+
+        if demuxer == "mkvmerge":
+            self._process = self._process_mkvmerge
+            self._finalize = self._finalize_mkvmerge
+        elif demuxer == "image2":
+            self._process = self._process_image2
+            self._finalize = None
+        else:
+            self._process = self._process_concat
+            self._finalize = None
+        self.log.debug("using %s demuxer", demuxer)
+
         rate = options.get("framerate", "auto")
         if rate != "auto":
             self.calculate_framerate = lambda _: (None, rate)
 
-        if options.get("ffmpeg-demuxer") == "image2":
-            self._process = self._image2
-        else:
-            self._process = self._concat
-
         if options.get("libx264-prevent-odd", True):
             # get last video-codec argument
             vcodec = None
@@ -88,13 +114,12 @@ class UgoiraPP(PostProcessor):
                 return
 
             # process frames and collect command-line arguments
-            args = self._process(tempdir)
+            pathfmt.set_extension(self.extension)
+            args = self._process(pathfmt, tempdir)
             if self.args:
                 args += self.args
-            self.log.debug("ffmpeg args: %s", args)
 
             # invoke ffmpeg
-            pathfmt.set_extension(self.extension)
             try:
                 if self.twopass:
                     if "-f" not in self.args:
@@ -105,48 +130,61 @@ class UgoiraPP(PostProcessor):
                 else:
                     args.append(pathfmt.realpath)
                     self._exec(args)
+                if self._finalize:
+                    self._finalize(pathfmt, tempdir)
             except OSError as exc:
                 print()
                 self.log.error("Unable to invoke FFmpeg (%s: %s)",
                                exc.__class__.__name__, exc)
                 pathfmt.realpath = pathfmt.temppath
             else:
+                if self.mtime:
+                    mtime = pathfmt.kwdict.get("_mtime")
+                    if mtime:
+                        util.set_mtime(pathfmt.realpath, mtime)
                 if self.delete:
                     pathfmt.delete = True
                 else:
                     pathfmt.set_extension("zip")
 
-    def _concat(self, path):
-        ffconcat = path + "/ffconcat.txt"
-
-        content = ["ffconcat version 1.0"]
-        append = content.append
-        for frame in self._frames:
-            append("file '{}'\nduration {}".format(
-                frame["file"], frame["delay"] / 1000))
-        if self.repeat:
-            append("file '{}'".format(frame["file"]))
-        append("")
-
-        with open(ffconcat, "w") as file:
-            file.write("\n".join(content))
+    def _exec(self, args):
+        self.log.debug(args)
+        out = None if self.output else subprocess.DEVNULL
+        return subprocess.Popen(args, stdout=out, stderr=out).wait()
 
+    def _process_concat(self, pathfmt, tempdir):
         rate_in, rate_out = self.calculate_framerate(self._frames)
         args = [self.ffmpeg, "-f", "concat"]
         if rate_in:
             args += ("-r", str(rate_in))
-        args += ("-i", ffconcat)
+        args += ("-i", self._write_ffmpeg_concat(tempdir))
         if rate_out:
             args += ("-r", str(rate_out))
         return args
 
-    def _image2(self, path):
-        path += "/"
+    def _process_image2(self, pathfmt, tempdir):
+        tempdir += "/"
+        frames = self._frames
+
+        # add extra frame if necessary
+        if self.repeat and not self._delay_is_uniform(frames):
+            last = frames[-1]
+            delay_gcd = self._delay_gcd(frames)
+            if last["delay"] - delay_gcd > 0:
+                last["delay"] -= delay_gcd
+
+                self.log.debug("non-uniform delays; inserting extra frame")
+                last_copy = last.copy()
+                frames.append(last_copy)
+                name, _, ext = last_copy["file"].rpartition(".")
+                last_copy["file"] = "{:>06}.{}".format(int(name)+1, ext)
+                shutil.copyfile(tempdir + last["file"],
+                                tempdir + last_copy["file"])
 
         # adjust frame mtime values
         ts = 0
-        for frame in self._frames:
-            os.utime(path + frame["file"], ns=(ts, ts))
+        for frame in frames:
+            os.utime(tempdir + frame["file"], ns=(ts, ts))
             ts += frame["delay"] * 1000000
 
         return [
@@ -155,18 +193,90 @@ class UgoiraPP(PostProcessor):
             "-ts_from_file", "2",
             "-pattern_type", "sequence",
             "-i", "{}%06d.{}".format(
-                path.replace("%", "%%"), frame["file"].rpartition(".")[2]),
+                tempdir.replace("%", "%%"),
+                frame["file"].rpartition(".")[2]
+            ),
         ]
 
-    def _exec(self, args):
-        out = None if self.output else subprocess.DEVNULL
-        return subprocess.Popen(args, stdout=out, stderr=out).wait()
+    def _process_mkvmerge(self, pathfmt, tempdir):
+        self._realpath = pathfmt.realpath
+        pathfmt.realpath = tempdir + "/temp." + self.extension
+
+        return [
+            self.ffmpeg,
+            "-f", "image2",
+            "-pattern_type", "sequence",
+            "-i", "{}/%06d.{}".format(
+                tempdir.replace("%", "%%"),
+                self._frames[0]["file"].rpartition(".")[2]
+            ),
+        ]
+
+    def _finalize_mkvmerge(self, pathfmt, tempdir):
+        args = [
+            self.mkvmerge,
+            "-o", self._realpath,
+            "--timecodes", "0:" + self._write_mkvmerge_timecodes(tempdir),
+        ]
+        if self.extension == "webm":
+            args.append("--webm")
+        args += ("=", pathfmt.realpath)
+
+        pathfmt.realpath = self._realpath
+        self._exec(args)
+
+    def _write_ffmpeg_concat(self, tempdir):
+        content = ["ffconcat version 1.0"]
+        append = content.append
+
+        for frame in self._frames:
+            append("file '{}'\nduration {}".format(
+                frame["file"], frame["delay"] / 1000))
+        if self.repeat:
+            append("file '{}'".format(frame["file"]))
+        append("")
+
+        ffconcat = tempdir + "/ffconcat.txt"
+        with open(ffconcat, "w") as file:
+            file.write("\n".join(content))
+        return ffconcat
+
+    def _write_mkvmerge_timecodes(self, tempdir):
+        content = ["# timecode format v2"]
+        append = content.append
+
+        delay_sum = 0
+        for frame in self._frames:
+            append(str(delay_sum))
+            delay_sum += frame["delay"]
+        append(str(delay_sum))
+        append("")
+
+        timecodes = tempdir + "/timecodes.tc"
+        with open(timecodes, "w") as file:
+            file.write("\n".join(content))
+        return timecodes
+
+    def calculate_framerate(self, frames):
+        uniform = self._delay_is_uniform(frames)
+        if uniform:
+            return ("1000/{}".format(frames[0]["delay"]), None)
+        return (None, "1000/{}".format(self._delay_gcd(frames)))
+
+    @staticmethod
+    def _delay_gcd(frames):
+        result = frames[0]["delay"]
+        for f in frames:
+            result = gcd(result, f["delay"])
+        return result
 
     @staticmethod
-    def calculate_framerate(framelist):
-        counter = collections.Counter(frame["delay"] for frame in framelist)
-        fps = "1000/{}".format(min(counter))
-        return (fps, None) if len(counter) == 1 else (None, fps)
+    def _delay_is_uniform(frames):
+        delay = frames[0]["delay"]
+        for f in frames:
+            if f["delay"] != delay:
+                return False
+        return True
 
 
 __postprocessor__ = UgoiraPP