Import Upstream version 1.8.7upstream/1.8.7

7 files changed, 423 insertions, 0 deletions

diff --git a/gallery_dl/postprocessor/__init__.py b/gallery_dl/postprocessor/__init__.py
new file mode 100644
index 0000000..093f8e0
--- /dev/null
+++ b/gallery_dl/postprocessor/__init__.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018-2019 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Post-processing modules"""
+
+import importlib
+import logging
+
+modules = [
+    "classify",
+    "exec",
+    "metadata",
+    "ugoira",
+    "zip",
+]
+
+log = logging.getLogger("postprocessor")
+
+
+def find(name):
+    """Return a postprocessor class with the given name"""
+    try:
+        return _cache[name]
+    except KeyError:
+        klass = None
+        try:
+            if name in modules:  # prevent unwanted imports
+                module = importlib.import_module("." + name, __package__)
+                klass = module.__postprocessor__
+        except (ImportError, AttributeError, TypeError):
+            pass
+        _cache[name] = klass
+        return klass
+
+
+# --------------------------------------------------------------------
+# internals
+
+_cache = {}
diff --git a/gallery_dl/postprocessor/classify.py b/gallery_dl/postprocessor/classify.py
new file mode 100644
index 0000000..62460d3
--- /dev/null
+++ b/gallery_dl/postprocessor/classify.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Categorize files by file extension"""
+
+from .common import PostProcessor
+import os
+
+
+class ClassifyPP(PostProcessor):
+
+    DEFAULT_MAPPING = {
+        "Music" : ("mp3", "aac", "flac", "ogg", "wma", "m4a", "wav"),
+        "Video" : ("flv", "ogv", "avi", "mp4", "mpg", "mpeg", "3gp", "mkv",
+                   "webm", "vob", "wmv"),
+        "Pictures" : ("jpg", "jpeg", "png", "gif", "bmp", "svg", "webp"),
+        "Archives" : ("zip", "rar", "7z", "tar", "gz", "bz2"),
+    }
+
+    def __init__(self, pathfmt, options):
+        PostProcessor.__init__(self)
+        mapping = options.get("mapping", self.DEFAULT_MAPPING)
+
+        self.mapping = {
+            ext: directory
+            for directory, exts in mapping.items()
+            for ext in exts
+        }
+
+    def prepare(self, pathfmt):
+        ext = pathfmt.keywords.get("extension")
+
+        if ext in self.mapping:
+            self._dir = pathfmt.realdirectory + os.sep + self.mapping[ext]
+            pathfmt.realpath = self._dir + os.sep + pathfmt.filename
+        else:
+            self._dir = None
+
+    def run(self, pathfmt):
+        if self._dir:
+            os.makedirs(self._dir, exist_ok=True)
+
+
+__postprocessor__ = ClassifyPP
diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py
new file mode 100644
index 0000000..c642f0f
--- /dev/null
+++ b/gallery_dl/postprocessor/common.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Common classes and constants used by postprocessor modules."""
+
+from . import log
+
+
+class PostProcessor():
+    """Base class for postprocessors"""
+    log = log
+
+    def prepare(self, pathfmt):
+        """ """
+
+    def run(self, pathfmt):
+        """Execute the postprocessor for a file"""
+
+    def finalize(self):
+        """Cleanup"""
diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py
new file mode 100644
index 0000000..c86b480
--- /dev/null
+++ b/gallery_dl/postprocessor/exec.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Execute processes"""
+
+from .common import PostProcessor
+import subprocess
+
+
+class ExecPP(PostProcessor):
+
+    def __init__(self, pathfmt, options):
+        PostProcessor.__init__(self)
+
+        try:
+            self.args = options["command"]
+            self.args[0]  # test if 'args' is subscriptable
+        except (KeyError, IndexError, TypeError):
+            raise TypeError("option 'command' must be a non-empty list")
+
+        if options.get("async", False):
+            self._exec = subprocess.Popen
+
+    def run(self, pathfmt):
+        self._exec([
+            arg.format_map(pathfmt.keywords)
+            for arg in self.args
+        ])
+
+    def _exec(self, args):
+        retcode = subprocess.Popen(args).wait()
+        if retcode:
+            self.log.warning(
+                "executing '%s' returned non-zero exit status %d",
+                " ".join(args), retcode)
+
+
+__postprocessor__ = ExecPP
diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py
new file mode 100644
index 0000000..77be9c7
--- /dev/null
+++ b/gallery_dl/postprocessor/metadata.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Write metadata to JSON files"""
+
+from .common import PostProcessor
+from .. import util
+
+
+class MetadataPP(PostProcessor):
+
+    def __init__(self, pathfmt, options):
+        PostProcessor.__init__(self)
+
+        mode = options.get("mode", "json")
+        ext = "txt"
+
+        if mode == "custom":
+            self.write = self._write_custom
+            self.formatter = util.Formatter(options.get("format"))
+        elif mode == "tags":
+            self.write = self._write_tags
+        else:
+            self.write = self._write_json
+            self.indent = options.get("indent", 4)
+            self.ascii = options.get("ascii", False)
+            ext = "json"
+
+        self.extension = options.get("extension", ext)
+
+    def run(self, pathfmt):
+        path = "{}.{}".format(pathfmt.realpath, self.extension)
+        with open(path, "w", encoding="utf-8") as file:
+            self.write(file, pathfmt)
+
+    def _write_custom(self, file, pathfmt):
+        output = self.formatter.format_map(pathfmt.keywords)
+        file.write(output)
+
+    def _write_tags(self, file, pathfmt):
+        kwds = pathfmt.keywords
+        tags = kwds.get("tags") or kwds.get("tag_string")
+
+        if not tags:
+            return
+
+        if not isinstance(tags, list):
+            taglist = tags.split(", ")
+            if len(taglist) < len(tags) / 16:
+                taglist = tags.split(" ")
+            tags = taglist
+
+        file.write("\n".join(tags))
+        file.write("\n")
+
+    def _write_json(self, file, pathfmt):
+        util.dump_json(pathfmt.keywords, file, self.ascii, self.indent)
+
+
+__postprocessor__ = MetadataPP
diff --git a/gallery_dl/postprocessor/ugoira.py b/gallery_dl/postprocessor/ugoira.py
new file mode 100644
index 0000000..bd8c5ad
--- /dev/null
+++ b/gallery_dl/postprocessor/ugoira.py
@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Convert pixiv ugoira to webm"""
+
+from .common import PostProcessor
+from .. import util
+import collections
+import subprocess
+import tempfile
+import zipfile
+import os
+
+
+class UgoiraPP(PostProcessor):
+
+    def __init__(self, pathfmt, options):
+        PostProcessor.__init__(self)
+        self.extension = options.get("extension") or "webm"
+        self.args = options.get("ffmpeg-args") or ()
+        self.twopass = options.get("ffmpeg-twopass", False)
+        self.output = options.get("ffmpeg-output", True)
+        self.delete = not options.get("keep-files", False)
+
+        ffmpeg = options.get("ffmpeg-location")
+        self.ffmpeg = util.expand_path(ffmpeg) if ffmpeg else "ffmpeg"
+
+        rate = options.get("framerate", "auto")
+        if rate != "auto":
+            self.calculate_framerate = lambda _: (None, rate)
+
+        if options.get("libx264-prevent-odd", True):
+            # get last video-codec argument
+            vcodec = None
+            for index, arg in enumerate(self.args):
+                arg, _, stream = arg.partition(":")
+                if arg == "-vcodec" or arg in ("-c", "-codec") and (
+                        not stream or stream.partition(":")[0] in ("v", "V")):
+                    vcodec = self.args[index + 1]
+            # use filter if libx264/5 is explicitly or implicitly used
+            self.prevent_odd = (
+                vcodec in ("libx264", "libx265") or
+                not vcodec and self.extension.lower() in ("mp4", "mkv"))
+        else:
+            self.prevent_odd = False
+
+    def prepare(self, pathfmt):
+        self._frames = None
+
+        if pathfmt.keywords["extension"] != "zip":
+            return
+
+        if "frames" in pathfmt.keywords:
+            self._frames = pathfmt.keywords["frames"]
+        elif "pixiv_ugoira_frame_data" in pathfmt.keywords:
+            self._frames = pathfmt.keywords["pixiv_ugoira_frame_data"]["data"]
+        else:
+            return
+
+        if self.delete:
+            pathfmt.set_extension(self.extension)
+
+    def run(self, pathfmt):
+        if not self._frames:
+            return
+
+        rate_in, rate_out = self.calculate_framerate(self._frames)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            # extract frames
+            with zipfile.ZipFile(pathfmt.temppath) as zfile:
+                zfile.extractall(tempdir)
+
+            # write ffconcat file
+            ffconcat = tempdir + "/ffconcat.txt"
+            with open(ffconcat, "w") as file:
+                file.write("ffconcat version 1.0\n")
+                for frame in self._frames:
+                    file.write("file '{}'\n".format(frame["file"]))
+                    file.write("duration {}\n".format(frame["delay"] / 1000))
+                if self.extension != "gif":
+                    # repeat the last frame to prevent it from only being
+                    # displayed for a very short amount of time
+                    file.write("file '{}'\n".format(self._frames[-1]["file"]))
+
+            # collect command-line arguments
+            args = [self.ffmpeg]
+            if rate_in:
+                args += ["-r", str(rate_in)]
+            args += ["-i", ffconcat]
+            if rate_out:
+                args += ["-r", str(rate_out)]
+            if self.prevent_odd:
+                args += ["-vf", "crop=iw-mod(iw\\,2):ih-mod(ih\\,2)"]
+            if self.args:
+                args += self.args
+            self.log.debug("ffmpeg args: %s", args)
+
+            # invoke ffmpeg
+            pathfmt.set_extension(self.extension)
+            if self.twopass:
+                if "-f" not in args:
+                    args += ["-f", self.extension]
+                args += ["-passlogfile", tempdir + "/ffmpeg2pass", "-pass"]
+                self._exec(args + ["1", "-y", os.devnull])
+                self._exec(args + ["2", pathfmt.realpath])
+            else:
+                args.append(pathfmt.realpath)
+                self._exec(args)
+
+        if self.delete:
+            pathfmt.delete = True
+        else:
+            pathfmt.set_extension("zip")
+
+    def _exec(self, args):
+        out = None if self.output else subprocess.DEVNULL
+        return subprocess.Popen(args, stdout=out, stderr=out).wait()
+
+    @staticmethod
+    def calculate_framerate(framelist):
+        counter = collections.Counter(frame["delay"] for frame in framelist)
+        fps = "1000/{}".format(min(counter))
+        return (fps, None) if len(counter) == 1 else (None, fps)
+
+
+__postprocessor__ = UgoiraPP
diff --git a/gallery_dl/postprocessor/zip.py b/gallery_dl/postprocessor/zip.py
new file mode 100644
index 0000000..3a0c323
--- /dev/null
+++ b/gallery_dl/postprocessor/zip.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Store files in ZIP archives"""
+
+from .common import PostProcessor
+import zipfile
+import os
+
+
+class ZipPP(PostProcessor):
+
+    COMPRESSION_ALGORITHMS = {
+        "store": zipfile.ZIP_STORED,
+        "zip": zipfile.ZIP_DEFLATED,
+        "bzip2": zipfile.ZIP_BZIP2,
+        "lzma": zipfile.ZIP_LZMA,
+    }
+
+    def __init__(self, pathfmt, options):
+        PostProcessor.__init__(self)
+        self.delete = not options.get("keep-files", False)
+        self.ext = "." + options.get("extension", "zip")
+        algorithm = options.get("compression", "store")
+        if algorithm not in self.COMPRESSION_ALGORITHMS:
+            self.log.warning(
+                "unknown compression algorithm '%s'; falling back to 'store'",
+                algorithm)
+            algorithm = "store"
+
+        self.path = pathfmt.realdirectory
+        self.zfile = zipfile.ZipFile(
+            self.path + self.ext, "a",
+            self.COMPRESSION_ALGORITHMS[algorithm], True)
+
+    def run(self, pathfmt):
+        # 'NameToInfo' is not officially documented, but it's available
+        # for all supported Python versions and using it directly is a lot
+        # better than calling getinfo()
+        if pathfmt.filename not in self.zfile.NameToInfo:
+            self.zfile.write(pathfmt.temppath, pathfmt.filename)
+            pathfmt.delete = self.delete
+
+    def finalize(self):
+        self.zfile.close()
+
+        if self.delete:
+            try:
+                os.rmdir(self.path)
+            except OSError:
+                pass
+
+            if not self.zfile.NameToInfo:
+                try:
+                    os.unlink(self.zfile.filename)
+                except OSError:
+                    pass
+
+
+__postprocessor__ = ZipPP