New upstream version 1.12.1upstream/1.12.1

1 files changed, 41 insertions, 16 deletions

diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 9c76336..88b6a55 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -8,6 +8,7 @@
 
 import sys
 import time
+import errno
 import logging
 from . import extractor, downloader, postprocessor
 from . import config, text, util, output, exception
@@ -97,6 +98,10 @@ class Job():
                 self.update_kwdict(kwds)
                 self.handle_urllist(urls, kwds)
 
+        elif msg[0] == Message.Metadata:
+            self.update_kwdict(msg[1])
+            self.handle_metadata(msg[1])
+
         elif msg[0] == Message.Version:
             if msg[1] != 1:
                 raise "unsupported message-version ({}, {})".format(
@@ -114,6 +119,9 @@ class Job():
     def handle_directory(self, kwdict):
         """Handle Message.Directory"""
 
+    def handle_metadata(self, kwdict):
+        """Handle Message.Metadata"""
+
     def handle_queue(self, url, kwdict):
         """Handle Message.Queue"""
 
@@ -161,11 +169,6 @@ class Job():
         if self.ulog:
             self.ulog.info(url)
 
-    @staticmethod
-    def _filter(kwdict):
-        """Return a copy of 'kwdict' without "private" entries"""
-        return {k: v for k, v in kwdict.items() if k[0] != "_"}
-
 
 class DownloadJob(Job):
     """Download images into appropriate directory/filename locations"""
@@ -247,6 +250,16 @@ class DownloadJob(Job):
         else:
             self.pathfmt.set_directory(kwdict)
 
+    def handle_metadata(self, kwdict):
+        """Run postprocessors with metadata from 'kwdict'"""
+        postprocessors = self.postprocessors
+
+        if postprocessors:
+            pathfmt = self.pathfmt
+            pathfmt.set_filename(kwdict)
+            for pp in postprocessors:
+                pp.run_metadata(pathfmt)
+
     def handle_queue(self, url, kwdict):
         if "_extractor" in kwdict:
             extr = kwdict["_extractor"].from_url(url)
@@ -280,7 +293,13 @@ class DownloadJob(Job):
         scheme = url.partition(":")[0]
         downloader = self.get_downloader(scheme)
         if downloader:
-            return downloader.download(url, self.pathfmt)
+            try:
+                return downloader.download(url, self.pathfmt)
+            except OSError as exc:
+                if exc.errno == errno.ENOSPC:
+                    raise
+                self.log.warning("%s: %s", exc.__class__.__name__, exc)
+                return False
         self._write_unsupported(url)
         return False
 
@@ -291,14 +310,14 @@ class DownloadJob(Job):
         except KeyError:
             pass
 
-        klass = downloader.find(scheme)
-        if klass and config.get(("downloader", klass.scheme, "enabled"), True):
-            instance = klass(self.extractor, self.out)
+        cls = downloader.find(scheme)
+        if cls and config.get(("downloader", cls.scheme), "enabled", True):
+            instance = cls(self.extractor, self.out)
         else:
             instance = None
             self.log.error("'%s:' URLs are not supported/enabled", scheme)
 
-        if klass and klass.scheme == "http":
+        if cls and cls.scheme == "http":
             self.downloaders["http"] = self.downloaders["https"] = instance
         else:
             self.downloaders[scheme] = instance
@@ -477,7 +496,10 @@ class DataJob(Job):
         Job.__init__(self, url, parent)
         self.file = file
         self.data = []
-        self.ascii = config.get(("output", "ascii"), ensure_ascii)
+        self.ascii = config.get(("output",), "ascii", ensure_ascii)
+
+        private = config.get(("output",), "private")
+        self.filter = (lambda x: x) if private else util.filter_dict
 
     def run(self):
         # collect data
@@ -492,7 +514,7 @@ class DataJob(Job):
             pass
 
         # convert numbers to string
-        if config.get(("output", "num-to-str"), False):
+        if config.get(("output",), "num-to-str", False):
             for msg in self.data:
                 util.transform_dict(msg[-1], util.number_to_string)
 
@@ -501,16 +523,19 @@ class DataJob(Job):
         return 0
 
     def handle_url(self, url, kwdict):
-        self.data.append((Message.Url, url, self._filter(kwdict)))
+        self.data.append((Message.Url, url, self.filter(kwdict)))
 
     def handle_urllist(self, urls, kwdict):
-        self.data.append((Message.Urllist, list(urls), self._filter(kwdict)))
+        self.data.append((Message.Urllist, list(urls), self.filter(kwdict)))
 
     def handle_directory(self, kwdict):
-        self.data.append((Message.Directory, self._filter(kwdict)))
+        self.data.append((Message.Directory, self.filter(kwdict)))
+
+    def handle_metadata(self, kwdict):
+        self.data.append((Message.Metadata, self.filter(kwdict)))
 
     def handle_queue(self, url, kwdict):
-        self.data.append((Message.Queue, url, self._filter(kwdict)))
+        self.data.append((Message.Queue, url, self.filter(kwdict)))
 
     def handle_finalize(self):
         self.file.close()