New upstream version 1.11.1upstream/1.11.1

1 files changed, 62 insertions, 65 deletions

diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 8b61024..9c76336 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -29,6 +29,7 @@ class Job():
         extr.log.job = self
         extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url)
 
+        self.status = 0
         self.pred_url = self._prepare_predicates("image", True)
         self.pred_queue = self._prepare_predicates("chapter", False)
 
@@ -46,34 +47,18 @@ class Job():
             log = self.extractor.log
             for msg in self.extractor:
                 self.dispatch(msg)
-        except exception.AuthenticationError as exc:
-            msg = str(exc) or "Please provide a valid username/password pair."
-            log.error("Authentication failed: %s", msg)
-        except exception.AuthorizationError:
-            log.error("You do not have permission to access the resource "
-                      "at '%s'", self.extractor.url)
-        except exception.NotFoundError as exc:
-            res = str(exc) or "resource (gallery/image/user)"
-            log.error("The %s at '%s' does not exist", res, self.extractor.url)
-        except exception.HttpError as exc:
-            err = exc.args[0]
-            if isinstance(err, Exception):
-                err = "{}: {}".format(err.__class__.__name__, err)
-            log.error("HTTP request failed:  %s", err)
-        except exception.FormatError as exc:
-            err, obj = exc.args
-            log.error("Applying %s format string failed:  %s: %s",
-                      obj, err.__class__.__name__, err)
-        except exception.FilterError as exc:
-            err = exc.args[0]
-            log.error("Evaluating filter expression failed:  %s: %s",
-                      err.__class__.__name__, err)
-        except exception.StopExtraction:
-            pass
+        except exception.StopExtraction as exc:
+            if exc.message:
+                log.error(exc.message)
+            self.status |= exc.code
+        except exception.GalleryDLException as exc:
+            log.error("%s: %s", exc.__class__.__name__, exc)
+            self.status |= exc.code
         except OSError as exc:
             log.error("Unable to download data:  %s: %s",
                       exc.__class__.__name__, exc)
             log.debug("", exc_info=True)
+            self.status |= 128
         except Exception as exc:
             log.error(("An unexpected error occurred: %s - %s. "
                        "Please run gallery-dl again with the --verbose flag, "
@@ -81,8 +66,13 @@ class Job():
                        "https://github.com/mikf/gallery-dl/issues ."),
                       exc.__class__.__name__, exc)
             log.debug("", exc_info=True)
+            self.status |= 1
+        except BaseException:
+            self.status |= 1
+            raise
         finally:
             self.handle_finalize()
+        return self.status
 
     def dispatch(self, msg):
         """Call the appropriate message handler"""
@@ -114,17 +104,17 @@ class Job():
                 )
             # TODO: support for multiple message versions
 
-    def handle_url(self, url, keywords):
+    def handle_url(self, url, kwdict):
         """Handle Message.Url"""
 
-    def handle_urllist(self, urls, keywords):
+    def handle_urllist(self, urls, kwdict):
         """Handle Message.Urllist"""
-        self.handle_url(urls[0], keywords)
+        self.handle_url(urls[0], kwdict)
 
-    def handle_directory(self, keywords):
+    def handle_directory(self, kwdict):
         """Handle Message.Directory"""
 
-    def handle_queue(self, url, keywords):
+    def handle_queue(self, url, kwdict):
         """Handle Message.Queue"""
 
     def handle_finalize(self):
@@ -132,8 +122,9 @@ class Job():
 
     def update_kwdict(self, kwdict):
         """Update 'kwdict' with additional metadata"""
-        kwdict["category"] = self.extractor.category
-        kwdict["subcategory"] = self.extractor.subcategory
+        extr = self.extractor
+        kwdict["category"] = extr.category
+        kwdict["subcategory"] = extr.subcategory
         if self.userkwds:
             kwdict.update(self.userkwds)
 
@@ -189,14 +180,14 @@ class DownloadJob(Job):
         self.postprocessors = None
         self.out = output.select()
 
-    def handle_url(self, url, keywords, fallback=None):
+    def handle_url(self, url, kwdict, fallback=None):
         """Download the resource specified in 'url'"""
         postprocessors = self.postprocessors
         pathfmt = self.pathfmt
         archive = self.archive
 
         # prepare download
-        pathfmt.set_filename(keywords)
+        pathfmt.set_filename(kwdict)
 
         if postprocessors:
             for pp in postprocessors:
@@ -219,6 +210,7 @@ class DownloadJob(Job):
                     break
             else:
                 # download failed
+                self.status |= 4
                 self.log.error("Failed to download %s",
                                pathfmt.filename or url)
                 return
@@ -236,41 +228,45 @@ class DownloadJob(Job):
         pathfmt.finalize()
         self.out.success(pathfmt.path, 0)
         if archive:
-            archive.add(keywords)
+            archive.add(kwdict)
         if postprocessors:
             for pp in postprocessors:
                 pp.run_after(pathfmt)
         self._skipcnt = 0
 
-    def handle_urllist(self, urls, keywords):
+    def handle_urllist(self, urls, kwdict):
         """Download the resource specified in 'url'"""
         fallback = iter(urls)
         url = next(fallback)
-        self.handle_url(url, keywords, fallback)
+        self.handle_url(url, kwdict, fallback)
 
-    def handle_directory(self, keywords):
+    def handle_directory(self, kwdict):
         """Set and create the target directory for downloads"""
         if not self.pathfmt:
-            self.initialize(keywords)
+            self.initialize(kwdict)
         else:
-            self.pathfmt.set_directory(keywords)
+            self.pathfmt.set_directory(kwdict)
 
-    def handle_queue(self, url, keywords):
-        if "_extractor" in keywords:
-            extr = keywords["_extractor"].from_url(url)
+    def handle_queue(self, url, kwdict):
+        if "_extractor" in kwdict:
+            extr = kwdict["_extractor"].from_url(url)
         else:
             extr = extractor.find(url)
         if extr:
-            self.__class__(extr, self).run()
+            self.status |= self.__class__(extr, self).run()
         else:
             self._write_unsupported(url)
 
     def handle_finalize(self):
-        if self.postprocessors:
-            for pp in self.postprocessors:
-                pp.finalize()
+        pathfmt = self.pathfmt
         if self.archive:
             self.archive.close()
+        if pathfmt:
+            self.extractor._store_cookies()
+            if self.postprocessors:
+                status = self.status
+                for pp in self.postprocessors:
+                    pp.run_final(pathfmt, status)
 
     def handle_skip(self):
         self.out.skip(self.pathfmt.path)
@@ -308,11 +304,11 @@ class DownloadJob(Job):
             self.downloaders[scheme] = instance
         return instance
 
-    def initialize(self, keywords=None):
+    def initialize(self, kwdict=None):
         """Delayed initialization of PathFormat, etc."""
         self.pathfmt = util.PathFormat(self.extractor)
-        if keywords:
-            self.pathfmt.set_directory(keywords)
+        if kwdict:
+            self.pathfmt.set_directory(kwdict)
 
         self.sleep = self.extractor.config("sleep")
         if not self.extractor.config("download", True):
@@ -379,15 +375,15 @@ class DownloadJob(Job):
 class SimulationJob(DownloadJob):
     """Simulate the extraction process without downloading anything"""
 
-    def handle_url(self, url, keywords, fallback=None):
-        self.pathfmt.set_filename(keywords)
+    def handle_url(self, url, kwdict, fallback=None):
+        self.pathfmt.set_filename(kwdict)
         self.out.skip(self.pathfmt.path)
         if self.sleep:
             time.sleep(self.sleep)
         if self.archive:
-            self.archive.add(keywords)
+            self.archive.add(kwdict)
 
-    def handle_directory(self, keywords):
+    def handle_directory(self, kwdict):
         if not self.pathfmt:
             self.initialize()
 
@@ -395,19 +391,19 @@ class SimulationJob(DownloadJob):
 class KeywordJob(Job):
     """Print available keywords"""
 
-    def handle_url(self, url, keywords):
+    def handle_url(self, url, kwdict):
         print("\nKeywords for filenames and --filter:")
         print("------------------------------------")
-        self.print_keywords(keywords)
+        self.print_kwdict(kwdict)
         raise exception.StopExtraction()
 
-    def handle_directory(self, keywords):
+    def handle_directory(self, kwdict):
         print("Keywords for directory names:")
         print("-----------------------------")
-        self.print_keywords(keywords)
+        self.print_kwdict(kwdict)
 
-    def handle_queue(self, url, keywords):
-        if not keywords:
+    def handle_queue(self, url, kwdict):
+        if not kwdict:
             self.extractor.log.info(
                 "This extractor delegates work to other extractors "
                 "and does not provide any keywords on its own. Try "
@@ -415,27 +411,27 @@ class KeywordJob(Job):
         else:
             print("Keywords for --chapter-filter:")
             print("------------------------------")
-            self.print_keywords(keywords)
+            self.print_kwdict(kwdict)
             if self.extractor.categorytransfer:
                 print()
                 KeywordJob(url, self).run()
         raise exception.StopExtraction()
 
     @staticmethod
-    def print_keywords(keywords, prefix=""):
-        """Print key-value pairs with formatting"""
+    def print_kwdict(kwdict, prefix=""):
+        """Print key-value pairs in 'kwdict' with formatting"""
         suffix = "]" if prefix else ""
-        for key, value in sorted(keywords.items()):
+        for key, value in sorted(kwdict.items()):
             if key[0] == "_":
                 continue
             key = prefix + key + suffix
 
             if isinstance(value, dict):
-                KeywordJob.print_keywords(value, key + "[")
+                KeywordJob.print_kwdict(value, key + "[")
 
             elif isinstance(value, list):
                 if value and isinstance(value[0], dict):
-                    KeywordJob.print_keywords(value[0], key + "[][")
+                    KeywordJob.print_kwdict(value[0], key + "[][")
                 else:
                     print(key, "[]", sep="")
                     for val in value:
@@ -502,6 +498,7 @@ class DataJob(Job):
 
         # dump to 'file'
         util.dump_json(self.data, self.file, self.ascii, 2)
+        return 0
 
     def handle_url(self, url, kwdict):
         self.data.append((Message.Url, url, self._filter(kwdict)))