New upstream version 1.26.0.upstream/1.26.0

1 files changed, 74 insertions, 13 deletions

diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index ca5785d..1e80cbf 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -18,7 +18,7 @@ from .output import stdout_write
 
 
 class Job():
-    """Base class for Job-types"""
+    """Base class for Job types"""
     ulog = None
 
     def __init__(self, extr, parent=None):
@@ -32,6 +32,35 @@ class Job():
         self.kwdict = {}
         self.status = 0
 
+        cfgpath = []
+        if parent:
+            if extr.category == parent.extractor.category or \
+                    extr.category in parent.parents:
+                parents = parent.parents
+            else:
+                parents = parent.parents + (parent.extractor.category,)
+
+            if parents:
+                for category in parents:
+                    cat = "{}>{}".format(category, extr.category)
+                    cfgpath.append((cat, extr.subcategory))
+                cfgpath.append((extr.category, extr.subcategory))
+                self.parents = parents
+            else:
+                self.parents = ()
+        else:
+            self.parents = ()
+
+        if extr.basecategory:
+            if not cfgpath:
+                cfgpath.append((extr.category, extr.subcategory))
+            cfgpath.append((extr.basecategory, extr.subcategory))
+
+        if cfgpath:
+            extr._cfgpath = cfgpath
+            extr.config = extr._config_shared
+            extr.config_accumulate = extr._config_shared_accumulate
+
         actions = extr.config("actions")
         if actions:
             from .actions import parse
@@ -77,16 +106,15 @@ class Job():
                 "current_git_head": util.git_head()
             }
 
-        # predicates
-        self.pred_url = self._prepare_predicates("image", True)
-        self.pred_queue = self._prepare_predicates("chapter", False)
-
     def run(self):
         """Execute or run the job"""
         extractor = self.extractor
         log = extractor.log
         msg = None
 
+        self._init()
+
+        # sleep before extractor start
         sleep = util.build_duration_func(
             extractor.config("sleep-extractor"))
         if sleep:
@@ -125,8 +153,7 @@ class Job():
                 log.info("No results for %s", extractor.url)
         finally:
             self.handle_finalize()
-            if extractor.finalize:
-                extractor.finalize()
+            extractor.finalize()
 
         return self.status
 
@@ -173,6 +200,11 @@ class Job():
         if self.kwdict:
             kwdict.update(self.kwdict)
 
+    def _init(self):
+        self.extractor.initialize()
+        self.pred_url = self._prepare_predicates("image", True)
+        self.pred_queue = self._prepare_predicates("chapter", False)
+
     def _prepare_predicates(self, target, skip=True):
         predicates = []
 
@@ -259,6 +291,10 @@ class DownloadJob(Job):
                 self.handle_skip()
                 return
 
+        if "prepare-after" in hooks:
+            for callback in hooks["prepare-after"]:
+                callback(pathfmt)
+
         if self.sleep:
             self.extractor.sleep(self.sleep(), "download")
 
@@ -356,10 +392,25 @@ class DownloadJob(Job):
                 try:
                     if pextr.config("parent-skip"):
                         job._skipcnt = self._skipcnt
-                        self.status |= job.run()
+                        status = job.run()
                         self._skipcnt = job._skipcnt
                     else:
-                        self.status |= job.run()
+                        status = job.run()
+
+                    if status:
+                        self.status |= status
+                        if "_fallback" in kwdict and self.fallback:
+                            fallback = kwdict["_fallback"] = \
+                                iter(kwdict["_fallback"])
+                            try:
+                                url = next(fallback)
+                            except StopIteration:
+                                pass
+                            else:
+                                text.nameext_from_url(url, kwdict)
+                                if url.startswith("ytdl:"):
+                                    kwdict["extension"] = ""
+                                self.handle_url(url, kwdict)
                     break
                 except exception.RestartExtraction:
                     pass
@@ -378,11 +429,19 @@ class DownloadJob(Job):
                 for callback in hooks["post-after"]:
                     callback(pathfmt)
 
-            self.extractor._store_cookies()
+            self.extractor.cookies_store()
+
             if "finalize" in hooks:
-                status = self.status
                 for callback in hooks["finalize"]:
-                    callback(pathfmt, status)
+                    callback(pathfmt)
+            if self.status:
+                if "finalize-error" in hooks:
+                    for callback in hooks["finalize-error"]:
+                        callback(pathfmt)
+            else:
+                if "finalize-success" in hooks:
+                    for callback in hooks["finalize-success"]:
+                        callback(pathfmt)
 
     def handle_skip(self):
         pathfmt = self.pathfmt
@@ -475,7 +534,7 @@ class DownloadJob(Job):
                 elif skip == "terminate":
                     self._skipexc = exception.TerminateExtraction
                 elif skip == "exit":
-                    self._skipexc = sys.exit
+                    self._skipexc = SystemExit
                 self._skipmax = text.parse_int(smax)
         else:
             # monkey-patch methods to always return False
@@ -755,6 +814,8 @@ class DataJob(Job):
         self.filter = dict.copy if private else util.filter_dict
 
     def run(self):
+        self._init()
+
         extractor = self.extractor
         sleep = util.build_duration_func(
             extractor.config("sleep-extractor"))