New upstream version 1.19.0.upstream/1.19.0

1 files changed, 20 insertions, 15 deletions

diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 32e9bb5..4e185d0 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -15,7 +15,7 @@ import operator
 import functools
 import collections
 from . import extractor, downloader, postprocessor
-from . import config, text, util, output, exception
+from . import config, text, util, path, formatter, output, exception
 from .extractor.message import Message
 
 
@@ -72,9 +72,9 @@ class Job():
         log = extractor.log
         msg = None
 
-        sleep = extractor.config("sleep-extractor")
+        sleep = util.build_duration_func(extractor.config("sleep-extractor"))
         if sleep:
-            time.sleep(sleep)
+            time.sleep(sleep())
 
         try:
             for msg in extractor:
@@ -109,6 +109,8 @@ class Job():
                 log.info("No results for %s", extractor.url)
         finally:
             self.handle_finalize()
+            if extractor.finalize:
+                extractor.finalize()
 
         return self.status
 
@@ -234,7 +236,7 @@ class DownloadJob(Job):
             return
 
         if self.sleep:
-            time.sleep(self.sleep)
+            time.sleep(self.sleep())
 
         # download from URL
         if not self.download(url):
@@ -392,11 +394,11 @@ class DownloadJob(Job):
     def initialize(self, kwdict=None):
         """Delayed initialization of PathFormat, etc."""
         cfg = self.extractor.config
-        pathfmt = self.pathfmt = util.PathFormat(self.extractor)
+        pathfmt = self.pathfmt = path.PathFormat(self.extractor)
         if kwdict:
             pathfmt.set_directory(kwdict)
 
-        self.sleep = cfg("sleep")
+        self.sleep = util.build_duration_func(cfg("sleep"))
         self.fallback = cfg("fallback", True)
         if not cfg("download", True):
             # monkey-patch method to do nothing and always return True
@@ -404,17 +406,18 @@ class DownloadJob(Job):
 
         archive = cfg("archive")
         if archive:
-            path = util.expand_path(archive)
+            archive = util.expand_path(archive)
             try:
-                if "{" in path:
-                    path = util.Formatter(path).format_map(kwdict)
-                self.archive = util.DownloadArchive(path, self.extractor)
+                if "{" in archive:
+                    archive = formatter.parse(archive).format_map(kwdict)
+                self.archive = util.DownloadArchive(archive, self.extractor)
             except Exception as exc:
                 self.extractor.log.warning(
                     "Failed to open download archive at '%s' ('%s: %s')",
-                    path, exc.__class__.__name__, exc)
+                    archive, exc.__class__.__name__, exc)
             else:
-                self.extractor.log.debug("Using download archive '%s'", path)
+                self.extractor.log.debug(
+                    "Using download archive '%s'", archive)
 
         skip = cfg("skip", True)
         if skip:
@@ -469,6 +472,7 @@ class DownloadJob(Job):
                 except Exception as exc:
                     pp_log.error("'%s' initialization failed:  %s: %s",
                                  name, exc.__class__.__name__, exc)
+                    pp_log.debug("", exc_info=True)
                 else:
                     pp_list.append(pp_obj)
 
@@ -539,7 +543,7 @@ class SimulationJob(DownloadJob):
         self.pathfmt.set_filename(kwdict)
         self.out.skip(self.pathfmt.path)
         if self.sleep:
-            time.sleep(self.sleep)
+            time.sleep(self.sleep())
         if self.archive:
             self.archive.add(kwdict)
 
@@ -693,9 +697,10 @@ class DataJob(Job):
         self.filter = util.identity if private else util.filter_dict
 
     def run(self):
-        sleep = self.extractor.config("sleep-extractor")
+        sleep = util.build_duration_func(
+            self.extractor.config("sleep-extractor"))
         if sleep:
-            time.sleep(sleep)
+            time.sleep(sleep())
 
         # collect data
         try: