diff options
Diffstat (limited to 'gallery_dl/job.py')
| -rw-r--r-- | gallery_dl/job.py | 35 |
1 files changed, 20 insertions, 15 deletions
diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 32e9bb5..4e185d0 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -15,7 +15,7 @@ import operator import functools import collections from . import extractor, downloader, postprocessor -from . import config, text, util, output, exception +from . import config, text, util, path, formatter, output, exception from .extractor.message import Message @@ -72,9 +72,9 @@ class Job(): log = extractor.log msg = None - sleep = extractor.config("sleep-extractor") + sleep = util.build_duration_func(extractor.config("sleep-extractor")) if sleep: - time.sleep(sleep) + time.sleep(sleep()) try: for msg in extractor: @@ -109,6 +109,8 @@ class Job(): log.info("No results for %s", extractor.url) finally: self.handle_finalize() + if extractor.finalize: + extractor.finalize() return self.status @@ -234,7 +236,7 @@ class DownloadJob(Job): return if self.sleep: - time.sleep(self.sleep) + time.sleep(self.sleep()) # download from URL if not self.download(url): @@ -392,11 +394,11 @@ class DownloadJob(Job): def initialize(self, kwdict=None): """Delayed initialization of PathFormat, etc.""" cfg = self.extractor.config - pathfmt = self.pathfmt = util.PathFormat(self.extractor) + pathfmt = self.pathfmt = path.PathFormat(self.extractor) if kwdict: pathfmt.set_directory(kwdict) - self.sleep = cfg("sleep") + self.sleep = util.build_duration_func(cfg("sleep")) self.fallback = cfg("fallback", True) if not cfg("download", True): # monkey-patch method to do nothing and always return True @@ -404,17 +406,18 @@ class DownloadJob(Job): archive = cfg("archive") if archive: - path = util.expand_path(archive) + archive = util.expand_path(archive) try: - if "{" in path: - path = util.Formatter(path).format_map(kwdict) - self.archive = util.DownloadArchive(path, self.extractor) + if "{" in archive: + archive = formatter.parse(archive).format_map(kwdict) + self.archive = util.DownloadArchive(archive, self.extractor) except Exception as exc: self.extractor.log.warning( "Failed to open download archive at '%s' ('%s: %s')", - path, exc.__class__.__name__, exc) + archive, exc.__class__.__name__, exc) else: - self.extractor.log.debug("Using download archive '%s'", path) + self.extractor.log.debug( + "Using download archive '%s'", archive) skip = cfg("skip", True) if skip: @@ -469,6 +472,7 @@ class DownloadJob(Job): except Exception as exc: pp_log.error("'%s' initialization failed: %s: %s", name, exc.__class__.__name__, exc) + pp_log.debug("", exc_info=True) else: pp_list.append(pp_obj) @@ -539,7 +543,7 @@ class SimulationJob(DownloadJob): self.pathfmt.set_filename(kwdict) self.out.skip(self.pathfmt.path) if self.sleep: - time.sleep(self.sleep) + time.sleep(self.sleep()) if self.archive: self.archive.add(kwdict) @@ -693,9 +697,10 @@ class DataJob(Job): self.filter = util.identity if private else util.filter_dict def run(self): - sleep = self.extractor.config("sleep-extractor") + sleep = util.build_duration_func( + self.extractor.config("sleep-extractor")) if sleep: - time.sleep(sleep) + time.sleep(sleep()) # collect data try: |