New upstream version 1.15.0.upstream/1.15.0

1 files changed, 46 insertions, 2 deletions

diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 163c3c6..7d08b86 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -59,6 +59,9 @@ class Job():
 
     def run(self):
         """Execute or run the job"""
+        sleep = self.extractor.config("sleep-extractor")
+        if sleep:
+            time.sleep(sleep)
         try:
             log = self.extractor.log
             for msg in self.extractor:
@@ -197,6 +200,7 @@ class DownloadJob(Job):
     def __init__(self, url, parent=None):
         Job.__init__(self, url, parent)
         self.log = self.get_logger("download")
+        self.blacklist = None
         self.archive = None
         self.sleep = None
         self.downloaders = {}
@@ -224,7 +228,14 @@ class DownloadJob(Job):
             for pp in postprocessors:
                 pp.prepare(pathfmt)
 
-        if pathfmt.exists(archive):
+        if archive and kwdict in archive:
+            pathfmt.fix_extension()
+            self.handle_skip()
+            return
+
+        if pathfmt.exists():
+            if archive:
+                archive.add(kwdict)
             self.handle_skip()
             return
 
@@ -248,6 +259,8 @@ class DownloadJob(Job):
                 return
 
         if not pathfmt.temppath:
+            if archive:
+                archive.add(kwdict)
             self.handle_skip()
             return
 
@@ -299,6 +312,12 @@ class DownloadJob(Job):
             extr = kwdict["_extractor"].from_url(url)
         else:
             extr = extractor.find(url)
+            if extr:
+                if self.blacklist is None:
+                    self.blacklist = self._build_blacklist()
+                if extr.category in self.blacklist:
+                    extr = None
+
         if extr:
             self.status |= self.__class__(extr, self).run()
         else:
@@ -388,6 +407,8 @@ class DownloadJob(Job):
         if archive:
             path = util.expand_path(archive)
             try:
+                if "{" in path:
+                    path = util.Formatter(path).format_map(kwdict)
                 self.archive = util.DownloadArchive(path, self.extractor)
             except Exception as exc:
                 self.extractor.log.warning(
@@ -396,7 +417,7 @@ class DownloadJob(Job):
             else:
                 self.extractor.log.debug("Using download archive '%s'", path)
 
-        postprocessors = config("postprocessors")
+        postprocessors = self.extractor.config_accumulate("postprocessors")
         if postprocessors:
             pp_log = self.get_logger("postprocessor")
             pp_list = []
@@ -426,6 +447,25 @@ class DownloadJob(Job):
                 self.extractor.log.debug(
                     "Active postprocessor modules: %s", pp_list)
 
+    def _build_blacklist(self):
+        wlist = self.extractor.config("whitelist")
+        if wlist:
+            if isinstance(wlist, str):
+                wlist = wlist.split(",")
+            blist = {e.category for e in extractor._list_classes()}
+            blist.difference_update(wlist)
+            return blist
+
+        blist = self.extractor.config("blacklist")
+        if blist:
+            if isinstance(blist, str):
+                blist = blist.split(",")
+            blist = set(blist)
+        else:
+            blist = {self.extractor.category}
+        blist |= util.SPECIAL_EXTRACTORS
+        return blist
+
 
 class SimulationJob(DownloadJob):
     """Simulate the extraction process without downloading anything"""
@@ -549,6 +589,10 @@ class DataJob(Job):
         self.filter = (lambda x: x) if private else util.filter_dict
 
     def run(self):
+        sleep = self.extractor.config("sleep-extractor")
+        if sleep:
+            time.sleep(sleep)
+
         # collect data
         try:
             for msg in self.extractor: