New upstream version 1.24.0.upstream/1.24.0

1 files changed, 46 insertions, 24 deletions

diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 2f48ffd..1f65438 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -32,11 +32,8 @@ class Job():
         self.pathfmt = None
         self.kwdict = {}
         self.status = 0
-        self.url_key = extr.config("url-metadata")
 
-        path_key = extr.config("path-metadata")
         path_proxy = output.PathfmtProxy(self)
-
         self._logger_extra = {
             "job"      : self,
             "extractor": extr,
@@ -56,12 +53,16 @@ class Job():
                 extr.category = pextr.category
                 extr.subcategory = pextr.subcategory
 
+        self.metadata_url = extr.config("url-metadata")
+        self.metadata_http = extr.config("http-metadata")
+        metadata_path = extr.config("path-metadata")
+
         # user-supplied metadata
         kwdict = extr.config("keywords")
         if kwdict:
             self.kwdict.update(kwdict)
-        if path_key:
-            self.kwdict[path_key] = path_proxy
+        if metadata_path:
+            self.kwdict[metadata_path] = path_proxy
 
         # predicates
         self.pred_url = self._prepare_predicates("image", True)
@@ -120,8 +121,8 @@ class Job():
         """Call the appropriate message handler"""
         if msg[0] == Message.Url:
             _, url, kwdict = msg
-            if self.url_key:
-                kwdict[self.url_key] = url
+            if self.metadata_url:
+                kwdict[self.metadata_url] = url
             if self.pred_url(url, kwdict):
                 self.update_kwdict(kwdict)
                 self.handle_url(url, kwdict)
@@ -132,8 +133,8 @@ class Job():
 
         elif msg[0] == Message.Queue:
             _, url, kwdict = msg
-            if self.url_key:
-                kwdict[self.url_key] = url
+            if self.metadata_url:
+                kwdict[self.metadata_url] = url
             if self.pred_queue(url, kwdict):
                 self.handle_queue(url, kwdict)
 
@@ -154,6 +155,8 @@ class Job():
         extr = self.extractor
         kwdict["category"] = extr.category
         kwdict["subcategory"] = extr.subcategory
+        if self.metadata_http:
+            kwdict.pop(self.metadata_http, None)
         if self.kwdict:
             kwdict.update(self.kwdict)
 
@@ -231,11 +234,14 @@ class DownloadJob(Job):
             self.handle_skip()
             return
 
-        if pathfmt.exists():
-            if archive:
-                archive.add(kwdict)
-            self.handle_skip()
-            return
+        if pathfmt.extension and not self.metadata_http:
+            pathfmt.build_path()
+
+            if pathfmt.exists():
+                if archive:
+                    archive.add(kwdict)
+                self.handle_skip()
+                return
 
         if self.sleep:
             self.extractor.sleep(self.sleep(), "download")
@@ -283,6 +289,9 @@ class DownloadJob(Job):
         if not self.pathfmt:
             self.initialize(kwdict)
         else:
+            if "post-after" in self.hooks:
+                for callback in self.hooks["post-after"]:
+                    callback(self.pathfmt)
             self.pathfmt.set_directory(kwdict)
         if "post" in self.hooks:
             for callback in self.hooks["post"]:
@@ -337,14 +346,20 @@ class DownloadJob(Job):
             self._write_unsupported(url)
 
     def handle_finalize(self):
-        pathfmt = self.pathfmt
         if self.archive:
             self.archive.close()
+
+        pathfmt = self.pathfmt
         if pathfmt:
+            hooks = self.hooks
+            if "post-after" in hooks:
+                for callback in hooks["post-after"]:
+                    callback(pathfmt)
+
             self.extractor._store_cookies()
-            if "finalize" in self.hooks:
+            if "finalize" in hooks:
                 status = self.status
-                for callback in self.hooks["finalize"]:
+                for callback in hooks["finalize"]:
                     callback(pathfmt, status)
 
     def handle_skip(self):
@@ -526,12 +541,11 @@ class SimulationJob(DownloadJob):
     def handle_url(self, url, kwdict):
         if not kwdict["extension"]:
             kwdict["extension"] = "jpg"
-        self.pathfmt.set_filename(kwdict)
         if self.sleep:
             self.extractor.sleep(self.sleep(), "download")
         if self.archive:
             self.archive.add(kwdict)
-        self.out.skip(self.pathfmt.path)
+        self.out.skip(self.pathfmt.build_filename(kwdict))
 
     def handle_directory(self, kwdict):
         if not self.pathfmt:
@@ -548,6 +562,11 @@ class KeywordJob(Job):
     def handle_url(self, url, kwdict):
         stdout_write("\nKeywords for filenames and --filter:\n"
                      "------------------------------------\n")
+
+        if self.metadata_http and url.startswith("http"):
+            kwdict[self.metadata_http] = util.extract_headers(
+                self.extractor.request(url, method="HEAD"))
+
         self.print_kwdict(kwdict)
         raise exception.StopExtraction()
 
@@ -605,12 +624,15 @@ class KeywordJob(Job):
                 self.print_kwdict(value, key + "[", markers)
 
             elif isinstance(value, list):
-                if value and isinstance(value[0], dict):
-                    self.print_kwdict(value[0], key + "[][", markers)
+                if not value:
+                    pass
+                elif isinstance(value[0], dict):
+                    self.print_kwdict(value[0], key + "[N][", markers)
                 else:
-                    write(key + "[]\n")
-                    for val in value:
-                        write("  - " + str(val) + "\n")
+                    fmt = ("  {:>%s} {}\n" % len(str(len(value)))).format
+                    write(key + "[N]\n")
+                    for idx, val in enumerate(value, 0):
+                        write(fmt(idx, val))
 
             else:
                 # string or number