diff options
Diffstat (limited to 'gallery_dl/job.py')
| -rw-r--r-- | gallery_dl/job.py | 70 |
1 files changed, 46 insertions, 24 deletions
diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 2f48ffd..1f65438 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -32,11 +32,8 @@ class Job(): self.pathfmt = None self.kwdict = {} self.status = 0 - self.url_key = extr.config("url-metadata") - path_key = extr.config("path-metadata") path_proxy = output.PathfmtProxy(self) - self._logger_extra = { "job" : self, "extractor": extr, @@ -56,12 +53,16 @@ class Job(): extr.category = pextr.category extr.subcategory = pextr.subcategory + self.metadata_url = extr.config("url-metadata") + self.metadata_http = extr.config("http-metadata") + metadata_path = extr.config("path-metadata") + # user-supplied metadata kwdict = extr.config("keywords") if kwdict: self.kwdict.update(kwdict) - if path_key: - self.kwdict[path_key] = path_proxy + if metadata_path: + self.kwdict[metadata_path] = path_proxy # predicates self.pred_url = self._prepare_predicates("image", True) @@ -120,8 +121,8 @@ class Job(): """Call the appropriate message handler""" if msg[0] == Message.Url: _, url, kwdict = msg - if self.url_key: - kwdict[self.url_key] = url + if self.metadata_url: + kwdict[self.metadata_url] = url if self.pred_url(url, kwdict): self.update_kwdict(kwdict) self.handle_url(url, kwdict) @@ -132,8 +133,8 @@ class Job(): elif msg[0] == Message.Queue: _, url, kwdict = msg - if self.url_key: - kwdict[self.url_key] = url + if self.metadata_url: + kwdict[self.metadata_url] = url if self.pred_queue(url, kwdict): self.handle_queue(url, kwdict) @@ -154,6 +155,8 @@ class Job(): extr = self.extractor kwdict["category"] = extr.category kwdict["subcategory"] = extr.subcategory + if self.metadata_http: + kwdict.pop(self.metadata_http, None) if self.kwdict: kwdict.update(self.kwdict) @@ -231,11 +234,14 @@ class DownloadJob(Job): self.handle_skip() return - if pathfmt.exists(): - if archive: - archive.add(kwdict) - self.handle_skip() - return + if pathfmt.extension and not self.metadata_http: + pathfmt.build_path() + + if pathfmt.exists(): + if archive: + archive.add(kwdict) + self.handle_skip() + return if self.sleep: self.extractor.sleep(self.sleep(), "download") @@ -283,6 +289,9 @@ class DownloadJob(Job): if not self.pathfmt: self.initialize(kwdict) else: + if "post-after" in self.hooks: + for callback in self.hooks["post-after"]: + callback(self.pathfmt) self.pathfmt.set_directory(kwdict) if "post" in self.hooks: for callback in self.hooks["post"]: @@ -337,14 +346,20 @@ class DownloadJob(Job): self._write_unsupported(url) def handle_finalize(self): - pathfmt = self.pathfmt if self.archive: self.archive.close() + + pathfmt = self.pathfmt if pathfmt: + hooks = self.hooks + if "post-after" in hooks: + for callback in hooks["post-after"]: + callback(pathfmt) + self.extractor._store_cookies() - if "finalize" in self.hooks: + if "finalize" in hooks: status = self.status - for callback in self.hooks["finalize"]: + for callback in hooks["finalize"]: callback(pathfmt, status) def handle_skip(self): @@ -526,12 +541,11 @@ class SimulationJob(DownloadJob): def handle_url(self, url, kwdict): if not kwdict["extension"]: kwdict["extension"] = "jpg" - self.pathfmt.set_filename(kwdict) if self.sleep: self.extractor.sleep(self.sleep(), "download") if self.archive: self.archive.add(kwdict) - self.out.skip(self.pathfmt.path) + self.out.skip(self.pathfmt.build_filename(kwdict)) def handle_directory(self, kwdict): if not self.pathfmt: @@ -548,6 +562,11 @@ class KeywordJob(Job): def handle_url(self, url, kwdict): stdout_write("\nKeywords for filenames and --filter:\n" "------------------------------------\n") + + if self.metadata_http and url.startswith("http"): + kwdict[self.metadata_http] = util.extract_headers( + self.extractor.request(url, method="HEAD")) + self.print_kwdict(kwdict) raise exception.StopExtraction() @@ -605,12 +624,15 @@ class KeywordJob(Job): self.print_kwdict(value, key + "[", markers) elif isinstance(value, list): - if value and isinstance(value[0], dict): - self.print_kwdict(value[0], key + "[][", markers) + if not value: + pass + elif isinstance(value[0], dict): + self.print_kwdict(value[0], key + "[N][", markers) else: - write(key + "[]\n") - for val in value: - write(" - " + str(val) + "\n") + fmt = (" {:>%s} {}\n" % len(str(len(value)))).format + write(key + "[N]\n") + for idx, val in enumerate(value, 0): + write(fmt(idx, val)) else: # string or number |