diff options
Diffstat (limited to 'gallery_dl/downloader')
| -rw-r--r-- | gallery_dl/downloader/common.py | 54 | ||||
| -rw-r--r-- | gallery_dl/downloader/http.py | 28 | ||||
| -rw-r--r-- | gallery_dl/downloader/ytdl.py | 7 |
3 files changed, 78 insertions, 11 deletions
diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py index 1168d83..8430884 100644 --- a/gallery_dl/downloader/common.py +++ b/gallery_dl/downloader/common.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2022 Mike Fährmann +# Copyright 2014-2025 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -10,6 +10,7 @@ import os from .. import config, util +_config = config._config class DownloaderBase(): @@ -17,8 +18,15 @@ class DownloaderBase(): scheme = "" def __init__(self, job): + extractor = job.extractor + + opts = self._extractor_config(extractor) + if opts: + self.opts = opts + self.config = self.config_opts + self.out = job.out - self.session = job.extractor.session + self.session = extractor.session self.part = self.config("part", True) self.partdir = self.config("part-directory") self.log = job.get_logger("downloader." + self.scheme) @@ -29,7 +37,7 @@ class DownloaderBase(): proxies = self.config("proxy", util.SENTINEL) if proxies is util.SENTINEL: - self.proxies = job.extractor._proxies + self.proxies = extractor._proxies else: self.proxies = util.build_proxy_map(proxies, self.log) @@ -37,5 +45,45 @@ class DownloaderBase(): """Interpolate downloader config value for 'key'""" return config.interpolate(("downloader", self.scheme), key, default) + def config_opts(self, key, default=None, conf=_config): + if key in conf: + return conf[key] + value = self.opts.get(key, util.SENTINEL) + if value is not util.SENTINEL: + return value + return config.interpolate(("downloader", self.scheme), key, default) + + def _extractor_config(self, extractor): + path = extractor._cfgpath + if not isinstance(path, list): + return self._extractor_opts(path[1], path[2]) + + opts = {} + for cat, sub in reversed(path): + popts = self._extractor_opts(cat, sub) + if popts: + opts.update(popts) + return opts + + def _extractor_opts(self, category, subcategory): + cfg = config.get(("extractor",), category) + if not cfg: + return None + + copts = cfg.get(self.scheme) + if copts: + if subcategory in cfg: + sopts = cfg[subcategory].get(self.scheme) + if sopts: + opts = copts.copy() + opts.update(sopts) + return opts + return copts + + if subcategory in cfg: + return cfg[subcategory].get(self.scheme) + + return None + def download(self, url, pathfmt): """Write data from 'url' into the file specified by 'pathfmt'""" diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index c8aeef8..449ffe8 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -12,7 +12,7 @@ import time import mimetypes from requests.exceptions import RequestException, ConnectionError, Timeout from .common import DownloaderBase -from .. import text, util +from .. import text, util, output from ssl import SSLError @@ -38,6 +38,7 @@ class HttpDownloader(DownloaderBase): self.verify = self.config("verify", extractor._verify) self.mtime = self.config("mtime", True) self.rate = self.config("rate") + interval_429 = self.config("sleep-429") if not self.config("consume-content", False): # this resets the underlying TCP connection, and therefore @@ -79,12 +80,16 @@ class HttpDownloader(DownloaderBase): self.receive = self._receive_rate if self.progress < 0.0: self.progress = 0.0 + if interval_429 is None: + self.interval_429 = extractor._interval_429 + else: + self.interval_429 = util.build_duration_func(interval_429) def download(self, url, pathfmt): try: return self._download_impl(url, pathfmt) except Exception: - print() + output.stderr_write("\n") raise finally: # remove file from incomplete downloads @@ -93,7 +98,7 @@ class HttpDownloader(DownloaderBase): def _download_impl(self, url, pathfmt): response = None - tries = 0 + tries = code = 0 msg = "" metadata = self.metadata @@ -111,10 +116,17 @@ class HttpDownloader(DownloaderBase): if response: self.release_conn(response) response = None + self.log.warning("%s (%s/%s)", msg, tries, self.retries+1) if tries > self.retries: return False - time.sleep(tries) + + if code == 429 and self.interval_429: + s = self.interval_429() + time.sleep(s if s > tries else tries) + else: + time.sleep(tries) + code = 0 tries += 1 file_header = None @@ -257,7 +269,7 @@ class HttpDownloader(DownloaderBase): else response.iter_content(16), b"") except (RequestException, SSLError) as exc: msg = str(exc) - print() + output.stderr_write("\n") continue if self._adjust_extension(pathfmt, file_header) and \ pathfmt.exists(): @@ -291,14 +303,14 @@ class HttpDownloader(DownloaderBase): self.receive(fp, content, size, offset) except (RequestException, SSLError) as exc: msg = str(exc) - print() + output.stderr_write("\n") continue # check file size if size and fp.tell() < size: msg = "file size mismatch ({} < {})".format( fp.tell(), size) - print() + output.stderr_write("\n") continue break @@ -317,7 +329,7 @@ class HttpDownloader(DownloaderBase): for _ in response.iter_content(self.chunk_size): pass except (RequestException, SSLError) as exc: - print() + output.stderr_write("\n") self.log.debug( "Unable to consume response body (%s: %s); " "closing the connection anyway", exc.__class__.__name__, exc) diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py index 40cddec..1242098 100644 --- a/gallery_dl/downloader/ytdl.py +++ b/gallery_dl/downloader/ytdl.py @@ -48,6 +48,13 @@ class YoutubeDLDownloader(DownloaderBase): self.log.debug("", exc_info=exc) self.download = lambda u, p: False return False + + try: + ytdl_version = module.version.__version__ + except Exception: + ytdl_version = "" + self.log.debug("Using %s version %s", module, ytdl_version) + self.ytdl_instance = ytdl_instance = ytdl.construct_YoutubeDL( module, self, self.ytdl_opts) if self.outtmpl == "default": |