New upstream version 1.29.0.upstream/1.29.0

3 files changed, 78 insertions, 11 deletions

diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py
index 1168d83..8430884 100644
--- a/gallery_dl/downloader/common.py
+++ b/gallery_dl/downloader/common.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2014-2022 Mike Fährmann
+# Copyright 2014-2025 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -10,6 +10,7 @@
 
 import os
 from .. import config, util
+_config = config._config
 
 
 class DownloaderBase():
@@ -17,8 +18,15 @@ class DownloaderBase():
     scheme = ""
 
     def __init__(self, job):
+        extractor = job.extractor
+
+        opts = self._extractor_config(extractor)
+        if opts:
+            self.opts = opts
+            self.config = self.config_opts
+
         self.out = job.out
-        self.session = job.extractor.session
+        self.session = extractor.session
         self.part = self.config("part", True)
         self.partdir = self.config("part-directory")
         self.log = job.get_logger("downloader." + self.scheme)
@@ -29,7 +37,7 @@ class DownloaderBase():
 
         proxies = self.config("proxy", util.SENTINEL)
         if proxies is util.SENTINEL:
-            self.proxies = job.extractor._proxies
+            self.proxies = extractor._proxies
         else:
             self.proxies = util.build_proxy_map(proxies, self.log)
 
@@ -37,5 +45,45 @@ class DownloaderBase():
         """Interpolate downloader config value for 'key'"""
         return config.interpolate(("downloader", self.scheme), key, default)
 
+    def config_opts(self, key, default=None, conf=_config):
+        if key in conf:
+            return conf[key]
+        value = self.opts.get(key, util.SENTINEL)
+        if value is not util.SENTINEL:
+            return value
+        return config.interpolate(("downloader", self.scheme), key, default)
+
+    def _extractor_config(self, extractor):
+        path = extractor._cfgpath
+        if not isinstance(path, list):
+            return self._extractor_opts(path[1], path[2])
+
+        opts = {}
+        for cat, sub in reversed(path):
+            popts = self._extractor_opts(cat, sub)
+            if popts:
+                opts.update(popts)
+        return opts
+
+    def _extractor_opts(self, category, subcategory):
+        cfg = config.get(("extractor",), category)
+        if not cfg:
+            return None
+
+        copts = cfg.get(self.scheme)
+        if copts:
+            if subcategory in cfg:
+                sopts = cfg[subcategory].get(self.scheme)
+                if sopts:
+                    opts = copts.copy()
+                    opts.update(sopts)
+                    return opts
+            return copts
+
+        if subcategory in cfg:
+            return cfg[subcategory].get(self.scheme)
+
+        return None
+
     def download(self, url, pathfmt):
         """Write data from 'url' into the file specified by 'pathfmt'"""
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index c8aeef8..449ffe8 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -12,7 +12,7 @@ import time
 import mimetypes
 from requests.exceptions import RequestException, ConnectionError, Timeout
 from .common import DownloaderBase
-from .. import text, util
+from .. import text, util, output
 from ssl import SSLError
 
 
@@ -38,6 +38,7 @@ class HttpDownloader(DownloaderBase):
         self.verify = self.config("verify", extractor._verify)
         self.mtime = self.config("mtime", True)
         self.rate = self.config("rate")
+        interval_429 = self.config("sleep-429")
 
         if not self.config("consume-content", False):
             # this resets the underlying TCP connection, and therefore
@@ -79,12 +80,16 @@ class HttpDownloader(DownloaderBase):
             self.receive = self._receive_rate
             if self.progress < 0.0:
                 self.progress = 0.0
+        if interval_429 is None:
+            self.interval_429 = extractor._interval_429
+        else:
+            self.interval_429 = util.build_duration_func(interval_429)
 
     def download(self, url, pathfmt):
         try:
             return self._download_impl(url, pathfmt)
         except Exception:
-            print()
+            output.stderr_write("\n")
             raise
         finally:
             # remove file from incomplete downloads
@@ -93,7 +98,7 @@ class HttpDownloader(DownloaderBase):
 
     def _download_impl(self, url, pathfmt):
         response = None
-        tries = 0
+        tries = code = 0
         msg = ""
 
         metadata = self.metadata
@@ -111,10 +116,17 @@ class HttpDownloader(DownloaderBase):
                 if response:
                     self.release_conn(response)
                     response = None
+
                 self.log.warning("%s (%s/%s)", msg, tries, self.retries+1)
                 if tries > self.retries:
                     return False
-                time.sleep(tries)
+
+                if code == 429 and self.interval_429:
+                    s = self.interval_429()
+                    time.sleep(s if s > tries else tries)
+                else:
+                    time.sleep(tries)
+                code = 0
 
             tries += 1
             file_header = None
@@ -257,7 +269,7 @@ class HttpDownloader(DownloaderBase):
                         else response.iter_content(16), b"")
                 except (RequestException, SSLError) as exc:
                     msg = str(exc)
-                    print()
+                    output.stderr_write("\n")
                     continue
                 if self._adjust_extension(pathfmt, file_header) and \
                         pathfmt.exists():
@@ -291,14 +303,14 @@ class HttpDownloader(DownloaderBase):
                     self.receive(fp, content, size, offset)
                 except (RequestException, SSLError) as exc:
                     msg = str(exc)
-                    print()
+                    output.stderr_write("\n")
                     continue
 
                 # check file size
                 if size and fp.tell() < size:
                     msg = "file size mismatch ({} < {})".format(
                         fp.tell(), size)
-                    print()
+                    output.stderr_write("\n")
                     continue
 
             break
@@ -317,7 +329,7 @@ class HttpDownloader(DownloaderBase):
             for _ in response.iter_content(self.chunk_size):
                 pass
         except (RequestException, SSLError) as exc:
-            print()
+            output.stderr_write("\n")
             self.log.debug(
                 "Unable to consume response body (%s: %s); "
                 "closing the connection anyway", exc.__class__.__name__, exc)
diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py
index 40cddec..1242098 100644
--- a/gallery_dl/downloader/ytdl.py
+++ b/gallery_dl/downloader/ytdl.py
@@ -48,6 +48,13 @@ class YoutubeDLDownloader(DownloaderBase):
                     self.log.debug("", exc_info=exc)
                     self.download = lambda u, p: False
                     return False
+
+                try:
+                    ytdl_version = module.version.__version__
+                except Exception:
+                    ytdl_version = ""
+                self.log.debug("Using %s version %s", module, ytdl_version)
+
                 self.ytdl_instance = ytdl_instance = ytdl.construct_YoutubeDL(
                     module, self, self.ytdl_opts)
                 if self.outtmpl == "default":