New upstream version 1.12.1upstream/1.12.1

2 files changed, 35 insertions, 23 deletions

diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py
index 6e5cd4c..596c956 100644
--- a/gallery_dl/downloader/common.py
+++ b/gallery_dl/downloader/common.py
@@ -30,7 +30,7 @@ class DownloaderBase():
 
     def config(self, key, default=None):
         """Interpolate downloader config value for 'key'"""
-        return config.interpolate(("downloader", self.scheme, key), default)
+        return config.interpolate(("downloader", self.scheme), key, default)
 
     def download(self, url, pathfmt):
         """Write data from 'url' into the file specified by 'pathfmt'"""
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 1c78cfb..fab96ba 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -15,10 +15,11 @@ from requests.exceptions import RequestException, ConnectionError, Timeout
 from .common import DownloaderBase
 from .. import text
 
+from ssl import SSLError
 try:
-    from OpenSSL.SSL import Error as SSLError
+    from OpenSSL.SSL import Error as OpenSSLError
 except ImportError:
-    from ssl import SSLError
+    OpenSSLError = SSLError
 
 
 class HttpDownloader(DownloaderBase):
@@ -39,11 +40,13 @@ class HttpDownloader(DownloaderBase):
             self.retries = float("inf")
         if self.rate:
             rate = text.parse_bytes(self.rate)
-            if not rate:
+            if rate:
+                if rate < self.chunk_size:
+                    self.chunk_size = rate
+                self.rate = rate
+                self.receive = self._receive_rate
+            else:
                 self.log.warning("Invalid rate limit (%r)", self.rate)
-            elif rate < self.chunk_size:
-                self.chunk_size = rate
-            self.rate = rate
 
     def download(self, url, pathfmt):
         try:
@@ -77,12 +80,15 @@ class HttpDownloader(DownloaderBase):
                 time.sleep(min(2 ** (tries-1), 1800))
             tries += 1
 
+            headers = {}
             # check for .part file
             filesize = pathfmt.part_size()
             if filesize:
-                headers = {"Range": "bytes={}-".format(filesize)}
-            else:
-                headers = None
+                headers["Range"] = "bytes={}-".format(filesize)
+            # file-specific headers
+            extra = pathfmt.kwdict.get("_http_headers")
+            if extra:
+                headers.update(extra)
 
             # connect to (remote) source
             try:
@@ -93,7 +99,7 @@ class HttpDownloader(DownloaderBase):
                 msg = str(exc)
                 continue
             except Exception as exc:
-                self.log.warning("%s", exc)
+                self.log.warning(exc)
                 return False
 
             # check response
@@ -110,7 +116,7 @@ class HttpDownloader(DownloaderBase):
                 msg = "'{} {}' for '{}'".format(code, response.reason, url)
                 if code == 429 or 500 <= code < 600:  # Server Error
                     continue
-                self.log.warning("%s", msg)
+                self.log.warning(msg)
                 return False
             size = text.parse_int(size)
 
@@ -140,7 +146,7 @@ class HttpDownloader(DownloaderBase):
                 # download content
                 try:
                     self.receive(response, file)
-                except (RequestException, SSLError) as exc:
+                except (RequestException, SSLError, OpenSSLError) as exc:
                     msg = str(exc)
                     print()
                     continue
@@ -166,20 +172,26 @@ class HttpDownloader(DownloaderBase):
         return True
 
     def receive(self, response, file):
-        if self.rate:
-            total = 0            # total amount of bytes received
-            start = time.time()  # start time
+        for data in response.iter_content(self.chunk_size):
+            file.write(data)
+
+    def _receive_rate(self, response, file):
+        t1 = time.time()
+        rt = self.rate
 
         for data in response.iter_content(self.chunk_size):
             file.write(data)
 
-            if self.rate:
-                total += len(data)
-                expected = total / self.rate  # expected elapsed time
-                delta = time.time() - start   # actual elapsed time since start
-                if delta < expected:
-                    # sleep if less time passed than expected
-                    time.sleep(expected - delta)
+            t2 = time.time()           # current time
+            actual = t2 - t1           # actual elapsed time
+            expected = len(data) / rt  # expected elapsed time
+
+            if actual < expected:
+                # sleep if less time elapsed than expected
+                time.sleep(expected - actual)
+                t1 = time.time()
+            else:
+                t1 = t2
 
     def get_extension(self, response):
         mtype = response.headers.get("Content-Type", "image/jpeg")