diff options
Diffstat (limited to 'gallery_dl/downloader/http.py')
| -rw-r--r-- | gallery_dl/downloader/http.py | 56 |
1 files changed, 34 insertions, 22 deletions
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 1c78cfb..fab96ba 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -15,10 +15,11 @@ from requests.exceptions import RequestException, ConnectionError, Timeout from .common import DownloaderBase from .. import text +from ssl import SSLError try: - from OpenSSL.SSL import Error as SSLError + from OpenSSL.SSL import Error as OpenSSLError except ImportError: - from ssl import SSLError + OpenSSLError = SSLError class HttpDownloader(DownloaderBase): @@ -39,11 +40,13 @@ class HttpDownloader(DownloaderBase): self.retries = float("inf") if self.rate: rate = text.parse_bytes(self.rate) - if not rate: + if rate: + if rate < self.chunk_size: + self.chunk_size = rate + self.rate = rate + self.receive = self._receive_rate + else: self.log.warning("Invalid rate limit (%r)", self.rate) - elif rate < self.chunk_size: - self.chunk_size = rate - self.rate = rate def download(self, url, pathfmt): try: @@ -77,12 +80,15 @@ class HttpDownloader(DownloaderBase): time.sleep(min(2 ** (tries-1), 1800)) tries += 1 + headers = {} # check for .part file filesize = pathfmt.part_size() if filesize: - headers = {"Range": "bytes={}-".format(filesize)} - else: - headers = None + headers["Range"] = "bytes={}-".format(filesize) + # file-specific headers + extra = pathfmt.kwdict.get("_http_headers") + if extra: + headers.update(extra) # connect to (remote) source try: @@ -93,7 +99,7 @@ class HttpDownloader(DownloaderBase): msg = str(exc) continue except Exception as exc: - self.log.warning("%s", exc) + self.log.warning(exc) return False # check response @@ -110,7 +116,7 @@ class HttpDownloader(DownloaderBase): msg = "'{} {}' for '{}'".format(code, response.reason, url) if code == 429 or 500 <= code < 600: # Server Error continue - self.log.warning("%s", msg) + self.log.warning(msg) return False size = text.parse_int(size) @@ -140,7 +146,7 @@ class HttpDownloader(DownloaderBase): # download content try: self.receive(response, file) - except (RequestException, SSLError) as exc: + except (RequestException, SSLError, OpenSSLError) as exc: msg = str(exc) print() continue @@ -166,20 +172,26 @@ class HttpDownloader(DownloaderBase): return True def receive(self, response, file): - if self.rate: - total = 0 # total amount of bytes received - start = time.time() # start time + for data in response.iter_content(self.chunk_size): + file.write(data) + + def _receive_rate(self, response, file): + t1 = time.time() + rt = self.rate for data in response.iter_content(self.chunk_size): file.write(data) - if self.rate: - total += len(data) - expected = total / self.rate # expected elapsed time - delta = time.time() - start # actual elapsed time since start - if delta < expected: - # sleep if less time passed than expected - time.sleep(expected - delta) + t2 = time.time() # current time + actual = t2 - t1 # actual elapsed time + expected = len(data) / rt # expected elapsed time + + if actual < expected: + # sleep if less time elapsed than expected + time.sleep(expected - actual) + t1 = time.time() + else: + t1 = t2 def get_extension(self, response): mtype = response.headers.get("Content-Type", "image/jpeg") |