diff options
Diffstat (limited to 'gallery_dl/downloader/http.py')
| -rw-r--r-- | gallery_dl/downloader/http.py | 46 |
1 files changed, 32 insertions, 14 deletions
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 78d8d34..d2efd3f 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -31,6 +31,7 @@ class HttpDownloader(DownloaderBase): self.downloading = False self.adjust_extension = self.config("adjust-extensions", True) + self.progress = self.config("progress", 3.0) self.headers = self.config("headers") self.minsize = self.config("filesize-min") self.maxsize = self.config("filesize-max") @@ -63,6 +64,8 @@ class HttpDownloader(DownloaderBase): self.receive = self._receive_rate else: self.log.warning("Invalid rate limit (%r)", self.rate) + if self.progress is not None: + self.receive = self._receive_rate def download(self, url, pathfmt): try: @@ -202,6 +205,7 @@ class HttpDownloader(DownloaderBase): with pathfmt.open(mode) as fp: if file_header: fp.write(file_header) + offset += len(file_header) elif offset: if adjust_extension and \ pathfmt.extension in FILE_SIGNATURES: @@ -210,7 +214,7 @@ class HttpDownloader(DownloaderBase): self.out.start(pathfmt.path) try: - self.receive(fp, content) + self.receive(fp, content, size, offset) except (RequestException, SSLError, OpenSSLError) as exc: msg = str(exc) print() @@ -234,28 +238,42 @@ class HttpDownloader(DownloaderBase): return True @staticmethod - def receive(fp, content): + def receive(fp, content, bytes_total, bytes_downloaded): write = fp.write for data in content: write(data) - def _receive_rate(self, fp, content): - rt = self.rate - t1 = time.time() + def _receive_rate(self, fp, content, bytes_total, bytes_downloaded): + rate = self.rate + progress = self.progress + bytes_start = bytes_downloaded + write = fp.write + t1 = tstart = time.time() for data in content: - fp.write(data) + write(data) t2 = time.time() # current time - actual = t2 - t1 # actual elapsed time - expected = len(data) / rt # expected elapsed time + elapsed = t2 - t1 # elapsed time + num_bytes = len(data) + + if progress is not None: + bytes_downloaded += num_bytes + tdiff = t2 - tstart + if tdiff >= progress: + self.out.progress( + bytes_total, bytes_downloaded, + int((bytes_downloaded - bytes_start) / tdiff), + ) - if actual < expected: - # sleep if less time elapsed than expected - time.sleep(expected - actual) - t1 = time.time() - else: - t1 = t2 + if rate: + expected = num_bytes / rate # expected elapsed time + if elapsed < expected: + # sleep if less time elapsed than expected + time.sleep(expected - elapsed) + t2 = time.time() + + t1 = t2 def _find_extension(self, response): """Get filename extension from MIME type""" |