diff options
Diffstat (limited to 'gallery_dl/downloader')
| -rw-r--r-- | gallery_dl/downloader/http.py | 46 | ||||
| -rw-r--r-- | gallery_dl/downloader/ytdl.py | 18 |
2 files changed, 49 insertions, 15 deletions
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 78d8d34..d2efd3f 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -31,6 +31,7 @@ class HttpDownloader(DownloaderBase): self.downloading = False self.adjust_extension = self.config("adjust-extensions", True) + self.progress = self.config("progress", 3.0) self.headers = self.config("headers") self.minsize = self.config("filesize-min") self.maxsize = self.config("filesize-max") @@ -63,6 +64,8 @@ class HttpDownloader(DownloaderBase): self.receive = self._receive_rate else: self.log.warning("Invalid rate limit (%r)", self.rate) + if self.progress is not None: + self.receive = self._receive_rate def download(self, url, pathfmt): try: @@ -202,6 +205,7 @@ class HttpDownloader(DownloaderBase): with pathfmt.open(mode) as fp: if file_header: fp.write(file_header) + offset += len(file_header) elif offset: if adjust_extension and \ pathfmt.extension in FILE_SIGNATURES: @@ -210,7 +214,7 @@ class HttpDownloader(DownloaderBase): self.out.start(pathfmt.path) try: - self.receive(fp, content) + self.receive(fp, content, size, offset) except (RequestException, SSLError, OpenSSLError) as exc: msg = str(exc) print() @@ -234,28 +238,42 @@ class HttpDownloader(DownloaderBase): return True @staticmethod - def receive(fp, content): + def receive(fp, content, bytes_total, bytes_downloaded): write = fp.write for data in content: write(data) - def _receive_rate(self, fp, content): - rt = self.rate - t1 = time.time() + def _receive_rate(self, fp, content, bytes_total, bytes_downloaded): + rate = self.rate + progress = self.progress + bytes_start = bytes_downloaded + write = fp.write + t1 = tstart = time.time() for data in content: - fp.write(data) + write(data) t2 = time.time() # current time - actual = t2 - t1 # actual elapsed time - expected = len(data) / rt # expected elapsed time + elapsed = t2 - t1 # elapsed time + num_bytes = len(data) + + if progress is not None: + bytes_downloaded += num_bytes + tdiff = t2 - tstart + if tdiff >= progress: + self.out.progress( + bytes_total, bytes_downloaded, + int((bytes_downloaded - bytes_start) / tdiff), + ) - if actual < expected: - # sleep if less time elapsed than expected - time.sleep(expected - actual) - t1 = time.time() - else: - t1 = t2 + if rate: + expected = num_bytes / rate # expected elapsed time + if elapsed < expected: + # sleep if less time elapsed than expected + time.sleep(expected - elapsed) + t2 = time.time() + + t1 = t2 def _find_extension(self, response): """Get filename extension from MIME type""" diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py index b1e1d58..86e247b 100644 --- a/gallery_dl/downloader/ytdl.py +++ b/gallery_dl/downloader/ytdl.py @@ -42,6 +42,10 @@ class YoutubeDLDownloader(DownloaderBase): if raw_options: options.update(raw_options) + self.progress = self.config("progress", 3.0) + if self.progress is not None: + options["progress_hooks"] = (self._progress_hook,) + if self.config("logging", True): options["logger"] = self.log self.forward_cookies = self.config("forward-cookies", False) @@ -56,7 +60,10 @@ class YoutubeDLDownloader(DownloaderBase): kwdict = pathfmt.kwdict ytdl = kwdict.pop("_ytdl_instance", None) - if not ytdl: + if ytdl: + if self.progress is not None and not ytdl._progress_hooks: + ytdl.add_progress_hook(self._progress_hook) + else: ytdl = self.ytdl if self.forward_cookies: set_cookie = ytdl.cookiejar.set_cookie @@ -126,6 +133,15 @@ class YoutubeDLDownloader(DownloaderBase): ytdl.process_info(entry) return True + def _progress_hook(self, info): + if info["status"] == "downloading" and \ + info["elapsed"] >= self.progress: + self.out.progress( + info["total_bytes"], + info["downloaded_bytes"], + int(info["speed"]), + ) + @staticmethod def _set_outtmpl(ytdl, outtmpl): try: |