New upstream version 1.19.0.upstream/1.19.0

2 files changed, 49 insertions, 15 deletions

diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 78d8d34..d2efd3f 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -31,6 +31,7 @@ class HttpDownloader(DownloaderBase):
         self.downloading = False
 
         self.adjust_extension = self.config("adjust-extensions", True)
+        self.progress = self.config("progress", 3.0)
         self.headers = self.config("headers")
         self.minsize = self.config("filesize-min")
         self.maxsize = self.config("filesize-max")
@@ -63,6 +64,8 @@ class HttpDownloader(DownloaderBase):
                 self.receive = self._receive_rate
             else:
                 self.log.warning("Invalid rate limit (%r)", self.rate)
+        if self.progress is not None:
+            self.receive = self._receive_rate
 
     def download(self, url, pathfmt):
         try:
@@ -202,6 +205,7 @@ class HttpDownloader(DownloaderBase):
             with pathfmt.open(mode) as fp:
                 if file_header:
                     fp.write(file_header)
+                    offset += len(file_header)
                 elif offset:
                     if adjust_extension and \
                             pathfmt.extension in FILE_SIGNATURES:
@@ -210,7 +214,7 @@ class HttpDownloader(DownloaderBase):
 
                 self.out.start(pathfmt.path)
                 try:
-                    self.receive(fp, content)
+                    self.receive(fp, content, size, offset)
                 except (RequestException, SSLError, OpenSSLError) as exc:
                     msg = str(exc)
                     print()
@@ -234,28 +238,42 @@ class HttpDownloader(DownloaderBase):
         return True
 
     @staticmethod
-    def receive(fp, content):
+    def receive(fp, content, bytes_total, bytes_downloaded):
         write = fp.write
         for data in content:
             write(data)
 
-    def _receive_rate(self, fp, content):
-        rt = self.rate
-        t1 = time.time()
+    def _receive_rate(self, fp, content, bytes_total, bytes_downloaded):
+        rate = self.rate
+        progress = self.progress
+        bytes_start = bytes_downloaded
+        write = fp.write
+        t1 = tstart = time.time()
 
         for data in content:
-            fp.write(data)
+            write(data)
 
             t2 = time.time()           # current time
-            actual = t2 - t1           # actual elapsed time
-            expected = len(data) / rt  # expected elapsed time
+            elapsed = t2 - t1          # elapsed time
+            num_bytes = len(data)
+
+            if progress is not None:
+                bytes_downloaded += num_bytes
+                tdiff = t2 - tstart
+                if tdiff >= progress:
+                    self.out.progress(
+                        bytes_total, bytes_downloaded,
+                        int((bytes_downloaded - bytes_start) / tdiff),
+                    )
 
-            if actual < expected:
-                # sleep if less time elapsed than expected
-                time.sleep(expected - actual)
-                t1 = time.time()
-            else:
-                t1 = t2
+            if rate:
+                expected = num_bytes / rate  # expected elapsed time
+                if elapsed < expected:
+                    # sleep if less time elapsed than expected
+                    time.sleep(expected - elapsed)
+                    t2 = time.time()
+
+            t1 = t2
 
     def _find_extension(self, response):
         """Get filename extension from MIME type"""
diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py
index b1e1d58..86e247b 100644
--- a/gallery_dl/downloader/ytdl.py
+++ b/gallery_dl/downloader/ytdl.py
@@ -42,6 +42,10 @@ class YoutubeDLDownloader(DownloaderBase):
         if raw_options:
             options.update(raw_options)
 
+        self.progress = self.config("progress", 3.0)
+        if self.progress is not None:
+            options["progress_hooks"] = (self._progress_hook,)
+
         if self.config("logging", True):
             options["logger"] = self.log
         self.forward_cookies = self.config("forward-cookies", False)
@@ -56,7 +60,10 @@ class YoutubeDLDownloader(DownloaderBase):
         kwdict = pathfmt.kwdict
 
         ytdl = kwdict.pop("_ytdl_instance", None)
-        if not ytdl:
+        if ytdl:
+            if self.progress is not None and not ytdl._progress_hooks:
+                ytdl.add_progress_hook(self._progress_hook)
+        else:
             ytdl = self.ytdl
             if self.forward_cookies:
                 set_cookie = ytdl.cookiejar.set_cookie
@@ -126,6 +133,15 @@ class YoutubeDLDownloader(DownloaderBase):
             ytdl.process_info(entry)
         return True
 
+    def _progress_hook(self, info):
+        if info["status"] == "downloading" and \
+                info["elapsed"] >= self.progress:
+            self.out.progress(
+                info["total_bytes"],
+                info["downloaded_bytes"],
+                int(info["speed"]),
+            )
+
     @staticmethod
     def _set_outtmpl(ytdl, outtmpl):
         try: