New upstream version 1.19.0.upstream/1.19.0

1 files changed, 32 insertions, 14 deletions

diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 78d8d34..d2efd3f 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -31,6 +31,7 @@ class HttpDownloader(DownloaderBase):
         self.downloading = False
 
         self.adjust_extension = self.config("adjust-extensions", True)
+        self.progress = self.config("progress", 3.0)
         self.headers = self.config("headers")
         self.minsize = self.config("filesize-min")
         self.maxsize = self.config("filesize-max")
@@ -63,6 +64,8 @@ class HttpDownloader(DownloaderBase):
                 self.receive = self._receive_rate
             else:
                 self.log.warning("Invalid rate limit (%r)", self.rate)
+        if self.progress is not None:
+            self.receive = self._receive_rate
 
     def download(self, url, pathfmt):
         try:
@@ -202,6 +205,7 @@ class HttpDownloader(DownloaderBase):
             with pathfmt.open(mode) as fp:
                 if file_header:
                     fp.write(file_header)
+                    offset += len(file_header)
                 elif offset:
                     if adjust_extension and \
                             pathfmt.extension in FILE_SIGNATURES:
@@ -210,7 +214,7 @@ class HttpDownloader(DownloaderBase):
 
                 self.out.start(pathfmt.path)
                 try:
-                    self.receive(fp, content)
+                    self.receive(fp, content, size, offset)
                 except (RequestException, SSLError, OpenSSLError) as exc:
                     msg = str(exc)
                     print()
@@ -234,28 +238,42 @@ class HttpDownloader(DownloaderBase):
         return True
 
     @staticmethod
-    def receive(fp, content):
+    def receive(fp, content, bytes_total, bytes_downloaded):
         write = fp.write
         for data in content:
             write(data)
 
-    def _receive_rate(self, fp, content):
-        rt = self.rate
-        t1 = time.time()
+    def _receive_rate(self, fp, content, bytes_total, bytes_downloaded):
+        rate = self.rate
+        progress = self.progress
+        bytes_start = bytes_downloaded
+        write = fp.write
+        t1 = tstart = time.time()
 
         for data in content:
-            fp.write(data)
+            write(data)
 
             t2 = time.time()           # current time
-            actual = t2 - t1           # actual elapsed time
-            expected = len(data) / rt  # expected elapsed time
+            elapsed = t2 - t1          # elapsed time
+            num_bytes = len(data)
+
+            if progress is not None:
+                bytes_downloaded += num_bytes
+                tdiff = t2 - tstart
+                if tdiff >= progress:
+                    self.out.progress(
+                        bytes_total, bytes_downloaded,
+                        int((bytes_downloaded - bytes_start) / tdiff),
+                    )
 
-            if actual < expected:
-                # sleep if less time elapsed than expected
-                time.sleep(expected - actual)
-                t1 = time.time()
-            else:
-                t1 = t2
+            if rate:
+                expected = num_bytes / rate  # expected elapsed time
+                if elapsed < expected:
+                    # sleep if less time elapsed than expected
+                    time.sleep(expected - elapsed)
+                    t2 = time.time()
+
+            t1 = t2
 
     def _find_extension(self, response):
         """Get filename extension from MIME type"""