New upstream version 1.31.1.upstream/1.31.1

4 files changed, 210 insertions, 116 deletions

diff --git a/gallery_dl/downloader/__init__.py b/gallery_dl/downloader/__init__.py
index e1b936e..79dc5cb 100644
--- a/gallery_dl/downloader/__init__.py
+++ b/gallery_dl/downloader/__init__.py
@@ -27,7 +27,7 @@ def find(scheme):
         scheme = "http"
     if scheme in modules:  # prevent unwanted imports
         try:
-            module = __import__(scheme, globals(), None, (), 1)
+            module = __import__(scheme, globals(), None, None, 1)
         except ImportError:
             pass
         else:
diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py
index 7cd8d10..66996f7 100644
--- a/gallery_dl/downloader/common.py
+++ b/gallery_dl/downloader/common.py
@@ -31,8 +31,15 @@ class DownloaderBase():
         self.partdir = self.config("part-directory")
 
         if self.partdir:
-            self.partdir = util.expand_path(self.partdir)
-            os.makedirs(self.partdir, exist_ok=True)
+            if isinstance(self.partdir, dict):
+                self.partdir = [
+                    (util.compile_filter(expr) if expr else util.true,
+                     util.expand_path(pdir))
+                    for expr, pdir in self.partdir.items()
+                ]
+            else:
+                self.partdir = util.expand_path(self.partdir)
+                os.makedirs(self.partdir, exist_ok=True)
 
         proxies = self.config("proxy", util.SENTINEL)
         if proxies is util.SENTINEL:
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 248bf70..703dcca 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -95,7 +95,7 @@ class HttpDownloader(DownloaderBase):
         except Exception as exc:
             if self.downloading:
                 output.stderr_write("\n")
-            self.log.debug("", exc_info=exc)
+            self.log.traceback(exc)
             raise
         finally:
             # remove file from incomplete downloads
@@ -230,6 +230,10 @@ class HttpDownloader(DownloaderBase):
             # check file size
             size = text.parse_int(size, None)
             if size is not None:
+                if not size:
+                    self.release_conn(response)
+                    self.log.warning("Empty file")
+                    return False
                 if self.minsize and size < self.minsize:
                     self.release_conn(response)
                     self.log.warning(
@@ -342,9 +346,15 @@ class HttpDownloader(DownloaderBase):
                     raise
 
                 # check file size
-                if size and fp.tell() < size:
-                    msg = f"file size mismatch ({fp.tell()} < {size})"
-                    output.stderr_write("\n")
+                if size and (fsize := fp.tell()) < size:
+                    if (segmented := kwdict.get("_http_segmented")) and \
+                            segmented is True or segmented == fsize:
+                        tries -= 1
+                        msg = "Resuming segmented download"
+                        output.stdout_write("\r")
+                    else:
+                        msg = f"file size mismatch ({fsize} < {size})"
+                        output.stderr_write("\n")
                     continue
 
             break
diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py
index a56a6be..e9b3294 100644
--- a/gallery_dl/downloader/ytdl.py
+++ b/gallery_dl/downloader/ytdl.py
@@ -22,9 +22,9 @@ class YoutubeDLDownloader(DownloaderBase):
         DownloaderBase.__init__(self, job)
 
         extractor = job.extractor
-        retries = self.config("retries", extractor._retries)
+        self.retries = self.config("retries", extractor._retries)
         self.ytdl_opts = {
-            "retries": retries+1 if retries >= 0 else float("inf"),
+            "retries": self.retries+1 if self.retries >= 0 else float("inf"),
             "socket_timeout": self.config("timeout", extractor._timeout),
             "nocheckcertificate": not self.config("verify", extractor._verify),
             "proxy": self.proxies.get("http") if self.proxies else None,
@@ -39,17 +39,25 @@ class YoutubeDLDownloader(DownloaderBase):
 
     def download(self, url, pathfmt):
         kwdict = pathfmt.kwdict
+        tries = 0
 
-        ytdl_instance = kwdict.pop("_ytdl_instance", None)
-        if not ytdl_instance:
+        if ytdl_instance := kwdict.pop("_ytdl_instance", None):
+            # 'ytdl' extractor
+            self._prepare(ytdl_instance)
+            info_dict = kwdict.pop("_ytdl_info_dict")
+        else:
+            # other extractors
             ytdl_instance = self.ytdl_instance
             if not ytdl_instance:
                 try:
                     module = ytdl.import_module(self.config("module"))
                 except (ImportError, SyntaxError) as exc:
-                    self.log.error("Cannot import module '%s'",
-                                   getattr(exc, "name", ""))
-                    self.log.debug("", exc_info=exc)
+                    if exc.__context__:
+                        self.log.error("Cannot import yt-dlp or youtube-dl")
+                    else:
+                        self.log.error("Cannot import module '%s'",
+                                       getattr(exc, "name", ""))
+                    self.log.traceback(exc)
                     self.download = lambda u, p: False
                     return False
 
@@ -63,6 +71,8 @@ class YoutubeDLDownloader(DownloaderBase):
                     module, self, self.ytdl_opts)
                 if self.outtmpl == "default":
                     self.outtmpl = module.DEFAULT_OUTTMPL
+                self._prepare(ytdl_instance)
+
             if self.forward_cookies:
                 self.log.debug("Forwarding cookies to %s",
                                ytdl_instance.__module__)
@@ -70,45 +80,150 @@ class YoutubeDLDownloader(DownloaderBase):
                 for cookie in self.session.cookies:
                     set_cookie(cookie)
 
-        if "__gdl_initialize" in ytdl_instance.params:
-            del ytdl_instance.params["__gdl_initialize"]
+            url = url[5:]
+            manifest = kwdict.get("_ytdl_manifest")
+            while True:
+                tries += 1
+                self.error = None
+                try:
+                    if manifest is None:
+                        info_dict = self._extract_url(
+                            ytdl_instance, url)
+                    else:
+                        info_dict = self._extract_manifest(
+                            ytdl_instance, url, kwdict)
+                except Exception as exc:
+                    self.log.traceback(exc)
+                    cls = exc.__class__
+                    if cls.__module__ == "builtins":
+                        tries = False
+                    msg = f"{cls.__name__}: {exc}"
+                else:
+                    if self.error is not None:
+                        msg = self.error
+                    elif not info_dict:
+                        msg = "Empty 'info_dict' data"
+                    else:
+                        break
+
+                if tries:
+                    self.log.error("%s (%s/%s)", msg, tries, self.retries+1)
+                else:
+                    self.log.error(msg)
+                    return False
+                if tries > self.retries:
+                    return False
 
-            if self.progress is not None:
-                ytdl_instance.add_progress_hook(self._progress_hook)
-            if rlf := ytdl_instance.params.pop("__gdl_ratelimit_func", False):
-                self.rate_dyn = rlf
+        if extra := kwdict.get("_ytdl_extra"):
+            info_dict.update(extra)
 
-        info_dict = kwdict.pop("_ytdl_info_dict", None)
-        if not info_dict:
-            url = url[5:]
+        while True:
+            tries += 1
+            self.error = None
             try:
-                if manifest := kwdict.pop("_ytdl_manifest", None):
-                    info_dict = self._extract_manifest(
-                        ytdl_instance, url, manifest,
-                        kwdict.pop("_ytdl_manifest_data", None),
-                        kwdict.pop("_ytdl_manifest_headers", None),
-                        kwdict.pop("_ytdl_manifest_cookies", None))
+                if "entries" in info_dict:
+                    success = self._download_playlist(
+                        ytdl_instance, pathfmt, info_dict)
                 else:
-                    info_dict = self._extract_info(ytdl_instance, url)
+                    success = self._download_video(
+                        ytdl_instance, pathfmt, info_dict)
             except Exception as exc:
-                self.log.debug("", exc_info=exc)
-                self.log.warning("%s: %s", exc.__class__.__name__, exc)
+                self.log.traceback(exc)
+                cls = exc.__class__
+                if cls.__module__ == "builtins":
+                    tries = False
+                msg = f"{cls.__name__}: {exc}"
+            else:
+                if self.error is not None:
+                    msg = self.error
+                elif not success:
+                    msg = "Error"
+                else:
+                    break
 
-            if not info_dict:
+            if tries:
+                self.log.error("%s (%s/%s)", msg, tries, self.retries+1)
+            else:
+                self.log.error(msg)
                 return False
+            if tries > self.retries:
+                return False
+        return True
+
+    def _extract_url(self, ytdl, url):
+        return ytdl.extract_info(url, download=False)
+
+    def _extract_manifest(self, ytdl, url, kwdict):
+        extr = ytdl.get_info_extractor("Generic")
+        video_id = extr._generic_id(url)
+
+        if cookies := kwdict.get("_ytdl_manifest_cookies"):
+            if isinstance(cookies, dict):
+                cookies = cookies.items()
+            set_cookie = ytdl.cookiejar.set_cookie
+            for name, value in cookies:
+                set_cookie(Cookie(
+                    0, name, value, None, False,
+                    "", False, False, "/", False,
+                    False, None, False, None, None, {},
+                ))
+
+        type = kwdict["_ytdl_manifest"]
+        data = kwdict.get("_ytdl_manifest_data")
+        headers = kwdict.get("_ytdl_manifest_headers")
+        if type == "hls":
+            if data is None:
+                try:
+                    fmts, subs = extr._extract_m3u8_formats_and_subtitles(
+                        url, video_id, "mp4", headers=headers)
+                except AttributeError:
+                    fmts = extr._extract_m3u8_formats(
+                        url, video_id, "mp4", headers=headers)
+                    subs = None
+            else:
+                try:
+                    fmts, subs = extr._parse_m3u8_formats_and_subtitles(
+                        data, url, "mp4", headers=headers)
+                except AttributeError:
+                    fmts = extr._parse_m3u8_formats(
+                        data, url, "mp4", headers=headers)
+                    subs = None
 
-        if "entries" in info_dict:
-            index = kwdict.get("_ytdl_index")
-            if index is None:
-                return self._download_playlist(
-                    ytdl_instance, pathfmt, info_dict)
+        elif type == "dash":
+            if data is None:
+                try:
+                    fmts, subs = extr._extract_mpd_formats_and_subtitles(
+                        url, video_id, headers=headers)
+                except AttributeError:
+                    fmts = extr._extract_mpd_formats(
+                        url, video_id, headers=headers)
+                    subs = None
             else:
-                info_dict = info_dict["entries"][index]
+                if isinstance(data, str):
+                    data = ElementTree.fromstring(data)
+                try:
+                    fmts, subs = extr._parse_mpd_formats_and_subtitles(
+                        data, mpd_id="dash")
+                except AttributeError:
+                    fmts = extr._parse_mpd_formats(
+                        data, mpd_id="dash")
+                    subs = None
 
-        if extra := kwdict.get("_ytdl_extra"):
-            info_dict.update(extra)
+        else:
+            raise ValueError(f"Unsupported manifest type '{type}'")
 
-        return self._download_video(ytdl_instance, pathfmt, info_dict)
+        if headers:
+            for fmt in fmts:
+                fmt["http_headers"] = headers
+
+        info_dict = {
+            "extractor": "",
+            "id"       : video_id,
+            "title"    : video_id,
+            "formats"  : fmts,
+            "subtitles": subs,
+        }
+        return ytdl.process_ie_result(info_dict, download=False)
 
     def _download_video(self, ytdl_instance, pathfmt, info_dict):
         if "url" in info_dict:
@@ -161,12 +276,7 @@ class YoutubeDLDownloader(DownloaderBase):
             path = pathfmt.realpath.replace("%", "%%")
 
         self._set_outtmpl(ytdl_instance, path)
-        try:
-            ytdl_instance.process_info(info_dict)
-        except Exception as exc:
-            self.log.debug("", exc_info=exc)
-            return False
-
+        ytdl_instance.process_info(info_dict)
         pathfmt.temppath = info_dict.get("filepath") or info_dict["_filename"]
         return True
 
@@ -188,78 +298,20 @@ class YoutubeDLDownloader(DownloaderBase):
                 ytdl_instance.process_info(entry)
                 status = True
             except Exception as exc:
-                self.log.debug("", exc_info=exc)
+                self.log.traceback(exc)
                 self.log.error("%s: %s", exc.__class__.__name__, exc)
         return status
 
-    def _extract_info(self, ytdl, url):
-        return ytdl.extract_info(url, download=False)
-
-    def _extract_manifest(self, ytdl, url, manifest_type, manifest_data=None,
-                          headers=None, cookies=None):
-        extr = ytdl.get_info_extractor("Generic")
-        video_id = extr._generic_id(url)
-
-        if cookies is not None:
-            if isinstance(cookies, dict):
-                cookies = cookies.items()
-            set_cookie = ytdl.cookiejar.set_cookie
-            for name, value in cookies:
-                set_cookie(Cookie(
-                    0, name, value, None, False,
-                    "", False, False, "/", False,
-                    False, None, False, None, None, {},
-                ))
+    def _prepare(self, ytdl_instance):
+        if "__gdl_initialize" not in ytdl_instance.params:
+            return
 
-        if manifest_type == "hls":
-            if manifest_data is None:
-                try:
-                    fmts, subs = extr._extract_m3u8_formats_and_subtitles(
-                        url, video_id, "mp4", headers=headers)
-                except AttributeError:
-                    fmts = extr._extract_m3u8_formats(
-                        url, video_id, "mp4", headers=headers)
-                    subs = None
-            else:
-                try:
-                    fmts, subs = extr._parse_m3u8_formats_and_subtitles(
-                        url, video_id, "mp4")
-                except AttributeError:
-                    fmts = extr._parse_m3u8_formats(url, video_id, "mp4")
-                    subs = None
-
-        elif manifest_type == "dash":
-            if manifest_data is None:
-                try:
-                    fmts, subs = extr._extract_mpd_formats_and_subtitles(
-                        url, video_id, headers=headers)
-                except AttributeError:
-                    fmts = extr._extract_mpd_formats(
-                        url, video_id, headers=headers)
-                    subs = None
-            else:
-                if isinstance(manifest_data, str):
-                    manifest_data = ElementTree.fromstring(manifest_data)
-                try:
-                    fmts, subs = extr._parse_mpd_formats_and_subtitles(
-                        manifest_data, mpd_id="dash")
-                except AttributeError:
-                    fmts = extr._parse_mpd_formats(
-                        manifest_data, mpd_id="dash")
-                    subs = None
-
-        else:
-            self.log.error("Unsupported manifest type '%s'", manifest_type)
-            return None
-
-        info_dict = {
-            "extractor": "",
-            "id"       : video_id,
-            "title"    : video_id,
-            "formats"  : fmts,
-            "subtitles": subs,
-        }
-        return ytdl.process_ie_result(info_dict, download=False)
+        del ytdl_instance.params["__gdl_initialize"]
+        if self.progress is not None:
+            ytdl_instance.add_progress_hook(self._progress_hook)
+        if rlf := ytdl_instance.params.pop("__gdl_ratelimit_func", False):
+            self.rate_dyn = rlf
+        ytdl_instance.params["logger"] = LoggerAdapter(self, ytdl_instance)
 
     def _progress_hook(self, info):
         if info["status"] == "downloading" and \
@@ -284,6 +336,31 @@ class YoutubeDLDownloader(DownloaderBase):
             ytdl_instance.params["outtmpl"] = {"default": outtmpl}
 
 
+class LoggerAdapter():
+    __slots__ = ("obj", "log")
+
+    def __init__(self, obj, ytdl_instance):
+        self.obj = obj
+        self.log = ytdl_instance.params.get("logger")
+
+    def debug(self, msg):
+        if self.log is not None:
+            if msg[0] == "[":
+                msg = msg[msg.find("]")+2:]
+            self.log.debug(msg)
+
+    def warning(self, msg):
+        if self.log is not None:
+            if "WARNING:" in msg:
+                msg = msg[msg.find(" ")+1:]
+            self.log.warning(msg)
+
+    def error(self, msg):
+        if "ERROR:" in msg:
+            msg = msg[msg.find(" ")+1:]
+        self.obj.error = msg
+
+
 def compatible_formats(formats):
     """Returns True if 'formats' are compatible for merge"""
     video_ext = formats[0].get("ext")