Update upstream source from tag 'upstream/1.29.7'

Update to upstream version '1.29.7'
with Debian dir 264267cd1ebd5c7205fe1f137a394d0ae1a2fb3b

1 files changed, 78 insertions, 28 deletions

diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py
index de8f86c..56fe851 100644
--- a/gallery_dl/extractor/civitai.py
+++ b/gallery_dl/extractor/civitai.py
@@ -45,6 +45,20 @@ class CivitaiExtractor(Extractor):
             self._image_quality = "original=true"
             self._image_ext = "png"
 
+        quality_video = self.config("quality-videos")
+        if quality_video:
+            if not isinstance(quality_video, str):
+                quality_video = ",".join(quality_video)
+            if quality_video[0] == "+":
+                quality_video = (self._image_quality + "," +
+                                 quality_video.lstrip("+,"))
+            self._video_quality = quality_video
+        elif quality_video is not None and quality:
+            self._video_quality = self._image_quality
+        else:
+            self._video_quality = "quality=100"
+        self._video_ext = "webm"
+
         metadata = self.config("metadata")
         if metadata:
             if isinstance(metadata, str):
@@ -82,9 +96,8 @@ class CivitaiExtractor(Extractor):
                     "user": post.pop("user"),
                 }
                 if self._meta_version:
-                    data["version"] = version = self.api.model_version(
-                        post["modelVersionId"]).copy()
-                    data["model"] = version.pop("model")
+                    data["model"], data["version"] = \
+                        self._extract_meta_version(post)
 
                 yield Message.Directory, data
                 for file in self._image_results(images):
@@ -95,26 +108,22 @@ class CivitaiExtractor(Extractor):
         images = self.images()
         if images:
             for image in images:
-                url = self._url(image)
+
                 if self._meta_generation:
-                    image["generation"] = self.api.image_generationdata(
-                        image["id"])
+                    image["generation"] = \
+                        self._extract_meta_generation(image)
                 if self._meta_version:
-                    if "modelVersionId" in image:
-                        version_id = image["modelVersionId"]
-                    else:
-                        post = image["post"] = self.api.post(
-                            image["postId"])
-                        post.pop("user", None)
-                        version_id = post["modelVersionId"]
-                    image["version"] = version = self.api.model_version(
-                        version_id).copy()
-                    image["model"] = version.pop("model")
-
+                    image["model"], image["version"] = \
+                        self._extract_meta_version(image, False)
                 image["date"] = text.parse_datetime(
                     image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
+
+                url = self._url(image)
                 text.nameext_from_url(url, image)
-                image["extension"] = self._image_ext
+                if not image["extension"]:
+                    image["extension"] = (
+                        self._video_ext if image.get("type") == "video" else
+                        self._image_ext)
                 yield Message.Directory, image
                 yield Message.Url, url, image
             return
@@ -130,20 +139,23 @@ class CivitaiExtractor(Extractor):
 
     def _url(self, image):
         url = image["url"]
+        video = image.get("type") == "video"
+        quality = self._video_quality if video else self._image_quality
+
         if "/" in url:
             parts = url.rsplit("/", 3)
             image["uuid"] = parts[1]
-            parts[2] = self._image_quality
+            parts[2] = quality
             return "/".join(parts)
-        image["uuid"] = url
 
+        image["uuid"] = url
         name = image.get("name")
         if not name:
             mime = image.get("mimeType") or self._image_ext
             name = "{}.{}".format(image.get("id"), mime.rpartition("/")[2])
         return (
             "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/{}/{}/{}".format(
-                url, self._image_quality, name)
+                url, quality, name)
         )
 
     def _image_results(self, images):
@@ -154,11 +166,13 @@ class CivitaiExtractor(Extractor):
                 "url" : self._url(file),
             })
             if not data["extension"]:
-                data["extension"] = self._image_ext
+                data["extension"] = (
+                    self._video_ext if file.get("type") == "video" else
+                    self._image_ext)
             if "id" not in file and data["filename"].isdecimal():
                 file["id"] = text.parse_int(data["filename"])
             if self._meta_generation:
-                file["generation"] = self.api.image_generationdata(file["id"])
+                file["generation"] = self._extract_meta_generation(file)
             yield data
 
     def _parse_query(self, value):
@@ -166,6 +180,38 @@ class CivitaiExtractor(Extractor):
             value, {"tags", "reactions", "baseModels", "tools", "techniques",
                     "types", "fileFormats"})
 
+    def _extract_meta_generation(self, image):
+        try:
+            return self.api.image_generationdata(image["id"])
+        except Exception as exc:
+            return self.log.debug("", exc_info=exc)
+
+    def _extract_meta_version(self, item, is_post=True):
+        try:
+            version_id = self._extract_version_id(item, is_post)
+            if version_id:
+                version = self.api.model_version(version_id).copy()
+                return version.pop("model", None), version
+        except Exception as exc:
+            self.log.debug("", exc_info=exc)
+        return None, None
+
+    def _extract_version_id(self, item, is_post=True):
+        version_id = item.get("modelVersionId")
+        if version_id:
+            return version_id
+
+        version_ids = item.get("modelVersionIds")
+        if version_ids:
+            return version_ids[0]
+
+        if is_post:
+            return None
+
+        item["post"] = post = self.api.post(item["postId"])
+        post.pop("user", None)
+        return self._extract_version_id(post)
+
 
 class CivitaiModelExtractor(CivitaiExtractor):
     subcategory = "model"
@@ -235,16 +281,20 @@ class CivitaiModelExtractor(CivitaiExtractor):
         files = []
 
         for num, file in enumerate(version["files"], 1):
+            name, sep, ext = file["name"].rpartition(".")
+            if not sep:
+                name = ext
+                ext = "bin"
             file["uuid"] = "model-{}-{}-{}".format(
                 model["id"], version["id"], file["id"])
             files.append({
                 "num"      : num,
                 "file"     : file,
-                "filename" : file["name"],
-                "extension": "bin",
-                "url"      : file.get("downloadUrl") or
-                             "{}/api/download/models/{}".format(
-                                 self.root, version["id"]),
+                "filename" : name,
+                "extension": ext,
+                "url"      : (file.get("downloadUrl") or
+                              "{}/api/download/models/{}".format(
+                              self.root, version["id"])),
                 "_http_headers" : {
                     "Authorization": self.api.headers.get("Authorization")},
                 "_http_validate": self._validate_file_model,