New upstream version 1.29.3.upstream/1.29.3

1 files changed, 76 insertions, 8 deletions

diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index 59b2d6d..3a862c1 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -687,10 +687,18 @@ x2="45.4107524%" y2="71.4898596%" id="app-root-3">\
             for folder in folders:
                 if match(folder["name"]):
                     return folder
+                elif folder["has_subfolders"]:
+                    for subfolder in folder["subfolders"]:
+                        if match(subfolder["name"]):
+                            return subfolder
         else:
             for folder in folders:
                 if folder["folderid"] == uuid:
                     return folder
+                elif folder["has_subfolders"]:
+                    for subfolder in folder["subfolders"]:
+                        if subfolder["folderid"] == uuid:
+                            return subfolder
         raise exception.NotFoundError("folder")
 
     def _folder_urls(self, folders, category, extractor):
@@ -891,7 +899,8 @@ class DeviantartGalleryExtractor(DeviantartExtractor):
     """Extractor for all deviations from an artist's gallery"""
     subcategory = "gallery"
     archive_fmt = "g_{_username}_{index}.{extension}"
-    pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$"
+    pattern = (BASE_PATTERN + r"/gallery"
+               r"(?:/all|/recommended-for-you|/?\?catpath=)?/?$")
     example = "https://www.deviantart.com/USER/gallery/"
 
     def deviations(self):
@@ -987,13 +996,36 @@ class DeviantartFolderExtractor(DeviantartExtractor):
     def deviations(self):
         folders = self.api.gallery_folders(self.user)
         folder = self._find_folder(folders, self.folder_name, self.folder_id)
+
+        # Leaving this here for backwards compatibility
         self.folder = {
             "title": folder["name"],
             "uuid" : folder["folderid"],
             "index": self.folder_id,
             "owner": self.user,
+            "parent_uuid": folder["parent"],
         }
-        return self.api.gallery(self.user, folder["folderid"], self.offset)
+
+        if folder.get("subfolder"):
+            self.folder["parent_folder"] = folder["parent_folder"]
+            self.archive_fmt = "F_{folder[parent_uuid]}_{index}.{extension}"
+
+            if self.flat:
+                self.directory_fmt = ("{category}", "{username}",
+                                      "{folder[parent_folder]}")
+            else:
+                self.directory_fmt = ("{category}", "{username}",
+                                      "{folder[parent_folder]}",
+                                      "{folder[title]}")
+
+        if folder.get("has_subfolders") and self.config("subfolders", True):
+            for subfolder in folder["subfolders"]:
+                subfolder["parent_folder"] = folder["name"]
+                subfolder["subfolder"] = True
+            yield from self._folder_urls(
+                folder["subfolders"], "gallery", DeviantartFolderExtractor)
+
+        yield from self.api.gallery(self.user, folder["folderid"], self.offset)
 
     def prepare(self, deviation):
         DeviantartExtractor.prepare(self, deviation)
@@ -1004,7 +1036,7 @@ class DeviantartStashExtractor(DeviantartExtractor):
     """Extractor for sta.sh-ed deviations"""
     subcategory = "stash"
     archive_fmt = "{index}.{extension}"
-    pattern = (r"(?:https?://)?(?:(?:www\.)?deviantart\.com/stash|sta\.sh)"
+    pattern = (r"(?:https?://)?(?:(?:www\.)?deviantart\.com/stash|sta\.s(h))"
                r"/([a-z0-9]+)")
     example = "https://www.deviantart.com/stash/abcde"
 
@@ -1016,9 +1048,18 @@ class DeviantartStashExtractor(DeviantartExtractor):
 
     def deviations(self, stash_id=None):
         if stash_id is None:
-            stash_id = self.groups[0]
-        url = "https://www.deviantart.com/stash/" + stash_id
-        page = self._limited_request(url).text
+            legacy_url, stash_id = self.groups
+        else:
+            legacy_url = False
+
+        if legacy_url and stash_id[0] == "2":
+            url = "https://sta.sh/" + stash_id
+            response = self._limited_request(url)
+            stash_id = response.url.rpartition("/")[2]
+            page = response.text
+        else:
+            url = "https://www.deviantart.com/stash/" + stash_id
+            page = self._limited_request(url).text
 
         if stash_id[0] == "0":
             uuid = text.extr(page, '//deviation/', '"')
@@ -1235,7 +1276,34 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
 
         deviation = self.api.deviation(uuid)
         deviation["_page"] = page
-        return (deviation,)
+
+        _dev_info = text.extr(
+            page, '\\"deviationExtended\\":', ',\\"deviation\\":', None)
+        # Clean up escaped quotes
+        _json_str = re.sub(
+            r'(?<!\\)\\{1}"', '"', _dev_info).replace("\\'", "'")
+        _extended_info = util.json_loads(_json_str)[self.deviation_id]
+        additional_media = _extended_info.get("additionalMedia") or ()
+
+        if additional_media:
+            self.filename_fmt = ("{category}_{index}_{index_file}_{title}_"
+                                 "{num:>02}.{extension}")
+            self.archive_fmt = ("g_{_username}_{index}{index_file:?_//}."
+                                "{extension}")
+
+        deviation["index_file"] = 0
+        deviation["count"] = 1 + len(additional_media)
+        deviation["num"] = 1
+        yield deviation
+
+        for index, post in enumerate(additional_media):
+            uri = post["media"]["baseUri"].encode().decode("unicode-escape")
+            deviation["content"]["src"] = uri
+            deviation["num"] += 1
+            deviation["index_file"] = post["fileId"]
+            # Download only works on purchased materials - no way to check
+            deviation["is_downloadable"] = False
+            yield deviation
 
 
 class DeviantartScrapsExtractor(DeviantartExtractor):
@@ -1366,7 +1434,7 @@ class DeviantartOAuthAPI():
     def __init__(self, extractor):
         self.extractor = extractor
         self.log = extractor.log
-        self.headers = {"dA-minor-version": "20200519"}
+        self.headers = {"dA-minor-version": "20210526"}
         self._warn_429 = True
 
         self.delay = extractor.config("wait-min", 0)