New upstream version 1.10.6upstream/1.10.6

1 files changed, 72 insertions, 42 deletions

diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index 525cc84..ab32a00 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -47,12 +47,6 @@ class DeviantartExtractor(Extractor):
         if self.quality:
             self.quality = "q_{}".format(self.quality)
 
-        if self.original != "image":
-            self._update_content = self._update_content_default
-        else:
-            self._update_content = self._update_content_image
-            self.original = True
-
         self.commit_journal = {
             "html": self._commit_journal_html,
             "text": self._commit_journal_text,
@@ -98,7 +92,8 @@ class DeviantartExtractor(Extractor):
                 yield self.commit(deviation, content)
 
             elif deviation["is_downloadable"]:
-                content = self.api.deviation_download(deviation["deviationid"])
+                content = {}
+                self._update_content(deviation, content)
                 yield self.commit(deviation, content)
 
             if "videos" in deviation:
@@ -240,15 +235,29 @@ class DeviantartExtractor(Extractor):
         url = "{}/{}/{}/0/".format(self.root, self.user, category)
         return [(url + folder["name"], folder) for folder in folders]
 
-    def _update_content_default(self, deviation, content):
-        content.update(self.api.deviation_download(deviation["deviationid"]))
-
-    def _update_content_image(self, deviation, content):
-        data = self.api.deviation_download(deviation["deviationid"])
-        url = data["src"].partition("?")[0]
-        mtype = mimetypes.guess_type(url, False)[0]
-        if mtype and mtype.startswith("image/"):
-            content.update(data)
+    def _update_content(self, deviation, content):
+        try:
+            data = self.api.deviation_extended_fetch(
+                deviation["index"],
+                deviation["author"]["username"],
+                "journal" if "excerpt" in deviation else "art",
+            )
+            download = data["deviation"]["extended"]["download"]
+            download["src"] = download["url"]
+        except Exception as e:
+            self.log.warning(
+                "Unable to fetch original download URL for ID %s ('%s: %s')",
+                deviation["index"], e.__class__.__name__, e,
+            )
+            self.log.debug("Server response: %s", data)
+        else:
+            if self.original == "image":
+                url = data["src"].partition("?")[0]
+                mtype = mimetypes.guess_type(url, False)[0]
+                if not mtype or not mtype.startswith("image/"):
+                    return
+            del download["url"]
+            content.update(download)
 
 
 class DeviantartGalleryExtractor(DeviantartExtractor):
@@ -258,8 +267,8 @@ class DeviantartGalleryExtractor(DeviantartExtractor):
     pattern = BASE_PATTERN + r"(?:/(?:gallery/?(?:\?catpath=/)?)?)?$"
     test = (
         ("https://www.deviantart.com/shimoda7/gallery/", {
-            "pattern": r"https://(s3.amazonaws.com/origin-(img|orig)"
-                       r".deviantart.net/|images-wixmp-\w+.wixmp.com/)",
+            "pattern": r"https://(www.deviantart.com/download/\d+/"
+                       r"|images-wixmp-[^.]+.wixmp.com/f/.+/.+.jpg\?token=.+)",
             "count": ">= 30",
             "keyword": {
                 "allows_comments": bool,
@@ -384,7 +393,7 @@ class DeviantartStashExtractor(DeviantartExtractor):
     pattern = r"(?:https?://)?sta\.sh/([a-z0-9]+)"
     test = (
         ("https://sta.sh/022c83odnaxc", {
-            "pattern": r"https://s3.amazonaws.com/origin-orig.deviantart.net",
+            "pattern": r"https://sta.sh/download/7549925030122512/.+\?token=",
             "count": 1,
         }),
         # multiple stash items
@@ -394,6 +403,7 @@ class DeviantartStashExtractor(DeviantartExtractor):
         }),
         # downloadable, but no "content" field (#307)
         ("https://sta.sh/024t4coz16mi", {
+            "pattern": r"https://sta.sh/download/7800709982190282/.+\?token=",
             "count": 1,
         }),
         ("https://sta.sh/abcdefghijkl", {
@@ -411,16 +421,34 @@ class DeviantartStashExtractor(DeviantartExtractor):
     def deviations(self):
         url = "https://sta.sh/" + self.stash_id
         page = self.request(url).text
-        deviation_id = text.extract(page, '//deviation/', '"')[0]
+        deviation_id, pos = text.extract(page, '//deviation/', '"')
 
         if deviation_id:
-            yield self.api.deviation(deviation_id)
+            deviation = self.api.deviation(deviation_id)
+            pos = page.find("dev-page-download", pos)
+            if pos >= 0:
+                deviation["_download"] = {
+                    "width" : text.parse_int(text.extract(
+                        page, 'data-download_width="' , '"', pos)[0]),
+                    "height": text.parse_int(text.extract(
+                        page, 'data-download_height="', '"', pos)[0]),
+                    "src"   : text.unescape(text.extract(
+                        page, 'data-download_url="'   , '"', pos)[0]),
+                }
+            return (deviation,)
         else:
             data = {"_extractor": DeviantartStashExtractor}
             page = text.extract(
-                page, '<div id="stash-body"', '<div class="footer"')[0]
-            for url in text.extract_iter(page, '<a href="', '"'):
-                yield url, data
+                page, 'id="stash-body"', 'class="footer"', pos)[0]
+            return [
+                (url, data)
+                for url in text.extract_iter(page, '<a href="', '"')
+            ]
+
+    def _update_content(self, deviation, content):
+        if "_download" in deviation:
+            content.update(deviation["_download"])
+            del deviation["_download"]
 
 
 class DeviantartFavoriteExtractor(DeviantartExtractor):
@@ -562,28 +590,17 @@ class DeviantartExtractorV2(DeviantartExtractor):
     """Base class for deviantart extractors using the NAPI"""
 
     def items(self):
-        url = (
-            self.root + "/_napi/da-browse/shared_api/deviation/extended_fetch"
-        )
-        params = {
-            "deviationid"    : None,
-            "username"       : None,
-            "type"           : None,
-            "include_session": "false",
-        }
-        headers = {
-            "Referer": self.root,
-        }
-
         yield Message.Version, 1
         for deviation in self.deviations():
-            params["deviationid"] = deviation["deviationId"]
-            params["username"] = deviation["author"]["username"]
-            params["type"] = "journal" if deviation["isJournal"] else "art"
-            data = self.request(url, params=params, headers=headers).json()
+            data = self.api.deviation_extended_fetch(
+                deviation["deviationId"],
+                deviation["author"]["username"],
+                "journal" if deviation["isJournal"] else "art",
+            )
 
             if "deviation" not in data:
-                self.log.warning("Skipping %s", params["deviationid"])
+                self.log.warning("Skipping ID %s", deviation["deviationId"])
+                self.log.debug("Server response: %s", data)
                 continue
             deviation = self._extract(data)
 
@@ -887,6 +904,19 @@ class DeviantartAPI():
         params = {"mature_content": self.mature}
         return self._call(endpoint, params)
 
+    def deviation_extended_fetch(self, deviation_id, user, kind):
+        url = ("https://www.deviantart.com/_napi/da-browse/shared_api"
+               "/deviation/extended_fetch")
+        headers = {"Referer": "https://www.deviantart.com/"}
+        params = {
+            "deviationid"    : deviation_id,
+            "username"       : user,
+            "type"           : kind,
+            "include_session": "false",
+        }
+        return self.extractor.request(
+            url, headers=headers, params=params, fatal=None).json()
+
     def deviation_metadata(self, deviations):
         """ Fetch deviation metadata for a set of deviations"""
         if not deviations: