diff options
Diffstat (limited to 'gallery_dl/extractor/deviantart.py')
| -rw-r--r-- | gallery_dl/extractor/deviantart.py | 114 |
1 files changed, 72 insertions, 42 deletions
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 525cc84..ab32a00 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -47,12 +47,6 @@ class DeviantartExtractor(Extractor): if self.quality: self.quality = "q_{}".format(self.quality) - if self.original != "image": - self._update_content = self._update_content_default - else: - self._update_content = self._update_content_image - self.original = True - self.commit_journal = { "html": self._commit_journal_html, "text": self._commit_journal_text, @@ -98,7 +92,8 @@ class DeviantartExtractor(Extractor): yield self.commit(deviation, content) elif deviation["is_downloadable"]: - content = self.api.deviation_download(deviation["deviationid"]) + content = {} + self._update_content(deviation, content) yield self.commit(deviation, content) if "videos" in deviation: @@ -240,15 +235,29 @@ class DeviantartExtractor(Extractor): url = "{}/{}/{}/0/".format(self.root, self.user, category) return [(url + folder["name"], folder) for folder in folders] - def _update_content_default(self, deviation, content): - content.update(self.api.deviation_download(deviation["deviationid"])) - - def _update_content_image(self, deviation, content): - data = self.api.deviation_download(deviation["deviationid"]) - url = data["src"].partition("?")[0] - mtype = mimetypes.guess_type(url, False)[0] - if mtype and mtype.startswith("image/"): - content.update(data) + def _update_content(self, deviation, content): + try: + data = self.api.deviation_extended_fetch( + deviation["index"], + deviation["author"]["username"], + "journal" if "excerpt" in deviation else "art", + ) + download = data["deviation"]["extended"]["download"] + download["src"] = download["url"] + except Exception as e: + self.log.warning( + "Unable to fetch original download URL for ID %s ('%s: %s')", + deviation["index"], e.__class__.__name__, e, + ) + self.log.debug("Server response: %s", data) + else: + if self.original == "image": + url = data["src"].partition("?")[0] + mtype = mimetypes.guess_type(url, False)[0] + if not mtype or not mtype.startswith("image/"): + return + del download["url"] + content.update(download) class DeviantartGalleryExtractor(DeviantartExtractor): @@ -258,8 +267,8 @@ class DeviantartGalleryExtractor(DeviantartExtractor): pattern = BASE_PATTERN + r"(?:/(?:gallery/?(?:\?catpath=/)?)?)?$" test = ( ("https://www.deviantart.com/shimoda7/gallery/", { - "pattern": r"https://(s3.amazonaws.com/origin-(img|orig)" - r".deviantart.net/|images-wixmp-\w+.wixmp.com/)", + "pattern": r"https://(www.deviantart.com/download/\d+/" + r"|images-wixmp-[^.]+.wixmp.com/f/.+/.+.jpg\?token=.+)", "count": ">= 30", "keyword": { "allows_comments": bool, @@ -384,7 +393,7 @@ class DeviantartStashExtractor(DeviantartExtractor): pattern = r"(?:https?://)?sta\.sh/([a-z0-9]+)" test = ( ("https://sta.sh/022c83odnaxc", { - "pattern": r"https://s3.amazonaws.com/origin-orig.deviantart.net", + "pattern": r"https://sta.sh/download/7549925030122512/.+\?token=", "count": 1, }), # multiple stash items @@ -394,6 +403,7 @@ class DeviantartStashExtractor(DeviantartExtractor): }), # downloadable, but no "content" field (#307) ("https://sta.sh/024t4coz16mi", { + "pattern": r"https://sta.sh/download/7800709982190282/.+\?token=", "count": 1, }), ("https://sta.sh/abcdefghijkl", { @@ -411,16 +421,34 @@ class DeviantartStashExtractor(DeviantartExtractor): def deviations(self): url = "https://sta.sh/" + self.stash_id page = self.request(url).text - deviation_id = text.extract(page, '//deviation/', '"')[0] + deviation_id, pos = text.extract(page, '//deviation/', '"') if deviation_id: - yield self.api.deviation(deviation_id) + deviation = self.api.deviation(deviation_id) + pos = page.find("dev-page-download", pos) + if pos >= 0: + deviation["_download"] = { + "width" : text.parse_int(text.extract( + page, 'data-download_width="' , '"', pos)[0]), + "height": text.parse_int(text.extract( + page, 'data-download_height="', '"', pos)[0]), + "src" : text.unescape(text.extract( + page, 'data-download_url="' , '"', pos)[0]), + } + return (deviation,) else: data = {"_extractor": DeviantartStashExtractor} page = text.extract( - page, '<div id="stash-body"', '<div class="footer"')[0] - for url in text.extract_iter(page, '<a href="', '"'): - yield url, data + page, 'id="stash-body"', 'class="footer"', pos)[0] + return [ + (url, data) + for url in text.extract_iter(page, '<a href="', '"') + ] + + def _update_content(self, deviation, content): + if "_download" in deviation: + content.update(deviation["_download"]) + del deviation["_download"] class DeviantartFavoriteExtractor(DeviantartExtractor): @@ -562,28 +590,17 @@ class DeviantartExtractorV2(DeviantartExtractor): """Base class for deviantart extractors using the NAPI""" def items(self): - url = ( - self.root + "/_napi/da-browse/shared_api/deviation/extended_fetch" - ) - params = { - "deviationid" : None, - "username" : None, - "type" : None, - "include_session": "false", - } - headers = { - "Referer": self.root, - } - yield Message.Version, 1 for deviation in self.deviations(): - params["deviationid"] = deviation["deviationId"] - params["username"] = deviation["author"]["username"] - params["type"] = "journal" if deviation["isJournal"] else "art" - data = self.request(url, params=params, headers=headers).json() + data = self.api.deviation_extended_fetch( + deviation["deviationId"], + deviation["author"]["username"], + "journal" if deviation["isJournal"] else "art", + ) if "deviation" not in data: - self.log.warning("Skipping %s", params["deviationid"]) + self.log.warning("Skipping ID %s", deviation["deviationId"]) + self.log.debug("Server response: %s", data) continue deviation = self._extract(data) @@ -887,6 +904,19 @@ class DeviantartAPI(): params = {"mature_content": self.mature} return self._call(endpoint, params) + def deviation_extended_fetch(self, deviation_id, user, kind): + url = ("https://www.deviantart.com/_napi/da-browse/shared_api" + "/deviation/extended_fetch") + headers = {"Referer": "https://www.deviantart.com/"} + params = { + "deviationid" : deviation_id, + "username" : user, + "type" : kind, + "include_session": "false", + } + return self.extractor.request( + url, headers=headers, params=params, fatal=None).json() + def deviation_metadata(self, deviations): """ Fetch deviation metadata for a set of deviations""" if not deviations: |