diff options
Diffstat (limited to 'gallery_dl/extractor/deviantart.py')
| -rw-r--r-- | gallery_dl/extractor/deviantart.py | 621 |
1 files changed, 168 insertions, 453 deletions
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 18d9867..9421096 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -32,20 +32,26 @@ class DeviantartExtractor(Extractor): root = "https://www.deviantart.com" directory_fmt = ("{category}", "{username}") filename_fmt = "{category}_{index}_{title}.{extension}" - cookiedomain = None - cookienames = ("auth", "auth_secure", "userinfo") + cookies_domain = None + cookies_names = ("auth", "auth_secure", "userinfo") _last_request = 0 def __init__(self, match): Extractor.__init__(self, match) + self.user = match.group(1) or match.group(2) + self.offset = 0 + + def _init(self): + self.jwt = self.config("jwt", True) self.flat = self.config("flat", True) self.extra = self.config("extra", False) + self.quality = self.config("quality", "100") self.original = self.config("original", True) self.comments = self.config("comments", False) - self.user = match.group(1) or match.group(2) + + self.api = DeviantartOAuthAPI(self) self.group = False - self.offset = 0 - self.api = None + self._premium_cache = {} unwatch = self.config("auto-unwatch") if unwatch: @@ -54,33 +60,37 @@ class DeviantartExtractor(Extractor): else: self.unwatch = None + if self.quality: + self.quality = ",q_{}".format(self.quality) + if self.original != "image": self._update_content = self._update_content_default else: self._update_content = self._update_content_image self.original = True - self._premium_cache = {} - self.commit_journal = { - "html": self._commit_journal_html, - "text": self._commit_journal_text, - }.get(self.config("journals", "html")) + journals = self.config("journals", "html") + if journals == "html": + self.commit_journal = self._commit_journal_html + elif journals == "text": + self.commit_journal = self._commit_journal_text + else: + self.commit_journal = None def skip(self, num): self.offset += num return num def login(self): - if not self._check_cookies(self.cookienames): - username, password = self._get_auth_info() - if not username: - return False - self._update_cookies(_login_impl(self, username, password)) - return True + if self.cookies_check(self.cookies_names): + return True - def items(self): - self.api = DeviantartOAuthAPI(self) + username, password = self._get_auth_info() + if username: + self.cookies_update(_login_impl(self, username, password)) + return True + def items(self): if self.user and self.config("group", True): profile = self.api.user_profile(self.user) self.group = not profile @@ -117,21 +127,36 @@ class DeviantartExtractor(Extractor): if self.original and deviation["is_downloadable"]: self._update_content(deviation, content) - else: + elif self.jwt: self._update_token(deviation, content) + elif content["src"].startswith("https://images-wixmp-"): + if deviation["index"] <= 790677560: + # https://github.com/r888888888/danbooru/issues/4069 + intermediary, count = re.subn( + r"(/f/[^/]+/[^/]+)/v\d+/.*", + r"/intermediary\1", content["src"], 1) + if count: + deviation["_fallback"] = (content["src"],) + content["src"] = intermediary + if self.quality: + content["src"] = re.sub( + r",q_\d+", self.quality, content["src"], 1) yield self.commit(deviation, content) elif deviation["is_downloadable"]: content = self.api.deviation_download(deviation["deviationid"]) + deviation["is_original"] = True yield self.commit(deviation, content) if "videos" in deviation and deviation["videos"]: video = max(deviation["videos"], key=lambda x: text.parse_int(x["quality"][:-1])) + deviation["is_original"] = False yield self.commit(deviation, video) if "flash" in deviation: + deviation["is_original"] = True yield self.commit(deviation, deviation["flash"]) if self.commit_journal: @@ -145,6 +170,7 @@ class DeviantartExtractor(Extractor): if journal: if self.extra: deviation["_journal"] = journal["html"] + deviation["is_original"] = True yield self.commit_journal(deviation, journal) if not self.extra: @@ -222,6 +248,8 @@ class DeviantartExtractor(Extractor): target["filename"] = deviation["filename"] deviation["target"] = target deviation["extension"] = target["extension"] = text.ext_from_url(name) + if "is_original" not in deviation: + deviation["is_original"] = ("/v1/" not in url) return Message.Url, url, deviation def _commit_journal_html(self, deviation, journal): @@ -320,9 +348,14 @@ class DeviantartExtractor(Extractor): yield url, folder def _update_content_default(self, deviation, content): - public = False if "premium_folder_data" in deviation else None + if "premium_folder_data" in deviation or deviation.get("is_mature"): + public = False + else: + public = None + data = self.api.deviation_download(deviation["deviationid"], public) content.update(data) + deviation["is_original"] = True def _update_content_image(self, deviation, content): data = self.api.deviation_download(deviation["deviationid"]) @@ -330,6 +363,7 @@ class DeviantartExtractor(Extractor): mtype = mimetypes.guess_type(url, False)[0] if mtype and mtype.startswith("image/"): content.update(data) + deviation["is_original"] = True def _update_token(self, deviation, content): """Replace JWT to be able to remove width/height limits @@ -341,6 +375,9 @@ class DeviantartExtractor(Extractor): if not sep: return + # 'images-wixmp' returns 401 errors, but just 'wixmp' still works + url = url.replace("//images-wixmp", "//wixmp", 1) + # header = b'{"typ":"JWT","alg":"none"}' payload = ( b'{"sub":"urn:app:","iss":"urn:app:","obj":[[{"path":"/f/' + @@ -349,11 +386,12 @@ class DeviantartExtractor(Extractor): ) deviation["_fallback"] = (content["src"],) + deviation["is_original"] = True content["src"] = ( "{}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.{}.".format( url, # base64 of 'header' is precomputed as 'eyJ0eX...' - # binascii.a2b_base64(header).rstrip(b"=\n").decode(), + # binascii.b2a_base64(header).rstrip(b"=\n").decode(), binascii.b2a_base64(payload).rstrip(b"=\n").decode()) ) @@ -435,18 +473,12 @@ class DeviantartUserExtractor(DeviantartExtractor): """Extractor for an artist's user profile""" subcategory = "user" pattern = BASE_PATTERN + r"/?$" - test = ( - ("https://www.deviantart.com/shimoda7", { - "pattern": r"/shimoda7/gallery$", - }), - ("https://www.deviantart.com/shimoda7", { - "options": (("include", "all"),), - "pattern": r"/shimoda7/" - r"(gallery(/scraps)?|posts(/statuses)?|favourites)$", - "count": 5, - }), - ("https://shimoda7.deviantart.com/"), - ) + example = "https://www.deviantart.com/USER" + + def initialize(self): + pass + + skip = Extractor.skip def items(self): base = "{}/{}/".format(self.root, self.user) @@ -467,84 +499,7 @@ class DeviantartGalleryExtractor(DeviantartExtractor): subcategory = "gallery" archive_fmt = "g_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$" - test = ( - ("https://www.deviantart.com/shimoda7/gallery/", { - "pattern": r"https://(images-)?wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.(jpg|png)\?token=.+", - "count": ">= 30", - "keyword": { - "allows_comments": bool, - "author": { - "type": "regular", - "usericon": str, - "userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", - "username": "shimoda7", - }, - "category_path": str, - "content": { - "filesize": int, - "height": int, - "src": str, - "transparency": bool, - "width": int, - }, - "da_category": str, - "date": "type:datetime", - "deviationid": str, - "?download_filesize": int, - "extension": str, - "index": int, - "is_deleted": bool, - "is_downloadable": bool, - "is_favourited": bool, - "is_mature": bool, - "preview": { - "height": int, - "src": str, - "transparency": bool, - "width": int, - }, - "published_time": int, - "stats": { - "comments": int, - "favourites": int, - }, - "target": dict, - "thumbs": list, - "title": str, - "url": r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", - "username": "shimoda7", - }, - }), - # group - ("https://www.deviantart.com/yakuzafc/gallery", { - "pattern": r"https://www.deviantart.com/yakuzafc/gallery" - r"/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", - "count": ">= 15", - }), - # 'folders' option (#276) - ("https://www.deviantart.com/justatest235723/gallery", { - "count": 3, - "options": (("metadata", 1), ("folders", 1), ("original", 0)), - "keyword": { - "description": str, - "folders": list, - "is_watching": bool, - "license": str, - "tags": list, - }, - }), - ("https://www.deviantart.com/shimoda8/gallery/", { - "exception": exception.NotFoundError, - }), - - ("https://www.deviantart.com/shimoda7/gallery"), - ("https://www.deviantart.com/shimoda7/gallery/all"), - ("https://www.deviantart.com/shimoda7/gallery/?catpath=/"), - ("https://shimoda7.deviantart.com/gallery/"), - ("https://shimoda7.deviantart.com/gallery/all/"), - ("https://shimoda7.deviantart.com/gallery/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/gallery/" def deviations(self): if self.flat and not self.group: @@ -559,32 +514,7 @@ class DeviantartFolderExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "{folder[title]}") archive_fmt = "F_{folder[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)/([^/?#]+)" - test = ( - # user - ("https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", { - "count": 5, - "options": (("original", False),), - }), - # group - ("https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", { - "count": ">= 4", - "options": (("original", False),), - }), - # uuid - (("https://www.deviantart.com/shimoda7/gallery" - "/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc"), { - "count": 5, - "options": (("original", False),), - }), - # name starts with '_', special characters (#1451) - (("https://www.deviantart.com/justatest235723" - "/gallery/69302698/-test-b-c-d-e-f-"), { - "count": 1, - "options": (("original", False),), - }), - ("https://shimoda7.deviantart.com/gallery/722019/Miscellaneous"), - ("https://yakuzafc.deviantart.com/gallery/37412168/Crafts"), - ) + example = "https://www.deviantart.com/USER/gallery/12345/TITLE" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -613,33 +543,7 @@ class DeviantartStashExtractor(DeviantartExtractor): subcategory = "stash" archive_fmt = "{index}.{extension}" pattern = r"(?:https?://)?sta\.sh/([a-z0-9]+)" - test = ( - ("https://sta.sh/022c83odnaxc", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.png\?token=.+", - "content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", - "count": 1, - }), - # multiple stash items - ("https://sta.sh/21jf51j7pzl2", { - "options": (("original", False),), - "count": 4, - }), - # downloadable, but no "content" field (#307) - ("https://sta.sh/024t4coz16mi", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.rar\?token=.+", - "count": 1, - }), - # mixed folders and images (#659) - ("https://sta.sh/215twi387vfj", { - "options": (("original", False),), - "count": 4, - }), - ("https://sta.sh/abcdefghijkl", { - "count": 0, - }), - ) + example = "https://sta.sh/abcde" skip = Extractor.skip @@ -684,20 +588,7 @@ class DeviantartFavoriteExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "Favourites") archive_fmt = "f_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites(?:/all|/?\?catpath=)?/?$" - test = ( - ("https://www.deviantart.com/h3813067/favourites/", { - "options": (("metadata", True), ("flat", False)), # issue #271 - "count": 1, - }), - ("https://www.deviantart.com/h3813067/favourites/", { - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/h3813067/favourites/all"), - ("https://www.deviantart.com/h3813067/favourites/?catpath=/"), - ("https://h3813067.deviantart.com/favourites/"), - ("https://h3813067.deviantart.com/favourites/all"), - ("https://h3813067.deviantart.com/favourites/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/favourites/" def deviations(self): if self.flat: @@ -714,20 +605,7 @@ class DeviantartCollectionExtractor(DeviantartExtractor): "{collection[title]}") archive_fmt = "C_{collection[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites/([^/?#]+)/([^/?#]+)" - test = ( - (("https://www.deviantart.com/pencilshadings/favourites" - "/70595441/3D-Favorites"), { - "count": ">= 15", - "options": (("original", False),), - }), - (("https://www.deviantart.com/pencilshadings/favourites" - "/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites"), { - "count": ">= 15", - "options": (("original", False),), - }), - ("https://pencilshadings.deviantart.com" - "/favourites/70595441/3D-Favorites"), - ) + example = "https://www.deviantart.com/USER/favourites/12345/TITLE" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -758,24 +636,7 @@ class DeviantartJournalExtractor(DeviantartExtractor): directory_fmt = ("{category}", "{username}", "Journal") archive_fmt = "j_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/(?:posts(?:/journals)?|journal)/?(?:\?.*)?$" - test = ( - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44", - }), - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", - "options": (("journals", "text"),), - }), - ("https://www.deviantart.com/angrywhitewanker/posts/journals/", { - "count": 0, - "options": (("journals", "none"),), - }), - ("https://www.deviantart.com/shimoda7/posts/"), - ("https://www.deviantart.com/shimoda7/journal/"), - ("https://www.deviantart.com/shimoda7/journal/?catpath=/"), - ("https://shimoda7.deviantart.com/journal/"), - ("https://shimoda7.deviantart.com/journal/?catpath=/"), - ) + example = "https://www.deviantart.com/USER/posts/journals/" def deviations(self): return self.api.browse_user_journals(self.user, self.offset) @@ -788,45 +649,7 @@ class DeviantartStatusExtractor(DeviantartExtractor): filename_fmt = "{category}_{index}_{title}_{date}.{extension}" archive_fmt = "S_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/posts/statuses" - test = ( - ("https://www.deviantart.com/t1na/posts/statuses", { - "count": 0, - }), - ("https://www.deviantart.com/justgalym/posts/statuses", { - "count": 4, - "url": "bf4c44c0c60ff2648a880f4c3723464ad3e7d074", - }), - # shared deviation - ("https://www.deviantart.com/justgalym/posts/statuses", { - "options": (("journals", "none"),), - "count": 1, - "pattern": r"https://images-wixmp-\w+\.wixmp\.com/f" - r"/[^/]+/[^.]+\.jpg\?token=", - }), - # shared sta.sh item - ("https://www.deviantart.com/vanillaghosties/posts/statuses", { - "options": (("journals", "none"), ("original", False)), - "range": "5-", - "count": 1, - "keyword": { - "index" : int, - "index_base36": "re:^[0-9a-z]+$", - "url" : "re:^https://sta.sh", - }, - }), - # "deleted" deviations in 'items' - ("https://www.deviantart.com/AndrejSKalin/posts/statuses", { - "options": (("journals", "none"), ("original", 0), - ("image-filter", "deviationid[:8] == '147C8B03'")), - "count": 2, - "archive": False, - "keyword": {"deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655"} - }), - ("https://www.deviantart.com/justgalym/posts/statuses", { - "options": (("journals", "text"),), - "url": "c8744f7f733a3029116607b826321233c5ca452d", - }), - ) + example = "https://www.deviantart.com/USER/posts/statuses/" def deviations(self): for status in self.api.user_statuses(self.user, self.offset): @@ -890,19 +713,7 @@ class DeviantartPopularExtractor(DeviantartExtractor): r"(?:deviations/?)?\?order=(popular-[^/?#]+)" r"|((?:[\w-]+/)*)(popular-[^/?#]+)" r")/?(?:\?([^#]*))?") - test = ( - ("https://www.deviantart.com/?order=popular-all-time", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }), - ("https://www.deviantart.com/popular-24-hours/?q=tree+house", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }), - ("https://www.deviantart.com/artisan/popular-all-time/?q=tree"), - ) + example = "https://www.deviantart.com/popular-24-hours/" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -947,11 +758,7 @@ class DeviantartTagExtractor(DeviantartExtractor): directory_fmt = ("{category}", "Tags", "{search_tags}") archive_fmt = "T_{search_tags}_{index}.{extension}" pattern = r"(?:https?://)?www\.deviantart\.com/tag/([^/?#]+)" - test = ("https://www.deviantart.com/tag/nature", { - "options": (("original", False),), - "range": "1-30", - "count": 30, - }) + example = "https://www.deviantart.com/tag/TAG" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -970,10 +777,7 @@ class DeviantartWatchExtractor(DeviantartExtractor): subcategory = "watch" pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com" r"/(?:watch/deviations|notifications/watch)()()") - test = ( - ("https://www.deviantart.com/watch/deviations"), - ("https://www.deviantart.com/notifications/watch"), - ) + example = "https://www.deviantart.com/watch/deviations" def deviations(self): return self.api.browse_deviantsyouwatch() @@ -983,7 +787,7 @@ class DeviantartWatchPostsExtractor(DeviantartExtractor): """Extractor for Posts from watched users""" subcategory = "watch-posts" pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()" - test = ("https://www.deviantart.com/watch/posts",) + example = "https://www.deviantart.com/watch/posts" def deviations(self): return self.api.browse_posts_deviantsyouwatch() @@ -1001,100 +805,7 @@ class DeviantartDeviationExtractor(DeviantartExtractor): r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" r"(\d+)" # bare deviation ID without slug r"|(?:https?://)?fav\.me/d([0-9a-z]+)") # base36 - test = ( - (("https://www.deviantart.com/shimoda7/art/For-the-sake-10073852"), { - "options": (("original", 0),), - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/zzz/art/zzz-1234567890", { - "exception": exception.NotFoundError, - }), - (("https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576"), { - "options": (("comments", True),), - "keyword": {"comments": list}, - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.jpg\?token=.+", - }), - # wixmp URL rewrite - (("https://www.deviantart.com/citizenfresh/art/Hverarond-789295466"), { - "pattern": (r"https://images-wixmp-\w+\.wixmp\.com/f" - r"/[^/]+/[^.]+\.jpg\?token="), - }), - # GIF (#242) - (("https://www.deviantart.com/skatergators/art/COM-Moni-781571783"), { - "pattern": r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-" - r"8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-" - r"f30961955bec\.gif\?token=ey...", - }), - # Flash animation with GIF preview (#1731) - ("https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", { - "pattern": r"https://wixmp-[^.]+\.wixmp\.com" - r"/f/.+/.+\.swf\?token=.+", - "keyword": { - "filename": "flash_comic_tutorial_by_yuumei-d3juatd", - "extension": "swf", - }, - }), - # sta.sh URLs from description (#302) - (("https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498"), { - "options": (("extra", 1), ("original", 0)), - "pattern": DeviantartStashExtractor.pattern, - "range": "2-", - "count": 4, - }), - # sta.sh URL from deviation["text_content"]["body"]["features"] - (("https://www.deviantart.com" - "/cimar-wildehopps/art/Honorary-Vixen-859809305"), { - "options": (("extra", 1),), - "pattern": ("text:<!DOCTYPE html>\n|" + - DeviantartStashExtractor.pattern), - "count": 2, - }), - # journal - ("https://www.deviantart.com/shimoda7/journal/ARTility-583755752", { - "url": "d34b2c9f873423e665a1b8ced20fcb75951694a3", - "pattern": "text:<!DOCTYPE html>\n", - }), - # journal-like post with isJournal == False (#419) - ("https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", { - "url": "e2e0044bd255304412179b6118536dbd9bb3bb0e", - "pattern": "text:<!DOCTYPE html>\n", - }), - # /view/ URLs - ("https://deviantart.com/view/904858796/", { - "content": "8770ec40ad1c1d60f6b602b16301d124f612948f", - }), - ("http://www.deviantart.com/view/890672057", { - "content": "1497e13d925caeb13a250cd666b779a640209236", - }), - ("https://www.deviantart.com/view/706871727", { - "content": "3f62ae0c2fca2294ac28e41888ea06bb37c22c65", - }), - ("https://www.deviantart.com/view/1", { - "exception": exception.NotFoundError, - }), - # /deviation/ (#3558) - ("https://www.deviantart.com/deviation/817215762"), - # fav.me (#3558) - ("https://fav.me/ddijrpu", { - "count": 1, - }), - ("https://fav.me/dddd", { - "exception": exception.NotFoundError, - }), - # old-style URLs - ("https://shimoda7.deviantart.com" - "/art/For-the-sake-of-a-memory-10073852"), - ("https://myria-moon.deviantart.com" - "/art/Aime-Moi-part-en-vadrouille-261986576"), - ("https://zzz.deviantart.com/art/zzz-1234567890"), - # old /view/ URLs from the Wayback Machine - ("https://www.deviantart.com/view.php?id=14864502"), - ("http://www.deviantart.com/view-full.php?id=100842"), - - ("https://www.fxdeviantart.com/zzz/art/zzz-1234567890"), - ("https://www.fxdeviantart.com/view/1234567890"), - ) + example = "https://www.deviantart.com/UsER/art/TITLE-12345" skip = Extractor.skip @@ -1105,11 +816,14 @@ class DeviantartDeviationExtractor(DeviantartExtractor): match.group(4) or match.group(5) or id_from_base36(match.group(6)) def deviations(self): - url = "{}/{}/{}/{}".format( - self.root, self.user or "u", self.type or "art", self.deviation_id) + if self.user: + url = "{}/{}/{}/{}".format( + self.root, self.user, self.type or "art", self.deviation_id) + else: + url = "{}/view/{}/".format(self.root, self.deviation_id) - uuid = text.extract(self._limited_request(url).text, - '"deviationUuid\\":\\"', '\\')[0] + uuid = text.extr(self._limited_request(url).text, + '"deviationUuid\\":\\"', '\\') if not uuid: raise exception.NotFoundError("deviation") return (self.api.deviation(uuid),) @@ -1120,15 +834,9 @@ class DeviantartScrapsExtractor(DeviantartExtractor): subcategory = "scraps" directory_fmt = ("{category}", "{username}", "Scraps") archive_fmt = "s_{_username}_{index}.{extension}" - cookiedomain = ".deviantart.com" + cookies_domain = ".deviantart.com" pattern = BASE_PATTERN + r"/gallery/(?:\?catpath=)?scraps\b" - test = ( - ("https://www.deviantart.com/shimoda7/gallery/scraps", { - "count": 12, - }), - ("https://www.deviantart.com/shimoda7/gallery/?catpath=scraps"), - ("https://shimoda7.deviantart.com/gallery/?catpath=scraps"), - ) + example = "https://www.deviantart.com/USER/gallery/scraps" def deviations(self): self.login() @@ -1143,14 +851,10 @@ class DeviantartSearchExtractor(DeviantartExtractor): subcategory = "search" directory_fmt = ("{category}", "Search", "{search_tags}") archive_fmt = "Q_{search_tags}_{index}.{extension}" - cookiedomain = ".deviantart.com" + cookies_domain = ".deviantart.com" pattern = (r"(?:https?://)?www\.deviantart\.com" r"/search(?:/deviations)?/?\?([^#]+)") - test = ( - ("https://www.deviantart.com/search?q=tree"), - ("https://www.deviantart.com/search/deviations?order=popular-1-week"), - ) - + example = "https://www.deviantart.com/search?q=QUERY" skip = Extractor.skip def __init__(self, match): @@ -1173,11 +877,6 @@ class DeviantartSearchExtractor(DeviantartExtractor): def _search_html(self, params): url = self.root + "/search" - deviation = { - "deviationId": None, - "author": {"username": "u"}, - "isJournal": False, - } while True: response = self.request(url, params=params) @@ -1186,13 +885,15 @@ class DeviantartSearchExtractor(DeviantartExtractor): raise exception.StopExtraction("HTTP redirect to login page") page = response.text - items , pos = text.rextract(page, r'\"items\":[', ']') - cursor, pos = text.extract(page, r'\"cursor\":\"', '\\', pos) - - for deviation_id in items.split(","): - deviation["deviationId"] = deviation_id - yield deviation + for dev in DeviantartDeviationExtractor.pattern.findall( + page)[2::3]: + yield { + "deviationId": dev[3], + "author": {"username": dev[0]}, + "isJournal": dev[2] == "journal", + } + cursor = text.extr(page, r'\"cursor\":\"', '\\',) if not cursor: return params["cursor"] = cursor @@ -1202,15 +903,9 @@ class DeviantartGallerySearchExtractor(DeviantartExtractor): """Extractor for deviantart gallery searches""" subcategory = "gallery-search" archive_fmt = "g_{_username}_{index}.{extension}" - cookiedomain = ".deviantart.com" + cookies_domain = ".deviantart.com" pattern = BASE_PATTERN + r"/gallery/?\?(q=[^#]+)" - test = ( - ("https://www.deviantart.com/shimoda7/gallery?q=memory", { - "options": (("original", 0),), - "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", - }), - ("https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular"), - ) + example = "https://www.deviantart.com/USER/gallery?q=QUERY" def __init__(self, match): DeviantartExtractor.__init__(self, match) @@ -1220,14 +915,12 @@ class DeviantartGallerySearchExtractor(DeviantartExtractor): self.login() eclipse_api = DeviantartEclipseAPI(self) - info = eclipse_api.user_info(self.user) - query = text.parse_query(self.query) self.search = query["q"] return self._eclipse_to_oauth( eclipse_api, eclipse_api.galleries_search( - info["user"]["userId"], + self.user, self.search, self.offset, query.get("sort", "most-recent"), @@ -1242,11 +935,7 @@ class DeviantartFollowingExtractor(DeviantartExtractor): """Extractor for user's watched users""" subcategory = "following" pattern = BASE_PATTERN + "/about#watching$" - test = ("https://www.deviantart.com/shimoda7/about#watching", { - "pattern": DeviantartUserExtractor.pattern, - "range": "1-50", - "count": 50, - }) + example = "https://www.deviantart.com/USER/about#watching" def items(self): eclipse_api = DeviantartEclipseAPI(self) @@ -1393,7 +1082,12 @@ class DeviantartOAuthAPI(): def deviation(self, deviation_id, public=None): """Query and return info about a single Deviation""" endpoint = "/deviation/" + deviation_id + deviation = self._call(endpoint, public=public) + if deviation.get("is_mature") and public is None and \ + self.refresh_token_key: + deviation = self._call(endpoint, public=False) + if self.metadata: self._metadata((deviation,)) if self.folders: @@ -1549,8 +1243,12 @@ class DeviantartOAuthAPI(): return data if not fatal and status != 429: return None - if data.get("error_description") == "User not found.": + + error = data.get("error_description") + if error == "User not found.": raise exception.NotFoundError("user or group") + if error == "Deviation not downloadable.": + raise exception.AuthorizationError() self.log.debug(response.text) msg = "API responded with {} {}".format( @@ -1574,6 +1272,17 @@ class DeviantartOAuthAPI(): self.log.error(msg) return data + def _switch_tokens(self, results, params): + if len(results) < params["limit"]: + return True + + if not self.extractor.jwt: + for item in results: + if item.get("is_mature"): + return True + + return False + def _pagination(self, endpoint, params, extend=True, public=None, unpack=False, key="results"): warn = True @@ -1592,7 +1301,7 @@ class DeviantartOAuthAPI(): results = [item["journal"] for item in results if "journal" in item] if extend: - if public and len(results) < params["limit"]: + if public and self._switch_tokens(results, params): if self.refresh_token_key: self.log.debug("Switching to private access token") public = False @@ -1600,9 +1309,10 @@ class DeviantartOAuthAPI(): elif data["has_more"] and warn: warn = False self.log.warning( - "Private deviations detected! Run 'gallery-dl " - "oauth:deviantart' and follow the instructions to " - "be able to access them.") + "Private or mature deviations detected! " + "Run 'gallery-dl oauth:deviantart' and follow the " + "instructions to be able to access them.") + # "statusid" cannot be used instead if results and "deviationid" in results[0]: if self.metadata: @@ -1711,70 +1421,70 @@ class DeviantartEclipseAPI(): self.request = self.extractor._limited_request self.csrf_token = None - def deviation_extended_fetch(self, deviation_id, user=None, kind=None): - endpoint = "/da-browse/shared_api/deviation/extended_fetch" + def deviation_extended_fetch(self, deviation_id, user, kind=None): + endpoint = "/_puppy/dadeviation/init" params = { - "deviationid" : deviation_id, - "username" : user, - "type" : kind, - "include_session": "false", + "deviationid" : deviation_id, + "username" : user, + "type" : kind, + "include_session" : "false", + "expand" : "deviation.related", + "da_minor_version": "20230710", } return self._call(endpoint, params) - def gallery_scraps(self, user, offset=None): - endpoint = "/da-user-profile/api/gallery/contents" + def gallery_scraps(self, user, offset=0): + endpoint = "/_puppy/dashared/gallection/contents" params = { "username" : user, + "type" : "gallery", "offset" : offset, "limit" : 24, "scraps_folder": "true", } return self._pagination(endpoint, params) - def galleries_search(self, user_id, query, - offset=None, order="most-recent"): - endpoint = "/shared_api/galleries/search" + def galleries_search(self, user, query, offset=0, order="most-recent"): + endpoint = "/_puppy/dashared/gallection/search" params = { - "userid": user_id, - "order" : order, - "q" : query, - "offset": offset, - "limit" : 24, + "username": user, + "type" : "gallery", + "order" : order, + "q" : query, + "offset" : offset, + "limit" : 24, } return self._pagination(endpoint, params) def search_deviations(self, params): - endpoint = "/da-browse/api/networkbar/search/deviations" + endpoint = "/_puppy/dabrowse/search/deviations" return self._pagination(endpoint, params, key="deviations") def user_info(self, user, expand=False): - endpoint = "/shared_api/user/info" + endpoint = "/_puppy/dauserprofile/init/about" params = {"username": user} - if expand: - params["expand"] = "user.stats,user.profile,user.watch" return self._call(endpoint, params) - def user_watching(self, user, offset=None): - endpoint = "/da-user-profile/api/module/watching" + def user_watching(self, user, offset=0): + gruserid, moduleid = self._ids_watching(user) + + endpoint = "/_puppy/gruser/module/watching" params = { - "username": user, - "moduleid": self._module_id_watching(user), - "offset" : offset, - "limit" : 24, + "gruserid" : gruserid, + "gruser_typeid": "4", + "username" : user, + "moduleid" : moduleid, + "offset" : offset, + "limit" : 24, } return self._pagination(endpoint, params) def _call(self, endpoint, params): - url = "https://www.deviantart.com/_napi" + endpoint - headers = {"Referer": "https://www.deviantart.com/"} + url = "https://www.deviantart.com" + endpoint params["csrf_token"] = self.csrf_token or self._fetch_csrf_token() - response = self.request( - url, params=params, headers=headers, fatal=None) + response = self.request(url, params=params, fatal=None) - if response.status_code == 404: - raise exception.StopExtraction( - "Your account must use the Eclipse interface.") try: return response.json() except Exception: @@ -1812,14 +1522,19 @@ class DeviantartEclipseAPI(): else: params["offset"] = int(params["offset"]) + len(results) - def _module_id_watching(self, user): + def _ids_watching(self, user): url = "{}/{}/about".format(self.extractor.root, user) page = self.request(url).text - pos = page.find('\\"type\\":\\"watching\\"') + + gruserid, pos = text.extract(page, ' data-userid="', '"') + + pos = page.find('\\"type\\":\\"watching\\"', pos) if pos < 0: raise exception.NotFoundError("module") + moduleid = text.rextract(page, '\\"id\\":', ',', pos)[0].strip('" ') + self._fetch_csrf_token(page) - return text.rextract(page, '\\"id\\":', ',', pos)[0].strip('" ') + return gruserid, moduleid def _fetch_csrf_token(self, page=None): if page is None: @@ -1866,7 +1581,7 @@ def _login_impl(extr, username, password): return { cookie.name: cookie.value - for cookie in extr.session.cookies + for cookie in extr.cookies } |