diff options
Diffstat (limited to 'gallery_dl/extractor/deviantart.py')
| -rw-r--r-- | gallery_dl/extractor/deviantart.py | 78 |
1 files changed, 60 insertions, 18 deletions
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 73ef20d..a0f4d1c 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -14,7 +14,6 @@ from ..cache import cache, memcache import collections import itertools import mimetypes -import math import time import re @@ -55,6 +54,7 @@ class DeviantartExtractor(Extractor): self._update_content = self._update_content_image self.original = True + self._premium_cache = {} self.commit_journal = { "html": self._commit_journal_html, "text": self._commit_journal_text, @@ -66,6 +66,8 @@ class DeviantartExtractor(Extractor): def items(self): self.api = DeviantartOAuthAPI(self) + if not self.api.refresh_token_key: + self._fetch_premium = self._fetch_premium_notoken if self.user: profile = self.api.user_profile(self.user) @@ -83,6 +85,10 @@ class DeviantartExtractor(Extractor): yield Message.Queue, url, data continue + if "premium_folder_data" in deviation: + if not self._fetch_premium(deviation): + continue + self.prepare(deviation) yield Message.Directory, deviation @@ -261,7 +267,9 @@ class DeviantartExtractor(Extractor): return [(url + folder["name"], folder) for folder in folders] def _update_content_default(self, deviation, content): - content.update(self.api.deviation_download(deviation["deviationid"])) + public = "premium_folder_data" not in deviation + data = self.api.deviation_download(deviation["deviationid"], public) + content.update(data) def _update_content_image(self, deviation, content): data = self.api.deviation_download(deviation["deviationid"]) @@ -290,6 +298,41 @@ class DeviantartExtractor(Extractor): return response self.wait(seconds=180) + def _fetch_premium(self, deviation): + cache = self._premium_cache + + if deviation["deviationid"] not in cache: + + # check accessibility + dev = self.api.deviation(deviation["deviationid"], False) + has_access = dev["premium_folder_data"]["has_access"] + + if has_access: + self.log.info("Fetching premium folder data") + else: + self.log.warning("Unable to access premium content (type: %s)", + dev["premium_folder_data"]["type"]) + # fill cache + for dev in self.api.gallery( + deviation["author"]["username"], + deviation["premium_folder_data"]["gallery_id"], + public=False, + ): + cache[dev["deviationid"]] = dev if has_access else None + + data = cache[deviation["deviationid"]] + if data: + deviation.update(data) + return True + return False + + def _fetch_premium_notoken(self, deviation): + if not self._premium_cache: + self.log.warning( + "Unable to access premium content (no refresh-token)") + self._premium_cache = True + return False + class DeviantartUserExtractor(DeviantartExtractor): """Extractor for an artist's user profile""" @@ -837,8 +880,7 @@ class DeviantartOAuthAPI(): self.log = extractor.log self.headers = {} - delay = extractor.config("wait-min", 0) - self.delay = math.ceil(math.log2(delay)) if delay >= 1 else -1 + self.delay = extractor.config("wait-min", 0) self.delay_min = max(2, self.delay) self.mature = extractor.config("mature", "true") @@ -897,27 +939,27 @@ class DeviantartOAuthAPI(): "mature_content": self.mature} return self._pagination_folders(endpoint, params) - def deviation(self, deviation_id): + def deviation(self, deviation_id, public=True): """Query and return info about a single Deviation""" endpoint = "deviation/" + deviation_id - deviation = self._call(endpoint) + deviation = self._call(endpoint, public=public) if self.metadata: self._metadata((deviation,)) if self.folders: self._folders((deviation,)) return deviation - def deviation_content(self, deviation_id): + def deviation_content(self, deviation_id, public=False): """Get extended content of a single Deviation""" endpoint = "deviation/content" params = {"deviationid": deviation_id} - return self._call(endpoint, params, public=False) + return self._call(endpoint, params, public=public) - def deviation_download(self, deviation_id): + def deviation_download(self, deviation_id, public=True): """Get the original file download (if allowed)""" endpoint = "deviation/download/" + deviation_id params = {"mature_content": self.mature} - return self._call(endpoint, params) + return self._call(endpoint, params, public=public) def deviation_metadata(self, deviations): """ Fetch deviation metadata for a set of deviations""" @@ -930,12 +972,12 @@ class DeviantartOAuthAPI(): params = {"mature_content": self.mature} return self._call(endpoint, params)["metadata"] - def gallery(self, username, folder_id="", offset=0, extend=True): + def gallery(self, username, folder_id, offset=0, extend=True, public=True): """Yield all Deviation-objects contained in a gallery folder""" endpoint = "gallery/" + folder_id params = {"username": username, "offset": offset, "limit": 24, "mature_content": self.mature, "mode": "newest"} - return self._pagination(endpoint, params, extend) + return self._pagination(endpoint, params, extend, public) def gallery_all(self, username, offset=0): """Yield all Deviation-objects of a specific user""" @@ -993,8 +1035,8 @@ class DeviantartOAuthAPI(): """Call an API endpoint""" url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint while True: - if self.delay >= 0: - time.sleep(2 ** self.delay) + if self.delay: + time.sleep(self.delay) self.authenticate(None if public else self.refresh_token_key) response = self.extractor.request( @@ -1015,15 +1057,15 @@ class DeviantartOAuthAPI(): msg = "API responded with {} {}".format( status, response.reason) if status == 429: - if self.delay < 9: + if self.delay < 30: self.delay += 1 - self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay) + self.log.warning("%s. Using %ds delay.", msg, self.delay) else: self.log.error(msg) return data - def _pagination(self, endpoint, params, extend=True): - public = warn = True + def _pagination(self, endpoint, params, extend=True, public=True): + warn = True while True: data = self._call(endpoint, params, public=public) if "results" not in data: |