diff options
Diffstat (limited to 'gallery_dl/extractor/deviantart.py')
| -rw-r--r-- | gallery_dl/extractor/deviantart.py | 78 |
1 files changed, 56 insertions, 22 deletions
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 47286b7..9d1701f 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -20,7 +20,7 @@ import re BASE_PATTERN = ( r"(?:https?://)?(?:" - r"(?:www\.)?deviantart\.com/([\w-]+)|" + r"(?:www\.)?deviantart\.com/(?!watch/)([\w-]+)|" r"(?!www\.)([\w-]+)\.deviantart\.com)" ) @@ -78,10 +78,6 @@ class DeviantartExtractor(Extractor): else: self.user = profile["user"]["username"] - if self.extra: - finditer_stash = DeviantartStashExtractor.pattern.finditer - finditer_deviation = DeviantartDeviationExtractor.pattern.finditer - yield Message.Version, 1 for deviation in self.deviations(): if isinstance(deviation, tuple): @@ -109,7 +105,8 @@ class DeviantartExtractor(Extractor): intermediary, count = re.subn( r"(/f/[^/]+/[^/]+)/v\d+/.*", r"/intermediary\1", content["src"], 1) - if count and self._check_url(intermediary): + if count: + deviation["_fallback"] = (content["src"],) content["src"] = intermediary if self.quality: content["src"] = re.sub( @@ -138,14 +135,10 @@ class DeviantartExtractor(Extractor): if self.extra: txt = (deviation.get("description", "") + deviation.get("_journal", "")) - for match in finditer_stash(txt): + for match in DeviantartStashExtractor.pattern.finditer(txt): url = text.ensure_http_scheme(match.group(0)) deviation["_extractor"] = DeviantartStashExtractor yield Message.Queue, url, deviation - for match in finditer_deviation(txt): - url = text.ensure_http_scheme(match.group(0)) - deviation["_extractor"] = DeviantartDeviationExtractor - yield Message.Queue, url, deviation def deviations(self): """Return an iterable containing all relevant Deviation-objects""" @@ -290,9 +283,6 @@ class DeviantartExtractor(Extractor): if mtype and mtype.startswith("image/"): content.update(data) - def _check_url(self, url): - return self.request(url, method="HEAD", fatal=False).status_code < 400 - def _limited_request(self, url, **kwargs): """Limits HTTP requests to one every 2 seconds""" kwargs["fatal"] = None @@ -718,15 +708,16 @@ class DeviantartPopularExtractor(DeviantartExtractor): if path: self.category_path = path.strip("/") if trange: - trange = trange[8:] if trange.startswith("popular-") else "" + if trange.startswith("popular-"): + trange = trange[8:] self.time_range = trange.replace("-", "").replace("hours", "hr") if query: self.search_term = query.get("q") self.popular = { "search": self.search_term or "", - "range": trange or "24-hours", - "path": self.category_path, + "range" : trange or "", + "path" : self.category_path, } def deviations(self): @@ -738,6 +729,30 @@ class DeviantartPopularExtractor(DeviantartExtractor): deviation["popular"] = self.popular +class DeviantartWatchExtractor(DeviantartExtractor): + """Extractor for Deviations from watched users""" + subcategory = "watch" + pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com" + r"/(?:watch/deviations|notifications/watch)()()") + test = ( + ("https://www.deviantart.com/watch/deviations"), + ("https://www.deviantart.com/notifications/watch"), + ) + + def deviations(self): + return self.api.browse_deviantsyouwatch() + + +class DeviantartWatchPostsExtractor(DeviantartExtractor): + """Extractor for Posts from watched users""" + subcategory = "watch-posts" + pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()" + test = ("https://www.deviantart.com/watch/posts",) + + def deviations(self): + return self.api.browse_posts_deviantsyouwatch() + + ############################################################################### # Eclipse ##################################################################### @@ -926,6 +941,20 @@ class DeviantartOAuthAPI(): self.client_id, ) + def browse_deviantsyouwatch(self, offset=0): + """Yield deviations from users you watch""" + endpoint = "browse/deviantsyouwatch" + params = {"limit": "50", "offset": offset, + "mature_content": self.mature} + return self._pagination(endpoint, params, public=False) + + def browse_posts_deviantsyouwatch(self, offset=0): + """Yield posts from users you watch""" + endpoint = "browse/posts/deviantsyouwatch" + params = {"limit": "50", "offset": offset, + "mature_content": self.mature} + return self._pagination(endpoint, params, public=False, unpack=True) + def browse_popular(self, query=None, timerange=None, offset=0): """Yield popular deviations""" endpoint = "browse/popular" @@ -1085,16 +1114,21 @@ class DeviantartOAuthAPI(): self.log.error(msg) return data - def _pagination(self, endpoint, params, extend=True, public=True): + def _pagination(self, endpoint, params, + extend=True, public=True, unpack=False): warn = True while True: data = self._call(endpoint, params, public=public) if "results" not in data: self.log.error("Unexpected API response: %s", data) return + results = data["results"] + if unpack: + results = [item["journal"] for item in results + if "journal" in item] if extend: - if public and len(data["results"]) < params["limit"]: + if public and len(results) < params["limit"]: if self.refresh_token_key: self.log.debug("Switching to private access token") public = False @@ -1106,10 +1140,10 @@ class DeviantartOAuthAPI(): "oauth:deviantart' and follow the instructions to " "be able to access them.") if self.metadata: - self._metadata(data["results"]) + self._metadata(results) if self.folders: - self._folders(data["results"]) - yield from data["results"] + self._folders(results) + yield from results if not data["has_more"]: return |