diff options
Diffstat (limited to 'gallery_dl/extractor/deviantart.py')
| -rw-r--r-- | gallery_dl/extractor/deviantart.py | 87 |
1 files changed, 71 insertions, 16 deletions
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 604966f..02a14e3 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2019 Mike Fährmann +# Copyright 2015-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -31,6 +31,7 @@ class DeviantartExtractor(Extractor): category = "deviantart" directory_fmt = ("{category}", "{username}") filename_fmt = "{category}_{index}_{title}.{extension}" + cookiedomain = None root = "https://www.deviantart.com" def __init__(self, match=None): @@ -475,7 +476,7 @@ class DeviantartFavoriteExtractor(DeviantartExtractor): subcategory = "favorite" directory_fmt = ("{category}", "{username}", "Favourites") archive_fmt = "f_{_username}_{index}.{extension}" - pattern = BASE_PATTERN + r"/favourites/?(?:\?catpath=/)?$" + pattern = BASE_PATTERN + r"/favourites(?:/all|/?\?catpath=)?/?$" test = ( ("https://www.deviantart.com/h3813067/favourites/", { "options": (("metadata", True), ("flat", False)), # issue #271 @@ -484,8 +485,10 @@ class DeviantartFavoriteExtractor(DeviantartExtractor): ("https://www.deviantart.com/h3813067/favourites/", { "content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", }), + ("https://www.deviantart.com/h3813067/favourites/all"), ("https://www.deviantart.com/h3813067/favourites/?catpath=/"), ("https://h3813067.deviantart.com/favourites/"), + ("https://h3813067.deviantart.com/favourites/all"), ("https://h3813067.deviantart.com/favourites/?catpath=/"), ) @@ -573,12 +576,22 @@ class DeviantartPopularExtractor(DeviantartExtractor): directory_fmt = ("{category}", "Popular", "{popular[range]}", "{popular[search]}") archive_fmt = "P_{popular[range]}_{popular[search]}_{index}.{extension}" - pattern = (r"(?:https?://)?www\.deviantart\.com" - r"((?:/\w+)*)/(?:popular-([^/?&#]+))/?(?:\?([^#]*))?") + pattern = (r"(?:https?://)?www\.deviantart\.com/(?:" + r"search(?:/deviations)?" + r"|(?:deviations/?)?\?order=(popular-[^/?&#]+)" + r"|((?:[\w-]+/)*)(popular-[^/?&#]+)" + r")/?(?:\?([^#]*))?") test = ( + ("https://www.deviantart.com/?order=popular-all-time", { + "options": (("original", False),), + "range": "1-30", + "count": 30, + }), ("https://www.deviantart.com/popular-24-hours/?q=tree+house", { "options": (("original", False),), }), + ("https://www.deviantart.com/search?q=tree"), + ("https://www.deviantart.com/search/deviations?order=popular-1-week"), ("https://www.deviantart.com/artisan/popular-all-time/?q=tree"), ) @@ -587,13 +600,20 @@ class DeviantartPopularExtractor(DeviantartExtractor): self.search_term = self.time_range = self.category_path = None self.user = "" - path, trange, query = match.groups() + trange1, path, trange2, query = match.groups() + trange = trange1 or trange2 + query = text.parse_query(query) + + if not trange: + trange = query.get("order") + if path: - self.category_path = path.lstrip("/") + self.category_path = path.strip("/") if trange: + trange = trange[8:] if trange.startswith("popular-") else "" self.time_range = trange.replace("-", "").replace("hours", "hr") if query: - self.search_term = text.parse_query(query).get("q") + self.search_term = query.get("q") self.popular = { "search": self.search_term or "", @@ -739,6 +759,15 @@ class DeviantartExtractorV2(DeviantartExtractor): deviation["target"] = target return deviation + def _pagination(self, url, params, headers=None): + while True: + data = self.request(url, params=params, headers=headers).json() + yield from data["results"] + + if not data["hasMore"]: + return + params["offset"] = data["nextOffset"] + class DeviantartDeviationExtractor(DeviantartExtractorV2): """Extractor for single deviations""" @@ -863,15 +892,40 @@ class DeviantartScrapsExtractor(DeviantartExtractorV2): "Referer": "{}/{}/gallery/scraps".format(self.root, self.user), } - while True: - data = self.request(url, params=params, headers=headers).json() + for obj in self._pagination(url, params, headers): + yield obj["deviation"] - for obj in data["results"]: - yield obj["deviation"] - if not data["hasMore"]: - return - params["offset"] = data["nextOffset"] +class DeviantartFollowingExtractor(DeviantartExtractorV2): + subcategory = "following" + pattern = BASE_PATTERN + "/about#watching$" + test = ("https://www.deviantart.com/shimoda7/about#watching", { + "pattern": DeviantartUserExtractor.pattern, + "range": "1-50", + "count": 50, + }) + + def items(self): + url = "{}/_napi/da-user-profile/api/module/watching".format(self.root) + params = { + "username": self.user, + "moduleid": self._module_id(self.user), + "offset" : "0", + "limit" : "24", + } + + yield Message.Version, 1 + for user in self._pagination(url, params): + url = "{}/{}".format(self.root, user["username"]) + yield Message.Queue, url, user + + def _module_id(self, username): + url = "{}/{}/about".format(self.root, username) + page = self.request(url).text + pos = page.find('\\"type\\":\\"watching\\"') + if pos < 0: + raise exception.NotFoundError("module") + return text.rextract(page, '\\"id\\":', ',', pos)[0].strip('" ') class DeviantartAPI(): @@ -1076,7 +1130,7 @@ class DeviantartAPI(): return data def _pagination(self, endpoint, params, extend=True): - public = True + public = warn = True while True: data = self._call(endpoint, params, public=public) if "results" not in data: @@ -1089,7 +1143,8 @@ class DeviantartAPI(): self.log.debug("Switching to private access token") public = False continue - elif data["has_more"]: + elif data["has_more"] and warn: + warn = False self.log.warning( "Private deviations detected! Run 'gallery-dl " "oauth:deviantart' and follow the instructions to " |