diff options
Diffstat (limited to 'gallery_dl/extractor/mangadex.py')
| -rw-r--r-- | gallery_dl/extractor/mangadex.py | 151 |
1 files changed, 111 insertions, 40 deletions
diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index 7f87cff..42a508d 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -29,11 +29,8 @@ class MangadexExtractor(Extractor): useragent = util.USERAGENT _cache = {} - def __init__(self, match): - Extractor.__init__(self, match) - self.uuid = match.group(1) - def _init(self): + self.uuid = self.groups[0] self.api = MangadexAPI(self) def items(self): @@ -44,6 +41,12 @@ class MangadexExtractor(Extractor): self._cache[uuid] = data yield Message.Queue, self.root + "/chapter/" + uuid, data + def _items_manga(self): + data = {"_extractor": MangadexMangaExtractor} + for manga in self.manga(): + url = "{}/title/{}".format(self.root, manga["id"]) + yield Message.Queue, url, data + def _transform(self, chapter): relationships = defaultdict(list) for item in chapter["relationships"]: @@ -130,7 +133,7 @@ class MangadexChapterExtractor(MangadexExtractor): class MangadexMangaExtractor(MangadexExtractor): """Extractor for manga from mangadex.org""" subcategory = "manga" - pattern = BASE_PATTERN + r"/(?:title|manga)/(?!feed$)([0-9a-f-]+)" + pattern = BASE_PATTERN + r"/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)" example = ("https://mangadex.org/title" "/01234567-89ab-cdef-0123-456789abcdef") @@ -139,17 +142,29 @@ class MangadexMangaExtractor(MangadexExtractor): class MangadexFeedExtractor(MangadexExtractor): - """Extractor for chapters from your Followed Feed""" + """Extractor for chapters from your Updates Feed""" subcategory = "feed" - pattern = BASE_PATTERN + r"/title/feed$()" + pattern = BASE_PATTERN + r"/titles?/feed$()" example = "https://mangadex.org/title/feed" def chapters(self): return self.api.user_follows_manga_feed() +class MangadexFollowingExtractor(MangadexExtractor): + """Extractor for followed manga from your Library""" + subcategory = "following" + pattern = BASE_PATTERN + r"/titles?/follows(?:\?([^#]+))?$" + example = "https://mangadex.org/title/follows" + + items = MangadexExtractor._items_manga + + def manga(self): + return self.api.user_follows_manga() + + class MangadexListExtractor(MangadexExtractor): - """Extractor for mangadex lists""" + """Extractor for mangadex MDLists""" subcategory = "list" pattern = (BASE_PATTERN + r"/list/([0-9a-f-]+)(?:/[^/?#]*)?(?:\?tab=(\w+))?") @@ -161,17 +176,17 @@ class MangadexListExtractor(MangadexExtractor): if match.group(2) == "feed": self.subcategory = "list-feed" else: - self.items = self._items_titles + self.items = self._items_manga def chapters(self): return self.api.list_feed(self.uuid) - def _items_titles(self): - data = {"_extractor": MangadexMangaExtractor} - for item in self.api.list(self.uuid)["relationships"]: - if item["type"] == "manga": - url = "{}/title/{}".format(self.root, item["id"]) - yield Message.Queue, url, data + def manga(self): + return [ + item + for item in self.api.list(self.uuid)["relationships"] + if item["type"] == "manga" + ] class MangadexAuthorExtractor(MangadexExtractor): @@ -196,10 +211,18 @@ class MangadexAPI(): def __init__(self, extr): self.extractor = extr - self.headers = {} + self.headers = None + self.headers_auth = {} self.username, self.password = extr._get_auth_info() - if not self.username: + if self.username: + self.client_id = cid = extr.config("client-id") + self.client_secret = extr.config("client-secret") + if cid: + self._authenticate_impl = self._authenticate_impl_client + else: + self._authenticate_impl = self._authenticate_impl_legacy + else: self.authenticate = util.noop server = extr.config("api-server") @@ -218,10 +241,10 @@ class MangadexAPI(): return self._call("/chapter/" + uuid, params)["data"] def list(self, uuid): - return self._call("/list/" + uuid)["data"] + return self._call("/list/" + uuid, None, True)["data"] def list_feed(self, uuid): - return self._pagination_chapters("/list/" + uuid + "/feed") + return self._pagination_chapters("/list/" + uuid + "/feed", None, True) @memcache(keyarg=1) def manga(self, uuid): @@ -240,28 +263,73 @@ class MangadexAPI(): } return self._pagination_chapters("/manga/" + uuid + "/feed", params) + def user_follows_manga(self): + params = {"contentRating": None} + return self._pagination_manga( + "/user/follows/manga", params, True) + def user_follows_manga_feed(self): params = {"order[publishAt]": "desc"} - return self._pagination_chapters("/user/follows/manga/feed", params) + return self._pagination_chapters( + "/user/follows/manga/feed", params, True) def authenticate(self): - self.headers["Authorization"] = \ + self.headers_auth["Authorization"] = \ self._authenticate_impl(self.username, self.password) @cache(maxage=900, keyarg=1) - def _authenticate_impl(self, username, password): + def _authenticate_impl_client(self, username, password): + refresh_token = _refresh_token_cache((username, "personal")) + if refresh_token: + self.extractor.log.info("Refreshing access token") + data = { + "grant_type" : "refresh_token", + "refresh_token": refresh_token, + "client_id" : self.client_id, + "client_secret": self.client_secret, + } + else: + self.extractor.log.info("Logging in as %s", username) + data = { + "grant_type" : "password", + "username" : self.username, + "password" : self.password, + "client_id" : self.client_id, + "client_secret": self.client_secret, + } + + self.extractor.log.debug("Using client-id '%s…'", self.client_id[:24]) + url = ("https://auth.mangadex.org/realms/mangadex" + "/protocol/openid-connect/token") + data = self.extractor.request( + url, method="POST", data=data, fatal=None).json() + + try: + access_token = data["access_token"] + except Exception: + raise exception.AuthenticationError(data.get("error_description")) + + if refresh_token != data.get("refresh_token"): + _refresh_token_cache.update( + (username, "personal"), data["refresh_token"]) + + return "Bearer " + access_token + + @cache(maxage=900, keyarg=1) + def _authenticate_impl_legacy(self, username, password): refresh_token = _refresh_token_cache(username) if refresh_token: self.extractor.log.info("Refreshing access token") url = self.root + "/auth/refresh" - data = {"token": refresh_token} + json = {"token": refresh_token} else: self.extractor.log.info("Logging in as %s", username) url = self.root + "/auth/login" - data = {"username": username, "password": password} + json = {"username": username, "password": password} + self.extractor.log.debug("Using legacy login method") data = self.extractor.request( - url, method="POST", json=data, fatal=None).json() + url, method="POST", json=json, fatal=None).json() if data.get("result") != "ok": raise exception.AuthenticationError() @@ -269,13 +337,15 @@ class MangadexAPI(): _refresh_token_cache.update(username, data["token"]["refresh"]) return "Bearer " + data["token"]["session"] - def _call(self, endpoint, params=None): + def _call(self, endpoint, params=None, auth=False): url = self.root + endpoint + headers = self.headers_auth if auth else self.headers while True: - self.authenticate() + if auth: + self.authenticate() response = self.extractor.request( - url, params=params, headers=self.headers, fatal=None) + url, params=params, headers=headers, fatal=None) if response.status_code < 400: return response.json() @@ -284,12 +354,12 @@ class MangadexAPI(): self.extractor.wait(until=until) continue - msg = ", ".join('{title}: {detail}'.format_map(error) + msg = ", ".join('{title}: "{detail}"'.format_map(error) for error in response.json()["errors"]) raise exception.StopExtraction( "%s %s (%s)", response.status_code, response.reason, msg) - def _pagination_chapters(self, endpoint, params=None): + def _pagination_chapters(self, endpoint, params=None, auth=False): if params is None: params = {} @@ -299,21 +369,22 @@ class MangadexAPI(): params["translatedLanguage[]"] = lang params["includes[]"] = ("scanlation_group",) - return self._pagination(endpoint, params) + return self._pagination(endpoint, params, auth) - def _pagination_manga(self, endpoint, params=None): + def _pagination_manga(self, endpoint, params=None, auth=False): if params is None: params = {} - return self._pagination(endpoint, params) + return self._pagination(endpoint, params, auth) - def _pagination(self, endpoint, params): + def _pagination(self, endpoint, params, auth=False): config = self.extractor.config - ratings = config("ratings") - if ratings is None: - ratings = ("safe", "suggestive", "erotica", "pornographic") - params["contentRating[]"] = ratings + if "contentRating" not in params: + ratings = config("ratings") + if ratings is None: + ratings = ("safe", "suggestive", "erotica", "pornographic") + params["contentRating[]"] = ratings params["offset"] = 0 api_params = config("api-parameters") @@ -321,7 +392,7 @@ class MangadexAPI(): params.update(api_params) while True: - data = self._call(endpoint, params) + data = self._call(endpoint, params, auth) yield from data["data"] params["offset"] = data["offset"] + data["limit"] @@ -329,6 +400,6 @@ class MangadexAPI(): return -@cache(maxage=28*86400, keyarg=0) +@cache(maxage=90*86400, keyarg=0) def _refresh_token_cache(username): return None |