New upstream version 1.29.7.upstream/1.29.7

1 files changed, 111 insertions, 40 deletions

diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py
index 7f87cff..42a508d 100644
--- a/gallery_dl/extractor/mangadex.py
+++ b/gallery_dl/extractor/mangadex.py
@@ -29,11 +29,8 @@ class MangadexExtractor(Extractor):
     useragent = util.USERAGENT
     _cache = {}
 
-    def __init__(self, match):
-        Extractor.__init__(self, match)
-        self.uuid = match.group(1)
-
     def _init(self):
+        self.uuid = self.groups[0]
         self.api = MangadexAPI(self)
 
     def items(self):
@@ -44,6 +41,12 @@ class MangadexExtractor(Extractor):
             self._cache[uuid] = data
             yield Message.Queue, self.root + "/chapter/" + uuid, data
 
+    def _items_manga(self):
+        data = {"_extractor": MangadexMangaExtractor}
+        for manga in self.manga():
+            url = "{}/title/{}".format(self.root, manga["id"])
+            yield Message.Queue, url, data
+
     def _transform(self, chapter):
         relationships = defaultdict(list)
         for item in chapter["relationships"]:
@@ -130,7 +133,7 @@ class MangadexChapterExtractor(MangadexExtractor):
 class MangadexMangaExtractor(MangadexExtractor):
     """Extractor for manga from mangadex.org"""
     subcategory = "manga"
-    pattern = BASE_PATTERN + r"/(?:title|manga)/(?!feed$)([0-9a-f-]+)"
+    pattern = BASE_PATTERN + r"/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)"
     example = ("https://mangadex.org/title"
                "/01234567-89ab-cdef-0123-456789abcdef")
 
@@ -139,17 +142,29 @@ class MangadexMangaExtractor(MangadexExtractor):
 
 
 class MangadexFeedExtractor(MangadexExtractor):
-    """Extractor for chapters from your Followed Feed"""
+    """Extractor for chapters from your Updates Feed"""
     subcategory = "feed"
-    pattern = BASE_PATTERN + r"/title/feed$()"
+    pattern = BASE_PATTERN + r"/titles?/feed$()"
     example = "https://mangadex.org/title/feed"
 
     def chapters(self):
         return self.api.user_follows_manga_feed()
 
 
+class MangadexFollowingExtractor(MangadexExtractor):
+    """Extractor for followed manga from your Library"""
+    subcategory = "following"
+    pattern = BASE_PATTERN + r"/titles?/follows(?:\?([^#]+))?$"
+    example = "https://mangadex.org/title/follows"
+
+    items = MangadexExtractor._items_manga
+
+    def manga(self):
+        return self.api.user_follows_manga()
+
+
 class MangadexListExtractor(MangadexExtractor):
-    """Extractor for mangadex lists"""
+    """Extractor for mangadex MDLists"""
     subcategory = "list"
     pattern = (BASE_PATTERN +
                r"/list/([0-9a-f-]+)(?:/[^/?#]*)?(?:\?tab=(\w+))?")
@@ -161,17 +176,17 @@ class MangadexListExtractor(MangadexExtractor):
         if match.group(2) == "feed":
             self.subcategory = "list-feed"
         else:
-            self.items = self._items_titles
+            self.items = self._items_manga
 
     def chapters(self):
         return self.api.list_feed(self.uuid)
 
-    def _items_titles(self):
-        data = {"_extractor": MangadexMangaExtractor}
-        for item in self.api.list(self.uuid)["relationships"]:
-            if item["type"] == "manga":
-                url = "{}/title/{}".format(self.root, item["id"])
-                yield Message.Queue, url, data
+    def manga(self):
+        return [
+            item
+            for item in self.api.list(self.uuid)["relationships"]
+            if item["type"] == "manga"
+        ]
 
 
 class MangadexAuthorExtractor(MangadexExtractor):
@@ -196,10 +211,18 @@ class MangadexAPI():
 
     def __init__(self, extr):
         self.extractor = extr
-        self.headers = {}
+        self.headers = None
+        self.headers_auth = {}
 
         self.username, self.password = extr._get_auth_info()
-        if not self.username:
+        if self.username:
+            self.client_id = cid = extr.config("client-id")
+            self.client_secret = extr.config("client-secret")
+            if cid:
+                self._authenticate_impl = self._authenticate_impl_client
+            else:
+                self._authenticate_impl = self._authenticate_impl_legacy
+        else:
             self.authenticate = util.noop
 
         server = extr.config("api-server")
@@ -218,10 +241,10 @@ class MangadexAPI():
         return self._call("/chapter/" + uuid, params)["data"]
 
     def list(self, uuid):
-        return self._call("/list/" + uuid)["data"]
+        return self._call("/list/" + uuid, None, True)["data"]
 
     def list_feed(self, uuid):
-        return self._pagination_chapters("/list/" + uuid + "/feed")
+        return self._pagination_chapters("/list/" + uuid + "/feed", None, True)
 
     @memcache(keyarg=1)
     def manga(self, uuid):
@@ -240,28 +263,73 @@ class MangadexAPI():
         }
         return self._pagination_chapters("/manga/" + uuid + "/feed", params)
 
+    def user_follows_manga(self):
+        params = {"contentRating": None}
+        return self._pagination_manga(
+            "/user/follows/manga", params, True)
+
     def user_follows_manga_feed(self):
         params = {"order[publishAt]": "desc"}
-        return self._pagination_chapters("/user/follows/manga/feed", params)
+        return self._pagination_chapters(
+            "/user/follows/manga/feed", params, True)
 
     def authenticate(self):
-        self.headers["Authorization"] = \
+        self.headers_auth["Authorization"] = \
             self._authenticate_impl(self.username, self.password)
 
     @cache(maxage=900, keyarg=1)
-    def _authenticate_impl(self, username, password):
+    def _authenticate_impl_client(self, username, password):
+        refresh_token = _refresh_token_cache((username, "personal"))
+        if refresh_token:
+            self.extractor.log.info("Refreshing access token")
+            data = {
+                "grant_type"   : "refresh_token",
+                "refresh_token": refresh_token,
+                "client_id"    : self.client_id,
+                "client_secret": self.client_secret,
+            }
+        else:
+            self.extractor.log.info("Logging in as %s", username)
+            data = {
+                "grant_type"   : "password",
+                "username"     : self.username,
+                "password"     : self.password,
+                "client_id"    : self.client_id,
+                "client_secret": self.client_secret,
+            }
+
+        self.extractor.log.debug("Using client-id '%s…'", self.client_id[:24])
+        url = ("https://auth.mangadex.org/realms/mangadex"
+               "/protocol/openid-connect/token")
+        data = self.extractor.request(
+            url, method="POST", data=data, fatal=None).json()
+
+        try:
+            access_token = data["access_token"]
+        except Exception:
+            raise exception.AuthenticationError(data.get("error_description"))
+
+        if refresh_token != data.get("refresh_token"):
+            _refresh_token_cache.update(
+                (username, "personal"), data["refresh_token"])
+
+        return "Bearer " + access_token
+
+    @cache(maxage=900, keyarg=1)
+    def _authenticate_impl_legacy(self, username, password):
         refresh_token = _refresh_token_cache(username)
         if refresh_token:
             self.extractor.log.info("Refreshing access token")
             url = self.root + "/auth/refresh"
-            data = {"token": refresh_token}
+            json = {"token": refresh_token}
         else:
             self.extractor.log.info("Logging in as %s", username)
             url = self.root + "/auth/login"
-            data = {"username": username, "password": password}
+            json = {"username": username, "password": password}
 
+        self.extractor.log.debug("Using legacy login method")
         data = self.extractor.request(
-            url, method="POST", json=data, fatal=None).json()
+            url, method="POST", json=json, fatal=None).json()
         if data.get("result") != "ok":
             raise exception.AuthenticationError()
 
@@ -269,13 +337,15 @@ class MangadexAPI():
             _refresh_token_cache.update(username, data["token"]["refresh"])
         return "Bearer " + data["token"]["session"]
 
-    def _call(self, endpoint, params=None):
+    def _call(self, endpoint, params=None, auth=False):
         url = self.root + endpoint
+        headers = self.headers_auth if auth else self.headers
 
         while True:
-            self.authenticate()
+            if auth:
+                self.authenticate()
             response = self.extractor.request(
-                url, params=params, headers=self.headers, fatal=None)
+                url, params=params, headers=headers, fatal=None)
 
             if response.status_code < 400:
                 return response.json()
@@ -284,12 +354,12 @@ class MangadexAPI():
                 self.extractor.wait(until=until)
                 continue
 
-            msg = ", ".join('{title}: {detail}'.format_map(error)
+            msg = ", ".join('{title}: "{detail}"'.format_map(error)
                             for error in response.json()["errors"])
             raise exception.StopExtraction(
                 "%s %s (%s)", response.status_code, response.reason, msg)
 
-    def _pagination_chapters(self, endpoint, params=None):
+    def _pagination_chapters(self, endpoint, params=None, auth=False):
         if params is None:
             params = {}
 
@@ -299,21 +369,22 @@ class MangadexAPI():
         params["translatedLanguage[]"] = lang
         params["includes[]"] = ("scanlation_group",)
 
-        return self._pagination(endpoint, params)
+        return self._pagination(endpoint, params, auth)
 
-    def _pagination_manga(self, endpoint, params=None):
+    def _pagination_manga(self, endpoint, params=None, auth=False):
         if params is None:
             params = {}
 
-        return self._pagination(endpoint, params)
+        return self._pagination(endpoint, params, auth)
 
-    def _pagination(self, endpoint, params):
+    def _pagination(self, endpoint, params, auth=False):
         config = self.extractor.config
 
-        ratings = config("ratings")
-        if ratings is None:
-            ratings = ("safe", "suggestive", "erotica", "pornographic")
-        params["contentRating[]"] = ratings
+        if "contentRating" not in params:
+            ratings = config("ratings")
+            if ratings is None:
+                ratings = ("safe", "suggestive", "erotica", "pornographic")
+            params["contentRating[]"] = ratings
         params["offset"] = 0
 
         api_params = config("api-parameters")
@@ -321,7 +392,7 @@ class MangadexAPI():
             params.update(api_params)
 
         while True:
-            data = self._call(endpoint, params)
+            data = self._call(endpoint, params, auth)
             yield from data["data"]
 
             params["offset"] = data["offset"] + data["limit"]
@@ -329,6 +400,6 @@ class MangadexAPI():
                 return
 
 
-@cache(maxage=28*86400, keyarg=0)
+@cache(maxage=90*86400, keyarg=0)
 def _refresh_token_cache(username):
     return None