New upstream version 1.26.9.upstream/1.26.9

1 files changed, 53 insertions, 20 deletions

diff --git a/gallery_dl/extractor/skeb.py b/gallery_dl/extractor/skeb.py
index 0b29ed0..38a2d16 100644
--- a/gallery_dl/extractor/skeb.py
+++ b/gallery_dl/extractor/skeb.py
@@ -7,7 +7,7 @@
 """Extractors for https://skeb.jp/"""
 
 from .common import Extractor, Message
-from .. import text
+from .. import text, exception
 import itertools
 
 
@@ -26,6 +26,19 @@ class SkebExtractor(Extractor):
     def _init(self):
         self.thumbnails = self.config("thumbnails", False)
         self.article = self.config("article", False)
+        self.headers = {"Accept": "application/json, text/plain, */*"}
+
+        if "Authorization" not in self.session.headers:
+            self.headers["Authorization"] = "Bearer null"
+
+    def request(self, url, **kwargs):
+        while True:
+            try:
+                return Extractor.request(self, url, **kwargs)
+            except exception.HttpError as exc:
+                if exc.status == 429 and "request_key" in exc.response.cookies:
+                    continue
+                raise
 
     def items(self):
         metadata = self.metadata()
@@ -42,6 +55,12 @@ class SkebExtractor(Extractor):
                 url = file["file_url"]
                 yield Message.Url, url, text.nameext_from_url(url, post)
 
+    def _items_users(self):
+        base = self.root + "/@"
+        for user in self.users():
+            user["_extractor"] = SkebUserExtractor
+            yield Message.Queue, base + user["screen_name"], user
+
     def posts(self):
         """Return post number"""
 
@@ -49,11 +68,11 @@ class SkebExtractor(Extractor):
         """Return additional metadata"""
 
     def _pagination(self, url, params):
-        headers = {"Authorization": "Bearer null"}
         params["offset"] = 0
 
         while True:
-            posts = self.request(url, params=params, headers=headers).json()
+            posts = self.request(
+                url, params=params, headers=self.headers).json()
 
             for post in posts:
                 parts = post["path"].split("/")
@@ -70,11 +89,24 @@ class SkebExtractor(Extractor):
                 return
             params["offset"] += 30
 
+    def _pagination_users(self, endpoint, params):
+        url = "{}/api{}".format(self.root, endpoint)
+        params["offset"] = 0
+        params["limit"] = 90
+
+        while True:
+            data = self.request(
+                url, params=params, headers=self.headers).json()
+            yield from data
+
+            if len(data) < params["limit"]:
+                return
+            params["offset"] += params["limit"]
+
     def _get_post_data(self, user_name, post_num):
         url = "{}/api/users/{}/works/{}".format(
             self.root, user_name, post_num)
-        headers = {"Authorization": "Bearer null"}
-        resp = self.request(url, headers=headers).json()
+        resp = self.request(url, headers=self.headers).json()
         creator = resp["creator"]
         post = {
             "post_id"          : resp["id"],
@@ -244,22 +276,23 @@ class SkebFollowingExtractor(SkebExtractor):
     pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/following_creators"
     example = "https://skeb.jp/@USER/following_creators"
 
-    def items(self):
-        for user in self.users():
-            url = "{}/@{}".format(self.root, user["screen_name"])
-            user["_extractor"] = SkebUserExtractor
-            yield Message.Queue, url, user
+    items = SkebExtractor._items_users
 
     def users(self):
-        url = "{}/api/users/{}/following_creators".format(
-            self.root, self.user_name)
-        params = {"sort": "date", "offset": 0, "limit": 90}
-        headers = {"Authorization": "Bearer null"}
+        endpoint = "/users/{}/following_creators".format(self.user_name)
+        params = {"sort": "date"}
+        return self._pagination_users(endpoint, params)
 
-        while True:
-            data = self.request(url, params=params, headers=headers).json()
-            yield from data
 
-            if len(data) < params["limit"]:
-                return
-            params["offset"] += params["limit"]
+class SkebFollowingUsersExtractor(SkebExtractor):
+    """Extractor for your followed users"""
+    subcategory = "following-users"
+    pattern = r"(?:https?://)?skeb\.jp/following_users()"
+    example = "https://skeb.jp/following_users"
+
+    items = SkebExtractor._items_users
+
+    def users(self):
+        endpoint = "/following_users"
+        params = {}
+        return self._pagination_users(endpoint, params)