diff options
Diffstat (limited to 'gallery_dl/extractor/gelbooru.py')
| -rw-r--r-- | gallery_dl/extractor/gelbooru.py | 66 |
1 files changed, 59 insertions, 7 deletions
diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index 80b0ae1..e2173de 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -21,18 +21,21 @@ class GelbooruBase(): category = "gelbooru" basecategory = "booru" root = "https://gelbooru.com" + offset = 0 - def _api_request(self, params): + def _api_request(self, params, key="post"): + if "s" not in params: + params["s"] = "post" params["api_key"] = self.api_key params["user_id"] = self.user_id - url = self.root + "/index.php?page=dapi&s=post&q=index&json=1" + url = self.root + "/index.php?page=dapi&q=index&json=1" data = self.request(url, params=params).json() - if "post" not in data: + if key not in data: return () - posts = data["post"] + posts = data[key] if not isinstance(posts, list): return (posts,) return posts @@ -57,7 +60,7 @@ class GelbooruBase(): def _pagination_html(self, params): url = self.root + "/index.php" - params["pid"] = self.page_start * self.per_page + params["pid"] = self.offset data = {} while True: @@ -103,6 +106,10 @@ class GelbooruBase(): "body" : extr(note, 'data-body="', '"')[0], }) + def _skip_offset(self, num): + self.offset += num + return num + class GelbooruTagExtractor(GelbooruBase, gelbooru_v02.GelbooruV02TagExtractor): @@ -133,13 +140,14 @@ class GelbooruPoolExtractor(GelbooruBase, }), ) + skip = GelbooruBase._skip_offset + def metadata(self): url = self.root + "/index.php" self._params = { "page": "pool", "s" : "show", "id" : self.pool_id, - "pid" : self.page_start, } page = self.request(url, params=self._params).text @@ -158,8 +166,52 @@ class GelbooruPoolExtractor(GelbooruBase, class GelbooruFavoriteExtractor(GelbooruBase, gelbooru_v02.GelbooruV02FavoriteExtractor): + """Extractor for gelbooru favorites""" + per_page = 100 pattern = BASE_PATTERN + r"page=favorites&s=view&id=(\d+)" - test = ("https://gelbooru.com/index.php?page=favorites&s=view&id=12345",) + test = ("https://gelbooru.com/index.php?page=favorites&s=view&id=279415", { + "count": 3, + }) + + skip = GelbooruBase._skip_offset + + def posts(self): + # get number of favorites + params = { + "s" : "favorite", + "id" : self.favorite_id, + "limit": "1", + } + count = self._api_request(params, "@attributes")[0]["count"] + + if count <= self.offset: + return + pnum, last = divmod(count + 1, self.per_page) + + if self.offset >= last: + self.offset -= last + diff, self.offset = divmod(self.offset, self.per_page) + pnum -= diff + 1 + skip = self.offset + + # paginate over them in reverse + params["pid"] = pnum + params["limit"] = self.per_page + + while True: + favs = self._api_request(params, "favorite") + + favs.reverse() + if skip: + favs = favs[skip:] + skip = 0 + + for fav in favs: + yield from self._api_request({"id": fav["favorite"]}) + + params["pid"] -= 1 + if params["pid"] < 0: + return class GelbooruPostExtractor(GelbooruBase, |
