diff options
Diffstat (limited to 'gallery_dl/extractor/sankaku.py')
| -rw-r--r-- | gallery_dl/extractor/sankaku.py | 56 |
1 files changed, 49 insertions, 7 deletions
diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index 2ce7f6c..3396e3a 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -53,12 +53,15 @@ class SankakuExtractor(BooruExtractor): url = "https://s.sankakucomplex.com" + url[url.index("/", 8):] return url - @staticmethod - def _prepare(post): + def _prepare(self, post): post["created_at"] = post["created_at"]["s"] post["date"] = text.parse_timestamp(post["created_at"]) post["tags"] = [tag["name"] for tag in post["tags"] if tag["name"]] post["tag_string"] = " ".join(post["tags"]) + post["_http_validate"] = self._check_expired + + def _check_expired(self, response): + return not response.history or '.com/expired.png' not in response.url def _extended_tags(self, post): tags = collections.defaultdict(list) @@ -219,7 +222,11 @@ class SankakuAPI(): def __init__(self, extractor): self.extractor = extractor - self.headers = {"Accept": "application/vnd.sankaku.api+json;v=2"} + self.headers = { + "Accept" : "application/vnd.sankaku.api+json;v=2", + "Origin" : extractor.root, + "Referer": extractor.root + "/", + } self.username, self.password = self.extractor._get_auth_info() if not self.username: @@ -253,11 +260,14 @@ class SankakuAPI(): for _ in range(5): self.authenticate() response = self.extractor.request( - url, params=params, headers=self.headers, fatal=False) + url, params=params, headers=self.headers, fatal=None) if response.status_code == 429: - self.extractor.wait( - until=response.headers.get("X-RateLimit-Reset")) + until = response.headers.get("X-RateLimit-Reset") + if not until and b"tags-limit" in response.content: + raise exception.StopExtraction("Search tag limit exceeded") + seconds = None if until else 60 + self.extractor.wait(until=until, seconds=seconds) continue data = response.json() @@ -278,9 +288,41 @@ class SankakuAPI(): params["lang"] = "en" params["limit"] = str(self.extractor.per_page) + refresh = self.extractor.config("refresh", False) + if refresh: + offset = expires = 0 + from time import time + while True: data = self._call(endpoint, params) - yield from data["data"] + + if refresh: + posts = data["data"] + if offset: + posts = util.advance(posts, offset) + + for post in posts: + if not expires: + url = post["file_url"] + if url: + expires = text.parse_int( + text.extract(url, "e=", "&")[0]) - 60 + + if 0 < expires <= time(): + self.extractor.log.debug("Refreshing download URLs") + expires = None + break + + offset += 1 + yield post + + if expires is None: + expires = 0 + continue + offset = expires = 0 + + else: + yield from data["data"] params["next"] = data["meta"]["next"] if not params["next"]: |