New upstream version 1.23.2upstream/1.23.2

1 files changed, 49 insertions, 7 deletions

diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py
index 2ce7f6c..3396e3a 100644
--- a/gallery_dl/extractor/sankaku.py
+++ b/gallery_dl/extractor/sankaku.py
@@ -53,12 +53,15 @@ class SankakuExtractor(BooruExtractor):
             url = "https://s.sankakucomplex.com" + url[url.index("/", 8):]
         return url
 
-    @staticmethod
-    def _prepare(post):
+    def _prepare(self, post):
         post["created_at"] = post["created_at"]["s"]
         post["date"] = text.parse_timestamp(post["created_at"])
         post["tags"] = [tag["name"] for tag in post["tags"] if tag["name"]]
         post["tag_string"] = " ".join(post["tags"])
+        post["_http_validate"] = self._check_expired
+
+    def _check_expired(self, response):
+        return not response.history or '.com/expired.png' not in response.url
 
     def _extended_tags(self, post):
         tags = collections.defaultdict(list)
@@ -219,7 +222,11 @@ class SankakuAPI():
 
     def __init__(self, extractor):
         self.extractor = extractor
-        self.headers = {"Accept": "application/vnd.sankaku.api+json;v=2"}
+        self.headers = {
+            "Accept" : "application/vnd.sankaku.api+json;v=2",
+            "Origin" : extractor.root,
+            "Referer": extractor.root + "/",
+        }
 
         self.username, self.password = self.extractor._get_auth_info()
         if not self.username:
@@ -253,11 +260,14 @@ class SankakuAPI():
         for _ in range(5):
             self.authenticate()
             response = self.extractor.request(
-                url, params=params, headers=self.headers, fatal=False)
+                url, params=params, headers=self.headers, fatal=None)
 
             if response.status_code == 429:
-                self.extractor.wait(
-                    until=response.headers.get("X-RateLimit-Reset"))
+                until = response.headers.get("X-RateLimit-Reset")
+                if not until and b"tags-limit" in response.content:
+                    raise exception.StopExtraction("Search tag limit exceeded")
+                seconds = None if until else 60
+                self.extractor.wait(until=until, seconds=seconds)
                 continue
 
             data = response.json()
@@ -278,9 +288,41 @@ class SankakuAPI():
         params["lang"] = "en"
         params["limit"] = str(self.extractor.per_page)
 
+        refresh = self.extractor.config("refresh", False)
+        if refresh:
+            offset = expires = 0
+            from time import time
+
         while True:
             data = self._call(endpoint, params)
-            yield from data["data"]
+
+            if refresh:
+                posts = data["data"]
+                if offset:
+                    posts = util.advance(posts, offset)
+
+                for post in posts:
+                    if not expires:
+                        url = post["file_url"]
+                        if url:
+                            expires = text.parse_int(
+                                text.extract(url, "e=", "&")[0]) - 60
+
+                    if 0 < expires <= time():
+                        self.extractor.log.debug("Refreshing download URLs")
+                        expires = None
+                        break
+
+                    offset += 1
+                    yield post
+
+                if expires is None:
+                    expires = 0
+                    continue
+                offset = expires = 0
+
+            else:
+                yield from data["data"]
 
             params["next"] = data["meta"]["next"]
             if not params["next"]: