New upstream version 1.25.5.upstream/1.25.5

1 files changed, 49 insertions, 30 deletions

diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py
index 326b53b..5cfbf5c 100644
--- a/gallery_dl/extractor/danbooru.py
+++ b/gallery_dl/extractor/danbooru.py
@@ -70,6 +70,8 @@ class DanbooruExtractor(BaseExtractor):
                 continue
 
             text.nameext_from_url(url, post)
+            post["date"] = text.parse_datetime(
+                post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
 
             if post["extension"] == "zip":
                 if self.ugoira:
@@ -92,42 +94,47 @@ class DanbooruExtractor(BaseExtractor):
     def posts(self):
         return ()
 
-    def _pagination(self, endpoint, params, pages=False):
+    def _pagination(self, endpoint, params, prefix=None):
         url = self.root + endpoint
         params["limit"] = self.per_page
         params["page"] = self.page_start
 
+        first = True
         while True:
             posts = self.request(url, params=params).json()
-            if "posts" in posts:
+            if isinstance(posts, dict):
                 posts = posts["posts"]
 
-            if self.includes and posts:
-                if not pages and "only" not in params:
-                    params["page"] = "b{}".format(posts[0]["id"] + 1)
-                params["only"] = self.includes
-                data = {
-                    meta["id"]: meta
-                    for meta in self.request(url, params=params).json()
-                }
-                for post in posts:
-                    post.update(data[post["id"]])
-                params["only"] = None
-
-            yield from posts
+            if posts:
+                if self.includes:
+                    params_meta = {
+                        "only" : self.includes,
+                        "limit": len(posts),
+                        "tags" : "id:" + ",".join(str(p["id"]) for p in posts),
+                    }
+                    data = {
+                        meta["id"]: meta
+                        for meta in self.request(
+                            url, params=params_meta).json()
+                    }
+                    for post in posts:
+                        post.update(data[post["id"]])
+
+                if prefix == "a" and not first:
+                    posts.reverse()
+
+                yield from posts
 
             if len(posts) < self.threshold:
                 return
 
-            if pages:
+            if prefix:
+                params["page"] = "{}{}".format(prefix, posts[-1]["id"])
+            elif params["page"]:
                 params["page"] += 1
             else:
-                for post in reversed(posts):
-                    if "id" in post:
-                        params["page"] = "b{}".format(post["id"])
-                        break
-                else:
-                    return
+                params["page"] = 2
+            first = False
 
     def _ugoira_frames(self, post):
         data = self.request("{}/posts/{}.json?only=media_metadata".format(
@@ -153,7 +160,7 @@ BASE_PATTERN = DanbooruExtractor.update({
     "aibooru": {
         "root": None,
         "pattern": r"(?:safe.)?aibooru\.online",
-    }
+    },
 })
 
 
@@ -181,7 +188,7 @@ class DanbooruTagExtractor(DanbooruExtractor):
             "count": 12,
         }),
         ("https://aibooru.online/posts?tags=center_frills&z=1", {
-            "pattern": r"https://aibooru\.online/data/original"
+            "pattern": r"https://cdn\.aibooru\.online/original"
                        r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+",
             "count": ">= 3",
         }),
@@ -200,7 +207,21 @@ class DanbooruTagExtractor(DanbooruExtractor):
         return {"search_tags": self.tags}
 
     def posts(self):
-        return self._pagination("/posts.json", {"tags": self.tags})
+        prefix = "b"
+        for tag in self.tags.split():
+            if tag.startswith("order:"):
+                if tag == "order:id" or tag == "order:id_asc":
+                    prefix = "a"
+                elif tag == "order:id_desc":
+                    prefix = "b"
+                else:
+                    prefix = None
+            elif tag.startswith(
+                    ("id:", "md5", "ordfav:", "ordfavgroup:", "ordpool:")):
+                prefix = None
+                break
+
+        return self._pagination("/posts.json", {"tags": self.tags}, prefix)
 
 
 class DanbooruPoolExtractor(DanbooruExtractor):
@@ -234,7 +255,7 @@ class DanbooruPoolExtractor(DanbooruExtractor):
 
     def posts(self):
         params = {"tags": "pool:" + self.pool_id}
-        return self._pagination("/posts.json", params)
+        return self._pagination("/posts.json", params, "b")
 
 
 class DanbooruPostExtractor(DanbooruExtractor):
@@ -245,6 +266,7 @@ class DanbooruPostExtractor(DanbooruExtractor):
     test = (
         ("https://danbooru.donmai.us/posts/294929", {
             "content": "5e255713cbf0a8e0801dc423563c34d896bb9229",
+            "keyword": {"date": "dt:2008-08-12 04:46:05"},
         }),
         ("https://danbooru.donmai.us/posts/3613024", {
             "pattern": r"https?://.+\.zip$",
@@ -307,7 +329,4 @@ class DanbooruPopularExtractor(DanbooruExtractor):
         return {"date": date, "scale": scale}
 
     def posts(self):
-        if self.page_start is None:
-            self.page_start = 1
-        return self._pagination(
-            "/explore/posts/popular.json", self.params, True)
+        return self._pagination("/explore/posts/popular.json", self.params)