New upstream version 1.26.8.upstream/1.26.8

1 files changed, 67 insertions, 37 deletions

diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py
index 10228b5..fd5a73a 100644
--- a/gallery_dl/extractor/kemonoparty.py
+++ b/gallery_dl/extractor/kemonoparty.py
@@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-"""Extractors for https://kemono.party/"""
+"""Extractors for https://kemono.su/"""
 
 from .common import Extractor, Message
 from .. import text, util, exception
@@ -23,11 +23,11 @@ HASH_PATTERN = r"/[0-9a-f]{2}/[0-9a-f]{2}/([0-9a-f]{64})"
 class KemonopartyExtractor(Extractor):
     """Base class for kemonoparty extractors"""
     category = "kemonoparty"
-    root = "https://kemono.party"
+    root = "https://kemono.su"
     directory_fmt = ("{category}", "{service}", "{user}")
     filename_fmt = "{id}_{title[:180]}_{num:>02}_{filename[:180]}.{extension}"
     archive_fmt = "{service}_{user}_{id}_{num}"
-    cookies_domain = ".kemono.party"
+    cookies_domain = ".kemono.su"
 
     def __init__(self, match):
         domain = match.group(1)
@@ -39,6 +39,8 @@ class KemonopartyExtractor(Extractor):
 
     def _init(self):
         self.revisions = self.config("revisions")
+        if self.revisions:
+            self.revisions_unique = (self.revisions == "unique")
         self._prepare_ddosguard_cookies()
         self._find_inline = re.compile(
             r'src="(?:https?://(?:kemono|coomer)\.(?:party|su))?(/inline/[^"]+'
@@ -162,7 +164,7 @@ class KemonopartyExtractor(Extractor):
         return post["attachments"]
 
     def _inline(self, post):
-        for path in self._find_inline(post["content"] or ""):
+        for path in self._find_inline(post.get("content") or ""):
             yield {"path": path, "name": path, "type": "inline"}
 
     def _build_file_generators(self, filetypes):
@@ -222,8 +224,37 @@ class KemonopartyExtractor(Extractor):
             self.root, server)
         return self.request(url).json()
 
-    @memcache(keyarg=1)
-    def _post_revisions(self, url):
+    def _revisions_post(self, post, url):
+        post["revision_id"] = 0
+
+        try:
+            revs = self.request(url + "/revisions").json()
+        except exception.HttpError:
+            post["revision_hash"] = self._revision_hash(post)
+            post["revision_index"] = 1
+            return (post,)
+        revs.insert(0, post)
+
+        for rev in revs:
+            rev["revision_hash"] = self._revision_hash(rev)
+
+        if self.revisions_unique:
+            uniq = []
+            last = None
+            for rev in revs:
+                if last != rev["revision_hash"]:
+                    last = rev["revision_hash"]
+                    uniq.append(rev)
+            revs = uniq
+
+        idx = len(revs)
+        for rev in revs:
+            rev["revision_index"] = idx
+            idx -= 1
+
+        return revs
+
+    def _revisions_all(self, url):
         revs = self.request(url + "/revisions").json()
 
         idx = len(revs)
@@ -240,7 +271,9 @@ class KemonopartyExtractor(Extractor):
         rev.pop("added", None)
         rev.pop("next", None)
         rev.pop("prev", None)
+        rev["file"] = rev["file"].copy()
         rev["file"].pop("name", None)
+        rev["attachments"] = [a.copy() for a in rev["attachments"]]
         for a in rev["attachments"]:
             a.pop("name", None)
         return util.sha1(self._json_dumps(rev))
@@ -252,10 +285,10 @@ def _validate(response):
 
 
 class KemonopartyUserExtractor(KemonopartyExtractor):
-    """Extractor for all posts from a kemono.party user listing"""
+    """Extractor for all posts from a kemono.su user listing"""
     subcategory = "user"
     pattern = USER_PATTERN + r"/?(?:\?([^#]+))?(?:$|[?#])"
-    example = "https://kemono.party/SERVICE/user/12345"
+    example = "https://kemono.su/SERVICE/user/12345"
 
     def __init__(self, match):
         _, _, service, user_id, self.query = match.groups()
@@ -275,18 +308,9 @@ class KemonopartyUserExtractor(KemonopartyExtractor):
 
             if self.revisions:
                 for post in posts:
-                    post["revision_hash"] = self._revision_hash(post)
-                    post["revision_id"] = 0
-                    post_url = "{}/post/{}".format(self.api_url, post["id"])
-                    try:
-                        revs = self._post_revisions(post_url)
-                    except exception.HttpError:
-                        post["revision_index"] = 1
-                        yield post
-                    else:
-                        post["revision_index"] = len(revs) + 1
-                        yield post
-                        yield from revs
+                    post_url = "{}/api/v1/{}/user/{}/post/{}".format(
+                        self.root, post["service"], post["user"], post["id"])
+                    yield from self._revisions_post(post, post_url)
             else:
                 yield from posts
 
@@ -295,11 +319,25 @@ class KemonopartyUserExtractor(KemonopartyExtractor):
             params["o"] += 50
 
 
+class KemonopartyPostsExtractor(KemonopartyExtractor):
+    """Extractor for kemono.su post listings"""
+    subcategory = "posts"
+    pattern = BASE_PATTERN + r"/posts(?:/?\?([^#]+))?"
+    example = "https://kemono.su/posts"
+
+    def __init__(self, match):
+        KemonopartyExtractor.__init__(self, match)
+        self.query = match.group(3)
+        self.api_url = self.root + "/api/v1/posts"
+
+    posts = KemonopartyUserExtractor.posts
+
+
 class KemonopartyPostExtractor(KemonopartyExtractor):
-    """Extractor for a single kemono.party post"""
+    """Extractor for a single kemono.su post"""
     subcategory = "post"
     pattern = USER_PATTERN + r"/post/([^/?#]+)(/revisions?(?:/(\d*))?)?"
-    example = "https://kemono.party/SERVICE/user/12345/post/12345"
+    example = "https://kemono.su/SERVICE/user/12345/post/12345"
 
     def __init__(self, match):
         _, _, service, user_id, post_id, self.revision, self.revision_id = \
@@ -314,18 +352,10 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
         if not self.revision:
             post = self.request(self.api_url).json()
             if self.revisions:
-                post["revision_hash"] = self._revision_hash(post)
-                post["revision_id"] = 0
-                try:
-                    revs = self._post_revisions(self.api_url)
-                except exception.HttpError:
-                    post["revision_index"] = 1
-                else:
-                    post["revision_index"] = len(revs) + 1
-                    return itertools.chain((post,), revs)
+                return self._revisions_post(post, self.api_url)
             return (post,)
 
-        revs = self._post_revisions(self.api_url)
+        revs = self._revisions_all(self.api_url)
         if not self.revision_id:
             return revs
 
@@ -337,14 +367,14 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
 
 
 class KemonopartyDiscordExtractor(KemonopartyExtractor):
-    """Extractor for kemono.party discord servers"""
+    """Extractor for kemono.su discord servers"""
     subcategory = "discord"
     directory_fmt = ("{category}", "discord", "{server}",
                      "{channel_name|channel}")
     filename_fmt = "{id}_{num:>02}_{filename}.{extension}"
     archive_fmt = "discord_{server}_{id}_{num}"
     pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)"
-    example = "https://kemono.party/discord/server/12345#CHANNEL"
+    example = "https://kemono.su/discord/server/12345#CHANNEL"
 
     def __init__(self, match):
         KemonopartyExtractor.__init__(self, match)
@@ -430,7 +460,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
 class KemonopartyDiscordServerExtractor(KemonopartyExtractor):
     subcategory = "discord-server"
     pattern = BASE_PATTERN + r"/discord/server/(\d+)$"
-    example = "https://kemono.party/discord/server/12345"
+    example = "https://kemono.su/discord/server/12345"
 
     def __init__(self, match):
         KemonopartyExtractor.__init__(self, match)
@@ -445,10 +475,10 @@ class KemonopartyDiscordServerExtractor(KemonopartyExtractor):
 
 
 class KemonopartyFavoriteExtractor(KemonopartyExtractor):
-    """Extractor for kemono.party favorites"""
+    """Extractor for kemono.su favorites"""
     subcategory = "favorite"
     pattern = BASE_PATTERN + r"/favorites(?:/?\?([^#]+))?"
-    example = "https://kemono.party/favorites"
+    example = "https://kemono.su/favorites"
 
     def __init__(self, match):
         KemonopartyExtractor.__init__(self, match)