diff options
Diffstat (limited to 'gallery_dl/extractor/kemonoparty.py')
| -rw-r--r-- | gallery_dl/extractor/kemonoparty.py | 104 |
1 files changed, 67 insertions, 37 deletions
diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 10228b5..fd5a73a 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -6,7 +6,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for https://kemono.party/""" +"""Extractors for https://kemono.su/""" from .common import Extractor, Message from .. import text, util, exception @@ -23,11 +23,11 @@ HASH_PATTERN = r"/[0-9a-f]{2}/[0-9a-f]{2}/([0-9a-f]{64})" class KemonopartyExtractor(Extractor): """Base class for kemonoparty extractors""" category = "kemonoparty" - root = "https://kemono.party" + root = "https://kemono.su" directory_fmt = ("{category}", "{service}", "{user}") filename_fmt = "{id}_{title[:180]}_{num:>02}_{filename[:180]}.{extension}" archive_fmt = "{service}_{user}_{id}_{num}" - cookies_domain = ".kemono.party" + cookies_domain = ".kemono.su" def __init__(self, match): domain = match.group(1) @@ -39,6 +39,8 @@ class KemonopartyExtractor(Extractor): def _init(self): self.revisions = self.config("revisions") + if self.revisions: + self.revisions_unique = (self.revisions == "unique") self._prepare_ddosguard_cookies() self._find_inline = re.compile( r'src="(?:https?://(?:kemono|coomer)\.(?:party|su))?(/inline/[^"]+' @@ -162,7 +164,7 @@ class KemonopartyExtractor(Extractor): return post["attachments"] def _inline(self, post): - for path in self._find_inline(post["content"] or ""): + for path in self._find_inline(post.get("content") or ""): yield {"path": path, "name": path, "type": "inline"} def _build_file_generators(self, filetypes): @@ -222,8 +224,37 @@ class KemonopartyExtractor(Extractor): self.root, server) return self.request(url).json() - @memcache(keyarg=1) - def _post_revisions(self, url): + def _revisions_post(self, post, url): + post["revision_id"] = 0 + + try: + revs = self.request(url + "/revisions").json() + except exception.HttpError: + post["revision_hash"] = self._revision_hash(post) + post["revision_index"] = 1 + return (post,) + revs.insert(0, post) + + for rev in revs: + rev["revision_hash"] = self._revision_hash(rev) + + if self.revisions_unique: + uniq = [] + last = None + for rev in revs: + if last != rev["revision_hash"]: + last = rev["revision_hash"] + uniq.append(rev) + revs = uniq + + idx = len(revs) + for rev in revs: + rev["revision_index"] = idx + idx -= 1 + + return revs + + def _revisions_all(self, url): revs = self.request(url + "/revisions").json() idx = len(revs) @@ -240,7 +271,9 @@ class KemonopartyExtractor(Extractor): rev.pop("added", None) rev.pop("next", None) rev.pop("prev", None) + rev["file"] = rev["file"].copy() rev["file"].pop("name", None) + rev["attachments"] = [a.copy() for a in rev["attachments"]] for a in rev["attachments"]: a.pop("name", None) return util.sha1(self._json_dumps(rev)) @@ -252,10 +285,10 @@ def _validate(response): class KemonopartyUserExtractor(KemonopartyExtractor): - """Extractor for all posts from a kemono.party user listing""" + """Extractor for all posts from a kemono.su user listing""" subcategory = "user" pattern = USER_PATTERN + r"/?(?:\?([^#]+))?(?:$|[?#])" - example = "https://kemono.party/SERVICE/user/12345" + example = "https://kemono.su/SERVICE/user/12345" def __init__(self, match): _, _, service, user_id, self.query = match.groups() @@ -275,18 +308,9 @@ class KemonopartyUserExtractor(KemonopartyExtractor): if self.revisions: for post in posts: - post["revision_hash"] = self._revision_hash(post) - post["revision_id"] = 0 - post_url = "{}/post/{}".format(self.api_url, post["id"]) - try: - revs = self._post_revisions(post_url) - except exception.HttpError: - post["revision_index"] = 1 - yield post - else: - post["revision_index"] = len(revs) + 1 - yield post - yield from revs + post_url = "{}/api/v1/{}/user/{}/post/{}".format( + self.root, post["service"], post["user"], post["id"]) + yield from self._revisions_post(post, post_url) else: yield from posts @@ -295,11 +319,25 @@ class KemonopartyUserExtractor(KemonopartyExtractor): params["o"] += 50 +class KemonopartyPostsExtractor(KemonopartyExtractor): + """Extractor for kemono.su post listings""" + subcategory = "posts" + pattern = BASE_PATTERN + r"/posts(?:/?\?([^#]+))?" + example = "https://kemono.su/posts" + + def __init__(self, match): + KemonopartyExtractor.__init__(self, match) + self.query = match.group(3) + self.api_url = self.root + "/api/v1/posts" + + posts = KemonopartyUserExtractor.posts + + class KemonopartyPostExtractor(KemonopartyExtractor): - """Extractor for a single kemono.party post""" + """Extractor for a single kemono.su post""" subcategory = "post" pattern = USER_PATTERN + r"/post/([^/?#]+)(/revisions?(?:/(\d*))?)?" - example = "https://kemono.party/SERVICE/user/12345/post/12345" + example = "https://kemono.su/SERVICE/user/12345/post/12345" def __init__(self, match): _, _, service, user_id, post_id, self.revision, self.revision_id = \ @@ -314,18 +352,10 @@ class KemonopartyPostExtractor(KemonopartyExtractor): if not self.revision: post = self.request(self.api_url).json() if self.revisions: - post["revision_hash"] = self._revision_hash(post) - post["revision_id"] = 0 - try: - revs = self._post_revisions(self.api_url) - except exception.HttpError: - post["revision_index"] = 1 - else: - post["revision_index"] = len(revs) + 1 - return itertools.chain((post,), revs) + return self._revisions_post(post, self.api_url) return (post,) - revs = self._post_revisions(self.api_url) + revs = self._revisions_all(self.api_url) if not self.revision_id: return revs @@ -337,14 +367,14 @@ class KemonopartyPostExtractor(KemonopartyExtractor): class KemonopartyDiscordExtractor(KemonopartyExtractor): - """Extractor for kemono.party discord servers""" + """Extractor for kemono.su discord servers""" subcategory = "discord" directory_fmt = ("{category}", "discord", "{server}", "{channel_name|channel}") filename_fmt = "{id}_{num:>02}_{filename}.{extension}" archive_fmt = "discord_{server}_{id}_{num}" pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)" - example = "https://kemono.party/discord/server/12345#CHANNEL" + example = "https://kemono.su/discord/server/12345#CHANNEL" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -430,7 +460,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor): class KemonopartyDiscordServerExtractor(KemonopartyExtractor): subcategory = "discord-server" pattern = BASE_PATTERN + r"/discord/server/(\d+)$" - example = "https://kemono.party/discord/server/12345" + example = "https://kemono.su/discord/server/12345" def __init__(self, match): KemonopartyExtractor.__init__(self, match) @@ -445,10 +475,10 @@ class KemonopartyDiscordServerExtractor(KemonopartyExtractor): class KemonopartyFavoriteExtractor(KemonopartyExtractor): - """Extractor for kemono.party favorites""" + """Extractor for kemono.su favorites""" subcategory = "favorite" pattern = BASE_PATTERN + r"/favorites(?:/?\?([^#]+))?" - example = "https://kemono.party/favorites" + example = "https://kemono.su/favorites" def __init__(self, match): KemonopartyExtractor.__init__(self, match) |