diff options
Diffstat (limited to 'gallery_dl/extractor')
| -rw-r--r-- | gallery_dl/extractor/__init__.py | 1 | ||||
| -rw-r--r-- | gallery_dl/extractor/chevereto.py | 2 | ||||
| -rw-r--r-- | gallery_dl/extractor/civitai.py | 29 | ||||
| -rw-r--r-- | gallery_dl/extractor/deviantart.py | 4 | ||||
| -rw-r--r-- | gallery_dl/extractor/kemonoparty.py | 25 | ||||
| -rw-r--r-- | gallery_dl/extractor/mangakakalot.py | 92 | ||||
| -rw-r--r-- | gallery_dl/extractor/manganelo.py | 179 | ||||
| -rw-r--r-- | gallery_dl/extractor/pixiv.py | 25 | ||||
| -rw-r--r-- | gallery_dl/extractor/scrolller.py | 7 | ||||
| -rw-r--r-- | gallery_dl/extractor/tumblr.py | 10 | ||||
| -rw-r--r-- | gallery_dl/extractor/twitter.py | 32 | ||||
| -rw-r--r-- | gallery_dl/extractor/weasyl.py | 2 |
12 files changed, 200 insertions, 208 deletions
diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 9a7ca53..2da471e 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -105,7 +105,6 @@ modules = [ "mangadex", "mangafox", "mangahere", - "mangakakalot", "manganelo", "mangapark", "mangaread", diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py index 600d231..dc963c5 100644 --- a/gallery_dl/extractor/chevereto.py +++ b/gallery_dl/extractor/chevereto.py @@ -78,6 +78,8 @@ class CheveretoImageExtractor(CheveretoExtractor): "id" : self.path.rpartition(".")[2], "url" : url, "album": text.extr(extr("Added to <a", "/a>"), ">", "<"), + "date" : text.parse_datetime(extr( + '<span title="', '"'), "%Y-%m-%d %H:%M:%S"), "user" : extr('username: "', '"'), } diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index 034a3c2..de8f86c 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -10,6 +10,7 @@ from .common import Extractor, Message from .. import text, util, exception +from ..cache import memcache import itertools import time @@ -49,10 +50,11 @@ class CivitaiExtractor(Extractor): if isinstance(metadata, str): metadata = metadata.split(",") elif not isinstance(metadata, (list, tuple)): - metadata = ("generation",) + metadata = ("generation", "version") self._meta_generation = ("generation" in metadata) + self._meta_version = ("version" in metadata) else: - self._meta_generation = False + self._meta_generation = self._meta_version = False def items(self): models = self.models() @@ -77,9 +79,12 @@ class CivitaiExtractor(Extractor): post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") data = { "post": post, - "user": post["user"], + "user": post.pop("user"), } - del post["user"] + if self._meta_version: + data["version"] = version = self.api.model_version( + post["modelVersionId"]).copy() + data["model"] = version.pop("model") yield Message.Directory, data for file in self._image_results(images): @@ -94,6 +99,18 @@ class CivitaiExtractor(Extractor): if self._meta_generation: image["generation"] = self.api.image_generationdata( image["id"]) + if self._meta_version: + if "modelVersionId" in image: + version_id = image["modelVersionId"] + else: + post = image["post"] = self.api.post( + image["postId"]) + post.pop("user", None) + version_id = post["modelVersionId"] + image["version"] = version = self.api.model_version( + version_id).copy() + image["model"] = version.pop("model") + image["date"] = text.parse_datetime( image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") text.nameext_from_url(url, image) @@ -464,6 +481,7 @@ class CivitaiRestAPI(): endpoint = "/v1/models/{}".format(model_id) return self._call(endpoint) + @memcache(keyarg=1) def model_version(self, model_version_id): endpoint = "/v1/model-versions/{}".format(model_version_id) return self._call(endpoint) @@ -504,7 +522,7 @@ class CivitaiTrpcAPI(): self.root = extractor.root + "/api/trpc/" self.headers = { "content-type" : "application/json", - "x-client-version": "5.0.542", + "x-client-version": "5.0.701", "x-client-date" : "", "x-client" : "web", "x-fingerprint" : "undefined", @@ -576,6 +594,7 @@ class CivitaiTrpcAPI(): params = {"id": int(model_id)} return self._call(endpoint, params) + @memcache(keyarg=1) def model_version(self, model_version_id): endpoint = "modelVersion.getById" params = {"id": int(model_version_id)} diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index ae475e2..37f57fe 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -868,7 +868,9 @@ x2="45.4107524%" y2="71.4898596%" id="app-root-3">\ yield self.api.deviation(deviation_uuid) def _unescape_json(self, json): - return json.replace('\\"', '"').replace("\\\\", "\\") + return json.replace('\\"', '"') \ + .replace("\\'", "'") \ + .replace("\\\\", "\\") class DeviantartUserExtractor(DeviantartExtractor): diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 79070ee..4893f19 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -317,11 +317,25 @@ class KemonopartyUserExtractor(KemonopartyExtractor): KemonopartyExtractor.__init__(self, match) def posts(self): + endpoint = self.config("endpoint") + if endpoint == "legacy": + endpoint = self.api.creator_posts_legacy + elif endpoint == "legacy+": + endpoint = self._posts_legacy_plus + else: + endpoint = self.api.creator_posts + _, _, service, creator_id, query = self.groups params = text.parse_query(query) - return self.api.creator_posts_legacy( - service, creator_id, - params.get("o"), params.get("q"), params.get("tag")) + return endpoint(service, creator_id, + params.get("o"), params.get("q"), params.get("tag")) + + def _posts_legacy_plus(self, service, creator_id, + offset=0, query=None, tags=None): + for post in self.api.creator_posts_legacy( + service, creator_id, offset, query, tags): + yield self.api.creator_post( + service, creator_id, post["id"])["post"] class KemonopartyPostsExtractor(KemonopartyExtractor): @@ -525,9 +539,10 @@ class KemonoAPI(): endpoint = "/file/" + file_hash return self._call(endpoint) - def creator_posts(self, service, creator_id, offset=0, query=None): + def creator_posts(self, service, creator_id, + offset=0, query=None, tags=None): endpoint = "/{}/user/{}".format(service, creator_id) - params = {"q": query, "o": offset} + params = {"q": query, "tag": tags, "o": offset} return self._pagination(endpoint, params, 50) def creator_posts_legacy(self, service, creator_id, diff --git a/gallery_dl/extractor/mangakakalot.py b/gallery_dl/extractor/mangakakalot.py deleted file mode 100644 index 9fc8681..0000000 --- a/gallery_dl/extractor/mangakakalot.py +++ /dev/null @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2020 Jake Mannens -# Copyright 2021-2023 Mike Fährmann -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Extractors for https://mangakakalot.tv/""" - -from .common import ChapterExtractor, MangaExtractor -from .. import text -import re - -BASE_PATTERN = r"(?:https?://)?(?:ww[\dw]?\.)?mangakakalot\.tv" - - -class MangakakalotBase(): - """Base class for mangakakalot extractors""" - category = "mangakakalot" - root = "https://ww8.mangakakalot.tv" - - -class MangakakalotChapterExtractor(MangakakalotBase, ChapterExtractor): - """Extractor for manga chapters from mangakakalot.tv""" - pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/chapter[_-][^/?#]+)" - example = "https://ww6.mangakakalot.tv/chapter/manga-ID/chapter-01" - - def __init__(self, match): - self.path = match.group(1) - ChapterExtractor.__init__(self, match, self.root + self.path) - - def metadata(self, page): - _ , pos = text.extract(page, '<span itemprop="title">', '<') - manga , pos = text.extract(page, '<span itemprop="title">', '<', pos) - info , pos = text.extract(page, '<span itemprop="title">', '<', pos) - author, pos = text.extract(page, '. Author:', ' already has ', pos) - - match = re.match( - r"(?:[Vv]ol\. *(\d+) )?" - r"[Cc]hapter *([^:]*)" - r"(?:: *(.+))?", info or "") - volume, chapter, title = match.groups() if match else ("", "", info) - chapter, sep, minor = chapter.partition(".") - - return { - "manga" : text.unescape(manga), - "title" : text.unescape(title) if title else "", - "author" : text.unescape(author).strip() if author else "", - "volume" : text.parse_int(volume), - "chapter" : text.parse_int(chapter), - "chapter_minor": sep + minor, - "lang" : "en", - "language" : "English", - } - - def images(self, page): - return [ - (url, None) - for url in text.extract_iter(page, '<img data-src="', '"') - ] - - -class MangakakalotMangaExtractor(MangakakalotBase, MangaExtractor): - """Extractor for manga from mangakakalot.tv""" - chapterclass = MangakakalotChapterExtractor - pattern = BASE_PATTERN + r"(/manga/[^/?#]+)" - example = "https://ww6.mangakakalot.tv/manga/manga-ID" - - def chapters(self, page): - data = {"lang": "en", "language": "English"} - data["manga"], pos = text.extract(page, "<h1>", "<") - author, pos = text.extract(page, "<li>Author(s) :", "</a>", pos) - data["author"] = text.remove_html(author) - - results = [] - for chapter in text.extract_iter(page, '<div class="row">', '</div>'): - url, pos = text.extract(chapter, '<a href="', '"') - title, pos = text.extract(chapter, '>', '</a>', pos) - data["title"] = title.partition(": ")[2] - data["date"] , pos = text.extract( - chapter, '<span title=" ', '"', pos) - - chapter, sep, minor = url.rpartition("/chapter-")[2].partition(".") - data["chapter"] = text.parse_int(chapter) - data["chapter_minor"] = sep + minor - - if url[0] == "/": - url = self.root + url - results.append((url, data.copy())) - return results diff --git a/gallery_dl/extractor/manganelo.py b/gallery_dl/extractor/manganelo.py index 232b98d..5e92aee 100644 --- a/gallery_dl/extractor/manganelo.py +++ b/gallery_dl/extractor/manganelo.py @@ -1,107 +1,128 @@ # -*- coding: utf-8 -*- +# Copyright 2020 Jake Mannens +# Copyright 2021-2025 Mike Fährmann +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for https://manganato.com/""" +"""Extractors for https://www.mangakakalot.gg/ and mirror sites""" -from .common import ChapterExtractor, MangaExtractor -from .. import text -import re +from .common import BaseExtractor, ChapterExtractor, MangaExtractor +from .. import text, util -BASE_PATTERN = ( - r"(?:https?://)?" - r"((?:chap|read|www\.|m\.)?mangan(?:at|el)o" - r"\.(?:to|com))" -) +class ManganeloExtractor(BaseExtractor): + basecategory = "manganelo" -class ManganeloBase(): - category = "manganelo" - root = "https://chapmanganato.com" - _match_chapter = None - def __init__(self, match): - domain, path = match.groups() - super().__init__(match, "https://" + domain + path) - - def _init(self): - if self._match_chapter is None: - ManganeloBase._match_chapter = re.compile( - r"(?:[Vv]ol\.?\s*(\d+)\s?)?" - r"[Cc]hapter\s*(\d+)([^:]*)" - r"(?::\s*(.+))?").match - - def _parse_chapter(self, info, manga, author, date=None): - match = self._match_chapter(info) - if match: - volume, chapter, minor, title = match.groups() - else: - volume = chapter = minor = "" - title = info - - return { - "manga" : manga, - "author" : author, - "date" : date, - "title" : text.unescape(title) if title else "", - "volume" : text.parse_int(volume), - "chapter" : text.parse_int(chapter), - "chapter_minor": minor, - "lang" : "en", - "language" : "English", - } +BASE_PATTERN = ManganeloExtractor.update({ + "nelomanga": { + "root" : "https://www.nelomanga.net", + "pattern": r"(?:www\.)?nelomanga\.net", + }, + "natomanga": { + "root" : "https://www.natomanga.com", + "pattern": r"(?:www\.)?natomanga\.com", + }, + "manganato": { + "root" : "https://www.manganato.gg", + "pattern": r"(?:www\.)?manganato\.gg", + }, + "mangakakalot": { + "root" : "https://www.mangakakalot.gg", + "pattern": r"(?:www\.)?mangakakalot\.gg", + }, +}) -class ManganeloChapterExtractor(ManganeloBase, ChapterExtractor): - """Extractor for manga chapters from manganelo.com""" - pattern = BASE_PATTERN + r"(/(?:manga-\w+|chapter/\w+)/chapter[-_][^/?#]+)" - example = "https://chapmanganato.com/manga-ID/chapter-01" +class ManganeloChapterExtractor(ManganeloExtractor, ChapterExtractor): + """Extractor for manganelo manga chapters""" + pattern = BASE_PATTERN + r"(/manga/[^/?#]+/chapter-[^/?#]+)" + example = "https://www.mangakakalot.gg/manga/MANGA_NAME/chapter-123" + + def __init__(self, match): + ManganeloExtractor.__init__(self, match) + self.gallery_url = self.root + self.groups[-1] def metadata(self, page): extr = text.extract_from(page) - extr('class="a-h"', ">") - manga = extr('title="', '"') - info = extr('title="', '"') - author = extr("- Author(s) : ", "</p>") - return self._parse_chapter( - info, text.unescape(manga), text.unescape(author)) + data = { + "date" : text.parse_datetime(extr( + '"datePublished": "', '"')[:19], "%Y-%m-%dT%H:%M:%S"), + "date_updated": text.parse_datetime(extr( + '"dateModified": "', '"')[:19], "%Y-%m-%dT%H:%M:%S"), + "manga_id" : text.parse_int(extr("comic_id =", ";")), + "chapter_id" : text.parse_int(extr("chapter_id =", ";")), + "manga" : extr("comic_name =", ";").strip('" '), + "lang" : "en", + "language" : "English", + } + + chapter_name = extr("chapter_name =", ";").strip('" ') + chapter, sep, minor = chapter_name.rpartition(" ")[2].partition(".") + data["chapter"] = text.parse_int(chapter) + data["chapter_minor"] = sep + minor + data["author"] = extr(". Author:", " already has ").strip() + + return data def images(self, page): - page = text.extr( - page, 'class="container-chapter-reader', 'class="container') + extr = text.extract_from(page) + cdns = util.json_loads(extr("var cdns =", ";"))[0] + imgs = util.json_loads(extr("var chapterImages =", ";")) + + if cdns[-1] != "/": + cdns += "/" + return [ - (url, None) - for url in text.extract_iter(page, '<img src="', '"') - if not url.endswith("/gohome.png") - ] or [ - (url, None) - for url in text.extract_iter( - page, '<img class="reader-content" src="', '"') + (cdns + path, None) + for path in imgs ] -class ManganeloMangaExtractor(ManganeloBase, MangaExtractor): - """Extractor for manga from manganelo.com""" +class ManganeloMangaExtractor(ManganeloExtractor, MangaExtractor): + """Extractor for manganelo manga""" chapterclass = ManganeloChapterExtractor - pattern = BASE_PATTERN + r"(/(?:manga[-/]|read_)\w+)/?$" - example = "https://manganato.com/manga-ID" + pattern = BASE_PATTERN + r"(/manga/[^/?#]+)$" + example = "https://www.mangakakalot.gg/manga/MANGA_NAME" - def chapters(self, page): - results = [] - append = results.append + def __init__(self, match): + ManganeloExtractor.__init__(self, match) + self.manga_url = self.root + self.groups[-1] + def chapters(self, page): extr = text.extract_from(page) + manga = text.unescape(extr("<h1>", "<")) - author = text.remove_html(extr("</i>Author(s) :</td>", "</tr>")) - - extr('class="row-content-chapter', '') - while True: - url = extr('class="chapter-name text-nowrap" href="', '"') - if not url: - return results - info = extr(">", "<") - date = extr('class="chapter-time text-nowrap" title="', '"') - append((url, self._parse_chapter(info, manga, author, date))) + author = text.remove_html(extr("<li>Author(s) :", "</a>")) + status = extr("<li>Status :", "<").strip() + update = text.parse_datetime(extr( + "<li>Last updated :", "<").strip(), "%b-%d-%Y %I:%M:%S %p") + tags = text.split_html(extr(">Genres :", "</li>"))[::2] + + results = [] + for chapter in text.extract_iter(page, '<div class="row">', '</div>'): + url, pos = text.extract(chapter, '<a href="', '"') + title, pos = text.extract(chapter, '>', '</a>', pos) + date, pos = text.extract(chapter, '<span title="', '"', pos) + chapter, sep, minor = url.rpartition("/chapter-")[2].partition("-") + + if url[0] == "/": + url = self.root + url + results.append((url, { + "manga" : manga, + "author" : author, + "status" : status, + "tags" : tags, + "date_updated": update, + "chapter" : text.parse_int(chapter), + "chapter_minor": (sep and ".") + minor, + "title" : title.partition(": ")[2], + "date" : text.parse_datetime(date, "%b-%d-%Y %H:%M"), + "lang" : "en", + "language": "English", + })) + return results diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index dfed1aa..c063216 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -866,16 +866,6 @@ class PixivNovelExtractor(PixivExtractor): embeds = self.config("embeds") covers = self.config("covers") - if embeds: - headers = { - "User-Agent" : "Mozilla/5.0", - "App-OS" : None, - "App-OS-Version": None, - "App-Version" : None, - "Referer" : self.root + "/", - "Authorization" : None, - } - novels = self.novels() if self.max_posts: novels = itertools.islice(novels, self.max_posts) @@ -935,15 +925,12 @@ class PixivNovelExtractor(PixivExtractor): if desktop: try: - novel_id = str(novel["id"]) - url = "{}/novel/show.php?id={}".format( - self.root, novel_id) - data = util.json_loads(text.extr( - self.request(url, headers=headers).text, - "id=\"meta-preload-data\" content='", "'")) - images = (data["novel"][novel_id] - ["textEmbeddedImages"]).values() - except Exception: + body = self._request_ajax("/novel/" + str(novel["id"])) + images = body["textEmbeddedImages"].values() + except Exception as exc: + self.log.warning( + "%s: Failed to get embedded novel images (%s: %s)", + novel["id"], exc.__class__.__name__, exc) images = () for image in images: diff --git a/gallery_dl/extractor/scrolller.py b/gallery_dl/extractor/scrolller.py index f97fa14..7bfc550 100644 --- a/gallery_dl/extractor/scrolller.py +++ b/gallery_dl/extractor/scrolller.py @@ -56,7 +56,12 @@ class ScrolllerExtractor(Extractor): files = [] for num, media in enumerate(album, 1): - src = max(media["mediaSources"], key=self._sort_key) + sources = media.get("mediaSources") + if not sources: + self.log.warning("%s/%s: Missing media file", + post.get("id"), num) + continue + src = max(sources, key=self._sort_key) src["num"] = num files.append(src) return files diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 6f2114e..a2cce83 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -474,8 +474,14 @@ class TumblrAPI(oauth.OAuth1API): board = False if board: - self.log.info("Run 'gallery-dl oauth:tumblr' " - "to access dashboard-only blogs") + if self.api_key is None: + self.log.info( + "Ensure your 'access-token' and " + "'access-token-secret' belong to the same " + "application as 'api-key' and 'api-secret'") + else: + self.log.info("Run 'gallery-dl oauth:tumblr' " + "to access dashboard-only blogs") raise exception.AuthorizationError(error) raise exception.NotFoundError("user or post") diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index e2fe000..896bf28 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1069,6 +1069,7 @@ class TwitterImageExtractor(Extractor): class TwitterAPI(): + client_transaction = None def __init__(self, extractor): self.extractor = extractor @@ -1101,6 +1102,7 @@ class TwitterAPI(): "x-csrf-token": csrf_token, "x-twitter-client-language": "en", "x-twitter-active-user": "yes", + "x-client-transaction-id": None, "Sec-Fetch-Dest": "empty", "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-origin", @@ -1503,12 +1505,38 @@ class TwitterAPI(): self.extractor.cookies.set( "gt", guest_token, domain=self.extractor.cookies_domain) + @cache(maxage=10800) + def _client_transaction(self): + self.log.info("Initializing client transaction keys") + + from .. import transaction_id + ct = transaction_id.ClientTransaction() + ct.initialize(self.extractor) + + # update 'x-csrf-token' header (#7467) + csrf_token = self.extractor.cookies.get( + "ct0", domain=self.extractor.cookies_domain) + if csrf_token: + self.headers["x-csrf-token"] = csrf_token + + return ct + + def _transaction_id(self, url, method="GET"): + if self.client_transaction is None: + TwitterAPI.client_transaction = self._client_transaction() + path = url[url.find("/", 8):] + self.headers["x-client-transaction-id"] = \ + self.client_transaction.generate_transaction_id(method, path) + def _call(self, endpoint, params, method="GET", auth=True, root=None): url = (root or self.root) + endpoint while True: - if not self.headers["x-twitter-auth-type"] and auth: - self._authenticate_guest() + if auth: + if self.headers["x-twitter-auth-type"]: + self._transaction_id(url, method) + else: + self._authenticate_guest() response = self.extractor.request( url, method=method, params=params, diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index ed2a395..9f6b021 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -72,7 +72,7 @@ class WeasylExtractor(Extractor): class WeasylSubmissionExtractor(WeasylExtractor): subcategory = "submission" - pattern = BASE_PATTERN + r"(?:~[\w~-]+/submissions|submission)/(\d+)" + pattern = BASE_PATTERN + r"(?:~[\w~-]+/submissions|submission|view)/(\d+)" example = "https://www.weasyl.com/~USER/submissions/12345/TITLE" def __init__(self, match): |