From 243b2597edb922fe7e0b0d887e80bb7ebbe72ab7 Mon Sep 17 00:00:00 2001 From: Unit 193 Date: Sun, 7 Sep 2025 20:40:45 -0400 Subject: New upstream version 1.30.6. --- gallery_dl/downloader/ytdl.py | 17 +- gallery_dl/extractor/__init__.py | 2 + gallery_dl/extractor/bunkr.py | 54 +++--- gallery_dl/extractor/chevereto.py | 77 ++++++-- gallery_dl/extractor/comick.py | 47 ++++- gallery_dl/extractor/common.py | 2 +- gallery_dl/extractor/deviantart.py | 12 +- gallery_dl/extractor/erome.py | 21 ++- gallery_dl/extractor/fansly.py | 318 ++++++++++++++++++++++++++++++++++ gallery_dl/extractor/hentaifoundry.py | 5 +- gallery_dl/extractor/imagebam.py | 1 + gallery_dl/extractor/instagram.py | 23 ++- gallery_dl/extractor/kemono.py | 3 +- gallery_dl/extractor/khinsider.py | 1 + gallery_dl/extractor/reddit.py | 110 +++++++----- gallery_dl/extractor/shimmie2.py | 11 +- gallery_dl/extractor/tumblr.py | 61 +++---- gallery_dl/extractor/tungsten.py | 100 +++++++++++ gallery_dl/extractor/twitter.py | 44 +++-- gallery_dl/extractor/vk.py | 79 ++++++++- gallery_dl/extractor/zerochan.py | 3 +- gallery_dl/formatter.py | 1 + gallery_dl/option.py | 6 +- gallery_dl/version.py | 2 +- 24 files changed, 824 insertions(+), 176 deletions(-) create mode 100644 gallery_dl/extractor/fansly.py create mode 100644 gallery_dl/extractor/tungsten.py (limited to 'gallery_dl') diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py index 9ef8816..a56a6be 100644 --- a/gallery_dl/downloader/ytdl.py +++ b/gallery_dl/downloader/ytdl.py @@ -11,6 +11,7 @@ from .common import DownloaderBase from .. import ytdl, text from xml.etree import ElementTree +from http.cookiejar import Cookie import os @@ -85,7 +86,8 @@ class YoutubeDLDownloader(DownloaderBase): info_dict = self._extract_manifest( ytdl_instance, url, manifest, kwdict.pop("_ytdl_manifest_data", None), - kwdict.pop("_ytdl_manifest_headers", None)) + kwdict.pop("_ytdl_manifest_headers", None), + kwdict.pop("_ytdl_manifest_cookies", None)) else: info_dict = self._extract_info(ytdl_instance, url) except Exception as exc: @@ -194,10 +196,21 @@ class YoutubeDLDownloader(DownloaderBase): return ytdl.extract_info(url, download=False) def _extract_manifest(self, ytdl, url, manifest_type, manifest_data=None, - headers=None): + headers=None, cookies=None): extr = ytdl.get_info_extractor("Generic") video_id = extr._generic_id(url) + if cookies is not None: + if isinstance(cookies, dict): + cookies = cookies.items() + set_cookie = ytdl.cookiejar.set_cookie + for name, value in cookies: + set_cookie(Cookie( + 0, name, value, None, False, + "", False, False, "/", False, + False, None, False, None, None, {}, + )) + if manifest_type == "hls": if manifest_data is None: try: diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index aabaa93..574d1e2 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -56,6 +56,7 @@ modules = [ "exhentai", "facebook", "fanbox", + "fansly", "fantia", "fapello", "fapachi", @@ -190,6 +191,7 @@ modules = [ "tsumino", "tumblr", "tumblrgallery", + "tungsten", "twibooru", "twitter", "urlgalleries", diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index eba1678..cf5bce1 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -62,7 +62,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): root = "https://bunkr.si" root_dl = "https://get.bunkrr.su" root_api = "https://apidl.bunkr.ru" - archive_fmt = "{album_id}_{id|id_url}" + archive_fmt = "{album_id}_{id|id_url|slug}" pattern = BASE_PATTERN + r"/a/([^/?#]+)" example = "https://bunkr.si/a/ID" @@ -134,13 +134,13 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): def fetch_album(self, album_id): # album metadata - page = self.request(self.root + "/a/" + album_id).text + page = self.request(f"{self.root}/a/{album_id}?advanced=1").text title = text.unescape(text.unescape(text.extr( page, 'property="og:title" content="', '"'))) # files - items = list(text.extract_iter( - page, '
")) + items = text.extr( + page, "window.albumFiles = [", "").split("\n},\n") return self._extract_files(items), { "album_id" : album_id, @@ -156,30 +156,29 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): for item in items: try: - url = text.unescape(text.extr(item, ' href="', '"')) - if url[0] == "/": - url = self.root + url - - file = self._extract_file(url) - info = text.split_html(item) - if not file["name"]: - file["name"] = info[-3] - file["size"] = info[-2] - file["date"] = text.parse_datetime( - info[-1], "%H:%M:%S %d/%m/%Y") + data_id = text.extr(item, " id: ", ",").strip() + file = self._extract_file(data_id) + + file["name"] = util.json_loads(text.extr( + item, 'original:', ',\n').replace("\\'", "'")) + file["slug"] = util.json_loads(text.extr( + item, 'slug: ', ',\n')) + file["uuid"] = text.extr( + item, 'name: "', ".") + file["size"] = text.parse_int(text.extr( + item, "size: ", " ,\n")) + file["date"] = text.parse_datetime(text.extr( + item, 'timestamp: "', '"'), "%H:%M:%S %d/%m/%Y") yield file except exception.ControlException: raise except Exception as exc: self.log.error("%s: %s", exc.__class__.__name__, exc) - self.log.debug("", exc_info=exc) - - def _extract_file(self, webpage_url): - page = self.request(webpage_url).text - data_id = text.extr(page, 'data-file-id="', '"') - referer = self.root_dl + "/file/" + data_id + self.log.debug("%s", item, exc_info=exc) + def _extract_file(self, data_id): + referer = f"{self.root_dl}/file/{data_id}" headers = {"Referer": referer, "Origin": self.root_dl} data = self.request_json(self.endpoint, method="POST", headers=headers, json={"id": data_id}) @@ -190,14 +189,9 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): else: file_url = data["url"] - file_name = text.extr(page, "")[2] - fallback = text.extr(page, 'property="og:url" content="', '"') - return { "file" : file_url, - "name" : text.unescape(file_name), "id_url" : data_id, - "_fallback" : (fallback,) if fallback else (), "_http_headers" : {"Referer": referer}, "_http_validate": self._validate, } @@ -222,7 +216,13 @@ class BunkrMediaExtractor(BunkrAlbumExtractor): def fetch_album(self, album_id): try: - file = self._extract_file(self.root + album_id) + page = self.request(f"{self.root}{album_id}").text + data_id = text.extr(page, 'data-file-id="', '"') + file = self._extract_file(data_id) + file["name"] = text.unquote(text.unescape(text.extr( + page, "")[2])) + file["slug"] = album_id.rpartition("/")[2] + file["uuid"] = text.extr(page, "/thumbs/", ".") except Exception as exc: self.log.error("%s: %s", exc.__class__.__name__, exc) return (), {} diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py index 1da7e23..6ba4d08 100644 --- a/gallery_dl/extractor/chevereto.py +++ b/gallery_dl/extractor/chevereto.py @@ -15,7 +15,7 @@ from .. import text, util class CheveretoExtractor(BaseExtractor): """Base class for chevereto extractors""" basecategory = "chevereto" - directory_fmt = ("{category}", "{user}", "{album}",) + directory_fmt = ("{category}", "{user}", "{album}") archive_fmt = "{id}" def _init(self): @@ -39,7 +39,7 @@ class CheveretoExtractor(BaseExtractor): BASE_PATTERN = CheveretoExtractor.update({ "jpgfish": { - "root": "https://jpg5.su", + "root": "https://jpg6.su", "pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)", }, "imgkiwi": { @@ -54,7 +54,7 @@ BASE_PATTERN = CheveretoExtractor.update({ class CheveretoImageExtractor(CheveretoExtractor): - """Extractor for chevereto Images""" + """Extractor for chevereto images""" subcategory = "image" pattern = BASE_PATTERN + r"(/im(?:g|age)/[^/?#]+)" example = "https://jpg2.su/img/TITLE.ID" @@ -74,22 +74,66 @@ class CheveretoImageExtractor(CheveretoExtractor): url, b"seltilovessimpcity@simpcityhatesscrapers", fromhex=True) - image = { + file = { "id" : self.path.rpartition(".")[2], "url" : url, - "album": text.extr(extr("Added to "), ">", "<"), + "album": text.remove_html(extr( + "Added to ").rpartition(">")[2]), "date" : text.parse_datetime(extr( '', "—"), + "album": text.remove_html(extr( + "Added to ").rpartition(">")[2]), + "date" : text.parse_datetime(extr( + '" + c + ":", "<").replace(",", "")) data["description"] = text.unescape(extr( - "class='storyDescript'>", "", '
')).replace( + "\r\n", "\n") + path = extr('class="pdfLink" href="', '"') data["src"] = self.root + path data["index"] = text.parse_int(path.rsplit("/", 2)[1]) data["ratings"] = [text.unescape(r) for r in text.extract_iter(extr( diff --git a/gallery_dl/extractor/imagebam.py b/gallery_dl/extractor/imagebam.py index 171feea..abba9df 100644 --- a/gallery_dl/extractor/imagebam.py +++ b/gallery_dl/extractor/imagebam.py @@ -23,6 +23,7 @@ class ImagebamExtractor(Extractor): def _init(self): self.cookies.set("nsfw_inter", "1", domain="www.imagebam.com") + self.cookies.set("sfw_inter", "1", domain="www.imagebam.com") def _parse_image_page(self, path): page = self.request(self.root + path).text diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index fa60f91..9b8f8c9 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -242,8 +242,7 @@ class InstagramExtractor(Extractor): if self._warn_video_ua: self._warn_video_ua = False pattern = text.re( - r"AppleWebKit/537\.36 \(KHTML, like Gecko\) " - r"Chrome/\d+\.\d+\.\d+\.\d+ Safari/537\.36$") + r"Chrome/\d{3,}\.\d+\.\d+\.\d+(?!\d* Mobile)") if not pattern.search(self.session.headers["User-Agent"]): self.log.warning("Potentially lowered video quality " "due to non-Chrome User-Agent") @@ -568,6 +567,20 @@ class InstagramCollectionExtractor(InstagramExtractor): return self.api.user_collection(self.collection_id) +class InstagramStoriesTrayExtractor(InstagramExtractor): + """Extractor for your Instagram account's stories tray""" + subcategory = "stories-tray" + pattern = rf"{BASE_PATTERN}/stories/me/?$()" + example = "https://www.instagram.com/stories/me/" + + def items(self): + base = f"{self.root}/stories/id:" + for story in self.api.reels_tray(): + story["date"] = text.parse_timestamp(story["latest_reel_media"]) + story["_extractor"] = InstagramStoriesExtractor + yield Message.Queue, f"{base}{story['id']}/", story + + class InstagramStoriesExtractor(InstagramExtractor): """Extractor for Instagram stories""" subcategory = "stories" @@ -793,7 +806,11 @@ class InstagramRestAPI(): try: return self._call(endpoint, params=params)["reels_media"] except KeyError: - raise exception.AuthorizationError("Login required") + raise exception.AuthRequired("authenticated cookies") + + def reels_tray(self): + endpoint = "/v1/feed/reels_tray/" + return self._call(endpoint)["tray"] def tags_media(self, tag): for section in self.tags_sections(tag): diff --git a/gallery_dl/extractor/kemono.py b/gallery_dl/extractor/kemono.py index a5e1f6d..fc5972c 100644 --- a/gallery_dl/extractor/kemono.py +++ b/gallery_dl/extractor/kemono.py @@ -151,7 +151,8 @@ class KemonoExtractor(Extractor): file["extension"] = ext elif ext == "txt" and file["extension"] != "txt": file["_http_validate"] = _validate - elif ext in exts_archive: + elif ext in exts_archive or \ + ext == "bin" and file["extension"] in exts_archive: file["type"] = "archive" if archives: try: diff --git a/gallery_dl/extractor/khinsider.py b/gallery_dl/extractor/khinsider.py index 9c33d4f..f22d54e 100644 --- a/gallery_dl/extractor/khinsider.py +++ b/gallery_dl/extractor/khinsider.py @@ -63,6 +63,7 @@ class KhinsiderSoundtrackExtractor(AsynchronousMixin, Extractor): "date" : extr("Date Added: ", "<"), "type" : text.remove_html(extr("Album type: ", "")), "uploader": text.remove_html(extr("Uploaded by: ", "Description", "

").strip(), }} def _extract_tracks(self, page): diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index c87430b..9febda9 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -17,7 +17,7 @@ class RedditExtractor(Extractor): """Base class for reddit extractors""" category = "reddit" directory_fmt = ("{category}", "{subreddit}") - filename_fmt = "{id}{num:? //>02} {title[:220]}.{extension}" + filename_fmt = "{id}{num:? //>02} {title|link_title:[:220]}.{extension}" archive_fmt = "{filename}" cookies_domain = ".reddit.com" request_interval = 0.6 @@ -84,6 +84,12 @@ class RedditExtractor(Extractor): text.nameext_from_url(url, submission) yield Message.Url, url, submission + elif embeds and "media_metadata" in media: + for embed in self._extract_embed(submission): + submission["num"] += 1 + text.nameext_from_url(embed, submission) + yield Message.Url, embed, submission + elif media["is_video"]: if videos: text.nameext_from_url(url, submission) @@ -102,6 +108,12 @@ class RedditExtractor(Extractor): urls.append((url, submission)) if self.api.comments: + if comments and not submission: + submission = comments[0] + submission.setdefault("num", 0) + if not parentdir: + yield Message.Directory, submission + for comment in comments: html = comment["body_html"] or "" href = (' href="' in html) @@ -193,19 +205,26 @@ class RedditExtractor(Extractor): return for mid, data in meta.items(): - if data["status"] != "valid" or "s" not in data: + if data["status"] != "valid": self.log.warning( "embed %s: skipping item %s (status: %s)", submission["id"], mid, data.get("status")) continue - src = data["s"] - if url := src.get("u") or src.get("gif") or src.get("mp4"): - yield url.partition("?")[0].replace("/preview.", "/i.", 1) - else: - self.log.error( - "embed %s: unable to fetch download URL for item %s", - submission["id"], mid) - self.log.debug(src) + + if src := data.get("s"): + if url := src.get("u") or src.get("gif") or src.get("mp4"): + yield url.partition("?")[0].replace("/preview.", "/i.", 1) + else: + self.log.error( + "embed %s: unable to fetch download URL for item %s", + submission["id"], mid) + self.log.debug(src) + elif url := data.get("dashUrl"): + submission["_ytdl_manifest"] = "dash" + yield f"ytdl:{url}" + elif url := data.get("hlsUrl"): + submission["_ytdl_manifest"] = "hls" + yield f"ytdl:{url}" def _extract_video_ytdl(self, submission): return "https://www.reddit.com" + submission["permalink"] @@ -361,6 +380,7 @@ class RedditAPI(): Ref: https://www.reddit.com/dev/api/ """ + ROOT = "https://oauth.reddit.com" CLIENT_ID = "6N9uN0krSDE-ig" USER_AGENT = "Python:gallery-dl:0.8.4 (by /u/mikf1)" @@ -369,41 +389,50 @@ class RedditAPI(): self.log = extractor.log config = extractor.config + self.comments = text.parse_int(config("comments", 0)) self.morecomments = config("morecomments", False) + self._warn_429 = False - client_id = config("client-id") - if client_id is None: - self.client_id = self.CLIENT_ID - self.headers = {"User-Agent": self.USER_AGENT} + if config("api") == "rest": + self.root = "https://www.reddit.com" + self.headers = None + self.authenticate = util.noop + self.log.debug("Using REST API") else: - self.client_id = client_id - self.headers = {"User-Agent": config("user-agent")} + self.root = self.ROOT - if self.client_id == self.CLIENT_ID: - client_id = self.client_id - self._warn_429 = True - kind = "default" - else: - client_id = client_id[:5] + "*" * (len(client_id)-5) - self._warn_429 = False - kind = "custom" + client_id = config("client-id") + if client_id is None: + self.client_id = self.CLIENT_ID + self.headers = {"User-Agent": self.USER_AGENT} + else: + self.client_id = client_id + self.headers = {"User-Agent": config("user-agent")} - self.log.debug( - "Using %s API credentials (client-id %s)", kind, client_id) + if self.client_id == self.CLIENT_ID: + client_id = self.client_id + self._warn_429 = True + kind = "default" + else: + client_id = client_id[:5] + "*" * (len(client_id)-5) + kind = "custom" - token = config("refresh-token") - if token is None or token == "cache": - key = "#" + self.client_id - self.refresh_token = _refresh_token_cache(key) - else: - self.refresh_token = token + self.log.debug( + "Using %s API credentials (client-id %s)", kind, client_id) - if not self.refresh_token: - # allow downloading from quarantined subreddits (#2180) - extractor.cookies.set( - "_options", '%7B%22pref_quarantine_optin%22%3A%20true%7D', - domain=extractor.cookies_domain) + token = config("refresh-token") + if token is None or token == "cache": + key = "#" + self.client_id + self.refresh_token = _refresh_token_cache(key) + else: + self.refresh_token = token + + if not self.refresh_token: + # allow downloading from quarantined subreddits (#2180) + extractor.cookies.set( + "_options", '%7B%22pref_quarantine_optin%22%3A%20true%7D', + domain=extractor.cookies_domain) def submission(self, submission_id): """Fetch the (submission, comments)=-tuple for a submission id""" @@ -416,13 +445,11 @@ class RedditAPI(): def submissions_subreddit(self, subreddit, params): """Collect all (submission, comments)-tuples of a subreddit""" endpoint = subreddit + "/.json" - params["limit"] = 100 return self._pagination(endpoint, params) def submissions_user(self, user, params): """Collect all (submission, comments)-tuples posted by a user""" endpoint = "/user/" + user + "/.json" - params["limit"] = 100 return self._pagination(endpoint, params) def morechildren(self, link_id, children): @@ -477,7 +504,7 @@ class RedditAPI(): return "Bearer " + data["access_token"] def _call(self, endpoint, params): - url = "https://oauth.reddit.com" + endpoint + url = f"{self.root}{endpoint}" params["raw_json"] = "1" while True: @@ -522,6 +549,9 @@ class RedditAPI(): id_max = float("inf") date_min, date_max = self.extractor._get_date_min_max(0, 253402210800) + if limit := self.extractor.config("limit"): + params["limit"] = limit + while True: data = self._call(endpoint, params)["data"] diff --git a/gallery_dl/extractor/shimmie2.py b/gallery_dl/extractor/shimmie2.py index b988646..36b083b 100644 --- a/gallery_dl/extractor/shimmie2.py +++ b/gallery_dl/extractor/shimmie2.py @@ -79,9 +79,12 @@ BASE_PATTERN = Shimmie2Extractor.update({ }, "nozrip": { "root": "https://noz.rip/booru", - "base": "https://noz.rip", "pattern": r"noz\.rip/booru", }, + "thecollectionS": { + "root": "https://co.llection.pics", + "pattern": r"co\.llection\.pics", + }, }) + r"/(?:index\.php\?q=/?)?" @@ -160,10 +163,12 @@ class Shimmie2PostExtractor(Shimmie2Extractor): def posts(self): post_id = self.groups[-1] - url = f"{self.root}/post/view/{post_id}" + root = self.root + base = root if (pos := root.find("/", 8)) < 0 else root[:pos] + + url = f"{root}/post/view/{post_id}" page = self.request(url).text extr = text.extract_from(page) - base = self.config_instance("base", self.root) qt = self._quote_type(page) post = { diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 6eea76c..92fc831 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -31,15 +31,12 @@ class TumblrExtractor(Extractor): filename_fmt = "{category}_{blog_name}_{id}_{num:>02}.{extension}" archive_fmt = "{id}_{num}" - def __init__(self, match): - Extractor.__init__(self, match) - - if name := match[2]: - self.blog = name + ".tumblr.com" + def _init(self): + if name := self.groups[1]: + self.blog = f"{name}.tumblr.com" else: - self.blog = match[1] or match[3] + self.blog = self.groups[0] or self.groups[2] - def _init(self): self.api = TumblrAPI(self) self.types = self._setup_posttypes() self.avatar = self.config("avatar", False) @@ -287,14 +284,10 @@ class TumblrPostExtractor(TumblrExtractor): pattern = BASE_PATTERN + r"/(?:post/|image/)?(\d+)" example = "https://www.tumblr.com/BLOG/12345" - def __init__(self, match): - TumblrExtractor.__init__(self, match) - self.post_id = match[4] + def posts(self): self.reblogs = True self.date_min = 0 - - def posts(self): - return self.api.posts(self.blog, {"id": self.post_id}) + return self.api.posts(self.blog, {"id": self.groups[3]}) def _setup_posttypes(self): return POST_TYPES @@ -303,15 +296,13 @@ class TumblrPostExtractor(TumblrExtractor): class TumblrTagExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by tag""" subcategory = "tag" - pattern = BASE_PATTERN + r"/tagged/([^/?#]+)" + pattern = BASE_PATTERN + r"(?:/archive)?/tagged/([^/?#]+)" example = "https://www.tumblr.com/BLOG/tagged/TAG" - def __init__(self, match): - TumblrExtractor.__init__(self, match) - self.tag = text.unquote(match[4].replace("-", " ")) - def posts(self): - return self.api.posts(self.blog, {"tag": self.tag}) + self.kwdict["search_tags"] = tag = text.unquote( + self.groups[3].replace("-", " ")) + return self.api.posts(self.blog, {"tag": tag}) class TumblrDayExtractor(TumblrExtractor): @@ -320,21 +311,13 @@ class TumblrDayExtractor(TumblrExtractor): pattern = BASE_PATTERN + r"/day/(\d\d\d\d/\d\d/\d\d)" example = "https://www.tumblr.com/BLOG/day/1970/01/01" - def __init__(self, match): - TumblrExtractor.__init__(self, match) - year, month, day = match[4].split("/") - self.ordinal = date(int(year), int(month), int(day)).toordinal() - - def _init(self): - TumblrExtractor._init(self) - - self.date_min = ( - # 719163 == date(1970, 1, 1).toordinal() - (self.ordinal - 719163) * 86400) + def posts(self): + year, month, day = self.groups[3].split("/") + ordinal = date(int(year), int(month), int(day)).toordinal() + # 719163 == date(1970, 1, 1).toordinal() + self.date_min = (ordinal - 719163) * 86400 self.api.before = self.date_min + 86400 - - def posts(self): return self.api.posts(self.blog, {}) @@ -550,8 +533,11 @@ class TumblrAPI(oauth.OAuth1API): params["api_key"] = self.api_key strategy = self.extractor.config("pagination") - if not strategy and "offset" not in params: - strategy = "api" + if not strategy: + if params.get("before"): + strategy = "before" + elif "offset" not in params: + strategy = "api" while True: data = self._call(endpoint, params) @@ -573,10 +559,9 @@ class TumblrAPI(oauth.OAuth1API): endpoint = data["_links"]["next"]["href"] except KeyError: return - - params = None - if self.api_key: - endpoint += "&api_key=" + self.api_key + if params is not None and self.api_key: + endpoint = f"{endpoint}&api_key={self.api_key}" + params = None elif strategy == "before": if not posts: diff --git a/gallery_dl/extractor/tungsten.py b/gallery_dl/extractor/tungsten.py new file mode 100644 index 0000000..20d5a59 --- /dev/null +++ b/gallery_dl/extractor/tungsten.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://tungsten.run/""" + +from .common import Extractor, Message +from .. import text + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?tungsten\.run" + + +class TungstenExtractor(Extractor): + """Base class for tungsten extractors""" + category = "tungsten" + root = "https://tungsten.run" + directory_fmt = ("{category}", "{user[username]}") + filename_fmt = "{date} {title:?/ /}{uuid}.{extension}" + archive_fmt = "{uuid}" + + def items(self): + for post in self.posts(): + url = post["original_url"] + post["date"] = text.parse_datetime(post["created_at"]) + post["filename"] = url[url.rfind("/")+1:] + post["extension"] = "webp" + yield Message.Directory, post + yield Message.Url, url, post + + def _pagination(self, url, params): + params["page"] = 1 + params["per_page"] = 40 + + headers = { + "Origin": self.root, + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", + } + + while True: + data = self.request_json(url, params=params, headers=headers) + + yield from data + + if len(data) < params["per_page"]: + break + params["page"] += 1 + + +class TungstenPostExtractor(TungstenExtractor): + subcategory = "post" + pattern = rf"{BASE_PATTERN}/post/(\w+)" + example = "https://tungsten.run/post/AbCdEfGhIjKlMnOp" + + def posts(self): + url = f"{self.root}/post/{self.groups[0]}" + page = self.request(url).text + data = self._extract_nextdata(page) + return (data["props"]["pageProps"]["post"],) + + +class TungstenModelExtractor(TungstenExtractor): + subcategory = "model" + pattern = rf"{BASE_PATTERN}/model/(\w+)(?:/?\?model_version=(\w+))?" + example = "https://tungsten.run/model/AbCdEfGhIjKlM" + + def posts(self): + uuid_model, uuid_version = self.groups + + if uuid_version is None: + url = f"{self.root}/model/{uuid_model}/" + page = self.request(url).text + uuid_version = text.extr(page, '"modelVersionUUID":"', '"') + + url = "https://api.tungsten.run/v1/posts" + params = { + "sort" : "top_all_time", + "tweakable_only": "false", + "following" : "false", + "model_version_uuid": uuid_version, + } + return self._pagination(url, params) + + +class TungstenUserExtractor(TungstenExtractor): + subcategory = "user" + pattern = rf"{BASE_PATTERN}/user/([^/?#]+)" + example = "https://tungsten.run/user/USER/posts" + + def posts(self): + url = f"{self.root}/user/{self.groups[0]}" + page = self.request(url).text + uuid_user = text.extr(page, '"user":{"uuid":"', '"') + + url = f"https://api.tungsten.run/v1/users/{uuid_user}/posts" + params = {"sort": "top_all_time"} + return self._pagination(url, params) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index c928507..c919cb8 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -471,21 +471,35 @@ class TwitterExtractor(Extractor): except KeyError: pass + admin = creator = banner = None + try: + if results := com.get("admin_results"): + admin = results["result"]["core"]["screen_name"] + except Exception: + pass + try: + if results := com.get("creator_results"): + creator = results["result"]["core"]["screen_name"] + except Exception: + pass + try: + if results := com.get("custom_banner_media"): + banner = results["media_info"]["original_img_url"] + except Exception: + pass + self._user_cache[f"C#{cid}"] = cdata = { "id": text.parse_int(cid), - "name": com["name"], - "description": com["description"], - "date": text.parse_timestamp(com["created_at"] // 1000), - "nsfw": com["is_nsfw"], - "role": com["role"], - "member_count": com["member_count"], - "rules": [rule["name"] for rule in com["rules"]], - "admin": (admin := com.get("admin_results")) and - admin["result"]["core"]["screen_name"], # noqa: E131 - "creator": (creator := com.get("creator_results")) and - creator["result"]["core"]["screen_name"], # noqa: E131 - "banner": (banner := com.get("custom_banner_media")) and - banner["media_info"]["original_img_url"], # noqa: E131 + "name": com.get("name"), + "description": com.get("description"), + "date": text.parse_timestamp(com.get("created_at", 0) // 1000), + "nsfw": com.get("is_nsfw"), + "role": com.get("role"), + "member_count": com.get("member_count"), + "rules": [rule["name"] for rule in com.get("rules", ())], + "admin" : admin, + "creator": creator, + "banner" : banner, } return cdata @@ -512,8 +526,8 @@ class TwitterExtractor(Extractor): entities = legacy["entities"] self._user_cache[uid] = udata = { "id" : text.parse_int(uid), - "name" : core["screen_name"], - "nick" : core["name"], + "name" : core.get("screen_name"), + "nick" : core.get("name"), "location" : user["location"]["location"], "date" : text.parse_datetime( core["created_at"], "%a %b %d %H:%M:%S %z %Y"), diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index 75a0137..22d4b9a 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -72,6 +72,10 @@ class VkExtractor(Extractor): photo["width"] = photo["height"] = 0 photo["id"] = photo["id"].rpartition("_")[2] + photo["date"] = text.parse_timestamp(text.extr( + photo["date"], 'data-date="', '"')) + photo["description"] = text.unescape(text.extr( + photo.get("desc", ""), ">", "<")) photo.update(data) text.nameext_from_url(url, photo) @@ -108,6 +112,10 @@ class VkExtractor(Extractor): total = payload[1] photos = payload[3] + for i in range(len(photos)): + photos[i]["num"] = self.offset + i + 1 + photos[i]["count"] = total + offset_next = self.offset + len(photos) if offset_next >= total: # the last chunk of photos also contains the first few photos @@ -128,7 +136,7 @@ class VkPhotosExtractor(VkExtractor): subcategory = "photos" pattern = (BASE_PATTERN + r"/(?:" r"(?:albums|photos|id)(-?\d+)" - r"|(?!(?:album|tag)-?\d+_?)([^/?#]+))") + r"|(?!(?:album|tag|wall)-?\d+_?)([^/?#]+))") example = "https://vk.com/id12345" def __init__(self, match): @@ -179,17 +187,40 @@ class VkAlbumExtractor(VkExtractor): pattern = BASE_PATTERN + r"/album(-?\d+)_(\d+)$" example = "https://vk.com/album12345_00" - def __init__(self, match): - VkExtractor.__init__(self, match) - self.user_id, self.album_id = match.groups() - def photos(self): - return self._pagination(f"album{self.user_id}_{self.album_id}") + user_id, album_id = self.groups + return self._pagination(f"album{user_id}_{album_id}") def metadata(self): + user_id, album_id = self.groups + + url = f"{self.root}/album{user_id}_{album_id}" + page = self.request(url).text + desc = text.extr(page, 'name="og:description" value="', '"') + try: + album_name, user_name, photos = desc.rsplit(" - ", 2) + except ValueError: + if msg := text.extr( + page, '
Error
', + "

"): + msg = f" ('{text.remove_html(msg)[:-5]}')" + self.log.warning("%s_%s: Failed to extract metadata%s", + user_id, album_id, msg) + return {"user": {"id": user_id}, "album": {"id": album_id}} + return { - "user": {"id": self.user_id}, - "album": {"id": self.album_id}, + "user": { + "id" : user_id, + "nick" : text.unescape(user_name), + "name" : text.unescape(text.extr( + page, 'class="ui_crumb" href="/', '"')), + "group": user_id[0] == "-", + }, + "album": { + "id" : album_id, + "name" : text.unescape(album_name), + "count": text.parse_int(photos[:-7]) + }, } @@ -209,3 +240,35 @@ class VkTaggedExtractor(VkExtractor): def metadata(self): return {"user": {"id": self.user_id}} + + +class VkWallPostExtractor(VkExtractor): + """Extractor for a vk wall post""" + subcategory = "wall-post" + directory_fmt = ("{category}", "{user[id]}", "wall") + filename_fmt = "{wall[id]}_{num}.{extension}" + pattern = BASE_PATTERN + r"/wall(-?\d+)_(\d+)" + example = "https://vk.com/wall12345_123" + + def photos(self): + user_id, wall_id = self.groups + return self._pagination(f"wall{user_id}_{wall_id}") + + def metadata(self): + user_id, wall_id = self.groups + + url = f"{self.root}/wall{user_id}_{wall_id}" + page = self.request(url).text + desc = text.unescape( + text.extr(page, 'data-testid="post_description">', "
") or + text.extr(page, 'name="description" content="', '"')) + + return { + "user": { + "id": user_id, + }, + "wall": { + "id": wall_id, + "description": desc, + }, + } diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index fca8911..e1b4897 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -55,7 +55,8 @@ class ZerochanExtractor(BooruExtractor): "login" : "Login", } - response = self.request(url, method="POST", headers=headers, data=data) + response = self.request( + url, method="POST", headers=headers, data=data, expected=(500,)) if not response.history: raise exception.AuthenticationError() diff --git a/gallery_dl/formatter.py b/gallery_dl/formatter.py index 7a49049..b09203f 100644 --- a/gallery_dl/formatter.py +++ b/gallery_dl/formatter.py @@ -565,6 +565,7 @@ _CONVERSIONS = { "U": text.unescape, "H": lambda s: text.unescape(text.remove_html(s)), "g": text.slugify, + "R": text.re(r"https?://[^\s\"']+").findall, "W": text.sanitize_whitespace, "S": util.to_string, "s": str, diff --git a/gallery_dl/option.py b/gallery_dl/option.py index fd664e6..05cc9d3 100644 --- a/gallery_dl/option.py +++ b/gallery_dl/option.py @@ -222,10 +222,6 @@ class Formatter(argparse.HelpFormatter): def _format_usage(self, usage, actions, groups, prefix): return f"Usage: {self._prog} [OPTIONS] URL [URL...]\n" - def format_help(self): - return self._long_break_matcher.sub( - "\n\n", self._root_section.format_help()) - def _parse_option(opt): key, _, value = opt.partition("=") @@ -276,7 +272,7 @@ def build_parser(): help="Load external extractors from PATH", ) general.add_argument( - "--user-agent", + "-a", "--user-agent", dest="user-agent", metavar="UA", action=ConfigAction, help="User-Agent request header", ) diff --git a/gallery_dl/version.py b/gallery_dl/version.py index 8020352..187ef92 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,5 +6,5 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.30.5" +__version__ = "1.30.6" __variant__ = None -- cgit v1.2.3