From 1f3ffe32342852fd9ea9e7704022488f3a1222bd Mon Sep 17 00:00:00 2001 From: Unit 193 Date: Sat, 7 Sep 2024 18:33:19 -0400 Subject: New upstream version 1.27.4. --- gallery_dl/__init__.py | 7 ++ gallery_dl/cookies.py | 9 +- gallery_dl/downloader/ytdl.py | 5 +- gallery_dl/extractor/batoto.py | 27 +++--- gallery_dl/extractor/bunkr.py | 50 +++++++---- gallery_dl/extractor/cyberdrop.py | 14 ++- gallery_dl/extractor/deviantart.py | 11 ++- gallery_dl/extractor/e621.py | 24 +++-- gallery_dl/extractor/exhentai.py | 2 +- gallery_dl/extractor/flickr.py | 38 +++++--- gallery_dl/extractor/furaffinity.py | 5 ++ gallery_dl/extractor/generic.py | 8 +- gallery_dl/extractor/gofile.py | 3 +- gallery_dl/extractor/hitomi.py | 1 + gallery_dl/extractor/instagram.py | 29 ++++-- gallery_dl/extractor/koharu.py | 25 ++++-- gallery_dl/extractor/lolisafe.py | 2 +- gallery_dl/extractor/newgrounds.py | 10 ++- gallery_dl/extractor/pixiv.py | 90 +++++++++++-------- gallery_dl/extractor/sankaku.py | 5 +- gallery_dl/extractor/sexcom.py | 19 ++++ gallery_dl/extractor/szurubooru.py | 8 +- gallery_dl/extractor/toyhouse.py | 3 +- gallery_dl/extractor/tumblr.py | 3 + gallery_dl/extractor/twitter.py | 29 +++++- gallery_dl/extractor/wikimedia.py | 124 ++++++++++++++++--------- gallery_dl/extractor/ytdl.py | 17 ++-- gallery_dl/formatter.py | 18 ++++ gallery_dl/job.py | 9 +- gallery_dl/option.py | 38 ++++++-- gallery_dl/path.py | 28 +++--- gallery_dl/postprocessor/__init__.py | 2 + gallery_dl/postprocessor/hash.py | 71 +++++++++++++++ gallery_dl/postprocessor/metadata.py | 34 ++++++- gallery_dl/postprocessor/rename.py | 91 +++++++++++++++++++ gallery_dl/postprocessor/ugoira.py | 169 +++++++++++++++++++++++------------ gallery_dl/util.py | 57 ++++++++---- gallery_dl/version.py | 2 +- gallery_dl/ytdl.py | 2 +- 39 files changed, 812 insertions(+), 277 deletions(-) create mode 100644 gallery_dl/postprocessor/hash.py create mode 100644 gallery_dl/postprocessor/rename.py (limited to 'gallery_dl') diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index 4b39c15..663fe99 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -238,6 +238,13 @@ def main(): return config.open_extern() else: + input_files = config.get((), "input-files") + if input_files: + for input_file in input_files: + if isinstance(input_file, str): + input_file = (input_file, None) + args.input_files.append(input_file) + if not args.urls and not args.input_files: parser.error( "The following arguments are required: URL\n" diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py index f017929..deb7c7b 100644 --- a/gallery_dl/cookies.py +++ b/gallery_dl/cookies.py @@ -179,11 +179,14 @@ def _firefox_cookies_database(profile=None, container=None): "{}".format(search_root)) _log_debug("Extracting cookies from %s", path) - if container == "none": + if not container or container == "none": container_id = False _log_debug("Only loading cookies not belonging to any container") - elif container: + elif container == "all": + container_id = None + + else: containers_path = os.path.join( os.path.dirname(path), "containers.json") @@ -207,8 +210,6 @@ def _firefox_cookies_database(profile=None, container=None): container)) _log_debug("Only loading cookies from container '%s' (ID %s)", container, container_id) - else: - container_id = None return path, container_id diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py index 87e7756..b3bec21 100644 --- a/gallery_dl/downloader/ytdl.py +++ b/gallery_dl/downloader/ytdl.py @@ -42,8 +42,9 @@ class YoutubeDLDownloader(DownloaderBase): if not ytdl_instance: try: module = ytdl.import_module(self.config("module")) - except ImportError as exc: - self.log.error("Cannot import module '%s'", exc.name) + except (ImportError, SyntaxError) as exc: + self.log.error("Cannot import module '%s'", + getattr(exc, "name", "")) self.log.debug("", exc_info=True) self.download = lambda u, p: False return False diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py index 2adb142..786acd9 100644 --- a/gallery_dl/extractor/batoto.py +++ b/gallery_dl/extractor/batoto.py @@ -51,28 +51,29 @@ class BatotoChapterExtractor(BatotoBase, ChapterExtractor): if not manga: manga = extr('link-hover">', "<") info = text.remove_html(extr('link-hover">', "", "", "", "") + current = text.extr(page, "", "").replace(",", "") self.log.debug("Image Limits: %s/%s", current, self.limits) self._remaining = self.limits - text.parse_int(current) diff --git a/gallery_dl/extractor/flickr.py b/gallery_dl/extractor/flickr.py index c94a110..1b4971c 100644 --- a/gallery_dl/extractor/flickr.py +++ b/gallery_dl/extractor/flickr.py @@ -75,11 +75,8 @@ class FlickrImageExtractor(FlickrExtractor): def items(self): photo = self.api.photos_getInfo(self.item_id) - if self.api.exif: - photo.update(self.api.photos_getExif(self.item_id)) - if self.api.contexts: - photo.update(self.api.photos_getAllContexts(self.item_id)) + self.api._extract_metadata(photo) if photo["media"] == "video" and self.api.videos: self.api._extract_video(photo) else: @@ -135,8 +132,13 @@ class FlickrAlbumExtractor(FlickrExtractor): def metadata(self): data = FlickrExtractor.metadata(self) - data["album"] = self.api.photosets_getInfo( - self.album_id, self.user["nsid"]) + try: + data["album"] = self.api.photosets_getInfo( + self.album_id, self.user["nsid"]) + except Exception: + data["album"] = {} + self.log.warning("%s: Unable to retrieve album metadata", + self.album_id) return data def photos(self): @@ -407,6 +409,8 @@ class FlickrAPI(oauth.OAuth1API): self.log.debug("Server response: %s", data) if data["code"] == 1: raise exception.NotFoundError(self.extractor.subcategory) + elif data["code"] == 2: + raise exception.AuthorizationError(msg) elif data["code"] == 98: raise exception.AuthenticationError(msg) elif data["code"] == 99: @@ -453,10 +457,7 @@ class FlickrAPI(oauth.OAuth1API): photo["date"] = text.parse_timestamp(photo["dateupload"]) photo["tags"] = photo["tags"].split() - if self.exif: - photo.update(self.photos_getExif(photo["id"])) - if self.contexts: - photo.update(self.photos_getAllContexts(photo["id"])) + self._extract_metadata(photo) photo["id"] = text.parse_int(photo["id"]) if "owner" in photo: @@ -512,6 +513,23 @@ class FlickrAPI(oauth.OAuth1API): photo["width"] = photo["height"] = 0 return photo + def _extract_metadata(self, photo): + if self.exif: + try: + photo.update(self.photos_getExif(photo["id"])) + except Exception as exc: + self.log.warning( + "Unable to retrieve 'exif' data for %s (%s: %s)", + photo["id"], exc.__class__.__name__, exc) + + if self.contexts: + try: + photo.update(self.photos_getAllContexts(photo["id"])) + except Exception as exc: + self.log.warning( + "Unable to retrieve 'contexts' data for %s (%s: %s)", + photo["id"], exc.__class__.__name__, exc) + @staticmethod def _clean_info(info): info["title"] = info["title"]["_content"] diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py index 3055426..d253582 100644 --- a/gallery_dl/extractor/furaffinity.py +++ b/gallery_dl/extractor/furaffinity.py @@ -179,6 +179,11 @@ class FuraffinityExtractor(Extractor): break self._favorite_id = text.parse_int(extr('data-fav-id="', '"')) yield post_id + + pos = page.find('type="submit">Next') + if pos >= 0: + path = text.rextract(page, '
', "") data['description'] = text.extr( diff --git a/gallery_dl/extractor/gofile.py b/gallery_dl/extractor/gofile.py index f0eb4e9..52b4ae6 100644 --- a/gallery_dl/extractor/gofile.py +++ b/gallery_dl/extractor/gofile.py @@ -47,8 +47,7 @@ class GofileFolderExtractor(Extractor): raise exception.AuthorizationError("Password required") num = 0 - for content_id in folder["childrenIds"]: - content = contents[content_id] + for content in contents.values(): content["folder"] = folder if content["type"] == "file": diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py index 9b74700..18df9df 100644 --- a/gallery_dl/extractor/hitomi.py +++ b/gallery_dl/extractor/hitomi.py @@ -89,6 +89,7 @@ class HitomiGalleryExtractor(GalleryExtractor): path = ext = "webp" ihash = image["hash"] idata = text.nameext_from_url(image["name"]) + idata["extension_original"] = idata["extension"] if ext: idata["extension"] = ext diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index c05fe72..422c865 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -12,6 +12,7 @@ from .common import Extractor, Message from .. import text, util, exception from ..cache import cache, memcache +import itertools import binascii import json import re @@ -57,12 +58,17 @@ class InstagramExtractor(Extractor): data = self.metadata() videos = self.config("videos", True) previews = self.config("previews", False) + max_posts = self.config("max-posts") video_headers = {"User-Agent": "Mozilla/5.0"} order = self.config("order-files") reverse = order[0] in ("r", "d") if order else False - for post in self.posts(): + posts = self.posts() + if max_posts: + posts = itertools.islice(posts, max_posts) + + for post in posts: if "__typename" in post: post = self._parse_post_graphql(post) @@ -159,15 +165,19 @@ class InstagramExtractor(Extractor): if "title" in post: data["highlight_title"] = post["title"] if "created_at" in post: - data["date"] = text.parse_timestamp(post.get("created_at")) + data["post_date"] = data["date"] = text.parse_timestamp( + post.get("created_at")) else: # regular image/video post + date = text.parse_timestamp(post.get("taken_at")) data = { "post_id" : post["pk"], "post_shortcode": post["code"], + "post_url": "{}/p/{}/".format(self.root, post["code"]), + "post_date": date, + "date": date, "likes": post.get("like_count", 0), "pinned": post.get("timeline_pinned_user_ids", ()), - "date": text.parse_timestamp(post.get("taken_at")), "liked": post.get("has_liked", False), } @@ -206,7 +216,6 @@ class InstagramExtractor(Extractor): data["owner_id"] = owner["pk"] data["username"] = owner.get("username") data["fullname"] = owner.get("full_name") - data["post_url"] = "{}/p/{}/".format(self.root, data["post_shortcode"]) data["_files"] = files = [] for num, item in enumerate(items, 1): @@ -269,7 +278,6 @@ class InstagramExtractor(Extractor): owner = post["owner"] data = { "typename" : typename, - "date" : text.parse_timestamp(post["taken_at_timestamp"]), "likes" : post["edge_media_preview_like"]["count"], "liked" : post.get("viewer_has_liked", False), "pinned" : pinned, @@ -279,11 +287,13 @@ class InstagramExtractor(Extractor): "post_id" : post["id"], "post_shortcode": post["shortcode"], "post_url" : "{}/p/{}/".format(self.root, post["shortcode"]), + "post_date" : text.parse_timestamp(post["taken_at_timestamp"]), "description": text.parse_unicode_escapes("\n".join( edge["node"]["text"] for edge in post["edge_media_to_caption"]["edges"] )), } + data["date"] = data["post_date"] tags = self._find_tags(data["description"]) if tags: @@ -313,6 +323,7 @@ class InstagramExtractor(Extractor): media = { "num": num, "media_id" : node["id"], + "date" : data["date"], "shortcode" : (node.get("shortcode") or shortcode_from_id(node["id"])), "display_url": node["display_url"], @@ -328,6 +339,7 @@ class InstagramExtractor(Extractor): dimensions = post["dimensions"] media = { "media_id" : post["id"], + "date" : data["date"], "shortcode" : post["shortcode"], "display_url": post["display_url"], "video_url" : post.get("video_url"), @@ -378,7 +390,11 @@ class InstagramExtractor(Extractor): "full_name": user["full_name"]}) def _init_cursor(self): - return self.config("cursor") or None + cursor = self.config("cursor", True) + if not cursor: + self._update_cursor = util.identity + elif isinstance(cursor, str): + return cursor def _update_cursor(self, cursor): self.log.debug("Cursor: %s", cursor) @@ -418,6 +434,7 @@ class InstagramUserExtractor(InstagramExtractor): base = "{}/{}/".format(self.root, self.item) stories = "{}/stories/{}/".format(self.root, self.item) return self._dispatch_extractors(( + (InstagramInfoExtractor , base + "info/"), (InstagramAvatarExtractor , base + "avatar/"), (InstagramStoriesExtractor , stories), (InstagramHighlightsExtractor, base + "highlights/"), diff --git a/gallery_dl/extractor/koharu.py b/gallery_dl/extractor/koharu.py index 979b1a2..cacf504 100644 --- a/gallery_dl/extractor/koharu.py +++ b/gallery_dl/extractor/koharu.py @@ -161,16 +161,29 @@ class KoharuGalleryExtractor(KoharuExtractor, GalleryExtractor): return results def _select_format(self, formats): - if not self.fmt or self.fmt == "original": - fmtid = "0" + fmt = self.fmt + + if not fmt or fmt == "best": + fmtids = ("0", "1600", "1280", "980", "780") + elif isinstance(fmt, str): + fmtids = fmt.split(",") + elif isinstance(fmt, list): + fmtids = fmt else: - fmtid = str(self.fmt) + fmtids = (str(self.fmt),) - try: - fmt = formats[fmtid] - except KeyError: + for fmtid in fmtids: + try: + fmt = formats[fmtid] + if fmt["id"]: + break + except KeyError: + self.log.debug("%s: Format %s is not available", + self.groups[0], fmtid) + else: raise exception.NotFoundError("format") + self.log.debug("%s: Selected format %s", self.groups[0], fmtid) fmt["w"] = fmtid return fmt diff --git a/gallery_dl/extractor/lolisafe.py b/gallery_dl/extractor/lolisafe.py index 3d7d685..117b88b 100644 --- a/gallery_dl/extractor/lolisafe.py +++ b/gallery_dl/extractor/lolisafe.py @@ -34,7 +34,7 @@ class LolisafeAlbumExtractor(LolisafeExtractor): def __init__(self, match): LolisafeExtractor.__init__(self, match) - self.album_id = match.group(match.lastindex) + self.album_id = self.groups[-1] def _init(self): domain = self.config("domain") diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index ecd6619..5fc0ce5 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -171,15 +171,17 @@ class NewgroundsExtractor(Extractor): if self.flash: url += "/format/flash" - with self.request(url, fatal=False) as response: - if response.status_code >= 400: - return {} - page = response.text + response = self.request(url, fatal=False) + page = response.text pos = page.find('id="adults_only"') if pos >= 0: msg = text.extract(page, 'class="highlight">', '<', pos)[0] self.log.warning('"%s"', msg) + return {} + + if response.status_code >= 400: + return {} extr = text.extract_from(page) data = extract_data(extr, post_url) diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index d732894..3479b88 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -94,12 +94,39 @@ class PixivExtractor(Extractor): work.get("id"), exc.message) continue - url = ugoira["zip_urls"]["medium"].replace( - "_ugoira600x600", "_ugoira1920x1080") - work["frames"] = ugoira["frames"] + url = ugoira["zip_urls"]["medium"] + work["frames"] = frames = ugoira["frames"] work["date_url"] = self._date_from_url(url) work["_http_adjust_extension"] = False - yield Message.Url, url, text.nameext_from_url(url, work) + + if self.load_ugoira == "original": + base, sep, _ = url.rpartition("_ugoira") + base = base.replace( + "/img-zip-ugoira/", "/img-original/", 1) + sep + + for ext in ("jpg", "png", "gif"): + try: + url = ("{}0.{}".format(base, ext)) + self.request(url, method="HEAD") + break + except exception.HttpError: + pass + else: + self.log.warning( + "Unable to find Ugoira frame URLs (%s)", + work.get("id")) + continue + + for num, frame in enumerate(frames): + url = ("{}{}.{}".format(base, num, ext)) + work["num"] = work["_ugoira_frame_index"] = num + work["suffix"] = "_p{:02}".format(num) + text.nameext_from_url(url, work) + yield Message.Url, url, work + + else: + url = url.replace("_ugoira600x600", "_ugoira1920x1080") + yield Message.Url, url, text.nameext_from_url(url, work) elif work["page_count"] == 1: url = meta_single_page["original_image_url"] @@ -551,9 +578,6 @@ class PixivSeriesExtractor(PixivExtractor): directory_fmt = ("{category}", "{user[id]} {user[account]}", "{series[id]} {series[title]}") filename_fmt = "{num_series:>03}_{id}_p{num}.{extension}" - cookies_domain = ".pixiv.net" - browser = "firefox" - tls12 = False pattern = BASE_PATTERN + r"/user/(\d+)/series/(\d+)" example = "https://www.pixiv.net/user/12345/series/12345" @@ -562,34 +586,18 @@ class PixivSeriesExtractor(PixivExtractor): self.user_id, self.series_id = match.groups() def works(self): - url = self.root + "/ajax/series/" + self.series_id - params = {"p": 1} - headers = { - "Accept": "application/json", - "Referer": "{}/user/{}/series/{}".format( - self.root, self.user_id, self.series_id), - "Alt-Used": "www.pixiv.net", - } + series = None - while True: - data = self.request(url, params=params, headers=headers).json() - body = data["body"] - page = body["page"] - - series = body["extraData"]["meta"] - series["id"] = self.series_id - series["total"] = page["total"] - series["title"] = text.extr(series["title"], '"', '"') - - for info in page["series"]: - work = self.api.illust_detail(info["workId"]) - work["num_series"] = info["order"] - work["series"] = series - yield work - - if len(page["series"]) < 10: - return - params["p"] += 1 + for work in self.api.illust_series(self.series_id): + if series is None: + series = self.api.data + series["total"] = num_series = series.pop("series_work_count") + else: + num_series -= 1 + + work["num_series"] = num_series + work["series"] = series + yield work class PixivNovelExtractor(PixivExtractor): @@ -916,6 +924,11 @@ class PixivAppAPI(): params = {"illust_id": illust_id} return self._pagination("/v2/illust/related", params) + def illust_series(self, series_id, offset=0): + params = {"illust_series_id": series_id, "offset": offset} + return self._pagination("/v1/illust/series", params, + key_data="illust_series_detail") + def novel_bookmark_detail(self, novel_id): params = {"novel_id": novel_id} return self._call( @@ -1013,10 +1026,15 @@ class PixivAppAPI(): raise exception.StopExtraction("API request failed: %s", error) - def _pagination(self, endpoint, params, key="illusts"): + def _pagination(self, endpoint, params, + key_items="illusts", key_data=None): while True: data = self._call(endpoint, params) - yield from data[key] + + if key_data: + self.data = data.get(key_data) + key_data = None + yield from data[key_items] if not data["next_url"]: return diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index ad3efa7..7db8172 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -66,7 +66,8 @@ class SankakuExtractor(BooruExtractor): def _prepare(self, post): post["created_at"] = post["created_at"]["s"] post["date"] = text.parse_timestamp(post["created_at"]) - post["tags"] = [tag["name"] for tag in post["tags"] if tag["name"]] + post["tags"] = [tag["name"].lower().replace(" ", "_") + for tag in post["tags"] if tag["name"]] post["tag_string"] = " ".join(post["tags"]) post["_http_validate"] = self._check_expired @@ -79,7 +80,7 @@ class SankakuExtractor(BooruExtractor): for tag in post["tags"]: name = tag["name"] if name: - tags[types[tag["type"]]].append(name) + tags[types[tag["type"]]].append(name.lower().replace(" ", "_")) for key, value in tags.items(): post["tags_" + key] = value post["tag_string_" + key] = " ".join(value) diff --git a/gallery_dl/extractor/sexcom.py b/gallery_dl/extractor/sexcom.py index 80f2aea..7708b5c 100644 --- a/gallery_dl/extractor/sexcom.py +++ b/gallery_dl/extractor/sexcom.py @@ -152,6 +152,25 @@ class SexcomPinsExtractor(SexcomExtractor): return self._pagination(url) +class SexcomLikesExtractor(SexcomExtractor): + """Extractor for a user's liked pins on www.sex.com""" + subcategory = "likes" + directory_fmt = ("{category}", "{user}", "Likes") + pattern = r"(?:https?://)?(?:www\.)?sex\.com/user/([^/?#]+)/likes/" + example = "https://www.sex.com/user/USER/likes/" + + def __init__(self, match): + SexcomExtractor.__init__(self, match) + self.user = match.group(1) + + def metadata(self): + return {"user": text.unquote(self.user)} + + def pins(self): + url = "{}/user/{}/likes/".format(self.root, self.user) + return self._pagination(url) + + class SexcomBoardExtractor(SexcomExtractor): """Extractor for pins from a board on www.sex.com""" subcategory = "board" diff --git a/gallery_dl/extractor/szurubooru.py b/gallery_dl/extractor/szurubooru.py index bba1ece..b6917cc 100644 --- a/gallery_dl/extractor/szurubooru.py +++ b/gallery_dl/extractor/szurubooru.py @@ -86,6 +86,7 @@ BASE_PATTERN = SzurubooruExtractor.update({ "bcbnsfw": { "root": "https://booru.bcbnsfw.space", "pattern": r"booru\.bcbnsfw\.space", + "query-all": "*", }, "snootbooru": { "root": "https://snootbooru.com", @@ -110,7 +111,12 @@ class SzurubooruTagExtractor(SzurubooruExtractor): return {"search_tags": self.query} def posts(self): - return self._pagination("/posts/", {"query": self.query}) + if self.query.strip(): + query = self.query + else: + query = self.config_instance("query-all") + + return self._pagination("/posts/", {"query": query}) class SzurubooruPostExtractor(SzurubooruExtractor): diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index 64fa951..44d87ee 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -123,4 +123,5 @@ class ToyhouseImageExtractor(ToyhouseExtractor): def posts(self): url = "{}/~images/{}".format(self.root, self.user) - return (self._parse_post(self.request(url).text, ' '%s'", name_old, name_new) + os.replace(path_old, path_new) + + def _apply_pathfmt(self, pathfmt): + return pathfmt.build_filename(pathfmt.kwdict) + + def _apply_format(self, format_string): + fmt = formatter.parse(format_string).format_map + + def apply(pathfmt): + return pathfmt.clean_path(pathfmt.clean_segment(fmt( + pathfmt.kwdict))) + + return apply + + +__postprocessor__ = RenamePP diff --git a/gallery_dl/postprocessor/ugoira.py b/gallery_dl/postprocessor/ugoira.py index 9e60ce2..f053afa 100644 --- a/gallery_dl/postprocessor/ugoira.py +++ b/gallery_dl/postprocessor/ugoira.py @@ -36,7 +36,8 @@ class UgoiraPP(PostProcessor): self.delete = not options.get("keep-files", False) self.repeat = options.get("repeat-last-frame", True) self.mtime = options.get("mtime", True) - self.uniform = False + self.skip = options.get("skip", True) + self.uniform = self._convert_zip = self._convert_files = False ffmpeg = options.get("ffmpeg-location") self.ffmpeg = util.expand_path(ffmpeg) if ffmpeg else "ffmpeg" @@ -90,33 +91,44 @@ class UgoiraPP(PostProcessor): if self.prevent_odd: args += ("-vf", "crop=iw-mod(iw\\,2):ih-mod(ih\\,2)") - job.register_hooks( - {"prepare": self.prepare, "file": self.convert}, options) + job.register_hooks({ + "prepare": self.prepare, + "file" : self.convert_zip, + "after" : self.convert_files, + }, options) def prepare(self, pathfmt): - self._frames = None - - if pathfmt.extension != "zip": + if "frames" not in pathfmt.kwdict: + self._frames = None return - kwdict = pathfmt.kwdict - if "frames" in kwdict: - self._frames = kwdict["frames"] - elif "pixiv_ugoira_frame_data" in kwdict: - self._frames = kwdict["pixiv_ugoira_frame_data"]["data"] + self._frames = pathfmt.kwdict["frames"] + if pathfmt.extension == "zip": + self._convert_zip = True + if self.delete: + pathfmt.set_extension(self.extension) + pathfmt.build_path() else: - return - - if self.delete: - pathfmt.set_extension(self.extension) pathfmt.build_path() + index = pathfmt.kwdict["_ugoira_frame_index"] + frame = self._frames[index].copy() + frame["index"] = index + frame["path"] = pathfmt.realpath + frame["ext"] = pathfmt.kwdict["extension"] + + if not index: + self._files = [frame] + else: + self._files.append(frame) + if len(self._files) >= len(self._frames): + self._convert_files = True - def convert(self, pathfmt): - if not self._frames: + def convert_zip(self, pathfmt): + if not self._convert_zip: return + self._convert_zip = False with tempfile.TemporaryDirectory() as tempdir: - # extract frames try: with zipfile.ZipFile(pathfmt.temppath) as zfile: zfile.extractall(tempdir) @@ -124,53 +136,89 @@ class UgoiraPP(PostProcessor): pathfmt.realpath = pathfmt.temppath return - # process frames and collect command-line arguments - pathfmt.set_extension(self.extension) - pathfmt.build_path() - - args = self._process(pathfmt, tempdir) - if self.args_pp: - args += self.args_pp - if self.args: - args += self.args - - # ensure target directory exists - os.makedirs(pathfmt.realdirectory, exist_ok=True) - - # invoke ffmpeg - try: - if self.twopass: - if "-f" not in self.args: - args += ("-f", self.extension) - args += ("-passlogfile", tempdir + "/ffmpeg2pass", "-pass") - self._exec(args + ["1", "-y", os.devnull]) - self._exec(args + ["2", pathfmt.realpath]) - else: - args.append(pathfmt.realpath) - self._exec(args) - if self._finalize: - self._finalize(pathfmt, tempdir) - except OSError as exc: - print() - self.log.error("Unable to invoke FFmpeg (%s: %s)", - exc.__class__.__name__, exc) - pathfmt.realpath = pathfmt.temppath - except Exception as exc: - print() - self.log.error("%s: %s", exc.__class__.__name__, exc) - self.log.debug("", exc_info=True) - pathfmt.realpath = pathfmt.temppath - else: - if self.mtime: - mtime = pathfmt.kwdict.get("_mtime") - if mtime: - util.set_mtime(pathfmt.realpath, mtime) + if self.convert(pathfmt, tempdir): if self.delete: pathfmt.delete = True else: + self.log.info(pathfmt.filename) pathfmt.set_extension("zip") pathfmt.build_path() + def convert_files(self, pathfmt): + if not self._convert_files: + return + self._convert_files = False + + with tempfile.TemporaryDirectory() as tempdir: + for frame in self._files: + + # update frame filename extension + frame["file"] = name = "{}.{}".format( + frame["file"].partition(".")[0], frame["ext"]) + + # move frame into tempdir + try: + self._copy_file(frame["path"], tempdir + "/" + name) + except OSError as exc: + self.log.debug("Unable to copy frame %s (%s: %s)", + name, exc.__class__.__name__, exc) + return + + pathfmt.kwdict["num"] = 0 + self._frames = self._files + if self.convert(pathfmt, tempdir): + self.log.info(pathfmt.filename) + if self.delete: + self.log.debug("Deleting frames") + for frame in self._files: + util.remove_file(frame["path"]) + + def convert(self, pathfmt, tempdir): + pathfmt.set_extension(self.extension) + pathfmt.build_path() + if self.skip and pathfmt.exists(): + return True + + # process frames and collect command-line arguments + args = self._process(pathfmt, tempdir) + if self.args_pp: + args += self.args_pp + if self.args: + args += self.args + + # ensure target directory exists + os.makedirs(pathfmt.realdirectory, exist_ok=True) + + # invoke ffmpeg + try: + if self.twopass: + if "-f" not in self.args: + args += ("-f", self.extension) + args += ("-passlogfile", tempdir + "/ffmpeg2pass", "-pass") + self._exec(args + ["1", "-y", os.devnull]) + self._exec(args + ["2", pathfmt.realpath]) + else: + args.append(pathfmt.realpath) + self._exec(args) + if self._finalize: + self._finalize(pathfmt, tempdir) + except OSError as exc: + print() + self.log.error("Unable to invoke FFmpeg (%s: %s)", + exc.__class__.__name__, exc) + pathfmt.realpath = pathfmt.temppath + except Exception as exc: + print() + self.log.error("%s: %s", exc.__class__.__name__, exc) + self.log.debug("", exc_info=True) + pathfmt.realpath = pathfmt.temppath + else: + if self.mtime: + mtime = pathfmt.kwdict.get("_mtime") + if mtime: + util.set_mtime(pathfmt.realpath, mtime) + return True + def _exec(self, args): self.log.debug(args) out = None if self.output else subprocess.DEVNULL @@ -182,6 +230,9 @@ class UgoiraPP(PostProcessor): raise ValueError() return retcode + def _copy_file(self, src, dst): + shutil.copyfile(src, dst) + def _process_concat(self, pathfmt, tempdir): rate_in, rate_out = self.calculate_framerate(self._frames) args = [self.ffmpeg, "-f", "concat"] diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 5744ef3..ecb496d 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -101,7 +101,7 @@ def raises(cls): return wrap -def identity(x): +def identity(x, _=None): """Returns its argument""" return x @@ -520,14 +520,9 @@ class CustomNone(): """None-style type that supports more operations than regular None""" __slots__ = () - def __getattribute__(self, _): - return self - - def __getitem__(self, _): - return self - - def __iter__(self): - return self + __getattribute__ = identity + __getitem__ = identity + __iter__ = identity def __call__(self, *args, **kwargs): return self @@ -536,10 +531,6 @@ class CustomNone(): def __next__(): raise StopIteration - @staticmethod - def __bool__(): - return False - def __eq__(self, other): return self is other @@ -550,14 +541,48 @@ class CustomNone(): __le__ = true __gt__ = false __ge__ = false + __bool__ = false + + __add__ = identity + __sub__ = identity + __mul__ = identity + __matmul__ = identity + __truediv__ = identity + __floordiv__ = identity + __mod__ = identity + + __radd__ = identity + __rsub__ = identity + __rmul__ = identity + __rmatmul__ = identity + __rtruediv__ = identity + __rfloordiv__ = identity + __rmod__ = identity + + __lshift__ = identity + __rshift__ = identity + __and__ = identity + __xor__ = identity + __or__ = identity + + __rlshift__ = identity + __rrshift__ = identity + __rand__ = identity + __rxor__ = identity + __ror__ = identity + + __neg__ = identity + __pos__ = identity + __abs__ = identity + __invert__ = identity @staticmethod def __len__(): return 0 - @staticmethod - def __hash__(): - return 0 + __int__ = __len__ + __hash__ = __len__ + __index__ = __len__ @staticmethod def __format__(_): diff --git a/gallery_dl/version.py b/gallery_dl/version.py index f2462ee..0f9f91b 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,5 +6,5 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.27.3" +__version__ = "1.27.4" __variant__ = None diff --git a/gallery_dl/ytdl.py b/gallery_dl/ytdl.py index d4fdedc..fe88c2c 100644 --- a/gallery_dl/ytdl.py +++ b/gallery_dl/ytdl.py @@ -18,7 +18,7 @@ def import_module(module_name): if module_name is None: try: return __import__("yt_dlp") - except ImportError: + except (ImportError, SyntaxError): return __import__("youtube_dl") return __import__(module_name.replace("-", "_")) -- cgit v1.2.3