From 3b7f8716690b7aa1994a9cb387bbc7215e01a4ed Mon Sep 17 00:00:00 2001 From: Unit 193 Date: Tue, 16 Sep 2025 02:12:49 -0400 Subject: New upstream version 1.30.7. --- CHANGELOG.md | 61 ++++----- PKG-INFO | 10 +- README.rst | 4 +- data/man/gallery-dl.1 | 2 +- data/man/gallery-dl.conf.5 | 84 +++++++++++-- docs/gallery-dl.conf | 9 +- gallery_dl.egg-info/PKG-INFO | 10 +- gallery_dl.egg-info/SOURCES.txt | 3 + gallery_dl/exception.py | 15 ++- gallery_dl/extractor/__init__.py | 3 + gallery_dl/extractor/ao3.py | 5 +- gallery_dl/extractor/bellazon.py | 165 +++++++++++++++++++++++++ gallery_dl/extractor/boosty.py | 2 +- gallery_dl/extractor/comick.py | 32 ++++- gallery_dl/extractor/common.py | 19 +-- gallery_dl/extractor/cyberfile.py | 125 +++++++++++++++++++ gallery_dl/extractor/danbooru.py | 5 +- gallery_dl/extractor/facebook.py | 56 ++++----- gallery_dl/extractor/fansly.py | 188 ++++++++++++++++++++-------- gallery_dl/extractor/imgbb.py | 253 ++++++++++++++++++-------------------- gallery_dl/extractor/simpcity.py | 145 ++++++++++++++++++++++ gallery_dl/extractor/tiktok.py | 3 +- gallery_dl/extractor/tungsten.py | 11 +- gallery_dl/extractor/twitter.py | 65 ++++++++-- gallery_dl/formatter.py | 4 +- gallery_dl/util.py | 15 ++- gallery_dl/version.py | 2 +- setup.py | 3 +- test/test_formatter.py | 9 +- 29 files changed, 996 insertions(+), 312 deletions(-) create mode 100644 gallery_dl/extractor/bellazon.py create mode 100644 gallery_dl/extractor/cyberfile.py create mode 100644 gallery_dl/extractor/simpcity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fb17a4..a5eaa4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,43 +1,28 @@ -## 1.30.6 - 2025-09-06 +## 1.30.7 - 2025-09-14 ### Extractors #### Additions -- [chevereto] add `video` extractor ([#8149](https://github.com/mikf/gallery-dl/issues/8149)) -- [comick] add `covers` extractor -- [fansly] add support ([#4401](https://github.com/mikf/gallery-dl/issues/4401)) -- [instagram] add `stories-tray` extractor ([#6582](https://github.com/mikf/gallery-dl/issues/6582)) -- [shimmie2] support `co.llection.pics` ([#8166](https://github.com/mikf/gallery-dl/issues/8166)) -- [tungsten] add support ([#8061](https://github.com/mikf/gallery-dl/issues/8061)) -- [vk] add `wall-post` extractor ([#474](https://github.com/mikf/gallery-dl/issues/474) [#6378](https://github.com/mikf/gallery-dl/issues/6378) [#8159](https://github.com/mikf/gallery-dl/issues/8159)) +- [bellazon] add support ([#7480](https://github.com/mikf/gallery-dl/issues/7480)) +- [cyberfile] add support ([#5015](https://github.com/mikf/gallery-dl/issues/5015)) +- [fansly] add `creator-media` extractor ([#4401](https://github.com/mikf/gallery-dl/issues/4401)) +- [simpcity] add support ([#3127](https://github.com/mikf/gallery-dl/issues/3127) [#5145](https://github.com/mikf/gallery-dl/issues/5145) [#5879](https://github.com/mikf/gallery-dl/issues/5879) [#8187](https://github.com/mikf/gallery-dl/issues/8187)) #### Fixes -- [bunkr] fix downloading albums with more than 100 files ([#8150](https://github.com/mikf/gallery-dl/issues/8150) [#8155](https://github.com/mikf/gallery-dl/issues/8155) [#8175](https://github.com/mikf/gallery-dl/issues/8175)) -- [chevereto:user] fix names starting with an `a` ([#8149](https://github.com/mikf/gallery-dl/issues/8149)) -- [common] prevent exception when using empty `user-agent` ([#8116](https://github.com/mikf/gallery-dl/issues/8116)) -- [deviantart:search] fix extraction ([#8083](https://github.com/mikf/gallery-dl/issues/8083)) -- [hentaifoundry:story] fix `src` & `description` extraction ([#8163](https://github.com/mikf/gallery-dl/issues/8163)) -- [imagebam] update guard page bypass cookies ([#8123](https://github.com/mikf/gallery-dl/issues/8123)) -- [kemono] fix `.bin` archive files not being added to archives list ([#8156](https://github.com/mikf/gallery-dl/issues/8156)) -- [reddit] fix `TypeaError` when processing comments ([#8139](https://github.com/mikf/gallery-dl/issues/8139)) -- [tumblr] fix pagination when using `date-max` -- [twitter] prevent exceptions in `_transform_community()` ([#8134](https://github.com/mikf/gallery-dl/issues/8134)) -- [twitter] prevent `KeyError: 'name'` in `_transform_user()` ([#8154](https://github.com/mikf/gallery-dl/issues/8154)) -- [twitter] fix `KeyError: 'core'` when processing communities ([#8141](https://github.com/mikf/gallery-dl/issues/8141)) -- [zerochan] fix `500 Internal Server Error` during login ([#8097](https://github.com/mikf/gallery-dl/issues/8097) [#8114](https://github.com/mikf/gallery-dl/issues/8114)) +- [aibooru] fix download URLs ([#8212](https://github.com/mikf/gallery-dl/issues/8212)) +- [ao3] fix pagination ([#8206](https://github.com/mikf/gallery-dl/issues/8206)) +- [boosty] fix extracting `accessToken` from cookies ([#8203](https://github.com/mikf/gallery-dl/issues/8203)) +- [comick] update `buildId` on `404` errors ([#8157](https://github.com/mikf/gallery-dl/issues/8157)) +- [facebook] fix `/photo/?fbid=…&set=…` URLs being handled as a set ([#8181](https://github.com/mikf/gallery-dl/issues/8181)) +- [fansly] fix & improve format selection ([#4401](https://github.com/mikf/gallery-dl/issues/4401)) +- [fansly] fix posts with more than 5 files ([#4401](https://github.com/mikf/gallery-dl/issues/4401)) +- [imgbb] fix & update ([#7936](https://github.com/mikf/gallery-dl/issues/7936)) +- [tiktok] fix `KeyError: 'author'` ([#8189](https://github.com/mikf/gallery-dl/issues/8189)) #### Improvements -- [comick] detect broken chapters ([#8054](https://github.com/mikf/gallery-dl/issues/8054)) -- [erome] handle reposts on user profiles ([#6582](https://github.com/mikf/gallery-dl/issues/6582)) -- [instagram] improve video quality warning regex ([#8078](https://github.com/mikf/gallery-dl/issues/8078)) -- [jpgfish] update domain to `jpg6.su` -- [reddit] add `api` & `limit` options ([#7997](https://github.com/mikf/gallery-dl/issues/7997) [#8012](https://github.com/mikf/gallery-dl/issues/8012) [#8092](https://github.com/mikf/gallery-dl/issues/8092)) -- [reddit] support video embeds ([#8139](https://github.com/mikf/gallery-dl/issues/8139)) -- [tumblr:tagged] support `/archive/tagged/` URLs ([#8160](https://github.com/mikf/gallery-dl/issues/8160)) -#### Metadata -- [khinsider] extract `description` metadata -- [tumblr:tagged] provide `search_tags` metadata ([#8160](https://github.com/mikf/gallery-dl/issues/8160)) -- [vk] parse `date` & `description` metadata ([#8029](https://github.com/mikf/gallery-dl/issues/8029)) -- [vk:album] extract more metadata ([#8029](https://github.com/mikf/gallery-dl/issues/8029)) -### Downloaders -- [ytdl] implement `_ytdl_manifest_cookies` +- [comick] handle redirects +- [fansly] provide fallback URL for manifest downloads ([#4401](https://github.com/mikf/gallery-dl/issues/4401)) +- [fansly:creator] support custom wall IDs ([#4401](https://github.com/mikf/gallery-dl/issues/4401)) +- [tungsten:user] support filtering results by tag ([#8061](https://github.com/mikf/gallery-dl/issues/8061)) +- [twitter] continue searches on empty response ([#8173](https://github.com/mikf/gallery-dl/issues/8173)) +- [twitter] implement various `search-…` options ([#8173](https://github.com/mikf/gallery-dl/issues/8173)) ### Miscellaneous -- [formatter] add `R` conversion - extract URLs ([#8125](https://github.com/mikf/gallery-dl/issues/8125)) -- [options] add `-a` as short option for `--user-agent` -- [scripts/init] implement `-s/--subcategory` +- [formatter] exclude `<>\` characters from `!R` results ([#8180](https://github.com/mikf/gallery-dl/issues/8180)) +- [formatter] support negative indicies +- [util] emit debug `Proxy Map` logging message ([#8195](https://github.com/mikf/gallery-dl/issues/8195)) diff --git a/PKG-INFO b/PKG-INFO index 353bfae..6abd758 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: gallery_dl -Version: 1.30.6 +Version: 1.30.7 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -8,7 +8,7 @@ Author: Mike Fährmann Author-email: mike_faehrmann@web.de Maintainer: Mike Fährmann Maintainer-email: mike_faehrmann@web.de -License: GPLv2 +License: GPL-2.0-only Keywords: image gallery downloader crawler scraper Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console @@ -30,6 +30,7 @@ Classifier: Topic :: Internet :: WWW/HTTP Classifier: Topic :: Multimedia :: Graphics Classifier: Topic :: Utilities Requires-Python: >=3.8 +Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: requests>=2.11.0 Provides-Extra: video @@ -46,6 +47,7 @@ Dynamic: author Dynamic: author-email Dynamic: classifier Dynamic: description +Dynamic: description-content-type Dynamic: download-url Dynamic: home-page Dynamic: keywords @@ -139,9 +141,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/README.rst b/README.rst index 71c7cf8..71cf809 100644 --- a/README.rst +++ b/README.rst @@ -79,9 +79,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1 index 3cda42a..36ca314 100644 --- a/data/man/gallery-dl.1 +++ b/data/man/gallery-dl.1 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL" "1" "2025-09-06" "1.30.6" "gallery-dl Manual" +.TH "GALLERY-DL" "1" "2025-09-14" "1.30.7" "gallery-dl Manual" .\" disable hyphenation .nh diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5 index 335195f..8d2f806 100644 --- a/data/man/gallery-dl.conf.5 +++ b/data/man/gallery-dl.conf.5 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL.CONF" "5" "2025-09-06" "1.30.6" "gallery-dl Manual" +.TH "GALLERY-DL.CONF" "5" "2025-09-14" "1.30.7" "gallery-dl Manual" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -1316,7 +1316,7 @@ for available \f[I]PRAGMA\f[] statements and further details. .br * \f[I]object\f[] (pattern -> \f[I]Action(s)\f[]) .br -* \f[I]list\f[] of \f[I]lists\f[] with pattern -> \f[I]Action(s)\f[] pairs as elements +* \f[I]list\f[] of [pattern, \f[I]Action(s)\f[]] pairs .IP "Example:" 4 .. code:: json @@ -1350,11 +1350,12 @@ Perform an \f[I]Action\f[] when logging a message matched by \f[I]pattern\f[]. \f[I]pattern\f[] is parsed as severity level (\f[I]debug\f[], \f[I]info\f[], \f[I]warning\f[], \f[I]error\f[], or integer value) followed by an optional \f[I]Python Regular Expression\f[] -separated by a colon \f[I]:\f[] +separated by a colon: +\f[I]:\f[] Using \f[I]*\f[] as level or leaving it empty -matches logging messages of all levels -(e.g. \f[I]*:\f[] or \f[I]:\f[]). +matches logging messages of all levels: +\f[I]*:\f[] or \f[I]:\f[] .SS extractor.*.postprocessors @@ -2413,6 +2414,21 @@ Setting this option to \f[I]"auto"\f[] uses the same domain as a given input URL. +.SS extractor.cyberfile.password +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Default:" 9 +\f[I]""\f[] + +.IP "Description:" 4 +Password value used to access protected files and folders. + +Note: Leave this value empty or undefined +to be interactively prompted for a password when needed +(see \f[I]getpass()\f[]). + + .SS extractor.[Danbooru].external .IP "Type:" 6 \f[I]bool\f[] @@ -3332,17 +3348,15 @@ Note: \f[I]comments\f[] can also be enabled via \f[I]fanbox.comments\f[] -.SS extractor.fansly.format +.SS extractor.fansly.formats .IP "Type:" 6 \f[I]list\f[] of \f[I]integers\f[] .IP "Default:" 9 -\f[I][303, 302, 1, 2, 4]\f[] +\f[I][1, 2, 3, 4, 302, 303]\f[] .IP "Description:" 4 -Selects the file format to extract. - -When more than one format is given, the first available one is selected. +List of file formats to consider during format selection. .SS extractor.fansly.token @@ -6781,6 +6795,49 @@ If this value is \f[I]"original"\f[], metadata for these files will be taken from the original Tweets, not the Retweets. +.SS extractor.twitter.search-limit +.IP "Type:" 6 +\f[I]integer\f[] + +.IP "Default:" 9 +\f[I]20\f[] + +.IP "Description:" 4 +Number of requested results per search query. + + +.SS extractor.twitter.search-pagination +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Default:" 9 +\f[I]"cursor"\f[] + +.IP "Description:" 4 +Selects how to paginate over search results. + +\f[I]"cursor"\f[] +Use \f[I]cursor\f[] values provided by the API +\f[I]"max_id"\f[] \f[I] \f[I]"maxid"\f[] \f[] \f[I]"id"\f[] +Update the \f[I]max_id\f[] search query parameter +to the Tweet ID value of the last retrieved Tweet. + + +.SS extractor.twitter.search-stop +.IP "Type:" 6 +\f[I]integer\f[] + +.IP "Default:" 9 +.br +* \f[I]3\f[] if \f[I]search-pagination\f[] is set to \f[I]"cursor"\f[] +.br +* \f[I]0\f[] otherwise + +.IP "Description:" 4 +Number of empty search result batches +to accept before stopping. + + .SS extractor.twitter.timeline.strategy .IP "Type:" 6 \f[I]string\f[] @@ -10263,10 +10320,13 @@ Store files in a ZIP archive * "raise AbortExtraction an error occured" .br * "flag file = terminate" +.br +* ["print Exiting", "exit 1"] .IP "Description:" 4 An \f[I]Action\f[] is parsed as Action Type -followed by (optional) arguments. +followed by (optional) arguments: +\f[I] …\f[] It is possible to specify more than one \f[I]action\f[] by providing them as a \f[I]list\f[]: \f[I]["", "", …]\f[] @@ -10289,6 +10349,8 @@ Modify severity level of the current logging message. .br Can be one of \f[I]debug\f[], \f[I]info\f[], \f[I]warning\f[], \f[I]error\f[] or an integer value. .br +Use \f[I]0\f[] to ignore a message (\f[I]level = 0\f[]). +.br \f[I]print\f[]: Write argument to stdout. \f[I]exec\f[]: diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index f1e3833..823fcc0 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -237,6 +237,10 @@ { "domain": null }, + "cyberfile": + { + "password": "" + }, "dankefuerslesen": { "zip": false @@ -332,7 +336,7 @@ { "token": "", - "format": [303, 302, 1, 2, 4] + "formats": [1, 2, 3, 4, 302, 303] }, "flickr": { @@ -818,6 +822,9 @@ "relogin" : true, "replies" : true, "retweets" : false, + "search-limit": 20, + "search-pagination": "cursor", + "search-stop" : "auto", "size" : ["orig", "4096x4096", "large", "medium", "small"], "text-tweets" : false, "tweet-endpoint": "auto", diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO index 353bfae..6abd758 100644 --- a/gallery_dl.egg-info/PKG-INFO +++ b/gallery_dl.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: gallery_dl -Version: 1.30.6 +Version: 1.30.7 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -8,7 +8,7 @@ Author: Mike Fährmann Author-email: mike_faehrmann@web.de Maintainer: Mike Fährmann Maintainer-email: mike_faehrmann@web.de -License: GPLv2 +License: GPL-2.0-only Keywords: image gallery downloader crawler scraper Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console @@ -30,6 +30,7 @@ Classifier: Topic :: Internet :: WWW/HTTP Classifier: Topic :: Multimedia :: Graphics Classifier: Topic :: Utilities Requires-Python: >=3.8 +Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: requests>=2.11.0 Provides-Extra: video @@ -46,6 +47,7 @@ Dynamic: author Dynamic: author-email Dynamic: classifier Dynamic: description +Dynamic: description-content-type Dynamic: download-url Dynamic: home-page Dynamic: keywords @@ -139,9 +141,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/gallery_dl.egg-info/SOURCES.txt b/gallery_dl.egg-info/SOURCES.txt index 1a0bc19..de3a9ed 100644 --- a/gallery_dl.egg-info/SOURCES.txt +++ b/gallery_dl.egg-info/SOURCES.txt @@ -66,6 +66,7 @@ gallery_dl/extractor/aryion.py gallery_dl/extractor/batoto.py gallery_dl/extractor/bbc.py gallery_dl/extractor/behance.py +gallery_dl/extractor/bellazon.py gallery_dl/extractor/bilibili.py gallery_dl/extractor/blogger.py gallery_dl/extractor/bluesky.py @@ -81,6 +82,7 @@ gallery_dl/extractor/comick.py gallery_dl/extractor/comicvine.py gallery_dl/extractor/common.py gallery_dl/extractor/cyberdrop.py +gallery_dl/extractor/cyberfile.py gallery_dl/extractor/danbooru.py gallery_dl/extractor/dankefuerslesen.py gallery_dl/extractor/desktopography.py @@ -220,6 +222,7 @@ gallery_dl/extractor/senmanga.py gallery_dl/extractor/sexcom.py gallery_dl/extractor/shimmie2.py gallery_dl/extractor/shopify.py +gallery_dl/extractor/simpcity.py gallery_dl/extractor/simplyhentai.py gallery_dl/extractor/sizebooru.py gallery_dl/extractor/skeb.py diff --git a/gallery_dl/exception.py b/gallery_dl/exception.py index 6adda0d..559fdd1 100644 --- a/gallery_dl/exception.py +++ b/gallery_dl/exception.py @@ -104,13 +104,16 @@ class AuthRequired(AuthorizationError): if auth: if not isinstance(auth, str): auth = " or ".join(auth) - if " " not in resource: - resource = "this " + resource - if message is None: - message = (f"{auth} needed to access {resource}") + + if resource: + if " " not in resource: + resource = f"this {resource}" + resource = f" to access {resource}" else: - message = (f"{auth} needed to access {resource} " - f"('{message}')") + resource = "" + + message = f" ('{message}')" if message else "" + message = f"{auth} needed{resource}{message}" AuthorizationError.__init__(self, message) diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 574d1e2..b32fcd1 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -31,6 +31,7 @@ modules = [ "batoto", "bbc", "behance", + "bellazon", "bilibili", "blogger", "bluesky", @@ -44,6 +45,7 @@ modules = [ "comick", "comicvine", "cyberdrop", + "cyberfile", "danbooru", "dankefuerslesen", "desktopography", @@ -170,6 +172,7 @@ modules = [ "senmanga", "sexcom", "shimmie2", + "simpcity", "simplyhentai", "sizebooru", "skeb", diff --git a/gallery_dl/extractor/ao3.py b/gallery_dl/extractor/ao3.py index 2652acb..60380c4 100644 --- a/gallery_dl/extractor/ao3.py +++ b/gallery_dl/extractor/ao3.py @@ -102,8 +102,11 @@ class Ao3Extractor(Extractor): def _pagination(self, path, needle='