diff options
| author | 2025-10-07 02:11:52 -0400 | |
|---|---|---|
| committer | 2025-10-07 02:11:52 -0400 | |
| commit | 83e1e051b8c0e622ef5f61c1955c47b4bde95b57 (patch) | |
| tree | 544a434cb398d2adb8b8a2d553dc1c9a44b4ee1d | |
| parent | f1612851ae9fe68c7444fb31e786503868aeaa7c (diff) | |
| parent | bbe7fac03d881662a458e7fbf870c9d71f5257f4 (diff) | |
Update upstream source from tag 'upstream/1.30.9'
Update to upstream version '1.30.9'
with Debian dir 46cc56e13f05f4465cc64f67b4d7b775a95bd87a
38 files changed, 2210 insertions, 1111 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aa64b9..1df5fff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,48 +1,44 @@ -## 1.30.8 - 2025-09-23 +## 1.30.9 - 2025-10-03 ### Extractors #### Additions -- [chevereto] support `imglike.com` ([#5179](https://github.com/mikf/gallery-dl/issues/5179)) -- [chevereto] add `category` extractor ([#5179](https://github.com/mikf/gallery-dl/issues/5179)) -- [Danbooru] add `random` extractor ([#8270](https://github.com/mikf/gallery-dl/issues/8270)) -- [hdoujin] add support ([#6810](https://github.com/mikf/gallery-dl/issues/6810)) -- [imgpile] add support ([#5044](https://github.com/mikf/gallery-dl/issues/5044)) -- [mangadex] add `covers` extractor ([#4994](https://github.com/mikf/gallery-dl/issues/4994)) -- [mangataro] add support ([#8237](https://github.com/mikf/gallery-dl/issues/8237)) -- [thehentaiworld] add support ([#274](https://github.com/mikf/gallery-dl/issues/274) [#8237](https://github.com/mikf/gallery-dl/issues/8237)) +- [mangafire] add support ([#7064](https://github.com/mikf/gallery-dl/issues/7064) [#7701](https://github.com/mikf/gallery-dl/issues/7701)) +- [mangareader] add support ([#6529](https://github.com/mikf/gallery-dl/issues/6529) [#6868](https://github.com/mikf/gallery-dl/issues/6868)) +- [patreon] add `collection` extractor ([#8286](https://github.com/mikf/gallery-dl/issues/8286)) +- [s3ndpics] add support ([#8322](https://github.com/mikf/gallery-dl/issues/8322)) #### Fixes -- [4archive] fix `TypeError` ([#8217](https://github.com/mikf/gallery-dl/issues/8217)) -- [bellazon] fix video attachments ([#8239](https://github.com/mikf/gallery-dl/issues/8239)) -- [bunkr] fix `JSONDecodeError` for files with URL slugs containing apostrophes `'` ([#8150](https://github.com/mikf/gallery-dl/issues/8150)) -- [instagram] ensure manifest data exists before attempting a DASH download ([#8267](https://github.com/mikf/gallery-dl/issues/8267)) -- [schalenetwork] fix extraction ([#6948](https://github.com/mikf/gallery-dl/issues/6948) [#7391](https://github.com/mikf/gallery-dl/issues/7391) [#7728](https://github.com/mikf/gallery-dl/issues/7728)) -- [twitter] fix quoted Tweets being marked as `deleted` ([#8225](https://github.com/mikf/gallery-dl/issues/8225)) +- [chevereto] fix `id` for links without file name ([#8307](https://github.com/mikf/gallery-dl/issues/8307)) +- [chevereto:album] fix video downloads ([#8149](https://github.com/mikf/gallery-dl/issues/8149) [#8295](https://github.com/mikf/gallery-dl/issues/8295)) +- [hdoujin] fix `KeyError: 13` by adding `reclass` tag type ([#8290](https://github.com/mikf/gallery-dl/issues/8290)) +- [misskey] include `withRenotes` parameter in API requests ([#8285](https://github.com/mikf/gallery-dl/issues/8285)) +- [nozomi] percent-encode search tags ([#8328](https://github.com/mikf/gallery-dl/issues/8328)) +- [simpcity] fix `KeyError: 'url'` when thread author is deleted ([#8323](https://github.com/mikf/gallery-dl/issues/8323)) +- [twitter] fix `quote_id` of individual Tweets ([#8284](https://github.com/mikf/gallery-dl/issues/8284)) +- [zerochan] prevent `HttpError: '503 Service Temporarily Unavailable'` ([#8288](https://github.com/mikf/gallery-dl/issues/8288)) #### Improvements -- [2ch] update domain to `2ch.su`, support `2ch.life` URLs ([#8216](https://github.com/mikf/gallery-dl/issues/8216)) -- [bellazon][simpcity][vipergirls] process threads in descending order ([#8248](https://github.com/mikf/gallery-dl/issues/8248)) -- [bellazon] extract `inline` images (##8247) -- [bellazon] support video embeds ([#8239](https://github.com/mikf/gallery-dl/issues/8239)) -- [bellazon] support `#comment-12345` post links ([#8239](https://github.com/mikf/gallery-dl/issues/8239)) -- [lensdump] support new direct file URL pattern ([#8251](https://github.com/mikf/gallery-dl/issues/8251)) -- [simpcity] extract URLs of `<iframe>` embeds ([#8214](https://github.com/mikf/gallery-dl/issues/8214) [#8256](https://github.com/mikf/gallery-dl/issues/8256)) -- [simpcity] improve post content extraction ([#8214](https://github.com/mikf/gallery-dl/issues/8214)) +- [chevereto] support URLs with `www` subdomain ([#8149](https://github.com/mikf/gallery-dl/issues/8149)) +- [imxto:gallery] support multiple pages ([#8282](https://github.com/mikf/gallery-dl/issues/8282)) +- [instagram] add `warn-images` & `warn-videos` options ([#8283](https://github.com/mikf/gallery-dl/issues/8283)) +- [instagram] use `reel` subcategory for `/reel/SHORTCODE` URLs ([#8274](https://github.com/mikf/gallery-dl/issues/8274)) +- [instagram] support `/reels/SHORTCODE` URLs ([#8318](https://github.com/mikf/gallery-dl/issues/8318)) +- [paheal] normalize `No results` output message ([#8313](https://github.com/mikf/gallery-dl/issues/8313)) +- [pixiv] implement searching past 5000 results ([#1686](https://github.com/mikf/gallery-dl/issues/1686) [#7082](https://github.com/mikf/gallery-dl/issues/7082) [#8298](https://github.com/mikf/gallery-dl/issues/8298)) +- [thehentaiworld] support more `post` URL formats ([#8277](https://github.com/mikf/gallery-dl/issues/8277)) +- [weibo] download `.m3u8` manifests with ytdl ([#8339](https://github.com/mikf/gallery-dl/issues/8339)) +- [weibo] resolve `wblive-out.api.weibo.com` URLs ([#8339](https://github.com/mikf/gallery-dl/issues/8339)) +- [weibo] use `replay_hd` URLs as video fallback ([#8339](https://github.com/mikf/gallery-dl/issues/8339)) +- [wikimedia] add ability to download image revisions ([#7283](https://github.com/mikf/gallery-dl/issues/7283) [#8330](https://github.com/mikf/gallery-dl/issues/8330)) +- [zerochan] normalize `No results` output message ([#8313](https://github.com/mikf/gallery-dl/issues/8313)) #### Metadata -- [facebook] extract `biography` metadata ([#8233](https://github.com/mikf/gallery-dl/issues/8233)) -- [instagram:tagged] provide full `tagged_…` metadata when using `id:…` URLs ([#8263](https://github.com/mikf/gallery-dl/issues/8263)) -- [iwara] extract more metadata ([#6582](https://github.com/mikf/gallery-dl/issues/6582)) -- [iwara] make `type` available for directories ([#8245](https://github.com/mikf/gallery-dl/issues/8245)) -- [reddit] provide `comment` metadata for all media files ([#8228](https://github.com/mikf/gallery-dl/issues/8228)) -#### Options -- [bellazon] add `quoted` option ([#8247](https://github.com/mikf/gallery-dl/issues/8247)) -- [bellazon] implement `order-posts` option ([#8248](https://github.com/mikf/gallery-dl/issues/8248)) -- [kemono:discord] implement `order-posts` option ([#8241](https://github.com/mikf/gallery-dl/issues/8241)) -- [simpcity] implement `order-posts` option ([#8248](https://github.com/mikf/gallery-dl/issues/8248)) -- [vipergirls] implement `order-posts` option ([#8248](https://github.com/mikf/gallery-dl/issues/8248)) +- [hdoujin] extract `source` metadata ([#8280](https://github.com/mikf/gallery-dl/issues/8280)) +- [instagram] provide `type` metadata ([#8274](https://github.com/mikf/gallery-dl/issues/8274)) +- [mangadex] extract more manga-related metadata ([#8325](https://github.com/mikf/gallery-dl/issues/8325)) +#### Removals +- [chevereto] remove `img.kiwi` ### Downloaders -- [ytdl] fix errors caused by deprecated options removal +- [http] add MIME type and signature for m3u8 & mpd files ([#8339](https://github.com/mikf/gallery-dl/issues/8339)) ### Post Processors -- [metadata] add `"mode": "print"` ([#2691](https://github.com/mikf/gallery-dl/issues/2691)) -- [python] add `"mode": "eval"` -- close archive database connections ([#8243](https://github.com/mikf/gallery-dl/issues/8243)) +- [python] restore `archive` functionality ### Miscellaneous -- [util] define `__enter__` & `__exit__` methods for `NullResponse` objects ([#8227](https://github.com/mikf/gallery-dl/issues/8227)) -- [util] extend list of ISO 639 language codes +- [cookies] add support for `Orion` browser ([#8303](https://github.com/mikf/gallery-dl/issues/8303)) +- [docker] include more optional Python dependencies ([#8026](https://github.com/mikf/gallery-dl/issues/8026)) +- [docs] update `configuration.rst` formatting @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: gallery_dl -Version: 1.30.8 +Version: 1.30.9 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -141,9 +141,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.exe>`__ +- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.exe>`__ (Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__) -- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.bin>`__ +- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.bin>`__ Nightly Builds @@ -79,9 +79,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.exe>`__ +- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.exe>`__ (Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__) -- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.bin>`__ +- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.bin>`__ Nightly Builds diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1 index 6560c3b..4137501 100644 --- a/data/man/gallery-dl.1 +++ b/data/man/gallery-dl.1 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL" "1" "2025-09-23" "1.30.8" "gallery-dl Manual" +.TH "GALLERY-DL" "1" "2025-10-03" "1.30.9" "gallery-dl Manual" .\" disable hyphenation .nh diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5 index fbf32bc..b827aeb 100644 --- a/data/man/gallery-dl.conf.5 +++ b/data/man/gallery-dl.conf.5 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL.CONF" "5" "2025-09-23" "1.30.8" "gallery-dl Manual" +.TH "GALLERY-DL.CONF" "5" "2025-10-03" "1.30.9" "gallery-dl Manual" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -76,9 +76,9 @@ those as makeshift comments by settings their values to arbitrary strings. .SS extractor.*.filename .IP "Type:" 6 .br -* \f[I]string\f[] +* \f[I]Format String\f[] .br -* \f[I]object\f[] (condition -> \f[I]format string\f[]) +* \f[I]object\f[] (\f[I]Condition\f[] → \f[I]Format String\f[]) .IP "Example:" 4 .. code:: json @@ -95,12 +95,15 @@ those as makeshift comments by settings their values to arbitrary strings. .IP "Description:" 4 -A \f[I]format string\f[] to build filenames for downloaded files with. +A \f[I]Format String\f[] to generate filenames for downloaded files. -If this is an \f[I]object\f[], it must contain Python expressions mapping to the -filename format strings to use. -These expressions are evaluated in the specified order until one evaluates -to \f[I]True\f[]. +If this is an \f[I]object\f[], +it must contain \f[I]Conditions\f[] mapping to the +\f[I]Format String\f[] to use. +These \f[I]Conditions\f[] are evaluated in the specified order +until one evaluates to \f[I]True\f[]. +When none match, the \f[I]""\f[] entry or +the extractor's default filename \f[I]Format String\f[] is used. The available replacement keys depend on the extractor used. A list of keys for a specific one can be acquired by calling *gallery-dl* @@ -128,7 +131,8 @@ image-id subcategory image -Note: Even if the value of the \f[I]extension\f[] key is missing or +.IP "Note:" 4 +Even if the value of the \f[I]extension\f[] key is missing or \f[I]None\f[], it will be filled in later when the file download is starting. This key is therefore always available to provide a valid filename extension. @@ -137,9 +141,9 @@ a valid filename extension. .SS extractor.*.directory .IP "Type:" 6 .br -* \f[I]list\f[] of \f[I]strings\f[] +* \f[I]list\f[] of \f[I]Format Strings\f[] .br -* \f[I]object\f[] (condition -> \f[I]format strings\f[]) +* \f[I]object\f[] (\f[I]Condition\f[] → \f[I]Format Strings\f[]) .IP "Example:" 4 .. code:: json @@ -156,10 +160,11 @@ a valid filename extension. .IP "Description:" 4 -A list of \f[I]format strings\f[] to build target directory paths with. +A list of \f[I]Format String(s)\f[] to generate the target directory path. -If this is an \f[I]object\f[], it must contain Python expressions mapping to the -list of format strings to use. +If this is an \f[I]object\f[], +it must contain \f[I]Conditions\f[] mapping to the +list of \f[I]Format Strings\f[] to use. Each individual string in such a list represents a single path segment, which will be joined together and appended to the @@ -233,7 +238,7 @@ Share number of skipped downloads between parent and child extractors. .br * \f[I]string\f[] .br -* \f[I]object\f[] (character -> replacement character(s)) +* \f[I]object\f[] (character → replacement character(s)) .IP "Default:" 9 \f[I]"auto"\f[] @@ -254,28 +259,39 @@ or character ranges to their replacements for generated path segment names. .br -Special values: - -.br -* \f[I]"auto"\f[]: Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[] +.IP "Special Values:" 4 +\f[I]"auto"\f[] +Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[] depending on the local operating system +\f[I]"unix"\f[] +\f[I]"/"\f[] +\f[I]"windows"\f[] +\f[I]"\\\\\\\\|/<>:\\"?*"\f[] +.br +(https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file) +.br +\f[I]"ascii"\f[] +\f[I]"^0-9A-Za-z_."\f[] .br -* \f[I]"unix"\f[]: \f[I]"/"\f[] +(only ASCII digits, letters, underscores, and dots) .br -* \f[I]"windows"\f[]: \f[I]"\\\\\\\\|/<>:\\"?*"\f[] +\f[I]"ascii+"\f[] +\f[I]"^0-9@-[\\\\]-{ #-)+-.;=!}~"\f[] .br -* \f[I]"ascii"\f[]: \f[I]"^0-9A-Za-z_."\f[] (only ASCII digits, letters, underscores, and dots) +(all ASCII characters except the ones not allowed by Windows) .br -* \f[I]"ascii+"\f[]: \f[I]"^0-9@-[\\\\]-{ #-)+-.;=!}~"\f[] (all ASCII characters except the ones not allowed by Windows) -Implementation Detail: For \f[I]strings\f[] with length >= 2, this option uses a +.IP "Implementation Detail:" 4 +For \f[I]strings\f[] with length >= 2, this option uses a \f[I]Regular Expression Character Set\f[], meaning that: .br -* using a caret \f[I]^\f[] as first character inverts the set +* Using a caret \f[I]^\f[] as first character inverts the set +(\f[I]"^..."\f[]) .br -* character ranges are supported (\f[I]0-9a-z\f[]) +* Character ranges are supported +(\f[I]"0-9a-z"\f[]) .br * \f[I]]\f[], \f[I]-\f[], and \f[I]\\\f[] need to be escaped as \f[I]\\\\]\f[], \f[I]\\\\-\f[], and \f[I]\\\\\\\\\f[] respectively @@ -304,7 +320,8 @@ The replacement character(s) for .IP "Description:" 4 Set of characters to remove from generated path names. -Note: In a string with 2 or more characters, \f[I][]^-\\\f[] need to be +.IP "Note:" 4 +In a string with 2 or more characters, \f[I][]^-\\\f[] need to be escaped with backslashes, e.g. \f[I]"\\\\[\\\\]"\f[] @@ -319,15 +336,14 @@ escaped with backslashes, e.g. \f[I]"\\\\[\\\\]"\f[] Set of characters to remove from the end of generated path segment names using \f[I]str.rstrip()\f[] -Special values: - -.br -* \f[I]"auto"\f[]: Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[] +.IP "Special Values:" 4 +\f[I]"auto"\f[] +Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[] depending on the local operating system -.br -* \f[I]"unix"\f[]: \f[I]""\f[] -.br -* \f[I]"windows"\f[]: \f[I]". "\f[] +\f[I]"unix"\f[] +\f[I]""\f[] +\f[I]"windows"\f[] +\f[I]". "\f[] .SS extractor.*.path-convert @@ -360,7 +376,7 @@ prefixed with \f[I]\\\\?\\\f[] to work around the 260 characters path length lim .SS extractor.*.extension-map .IP "Type:" 6 -\f[I]object\f[] (extension -> replacement) +\f[I]object\f[] (extension → replacement) .IP "Default:" 9 .. code:: json @@ -445,10 +461,10 @@ filename extension (\f[I]file.1.ext\f[], \f[I]file.2.ext\f[], etc.) .SS extractor.*.skip-filter .IP "Type:" 6 -\f[I]string\f[] +\f[I]Condition\f[] .IP "Description:" 4 -Python expression controlling which skipped files to count towards +Python \f[I]Expression\f[] controlling which skipped files to count towards \f[I]"abort"\f[] / \f[I]"terminate"\f[] / \f[I]"exit"\f[]. @@ -492,66 +508,57 @@ response before \f[I]retrying\f[] the request. \f[I]Duration\f[] .IP "Default:" 9 -.br -* \f[I]"0.5-1.5"\f[] -\f[I]ao3\f[], -\f[I]arcalive\f[], -\f[I]booth\f[], -\f[I]civitai\f[], -\f[I][Danbooru]\f[], -\f[I][E621]\f[], -\f[I][foolfuuka]:search\f[], -\f[I]hdoujin\f[], -\f[I]itaku\f[], -\f[I]newgrounds\f[], -\f[I][philomena]\f[], -\f[I]pixiv-novel\f[], -\f[I]plurk\f[], -\f[I]poipiku\f[] , -\f[I]pornpics\f[], -\f[I]schalenetwork\f[], -\f[I]scrolller\f[], -\f[I]sizebooru\f[], -\f[I]soundgasm\f[], -\f[I]thehentaiworld\f[], -\f[I]urlgalleries\f[], -\f[I]vk\f[], -\f[I]webtoons\f[], -\f[I]weebcentral\f[], -\f[I]xfolio\f[], +\f[I]"0.5-1.5"\f[] +\f[I]ao3\f[] \f[I] +\f[I]arcalive\f[] \f[] +\f[I]booth\f[] \f[I] +\f[I]civitai\f[] \f[] +\f[I][Danbooru]\f[] \f[I] +\f[I][E621]\f[] \f[] +\f[I][foolfuuka]:search\f[] \f[I] +\f[I]hdoujin\f[] \f[] +\f[I]itaku\f[] \f[I] +\f[I]newgrounds\f[] \f[] +\f[I][philomena]\f[] \f[I] +\f[I]pixiv-novel\f[] \f[] +\f[I]plurk\f[] \f[I] +\f[I]poipiku\f[] \f[] +\f[I]pornpics\f[] \f[I] +\f[I]schalenetwork\f[] \f[] +\f[I]scrolller\f[] \f[I] +\f[I]sizebooru\f[] \f[] +\f[I]soundgasm\f[] \f[I] +\f[I]thehentaiworld\f[] \f[] +\f[I]urlgalleries\f[] \f[I] +\f[I]vk\f[] \f[] +\f[I]webtoons\f[] \f[I] +\f[I]weebcentral\f[] \f[] +\f[I]xfolio\f[] \f[I] \f[I]zerochan\f[] -.br -* \f[I]"1.0"\f[] -\f[I]furaffinity\f[] +\f[I]"1.0"\f[] +\f[I]furaffinity\f[] \f[] \f[I]rule34\f[] -.br -* \f[I]"1.0-2.0"\f[] -\f[I]flickr\f[], -\f[I]pexels\f[], -\f[I]weibo\f[], +\f[I]"1.0-2.0"\f[] +\f[I]flickr\f[] \f[I] +\f[I]pexels\f[] \f[] +\f[I]weibo\f[] \f[I] \f[I][wikimedia]\f[] -.br -* \f[I]"1.4"\f[] +\f[I]"1.4"\f[] \f[I]wallhaven\f[] -.br -* \f[I]"2.0-4.0"\f[] -\f[I]behance\f[], -\f[I]imagefap\f[], +\f[I]"2.0-4.0"\f[] +\f[I]behance\f[] \f[] +\f[I]imagefap\f[] \f[I] \f[I][Nijie]\f[] -.br -* \f[I]"3.0-6.0"\f[] -\f[I]bilibili\f[], -\f[I]exhentai\f[], -\f[I][reactor]\f[], +\f[I]"3.0-6.0"\f[] +\f[I]bilibili\f[] \f[] +\f[I]exhentai\f[] \f[I] +\f[I][reactor]\f[] \f[] \f[I]readcomiconline\f[] -.br -* \f[I]"6.0-6.1"\f[] +\f[I]"6.0-6.1"\f[] \f[I]twibooru\f[] -.br -* \f[I]"6.0-12.0"\f[] +\f[I]"6.0-12.0"\f[] \f[I]instagram\f[] -.br -* \f[I]0\f[] +\f[I]0\f[] otherwise .IP "Description:" 4 @@ -573,35 +580,35 @@ another site. This is supported for .br -* \f[I]aibooru\f[] (*) +* \f[I]aibooru\f[] (\f[I]*\f[]) .br * \f[I]ao3\f[] .br * \f[I]aryion\f[] .br -* \f[I]atfbooru\f[] (*) +* \f[I]atfbooru\f[] (\f[I]*\f[]) .br * \f[I]bluesky\f[] .br -* \f[I]booruvar\f[] (*) +* \f[I]booruvar\f[] (\f[I]*\f[]) .br * \f[I]coomer\f[] .br -* \f[I]danbooru\f[] (*) +* \f[I]danbooru\f[] (\f[I]*\f[]) .br * \f[I]deviantart\f[] .br -* \f[I]e621\f[] (*) +* \f[I]e621\f[] (\f[I]*\f[]) .br -* \f[I]e6ai\f[] (*) +* \f[I]e6ai\f[] (\f[I]*\f[]) .br -* \f[I]e926\f[] (*) +* \f[I]e926\f[] (\f[I]*\f[]) .br * \f[I]exhentai\f[] .br * \f[I]girlswithmuscle\f[] .br -* \f[I]horne\f[] (R) +* \f[I]horne\f[] (\f[I]R\f[]) .br * \f[I]idolcomplex\f[] .br @@ -613,7 +620,7 @@ This is supported for .br * \f[I]kemono\f[] .br -* \f[I]madokami\f[] (R) +* \f[I]madokami\f[] (\f[I]R\f[]) .br * \f[I]mangadex\f[] .br @@ -621,7 +628,7 @@ This is supported for .br * \f[I]newgrounds\f[] .br -* \f[I]nijie\f[] (R) +* \f[I]nijie\f[] (\f[I]R\f[]) .br * \f[I]pillowfort\f[] .br @@ -629,8 +636,6 @@ This is supported for .br * \f[I]sankaku\f[] .br -* \f[I]schalenetwork\f[] -.br * \f[I]scrolller\f[] .br * \f[I]seiga\f[] @@ -651,16 +656,24 @@ These values can also be specified via the \f[I]-u/--username\f[] and \f[I]-p/--password\f[] command-line options or by using a \f[I].netrc\f[] file. (see Authentication_) -(*) The password value for these sites should be -the API key found in your user profile, not the actual account password. - -(R) Login with username & password or supplying logged-in -\f[I]cookies\f[] is required -Note: Leave the \f[I]password\f[] value empty or undefined +.IP "Note:" 4 +Leave the \f[I]password\f[] value empty or undefined to be prompted for a password when performing a login (see \f[I]getpass()\f[]). +.. _pw-apikey: + +(*) The \f[I]password\f[] value for these sites should be +the API key found in your user profile, not the actual account password. + +.. _pw-required: + +(R) Login with username & password or +supplying authenticated +\f[I]cookies\f[] +is *required* + .SS extractor.*.input .IP "Type:" 6 @@ -690,7 +703,7 @@ Enable the use of \f[I].netrc\f[] authentication data. .br * \f[I]Path\f[] .br -* \f[I]object\f[] (name -> value) +* \f[I]object\f[] (name → value) .br * \f[I]list\f[] @@ -741,17 +754,12 @@ Source to read additional cookies from. This can be \f[I]string\f[] .IP "Default:" 9 -\f[I]"random"\f[] +\f[I]null\f[] .IP "Description:" 4 Interpret \f[I]extractor.cookies\f[] as a list of cookie sources and select one of them for each extractor run. -.br -* \f[I]"random"\f[]: Select cookies \f[I]randomly\f[] -.br -* \f[I]"rotate"\f[]: Select cookies in sequence. Start over from the beginning after reaching the end of the list. - .. code:: json [ @@ -762,6 +770,13 @@ as a list of cookie sources and select one of them for each extractor run. ] +.IP "Supported Values:" 4 +\f[I]"random"\f[] +Select cookies \f[I]randomly\f[]. +\f[I]"rotate"\f[] +Select cookies in sequence. Start over from the beginning after reaching the end of the list. + + .SS extractor.*.cookies-update .IP "Type:" 6 .br @@ -788,7 +803,7 @@ of a valid cookies.txt file, update its contents. .br * \f[I]string\f[] .br -* \f[I]object\f[] (scheme -> proxy) +* \f[I]object\f[] (scheme → proxy) .IP "Example:" 4 .. code:: json @@ -817,8 +832,8 @@ It is also possible to set a proxy for a specific host by using \f[I]scheme://host\f[] as key. See \f[I]Requests' proxy documentation\f[] for more details. -Note: If a proxy URL does not include a scheme, -\f[I]http://\f[] is assumed. +.IP "Note:" 4 +If a proxy URL does not include a scheme, \f[I]http://\f[] is assumed. .SS extractor.*.proxy-env @@ -861,18 +876,30 @@ or a \f[I]list\f[] with IP and explicit port number as elements. \f[I]string\f[] .IP "Default:" 9 +\f[I]"gallery-dl/VERSION"\f[] .br -* \f[I]"gallery-dl/VERSION"\f[]: \f[I][Danbooru]\f[], \f[I]mangadex\f[], \f[I]weasyl\f[] +* \f[I][Danbooru]\f[] .br -* \f[I]"gallery-dl/VERSION (by mikf)"\f[]: \f[I][E621]\f[] +* \f[I]mangadex\f[] +.br +* \f[I]weasyl\f[] +.br +* \f[I]zerochan\f[] +\f[I]"gallery-dl/VERSION (by mikf)"\f[] +.br +* \f[I][E621]\f[] +\f[I]"net.umanle.arca.android.playstore/0.9.75"\f[] .br -* \f[I]"net.umanle.arca.android.playstore/0.9.75"\f[]: \f[I]arcalive\f[] +* \f[I]arcalive\f[] +\f[I]"Patreon/72.2.28 (Android; Android 14; Scale/2.10)"\f[] .br -* \f[I]"Patreon/72.2.28 (Android; Android 14; Scale/2.10)"\f[]: \f[I]patreon\f[] +* \f[I]patreon\f[] +\f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/LATEST.0.0.0 Safari/537.36"\f[] .br -* \f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/LATEST.0.0.0 Safari/537.36"\f[]: \f[I]instagram\f[] +* \f[I]instagram\f[] +\f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:LATEST) Gecko/20100101 Firefox/LATEST"\f[] .br -* \f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:LATEST) Gecko/20100101 Firefox/LATEST"\f[]: otherwise +* otherwise .IP "Example:" 4 .br @@ -897,23 +924,37 @@ and use the \f[I]User-Agent\f[] header of this installed browser. \f[I]string\f[] .IP "Default:" 9 -.br -* \f[I]"firefox"\f[]: \f[I]artstation\f[], \f[I]behance\f[], \f[I]fanbox\f[], \f[I]twitter\f[] -.br -* \f[I]null\f[]: otherwise +\f[I]"firefox"\f[] +\f[I]artstation\f[] \f[I] +\f[I]behance\f[] \f[] +\f[I]fanbox\f[] | +\f[I]twitter\f[] +\f[I]null\f[] +otherwise .IP "Example:" 4 .br -* "firefox/128:linux" +* "firefox" +.br +* "firefox/128" .br * "chrome:macos" +.br +* "chrome/138:macos" .IP "Description:" 4 Try to emulate a real browser (\f[I]firefox\f[] or \f[I]chrome\f[]) -by using their default HTTP headers and TLS ciphers for HTTP requests. +.br +by using its HTTP headers and TLS ciphers for HTTP requests +.br +by setting specialized defaults for -Optionally, the operating system used in the \f[I]User-Agent\f[] header can be -specified after a \f[I]:\f[] (\f[I]windows\f[], \f[I]linux\f[], or \f[I]macos\f[]). +.br +* \f[I]user-agent\f[] +.br +* \f[I]headers\f[] +.br +* \f[I]ciphers\f[] Supported browsers: @@ -930,15 +971,16 @@ Supported browsers: .br * \f[I]chrome/111\f[] -Note: -This option sets custom -\f[I]headers\f[] -and -\f[I]ciphers\f[] -defaults. +The operating system used in the \f[I]User-Agent\f[] header can be +specified after a colon \f[I]:\f[] (\f[I]windows\f[], \f[I]linux\f[], \f[I]macos\f[]), +for example \f[I]chrome:windows\f[]. -Note: \f[I]requests\f[] and \f[I]urllib3\f[] only support HTTP/1.1, while a real -browser would use HTTP/2. +.IP "Note:" 4 +Any value *not* matching a supported browser +will fall back to \f[I]"firefox"\f[]. + +\f[I]requests\f[] and \f[I]urllib3\f[] only support HTTP/1.1, while a real +browser would use HTTP/2 and HTTP/3. .SS extractor.*.referer @@ -949,7 +991,14 @@ browser would use HTTP/2. * \f[I]string\f[] .IP "Default:" 9 +\f[I]false\f[] +\f[I]4archive\f[] \f[I] +\f[I]4chanarchives\f[] \f[] +\f[I]archivedmoe\f[] \f[I] +\f[I]nsfwalbum\f[] \f[] +\f[I]tumblrgallery\f[] \f[I]true\f[] +otherwise .IP "Description:" 4 Send \f[I]Referer\f[] @@ -964,7 +1013,7 @@ instead of the extractor's \f[I]root\f[] domain. .br * \f[I]"string"\f[] .br -* \f[I]object\f[] (name -> value) +* \f[I]object\f[] (name → value) .IP "Default:" 9 .. code:: json @@ -1022,10 +1071,12 @@ to use these browser's default ciphers. \f[I]bool\f[] .IP "Default:" 9 -.br -* \f[I]false\f[]: \f[I]artstation\f[], \f[I]behance\f[], \f[I]vsco\f[] -.br -* \f[I]true\f[]: otherwise +\f[I]false\f[] +\f[I]artstation\f[] \f[I] +\f[I]behance\f[] \f[] +\f[I]vsco\f[] +\f[I]true\f[] +otherwise .IP "Description:" 4 Allow selecting TLS 1.2 cipher suites. @@ -1036,7 +1087,7 @@ and potentially bypass Cloudflare blocks. .SS extractor.*.keywords .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 {"type": "Pixel Art", "type_id": 123} @@ -1054,7 +1105,7 @@ Additional name-value pairs to be added to each metadata dictionary. .IP "Description:" 4 Evaluate each \f[I]keywords\f[] \f[I]string\f[] value -as a \f[I]format string\f[]. +as a \f[I]Format String\f[]. .SS extractor.*.keywords-default @@ -1065,8 +1116,8 @@ any \f[I]"None"\f[] .IP "Description:" 4 -Default value used for missing or undefined keyword names in -\f[I]format strings\f[]. +Default value used for missing or undefined keyword names in a +\f[I]Format String\f[]. .SS extractor.*.url-metadata @@ -1176,7 +1227,8 @@ Each identifier can be Both names can be a * or left empty, matching all possible names (\f[I]"*:image"\f[], \f[I]":user"\f[]). .br -Note: Any \f[I]blacklist\f[] setting will automatically include +.IP "Note:" 4 +Any \f[I]blacklist\f[] setting will automatically include \f[I]"oauth"\f[], \f[I]"recursive"\f[], and \f[I]"test"\f[]. @@ -1211,9 +1263,10 @@ If this value is a the archive will use this PostgreSQL database as backend (requires \f[I]Psycopg\f[]). -Note: Archive files that do not already exist get generated automatically. +.IP "Note:" 4 +Archive files that do not already exist get generated automatically. -Note: Archive paths support regular \f[I]format string\f[] replacements, +Archive paths support basic \f[I]Format String\f[] replacements, but be aware that using external inputs for building local paths may pose a security risk. @@ -1243,13 +1296,13 @@ Available events are: .SS extractor.*.archive-format .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Example:" 4 "{id}_{offset}" .IP "Description:" 4 -An alternative \f[I]format string\f[] to build archive IDs with. +An alternative \f[I]Format String\f[] to build archive IDs with. .SS extractor.*.archive-mode @@ -1263,17 +1316,17 @@ An alternative \f[I]format string\f[] to build archive IDs with. Controls when to write \f[I]archive IDs\f[] to the archive database. -.br -* \f[I]"file"\f[]: Write IDs immediately +\f[I]"file"\f[] +Write IDs immediately after completing or skipping a file download. -.br -* \f[I]"memory"\f[]: Keep IDs in memory +\f[I]"memory"\f[] +Keep IDs in memory and only write them after successful job completion. .SS extractor.*.archive-prefix .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Default:" 9 .br @@ -1301,7 +1354,7 @@ for available \f[I]PRAGMA\f[] statements and further details. .SS extractor.*.archive-table .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Default:" 9 \f[I]"archive"\f[] @@ -1310,13 +1363,13 @@ for available \f[I]PRAGMA\f[] statements and further details. "{category}" .IP "Description:" 4 -\f[I]Format string\f[] selecting the archive database table name. +\f[I]Format String\f[] selecting the archive database table name. .SS extractor.*.actions .IP "Type:" 6 .br -* \f[I]object\f[] (pattern -> \f[I]Action(s)\f[]) +* \f[I]object\f[] (pattern → \f[I]Action(s)\f[]) .br * \f[I]list\f[] of [pattern, \f[I]Action(s)\f[]] pairs @@ -1408,7 +1461,7 @@ for each downloaded \f[I]pixiv\f[] file. .SS extractor.*.postprocessor-options .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 .. code:: json @@ -1544,15 +1597,15 @@ Use fallback download URLs when a download fails. .br * \f[I]list\f[] of \f[I]strings\f[] -.IP "Examples:" 4 +.IP "Example:" 4 .br -* \f[I]"10-20"\f[] +* "10-20" .br -* \f[I]"-5, 10, 30-50, 100-"\f[] +* "-5, 10, 30-50, 100-" .br -* \f[I]"10:21, 30:51:2, :5, 100:"\f[] +* "10:21, 30:51:2, :5, 100:" .br -* \f[I]["-5", "10", "30-50", "100-"]\f[] +* ["-5", "10", "30-50", "100-"] .IP "Description:" 4 Index range(s) selecting which files to download. @@ -1572,7 +1625,8 @@ and will default to begin (\f[I]1\f[]) or end (\f[I]sys.maxsize\f[]) if omitted. For example \f[I]5-\f[], \f[I]5:\f[], and \f[I]5::\f[] all mean "Start at file number 5". .br -Note: The index of the first file is \f[I]1\f[]. +.IP "Note:" 4 +The index of the first file is \f[I]1\f[]. .SS extractor.*.chapter-range @@ -1587,20 +1641,20 @@ but applies to delegated URLs like manga chapters, etc. .SS extractor.*.image-filter .IP "Type:" 6 .br -* \f[I]string\f[] +* \f[I]Condition\f[] .br -* \f[I]list\f[] of \f[I]strings\f[] +* \f[I]list\f[] of \f[I]Conditions\f[] -.IP "Examples:" 4 +.IP "Example:" 4 .br -* \f[I]"re.search(r'foo(bar)+', description)"\f[] +* "re.search(r'foo(bar)+', description)" .br -* \f[I]["width >= 1200", "width/height > 1.2"]\f[] +* ["width >= 1200", "width/height > 1.2"] .IP "Description:" 4 -Python expression controlling which files to download. +Python \f[I]Expression\f[] controlling which files to download. -A file only gets downloaded when *all* of the given expressions evaluate to \f[I]True\f[]. +A file only gets downloaded when *all* of the given \f[I]Expressions\f[] evaluate to \f[I]True\f[]. Available values are the filename-specific ones listed by \f[I]-K\f[] or \f[I]-j\f[]. @@ -1608,15 +1662,15 @@ Available values are the filename-specific ones listed by \f[I]-K\f[] or \f[I]-j .SS extractor.*.chapter-filter .IP "Type:" 6 .br -* \f[I]string\f[] +* \f[I]Condition\f[] .br -* \f[I]list\f[] of \f[I]strings\f[] +* \f[I]list\f[] of \f[I]Conditions\f[] -.IP "Examples:" 4 +.IP "Example:" 4 .br -* \f[I]"lang == 'en'"\f[] +* "lang == 'en'" .br -* \f[I]["language == 'French'", "10 <= chapter < 20"]\f[] +* ["language == 'French'", "10 <= chapter < 20"] .IP "Description:" 4 Like \f[I]image-filter\f[], @@ -1660,7 +1714,8 @@ date-min and date-max. See \f[I]strptime\f[] for a list of formatting directives. -Note: Despite its name, this option does **not** control how +.IP "Note:" 4 +Despite its name, this option does **not** control how \f[I]{date}\f[] metadata fields are formatted. To use a different formatting for those values other than the default \f[I]%Y-%m-%d %H:%M:%S\f[], put \f[I]strptime\f[] formatting directives @@ -1682,12 +1737,14 @@ During data extraction, write received HTTP request data to enumerated files in the current working directory. -Special values: - +.IP "Special Values:" 4 +\f[I]"all"\f[] +Include HTTP request and response headers. .br -* \f[I]"all"\f[]: Include HTTP request and response headers. Hide \f[I]Authorization\f[], \f[I]Cookie\f[], and \f[I]Set-Cookie\f[] values. +Hide \f[I]Authorization\f[], \f[I]Cookie\f[], and \f[I]Set-Cookie\f[] values. .br -* \f[I]"ALL"\f[]: Include all HTTP request and response headers. +\f[I]"ALL"\f[] +Include all HTTP request and response headers. .SH EXTRACTOR-SPECIFIC OPTIONS @@ -1823,11 +1880,11 @@ when retrieving search results. .IP "Description:" 4 Controls the post extraction strategy. -.br -* \f[I]true\f[]: Start on users' main gallery pages and recursively -descend into subfolders -.br -* \f[I]false\f[]: Get posts from "Latest Updates" pages +\f[I]true\f[] +Start on users' main gallery pages and +recursively descend into subfolders +\f[I]false\f[] +Get posts from "Latest Updates" pages .SS extractor.batoto.domain @@ -1881,8 +1938,17 @@ The maximum possible value appears to be \f[I]1920\f[]. .IP "Description:" 4 Selects which gallery modules to download from. -Supported module types are -\f[I]image\f[], \f[I]video\f[], \f[I]mediacollection\f[], \f[I]embed\f[], \f[I]text\f[]. +.IP "Supported Types:" 4 +.br +* \f[I]"image"\f[] +.br +* \f[I]"video"\f[] +.br +* \f[I]"mediacollection"\f[] +.br +* \f[I]"embed"\f[] +.br +* \f[I]"text"\f[] .SS extractor.bellazon.order-posts @@ -1959,16 +2025,25 @@ Download embedded videos hosted on https://www.blogger.com/ A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"info"\f[], -\f[I]"avatar"\f[], -\f[I]"background"\f[], -\f[I]"posts"\f[], -\f[I]"replies"\f[], -\f[I]"media"\f[], -\f[I]"video"\f[], -\f[I]"likes"\f[], +.IP "Supported Values:" 4 +.br +* \f[I]info\f[] +.br +* \f[I]avatar\f[] +.br +* \f[I]background\f[] +.br +* \f[I]posts\f[] +.br +* \f[I]replies\f[] +.br +* \f[I]media\f[] +.br +* \f[I]video\f[] +.br +* \f[I]likes\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -2020,11 +2095,13 @@ Requires login and only allows accessing your own likes. .IP "Description:" 4 Extract additional metadata. +\f[I]facets\f[] +\f[I]hashtags\f[], \f[I]mentions\f[], \f[I]uris\f[] +\f[I]user\f[] +Detailed \f[I]user\f[] metadata for the user referenced in the input URL. .br -* \f[I]facets\f[]: \f[I]hashtags\f[], \f[I]mentions\f[], and \f[I]uris\f[] +(\f[I]app.bsky.actor.getProfile\f[]) .br -* \f[I]user\f[]: detailed \f[I]user\f[] metadata for the user referenced in the input URL -(See \f[I]app.bsky.actor.getProfile\f[]). .SS extractor.bluesky.likes.depth @@ -2037,7 +2114,7 @@ Extract additional metadata. .IP "Description:" 4 Sets the maximum depth of returned reply posts. -(See depth parameter of \f[I]app.bsky.feed.getPostThread\f[]) +(See the \f[I]depth\f[] parameter of \f[I]app.bsky.feed.getPostThread\f[]) .SS extractor.bluesky.quoted @@ -2123,26 +2200,24 @@ Provide detailed \f[I]user\f[] metadata. Download videos. If this is a \f[I]list\f[], it selects which format to try to download. -.br -Possibly available formats are -.br +.IP "Possible Formats:" 4 .br * \f[I]ultra_hd\f[] (2160p) .br -* \f[I]quad_hd\f[] (1440p) +* \f[I]quad_hd\f[] (1440p) .br -* \f[I]full_hd\f[] (1080p) +* \f[I]full_hd\f[] (1080p) .br -* \f[I]high\f[] (720p) +* \f[I]high\f[] (720p) .br -* \f[I]medium\f[] (480p) +* \f[I]medium\f[] (480p) .br -* \f[I]low\f[] (360p) +* \f[I]low\f[] (360p) .br -* \f[I]lowest\f[] (240p) +* \f[I]lowest\f[] (240p) .br -* \f[I]tiny\f[] (144p) +* \f[I]tiny\f[] (144p) .SS extractor.booth.strategy @@ -2184,10 +2259,10 @@ API endpoint for retrieving file URLs. .IP "Description:" 4 Controls which \f[I]bunkr\f[] TLDs to accept. -.br -* \f[I]true\f[]: Match URLs with *all* possible TLDs (e.g. \f[I]bunkr.xyz\f[] or \f[I]bunkrrr.duck\f[]) -.br -* \f[I]false\f[]: Match only URLs with known TLDs +\f[I]true\f[] +Match URLs with *all* possible TLDs (e.g. \f[I]bunkr.xyz\f[] or \f[I]bunkrrr.duck\f[]) +\f[I]false\f[] +Match only URLs with known TLDs .SS extractor.cien.files @@ -2200,11 +2275,15 @@ Controls which \f[I]bunkr\f[] TLDs to accept. .IP "Description:" 4 Determines the type and order of files to download. -Available types are -\f[I]image\f[], -\f[I]video\f[], -\f[I]download\f[], -\f[I]gallery\f[]. +.IP "Available Types:" 4 +.br +* \f[I]image\f[] +.br +* \f[I]video\f[] +.br +* \f[I]download\f[] +.br +* \f[I]gallery\f[] .SS extractor.civitai.api @@ -2217,10 +2296,10 @@ Available types are .IP "Description:" 4 Selects which API endpoints to use. -.br -* \f[I]"rest"\f[]: \f[I]Public REST API\f[] -.br -* \f[I]"trpc"\f[]: Internal tRPC API +\f[I]"rest"\f[] +\f[I]Public REST API\f[] +\f[I]"trpc"\f[] +Internal tRPC API .SS extractor.civitai.api-key @@ -2246,10 +2325,13 @@ for details. .IP "Description:" 4 Determines the type and order of files to download when processing models. -Available types are -\f[I]model\f[], -\f[I]image\f[], -\f[I]gallery\f[]. +.IP "Available Types:" 4 +.br +* \f[I]model\f[] +.br +* \f[I]image\f[] +.br +* \f[I]gallery\f[] .SS extractor.civitai.include @@ -2266,22 +2348,21 @@ Available types are A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are - +.IP "Supported Values:" 4 .br -* \f[I]"user-models"\f[] +* \f[I]user-models\f[] .br -* \f[I]"user-posts"\f[] +* \f[I]user-posts\f[] .br -* \f[I]"user-images"\f[] +* \f[I]user-images\f[] .br -* \f[I]"user-videos"\f[] +* \f[I]user-videos\f[] .br -* \f[I]"user-collections"\f[] +* \f[I]user-collections\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. -.IP "Note:" 4 To get a more complete set of metadata like \f[I]model['name']\f[] and \f[I]post['title']\f[], include \f[I]user-models\f[] and \f[I]user-posts\f[] @@ -2311,7 +2392,8 @@ as well as the default \f[I]user-images\f[] and \f[I]user-videos\f[]: .IP "Description:" 4 Extract additional \f[I]generation\f[], \f[I]version\f[], and \f[I]post\f[] metadata. -Note: This requires 1 or more additional API requests per image or video. +.IP "Note:" 4 +This requires 1 or more additional API requests per image or video. .SS extractor.civitai.nsfw @@ -2366,7 +2448,8 @@ to pass with every image URL. Known available options include \f[I]original\f[], \f[I]quality\f[], \f[I]width\f[] -Note: Set this option to an arbitrary letter, e.g., \f[I]"w"\f[], +.IP "Note:" 4 +Set this option to an arbitrary letter, e.g., \f[I]"w"\f[], to download images in JPEG format at their original resolution. @@ -2423,8 +2506,7 @@ Use \f[I]+\f[] as first character to add the given options to the * ["fr", "it", "pl"] .IP "Description:" 4 -\f[I]ISO 639-1\f[] language codes -to filter chapters by. +\f[I]ISO 639-1\f[] code(s) to filter chapters by. .SS extractor.cyberdrop.domain @@ -2454,7 +2536,7 @@ uses the same domain as a given input URL. .IP "Description:" 4 Password value used to access protected files and folders. -Note: Leave this value empty or undefined +Leave this value empty or undefined to be interactively prompted for a password when needed (see \f[I]getpass()\f[]). @@ -2501,10 +2583,10 @@ Ascending Post ID order .IP "Description:" 4 Controls the download target for Ugoira posts. -.br -* \f[I]true\f[]: Original ZIP archives -.br -* \f[I]false\f[]: Converted video files +\f[I]true\f[] +ZIP archives +\f[I]false\f[] +Converted video files .SS extractor.[Danbooru].metadata @@ -2533,7 +2615,8 @@ It is possible to specify a custom list of metadata includes. See \f[I]available_includes\f[] for possible field names. \f[I]aibooru\f[] also supports \f[I]ai_metadata\f[]. -Note: This requires 1 additional HTTP request per 200-post batch. +.IP "Note:" 4 +This requires 1 additional HTTP request per 200-post batch. .SS extractor.[Danbooru].threshold @@ -2551,7 +2634,8 @@ Stop paginating over API results if the length of a batch of returned posts is less than the specified number. Defaults to the per-page limit of the current instance, which is 200. -Note: Changing this setting is normally not necessary. When the value is +.IP "Note:" 4 +Changing this setting is normally not necessary. When the value is greater than the per-page limit, gallery-dl will stop after the first batch. The value cannot be less than 1. @@ -2612,7 +2696,8 @@ Extract \f[I]comments\f[] metadata. .IP "Description:" 4 Download the avatar of each commenting user. -Note: Enabling this option also enables deviantart.comments_. +.IP "Note:" 4 +Enabling this option also enables deviantart.comments_. .SS extractor.deviantart.extra @@ -2626,7 +2711,8 @@ Note: Enabling this option also enables deviantart.comments_. Download extra Sta.sh resources from description texts and journals. -Note: Enabling this option also enables deviantart.metadata_. +.IP "Note:" 4 +Enabling this option also enables deviantart.metadata_. .SS extractor.deviantart.flat @@ -2640,16 +2726,17 @@ Note: Enabling this option also enables deviantart.metadata_. Select the directory structure created by the Gallery- and Favorite-Extractors. -.br -* \f[I]true\f[]: Use a flat directory structure. -.br -* \f[I]false\f[]: Collect a list of all gallery-folders or -favorites-collections and transfer any further work to other +\f[I]true\f[] +Use a flat directory structure. +\f[I]false\f[] +Collect a list of all gallery \f[I]folders\f[] or +favorites \f[I]collections\f[] and transfer any further work to other extractors (\f[I]folder\f[] or \f[I]collection\f[]), which will then create individual subdirectories for each of them. -Note: Going through all gallery folders will not be able to -fetch deviations which aren't in any folder. +.IP "Note:" 4 +Going through all gallery folders won't +fetch deviations not contained in any folder. .SS extractor.deviantart.folders @@ -2663,7 +2750,8 @@ fetch deviations which aren't in any folder. Provide a \f[I]folders\f[] metadata field that contains the names of all folders a deviation is present in. -Note: Gathering this information requires a lot of API calls. +.IP "Note:" 4 +Gathering this information requires a lot of API calls. Use with caution. @@ -2684,10 +2772,9 @@ belongs to a group or a regular user. When disabled, assume every given profile name belongs to a regular user. -Special values: - -.br -* \f[I]"skip"\f[]: Skip groups +.IP "Special Values:" 4 +\f[I]"skip"\f[] +Skip groups .SS extractor.deviantart.include @@ -2710,15 +2797,23 @@ Special values: A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"avatar"\f[], -\f[I]"background"\f[], -\f[I]"gallery"\f[], -\f[I]"scraps"\f[], -\f[I]"journal"\f[], -\f[I]"favorite"\f[], -\f[I]"status"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]avatar\f[] +.br +* \f[I]background\f[] +.br +* \f[I]gallery\f[] +.br +* \f[I]scraps\f[] +.br +* \f[I]journal\f[] +.br +* \f[I]favorite\f[] +.br +* \f[I]status\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -2745,27 +2840,12 @@ download a higher-quality \f[I]/intermediary/\f[] version. Selects the output format for textual content. This includes journals, literature and status updates. -.br -* \f[I]"html"\f[]: HTML with (roughly) the same layout as on DeviantArt. -.br -* \f[I]"text"\f[]: Plain text with image references and HTML tags removed. -.br -* \f[I]"none"\f[]: Don't download textual content. - - -.SS extractor.deviantart.jwt -.IP "Type:" 6 -\f[I]bool\f[] - -.IP "Default:" 9 -\f[I]false\f[] - -.IP "Description:" 4 -Update \f[I]JSON Web Tokens\f[] (the \f[I]token\f[] URL parameter) -of otherwise non-downloadable, low-resolution images -to be able to download them in full resolution. - -Note: No longer functional as of 2023-10-11 +\f[I]"html"\f[] +HTML with (roughly) the same layout as on DeviantArt. +\f[I]"text"\f[] +Plain text with image references and HTML tags removed. +\f[I]"none"\f[] +Don't download textual content. .SS extractor.deviantart.mature @@ -2810,17 +2890,18 @@ fields when enabled. It is possible to request extended metadata by specifying a list of -.br -* \f[I]camera\f[] : EXIF information (if available) -.br -* \f[I]stats\f[] : deviation statistics -.br -* \f[I]submission\f[] : submission information -.br -* \f[I]collection\f[] : favourited folder information (requires a \f[I]refresh token\f[]) -.br -* \f[I]gallery\f[] : gallery folder information (requires a \f[I]refresh token\f[]) +\f[I]camera\f[] +EXIF information if available +\f[I]stats\f[] +Deviation statistics +\f[I]submission\f[] +Submission information +\f[I]collection\f[] +Favourited folder information (requires a \f[I]refresh token\f[]) +\f[I]gallery\f[] +Gallery folder information (requires a \f[I]refresh token\f[]) +.IP "Note:" 4 Set this option to \f[I]"all"\f[] to request all extended metadata categories. See \f[I]/deviation/metadata\f[] @@ -2855,10 +2936,10 @@ everything else (archives, videos, etc.). .IP "Description:" 4 Controls when to stop paginating over API results. -.br -* \f[I]"api"\f[]: Trust the API and stop when \f[I]has_more\f[] is \f[I]false\f[]. -.br -* \f[I]"manual"\f[]: Disregard \f[I]has_more\f[] and only stop when a batch of results is empty. +\f[I]"api"\f[] +Trust the API and stop when \f[I]has_more\f[] is \f[I]false\f[]. +\f[I]"manual"\f[] +Disregard \f[I]has_more\f[] and only stop when a batch of results is empty. .SS extractor.deviantart.previews @@ -2920,7 +3001,8 @@ The \f[I]refresh-token\f[] value you get from Using a \f[I]refresh-token\f[] allows you to access private or otherwise not publicly available deviations. -Note: The \f[I]refresh-token\f[] becomes invalid +.IP "Note:" 4 +The \f[I]refresh-token\f[] becomes invalid \f[I]after 3 months\f[] or whenever your \f[I]cache file\f[] is deleted or cleared. @@ -3043,7 +3125,8 @@ from an anthology's HTML page. .IP "Description:" 4 Extract additional metadata (notes, pool metadata) if available. -Note: This requires 0-2 additional HTTP requests per post. +.IP "Note:" 4 +This requires 0-2 additional HTTP requests per post. .SS extractor.[E621].threshold @@ -3061,7 +3144,8 @@ Stop paginating over API results if the length of a batch of returned posts is less than the specified number. Defaults to the per-page limit of the current instance, which is 320. -Note: Changing this setting is normally not necessary. When the value is +.IP "Note:" 4 +Changing this setting is normally not necessary. When the value is greater than the per-page limit, gallery-dl will stop after the first batch. The value cannot be less than 1. @@ -3085,13 +3169,13 @@ Include reposts when extracting albums from a user profile. \f[I]"auto"\f[] .IP "Description:" 4 -.br -* \f[I]"auto"\f[]: Use \f[I]e-hentai.org\f[] or \f[I]exhentai.org\f[] +\f[I]"auto"\f[] +Use \f[I]e-hentai.org\f[] or \f[I]exhentai.org\f[] depending on the input URL -.br -* \f[I]"e-hentai.org"\f[]: Use \f[I]e-hentai.org\f[] for all URLs -.br -* \f[I]"exhentai.org"\f[]: Use \f[I]exhentai.org\f[] for all URLs +\f[I]"e-hentai.org"\f[] +Use \f[I]e-hentai.org\f[] for all URLs +\f[I]"exhentai.org"\f[] +Use \f[I]exhentai.org\f[] for all URLs .SS extractor.exhentai.fallback-retries @@ -3117,9 +3201,10 @@ or \f[I]-1\f[] for infinite retries. After downloading a gallery, add it to your account's favorites as the given category number. -Note: Set this to "favdel" to remove galleries from your favorites. +.IP "Note:" 4 +Set this to "favdel" to remove galleries from your favorites. -Note: This will remove any Favorite Notes when applied +This will remove any Favorite Notes when applied to already favorited galleries. @@ -3210,10 +3295,10 @@ Download full-sized original images if available. .IP "Description:" 4 Selects an alternative source to download files from. -.br -* \f[I]"hitomi"\f[]: Download the corresponding gallery from \f[I]hitomi.la\f[] -.br -* \f[I]"metadata"\f[]: Load only a gallery's metadata from the +\f[I]"hitomi"\f[] +Download the corresponding gallery from \f[I]hitomi.la\f[] +\f[I]"metadata"\f[] +Load only a gallery's metadata from the \f[I]API\f[] @@ -3261,8 +3346,7 @@ Extract comments that include photo attachments made by the author of the post. A (comma-separated) list of subcategories to include when processing a user profile. -Supported values are - +.IP "Supported Values:" 4 .br * \f[I]info\f[] .br @@ -3272,6 +3356,7 @@ Supported values are .br * \f[I]albums\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -3288,12 +3373,12 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately. .IP "Description:" 4 Control video download behavior. -.br -* \f[I]true\f[]: Extract and download video & audio separately. -.br -* \f[I]"ytdl"\f[]: Let \f[I]ytdl\f[] handle video extraction and download, and merge video & audio streams. -.br -* \f[I]false\f[]: Ignore videos. +\f[I]true\f[] +Extract and download video & audio separately. +\f[I]"ytdl"\f[] +Let \f[I]ytdl\f[] handle video extraction and download, and merge video & audio streams. +\f[I]false\f[] +Ignore videos. .SS extractor.fanbox.comments @@ -3306,7 +3391,8 @@ Control video download behavior. .IP "Description:" 4 Extract \f[I]comments\f[] metadata. -Note: This requires 1 or more additional API requests per post, +.IP "Note:" 4 +This requires 1 or more additional API requests per post, depending on the number of comments. @@ -3323,14 +3409,14 @@ depending on the number of comments. .IP "Description:" 4 Control behavior on embedded content from external sites. -.br -* \f[I]true\f[]: Extract embed URLs and download them if supported +\f[I]true\f[] +Extract embed URLs and download them if supported (videos are not downloaded). -.br -* \f[I]"ytdl"\f[]: Like \f[I]true\f[], but let \f[I]ytdl\f[] handle video +\f[I]"ytdl"\f[] +Like \f[I]true\f[], but let \f[I]ytdl\f[] handle video extraction and download for YouTube, Vimeo, and SoundCloud embeds. -.br -* \f[I]false\f[]: Ignore embeds. +\f[I]false\f[] +Ignore embeds. .SS extractor.fanbox.fee-max @@ -3341,7 +3427,8 @@ extraction and download for YouTube, Vimeo, and SoundCloud embeds. Do not request API data or extract files from posts that require a fee (\f[I]feeRequired\f[]) greater than the specified amount. -Note: This option has no effect on individual post URLs. +.IP "Note:" 4 +This option has no effect on individual post URLs. .SS extractor.fanbox.metadata @@ -3365,8 +3452,7 @@ Note: This option has no effect on individual post URLs. .IP "Description:" 4 Extract \f[I]plan\f[] and extended \f[I]user\f[] metadata. -Supported fields when selecting which data to extract are - +.IP "Supported Fields:" 4 .br * \f[I]comments\f[] .br @@ -3374,7 +3460,8 @@ Supported fields when selecting which data to extract are .br * \f[I]user\f[] -Note: \f[I]comments\f[] can also be enabled via +.IP "Note:" 4 +\f[I]comments\f[] can also be enabled via \f[I]fanbox.comments\f[] @@ -3425,7 +3512,8 @@ from \f[I]linking your Flickr account to gallery-dl\f[]. For each photo, return the albums and pools it belongs to as \f[I]set\f[] and \f[I]pool\f[] metadata. -Note: This requires 1 additional API call per photo. +.IP "Note:" 4 +This requires 1 additional API call per photo. See \f[I]flickr.photos.getAllContexts\f[] for details. @@ -3440,7 +3528,8 @@ See \f[I]flickr.photos.getAllContexts\f[] for details. For each photo, return its EXIF/TIFF/GPS tags as \f[I]exif\f[] and \f[I]camera\f[] metadata. -Note: This requires 1 additional API call per photo. +.IP "Note:" 4 +This requires 1 additional API call per photo. See \f[I]flickr.photos.getExif\f[] for details. @@ -3455,7 +3544,8 @@ See \f[I]flickr.photos.getExif\f[] for details. For each photo, retrieve its "full" metadata as provided by \f[I]flickr.photos.getInfo\f[] -Note: This requires 1 additional API call per photo. +.IP "Note:" 4 +This requires 1 additional API call per photo. .SS extractor.flickr.metadata @@ -3496,7 +3586,8 @@ for possible field names. .IP "Description:" 4 Extract additional \f[I]user\f[] profile metadata. -Note: This requires 1 additional API call per user profile. +.IP "Note:" 4 +This requires 1 additional API call per user profile. See \f[I]flickr.people.getInfo\f[] for details. @@ -3543,10 +3634,10 @@ Sets the maximum allowed size for downloaded images. .IP "Description:" 4 Controls the format of \f[I]description\f[] metadata fields. -.br -* \f[I]"text"\f[]: Plain text with HTML tags removed -.br -* \f[I]"html"\f[]: Raw HTML content +\f[I]"text"\f[] +Plain text with HTML tags removed +\f[I]"html"\f[] +Raw HTML content .SS extractor.furaffinity.external @@ -3580,9 +3671,15 @@ Follow external URLs linked in descriptions. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"gallery"\f[], \f[I]"scraps"\f[], \f[I]"favorite"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]gallery\f[] +.br +* \f[I]scraps\f[] +.br +* \f[I]favorite\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -3596,12 +3693,12 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately. .IP "Description:" 4 Selects which site layout to expect when parsing posts. -.br -* \f[I]"auto"\f[]: Automatically differentiate between \f[I]"old"\f[] and \f[I]"new"\f[] -.br -* \f[I]"old"\f[]: Expect the *old* site layout -.br -* \f[I]"new"\f[]: Expect the *new* site layout +\f[I]"auto"\f[] +Automatically differentiate between \f[I]"old"\f[] and \f[I]"new"\f[] +\f[I]"old"\f[] +Expect the *old* site layout +\f[I]"new"\f[] +Expect the *new* site layout .SS extractor.gelbooru.api-key & .user-id @@ -3628,12 +3725,10 @@ page. .IP "Description:" 4 Controls the order in which favorited posts are returned. -.br -* \f[I]"asc"\f[]: Ascending favorite date order (oldest first) -.br -* \f[I]"desc"\f[]: Descending favorite date order (newest first) -.br -* \f[I]"reverse"\f[]: Same as \f[I]"asc"\f[] +\f[I]"asc"\f[] +Ascending favorite date order (oldest first) +\f[I]"desc"\f[] | \f[I]"reverse"\f[] +Descending favorite date order (newest first) .SS extractor.generic.enabled @@ -3705,13 +3800,14 @@ To get this value: .br * Open your browser's Developer Tools (F12) .br -* Select Network -> XHR +* Select Network → XHR .br * Open a gallery page .br * Select the last Network entry and copy its \f[I]crt\f[] value -Note: You will also need your browser's +.IP "Note:" 4 +You will also need your browser's \f[I]user-agent\f[] @@ -3777,10 +3873,10 @@ to access \f[I]favorite\f[] galleries. .IP "Description:" 4 Controls the format of \f[I]description\f[] metadata fields. -.br -* \f[I]"text"\f[]: Plain text with HTML tags removed -.br -* \f[I]"html"\f[]: Raw HTML content +\f[I]"text"\f[] +Plain text with HTML tags removed +\f[I]"html"\f[] +Raw HTML content .SS extractor.hentaifoundry.include @@ -3803,9 +3899,17 @@ Controls the format of \f[I]description\f[] metadata fields. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"pictures"\f[], \f[I]"scraps"\f[], \f[I]"stories"\f[], \f[I]"favorite"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]pictures\f[] +.br +* \f[I]scraps\f[] +.br +* \f[I]stories\f[] +.br +* \f[I]favorite\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -3819,7 +3923,11 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately. .IP "Description:" 4 Selects which image format to download. -Available formats are \f[I]"webp"\f[] and \f[I]"avif"\f[]. +.IP "Available Formats:" 4 +.br +* \f[I]"webp"\f[] +.br +* \f[I]"avif"\f[] .SS extractor.imagechest.access-token @@ -3858,13 +3966,13 @@ Custom Client ID value for API requests. .IP "Description:" 4 Controls whether to choose the GIF or MP4 version of an animation. -.br -* \f[I]true\f[]: Follow Imgur's advice and choose MP4 if the +\f[I]true\f[] +Follow Imgur's advice and choose MP4 if the \f[I]prefer_video\f[] flag in an image's metadata is set. -.br -* \f[I]false\f[]: Always choose GIF. -.br -* \f[I]"always"\f[]: Always choose MP4. +\f[I]false\f[] +Always choose GIF. +\f[I]"always"\f[] +Always choose MP4. .SS extractor.inkbunny.orderby @@ -3891,10 +3999,10 @@ for details) .IP "Description:" 4 Selects which API endpoints to use. -.br -* \f[I]"rest"\f[]: REST API - higher-resolution media -.br -* \f[I]"graphql"\f[]: GraphQL API - lower-resolution media +\f[I]"rest"\f[] +REST API - higher-resolution media +\f[I]"graphql"\f[] +GraphQL API - lower-resolution media .SS extractor.instagram.cursor @@ -3944,15 +4052,23 @@ Start from the position defined by this value. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"posts"\f[], -\f[I]"reels"\f[], -\f[I]"tagged"\f[], -\f[I]"stories"\f[], -\f[I]"highlights"\f[], -\f[I]"info"\f[], -\f[I]"avatar"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]posts\f[] +.br +* \f[I]reels\f[] +.br +* \f[I]tagged\f[] +.br +* \f[I]stories\f[] +.br +* \f[I]highlights\f[] +.br +* \f[I]info\f[] +.br +* \f[I]avatar\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -3978,7 +4094,8 @@ Limit the number of posts to download. Provide extended \f[I]user\f[] metadata even when referring to a user by ID, e.g. \f[I]instagram.com/id:12345678\f[]. -Note: This metadata is always available when referring to a user by name, +.IP "Note:" 4 +This metadata is always available when referring to a user by name, e.g. \f[I]instagram.com/USERNAME\f[]. @@ -3992,14 +4109,13 @@ e.g. \f[I]instagram.com/USERNAME\f[]. .IP "Description:" 4 Controls the order in which files of each post are returned. -.br -* \f[I]"asc"\f[]: Same order as displayed in a post -.br -* \f[I]"desc"\f[]: Reverse order as displayed in a post -.br -* \f[I]"reverse"\f[]: Same as \f[I]"desc"\f[] +\f[I]"asc"\f[] +Same order as displayed in a post +\f[I]"desc"\f[] | \f[I]"reverse"\f[] +Reverse order as displayed in a post -Note: This option does *not* affect \f[I]{num}\f[]. +.IP "Note:" 4 +This option does *not* affect \f[I]{num}\f[]. To enumerate files in reverse order, use \f[I]count - num + 1\f[]. @@ -4013,18 +4129,17 @@ To enumerate files in reverse order, use \f[I]count - num + 1\f[]. .IP "Description:" 4 Controls the order in which posts are returned. -.br -* \f[I]"asc"\f[]: Same order as displayed -.br -* \f[I]"desc"\f[]: Reverse order as displayed -.br -* \f[I]"id"\f[] or \f[I]"id_asc"\f[]: Ascending order by ID -.br -* \f[I]"id_desc"\f[]: Descending order by ID -.br -* \f[I]"reverse"\f[]: Same as \f[I]"desc"\f[] +\f[I]"asc"\f[] +Same order as displayed +\f[I]"desc"\f[] | \f[I]"reverse"\f[] +Reverse order as displayed +\f[I]"id"\f[] or \f[I]"id_asc"\f[] +Ascending order by ID +\f[I]"id_desc"\f[] +Descending order by ID -Note: This option only affects \f[I]highlights\f[]. +.IP "Note:" 4 +This option only affects \f[I]highlights\f[]. .SS extractor.instagram.previews @@ -4059,6 +4174,43 @@ Download pre-merged video formats Do not download videos +.SS extractor.instagram.warn-images +.IP "Type:" 6 +.br +* \f[I]bool\f[] +.br +* \f[I]string\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Show a warning when downloading images +with a resolution smaller than the original. + +\f[I]true\f[] +Show a warning when at least one dimension +is smaller than the reported original resolution +\f[I]"all"\f[] | \f[I]"both"\f[] +Show a warning only when both \f[I]width\f[] and \f[I]height\f[] +are smaller than the reported original resolution +\f[I]false\f[] +Do not show a warning + + +.SS extractor.instagram.warn-videos +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Show a warning when downloading videos with a +\f[I]User-Agent\f[] +header causing potentially lowered video quality. + + .SS extractor.instagram.stories.split .IP "Type:" 6 .br @@ -4091,8 +4243,7 @@ Split \f[I]stories\f[] elements into separate posts. A (comma-separated) list of subcategories to include when processing a user profile. -Supported values are - +.IP "Supported Values:" 4 .br * \f[I]gallery\f[] .br @@ -4104,6 +4255,7 @@ Supported values are .br * \f[I]stars\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -4132,15 +4284,15 @@ Download video files. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are - +.IP "Supported Values:" 4 .br -* \f[I]"user-images"\f[] +* \f[I]user-images\f[] .br -* \f[I]"user-videos"\f[] +* \f[I]user-videos\f[] .br -* \f[I]"user-playlists"\f[] +* \f[I]user-playlists\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -4155,7 +4307,8 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately. Extract additional metadata for \f[I]archives\f[] files, including \f[I]file\f[], \f[I]file_list\f[], and \f[I]password\f[]. -Note: This requires 1 additional HTTP request per \f[I]archives\f[] file. +.IP "Note:" 4 +This requires 1 additional HTTP request per \f[I]archives\f[] file. .SS extractor.kemono.comments @@ -4168,7 +4321,8 @@ Note: This requires 1 additional HTTP request per \f[I]archives\f[] file. .IP "Description:" 4 Extract \f[I]comments\f[] metadata. -Note: This requires 1 additional HTTP request per post. +.IP "Note:" 4 +This requires 1 additional HTTP request per post. .SS extractor.kemono.duplicates @@ -4265,7 +4419,13 @@ Available types are \f[I]artist\f[], and \f[I]post\f[]. .IP "Description:" 4 Determines the type and order of files to be downloaded. -Available types are \f[I]file\f[], \f[I]attachments\f[], and \f[I]inline\f[]. +.IP "Available Types:" 4 +.br +* \f[I]file\f[] +.br +* \f[I]attachments\f[] +.br +* \f[I]inline\f[] .SS extractor.kemono.max-posts @@ -4305,7 +4465,8 @@ Extract post revisions. Set this to \f[I]"unique"\f[] to filter out duplicate revisions. -Note: This requires 1 additional HTTP request per post. +.IP "Note:" 4 +This requires 1 additional HTTP request per post. .SS extractor.kemono.order-revisions @@ -4414,7 +4575,7 @@ The server to use for API requests. .SS extractor.mangadex.api-parameters .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 {"order[updatedAt]": "desc"} @@ -4442,8 +4603,7 @@ and \f[I]/user/follows/manga/feed\f[]) * ["fr", "it"] .IP "Description:" 4 -\f[I]ISO 639-1\f[] language codes -to filter chapters by. +\f[I]ISO 639-1\f[] code(s) to filter chapters by. .SS extractor.mangadex.ratings @@ -4468,6 +4628,31 @@ to filter chapters by. List of acceptable content ratings for returned chapters. +.SS extractor.mangafire.manga.lang +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Default:" 9 +\f[I]"en"\f[] + +.IP "Description:" 4 +\f[I]ISO 639-1\f[] code selecting which chapters to download. + + +.SS extractor.mangareader.manga.lang +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Default:" 9 +\f[I]"en"\f[] + +.IP "Example:" 4 +"pt-br" + +.IP "Description:" 4 +\f[I]ISO 639-1\f[] code selecting which chapters to download. + + .SS extractor.mangapark.source .IP "Type:" 6 .br @@ -4504,10 +4689,9 @@ Specifying the numeric \f[I]ID\f[] of a source is also supported. The \f[I]access-token\f[] value you get from \f[I]linking your account to gallery-dl\f[]. -Note: gallery-dl comes with built-in tokens for \f[I]mastodon.social\f[], -\f[I]pawoo\f[] and \f[I]baraag\f[]. For other instances, you need to obtain an -\f[I]access-token\f[] in order to use usernames in place of numerical -user IDs. +.IP "Note:" 4 +gallery-dl comes with built-in tokens for +\f[I]mastodon.social\f[], \f[I]pawoo\f[], and \f[I]baraag\f[]. .SS extractor.[mastodon].cards @@ -4582,12 +4766,17 @@ Your access token, necessary to fetch favorited notes. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"info"\f[], -\f[I]"avatar"\f[], -\f[I]"background"\f[], -\f[I]"notes"\f[], +.IP "Supported Values:" 4 +.br +* \f[I]info\f[] +.br +* \f[I]avatar\f[] +.br +* \f[I]background\f[] +.br +* \f[I]notes\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -4623,7 +4812,8 @@ Fetch media from replies to other notes. .IP "Description:" 4 Extract extended \f[I]pool\f[] metadata. -Note: Not supported by all \f[I]moebooru\f[] instances. +.IP "Note:" 4 +Not supported by all \f[I]moebooru\f[] instances. .SS extractor.naver-blog.videos @@ -4706,9 +4896,17 @@ until an available format is found. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"art"\f[], \f[I]"audio"\f[], \f[I]"games"\f[], \f[I]"movies"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]art\f[] +.br +* \f[I]audio\f[] +.br +* \f[I]games\f[] +.br +* \f[I]movies\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -4726,9 +4924,17 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"illustration"\f[], \f[I]"doujin"\f[], \f[I]"favorite"\f[], \f[I]"nuita"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]illustration\f[] +.br +* \f[I]doujin\f[] +.br +* \f[I]favorite\f[] +.br +* \f[I]nuita\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -4767,12 +4973,12 @@ Fetch media from Retweets. .IP "Description:" 4 Control video download behavior. -.br -* \f[I]true\f[]: Download videos -.br -* \f[I]"ytdl"\f[]: Download videos using \f[I]ytdl\f[] -.br -* \f[I]false\f[]: Skip video Tweets +\f[I]true\f[] +Download videos +\f[I]"ytdl"\f[] +Download videos using \f[I]ytdl\f[] +\f[I]false\f[] +Skip video Tweets .SS extractor.oauth.browser @@ -4785,11 +4991,11 @@ Control video download behavior. .IP "Description:" 4 Controls how a user is directed to an OAuth authorization page. -.br -* \f[I]true\f[]: Use Python's \f[I]webbrowser.open()\f[] method to automatically +\f[I]true\f[] +Use Python's \f[I]webbrowser.open()\f[] method to automatically open the URL in the user's default browser. -.br -* \f[I]false\f[]: Ask the user to copy & paste an URL from the terminal. +\f[I]false\f[] +Ask the user to copy & paste an URL from the terminal. .SS extractor.oauth.cache @@ -4825,7 +5031,8 @@ Host name / IP address to bind to during OAuth authorization. .IP "Description:" 4 Port number to listen on during OAuth authorization. -Note: All redirects will go to port \f[I]6414\f[], regardless +.IP "Note:" 4 +All redirects will go to port \f[I]6414\f[], regardless of the port specified here. You'll have to manually adjust the port number in your browser's address bar when using a different port than the default. @@ -4841,7 +5048,8 @@ port than the default. .IP "Description:" 4 Extract additional metadata (\f[I]source\f[], \f[I]uploader\f[]) -Note: This requires 1 additional HTTP request per post. +.IP "Note:" 4 +This requires 1 additional HTTP request per post. .SS extractor.patreon.cursor @@ -4881,8 +5089,7 @@ Start from the position defined by this value. .IP "Description:" 4 Determines types and order of files to download. -Available types: - +.IP "Available Types:" 4 .br * \f[I]postfile\f[] .br @@ -4905,8 +5112,7 @@ Available types: .IP "Description:" 4 Selects the format of \f[I]images\f[] \f[I]files\f[]. -Possible formats: - +.IP "Available Formats:" 4 .br * \f[I]download_url\f[] (\f[I]"a":1,"p":1\f[]) .br @@ -5107,14 +5313,21 @@ Recursively download files from subfolders. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"artworks"\f[], -\f[I]"avatar"\f[], -\f[I]"background"\f[], -\f[I]"favorite"\f[], -\f[I]"novel-user"\f[], -\f[I]"novel-bookmark"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]artworks\f[] +.br +* \f[I]avatar\f[] +.br +* \f[I]background\f[] +.br +* \f[I]favorite\f[] +.br +* \f[I]novel-user\f[] +.br +* \f[I]novel-bookmark\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -5152,7 +5365,8 @@ For works bookmarked by \f[I]your own account\f[], fetch bookmark tags as \f[I]tags_bookmark\f[] metadata. -Note: This requires 1 additional API request per bookmarked post. +.IP "Note:" 4 +This requires 1 additional API request per bookmarked post. .SS extractor.pixiv.captions @@ -5177,7 +5391,8 @@ try to grab the actual \f[I]caption\f[] value using the AJAX API. .IP "Description:" 4 Fetch \f[I]comments\f[] metadata. -Note: This requires 1 or more additional API requests per post, +.IP "Note:" 4 +This requires 1 or more additional API requests per post, depending on the number of comments. @@ -5266,7 +5481,8 @@ Try to fetch \f[I]limit_sanity_level\f[] works via web API. .IP "Description:" 4 Fetch \f[I]comments\f[] metadata. -Note: This requires 1 or more additional API requests per novel, +.IP "Note:" 4 +This requires 1 or more additional API requests per novel, depending on the number of comments. @@ -5341,7 +5557,8 @@ For novels bookmarked by \f[I]your own account\f[], fetch bookmark tags as \f[I]tags_bookmark\f[] metadata. -Note: This requires 1 additional API request per bookmarked post. +.IP "Note:" 4 +This requires 1 additional API request per bookmarked post. .SS extractor.pixiv-novel.refresh-token @@ -5421,10 +5638,10 @@ to download as mp4 videos. .IP "Description:" 4 Controls how to handle redirects to CAPTCHA pages. -.br -* \f[I]"stop\f[]: Stop the current extractor run. -.br -* \f[I]"wait\f[]: Ask the user to solve the CAPTCHA and wait. +\f[I]"stop\f[] +Stop the current extractor run. +\f[I]"wait\f[] +Ask the user to solve the CAPTCHA and wait. .SS extractor.readcomiconline.quality @@ -5499,7 +5716,8 @@ time required when scanning a subreddit. Retrieve additional comments by resolving the \f[I]more\f[] comment stubs in the base comment tree. -Note: This requires 1 additional API call for every 100 extra comments. +.IP "Note:" 4 +This requires 1 additional API call for every 100 extra comments. .SS extractor.reddit.embeds @@ -5576,12 +5794,11 @@ Reddit extractors can recursively visit other submissions linked to in the initial set of submissions. This value sets the maximum recursion depth. -Special values: - -.br -* \f[I]0\f[]: Recursion is disabled -.br -* \f[I]-1\f[]: Infinite recursion (don't do this) +.IP "Special Values:" 4 +\f[I]0\f[] +Recursion is disabled +\f[I]-1\f[] +Infinite recursion (don't do this) .SS extractor.reddit.refresh-token @@ -5629,18 +5846,19 @@ Follow links in the original post's \f[I]selftext\f[]. .IP "Description:" 4 Control video download behavior. -.br -* \f[I]true\f[]: Download videos and use \f[I]ytdl\f[] to handle +\f[I]true\f[] +Download videos and use \f[I]ytdl\f[] to handle HLS and DASH manifests -.br -* \f[I]"ytdl"\f[]: Download videos and let \f[I]ytdl\f[] handle all of +\f[I]"ytdl"\f[] +Download videos and let \f[I]ytdl\f[] handle all of video extraction and download -.br -* \f[I]"dash"\f[]: Extract DASH manifest URLs and use \f[I]ytdl\f[] +\f[I]"dash"\f[] +Extract DASH manifest URLs and use \f[I]ytdl\f[] to download and merge them. (*) -.br -* \f[I]false\f[]: Ignore videos +\f[I]false\f[] +Ignore videos +.IP "Note:" 4 (*) This saves 1 HTTP request per video and might potentially be able to download otherwise deleted videos, @@ -5658,13 +5876,7 @@ but it will not always get the best video quality available. \f[I]["hd", "sd", "gif"]\f[] .IP "Description:" 4 -List of names of the preferred animation format, which can be -\f[I]"hd"\f[], -\f[I]"sd"\f[], -\f[I]"gif"\f[], -\f[I]"thumbnail"\f[], -\f[I]"vthumbnail"\f[], or -\f[I]"poster"\f[]. +List of names of the preferred animation format.` If a selected format is not available, the next one in the list will be tried until an available format is found. @@ -5673,6 +5885,20 @@ If the format is given as \f[I]string\f[], it will be extended with \f[I]["hd", "sd", "gif"]\f[]. Use a list with one element to restrict it to only one possible format. +.IP "Available Formats:" 4 +.br +* \f[I]"hd"\f[] +.br +* \f[I]"sd"\f[] +.br +* \f[I]"gif"\f[] +.br +* \f[I]"thumbnail"\f[] +.br +* \f[I]"vthumbnail"\f[] +.br +* \f[I]"poster"\f[] + .SS extractor.rule34.api-key & .user-id .IP "Type:" 6 @@ -5735,7 +5961,7 @@ Refresh download URLs before they expire. .IP "Description:" 4 Group \f[I]tags\f[] by type and .br -provide them as \f[I]tags_TYPE\f[] and \f[I]tag_string_TYPE\f[] metadata fields, +provide them as \f[I]tags_<type>\f[] and \f[I]tag_string_TYPE\f[] metadata fields, for example \f[I]tags_artist\f[] and \f[I]tags_character\f[]. .br @@ -5756,7 +5982,7 @@ Requires: .br * 1 additional HTTP request per post .br -* logged-in \f[I]cookies\f[] +* authenticated \f[I]cookies\f[] to fetch full \f[I]tags\f[] category data \f[I]false\f[] @@ -5804,13 +6030,14 @@ To get this value: .br * Open your browser's Developer Tools (F12) .br -* Select Network -> XHR +* Select Network → XHR .br * Open a gallery page .br * Select the last Network entry and copy its \f[I]crt\f[] value -Note: You will also need your browser's +.IP "Note:" 4 +You will also need your browser's \f[I]user-agent\f[] @@ -5829,10 +6056,17 @@ Name(s) of the image format to download. When more than one format is given, the first available one is selected. -Possible formats are +.IP "Formats:" 4 .br -\f[I]"780"\f[], \f[I]"980"\f[], \f[I]"1280"\f[], \f[I]"1600"\f[], \f[I]"0"\f[] (original) +* \f[I]"780"\f[] +.br +* \f[I]"980"\f[] .br +* \f[I]"1280"\f[] +.br +* \f[I]"1600"\f[] +.br +* \f[I]"0"\f[] (original) .SS extractor.schalenetwork.tags @@ -5962,13 +6196,13 @@ if \f[I]sent-requests\f[] are enabled A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are - +.IP "Supported Values:" 4 .br -* \f[I]"works"\f[] +* \f[I]works\f[] .br -* \f[I]"sentrequests"\f[] +* \f[I]sentrequests\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -6030,7 +6264,8 @@ Download video files. \f[I]true\f[] .IP "Description:" 4 -Include animated assets when downloading from a list of assets. +Include \f[I]animated\f[] assets +when downloading from a list of assets. .SS extractor.steamgriddb.epilepsy @@ -6041,7 +6276,8 @@ Include animated assets when downloading from a list of assets. \f[I]true\f[] .IP "Description:" 4 -Include assets tagged with epilepsy when downloading from a list of assets. +Include assets tagged with \f[I]epilepsy\f[] +when downloading from a list of assets. .SS extractor.steamgriddb.dimensions @@ -6054,31 +6290,39 @@ Include assets tagged with epilepsy when downloading from a list of assets. .IP "Default:" 9 \f[I]"all"\f[] -.IP "Examples:" 4 +.IP "Example:" 4 .br -* \f[I]"1024x512,512x512"\f[] +* "1024x512,512x512" .br -* \f[I]["460x215", "920x430"]\f[] +* ["460x215", "920x430"] .IP "Description:" 4 Only include assets that are in the specified dimensions. \f[I]all\f[] can be -used to specify all dimensions. Valid values are: - -.br -* Grids: \f[I]460x215\f[], \f[I]920x430\f[], \f[I]600x900\f[], \f[I]342x482\f[], \f[I]660x930\f[], -\f[I]512x512\f[], \f[I]1024x1024\f[] -.br -* Heroes: \f[I]1920x620\f[], \f[I]3840x1240\f[], \f[I]1600x650\f[] -.br -* Logos: N/A (will be ignored) -.br -* Icons: \f[I]8x8\f[], \f[I]10x10\f[], \f[I]14x14\f[], \f[I]16x16\f[], \f[I]20x20\f[], \f[I]24x24\f[], -\f[I]28x28\f[], \f[I]32x32\f[], \f[I]35x35\f[], \f[I]40x40\f[], \f[I]48x48\f[], \f[I]54x54\f[], -\f[I]56x56\f[], \f[I]57x57\f[], \f[I]60x60\f[], \f[I]64x64\f[], \f[I]72x72\f[], \f[I]76x76\f[], -\f[I]80x80\f[], \f[I]90x90\f[], \f[I]96x96\f[], \f[I]100x100\f[], \f[I]114x114\f[], \f[I]120x120\f[], -\f[I]128x128\f[], \f[I]144x144\f[], \f[I]150x150\f[], \f[I]152x152\f[], \f[I]160x160\f[], -\f[I]180x180\f[], \f[I]192x192\f[], \f[I]194x194\f[], \f[I]256x256\f[], \f[I]310x310\f[], -\f[I]512x512\f[], \f[I]768x768\f[], \f[I]1024x1024\f[] +used to specify all dimensions. + +.IP "Valid Values:" 4 +Grids +\f[I]460x215\f[] \f[I] +\f[I]920x430\f[] \f[] +\f[I]342x482\f[] \f[I] +\f[I]600x900\f[] \f[] +\f[I]660x930\f[] \f[I] +\f[I]512x512\f[] \f[] +\f[I]1024x1024\f[] +Heroes +\f[I]1600x650\f[] \f[I] +\f[I]1920x620\f[] \f[] +\f[I]3840x1240\f[] +Logos +N/A (will be ignored) +Icons +\f[I]8x8\f[] \f[I] \f[I]10x10\f[] \f[] \f[I]14x14\f[] \f[I] \f[I]16x16\f[] \f[] \f[I]20x20\f[] \f[I] \f[I]24x24\f[] \f[] +\f[I]28x28\f[] \f[I] \f[I]32x32\f[] \f[] \f[I]35x35\f[] \f[I] \f[I]40x40\f[] \f[] \f[I]48x48\f[] \f[I] \f[I]54x54\f[] \f[] +\f[I]56x56\f[] \f[I] \f[I]57x57\f[] \f[] \f[I]60x60\f[] \f[I] \f[I]64x64\f[] \f[] \f[I]72x72\f[] \f[I] \f[I]76x76\f[] \f[] +\f[I]80x80\f[] \f[I] \f[I]90x90\f[] \f[] \f[I]96x96\f[] \f[I] \f[I]100x100\f[] \f[] \f[I]114x114\f[] \f[I] \f[I]120x120\f[] \f[] +\f[I]128x128\f[] \f[I] \f[I]144x144\f[] \f[] \f[I]150x150\f[] \f[I] \f[I]152x152\f[] \f[] \f[I]160x160\f[] \f[I] +\f[I]180x180\f[] \f[] \f[I]192x192\f[] \f[I] \f[I]194x194\f[] \f[] \f[I]256x256\f[] \f[I] \f[I]310x310\f[] \f[] +\f[I]512x512\f[] \f[I] \f[I]768x768\f[] \f[] \f[I]1024x1024\f[] .SS extractor.steamgriddb.file-types @@ -6091,24 +6335,25 @@ used to specify all dimensions. Valid values are: .IP "Default:" 9 \f[I]"all"\f[] -.IP "Examples:" 4 +.IP "Example:" 4 .br -* \f[I]"png,jpeg"\f[] +* "png,jpeg" .br -* \f[I]["jpeg", "webp"]\f[] +* ["jpeg", "webp"] .IP "Description:" 4 Only include assets that are in the specified file types. \f[I]all\f[] can be -used to specify all file types. Valid values are: +used to specify all file types. -.br -* Grids: \f[I]png\f[], \f[I]jpeg\f[], \f[I]jpg\f[], \f[I]webp\f[] -.br -* Heroes: \f[I]png\f[], \f[I]jpeg\f[], \f[I]jpg\f[], \f[I]webp\f[] -.br -* Logos: \f[I]png\f[], \f[I]webp\f[] -.br -* Icons: \f[I]png\f[], \f[I]ico\f[] +.IP "Valid Values:" 4 +Grids +\f[I]png\f[] \f[I] \f[I]jpeg\f[] \f[] \f[I]jpg\f[] \f[I] \f[I]webp\f[] +Heroes +\f[I]png\f[] \f[] \f[I]jpeg\f[] \f[I] \f[I]jpg\f[] \f[] \f[I]webp\f[] +Logos +\f[I]png\f[] \f[I] \f[I]webp\f[] +Icons +\f[I]png\f[] \f[] \f[I]ico\f[] .SS extractor.steamgriddb.download-fake-png @@ -6130,7 +6375,8 @@ Download fake PNGs alongside the real file. \f[I]true\f[] .IP "Description:" 4 -Include assets tagged with humor when downloading from a list of assets. +Include assets tagged with \f[I]humor\f[] +when downloading from a list of assets. .SS extractor.steamgriddb.languages @@ -6143,16 +6389,20 @@ Include assets tagged with humor when downloading from a list of assets. .IP "Default:" 9 \f[I]"all"\f[] -.IP "Examples:" 4 +.IP "Example:" 4 .br -* \f[I]"en,km"\f[] +* "en,km" .br -* \f[I]["fr", "it"]\f[] +* ["fr", "it"] .IP "Description:" 4 -Only include assets that are in the specified languages. \f[I]all\f[] can be -used to specify all languages. Valid values are \f[I]ISO 639-1\f[] -language codes. +Only include assets that are in the specified languages. + +.IP "Valid Values:" 4 +\f[I]ISO 639-1\f[] codes + +.IP "Note:" 4 +\f[I]all\f[] can be used to specify all languages. .SS extractor.steamgriddb.nsfw @@ -6171,23 +6421,24 @@ Include assets tagged with adult content when downloading from a list of assets. \f[I]string\f[] .IP "Default:" 9 -\f[I]score_desc\f[] +\f[I]"score_desc"\f[] .IP "Description:" 4 -Set the chosen sorting method when downloading from a list of assets. Can be one of: +Set the chosen sorting method when downloading from a list of assets. +.IP "Supported Values:" 4 .br -* \f[I]score_desc\f[] (Highest Score (Beta)) +* \f[I]score_desc\f[] (Highest Score (Beta)) .br -* \f[I]score_asc\f[] (Lowest Score (Beta)) +* \f[I]score_asc\f[] (Lowest Score (Beta)) .br * \f[I]score_old_desc\f[] (Highest Score (Old)) .br -* \f[I]score_old_asc\f[] (Lowest Score (Old)) +* \f[I]score_old_asc\f[] (Lowest Score (Old)) .br -* \f[I]age_desc\f[] (Newest First) +* \f[I]age_desc\f[] (Newest First) .br -* \f[I]age_asc\f[] (Oldest First) +* \f[I]age_asc\f[] (Oldest First) .SS extractor.steamgriddb.static @@ -6201,36 +6452,6 @@ Set the chosen sorting method when downloading from a list of assets. Can be one Include static assets when downloading from a list of assets. -.SS extractor.steamgriddb.styles -.IP "Type:" 6 -.br -* \f[I]string\f[] -.br -* \f[I]list\f[] of \f[I]strings\f[] - -.IP "Default:" 9 -\f[I]all\f[] - -.IP "Examples:" 4 -.br -* \f[I]white,black\f[] -.br -* \f[I]["no_logo", "white_logo"]\f[] - -.IP "Description:" 4 -Only include assets that are in the specified styles. \f[I]all\f[] can be used -to specify all styles. Valid values are: - -.br -* Grids: \f[I]alternate\f[], \f[I]blurred\f[], \f[I]no_logo\f[], \f[I]material\f[], \f[I]white_logo\f[] -.br -* Heroes: \f[I]alternate\f[], \f[I]blurred\f[], \f[I]material\f[] -.br -* Logos: \f[I]official\f[], \f[I]white\f[], \f[I]black\f[], \f[I]custom\f[] -.br -* Icons: \f[I]official\f[], \f[I]custom\f[] - - .SS extractor.steamgriddb.untagged .IP "Type:" 6 \f[I]bool\f[] @@ -6269,8 +6490,7 @@ List of names of the preferred animation format. If a selected format is not available, the next one in the list will be tried until a format is found. -Possible formats include - +.IP "Available Formats:" 4 .br * \f[I]gif\f[] .br @@ -6312,12 +6532,12 @@ Possible formats include .IP "Description:" 4 Controls audio download behavior. -.br -* \f[I]true\f[]: Download audio tracks -.br -* \f[I]"ytdl"\f[]: Download audio tracks using \f[I]ytdl\f[] -.br -* \f[I]false\f[]: Ignore audio tracks +\f[I]true\f[] +Download audio tracks +\f[I]"ytdl"\f[] +Download audio tracks using \f[I]ytdl\f[] +\f[I]false\f[] +Ignore audio tracks .SS extractor.tiktok.videos @@ -6461,15 +6681,15 @@ use an extra HTTP request to find the URL to its full-resolution version. .IP "Description:" 4 Controls how to paginate over blog posts. -.br -* \f[I]"api"\f[]: \f[I]next\f[] parameter provided by the API +\f[I]"api"\f[] +\f[I]next\f[] parameter provided by the API (potentially misses posts due to a \f[I]bug\f[] in Tumblr's API) -.br -* \f[I]"before"\f[]: timestamp of last post -.br -* \f[I]"offset"\f[]: post offset number +\f[I]"before"\f[] +Timestamp of last post +\f[I]"offset"\f[] +Post offset number .SS extractor.tumblr.ratelimit @@ -6482,10 +6702,10 @@ in Tumblr's API) .IP "Description:" 4 Selects how to handle exceeding the daily API rate limit. -.br -* \f[I]"abort"\f[]: Raise an error and stop extraction -.br -* \f[I]"wait"\f[]: Wait until rate limit reset +\f[I]"abort"\f[] +Raise an error and stop extraction +\f[I]"wait"\f[] +Wait until rate limit reset .SS extractor.tumblr.reblogs @@ -6499,12 +6719,12 @@ Selects how to handle exceeding the daily API rate limit. \f[I]true\f[] .IP "Description:" 4 -.br -* \f[I]true\f[]: Extract media from reblogged posts -.br -* \f[I]false\f[]: Skip reblogged posts -.br -* \f[I]"same-blog"\f[]: Skip reblogged posts unless the original post +\f[I]true\f[] +Extract media from reblogged posts +\f[I]false\f[] +Skip reblogged posts +\f[I]"same-blog"\f[] +Skip reblogged posts unless the original post is from the same blog @@ -6623,13 +6843,12 @@ Fetch media from promoted Tweets. .IP "Description:" 4 Controls how to handle \f[I]Twitter Cards\f[]. -.br -* \f[I]false\f[]: Ignore cards -.br -* \f[I]true\f[]: Download image content from supported cards -.br -* \f[I]"ytdl"\f[]: Additionally download video content from unsupported cards -using \f[I]ytdl\f[] +\f[I]false\f[] +Ignore cards +\f[I]true\f[] +Download image content from supported cards +\f[I]"ytdl"\f[] +Additionally download video content from unsupported cards using \f[I]ytdl\f[] .SS extractor.twitter.cards-blacklist @@ -6683,10 +6902,10 @@ if the given initial Tweet is accessible. .IP "Description:" 4 Controls how to handle Cross Site Request Forgery (CSRF) tokens. -.br -* \f[I]"auto"\f[]: Always auto-generate a token. -.br -* \f[I]"cookies"\f[]: Use token given by the \f[I]ct0\f[] cookie if present. +\f[I]"auto"\f[] +Always auto-generate a token. +\f[I]"cookies"\f[] +Use token given by the \f[I]ct0\f[] cookie if present. .SS extractor.twitter.cursor @@ -6735,7 +6954,8 @@ as running \f[I]gallery-dl https://twitter.com/i/web/status/<TweetID>\f[] with enabled \f[I]conversations\f[] option for each Tweet in said timeline. -Note: This requires at least 1 additional API call per initial Tweet. +.IP "Note:" 4 +This requires at least 1 additional API call per initial Tweet. .SS extractor.twitter.unavailable @@ -6770,27 +6990,27 @@ e.g. \f[I]Geoblocked\f[] videos. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are - +.IP "Supported Values:" 4 .br -* \f[I]"info"\f[] +* \f[I]info\f[] .br -* \f[I]"avatar"\f[] +* \f[I]avatar\f[] .br -* \f[I]"background"\f[] +* \f[I]background\f[] .br -* \f[I]"timeline"\f[] +* \f[I]timeline\f[] .br -* \f[I]"tweets"\f[] +* \f[I]tweets\f[] .br -* \f[I]"media"\f[] +* \f[I]media\f[] .br -* \f[I]"replies"\f[] +* \f[I]replies\f[] .br -* \f[I]"highlights"\f[] +* \f[I]highlights\f[] .br -* \f[I]"likes"\f[] +* \f[I]likes\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -6815,12 +7035,12 @@ Transform Tweet and User metadata into a simpler, uniform format. .IP "Description:" 4 Selects the API endpoint used to retrieve single Tweets. -.br -* \f[I]"restid"\f[]: \f[I]/TweetResultByRestId\f[] - accessible to guest users -.br -* \f[I]"detail"\f[]: \f[I]/TweetDetail\f[] - more stable -.br -* \f[I]"auto"\f[]: \f[I]"detail"\f[] when logged in, \f[I]"restid"\f[] otherwise +\f[I]"restid"\f[] +\f[I]/TweetResultByRestId\f[] - accessible to guest users +\f[I]"detail"\f[] +\f[I]/TweetDetail\f[] - more stable +\f[I]"auto"\f[] +\f[I]"detail"\f[] when logged in, \f[I]"restid"\f[] otherwise .SS extractor.twitter.size @@ -6899,12 +7119,12 @@ a quoted (original) Tweet when it sees the Tweet which quotes it. .IP "Description:" 4 Selects how to handle exceeding the API rate limit. -.br -* \f[I]"abort"\f[]: Raise an error and stop extraction -.br -* \f[I]"wait"\f[]: Wait until rate limit reset -.br -* \f[I]"wait:N"\f[]: Wait for \f[I]N\f[] seconds +\f[I]"abort"\f[] +Raise an error and stop extraction +\f[I]"wait"\f[] +Wait until rate limit reset +\f[I]"wait:N"\f[] +Wait for \f[I]N\f[] seconds .SS extractor.twitter.relogin @@ -6931,10 +7151,10 @@ try to continue from where it left off. .IP "Description:" 4 Selects how to handle "account is temporarily locked" errors. -.br -* \f[I]"abort"\f[]: Raise an error and stop extraction -.br -* \f[I]"wait"\f[]: Wait until the account is unlocked and retry +\f[I]"abort"\f[] +Raise an error and stop extraction +\f[I]"wait"\f[] +Wait until the account is unlocked and retry .SS extractor.twitter.replies @@ -6950,7 +7170,8 @@ Fetch media from replies to other Tweets. If this value is \f[I]"self"\f[], only consider replies where reply and original Tweet are from the same user. -Note: Twitter will automatically expand conversations if you +.IP "Note:" 4 +Twitter will automatically expand conversations if you use the \f[I]/with_replies\f[] timeline while logged in. For example, media from Tweets which the user replied to will also be downloaded. @@ -7026,14 +7247,14 @@ to accept before stopping. Controls the strategy / tweet source used for timeline URLs (\f[I]https://twitter.com/USER/timeline\f[]). -.br -* \f[I]"tweets"\f[]: \f[I]/tweets\f[] timeline + search -.br -* \f[I]"media"\f[]: \f[I]/media\f[] timeline + search -.br -* \f[I]"with_replies"\f[]: \f[I]/with_replies\f[] timeline + search -.br -* \f[I]"auto"\f[]: \f[I]"tweets"\f[] or \f[I]"media"\f[], depending on \f[I]retweets\f[] and \f[I]text-tweets\f[] settings +\f[I]"tweets"\f[] +\f[I]/tweets\f[] timeline + search +\f[I]"media"\f[] +\f[I]/media\f[] timeline + search +\f[I]"with_replies"\f[] +\f[I]/with_replies\f[] timeline + search +\f[I]"auto"\f[] +\f[I]"tweets"\f[] or \f[I]"media"\f[], depending on \f[I]retweets\f[] and \f[I]text-tweets\f[] settings .SS extractor.twitter.text-tweets @@ -7082,7 +7303,7 @@ Alternate Identifier (username, email, phone number) when \f[I]logging in\f[]. When not specified and asked for by Twitter, -this identifier will need to entered in an interactive prompt. +this identifier will need to be entered in an interactive prompt. .SS extractor.twitter.users @@ -7096,25 +7317,25 @@ this identifier will need to entered in an interactive prompt. "https://twitter.com/search?q=from:{legacy[screen_name]}" .IP "Description:" 4 -Format string for user URLs generated from +Basic format string for user URLs generated from .br \f[I]following\f[] and \f[I]list-members\f[] queries, whose replacement field values come from Twitter \f[I]user\f[] objects .br (\f[I]Example\f[]) -Special values: - -.br -* \f[I]"user"\f[]: \f[I]https://twitter.com/i/user/{rest_id}\f[] -.br -* \f[I]"timeline"\f[]: \f[I]https://twitter.com/id:{rest_id}/timeline\f[] -.br -* \f[I]"tweets"\f[]: \f[I]https://twitter.com/id:{rest_id}/tweets\f[] -.br -* \f[I]"media"\f[]: \f[I]https://twitter.com/id:{rest_id}/media\f[] +.IP "Special Values:" 4 +\f[I]"user"\f[] +\f[I]https://twitter.com/i/user/{rest_id}\f[] +\f[I]"timeline"\f[] +\f[I]https://twitter.com/id:{rest_id}/timeline\f[] +\f[I]"tweets"\f[] +\f[I]https://twitter.com/id:{rest_id}/tweets\f[] +\f[I]"media"\f[] +\f[I]https://twitter.com/id:{rest_id}/media\f[] -Note: To allow gallery-dl to follow custom URL formats, set the \f[I]blacklist\f[] +.IP "Note:" 4 +To allow gallery-dl to follow custom URL formats, set the \f[I]blacklist\f[] for \f[I]twitter\f[] to a non-default value, e.g. an empty string \f[I]""\f[]. @@ -7131,12 +7352,12 @@ for \f[I]twitter\f[] to a non-default value, e.g. an empty string \f[I]""\f[]. .IP "Description:" 4 Control video download behavior. -.br -* \f[I]true\f[]: Download videos -.br -* \f[I]"ytdl"\f[]: Download videos using \f[I]ytdl\f[] -.br -* \f[I]false\f[]: Skip video Tweets +\f[I]true\f[] +Download videos +\f[I]"ytdl"\f[] +Download videos using \f[I]ytdl\f[] +\f[I]false\f[] +Skip video Tweets .SS extractor.unsplash.format @@ -7149,8 +7370,17 @@ Control video download behavior. .IP "Description:" 4 Name of the image format to download. -Available formats are -\f[I]"raw"\f[], \f[I]"full"\f[], \f[I]"regular"\f[], \f[I]"small"\f[], and \f[I]"thumb"\f[]. +.IP "Available Formats:" 4 +.br +* \f[I]"raw"\f[] +.br +* \f[I]"full"\f[] +.br +* \f[I]"regular"\f[] +.br +* \f[I]"small"\f[] +.br +* \f[I]"thumb"\f[] .SS extractor.vipergirls.domain @@ -7176,7 +7406,8 @@ For example \f[I]"viper.click"\f[] if the main domain is blocked or to bypass Cl .IP "Description:" 4 Automatically like posts after downloading their images. -Note: Requires \f[I]login\f[] +.IP "Note:" 4 +Requires \f[I]login\f[] or \f[I]cookies\f[] @@ -7228,12 +7459,17 @@ Custom \f[I]offset\f[] starting value when paginating over image results. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"avatar"\f[], -\f[I]"gallery"\f[], -\f[I]"spaces"\f[], -\f[I]"collection"\f[], +.IP "Supported Values:" 4 +.br +* \f[I]avatar\f[] +.br +* \f[I]gallery\f[] +.br +* \f[I]spaces\f[] +.br +* \f[I]collection\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -7282,9 +7518,13 @@ See https://wallhaven.cc/help/api for more information. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"uploads"\f[], \f[I]"collections"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]uploads\f[] +.br +* \f[I]collections\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -7298,7 +7538,8 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately. .IP "Description:" 4 Extract additional metadata (tags, uploader) -Note: This requires 1 additional HTTP request per post. +.IP "Note:" 4 +This requires 1 additional HTTP request per post. .SS extractor.weasyl.api-key @@ -7327,7 +7568,8 @@ Fetch extra submission metadata during gallery downloads. .br \f[I]tags\f[], \f[I]views\f[]) -Note: This requires 1 additional HTTP request per submission. +.IP "Note:" 4 +This requires 1 additional HTTP request per submission. .SS extractor.webtoons.quality @@ -7337,7 +7579,7 @@ Note: This requires 1 additional HTTP request per submission. .br * \f[I]string\f[] .br -* \f[I]object\f[] (ext -> type) +* \f[I]object\f[] (ext → type) .IP "Default:" 9 @@ -7423,14 +7665,21 @@ Set this to \f[I]"video"\f[] to download GIFs as video files. A (comma-separated) list of subcategories to include when processing a user profile. -Possible values are -\f[I]"home"\f[], -\f[I]"feed"\f[], -\f[I]"videos"\f[], -\f[I]"newvideo"\f[], -\f[I]"article"\f[], -\f[I]"album"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]home\f[] +.br +* \f[I]feed\f[] +.br +* \f[I]videos\f[] +.br +* \f[I]newvideo\f[] +.br +* \f[I]article\f[] +.br +* \f[I]album\f[] +.IP "Note:" 4 It is possible to use \f[I]"all"\f[] instead of listing all values separately. @@ -7481,6 +7730,25 @@ will be taken from the original posts, not the retweeted posts. Download video files. +.SS extractor.wikimedia.image-revisions +.IP "Type:" 6 +\f[I]integer\f[] + +.IP "Default:" 9 +\f[I]1\f[] + +.IP "Description:" 4 +Number of revisions to return for a single image. + +The dafault value of 1 only returns the latest revision. + +The value must be between 1 and 500. + +.IP "Note:" 4 +The API sometimes returns image revisions on article pages even when this option is +set to 1. However, setting it to a higher value may reduce the number of API requests. + + .SS extractor.wikimedia.limit .IP "Type:" 6 \f[I]integer\f[] @@ -7621,7 +7889,8 @@ set subcategory to the input URL's domain. Route \f[I]ytdl's\f[] output through gallery-dl's logging system. Otherwise it will be written directly to stdout/stderr. -Note: Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in +.IP "Note:" 4 +Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in \f[I]extractor.ytdl.raw-options\f[] to \f[I]true\f[] to suppress all output. @@ -7647,7 +7916,7 @@ followed by \f[I]"youtube_dl"\f[] as fallback. .SS extractor.ytdl.raw-options .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 .. code:: json @@ -7699,7 +7968,8 @@ List of filename extensions to try when dynamically building download URLs .IP "Description:" 4 Extract additional metadata (date, md5, tags, ...) -Note: This requires 1-2 additional HTTP requests per post. +.IP "Note:" 4 +This requires 1-2 additional HTTP requests per post. .SS extractor.zerochan.pagination @@ -7712,11 +7982,11 @@ Note: This requires 1-2 additional HTTP requests per post. .IP "Description:" 4 Controls how to paginate over tag search results. -.br -* \f[I]"api"\f[]: Use the \f[I]JSON API\f[] +\f[I]"api"\f[] +Use the \f[I]JSON API\f[] (no \f[I]extension\f[] metadata) -.br -* \f[I]"html"\f[]: Parse HTML pages +\f[I]"html"\f[] +Parse HTML pages (limited to 100 pages * 24 posts) @@ -7743,7 +8013,8 @@ Group \f[I]tags\f[] by type and provide them as \f[I]tags_<type>\f[] metadata fields, for example \f[I]tags_artist\f[] or \f[I]tags_character\f[]. -Note: This requires 1 additional HTTP request per post. +.IP "Note:" 4 +This requires 1 additional HTTP request per post. .SS extractor.[booru].notes @@ -7756,7 +8027,8 @@ Note: This requires 1 additional HTTP request per post. .IP "Description:" 4 Extract overlay notes (position and text). -Note: This requires 1 additional HTTP request per post. +.IP "Note:" 4 +This requires 1 additional HTTP request per post. .SS extractor.[booru].url @@ -7791,10 +8063,10 @@ When multiple names are given, download the first available one. .IP "Description:" 4 Reverse the order of chapter URLs extracted from manga pages. -.br -* \f[I]true\f[]: Start with the latest chapter -.br -* \f[I]false\f[]: Start with the first chapter +\f[I]true\f[] +Start with the latest chapter +\f[I]false\f[] +Start with the first chapter .SS extractor.[manga-extractor].page-reverse @@ -7861,12 +8133,12 @@ to set file modification times. .IP "Description:" 4 Controls the use of \f[I].part\f[] files during file downloads. -.br -* \f[I]true\f[]: Write downloaded data into \f[I].part\f[] files and rename +\f[I]true\f[] +Write downloaded data into \f[I].part\f[] files and rename them upon download completion. This mode additionally supports resuming incomplete downloads. -.br -* \f[I]false\f[]: Do not use \f[I].part\f[] files and write data directly +\f[I]false\f[] +Do not use \f[I].part\f[] files and write data directly into the actual output files. @@ -7973,7 +8245,7 @@ Certificate validation during file downloads. .br * \f[I]string\f[] .br -* \f[I]object\f[] (scheme -> proxy) +* \f[I]object\f[] (scheme → proxy) .IP "Default:" 9 \f[I]extractor.*.proxy\f[] @@ -8044,7 +8316,7 @@ These suffixes are case-insensitive. .SS downloader.http.headers .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 {"Accept": "image/webp,*/*", "Referer": "https://example.org/"} @@ -8083,7 +8355,8 @@ regardless of this option. Number of seconds to sleep when receiving a 429 Too Many Requests response before \f[I]retrying\f[] the request. -Note: Requires +.IP "Note:" 4 +Requires \f[I]retry-codes\f[] to include \f[I]429\f[]. @@ -8203,7 +8476,8 @@ Forward gallery-dl's cookies to \f[I]ytdl\f[]. Route \f[I]ytdl's\f[] output through gallery-dl's logging system. Otherwise it will be written directly to stdout/stderr. -Note: Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in +.IP "Note:" 4 +Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in \f[I]downloader.ytdl.raw-options\f[] to \f[I]true\f[] to suppress all output. @@ -8243,23 +8517,23 @@ See / \f[I]youtube-dl output template\f[]. -Special values: - -.br -* \f[I]null\f[]: generate filenames with \f[I]extractor.*.filename\f[] -.br -* \f[I]"default"\f[]: use \f[I]ytdl's\f[] default, currently +.IP "Special Values:" 4 +\f[I]null\f[] +generate filenames with \f[I]extractor.*.filename\f[] +\f[I]"default"\f[] +use \f[I]ytdl's\f[] default, currently \f[I]"%(title)s [%(id)s].%(ext)s"\f[] for \f[I]yt-dlp\f[] / \f[I]"%(title)s-%(id)s.%(ext)s"\f[] for \f[I]youtube-dl\f[] -Note: An output template other than \f[I]null\f[] might +.IP "Note:" 4 +An output template other than \f[I]null\f[] might cause unexpected results in combination with certain options (e.g. \f[I]"skip": "enumerate"\f[]) .SS downloader.ytdl.raw-options .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 .. code:: json @@ -8286,7 +8560,7 @@ Available options can be found in .br * \f[I]string\f[] .br -* \f[I]object\f[] (key -> format string) +* \f[I]object\f[] (key → format string) .IP "Default:" 9 \f[I]"auto"\f[] @@ -8294,17 +8568,18 @@ Available options can be found in .IP "Description:" 4 Controls the output string format and status indicators. -.br -* \f[I]"null"\f[]: No output -.br -* \f[I]"pipe"\f[]: Suitable for piping to other processes or files -.br -* \f[I]"terminal"\f[]: Suitable for the standard Windows console -.br -* \f[I]"color"\f[]: Suitable for terminals that understand ANSI escape codes and colors -.br -* \f[I]"auto"\f[]: \f[I]"pipe"\f[] if not on a TTY, \f[I]"terminal"\f[] on Windows with -\f[I]output.ansi\f[] disabled, \f[I]"color"\f[] otherwise. +\f[I]"null"\f[] +No output +\f[I]"pipe"\f[] +Suitable for piping to other processes or files +\f[I]"terminal"\f[] +Suitable for the standard Windows console +\f[I]"color"\f[] +Suitable for terminals that understand ANSI escape codes and colors +\f[I]"auto"\f[] +\f[I]"pipe"\f[] if not on a TTY, +\f[I]"terminal"\f[] on Windows with \f[I]output.ansi\f[] disabled, +\f[I]"color"\f[] otherwise. It is possible to use custom output format strings .br @@ -8394,7 +8669,8 @@ Possible options are When this option is specified as a simple \f[I]string\f[], it is interpreted as \f[I]{"encoding": "<string-value>", "errors": "replace"}\f[] -Note: \f[I]errors\f[] always defaults to \f[I]"replace"\f[] +.IP "Note:" 4 +\f[I]errors\f[] always defaults to \f[I]"replace"\f[] .SS output.shorten @@ -8414,7 +8690,7 @@ with a display width greater than 1. .SS output.colors .IP "Type:" 6 -\f[I]object\f[] (key -> ANSI color) +\f[I]object\f[] (key → ANSI color) .IP "Default:" 9 .. code:: json @@ -8516,13 +8792,13 @@ in the output of \f[I]-K/--list-keywords\f[] and \f[I]-j/--dump-json\f[]. Controls the progress indicator when *gallery-dl* is run with multiple URLs as arguments. -.br -* \f[I]true\f[]: Show the default progress indicator +\f[I]true\f[] +Show the default progress indicator (\f[I]"[{current}/{total}] {url}"\f[]) -.br -* \f[I]false\f[]: Do not show any progress indicator -.br -* Any \f[I]string\f[]: Show the progress indicator using this +\f[I]false\f[] +Do not show any progress indicator +Any \f[I]string\f[] +Show the progress indicator using this as a custom \f[I]format string\f[]. Possible replacement keys are \f[I]current\f[], \f[I]total\f[] and \f[I]url\f[]. @@ -8603,7 +8879,7 @@ before outputting them as JSON. .SH POSTPROCESSOR OPTIONS .SS classify.mapping .IP "Type:" 6 -\f[I]object\f[] (directory -> extensions) +\f[I]object\f[] (directory → extensions) .IP "Default:" 9 .. code:: json @@ -8637,11 +8913,10 @@ in their default location. .IP "Description:" 4 The action to take when files do **not** compare as equal. -.br -* \f[I]"replace"\f[]: Replace/Overwrite the old version with the new one - -.br -* \f[I]"enumerate"\f[]: Add an enumeration index to the filename of the new +\f[I]"replace"\f[] +Replace/Overwrite the old version with the new one +\f[I]"enumerate"\f[] +Add an enumeration index to the filename of the new version like \f[I]skip = "enumerate"\f[] @@ -8655,17 +8930,15 @@ version like \f[I]skip = "enumerate"\f[] .IP "Description:" 4 The action to take when files do compare as equal. -.br -* \f[I]"abort:N"\f[]: Stop the current extractor run +\f[I]"abort:N"\f[] +Stop the current extractor run after \f[I]N\f[] consecutive files compared as equal. - -.br -* \f[I]"terminate:N"\f[]: Stop the current extractor run, +\f[I]"terminate:N"\f[] +Stop the current extractor run, including parent extractors, after \f[I]N\f[] consecutive files compared as equal. - -.br -* \f[I]"exit:N"\f[]: Exit the program +\f[I]"exit:N"\f[] +Exit the program after \f[I]N\f[] consecutive files compared as equal. @@ -8691,7 +8964,7 @@ Only compare file sizes. Do not read and compare their content. \f[I]"prepare"\f[] .IP "Description:" 4 -The event(s) for which \f[I]directory\f[] format strings are (re)evaluated. +The event(s) for which \f[I]directory\f[] \f[I]Format Strings\f[] are (re)evaluated. See \f[I]metadata.event\f[] for a list of available events. @@ -8756,7 +9029,8 @@ with the full path of a file or target directory, depending on .br * If this is a \f[I]list\f[], the first element specifies the program name and any further elements its arguments. -Each element of this list is treated as a \f[I]format string\f[] using + +Each element of this list is evaluated as a \f[I]Format String\f[] using the files' metadata as well as \f[I]{_path}\f[], \f[I]{_directory}\f[], and \f[I]{_filename}\f[]. @@ -8863,7 +9137,7 @@ Rebuild \f[I]filenames\f[] after computing .br * \f[I]string\f[] .br -* \f[I]object\f[] (field name -> hash algorithm) +* \f[I]object\f[] (field name → hash algorithm) .IP "Default:" 9 \f[I]"md5,sha1"\f[] @@ -8919,25 +9193,24 @@ for hash digests to compute. .IP "Description:" 4 Selects how to process metadata. -.br -* \f[I]"json"\f[]: write metadata using \f[I]json.dump()\f[] -.br -* \f[I]"jsonl"\f[]: write metadata in \f[I]JSON Lines -<https://jsonlines.org/>\f[] format -.br -* \f[I]"tags"\f[]: write \f[I]tags\f[] separated by newlines -.br -* \f[I]"custom"\f[]: write the result of applying \f[I]metadata.content-format\f[] +\f[I]"json"\f[] +Write metadata using \f[I]json.dump()\f[] +\f[I]"jsonl"\f[] +Write metadata in \f[I]JSON Lines\f[] format +\f[I]"tags"\f[] +Write \f[I]tags\f[] separated by newlines +\f[I]"custom"\f[] +Write the result of applying \f[I]metadata.content-format\f[] to a file's metadata dictionary -.br -* \f[I]"modify"\f[]: add or modify metadata entries -.br -* \f[I]"delete"\f[]: remove metadata entries +\f[I]"modify"\f[] +Add or modify metadata entries +\f[I]"delete"\f[] +Remove metadata entries .SS metadata.filename .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Default:" 9 \f[I]null\f[] @@ -8946,7 +9219,7 @@ to a file's metadata dictionary "{id}.data.json" .IP "Description:" 4 -A \f[I]format string\f[] to build the filenames for metadata files with. +A \f[I]Format String\f[] to generate filenames for metadata files. (see \f[I]extractor.filename\f[]) Using \f[I]"-"\f[] as filename will write all output to \f[I]stdout\f[]. @@ -8989,12 +9262,12 @@ relative to \f[I]metadata.base-directory\f[]. .IP "Description:" 4 Selects the relative location for metadata files. -.br -* \f[I]false\f[]: current target location for file downloads (\f[I]base-directory\f[] + directory_) -.br -* \f[I]true\f[]: current \f[I]base-directory\f[] location -.br -* any \f[I]Path\f[]: custom location +\f[I]false\f[] +Current target location for file downloads (\f[I]base-directory\f[] + directory_) +\f[I]true\f[] +Current \f[I]base-directory\f[] location +any \f[I]Path\f[] +Custom location .SS metadata.extension @@ -9011,7 +9284,7 @@ original file names. .SS metadata.extension-format .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Example:" 4 .br @@ -9020,10 +9293,11 @@ original file names. * "json" .IP "Description:" 4 -Custom format string to build filename extensions for metadata -files with, which will replace the original filename extensions. +Custom \f[I]Format String\f[] to generate filename extensions +for metadata files, which will replace the original filename extension. -Note: \f[I]metadata.extension\f[] is ignored if this option is set. +.IP "Note:" 4 +When this option is set, \f[I]metadata.extension\f[] is ignored. .SS metadata.metadata-path @@ -9099,7 +9373,8 @@ After downloading all files of a post .IP "Description:" 4 Include only the given top-level keys when writing JSON data. -Note: Missing or undefined fields will be silently ignored. +.IP "Note:" 4 +Missing or undefined fields will be silently ignored. .SS metadata.exclude @@ -9112,7 +9387,8 @@ Note: Missing or undefined fields will be silently ignored. .IP "Description:" 4 Exclude all given keys from written JSON data. -Note: Cannot be used with \f[I]metadata.include\f[]. +.IP "Note:" 4 +Cannot be used with \f[I]metadata.include\f[]. .SS metadata.fields @@ -9120,7 +9396,7 @@ Note: Cannot be used with \f[I]metadata.include\f[]. .br * \f[I]list\f[] of \f[I]strings\f[] .br -* \f[I]object\f[] (field name -> \f[I]format string\f[]) +* \f[I]object\f[] (field name → \f[I]Format String\f[]) .IP "Example:" 4 .. code:: json @@ -9137,21 +9413,19 @@ Note: Cannot be used with \f[I]metadata.include\f[]. .IP "Description:" 4 -.br -* \f[I]"mode": "delete"\f[]: +\f[I]"mode": "delete"\f[] A list of metadata field names to remove. -.br -* \f[I]"mode": "modify"\f[]: -An object with metadata field names mapping to a \f[I]format string\f[] -whose result is assigned to said field name. +\f[I]"mode": "modify"\f[] +An object with metadata field names mapping to a \f[I]Format String\f[] +whose result is assigned to that field name. .SS metadata.content-format .IP "Type:" 6 .br -* \f[I]string\f[] +* \f[I]Format String\f[] .br -* \f[I]list\f[] of \f[I]strings\f[] +* \f[I]list\f[] of \f[I]Format Strings\f[] .IP "Example:" 4 .br @@ -9160,9 +9434,10 @@ whose result is assigned to said field name. * ["tags:", "", "{tags:J\\n}"] .IP "Description:" 4 -Custom format string to build the content of metadata files with. +Custom \f[I]Format String(s)\f[] to build the content of metadata files with. -Note: Only applies for \f[I]"mode": "custom"\f[]. +.IP "Note:" 4 +Only applies to \f[I]"mode": "custom"\f[]. .SS metadata.ascii @@ -9177,7 +9452,8 @@ Escape all non-ASCII characters. See the \f[I]ensure_ascii\f[] argument of \f[I]json.dump()\f[] for further details. -Note: Only applies for \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[]. +.IP "Note:" 4 +Only applies to \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[]. .SS metadata.indent @@ -9195,7 +9471,8 @@ Indentation level of JSON output. See the \f[I]indent\f[] argument of \f[I]json.dump()\f[] for further details. -Note: Only applies for \f[I]"mode": "json"\f[]. +.IP "Note:" 4 +Only applies to \f[I]"mode": "json"\f[]. .SS metadata.separators @@ -9211,7 +9488,8 @@ to separate JSON keys and values with. See the \f[I]separators\f[] argument of \f[I]json.dump()\f[] for further details. -Note: Only applies for \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[]. +.IP "Note:" 4 +Only applies to \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[]. .SS metadata.sort @@ -9226,7 +9504,8 @@ Sort output by key. See the \f[I]sort_keys\f[] argument of \f[I]json.dump()\f[] for further details. -Note: Only applies for \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[]. +.IP "Note:" 4 +Only applies to \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[]. .SS metadata.open @@ -9358,12 +9637,13 @@ Name of the metadata field whose value should be used. This value must be either a UNIX timestamp or a \f[I]datetime\f[] object. -Note: This option gets ignored if \f[I]mtime.value\f[] is set. +.IP "Note:" 4 +This option is ignored if \f[I]mtime.value\f[] is set. .SS mtime.value .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Default:" 9 \f[I]null\f[] @@ -9375,7 +9655,7 @@ Note: This option gets ignored if \f[I]mtime.value\f[] is set. * "{content[0:6]:R22/2022/D%Y%m%d/}" .IP "Description:" 4 -A \f[I]format string\f[] whose value should be used. +The \f[I]Format String\f[] whose value should be used. The resulting value must be either a UNIX timestamp or a \f[I]datetime\f[] object. @@ -9422,7 +9702,7 @@ See \f[I]metadata.event\f[] for a list of available events. .SS python.expression .IP "Type:" 6 -\f[I]string\f[] +\f[I]Expression\f[] .IP "Example:" 4 .br @@ -9431,13 +9711,11 @@ See \f[I]metadata.event\f[] for a list of available events. * "terminate()" .IP "Description:" 4 -A -\f[I]Python expression\f[] -to +A Python \f[I]Expression\f[] to \f[I]evaluate\f[]. -Note: Only used with -\f[I]"mode": "eval"\f[] +.IP "Note:" 4 +Only used with \f[I]"mode": "eval"\f[] .SS python.function @@ -9482,20 +9760,20 @@ Call a .SS rename.from .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Description:" 4 -The \f[I]format string\f[] for filenames to rename. +The \f[I]Format String\f[] for filenames to rename. When no value is given, \f[I]extractor.*.filename\f[] is used. .SS rename.to .IP "Type:" 6 -\f[I]string\f[] +\f[I]Format String\f[] .IP "Description:" 4 -The \f[I]format string\f[] for target filenames. +The \f[I]Format String\f[] for target filenames. When no value is given, \f[I]extractor.*.filename\f[] is used. @@ -9541,23 +9819,28 @@ Additional \f[I]ffmpeg\f[] command-line arguments. \f[I]string\f[] .IP "Default:" 9 -\f[I]auto\f[] +\f[I]"auto"\f[] .IP "Description:" 4 \f[I]ffmpeg\f[] demuxer to read and process input files with. -Possible values are - +.IP "Supported Values:" 4 +\f[I]"auto"\f[] +use \f[I]mkvmerge\f[] if available, fall back to \f[I]concat\f[] otherwise +\f[I]"concat"\f[] +https://ffmpeg.org/ffmpeg-formats.html#concat-1 .br -* "\f[I]concat\f[]" (inaccurate frame timecodes for non-uniform frame delays) +Inaccurate frame timecodes for non-uniform frame delays .br -* "\f[I]image2\f[]" (accurate timecodes, requires nanosecond file timestamps, i.e. no Windows or macOS) +\f[I]"image2"\f[] +https://ffmpeg.org/ffmpeg-formats.html#image2-1 .br -* "mkvmerge" (accurate timecodes, only WebM or MKV, requires \f[I]mkvmerge\f[]) +Accurate timecodes, requires nanosecond file timestamps, i.e. no Windows or macOS) .br -* "archive" (store "original" frames in a \f[I].zip\f[] archive) - -"auto" will select mkvmerge if available and fall back to concat otherwise. +\f[I]"mkvmerge"\f[] +Accurate timecodes, only WebM or MKV, requires \f[I]mkvmerge\f[]) +\f[I]"archive"\f[] +Store "original" frames in a \f[I].zip\f[] archive .SS ugoira.ffmpeg-location @@ -9596,12 +9879,12 @@ Location of the \f[I]mkvmerge\f[] executable for use with the .IP "Description:" 4 Controls \f[I]ffmpeg\f[] output. -.br -* \f[I]true\f[]: Enable \f[I]ffmpeg\f[] output -.br -* \f[I]false\f[]: Disable all \f[I]ffmpeg\f[] output -.br -* any \f[I]string\f[]: Pass \f[I]-hide_banner\f[] and \f[I]-loglevel\f[] +\f[I]true\f[] +Enable \f[I]ffmpeg\f[] output +\f[I]false\f[] +Disable all \f[I]ffmpeg\f[] output +any \f[I]string\f[] +Pass \f[I]-hide_banner\f[] and \f[I]-loglevel\f[] with this value as argument to \f[I]ffmpeg\f[] @@ -9626,16 +9909,16 @@ Enable Two-Pass encoding. .IP "Description:" 4 Controls the frame rate argument (\f[I]-r\f[]) for \f[I]ffmpeg\f[] -.br -* \f[I]"auto"\f[]: Automatically assign a fitting frame rate +\f[I]"auto"\f[] +Automatically assign a fitting frame rate based on delays between frames. -.br -* \f[I]"uniform"\f[]: Like \f[I]auto\f[], but assign an explicit frame rate +\f[I]"uniform"\f[] +Like \f[I]auto\f[], but assign an explicit frame rate only to Ugoira with uniform frame delays. -.br -* any other \f[I]string\f[]: Use this value as argument for \f[I]-r\f[]. -.br -* \f[I]null\f[] or an empty \f[I]string\f[]: Don't set an explicit frame rate. +any other \f[I]string\f[] +Use this value as argument for \f[I]-r\f[]. +\f[I]null\f[] or an empty \f[I]string\f[] +Don't set an explicit frame rate. .SS ugoira.keep-files @@ -9731,7 +10014,15 @@ Do not convert frames if target file already exists. .IP "Description:" 4 Compression method to use when writing the archive. -Possible values are \f[I]"store"\f[], \f[I]"zip"\f[], \f[I]"bzip2"\f[], \f[I]"lzma"\f[]. +.IP "Supported Values:" 4 +.br +* \f[I]"store"\f[] +.br +* \f[I]"zip"\f[] +.br +* \f[I]"bzip2"\f[] +.br +* \f[I]"lzma"\f[] .SS zip.extension @@ -9755,7 +10046,8 @@ Filename extension for the created ZIP archive. .IP "Description:" 4 List of extra files to be added to a ZIP archive. -Note: Relative paths are relative to the current +.IP "Note:" 4 +Relative paths are relative to the current \f[I]download directory\f[]. @@ -9778,12 +10070,12 @@ Keep the actual files after writing them to a ZIP archive. \f[I]"default"\f[] .IP "Description:" 4 -.br -* \f[I]"default"\f[]: Write the central directory file header +\f[I]"default"\f[] +Write the central directory file header once after everything is done or an exception is raised. -.br -* \f[I]"safe"\f[]: Update the central directory file header +\f[I]"safe"\f[] +Update the central directory file header each time a file is stored in a ZIP archive. This greatly reduces the chance a ZIP archive gets corrupted in @@ -9823,7 +10115,8 @@ gets \f[I]imported\f[] and searched for potential extractors, i.e. classes with a \f[I]pattern\f[] attribute. -Note: \f[I]null\f[] references internal extractors defined in +.IP "Note:" 4 +\f[I]null\f[] references internal extractors defined in \f[I]extractor/__init__.py\f[] or by \f[I]extractor.modules\f[]. @@ -9831,7 +10124,7 @@ or by \f[I]extractor.modules\f[]. .SS extractor.category-map .IP "Type:" 6 .br -* \f[I]object\f[] (category -> category) +* \f[I]object\f[] (category → category) .br * \f[I]string\f[] @@ -9846,8 +10139,7 @@ or by \f[I]extractor.modules\f[]. .IP "Description:" 4 A JSON object mapping category names to their replacements. -Special values: - +.IP "Special Values:" 4 .br * \f[I]"compat"\f[] .. code:: json @@ -9869,7 +10161,7 @@ Special values: .SS extractor.config-map .IP "Type:" 6 -\f[I]object\f[] (category -> category) +\f[I]object\f[] (category → category) .IP "Default:" 9 .. code:: json @@ -9895,7 +10187,7 @@ For example, a \f[I]"naver": "naver-blog"\f[] key-value pair will make all .SS jinja.environment .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 .. code:: json @@ -9914,7 +10206,7 @@ object. .SS jinja.policies .IP "Type:" 6 -\f[I]object\f[] (name -> value) +\f[I]object\f[] (name → value) .IP "Example:" 4 .. code:: json @@ -9955,7 +10247,7 @@ A Python \f[I]Module\f[] containing custom \f[I]jinja\f[] A Python \f[I]Module\f[] whose namespace, in addition to the \f[I]GLOBALS\f[] dict in \f[I]util.py\f[], -is used as \f[I]globals parameter\f[] for compiled Python expressions. +is used as \f[I]globals parameter\f[] for compiled Expressions_. .SS cache.file @@ -9987,15 +10279,15 @@ this cache. \f[I]true\f[] .IP "Description:" 4 -Evaluate filter expressions in a special environment +Evaluate \f[I]Expressions\f[] in a special environment preventing them from raising fatal exceptions. -\f[I]true\f[] or \f[I]"tryexcept"\f[]: +\f[I]true\f[] \f[I] \f[I]"tryexcept"\f[] Wrap expressions in a try/except block; Evaluate expressions raising an exception as \f[I]false\f[] -\f[I]false\f[] or \f[I]"raw"\f[]: +\f[I]false\f[] \f[] \f[I]"raw"\f[] Do not wrap expressions in a special environment -\f[I]"defaultdict"\f[]: +\f[I]"defaultdict"\f[] Prevent exceptions when accessing undefined variables by using a \f[I]defaultdict\f[] @@ -10008,7 +10300,7 @@ by using a \f[I]defaultdict\f[] \f[I]"/"\f[] .IP "Description:" 4 -Character(s) used as argument separator in format string +Character(s) used as argument separator in \f[I]Format String\f[] \f[I]format specifiers\f[]. For example, setting this option to \f[I]"#"\f[] would allow a replacement @@ -10041,7 +10333,7 @@ as signal handler for. .SS signals-actions .IP "Type:" 6 -\f[I]object\f[] (signal -> \f[I]Action(s)\f[]) +\f[I]object\f[] (signal → \f[I]Action(s)\f[]) .IP "Example:" 4 .. code:: json @@ -10378,7 +10670,7 @@ Simple \f[I]tilde expansion\f[] and \f[I]environment variable expansion\f[] is supported. -.IP "Note::" 4 +.IP "Note:" 4 In Windows environments, both backslashes \f[I]\\\f[] as well as forward slashes \f[I]/\f[] can be used as path separators. @@ -10475,7 +10767,8 @@ use \f[I]"w"\f[] to truncate or \f[I]"a"\f[] to append .br * Default: \f[I]"utf-8"\f[] -Note: path, mode, and encoding are only applied when configuring +.IP "Note:" 4 +path, mode, and encoding are only applied when configuring logging output to a file. @@ -10502,9 +10795,9 @@ logging output to a file. An \f[I]object\f[] containing a \f[I]"name"\f[] attribute specifying the post-processor type, as well as any of its \f[I]options\f[]. -It is possible to set a \f[I]"filter"\f[] expression similar to -\f[I]image-filter\f[] to only run a post-processor -conditionally. +It is possible to set a \f[I]"filter"\f[] \f[I]Condition\f[] similar to +\f[I]image-filter\f[] +to only run a post-processor conditionally. It is also possible set a \f[I]"whitelist"\f[] or \f[I]"blacklist"\f[] to only enable or disable a post-processor for the specified @@ -10520,7 +10813,7 @@ Compare versions of the same file and replace/enumerate them on mismatch (requires \f[I]downloader.*.part\f[] = \f[I]true\f[] and \f[I]extractor.*.skip\f[] = \f[I]false\f[]) .br \f[I]directory\f[] -Reevaluate \f[I]directory\f[] format strings +Reevaluate \f[I]directory\f[] \f[I]Format Strings\f[] \f[I]exec\f[] Execute external commands \f[I]hash\f[] @@ -10619,6 +10912,80 @@ wait until Enter is pressed when no argument was given. Exit the program with the given argument as exit status. +.SS Expression +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Example:" 4 +.br +* "1 + 2 + 3" +.br +* "str(id) + '_' + title" +.br +* "' - '.join(tags[:3]) if tags else 'no tags'" + +.IP "Description:" 4 +A Python \f[I]Expression\f[] is a combination of +values, variables, operators, and function calls +that evaluate to a single value. + +.IP "Reference:" 4 +.br +* https://docs.python.org/3/reference/expressions.html + + +.SS Condition +.IP "Type:" 6 +.br +* \f[I]Expression\f[] +.br +* \f[I]list\f[] of \f[I]Expressions\f[] + +.IP "Example:" 4 +.br +* "not is_watching" +.br +* "locals().get('optional')" +.br +* "date >= datetime(2025, 7, 1) or abort()" +.br +* ["width > 800", "0.9 < width/height < 1.1"] + +.IP "Description:" 4 +A \f[I]Condition\f[] is an \f[I]Expression\f[] +whose result is evaluated as a +\f[I]boolean\f[] +value. + + +.SS Format String +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Example:" 4 +.br +* "foo" +.br +* "{username}" +.br +* "{title} ({id}).{extension}" +.br +* "\\fF {title.title()} ({num:>0:>0{len(str(a))}} / {count}).{extension}" + +.IP "Description:" 4 +A \f[I]Format String\f[] allows creating dynamic text +by embedding metadata values directly into replacement fields +marked by curly braces \f[I]{...}\f[]. + +.IP "Reference:" 4 +.br +* \f[I]docs/formatting\f[] +.br +* https://docs.python.org/3/library/string.html#formatstrings +.br +* https://docs.python.org/3/library/string.html#formatspec + + .SH BUGS https://github.com/mikf/gallery-dl/issues diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 5a1b604..2205ecb 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -455,6 +455,8 @@ "order-posts": "asc", "previews" : false, "videos" : true, + "warn-images": true, + "warn-videos": true, "stories": { "split": false @@ -521,6 +523,14 @@ "lang": null, "ratings": ["safe", "suggestive", "erotica", "pornographic"] }, + "mangafire": + { + "lang": "en" + }, + "mangareader": + { + "lang": "en" + }, "mangoxo": { "username": "", @@ -1112,6 +1122,7 @@ "wikimedia": { "sleep-request": "1.0-2.0", + "image-revisions": 1, "limit": 50, "subcategories": true }, diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO index a339b24..6878d26 100644 --- a/gallery_dl.egg-info/PKG-INFO +++ b/gallery_dl.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: gallery_dl -Version: 1.30.8 +Version: 1.30.9 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -141,9 +141,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.exe>`__ +- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.exe>`__ (Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__) -- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.bin>`__ +- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.bin>`__ Nightly Builds diff --git a/gallery_dl.egg-info/SOURCES.txt b/gallery_dl.egg-info/SOURCES.txt index 93a6880..336ef00 100644 --- a/gallery_dl.egg-info/SOURCES.txt +++ b/gallery_dl.egg-info/SOURCES.txt @@ -158,11 +158,13 @@ gallery_dl/extractor/luscious.py gallery_dl/extractor/lynxchan.py gallery_dl/extractor/madokami.py gallery_dl/extractor/mangadex.py +gallery_dl/extractor/mangafire.py gallery_dl/extractor/mangafox.py gallery_dl/extractor/mangahere.py gallery_dl/extractor/manganelo.py gallery_dl/extractor/mangapark.py gallery_dl/extractor/mangaread.py +gallery_dl/extractor/mangareader.py gallery_dl/extractor/mangataro.py gallery_dl/extractor/mangoxo.py gallery_dl/extractor/mastodon.py @@ -215,6 +217,7 @@ gallery_dl/extractor/redgifs.py gallery_dl/extractor/rule34us.py gallery_dl/extractor/rule34vault.py gallery_dl/extractor/rule34xyz.py +gallery_dl/extractor/s3ndpics.py gallery_dl/extractor/saint.py gallery_dl/extractor/sankaku.py gallery_dl/extractor/sankakucomplex.py diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py index 5d6c3d7..6c19e23 100644 --- a/gallery_dl/cookies.py +++ b/gallery_dl/cookies.py @@ -27,8 +27,11 @@ from . import aes, text, util SUPPORTED_BROWSERS_CHROMIUM = { "brave", "chrome", "chromium", "edge", "opera", "thorium", "vivaldi"} SUPPORTED_BROWSERS_FIREFOX = {"firefox", "librewolf", "zen"} +SUPPORTED_BROWSERS_WEBKIT = {"safari", "orion"} SUPPORTED_BROWSERS = \ - SUPPORTED_BROWSERS_CHROMIUM | SUPPORTED_BROWSERS_FIREFOX | {"safari"} + SUPPORTED_BROWSERS_CHROMIUM \ + | SUPPORTED_BROWSERS_FIREFOX \ + | SUPPORTED_BROWSERS_WEBKIT logger = logging.getLogger("cookies") @@ -38,8 +41,8 @@ def load_cookies(browser_specification): _parse_browser_specification(*browser_specification) if browser_name in SUPPORTED_BROWSERS_FIREFOX: return load_cookies_firefox(browser_name, profile, container, domain) - elif browser_name == "safari": - return load_cookies_safari(profile, domain) + elif browser_name in SUPPORTED_BROWSERS_WEBKIT: + return load_cookies_webkit(browser_name, profile, domain) elif browser_name in SUPPORTED_BROWSERS_CHROMIUM: return load_cookies_chromium(browser_name, profile, keyring, domain) else: @@ -92,7 +95,7 @@ def load_cookies_firefox(browser_name, profile=None, return cookies -def load_cookies_safari(profile=None, domain=None): +def load_cookies_webkit(browser_name, profile=None, domain=None): """Ref.: https://github.com/libyal/dtformats/blob /main/documentation/Safari%20Cookies.asciidoc - This data appears to be out of date @@ -100,15 +103,24 @@ def load_cookies_safari(profile=None, domain=None): - There are a few bytes here and there which are skipped during parsing """ - with _safari_cookies_database() as fp: - data = fp.read() - page_sizes, body_start = _safari_parse_cookies_header(data) + if browser_name == "safari": + with _safari_cookies_database() as fp: + data = fp.read() + elif browser_name == "orion": + with _orion_cookies_database() as fp: + data = fp.read() + else: + raise ValueError(f"unknown webkit browser '{browser_name}'") + + page_sizes, body_start = _webkit_parse_cookies_header(data) p = DataParser(data[body_start:]) cookies = [] for page_size in page_sizes: - _safari_parse_cookies_page(p.read_bytes(page_size), cookies) - _log_info("Extracted %s cookies from Safari", len(cookies)) + _webkit_parse_cookies_page(p.read_bytes(page_size), cookies) + _log_info("Extracted %s cookies from %s", + browser_name.capitalize(), len(cookies)) + return cookies @@ -278,7 +290,8 @@ def _firefox_browser_directory(browser_name): # -------------------------------------------------------------------- -# safari +# safari/orion/webkit + def _safari_cookies_database(): try: @@ -291,7 +304,13 @@ def _safari_cookies_database(): return open(path, "rb") -def _safari_parse_cookies_header(data): +def _orion_cookies_database(): + path = os.path.expanduser( + "~/Library/HTTPStorages/com.kagi.kagimacOS.binarycookies") + return open(path, "rb") + + +def _webkit_parse_cookies_header(data): p = DataParser(data) p.expect_bytes(b"cook", "database signature") number_of_pages = p.read_uint(big_endian=True) @@ -300,7 +319,7 @@ def _safari_parse_cookies_header(data): return page_sizes, p.cursor -def _safari_parse_cookies_page(data, cookies, domain=None): +def _webkit_parse_cookies_page(data, cookies, domain=None): p = DataParser(data) p.expect_bytes(b"\x00\x00\x01\x00", "page signature") number_of_cookies = p.read_uint() @@ -313,13 +332,13 @@ def _safari_parse_cookies_page(data, cookies, domain=None): for i, record_offset in enumerate(record_offsets): p.skip_to(record_offset, "space between records") - record_length = _safari_parse_cookies_record( + record_length = _webkit_parse_cookies_record( data[record_offset:], cookies, domain) p.read_bytes(record_length) p.skip_to_end("space in between pages") -def _safari_parse_cookies_record(data, cookies, host=None): +def _webkit_parse_cookies_record(data, cookies, host=None): p = DataParser(data) record_size = p.read_uint() p.skip(4, "unknown record field 1") @@ -355,7 +374,7 @@ def _safari_parse_cookies_record(data, cookies, host=None): p.skip_to(value_offset) value = p.read_cstring() except UnicodeDecodeError: - _log_warning("Failed to parse Safari cookie") + _log_warning("Failed to parse WebKit cookie") return record_size p.skip_to(record_size, "space at the end of the record") diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 111fd9b..248bf70 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -413,7 +413,7 @@ class HttpDownloader(DownloaderBase): def _find_extension(self, response): """Get filename extension from MIME type""" mtype = response.headers.get("Content-Type", "image/jpeg") - mtype = mtype.partition(";")[0] + mtype = mtype.partition(";")[0].lower() if "/" not in mtype: mtype = "image/" + mtype @@ -475,6 +475,10 @@ MIME_TYPES = { "audio/ogg" : "ogg", "audio/mpeg" : "mp3", + "application/vnd.apple.mpegurl": "m3u8", + "application/x-mpegurl" : "m3u8", + "application/dash+xml" : "mpd", + "application/zip" : "zip", "application/x-zip": "zip", "application/x-zip-compressed": "zip", @@ -526,6 +530,8 @@ SIGNATURE_CHECKS = { s[8:12] == b"WAVE"), "mp3" : lambda s: (s[0:3] == b"ID3" or s[0:2] in (b"\xFF\xFB", b"\xFF\xF3", b"\xFF\xF2")), + "m3u8": lambda s: s[0:7] == b"#EXTM3U", + "mpd" : lambda s: b"<MPD" in s, "zip" : lambda s: s[0:4] in (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"), "rar" : lambda s: s[0:6] == b"Rar!\x1A\x07", "7z" : lambda s: s[0:6] == b"\x37\x7A\xBC\xAF\x27\x1C", diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index abdb6cc..a3df634 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -115,11 +115,13 @@ modules = [ "lynxchan", "madokami", "mangadex", + "mangafire", "mangafox", "mangahere", "manganelo", "mangapark", "mangaread", + "mangareader", "mangataro", "mangoxo", "misskey", @@ -166,6 +168,7 @@ modules = [ "rule34us", "rule34vault", "rule34xyz", + "s3ndpics", "saint", "sankaku", "sankakucomplex", diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py index 67fdb39..1552899 100644 --- a/gallery_dl/extractor/chevereto.py +++ b/gallery_dl/extractor/chevereto.py @@ -40,19 +40,15 @@ class CheveretoExtractor(BaseExtractor): BASE_PATTERN = CheveretoExtractor.update({ "jpgfish": { "root": "https://jpg6.su", - "pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)", - }, - "imgkiwi": { - "root": "https://img.kiwi", - "pattern": r"img\.kiwi", + "pattern": r"(?:www\.)?jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)", }, "imagepond": { "root": "https://imagepond.net", - "pattern": r"imagepond\.net", + "pattern": r"(?:www\.)?imagepond\.net", }, "imglike": { "root": "https://imglike.com", - "pattern": r"imglike\.com", + "pattern": r"(?:www\.)?imglike\.com", }, }) @@ -79,7 +75,7 @@ class CheveretoImageExtractor(CheveretoExtractor): fromhex=True) file = { - "id" : self.path.rpartition(".")[2], + "id" : self.path.rpartition("/")[2].rpartition(".")[2], "url" : url, "album": text.remove_html(extr( "Added to <a", "</a>").rpartition(">")[2]), @@ -144,7 +140,8 @@ class CheveretoAlbumExtractor(CheveretoExtractor): def items(self): url = self.root + self.path - data = {"_extractor": CheveretoImageExtractor} + data_image = {"_extractor": CheveretoImageExtractor} + data_video = {"_extractor": CheveretoVideoExtractor} if self.path.endswith("/sub"): albums = self._pagination(url) @@ -152,8 +149,9 @@ class CheveretoAlbumExtractor(CheveretoExtractor): albums = (url,) for album in albums: - for image in self._pagination(album): - yield Message.Queue, image, data + for item_url in self._pagination(album): + data = data_video if "/video/" in item_url else data_image + yield Message.Queue, item_url, data class CheveretoCategoryExtractor(CheveretoExtractor): diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index fccc466..817d2c4 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -125,8 +125,18 @@ class ImxtoGalleryExtractor(ImagehostImageExtractor): "title": text.unescape(title.partition(">")[2]).strip(), } - for url in text.extract_iter(page, "<a href=", " ", pos): - yield Message.Queue, url.strip("\"'"), data + params = {"page": 1} + while True: + for url in text.extract_iter(page, "<a href=", " ", pos): + if "/i/" in url: + yield Message.Queue, url.strip("\"'"), data + + if 'class="pagination' not in page or \ + 'class="disabled">Last' in page: + return + + params["page"] += 1 + page = self.request(self.page_url, params=params).text class AcidimgImageExtractor(ImagehostImageExtractor): diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py index 00e06b5..0e6c480 100644 --- a/gallery_dl/extractor/instagram.py +++ b/gallery_dl/extractor/instagram.py @@ -39,7 +39,6 @@ class InstagramExtractor(Extractor): self.www_claim = "0" self.csrf_token = util.generate_token() self._find_tags = util.re(r"#\w+").findall - self._warn_video_ua = True self._logged_in = True self._cursor = None self._user = None @@ -52,6 +51,12 @@ class InstagramExtractor(Extractor): else: self.api = InstagramRestAPI(self) + self._warn_video = True if self.config("warn-videos", True) else False + self._warn_image = ( + 9 if not (wi := self.config("warn-images", True)) else + 1 if wi in ("all", "both") else + 0) + def items(self): self.login() @@ -172,6 +177,7 @@ class InstagramExtractor(Extractor): "post_id": reel_id, "post_shortcode": shortcode_from_id(reel_id), "post_url": post_url, + "type": "story" if expires else "highlight", } if "title" in post: data["highlight_title"] = post["title"] @@ -182,7 +188,6 @@ class InstagramExtractor(Extractor): data = { "post_id" : post["pk"], "post_shortcode": post["code"], - "post_url": f"{self.root}/p/{post['code']}/", "likes": post.get("like_count", 0), "liked": post.get("has_liked", False), "pinned": self._extract_pinned(post), @@ -239,8 +244,8 @@ class InstagramExtractor(Extractor): manifest = item.get("video_dash_manifest") media = video - if self._warn_video_ua: - self._warn_video_ua = False + if self._warn_video: + self._warn_video = False pattern = text.re( r"Chrome/\d{3,}\.\d+\.\d+\.\d+(?!\d* Mobile)") if not pattern.search(self.session.headers["User-Agent"]): @@ -250,8 +255,9 @@ class InstagramExtractor(Extractor): video = manifest = None media = image - if image["width"] < item.get("original_width", 0) or \ - image["height"] < item.get("original_height", 0): + if self._warn_image < ( + (image["width"] < item.get("original_width", 0)) + + (image["height"] < item.get("original_height", 0))): self.log.warning( "%s: Available image resolutions lower than the " "original (%sx%s < %sx%s). " @@ -278,7 +284,7 @@ class InstagramExtractor(Extractor): if manifest is not None: media["_ytdl_manifest_data"] = manifest if "owner" in item: - media["owner2"] = item["owner"] + media["owner"] = item["owner"] if "reshared_story_media_author" in item: media["author"] = item["reshared_story_media_author"] if "expiring_at" in item: @@ -287,6 +293,14 @@ class InstagramExtractor(Extractor): self._extract_tagged_users(item, media) files.append(media) + if "type" not in data: + if len(files) == 1 and files[0]["video_url"]: + data["type"] = "reel" + data["post_url"] = f"{self.root}/reel/{post['code']}/" + else: + data["type"] = "post" + data["post_url"] = f"{self.root}/p/{post['code']}/" + return data def _parse_post_graphql(self, post): @@ -443,6 +457,32 @@ class InstagramExtractor(Extractor): user[key] = 0 +class InstagramPostExtractor(InstagramExtractor): + """Extractor for an Instagram post""" + subcategory = "post" + pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" + r"/(?:share/()|[^/?#]+/)?(?:p|tv|reels?())/([^/?#]+)") + example = "https://www.instagram.com/p/abcdefg/" + + def __init__(self, match): + if match[2] is not None: + self.subcategory = "reel" + InstagramExtractor.__init__(self, match) + + def posts(self): + share, reel, shortcode = self.groups + if share is not None: + url = text.ensure_http_scheme(self.url) + headers = { + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "same-origin", + } + location = self.request_location(url, headers=headers) + shortcode = location.split("/")[-2] + return self.api.media(shortcode) + + class InstagramUserExtractor(Dispatch, InstagramExtractor): """Extractor for an Instagram user profile""" pattern = USER_PATTERN + r"/?(?:$|[?#])" @@ -740,27 +780,6 @@ class InstagramAvatarExtractor(InstagramExtractor): },) -class InstagramPostExtractor(InstagramExtractor): - """Extractor for an Instagram post""" - subcategory = "post" - pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" - r"/(?:share/()|[^/?#]+/)?(?:p|tv|reel)/([^/?#]+)") - example = "https://www.instagram.com/p/abcdefg/" - - def posts(self): - share, shortcode = self.groups - if share is not None: - url = text.ensure_http_scheme(self.url) - headers = { - "Sec-Fetch-Dest": "empty", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "same-origin", - } - location = self.request_location(url, headers=headers) - shortcode = location.split("/")[-2] - return self.api.media(shortcode) - - class InstagramRestAPI(): def __init__(self, extractor): diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py index fbed328..30d6848 100644 --- a/gallery_dl/extractor/mangadex.py +++ b/gallery_dl/extractor/mangadex.py @@ -39,7 +39,7 @@ class MangadexExtractor(Extractor): data = self._transform(chapter) data["_extractor"] = MangadexChapterExtractor self._cache[uuid] = data - yield Message.Queue, self.root + "/chapter/" + uuid, data + yield Message.Queue, f"{self.root}/chapter/{uuid}", data def _items_manga(self): data = {"_extractor": MangadexMangaExtractor} @@ -51,13 +51,8 @@ class MangadexExtractor(Extractor): relationships = defaultdict(list) for item in chapter["relationships"]: relationships[item["type"]].append(item) - manga = self.api.manga(relationships["manga"][0]["id"]) - for item in manga["relationships"]: - relationships[item["type"]].append(item) cattributes = chapter["attributes"] - mattributes = manga["attributes"] - if lang := cattributes.get("translatedLanguage"): lang = lang.partition("-")[0] @@ -66,35 +61,21 @@ class MangadexExtractor(Extractor): else: chnum, sep, minor = 0, "", "" - data = { - "manga" : (mattributes["title"].get("en") or - next(iter(mattributes["title"].values()))), - "manga_id": manga["id"], + return { + **_manga_info(self, relationships["manga"][0]["id"]), "title" : cattributes["title"], "volume" : text.parse_int(cattributes["volume"]), "chapter" : text.parse_int(chnum), - "chapter_minor": sep + minor, + "chapter_minor": f"{sep}{minor}", "chapter_id": chapter["id"], "date" : text.parse_datetime(cattributes["publishAt"]), + "group" : [group["attributes"]["name"] + for group in relationships["scanlation_group"]], "lang" : lang, - "language": util.code_to_language(lang), "count" : cattributes["pages"], "_external_url": cattributes.get("externalUrl"), } - data["artist"] = [artist["attributes"]["name"] - for artist in relationships["artist"]] - data["author"] = [author["attributes"]["name"] - for author in relationships["author"]] - data["group"] = [group["attributes"]["name"] - for group in relationships["scanlation_group"]] - - data["status"] = mattributes["status"] - data["tags"] = [tag["attributes"]["name"]["en"] - for tag in mattributes["tags"]] - - return data - class MangadexCoversExtractor(MangadexExtractor): """Extractor for mangadex manga covers""" @@ -103,7 +84,7 @@ class MangadexCoversExtractor(MangadexExtractor): filename_fmt = "{volume:>02}_{lang}.{extension}" archive_fmt = "c_{cover_id}" pattern = (rf"{BASE_PATTERN}/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)" - r"(?:/[^/?#]+)?\?tab=art") + rf"(?:/[^/?#]+)?\?tab=art") example = ("https://mangadex.org/title" "/01234567-89ab-cdef-0123-456789abcdef?tab=art") @@ -121,24 +102,10 @@ class MangadexCoversExtractor(MangadexExtractor): relationships = defaultdict(list) for item in cover["relationships"]: relationships[item["type"]].append(item) - manga = self.api.manga(relationships["manga"][0]["id"]) - for item in manga["relationships"]: - relationships[item["type"]].append(item) - cattributes = cover["attributes"] - mattributes = manga["attributes"] return { - "manga" : (mattributes["title"].get("en") or - next(iter(mattributes["title"].values()))), - "manga_id": manga["id"], - "status" : mattributes["status"], - "author" : [author["attributes"]["name"] - for author in relationships["author"]], - "artist" : [artist["attributes"]["name"] - for artist in relationships["artist"]], - "tags" : [tag["attributes"]["name"]["en"] - for tag in mattributes["tags"]], + **_manga_info(self, relationships["manga"][0]["id"]), "cover" : cattributes["fileName"], "lang" : cattributes.get("locale"), "volume" : text.parse_int(cattributes["volume"]), @@ -150,7 +117,7 @@ class MangadexCoversExtractor(MangadexExtractor): class MangadexChapterExtractor(MangadexExtractor): """Extractor for manga-chapters from mangadex.org""" subcategory = "chapter" - pattern = BASE_PATTERN + r"/chapter/([0-9a-f-]+)" + pattern = rf"{BASE_PATTERN}/chapter/([0-9a-f-]+)" example = ("https://mangadex.org/chapter" "/01234567-89ab-cdef-0123-456789abcdef") @@ -177,13 +144,13 @@ class MangadexChapterExtractor(MangadexExtractor): "page-reverse") else enumerate for data["page"], page in enum(chapter["data"], 1): text.nameext_from_url(page, data) - yield Message.Url, base + page, data + yield Message.Url, f"{base}{page}", data class MangadexMangaExtractor(MangadexExtractor): """Extractor for manga from mangadex.org""" subcategory = "manga" - pattern = BASE_PATTERN + r"/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)" + pattern = rf"{BASE_PATTERN}/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)" example = ("https://mangadex.org/title" "/01234567-89ab-cdef-0123-456789abcdef") @@ -194,7 +161,7 @@ class MangadexMangaExtractor(MangadexExtractor): class MangadexFeedExtractor(MangadexExtractor): """Extractor for chapters from your Updates Feed""" subcategory = "feed" - pattern = BASE_PATTERN + r"/titles?/feed$()" + pattern = rf"{BASE_PATTERN}/titles?/feed$()" example = "https://mangadex.org/title/feed" def chapters(self): @@ -204,7 +171,7 @@ class MangadexFeedExtractor(MangadexExtractor): class MangadexFollowingExtractor(MangadexExtractor): """Extractor for followed manga from your Library""" subcategory = "following" - pattern = BASE_PATTERN + r"/titles?/follows(?:\?([^#]+))?$" + pattern = rf"{BASE_PATTERN}/titles?/follows(?:\?([^#]+))?$" example = "https://mangadex.org/title/follows" items = MangadexExtractor._items_manga @@ -216,8 +183,8 @@ class MangadexFollowingExtractor(MangadexExtractor): class MangadexListExtractor(MangadexExtractor): """Extractor for mangadex MDLists""" subcategory = "list" - pattern = (BASE_PATTERN + - r"/list/([0-9a-f-]+)(?:/[^/?#]*)?(?:\?tab=(\w+))?") + pattern = (rf"{BASE_PATTERN}" + rf"/list/([0-9a-f-]+)(?:/[^/?#]*)?(?:\?tab=(\w+))?") example = ("https://mangadex.org/list" "/01234567-89ab-cdef-0123-456789abcdef/NAME") @@ -242,7 +209,7 @@ class MangadexListExtractor(MangadexExtractor): class MangadexAuthorExtractor(MangadexExtractor): """Extractor for mangadex authors""" subcategory = "author" - pattern = BASE_PATTERN + r"/author/([0-9a-f-]+)" + pattern = rf"{BASE_PATTERN}/author/([0-9a-f-]+)" example = ("https://mangadex.org/author" "/01234567-89ab-cdef-0123-456789abcdef/NAME") @@ -280,30 +247,30 @@ class MangadexAPI(): else text.ensure_http_scheme(server).rstrip("/")) def athome_server(self, uuid): - return self._call("/at-home/server/" + uuid) + return self._call(f"/at-home/server/{uuid}") def author(self, uuid, manga=False): params = {"includes[]": ("manga",)} if manga else None - return self._call("/author/" + uuid, params)["data"] + return self._call(f"/author/{uuid}", params)["data"] def chapter(self, uuid): params = {"includes[]": ("scanlation_group",)} - return self._call("/chapter/" + uuid, params)["data"] + return self._call(f"/chapter/{uuid}", params)["data"] def covers_manga(self, uuid): params = {"manga[]": uuid} return self._pagination_covers("/cover", params) def list(self, uuid): - return self._call("/list/" + uuid, None, True)["data"] + return self._call(f"/list/{uuid}", None, True)["data"] def list_feed(self, uuid): - return self._pagination_chapters("/list/" + uuid + "/feed", None, True) + return self._pagination_chapters(f"/list/{uuid}/feed", None, True) @memcache(keyarg=1) def manga(self, uuid): params = {"includes[]": ("artist", "author")} - return self._call("/manga/" + uuid, params)["data"] + return self._call(f"/manga/{uuid}", params)["data"] def manga_author(self, uuid_author): params = {"authorOrArtist": uuid_author} @@ -315,7 +282,7 @@ class MangadexAPI(): "order[volume]" : order, "order[chapter]": order, } - return self._pagination_chapters("/manga/" + uuid + "/feed", params) + return self._pagination_chapters(f"/manga/{uuid}/feed", params) def user_follows_manga(self): params = {"contentRating": None} @@ -366,17 +333,17 @@ class MangadexAPI(): _refresh_token_cache.update( (username, "personal"), data["refresh_token"]) - return "Bearer " + access_token + return f"Bearer {access_token}" @cache(maxage=900, keyarg=1) def _authenticate_impl_legacy(self, username, password): if refresh_token := _refresh_token_cache(username): self.extractor.log.info("Refreshing access token") - url = self.root + "/auth/refresh" + url = f"{self.root}/auth/refresh" json = {"token": refresh_token} else: self.extractor.log.info("Logging in as %s", username) - url = self.root + "/auth/login" + url = f"{self.root}/auth/login" json = {"username": username, "password": password} self.extractor.log.debug("Using legacy login method") @@ -387,10 +354,10 @@ class MangadexAPI(): if refresh_token != data["token"]["refresh"]: _refresh_token_cache.update(username, data["token"]["refresh"]) - return "Bearer " + data["token"]["session"] + return f"Bearer {data['token']['session']}" def _call(self, endpoint, params=None, auth=False): - url = self.root + endpoint + url = f"{self.root}{endpoint}" headers = self.headers_auth if auth else self.headers while True: @@ -470,3 +437,33 @@ class MangadexAPI(): @cache(maxage=90*86400, keyarg=0) def _refresh_token_cache(username): return None + + +@memcache(keyarg=1) +def _manga_info(self, uuid): + manga = self.api.manga(uuid) + + rel = defaultdict(list) + for item in manga["relationships"]: + rel[item["type"]].append(item) + mattr = manga["attributes"] + + return { + "manga" : (mattr["title"].get("en") or + next(iter(mattr["title"].values()))), + "manga_id": manga["id"], + "manga_titles": [t.popitem()[1] + for t in mattr.get("altTitles") or ()], + "manga_date" : text.parse_datetime(mattr.get("createdAt")), + "description" : (mattr["description"].get("en") or + next(iter(mattr["description"].values()))), + "demographic": mattr.get("publicationDemographic"), + "origin": mattr.get("originalLanguage"), + "status": mattr.get("status"), + "year" : mattr.get("year"), + "rating": mattr.get("contentRating"), + "links" : mattr.get("links"), + "tags" : [tag["attributes"]["name"]["en"] for tag in mattr["tags"]], + "artist": [artist["attributes"]["name"] for artist in rel["artist"]], + "author": [author["attributes"]["name"] for author in rel["author"]], + } diff --git a/gallery_dl/extractor/mangafire.py b/gallery_dl/extractor/mangafire.py new file mode 100644 index 0000000..5ccb732 --- /dev/null +++ b/gallery_dl/extractor/mangafire.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://mangafire.to/""" + +from .common import ChapterExtractor, MangaExtractor +from .. import text, exception +from ..cache import memcache + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?mangafire\.to" + + +class MangafireBase(): + """Base class for mangafire extractors""" + category = "mangafire" + root = "https://mangafire.to" + + +class MangafireChapterExtractor(MangafireBase, ChapterExtractor): + """Extractor for mangafire manga chapters""" + directory_fmt = ( + "{category}", "{manga}", + "{volume:?v/ />02}{chapter:?c//>03}{chapter_minor:?//}{title:?: //}") + filename_fmt = ( + "{manga}{volume:?_v//>02}{chapter:?_c//>03}{chapter_minor:?//}_" + "{page:>03}.{extension}") + archive_fmt = ( + "{manga_id}_{chapter_id}_{page}") + pattern = (rf"{BASE_PATTERN}/read/([\w-]+\.(\w+))/([\w-]+)" + rf"/((chapter|volume)-\d+(?:\D.*)?)") + example = "https://mangafire.to/read/MANGA.ID/LANG/chapter-123" + + def metadata(self, _): + manga_path, manga_id, lang, chapter_info, self.type = self.groups + + try: + chapters = _manga_chapters(self, (manga_id, self.type, lang)) + anchor = chapters[chapter_info] + except KeyError: + raise exception.NotFoundError("chapter") + self.chapter_id = text.extr(anchor, 'data-id="', '"') + + return { + **_manga_info(self, manga_path), + **_chapter_info(anchor), + } + + def images(self, page): + url = f"{self.root}/ajax/read/{self.type}/{self.chapter_id}" + headers = {"x-requested-with": "XMLHttpRequest"} + data = self.request_json(url, headers=headers) + + return [ + (image[0], None) + for image in data["result"]["images"] + ] + + +class MangafireMangaExtractor(MangafireBase, MangaExtractor): + """Extractor for mangafire manga""" + chapterclass = MangafireChapterExtractor + pattern = rf"{BASE_PATTERN}/manga/([\w-]+)\.(\w+)" + example = "https://mangafire.to/manga/MANGA.ID" + + def chapters(self, page): + manga_slug, manga_id = self.groups + lang = self.config("lang") or "en" + + manga = _manga_info(self, f"{manga_slug}.{manga_id}") + chapters = _manga_chapters(self, (manga_id, "chapter", lang)) + + return [ + (f"""{self.root}{text.extr(anchor, 'href="', '"')}""", { + **manga, + **_chapter_info(anchor), + }) + for anchor in chapters.values() + ] + + +@memcache(keyarg=1) +def _manga_info(self, manga_path, page=None): + if page is None: + url = f"{self.root}/manga/{manga_path}" + page = self.request(url).text + slug, _, mid = manga_path.rpartition(".") + + extr = text.extract_from(page) + manga = { + "cover": text.extr(extr( + 'class="poster">', '</div>'), 'src="', '"'), + "status": extr("<p>", "<").replace("_", " ").title(), + "manga" : text.unescape(extr( + 'itemprop="name">', "<")), + "manga_id": mid, + "manga_slug": slug, + "manga_titles": text.unescape(extr( + "<h6>", "<")).split("; "), + "type": text.remove_html(extr( + 'class="min-info">', "</a>")), + "author": text.unescape(text.remove_html(extr( + "<span>Author:</span>", "</div>"))).split(" , "), + "published": text.remove_html(extr( + "<span>Published:</span>", "</div>")), + "tags": text.split_html(extr( + "<span>Genres:</span>", "</div>"))[::2], + "publisher": text.unescape(text.remove_html(extr( + "<span>Mangazines:</span>", "</div>"))).split(" , "), + "score": text.parse_float(text.remove_html(extr( + 'class="score">', " / "))), + "description": text.remove_html(extr( + 'id="synopsis">', "<script>")), + } + + if len(lst := manga["author"]) == 1 and not lst[0]: + manga["author"] = () + if len(lst := manga["publisher"]) == 1 and not lst[0]: + manga["publisher"] = () + + return manga + + +@memcache(keyarg=1) +def _manga_chapters(self, manga_info): + manga_id, type, lang = manga_info + url = f"{self.root}/ajax/read/{manga_id}/{type}/{lang}" + headers = {"x-requested-with": "XMLHttpRequest"} + data = self.request_json(url, headers=headers) + + needle = f"{manga_id}/{lang}/" + return { + text.extr(anchor, needle, '"'): anchor + for anchor in text.extract_iter(data["result"]["html"], "<a ", ">") + } + + +@memcache(keyarg=0) +def _chapter_info(info): + _, lang, chapter_info = text.extr(info, 'href="', '"').rsplit("/", 2) + + if chapter_info.startswith("vol"): + volume = text.extr(info, 'data-number="', '"') + volume_id = text.parse_int(text.extr(info, 'data-id="', '"')) + return { + "volume" : text.parse_int(volume), + "volume_id" : volume_id, + "chapter" : 0, + "chapter_minor" : "", + "chapter_string": chapter_info, + "chapter_id" : volume_id, + "title" : text.unescape(text.extr(info, 'title="', '"')), + "lang" : lang, + } + + chapter, sep, minor = text.extr(info, 'data-number="', '"').partition(".") + return { + "chapter" : text.parse_int(chapter), + "chapter_minor" : f"{sep}{minor}", + "chapter_string": chapter_info, + "chapter_id" : text.parse_int(text.extr(info, 'data-id="', '"')), + "title" : text.unescape(text.extr(info, 'title="', '"')), + "lang" : lang, + } diff --git a/gallery_dl/extractor/mangareader.py b/gallery_dl/extractor/mangareader.py new file mode 100644 index 0000000..eb53998 --- /dev/null +++ b/gallery_dl/extractor/mangareader.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://mangareader.to/""" + +from .common import ChapterExtractor, MangaExtractor +from .. import text, util +from ..cache import memcache + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?mangareader\.to" + + +class MangareaderBase(): + """Base class for mangareader extractors""" + category = "mangareader" + root = "https://mangareader.to" + + +class MangareaderChapterExtractor(MangareaderBase, ChapterExtractor): + """Extractor for mangareader manga chapters""" + directory_fmt = ( + "{category}", "{manga}", + "{volume:?v/ />02}{chapter:?c//>03}{chapter_minor:?//}{title:?: //}") + filename_fmt = ( + "{manga}{volume:?_v//>02}{chapter:?_c//>03}{chapter_minor:?//}_" + "{page:>03}.{extension}") + archive_fmt = ( + "{manga_id}_{chapter_id}_{page}") + pattern = (rf"{BASE_PATTERN}/read/([\w-]+-\d+)/([^/?#]+)" + rf"/(chapter|volume)-(\d+[^/?#]*)") + example = "https://mangareader.to/read/MANGA-123/LANG/chapter-123" + + def metadata(self, _): + path, lang, type, chstr = self.groups + + settings = util.json_dumps({ + "readingMode" : "vertical", + "readingDirection": "rtl", + "quality" : "high", + }) + self.cookies.set("mr_settings", settings, domain="mangareader.to") + + url = f"{self.root}/read/{path}/{lang}/{type}-{chstr}" + page = self.request(url).text + self.cid = cid = text.extr(page, 'data-reading-id="', '"') + + manga = _manga_info(self, path) + return { + **manga, + **manga[f"_{type}s"][lang][chstr], + "chapter_id": text.parse_int(cid), + } + + def images(self, page): + key = "chap" if self.groups[2] == "chapter" else "vol" + url = f"{self.root}/ajax/image/list/{key}/{self.cid}" + params = { + "mode" : "vertical,", + "quality" : "high,", + "hozPageSize": "1,", + } + headers = { + "X-Requested-With": "XMLHttpRequest", + "Sec-Fetch-Dest" : "empty", + "Sec-Fetch-Mode" : "cors", + "Sec-Fetch-Site" : "same-origin", + } + html = self.request_json(url, params=params, headers=headers)["html"] + + return [ + (url, None) + for url in text.extract_iter(html, 'data-url="', '"') + ] + + +class MangareaderMangaExtractor(MangareaderBase, MangaExtractor): + """Extractor for mangareader manga""" + chapterclass = MangareaderChapterExtractor + pattern = rf"{BASE_PATTERN}/([\w-]+-\d+)" + example = "https://mangareader.to/MANGA-123" + + def chapters(self, page): + manga = _manga_info(self, self.groups[0]) + lang = self.config("lang") or "en" + + return [ + (info["chapter_url"], {**manga, **info}) + for info in manga["_chapters"][lang].values() + ] + + +@memcache(keyarg=1) +def _manga_info(self, manga_path): + url = f"{self.root}/{manga_path}" + html = self.request(url).text + + slug, _, mid = manga_path.rpartition("-") + extr = text.extract_from(html) + url = extr('property="og:url" content="', '"') + manga = { + "manga_url": url, + "manga_slug": url.rpartition("/")[2].rpartition("-")[0], + "manga_id": text.parse_int(mid), + "manga": text.unescape(extr('class="manga-name">', "<")), + "manga_alt": text.unescape(extr('class="manga-name-or">', "<")), + "tags": text.split_html(extr('class="genres">', "</div>")), + "type": text.remove_html(extr('>Type:', "</div>")), + "status": text.remove_html(extr('>Status:', "</div>")), + "author": text.split_html(extr('>Authors:', "</div>"))[0::2], + "published": text.remove_html(extr('>Published:', "</div>")), + "score": text.parse_float(text.remove_html(extr( + '>Score:', "</div>"))), + "views": text.parse_int(text.remove_html(extr( + '>Views:', "</div>")).replace(",", "")), + } + + base = self.root + + # extract all chapters + html = extr('class="chapters-list-ul">', " </div>") + manga["_chapters"] = chapters = {} + for group in text.extract_iter(html, "<ul", "</ul>"): + lang = text.extr(group, ' id="', '-chapters"') + + chapters[lang] = current = {} + lang = lang.partition("-")[0] + for ch in text.extract_iter(group, "<li ", "</li>"): + path = text.extr(ch, 'href="', '"') + chap = text.extr(ch, 'data-number="', '"') + name = text.unescape(text.extr(ch, 'class="name">', "<")) + + chapter, sep, minor = chap.partition(".") + current[chap] = { + "title" : name.partition(":")[2].strip(), + "chapter" : text.parse_int(chapter), + "chapter_minor" : f"{sep}{minor}", + "chapter_string": chap, + "chapter_url" : f"{base}{path}", + "lang" : lang, + } + + # extract all volumes + html = extr('class="volume-list-ul">', "</section>") + manga["_volumes"] = volumes = {} + for group in html.split('<div class="manga_list-wrap')[1:]: + lang = text.extr(group, ' id="', '-volumes"') + + volumes[lang] = current = {} + lang = lang.partition("-")[0] + for vol in text.extract_iter(group, 'class="item">', "</div>"): + path = text.extr(vol, 'href="', '"') + voln = text.extr(vol, 'tick-vol">', '<').rpartition(" ")[2] + + current[voln] = { + "volume" : text.parse_int(voln), + "volume_cover" : text.extr(vol, ' src="', '"'), + "chapter" : 0, + "chapter_minor" : "", + "chapter_string": voln, + "chapter_url" : f"{base}{path}", + "lang" : lang, + } + + # extract remaining metadata + manga["description"] = text.unescape(extr( + 'class="description-modal">', "</div>")).strip() + + return manga diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py index 5ff601a..42eaeef 100644 --- a/gallery_dl/extractor/misskey.py +++ b/gallery_dl/extractor/misskey.py @@ -25,8 +25,8 @@ class MisskeyExtractor(BaseExtractor): def _init(self): self.api = MisskeyAPI(self) self.instance = self.root.rpartition("://")[2] - self.renotes = self.config("renotes", False) - self.replies = self.config("replies", True) + self.renotes = True if self.config("renotes", False) else False + self.replies = True if self.config("replies", True) else False def items(self): for note in self.notes(): @@ -254,6 +254,8 @@ class MisskeyAPI(): def _pagination(self, endpoint, data): data["limit"] = 100 + data["withRenotes"] = self.extractor.renotes + while True: notes = self._call(endpoint, data) if not notes: diff --git a/gallery_dl/extractor/nozomi.py b/gallery_dl/extractor/nozomi.py index 21c361c..528aff2 100644 --- a/gallery_dl/extractor/nozomi.py +++ b/gallery_dl/extractor/nozomi.py @@ -173,7 +173,7 @@ class NozomiSearchExtractor(NozomiExtractor): for tag in self.tags: (negative if tag[0] == "-" else positive).append( - tag.replace("/", "")) + text.quote(tag.replace("/", ""))) for tag in positive: ids = nozomi("nozomi/" + tag) diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py index 5245f31..490243a 100644 --- a/gallery_dl/extractor/paheal.py +++ b/gallery_dl/extractor/paheal.py @@ -9,7 +9,7 @@ """Extractors for https://rule34.paheal.net/""" from .common import Extractor, Message -from .. import text +from .. import text, exception class PahealExtractor(Extractor): @@ -97,7 +97,12 @@ class PahealTagExtractor(PahealExtractor): base = f"{self.root}/post/list/{self.groups[0]}/" while True: - page = self.request(base + str(pnum)).text + try: + page = self.request(f"{base}{pnum}").text + except exception.HttpError as exc: + if exc.status == 404: + return + raise pos = page.find("id='image-list'") for post in text.extract_iter( @@ -146,4 +151,9 @@ class PahealPostExtractor(PahealExtractor): example = "https://rule34.paheal.net/post/view/12345" def get_posts(self): - return (self._extract_post(self.groups[0]),) + try: + return (self._extract_post(self.groups[0]),) + except exception.HttpError as exc: + if exc.status == 404: + return () + raise diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index fb2f32c..cf1a6d6 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -230,6 +230,16 @@ class PatreonExtractor(Extractor): attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z") return attr + def _collection(self, collection_id): + url = f"{self.root}/api/collection/{collection_id}" + data = self.request_json(url) + coll = data["data"] + attr = coll["attributes"] + attr["id"] = coll["id"] + attr["date"] = text.parse_datetime( + attr["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + return attr + def _filename(self, url): """Fetch filename from an URL's Content-Disposition header""" response = self.request(url, method="HEAD", fatal=False) @@ -333,6 +343,33 @@ class PatreonExtractor(Extractor): raise exception.AbortExtraction("Unable to extract bootstrap data") +class PatreonCollectionExtractor(PatreonExtractor): + """Extractor for a patreon collection""" + subcategory = "collection" + directory_fmt = ("{category}", "{creator[full_name]}", + "Collections", "{collection[title]} ({collection[id]})") + pattern = r"(?:https?://)?(?:www\.)?patreon\.com/collection/(\d+)" + example = "https://www.patreon.com/collection/12345" + + def posts(self): + collection_id = self.groups[0] + self.kwdict["collection"] = collection = \ + self._collection(collection_id) + campaign_id = text.extr( + collection["thumbnail"]["url"], "/campaign/", "/") + + url = self._build_url("posts", ( + # patreon returns '400 Bad Request' without campaign_id filter + f"&filter[campaign_id]={campaign_id}" + "&filter[contains_exclusive_posts]=true" + "&filter[is_draft]=false" + f"&filter[collection_id]={collection_id}" + "&filter[include_drops]=true" + "&sort=collection_order" + )) + return self._pagination(url) + + class PatreonCreatorExtractor(PatreonExtractor): """Extractor for a creator's works""" subcategory = "creator" diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index a72042c..6276a2a 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -1232,7 +1232,7 @@ class PixivAppAPI(): params = {"word": word, "search_target": target, "sort": sort, "duration": duration, "start_date": date_start, "end_date": date_end} - return self._pagination("/v1/search/illust", params) + return self._pagination_search("/v1/search/illust", params) def user_bookmarks_illust(self, user_id, tag=None, restrict="public"): """Return illusts bookmarked by a user""" @@ -1322,6 +1322,48 @@ class PixivAppAPI(): params = text.parse_query(query) data = self._call(endpoint, params) + def _pagination_search(self, endpoint, params): + sort = params["sort"] + if sort == "date_desc": + date_key = "end_date" + date_off = timedelta(days=1) + date_cmp = lambda lhs, rhs: lhs >= rhs # noqa E731 + elif sort == "date_asc": + date_key = "start_date" + date_off = timedelta(days=-1) + date_cmp = lambda lhs, rhs: lhs <= rhs # noqa E731 + else: + date_key = None + date_last = None + + while True: + data = self._call(endpoint, params) + + if date_last is None: + yield from data["illusts"] + else: + works = data["illusts"] + if date_cmp(date_last, works[-1]["create_date"]): + for work in works: + if date_last is None: + yield work + elif date_cmp(date_last, work["create_date"]): + date_last = None + + if not (next_url := data.get("next_url")): + return + query = next_url.rpartition("?")[2] + params = text.parse_query(query) + + if date_key and text.parse_int(params.get("offset")) >= 5000: + date_last = data["illusts"][-1]["create_date"] + date_val = (text.parse_datetime( + date_last) + date_off).strftime("%Y-%m-%d") + self.log.info("Reached 'offset' >= 5000; " + "Updating '%s' to '%s'", date_key, date_val) + params[date_key] = date_val + params.pop("offset", None) + @cache(maxage=36500*86400, keyarg=0) def _refresh_token_cache(username): diff --git a/gallery_dl/extractor/s3ndpics.py b/gallery_dl/extractor/s3ndpics.py new file mode 100644 index 0000000..215f160 --- /dev/null +++ b/gallery_dl/extractor/s3ndpics.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://s3nd.pics/""" + +from .common import Extractor, Message +from .. import text + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?s3nd\.pics" + + +class S3ndpicsExtractor(Extractor): + """Base class for s3ndpics extractors""" + category = "s3ndpics" + root = "https://s3nd.pics" + root_api = f"{root}/api" + directory_fmt = ("{category}", "{user[username]}", + "{date} {title:?/ /}({id})") + filename_fmt = "{num:>02}.{extension}" + archive_fmt = "{id}_{num}" + + def items(self): + base = "https://s3.s3nd.pics/s3nd-pics/" + + for post in self.posts(): + post["id"] = post.pop("_id", None) + post["user"] = post.pop("userId", None) + post["date"] = text.parse_datetime( + post["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date_updated"] = text.parse_datetime( + post["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + + files = post.pop("files", ()) + post["count"] = len(files) + + yield Message.Directory, post + for post["num"], file in enumerate(files, 1): + post["type"] = file["type"] + path = file["url"] + text.nameext_from_url(path, post) + yield Message.Url, f"{base}{path}", post + + def _pagination(self, url, params): + params["page"] = 1 + + while True: + data = self.request_json(url, params=params) + + self.kwdict["total"] = data["pagination"]["total"] + yield from data["posts"] + + if params["page"] >= data["pagination"]["pages"]: + return + params["page"] += 1 + + +class S3ndpicsPostExtractor(S3ndpicsExtractor): + subcategory = "post" + pattern = rf"{BASE_PATTERN}/post/([0-9a-f]+)" + example = "https://s3nd.pics/post/0123456789abcdef01234567" + + def posts(self): + url = f"{self.root_api}/posts/{self.groups[0]}" + return (self.request_json(url)["post"],) + + +class S3ndpicsUserExtractor(S3ndpicsExtractor): + subcategory = "user" + pattern = rf"{BASE_PATTERN}/user/(\w+)" + example = "https://s3nd.pics/user/USER" + + def posts(self): + url = f"{self.root_api}/users/username/{self.groups[0]}" + self.kwdict["user"] = user = self.request_json(url)["user"] + + url = f"{self.root_api}/posts" + params = { + "userId": user["_id"], + "limit" : "12", + "sortBy": "newest", + } + return self._pagination(url, params) + + +class S3ndpicsSearchExtractor(S3ndpicsExtractor): + subcategory = "search" + pattern = rf"{BASE_PATTERN}/search/?\?([^#]+)" + example = "https://s3nd.pics/search?QUERY" + + def posts(self): + url = f"{self.root_api}/posts" + params = text.parse_query(self.groups[0]) + params.setdefault("limit", "20") + self.kwdict["search_tags"] = \ + params.get("tag") or params.get("tags") or params.get("q") + return self._pagination(url, params) diff --git a/gallery_dl/extractor/schalenetwork.py b/gallery_dl/extractor/schalenetwork.py index dc42417..a4ef3b0 100644 --- a/gallery_dl/extractor/schalenetwork.py +++ b/gallery_dl/extractor/schalenetwork.py @@ -62,10 +62,11 @@ class SchalenetworkExtractor(Extractor): pass params["page"] += 1 - def _token(self): + def _token(self, required=True): if token := self.config("token"): return f"Bearer {token.rpartition(' ')[2]}" - raise exception.AuthRequired("'token'", "your favorites") + if required: + raise exception.AuthRequired("'token'", "your favorites") def _crt(self): crt = self.config("crt") @@ -88,7 +89,7 @@ class SchalenetworkExtractor(Extractor): else: msg = f"{exc.status} {exc.response.reason}" raise exception.AuthRequired( - "'crt' query parameter & matching '--user-agent'", None, msg) + "'crt' query parameter & matching 'user-agent'", None, msg) class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor): @@ -114,19 +115,26 @@ class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor): 10: "mixed", 11: "language", 12: "other", + 13: "reclass", } def metadata(self, _): _, gid, gkey = self.groups + url = f"{self.root_api}/books/detail/{gid}/{gkey}" - data = self.request_json(url, headers=self.headers) - data["date"] = text.parse_timestamp(data["created_at"] // 1000) + headers = self.headers + data = self.request_json(url, headers=headers) + + try: + data["date"] = text.parse_timestamp(data["created_at"] // 1000) + data["count"] = len(data["thumbnails"]["entries"]) + del data["thumbnails"] + except Exception: + pass tags = [] types = self.TAG_TYPES - tags_data = data["tags"] - - for tag in tags_data: + for tag in data["tags"]: name = tag["name"] namespace = tag.get("namespace", 0) tags.append(types[namespace] + ":" + name) @@ -134,33 +142,34 @@ class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor): if self.config("tags", False): tags = collections.defaultdict(list) - for tag in tags_data : + for tag in data["tags"]: tags[tag.get("namespace", 0)].append(tag["name"]) for type, values in tags.items(): data["tags_" + types[type]] = values + url = f"{self.root_api}/books/detail/{gid}/{gkey}?crt={self._crt()}" + if token := self._token(False): + headers = headers.copy() + headers["Authorization"] = token try: - data["count"] = len(data["thumbnails"]["entries"]) - del data["thumbnails"] - except Exception: - pass + data_fmt = self.request_json( + url, method="POST", headers=headers) + except exception.HttpError as exc: + self._require_auth(exc) + + self.fmt = self._select_format(data_fmt["data"]) + data["source"] = data_fmt.get("source") return data def images(self, _): - crt = self._crt() _, gid, gkey = self.groups - url = f"{self.root_api}/books/detail/{gid}/{gkey}?crt={crt}" - try: - data = self.request_json(url, method="POST", headers=self.headers) - except exception.HttpError as exc: - self._require_auth(exc) - - fmt = self._select_format(data["data"]) + fmt = self.fmt url = (f"{self.root_api}/books/data/{gid}/{gkey}" - f"/{fmt['id']}/{fmt['key']}/{fmt['w']}?crt={crt}") - data = self.request_json(url, headers=self.headers) + f"/{fmt['id']}/{fmt['key']}/{fmt['w']}?crt={self._crt()}") + headers = self.headers + data = self.request_json(url, headers=headers) base = data["base"] results = [] @@ -169,7 +178,7 @@ class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor): info = { "width" : dimensions[0], "height": dimensions[1], - "_http_headers": self.headers, + "_http_headers": headers, } results.append((base + entry["path"], info)) return results diff --git a/gallery_dl/extractor/simpcity.py b/gallery_dl/extractor/simpcity.py index 3354289..d8227fa 100644 --- a/gallery_dl/extractor/simpcity.py +++ b/gallery_dl/extractor/simpcity.py @@ -92,7 +92,7 @@ class SimpcityExtractor(Extractor): author = schema["author"] stats = schema["interactionStatistic"] url_t = schema["url"] - url_a = author["url"] + url_a = author.get("url") or "" thread = { "id" : url_t[url_t.rfind(".")+1:-1], @@ -104,8 +104,9 @@ class SimpcityExtractor(Extractor): "tags" : (schema["keywords"].split(", ") if "keywords" in schema else ()), "section" : schema["articleSection"], - "author" : author["name"], - "author_id" : url_a[url_a.rfind(".")+1:-1], + "author" : author.get("name") or "", + "author_id" : (url_a[url_a.rfind(".")+1:-1] if url_a else + (author.get("name") or "")[15:]), "author_url": url_a, } diff --git a/gallery_dl/extractor/thehentaiworld.py b/gallery_dl/extractor/thehentaiworld.py index 055d7d8..9a30654 100644 --- a/gallery_dl/extractor/thehentaiworld.py +++ b/gallery_dl/extractor/thehentaiworld.py @@ -60,14 +60,16 @@ class ThehentaiworldExtractor(Extractor): "<li>Posted: ", "<"), "%Y-%m-%d"), } - if "/videos/" in url: + if (c := url[27]) == "v": post["type"] = "video" post["width"] = post["height"] = 0 post["votes"] = text.parse_int(extr("(<strong>", "</strong>")) post["score"] = text.parse_float(extr("<strong>", "<")) post["file_url"] = extr('<source src="', '"') else: - post["type"] = "image" + post["type"] = ("animated" if c == "g" else + "3d cgi" if c == "3" else + "image") post["width"] = text.parse_int(extr("<li>Size: ", " ")) post["height"] = text.parse_int(extr("x ", "<")) post["file_url"] = extr('a href="', '"') @@ -109,16 +111,6 @@ class ThehentaiworldExtractor(Extractor): pnum += 1 -class ThehentaiworldPostExtractor(ThehentaiworldExtractor): - subcategory = "post" - pattern = (rf"{BASE_PATTERN}" - rf"(/(?:(?:3d-cgi-)?hentai-image|video)s/([^/?#]+))") - example = "https://thehentaiworld.com/hentai-images/SLUG/" - - def posts(self): - return (f"{self.root}{self.groups[0]}/",) - - class ThehentaiworldTagExtractor(ThehentaiworldExtractor): subcategory = "tag" per_page = 24 @@ -137,3 +129,13 @@ class ThehentaiworldTagExtractor(ThehentaiworldExtractor): self.page_start += pages self.post_start += posts return num + + +class ThehentaiworldPostExtractor(ThehentaiworldExtractor): + subcategory = "post" + pattern = (rf"{BASE_PATTERN}(" + rf"/(?:video|(?:[\w-]+-)?hentai-image)s/([^/?#]+))") + example = "https://thehentaiworld.com/hentai-images/SLUG/" + + def posts(self): + return (f"{self.root}{self.groups[0]}/",) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index e6c84d1..e7df4a3 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1026,11 +1026,12 @@ class TwitterTweetExtractor(TwitterExtractor): return while True: + parent_id = tweet["rest_id"] tweet_id = tweet["legacy"].get("quoted_status_id_str") if not tweet_id: break tweet = self.api.tweet_result_by_rest_id(tweet_id) - tweet["legacy"]["quoted_by_id_str"] = tweet_id + tweet["legacy"]["quoted_by_id_str"] = parent_id yield tweet def _tweets_detail(self, tweet_id): diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 823e8e0..07bed79 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -86,16 +86,25 @@ class WeiboExtractor(Extractor): status["count"] = len(files) yield Message.Directory, status - for num, file in enumerate(files, 1): - if file["url"].startswith("http:"): - file["url"] = "https:" + file["url"][5:] + num = 0 + for file in files: + url = file["url"] + if not url: + continue + if url.startswith("http:"): + url = f"https:{url[5:]}" if "filename" not in file: - text.nameext_from_url(file["url"], file) + text.nameext_from_url(url, file) if file["extension"] == "json": file["extension"] = "mp4" + if file["extension"] == "m3u8": + url = f"ytdl:{url}" + file["_ytdl_manifest"] = "hls" + file["extension"] = "mp4" + num += 1 file["status"] = status file["num"] = num - yield Message.Url, file["url"], file + yield Message.Url, url, file def _extract_status(self, status, files): if "mix_media_info" in status: @@ -143,10 +152,21 @@ class WeiboExtractor(Extractor): media = max(info["playback_list"], key=lambda m: m["meta"]["quality_index"]) except Exception: - return {"url": (info.get("stream_url_hd") or - info.get("stream_url") or "")} + video = {"url": (info.get("replay_hd") or + info.get("stream_url_hd") or + info.get("stream_url") or "")} else: - return media["play_info"].copy() + video = media["play_info"].copy() + + if "//wblive-out." in video["url"] and \ + not text.ext_from_url(video["url"]): + try: + video["url"] = self.request_location(video["url"]) + except exception.HttpError as exc: + self.log.warning("%s: %s", exc.__class__.__name__, exc) + video["url"] = "" + + return video def _status_by_id(self, status_id): url = f"{self.root}/ajax/statuses/show?id={status_id}" diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py index 00266bd..5ba47d2 100644 --- a/gallery_dl/extractor/wikimedia.py +++ b/gallery_dl/extractor/wikimedia.py @@ -46,6 +46,12 @@ class WikimediaExtractor(BaseExtractor): else: self.api_url = None + # note: image revisions are different from page revisions + # ref: + # https://www.mediawiki.org/wiki/API:Revisions + # https://www.mediawiki.org/wiki/API:Imageinfo + self.image_revisions = self.config("image-revisions", 1) + @cache(maxage=36500*86400, keyarg=1) def _search_api_path(self, root): self.log.debug("Probing possible API endpoints") @@ -56,7 +62,10 @@ class WikimediaExtractor(BaseExtractor): return url raise exception.AbortExtraction("Unable to find API endpoint") - def prepare(self, image): + def prepare_info(self, info): + """Adjust the content of an image info object""" + + def prepare_image(self, image): """Adjust the content of an image object""" image["metadata"] = { m["name"]: m["value"] @@ -74,14 +83,19 @@ class WikimediaExtractor(BaseExtractor): def items(self): for info in self._pagination(self.params): try: - image = info["imageinfo"][0] - except LookupError: + images = info.pop("imageinfo") + except KeyError: self.log.debug("Missing 'imageinfo' for %s", info) - continue + images = () + + info["count"] = len(images) + self.prepare_info(info) + yield Message.Directory, info - self.prepare(image) - yield Message.Directory, image - yield Message.Url, image["url"], image + for info["num"], image in enumerate(images, 1): + self.prepare_image(image) + image.update(info) + yield Message.Url, image["url"], image if self.subcategories: base = self.root + "/wiki/" @@ -108,6 +122,7 @@ class WikimediaExtractor(BaseExtractor): "timestamp|user|userid|comment|canonicaltitle|url|size|" "sha1|mime|metadata|commonmetadata|extmetadata|bitdepth" ) + params["iilimit"] = self.image_revisions while True: data = self.request_json(url, params=params) @@ -237,9 +252,8 @@ class WikimediaArticleExtractor(WikimediaExtractor): "titles" : path, } - def prepare(self, image): - WikimediaExtractor.prepare(self, image) - image["page"] = self.title + def prepare_info(self, info): + info["page"] = self.title class WikimediaWikiExtractor(WikimediaExtractor): diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index e1b4897..98c9331 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -26,6 +26,7 @@ class ZerochanExtractor(BooruExtractor): per_page = 250 cookies_domain = ".zerochan.net" cookies_names = ("z_id", "z_hash") + useragent = util.USERAGENT request_interval = (0.5, 1.5) def login(self): @@ -192,7 +193,13 @@ class ZerochanTagExtractor(ZerochanExtractor): metadata = self.config("metadata") while True: - page = self.request(url, params=params, expected=(500,)).text + try: + page = self.request( + url, params=params, expected=(500,)).text + except exception.HttpError as exc: + if exc.status == 404: + return + raise thumbs = text.extr(page, '<ul id="thumbs', '</ul>') extr = text.extract_from(thumbs) @@ -231,7 +238,13 @@ class ZerochanTagExtractor(ZerochanExtractor): } while True: - response = self.request(url, params=params, allow_redirects=False) + try: + response = self.request( + url, params=params, allow_redirects=False) + except exception.HttpError as exc: + if exc.status == 404: + return + raise if response.status_code >= 300: url = text.urljoin(self.root, response.headers["location"]) @@ -275,12 +288,18 @@ class ZerochanImageExtractor(ZerochanExtractor): pattern = BASE_PATTERN + r"/(\d+)" example = "https://www.zerochan.net/12345" - def __init__(self, match): - ZerochanExtractor.__init__(self, match) - self.image_id = match[1] - def posts(self): - post = self._parse_entry_html(self.image_id) + image_id = self.groups[0] + + try: + post = self._parse_entry_html(image_id) + except exception.HttpError as exc: + if exc.status in (404, 410): + if msg := text.extr(exc.response.text, "<h2>", "<"): + self.log.warning(f"'{msg}'") + return () + raise + if self.config("metadata"): - post.update(self._parse_entry_api(self.image_id)) + post.update(self._parse_entry_api(image_id)) return (post,) diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py index 9992c56..82ad388 100644 --- a/gallery_dl/postprocessor/common.py +++ b/gallery_dl/postprocessor/common.py @@ -21,7 +21,7 @@ class PostProcessor(): def __repr__(self): return self.__class__.__name__ - def _init_archive(self, job, options, prefix=None): + def _archive_init(self, job, options, prefix=None): if archive_path := options.get("archive"): extr = job.extractor @@ -54,11 +54,13 @@ class PostProcessor(): else: self.log.debug( "Using %s archive '%s'", self.name, archive_path) - job.register_hooks({"finalize": self._close_archive}) return True self.archive = None return False - def _close_archive(self, _): + def _archive_register(self, job): + job.register_hooks({"finalize": self._archive_close}) + + def _archive_close(self, _): self.archive.close() diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py index 0bfe1a2..ef11bff 100644 --- a/gallery_dl/postprocessor/exec.py +++ b/gallery_dl/postprocessor/exec.py @@ -50,7 +50,8 @@ class ExecPP(PostProcessor): events = events.split(",") job.register_hooks({event: execute for event in events}, options) - self._init_archive(job, options) + if self._archive_init(job, options): + self._archive_register(job) def _prepare_cmd(self, cmd): if isinstance(cmd, str): diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index a6d2b7f..90e6e3d 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -110,7 +110,9 @@ class MetadataPP(PostProcessor): events = events.split(",") job.register_hooks({event: self.run for event in events}, options) - self._init_archive(job, options, "_MD_") + if self._archive_init(job, options, "_MD_"): + self._archive_register(job) + self.filter = self._make_filter(options) self.mtime = options.get("mtime") self.omode = options.get("open", omode) diff --git a/gallery_dl/postprocessor/python.py b/gallery_dl/postprocessor/python.py index 66d9343..5011ce8 100644 --- a/gallery_dl/postprocessor/python.py +++ b/gallery_dl/postprocessor/python.py @@ -26,6 +26,9 @@ class PythonPP(PostProcessor): module = util.import_file(module_name) self.function = getattr(module, function_name) + if archive := self._archive_init(job, options): + self.run = self.run_archive + events = options.get("event") if events is None: events = ("file",) @@ -33,8 +36,8 @@ class PythonPP(PostProcessor): events = events.split(",") job.register_hooks({event: self.run for event in events}, options) - if self._init_archive(job, options): - self.run = self.run_archive + if archive: + self._archive_register(job) def run(self, pathfmt): self.function(pathfmt.kwdict) diff --git a/gallery_dl/version.py b/gallery_dl/version.py index 4861a9d..d3e0277 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,5 +6,5 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.30.8" +__version__ = "1.30.9" __variant__ = None diff --git a/test/test_downloader.py b/test/test_downloader.py index ecd8b85..f6c3dbe 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -386,6 +386,8 @@ SAMPLES = { ("mp3" , b"\xFF\xFB"), ("mp3" , b"\xFF\xF3"), ("mp3" , b"\xFF\xF2"), + ("m3u8", b"#EXTM3U\n#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=200000"), + ("mpd" , b'<MPD xmlns="urn:mpeg:dash:schema:mpd:2011"'), ("zip" , b"PK\x03\x04"), ("zip" , b"PK\x05\x06"), ("zip" , b"PK\x07\x08"), diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 2902fea..17b36b6 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -21,7 +21,7 @@ from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import extractor, output, path, util, exception # noqa E402 -from gallery_dl import postprocessor, config # noqa E402 +from gallery_dl import postprocessor, config, archive # noqa E402 from gallery_dl.postprocessor.common import PostProcessor # noqa E402 @@ -39,7 +39,7 @@ class FakeJob(): self.get_logger = logging.getLogger self.hooks = collections.defaultdict(list) - def register_hooks(self, hooks, options): + def register_hooks(self, hooks, options=None): for hook, callback in hooks.items(): self.hooks[hook].append(callback) @@ -353,6 +353,23 @@ class ExecTest(BasePostprocessorTest): ) i.wait.assert_called_once_with() + def test_archive(self): + pp = self._create({ + "command": ["echo", "foobar"], + "archive": ":memory:", + "event" : "finalize", + }) + + self.assertIsInstance(pp.archive, archive.DownloadArchive) + + with patch.object(pp.archive, "add") as m_aa, \ + patch.object(pp.archive, "close") as m_ac: + self._trigger(("finalize",)) + pp.archive.close() + + m_aa.assert_called_once_with(self.pathfmt.kwdict) + m_ac.assert_called_once() + class HashTest(BasePostprocessorTest): @@ -811,6 +828,22 @@ class MetadataTest(BasePostprocessorTest): } """) + def test_archive(self): + pp = self._create({ + "archive": ":memory:", + "event" : "finalize", + }) + + self.assertIsInstance(pp.archive, archive.DownloadArchive) + + with patch.object(pp.archive, "add") as m_aa, \ + patch.object(pp.archive, "close") as m_ac: + self._trigger(("finalize",)) + pp.archive.close() + + m_aa.assert_called_once_with(self.pathfmt.kwdict) + m_ac.assert_called_once() + def _output(self, mock): return "".join( call[1][0] @@ -890,6 +923,23 @@ class PythonTest(BasePostprocessorTest): with self.assertRaises(exception.StopExtraction): self._trigger() + def test_archive(self): + pp = self._create({ + "expression": "True", + "archive" : ":memory:", + "event" : "finalize", + }) + + self.assertIsInstance(pp.archive, archive.DownloadArchive) + + with patch.object(pp.archive, "add") as m_aa, \ + patch.object(pp.archive, "close") as m_ac: + self._trigger(("finalize",)) + pp.archive.close() + + m_aa.assert_called_once_with(self.pathfmt.kwdict) + m_ac.assert_called_once() + def _write_module(self, path): with open(path, "w") as fp: fp.write(""" diff --git a/test/test_results.py b/test/test_results.py index 7e024b8..2e2eaa9 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -348,6 +348,17 @@ class TestExtractorResults(unittest.TestCase): for _ in value: len_value += 1 self.assertEqual(int(length), len_value, msg=path) + elif test.startswith("hash:"): + digest = test[5:].lower() + msg = f"{path} / {digest}" + if digest == "md5": + self.assertRegex(value, r"^[0-9a-fA-F]{32}$", msg) + elif digest == "sha1": + self.assertRegex(value, r"^[0-9a-fA-F]{40}$", msg) + elif digest == "sha256": + self.assertRegex(value, r"^[0-9a-fA-F]{64}$", msg) + elif digest == "sha512": + self.assertRegex(value, r"^[0-9a-fA-F]{128}$", msg) elif test.startswith("iso:"): iso = test[4:] if iso in ("dt", "datetime", "8601"): |
