summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLibravatarUnit 193 <unit193@unit193.net>2025-10-07 02:11:45 -0400
committerLibravatarUnit 193 <unit193@unit193.net>2025-10-07 02:11:45 -0400
commitbbe7fac03d881662a458e7fbf870c9d71f5257f4 (patch)
treeb90b8974242d7fcb381e43c69c215c97c2e99197
parent42b62671fabfdcf983a9575221420d85f7fbcac1 (diff)
New upstream version 1.30.9.upstream/1.30.9
-rw-r--r--CHANGELOG.md76
-rw-r--r--PKG-INFO6
-rw-r--r--README.rst4
-rw-r--r--data/man/gallery-dl.12
-rw-r--r--data/man/gallery-dl.conf.52089
-rw-r--r--docs/gallery-dl.conf11
-rw-r--r--gallery_dl.egg-info/PKG-INFO6
-rw-r--r--gallery_dl.egg-info/SOURCES.txt3
-rw-r--r--gallery_dl/cookies.py49
-rw-r--r--gallery_dl/downloader/http.py8
-rw-r--r--gallery_dl/extractor/__init__.py3
-rw-r--r--gallery_dl/extractor/chevereto.py20
-rw-r--r--gallery_dl/extractor/imagehosts.py14
-rw-r--r--gallery_dl/extractor/instagram.py75
-rw-r--r--gallery_dl/extractor/mangadex.py119
-rw-r--r--gallery_dl/extractor/mangafire.py168
-rw-r--r--gallery_dl/extractor/mangareader.py173
-rw-r--r--gallery_dl/extractor/misskey.py6
-rw-r--r--gallery_dl/extractor/nozomi.py2
-rw-r--r--gallery_dl/extractor/paheal.py16
-rw-r--r--gallery_dl/extractor/patreon.py37
-rw-r--r--gallery_dl/extractor/pixiv.py44
-rw-r--r--gallery_dl/extractor/s3ndpics.py101
-rw-r--r--gallery_dl/extractor/schalenetwork.py57
-rw-r--r--gallery_dl/extractor/simpcity.py7
-rw-r--r--gallery_dl/extractor/thehentaiworld.py26
-rw-r--r--gallery_dl/extractor/twitter.py3
-rw-r--r--gallery_dl/extractor/weibo.py36
-rw-r--r--gallery_dl/extractor/wikimedia.py34
-rw-r--r--gallery_dl/extractor/zerochan.py35
-rw-r--r--gallery_dl/postprocessor/common.py8
-rw-r--r--gallery_dl/postprocessor/exec.py3
-rw-r--r--gallery_dl/postprocessor/metadata.py4
-rw-r--r--gallery_dl/postprocessor/python.py7
-rw-r--r--gallery_dl/version.py2
-rw-r--r--test/test_downloader.py2
-rw-r--r--test/test_postprocessor.py54
-rw-r--r--test/test_results.py11
38 files changed, 2210 insertions, 1111 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5aa64b9..1df5fff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,48 +1,44 @@
-## 1.30.8 - 2025-09-23
+## 1.30.9 - 2025-10-03
### Extractors
#### Additions
-- [chevereto] support `imglike.com` ([#5179](https://github.com/mikf/gallery-dl/issues/5179))
-- [chevereto] add `category` extractor ([#5179](https://github.com/mikf/gallery-dl/issues/5179))
-- [Danbooru] add `random` extractor ([#8270](https://github.com/mikf/gallery-dl/issues/8270))
-- [hdoujin] add support ([#6810](https://github.com/mikf/gallery-dl/issues/6810))
-- [imgpile] add support ([#5044](https://github.com/mikf/gallery-dl/issues/5044))
-- [mangadex] add `covers` extractor ([#4994](https://github.com/mikf/gallery-dl/issues/4994))
-- [mangataro] add support ([#8237](https://github.com/mikf/gallery-dl/issues/8237))
-- [thehentaiworld] add support ([#274](https://github.com/mikf/gallery-dl/issues/274) [#8237](https://github.com/mikf/gallery-dl/issues/8237))
+- [mangafire] add support ([#7064](https://github.com/mikf/gallery-dl/issues/7064) [#7701](https://github.com/mikf/gallery-dl/issues/7701))
+- [mangareader] add support ([#6529](https://github.com/mikf/gallery-dl/issues/6529) [#6868](https://github.com/mikf/gallery-dl/issues/6868))
+- [patreon] add `collection` extractor ([#8286](https://github.com/mikf/gallery-dl/issues/8286))
+- [s3ndpics] add support ([#8322](https://github.com/mikf/gallery-dl/issues/8322))
#### Fixes
-- [4archive] fix `TypeError` ([#8217](https://github.com/mikf/gallery-dl/issues/8217))
-- [bellazon] fix video attachments ([#8239](https://github.com/mikf/gallery-dl/issues/8239))
-- [bunkr] fix `JSONDecodeError` for files with URL slugs containing apostrophes `'` ([#8150](https://github.com/mikf/gallery-dl/issues/8150))
-- [instagram] ensure manifest data exists before attempting a DASH download ([#8267](https://github.com/mikf/gallery-dl/issues/8267))
-- [schalenetwork] fix extraction ([#6948](https://github.com/mikf/gallery-dl/issues/6948) [#7391](https://github.com/mikf/gallery-dl/issues/7391) [#7728](https://github.com/mikf/gallery-dl/issues/7728))
-- [twitter] fix quoted Tweets being marked as `deleted` ([#8225](https://github.com/mikf/gallery-dl/issues/8225))
+- [chevereto] fix `id` for links without file name ([#8307](https://github.com/mikf/gallery-dl/issues/8307))
+- [chevereto:album] fix video downloads ([#8149](https://github.com/mikf/gallery-dl/issues/8149) [#8295](https://github.com/mikf/gallery-dl/issues/8295))
+- [hdoujin] fix `KeyError: 13` by adding `reclass` tag type ([#8290](https://github.com/mikf/gallery-dl/issues/8290))
+- [misskey] include `withRenotes` parameter in API requests ([#8285](https://github.com/mikf/gallery-dl/issues/8285))
+- [nozomi] percent-encode search tags ([#8328](https://github.com/mikf/gallery-dl/issues/8328))
+- [simpcity] fix `KeyError: 'url'` when thread author is deleted ([#8323](https://github.com/mikf/gallery-dl/issues/8323))
+- [twitter] fix `quote_id` of individual Tweets ([#8284](https://github.com/mikf/gallery-dl/issues/8284))
+- [zerochan] prevent `HttpError: '503 Service Temporarily Unavailable'` ([#8288](https://github.com/mikf/gallery-dl/issues/8288))
#### Improvements
-- [2ch] update domain to `2ch.su`, support `2ch.life` URLs ([#8216](https://github.com/mikf/gallery-dl/issues/8216))
-- [bellazon][simpcity][vipergirls] process threads in descending order ([#8248](https://github.com/mikf/gallery-dl/issues/8248))
-- [bellazon] extract `inline` images (##8247)
-- [bellazon] support video embeds ([#8239](https://github.com/mikf/gallery-dl/issues/8239))
-- [bellazon] support `#comment-12345` post links ([#8239](https://github.com/mikf/gallery-dl/issues/8239))
-- [lensdump] support new direct file URL pattern ([#8251](https://github.com/mikf/gallery-dl/issues/8251))
-- [simpcity] extract URLs of `<iframe>` embeds ([#8214](https://github.com/mikf/gallery-dl/issues/8214) [#8256](https://github.com/mikf/gallery-dl/issues/8256))
-- [simpcity] improve post content extraction ([#8214](https://github.com/mikf/gallery-dl/issues/8214))
+- [chevereto] support URLs with `www` subdomain ([#8149](https://github.com/mikf/gallery-dl/issues/8149))
+- [imxto:gallery] support multiple pages ([#8282](https://github.com/mikf/gallery-dl/issues/8282))
+- [instagram] add `warn-images` & `warn-videos` options ([#8283](https://github.com/mikf/gallery-dl/issues/8283))
+- [instagram] use `reel` subcategory for `/reel/SHORTCODE` URLs ([#8274](https://github.com/mikf/gallery-dl/issues/8274))
+- [instagram] support `/reels/SHORTCODE` URLs ([#8318](https://github.com/mikf/gallery-dl/issues/8318))
+- [paheal] normalize `No results` output message ([#8313](https://github.com/mikf/gallery-dl/issues/8313))
+- [pixiv] implement searching past 5000 results ([#1686](https://github.com/mikf/gallery-dl/issues/1686) [#7082](https://github.com/mikf/gallery-dl/issues/7082) [#8298](https://github.com/mikf/gallery-dl/issues/8298))
+- [thehentaiworld] support more `post` URL formats ([#8277](https://github.com/mikf/gallery-dl/issues/8277))
+- [weibo] download `.m3u8` manifests with ytdl ([#8339](https://github.com/mikf/gallery-dl/issues/8339))
+- [weibo] resolve `wblive-out.api.weibo.com` URLs ([#8339](https://github.com/mikf/gallery-dl/issues/8339))
+- [weibo] use `replay_hd` URLs as video fallback ([#8339](https://github.com/mikf/gallery-dl/issues/8339))
+- [wikimedia] add ability to download image revisions ([#7283](https://github.com/mikf/gallery-dl/issues/7283) [#8330](https://github.com/mikf/gallery-dl/issues/8330))
+- [zerochan] normalize `No results` output message ([#8313](https://github.com/mikf/gallery-dl/issues/8313))
#### Metadata
-- [facebook] extract `biography` metadata ([#8233](https://github.com/mikf/gallery-dl/issues/8233))
-- [instagram:tagged] provide full `tagged_…` metadata when using `id:…` URLs ([#8263](https://github.com/mikf/gallery-dl/issues/8263))
-- [iwara] extract more metadata ([#6582](https://github.com/mikf/gallery-dl/issues/6582))
-- [iwara] make `type` available for directories ([#8245](https://github.com/mikf/gallery-dl/issues/8245))
-- [reddit] provide `comment` metadata for all media files ([#8228](https://github.com/mikf/gallery-dl/issues/8228))
-#### Options
-- [bellazon] add `quoted` option ([#8247](https://github.com/mikf/gallery-dl/issues/8247))
-- [bellazon] implement `order-posts` option ([#8248](https://github.com/mikf/gallery-dl/issues/8248))
-- [kemono:discord] implement `order-posts` option ([#8241](https://github.com/mikf/gallery-dl/issues/8241))
-- [simpcity] implement `order-posts` option ([#8248](https://github.com/mikf/gallery-dl/issues/8248))
-- [vipergirls] implement `order-posts` option ([#8248](https://github.com/mikf/gallery-dl/issues/8248))
+- [hdoujin] extract `source` metadata ([#8280](https://github.com/mikf/gallery-dl/issues/8280))
+- [instagram] provide `type` metadata ([#8274](https://github.com/mikf/gallery-dl/issues/8274))
+- [mangadex] extract more manga-related metadata ([#8325](https://github.com/mikf/gallery-dl/issues/8325))
+#### Removals
+- [chevereto] remove `img.kiwi`
### Downloaders
-- [ytdl] fix errors caused by deprecated options removal
+- [http] add MIME type and signature for m3u8 & mpd files ([#8339](https://github.com/mikf/gallery-dl/issues/8339))
### Post Processors
-- [metadata] add `"mode": "print"` ([#2691](https://github.com/mikf/gallery-dl/issues/2691))
-- [python] add `"mode": "eval"`
-- close archive database connections ([#8243](https://github.com/mikf/gallery-dl/issues/8243))
+- [python] restore `archive` functionality
### Miscellaneous
-- [util] define `__enter__` & `__exit__` methods for `NullResponse` objects ([#8227](https://github.com/mikf/gallery-dl/issues/8227))
-- [util] extend list of ISO 639 language codes
+- [cookies] add support for `Orion` browser ([#8303](https://github.com/mikf/gallery-dl/issues/8303))
+- [docker] include more optional Python dependencies ([#8026](https://github.com/mikf/gallery-dl/issues/8026))
+- [docs] update `configuration.rst` formatting
diff --git a/PKG-INFO b/PKG-INFO
index a339b24..6878d26 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: gallery_dl
-Version: 1.30.8
+Version: 1.30.9
Summary: Command-line program to download image galleries and collections from several image hosting sites
Home-page: https://github.com/mikf/gallery-dl
Download-URL: https://github.com/mikf/gallery-dl/releases/latest
@@ -141,9 +141,9 @@ Standalone Executable
Prebuilt executable files with a Python interpreter and
required Python packages included are available for
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.exe>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.exe>`__
(Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__)
-- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.bin>`__
+- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.bin>`__
Nightly Builds
diff --git a/README.rst b/README.rst
index 554fe51..b2d6118 100644
--- a/README.rst
+++ b/README.rst
@@ -79,9 +79,9 @@ Standalone Executable
Prebuilt executable files with a Python interpreter and
required Python packages included are available for
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.exe>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.exe>`__
(Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__)
-- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.bin>`__
+- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.bin>`__
Nightly Builds
diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1
index 6560c3b..4137501 100644
--- a/data/man/gallery-dl.1
+++ b/data/man/gallery-dl.1
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL" "1" "2025-09-23" "1.30.8" "gallery-dl Manual"
+.TH "GALLERY-DL" "1" "2025-10-03" "1.30.9" "gallery-dl Manual"
.\" disable hyphenation
.nh
diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5
index fbf32bc..b827aeb 100644
--- a/data/man/gallery-dl.conf.5
+++ b/data/man/gallery-dl.conf.5
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL.CONF" "5" "2025-09-23" "1.30.8" "gallery-dl Manual"
+.TH "GALLERY-DL.CONF" "5" "2025-10-03" "1.30.9" "gallery-dl Manual"
.\" disable hyphenation
.nh
.\" disable justification (adjust text to left margin only)
@@ -76,9 +76,9 @@ those as makeshift comments by settings their values to arbitrary strings.
.SS extractor.*.filename
.IP "Type:" 6
.br
-* \f[I]string\f[]
+* \f[I]Format String\f[]
.br
-* \f[I]object\f[] (condition -> \f[I]format string\f[])
+* \f[I]object\f[] (\f[I]Condition\f[] → \f[I]Format String\f[])
.IP "Example:" 4
.. code:: json
@@ -95,12 +95,15 @@ those as makeshift comments by settings their values to arbitrary strings.
.IP "Description:" 4
-A \f[I]format string\f[] to build filenames for downloaded files with.
+A \f[I]Format String\f[] to generate filenames for downloaded files.
-If this is an \f[I]object\f[], it must contain Python expressions mapping to the
-filename format strings to use.
-These expressions are evaluated in the specified order until one evaluates
-to \f[I]True\f[].
+If this is an \f[I]object\f[],
+it must contain \f[I]Conditions\f[] mapping to the
+\f[I]Format String\f[] to use.
+These \f[I]Conditions\f[] are evaluated in the specified order
+until one evaluates to \f[I]True\f[].
+When none match, the \f[I]""\f[] entry or
+the extractor's default filename \f[I]Format String\f[] is used.
The available replacement keys depend on the extractor used. A list
of keys for a specific one can be acquired by calling *gallery-dl*
@@ -128,7 +131,8 @@ image-id
subcategory
image
-Note: Even if the value of the \f[I]extension\f[] key is missing or
+.IP "Note:" 4
+Even if the value of the \f[I]extension\f[] key is missing or
\f[I]None\f[], it will be filled in later when the file download is
starting. This key is therefore always available to provide
a valid filename extension.
@@ -137,9 +141,9 @@ a valid filename extension.
.SS extractor.*.directory
.IP "Type:" 6
.br
-* \f[I]list\f[] of \f[I]strings\f[]
+* \f[I]list\f[] of \f[I]Format Strings\f[]
.br
-* \f[I]object\f[] (condition -> \f[I]format strings\f[])
+* \f[I]object\f[] (\f[I]Condition\f[] → \f[I]Format Strings\f[])
.IP "Example:" 4
.. code:: json
@@ -156,10 +160,11 @@ a valid filename extension.
.IP "Description:" 4
-A list of \f[I]format strings\f[] to build target directory paths with.
+A list of \f[I]Format String(s)\f[] to generate the target directory path.
-If this is an \f[I]object\f[], it must contain Python expressions mapping to the
-list of format strings to use.
+If this is an \f[I]object\f[],
+it must contain \f[I]Conditions\f[] mapping to the
+list of \f[I]Format Strings\f[] to use.
Each individual string in such a list represents a single path
segment, which will be joined together and appended to the
@@ -233,7 +238,7 @@ Share number of skipped downloads between parent and child extractors.
.br
* \f[I]string\f[]
.br
-* \f[I]object\f[] (character -> replacement character(s))
+* \f[I]object\f[] (character → replacement character(s))
.IP "Default:" 9
\f[I]"auto"\f[]
@@ -254,28 +259,39 @@ or character ranges to their replacements
for generated path segment names.
.br
-Special values:
-
-.br
-* \f[I]"auto"\f[]: Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[]
+.IP "Special Values:" 4
+\f[I]"auto"\f[]
+Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[]
depending on the local operating system
+\f[I]"unix"\f[]
+\f[I]"/"\f[]
+\f[I]"windows"\f[]
+\f[I]"\\\\\\\\|/<>:\\"?*"\f[]
+.br
+(https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file)
+.br
+\f[I]"ascii"\f[]
+\f[I]"^0-9A-Za-z_."\f[]
.br
-* \f[I]"unix"\f[]: \f[I]"/"\f[]
+(only ASCII digits, letters, underscores, and dots)
.br
-* \f[I]"windows"\f[]: \f[I]"\\\\\\\\|/<>:\\"?*"\f[]
+\f[I]"ascii+"\f[]
+\f[I]"^0-9@-[\\\\]-{ #-)+-.;=!}~"\f[]
.br
-* \f[I]"ascii"\f[]: \f[I]"^0-9A-Za-z_."\f[] (only ASCII digits, letters, underscores, and dots)
+(all ASCII characters except the ones not allowed by Windows)
.br
-* \f[I]"ascii+"\f[]: \f[I]"^0-9@-[\\\\]-{ #-)+-.;=!}~"\f[] (all ASCII characters except the ones not allowed by Windows)
-Implementation Detail: For \f[I]strings\f[] with length >= 2, this option uses a
+.IP "Implementation Detail:" 4
+For \f[I]strings\f[] with length >= 2, this option uses a
\f[I]Regular Expression Character Set\f[],
meaning that:
.br
-* using a caret \f[I]^\f[] as first character inverts the set
+* Using a caret \f[I]^\f[] as first character inverts the set
+(\f[I]"^..."\f[])
.br
-* character ranges are supported (\f[I]0-9a-z\f[])
+* Character ranges are supported
+(\f[I]"0-9a-z"\f[])
.br
* \f[I]]\f[], \f[I]-\f[], and \f[I]\\\f[] need to be escaped as
\f[I]\\\\]\f[], \f[I]\\\\-\f[], and \f[I]\\\\\\\\\f[] respectively
@@ -304,7 +320,8 @@ The replacement character(s) for
.IP "Description:" 4
Set of characters to remove from generated path names.
-Note: In a string with 2 or more characters, \f[I][]^-\\\f[] need to be
+.IP "Note:" 4
+In a string with 2 or more characters, \f[I][]^-\\\f[] need to be
escaped with backslashes, e.g. \f[I]"\\\\[\\\\]"\f[]
@@ -319,15 +336,14 @@ escaped with backslashes, e.g. \f[I]"\\\\[\\\\]"\f[]
Set of characters to remove from the end of generated path segment names
using \f[I]str.rstrip()\f[]
-Special values:
-
-.br
-* \f[I]"auto"\f[]: Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[]
+.IP "Special Values:" 4
+\f[I]"auto"\f[]
+Use characters from \f[I]"unix"\f[] or \f[I]"windows"\f[]
depending on the local operating system
-.br
-* \f[I]"unix"\f[]: \f[I]""\f[]
-.br
-* \f[I]"windows"\f[]: \f[I]". "\f[]
+\f[I]"unix"\f[]
+\f[I]""\f[]
+\f[I]"windows"\f[]
+\f[I]". "\f[]
.SS extractor.*.path-convert
@@ -360,7 +376,7 @@ prefixed with \f[I]\\\\?\\\f[] to work around the 260 characters path length lim
.SS extractor.*.extension-map
.IP "Type:" 6
-\f[I]object\f[] (extension -> replacement)
+\f[I]object\f[] (extension → replacement)
.IP "Default:" 9
.. code:: json
@@ -445,10 +461,10 @@ filename extension (\f[I]file.1.ext\f[], \f[I]file.2.ext\f[], etc.)
.SS extractor.*.skip-filter
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Condition\f[]
.IP "Description:" 4
-Python expression controlling which skipped files to count towards
+Python \f[I]Expression\f[] controlling which skipped files to count towards
\f[I]"abort"\f[] / \f[I]"terminate"\f[] / \f[I]"exit"\f[].
@@ -492,66 +508,57 @@ response before \f[I]retrying\f[] the request.
\f[I]Duration\f[]
.IP "Default:" 9
-.br
-* \f[I]"0.5-1.5"\f[]
-\f[I]ao3\f[],
-\f[I]arcalive\f[],
-\f[I]booth\f[],
-\f[I]civitai\f[],
-\f[I][Danbooru]\f[],
-\f[I][E621]\f[],
-\f[I][foolfuuka]:search\f[],
-\f[I]hdoujin\f[],
-\f[I]itaku\f[],
-\f[I]newgrounds\f[],
-\f[I][philomena]\f[],
-\f[I]pixiv-novel\f[],
-\f[I]plurk\f[],
-\f[I]poipiku\f[] ,
-\f[I]pornpics\f[],
-\f[I]schalenetwork\f[],
-\f[I]scrolller\f[],
-\f[I]sizebooru\f[],
-\f[I]soundgasm\f[],
-\f[I]thehentaiworld\f[],
-\f[I]urlgalleries\f[],
-\f[I]vk\f[],
-\f[I]webtoons\f[],
-\f[I]weebcentral\f[],
-\f[I]xfolio\f[],
+\f[I]"0.5-1.5"\f[]
+\f[I]ao3\f[] \f[I]
+\f[I]arcalive\f[] \f[]
+\f[I]booth\f[] \f[I]
+\f[I]civitai\f[] \f[]
+\f[I][Danbooru]\f[] \f[I]
+\f[I][E621]\f[] \f[]
+\f[I][foolfuuka]:search\f[] \f[I]
+\f[I]hdoujin\f[] \f[]
+\f[I]itaku\f[] \f[I]
+\f[I]newgrounds\f[] \f[]
+\f[I][philomena]\f[] \f[I]
+\f[I]pixiv-novel\f[] \f[]
+\f[I]plurk\f[] \f[I]
+\f[I]poipiku\f[] \f[]
+\f[I]pornpics\f[] \f[I]
+\f[I]schalenetwork\f[] \f[]
+\f[I]scrolller\f[] \f[I]
+\f[I]sizebooru\f[] \f[]
+\f[I]soundgasm\f[] \f[I]
+\f[I]thehentaiworld\f[] \f[]
+\f[I]urlgalleries\f[] \f[I]
+\f[I]vk\f[] \f[]
+\f[I]webtoons\f[] \f[I]
+\f[I]weebcentral\f[] \f[]
+\f[I]xfolio\f[] \f[I]
\f[I]zerochan\f[]
-.br
-* \f[I]"1.0"\f[]
-\f[I]furaffinity\f[]
+\f[I]"1.0"\f[]
+\f[I]furaffinity\f[] \f[]
\f[I]rule34\f[]
-.br
-* \f[I]"1.0-2.0"\f[]
-\f[I]flickr\f[],
-\f[I]pexels\f[],
-\f[I]weibo\f[],
+\f[I]"1.0-2.0"\f[]
+\f[I]flickr\f[] \f[I]
+\f[I]pexels\f[] \f[]
+\f[I]weibo\f[] \f[I]
\f[I][wikimedia]\f[]
-.br
-* \f[I]"1.4"\f[]
+\f[I]"1.4"\f[]
\f[I]wallhaven\f[]
-.br
-* \f[I]"2.0-4.0"\f[]
-\f[I]behance\f[],
-\f[I]imagefap\f[],
+\f[I]"2.0-4.0"\f[]
+\f[I]behance\f[] \f[]
+\f[I]imagefap\f[] \f[I]
\f[I][Nijie]\f[]
-.br
-* \f[I]"3.0-6.0"\f[]
-\f[I]bilibili\f[],
-\f[I]exhentai\f[],
-\f[I][reactor]\f[],
+\f[I]"3.0-6.0"\f[]
+\f[I]bilibili\f[] \f[]
+\f[I]exhentai\f[] \f[I]
+\f[I][reactor]\f[] \f[]
\f[I]readcomiconline\f[]
-.br
-* \f[I]"6.0-6.1"\f[]
+\f[I]"6.0-6.1"\f[]
\f[I]twibooru\f[]
-.br
-* \f[I]"6.0-12.0"\f[]
+\f[I]"6.0-12.0"\f[]
\f[I]instagram\f[]
-.br
-* \f[I]0\f[]
+\f[I]0\f[]
otherwise
.IP "Description:" 4
@@ -573,35 +580,35 @@ another site.
This is supported for
.br
-* \f[I]aibooru\f[] (*)
+* \f[I]aibooru\f[] (\f[I]*\f[])
.br
* \f[I]ao3\f[]
.br
* \f[I]aryion\f[]
.br
-* \f[I]atfbooru\f[] (*)
+* \f[I]atfbooru\f[] (\f[I]*\f[])
.br
* \f[I]bluesky\f[]
.br
-* \f[I]booruvar\f[] (*)
+* \f[I]booruvar\f[] (\f[I]*\f[])
.br
* \f[I]coomer\f[]
.br
-* \f[I]danbooru\f[] (*)
+* \f[I]danbooru\f[] (\f[I]*\f[])
.br
* \f[I]deviantart\f[]
.br
-* \f[I]e621\f[] (*)
+* \f[I]e621\f[] (\f[I]*\f[])
.br
-* \f[I]e6ai\f[] (*)
+* \f[I]e6ai\f[] (\f[I]*\f[])
.br
-* \f[I]e926\f[] (*)
+* \f[I]e926\f[] (\f[I]*\f[])
.br
* \f[I]exhentai\f[]
.br
* \f[I]girlswithmuscle\f[]
.br
-* \f[I]horne\f[] (R)
+* \f[I]horne\f[] (\f[I]R\f[])
.br
* \f[I]idolcomplex\f[]
.br
@@ -613,7 +620,7 @@ This is supported for
.br
* \f[I]kemono\f[]
.br
-* \f[I]madokami\f[] (R)
+* \f[I]madokami\f[] (\f[I]R\f[])
.br
* \f[I]mangadex\f[]
.br
@@ -621,7 +628,7 @@ This is supported for
.br
* \f[I]newgrounds\f[]
.br
-* \f[I]nijie\f[] (R)
+* \f[I]nijie\f[] (\f[I]R\f[])
.br
* \f[I]pillowfort\f[]
.br
@@ -629,8 +636,6 @@ This is supported for
.br
* \f[I]sankaku\f[]
.br
-* \f[I]schalenetwork\f[]
-.br
* \f[I]scrolller\f[]
.br
* \f[I]seiga\f[]
@@ -651,16 +656,24 @@ These values can also be specified via the
\f[I]-u/--username\f[] and \f[I]-p/--password\f[] command-line options or
by using a \f[I].netrc\f[] file. (see Authentication_)
-(*) The password value for these sites should be
-the API key found in your user profile, not the actual account password.
-
-(R) Login with username & password or supplying logged-in
-\f[I]cookies\f[] is required
-Note: Leave the \f[I]password\f[] value empty or undefined
+.IP "Note:" 4
+Leave the \f[I]password\f[] value empty or undefined
to be prompted for a password when performing a login
(see \f[I]getpass()\f[]).
+.. _pw-apikey:
+
+(*) The \f[I]password\f[] value for these sites should be
+the API key found in your user profile, not the actual account password.
+
+.. _pw-required:
+
+(R) Login with username & password or
+supplying authenticated
+\f[I]cookies\f[]
+is *required*
+
.SS extractor.*.input
.IP "Type:" 6
@@ -690,7 +703,7 @@ Enable the use of \f[I].netrc\f[] authentication data.
.br
* \f[I]Path\f[]
.br
-* \f[I]object\f[] (name -> value)
+* \f[I]object\f[] (name → value)
.br
* \f[I]list\f[]
@@ -741,17 +754,12 @@ Source to read additional cookies from. This can be
\f[I]string\f[]
.IP "Default:" 9
-\f[I]"random"\f[]
+\f[I]null\f[]
.IP "Description:" 4
Interpret \f[I]extractor.cookies\f[]
as a list of cookie sources and select one of them for each extractor run.
-.br
-* \f[I]"random"\f[]: Select cookies \f[I]randomly\f[]
-.br
-* \f[I]"rotate"\f[]: Select cookies in sequence. Start over from the beginning after reaching the end of the list.
-
.. code:: json
[
@@ -762,6 +770,13 @@ as a list of cookie sources and select one of them for each extractor run.
]
+.IP "Supported Values:" 4
+\f[I]"random"\f[]
+Select cookies \f[I]randomly\f[].
+\f[I]"rotate"\f[]
+Select cookies in sequence. Start over from the beginning after reaching the end of the list.
+
+
.SS extractor.*.cookies-update
.IP "Type:" 6
.br
@@ -788,7 +803,7 @@ of a valid cookies.txt file, update its contents.
.br
* \f[I]string\f[]
.br
-* \f[I]object\f[] (scheme -> proxy)
+* \f[I]object\f[] (scheme → proxy)
.IP "Example:" 4
.. code:: json
@@ -817,8 +832,8 @@ It is also possible to set a proxy for a specific host by using
\f[I]scheme://host\f[] as key.
See \f[I]Requests' proxy documentation\f[] for more details.
-Note: If a proxy URL does not include a scheme,
-\f[I]http://\f[] is assumed.
+.IP "Note:" 4
+If a proxy URL does not include a scheme, \f[I]http://\f[] is assumed.
.SS extractor.*.proxy-env
@@ -861,18 +876,30 @@ or a \f[I]list\f[] with IP and explicit port number as elements.
\f[I]string\f[]
.IP "Default:" 9
+\f[I]"gallery-dl/VERSION"\f[]
.br
-* \f[I]"gallery-dl/VERSION"\f[]: \f[I][Danbooru]\f[], \f[I]mangadex\f[], \f[I]weasyl\f[]
+* \f[I][Danbooru]\f[]
.br
-* \f[I]"gallery-dl/VERSION (by mikf)"\f[]: \f[I][E621]\f[]
+* \f[I]mangadex\f[]
+.br
+* \f[I]weasyl\f[]
+.br
+* \f[I]zerochan\f[]
+\f[I]"gallery-dl/VERSION (by mikf)"\f[]
+.br
+* \f[I][E621]\f[]
+\f[I]"net.umanle.arca.android.playstore/0.9.75"\f[]
.br
-* \f[I]"net.umanle.arca.android.playstore/0.9.75"\f[]: \f[I]arcalive\f[]
+* \f[I]arcalive\f[]
+\f[I]"Patreon/72.2.28 (Android; Android 14; Scale/2.10)"\f[]
.br
-* \f[I]"Patreon/72.2.28 (Android; Android 14; Scale/2.10)"\f[]: \f[I]patreon\f[]
+* \f[I]patreon\f[]
+\f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/LATEST.0.0.0 Safari/537.36"\f[]
.br
-* \f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/LATEST.0.0.0 Safari/537.36"\f[]: \f[I]instagram\f[]
+* \f[I]instagram\f[]
+\f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:LATEST) Gecko/20100101 Firefox/LATEST"\f[]
.br
-* \f[I]"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:LATEST) Gecko/20100101 Firefox/LATEST"\f[]: otherwise
+* otherwise
.IP "Example:" 4
.br
@@ -897,23 +924,37 @@ and use the \f[I]User-Agent\f[] header of this installed browser.
\f[I]string\f[]
.IP "Default:" 9
-.br
-* \f[I]"firefox"\f[]: \f[I]artstation\f[], \f[I]behance\f[], \f[I]fanbox\f[], \f[I]twitter\f[]
-.br
-* \f[I]null\f[]: otherwise
+\f[I]"firefox"\f[]
+\f[I]artstation\f[] \f[I]
+\f[I]behance\f[] \f[]
+\f[I]fanbox\f[] |
+\f[I]twitter\f[]
+\f[I]null\f[]
+otherwise
.IP "Example:" 4
.br
-* "firefox/128:linux"
+* "firefox"
+.br
+* "firefox/128"
.br
* "chrome:macos"
+.br
+* "chrome/138:macos"
.IP "Description:" 4
Try to emulate a real browser (\f[I]firefox\f[] or \f[I]chrome\f[])
-by using their default HTTP headers and TLS ciphers for HTTP requests.
+.br
+by using its HTTP headers and TLS ciphers for HTTP requests
+.br
+by setting specialized defaults for
-Optionally, the operating system used in the \f[I]User-Agent\f[] header can be
-specified after a \f[I]:\f[] (\f[I]windows\f[], \f[I]linux\f[], or \f[I]macos\f[]).
+.br
+* \f[I]user-agent\f[]
+.br
+* \f[I]headers\f[]
+.br
+* \f[I]ciphers\f[]
Supported browsers:
@@ -930,15 +971,16 @@ Supported browsers:
.br
* \f[I]chrome/111\f[]
-Note:
-This option sets custom
-\f[I]headers\f[]
-and
-\f[I]ciphers\f[]
-defaults.
+The operating system used in the \f[I]User-Agent\f[] header can be
+specified after a colon \f[I]:\f[] (\f[I]windows\f[], \f[I]linux\f[], \f[I]macos\f[]),
+for example \f[I]chrome:windows\f[].
-Note: \f[I]requests\f[] and \f[I]urllib3\f[] only support HTTP/1.1, while a real
-browser would use HTTP/2.
+.IP "Note:" 4
+Any value *not* matching a supported browser
+will fall back to \f[I]"firefox"\f[].
+
+\f[I]requests\f[] and \f[I]urllib3\f[] only support HTTP/1.1, while a real
+browser would use HTTP/2 and HTTP/3.
.SS extractor.*.referer
@@ -949,7 +991,14 @@ browser would use HTTP/2.
* \f[I]string\f[]
.IP "Default:" 9
+\f[I]false\f[]
+\f[I]4archive\f[] \f[I]
+\f[I]4chanarchives\f[] \f[]
+\f[I]archivedmoe\f[] \f[I]
+\f[I]nsfwalbum\f[] \f[]
+\f[I]tumblrgallery\f[]
\f[I]true\f[]
+otherwise
.IP "Description:" 4
Send \f[I]Referer\f[]
@@ -964,7 +1013,7 @@ instead of the extractor's \f[I]root\f[] domain.
.br
* \f[I]"string"\f[]
.br
-* \f[I]object\f[] (name -> value)
+* \f[I]object\f[] (name → value)
.IP "Default:" 9
.. code:: json
@@ -1022,10 +1071,12 @@ to use these browser's default ciphers.
\f[I]bool\f[]
.IP "Default:" 9
-.br
-* \f[I]false\f[]: \f[I]artstation\f[], \f[I]behance\f[], \f[I]vsco\f[]
-.br
-* \f[I]true\f[]: otherwise
+\f[I]false\f[]
+\f[I]artstation\f[] \f[I]
+\f[I]behance\f[] \f[]
+\f[I]vsco\f[]
+\f[I]true\f[]
+otherwise
.IP "Description:" 4
Allow selecting TLS 1.2 cipher suites.
@@ -1036,7 +1087,7 @@ and potentially bypass Cloudflare blocks.
.SS extractor.*.keywords
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
{"type": "Pixel Art", "type_id": 123}
@@ -1054,7 +1105,7 @@ Additional name-value pairs to be added to each metadata dictionary.
.IP "Description:" 4
Evaluate each \f[I]keywords\f[] \f[I]string\f[] value
-as a \f[I]format string\f[].
+as a \f[I]Format String\f[].
.SS extractor.*.keywords-default
@@ -1065,8 +1116,8 @@ any
\f[I]"None"\f[]
.IP "Description:" 4
-Default value used for missing or undefined keyword names in
-\f[I]format strings\f[].
+Default value used for missing or undefined keyword names in a
+\f[I]Format String\f[].
.SS extractor.*.url-metadata
@@ -1176,7 +1227,8 @@ Each identifier can be
Both names can be a * or left empty, matching all possible names (\f[I]"*:image"\f[], \f[I]":user"\f[]).
.br
-Note: Any \f[I]blacklist\f[] setting will automatically include
+.IP "Note:" 4
+Any \f[I]blacklist\f[] setting will automatically include
\f[I]"oauth"\f[], \f[I]"recursive"\f[], and \f[I]"test"\f[].
@@ -1211,9 +1263,10 @@ If this value is a
the archive will use this PostgreSQL database as backend (requires
\f[I]Psycopg\f[]).
-Note: Archive files that do not already exist get generated automatically.
+.IP "Note:" 4
+Archive files that do not already exist get generated automatically.
-Note: Archive paths support regular \f[I]format string\f[] replacements,
+Archive paths support basic \f[I]Format String\f[] replacements,
but be aware that using external inputs for building local paths
may pose a security risk.
@@ -1243,13 +1296,13 @@ Available events are:
.SS extractor.*.archive-format
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Example:" 4
"{id}_{offset}"
.IP "Description:" 4
-An alternative \f[I]format string\f[] to build archive IDs with.
+An alternative \f[I]Format String\f[] to build archive IDs with.
.SS extractor.*.archive-mode
@@ -1263,17 +1316,17 @@ An alternative \f[I]format string\f[] to build archive IDs with.
Controls when to write \f[I]archive IDs\f[]
to the archive database.
-.br
-* \f[I]"file"\f[]: Write IDs immediately
+\f[I]"file"\f[]
+Write IDs immediately
after completing or skipping a file download.
-.br
-* \f[I]"memory"\f[]: Keep IDs in memory
+\f[I]"memory"\f[]
+Keep IDs in memory
and only write them after successful job completion.
.SS extractor.*.archive-prefix
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Default:" 9
.br
@@ -1301,7 +1354,7 @@ for available \f[I]PRAGMA\f[] statements and further details.
.SS extractor.*.archive-table
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Default:" 9
\f[I]"archive"\f[]
@@ -1310,13 +1363,13 @@ for available \f[I]PRAGMA\f[] statements and further details.
"{category}"
.IP "Description:" 4
-\f[I]Format string\f[] selecting the archive database table name.
+\f[I]Format String\f[] selecting the archive database table name.
.SS extractor.*.actions
.IP "Type:" 6
.br
-* \f[I]object\f[] (pattern -> \f[I]Action(s)\f[])
+* \f[I]object\f[] (pattern → \f[I]Action(s)\f[])
.br
* \f[I]list\f[] of [pattern, \f[I]Action(s)\f[]] pairs
@@ -1408,7 +1461,7 @@ for each downloaded \f[I]pixiv\f[] file.
.SS extractor.*.postprocessor-options
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
.. code:: json
@@ -1544,15 +1597,15 @@ Use fallback download URLs when a download fails.
.br
* \f[I]list\f[] of \f[I]strings\f[]
-.IP "Examples:" 4
+.IP "Example:" 4
.br
-* \f[I]"10-20"\f[]
+* "10-20"
.br
-* \f[I]"-5, 10, 30-50, 100-"\f[]
+* "-5, 10, 30-50, 100-"
.br
-* \f[I]"10:21, 30:51:2, :5, 100:"\f[]
+* "10:21, 30:51:2, :5, 100:"
.br
-* \f[I]["-5", "10", "30-50", "100-"]\f[]
+* ["-5", "10", "30-50", "100-"]
.IP "Description:" 4
Index range(s) selecting which files to download.
@@ -1572,7 +1625,8 @@ and will default to begin (\f[I]1\f[]) or end (\f[I]sys.maxsize\f[]) if omitted.
For example \f[I]5-\f[], \f[I]5:\f[], and \f[I]5::\f[] all mean "Start at file number 5".
.br
-Note: The index of the first file is \f[I]1\f[].
+.IP "Note:" 4
+The index of the first file is \f[I]1\f[].
.SS extractor.*.chapter-range
@@ -1587,20 +1641,20 @@ but applies to delegated URLs like manga chapters, etc.
.SS extractor.*.image-filter
.IP "Type:" 6
.br
-* \f[I]string\f[]
+* \f[I]Condition\f[]
.br
-* \f[I]list\f[] of \f[I]strings\f[]
+* \f[I]list\f[] of \f[I]Conditions\f[]
-.IP "Examples:" 4
+.IP "Example:" 4
.br
-* \f[I]"re.search(r'foo(bar)+', description)"\f[]
+* "re.search(r'foo(bar)+', description)"
.br
-* \f[I]["width >= 1200", "width/height > 1.2"]\f[]
+* ["width >= 1200", "width/height > 1.2"]
.IP "Description:" 4
-Python expression controlling which files to download.
+Python \f[I]Expression\f[] controlling which files to download.
-A file only gets downloaded when *all* of the given expressions evaluate to \f[I]True\f[].
+A file only gets downloaded when *all* of the given \f[I]Expressions\f[] evaluate to \f[I]True\f[].
Available values are the filename-specific ones listed by \f[I]-K\f[] or \f[I]-j\f[].
@@ -1608,15 +1662,15 @@ Available values are the filename-specific ones listed by \f[I]-K\f[] or \f[I]-j
.SS extractor.*.chapter-filter
.IP "Type:" 6
.br
-* \f[I]string\f[]
+* \f[I]Condition\f[]
.br
-* \f[I]list\f[] of \f[I]strings\f[]
+* \f[I]list\f[] of \f[I]Conditions\f[]
-.IP "Examples:" 4
+.IP "Example:" 4
.br
-* \f[I]"lang == 'en'"\f[]
+* "lang == 'en'"
.br
-* \f[I]["language == 'French'", "10 <= chapter < 20"]\f[]
+* ["language == 'French'", "10 <= chapter < 20"]
.IP "Description:" 4
Like \f[I]image-filter\f[],
@@ -1660,7 +1714,8 @@ date-min and date-max.
See \f[I]strptime\f[] for a list of formatting directives.
-Note: Despite its name, this option does **not** control how
+.IP "Note:" 4
+Despite its name, this option does **not** control how
\f[I]{date}\f[] metadata fields are formatted.
To use a different formatting for those values other than the default
\f[I]%Y-%m-%d %H:%M:%S\f[], put \f[I]strptime\f[] formatting directives
@@ -1682,12 +1737,14 @@ During data extraction,
write received HTTP request data
to enumerated files in the current working directory.
-Special values:
-
+.IP "Special Values:" 4
+\f[I]"all"\f[]
+Include HTTP request and response headers.
.br
-* \f[I]"all"\f[]: Include HTTP request and response headers. Hide \f[I]Authorization\f[], \f[I]Cookie\f[], and \f[I]Set-Cookie\f[] values.
+Hide \f[I]Authorization\f[], \f[I]Cookie\f[], and \f[I]Set-Cookie\f[] values.
.br
-* \f[I]"ALL"\f[]: Include all HTTP request and response headers.
+\f[I]"ALL"\f[]
+Include all HTTP request and response headers.
.SH EXTRACTOR-SPECIFIC OPTIONS
@@ -1823,11 +1880,11 @@ when retrieving search results.
.IP "Description:" 4
Controls the post extraction strategy.
-.br
-* \f[I]true\f[]: Start on users' main gallery pages and recursively
-descend into subfolders
-.br
-* \f[I]false\f[]: Get posts from "Latest Updates" pages
+\f[I]true\f[]
+Start on users' main gallery pages and
+recursively descend into subfolders
+\f[I]false\f[]
+Get posts from "Latest Updates" pages
.SS extractor.batoto.domain
@@ -1881,8 +1938,17 @@ The maximum possible value appears to be \f[I]1920\f[].
.IP "Description:" 4
Selects which gallery modules to download from.
-Supported module types are
-\f[I]image\f[], \f[I]video\f[], \f[I]mediacollection\f[], \f[I]embed\f[], \f[I]text\f[].
+.IP "Supported Types:" 4
+.br
+* \f[I]"image"\f[]
+.br
+* \f[I]"video"\f[]
+.br
+* \f[I]"mediacollection"\f[]
+.br
+* \f[I]"embed"\f[]
+.br
+* \f[I]"text"\f[]
.SS extractor.bellazon.order-posts
@@ -1959,16 +2025,25 @@ Download embedded videos hosted on https://www.blogger.com/
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"info"\f[],
-\f[I]"avatar"\f[],
-\f[I]"background"\f[],
-\f[I]"posts"\f[],
-\f[I]"replies"\f[],
-\f[I]"media"\f[],
-\f[I]"video"\f[],
-\f[I]"likes"\f[],
+.IP "Supported Values:" 4
+.br
+* \f[I]info\f[]
+.br
+* \f[I]avatar\f[]
+.br
+* \f[I]background\f[]
+.br
+* \f[I]posts\f[]
+.br
+* \f[I]replies\f[]
+.br
+* \f[I]media\f[]
+.br
+* \f[I]video\f[]
+.br
+* \f[I]likes\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -2020,11 +2095,13 @@ Requires login and only allows accessing your own likes.
.IP "Description:" 4
Extract additional metadata.
+\f[I]facets\f[]
+\f[I]hashtags\f[], \f[I]mentions\f[], \f[I]uris\f[]
+\f[I]user\f[]
+Detailed \f[I]user\f[] metadata for the user referenced in the input URL.
.br
-* \f[I]facets\f[]: \f[I]hashtags\f[], \f[I]mentions\f[], and \f[I]uris\f[]
+(\f[I]app.bsky.actor.getProfile\f[])
.br
-* \f[I]user\f[]: detailed \f[I]user\f[] metadata for the user referenced in the input URL
-(See \f[I]app.bsky.actor.getProfile\f[]).
.SS extractor.bluesky.likes.depth
@@ -2037,7 +2114,7 @@ Extract additional metadata.
.IP "Description:" 4
Sets the maximum depth of returned reply posts.
-(See depth parameter of \f[I]app.bsky.feed.getPostThread\f[])
+(See the \f[I]depth\f[] parameter of \f[I]app.bsky.feed.getPostThread\f[])
.SS extractor.bluesky.quoted
@@ -2123,26 +2200,24 @@ Provide detailed \f[I]user\f[] metadata.
Download videos.
If this is a \f[I]list\f[], it selects which format to try to download.
-.br
-Possibly available formats are
-.br
+.IP "Possible Formats:" 4
.br
* \f[I]ultra_hd\f[] (2160p)
.br
-* \f[I]quad_hd\f[] (1440p)
+* \f[I]quad_hd\f[] (1440p)
.br
-* \f[I]full_hd\f[] (1080p)
+* \f[I]full_hd\f[] (1080p)
.br
-* \f[I]high\f[] (720p)
+* \f[I]high\f[] (720p)
.br
-* \f[I]medium\f[] (480p)
+* \f[I]medium\f[] (480p)
.br
-* \f[I]low\f[] (360p)
+* \f[I]low\f[] (360p)
.br
-* \f[I]lowest\f[] (240p)
+* \f[I]lowest\f[] (240p)
.br
-* \f[I]tiny\f[] (144p)
+* \f[I]tiny\f[] (144p)
.SS extractor.booth.strategy
@@ -2184,10 +2259,10 @@ API endpoint for retrieving file URLs.
.IP "Description:" 4
Controls which \f[I]bunkr\f[] TLDs to accept.
-.br
-* \f[I]true\f[]: Match URLs with *all* possible TLDs (e.g. \f[I]bunkr.xyz\f[] or \f[I]bunkrrr.duck\f[])
-.br
-* \f[I]false\f[]: Match only URLs with known TLDs
+\f[I]true\f[]
+Match URLs with *all* possible TLDs (e.g. \f[I]bunkr.xyz\f[] or \f[I]bunkrrr.duck\f[])
+\f[I]false\f[]
+Match only URLs with known TLDs
.SS extractor.cien.files
@@ -2200,11 +2275,15 @@ Controls which \f[I]bunkr\f[] TLDs to accept.
.IP "Description:" 4
Determines the type and order of files to download.
-Available types are
-\f[I]image\f[],
-\f[I]video\f[],
-\f[I]download\f[],
-\f[I]gallery\f[].
+.IP "Available Types:" 4
+.br
+* \f[I]image\f[]
+.br
+* \f[I]video\f[]
+.br
+* \f[I]download\f[]
+.br
+* \f[I]gallery\f[]
.SS extractor.civitai.api
@@ -2217,10 +2296,10 @@ Available types are
.IP "Description:" 4
Selects which API endpoints to use.
-.br
-* \f[I]"rest"\f[]: \f[I]Public REST API\f[]
-.br
-* \f[I]"trpc"\f[]: Internal tRPC API
+\f[I]"rest"\f[]
+\f[I]Public REST API\f[]
+\f[I]"trpc"\f[]
+Internal tRPC API
.SS extractor.civitai.api-key
@@ -2246,10 +2325,13 @@ for details.
.IP "Description:" 4
Determines the type and order of files to download when processing models.
-Available types are
-\f[I]model\f[],
-\f[I]image\f[],
-\f[I]gallery\f[].
+.IP "Available Types:" 4
+.br
+* \f[I]model\f[]
+.br
+* \f[I]image\f[]
+.br
+* \f[I]gallery\f[]
.SS extractor.civitai.include
@@ -2266,22 +2348,21 @@ Available types are
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-
+.IP "Supported Values:" 4
.br
-* \f[I]"user-models"\f[]
+* \f[I]user-models\f[]
.br
-* \f[I]"user-posts"\f[]
+* \f[I]user-posts\f[]
.br
-* \f[I]"user-images"\f[]
+* \f[I]user-images\f[]
.br
-* \f[I]"user-videos"\f[]
+* \f[I]user-videos\f[]
.br
-* \f[I]"user-collections"\f[]
+* \f[I]user-collections\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
-.IP "Note:" 4
To get a more complete set of metadata
like \f[I]model['name']\f[] and \f[I]post['title']\f[],
include \f[I]user-models\f[] and \f[I]user-posts\f[]
@@ -2311,7 +2392,8 @@ as well as the default \f[I]user-images\f[] and \f[I]user-videos\f[]:
.IP "Description:" 4
Extract additional \f[I]generation\f[], \f[I]version\f[], and \f[I]post\f[] metadata.
-Note: This requires 1 or more additional API requests per image or video.
+.IP "Note:" 4
+This requires 1 or more additional API requests per image or video.
.SS extractor.civitai.nsfw
@@ -2366,7 +2448,8 @@ to pass with every image URL.
Known available options include \f[I]original\f[], \f[I]quality\f[], \f[I]width\f[]
-Note: Set this option to an arbitrary letter, e.g., \f[I]"w"\f[],
+.IP "Note:" 4
+Set this option to an arbitrary letter, e.g., \f[I]"w"\f[],
to download images in JPEG format at their original resolution.
@@ -2423,8 +2506,7 @@ Use \f[I]+\f[] as first character to add the given options to the
* ["fr", "it", "pl"]
.IP "Description:" 4
-\f[I]ISO 639-1\f[] language codes
-to filter chapters by.
+\f[I]ISO 639-1\f[] code(s) to filter chapters by.
.SS extractor.cyberdrop.domain
@@ -2454,7 +2536,7 @@ uses the same domain as a given input URL.
.IP "Description:" 4
Password value used to access protected files and folders.
-Note: Leave this value empty or undefined
+Leave this value empty or undefined
to be interactively prompted for a password when needed
(see \f[I]getpass()\f[]).
@@ -2501,10 +2583,10 @@ Ascending Post ID order
.IP "Description:" 4
Controls the download target for Ugoira posts.
-.br
-* \f[I]true\f[]: Original ZIP archives
-.br
-* \f[I]false\f[]: Converted video files
+\f[I]true\f[]
+ZIP archives
+\f[I]false\f[]
+Converted video files
.SS extractor.[Danbooru].metadata
@@ -2533,7 +2615,8 @@ It is possible to specify a custom list of metadata includes.
See \f[I]available_includes\f[]
for possible field names. \f[I]aibooru\f[] also supports \f[I]ai_metadata\f[].
-Note: This requires 1 additional HTTP request per 200-post batch.
+.IP "Note:" 4
+This requires 1 additional HTTP request per 200-post batch.
.SS extractor.[Danbooru].threshold
@@ -2551,7 +2634,8 @@ Stop paginating over API results if the length of a batch of returned
posts is less than the specified number. Defaults to the per-page limit
of the current instance, which is 200.
-Note: Changing this setting is normally not necessary. When the value is
+.IP "Note:" 4
+Changing this setting is normally not necessary. When the value is
greater than the per-page limit, gallery-dl will stop after the first
batch. The value cannot be less than 1.
@@ -2612,7 +2696,8 @@ Extract \f[I]comments\f[] metadata.
.IP "Description:" 4
Download the avatar of each commenting user.
-Note: Enabling this option also enables deviantart.comments_.
+.IP "Note:" 4
+Enabling this option also enables deviantart.comments_.
.SS extractor.deviantart.extra
@@ -2626,7 +2711,8 @@ Note: Enabling this option also enables deviantart.comments_.
Download extra Sta.sh resources from
description texts and journals.
-Note: Enabling this option also enables deviantart.metadata_.
+.IP "Note:" 4
+Enabling this option also enables deviantart.metadata_.
.SS extractor.deviantart.flat
@@ -2640,16 +2726,17 @@ Note: Enabling this option also enables deviantart.metadata_.
Select the directory structure created by the Gallery- and
Favorite-Extractors.
-.br
-* \f[I]true\f[]: Use a flat directory structure.
-.br
-* \f[I]false\f[]: Collect a list of all gallery-folders or
-favorites-collections and transfer any further work to other
+\f[I]true\f[]
+Use a flat directory structure.
+\f[I]false\f[]
+Collect a list of all gallery \f[I]folders\f[] or
+favorites \f[I]collections\f[] and transfer any further work to other
extractors (\f[I]folder\f[] or \f[I]collection\f[]), which will then
create individual subdirectories for each of them.
-Note: Going through all gallery folders will not be able to
-fetch deviations which aren't in any folder.
+.IP "Note:" 4
+Going through all gallery folders won't
+fetch deviations not contained in any folder.
.SS extractor.deviantart.folders
@@ -2663,7 +2750,8 @@ fetch deviations which aren't in any folder.
Provide a \f[I]folders\f[] metadata field that contains the names of all
folders a deviation is present in.
-Note: Gathering this information requires a lot of API calls.
+.IP "Note:" 4
+Gathering this information requires a lot of API calls.
Use with caution.
@@ -2684,10 +2772,9 @@ belongs to a group or a regular user.
When disabled, assume every given profile name
belongs to a regular user.
-Special values:
-
-.br
-* \f[I]"skip"\f[]: Skip groups
+.IP "Special Values:" 4
+\f[I]"skip"\f[]
+Skip groups
.SS extractor.deviantart.include
@@ -2710,15 +2797,23 @@ Special values:
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"avatar"\f[],
-\f[I]"background"\f[],
-\f[I]"gallery"\f[],
-\f[I]"scraps"\f[],
-\f[I]"journal"\f[],
-\f[I]"favorite"\f[],
-\f[I]"status"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]avatar\f[]
+.br
+* \f[I]background\f[]
+.br
+* \f[I]gallery\f[]
+.br
+* \f[I]scraps\f[]
+.br
+* \f[I]journal\f[]
+.br
+* \f[I]favorite\f[]
+.br
+* \f[I]status\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -2745,27 +2840,12 @@ download a higher-quality \f[I]/intermediary/\f[] version.
Selects the output format for textual content. This includes journals,
literature and status updates.
-.br
-* \f[I]"html"\f[]: HTML with (roughly) the same layout as on DeviantArt.
-.br
-* \f[I]"text"\f[]: Plain text with image references and HTML tags removed.
-.br
-* \f[I]"none"\f[]: Don't download textual content.
-
-
-.SS extractor.deviantart.jwt
-.IP "Type:" 6
-\f[I]bool\f[]
-
-.IP "Default:" 9
-\f[I]false\f[]
-
-.IP "Description:" 4
-Update \f[I]JSON Web Tokens\f[] (the \f[I]token\f[] URL parameter)
-of otherwise non-downloadable, low-resolution images
-to be able to download them in full resolution.
-
-Note: No longer functional as of 2023-10-11
+\f[I]"html"\f[]
+HTML with (roughly) the same layout as on DeviantArt.
+\f[I]"text"\f[]
+Plain text with image references and HTML tags removed.
+\f[I]"none"\f[]
+Don't download textual content.
.SS extractor.deviantart.mature
@@ -2810,17 +2890,18 @@ fields when enabled.
It is possible to request extended metadata by specifying a list of
-.br
-* \f[I]camera\f[] : EXIF information (if available)
-.br
-* \f[I]stats\f[] : deviation statistics
-.br
-* \f[I]submission\f[] : submission information
-.br
-* \f[I]collection\f[] : favourited folder information (requires a \f[I]refresh token\f[])
-.br
-* \f[I]gallery\f[] : gallery folder information (requires a \f[I]refresh token\f[])
+\f[I]camera\f[]
+EXIF information if available
+\f[I]stats\f[]
+Deviation statistics
+\f[I]submission\f[]
+Submission information
+\f[I]collection\f[]
+Favourited folder information (requires a \f[I]refresh token\f[])
+\f[I]gallery\f[]
+Gallery folder information (requires a \f[I]refresh token\f[])
+.IP "Note:" 4
Set this option to \f[I]"all"\f[] to request all extended metadata categories.
See \f[I]/deviation/metadata\f[]
@@ -2855,10 +2936,10 @@ everything else (archives, videos, etc.).
.IP "Description:" 4
Controls when to stop paginating over API results.
-.br
-* \f[I]"api"\f[]: Trust the API and stop when \f[I]has_more\f[] is \f[I]false\f[].
-.br
-* \f[I]"manual"\f[]: Disregard \f[I]has_more\f[] and only stop when a batch of results is empty.
+\f[I]"api"\f[]
+Trust the API and stop when \f[I]has_more\f[] is \f[I]false\f[].
+\f[I]"manual"\f[]
+Disregard \f[I]has_more\f[] and only stop when a batch of results is empty.
.SS extractor.deviantart.previews
@@ -2920,7 +3001,8 @@ The \f[I]refresh-token\f[] value you get from
Using a \f[I]refresh-token\f[] allows you to access private or otherwise
not publicly available deviations.
-Note: The \f[I]refresh-token\f[] becomes invalid
+.IP "Note:" 4
+The \f[I]refresh-token\f[] becomes invalid
\f[I]after 3 months\f[]
or whenever your \f[I]cache file\f[] is deleted or cleared.
@@ -3043,7 +3125,8 @@ from an anthology's HTML page.
.IP "Description:" 4
Extract additional metadata (notes, pool metadata) if available.
-Note: This requires 0-2 additional HTTP requests per post.
+.IP "Note:" 4
+This requires 0-2 additional HTTP requests per post.
.SS extractor.[E621].threshold
@@ -3061,7 +3144,8 @@ Stop paginating over API results if the length of a batch of returned
posts is less than the specified number. Defaults to the per-page limit
of the current instance, which is 320.
-Note: Changing this setting is normally not necessary. When the value is
+.IP "Note:" 4
+Changing this setting is normally not necessary. When the value is
greater than the per-page limit, gallery-dl will stop after the first
batch. The value cannot be less than 1.
@@ -3085,13 +3169,13 @@ Include reposts when extracting albums from a user profile.
\f[I]"auto"\f[]
.IP "Description:" 4
-.br
-* \f[I]"auto"\f[]: Use \f[I]e-hentai.org\f[] or \f[I]exhentai.org\f[]
+\f[I]"auto"\f[]
+Use \f[I]e-hentai.org\f[] or \f[I]exhentai.org\f[]
depending on the input URL
-.br
-* \f[I]"e-hentai.org"\f[]: Use \f[I]e-hentai.org\f[] for all URLs
-.br
-* \f[I]"exhentai.org"\f[]: Use \f[I]exhentai.org\f[] for all URLs
+\f[I]"e-hentai.org"\f[]
+Use \f[I]e-hentai.org\f[] for all URLs
+\f[I]"exhentai.org"\f[]
+Use \f[I]exhentai.org\f[] for all URLs
.SS extractor.exhentai.fallback-retries
@@ -3117,9 +3201,10 @@ or \f[I]-1\f[] for infinite retries.
After downloading a gallery,
add it to your account's favorites as the given category number.
-Note: Set this to "favdel" to remove galleries from your favorites.
+.IP "Note:" 4
+Set this to "favdel" to remove galleries from your favorites.
-Note: This will remove any Favorite Notes when applied
+This will remove any Favorite Notes when applied
to already favorited galleries.
@@ -3210,10 +3295,10 @@ Download full-sized original images if available.
.IP "Description:" 4
Selects an alternative source to download files from.
-.br
-* \f[I]"hitomi"\f[]: Download the corresponding gallery from \f[I]hitomi.la\f[]
-.br
-* \f[I]"metadata"\f[]: Load only a gallery's metadata from the
+\f[I]"hitomi"\f[]
+Download the corresponding gallery from \f[I]hitomi.la\f[]
+\f[I]"metadata"\f[]
+Load only a gallery's metadata from the
\f[I]API\f[]
@@ -3261,8 +3346,7 @@ Extract comments that include photo attachments made by the author of the post.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Supported values are
-
+.IP "Supported Values:" 4
.br
* \f[I]info\f[]
.br
@@ -3272,6 +3356,7 @@ Supported values are
.br
* \f[I]albums\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -3288,12 +3373,12 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately.
.IP "Description:" 4
Control video download behavior.
-.br
-* \f[I]true\f[]: Extract and download video & audio separately.
-.br
-* \f[I]"ytdl"\f[]: Let \f[I]ytdl\f[] handle video extraction and download, and merge video & audio streams.
-.br
-* \f[I]false\f[]: Ignore videos.
+\f[I]true\f[]
+Extract and download video & audio separately.
+\f[I]"ytdl"\f[]
+Let \f[I]ytdl\f[] handle video extraction and download, and merge video & audio streams.
+\f[I]false\f[]
+Ignore videos.
.SS extractor.fanbox.comments
@@ -3306,7 +3391,8 @@ Control video download behavior.
.IP "Description:" 4
Extract \f[I]comments\f[] metadata.
-Note: This requires 1 or more additional API requests per post,
+.IP "Note:" 4
+This requires 1 or more additional API requests per post,
depending on the number of comments.
@@ -3323,14 +3409,14 @@ depending on the number of comments.
.IP "Description:" 4
Control behavior on embedded content from external sites.
-.br
-* \f[I]true\f[]: Extract embed URLs and download them if supported
+\f[I]true\f[]
+Extract embed URLs and download them if supported
(videos are not downloaded).
-.br
-* \f[I]"ytdl"\f[]: Like \f[I]true\f[], but let \f[I]ytdl\f[] handle video
+\f[I]"ytdl"\f[]
+Like \f[I]true\f[], but let \f[I]ytdl\f[] handle video
extraction and download for YouTube, Vimeo, and SoundCloud embeds.
-.br
-* \f[I]false\f[]: Ignore embeds.
+\f[I]false\f[]
+Ignore embeds.
.SS extractor.fanbox.fee-max
@@ -3341,7 +3427,8 @@ extraction and download for YouTube, Vimeo, and SoundCloud embeds.
Do not request API data or extract files from posts
that require a fee (\f[I]feeRequired\f[]) greater than the specified amount.
-Note: This option has no effect on individual post URLs.
+.IP "Note:" 4
+This option has no effect on individual post URLs.
.SS extractor.fanbox.metadata
@@ -3365,8 +3452,7 @@ Note: This option has no effect on individual post URLs.
.IP "Description:" 4
Extract \f[I]plan\f[] and extended \f[I]user\f[] metadata.
-Supported fields when selecting which data to extract are
-
+.IP "Supported Fields:" 4
.br
* \f[I]comments\f[]
.br
@@ -3374,7 +3460,8 @@ Supported fields when selecting which data to extract are
.br
* \f[I]user\f[]
-Note: \f[I]comments\f[] can also be enabled via
+.IP "Note:" 4
+\f[I]comments\f[] can also be enabled via
\f[I]fanbox.comments\f[]
@@ -3425,7 +3512,8 @@ from \f[I]linking your Flickr account to gallery-dl\f[].
For each photo, return the albums and pools it belongs to
as \f[I]set\f[] and \f[I]pool\f[] metadata.
-Note: This requires 1 additional API call per photo.
+.IP "Note:" 4
+This requires 1 additional API call per photo.
See \f[I]flickr.photos.getAllContexts\f[] for details.
@@ -3440,7 +3528,8 @@ See \f[I]flickr.photos.getAllContexts\f[] for details.
For each photo, return its EXIF/TIFF/GPS tags
as \f[I]exif\f[] and \f[I]camera\f[] metadata.
-Note: This requires 1 additional API call per photo.
+.IP "Note:" 4
+This requires 1 additional API call per photo.
See \f[I]flickr.photos.getExif\f[] for details.
@@ -3455,7 +3544,8 @@ See \f[I]flickr.photos.getExif\f[] for details.
For each photo, retrieve its "full" metadata as provided by
\f[I]flickr.photos.getInfo\f[]
-Note: This requires 1 additional API call per photo.
+.IP "Note:" 4
+This requires 1 additional API call per photo.
.SS extractor.flickr.metadata
@@ -3496,7 +3586,8 @@ for possible field names.
.IP "Description:" 4
Extract additional \f[I]user\f[] profile metadata.
-Note: This requires 1 additional API call per user profile.
+.IP "Note:" 4
+This requires 1 additional API call per user profile.
See \f[I]flickr.people.getInfo\f[] for details.
@@ -3543,10 +3634,10 @@ Sets the maximum allowed size for downloaded images.
.IP "Description:" 4
Controls the format of \f[I]description\f[] metadata fields.
-.br
-* \f[I]"text"\f[]: Plain text with HTML tags removed
-.br
-* \f[I]"html"\f[]: Raw HTML content
+\f[I]"text"\f[]
+Plain text with HTML tags removed
+\f[I]"html"\f[]
+Raw HTML content
.SS extractor.furaffinity.external
@@ -3580,9 +3671,15 @@ Follow external URLs linked in descriptions.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"gallery"\f[], \f[I]"scraps"\f[], \f[I]"favorite"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]gallery\f[]
+.br
+* \f[I]scraps\f[]
+.br
+* \f[I]favorite\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -3596,12 +3693,12 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately.
.IP "Description:" 4
Selects which site layout to expect when parsing posts.
-.br
-* \f[I]"auto"\f[]: Automatically differentiate between \f[I]"old"\f[] and \f[I]"new"\f[]
-.br
-* \f[I]"old"\f[]: Expect the *old* site layout
-.br
-* \f[I]"new"\f[]: Expect the *new* site layout
+\f[I]"auto"\f[]
+Automatically differentiate between \f[I]"old"\f[] and \f[I]"new"\f[]
+\f[I]"old"\f[]
+Expect the *old* site layout
+\f[I]"new"\f[]
+Expect the *new* site layout
.SS extractor.gelbooru.api-key & .user-id
@@ -3628,12 +3725,10 @@ page.
.IP "Description:" 4
Controls the order in which favorited posts are returned.
-.br
-* \f[I]"asc"\f[]: Ascending favorite date order (oldest first)
-.br
-* \f[I]"desc"\f[]: Descending favorite date order (newest first)
-.br
-* \f[I]"reverse"\f[]: Same as \f[I]"asc"\f[]
+\f[I]"asc"\f[]
+Ascending favorite date order (oldest first)
+\f[I]"desc"\f[] | \f[I]"reverse"\f[]
+Descending favorite date order (newest first)
.SS extractor.generic.enabled
@@ -3705,13 +3800,14 @@ To get this value:
.br
* Open your browser's Developer Tools (F12)
.br
-* Select Network -> XHR
+* Select Network → XHR
.br
* Open a gallery page
.br
* Select the last Network entry and copy its \f[I]crt\f[] value
-Note: You will also need your browser's
+.IP "Note:" 4
+You will also need your browser's
\f[I]user-agent\f[]
@@ -3777,10 +3873,10 @@ to access \f[I]favorite\f[] galleries.
.IP "Description:" 4
Controls the format of \f[I]description\f[] metadata fields.
-.br
-* \f[I]"text"\f[]: Plain text with HTML tags removed
-.br
-* \f[I]"html"\f[]: Raw HTML content
+\f[I]"text"\f[]
+Plain text with HTML tags removed
+\f[I]"html"\f[]
+Raw HTML content
.SS extractor.hentaifoundry.include
@@ -3803,9 +3899,17 @@ Controls the format of \f[I]description\f[] metadata fields.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"pictures"\f[], \f[I]"scraps"\f[], \f[I]"stories"\f[], \f[I]"favorite"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]pictures\f[]
+.br
+* \f[I]scraps\f[]
+.br
+* \f[I]stories\f[]
+.br
+* \f[I]favorite\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -3819,7 +3923,11 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately.
.IP "Description:" 4
Selects which image format to download.
-Available formats are \f[I]"webp"\f[] and \f[I]"avif"\f[].
+.IP "Available Formats:" 4
+.br
+* \f[I]"webp"\f[]
+.br
+* \f[I]"avif"\f[]
.SS extractor.imagechest.access-token
@@ -3858,13 +3966,13 @@ Custom Client ID value for API requests.
.IP "Description:" 4
Controls whether to choose the GIF or MP4 version of an animation.
-.br
-* \f[I]true\f[]: Follow Imgur's advice and choose MP4 if the
+\f[I]true\f[]
+Follow Imgur's advice and choose MP4 if the
\f[I]prefer_video\f[] flag in an image's metadata is set.
-.br
-* \f[I]false\f[]: Always choose GIF.
-.br
-* \f[I]"always"\f[]: Always choose MP4.
+\f[I]false\f[]
+Always choose GIF.
+\f[I]"always"\f[]
+Always choose MP4.
.SS extractor.inkbunny.orderby
@@ -3891,10 +3999,10 @@ for details)
.IP "Description:" 4
Selects which API endpoints to use.
-.br
-* \f[I]"rest"\f[]: REST API - higher-resolution media
-.br
-* \f[I]"graphql"\f[]: GraphQL API - lower-resolution media
+\f[I]"rest"\f[]
+REST API - higher-resolution media
+\f[I]"graphql"\f[]
+GraphQL API - lower-resolution media
.SS extractor.instagram.cursor
@@ -3944,15 +4052,23 @@ Start from the position defined by this value.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"posts"\f[],
-\f[I]"reels"\f[],
-\f[I]"tagged"\f[],
-\f[I]"stories"\f[],
-\f[I]"highlights"\f[],
-\f[I]"info"\f[],
-\f[I]"avatar"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]posts\f[]
+.br
+* \f[I]reels\f[]
+.br
+* \f[I]tagged\f[]
+.br
+* \f[I]stories\f[]
+.br
+* \f[I]highlights\f[]
+.br
+* \f[I]info\f[]
+.br
+* \f[I]avatar\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -3978,7 +4094,8 @@ Limit the number of posts to download.
Provide extended \f[I]user\f[] metadata even when referring to a user by ID,
e.g. \f[I]instagram.com/id:12345678\f[].
-Note: This metadata is always available when referring to a user by name,
+.IP "Note:" 4
+This metadata is always available when referring to a user by name,
e.g. \f[I]instagram.com/USERNAME\f[].
@@ -3992,14 +4109,13 @@ e.g. \f[I]instagram.com/USERNAME\f[].
.IP "Description:" 4
Controls the order in which files of each post are returned.
-.br
-* \f[I]"asc"\f[]: Same order as displayed in a post
-.br
-* \f[I]"desc"\f[]: Reverse order as displayed in a post
-.br
-* \f[I]"reverse"\f[]: Same as \f[I]"desc"\f[]
+\f[I]"asc"\f[]
+Same order as displayed in a post
+\f[I]"desc"\f[] | \f[I]"reverse"\f[]
+Reverse order as displayed in a post
-Note: This option does *not* affect \f[I]{num}\f[].
+.IP "Note:" 4
+This option does *not* affect \f[I]{num}\f[].
To enumerate files in reverse order, use \f[I]count - num + 1\f[].
@@ -4013,18 +4129,17 @@ To enumerate files in reverse order, use \f[I]count - num + 1\f[].
.IP "Description:" 4
Controls the order in which posts are returned.
-.br
-* \f[I]"asc"\f[]: Same order as displayed
-.br
-* \f[I]"desc"\f[]: Reverse order as displayed
-.br
-* \f[I]"id"\f[] or \f[I]"id_asc"\f[]: Ascending order by ID
-.br
-* \f[I]"id_desc"\f[]: Descending order by ID
-.br
-* \f[I]"reverse"\f[]: Same as \f[I]"desc"\f[]
+\f[I]"asc"\f[]
+Same order as displayed
+\f[I]"desc"\f[] | \f[I]"reverse"\f[]
+Reverse order as displayed
+\f[I]"id"\f[] or \f[I]"id_asc"\f[]
+Ascending order by ID
+\f[I]"id_desc"\f[]
+Descending order by ID
-Note: This option only affects \f[I]highlights\f[].
+.IP "Note:" 4
+This option only affects \f[I]highlights\f[].
.SS extractor.instagram.previews
@@ -4059,6 +4174,43 @@ Download pre-merged video formats
Do not download videos
+.SS extractor.instagram.warn-images
+.IP "Type:" 6
+.br
+* \f[I]bool\f[]
+.br
+* \f[I]string\f[]
+
+.IP "Default:" 9
+\f[I]true\f[]
+
+.IP "Description:" 4
+Show a warning when downloading images
+with a resolution smaller than the original.
+
+\f[I]true\f[]
+Show a warning when at least one dimension
+is smaller than the reported original resolution
+\f[I]"all"\f[] | \f[I]"both"\f[]
+Show a warning only when both \f[I]width\f[] and \f[I]height\f[]
+are smaller than the reported original resolution
+\f[I]false\f[]
+Do not show a warning
+
+
+.SS extractor.instagram.warn-videos
+.IP "Type:" 6
+\f[I]bool\f[]
+
+.IP "Default:" 9
+\f[I]true\f[]
+
+.IP "Description:" 4
+Show a warning when downloading videos with a
+\f[I]User-Agent\f[]
+header causing potentially lowered video quality.
+
+
.SS extractor.instagram.stories.split
.IP "Type:" 6
.br
@@ -4091,8 +4243,7 @@ Split \f[I]stories\f[] elements into separate posts.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Supported values are
-
+.IP "Supported Values:" 4
.br
* \f[I]gallery\f[]
.br
@@ -4104,6 +4255,7 @@ Supported values are
.br
* \f[I]stars\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -4132,15 +4284,15 @@ Download video files.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-
+.IP "Supported Values:" 4
.br
-* \f[I]"user-images"\f[]
+* \f[I]user-images\f[]
.br
-* \f[I]"user-videos"\f[]
+* \f[I]user-videos\f[]
.br
-* \f[I]"user-playlists"\f[]
+* \f[I]user-playlists\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -4155,7 +4307,8 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately.
Extract additional metadata for \f[I]archives\f[] files, including
\f[I]file\f[], \f[I]file_list\f[], and \f[I]password\f[].
-Note: This requires 1 additional HTTP request per \f[I]archives\f[] file.
+.IP "Note:" 4
+This requires 1 additional HTTP request per \f[I]archives\f[] file.
.SS extractor.kemono.comments
@@ -4168,7 +4321,8 @@ Note: This requires 1 additional HTTP request per \f[I]archives\f[] file.
.IP "Description:" 4
Extract \f[I]comments\f[] metadata.
-Note: This requires 1 additional HTTP request per post.
+.IP "Note:" 4
+This requires 1 additional HTTP request per post.
.SS extractor.kemono.duplicates
@@ -4265,7 +4419,13 @@ Available types are \f[I]artist\f[], and \f[I]post\f[].
.IP "Description:" 4
Determines the type and order of files to be downloaded.
-Available types are \f[I]file\f[], \f[I]attachments\f[], and \f[I]inline\f[].
+.IP "Available Types:" 4
+.br
+* \f[I]file\f[]
+.br
+* \f[I]attachments\f[]
+.br
+* \f[I]inline\f[]
.SS extractor.kemono.max-posts
@@ -4305,7 +4465,8 @@ Extract post revisions.
Set this to \f[I]"unique"\f[] to filter out duplicate revisions.
-Note: This requires 1 additional HTTP request per post.
+.IP "Note:" 4
+This requires 1 additional HTTP request per post.
.SS extractor.kemono.order-revisions
@@ -4414,7 +4575,7 @@ The server to use for API requests.
.SS extractor.mangadex.api-parameters
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
{"order[updatedAt]": "desc"}
@@ -4442,8 +4603,7 @@ and \f[I]/user/follows/manga/feed\f[])
* ["fr", "it"]
.IP "Description:" 4
-\f[I]ISO 639-1\f[] language codes
-to filter chapters by.
+\f[I]ISO 639-1\f[] code(s) to filter chapters by.
.SS extractor.mangadex.ratings
@@ -4468,6 +4628,31 @@ to filter chapters by.
List of acceptable content ratings for returned chapters.
+.SS extractor.mangafire.manga.lang
+.IP "Type:" 6
+\f[I]string\f[]
+
+.IP "Default:" 9
+\f[I]"en"\f[]
+
+.IP "Description:" 4
+\f[I]ISO 639-1\f[] code selecting which chapters to download.
+
+
+.SS extractor.mangareader.manga.lang
+.IP "Type:" 6
+\f[I]string\f[]
+
+.IP "Default:" 9
+\f[I]"en"\f[]
+
+.IP "Example:" 4
+"pt-br"
+
+.IP "Description:" 4
+\f[I]ISO 639-1\f[] code selecting which chapters to download.
+
+
.SS extractor.mangapark.source
.IP "Type:" 6
.br
@@ -4504,10 +4689,9 @@ Specifying the numeric \f[I]ID\f[] of a source is also supported.
The \f[I]access-token\f[] value you get from \f[I]linking your account to
gallery-dl\f[].
-Note: gallery-dl comes with built-in tokens for \f[I]mastodon.social\f[],
-\f[I]pawoo\f[] and \f[I]baraag\f[]. For other instances, you need to obtain an
-\f[I]access-token\f[] in order to use usernames in place of numerical
-user IDs.
+.IP "Note:" 4
+gallery-dl comes with built-in tokens for
+\f[I]mastodon.social\f[], \f[I]pawoo\f[], and \f[I]baraag\f[].
.SS extractor.[mastodon].cards
@@ -4582,12 +4766,17 @@ Your access token, necessary to fetch favorited notes.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"info"\f[],
-\f[I]"avatar"\f[],
-\f[I]"background"\f[],
-\f[I]"notes"\f[],
+.IP "Supported Values:" 4
+.br
+* \f[I]info\f[]
+.br
+* \f[I]avatar\f[]
+.br
+* \f[I]background\f[]
+.br
+* \f[I]notes\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -4623,7 +4812,8 @@ Fetch media from replies to other notes.
.IP "Description:" 4
Extract extended \f[I]pool\f[] metadata.
-Note: Not supported by all \f[I]moebooru\f[] instances.
+.IP "Note:" 4
+Not supported by all \f[I]moebooru\f[] instances.
.SS extractor.naver-blog.videos
@@ -4706,9 +4896,17 @@ until an available format is found.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"art"\f[], \f[I]"audio"\f[], \f[I]"games"\f[], \f[I]"movies"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]art\f[]
+.br
+* \f[I]audio\f[]
+.br
+* \f[I]games\f[]
+.br
+* \f[I]movies\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -4726,9 +4924,17 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"illustration"\f[], \f[I]"doujin"\f[], \f[I]"favorite"\f[], \f[I]"nuita"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]illustration\f[]
+.br
+* \f[I]doujin\f[]
+.br
+* \f[I]favorite\f[]
+.br
+* \f[I]nuita\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -4767,12 +4973,12 @@ Fetch media from Retweets.
.IP "Description:" 4
Control video download behavior.
-.br
-* \f[I]true\f[]: Download videos
-.br
-* \f[I]"ytdl"\f[]: Download videos using \f[I]ytdl\f[]
-.br
-* \f[I]false\f[]: Skip video Tweets
+\f[I]true\f[]
+Download videos
+\f[I]"ytdl"\f[]
+Download videos using \f[I]ytdl\f[]
+\f[I]false\f[]
+Skip video Tweets
.SS extractor.oauth.browser
@@ -4785,11 +4991,11 @@ Control video download behavior.
.IP "Description:" 4
Controls how a user is directed to an OAuth authorization page.
-.br
-* \f[I]true\f[]: Use Python's \f[I]webbrowser.open()\f[] method to automatically
+\f[I]true\f[]
+Use Python's \f[I]webbrowser.open()\f[] method to automatically
open the URL in the user's default browser.
-.br
-* \f[I]false\f[]: Ask the user to copy & paste an URL from the terminal.
+\f[I]false\f[]
+Ask the user to copy & paste an URL from the terminal.
.SS extractor.oauth.cache
@@ -4825,7 +5031,8 @@ Host name / IP address to bind to during OAuth authorization.
.IP "Description:" 4
Port number to listen on during OAuth authorization.
-Note: All redirects will go to port \f[I]6414\f[], regardless
+.IP "Note:" 4
+All redirects will go to port \f[I]6414\f[], regardless
of the port specified here. You'll have to manually adjust the
port number in your browser's address bar when using a different
port than the default.
@@ -4841,7 +5048,8 @@ port than the default.
.IP "Description:" 4
Extract additional metadata (\f[I]source\f[], \f[I]uploader\f[])
-Note: This requires 1 additional HTTP request per post.
+.IP "Note:" 4
+This requires 1 additional HTTP request per post.
.SS extractor.patreon.cursor
@@ -4881,8 +5089,7 @@ Start from the position defined by this value.
.IP "Description:" 4
Determines types and order of files to download.
-Available types:
-
+.IP "Available Types:" 4
.br
* \f[I]postfile\f[]
.br
@@ -4905,8 +5112,7 @@ Available types:
.IP "Description:" 4
Selects the format of \f[I]images\f[] \f[I]files\f[].
-Possible formats:
-
+.IP "Available Formats:" 4
.br
* \f[I]download_url\f[] (\f[I]"a":1,"p":1\f[])
.br
@@ -5107,14 +5313,21 @@ Recursively download files from subfolders.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"artworks"\f[],
-\f[I]"avatar"\f[],
-\f[I]"background"\f[],
-\f[I]"favorite"\f[],
-\f[I]"novel-user"\f[],
-\f[I]"novel-bookmark"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]artworks\f[]
+.br
+* \f[I]avatar\f[]
+.br
+* \f[I]background\f[]
+.br
+* \f[I]favorite\f[]
+.br
+* \f[I]novel-user\f[]
+.br
+* \f[I]novel-bookmark\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -5152,7 +5365,8 @@ For works bookmarked by
\f[I]your own account\f[],
fetch bookmark tags as \f[I]tags_bookmark\f[] metadata.
-Note: This requires 1 additional API request per bookmarked post.
+.IP "Note:" 4
+This requires 1 additional API request per bookmarked post.
.SS extractor.pixiv.captions
@@ -5177,7 +5391,8 @@ try to grab the actual \f[I]caption\f[] value using the AJAX API.
.IP "Description:" 4
Fetch \f[I]comments\f[] metadata.
-Note: This requires 1 or more additional API requests per post,
+.IP "Note:" 4
+This requires 1 or more additional API requests per post,
depending on the number of comments.
@@ -5266,7 +5481,8 @@ Try to fetch \f[I]limit_sanity_level\f[] works via web API.
.IP "Description:" 4
Fetch \f[I]comments\f[] metadata.
-Note: This requires 1 or more additional API requests per novel,
+.IP "Note:" 4
+This requires 1 or more additional API requests per novel,
depending on the number of comments.
@@ -5341,7 +5557,8 @@ For novels bookmarked by
\f[I]your own account\f[],
fetch bookmark tags as \f[I]tags_bookmark\f[] metadata.
-Note: This requires 1 additional API request per bookmarked post.
+.IP "Note:" 4
+This requires 1 additional API request per bookmarked post.
.SS extractor.pixiv-novel.refresh-token
@@ -5421,10 +5638,10 @@ to download as mp4 videos.
.IP "Description:" 4
Controls how to handle redirects to CAPTCHA pages.
-.br
-* \f[I]"stop\f[]: Stop the current extractor run.
-.br
-* \f[I]"wait\f[]: Ask the user to solve the CAPTCHA and wait.
+\f[I]"stop\f[]
+Stop the current extractor run.
+\f[I]"wait\f[]
+Ask the user to solve the CAPTCHA and wait.
.SS extractor.readcomiconline.quality
@@ -5499,7 +5716,8 @@ time required when scanning a subreddit.
Retrieve additional comments by resolving the \f[I]more\f[] comment
stubs in the base comment tree.
-Note: This requires 1 additional API call for every 100 extra comments.
+.IP "Note:" 4
+This requires 1 additional API call for every 100 extra comments.
.SS extractor.reddit.embeds
@@ -5576,12 +5794,11 @@ Reddit extractors can recursively visit other submissions
linked to in the initial set of submissions.
This value sets the maximum recursion depth.
-Special values:
-
-.br
-* \f[I]0\f[]: Recursion is disabled
-.br
-* \f[I]-1\f[]: Infinite recursion (don't do this)
+.IP "Special Values:" 4
+\f[I]0\f[]
+Recursion is disabled
+\f[I]-1\f[]
+Infinite recursion (don't do this)
.SS extractor.reddit.refresh-token
@@ -5629,18 +5846,19 @@ Follow links in the original post's \f[I]selftext\f[].
.IP "Description:" 4
Control video download behavior.
-.br
-* \f[I]true\f[]: Download videos and use \f[I]ytdl\f[] to handle
+\f[I]true\f[]
+Download videos and use \f[I]ytdl\f[] to handle
HLS and DASH manifests
-.br
-* \f[I]"ytdl"\f[]: Download videos and let \f[I]ytdl\f[] handle all of
+\f[I]"ytdl"\f[]
+Download videos and let \f[I]ytdl\f[] handle all of
video extraction and download
-.br
-* \f[I]"dash"\f[]: Extract DASH manifest URLs and use \f[I]ytdl\f[]
+\f[I]"dash"\f[]
+Extract DASH manifest URLs and use \f[I]ytdl\f[]
to download and merge them. (*)
-.br
-* \f[I]false\f[]: Ignore videos
+\f[I]false\f[]
+Ignore videos
+.IP "Note:" 4
(*)
This saves 1 HTTP request per video
and might potentially be able to download otherwise deleted videos,
@@ -5658,13 +5876,7 @@ but it will not always get the best video quality available.
\f[I]["hd", "sd", "gif"]\f[]
.IP "Description:" 4
-List of names of the preferred animation format, which can be
-\f[I]"hd"\f[],
-\f[I]"sd"\f[],
-\f[I]"gif"\f[],
-\f[I]"thumbnail"\f[],
-\f[I]"vthumbnail"\f[], or
-\f[I]"poster"\f[].
+List of names of the preferred animation format.`
If a selected format is not available, the next one in the list will be
tried until an available format is found.
@@ -5673,6 +5885,20 @@ If the format is given as \f[I]string\f[], it will be extended with
\f[I]["hd", "sd", "gif"]\f[]. Use a list with one element to
restrict it to only one possible format.
+.IP "Available Formats:" 4
+.br
+* \f[I]"hd"\f[]
+.br
+* \f[I]"sd"\f[]
+.br
+* \f[I]"gif"\f[]
+.br
+* \f[I]"thumbnail"\f[]
+.br
+* \f[I]"vthumbnail"\f[]
+.br
+* \f[I]"poster"\f[]
+
.SS extractor.rule34.api-key & .user-id
.IP "Type:" 6
@@ -5735,7 +5961,7 @@ Refresh download URLs before they expire.
.IP "Description:" 4
Group \f[I]tags\f[] by type and
.br
-provide them as \f[I]tags_TYPE\f[] and \f[I]tag_string_TYPE\f[] metadata fields,
+provide them as \f[I]tags_<type>\f[] and \f[I]tag_string_TYPE\f[] metadata fields,
for example \f[I]tags_artist\f[] and \f[I]tags_character\f[].
.br
@@ -5756,7 +5982,7 @@ Requires:
.br
* 1 additional HTTP request per post
.br
-* logged-in \f[I]cookies\f[]
+* authenticated \f[I]cookies\f[]
to fetch full \f[I]tags\f[] category data
\f[I]false\f[]
@@ -5804,13 +6030,14 @@ To get this value:
.br
* Open your browser's Developer Tools (F12)
.br
-* Select Network -> XHR
+* Select Network → XHR
.br
* Open a gallery page
.br
* Select the last Network entry and copy its \f[I]crt\f[] value
-Note: You will also need your browser's
+.IP "Note:" 4
+You will also need your browser's
\f[I]user-agent\f[]
@@ -5829,10 +6056,17 @@ Name(s) of the image format to download.
When more than one format is given, the first available one is selected.
-Possible formats are
+.IP "Formats:" 4
.br
-\f[I]"780"\f[], \f[I]"980"\f[], \f[I]"1280"\f[], \f[I]"1600"\f[], \f[I]"0"\f[] (original)
+* \f[I]"780"\f[]
+.br
+* \f[I]"980"\f[]
.br
+* \f[I]"1280"\f[]
+.br
+* \f[I]"1600"\f[]
+.br
+* \f[I]"0"\f[] (original)
.SS extractor.schalenetwork.tags
@@ -5962,13 +6196,13 @@ if \f[I]sent-requests\f[] are enabled
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-
+.IP "Supported Values:" 4
.br
-* \f[I]"works"\f[]
+* \f[I]works\f[]
.br
-* \f[I]"sentrequests"\f[]
+* \f[I]sentrequests\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -6030,7 +6264,8 @@ Download video files.
\f[I]true\f[]
.IP "Description:" 4
-Include animated assets when downloading from a list of assets.
+Include \f[I]animated\f[] assets
+when downloading from a list of assets.
.SS extractor.steamgriddb.epilepsy
@@ -6041,7 +6276,8 @@ Include animated assets when downloading from a list of assets.
\f[I]true\f[]
.IP "Description:" 4
-Include assets tagged with epilepsy when downloading from a list of assets.
+Include assets tagged with \f[I]epilepsy\f[]
+when downloading from a list of assets.
.SS extractor.steamgriddb.dimensions
@@ -6054,31 +6290,39 @@ Include assets tagged with epilepsy when downloading from a list of assets.
.IP "Default:" 9
\f[I]"all"\f[]
-.IP "Examples:" 4
+.IP "Example:" 4
.br
-* \f[I]"1024x512,512x512"\f[]
+* "1024x512,512x512"
.br
-* \f[I]["460x215", "920x430"]\f[]
+* ["460x215", "920x430"]
.IP "Description:" 4
Only include assets that are in the specified dimensions. \f[I]all\f[] can be
-used to specify all dimensions. Valid values are:
-
-.br
-* Grids: \f[I]460x215\f[], \f[I]920x430\f[], \f[I]600x900\f[], \f[I]342x482\f[], \f[I]660x930\f[],
-\f[I]512x512\f[], \f[I]1024x1024\f[]
-.br
-* Heroes: \f[I]1920x620\f[], \f[I]3840x1240\f[], \f[I]1600x650\f[]
-.br
-* Logos: N/A (will be ignored)
-.br
-* Icons: \f[I]8x8\f[], \f[I]10x10\f[], \f[I]14x14\f[], \f[I]16x16\f[], \f[I]20x20\f[], \f[I]24x24\f[],
-\f[I]28x28\f[], \f[I]32x32\f[], \f[I]35x35\f[], \f[I]40x40\f[], \f[I]48x48\f[], \f[I]54x54\f[],
-\f[I]56x56\f[], \f[I]57x57\f[], \f[I]60x60\f[], \f[I]64x64\f[], \f[I]72x72\f[], \f[I]76x76\f[],
-\f[I]80x80\f[], \f[I]90x90\f[], \f[I]96x96\f[], \f[I]100x100\f[], \f[I]114x114\f[], \f[I]120x120\f[],
-\f[I]128x128\f[], \f[I]144x144\f[], \f[I]150x150\f[], \f[I]152x152\f[], \f[I]160x160\f[],
-\f[I]180x180\f[], \f[I]192x192\f[], \f[I]194x194\f[], \f[I]256x256\f[], \f[I]310x310\f[],
-\f[I]512x512\f[], \f[I]768x768\f[], \f[I]1024x1024\f[]
+used to specify all dimensions.
+
+.IP "Valid Values:" 4
+Grids
+\f[I]460x215\f[] \f[I]
+\f[I]920x430\f[] \f[]
+\f[I]342x482\f[] \f[I]
+\f[I]600x900\f[] \f[]
+\f[I]660x930\f[] \f[I]
+\f[I]512x512\f[] \f[]
+\f[I]1024x1024\f[]
+Heroes
+\f[I]1600x650\f[] \f[I]
+\f[I]1920x620\f[] \f[]
+\f[I]3840x1240\f[]
+Logos
+N/A (will be ignored)
+Icons
+\f[I]8x8\f[] \f[I] \f[I]10x10\f[] \f[] \f[I]14x14\f[] \f[I] \f[I]16x16\f[] \f[] \f[I]20x20\f[] \f[I] \f[I]24x24\f[] \f[]
+\f[I]28x28\f[] \f[I] \f[I]32x32\f[] \f[] \f[I]35x35\f[] \f[I] \f[I]40x40\f[] \f[] \f[I]48x48\f[] \f[I] \f[I]54x54\f[] \f[]
+\f[I]56x56\f[] \f[I] \f[I]57x57\f[] \f[] \f[I]60x60\f[] \f[I] \f[I]64x64\f[] \f[] \f[I]72x72\f[] \f[I] \f[I]76x76\f[] \f[]
+\f[I]80x80\f[] \f[I] \f[I]90x90\f[] \f[] \f[I]96x96\f[] \f[I] \f[I]100x100\f[] \f[] \f[I]114x114\f[] \f[I] \f[I]120x120\f[] \f[]
+\f[I]128x128\f[] \f[I] \f[I]144x144\f[] \f[] \f[I]150x150\f[] \f[I] \f[I]152x152\f[] \f[] \f[I]160x160\f[] \f[I]
+\f[I]180x180\f[] \f[] \f[I]192x192\f[] \f[I] \f[I]194x194\f[] \f[] \f[I]256x256\f[] \f[I] \f[I]310x310\f[] \f[]
+\f[I]512x512\f[] \f[I] \f[I]768x768\f[] \f[] \f[I]1024x1024\f[]
.SS extractor.steamgriddb.file-types
@@ -6091,24 +6335,25 @@ used to specify all dimensions. Valid values are:
.IP "Default:" 9
\f[I]"all"\f[]
-.IP "Examples:" 4
+.IP "Example:" 4
.br
-* \f[I]"png,jpeg"\f[]
+* "png,jpeg"
.br
-* \f[I]["jpeg", "webp"]\f[]
+* ["jpeg", "webp"]
.IP "Description:" 4
Only include assets that are in the specified file types. \f[I]all\f[] can be
-used to specify all file types. Valid values are:
+used to specify all file types.
-.br
-* Grids: \f[I]png\f[], \f[I]jpeg\f[], \f[I]jpg\f[], \f[I]webp\f[]
-.br
-* Heroes: \f[I]png\f[], \f[I]jpeg\f[], \f[I]jpg\f[], \f[I]webp\f[]
-.br
-* Logos: \f[I]png\f[], \f[I]webp\f[]
-.br
-* Icons: \f[I]png\f[], \f[I]ico\f[]
+.IP "Valid Values:" 4
+Grids
+\f[I]png\f[] \f[I] \f[I]jpeg\f[] \f[] \f[I]jpg\f[] \f[I] \f[I]webp\f[]
+Heroes
+\f[I]png\f[] \f[] \f[I]jpeg\f[] \f[I] \f[I]jpg\f[] \f[] \f[I]webp\f[]
+Logos
+\f[I]png\f[] \f[I] \f[I]webp\f[]
+Icons
+\f[I]png\f[] \f[] \f[I]ico\f[]
.SS extractor.steamgriddb.download-fake-png
@@ -6130,7 +6375,8 @@ Download fake PNGs alongside the real file.
\f[I]true\f[]
.IP "Description:" 4
-Include assets tagged with humor when downloading from a list of assets.
+Include assets tagged with \f[I]humor\f[]
+when downloading from a list of assets.
.SS extractor.steamgriddb.languages
@@ -6143,16 +6389,20 @@ Include assets tagged with humor when downloading from a list of assets.
.IP "Default:" 9
\f[I]"all"\f[]
-.IP "Examples:" 4
+.IP "Example:" 4
.br
-* \f[I]"en,km"\f[]
+* "en,km"
.br
-* \f[I]["fr", "it"]\f[]
+* ["fr", "it"]
.IP "Description:" 4
-Only include assets that are in the specified languages. \f[I]all\f[] can be
-used to specify all languages. Valid values are \f[I]ISO 639-1\f[]
-language codes.
+Only include assets that are in the specified languages.
+
+.IP "Valid Values:" 4
+\f[I]ISO 639-1\f[] codes
+
+.IP "Note:" 4
+\f[I]all\f[] can be used to specify all languages.
.SS extractor.steamgriddb.nsfw
@@ -6171,23 +6421,24 @@ Include assets tagged with adult content when downloading from a list of assets.
\f[I]string\f[]
.IP "Default:" 9
-\f[I]score_desc\f[]
+\f[I]"score_desc"\f[]
.IP "Description:" 4
-Set the chosen sorting method when downloading from a list of assets. Can be one of:
+Set the chosen sorting method when downloading from a list of assets.
+.IP "Supported Values:" 4
.br
-* \f[I]score_desc\f[] (Highest Score (Beta))
+* \f[I]score_desc\f[] (Highest Score (Beta))
.br
-* \f[I]score_asc\f[] (Lowest Score (Beta))
+* \f[I]score_asc\f[] (Lowest Score (Beta))
.br
* \f[I]score_old_desc\f[] (Highest Score (Old))
.br
-* \f[I]score_old_asc\f[] (Lowest Score (Old))
+* \f[I]score_old_asc\f[] (Lowest Score (Old))
.br
-* \f[I]age_desc\f[] (Newest First)
+* \f[I]age_desc\f[] (Newest First)
.br
-* \f[I]age_asc\f[] (Oldest First)
+* \f[I]age_asc\f[] (Oldest First)
.SS extractor.steamgriddb.static
@@ -6201,36 +6452,6 @@ Set the chosen sorting method when downloading from a list of assets. Can be one
Include static assets when downloading from a list of assets.
-.SS extractor.steamgriddb.styles
-.IP "Type:" 6
-.br
-* \f[I]string\f[]
-.br
-* \f[I]list\f[] of \f[I]strings\f[]
-
-.IP "Default:" 9
-\f[I]all\f[]
-
-.IP "Examples:" 4
-.br
-* \f[I]white,black\f[]
-.br
-* \f[I]["no_logo", "white_logo"]\f[]
-
-.IP "Description:" 4
-Only include assets that are in the specified styles. \f[I]all\f[] can be used
-to specify all styles. Valid values are:
-
-.br
-* Grids: \f[I]alternate\f[], \f[I]blurred\f[], \f[I]no_logo\f[], \f[I]material\f[], \f[I]white_logo\f[]
-.br
-* Heroes: \f[I]alternate\f[], \f[I]blurred\f[], \f[I]material\f[]
-.br
-* Logos: \f[I]official\f[], \f[I]white\f[], \f[I]black\f[], \f[I]custom\f[]
-.br
-* Icons: \f[I]official\f[], \f[I]custom\f[]
-
-
.SS extractor.steamgriddb.untagged
.IP "Type:" 6
\f[I]bool\f[]
@@ -6269,8 +6490,7 @@ List of names of the preferred animation format.
If a selected format is not available, the next one in the list will be
tried until a format is found.
-Possible formats include
-
+.IP "Available Formats:" 4
.br
* \f[I]gif\f[]
.br
@@ -6312,12 +6532,12 @@ Possible formats include
.IP "Description:" 4
Controls audio download behavior.
-.br
-* \f[I]true\f[]: Download audio tracks
-.br
-* \f[I]"ytdl"\f[]: Download audio tracks using \f[I]ytdl\f[]
-.br
-* \f[I]false\f[]: Ignore audio tracks
+\f[I]true\f[]
+Download audio tracks
+\f[I]"ytdl"\f[]
+Download audio tracks using \f[I]ytdl\f[]
+\f[I]false\f[]
+Ignore audio tracks
.SS extractor.tiktok.videos
@@ -6461,15 +6681,15 @@ use an extra HTTP request to find the URL to its full-resolution version.
.IP "Description:" 4
Controls how to paginate over blog posts.
-.br
-* \f[I]"api"\f[]: \f[I]next\f[] parameter provided by the API
+\f[I]"api"\f[]
+\f[I]next\f[] parameter provided by the API
(potentially misses posts due to a
\f[I]bug\f[]
in Tumblr's API)
-.br
-* \f[I]"before"\f[]: timestamp of last post
-.br
-* \f[I]"offset"\f[]: post offset number
+\f[I]"before"\f[]
+Timestamp of last post
+\f[I]"offset"\f[]
+Post offset number
.SS extractor.tumblr.ratelimit
@@ -6482,10 +6702,10 @@ in Tumblr's API)
.IP "Description:" 4
Selects how to handle exceeding the daily API rate limit.
-.br
-* \f[I]"abort"\f[]: Raise an error and stop extraction
-.br
-* \f[I]"wait"\f[]: Wait until rate limit reset
+\f[I]"abort"\f[]
+Raise an error and stop extraction
+\f[I]"wait"\f[]
+Wait until rate limit reset
.SS extractor.tumblr.reblogs
@@ -6499,12 +6719,12 @@ Selects how to handle exceeding the daily API rate limit.
\f[I]true\f[]
.IP "Description:" 4
-.br
-* \f[I]true\f[]: Extract media from reblogged posts
-.br
-* \f[I]false\f[]: Skip reblogged posts
-.br
-* \f[I]"same-blog"\f[]: Skip reblogged posts unless the original post
+\f[I]true\f[]
+Extract media from reblogged posts
+\f[I]false\f[]
+Skip reblogged posts
+\f[I]"same-blog"\f[]
+Skip reblogged posts unless the original post
is from the same blog
@@ -6623,13 +6843,12 @@ Fetch media from promoted Tweets.
.IP "Description:" 4
Controls how to handle \f[I]Twitter Cards\f[].
-.br
-* \f[I]false\f[]: Ignore cards
-.br
-* \f[I]true\f[]: Download image content from supported cards
-.br
-* \f[I]"ytdl"\f[]: Additionally download video content from unsupported cards
-using \f[I]ytdl\f[]
+\f[I]false\f[]
+Ignore cards
+\f[I]true\f[]
+Download image content from supported cards
+\f[I]"ytdl"\f[]
+Additionally download video content from unsupported cards using \f[I]ytdl\f[]
.SS extractor.twitter.cards-blacklist
@@ -6683,10 +6902,10 @@ if the given initial Tweet is accessible.
.IP "Description:" 4
Controls how to handle Cross Site Request Forgery (CSRF) tokens.
-.br
-* \f[I]"auto"\f[]: Always auto-generate a token.
-.br
-* \f[I]"cookies"\f[]: Use token given by the \f[I]ct0\f[] cookie if present.
+\f[I]"auto"\f[]
+Always auto-generate a token.
+\f[I]"cookies"\f[]
+Use token given by the \f[I]ct0\f[] cookie if present.
.SS extractor.twitter.cursor
@@ -6735,7 +6954,8 @@ as running \f[I]gallery-dl https://twitter.com/i/web/status/<TweetID>\f[]
with enabled \f[I]conversations\f[] option
for each Tweet in said timeline.
-Note: This requires at least 1 additional API call per initial Tweet.
+.IP "Note:" 4
+This requires at least 1 additional API call per initial Tweet.
.SS extractor.twitter.unavailable
@@ -6770,27 +6990,27 @@ e.g. \f[I]Geoblocked\f[] videos.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-
+.IP "Supported Values:" 4
.br
-* \f[I]"info"\f[]
+* \f[I]info\f[]
.br
-* \f[I]"avatar"\f[]
+* \f[I]avatar\f[]
.br
-* \f[I]"background"\f[]
+* \f[I]background\f[]
.br
-* \f[I]"timeline"\f[]
+* \f[I]timeline\f[]
.br
-* \f[I]"tweets"\f[]
+* \f[I]tweets\f[]
.br
-* \f[I]"media"\f[]
+* \f[I]media\f[]
.br
-* \f[I]"replies"\f[]
+* \f[I]replies\f[]
.br
-* \f[I]"highlights"\f[]
+* \f[I]highlights\f[]
.br
-* \f[I]"likes"\f[]
+* \f[I]likes\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -6815,12 +7035,12 @@ Transform Tweet and User metadata into a simpler, uniform format.
.IP "Description:" 4
Selects the API endpoint used to retrieve single Tweets.
-.br
-* \f[I]"restid"\f[]: \f[I]/TweetResultByRestId\f[] - accessible to guest users
-.br
-* \f[I]"detail"\f[]: \f[I]/TweetDetail\f[] - more stable
-.br
-* \f[I]"auto"\f[]: \f[I]"detail"\f[] when logged in, \f[I]"restid"\f[] otherwise
+\f[I]"restid"\f[]
+\f[I]/TweetResultByRestId\f[] - accessible to guest users
+\f[I]"detail"\f[]
+\f[I]/TweetDetail\f[] - more stable
+\f[I]"auto"\f[]
+\f[I]"detail"\f[] when logged in, \f[I]"restid"\f[] otherwise
.SS extractor.twitter.size
@@ -6899,12 +7119,12 @@ a quoted (original) Tweet when it sees the Tweet which quotes it.
.IP "Description:" 4
Selects how to handle exceeding the API rate limit.
-.br
-* \f[I]"abort"\f[]: Raise an error and stop extraction
-.br
-* \f[I]"wait"\f[]: Wait until rate limit reset
-.br
-* \f[I]"wait:N"\f[]: Wait for \f[I]N\f[] seconds
+\f[I]"abort"\f[]
+Raise an error and stop extraction
+\f[I]"wait"\f[]
+Wait until rate limit reset
+\f[I]"wait:N"\f[]
+Wait for \f[I]N\f[] seconds
.SS extractor.twitter.relogin
@@ -6931,10 +7151,10 @@ try to continue from where it left off.
.IP "Description:" 4
Selects how to handle "account is temporarily locked" errors.
-.br
-* \f[I]"abort"\f[]: Raise an error and stop extraction
-.br
-* \f[I]"wait"\f[]: Wait until the account is unlocked and retry
+\f[I]"abort"\f[]
+Raise an error and stop extraction
+\f[I]"wait"\f[]
+Wait until the account is unlocked and retry
.SS extractor.twitter.replies
@@ -6950,7 +7170,8 @@ Fetch media from replies to other Tweets.
If this value is \f[I]"self"\f[], only consider replies where
reply and original Tweet are from the same user.
-Note: Twitter will automatically expand conversations if you
+.IP "Note:" 4
+Twitter will automatically expand conversations if you
use the \f[I]/with_replies\f[] timeline while logged in. For example,
media from Tweets which the user replied to will also be downloaded.
@@ -7026,14 +7247,14 @@ to accept before stopping.
Controls the strategy / tweet source used for timeline URLs
(\f[I]https://twitter.com/USER/timeline\f[]).
-.br
-* \f[I]"tweets"\f[]: \f[I]/tweets\f[] timeline + search
-.br
-* \f[I]"media"\f[]: \f[I]/media\f[] timeline + search
-.br
-* \f[I]"with_replies"\f[]: \f[I]/with_replies\f[] timeline + search
-.br
-* \f[I]"auto"\f[]: \f[I]"tweets"\f[] or \f[I]"media"\f[], depending on \f[I]retweets\f[] and \f[I]text-tweets\f[] settings
+\f[I]"tweets"\f[]
+\f[I]/tweets\f[] timeline + search
+\f[I]"media"\f[]
+\f[I]/media\f[] timeline + search
+\f[I]"with_replies"\f[]
+\f[I]/with_replies\f[] timeline + search
+\f[I]"auto"\f[]
+\f[I]"tweets"\f[] or \f[I]"media"\f[], depending on \f[I]retweets\f[] and \f[I]text-tweets\f[] settings
.SS extractor.twitter.text-tweets
@@ -7082,7 +7303,7 @@ Alternate Identifier (username, email, phone number)
when \f[I]logging in\f[].
When not specified and asked for by Twitter,
-this identifier will need to entered in an interactive prompt.
+this identifier will need to be entered in an interactive prompt.
.SS extractor.twitter.users
@@ -7096,25 +7317,25 @@ this identifier will need to entered in an interactive prompt.
"https://twitter.com/search?q=from:{legacy[screen_name]}"
.IP "Description:" 4
-Format string for user URLs generated from
+Basic format string for user URLs generated from
.br
\f[I]following\f[] and \f[I]list-members\f[] queries,
whose replacement field values come from Twitter \f[I]user\f[] objects
.br
(\f[I]Example\f[])
-Special values:
-
-.br
-* \f[I]"user"\f[]: \f[I]https://twitter.com/i/user/{rest_id}\f[]
-.br
-* \f[I]"timeline"\f[]: \f[I]https://twitter.com/id:{rest_id}/timeline\f[]
-.br
-* \f[I]"tweets"\f[]: \f[I]https://twitter.com/id:{rest_id}/tweets\f[]
-.br
-* \f[I]"media"\f[]: \f[I]https://twitter.com/id:{rest_id}/media\f[]
+.IP "Special Values:" 4
+\f[I]"user"\f[]
+\f[I]https://twitter.com/i/user/{rest_id}\f[]
+\f[I]"timeline"\f[]
+\f[I]https://twitter.com/id:{rest_id}/timeline\f[]
+\f[I]"tweets"\f[]
+\f[I]https://twitter.com/id:{rest_id}/tweets\f[]
+\f[I]"media"\f[]
+\f[I]https://twitter.com/id:{rest_id}/media\f[]
-Note: To allow gallery-dl to follow custom URL formats, set the \f[I]blacklist\f[]
+.IP "Note:" 4
+To allow gallery-dl to follow custom URL formats, set the \f[I]blacklist\f[]
for \f[I]twitter\f[] to a non-default value, e.g. an empty string \f[I]""\f[].
@@ -7131,12 +7352,12 @@ for \f[I]twitter\f[] to a non-default value, e.g. an empty string \f[I]""\f[].
.IP "Description:" 4
Control video download behavior.
-.br
-* \f[I]true\f[]: Download videos
-.br
-* \f[I]"ytdl"\f[]: Download videos using \f[I]ytdl\f[]
-.br
-* \f[I]false\f[]: Skip video Tweets
+\f[I]true\f[]
+Download videos
+\f[I]"ytdl"\f[]
+Download videos using \f[I]ytdl\f[]
+\f[I]false\f[]
+Skip video Tweets
.SS extractor.unsplash.format
@@ -7149,8 +7370,17 @@ Control video download behavior.
.IP "Description:" 4
Name of the image format to download.
-Available formats are
-\f[I]"raw"\f[], \f[I]"full"\f[], \f[I]"regular"\f[], \f[I]"small"\f[], and \f[I]"thumb"\f[].
+.IP "Available Formats:" 4
+.br
+* \f[I]"raw"\f[]
+.br
+* \f[I]"full"\f[]
+.br
+* \f[I]"regular"\f[]
+.br
+* \f[I]"small"\f[]
+.br
+* \f[I]"thumb"\f[]
.SS extractor.vipergirls.domain
@@ -7176,7 +7406,8 @@ For example \f[I]"viper.click"\f[] if the main domain is blocked or to bypass Cl
.IP "Description:" 4
Automatically like posts after downloading their images.
-Note: Requires \f[I]login\f[]
+.IP "Note:" 4
+Requires \f[I]login\f[]
or \f[I]cookies\f[]
@@ -7228,12 +7459,17 @@ Custom \f[I]offset\f[] starting value when paginating over image results.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"avatar"\f[],
-\f[I]"gallery"\f[],
-\f[I]"spaces"\f[],
-\f[I]"collection"\f[],
+.IP "Supported Values:" 4
+.br
+* \f[I]avatar\f[]
+.br
+* \f[I]gallery\f[]
+.br
+* \f[I]spaces\f[]
+.br
+* \f[I]collection\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -7282,9 +7518,13 @@ See https://wallhaven.cc/help/api for more information.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"uploads"\f[], \f[I]"collections"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]uploads\f[]
+.br
+* \f[I]collections\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -7298,7 +7538,8 @@ It is possible to use \f[I]"all"\f[] instead of listing all values separately.
.IP "Description:" 4
Extract additional metadata (tags, uploader)
-Note: This requires 1 additional HTTP request per post.
+.IP "Note:" 4
+This requires 1 additional HTTP request per post.
.SS extractor.weasyl.api-key
@@ -7327,7 +7568,8 @@ Fetch extra submission metadata during gallery downloads.
.br
\f[I]tags\f[], \f[I]views\f[])
-Note: This requires 1 additional HTTP request per submission.
+.IP "Note:" 4
+This requires 1 additional HTTP request per submission.
.SS extractor.webtoons.quality
@@ -7337,7 +7579,7 @@ Note: This requires 1 additional HTTP request per submission.
.br
* \f[I]string\f[]
.br
-* \f[I]object\f[] (ext -> type)
+* \f[I]object\f[] (ext → type)
.IP "Default:" 9
@@ -7423,14 +7665,21 @@ Set this to \f[I]"video"\f[] to download GIFs as video files.
A (comma-separated) list of subcategories to include
when processing a user profile.
-Possible values are
-\f[I]"home"\f[],
-\f[I]"feed"\f[],
-\f[I]"videos"\f[],
-\f[I]"newvideo"\f[],
-\f[I]"article"\f[],
-\f[I]"album"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]home\f[]
+.br
+* \f[I]feed\f[]
+.br
+* \f[I]videos\f[]
+.br
+* \f[I]newvideo\f[]
+.br
+* \f[I]article\f[]
+.br
+* \f[I]album\f[]
+.IP "Note:" 4
It is possible to use \f[I]"all"\f[] instead of listing all values separately.
@@ -7481,6 +7730,25 @@ will be taken from the original posts, not the retweeted posts.
Download video files.
+.SS extractor.wikimedia.image-revisions
+.IP "Type:" 6
+\f[I]integer\f[]
+
+.IP "Default:" 9
+\f[I]1\f[]
+
+.IP "Description:" 4
+Number of revisions to return for a single image.
+
+The dafault value of 1 only returns the latest revision.
+
+The value must be between 1 and 500.
+
+.IP "Note:" 4
+The API sometimes returns image revisions on article pages even when this option is
+set to 1. However, setting it to a higher value may reduce the number of API requests.
+
+
.SS extractor.wikimedia.limit
.IP "Type:" 6
\f[I]integer\f[]
@@ -7621,7 +7889,8 @@ set subcategory to the input URL's domain.
Route \f[I]ytdl's\f[] output through gallery-dl's logging system.
Otherwise it will be written directly to stdout/stderr.
-Note: Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in
+.IP "Note:" 4
+Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in
\f[I]extractor.ytdl.raw-options\f[] to \f[I]true\f[] to suppress all output.
@@ -7647,7 +7916,7 @@ followed by \f[I]"youtube_dl"\f[] as fallback.
.SS extractor.ytdl.raw-options
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
.. code:: json
@@ -7699,7 +7968,8 @@ List of filename extensions to try when dynamically building download URLs
.IP "Description:" 4
Extract additional metadata (date, md5, tags, ...)
-Note: This requires 1-2 additional HTTP requests per post.
+.IP "Note:" 4
+This requires 1-2 additional HTTP requests per post.
.SS extractor.zerochan.pagination
@@ -7712,11 +7982,11 @@ Note: This requires 1-2 additional HTTP requests per post.
.IP "Description:" 4
Controls how to paginate over tag search results.
-.br
-* \f[I]"api"\f[]: Use the \f[I]JSON API\f[]
+\f[I]"api"\f[]
+Use the \f[I]JSON API\f[]
(no \f[I]extension\f[] metadata)
-.br
-* \f[I]"html"\f[]: Parse HTML pages
+\f[I]"html"\f[]
+Parse HTML pages
(limited to 100 pages * 24 posts)
@@ -7743,7 +8013,8 @@ Group \f[I]tags\f[] by type and
provide them as \f[I]tags_<type>\f[] metadata fields,
for example \f[I]tags_artist\f[] or \f[I]tags_character\f[].
-Note: This requires 1 additional HTTP request per post.
+.IP "Note:" 4
+This requires 1 additional HTTP request per post.
.SS extractor.[booru].notes
@@ -7756,7 +8027,8 @@ Note: This requires 1 additional HTTP request per post.
.IP "Description:" 4
Extract overlay notes (position and text).
-Note: This requires 1 additional HTTP request per post.
+.IP "Note:" 4
+This requires 1 additional HTTP request per post.
.SS extractor.[booru].url
@@ -7791,10 +8063,10 @@ When multiple names are given, download the first available one.
.IP "Description:" 4
Reverse the order of chapter URLs extracted from manga pages.
-.br
-* \f[I]true\f[]: Start with the latest chapter
-.br
-* \f[I]false\f[]: Start with the first chapter
+\f[I]true\f[]
+Start with the latest chapter
+\f[I]false\f[]
+Start with the first chapter
.SS extractor.[manga-extractor].page-reverse
@@ -7861,12 +8133,12 @@ to set file modification times.
.IP "Description:" 4
Controls the use of \f[I].part\f[] files during file downloads.
-.br
-* \f[I]true\f[]: Write downloaded data into \f[I].part\f[] files and rename
+\f[I]true\f[]
+Write downloaded data into \f[I].part\f[] files and rename
them upon download completion. This mode additionally supports
resuming incomplete downloads.
-.br
-* \f[I]false\f[]: Do not use \f[I].part\f[] files and write data directly
+\f[I]false\f[]
+Do not use \f[I].part\f[] files and write data directly
into the actual output files.
@@ -7973,7 +8245,7 @@ Certificate validation during file downloads.
.br
* \f[I]string\f[]
.br
-* \f[I]object\f[] (scheme -> proxy)
+* \f[I]object\f[] (scheme → proxy)
.IP "Default:" 9
\f[I]extractor.*.proxy\f[]
@@ -8044,7 +8316,7 @@ These suffixes are case-insensitive.
.SS downloader.http.headers
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
{"Accept": "image/webp,*/*", "Referer": "https://example.org/"}
@@ -8083,7 +8355,8 @@ regardless of this option.
Number of seconds to sleep when receiving a 429 Too Many Requests
response before \f[I]retrying\f[] the request.
-Note: Requires
+.IP "Note:" 4
+Requires
\f[I]retry-codes\f[]
to include \f[I]429\f[].
@@ -8203,7 +8476,8 @@ Forward gallery-dl's cookies to \f[I]ytdl\f[].
Route \f[I]ytdl's\f[] output through gallery-dl's logging system.
Otherwise it will be written directly to stdout/stderr.
-Note: Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in
+.IP "Note:" 4
+Set \f[I]quiet\f[] and \f[I]no_warnings\f[] in
\f[I]downloader.ytdl.raw-options\f[] to \f[I]true\f[] to suppress all output.
@@ -8243,23 +8517,23 @@ See
/
\f[I]youtube-dl output template\f[].
-Special values:
-
-.br
-* \f[I]null\f[]: generate filenames with \f[I]extractor.*.filename\f[]
-.br
-* \f[I]"default"\f[]: use \f[I]ytdl's\f[] default, currently
+.IP "Special Values:" 4
+\f[I]null\f[]
+generate filenames with \f[I]extractor.*.filename\f[]
+\f[I]"default"\f[]
+use \f[I]ytdl's\f[] default, currently
\f[I]"%(title)s [%(id)s].%(ext)s"\f[] for \f[I]yt-dlp\f[] /
\f[I]"%(title)s-%(id)s.%(ext)s"\f[] for \f[I]youtube-dl\f[]
-Note: An output template other than \f[I]null\f[] might
+.IP "Note:" 4
+An output template other than \f[I]null\f[] might
cause unexpected results in combination with certain options
(e.g. \f[I]"skip": "enumerate"\f[])
.SS downloader.ytdl.raw-options
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
.. code:: json
@@ -8286,7 +8560,7 @@ Available options can be found in
.br
* \f[I]string\f[]
.br
-* \f[I]object\f[] (key -> format string)
+* \f[I]object\f[] (key → format string)
.IP "Default:" 9
\f[I]"auto"\f[]
@@ -8294,17 +8568,18 @@ Available options can be found in
.IP "Description:" 4
Controls the output string format and status indicators.
-.br
-* \f[I]"null"\f[]: No output
-.br
-* \f[I]"pipe"\f[]: Suitable for piping to other processes or files
-.br
-* \f[I]"terminal"\f[]: Suitable for the standard Windows console
-.br
-* \f[I]"color"\f[]: Suitable for terminals that understand ANSI escape codes and colors
-.br
-* \f[I]"auto"\f[]: \f[I]"pipe"\f[] if not on a TTY, \f[I]"terminal"\f[] on Windows with
-\f[I]output.ansi\f[] disabled, \f[I]"color"\f[] otherwise.
+\f[I]"null"\f[]
+No output
+\f[I]"pipe"\f[]
+Suitable for piping to other processes or files
+\f[I]"terminal"\f[]
+Suitable for the standard Windows console
+\f[I]"color"\f[]
+Suitable for terminals that understand ANSI escape codes and colors
+\f[I]"auto"\f[]
+\f[I]"pipe"\f[] if not on a TTY,
+\f[I]"terminal"\f[] on Windows with \f[I]output.ansi\f[] disabled,
+\f[I]"color"\f[] otherwise.
It is possible to use custom output format strings
.br
@@ -8394,7 +8669,8 @@ Possible options are
When this option is specified as a simple \f[I]string\f[],
it is interpreted as \f[I]{"encoding": "<string-value>", "errors": "replace"}\f[]
-Note: \f[I]errors\f[] always defaults to \f[I]"replace"\f[]
+.IP "Note:" 4
+\f[I]errors\f[] always defaults to \f[I]"replace"\f[]
.SS output.shorten
@@ -8414,7 +8690,7 @@ with a display width greater than 1.
.SS output.colors
.IP "Type:" 6
-\f[I]object\f[] (key -> ANSI color)
+\f[I]object\f[] (key → ANSI color)
.IP "Default:" 9
.. code:: json
@@ -8516,13 +8792,13 @@ in the output of \f[I]-K/--list-keywords\f[] and \f[I]-j/--dump-json\f[].
Controls the progress indicator when *gallery-dl* is run with
multiple URLs as arguments.
-.br
-* \f[I]true\f[]: Show the default progress indicator
+\f[I]true\f[]
+Show the default progress indicator
(\f[I]"[{current}/{total}] {url}"\f[])
-.br
-* \f[I]false\f[]: Do not show any progress indicator
-.br
-* Any \f[I]string\f[]: Show the progress indicator using this
+\f[I]false\f[]
+Do not show any progress indicator
+Any \f[I]string\f[]
+Show the progress indicator using this
as a custom \f[I]format string\f[]. Possible replacement keys are
\f[I]current\f[], \f[I]total\f[] and \f[I]url\f[].
@@ -8603,7 +8879,7 @@ before outputting them as JSON.
.SH POSTPROCESSOR OPTIONS
.SS classify.mapping
.IP "Type:" 6
-\f[I]object\f[] (directory -> extensions)
+\f[I]object\f[] (directory → extensions)
.IP "Default:" 9
.. code:: json
@@ -8637,11 +8913,10 @@ in their default location.
.IP "Description:" 4
The action to take when files do **not** compare as equal.
-.br
-* \f[I]"replace"\f[]: Replace/Overwrite the old version with the new one
-
-.br
-* \f[I]"enumerate"\f[]: Add an enumeration index to the filename of the new
+\f[I]"replace"\f[]
+Replace/Overwrite the old version with the new one
+\f[I]"enumerate"\f[]
+Add an enumeration index to the filename of the new
version like \f[I]skip = "enumerate"\f[]
@@ -8655,17 +8930,15 @@ version like \f[I]skip = "enumerate"\f[]
.IP "Description:" 4
The action to take when files do compare as equal.
-.br
-* \f[I]"abort:N"\f[]: Stop the current extractor run
+\f[I]"abort:N"\f[]
+Stop the current extractor run
after \f[I]N\f[] consecutive files compared as equal.
-
-.br
-* \f[I]"terminate:N"\f[]: Stop the current extractor run,
+\f[I]"terminate:N"\f[]
+Stop the current extractor run,
including parent extractors,
after \f[I]N\f[] consecutive files compared as equal.
-
-.br
-* \f[I]"exit:N"\f[]: Exit the program
+\f[I]"exit:N"\f[]
+Exit the program
after \f[I]N\f[] consecutive files compared as equal.
@@ -8691,7 +8964,7 @@ Only compare file sizes. Do not read and compare their content.
\f[I]"prepare"\f[]
.IP "Description:" 4
-The event(s) for which \f[I]directory\f[] format strings are (re)evaluated.
+The event(s) for which \f[I]directory\f[] \f[I]Format Strings\f[] are (re)evaluated.
See \f[I]metadata.event\f[] for a list of available events.
@@ -8756,7 +9029,8 @@ with the full path of a file or target directory, depending on
.br
* If this is a \f[I]list\f[], the first element specifies the program
name and any further elements its arguments.
-Each element of this list is treated as a \f[I]format string\f[] using
+
+Each element of this list is evaluated as a \f[I]Format String\f[] using
the files' metadata as well as \f[I]{_path}\f[], \f[I]{_directory}\f[],
and \f[I]{_filename}\f[].
@@ -8863,7 +9137,7 @@ Rebuild \f[I]filenames\f[] after computing
.br
* \f[I]string\f[]
.br
-* \f[I]object\f[] (field name -> hash algorithm)
+* \f[I]object\f[] (field name → hash algorithm)
.IP "Default:" 9
\f[I]"md5,sha1"\f[]
@@ -8919,25 +9193,24 @@ for hash digests to compute.
.IP "Description:" 4
Selects how to process metadata.
-.br
-* \f[I]"json"\f[]: write metadata using \f[I]json.dump()\f[]
-.br
-* \f[I]"jsonl"\f[]: write metadata in \f[I]JSON Lines
-<https://jsonlines.org/>\f[] format
-.br
-* \f[I]"tags"\f[]: write \f[I]tags\f[] separated by newlines
-.br
-* \f[I]"custom"\f[]: write the result of applying \f[I]metadata.content-format\f[]
+\f[I]"json"\f[]
+Write metadata using \f[I]json.dump()\f[]
+\f[I]"jsonl"\f[]
+Write metadata in \f[I]JSON Lines\f[] format
+\f[I]"tags"\f[]
+Write \f[I]tags\f[] separated by newlines
+\f[I]"custom"\f[]
+Write the result of applying \f[I]metadata.content-format\f[]
to a file's metadata dictionary
-.br
-* \f[I]"modify"\f[]: add or modify metadata entries
-.br
-* \f[I]"delete"\f[]: remove metadata entries
+\f[I]"modify"\f[]
+Add or modify metadata entries
+\f[I]"delete"\f[]
+Remove metadata entries
.SS metadata.filename
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Default:" 9
\f[I]null\f[]
@@ -8946,7 +9219,7 @@ to a file's metadata dictionary
"{id}.data.json"
.IP "Description:" 4
-A \f[I]format string\f[] to build the filenames for metadata files with.
+A \f[I]Format String\f[] to generate filenames for metadata files.
(see \f[I]extractor.filename\f[])
Using \f[I]"-"\f[] as filename will write all output to \f[I]stdout\f[].
@@ -8989,12 +9262,12 @@ relative to \f[I]metadata.base-directory\f[].
.IP "Description:" 4
Selects the relative location for metadata files.
-.br
-* \f[I]false\f[]: current target location for file downloads (\f[I]base-directory\f[] + directory_)
-.br
-* \f[I]true\f[]: current \f[I]base-directory\f[] location
-.br
-* any \f[I]Path\f[]: custom location
+\f[I]false\f[]
+Current target location for file downloads (\f[I]base-directory\f[] + directory_)
+\f[I]true\f[]
+Current \f[I]base-directory\f[] location
+any \f[I]Path\f[]
+Custom location
.SS metadata.extension
@@ -9011,7 +9284,7 @@ original file names.
.SS metadata.extension-format
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Example:" 4
.br
@@ -9020,10 +9293,11 @@ original file names.
* "json"
.IP "Description:" 4
-Custom format string to build filename extensions for metadata
-files with, which will replace the original filename extensions.
+Custom \f[I]Format String\f[] to generate filename extensions
+for metadata files, which will replace the original filename extension.
-Note: \f[I]metadata.extension\f[] is ignored if this option is set.
+.IP "Note:" 4
+When this option is set, \f[I]metadata.extension\f[] is ignored.
.SS metadata.metadata-path
@@ -9099,7 +9373,8 @@ After downloading all files of a post
.IP "Description:" 4
Include only the given top-level keys when writing JSON data.
-Note: Missing or undefined fields will be silently ignored.
+.IP "Note:" 4
+Missing or undefined fields will be silently ignored.
.SS metadata.exclude
@@ -9112,7 +9387,8 @@ Note: Missing or undefined fields will be silently ignored.
.IP "Description:" 4
Exclude all given keys from written JSON data.
-Note: Cannot be used with \f[I]metadata.include\f[].
+.IP "Note:" 4
+Cannot be used with \f[I]metadata.include\f[].
.SS metadata.fields
@@ -9120,7 +9396,7 @@ Note: Cannot be used with \f[I]metadata.include\f[].
.br
* \f[I]list\f[] of \f[I]strings\f[]
.br
-* \f[I]object\f[] (field name -> \f[I]format string\f[])
+* \f[I]object\f[] (field name → \f[I]Format String\f[])
.IP "Example:" 4
.. code:: json
@@ -9137,21 +9413,19 @@ Note: Cannot be used with \f[I]metadata.include\f[].
.IP "Description:" 4
-.br
-* \f[I]"mode": "delete"\f[]:
+\f[I]"mode": "delete"\f[]
A list of metadata field names to remove.
-.br
-* \f[I]"mode": "modify"\f[]:
-An object with metadata field names mapping to a \f[I]format string\f[]
-whose result is assigned to said field name.
+\f[I]"mode": "modify"\f[]
+An object with metadata field names mapping to a \f[I]Format String\f[]
+whose result is assigned to that field name.
.SS metadata.content-format
.IP "Type:" 6
.br
-* \f[I]string\f[]
+* \f[I]Format String\f[]
.br
-* \f[I]list\f[] of \f[I]strings\f[]
+* \f[I]list\f[] of \f[I]Format Strings\f[]
.IP "Example:" 4
.br
@@ -9160,9 +9434,10 @@ whose result is assigned to said field name.
* ["tags:", "", "{tags:J\\n}"]
.IP "Description:" 4
-Custom format string to build the content of metadata files with.
+Custom \f[I]Format String(s)\f[] to build the content of metadata files with.
-Note: Only applies for \f[I]"mode": "custom"\f[].
+.IP "Note:" 4
+Only applies to \f[I]"mode": "custom"\f[].
.SS metadata.ascii
@@ -9177,7 +9452,8 @@ Escape all non-ASCII characters.
See the \f[I]ensure_ascii\f[] argument of \f[I]json.dump()\f[] for further details.
-Note: Only applies for \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[].
+.IP "Note:" 4
+Only applies to \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[].
.SS metadata.indent
@@ -9195,7 +9471,8 @@ Indentation level of JSON output.
See the \f[I]indent\f[] argument of \f[I]json.dump()\f[] for further details.
-Note: Only applies for \f[I]"mode": "json"\f[].
+.IP "Note:" 4
+Only applies to \f[I]"mode": "json"\f[].
.SS metadata.separators
@@ -9211,7 +9488,8 @@ to separate JSON keys and values with.
See the \f[I]separators\f[] argument of \f[I]json.dump()\f[] for further details.
-Note: Only applies for \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[].
+.IP "Note:" 4
+Only applies to \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[].
.SS metadata.sort
@@ -9226,7 +9504,8 @@ Sort output by key.
See the \f[I]sort_keys\f[] argument of \f[I]json.dump()\f[] for further details.
-Note: Only applies for \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[].
+.IP "Note:" 4
+Only applies to \f[I]"mode": "json"\f[] and \f[I]"jsonl"\f[].
.SS metadata.open
@@ -9358,12 +9637,13 @@ Name of the metadata field whose value should be used.
This value must be either a UNIX timestamp or a
\f[I]datetime\f[] object.
-Note: This option gets ignored if \f[I]mtime.value\f[] is set.
+.IP "Note:" 4
+This option is ignored if \f[I]mtime.value\f[] is set.
.SS mtime.value
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Default:" 9
\f[I]null\f[]
@@ -9375,7 +9655,7 @@ Note: This option gets ignored if \f[I]mtime.value\f[] is set.
* "{content[0:6]:R22/2022/D%Y%m%d/}"
.IP "Description:" 4
-A \f[I]format string\f[] whose value should be used.
+The \f[I]Format String\f[] whose value should be used.
The resulting value must be either a UNIX timestamp or a
\f[I]datetime\f[] object.
@@ -9422,7 +9702,7 @@ See \f[I]metadata.event\f[] for a list of available events.
.SS python.expression
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Expression\f[]
.IP "Example:" 4
.br
@@ -9431,13 +9711,11 @@ See \f[I]metadata.event\f[] for a list of available events.
* "terminate()"
.IP "Description:" 4
-A
-\f[I]Python expression\f[]
-to
+A Python \f[I]Expression\f[] to
\f[I]evaluate\f[].
-Note: Only used with
-\f[I]"mode": "eval"\f[]
+.IP "Note:" 4
+Only used with \f[I]"mode": "eval"\f[]
.SS python.function
@@ -9482,20 +9760,20 @@ Call a
.SS rename.from
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Description:" 4
-The \f[I]format string\f[] for filenames to rename.
+The \f[I]Format String\f[] for filenames to rename.
When no value is given, \f[I]extractor.*.filename\f[] is used.
.SS rename.to
.IP "Type:" 6
-\f[I]string\f[]
+\f[I]Format String\f[]
.IP "Description:" 4
-The \f[I]format string\f[] for target filenames.
+The \f[I]Format String\f[] for target filenames.
When no value is given, \f[I]extractor.*.filename\f[] is used.
@@ -9541,23 +9819,28 @@ Additional \f[I]ffmpeg\f[] command-line arguments.
\f[I]string\f[]
.IP "Default:" 9
-\f[I]auto\f[]
+\f[I]"auto"\f[]
.IP "Description:" 4
\f[I]ffmpeg\f[] demuxer to read and process input files with.
-Possible values are
-
+.IP "Supported Values:" 4
+\f[I]"auto"\f[]
+use \f[I]mkvmerge\f[] if available, fall back to \f[I]concat\f[] otherwise
+\f[I]"concat"\f[]
+https://ffmpeg.org/ffmpeg-formats.html#concat-1
.br
-* "\f[I]concat\f[]" (inaccurate frame timecodes for non-uniform frame delays)
+Inaccurate frame timecodes for non-uniform frame delays
.br
-* "\f[I]image2\f[]" (accurate timecodes, requires nanosecond file timestamps, i.e. no Windows or macOS)
+\f[I]"image2"\f[]
+https://ffmpeg.org/ffmpeg-formats.html#image2-1
.br
-* "mkvmerge" (accurate timecodes, only WebM or MKV, requires \f[I]mkvmerge\f[])
+Accurate timecodes, requires nanosecond file timestamps, i.e. no Windows or macOS)
.br
-* "archive" (store "original" frames in a \f[I].zip\f[] archive)
-
-"auto" will select mkvmerge if available and fall back to concat otherwise.
+\f[I]"mkvmerge"\f[]
+Accurate timecodes, only WebM or MKV, requires \f[I]mkvmerge\f[])
+\f[I]"archive"\f[]
+Store "original" frames in a \f[I].zip\f[] archive
.SS ugoira.ffmpeg-location
@@ -9596,12 +9879,12 @@ Location of the \f[I]mkvmerge\f[] executable for use with the
.IP "Description:" 4
Controls \f[I]ffmpeg\f[] output.
-.br
-* \f[I]true\f[]: Enable \f[I]ffmpeg\f[] output
-.br
-* \f[I]false\f[]: Disable all \f[I]ffmpeg\f[] output
-.br
-* any \f[I]string\f[]: Pass \f[I]-hide_banner\f[] and \f[I]-loglevel\f[]
+\f[I]true\f[]
+Enable \f[I]ffmpeg\f[] output
+\f[I]false\f[]
+Disable all \f[I]ffmpeg\f[] output
+any \f[I]string\f[]
+Pass \f[I]-hide_banner\f[] and \f[I]-loglevel\f[]
with this value as argument to \f[I]ffmpeg\f[]
@@ -9626,16 +9909,16 @@ Enable Two-Pass encoding.
.IP "Description:" 4
Controls the frame rate argument (\f[I]-r\f[]) for \f[I]ffmpeg\f[]
-.br
-* \f[I]"auto"\f[]: Automatically assign a fitting frame rate
+\f[I]"auto"\f[]
+Automatically assign a fitting frame rate
based on delays between frames.
-.br
-* \f[I]"uniform"\f[]: Like \f[I]auto\f[], but assign an explicit frame rate
+\f[I]"uniform"\f[]
+Like \f[I]auto\f[], but assign an explicit frame rate
only to Ugoira with uniform frame delays.
-.br
-* any other \f[I]string\f[]: Use this value as argument for \f[I]-r\f[].
-.br
-* \f[I]null\f[] or an empty \f[I]string\f[]: Don't set an explicit frame rate.
+any other \f[I]string\f[]
+Use this value as argument for \f[I]-r\f[].
+\f[I]null\f[] or an empty \f[I]string\f[]
+Don't set an explicit frame rate.
.SS ugoira.keep-files
@@ -9731,7 +10014,15 @@ Do not convert frames if target file already exists.
.IP "Description:" 4
Compression method to use when writing the archive.
-Possible values are \f[I]"store"\f[], \f[I]"zip"\f[], \f[I]"bzip2"\f[], \f[I]"lzma"\f[].
+.IP "Supported Values:" 4
+.br
+* \f[I]"store"\f[]
+.br
+* \f[I]"zip"\f[]
+.br
+* \f[I]"bzip2"\f[]
+.br
+* \f[I]"lzma"\f[]
.SS zip.extension
@@ -9755,7 +10046,8 @@ Filename extension for the created ZIP archive.
.IP "Description:" 4
List of extra files to be added to a ZIP archive.
-Note: Relative paths are relative to the current
+.IP "Note:" 4
+Relative paths are relative to the current
\f[I]download directory\f[].
@@ -9778,12 +10070,12 @@ Keep the actual files after writing them to a ZIP archive.
\f[I]"default"\f[]
.IP "Description:" 4
-.br
-* \f[I]"default"\f[]: Write the central directory file header
+\f[I]"default"\f[]
+Write the central directory file header
once after everything is done or an exception is raised.
-.br
-* \f[I]"safe"\f[]: Update the central directory file header
+\f[I]"safe"\f[]
+Update the central directory file header
each time a file is stored in a ZIP archive.
This greatly reduces the chance a ZIP archive gets corrupted in
@@ -9823,7 +10115,8 @@ gets \f[I]imported\f[]
and searched for potential extractors,
i.e. classes with a \f[I]pattern\f[] attribute.
-Note: \f[I]null\f[] references internal extractors defined in
+.IP "Note:" 4
+\f[I]null\f[] references internal extractors defined in
\f[I]extractor/__init__.py\f[]
or by \f[I]extractor.modules\f[].
@@ -9831,7 +10124,7 @@ or by \f[I]extractor.modules\f[].
.SS extractor.category-map
.IP "Type:" 6
.br
-* \f[I]object\f[] (category -> category)
+* \f[I]object\f[] (category → category)
.br
* \f[I]string\f[]
@@ -9846,8 +10139,7 @@ or by \f[I]extractor.modules\f[].
.IP "Description:" 4
A JSON object mapping category names to their replacements.
-Special values:
-
+.IP "Special Values:" 4
.br
* \f[I]"compat"\f[]
.. code:: json
@@ -9869,7 +10161,7 @@ Special values:
.SS extractor.config-map
.IP "Type:" 6
-\f[I]object\f[] (category -> category)
+\f[I]object\f[] (category → category)
.IP "Default:" 9
.. code:: json
@@ -9895,7 +10187,7 @@ For example, a \f[I]"naver": "naver-blog"\f[] key-value pair will make all
.SS jinja.environment
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
.. code:: json
@@ -9914,7 +10206,7 @@ object.
.SS jinja.policies
.IP "Type:" 6
-\f[I]object\f[] (name -> value)
+\f[I]object\f[] (name → value)
.IP "Example:" 4
.. code:: json
@@ -9955,7 +10247,7 @@ A Python \f[I]Module\f[] containing custom \f[I]jinja\f[]
A Python \f[I]Module\f[] whose namespace,
in addition to the \f[I]GLOBALS\f[] dict in
\f[I]util.py\f[],
-is used as \f[I]globals parameter\f[] for compiled Python expressions.
+is used as \f[I]globals parameter\f[] for compiled Expressions_.
.SS cache.file
@@ -9987,15 +10279,15 @@ this cache.
\f[I]true\f[]
.IP "Description:" 4
-Evaluate filter expressions in a special environment
+Evaluate \f[I]Expressions\f[] in a special environment
preventing them from raising fatal exceptions.
-\f[I]true\f[] or \f[I]"tryexcept"\f[]:
+\f[I]true\f[] \f[I] \f[I]"tryexcept"\f[]
Wrap expressions in a try/except block;
Evaluate expressions raising an exception as \f[I]false\f[]
-\f[I]false\f[] or \f[I]"raw"\f[]:
+\f[I]false\f[] \f[] \f[I]"raw"\f[]
Do not wrap expressions in a special environment
-\f[I]"defaultdict"\f[]:
+\f[I]"defaultdict"\f[]
Prevent exceptions when accessing undefined variables
by using a \f[I]defaultdict\f[]
@@ -10008,7 +10300,7 @@ by using a \f[I]defaultdict\f[]
\f[I]"/"\f[]
.IP "Description:" 4
-Character(s) used as argument separator in format string
+Character(s) used as argument separator in \f[I]Format String\f[]
\f[I]format specifiers\f[].
For example, setting this option to \f[I]"#"\f[] would allow a replacement
@@ -10041,7 +10333,7 @@ as signal handler for.
.SS signals-actions
.IP "Type:" 6
-\f[I]object\f[] (signal -> \f[I]Action(s)\f[])
+\f[I]object\f[] (signal → \f[I]Action(s)\f[])
.IP "Example:" 4
.. code:: json
@@ -10378,7 +10670,7 @@ Simple \f[I]tilde expansion\f[]
and \f[I]environment variable expansion\f[]
is supported.
-.IP "Note::" 4
+.IP "Note:" 4
In Windows environments,
both backslashes \f[I]\\\f[] as well as forward slashes \f[I]/\f[]
can be used as path separators.
@@ -10475,7 +10767,8 @@ use \f[I]"w"\f[] to truncate or \f[I]"a"\f[] to append
.br
* Default: \f[I]"utf-8"\f[]
-Note: path, mode, and encoding are only applied when configuring
+.IP "Note:" 4
+path, mode, and encoding are only applied when configuring
logging output to a file.
@@ -10502,9 +10795,9 @@ logging output to a file.
An \f[I]object\f[] containing a \f[I]"name"\f[] attribute specifying the
post-processor type, as well as any of its \f[I]options\f[].
-It is possible to set a \f[I]"filter"\f[] expression similar to
-\f[I]image-filter\f[] to only run a post-processor
-conditionally.
+It is possible to set a \f[I]"filter"\f[] \f[I]Condition\f[] similar to
+\f[I]image-filter\f[]
+to only run a post-processor conditionally.
It is also possible set a \f[I]"whitelist"\f[] or \f[I]"blacklist"\f[] to
only enable or disable a post-processor for the specified
@@ -10520,7 +10813,7 @@ Compare versions of the same file and replace/enumerate them on mismatch
(requires \f[I]downloader.*.part\f[] = \f[I]true\f[] and \f[I]extractor.*.skip\f[] = \f[I]false\f[])
.br
\f[I]directory\f[]
-Reevaluate \f[I]directory\f[] format strings
+Reevaluate \f[I]directory\f[] \f[I]Format Strings\f[]
\f[I]exec\f[]
Execute external commands
\f[I]hash\f[]
@@ -10619,6 +10912,80 @@ wait until Enter is pressed when no argument was given.
Exit the program with the given argument as exit status.
+.SS Expression
+.IP "Type:" 6
+\f[I]string\f[]
+
+.IP "Example:" 4
+.br
+* "1 + 2 + 3"
+.br
+* "str(id) + '_' + title"
+.br
+* "' - '.join(tags[:3]) if tags else 'no tags'"
+
+.IP "Description:" 4
+A Python \f[I]Expression\f[] is a combination of
+values, variables, operators, and function calls
+that evaluate to a single value.
+
+.IP "Reference:" 4
+.br
+* https://docs.python.org/3/reference/expressions.html
+
+
+.SS Condition
+.IP "Type:" 6
+.br
+* \f[I]Expression\f[]
+.br
+* \f[I]list\f[] of \f[I]Expressions\f[]
+
+.IP "Example:" 4
+.br
+* "not is_watching"
+.br
+* "locals().get('optional')"
+.br
+* "date >= datetime(2025, 7, 1) or abort()"
+.br
+* ["width > 800", "0.9 < width/height < 1.1"]
+
+.IP "Description:" 4
+A \f[I]Condition\f[] is an \f[I]Expression\f[]
+whose result is evaluated as a
+\f[I]boolean\f[]
+value.
+
+
+.SS Format String
+.IP "Type:" 6
+\f[I]string\f[]
+
+.IP "Example:" 4
+.br
+* "foo"
+.br
+* "{username}"
+.br
+* "{title} ({id}).{extension}"
+.br
+* "\\fF {title.title()} ({num:>0:>0{len(str(a))}} / {count}).{extension}"
+
+.IP "Description:" 4
+A \f[I]Format String\f[] allows creating dynamic text
+by embedding metadata values directly into replacement fields
+marked by curly braces \f[I]{...}\f[].
+
+.IP "Reference:" 4
+.br
+* \f[I]docs/formatting\f[]
+.br
+* https://docs.python.org/3/library/string.html#formatstrings
+.br
+* https://docs.python.org/3/library/string.html#formatspec
+
+
.SH BUGS
https://github.com/mikf/gallery-dl/issues
diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf
index 5a1b604..2205ecb 100644
--- a/docs/gallery-dl.conf
+++ b/docs/gallery-dl.conf
@@ -455,6 +455,8 @@
"order-posts": "asc",
"previews" : false,
"videos" : true,
+ "warn-images": true,
+ "warn-videos": true,
"stories": {
"split": false
@@ -521,6 +523,14 @@
"lang": null,
"ratings": ["safe", "suggestive", "erotica", "pornographic"]
},
+ "mangafire":
+ {
+ "lang": "en"
+ },
+ "mangareader":
+ {
+ "lang": "en"
+ },
"mangoxo":
{
"username": "",
@@ -1112,6 +1122,7 @@
"wikimedia":
{
"sleep-request": "1.0-2.0",
+ "image-revisions": 1,
"limit": 50,
"subcategories": true
},
diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO
index a339b24..6878d26 100644
--- a/gallery_dl.egg-info/PKG-INFO
+++ b/gallery_dl.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: gallery_dl
-Version: 1.30.8
+Version: 1.30.9
Summary: Command-line program to download image galleries and collections from several image hosting sites
Home-page: https://github.com/mikf/gallery-dl
Download-URL: https://github.com/mikf/gallery-dl/releases/latest
@@ -141,9 +141,9 @@ Standalone Executable
Prebuilt executable files with a Python interpreter and
required Python packages included are available for
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.exe>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.exe>`__
(Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__)
-- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.8/gallery-dl.bin>`__
+- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.30.9/gallery-dl.bin>`__
Nightly Builds
diff --git a/gallery_dl.egg-info/SOURCES.txt b/gallery_dl.egg-info/SOURCES.txt
index 93a6880..336ef00 100644
--- a/gallery_dl.egg-info/SOURCES.txt
+++ b/gallery_dl.egg-info/SOURCES.txt
@@ -158,11 +158,13 @@ gallery_dl/extractor/luscious.py
gallery_dl/extractor/lynxchan.py
gallery_dl/extractor/madokami.py
gallery_dl/extractor/mangadex.py
+gallery_dl/extractor/mangafire.py
gallery_dl/extractor/mangafox.py
gallery_dl/extractor/mangahere.py
gallery_dl/extractor/manganelo.py
gallery_dl/extractor/mangapark.py
gallery_dl/extractor/mangaread.py
+gallery_dl/extractor/mangareader.py
gallery_dl/extractor/mangataro.py
gallery_dl/extractor/mangoxo.py
gallery_dl/extractor/mastodon.py
@@ -215,6 +217,7 @@ gallery_dl/extractor/redgifs.py
gallery_dl/extractor/rule34us.py
gallery_dl/extractor/rule34vault.py
gallery_dl/extractor/rule34xyz.py
+gallery_dl/extractor/s3ndpics.py
gallery_dl/extractor/saint.py
gallery_dl/extractor/sankaku.py
gallery_dl/extractor/sankakucomplex.py
diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py
index 5d6c3d7..6c19e23 100644
--- a/gallery_dl/cookies.py
+++ b/gallery_dl/cookies.py
@@ -27,8 +27,11 @@ from . import aes, text, util
SUPPORTED_BROWSERS_CHROMIUM = {
"brave", "chrome", "chromium", "edge", "opera", "thorium", "vivaldi"}
SUPPORTED_BROWSERS_FIREFOX = {"firefox", "librewolf", "zen"}
+SUPPORTED_BROWSERS_WEBKIT = {"safari", "orion"}
SUPPORTED_BROWSERS = \
- SUPPORTED_BROWSERS_CHROMIUM | SUPPORTED_BROWSERS_FIREFOX | {"safari"}
+ SUPPORTED_BROWSERS_CHROMIUM \
+ | SUPPORTED_BROWSERS_FIREFOX \
+ | SUPPORTED_BROWSERS_WEBKIT
logger = logging.getLogger("cookies")
@@ -38,8 +41,8 @@ def load_cookies(browser_specification):
_parse_browser_specification(*browser_specification)
if browser_name in SUPPORTED_BROWSERS_FIREFOX:
return load_cookies_firefox(browser_name, profile, container, domain)
- elif browser_name == "safari":
- return load_cookies_safari(profile, domain)
+ elif browser_name in SUPPORTED_BROWSERS_WEBKIT:
+ return load_cookies_webkit(browser_name, profile, domain)
elif browser_name in SUPPORTED_BROWSERS_CHROMIUM:
return load_cookies_chromium(browser_name, profile, keyring, domain)
else:
@@ -92,7 +95,7 @@ def load_cookies_firefox(browser_name, profile=None,
return cookies
-def load_cookies_safari(profile=None, domain=None):
+def load_cookies_webkit(browser_name, profile=None, domain=None):
"""Ref.: https://github.com/libyal/dtformats/blob
/main/documentation/Safari%20Cookies.asciidoc
- This data appears to be out of date
@@ -100,15 +103,24 @@ def load_cookies_safari(profile=None, domain=None):
- There are a few bytes here and there
which are skipped during parsing
"""
- with _safari_cookies_database() as fp:
- data = fp.read()
- page_sizes, body_start = _safari_parse_cookies_header(data)
+ if browser_name == "safari":
+ with _safari_cookies_database() as fp:
+ data = fp.read()
+ elif browser_name == "orion":
+ with _orion_cookies_database() as fp:
+ data = fp.read()
+ else:
+ raise ValueError(f"unknown webkit browser '{browser_name}'")
+
+ page_sizes, body_start = _webkit_parse_cookies_header(data)
p = DataParser(data[body_start:])
cookies = []
for page_size in page_sizes:
- _safari_parse_cookies_page(p.read_bytes(page_size), cookies)
- _log_info("Extracted %s cookies from Safari", len(cookies))
+ _webkit_parse_cookies_page(p.read_bytes(page_size), cookies)
+ _log_info("Extracted %s cookies from %s",
+ browser_name.capitalize(), len(cookies))
+
return cookies
@@ -278,7 +290,8 @@ def _firefox_browser_directory(browser_name):
# --------------------------------------------------------------------
-# safari
+# safari/orion/webkit
+
def _safari_cookies_database():
try:
@@ -291,7 +304,13 @@ def _safari_cookies_database():
return open(path, "rb")
-def _safari_parse_cookies_header(data):
+def _orion_cookies_database():
+ path = os.path.expanduser(
+ "~/Library/HTTPStorages/com.kagi.kagimacOS.binarycookies")
+ return open(path, "rb")
+
+
+def _webkit_parse_cookies_header(data):
p = DataParser(data)
p.expect_bytes(b"cook", "database signature")
number_of_pages = p.read_uint(big_endian=True)
@@ -300,7 +319,7 @@ def _safari_parse_cookies_header(data):
return page_sizes, p.cursor
-def _safari_parse_cookies_page(data, cookies, domain=None):
+def _webkit_parse_cookies_page(data, cookies, domain=None):
p = DataParser(data)
p.expect_bytes(b"\x00\x00\x01\x00", "page signature")
number_of_cookies = p.read_uint()
@@ -313,13 +332,13 @@ def _safari_parse_cookies_page(data, cookies, domain=None):
for i, record_offset in enumerate(record_offsets):
p.skip_to(record_offset, "space between records")
- record_length = _safari_parse_cookies_record(
+ record_length = _webkit_parse_cookies_record(
data[record_offset:], cookies, domain)
p.read_bytes(record_length)
p.skip_to_end("space in between pages")
-def _safari_parse_cookies_record(data, cookies, host=None):
+def _webkit_parse_cookies_record(data, cookies, host=None):
p = DataParser(data)
record_size = p.read_uint()
p.skip(4, "unknown record field 1")
@@ -355,7 +374,7 @@ def _safari_parse_cookies_record(data, cookies, host=None):
p.skip_to(value_offset)
value = p.read_cstring()
except UnicodeDecodeError:
- _log_warning("Failed to parse Safari cookie")
+ _log_warning("Failed to parse WebKit cookie")
return record_size
p.skip_to(record_size, "space at the end of the record")
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 111fd9b..248bf70 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -413,7 +413,7 @@ class HttpDownloader(DownloaderBase):
def _find_extension(self, response):
"""Get filename extension from MIME type"""
mtype = response.headers.get("Content-Type", "image/jpeg")
- mtype = mtype.partition(";")[0]
+ mtype = mtype.partition(";")[0].lower()
if "/" not in mtype:
mtype = "image/" + mtype
@@ -475,6 +475,10 @@ MIME_TYPES = {
"audio/ogg" : "ogg",
"audio/mpeg" : "mp3",
+ "application/vnd.apple.mpegurl": "m3u8",
+ "application/x-mpegurl" : "m3u8",
+ "application/dash+xml" : "mpd",
+
"application/zip" : "zip",
"application/x-zip": "zip",
"application/x-zip-compressed": "zip",
@@ -526,6 +530,8 @@ SIGNATURE_CHECKS = {
s[8:12] == b"WAVE"),
"mp3" : lambda s: (s[0:3] == b"ID3" or
s[0:2] in (b"\xFF\xFB", b"\xFF\xF3", b"\xFF\xF2")),
+ "m3u8": lambda s: s[0:7] == b"#EXTM3U",
+ "mpd" : lambda s: b"<MPD" in s,
"zip" : lambda s: s[0:4] in (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"),
"rar" : lambda s: s[0:6] == b"Rar!\x1A\x07",
"7z" : lambda s: s[0:6] == b"\x37\x7A\xBC\xAF\x27\x1C",
diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py
index abdb6cc..a3df634 100644
--- a/gallery_dl/extractor/__init__.py
+++ b/gallery_dl/extractor/__init__.py
@@ -115,11 +115,13 @@ modules = [
"lynxchan",
"madokami",
"mangadex",
+ "mangafire",
"mangafox",
"mangahere",
"manganelo",
"mangapark",
"mangaread",
+ "mangareader",
"mangataro",
"mangoxo",
"misskey",
@@ -166,6 +168,7 @@ modules = [
"rule34us",
"rule34vault",
"rule34xyz",
+ "s3ndpics",
"saint",
"sankaku",
"sankakucomplex",
diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py
index 67fdb39..1552899 100644
--- a/gallery_dl/extractor/chevereto.py
+++ b/gallery_dl/extractor/chevereto.py
@@ -40,19 +40,15 @@ class CheveretoExtractor(BaseExtractor):
BASE_PATTERN = CheveretoExtractor.update({
"jpgfish": {
"root": "https://jpg6.su",
- "pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)",
- },
- "imgkiwi": {
- "root": "https://img.kiwi",
- "pattern": r"img\.kiwi",
+ "pattern": r"(?:www\.)?jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)",
},
"imagepond": {
"root": "https://imagepond.net",
- "pattern": r"imagepond\.net",
+ "pattern": r"(?:www\.)?imagepond\.net",
},
"imglike": {
"root": "https://imglike.com",
- "pattern": r"imglike\.com",
+ "pattern": r"(?:www\.)?imglike\.com",
},
})
@@ -79,7 +75,7 @@ class CheveretoImageExtractor(CheveretoExtractor):
fromhex=True)
file = {
- "id" : self.path.rpartition(".")[2],
+ "id" : self.path.rpartition("/")[2].rpartition(".")[2],
"url" : url,
"album": text.remove_html(extr(
"Added to <a", "</a>").rpartition(">")[2]),
@@ -144,7 +140,8 @@ class CheveretoAlbumExtractor(CheveretoExtractor):
def items(self):
url = self.root + self.path
- data = {"_extractor": CheveretoImageExtractor}
+ data_image = {"_extractor": CheveretoImageExtractor}
+ data_video = {"_extractor": CheveretoVideoExtractor}
if self.path.endswith("/sub"):
albums = self._pagination(url)
@@ -152,8 +149,9 @@ class CheveretoAlbumExtractor(CheveretoExtractor):
albums = (url,)
for album in albums:
- for image in self._pagination(album):
- yield Message.Queue, image, data
+ for item_url in self._pagination(album):
+ data = data_video if "/video/" in item_url else data_image
+ yield Message.Queue, item_url, data
class CheveretoCategoryExtractor(CheveretoExtractor):
diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py
index fccc466..817d2c4 100644
--- a/gallery_dl/extractor/imagehosts.py
+++ b/gallery_dl/extractor/imagehosts.py
@@ -125,8 +125,18 @@ class ImxtoGalleryExtractor(ImagehostImageExtractor):
"title": text.unescape(title.partition(">")[2]).strip(),
}
- for url in text.extract_iter(page, "<a href=", " ", pos):
- yield Message.Queue, url.strip("\"'"), data
+ params = {"page": 1}
+ while True:
+ for url in text.extract_iter(page, "<a href=", " ", pos):
+ if "/i/" in url:
+ yield Message.Queue, url.strip("\"'"), data
+
+ if 'class="pagination' not in page or \
+ 'class="disabled">Last' in page:
+ return
+
+ params["page"] += 1
+ page = self.request(self.page_url, params=params).text
class AcidimgImageExtractor(ImagehostImageExtractor):
diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py
index 00e06b5..0e6c480 100644
--- a/gallery_dl/extractor/instagram.py
+++ b/gallery_dl/extractor/instagram.py
@@ -39,7 +39,6 @@ class InstagramExtractor(Extractor):
self.www_claim = "0"
self.csrf_token = util.generate_token()
self._find_tags = util.re(r"#\w+").findall
- self._warn_video_ua = True
self._logged_in = True
self._cursor = None
self._user = None
@@ -52,6 +51,12 @@ class InstagramExtractor(Extractor):
else:
self.api = InstagramRestAPI(self)
+ self._warn_video = True if self.config("warn-videos", True) else False
+ self._warn_image = (
+ 9 if not (wi := self.config("warn-images", True)) else
+ 1 if wi in ("all", "both") else
+ 0)
+
def items(self):
self.login()
@@ -172,6 +177,7 @@ class InstagramExtractor(Extractor):
"post_id": reel_id,
"post_shortcode": shortcode_from_id(reel_id),
"post_url": post_url,
+ "type": "story" if expires else "highlight",
}
if "title" in post:
data["highlight_title"] = post["title"]
@@ -182,7 +188,6 @@ class InstagramExtractor(Extractor):
data = {
"post_id" : post["pk"],
"post_shortcode": post["code"],
- "post_url": f"{self.root}/p/{post['code']}/",
"likes": post.get("like_count", 0),
"liked": post.get("has_liked", False),
"pinned": self._extract_pinned(post),
@@ -239,8 +244,8 @@ class InstagramExtractor(Extractor):
manifest = item.get("video_dash_manifest")
media = video
- if self._warn_video_ua:
- self._warn_video_ua = False
+ if self._warn_video:
+ self._warn_video = False
pattern = text.re(
r"Chrome/\d{3,}\.\d+\.\d+\.\d+(?!\d* Mobile)")
if not pattern.search(self.session.headers["User-Agent"]):
@@ -250,8 +255,9 @@ class InstagramExtractor(Extractor):
video = manifest = None
media = image
- if image["width"] < item.get("original_width", 0) or \
- image["height"] < item.get("original_height", 0):
+ if self._warn_image < (
+ (image["width"] < item.get("original_width", 0)) +
+ (image["height"] < item.get("original_height", 0))):
self.log.warning(
"%s: Available image resolutions lower than the "
"original (%sx%s < %sx%s). "
@@ -278,7 +284,7 @@ class InstagramExtractor(Extractor):
if manifest is not None:
media["_ytdl_manifest_data"] = manifest
if "owner" in item:
- media["owner2"] = item["owner"]
+ media["owner"] = item["owner"]
if "reshared_story_media_author" in item:
media["author"] = item["reshared_story_media_author"]
if "expiring_at" in item:
@@ -287,6 +293,14 @@ class InstagramExtractor(Extractor):
self._extract_tagged_users(item, media)
files.append(media)
+ if "type" not in data:
+ if len(files) == 1 and files[0]["video_url"]:
+ data["type"] = "reel"
+ data["post_url"] = f"{self.root}/reel/{post['code']}/"
+ else:
+ data["type"] = "post"
+ data["post_url"] = f"{self.root}/p/{post['code']}/"
+
return data
def _parse_post_graphql(self, post):
@@ -443,6 +457,32 @@ class InstagramExtractor(Extractor):
user[key] = 0
+class InstagramPostExtractor(InstagramExtractor):
+ """Extractor for an Instagram post"""
+ subcategory = "post"
+ pattern = (r"(?:https?://)?(?:www\.)?instagram\.com"
+ r"/(?:share/()|[^/?#]+/)?(?:p|tv|reels?())/([^/?#]+)")
+ example = "https://www.instagram.com/p/abcdefg/"
+
+ def __init__(self, match):
+ if match[2] is not None:
+ self.subcategory = "reel"
+ InstagramExtractor.__init__(self, match)
+
+ def posts(self):
+ share, reel, shortcode = self.groups
+ if share is not None:
+ url = text.ensure_http_scheme(self.url)
+ headers = {
+ "Sec-Fetch-Dest": "empty",
+ "Sec-Fetch-Mode": "navigate",
+ "Sec-Fetch-Site": "same-origin",
+ }
+ location = self.request_location(url, headers=headers)
+ shortcode = location.split("/")[-2]
+ return self.api.media(shortcode)
+
+
class InstagramUserExtractor(Dispatch, InstagramExtractor):
"""Extractor for an Instagram user profile"""
pattern = USER_PATTERN + r"/?(?:$|[?#])"
@@ -740,27 +780,6 @@ class InstagramAvatarExtractor(InstagramExtractor):
},)
-class InstagramPostExtractor(InstagramExtractor):
- """Extractor for an Instagram post"""
- subcategory = "post"
- pattern = (r"(?:https?://)?(?:www\.)?instagram\.com"
- r"/(?:share/()|[^/?#]+/)?(?:p|tv|reel)/([^/?#]+)")
- example = "https://www.instagram.com/p/abcdefg/"
-
- def posts(self):
- share, shortcode = self.groups
- if share is not None:
- url = text.ensure_http_scheme(self.url)
- headers = {
- "Sec-Fetch-Dest": "empty",
- "Sec-Fetch-Mode": "navigate",
- "Sec-Fetch-Site": "same-origin",
- }
- location = self.request_location(url, headers=headers)
- shortcode = location.split("/")[-2]
- return self.api.media(shortcode)
-
-
class InstagramRestAPI():
def __init__(self, extractor):
diff --git a/gallery_dl/extractor/mangadex.py b/gallery_dl/extractor/mangadex.py
index fbed328..30d6848 100644
--- a/gallery_dl/extractor/mangadex.py
+++ b/gallery_dl/extractor/mangadex.py
@@ -39,7 +39,7 @@ class MangadexExtractor(Extractor):
data = self._transform(chapter)
data["_extractor"] = MangadexChapterExtractor
self._cache[uuid] = data
- yield Message.Queue, self.root + "/chapter/" + uuid, data
+ yield Message.Queue, f"{self.root}/chapter/{uuid}", data
def _items_manga(self):
data = {"_extractor": MangadexMangaExtractor}
@@ -51,13 +51,8 @@ class MangadexExtractor(Extractor):
relationships = defaultdict(list)
for item in chapter["relationships"]:
relationships[item["type"]].append(item)
- manga = self.api.manga(relationships["manga"][0]["id"])
- for item in manga["relationships"]:
- relationships[item["type"]].append(item)
cattributes = chapter["attributes"]
- mattributes = manga["attributes"]
-
if lang := cattributes.get("translatedLanguage"):
lang = lang.partition("-")[0]
@@ -66,35 +61,21 @@ class MangadexExtractor(Extractor):
else:
chnum, sep, minor = 0, "", ""
- data = {
- "manga" : (mattributes["title"].get("en") or
- next(iter(mattributes["title"].values()))),
- "manga_id": manga["id"],
+ return {
+ **_manga_info(self, relationships["manga"][0]["id"]),
"title" : cattributes["title"],
"volume" : text.parse_int(cattributes["volume"]),
"chapter" : text.parse_int(chnum),
- "chapter_minor": sep + minor,
+ "chapter_minor": f"{sep}{minor}",
"chapter_id": chapter["id"],
"date" : text.parse_datetime(cattributes["publishAt"]),
+ "group" : [group["attributes"]["name"]
+ for group in relationships["scanlation_group"]],
"lang" : lang,
- "language": util.code_to_language(lang),
"count" : cattributes["pages"],
"_external_url": cattributes.get("externalUrl"),
}
- data["artist"] = [artist["attributes"]["name"]
- for artist in relationships["artist"]]
- data["author"] = [author["attributes"]["name"]
- for author in relationships["author"]]
- data["group"] = [group["attributes"]["name"]
- for group in relationships["scanlation_group"]]
-
- data["status"] = mattributes["status"]
- data["tags"] = [tag["attributes"]["name"]["en"]
- for tag in mattributes["tags"]]
-
- return data
-
class MangadexCoversExtractor(MangadexExtractor):
"""Extractor for mangadex manga covers"""
@@ -103,7 +84,7 @@ class MangadexCoversExtractor(MangadexExtractor):
filename_fmt = "{volume:>02}_{lang}.{extension}"
archive_fmt = "c_{cover_id}"
pattern = (rf"{BASE_PATTERN}/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)"
- r"(?:/[^/?#]+)?\?tab=art")
+ rf"(?:/[^/?#]+)?\?tab=art")
example = ("https://mangadex.org/title"
"/01234567-89ab-cdef-0123-456789abcdef?tab=art")
@@ -121,24 +102,10 @@ class MangadexCoversExtractor(MangadexExtractor):
relationships = defaultdict(list)
for item in cover["relationships"]:
relationships[item["type"]].append(item)
- manga = self.api.manga(relationships["manga"][0]["id"])
- for item in manga["relationships"]:
- relationships[item["type"]].append(item)
-
cattributes = cover["attributes"]
- mattributes = manga["attributes"]
return {
- "manga" : (mattributes["title"].get("en") or
- next(iter(mattributes["title"].values()))),
- "manga_id": manga["id"],
- "status" : mattributes["status"],
- "author" : [author["attributes"]["name"]
- for author in relationships["author"]],
- "artist" : [artist["attributes"]["name"]
- for artist in relationships["artist"]],
- "tags" : [tag["attributes"]["name"]["en"]
- for tag in mattributes["tags"]],
+ **_manga_info(self, relationships["manga"][0]["id"]),
"cover" : cattributes["fileName"],
"lang" : cattributes.get("locale"),
"volume" : text.parse_int(cattributes["volume"]),
@@ -150,7 +117,7 @@ class MangadexCoversExtractor(MangadexExtractor):
class MangadexChapterExtractor(MangadexExtractor):
"""Extractor for manga-chapters from mangadex.org"""
subcategory = "chapter"
- pattern = BASE_PATTERN + r"/chapter/([0-9a-f-]+)"
+ pattern = rf"{BASE_PATTERN}/chapter/([0-9a-f-]+)"
example = ("https://mangadex.org/chapter"
"/01234567-89ab-cdef-0123-456789abcdef")
@@ -177,13 +144,13 @@ class MangadexChapterExtractor(MangadexExtractor):
"page-reverse") else enumerate
for data["page"], page in enum(chapter["data"], 1):
text.nameext_from_url(page, data)
- yield Message.Url, base + page, data
+ yield Message.Url, f"{base}{page}", data
class MangadexMangaExtractor(MangadexExtractor):
"""Extractor for manga from mangadex.org"""
subcategory = "manga"
- pattern = BASE_PATTERN + r"/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)"
+ pattern = rf"{BASE_PATTERN}/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)"
example = ("https://mangadex.org/title"
"/01234567-89ab-cdef-0123-456789abcdef")
@@ -194,7 +161,7 @@ class MangadexMangaExtractor(MangadexExtractor):
class MangadexFeedExtractor(MangadexExtractor):
"""Extractor for chapters from your Updates Feed"""
subcategory = "feed"
- pattern = BASE_PATTERN + r"/titles?/feed$()"
+ pattern = rf"{BASE_PATTERN}/titles?/feed$()"
example = "https://mangadex.org/title/feed"
def chapters(self):
@@ -204,7 +171,7 @@ class MangadexFeedExtractor(MangadexExtractor):
class MangadexFollowingExtractor(MangadexExtractor):
"""Extractor for followed manga from your Library"""
subcategory = "following"
- pattern = BASE_PATTERN + r"/titles?/follows(?:\?([^#]+))?$"
+ pattern = rf"{BASE_PATTERN}/titles?/follows(?:\?([^#]+))?$"
example = "https://mangadex.org/title/follows"
items = MangadexExtractor._items_manga
@@ -216,8 +183,8 @@ class MangadexFollowingExtractor(MangadexExtractor):
class MangadexListExtractor(MangadexExtractor):
"""Extractor for mangadex MDLists"""
subcategory = "list"
- pattern = (BASE_PATTERN +
- r"/list/([0-9a-f-]+)(?:/[^/?#]*)?(?:\?tab=(\w+))?")
+ pattern = (rf"{BASE_PATTERN}"
+ rf"/list/([0-9a-f-]+)(?:/[^/?#]*)?(?:\?tab=(\w+))?")
example = ("https://mangadex.org/list"
"/01234567-89ab-cdef-0123-456789abcdef/NAME")
@@ -242,7 +209,7 @@ class MangadexListExtractor(MangadexExtractor):
class MangadexAuthorExtractor(MangadexExtractor):
"""Extractor for mangadex authors"""
subcategory = "author"
- pattern = BASE_PATTERN + r"/author/([0-9a-f-]+)"
+ pattern = rf"{BASE_PATTERN}/author/([0-9a-f-]+)"
example = ("https://mangadex.org/author"
"/01234567-89ab-cdef-0123-456789abcdef/NAME")
@@ -280,30 +247,30 @@ class MangadexAPI():
else text.ensure_http_scheme(server).rstrip("/"))
def athome_server(self, uuid):
- return self._call("/at-home/server/" + uuid)
+ return self._call(f"/at-home/server/{uuid}")
def author(self, uuid, manga=False):
params = {"includes[]": ("manga",)} if manga else None
- return self._call("/author/" + uuid, params)["data"]
+ return self._call(f"/author/{uuid}", params)["data"]
def chapter(self, uuid):
params = {"includes[]": ("scanlation_group",)}
- return self._call("/chapter/" + uuid, params)["data"]
+ return self._call(f"/chapter/{uuid}", params)["data"]
def covers_manga(self, uuid):
params = {"manga[]": uuid}
return self._pagination_covers("/cover", params)
def list(self, uuid):
- return self._call("/list/" + uuid, None, True)["data"]
+ return self._call(f"/list/{uuid}", None, True)["data"]
def list_feed(self, uuid):
- return self._pagination_chapters("/list/" + uuid + "/feed", None, True)
+ return self._pagination_chapters(f"/list/{uuid}/feed", None, True)
@memcache(keyarg=1)
def manga(self, uuid):
params = {"includes[]": ("artist", "author")}
- return self._call("/manga/" + uuid, params)["data"]
+ return self._call(f"/manga/{uuid}", params)["data"]
def manga_author(self, uuid_author):
params = {"authorOrArtist": uuid_author}
@@ -315,7 +282,7 @@ class MangadexAPI():
"order[volume]" : order,
"order[chapter]": order,
}
- return self._pagination_chapters("/manga/" + uuid + "/feed", params)
+ return self._pagination_chapters(f"/manga/{uuid}/feed", params)
def user_follows_manga(self):
params = {"contentRating": None}
@@ -366,17 +333,17 @@ class MangadexAPI():
_refresh_token_cache.update(
(username, "personal"), data["refresh_token"])
- return "Bearer " + access_token
+ return f"Bearer {access_token}"
@cache(maxage=900, keyarg=1)
def _authenticate_impl_legacy(self, username, password):
if refresh_token := _refresh_token_cache(username):
self.extractor.log.info("Refreshing access token")
- url = self.root + "/auth/refresh"
+ url = f"{self.root}/auth/refresh"
json = {"token": refresh_token}
else:
self.extractor.log.info("Logging in as %s", username)
- url = self.root + "/auth/login"
+ url = f"{self.root}/auth/login"
json = {"username": username, "password": password}
self.extractor.log.debug("Using legacy login method")
@@ -387,10 +354,10 @@ class MangadexAPI():
if refresh_token != data["token"]["refresh"]:
_refresh_token_cache.update(username, data["token"]["refresh"])
- return "Bearer " + data["token"]["session"]
+ return f"Bearer {data['token']['session']}"
def _call(self, endpoint, params=None, auth=False):
- url = self.root + endpoint
+ url = f"{self.root}{endpoint}"
headers = self.headers_auth if auth else self.headers
while True:
@@ -470,3 +437,33 @@ class MangadexAPI():
@cache(maxage=90*86400, keyarg=0)
def _refresh_token_cache(username):
return None
+
+
+@memcache(keyarg=1)
+def _manga_info(self, uuid):
+ manga = self.api.manga(uuid)
+
+ rel = defaultdict(list)
+ for item in manga["relationships"]:
+ rel[item["type"]].append(item)
+ mattr = manga["attributes"]
+
+ return {
+ "manga" : (mattr["title"].get("en") or
+ next(iter(mattr["title"].values()))),
+ "manga_id": manga["id"],
+ "manga_titles": [t.popitem()[1]
+ for t in mattr.get("altTitles") or ()],
+ "manga_date" : text.parse_datetime(mattr.get("createdAt")),
+ "description" : (mattr["description"].get("en") or
+ next(iter(mattr["description"].values()))),
+ "demographic": mattr.get("publicationDemographic"),
+ "origin": mattr.get("originalLanguage"),
+ "status": mattr.get("status"),
+ "year" : mattr.get("year"),
+ "rating": mattr.get("contentRating"),
+ "links" : mattr.get("links"),
+ "tags" : [tag["attributes"]["name"]["en"] for tag in mattr["tags"]],
+ "artist": [artist["attributes"]["name"] for artist in rel["artist"]],
+ "author": [author["attributes"]["name"] for author in rel["author"]],
+ }
diff --git a/gallery_dl/extractor/mangafire.py b/gallery_dl/extractor/mangafire.py
new file mode 100644
index 0000000..5ccb732
--- /dev/null
+++ b/gallery_dl/extractor/mangafire.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2025 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://mangafire.to/"""
+
+from .common import ChapterExtractor, MangaExtractor
+from .. import text, exception
+from ..cache import memcache
+
+BASE_PATTERN = r"(?:https?://)?(?:www\.)?mangafire\.to"
+
+
+class MangafireBase():
+ """Base class for mangafire extractors"""
+ category = "mangafire"
+ root = "https://mangafire.to"
+
+
+class MangafireChapterExtractor(MangafireBase, ChapterExtractor):
+ """Extractor for mangafire manga chapters"""
+ directory_fmt = (
+ "{category}", "{manga}",
+ "{volume:?v/ />02}{chapter:?c//>03}{chapter_minor:?//}{title:?: //}")
+ filename_fmt = (
+ "{manga}{volume:?_v//>02}{chapter:?_c//>03}{chapter_minor:?//}_"
+ "{page:>03}.{extension}")
+ archive_fmt = (
+ "{manga_id}_{chapter_id}_{page}")
+ pattern = (rf"{BASE_PATTERN}/read/([\w-]+\.(\w+))/([\w-]+)"
+ rf"/((chapter|volume)-\d+(?:\D.*)?)")
+ example = "https://mangafire.to/read/MANGA.ID/LANG/chapter-123"
+
+ def metadata(self, _):
+ manga_path, manga_id, lang, chapter_info, self.type = self.groups
+
+ try:
+ chapters = _manga_chapters(self, (manga_id, self.type, lang))
+ anchor = chapters[chapter_info]
+ except KeyError:
+ raise exception.NotFoundError("chapter")
+ self.chapter_id = text.extr(anchor, 'data-id="', '"')
+
+ return {
+ **_manga_info(self, manga_path),
+ **_chapter_info(anchor),
+ }
+
+ def images(self, page):
+ url = f"{self.root}/ajax/read/{self.type}/{self.chapter_id}"
+ headers = {"x-requested-with": "XMLHttpRequest"}
+ data = self.request_json(url, headers=headers)
+
+ return [
+ (image[0], None)
+ for image in data["result"]["images"]
+ ]
+
+
+class MangafireMangaExtractor(MangafireBase, MangaExtractor):
+ """Extractor for mangafire manga"""
+ chapterclass = MangafireChapterExtractor
+ pattern = rf"{BASE_PATTERN}/manga/([\w-]+)\.(\w+)"
+ example = "https://mangafire.to/manga/MANGA.ID"
+
+ def chapters(self, page):
+ manga_slug, manga_id = self.groups
+ lang = self.config("lang") or "en"
+
+ manga = _manga_info(self, f"{manga_slug}.{manga_id}")
+ chapters = _manga_chapters(self, (manga_id, "chapter", lang))
+
+ return [
+ (f"""{self.root}{text.extr(anchor, 'href="', '"')}""", {
+ **manga,
+ **_chapter_info(anchor),
+ })
+ for anchor in chapters.values()
+ ]
+
+
+@memcache(keyarg=1)
+def _manga_info(self, manga_path, page=None):
+ if page is None:
+ url = f"{self.root}/manga/{manga_path}"
+ page = self.request(url).text
+ slug, _, mid = manga_path.rpartition(".")
+
+ extr = text.extract_from(page)
+ manga = {
+ "cover": text.extr(extr(
+ 'class="poster">', '</div>'), 'src="', '"'),
+ "status": extr("<p>", "<").replace("_", " ").title(),
+ "manga" : text.unescape(extr(
+ 'itemprop="name">', "<")),
+ "manga_id": mid,
+ "manga_slug": slug,
+ "manga_titles": text.unescape(extr(
+ "<h6>", "<")).split("; "),
+ "type": text.remove_html(extr(
+ 'class="min-info">', "</a>")),
+ "author": text.unescape(text.remove_html(extr(
+ "<span>Author:</span>", "</div>"))).split(" , "),
+ "published": text.remove_html(extr(
+ "<span>Published:</span>", "</div>")),
+ "tags": text.split_html(extr(
+ "<span>Genres:</span>", "</div>"))[::2],
+ "publisher": text.unescape(text.remove_html(extr(
+ "<span>Mangazines:</span>", "</div>"))).split(" , "),
+ "score": text.parse_float(text.remove_html(extr(
+ 'class="score">', " / "))),
+ "description": text.remove_html(extr(
+ 'id="synopsis">', "<script>")),
+ }
+
+ if len(lst := manga["author"]) == 1 and not lst[0]:
+ manga["author"] = ()
+ if len(lst := manga["publisher"]) == 1 and not lst[0]:
+ manga["publisher"] = ()
+
+ return manga
+
+
+@memcache(keyarg=1)
+def _manga_chapters(self, manga_info):
+ manga_id, type, lang = manga_info
+ url = f"{self.root}/ajax/read/{manga_id}/{type}/{lang}"
+ headers = {"x-requested-with": "XMLHttpRequest"}
+ data = self.request_json(url, headers=headers)
+
+ needle = f"{manga_id}/{lang}/"
+ return {
+ text.extr(anchor, needle, '"'): anchor
+ for anchor in text.extract_iter(data["result"]["html"], "<a ", ">")
+ }
+
+
+@memcache(keyarg=0)
+def _chapter_info(info):
+ _, lang, chapter_info = text.extr(info, 'href="', '"').rsplit("/", 2)
+
+ if chapter_info.startswith("vol"):
+ volume = text.extr(info, 'data-number="', '"')
+ volume_id = text.parse_int(text.extr(info, 'data-id="', '"'))
+ return {
+ "volume" : text.parse_int(volume),
+ "volume_id" : volume_id,
+ "chapter" : 0,
+ "chapter_minor" : "",
+ "chapter_string": chapter_info,
+ "chapter_id" : volume_id,
+ "title" : text.unescape(text.extr(info, 'title="', '"')),
+ "lang" : lang,
+ }
+
+ chapter, sep, minor = text.extr(info, 'data-number="', '"').partition(".")
+ return {
+ "chapter" : text.parse_int(chapter),
+ "chapter_minor" : f"{sep}{minor}",
+ "chapter_string": chapter_info,
+ "chapter_id" : text.parse_int(text.extr(info, 'data-id="', '"')),
+ "title" : text.unescape(text.extr(info, 'title="', '"')),
+ "lang" : lang,
+ }
diff --git a/gallery_dl/extractor/mangareader.py b/gallery_dl/extractor/mangareader.py
new file mode 100644
index 0000000..eb53998
--- /dev/null
+++ b/gallery_dl/extractor/mangareader.py
@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2025 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://mangareader.to/"""
+
+from .common import ChapterExtractor, MangaExtractor
+from .. import text, util
+from ..cache import memcache
+
+BASE_PATTERN = r"(?:https?://)?(?:www\.)?mangareader\.to"
+
+
+class MangareaderBase():
+ """Base class for mangareader extractors"""
+ category = "mangareader"
+ root = "https://mangareader.to"
+
+
+class MangareaderChapterExtractor(MangareaderBase, ChapterExtractor):
+ """Extractor for mangareader manga chapters"""
+ directory_fmt = (
+ "{category}", "{manga}",
+ "{volume:?v/ />02}{chapter:?c//>03}{chapter_minor:?//}{title:?: //}")
+ filename_fmt = (
+ "{manga}{volume:?_v//>02}{chapter:?_c//>03}{chapter_minor:?//}_"
+ "{page:>03}.{extension}")
+ archive_fmt = (
+ "{manga_id}_{chapter_id}_{page}")
+ pattern = (rf"{BASE_PATTERN}/read/([\w-]+-\d+)/([^/?#]+)"
+ rf"/(chapter|volume)-(\d+[^/?#]*)")
+ example = "https://mangareader.to/read/MANGA-123/LANG/chapter-123"
+
+ def metadata(self, _):
+ path, lang, type, chstr = self.groups
+
+ settings = util.json_dumps({
+ "readingMode" : "vertical",
+ "readingDirection": "rtl",
+ "quality" : "high",
+ })
+ self.cookies.set("mr_settings", settings, domain="mangareader.to")
+
+ url = f"{self.root}/read/{path}/{lang}/{type}-{chstr}"
+ page = self.request(url).text
+ self.cid = cid = text.extr(page, 'data-reading-id="', '"')
+
+ manga = _manga_info(self, path)
+ return {
+ **manga,
+ **manga[f"_{type}s"][lang][chstr],
+ "chapter_id": text.parse_int(cid),
+ }
+
+ def images(self, page):
+ key = "chap" if self.groups[2] == "chapter" else "vol"
+ url = f"{self.root}/ajax/image/list/{key}/{self.cid}"
+ params = {
+ "mode" : "vertical,",
+ "quality" : "high,",
+ "hozPageSize": "1,",
+ }
+ headers = {
+ "X-Requested-With": "XMLHttpRequest",
+ "Sec-Fetch-Dest" : "empty",
+ "Sec-Fetch-Mode" : "cors",
+ "Sec-Fetch-Site" : "same-origin",
+ }
+ html = self.request_json(url, params=params, headers=headers)["html"]
+
+ return [
+ (url, None)
+ for url in text.extract_iter(html, 'data-url="', '"')
+ ]
+
+
+class MangareaderMangaExtractor(MangareaderBase, MangaExtractor):
+ """Extractor for mangareader manga"""
+ chapterclass = MangareaderChapterExtractor
+ pattern = rf"{BASE_PATTERN}/([\w-]+-\d+)"
+ example = "https://mangareader.to/MANGA-123"
+
+ def chapters(self, page):
+ manga = _manga_info(self, self.groups[0])
+ lang = self.config("lang") or "en"
+
+ return [
+ (info["chapter_url"], {**manga, **info})
+ for info in manga["_chapters"][lang].values()
+ ]
+
+
+@memcache(keyarg=1)
+def _manga_info(self, manga_path):
+ url = f"{self.root}/{manga_path}"
+ html = self.request(url).text
+
+ slug, _, mid = manga_path.rpartition("-")
+ extr = text.extract_from(html)
+ url = extr('property="og:url" content="', '"')
+ manga = {
+ "manga_url": url,
+ "manga_slug": url.rpartition("/")[2].rpartition("-")[0],
+ "manga_id": text.parse_int(mid),
+ "manga": text.unescape(extr('class="manga-name">', "<")),
+ "manga_alt": text.unescape(extr('class="manga-name-or">', "<")),
+ "tags": text.split_html(extr('class="genres">', "</div>")),
+ "type": text.remove_html(extr('>Type:', "</div>")),
+ "status": text.remove_html(extr('>Status:', "</div>")),
+ "author": text.split_html(extr('>Authors:', "</div>"))[0::2],
+ "published": text.remove_html(extr('>Published:', "</div>")),
+ "score": text.parse_float(text.remove_html(extr(
+ '>Score:', "</div>"))),
+ "views": text.parse_int(text.remove_html(extr(
+ '>Views:', "</div>")).replace(",", "")),
+ }
+
+ base = self.root
+
+ # extract all chapters
+ html = extr('class="chapters-list-ul">', " </div>")
+ manga["_chapters"] = chapters = {}
+ for group in text.extract_iter(html, "<ul", "</ul>"):
+ lang = text.extr(group, ' id="', '-chapters"')
+
+ chapters[lang] = current = {}
+ lang = lang.partition("-")[0]
+ for ch in text.extract_iter(group, "<li ", "</li>"):
+ path = text.extr(ch, 'href="', '"')
+ chap = text.extr(ch, 'data-number="', '"')
+ name = text.unescape(text.extr(ch, 'class="name">', "<"))
+
+ chapter, sep, minor = chap.partition(".")
+ current[chap] = {
+ "title" : name.partition(":")[2].strip(),
+ "chapter" : text.parse_int(chapter),
+ "chapter_minor" : f"{sep}{minor}",
+ "chapter_string": chap,
+ "chapter_url" : f"{base}{path}",
+ "lang" : lang,
+ }
+
+ # extract all volumes
+ html = extr('class="volume-list-ul">', "</section>")
+ manga["_volumes"] = volumes = {}
+ for group in html.split('<div class="manga_list-wrap')[1:]:
+ lang = text.extr(group, ' id="', '-volumes"')
+
+ volumes[lang] = current = {}
+ lang = lang.partition("-")[0]
+ for vol in text.extract_iter(group, 'class="item">', "</div>"):
+ path = text.extr(vol, 'href="', '"')
+ voln = text.extr(vol, 'tick-vol">', '<').rpartition(" ")[2]
+
+ current[voln] = {
+ "volume" : text.parse_int(voln),
+ "volume_cover" : text.extr(vol, ' src="', '"'),
+ "chapter" : 0,
+ "chapter_minor" : "",
+ "chapter_string": voln,
+ "chapter_url" : f"{base}{path}",
+ "lang" : lang,
+ }
+
+ # extract remaining metadata
+ manga["description"] = text.unescape(extr(
+ 'class="description-modal">', "</div>")).strip()
+
+ return manga
diff --git a/gallery_dl/extractor/misskey.py b/gallery_dl/extractor/misskey.py
index 5ff601a..42eaeef 100644
--- a/gallery_dl/extractor/misskey.py
+++ b/gallery_dl/extractor/misskey.py
@@ -25,8 +25,8 @@ class MisskeyExtractor(BaseExtractor):
def _init(self):
self.api = MisskeyAPI(self)
self.instance = self.root.rpartition("://")[2]
- self.renotes = self.config("renotes", False)
- self.replies = self.config("replies", True)
+ self.renotes = True if self.config("renotes", False) else False
+ self.replies = True if self.config("replies", True) else False
def items(self):
for note in self.notes():
@@ -254,6 +254,8 @@ class MisskeyAPI():
def _pagination(self, endpoint, data):
data["limit"] = 100
+ data["withRenotes"] = self.extractor.renotes
+
while True:
notes = self._call(endpoint, data)
if not notes:
diff --git a/gallery_dl/extractor/nozomi.py b/gallery_dl/extractor/nozomi.py
index 21c361c..528aff2 100644
--- a/gallery_dl/extractor/nozomi.py
+++ b/gallery_dl/extractor/nozomi.py
@@ -173,7 +173,7 @@ class NozomiSearchExtractor(NozomiExtractor):
for tag in self.tags:
(negative if tag[0] == "-" else positive).append(
- tag.replace("/", ""))
+ text.quote(tag.replace("/", "")))
for tag in positive:
ids = nozomi("nozomi/" + tag)
diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py
index 5245f31..490243a 100644
--- a/gallery_dl/extractor/paheal.py
+++ b/gallery_dl/extractor/paheal.py
@@ -9,7 +9,7 @@
"""Extractors for https://rule34.paheal.net/"""
from .common import Extractor, Message
-from .. import text
+from .. import text, exception
class PahealExtractor(Extractor):
@@ -97,7 +97,12 @@ class PahealTagExtractor(PahealExtractor):
base = f"{self.root}/post/list/{self.groups[0]}/"
while True:
- page = self.request(base + str(pnum)).text
+ try:
+ page = self.request(f"{base}{pnum}").text
+ except exception.HttpError as exc:
+ if exc.status == 404:
+ return
+ raise
pos = page.find("id='image-list'")
for post in text.extract_iter(
@@ -146,4 +151,9 @@ class PahealPostExtractor(PahealExtractor):
example = "https://rule34.paheal.net/post/view/12345"
def get_posts(self):
- return (self._extract_post(self.groups[0]),)
+ try:
+ return (self._extract_post(self.groups[0]),)
+ except exception.HttpError as exc:
+ if exc.status == 404:
+ return ()
+ raise
diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py
index fb2f32c..cf1a6d6 100644
--- a/gallery_dl/extractor/patreon.py
+++ b/gallery_dl/extractor/patreon.py
@@ -230,6 +230,16 @@ class PatreonExtractor(Extractor):
attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z")
return attr
+ def _collection(self, collection_id):
+ url = f"{self.root}/api/collection/{collection_id}"
+ data = self.request_json(url)
+ coll = data["data"]
+ attr = coll["attributes"]
+ attr["id"] = coll["id"]
+ attr["date"] = text.parse_datetime(
+ attr["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
+ return attr
+
def _filename(self, url):
"""Fetch filename from an URL's Content-Disposition header"""
response = self.request(url, method="HEAD", fatal=False)
@@ -333,6 +343,33 @@ class PatreonExtractor(Extractor):
raise exception.AbortExtraction("Unable to extract bootstrap data")
+class PatreonCollectionExtractor(PatreonExtractor):
+ """Extractor for a patreon collection"""
+ subcategory = "collection"
+ directory_fmt = ("{category}", "{creator[full_name]}",
+ "Collections", "{collection[title]} ({collection[id]})")
+ pattern = r"(?:https?://)?(?:www\.)?patreon\.com/collection/(\d+)"
+ example = "https://www.patreon.com/collection/12345"
+
+ def posts(self):
+ collection_id = self.groups[0]
+ self.kwdict["collection"] = collection = \
+ self._collection(collection_id)
+ campaign_id = text.extr(
+ collection["thumbnail"]["url"], "/campaign/", "/")
+
+ url = self._build_url("posts", (
+ # patreon returns '400 Bad Request' without campaign_id filter
+ f"&filter[campaign_id]={campaign_id}"
+ "&filter[contains_exclusive_posts]=true"
+ "&filter[is_draft]=false"
+ f"&filter[collection_id]={collection_id}"
+ "&filter[include_drops]=true"
+ "&sort=collection_order"
+ ))
+ return self._pagination(url)
+
+
class PatreonCreatorExtractor(PatreonExtractor):
"""Extractor for a creator's works"""
subcategory = "creator"
diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index a72042c..6276a2a 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -1232,7 +1232,7 @@ class PixivAppAPI():
params = {"word": word, "search_target": target,
"sort": sort, "duration": duration,
"start_date": date_start, "end_date": date_end}
- return self._pagination("/v1/search/illust", params)
+ return self._pagination_search("/v1/search/illust", params)
def user_bookmarks_illust(self, user_id, tag=None, restrict="public"):
"""Return illusts bookmarked by a user"""
@@ -1322,6 +1322,48 @@ class PixivAppAPI():
params = text.parse_query(query)
data = self._call(endpoint, params)
+ def _pagination_search(self, endpoint, params):
+ sort = params["sort"]
+ if sort == "date_desc":
+ date_key = "end_date"
+ date_off = timedelta(days=1)
+ date_cmp = lambda lhs, rhs: lhs >= rhs # noqa E731
+ elif sort == "date_asc":
+ date_key = "start_date"
+ date_off = timedelta(days=-1)
+ date_cmp = lambda lhs, rhs: lhs <= rhs # noqa E731
+ else:
+ date_key = None
+ date_last = None
+
+ while True:
+ data = self._call(endpoint, params)
+
+ if date_last is None:
+ yield from data["illusts"]
+ else:
+ works = data["illusts"]
+ if date_cmp(date_last, works[-1]["create_date"]):
+ for work in works:
+ if date_last is None:
+ yield work
+ elif date_cmp(date_last, work["create_date"]):
+ date_last = None
+
+ if not (next_url := data.get("next_url")):
+ return
+ query = next_url.rpartition("?")[2]
+ params = text.parse_query(query)
+
+ if date_key and text.parse_int(params.get("offset")) >= 5000:
+ date_last = data["illusts"][-1]["create_date"]
+ date_val = (text.parse_datetime(
+ date_last) + date_off).strftime("%Y-%m-%d")
+ self.log.info("Reached 'offset' >= 5000; "
+ "Updating '%s' to '%s'", date_key, date_val)
+ params[date_key] = date_val
+ params.pop("offset", None)
+
@cache(maxage=36500*86400, keyarg=0)
def _refresh_token_cache(username):
diff --git a/gallery_dl/extractor/s3ndpics.py b/gallery_dl/extractor/s3ndpics.py
new file mode 100644
index 0000000..215f160
--- /dev/null
+++ b/gallery_dl/extractor/s3ndpics.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2025 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://s3nd.pics/"""
+
+from .common import Extractor, Message
+from .. import text
+
+BASE_PATTERN = r"(?:https?://)?(?:www\.)?s3nd\.pics"
+
+
+class S3ndpicsExtractor(Extractor):
+ """Base class for s3ndpics extractors"""
+ category = "s3ndpics"
+ root = "https://s3nd.pics"
+ root_api = f"{root}/api"
+ directory_fmt = ("{category}", "{user[username]}",
+ "{date} {title:?/ /}({id})")
+ filename_fmt = "{num:>02}.{extension}"
+ archive_fmt = "{id}_{num}"
+
+ def items(self):
+ base = "https://s3.s3nd.pics/s3nd-pics/"
+
+ for post in self.posts():
+ post["id"] = post.pop("_id", None)
+ post["user"] = post.pop("userId", None)
+ post["date"] = text.parse_datetime(
+ post["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
+ post["date_updated"] = text.parse_datetime(
+ post["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
+
+ files = post.pop("files", ())
+ post["count"] = len(files)
+
+ yield Message.Directory, post
+ for post["num"], file in enumerate(files, 1):
+ post["type"] = file["type"]
+ path = file["url"]
+ text.nameext_from_url(path, post)
+ yield Message.Url, f"{base}{path}", post
+
+ def _pagination(self, url, params):
+ params["page"] = 1
+
+ while True:
+ data = self.request_json(url, params=params)
+
+ self.kwdict["total"] = data["pagination"]["total"]
+ yield from data["posts"]
+
+ if params["page"] >= data["pagination"]["pages"]:
+ return
+ params["page"] += 1
+
+
+class S3ndpicsPostExtractor(S3ndpicsExtractor):
+ subcategory = "post"
+ pattern = rf"{BASE_PATTERN}/post/([0-9a-f]+)"
+ example = "https://s3nd.pics/post/0123456789abcdef01234567"
+
+ def posts(self):
+ url = f"{self.root_api}/posts/{self.groups[0]}"
+ return (self.request_json(url)["post"],)
+
+
+class S3ndpicsUserExtractor(S3ndpicsExtractor):
+ subcategory = "user"
+ pattern = rf"{BASE_PATTERN}/user/(\w+)"
+ example = "https://s3nd.pics/user/USER"
+
+ def posts(self):
+ url = f"{self.root_api}/users/username/{self.groups[0]}"
+ self.kwdict["user"] = user = self.request_json(url)["user"]
+
+ url = f"{self.root_api}/posts"
+ params = {
+ "userId": user["_id"],
+ "limit" : "12",
+ "sortBy": "newest",
+ }
+ return self._pagination(url, params)
+
+
+class S3ndpicsSearchExtractor(S3ndpicsExtractor):
+ subcategory = "search"
+ pattern = rf"{BASE_PATTERN}/search/?\?([^#]+)"
+ example = "https://s3nd.pics/search?QUERY"
+
+ def posts(self):
+ url = f"{self.root_api}/posts"
+ params = text.parse_query(self.groups[0])
+ params.setdefault("limit", "20")
+ self.kwdict["search_tags"] = \
+ params.get("tag") or params.get("tags") or params.get("q")
+ return self._pagination(url, params)
diff --git a/gallery_dl/extractor/schalenetwork.py b/gallery_dl/extractor/schalenetwork.py
index dc42417..a4ef3b0 100644
--- a/gallery_dl/extractor/schalenetwork.py
+++ b/gallery_dl/extractor/schalenetwork.py
@@ -62,10 +62,11 @@ class SchalenetworkExtractor(Extractor):
pass
params["page"] += 1
- def _token(self):
+ def _token(self, required=True):
if token := self.config("token"):
return f"Bearer {token.rpartition(' ')[2]}"
- raise exception.AuthRequired("'token'", "your favorites")
+ if required:
+ raise exception.AuthRequired("'token'", "your favorites")
def _crt(self):
crt = self.config("crt")
@@ -88,7 +89,7 @@ class SchalenetworkExtractor(Extractor):
else:
msg = f"{exc.status} {exc.response.reason}"
raise exception.AuthRequired(
- "'crt' query parameter & matching '--user-agent'", None, msg)
+ "'crt' query parameter & matching 'user-agent'", None, msg)
class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor):
@@ -114,19 +115,26 @@ class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor):
10: "mixed",
11: "language",
12: "other",
+ 13: "reclass",
}
def metadata(self, _):
_, gid, gkey = self.groups
+
url = f"{self.root_api}/books/detail/{gid}/{gkey}"
- data = self.request_json(url, headers=self.headers)
- data["date"] = text.parse_timestamp(data["created_at"] // 1000)
+ headers = self.headers
+ data = self.request_json(url, headers=headers)
+
+ try:
+ data["date"] = text.parse_timestamp(data["created_at"] // 1000)
+ data["count"] = len(data["thumbnails"]["entries"])
+ del data["thumbnails"]
+ except Exception:
+ pass
tags = []
types = self.TAG_TYPES
- tags_data = data["tags"]
-
- for tag in tags_data:
+ for tag in data["tags"]:
name = tag["name"]
namespace = tag.get("namespace", 0)
tags.append(types[namespace] + ":" + name)
@@ -134,33 +142,34 @@ class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor):
if self.config("tags", False):
tags = collections.defaultdict(list)
- for tag in tags_data :
+ for tag in data["tags"]:
tags[tag.get("namespace", 0)].append(tag["name"])
for type, values in tags.items():
data["tags_" + types[type]] = values
+ url = f"{self.root_api}/books/detail/{gid}/{gkey}?crt={self._crt()}"
+ if token := self._token(False):
+ headers = headers.copy()
+ headers["Authorization"] = token
try:
- data["count"] = len(data["thumbnails"]["entries"])
- del data["thumbnails"]
- except Exception:
- pass
+ data_fmt = self.request_json(
+ url, method="POST", headers=headers)
+ except exception.HttpError as exc:
+ self._require_auth(exc)
+
+ self.fmt = self._select_format(data_fmt["data"])
+ data["source"] = data_fmt.get("source")
return data
def images(self, _):
- crt = self._crt()
_, gid, gkey = self.groups
- url = f"{self.root_api}/books/detail/{gid}/{gkey}?crt={crt}"
- try:
- data = self.request_json(url, method="POST", headers=self.headers)
- except exception.HttpError as exc:
- self._require_auth(exc)
-
- fmt = self._select_format(data["data"])
+ fmt = self.fmt
url = (f"{self.root_api}/books/data/{gid}/{gkey}"
- f"/{fmt['id']}/{fmt['key']}/{fmt['w']}?crt={crt}")
- data = self.request_json(url, headers=self.headers)
+ f"/{fmt['id']}/{fmt['key']}/{fmt['w']}?crt={self._crt()}")
+ headers = self.headers
+ data = self.request_json(url, headers=headers)
base = data["base"]
results = []
@@ -169,7 +178,7 @@ class SchalenetworkGalleryExtractor(SchalenetworkExtractor, GalleryExtractor):
info = {
"width" : dimensions[0],
"height": dimensions[1],
- "_http_headers": self.headers,
+ "_http_headers": headers,
}
results.append((base + entry["path"], info))
return results
diff --git a/gallery_dl/extractor/simpcity.py b/gallery_dl/extractor/simpcity.py
index 3354289..d8227fa 100644
--- a/gallery_dl/extractor/simpcity.py
+++ b/gallery_dl/extractor/simpcity.py
@@ -92,7 +92,7 @@ class SimpcityExtractor(Extractor):
author = schema["author"]
stats = schema["interactionStatistic"]
url_t = schema["url"]
- url_a = author["url"]
+ url_a = author.get("url") or ""
thread = {
"id" : url_t[url_t.rfind(".")+1:-1],
@@ -104,8 +104,9 @@ class SimpcityExtractor(Extractor):
"tags" : (schema["keywords"].split(", ")
if "keywords" in schema else ()),
"section" : schema["articleSection"],
- "author" : author["name"],
- "author_id" : url_a[url_a.rfind(".")+1:-1],
+ "author" : author.get("name") or "",
+ "author_id" : (url_a[url_a.rfind(".")+1:-1] if url_a else
+ (author.get("name") or "")[15:]),
"author_url": url_a,
}
diff --git a/gallery_dl/extractor/thehentaiworld.py b/gallery_dl/extractor/thehentaiworld.py
index 055d7d8..9a30654 100644
--- a/gallery_dl/extractor/thehentaiworld.py
+++ b/gallery_dl/extractor/thehentaiworld.py
@@ -60,14 +60,16 @@ class ThehentaiworldExtractor(Extractor):
"<li>Posted: ", "<"), "%Y-%m-%d"),
}
- if "/videos/" in url:
+ if (c := url[27]) == "v":
post["type"] = "video"
post["width"] = post["height"] = 0
post["votes"] = text.parse_int(extr("(<strong>", "</strong>"))
post["score"] = text.parse_float(extr("<strong>", "<"))
post["file_url"] = extr('<source src="', '"')
else:
- post["type"] = "image"
+ post["type"] = ("animated" if c == "g" else
+ "3d cgi" if c == "3" else
+ "image")
post["width"] = text.parse_int(extr("<li>Size: ", " "))
post["height"] = text.parse_int(extr("x ", "<"))
post["file_url"] = extr('a href="', '"')
@@ -109,16 +111,6 @@ class ThehentaiworldExtractor(Extractor):
pnum += 1
-class ThehentaiworldPostExtractor(ThehentaiworldExtractor):
- subcategory = "post"
- pattern = (rf"{BASE_PATTERN}"
- rf"(/(?:(?:3d-cgi-)?hentai-image|video)s/([^/?#]+))")
- example = "https://thehentaiworld.com/hentai-images/SLUG/"
-
- def posts(self):
- return (f"{self.root}{self.groups[0]}/",)
-
-
class ThehentaiworldTagExtractor(ThehentaiworldExtractor):
subcategory = "tag"
per_page = 24
@@ -137,3 +129,13 @@ class ThehentaiworldTagExtractor(ThehentaiworldExtractor):
self.page_start += pages
self.post_start += posts
return num
+
+
+class ThehentaiworldPostExtractor(ThehentaiworldExtractor):
+ subcategory = "post"
+ pattern = (rf"{BASE_PATTERN}("
+ rf"/(?:video|(?:[\w-]+-)?hentai-image)s/([^/?#]+))")
+ example = "https://thehentaiworld.com/hentai-images/SLUG/"
+
+ def posts(self):
+ return (f"{self.root}{self.groups[0]}/",)
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
index e6c84d1..e7df4a3 100644
--- a/gallery_dl/extractor/twitter.py
+++ b/gallery_dl/extractor/twitter.py
@@ -1026,11 +1026,12 @@ class TwitterTweetExtractor(TwitterExtractor):
return
while True:
+ parent_id = tweet["rest_id"]
tweet_id = tweet["legacy"].get("quoted_status_id_str")
if not tweet_id:
break
tweet = self.api.tweet_result_by_rest_id(tweet_id)
- tweet["legacy"]["quoted_by_id_str"] = tweet_id
+ tweet["legacy"]["quoted_by_id_str"] = parent_id
yield tweet
def _tweets_detail(self, tweet_id):
diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py
index 823e8e0..07bed79 100644
--- a/gallery_dl/extractor/weibo.py
+++ b/gallery_dl/extractor/weibo.py
@@ -86,16 +86,25 @@ class WeiboExtractor(Extractor):
status["count"] = len(files)
yield Message.Directory, status
- for num, file in enumerate(files, 1):
- if file["url"].startswith("http:"):
- file["url"] = "https:" + file["url"][5:]
+ num = 0
+ for file in files:
+ url = file["url"]
+ if not url:
+ continue
+ if url.startswith("http:"):
+ url = f"https:{url[5:]}"
if "filename" not in file:
- text.nameext_from_url(file["url"], file)
+ text.nameext_from_url(url, file)
if file["extension"] == "json":
file["extension"] = "mp4"
+ if file["extension"] == "m3u8":
+ url = f"ytdl:{url}"
+ file["_ytdl_manifest"] = "hls"
+ file["extension"] = "mp4"
+ num += 1
file["status"] = status
file["num"] = num
- yield Message.Url, file["url"], file
+ yield Message.Url, url, file
def _extract_status(self, status, files):
if "mix_media_info" in status:
@@ -143,10 +152,21 @@ class WeiboExtractor(Extractor):
media = max(info["playback_list"],
key=lambda m: m["meta"]["quality_index"])
except Exception:
- return {"url": (info.get("stream_url_hd") or
- info.get("stream_url") or "")}
+ video = {"url": (info.get("replay_hd") or
+ info.get("stream_url_hd") or
+ info.get("stream_url") or "")}
else:
- return media["play_info"].copy()
+ video = media["play_info"].copy()
+
+ if "//wblive-out." in video["url"] and \
+ not text.ext_from_url(video["url"]):
+ try:
+ video["url"] = self.request_location(video["url"])
+ except exception.HttpError as exc:
+ self.log.warning("%s: %s", exc.__class__.__name__, exc)
+ video["url"] = ""
+
+ return video
def _status_by_id(self, status_id):
url = f"{self.root}/ajax/statuses/show?id={status_id}"
diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py
index 00266bd..5ba47d2 100644
--- a/gallery_dl/extractor/wikimedia.py
+++ b/gallery_dl/extractor/wikimedia.py
@@ -46,6 +46,12 @@ class WikimediaExtractor(BaseExtractor):
else:
self.api_url = None
+ # note: image revisions are different from page revisions
+ # ref:
+ # https://www.mediawiki.org/wiki/API:Revisions
+ # https://www.mediawiki.org/wiki/API:Imageinfo
+ self.image_revisions = self.config("image-revisions", 1)
+
@cache(maxage=36500*86400, keyarg=1)
def _search_api_path(self, root):
self.log.debug("Probing possible API endpoints")
@@ -56,7 +62,10 @@ class WikimediaExtractor(BaseExtractor):
return url
raise exception.AbortExtraction("Unable to find API endpoint")
- def prepare(self, image):
+ def prepare_info(self, info):
+ """Adjust the content of an image info object"""
+
+ def prepare_image(self, image):
"""Adjust the content of an image object"""
image["metadata"] = {
m["name"]: m["value"]
@@ -74,14 +83,19 @@ class WikimediaExtractor(BaseExtractor):
def items(self):
for info in self._pagination(self.params):
try:
- image = info["imageinfo"][0]
- except LookupError:
+ images = info.pop("imageinfo")
+ except KeyError:
self.log.debug("Missing 'imageinfo' for %s", info)
- continue
+ images = ()
+
+ info["count"] = len(images)
+ self.prepare_info(info)
+ yield Message.Directory, info
- self.prepare(image)
- yield Message.Directory, image
- yield Message.Url, image["url"], image
+ for info["num"], image in enumerate(images, 1):
+ self.prepare_image(image)
+ image.update(info)
+ yield Message.Url, image["url"], image
if self.subcategories:
base = self.root + "/wiki/"
@@ -108,6 +122,7 @@ class WikimediaExtractor(BaseExtractor):
"timestamp|user|userid|comment|canonicaltitle|url|size|"
"sha1|mime|metadata|commonmetadata|extmetadata|bitdepth"
)
+ params["iilimit"] = self.image_revisions
while True:
data = self.request_json(url, params=params)
@@ -237,9 +252,8 @@ class WikimediaArticleExtractor(WikimediaExtractor):
"titles" : path,
}
- def prepare(self, image):
- WikimediaExtractor.prepare(self, image)
- image["page"] = self.title
+ def prepare_info(self, info):
+ info["page"] = self.title
class WikimediaWikiExtractor(WikimediaExtractor):
diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py
index e1b4897..98c9331 100644
--- a/gallery_dl/extractor/zerochan.py
+++ b/gallery_dl/extractor/zerochan.py
@@ -26,6 +26,7 @@ class ZerochanExtractor(BooruExtractor):
per_page = 250
cookies_domain = ".zerochan.net"
cookies_names = ("z_id", "z_hash")
+ useragent = util.USERAGENT
request_interval = (0.5, 1.5)
def login(self):
@@ -192,7 +193,13 @@ class ZerochanTagExtractor(ZerochanExtractor):
metadata = self.config("metadata")
while True:
- page = self.request(url, params=params, expected=(500,)).text
+ try:
+ page = self.request(
+ url, params=params, expected=(500,)).text
+ except exception.HttpError as exc:
+ if exc.status == 404:
+ return
+ raise
thumbs = text.extr(page, '<ul id="thumbs', '</ul>')
extr = text.extract_from(thumbs)
@@ -231,7 +238,13 @@ class ZerochanTagExtractor(ZerochanExtractor):
}
while True:
- response = self.request(url, params=params, allow_redirects=False)
+ try:
+ response = self.request(
+ url, params=params, allow_redirects=False)
+ except exception.HttpError as exc:
+ if exc.status == 404:
+ return
+ raise
if response.status_code >= 300:
url = text.urljoin(self.root, response.headers["location"])
@@ -275,12 +288,18 @@ class ZerochanImageExtractor(ZerochanExtractor):
pattern = BASE_PATTERN + r"/(\d+)"
example = "https://www.zerochan.net/12345"
- def __init__(self, match):
- ZerochanExtractor.__init__(self, match)
- self.image_id = match[1]
-
def posts(self):
- post = self._parse_entry_html(self.image_id)
+ image_id = self.groups[0]
+
+ try:
+ post = self._parse_entry_html(image_id)
+ except exception.HttpError as exc:
+ if exc.status in (404, 410):
+ if msg := text.extr(exc.response.text, "<h2>", "<"):
+ self.log.warning(f"'{msg}'")
+ return ()
+ raise
+
if self.config("metadata"):
- post.update(self._parse_entry_api(self.image_id))
+ post.update(self._parse_entry_api(image_id))
return (post,)
diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py
index 9992c56..82ad388 100644
--- a/gallery_dl/postprocessor/common.py
+++ b/gallery_dl/postprocessor/common.py
@@ -21,7 +21,7 @@ class PostProcessor():
def __repr__(self):
return self.__class__.__name__
- def _init_archive(self, job, options, prefix=None):
+ def _archive_init(self, job, options, prefix=None):
if archive_path := options.get("archive"):
extr = job.extractor
@@ -54,11 +54,13 @@ class PostProcessor():
else:
self.log.debug(
"Using %s archive '%s'", self.name, archive_path)
- job.register_hooks({"finalize": self._close_archive})
return True
self.archive = None
return False
- def _close_archive(self, _):
+ def _archive_register(self, job):
+ job.register_hooks({"finalize": self._archive_close})
+
+ def _archive_close(self, _):
self.archive.close()
diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py
index 0bfe1a2..ef11bff 100644
--- a/gallery_dl/postprocessor/exec.py
+++ b/gallery_dl/postprocessor/exec.py
@@ -50,7 +50,8 @@ class ExecPP(PostProcessor):
events = events.split(",")
job.register_hooks({event: execute for event in events}, options)
- self._init_archive(job, options)
+ if self._archive_init(job, options):
+ self._archive_register(job)
def _prepare_cmd(self, cmd):
if isinstance(cmd, str):
diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py
index a6d2b7f..90e6e3d 100644
--- a/gallery_dl/postprocessor/metadata.py
+++ b/gallery_dl/postprocessor/metadata.py
@@ -110,7 +110,9 @@ class MetadataPP(PostProcessor):
events = events.split(",")
job.register_hooks({event: self.run for event in events}, options)
- self._init_archive(job, options, "_MD_")
+ if self._archive_init(job, options, "_MD_"):
+ self._archive_register(job)
+
self.filter = self._make_filter(options)
self.mtime = options.get("mtime")
self.omode = options.get("open", omode)
diff --git a/gallery_dl/postprocessor/python.py b/gallery_dl/postprocessor/python.py
index 66d9343..5011ce8 100644
--- a/gallery_dl/postprocessor/python.py
+++ b/gallery_dl/postprocessor/python.py
@@ -26,6 +26,9 @@ class PythonPP(PostProcessor):
module = util.import_file(module_name)
self.function = getattr(module, function_name)
+ if archive := self._archive_init(job, options):
+ self.run = self.run_archive
+
events = options.get("event")
if events is None:
events = ("file",)
@@ -33,8 +36,8 @@ class PythonPP(PostProcessor):
events = events.split(",")
job.register_hooks({event: self.run for event in events}, options)
- if self._init_archive(job, options):
- self.run = self.run_archive
+ if archive:
+ self._archive_register(job)
def run(self, pathfmt):
self.function(pathfmt.kwdict)
diff --git a/gallery_dl/version.py b/gallery_dl/version.py
index 4861a9d..d3e0277 100644
--- a/gallery_dl/version.py
+++ b/gallery_dl/version.py
@@ -6,5 +6,5 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
-__version__ = "1.30.8"
+__version__ = "1.30.9"
__variant__ = None
diff --git a/test/test_downloader.py b/test/test_downloader.py
index ecd8b85..f6c3dbe 100644
--- a/test/test_downloader.py
+++ b/test/test_downloader.py
@@ -386,6 +386,8 @@ SAMPLES = {
("mp3" , b"\xFF\xFB"),
("mp3" , b"\xFF\xF3"),
("mp3" , b"\xFF\xF2"),
+ ("m3u8", b"#EXTM3U\n#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=200000"),
+ ("mpd" , b'<MPD xmlns="urn:mpeg:dash:schema:mpd:2011"'),
("zip" , b"PK\x03\x04"),
("zip" , b"PK\x05\x06"),
("zip" , b"PK\x07\x08"),
diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py
index 2902fea..17b36b6 100644
--- a/test/test_postprocessor.py
+++ b/test/test_postprocessor.py
@@ -21,7 +21,7 @@ from datetime import datetime
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from gallery_dl import extractor, output, path, util, exception # noqa E402
-from gallery_dl import postprocessor, config # noqa E402
+from gallery_dl import postprocessor, config, archive # noqa E402
from gallery_dl.postprocessor.common import PostProcessor # noqa E402
@@ -39,7 +39,7 @@ class FakeJob():
self.get_logger = logging.getLogger
self.hooks = collections.defaultdict(list)
- def register_hooks(self, hooks, options):
+ def register_hooks(self, hooks, options=None):
for hook, callback in hooks.items():
self.hooks[hook].append(callback)
@@ -353,6 +353,23 @@ class ExecTest(BasePostprocessorTest):
)
i.wait.assert_called_once_with()
+ def test_archive(self):
+ pp = self._create({
+ "command": ["echo", "foobar"],
+ "archive": ":memory:",
+ "event" : "finalize",
+ })
+
+ self.assertIsInstance(pp.archive, archive.DownloadArchive)
+
+ with patch.object(pp.archive, "add") as m_aa, \
+ patch.object(pp.archive, "close") as m_ac:
+ self._trigger(("finalize",))
+ pp.archive.close()
+
+ m_aa.assert_called_once_with(self.pathfmt.kwdict)
+ m_ac.assert_called_once()
+
class HashTest(BasePostprocessorTest):
@@ -811,6 +828,22 @@ class MetadataTest(BasePostprocessorTest):
}
""")
+ def test_archive(self):
+ pp = self._create({
+ "archive": ":memory:",
+ "event" : "finalize",
+ })
+
+ self.assertIsInstance(pp.archive, archive.DownloadArchive)
+
+ with patch.object(pp.archive, "add") as m_aa, \
+ patch.object(pp.archive, "close") as m_ac:
+ self._trigger(("finalize",))
+ pp.archive.close()
+
+ m_aa.assert_called_once_with(self.pathfmt.kwdict)
+ m_ac.assert_called_once()
+
def _output(self, mock):
return "".join(
call[1][0]
@@ -890,6 +923,23 @@ class PythonTest(BasePostprocessorTest):
with self.assertRaises(exception.StopExtraction):
self._trigger()
+ def test_archive(self):
+ pp = self._create({
+ "expression": "True",
+ "archive" : ":memory:",
+ "event" : "finalize",
+ })
+
+ self.assertIsInstance(pp.archive, archive.DownloadArchive)
+
+ with patch.object(pp.archive, "add") as m_aa, \
+ patch.object(pp.archive, "close") as m_ac:
+ self._trigger(("finalize",))
+ pp.archive.close()
+
+ m_aa.assert_called_once_with(self.pathfmt.kwdict)
+ m_ac.assert_called_once()
+
def _write_module(self, path):
with open(path, "w") as fp:
fp.write("""
diff --git a/test/test_results.py b/test/test_results.py
index 7e024b8..2e2eaa9 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -348,6 +348,17 @@ class TestExtractorResults(unittest.TestCase):
for _ in value:
len_value += 1
self.assertEqual(int(length), len_value, msg=path)
+ elif test.startswith("hash:"):
+ digest = test[5:].lower()
+ msg = f"{path} / {digest}"
+ if digest == "md5":
+ self.assertRegex(value, r"^[0-9a-fA-F]{32}$", msg)
+ elif digest == "sha1":
+ self.assertRegex(value, r"^[0-9a-fA-F]{40}$", msg)
+ elif digest == "sha256":
+ self.assertRegex(value, r"^[0-9a-fA-F]{64}$", msg)
+ elif digest == "sha512":
+ self.assertRegex(value, r"^[0-9a-fA-F]{128}$", msg)
elif test.startswith("iso:"):
iso = test[4:]
if iso in ("dt", "datetime", "8601"):