New upstream version 1.24.0.upstream/1.24.0

5 files changed, 133 insertions, 14 deletions

diff --git a/data/completion/_gallery-dl b/data/completion/_gallery-dl
index 5e46dc5..eb5c0f4 100644
--- a/data/completion/_gallery-dl
+++ b/data/completion/_gallery-dl
@@ -13,6 +13,7 @@ _arguments -C -S \
 {-f,--filename}'[Filename format string for downloaded files ("/O" for "original" filenames)]':'<format>' \
 --proxy'[Use the specified proxy]':'<url>' \
 --source-address'[Client-side IP address to bind to]':'<ip>' \
+--user-agent'[User-Agent request header]':'<ua>' \
 --clear-cache'[Delete cached login sessions, cookies, etc. for MODULE (ALL to delete everything)]':'<module>' \
 --cookies'[File to load additional cookies from]':'<file>':_files \
 --cookies-from-browser'[Name of the browser to load cookies from, with optional keyring name prefixed with "+" and profile prefixed with ":"]':'<browser[+keyring][:profile]>' \
@@ -37,6 +38,7 @@ _arguments -C -S \
 --sleep-extractor'[Number of seconds to wait before starting data extraction for an input URL]':'<seconds>' \
 --filesize-min'[Do not download files smaller than SIZE (e.g. 500k or 2.5M)]':'<size>' \
 --filesize-max'[Do not download files larger than SIZE (e.g. 500k or 2.5M)]':'<size>' \
+--chunk-size'[Size of in-memory data chunks (default: 32k)]':'<size>' \
 --no-part'[Do not use .part files]' \
 --no-skip'[Do not skip downloads; overwrite existing files]' \
 --no-mtime'[Do not set file modification times according to Last-Modified HTTP response headers]' \
diff --git a/data/completion/gallery-dl b/data/completion/gallery-dl
index 40280d5..f57306e 100644
--- a/data/completion/gallery-dl
+++ b/data/completion/gallery-dl
@@ -10,7 +10,7 @@ _gallery_dl()
     elif [[ "${prev}" =~ ^()$ ]]; then
         COMPREPLY=( $(compgen -d -- "${cur}") )
     else
-        COMPREPLY=( $(compgen -W "--help --version --input-file --destination --directory --filename --proxy --source-address --clear-cache --cookies --cookies-from-browser --quiet --verbose --get-urls --resolve-urls --dump-json --simulate --extractor-info --list-keywords --list-modules --list-extractors --write-log --write-unsupported --write-pages --limit-rate --retries --http-timeout --sleep --sleep-request --sleep-extractor --filesize-min --filesize-max --no-part --no-skip --no-mtime --no-download --no-postprocessors --no-check-certificate --config --config-yaml --option --ignore-config --username --password --netrc --download-archive --abort --terminate --range --chapter-range --filter --chapter-filter --zip --ugoira-conv --ugoira-conv-lossless --ugoira-conv-copy --write-metadata --write-info-json --write-infojson --write-tags --mtime-from-date --exec --exec-after --postprocessor" -- "${cur}") )
+        COMPREPLY=( $(compgen -W "--help --version --input-file --destination --directory --filename --proxy --source-address --user-agent --clear-cache --cookies --cookies-from-browser --quiet --verbose --get-urls --resolve-urls --dump-json --simulate --extractor-info --list-keywords --list-modules --list-extractors --write-log --write-unsupported --write-pages --limit-rate --retries --http-timeout --sleep --sleep-request --sleep-extractor --filesize-min --filesize-max --chunk-size --no-part --no-skip --no-mtime --no-download --no-postprocessors --no-check-certificate --config --config-yaml --option --ignore-config --username --password --netrc --download-archive --abort --terminate --range --chapter-range --filter --chapter-filter --zip --ugoira-conv --ugoira-conv-lossless --ugoira-conv-copy --write-metadata --write-info-json --write-infojson --write-tags --mtime-from-date --exec --exec-after --postprocessor" -- "${cur}") )
     fi
 }
 
diff --git a/data/completion/gallery-dl.fish b/data/completion/gallery-dl.fish
index 587ff12..87e625a 100644
--- a/data/completion/gallery-dl.fish
+++ b/data/completion/gallery-dl.fish
@@ -7,6 +7,7 @@ complete -c gallery-dl -x -a '(__fish_complete_directories)' -s 'D' -l 'director
 complete -c gallery-dl -x -s 'f' -l 'filename' -d 'Filename format string for downloaded files ("/O" for "original" filenames)'
 complete -c gallery-dl -x -l 'proxy' -d 'Use the specified proxy'
 complete -c gallery-dl -x -l 'source-address' -d 'Client-side IP address to bind to'
+complete -c gallery-dl -x -l 'user-agent' -d 'User-Agent request header'
 complete -c gallery-dl -x -l 'clear-cache' -d 'Delete cached login sessions, cookies, etc. for MODULE (ALL to delete everything)'
 complete -c gallery-dl -r -F -l 'cookies' -d 'File to load additional cookies from'
 complete -c gallery-dl -x -l 'cookies-from-browser' -d 'Name of the browser to load cookies from, with optional keyring name prefixed with "+" and profile prefixed with ":"'
@@ -31,6 +32,7 @@ complete -c gallery-dl -x -l 'sleep-request' -d 'Number of seconds to wait betwe
 complete -c gallery-dl -x -l 'sleep-extractor' -d 'Number of seconds to wait before starting data extraction for an input URL'
 complete -c gallery-dl -x -l 'filesize-min' -d 'Do not download files smaller than SIZE (e.g. 500k or 2.5M)'
 complete -c gallery-dl -x -l 'filesize-max' -d 'Do not download files larger than SIZE (e.g. 500k or 2.5M)'
+complete -c gallery-dl -x -l 'chunk-size' -d 'Size of in-memory data chunks (default: 32k)'
 complete -c gallery-dl -l 'no-part' -d 'Do not use .part files'
 complete -c gallery-dl -l 'no-skip' -d 'Do not skip downloads; overwrite existing files'
 complete -c gallery-dl -l 'no-mtime' -d 'Do not set file modification times according to Last-Modified HTTP response headers'
diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1
index 0b27854..059b726 100644
--- a/data/man/gallery-dl.1
+++ b/data/man/gallery-dl.1
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL" "1" "2022-10-30" "1.23.5" "gallery-dl Manual"
+.TH "GALLERY-DL" "1" "2022-11-20" "1.24.0" "gallery-dl Manual"
 .\" disable hyphenation
 .nh
 
@@ -41,6 +41,9 @@ Use the specified proxy
 .B "\-\-source\-address" \f[I]IP\f[]
 Client-side IP address to bind to
 .TP
+.B "\-\-user\-agent" \f[I]UA\f[]
+User-Agent request header
+.TP
 .B "\-\-clear\-cache" \f[I]MODULE\f[]
 Delete cached login sessions, cookies, etc. for MODULE (ALL to delete everything)
 .TP
@@ -113,6 +116,9 @@ Do not download files smaller than SIZE (e.g. 500k or 2.5M)
 .B "\-\-filesize\-max" \f[I]SIZE\f[]
 Do not download files larger than SIZE (e.g. 500k or 2.5M)
 .TP
+.B "\-\-chunk\-size" \f[I]SIZE\f[]
+Size of in-memory data chunks (default: 32k)
+.TP
 .B "\-\-no\-part" 
 Do not use .part files
 .TP
diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5
index 8944195..847d665 100644
--- a/data/man/gallery-dl.conf.5
+++ b/data/man/gallery-dl.conf.5
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL.CONF" "5" "2022-10-30" "1.23.5" "gallery-dl Manual"
+.TH "GALLERY-DL.CONF" "5" "2022-11-20" "1.24.0" "gallery-dl Manual"
 .\" disable hyphenation
 .nh
 .\" disable justification (adjust text to left margin only)
@@ -449,8 +449,6 @@ and optional for
 .br
 * \f[I]inkbunny\f[]
 .br
-* \f[I]instagram\f[]
-.br
 * \f[I]kemonoparty\f[]
 .br
 * \f[I]mangadex\f[]
@@ -615,6 +613,9 @@ or a \f[I]list\f[] with IP and explicit port number as elements.
 .IP "Description:" 4
 User-Agent header value to be used for HTTP requests.
 
+Setting this value to \f[I]"browser"\f[] will try to automatically detect
+and use the User-Agent used by the system's default browser.
+
 Note: This option has no effect on pixiv extractors,
 as these need specific values to function correctly.
 
@@ -624,7 +625,10 @@ as these need specific values to function correctly.
 \f[I]string\f[]
 
 .IP "Default:" 9
-\f[I]"firefox"\f[] for \f[I]patreon\f[], \f[I]null\f[] everywhere else
+.br
+* \f[I]"firefox"\f[] for \f[I]patreon\f[], \f[I]mangapark\f[], and \f[I]mangasee\f[]
+.br
+* \f[I]null\f[] everywhere else
 
 .IP "Example:" 4
 .br
@@ -696,6 +700,23 @@ For example, setting this option to \f[I]"gdl_path"\f[] would make it possible
 to access the current file's filename as \f[I]"[gdl_path.filename}"\f[].
 
 
+.SS extractor.*.http-metadata
+.IP "Type:" 6
+\f[I]string\f[]
+
+.IP "Default:" 9
+\f[I]null\f[]
+
+.IP "Description:" 4
+Insert an \f[I]object\f[] containing a file's HTTP headers and
+\f[I]filename\f[], \f[I]extension\f[], and \f[I]date\f[] parsed from them
+into metadata dictionaries as the given name.
+
+For example, setting this option to \f[I]"gdl_http"\f[] would make it possible
+to access the current file's \f[I]Last-Modified\f[] header as \f[I]"{gdl_http[Last-Modified]}"\f[]
+and its parsed form as \f[I]"{gdl_http[date]}"\f[].
+
+
 .SS extractor.*.category-transfer
 .IP "Type:" 6
 \f[I]bool\f[]
@@ -1718,17 +1739,15 @@ for details)
 \f[I]string\f[]
 
 .IP "Default:" 9
-\f[I]"auto"\f[]
+\f[I]"rest"\f[]
 
 .IP "Description:" 4
 Selects which API endpoints to use.
 
 .br
-* \f[I]"rest"\f[]: REST API - higher-resolution media, only usable when logged in
+* \f[I]"rest"\f[]: REST API - higher-resolution media
 .br
-* \f[I]"graphql"\f[]: GraphQL API - lower-resolution media, partially accessible when not logged in
-.br
-* \f[I]"auto"\f[]: Use REST API when logged in, GraphQL API otherwise
+* \f[I]"graphql"\f[]: GraphQL API - lower-resolution media
 
 
 .SS extractor.instagram.include
@@ -1748,7 +1767,6 @@ when processing a user profile.
 Possible values are
 \f[I]"posts"\f[],
 \f[I]"reels"\f[],
-\f[I]"channel"\f[]
 \f[I]"tagged"\f[],
 \f[I]"stories"\f[],
 \f[I]"highlights"\f[],
@@ -3105,6 +3123,32 @@ to use your account's browsing settings and default filters when searching.
 See https://wallhaven.cc/help/api for more information.
 
 
+.SS extractor.wallhaven.include
+.IP "Type:" 6
+.br
+* \f[I]string\f[]
+.br
+* \f[I]list\f[] of \f[I]strings\f[]
+
+.IP "Default:" 9
+\f[I]"uploads"\f[]
+
+.IP "Example:" 4
+.br
+* "uploads,collections"
+.br
+* ["uploads", "collections"]
+
+.IP "Description:" 4
+A (comma-separated) list of subcategories to include
+when processing a user profile.
+
+Possible values are
+\f[I]"uploads"\f[], \f[I]"collections"\f[].
+
+It is possible to use \f[I]"all"\f[] instead of listing all values separately.
+
+
 .SS extractor.wallhaven.metadata
 .IP "Type:" 6
 \f[I]bool\f[]
@@ -3413,7 +3457,7 @@ Minimum/Maximum allowed file size in bytes.
 Any file smaller/larger than this limit will not be downloaded.
 
 Possible values are valid integer or floating-point numbers
-optionally followed by one of \f[I]k\f[], \f[I]m\f[]. \f[I]g\f[], \f[I]t\f[] or \f[I]p\f[].
+optionally followed by one of \f[I]k\f[], \f[I]m\f[]. \f[I]g\f[], \f[I]t\f[], or \f[I]p\f[].
 These suffixes are case-insensitive.
 
 
@@ -3491,7 +3535,7 @@ Set this option to \f[I]null\f[] to disable this indicator.
 Maximum download rate in bytes per second.
 
 Possible values are valid integer or floating-point numbers
-optionally followed by one of \f[I]k\f[], \f[I]m\f[]. \f[I]g\f[], \f[I]t\f[] or \f[I]p\f[].
+optionally followed by one of \f[I]k\f[], \f[I]m\f[]. \f[I]g\f[], \f[I]t\f[], or \f[I]p\f[].
 These suffixes are case-insensitive.
 
 
@@ -3559,6 +3603,24 @@ of a file called \f[I]example.png\f[] from \f[I]png\f[] to \f[I]jpg\f[] when sai
 contains JPEG/JFIF data.
 
 
+.SS downloader.http.chunk-size
+.IP "Type:" 6
+\f[I]integer\f[] or \f[I]string\f[]
+
+.IP "Default:" 9
+\f[I]32768\f[]
+
+.IP "Example:" 4
+"50k", "0.8M"
+
+.IP "Description:" 4
+Number of bytes per downloaded chunk.
+
+Possible values are integer numbers
+optionally followed by one of \f[I]k\f[], \f[I]m\f[]. \f[I]g\f[], \f[I]t\f[], or \f[I]p\f[].
+These suffixes are case-insensitive.
+
+
 .SS downloader.http.headers
 .IP "Type:" 6
 \f[I]object\f[]
@@ -4009,6 +4071,9 @@ Selects how to process metadata.
 * \f[I]"json"\f[]: write metadata using \f[I]json.dump()
 <https://docs.python.org/3/library/json.html#json.dump>\f[]
 .br
+* \f[I]"jsonl"\f[]: write metadata in \f[I]JSON Lines
+<https://jsonlines.org/>\f[] format
+.br
 * \f[I]"tags"\f[]: write \f[I]tags\f[] separated by newlines
 .br
 * \f[I]"custom"\f[]: write the result of applying \f[I]metadata.content-format\f[]
@@ -4112,6 +4177,8 @@ When skipping a file download
 \f[I]post\f[]
 When starting to download all files of a post,
 e.g. a Tweet on Twitter or a post on Patreon.
+\f[I]post-after\f[]
+After downloading all files of a post
 
 
 .SS metadata.fields
@@ -4163,6 +4230,48 @@ Custom format string to build the content of metadata files with.
 Note: Only applies for \f[I]"mode": "custom"\f[].
 
 
+.SS metadata.open
+.IP "Type:" 6
+\f[I]string\f[]
+
+.IP "Defsult:" 4
+\f[I]"w"\f[]
+
+.IP "Description:" 4
+The \f[I]mode\f[] in which metadata files get opened.
+
+For example,
+use \f[I]"a"\f[] to append to a file's content
+or \f[I]"w"\f[] to truncate it.
+
+See the \f[I]mode\f[] parameter of \f[I]open()\f[] for further details.
+
+
+.SS metadata.private
+.IP "Type:" 6
+\f[I]bool\f[]
+
+.IP "Default:" 9
+\f[I]false\f[]
+
+.IP "Description:" 4
+Include private fields,
+i.e. fields whose name starts with an underscore.
+
+
+.SS metadata.encoding
+.IP "Type:" 6
+\f[I]string\f[]
+
+.IP "Defsult:" 4
+\f[I]"utf-8"\f[]
+
+.IP "Description:" 4
+Name of the encoding used to encode a file's content.
+
+See the \f[I]encoding\f[] parameter of \f[I]open()\f[] for further details.
+
+
 .SS metadata.archive
 .IP "Type:" 6
 \f[I]Path\f[]