From 1a457ed68769880ab7760e0746f0cbbd9ca00487 Mon Sep 17 00:00:00 2001 From: Unit 193 Date: Sat, 28 Sep 2024 20:01:25 -0400 Subject: New upstream version 1.27.5. --- CHANGELOG.md | 78 ++---- PKG-INFO | 6 +- README.rst | 4 +- data/completion/_gallery-dl | 4 +- data/completion/gallery-dl.fish | 4 +- data/man/gallery-dl.1 | 6 +- data/man/gallery-dl.conf.5 | 287 ++++++++++++++++++++-- gallery_dl.egg-info/PKG-INFO | 6 +- gallery_dl.egg-info/SOURCES.txt | 3 + gallery_dl/__init__.py | 10 +- gallery_dl/cookies.py | 24 +- gallery_dl/downloader/ytdl.py | 10 +- gallery_dl/extractor/8chan.py | 16 +- gallery_dl/extractor/__init__.py | 3 + gallery_dl/extractor/ao3.py | 302 +++++++++++++++++++++++ gallery_dl/extractor/bluesky.py | 78 +++--- gallery_dl/extractor/chevereto.py | 2 +- gallery_dl/extractor/civitai.py | 490 +++++++++++++++++++++++++++++++++++++ gallery_dl/extractor/cohost.py | 223 +++++++++++++++++ gallery_dl/extractor/common.py | 80 +++--- gallery_dl/extractor/deviantart.py | 44 +++- gallery_dl/extractor/flickr.py | 6 +- gallery_dl/extractor/inkbunny.py | 13 +- gallery_dl/extractor/newgrounds.py | 4 +- gallery_dl/extractor/pixiv.py | 5 +- gallery_dl/extractor/skeb.py | 6 +- gallery_dl/extractor/weasyl.py | 28 ++- gallery_dl/extractor/wikimedia.py | 21 +- gallery_dl/extractor/zzup.py | 30 ++- gallery_dl/formatter.py | 2 + gallery_dl/job.py | 7 +- gallery_dl/option.py | 11 +- gallery_dl/postprocessor/ugoira.py | 128 +++++++--- gallery_dl/text.py | 25 +- gallery_dl/util.py | 34 ++- gallery_dl/version.py | 2 +- test/test_extractor.py | 19 +- test/test_formatter.py | 11 + test/test_results.py | 42 +++- test/test_text.py | 8 +- test/test_util.py | 16 ++ 41 files changed, 1826 insertions(+), 272 deletions(-) create mode 100644 gallery_dl/extractor/ao3.py create mode 100644 gallery_dl/extractor/civitai.py create mode 100644 gallery_dl/extractor/cohost.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7397593..ab8f174 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,57 +1,33 @@ -## 1.27.4 - 2024-09-06 +## 1.27.5 - 2024-09-28 ### Extractors #### Additions -- [sexcom] add `likes` extractor ([#6149](https://github.com/mikf/gallery-dl/issues/6149)) -- [wikimedia] add `wiki` extractor ([#6050](https://github.com/mikf/gallery-dl/issues/6050)) +- [ao3] add support ([#6013](https://github.com/mikf/gallery-dl/issues/6013)) +- [civitai] add support ([#3706](https://github.com/mikf/gallery-dl/issues/3706), [#3787](https://github.com/mikf/gallery-dl/issues/3787), [#4129](https://github.com/mikf/gallery-dl/issues/4129), [#5995](https://github.com/mikf/gallery-dl/issues/5995), [#6220](https://github.com/mikf/gallery-dl/issues/6220)) +- [cohost] add support ([#4483](https://github.com/mikf/gallery-dl/issues/4483), [#6191](https://github.com/mikf/gallery-dl/issues/6191)) #### Fixes -- [bunkr] fix file downloads ([#6037](https://github.com/mikf/gallery-dl/issues/6037)) -- [cyberdrop] fix extraction -- [deviantart] fix `"pagination": "manual"` for cursor-based endpoints ([#6079](https://github.com/mikf/gallery-dl/issues/6079)) -- [deviantart] fix `"original": "images"` ([#6124](https://github.com/mikf/gallery-dl/issues/6124)) -- [exhentai] fix `limits` option ([#6090](https://github.com/mikf/gallery-dl/issues/6090)) -- [flickr] make `exif` and `context` metadata extraction non-fatal ([#6002](https://github.com/mikf/gallery-dl/issues/6002), [#6077](https://github.com/mikf/gallery-dl/issues/6077)) -- [flickr] make `album` metadata extraction non-fatal ([#3441](https://github.com/mikf/gallery-dl/issues/3441)) -- [furaffinity] fix `favorite` pagination ([#6151](https://github.com/mikf/gallery-dl/issues/6151)) -- [gofile] fix `KeyError: 'childrenIds'` ([#5993](https://github.com/mikf/gallery-dl/issues/5993)) -- [newgrounds] fix warning for age-restricted posts ([#6005](https://github.com/mikf/gallery-dl/issues/6005)) -- [toyhouse] fix extraction of image URLs -- [tumblr] fix `401 Unauthorized` for likes when using api-key ([#5994](https://github.com/mikf/gallery-dl/issues/5994)) -- [twitter] fix pinned Tweet extraction ([#6102](https://github.com/mikf/gallery-dl/issues/6102)) -- [ytdl] fix processing playlists of playlists ([#6127](https://github.com/mikf/gallery-dl/issues/6127)) +- [8chan] update `TOS` cookie name +- [deviantart] work around OAuth API returning empty journal texts ([#6196](https://github.com/mikf/gallery-dl/issues/6196), [#6207](https://github.com/mikf/gallery-dl/issues/6207), [#5916](https://github.com/mikf/gallery-dl/issues/5916)) +- [weasyl:favorite] fix pagination ([#6113](https://github.com/mikf/gallery-dl/issues/6113)) #### Improvements -- [bcbnsfw] use `*` as query when retrieving all posts ([#6135](https://github.com/mikf/gallery-dl/issues/6135)) -- [bunkr] support `bunkr:` URL prefix ([#6017](https://github.com/mikf/gallery-dl/issues/6017)) -- [e621] cache pool metadata API calls ([#6001](https://github.com/mikf/gallery-dl/issues/6001)) -- [generic] better directory names ([#6104](https://github.com/mikf/gallery-dl/issues/6104)) -- [koharu] improve format selection ([#6088](https://github.com/mikf/gallery-dl/issues/6088)) -- [pixiv] implement downloading "original" ugoira frames ([#6056](https://github.com/mikf/gallery-dl/issues/6056)) -- [pixiv] use mobile API for `series` ([#5983](https://github.com/mikf/gallery-dl/issues/5983)) -#### Metadata -- [batoto] improve chapter info regex ([#5988](https://github.com/mikf/gallery-dl/issues/5988), [#5997](https://github.com/mikf/gallery-dl/issues/5997)) -- [batoto] extract `chapter_url` metadata ([#5562](https://github.com/mikf/gallery-dl/issues/5562)) -- [batoto] improve `title` extraction ([#5988](https://github.com/mikf/gallery-dl/issues/5988)) -- [hitomi] extract `extension_original` metadata ([#6049](https://github.com/mikf/gallery-dl/issues/6049)) -- [instagram] add `post_date` metadata field ([#6081](https://github.com/mikf/gallery-dl/issues/6081), [#6091](https://github.com/mikf/gallery-dl/issues/6091)) -- [sankaku] restore old `tags` format ([#6043](https://github.com/mikf/gallery-dl/issues/6043)) -- [twitter] extract `type` metadata ([#6111](https://github.com/mikf/gallery-dl/issues/6111)) -#### Options -- [bunkr] add `tlds` option to match URLs with all possible TLDs ([#5875](https://github.com/mikf/gallery-dl/issues/5875), [#6017](https://github.com/mikf/gallery-dl/issues/6017)) -- [instagram] add `max-posts` option ([#6054](https://github.com/mikf/gallery-dl/issues/6054)) -- [instagram] add `info` as a possible `include` value -- [instagram] allow disabling `cursor` output -- [twitter] add `info` as a possible `include` value ([#6114](https://github.com/mikf/gallery-dl/issues/6114)) -- [twitter] allow disabling `cursor` output ([#5990](https://github.com/mikf/gallery-dl/issues/5990)) +- [bluesky] support video downloads ([#6183](https://github.com/mikf/gallery-dl/issues/6183)) +- [deviantart] add `previews` option ([#3782](https://github.com/mikf/gallery-dl/issues/3782), [#6124](https://github.com/mikf/gallery-dl/issues/6124)) +- [deviantart] warn about empty journal texts ([#5916](https://github.com/mikf/gallery-dl/issues/5916)) +- [inkbunny:favorite] update default directory ([#6115](https://github.com/mikf/gallery-dl/issues/6115)) +- [jpgfish] update domain to `jpg5.su` ([#6231](https://github.com/mikf/gallery-dl/issues/6231)) +- [skeb] prevent 429 errors and need for `request_key` cookie +- [weasyl:favorite] support readable URL format ([#6113](https://github.com/mikf/gallery-dl/issues/6113)) +- [wikimedia] automatically detect API endpoint when none is defined +- [zzup] support `up.zzup.com` galleries ([#6181](https://github.com/mikf/gallery-dl/issues/6181)) ### Post Processors -- [hash] add `hash` post processor to compute file hash metadata ([#6099](https://github.com/mikf/gallery-dl/issues/6099)) -- [metadata] add `include` and `exclude` options ([#6058](https://github.com/mikf/gallery-dl/issues/6058)) -- [metadata] fix using `..` in directories on Windows ([#5942](https://github.com/mikf/gallery-dl/issues/5942), [#6094](https://github.com/mikf/gallery-dl/issues/6094)) -- [rename] add `rename` post processor to rename previously downloaded files ([#5846](https://github.com/mikf/gallery-dl/issues/5846), [#6044](https://github.com/mikf/gallery-dl/issues/6044)) -- [ugoira] support converting "original" frames ([#6056](https://github.com/mikf/gallery-dl/issues/6056)) -- [ugoira] add `skip` option ([#6056](https://github.com/mikf/gallery-dl/issues/6056)) +- [ugoira] implement storing "original" frames in ZIP archives ([#6147](https://github.com/mikf/gallery-dl/issues/6147)) +- [ugoira] fix `KeyError: '_ugoira_frame_index'` ([#6154](https://github.com/mikf/gallery-dl/issues/6154)) +### Formatter +- add `L` conversion - returns the length of a value +- allow accessing `util.NONE` via global `_nul` ### Miscellaneous -- [cookies:firefox] extract only cookies without container by default ([#5957](https://github.com/mikf/gallery-dl/issues/5957)) -- [formatter] implement `A` format specifier ([#6036](https://github.com/mikf/gallery-dl/issues/6036)) -- [tests] fix bug when running tests in a certain order -- [util] extend `CustomNone` with arithmetic operators ([#6007](https://github.com/mikf/gallery-dl/issues/6007), [#6009](https://github.com/mikf/gallery-dl/issues/6009)) -- add `--rename` and `--rename-to` command-line options ([#5846](https://github.com/mikf/gallery-dl/issues/5846), [#6044](https://github.com/mikf/gallery-dl/issues/6044)) -- add `input-files` config option ([#6059](https://github.com/mikf/gallery-dl/issues/6059)) +- [cookies] add `cookies-select` option +- [cookies:firefox] support using domain & container filters together +- [docker] prevent errors in Dockerfile build +- [tests] make `#category` result entries optional +- allow filtering `--list-extractors` results +- implement alternatives for deprecated `utc` datetime functions diff --git a/PKG-INFO b/PKG-INFO index 35dd6ce..32ff8bc 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery_dl -Version: 1.27.4 +Version: 1.27.5 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -114,9 +114,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/README.rst b/README.rst index ab85c75..d0e4a72 100644 --- a/README.rst +++ b/README.rst @@ -74,9 +74,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/data/completion/_gallery-dl b/data/completion/_gallery-dl index 1353fa8..5e1b1e0 100644 --- a/data/completion/_gallery-dl +++ b/data/completion/_gallery-dl @@ -30,7 +30,7 @@ _arguments -s -S \ {-K,--list-keywords}'[Print a list of available keywords and example values for the given URLs]' \ {-e,--error-file}'[Add input URLs which returned an error to FILE]':'':_files \ --list-modules'[Print a list of available extractor modules]' \ ---list-extractors'[Print a list of extractor classes with description, (sub)category and example URL]' \ +--list-extractors'[Print a list of extractor classes with description, (sub)category and example URL]':'' \ --write-log'[Write logging output to FILE]':'':_files \ --write-unsupported'[Write URLs, which get emitted by other extractors but cannot be handled, to FILE]':'':_files \ --write-pages'[Write downloaded intermediary pages to files in the current directory to debug problems]' \ @@ -84,7 +84,7 @@ _arguments -s -S \ --mtime'[Set file modification times according to metadata selected by NAME. Examples: '\''date'\'' or '\''status\[date\]'\'']':'' \ --rename'[Rename previously downloaded files from FORMAT to the current filename format]':'' \ --rename-to'[Rename previously downloaded files from the current filename format to FORMAT]':'' \ ---ugoira'[Convert Pixiv Ugoira to FMT using FFmpeg. Supported formats are '\''webm'\'', '\''mp4'\'', '\''gif'\'', '\''vp8'\'', '\''vp9'\'', '\''vp9-lossless'\'', '\''copy'\''.]':'' \ +--ugoira'[Convert Pixiv Ugoira to FMT using FFmpeg. Supported formats are '\''webm'\'', '\''mp4'\'', '\''gif'\'', '\''vp8'\'', '\''vp9'\'', '\''vp9-lossless'\'', '\''copy'\'', '\''zip'\''.]':'' \ --exec'[Execute CMD for each downloaded file. Supported replacement fields are {} or {_path}, {_directory}, {_filename}. Example: --exec "convert {} {}.png && rm {}"]':'' \ --exec-after'[Execute CMD after all files were downloaded. Example: --exec-after "cd {_directory} && convert * ../doc.pdf"]':'' && rc=0 diff --git a/data/completion/gallery-dl.fish b/data/completion/gallery-dl.fish index 971ba68..7734f40 100644 --- a/data/completion/gallery-dl.fish +++ b/data/completion/gallery-dl.fish @@ -24,7 +24,7 @@ complete -c gallery-dl -s 'E' -l 'extractor-info' -d 'Print extractor defaults a complete -c gallery-dl -s 'K' -l 'list-keywords' -d 'Print a list of available keywords and example values for the given URLs' complete -c gallery-dl -r -F -s 'e' -l 'error-file' -d 'Add input URLs which returned an error to FILE' complete -c gallery-dl -l 'list-modules' -d 'Print a list of available extractor modules' -complete -c gallery-dl -l 'list-extractors' -d 'Print a list of extractor classes with description, (sub)category and example URL' +complete -c gallery-dl -x -l 'list-extractors' -d 'Print a list of extractor classes with description, (sub)category and example URL' complete -c gallery-dl -r -F -l 'write-log' -d 'Write logging output to FILE' complete -c gallery-dl -r -F -l 'write-unsupported' -d 'Write URLs, which get emitted by other extractors but cannot be handled, to FILE' complete -c gallery-dl -l 'write-pages' -d 'Write downloaded intermediary pages to files in the current directory to debug problems' @@ -81,7 +81,7 @@ complete -c gallery-dl -x -l 'mtime' -d 'Set file modification times according t complete -c gallery-dl -l 'mtime-from-date' -d '==SUPPRESS==' complete -c gallery-dl -x -l 'rename' -d 'Rename previously downloaded files from FORMAT to the current filename format' complete -c gallery-dl -x -l 'rename-to' -d 'Rename previously downloaded files from the current filename format to FORMAT' -complete -c gallery-dl -x -l 'ugoira' -d 'Convert Pixiv Ugoira to FMT using FFmpeg. Supported formats are "webm", "mp4", "gif", "vp8", "vp9", "vp9-lossless", "copy".' +complete -c gallery-dl -x -l 'ugoira' -d 'Convert Pixiv Ugoira to FMT using FFmpeg. Supported formats are "webm", "mp4", "gif", "vp8", "vp9", "vp9-lossless", "copy", "zip".' complete -c gallery-dl -l 'ugoira-conv' -d '==SUPPRESS==' complete -c gallery-dl -l 'ugoira-conv-lossless' -d '==SUPPRESS==' complete -c gallery-dl -l 'ugoira-conv-copy' -d '==SUPPRESS==' diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1 index 591daae..06effd6 100644 --- a/data/man/gallery-dl.1 +++ b/data/man/gallery-dl.1 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL" "1" "2024-09-06" "1.27.4" "gallery-dl Manual" +.TH "GALLERY-DL" "1" "2024-09-28" "1.27.5" "gallery-dl Manual" .\" disable hyphenation .nh @@ -92,7 +92,7 @@ Add input URLs which returned an error to FILE .B "\-\-list\-modules" Print a list of available extractor modules .TP -.B "\-\-list\-extractors" +.B "\-\-list\-extractors" \f[I]CATEGORIES\f[] Print a list of extractor classes with description, (sub)category and example URL .TP .B "\-\-write\-log" \f[I]FILE\f[] @@ -255,7 +255,7 @@ Rename previously downloaded files from FORMAT to the current filename format Rename previously downloaded files from the current filename format to FORMAT .TP .B "\-\-ugoira" \f[I]FMT\f[] -Convert Pixiv Ugoira to FMT using FFmpeg. Supported formats are 'webm', 'mp4', 'gif', 'vp8', 'vp9', 'vp9-lossless', 'copy'. +Convert Pixiv Ugoira to FMT using FFmpeg. Supported formats are 'webm', 'mp4', 'gif', 'vp8', 'vp9', 'vp9-lossless', 'copy', 'zip'. .TP .B "\-\-exec" \f[I]CMD\f[] Execute CMD for each downloaded file. Supported replacement fields are {} or {_path}, {_directory}, {_filename}. Example: --exec "convert {} {}.png && rm {}" diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5 index e0d75ac..a36e108 100644 --- a/data/man/gallery-dl.conf.5 +++ b/data/man/gallery-dl.conf.5 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL.CONF" "5" "2024-09-06" "1.27.4" "gallery-dl Manual" +.TH "GALLERY-DL.CONF" "5" "2024-09-28" "1.27.5" "gallery-dl Manual" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -454,6 +454,7 @@ response before \f[I]retrying\f[] the request. .IP "Default:" 9 .br * \f[I]"0.5-1.5"\f[] +\f[I]ao3\f[], \f[I]civitai\f[], \f[I][Danbooru]\f[], \f[I][E621]\f[], \f[I][foolfuuka]:search\f[], \f[I]itaku\f[], \f[I]koharu\f[], \f[I]newgrounds\f[], \f[I][philomena]\f[], \f[I]pixiv:novel\f[], \f[I]plurk\f[], @@ -494,18 +495,13 @@ during data extraction. The username and password to use when attempting to log in to another site. -Specifying username and password is required for - -.br -* \f[I]nijie\f[] -.br -* \f[I]horne\f[] - -and optional for +This is supported for .br * \f[I]aibooru\f[] (*) .br +* \f[I]ao3\f[] +.br * \f[I]aryion\f[] .br * \f[I]atfbooru\f[] (*) @@ -528,6 +524,8 @@ and optional for .br * \f[I]exhentai\f[] .br +* \f[I]horne\f[] (R) +.br * \f[I]idolcomplex\f[] .br * \f[I]imgbb\f[] @@ -542,10 +540,16 @@ and optional for .br * \f[I]mangoxo\f[] .br +* \f[I]newgrounds\f[] +.br +* \f[I]nijie\f[] (R) +.br * \f[I]pillowfort\f[] .br * \f[I]sankaku\f[] .br +* \f[I]seiga\f[] +.br * \f[I]subscribestar\f[] .br * \f[I]tapas\f[] @@ -565,6 +569,9 @@ by using a \f[I].netrc\f[] file. (see Authentication_) (*) The password value for these sites should be the API key found in your user profile, not the actual account password. +(R) Login with username & password or supplying logged-in +\f[I]cookies\f[] is required + Note: Leave the \f[I]password\f[] value empty or undefined to be prompted for a passeword when performing a login (see \f[I]getpass()\f[]). @@ -575,7 +582,7 @@ to be prompted for a passeword when performing a login \f[I]bool\f[] .IP "Default:" 9 -\f[I]true\f[] if stdin is attached to a terminal , +\f[I]true\f[] if stdin is attached to a terminal, \f[I]false\f[] otherwise .IP "Description:" 4 @@ -633,9 +640,9 @@ Source to read additional cookies from. This can be .br * The optional third entry is the keyring to retrieve passwords for decrypting cookies from .br -* The optional fourth entry is a (Firefox) container name (\f[I]"none"\f[] for only cookies with no container) +* The optional fourth entry is a (Firefox) container name (\f[I]"none"\f[] for only cookies with no container (default)) .br -* The optional fifth entry is the domain to extract cookies for. Prefix it with a dot \f[I].\f[] to include cookies for subdomains. Has no effect when also specifying a container. +* The optional fifth entry is the domain to extract cookies for. Prefix it with a dot \f[I].\f[] to include cookies for subdomains. .. code:: json @@ -644,6 +651,32 @@ Source to read additional cookies from. This can be ["chromium", "Private", "kwallet", null, ".twitter.com"] +.SS extractor.*.cookies-select +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Default:" 9 +\f[I]"random"\f[] + +.IP "Description:" 4 +Interpret \f[I]extractor.cookies\f[] +as a list of cookie sources and select one of them for each extractor run. + +.br +* \f[I]"random"\f[]: Select cookies \f[I]randomly\f[] +.br +* \f[I]"rotate"\f[]: Select cookies in sequence. Start over from the beginning after reaching the end of the list. + +.. code:: json + +[ +"~/.local/share/cookies-instagram-com-1.txt", +"~/.local/share/cookies-instagram-com-2.txt", +"~/.local/share/cookies-instagram-com-3.txt", +["firefox", null, null, "c1", ".instagram-com"], +] + + .SS extractor.*.cookies-update .IP "Type:" 6 .br @@ -1492,6 +1525,26 @@ Special values: .SH EXTRACTOR-SPECIFIC OPTIONS +.SS extractor.ao3.formats +.IP "Type:" 6 +.br +* \f[I]string\f[] +.br +* \f[I]list\f[] of \f[I]strings\f[] + +.IP "Default:" 9 +\f[I]"pdf"\f[] + +.IP "Example:" 4 +.br +* "azw3,epub,mobi,pdf,html" +.br +* ["azw3", "epub", "mobi", "pdf", "html"] + +.IP "Description:" 4 +Format(s) to download. + + .SS extractor.artstation.external .IP "Type:" 6 \f[I]bool\f[] @@ -1697,6 +1750,17 @@ Sets the maximum depth of returned reply posts. Process reposts. +.SS extractor.bluesky.videos +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Download videos. + + .SS extractor.bunkr.tlds .IP "Type:" 6 \f[I]bool\f[] @@ -1721,7 +1785,7 @@ Controls which \f[I]bunkr\f[] TLDs to accept. \f[I]["image", "video", "download", "gallery"]\f[] .IP "Description:" 4 -Determines the type and order of files to be downloaded. +Determines the type and order of files to download. Available types are \f[I]image\f[], @@ -1730,6 +1794,151 @@ Available types are \f[I]gallery\f[]. +.SS extractor.civitai.api +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Default:" 9 +\f[I]"rest"\f[] + +.IP "Description:" 4 +Selects which API endpoints to use. + +.br +* \f[I]"rest"\f[]: \f[I]Public REST API\f[] +.br +* \f[I]"trpc"\f[]: Internal TRPC API + + +.SS extractor.civitai.api-key +.IP "Type:" 6 +\f[I]string\f[] + +.IP "Description:" 4 +The API Key value generated in your +\f[I]User Account Settings\f[] +to make authorized API requests. + +See \f[I]API/Authorization\f[] +for details. + + +.SS extractor.civitai.files +.IP "Type:" 6 +\f[I]list\f[] of \f[I]strings\f[] + +.IP "Default:" 9 +\f[I]["image"]\f[] + +.IP "Description:" 4 +Determines the type and order of files to download when processing models. + +Available types are +\f[I]model\f[], +\f[I]image\f[], +\f[I]gallery\f[]. + + +.SS extractor.civitai.nsfw +.IP "Type:" 6 +.br +* \f[I]bool\f[] +.br +* \f[I]string\f[] (\f[I]"api": "rest"\f[]) +.br +* \f[I]integer\f[] (\f[I]"api": "trpc"\f[]) + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Download images rated NSFW. + +.br +* For \f[I]"api": "rest"\f[], this can be one of +\f[I]"None"\f[], \f[I]"Soft"\f[], \f[I]"Mature"\f[], \f[I]"X"\f[] +to set the highest returned mature content flag. + +.br +* For \f[I]"api": "trpc"\f[], this can be an \f[I]integer\f[] +whose bits select the returned mature content flags. + +For example, \f[I]12\f[] (\f[I]4\f[I]8\f[]) would return only +\f[I]Mature\f[] and \f[I]X\f[] rated images, +while \f[I]3\f[] (\f[I]1\f[]2\f[]) would return only +\f[I]None\f[] and \f[I]Soft\f[] rated images, + + +.SS extractor.civitai.quality +.IP "Type:" 6 +.br +* \f[I]string\f[] +.br +* \f[I]list\f[] of \f[I]strings\f[] + +.IP "Default:" 9 +\f[I]"original=true"\f[] + +.IP "Example:" 4 +.br +* "width=1280,quality=90" +.br +* ["width=1280", "quality=90"] + +.IP "Description:" 4 +A (comma-separated) list of image quality options +to pass with every image URL. + +Known available options include \f[I]original\f[], \f[I]quality\f[], \f[I]width\f[] + +Note: Set this option to an arbitrary letter, e.g., \f[I]"w"\f[], +to download images in JPEG format at their original resolution. + + +.SS extractor.cohost.asks +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Extract \f[I]ask\f[] posts. + + +.SS extractor.cohost.pinned +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]false\f[] + +.IP "Description:" 4 +Extract pinned posts. + + +.SS extractor.cohost.replies +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Extract reply posts. + + +.SS extractor.cohost.shares +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]false\f[] + +.IP "Description:" 4 +Extract shared posts. + + .SS extractor.cyberdrop.domain .IP "Type:" 6 \f[I]string\f[] @@ -2141,7 +2350,7 @@ Download original files if available. Setting this option to \f[I]"images"\f[] only downloads original files if they are images and falls back to preview versions for -everything else (archives, etc.). +everything else (archives, videos, etc.). .SS extractor.deviantart.pagination @@ -2160,6 +2369,20 @@ Controls when to stop paginating over API results. * \f[I]"manual"\f[]: Disregard \f[I]has_more\f[] and only stop when a batch of results is empty. +.SS extractor.deviantart.previews +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]false\f[] + +.IP "Description:" 4 +For non-image files (archives, videos, etc.), +also download the file's preview image. + +Set this option to \f[I]"all"\f[] to download previews for all files. + + .SS extractor.deviantart.public .IP "Type:" 6 \f[I]bool\f[] @@ -3798,19 +4021,25 @@ Controls the \f[I]tags\f[] metadata field. .SS extractor.pixiv.ugoira .IP "Type:" 6 -\f[I]bool\f[] +.br +* \f[I]bool\f[] +.br +* \f[I]string\f[] .IP "Default:" 9 \f[I]true\f[] .IP "Description:" 4 -Download Pixiv's Ugoira animations or ignore them. +Download Pixiv's Ugoira animations. + +These animations come as a \f[I].zip\f[] archive containing all +animation frames in JPEG format by default. -These animations come as a \f[I].zip\f[] file containing all -animation frames in JPEG format. +Set this option to \f[I]"original"\f[] +to download them as individual, higher-quality frames. Use an ugoira post processor to convert them -to watchable videos. (Example__) +to watchable animations. (Example__) .SS extractor.pixiv.max-posts @@ -7137,6 +7366,8 @@ Possible values are * "\f[I]image2\f[]" (accurate timecodes, requires nanosecond file timestamps, i.e. no Windows or macOS) .br * "mkvmerge" (accurate timecodes, only WebM or MKV, requires \f[I]mkvmerge\f[]) +.br +* "archive" (store "original" frames in a \f[I].zip\f[] archive) "auto" will select mkvmerge if available and fall back to concat otherwise. @@ -7250,6 +7481,24 @@ to the list of \f[I]ffmpeg\f[] command-line arguments to reduce an odd width/height by 1 pixel and make them even. +.SS ugoira.metadata +.IP "Type:" 6 +.br +* \f[I]bool\f[] +.br +* \f[I]string\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +When using \f[I]"mode": "archive"\f[], save Ugoira frame delay data as +\f[I]animation.json\f[] within the archive file. + +If this is a \f[I]string\f[], +use it as alternate filename for frame delay files. + + .SS ugoira.mtime .IP "Type:" 6 \f[I]bool\f[] diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO index 35dd6ce..32ff8bc 100644 --- a/gallery_dl.egg-info/PKG-INFO +++ b/gallery_dl.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery_dl -Version: 1.27.4 +Version: 1.27.5 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -114,9 +114,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/gallery_dl.egg-info/SOURCES.txt b/gallery_dl.egg-info/SOURCES.txt index 854bfaa..8ae8026 100644 --- a/gallery_dl.egg-info/SOURCES.txt +++ b/gallery_dl.egg-info/SOURCES.txt @@ -57,6 +57,7 @@ gallery_dl/extractor/8muses.py gallery_dl/extractor/__init__.py gallery_dl/extractor/adultempire.py gallery_dl/extractor/agnph.py +gallery_dl/extractor/ao3.py gallery_dl/extractor/architizer.py gallery_dl/extractor/artstation.py gallery_dl/extractor/aryion.py @@ -70,6 +71,8 @@ gallery_dl/extractor/bunkr.py gallery_dl/extractor/catbox.py gallery_dl/extractor/chevereto.py gallery_dl/extractor/cien.py +gallery_dl/extractor/civitai.py +gallery_dl/extractor/cohost.py gallery_dl/extractor/comicvine.py gallery_dl/extractor/common.py gallery_dl/extractor/cyberdrop.py diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index 663fe99..7a9e0be 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -202,12 +202,18 @@ def main(): extractor.modules.append("") sys.stdout.write("\n".join(extractor.modules)) - elif args.list_extractors: + elif args.list_extractors is not None: write = sys.stdout.write fmt = ("{}{}\nCategory: {} - Subcategory: {}" "\nExample : {}\n\n").format - for extr in extractor.extractors(): + extractors = extractor.extractors() + if args.list_extractors: + fltr = util.build_extractor_filter( + args.list_extractors, negate=False) + extractors = filter(fltr, extractors) + + for extr in extractors: write(fmt( extr.__name__, "\n" + extr.__doc__ if extr.__doc__ else "", diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py index deb7c7b..0ffd29a 100644 --- a/gallery_dl/cookies.py +++ b/gallery_dl/cookies.py @@ -50,21 +50,27 @@ def load_cookies_firefox(cookiejar, profile=None, container=None, domain=None): sql = ("SELECT name, value, host, path, isSecure, expiry " "FROM moz_cookies") - parameters = () + conditions = [] + parameters = [] if container_id is False: - sql += " WHERE NOT INSTR(originAttributes,'userContextId=')" + conditions.append("NOT INSTR(originAttributes,'userContextId=')") elif container_id: - sql += " WHERE originAttributes LIKE ? OR originAttributes LIKE ?" + conditions.append( + "originAttributes LIKE ? OR originAttributes LIKE ?") uid = "%userContextId={}".format(container_id) - parameters = (uid, uid + "&%") - elif domain: + parameters += (uid, uid + "&%") + + if domain: if domain[0] == ".": - sql += " WHERE host == ? OR host LIKE ?" - parameters = (domain[1:], "%" + domain) + conditions.append("host == ? OR host LIKE ?") + parameters += (domain[1:], "%" + domain) else: - sql += " WHERE host == ? OR host == ?" - parameters = (domain, "." + domain) + conditions.append("host == ? OR host == ?") + parameters += (domain, "." + domain) + + if conditions: + sql = "{} WHERE ( {} )".format(sql, " ) AND ( ".join(conditions)) set_cookie = cookiejar.set_cookie for name, value, domain, path, secure, expires in db.execute( diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py index b3bec21..950a72f 100644 --- a/gallery_dl/downloader/ytdl.py +++ b/gallery_dl/downloader/ytdl.py @@ -45,7 +45,7 @@ class YoutubeDLDownloader(DownloaderBase): except (ImportError, SyntaxError) as exc: self.log.error("Cannot import module '%s'", getattr(exc, "name", "")) - self.log.debug("", exc_info=True) + self.log.debug("", exc_info=exc) self.download = lambda u, p: False return False self.ytdl_instance = ytdl_instance = ytdl.construct_YoutubeDL( @@ -64,8 +64,8 @@ class YoutubeDLDownloader(DownloaderBase): if not info_dict: try: info_dict = ytdl_instance.extract_info(url[5:], download=False) - except Exception: - pass + except Exception as exc: + self.log.debug("", exc_info=exc) if not info_dict: return False @@ -120,8 +120,8 @@ class YoutubeDLDownloader(DownloaderBase): self.out.start(pathfmt.path) try: ytdl_instance.process_info(info_dict) - except Exception: - self.log.debug("Traceback", exc_info=True) + except Exception as exc: + self.log.debug("", exc_info=exc) return False return True diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index a5e8b27..afa3a69 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -9,9 +9,9 @@ """Extractors for https://8chan.moe/""" from .common import Extractor, Message -from .. import text +from .. import text, util from ..cache import memcache -from datetime import datetime, timedelta +from datetime import timedelta import itertools BASE_PATTERN = r"(?:https?://)?8chan\.(moe|se|cc)" @@ -27,21 +27,23 @@ class _8chanExtractor(Extractor): Extractor.__init__(self, match) def _init(self): - self.cookies.set( - "TOS20240718", "1", domain=self.root.rpartition("/")[2]) + now = util.datetime_utcnow() + domain = self.root.rpartition("/")[2] + self.cookies.set("TOS20240928", "1", domain=domain) + self.cookies.set(now.strftime("TOS%Y%m%d"), "1", domain=domain) @memcache() def cookies_prepare(self): # fetch captcha cookies # (necessary to download without getting interrupted) - now = datetime.utcnow() + now = util.datetime_utcnow() url = self.root + "/captcha.js" params = {"d": now.strftime("%a %b %d %Y %H:%M:%S GMT+0000 (UTC)")} self.request(url, params=params).content # adjust cookies # - remove 'expires' timestamp - # - move 'captchaexpiration' value forward by 1 month) + # - move 'captchaexpiration' value forward by 1 month domain = self.root.rpartition("/")[2] for cookie in self.cookies: if cookie.domain.endswith(domain): @@ -79,7 +81,7 @@ class _8chanThreadExtractor(_8chanExtractor): self.cookies = self.cookies_prepare() except Exception as exc: self.log.debug("Failed to fetch captcha cookies: %s: %s", - exc.__class__.__name__, exc, exc_info=True) + exc.__class__.__name__, exc, exc_info=exc) # download files posts = thread.pop("posts", ()) diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index e103cb1..826771c 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -23,6 +23,7 @@ modules = [ "8muses", "adultempire", "agnph", + "ao3", "architizer", "artstation", "aryion", @@ -35,6 +36,8 @@ modules = [ "catbox", "chevereto", "cien", + "civitai", + "cohost", "comicvine", "cyberdrop", "danbooru", diff --git a/gallery_dl/extractor/ao3.py b/gallery_dl/extractor/ao3.py new file mode 100644 index 0000000..1f570e8 --- /dev/null +++ b/gallery_dl/extractor/ao3.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://archiveofourown.org/""" + +from .common import Extractor, Message +from .. import text, util, exception +from ..cache import cache + +BASE_PATTERN = (r"(?:https?://)?(?:www\.)?" + r"a(?:rchiveofourown|o3)\.(?:org|com|net)") + + +class Ao3Extractor(Extractor): + """Base class for ao3 extractors""" + category = "ao3" + root = "https://archiveofourown.org" + categorytransfer = True + cookies_domain = ".archiveofourown.org" + cookies_names = ("remember_user_token",) + request_interval = (0.5, 1.5) + + def items(self): + self.login() + + base = self.root + "/works/" + data = {"_extractor": Ao3WorkExtractor} + + for work_id in self.works(): + yield Message.Queue, base + work_id, data + + def works(self): + return self._pagination(self.groups[0]) + + def login(self): + if self.cookies_check(self.cookies_names): + return + + username, password = self._get_auth_info() + if username: + return self.cookies_update(self._login_impl(username, password)) + + @cache(maxage=90*86400, keyarg=1) + def _login_impl(self, username, password): + self.log.info("Logging in as %s", username) + + url = self.root + "/users/login" + page = self.request(url).text + + pos = page.find('id="loginform"') + token = text.extract( + page, ' name="authenticity_token" value="', '"', pos)[0] + if not token: + self.log.error("Unable to extract 'authenticity_token'") + + data = { + "authenticity_token": text.unescape(token), + "user[login]" : username, + "user[password]" : password, + "user[remember_me]" : "1", + "commit" : "Log In", + } + + response = self.request(url, method="POST", data=data) + if not response.history: + raise exception.AuthenticationError() + + remember = response.history[0].cookies.get("remember_user_token") + if not remember: + raise exception.AuthenticationError() + + return { + "remember_user_token": remember, + "user_credentials" : "1", + } + + def _pagination(self, path, needle='