From f1051085013c0d702ef974b9b27ea43b3fc73259 Mon Sep 17 00:00:00 2001 From: Unit 193 Date: Thu, 12 Jan 2023 03:41:59 -0500 Subject: New upstream version 1.24.4. --- CHANGELOG.md | 8 +++++++- PKG-INFO | 6 +++--- README.rst | 4 ++-- data/man/gallery-dl.1 | 2 +- data/man/gallery-dl.conf.5 | 16 +++++++++++++++- docs/gallery-dl.conf | 3 ++- gallery_dl.egg-info/PKG-INFO | 6 +++--- gallery_dl/downloader/http.py | 5 +++-- gallery_dl/extractor/kemonoparty.py | 14 +++++++++----- gallery_dl/version.py | 2 +- 10 files changed, 46 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a62a8ab..f9555ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.24.4 - 2023-01-11 +### Additions +- [downloader:http] add `validate` option +### Fixes +- [kemonoparty] fix regression from commit 473bd380 ([#3519](https://github.com/mikf/gallery-dl/issues/3519)) + ## 1.24.3 - 2023-01-10 ### Additions - [danbooru] extract `uploader` metadata ([#3457](https://github.com/mikf/gallery-dl/issues/3457)) @@ -29,7 +35,7 @@ - [twitter] apply tweet type checks before uniqueness check ([#3439](https://github.com/mikf/gallery-dl/issues/3439), [#3455](https://github.com/mikf/gallery-dl/issues/3455)) - [twitter] force `https://` for TwitPic URLs ([#3449](https://github.com/mikf/gallery-dl/issues/3449)) - [ytdl] adapt to yt-dlp changes -- updste and improve documentation ([#3453](https://github.com/mikf/gallery-dl/issues/3453), [#3462](https://github.com/mikf/gallery-dl/issues/3462), [#3496](https://github.com/mikf/gallery-dl/issues/3496)) +- update and improve documentation ([#3453](https://github.com/mikf/gallery-dl/issues/3453), [#3462](https://github.com/mikf/gallery-dl/issues/3462), [#3496](https://github.com/mikf/gallery-dl/issues/3496)) ## 1.24.2 - 2022-12-18 ### Additions diff --git a/PKG-INFO b/PKG-INFO index 508ba02..22c1c63 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery_dl -Version: 1.24.3 +Version: 1.24.4 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -106,9 +106,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/README.rst b/README.rst index 56641a4..5028f9b 100644 --- a/README.rst +++ b/README.rst @@ -69,9 +69,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1 index e88dd4f..00723f3 100644 --- a/data/man/gallery-dl.1 +++ b/data/man/gallery-dl.1 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL" "1" "2023-01-10" "1.24.3" "gallery-dl Manual" +.TH "GALLERY-DL" "1" "2023-01-11" "1.24.4" "gallery-dl Manual" .\" disable hyphenation .nh diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5 index ff0067b..e5742b7 100644 --- a/data/man/gallery-dl.conf.5 +++ b/data/man/gallery-dl.conf.5 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL.CONF" "5" "2023-01-10" "1.24.3" "gallery-dl Manual" +.TH "GALLERY-DL.CONF" "5" "2023-01-11" "1.24.4" "gallery-dl Manual" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -3992,6 +3992,20 @@ Codes \f[I]500\f[] - \f[I]599\f[] (server error responses) will always be retri regardless of this option. +.SS downloader.http.validate +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Check for invalid responses. + +Fail a download when a file does not pass +instead of downloading a potentially broken file. + + .SS downloader.ytdl.format .IP "Type:" 6 \f[I]string\f[] diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 2d2adbb..3012e71 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -390,7 +390,8 @@ { "adjust-extensions": true, "chunk-size": 32768, - "headers": null + "headers": null, + "validate": true }, "ytdl": diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO index 17442cc..9eddd2f 100644 --- a/gallery_dl.egg-info/PKG-INFO +++ b/gallery_dl.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery-dl -Version: 1.24.3 +Version: 1.24.4 Summary: Command-line program to download image galleries and collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Download-URL: https://github.com/mikf/gallery-dl/releases/latest @@ -106,9 +106,9 @@ Standalone Executable Prebuilt executable files with a Python interpreter and required Python packages included are available for -- `Windows `__ +- `Windows `__ (Requires `Microsoft Visual C++ Redistributable Package (x86) `__) -- `Linux `__ +- `Linux `__ Nightly Builds diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 4037420..0bf19c2 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2022 Mike Fährmann +# Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -33,6 +33,7 @@ class HttpDownloader(DownloaderBase): self.chunk_size = self.config("chunk-size", 32768) self.metadata = extractor.config("http-metadata") self.progress = self.config("progress", 3.0) + self.validate = self.config("validate", True) self.headers = self.config("headers") self.minsize = self.config("filesize-min") self.maxsize = self.config("filesize-max") @@ -175,7 +176,7 @@ class HttpDownloader(DownloaderBase): # check for invalid responses validate = kwdict.get("_http_validate") - if validate: + if validate and self.validate: result = validate(response) if isinstance(result, str): url = result diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 541e427..63e3084 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021-2022 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -67,7 +67,6 @@ class KemonopartyExtractor(Extractor): headers["Referer"] = "{}/{}/user/{}/post/{}".format( self.root, post["service"], post["user"], post["id"]) post["_http_headers"] = headers - post["_http_validate"] = _validate post["date"] = text.parse_datetime( post["published"] or post["added"], "%a, %d %b %Y %H:%M:%S %Z") @@ -103,13 +102,17 @@ class KemonopartyExtractor(Extractor): yield Message.Directory, post for post["num"], file in enumerate(files, 1): + post["_http_validate"] = None post["hash"] = file["hash"] post["type"] = file["type"] url = file["path"] text.nameext_from_url(file.get("name", url), post) + ext = text.ext_from_url(url) if not post["extension"]: - post["extension"] = text.ext_from_url(url) + post["extension"] = ext + elif ext == "txt" and post["extension"] != "txt": + post["_http_validate"] = _validate if url[0] == "/": url = self.root + "/data" + url @@ -199,7 +202,7 @@ class KemonopartyExtractor(Extractor): def _validate(response): - return (response.headers["content-length"] != "9" and + return (response.headers["content-length"] != "9" or response.content != b"not found") @@ -250,6 +253,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor): ("https://kemono.party/fanbox/user/6993449/post/506575", { "pattern": r"https://kemono.party/data/21/0f" r"/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", + "content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", "keyword": { "added": "Wed, 06 May 2020 20:28:02 GMT", "content": str, @@ -319,7 +323,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor): ("https://kemono.party/patreon/user/19623797/post/29035449", { "pattern": r"907ba78b4545338d3539683e63ecb51c" r"f51c10adc9dabd86e92bd52339f298b9\.txt", - "content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + "content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", # empty }), ("https://kemono.party/subscribestar/user/alcorart/post/184330"), ("https://www.kemono.party/subscribestar/user/alcorart/post/184330"), diff --git a/gallery_dl/version.py b/gallery_dl/version.py index 5e3b507..aec56c9 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,4 +6,4 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.24.3" +__version__ = "1.24.4" -- cgit v1.2.3