New upstream version 1.24.4.upstream/1.24.4

10 files changed, 46 insertions, 20 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a62a8ab..f9555ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## 1.24.4 - 2023-01-11
+### Additions
+- [downloader:http] add `validate` option
+### Fixes
+- [kemonoparty] fix regression from commit 473bd380 ([#3519](https://github.com/mikf/gallery-dl/issues/3519))
+
 ## 1.24.3 - 2023-01-10
 ### Additions
 - [danbooru] extract `uploader` metadata ([#3457](https://github.com/mikf/gallery-dl/issues/3457))
@@ -29,7 +35,7 @@
 - [twitter] apply tweet type checks before uniqueness check ([#3439](https://github.com/mikf/gallery-dl/issues/3439), [#3455](https://github.com/mikf/gallery-dl/issues/3455))
 - [twitter] force `https://` for TwitPic URLs ([#3449](https://github.com/mikf/gallery-dl/issues/3449))
 - [ytdl] adapt to yt-dlp changes
-- updste and improve documentation ([#3453](https://github.com/mikf/gallery-dl/issues/3453), [#3462](https://github.com/mikf/gallery-dl/issues/3462), [#3496](https://github.com/mikf/gallery-dl/issues/3496))
+- update and improve documentation ([#3453](https://github.com/mikf/gallery-dl/issues/3453), [#3462](https://github.com/mikf/gallery-dl/issues/3462), [#3496](https://github.com/mikf/gallery-dl/issues/3496))
 
 ## 1.24.2 - 2022-12-18
 ### Additions
diff --git a/PKG-INFO b/PKG-INFO
index 508ba02..22c1c63 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gallery_dl
-Version: 1.24.3
+Version: 1.24.4
 Summary: Command-line program to download image galleries and collections from several image hosting sites
 Home-page: https://github.com/mikf/gallery-dl
 Download-URL: https://github.com/mikf/gallery-dl/releases/latest
@@ -106,9 +106,9 @@ Standalone Executable
 Prebuilt executable files with a Python interpreter and
 required Python packages included are available for
 
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.24.3/gallery-dl.exe>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.24.4/gallery-dl.exe>`__
   (Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__)
-- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.24.3/gallery-dl.bin>`__
+- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.24.4/gallery-dl.bin>`__
 
 
 Nightly Builds
diff --git a/README.rst b/README.rst
index 56641a4..5028f9b 100644
--- a/README.rst
+++ b/README.rst
@@ -69,9 +69,9 @@ Standalone Executable
 Prebuilt executable files with a Python interpreter and
 required Python packages included are available for
 
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.24.3/gallery-dl.exe>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.24.4/gallery-dl.exe>`__
   (Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__)
-- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.24.3/gallery-dl.bin>`__
+- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.24.4/gallery-dl.bin>`__
 
 
 Nightly Builds
diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1
index e88dd4f..00723f3 100644
--- a/data/man/gallery-dl.1
+++ b/data/man/gallery-dl.1
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL" "1" "2023-01-10" "1.24.3" "gallery-dl Manual"
+.TH "GALLERY-DL" "1" "2023-01-11" "1.24.4" "gallery-dl Manual"
 .\" disable hyphenation
 .nh
 
diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5
index ff0067b..e5742b7 100644
--- a/data/man/gallery-dl.conf.5
+++ b/data/man/gallery-dl.conf.5
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL.CONF" "5" "2023-01-10" "1.24.3" "gallery-dl Manual"
+.TH "GALLERY-DL.CONF" "5" "2023-01-11" "1.24.4" "gallery-dl Manual"
 .\" disable hyphenation
 .nh
 .\" disable justification (adjust text to left margin only)
@@ -3992,6 +3992,20 @@ Codes \f[I]500\f[] - \f[I]599\f[] (server error responses)  will always be retri
 regardless of this option.
 
 
+.SS downloader.http.validate
+.IP "Type:" 6
+\f[I]bool\f[]
+
+.IP "Default:" 9
+\f[I]true\f[]
+
+.IP "Description:" 4
+Check for invalid responses.
+
+Fail a download when a file does not pass
+instead of downloading a potentially broken file.
+
+
 .SS downloader.ytdl.format
 .IP "Type:" 6
 \f[I]string\f[]
diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf
index 2d2adbb..3012e71 100644
--- a/docs/gallery-dl.conf
+++ b/docs/gallery-dl.conf
@@ -390,7 +390,8 @@
         {
             "adjust-extensions": true,
             "chunk-size": 32768,
-            "headers": null
+            "headers": null,
+            "validate": true
         },
 
         "ytdl":
diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO
index 17442cc..9eddd2f 100644
--- a/gallery_dl.egg-info/PKG-INFO
+++ b/gallery_dl.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gallery-dl
-Version: 1.24.3
+Version: 1.24.4
 Summary: Command-line program to download image galleries and collections from several image hosting sites
 Home-page: https://github.com/mikf/gallery-dl
 Download-URL: https://github.com/mikf/gallery-dl/releases/latest
@@ -106,9 +106,9 @@ Standalone Executable
 Prebuilt executable files with a Python interpreter and
 required Python packages included are available for
 
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.24.3/gallery-dl.exe>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.24.4/gallery-dl.exe>`__
   (Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__)
-- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.24.3/gallery-dl.bin>`__
+- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.24.4/gallery-dl.bin>`__
 
 
 Nightly Builds
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 4037420..0bf19c2 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2014-2022 Mike Fährmann
+# Copyright 2014-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -33,6 +33,7 @@ class HttpDownloader(DownloaderBase):
         self.chunk_size = self.config("chunk-size", 32768)
         self.metadata = extractor.config("http-metadata")
         self.progress = self.config("progress", 3.0)
+        self.validate = self.config("validate", True)
         self.headers = self.config("headers")
         self.minsize = self.config("filesize-min")
         self.maxsize = self.config("filesize-max")
@@ -175,7 +176,7 @@ class HttpDownloader(DownloaderBase):
 
             # check for invalid responses
             validate = kwdict.get("_http_validate")
-            if validate:
+            if validate and self.validate:
                 result = validate(response)
                 if isinstance(result, str):
                     url = result
diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py
index 541e427..63e3084 100644
--- a/gallery_dl/extractor/kemonoparty.py
+++ b/gallery_dl/extractor/kemonoparty.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2021-2022 Mike Fährmann
+# Copyright 2021-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -67,7 +67,6 @@ class KemonopartyExtractor(Extractor):
             headers["Referer"] = "{}/{}/user/{}/post/{}".format(
                 self.root, post["service"], post["user"], post["id"])
             post["_http_headers"] = headers
-            post["_http_validate"] = _validate
             post["date"] = text.parse_datetime(
                 post["published"] or post["added"],
                 "%a, %d %b %Y %H:%M:%S %Z")
@@ -103,13 +102,17 @@ class KemonopartyExtractor(Extractor):
             yield Message.Directory, post
 
             for post["num"], file in enumerate(files, 1):
+                post["_http_validate"] = None
                 post["hash"] = file["hash"]
                 post["type"] = file["type"]
                 url = file["path"]
 
                 text.nameext_from_url(file.get("name", url), post)
+                ext = text.ext_from_url(url)
                 if not post["extension"]:
-                    post["extension"] = text.ext_from_url(url)
+                    post["extension"] = ext
+                elif ext == "txt" and post["extension"] != "txt":
+                    post["_http_validate"] = _validate
 
                 if url[0] == "/":
                     url = self.root + "/data" + url
@@ -199,7 +202,7 @@ class KemonopartyExtractor(Extractor):
 
 
 def _validate(response):
-    return (response.headers["content-length"] != "9" and
+    return (response.headers["content-length"] != "9" or
             response.content != b"not found")
 
 
@@ -250,6 +253,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
         ("https://kemono.party/fanbox/user/6993449/post/506575", {
             "pattern": r"https://kemono.party/data/21/0f"
                        r"/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg",
+            "content": "900949cefc97ab8dc1979cc3664785aac5ba70dd",
             "keyword": {
                 "added": "Wed, 06 May 2020 20:28:02 GMT",
                 "content": str,
@@ -319,7 +323,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
         ("https://kemono.party/patreon/user/19623797/post/29035449", {
             "pattern": r"907ba78b4545338d3539683e63ecb51c"
                        r"f51c10adc9dabd86e92bd52339f298b9\.txt",
-            "content": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+            "content": "da39a3ee5e6b4b0d3255bfef95601890afd80709",  # empty
         }),
         ("https://kemono.party/subscribestar/user/alcorart/post/184330"),
         ("https://www.kemono.party/subscribestar/user/alcorart/post/184330"),
diff --git a/gallery_dl/version.py b/gallery_dl/version.py
index 5e3b507..aec56c9 100644
--- a/gallery_dl/version.py
+++ b/gallery_dl/version.py
@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.24.3"
+__version__ = "1.24.4"