New upstream version 1.24.0.upstream/1.24.0

4 files changed, 130 insertions, 64 deletions

diff --git a/test/test_downloader.py b/test/test_downloader.py
index 9350ce4..0703754 100644
--- a/test/test_downloader.py
+++ b/test/test_downloader.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2021 Mike Fährmann
+# Copyright 2018-2022 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -13,9 +13,9 @@ import unittest
 from unittest.mock import Mock, MagicMock, patch
 
 import re
-import base64
 import logging
 import os.path
+import binascii
 import tempfile
 import threading
 import http.server
@@ -23,6 +23,7 @@ import http.server
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from gallery_dl import downloader, extractor, output, config, path  # noqa E402
+from gallery_dl.downloader.http import MIME_TYPES, SIGNATURE_CHECKS # noqa E402
 
 
 class MockDownloaderModule(Mock):
@@ -130,6 +131,7 @@ class TestDownloaderBase(unittest.TestCase):
         pathfmt = cls.job.pathfmt
         pathfmt.set_directory(kwdict)
         pathfmt.set_filename(kwdict)
+        pathfmt.build_path()
 
         if content:
             mode = "w" + ("b" if isinstance(content, bytes) else "")
@@ -156,6 +158,7 @@ class TestDownloaderBase(unittest.TestCase):
         self.assertEqual(
             pathfmt.extension,
             expected_extension,
+            content[0:16],
         )
         self.assertEqual(
             os.path.splitext(pathfmt.realpath)[1][1:],
@@ -172,48 +175,52 @@ class TestHTTPDownloader(TestDownloaderBase):
 
         port = 8088
         cls.address = "http://127.0.0.1:{}".format(port)
-        cls._jpg = cls.address + "/image.jpg"
-        cls._png = cls.address + "/image.png"
-        cls._gif = cls.address + "/image.gif"
-
         server = http.server.HTTPServer(("", port), HttpRequestHandler)
         threading.Thread(target=server.serve_forever, daemon=True).start()
 
+    def _run_test(self, ext, input, output,
+                  extension, expected_extension=None):
+        TestDownloaderBase._run_test(
+            self, self.address + "/" + ext, input, output,
+            extension, expected_extension)
+
     def tearDown(self):
         self.downloader.minsize = self.downloader.maxsize = None
 
     def test_http_download(self):
-        self._run_test(self._jpg, None, DATA_JPG, "jpg", "jpg")
-        self._run_test(self._png, None, DATA_PNG, "png", "png")
-        self._run_test(self._gif, None, DATA_GIF, "gif", "gif")
+        self._run_test("jpg", None, DATA["jpg"], "jpg", "jpg")
+        self._run_test("png", None, DATA["png"], "png", "png")
+        self._run_test("gif", None, DATA["gif"], "gif", "gif")
 
     def test_http_offset(self):
-        self._run_test(self._jpg, DATA_JPG[:123], DATA_JPG, "jpg", "jpg")
-        self._run_test(self._png, DATA_PNG[:12] , DATA_PNG, "png", "png")
-        self._run_test(self._gif, DATA_GIF[:1]  , DATA_GIF, "gif", "gif")
+        self._run_test("jpg", DATA["jpg"][:123], DATA["jpg"], "jpg", "jpg")
+        self._run_test("png", DATA["png"][:12] , DATA["png"], "png", "png")
+        self._run_test("gif", DATA["gif"][:1]  , DATA["gif"], "gif", "gif")
 
     def test_http_extension(self):
-        self._run_test(self._jpg, None, DATA_JPG, None, "jpg")
-        self._run_test(self._png, None, DATA_PNG, None, "png")
-        self._run_test(self._gif, None, DATA_GIF, None, "gif")
+        self._run_test("jpg", None, DATA["jpg"], None, "jpg")
+        self._run_test("png", None, DATA["png"], None, "png")
+        self._run_test("gif", None, DATA["gif"], None, "gif")
 
     def test_http_adjust_extension(self):
-        self._run_test(self._jpg, None, DATA_JPG, "png", "jpg")
-        self._run_test(self._png, None, DATA_PNG, "gif", "png")
-        self._run_test(self._gif, None, DATA_GIF, "jpg", "gif")
+        self._run_test("jpg", None, DATA["jpg"], "png", "jpg")
+        self._run_test("png", None, DATA["png"], "gif", "png")
+        self._run_test("gif", None, DATA["gif"], "jpg", "gif")
 
     def test_http_filesize_min(self):
+        url = self.address + "/gif"
         pathfmt = self._prepare_destination(None, extension=None)
         self.downloader.minsize = 100
         with self.assertLogs(self.downloader.log, "WARNING"):
-            success = self.downloader.download(self._gif, pathfmt)
+            success = self.downloader.download(url, pathfmt)
         self.assertFalse(success)
 
     def test_http_filesize_max(self):
+        url = self.address + "/jpg"
         pathfmt = self._prepare_destination(None, extension=None)
         self.downloader.maxsize = 100
         with self.assertLogs(self.downloader.log, "WARNING"):
-            success = self.downloader.download(self._jpg, pathfmt)
+            success = self.downloader.download(url, pathfmt)
         self.assertFalse(success)
 
 
@@ -237,24 +244,14 @@ class TestTextDownloader(TestDownloaderBase):
 class HttpRequestHandler(http.server.BaseHTTPRequestHandler):
 
     def do_GET(self):
-        if self.path == "/image.jpg":
-            content_type = "image/jpeg"
-            output = DATA_JPG
-        elif self.path == "/image.png":
-            content_type = "image/png"
-            output = DATA_PNG
-        elif self.path == "/image.gif":
-            content_type = "image/gif"
-            output = DATA_GIF
-        else:
+        try:
+            output = DATA[self.path[1:]]
+        except KeyError:
             self.send_response(404)
             self.wfile.write(self.path.encode())
             return
 
-        headers = {
-            "Content-Type": content_type,
-            "Content-Length": len(output),
-        }
+        headers = {"Content-Length": len(output)}
 
         if "Range" in self.headers:
             status = 206
@@ -275,31 +272,79 @@ class HttpRequestHandler(http.server.BaseHTTPRequestHandler):
         self.wfile.write(output)
 
 
-DATA_JPG = base64.standard_b64decode("""
-/9j/4AAQSkZJRgABAQEASABIAAD/2wBD
-AAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB
-AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB
-AQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEB
-AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB
-AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB
-AQEBAQEBAQEBAQEBAQH/wAARCAABAAED
-AREAAhEBAxEB/8QAFAABAAAAAAAAAAAA
-AAAAAAAACv/EABQQAQAAAAAAAAAAAAAA
-AAAAAAD/xAAUAQEAAAAAAAAAAAAAAAAA
-AAAA/8QAFBEBAAAAAAAAAAAAAAAAAAAA
-AP/aAAwDAQACEQMRAD8AfwD/2Q==""")
-
-
-DATA_PNG = base64.standard_b64decode("""
-iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB
-CAAAAAA6fptVAAAACklEQVQIHWP4DwAB
-AQEANl9ngAAAAABJRU5ErkJggg==""")
-
-
-DATA_GIF = base64.standard_b64decode("""
-R0lGODdhAQABAIAAAP///////ywAAAAA
-AQABAAACAkQBADs=""")
-
-
+SAMPLES = {
+    ("jpg" , binascii.a2b_base64(
+        "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB"
+        "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEB"
+        "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB"
+        "AQEBAQEBAQEBAQEBAQH/wAARCAABAAEDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAA"
+        "AAAAAAAACv/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAUAQEAAAAAAAAAAAAAAAAA"
+        "AAAA/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEQMRAD8AfwD/2Q==")),
+    ("png" , binascii.a2b_base64(
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVQIHWP4DwAB"
+        "AQEANl9ngAAAAABJRU5ErkJggg==")),
+    ("gif" , binascii.a2b_base64(
+        "R0lGODdhAQABAIAAAP///////ywAAAAAAQABAAACAkQBADs=")),
+    ("bmp" , b"BM"),
+    ("webp", b"RIFF????WEBP"),
+    ("avif", b"????ftypavif"),
+    ("avif", b"????ftypavis"),
+    ("svg" , b"<?xml"),
+    ("ico" , b"\x00\x00\x01\x00"),
+    ("cur" , b"\x00\x00\x02\x00"),
+    ("psd" , b"8BPS"),
+    ("mp4" , b"????ftypmp4"),
+    ("mp4" , b"????ftypavc1"),
+    ("mp4" , b"????ftypiso3"),
+    ("mp4" , b"????ftypM4V"),
+    ("webm", b"\x1A\x45\xDF\xA3"),
+    ("ogg" , b"OggS"),
+    ("wav" , b"RIFF????WAVE"),
+    ("mp3" , b"ID3"),
+    ("mp3" , b"\xFF\xFB"),
+    ("mp3" , b"\xFF\xF3"),
+    ("mp3" , b"\xFF\xF2"),
+    ("zip" , b"PK\x03\x04"),
+    ("zip" , b"PK\x05\x06"),
+    ("zip" , b"PK\x07\x08"),
+    ("rar" , b"Rar!\x1A\x07"),
+    ("rar" , b"\x52\x61\x72\x21\x1A\x07"),
+    ("7z"  , b"\x37\x7A\xBC\xAF\x27\x1C"),
+    ("pdf" , b"%PDF-"),
+    ("swf" , b"FWS"),
+    ("swf" , b"CWS"),
+}
+
+
+DATA = {}
+
+for ext, content in SAMPLES:
+    if ext not in DATA:
+        DATA[ext] = content
+
+for idx, (_, content) in enumerate(SAMPLES):
+    DATA["S{:>02}".format(idx)] = content
+
+
+# reverse mime types mapping
+MIME_TYPES = {
+    ext: mtype
+    for mtype, ext in MIME_TYPES.items()
+}
+
+
+def generate_tests():
+    def generate_test(idx, ext, content):
+        def test(self):
+            self._run_test("S{:>02}".format(idx), None, content, "bin", ext)
+        test.__name__ = "test_http_ext_{:>02}_{}".format(idx, ext)
+        return test
+
+    for idx, (ext, content) in enumerate(SAMPLES):
+        test = generate_test(idx, ext, content)
+        setattr(TestHTTPDownloader, test.__name__, test)
+
+
+generate_tests()
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/test_job.py b/test/test_job.py
index fec6997..1bd9ccc 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -87,10 +87,10 @@ num
   1
 subcategory
   test_subcategory
-tags[]
-  - foo
-  - bar
-  - テスト
+tags[N]
+  0 foo
+  1 bar
+  2 テスト
 user[id]
   123
 user[name]
diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py
index ba37ee0..7da2089 100644
--- a/test/test_postprocessor.py
+++ b/test/test_postprocessor.py
@@ -97,6 +97,7 @@ class BasePostprocessorTest(unittest.TestCase):
         self.pathfmt = self.job.pathfmt
         self.pathfmt.set_directory(kwdict)
         self.pathfmt.set_filename(kwdict)
+        self.pathfmt.build_path()
 
         pp = postprocessor.find(self.__class__.__name__[:-4].lower())
         return pp(self.job, options)
@@ -118,6 +119,7 @@ class ClassifyTest(BasePostprocessorTest):
             for ext in exts
         })
         self.pathfmt.set_extension("jpg")
+        self.pathfmt.build_path()
 
         pp.prepare(self.pathfmt)
         path = os.path.join(self.dir.name, "test", "Pictures")
@@ -150,6 +152,7 @@ class ClassifyTest(BasePostprocessorTest):
             "bar": "foo/bar",
         })
         self.pathfmt.set_extension("foo")
+        self.pathfmt.build_path()
 
         pp.prepare(self.pathfmt)
         path = os.path.join(self.dir.name, "test", "foo", "bar")
diff --git a/test/test_text.py b/test/test_text.py
index 0ac7767..2c0be3b 100644
--- a/test/test_text.py
+++ b/test/test_text.py
@@ -203,6 +203,24 @@ class TestText(unittest.TestCase):
             self.assertEqual(f(txt  , value, ">")  , (None, 0))
             self.assertEqual(f(txt  , "<"  , value), (None, 0))
 
+    def test_extr(self, f=text.extr):
+        txt = "<a><b>"
+        self.assertEqual(f(txt, "X", ">"), "")
+        self.assertEqual(f(txt, "<", "X"), "")
+        self.assertEqual(f(txt, "<", ">"), "a")
+        self.assertEqual(f(txt, "><", ">"), "b")
+
+        # 'default' argument
+        self.assertEqual(f(txt, "<", "X", None), None)
+        self.assertEqual(f(txt, "<", "X", default=None), None)
+        self.assertEqual(f(txt, "<", "X", default=()), ())
+
+        # invalid arguments
+        for value in INVALID:
+            self.assertEqual(f(value, "<"  , ">")  , "")
+            self.assertEqual(f(txt  , value, ">")  , "")
+            self.assertEqual(f(txt  , "<"  , value), "")
+
     def test_rextract(self, f=text.rextract):
         txt = "<a><b>"
         self.assertEqual(f(txt, "<", ">"), ("b" , 3))