New upstream version 1.25.0.upstream/1.25.0

7 files changed, 222 insertions, 42 deletions

diff --git a/test/test_config.py b/test/test_config.py
index 7cbb12b..859faf5 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2020 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -11,12 +11,11 @@ import os
 import sys
 import unittest
 
-import json
 import tempfile
 
 ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, ROOTDIR)
-from gallery_dl import config  # noqa E402
+from gallery_dl import config, util  # noqa E402
 
 
 class TestConfig(unittest.TestCase):
@@ -209,8 +208,8 @@ class TestConfigFiles(unittest.TestCase):
     def _load(name):
         path = os.path.join(ROOTDIR, "docs", name)
         try:
-            with open(path) as fp:
-                return json.load(fp)
+            with open(path) as file:
+                return util.json_loads(file.read())
         except FileNotFoundError:
             raise unittest.SkipTest(path + " not available")
 
diff --git a/test/test_extractor.py b/test/test_extractor.py
index 144c6f9..6516fa8 100644
--- a/test/test_extractor.py
+++ b/test/test_extractor.py
@@ -46,7 +46,7 @@ class TestExtractorModule(unittest.TestCase):
 
     def setUp(self):
         extractor._cache.clear()
-        extractor._module_iter = iter(extractor.modules)
+        extractor._module_iter = extractor._modules_internal()
         extractor._list_classes = _list_classes
 
     def test_find(self):
diff --git a/test/test_oauth.py b/test/test_oauth.py
index 7455928..0082419 100644
--- a/test/test_oauth.py
+++ b/test/test_oauth.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2020 Mike Fährmann
+# Copyright 2018-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -10,6 +10,7 @@
 import os
 import sys
 import unittest
+from unittest.mock import patch
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from gallery_dl import oauth, text  # noqa E402
@@ -66,6 +67,53 @@ class TestOAuthSession(unittest.TestCase):
             self.assertTrue(len(quoted) >= 3)
             self.assertEqual(quoted_hex.upper(), quoted_hex)
 
+    def test_generate_signature(self):
+        client = oauth.OAuth1Client(
+            CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
+
+        request = MockRequest()
+        params = []
+        self.assertEqual(
+            client.generate_signature(request, params),
+            "Wt2xo49dM5pkL4gsnCakNdHaVUo%3D")
+
+        request = MockRequest("https://example.org/")
+        params = [("hello", "world"), ("foo", "bar")]
+        self.assertEqual(
+            client.generate_signature(request, params),
+            "ay2269%2F8uKpZqKJR1doTtpv%2Bzn0%3D")
+
+        request = MockRequest("https://example.org/index.html"
+                              "?hello=world&foo=bar", method="POST")
+        params = [("oauth_signature_method", "HMAC-SHA1")]
+        self.assertEqual(
+            client.generate_signature(request, params),
+            "yVZWb1ts4smdMmXxMlhaXrkoOng%3D")
+
+    def test_dunder_call(self):
+        client = oauth.OAuth1Client(
+            CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
+        request = MockRequest("https://example.org/")
+
+        with patch("time.time") as tmock, \
+             patch("gallery_dl.oauth.nonce") as nmock:
+            tmock.return_value = 123456789.123
+            nmock.return_value = "abcdefghijklmno"
+
+            client(request)
+
+        self.assertEqual(
+            request.headers["Authorization"],
+            """OAuth \
+oauth_consumer_key="key",\
+oauth_nonce="abcdefghijklmno",\
+oauth_signature_method="HMAC-SHA1",\
+oauth_timestamp="123456789",\
+oauth_version="1.0",\
+oauth_token="accesskey",\
+oauth_signature="DjtTk5j5P3BDZFnstZ%2FtEYcwD6c%3D"\
+""")
+
     def test_request_token(self):
         response = self._oauth_request(
             "/request_token.php", {})
@@ -110,5 +158,13 @@ class TestOAuthSession(unittest.TestCase):
             raise unittest.SkipTest()
 
 
+class MockRequest():
+
+    def __init__(self, url="", method="GET"):
+        self.url = url
+        self.method = method
+        self.headers = {}
+
+
 if __name__ == "__main__":
     unittest.main(warnings="ignore")
diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py
index 7da2089..650bf59 100644
--- a/test/test_postprocessor.py
+++ b/test/test_postprocessor.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2019-2022 Mike Fährmann
+# Copyright 2019-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -171,39 +171,71 @@ class MetadataTest(BasePostprocessorTest):
 
         # default arguments
         self.assertEqual(pp.write    , pp._write_json)
-        self.assertEqual(pp.ascii    , False)
-        self.assertEqual(pp.indent   , 4)
         self.assertEqual(pp.extension, "json")
+        self.assertTrue(callable(pp._json_encode))
 
     def test_metadata_json(self):
         pp = self._create({
-            "mode"     : "json",
-            "ascii"    : True,
-            "indent"   : 2,
-            "extension": "JSON",
+            "mode"      : "json",
+            "extension" : "JSON",
         }, {
-            "public"   : "hello",
-            "_private" : "world",
+            "public"    : "hello ワールド",
+            "_private"  : "foo バー",
         })
 
         self.assertEqual(pp.write    , pp._write_json)
-        self.assertEqual(pp.ascii    , True)
-        self.assertEqual(pp.indent   , 2)
         self.assertEqual(pp.extension, "JSON")
+        self.assertTrue(callable(pp._json_encode))
 
         with patch("builtins.open", mock_open()) as m:
             self._trigger()
 
         path = self.pathfmt.realpath + ".JSON"
         m.assert_called_once_with(path, "w", encoding="utf-8")
-        self.assertEqual(self._output(m), """{
-  "category": "test",
-  "extension": "ext",
-  "filename": "file",
-  "public": "hello"
+
+        if sys.hexversion >= 0x3060000:
+            # python 3.4 & 3.5 have random order without 'sort: True'
+            self.assertEqual(self._output(m), """{
+    "category": "test",
+    "filename": "file",
+    "extension": "ext",
+    "public": "hello ワールド"
 }
 """)
 
+    def test_metadata_json_options(self):
+        pp = self._create({
+            "mode"      : "json",
+            "ascii"     : True,
+            "sort"      : True,
+            "separators": [",", " : "],
+            "private"   : True,
+            "indent"    : None,
+            "open"      : "a",
+            "encoding"  : "UTF-8",
+            "extension" : "JSON",
+        }, {
+            "public"    : "hello ワールド",
+            "_private"  : "foo バー",
+        })
+
+        self.assertEqual(pp.write    , pp._write_json)
+        self.assertEqual(pp.extension, "JSON")
+        self.assertTrue(callable(pp._json_encode))
+
+        with patch("builtins.open", mock_open()) as m:
+            self._trigger()
+
+        path = self.pathfmt.realpath + ".JSON"
+        m.assert_called_once_with(path, "a", encoding="UTF-8")
+        self.assertEqual(self._output(m), """{\
+"_private" : "foo \\u30d0\\u30fc",\
+"category" : "test",\
+"extension" : "ext",\
+"filename" : "file",\
+"public" : "hello \\u30ef\\u30fc\\u30eb\\u30c9"}
+""")
+
     def test_metadata_tags(self):
         pp = self._create(
             {"mode": "tags"},
@@ -255,6 +287,18 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
         self.assertEqual(self._output(m), "foobar1\nfoobar2\nfoobarbaz\n")
 
+    def test_metadata_tags_list_of_dict(self):
+        self._create(
+            {"mode": "tags"},
+            {"tags": [
+                {"g": "foobar1", "m": "foobar2"},
+                {"g": None, "m": "foobarbaz"}
+            ]},
+        )
+        with patch("builtins.open", mock_open()) as m:
+            self._trigger()
+        self.assertEqual(self._output(m), "foobar1\nfoobar2\nfoobarbaz\n")
+
     def test_metadata_custom(self):
         def test(pp_info):
             pp = self._create(pp_info, {"foo": "bar"})
@@ -334,7 +378,7 @@ class MetadataTest(BasePostprocessorTest):
         m.assert_called_once_with(path, "w", encoding="utf-8")
 
     def test_metadata_stdout(self):
-        self._create({"filename": "-", "indent": None})
+        self._create({"filename": "-", "indent": None, "sort": True})
 
         with patch("sys.stdout", Mock()) as m:
             self._trigger()
diff --git a/test/test_results.py b/test/test_results.py
index a42de09..d28496b 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -322,9 +322,9 @@ def setup_test_config():
     config.set(("extractor", "mangoxo")   , "username", "LiQiang3")
     config.set(("extractor", "mangoxo")   , "password", "5zbQF10_5u25259Ma")
 
-    for category in ("danbooru", "instagram", "twitter", "subscribestar",
-                     "e621", "atfbooru", "inkbunny", "tapas", "pillowfort",
-                     "mangadex", "aibooru"):
+    for category in ("danbooru", "atfbooru", "aibooru", "e621", "e926",
+                     "instagram", "twitter", "subscribestar", "deviantart",
+                     "inkbunny", "tapas", "pillowfort", "mangadex"):
         config.set(("extractor", category), "username", None)
 
     config.set(("extractor", "mastodon.social"), "access-token",
diff --git a/test/test_util.py b/test/test_util.py
index 67fdf60..0813a0b 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2022 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -15,6 +15,7 @@ import io
 import random
 import string
 import datetime
+import tempfile
 import itertools
 import http.cookiejar
 
@@ -394,6 +395,46 @@ class TestOther(unittest.TestCase):
     def test_noop(self):
         self.assertEqual(util.noop(), None)
 
+    def test_md5(self):
+        self.assertEqual(util.md5(b""),
+                         "d41d8cd98f00b204e9800998ecf8427e")
+        self.assertEqual(util.md5(b"hello"),
+                         "5d41402abc4b2a76b9719d911017c592")
+
+        self.assertEqual(util.md5(""),
+                         "d41d8cd98f00b204e9800998ecf8427e")
+        self.assertEqual(util.md5("hello"),
+                         "5d41402abc4b2a76b9719d911017c592")
+        self.assertEqual(util.md5("ワルド"),
+                         "051f29cd6c942cf110a0ccc5729871d2")
+
+        self.assertEqual(util.md5(0),
+                         "d41d8cd98f00b204e9800998ecf8427e")
+        self.assertEqual(util.md5(()),
+                         "d41d8cd98f00b204e9800998ecf8427e")
+        self.assertEqual(util.md5(None),
+                         "d41d8cd98f00b204e9800998ecf8427e")
+
+    def test_sha1(self):
+        self.assertEqual(util.sha1(b""),
+                         "da39a3ee5e6b4b0d3255bfef95601890afd80709")
+        self.assertEqual(util.sha1(b"hello"),
+                         "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d")
+
+        self.assertEqual(util.sha1(""),
+                         "da39a3ee5e6b4b0d3255bfef95601890afd80709")
+        self.assertEqual(util.sha1("hello"),
+                         "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d")
+        self.assertEqual(util.sha1("ワルド"),
+                         "0cbe319081aa0e9298448ec2bb16df8c494aa04e")
+
+        self.assertEqual(util.sha1(0),
+                         "da39a3ee5e6b4b0d3255bfef95601890afd80709")
+        self.assertEqual(util.sha1(()),
+                         "da39a3ee5e6b4b0d3255bfef95601890afd80709")
+        self.assertEqual(util.sha1(None),
+                         "da39a3ee5e6b4b0d3255bfef95601890afd80709")
+
     def test_compile_expression(self):
         expr = util.compile_expression("1 + 2 * 3")
         self.assertEqual(expr(), 7)
@@ -418,6 +459,56 @@ class TestOther(unittest.TestCase):
         with self.assertRaises(exception.StopExtraction):
             expr()
 
+    def test_import_file(self):
+        module = util.import_file("datetime")
+        self.assertIs(module, datetime)
+
+        with tempfile.TemporaryDirectory() as path:
+            file = path + "/module_test.py"
+            with open(file, "w") as fp:
+                fp.write("""
+import datetime
+key = "foobar"
+value = 123
+""")
+            module = util.import_file(file)
+
+        self.assertEqual(module.__name__, "module_test")
+        self.assertEqual(module.key, "foobar")
+        self.assertEqual(module.value, 123)
+        self.assertIs(module.datetime, datetime)
+
+    def test_custom_globals(self):
+        value = {"v": "foobar"}
+        result = "8843d7f92416211de9ebb963ff4ce28125932878"
+
+        expr = util.compile_expression("hash_sha1(v)")
+        self.assertEqual(expr(value), result)
+
+        expr = util.compile_expression("hs(v)", globals={"hs": util.sha1})
+        self.assertEqual(expr(value), result)
+
+        with tempfile.TemporaryDirectory() as path:
+            file = path + "/module_sha1.py"
+            with open(file, "w") as fp:
+                fp.write("""
+import hashlib
+def hash(value):
+    return hashlib.sha1(value.encode()).hexdigest()
+""")
+            module = util.import_file(file)
+
+        expr = util.compile_expression("hash(v)", globals=module.__dict__)
+        self.assertEqual(expr(value), result)
+
+        GLOBALS_ORIG = util.GLOBALS
+        try:
+            util.GLOBALS = module.__dict__
+            expr = util.compile_expression("hash(v)")
+        finally:
+            util.GLOBALS = GLOBALS_ORIG
+        self.assertEqual(expr(value), result)
+
     def test_build_duration_func(self, f=util.build_duration_func):
 
         def test_single(df, v):
diff --git a/test/test_ytdl.py b/test/test_ytdl.py
index a273604..7b82a0f 100644
--- a/test/test_ytdl.py
+++ b/test/test_ytdl.py
@@ -166,7 +166,7 @@ class Test_CommandlineArguments(unittest.TestCase):
             subs["already_have_subtitle"] = False
 
         opts = self._(["--embed-subs", "--embed-thumbnail"])
-        self.assertEqual(opts["postprocessors"], [subs, thumb])
+        self.assertEqual(opts["postprocessors"][:2], [subs, thumb])
 
         thumb["already_have_thumbnail"] = True
         if self.module_name == "yt_dlp":
@@ -179,7 +179,7 @@ class Test_CommandlineArguments(unittest.TestCase):
             "--write-sub",
             "--write-all-thumbnails",
         ])
-        self.assertEqual(opts["postprocessors"], [subs, thumb])
+        self.assertEqual(opts["postprocessors"][:2], [subs, thumb])
 
     def test_metadata(self):
         opts = self._("--add-metadata")
@@ -262,21 +262,11 @@ class Test_CommandlineArguments_YtDlp(Test_CommandlineArguments):
 
     def test_metadata_from_title(self):
         opts = self._(["--metadata-from-title", "%(artist)s - %(title)s"])
-
-        try:
-            legacy = (self.module.version.__version__ < "2023.01.01")
-        except AttributeError:
-            legacy = True
-
-        actions = [self.module.MetadataFromFieldPP.to_action(
-                   "title:%(artist)s - %(title)s")]
-        if not legacy:
-            actions = {"pre_process": actions}
-
         self.assertEqual(opts["postprocessors"][0], {
             "key"    : "MetadataParser",
             "when"   : "pre_process",
-            "actions": actions,
+            "actions": [self.module.MetadataFromFieldPP.to_action(
+                "title:%(artist)s - %(title)s")],
         })