diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/test_config.py | 140 | ||||
| -rw-r--r-- | test/test_cookies.py | 12 | ||||
| -rw-r--r-- | test/test_downloader.py | 2 | ||||
| -rw-r--r-- | test/test_postprocessor.py | 39 | ||||
| -rw-r--r-- | test/test_results.py | 77 | ||||
| -rw-r--r-- | test/test_util.py | 15 |
6 files changed, 199 insertions, 86 deletions
diff --git a/test/test_config.py b/test/test_config.py index 8cdb3da..a9d3f54 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2015-2017 Mike Fährmann +# Copyright 2015-2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -16,65 +16,125 @@ import tempfile class TestConfig(unittest.TestCase): def setUp(self): - fd, self._configfile = tempfile.mkstemp() - with os.fdopen(fd, "w") as file: - file.write('{"a": "1", "b": {"a": 2, "c": "text"}}') - config.load((self._configfile,)) + config.set(() , "a", 1) + config.set(("b",) , "a", 2) + config.set(("b", "b"), "a", 3) + config.set(("b",) , "c", "text") + config.set(("b", "b"), "c", [8, 9]) def tearDown(self): config.clear() - os.remove(self._configfile) def test_get(self): - self.assertEqual(config.get(["a"]), "1") - self.assertEqual(config.get(["b", "c"]), "text") - self.assertEqual(config.get(["d"]), None) - self.assertEqual(config.get(["e", "f", "g"], 123), 123) + self.assertEqual(config.get(() , "a") , 1) + self.assertEqual(config.get(("b",) , "a") , 2) + self.assertEqual(config.get(("b", "b"), "a") , 3) + + self.assertEqual(config.get(() , "c") , None) + self.assertEqual(config.get(("b",) , "c") , "text") + self.assertEqual(config.get(("b", "b"), "c") , [8, 9]) + + self.assertEqual(config.get(("a",) , "g") , None) + self.assertEqual(config.get(("a", "a"), "g") , None) + self.assertEqual(config.get(("e", "f"), "g") , None) + self.assertEqual(config.get(("e", "f"), "g", 4), 4) def test_interpolate(self): - self.assertEqual(config.interpolate(["a"]), "1") - self.assertEqual(config.interpolate(["b", "a"]), "1") - self.assertEqual(config.interpolate(["b", "c"], "2"), "text") - self.assertEqual(config.interpolate(["b", "d"], "2"), "2") - config.set(["d"], 123) - self.assertEqual(config.interpolate(["b", "d"], "2"), 123) - self.assertEqual(config.interpolate(["d", "d"], "2"), 123) + self.assertEqual(config.interpolate(() , "a"), 1) + self.assertEqual(config.interpolate(("b",) , "a"), 1) + self.assertEqual(config.interpolate(("b", "b"), "a"), 1) + + self.assertEqual(config.interpolate(() , "c"), None) + self.assertEqual(config.interpolate(("b",) , "c"), "text") + self.assertEqual(config.interpolate(("b", "b"), "c"), [8, 9]) + + self.assertEqual(config.interpolate(("a",) , "g") , None) + self.assertEqual(config.interpolate(("a", "a"), "g") , None) + self.assertEqual(config.interpolate(("e", "f"), "g") , None) + self.assertEqual(config.interpolate(("e", "f"), "g", 4), 4) + + self.assertEqual(config.interpolate(("b",), "d", 1) , 1) + self.assertEqual(config.interpolate(("d",), "d", 1) , 1) + config.set(() , "d", 2) + self.assertEqual(config.interpolate(("b",), "d", 1) , 2) + self.assertEqual(config.interpolate(("d",), "d", 1) , 2) + config.set(("b",), "d", 3) + self.assertEqual(config.interpolate(("b",), "d", 1) , 2) + self.assertEqual(config.interpolate(("d",), "d", 1) , 2) def test_set(self): - config.set(["b", "c"], [1, 2, 3]) - config.set(["e", "f", "g"], value=234) - self.assertEqual(config.get(["b", "c"]), [1, 2, 3]) - self.assertEqual(config.get(["e", "f", "g"]), 234) + config.set(() , "c", [1, 2, 3]) + config.set(("b",) , "c", [1, 2, 3]) + config.set(("e", "f"), "g", value=234) + self.assertEqual(config.get(() , "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",) , "c"), [1, 2, 3]) + self.assertEqual(config.get(("e", "f"), "g"), 234) def test_setdefault(self): - config.setdefault(["b", "c"], [1, 2, 3]) - config.setdefault(["e", "f", "g"], value=234) - self.assertEqual(config.get(["b", "c"]), "text") - self.assertEqual(config.get(["e", "f", "g"]), 234) + config.setdefault(() , "c", [1, 2, 3]) + config.setdefault(("b",) , "c", [1, 2, 3]) + config.setdefault(("e", "f"), "g", value=234) + self.assertEqual(config.get(() , "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get(("e", "f"), "g"), 234) def test_unset(self): - config.unset(["a"]) - config.unset(["b", "c"]) - config.unset(["c", "d"]) - self.assertEqual(config.get(["a"]), None) - self.assertEqual(config.get(["b", "a"]), 2) - self.assertEqual(config.get(["b", "c"]), None) + config.unset(() , "a") + config.unset(("b",), "c") + config.unset(("a",), "d") + config.unset(("b",), "d") + config.unset(("c",), "d") + self.assertEqual(config.get(() , "a"), None) + self.assertEqual(config.get(("b",), "a"), 2) + self.assertEqual(config.get(("b",), "c"), None) + self.assertEqual(config.get(("a",), "d"), None) + self.assertEqual(config.get(("b",), "d"), None) + self.assertEqual(config.get(("c",), "d"), None) def test_apply(self): options = ( - (["b", "c"], [1, 2, 3]), - (["e", "f", "g"], 234), + (("b",) , "c", [1, 2, 3]), + (("e", "f"), "g", 234), ) - self.assertEqual(config.get(["b", "c"]), "text") - self.assertEqual(config.get(["e", "f", "g"]), None) + self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get(("e", "f"), "g"), None) with config.apply(options): - self.assertEqual(config.get(["b", "c"]), [1, 2, 3]) - self.assertEqual(config.get(["e", "f", "g"]), 234) - - self.assertEqual(config.get(["b", "c"]), "text") - self.assertEqual(config.get(["e", "f", "g"]), None) + self.assertEqual(config.get(("b",) , "c"), [1, 2, 3]) + self.assertEqual(config.get(("e", "f"), "g"), 234) + + self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get(("e", "f"), "g"), None) + + def test_load(self): + with tempfile.TemporaryDirectory() as base: + path1 = os.path.join(base, "cfg1") + with open(path1, "w") as file: + file.write('{"a": 1, "b": {"a": 2, "c": "text"}}') + + path2 = os.path.join(base, "cfg2") + with open(path2, "w") as file: + file.write('{"a": 7, "b": {"a": 8, "e": "foo"}}') + + config.clear() + config.load((path1,)) + self.assertEqual(config.get(() , "a"), 1) + self.assertEqual(config.get(("b",), "a"), 2) + self.assertEqual(config.get(("b",), "c"), "text") + + config.load((path2,)) + self.assertEqual(config.get(() , "a"), 7) + self.assertEqual(config.get(("b",), "a"), 8) + self.assertEqual(config.get(("b",), "c"), "text") + self.assertEqual(config.get(("b",), "e"), "foo") + + config.clear() + config.load((path1, path2)) + self.assertEqual(config.get(() , "a"), 7) + self.assertEqual(config.get(("b",), "a"), 8) + self.assertEqual(config.get(("b",), "c"), "text") + self.assertEqual(config.get(("b",), "e"), "foo") if __name__ == '__main__': diff --git a/test/test_cookies.py b/test/test_cookies.py index a786df6..4f294bf 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2017 Mike Fährmann +# Copyright 2017-2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -18,8 +18,6 @@ from os.path import join import gallery_dl.config as config import gallery_dl.extractor as extractor -CKEY = ("cookies",) - class TestCookiejar(unittest.TestCase): @@ -45,7 +43,7 @@ class TestCookiejar(unittest.TestCase): config.clear() def test_cookiefile(self): - config.set(CKEY, self.cookiefile) + config.set((), "cookies", self.cookiefile) cookies = extractor.find("test:").session.cookies self.assertEqual(len(cookies), 1) @@ -63,7 +61,7 @@ class TestCookiejar(unittest.TestCase): self._test_warning(join(self.path.name, "nothing"), FileNotFoundError) def _test_warning(self, filename, exc): - config.set(CKEY, filename) + config.set((), "cookies", filename) log = logging.getLogger("test") with mock.patch.object(log, "warning") as mock_warning: cookies = extractor.find("test:").session.cookies @@ -77,7 +75,7 @@ class TestCookiedict(unittest.TestCase): def setUp(self): self.cdict = {"NAME1": "VALUE1", "NAME2": "VALUE2"} - config.set(CKEY, self.cdict) + config.set((), "cookies", self.cdict) def tearDown(self): config.clear() @@ -112,7 +110,7 @@ class TestCookieLogin(unittest.TestCase): } for category, cookienames in extr_cookies.items(): cookies = {name: "value" for name in cookienames} - config.set(CKEY, cookies) + config.set((), "cookies", cookies) extr = _get_extractor(category) with mock.patch.object(extr, "_login_impl") as mock_login: extr.login() diff --git a/test/test_downloader.py b/test/test_downloader.py index 0f58d4e..a7c4ce6 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -100,7 +100,7 @@ class TestDownloaderBase(unittest.TestCase): cls.extractor = extractor.find("test:") cls.dir = tempfile.TemporaryDirectory() cls.fnum = 0 - config.set(("base-directory",), cls.dir.name) + config.set((), "base-directory", cls.dir.name) @classmethod def tearDownClass(cls): diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 0ab89db..17f82c9 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -58,7 +58,7 @@ class BasePostprocessorTest(unittest.TestCase): def setUpClass(cls): cls.extractor = extractor.find("test:") cls.dir = tempfile.TemporaryDirectory() - config.set(("base-directory",), cls.dir.name) + config.set((), "base-directory", cls.dir.name) @classmethod def tearDownClass(cls): @@ -151,8 +151,12 @@ class MetadataTest(BasePostprocessorTest): "ascii" : True, "indent" : 2, "extension": "JSON", + }, { + "public" : "hello", + "_private" : "world", }) + self.assertEqual(pp.path , pp._path_append) self.assertEqual(pp.write , pp._write_json) self.assertEqual(pp.ascii , True) self.assertEqual(pp.indent , 2) @@ -167,7 +171,8 @@ class MetadataTest(BasePostprocessorTest): self.assertEqual(self._output(m), """{ "category": "test", "extension": "ext", - "filename": "file" + "filename": "file", + "public": "hello" } """) @@ -224,13 +229,41 @@ class MetadataTest(BasePostprocessorTest): ) self.assertEqual(pp.write, pp._write_custom) self.assertEqual(pp.extension, "txt") - self.assertTrue(pp.formatter) + self.assertTrue(pp.contentfmt) with patch("builtins.open", mock_open()) as m: pp.prepare(self.pathfmt) pp.run(self.pathfmt) self.assertEqual(self._output(m), "bar\nNone\n") + def test_metadata_extfmt(self): + pp = self._create({ + "extension" : "ignored", + "extension-format": "json", + }) + + self.assertEqual(pp.path, pp._path_format) + + with patch("builtins.open", mock_open()) as m: + pp.prepare(self.pathfmt) + pp.run(self.pathfmt) + + path = self.pathfmt.realdirectory + "file.json" + m.assert_called_once_with(path, "w", encoding="utf-8") + + def test_metadata_extfmt_2(self): + pp = self._create({ + "extension-format": "{extension!u}-data:{category:Res/ES/}", + }) + + self.pathfmt.prefix = "2." + with patch("builtins.open", mock_open()) as m: + pp.prepare(self.pathfmt) + pp.run(self.pathfmt) + + path = self.pathfmt.realdirectory + "file.2.EXT-data:tESt" + m.assert_called_once_with(path, "w", encoding="utf-8") + @staticmethod def _output(mock): return "".join( diff --git a/test/test_results.py b/test/test_results.py index 6d628c3..869ff83 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -26,10 +26,9 @@ TRAVIS_SKIP = { # temporary issues, etc. BROKEN = { - "hentaifox", - "livedoor", + "erolord", "mangapark", - "yaplog", + "photobucket", } @@ -56,10 +55,11 @@ class TestExtractorResults(unittest.TestCase): if result: if "options" in result: for key, value in result["options"]: - config.set(key.split("."), value) + key = key.split(".") + config.set(key[:-1], key[-1], value) if "range" in result: - config.set(("image-range",), result["range"]) - config.set(("chapter-range",), result["range"]) + config.set((), "image-range" , result["range"]) + config.set((), "chapter-range", result["range"]) content = "content" in result else: content = False @@ -79,14 +79,18 @@ class TestExtractorResults(unittest.TestCase): pass except exception.HttpError as exc: exc = str(exc) - if re.match(r"5\d\d: ", exc) or \ + if re.match(r"'5\d\d ", exc) or \ re.search(r"\bRead timed out\b", exc): self._skipped.append((url, exc)) self.skipTest(exc) raise - # test archive-id uniqueness - self.assertEqual(len(set(tjob.archive_list)), len(tjob.archive_list)) + if result.get("archive", True): + self.assertEqual( + len(set(tjob.archive_list)), + len(tjob.archive_list), + "archive-id uniqueness", + ) if tjob.queue: # test '_extractor' entries @@ -185,7 +189,7 @@ class ResultJob(job.DownloadJob): self._update_url(url) self._update_kwdict(kwdict) self._update_archive(kwdict) - self._update_content(url) + self._update_content(url, kwdict) self.format_filename(kwdict) def handle_directory(self, kwdict): @@ -204,7 +208,7 @@ class ResultJob(job.DownloadJob): def _update_kwdict(self, kwdict, to_list=True): if to_list: self.kwdict_list.append(kwdict.copy()) - kwdict = self._filter(kwdict) + kwdict = util.filter_dict(kwdict) self.kwdict_hash.update( json.dumps(kwdict, sort_keys=True, default=str).encode()) @@ -213,9 +217,10 @@ class ResultJob(job.DownloadJob): self.archive_list.append(archive_id) self.archive_hash.update(archive_id.encode()) - def _update_content(self, url): + def _update_content(self, url, kwdict): if self.content: scheme = url.partition(":")[0] + self.fileobj.kwdict = kwdict self.get_downloader(scheme).download(url, self.fileobj) @@ -281,34 +286,36 @@ def setup_test_config(): email = "gallerydl@openaliasbox.org" config.clear() - config.set(("cache", "file"), ":memory:") - config.set(("downloader", "part"), False) - config.set(("downloader", "adjust-extensions"), False) - config.set(("extractor", "timeout"), 60) - config.set(("extractor", "username"), name) - config.set(("extractor", "password"), name) - config.set(("extractor", "nijie" , "username"), email) - config.set(("extractor", "seiga" , "username"), email) - - config.set(("extractor", "danbooru" , "username"), None) - config.set(("extractor", "instagram", "username"), None) - config.set(("extractor", "imgur" , "username"), None) - config.set(("extractor", "twitter" , "username"), None) - - config.set(("extractor", "mangoxo" , "username"), "LiQiang3") - config.set(("extractor", "mangoxo" , "password"), "5zbQF10_5u25259Ma") - - config.set(("extractor", "deviantart", "client-id"), "7777") - config.set(("extractor", "deviantart", "client-secret"), + config.set(("cache",), "file", None) + config.set(("downloader",), "part", False) + config.set(("downloader",), "adjust-extensions", False) + config.set(("extractor" ,), "timeout" , 60) + config.set(("extractor" ,), "username", name) + config.set(("extractor" ,), "password", name) + + config.set(("extractor", "nijie") , "username", email) + config.set(("extractor", "seiga") , "username", email) + config.set(("extractor", "danbooru") , "username", None) + config.set(("extractor", "instagram") , "username", None) + config.set(("extractor", "twitter") , "username", None) + + config.set(("extractor", "newgrounds"), "username", "d1618111") + config.set(("extractor", "newgrounds"), "password", "d1618111") + + config.set(("extractor", "mangoxo") , "username", "LiQiang3") + config.set(("extractor", "mangoxo") , "password", "5zbQF10_5u25259Ma") + + config.set(("extractor", "deviantart"), "client-id", "7777") + config.set(("extractor", "deviantart"), "client-secret", "ff14994c744d9208e5caeec7aab4a026") - config.set(("extractor", "tumblr", "api-key"), + config.set(("extractor", "tumblr"), "api-key", "0cXoHfIqVzMQcc3HESZSNsVlulGxEXGDTTZCDrRrjaa0jmuTc6") - config.set(("extractor", "tumblr", "api-secret"), + config.set(("extractor", "tumblr"), "api-secret", "6wxAK2HwrXdedn7VIoZWxGqVhZ8JdYKDLjiQjL46MLqGuEtyVj") - config.set(("extractor", "tumblr", "access-token"), + config.set(("extractor", "tumblr"), "access-token", "N613fPV6tOZQnyn0ERTuoEZn0mEqG8m2K8M3ClSJdEHZJuqFdG") - config.set(("extractor", "tumblr", "access-token-secret"), + config.set(("extractor", "tumblr"), "access-token-secret", "sgOA7ZTT4FBXdOGGVV331sSp0jHYp4yMDRslbhaQf7CaS71i4O") diff --git a/test/test_util.py b/test/test_util.py index 9b252a3..5a103cf 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -358,6 +358,21 @@ class TestOther(unittest.TestCase): self.assertEqual( d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}}) + def test_filter_dict(self): + d = {} + r = util.filter_dict(d) + self.assertEqual(r, d) + self.assertIsNot(r, d) + + d = {"foo": 123, "bar": [], "baz": None} + r = util.filter_dict(d) + self.assertEqual(r, d) + self.assertIsNot(r, d) + + d = {"foo": 123, "_bar": [], "__baz__": None} + r = util.filter_dict(d) + self.assertEqual(r, {"foo": 123}) + def test_number_to_string(self, f=util.number_to_string): self.assertEqual(f(1) , "1") self.assertEqual(f(1.0) , "1.0") |