aboutsummaryrefslogtreecommitdiffstats
path: root/test
diff options
context:
space:
mode:
Diffstat (limited to 'test')
-rw-r--r--test/test_cache.py202
-rw-r--r--test/test_cookies.py7
-rw-r--r--test/test_results.py16
-rw-r--r--test/test_util.py131
4 files changed, 347 insertions, 9 deletions
diff --git a/test/test_cache.py b/test/test_cache.py
new file mode 100644
index 0000000..31ece7e
--- /dev/null
+++ b/test/test_cache.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2020 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+import unittest
+import tempfile
+import time
+
+from gallery_dl import config, util
+dbpath = tempfile.mkstemp()[1]
+config.set(("cache",), "file", dbpath)
+from gallery_dl import cache # noqa
+
+
+def tearDownModule():
+ util.remove_file(dbpath)
+
+
+class TestCache(unittest.TestCase):
+
+ def test_decorator(self):
+
+ @cache.memcache()
+ def mc1():
+ pass
+
+ @cache.memcache(maxage=10)
+ def mc2():
+ pass
+
+ @cache.cache()
+ def dbc():
+ pass
+
+ self.assertIsInstance(mc1, cache.CacheDecorator)
+ self.assertIsInstance(mc2, cache.MemoryCacheDecorator)
+ self.assertIsInstance(dbc, cache.DatabaseCacheDecorator)
+
+ def test_keyarg_mem_simple(self):
+ @cache.memcache(keyarg=2)
+ def ka(a, b, c):
+ return a+b+c
+
+ self.assertEqual(ka(1, 1, 1), 3)
+ self.assertEqual(ka(2, 2, 2), 6)
+
+ self.assertEqual(ka(0, 0, 1), 3)
+ self.assertEqual(ka(9, 9, 1), 3)
+ self.assertEqual(ka(0, 0, 2), 6)
+ self.assertEqual(ka(9, 9, 2), 6)
+
+ def test_keyarg_mem(self):
+ @cache.memcache(keyarg=2, maxage=10)
+ def ka(a, b, c):
+ return a+b+c
+
+ self.assertEqual(ka(1, 1, 1), 3)
+ self.assertEqual(ka(2, 2, 2), 6)
+
+ self.assertEqual(ka(0, 0, 1), 3)
+ self.assertEqual(ka(9, 9, 1), 3)
+ self.assertEqual(ka(0, 0, 2), 6)
+ self.assertEqual(ka(9, 9, 2), 6)
+
+ def test_keyarg_db(self):
+ @cache.cache(keyarg=2, maxage=10)
+ def ka(a, b, c):
+ return a+b+c
+
+ self.assertEqual(ka(1, 1, 1), 3)
+ self.assertEqual(ka(2, 2, 2), 6)
+
+ self.assertEqual(ka(0, 0, 1), 3)
+ self.assertEqual(ka(9, 9, 1), 3)
+ self.assertEqual(ka(0, 0, 2), 6)
+ self.assertEqual(ka(9, 9, 2), 6)
+
+ def test_expires_mem(self):
+ @cache.memcache(maxage=1)
+ def ex(a, b, c):
+ return a+b+c
+
+ self.assertEqual(ex(1, 1, 1), 3)
+ self.assertEqual(ex(2, 2, 2), 3)
+ self.assertEqual(ex(3, 3, 3), 3)
+
+ time.sleep(2)
+ self.assertEqual(ex(3, 3, 3), 9)
+ self.assertEqual(ex(2, 2, 2), 9)
+ self.assertEqual(ex(1, 1, 1), 9)
+
+ def test_expires_db(self):
+ @cache.cache(maxage=1)
+ def ex(a, b, c):
+ return a+b+c
+
+ self.assertEqual(ex(1, 1, 1), 3)
+ self.assertEqual(ex(2, 2, 2), 3)
+ self.assertEqual(ex(3, 3, 3), 3)
+
+ time.sleep(2)
+ self.assertEqual(ex(3, 3, 3), 9)
+ self.assertEqual(ex(2, 2, 2), 9)
+ self.assertEqual(ex(1, 1, 1), 9)
+
+ def test_update_mem_simple(self):
+ @cache.memcache(keyarg=0)
+ def up(a, b, c):
+ return a+b+c
+
+ self.assertEqual(up(1, 1, 1), 3)
+ up.update(1, 0)
+ up.update(2, 9)
+ self.assertEqual(up(1, 0, 0), 0)
+ self.assertEqual(up(2, 0, 0), 9)
+
+ def test_update_mem(self):
+ @cache.memcache(keyarg=0, maxage=10)
+ def up(a, b, c):
+ return a+b+c
+
+ self.assertEqual(up(1, 1, 1), 3)
+ up.update(1, 0)
+ up.update(2, 9)
+ self.assertEqual(up(1, 0, 0), 0)
+ self.assertEqual(up(2, 0, 0), 9)
+
+ def test_update_db(self):
+ @cache.cache(keyarg=0, maxage=10)
+ def up(a, b, c):
+ return a+b+c
+
+ self.assertEqual(up(1, 1, 1), 3)
+ up.update(1, 0)
+ up.update(2, 9)
+ self.assertEqual(up(1, 0, 0), 0)
+ self.assertEqual(up(2, 0, 0), 9)
+
+ def test_invalidate_mem_simple(self):
+ @cache.memcache(keyarg=0)
+ def inv(a, b, c):
+ return a+b+c
+
+ self.assertEqual(inv(1, 1, 1), 3)
+ inv.invalidate(1)
+ inv.invalidate(2)
+ self.assertEqual(inv(1, 0, 0), 1)
+ self.assertEqual(inv(2, 0, 0), 2)
+
+ def test_invalidate_mem(self):
+ @cache.memcache(keyarg=0, maxage=10)
+ def inv(a, b, c):
+ return a+b+c
+
+ self.assertEqual(inv(1, 1, 1), 3)
+ inv.invalidate(1)
+ inv.invalidate(2)
+ self.assertEqual(inv(1, 0, 0), 1)
+ self.assertEqual(inv(2, 0, 0), 2)
+
+ def test_invalidate_db(self):
+ @cache.cache(keyarg=0, maxage=10)
+ def inv(a, b, c):
+ return a+b+c
+
+ self.assertEqual(inv(1, 1, 1), 3)
+ inv.invalidate(1)
+ inv.invalidate(2)
+ self.assertEqual(inv(1, 0, 0), 1)
+ self.assertEqual(inv(2, 0, 0), 2)
+
+ def test_database_read(self):
+ @cache.cache(keyarg=0, maxage=10)
+ def db(a, b, c):
+ return a+b+c
+
+ # initialize cache
+ self.assertEqual(db(1, 1, 1), 3)
+ db.update(2, 6)
+
+ # check and clear the in-memory portion of said cache
+ self.assertEqual(db.cache[1][0], 3)
+ self.assertEqual(db.cache[2][0], 6)
+ db.cache.clear()
+ self.assertEqual(db.cache, {})
+
+ # fetch results from database
+ self.assertEqual(db(1, 0, 0), 3)
+ self.assertEqual(db(2, 0, 0), 6)
+
+ # check in-memory cache updates
+ self.assertEqual(db.cache[1][0], 3)
+ self.assertEqual(db.cache[2][0], 6)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_cookies.py b/test/test_cookies.py
index 4f294bf..c39a5e6 100644
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
-# Copyright 2017-2019 Mike Fährmann
+# Copyright 2017-2020 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -12,7 +12,6 @@ from unittest import mock
import logging
import tempfile
-import http.cookiejar
from os.path import join
import gallery_dl.config as config
@@ -34,7 +33,7 @@ class TestCookiejar(unittest.TestCase):
cls.invalid_cookiefile = join(cls.path.name, "invalid.txt")
with open(cls.invalid_cookiefile, "w") as file:
file.write("""# asd
-.example.org\tTRUE\t/\tFALSE\t253402210800\tNAME\tVALUE
+.example.org\tTRUE/FALSE\t253402210800\tNAME\tVALUE
""")
@classmethod
@@ -55,7 +54,7 @@ class TestCookiejar(unittest.TestCase):
self.assertEqual(cookie.value , "VALUE")
def test_invalid_cookiefile(self):
- self._test_warning(self.invalid_cookiefile, http.cookiejar.LoadError)
+ self._test_warning(self.invalid_cookiefile, ValueError)
def test_invalid_filename(self):
self._test_warning(join(self.path.name, "nothing"), FileNotFoundError)
diff --git a/test/test_results.py b/test/test_results.py
index e87b4b8..538abfa 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -12,6 +12,7 @@ import sys
import re
import json
import hashlib
+import datetime
import unittest
from gallery_dl import extractor, util, job, config, exception
@@ -21,14 +22,17 @@ TRAVIS_SKIP = {
"exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie", "bobx",
"archivedmoe", "archiveofsins", "thebarchive", "fireden", "4plebs",
"sankaku", "idolcomplex", "mangahere", "readcomiconline", "mangadex",
- "sankakucomplex", "warosu", "fuskator", "patreon",
+ "sankakucomplex", "warosu", "fuskator", "patreon", "komikcast",
}
# temporary issues, etc.
BROKEN = {
- "imxto",
+ "35photo",
"mangapark",
"photobucket",
+ "sexcom",
+ "hentaicafe",
+ "worldthree",
}
@@ -154,6 +158,9 @@ class TestExtractorResults(unittest.TestCase):
elif isinstance(test, str):
if test.startswith("re:"):
self.assertRegex(value, test[3:], msg=key)
+ elif test.startswith("dt:"):
+ self.assertIsInstance(value, datetime.datetime, msg=key)
+ self.assertEqual(str(value), test[3:], msg=key)
elif test.startswith("type:"):
self.assertEqual(type(value).__name__, test[5:], msg=key)
else:
@@ -267,7 +274,7 @@ class TestFormatter(util.Formatter):
return ""
def _apply_simple(self, key, fmt):
- if key == "extension" or "._format_optional." in repr(fmt):
+ if key == "extension" or "._parse_optional." in repr(fmt):
return self._noop
def wrap(obj):
@@ -275,7 +282,7 @@ class TestFormatter(util.Formatter):
return wrap
def _apply(self, key, funcs, fmt):
- if key == "extension" or "._format_optional." in repr(fmt):
+ if key == "extension" or "._parse_optional." in repr(fmt):
return self._noop
def wrap(obj):
@@ -301,6 +308,7 @@ def setup_test_config():
config.set(("extractor", "nijie") , "username", email)
config.set(("extractor", "seiga") , "username", email)
config.set(("extractor", "danbooru") , "username", None)
+ config.set(("extractor", "e621") , "username", None)
config.set(("extractor", "instagram") , "username", None)
config.set(("extractor", "twitter") , "username", None)
diff --git a/test/test_util.py b/test/test_util.py
index 5a103cf..ffabd37 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
-# Copyright 2015-2019 Mike Fährmann
+# Copyright 2015-2020 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,10 @@
import unittest
import sys
+import io
import random
import string
+import http.cookiejar
from gallery_dl import util, text, exception
@@ -158,11 +160,106 @@ class TestISO639_1(unittest.TestCase):
self.assertEqual(func(*args), result)
+class TestCookiesTxt(unittest.TestCase):
+
+ def test_load_cookiestxt(self):
+
+ def _assert(content, expected):
+ cookies = util.load_cookiestxt(io.StringIO(content, None))
+ for c, e in zip(cookies, expected):
+ self.assertEqual(c.__dict__, e.__dict__)
+
+ _assert("", [])
+ _assert("\n\n\n", [])
+ _assert("$ Comment", [])
+ _assert("# Comment", [])
+ _assert(" # Comment \n\n $ Comment ", [])
+ _assert(
+ ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue",
+ [self._cookie("name", "value", ".example.org")],
+ )
+ _assert(
+ ".example.org\tTRUE\t/\tTRUE\t\tname\t",
+ [self._cookie("name", "", ".example.org")],
+ )
+ _assert(
+ "# Netscape HTTP Cookie File\n"
+ "\n"
+ "# default\n"
+ ".example.org TRUE / FALSE 0 n1 v1\n"
+ ".example.org TRUE / TRUE 2145945600 n2 v2\n"
+ ".example.org TRUE /path FALSE 0 n3\n"
+ "\n"
+ " # # extra # # \n"
+ "www.example.org FALSE / FALSE n4 \n"
+ "www.example.org FALSE /path FALSE 100 n5 v5\n",
+ [
+ self._cookie(
+ "n1", "v1", ".example.org", True, "/", False),
+ self._cookie(
+ "n2", "v2", ".example.org", True, "/", True, 2145945600),
+ self._cookie(
+ "n3", None, ".example.org", True, "/path", False),
+ self._cookie(
+ "n4", "" , "www.example.org", False, "/", False),
+ self._cookie(
+ "n5", "v5", "www.example.org", False, "/path", False, 100),
+ ],
+ )
+
+ with self.assertRaises(ValueError):
+ util.load_cookiestxt("example.org\tTRUE\t/\tTRUE\t0\tname")
+
+ def test_save_cookiestxt(self):
+
+ def _assert(cookies, expected):
+ fp = io.StringIO(newline=None)
+ util.save_cookiestxt(fp, cookies)
+ self.assertMultiLineEqual(fp.getvalue(), expected)
+
+ _assert([], "# Netscape HTTP Cookie File\n\n")
+ _assert(
+ [self._cookie("name", "value", ".example.org")],
+ "# Netscape HTTP Cookie File\n\n"
+ ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n",
+ )
+ _assert(
+ [
+ self._cookie(
+ "n1", "v1", ".example.org", True, "/", False),
+ self._cookie(
+ "n2", "v2", ".example.org", True, "/", True, 2145945600),
+ self._cookie(
+ "n3", None, ".example.org", True, "/path", False),
+ self._cookie(
+ "n4", "" , "www.example.org", False, "/", False),
+ self._cookie(
+ "n5", "v5", "www.example.org", False, "/path", False, 100),
+ ],
+ "# Netscape HTTP Cookie File\n"
+ "\n"
+ ".example.org TRUE / FALSE 0 n1 v1\n"
+ ".example.org TRUE / TRUE 2145945600 n2 v2\n"
+ ".example.org TRUE /path FALSE 0 n3\n"
+ "www.example.org FALSE / FALSE 0 n4 \n"
+ "www.example.org FALSE /path FALSE 100 n5 v5\n",
+ )
+
+ def _cookie(self, name, value, domain, domain_specified=True,
+ path="/", secure=True, expires=None):
+ return http.cookiejar.Cookie(
+ 0, name, value, None, False,
+ domain, domain_specified, domain.startswith("."),
+ path, False, secure, expires, False, None, None, {},
+ )
+
+
class TestFormatter(unittest.TestCase):
kwdict = {
"a": "hElLo wOrLd",
"b": "äöü",
+ "d": {"a": "foo", "b": 0, "c": None},
"l": ["a", "b", "c"],
"n": None,
"u": "%27%3C%20/%20%3E%27",
@@ -227,6 +324,26 @@ class TestFormatter(unittest.TestCase):
self._run_test("{missing[key]}", replacement, default)
self._run_test("{missing:?a//}", "a" + default, default)
+ def test_alternative(self):
+ self._run_test("{a|z}" , "hElLo wOrLd")
+ self._run_test("{z|a}" , "hElLo wOrLd")
+ self._run_test("{z|y|a}" , "hElLo wOrLd")
+ self._run_test("{z|y|x|a}", "hElLo wOrLd")
+ self._run_test("{z|n|a|y}", "hElLo wOrLd")
+
+ self._run_test("{z|a!C}" , "Hello World")
+ self._run_test("{z|a:Rh/C/}" , "CElLo wOrLd")
+ self._run_test("{z|a!C:RH/C/}", "Cello World")
+ self._run_test("{z|y|x:?</>/}", "")
+
+ self._run_test("{d[c]|d[b]|d[a]}", "0")
+ self._run_test("{d[a]|d[b]|d[c]}", "foo")
+ self._run_test("{d[z]|d[y]|d[x]}", "None")
+
+ def test_indexing(self):
+ self._run_test("{l[0]}" , "a")
+ self._run_test("{a[6]}" , "w")
+
def test_slicing(self):
v = self.kwdict["a"]
self._run_test("{a[1:10]}" , v[1:10])
@@ -273,6 +390,18 @@ class TestFormatter(unittest.TestCase):
self._run_test("{a!l:Rl//}" , "heo word")
self._run_test("{name:Rame/othing/}", "Nothing")
+ def test_chain_special(self):
+ # multiple replacements
+ self._run_test("{a:Rh/C/RE/e/RL/l/}", "Cello wOrld")
+ self._run_test("{d[b]!s:R1/Q/R2/A/R0/Y/}", "Y")
+
+ # join-and-replace
+ self._run_test("{l:J-/Rb/E/}", "a-E-c")
+
+ # optional-and-maxlen
+ self._run_test("{d[a]:?</>/L1/too long/}", "<too long>")
+ self._run_test("{d[c]:?</>/L5/too long/}", "")
+
def _run_test(self, format_string, result, default=None):
formatter = util.Formatter(format_string, default)
output = formatter.format_map(self.kwdict)
generated by cgit v1.2.3 (git 2.46.0) at 2026-01-26 19:45:54 +0000