New upstream version 1.13.2upstream/1.13.2

4 files changed, 347 insertions, 9 deletions

diff --git a/test/test_cache.py b/test/test_cache.py
new file mode 100644
index 0000000..31ece7e
--- /dev/null
+++ b/test/test_cache.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2020 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+import unittest
+import tempfile
+import time
+
+from gallery_dl import config, util
+dbpath = tempfile.mkstemp()[1]
+config.set(("cache",), "file", dbpath)
+from gallery_dl import cache  # noqa
+
+
+def tearDownModule():
+    util.remove_file(dbpath)
+
+
+class TestCache(unittest.TestCase):
+
+    def test_decorator(self):
+
+        @cache.memcache()
+        def mc1():
+            pass
+
+        @cache.memcache(maxage=10)
+        def mc2():
+            pass
+
+        @cache.cache()
+        def dbc():
+            pass
+
+        self.assertIsInstance(mc1, cache.CacheDecorator)
+        self.assertIsInstance(mc2, cache.MemoryCacheDecorator)
+        self.assertIsInstance(dbc, cache.DatabaseCacheDecorator)
+
+    def test_keyarg_mem_simple(self):
+        @cache.memcache(keyarg=2)
+        def ka(a, b, c):
+            return a+b+c
+
+        self.assertEqual(ka(1, 1, 1), 3)
+        self.assertEqual(ka(2, 2, 2), 6)
+
+        self.assertEqual(ka(0, 0, 1), 3)
+        self.assertEqual(ka(9, 9, 1), 3)
+        self.assertEqual(ka(0, 0, 2), 6)
+        self.assertEqual(ka(9, 9, 2), 6)
+
+    def test_keyarg_mem(self):
+        @cache.memcache(keyarg=2, maxage=10)
+        def ka(a, b, c):
+            return a+b+c
+
+        self.assertEqual(ka(1, 1, 1), 3)
+        self.assertEqual(ka(2, 2, 2), 6)
+
+        self.assertEqual(ka(0, 0, 1), 3)
+        self.assertEqual(ka(9, 9, 1), 3)
+        self.assertEqual(ka(0, 0, 2), 6)
+        self.assertEqual(ka(9, 9, 2), 6)
+
+    def test_keyarg_db(self):
+        @cache.cache(keyarg=2, maxage=10)
+        def ka(a, b, c):
+            return a+b+c
+
+        self.assertEqual(ka(1, 1, 1), 3)
+        self.assertEqual(ka(2, 2, 2), 6)
+
+        self.assertEqual(ka(0, 0, 1), 3)
+        self.assertEqual(ka(9, 9, 1), 3)
+        self.assertEqual(ka(0, 0, 2), 6)
+        self.assertEqual(ka(9, 9, 2), 6)
+
+    def test_expires_mem(self):
+        @cache.memcache(maxage=1)
+        def ex(a, b, c):
+            return a+b+c
+
+        self.assertEqual(ex(1, 1, 1), 3)
+        self.assertEqual(ex(2, 2, 2), 3)
+        self.assertEqual(ex(3, 3, 3), 3)
+
+        time.sleep(2)
+        self.assertEqual(ex(3, 3, 3), 9)
+        self.assertEqual(ex(2, 2, 2), 9)
+        self.assertEqual(ex(1, 1, 1), 9)
+
+    def test_expires_db(self):
+        @cache.cache(maxage=1)
+        def ex(a, b, c):
+            return a+b+c
+
+        self.assertEqual(ex(1, 1, 1), 3)
+        self.assertEqual(ex(2, 2, 2), 3)
+        self.assertEqual(ex(3, 3, 3), 3)
+
+        time.sleep(2)
+        self.assertEqual(ex(3, 3, 3), 9)
+        self.assertEqual(ex(2, 2, 2), 9)
+        self.assertEqual(ex(1, 1, 1), 9)
+
+    def test_update_mem_simple(self):
+        @cache.memcache(keyarg=0)
+        def up(a, b, c):
+            return a+b+c
+
+        self.assertEqual(up(1, 1, 1), 3)
+        up.update(1, 0)
+        up.update(2, 9)
+        self.assertEqual(up(1, 0, 0), 0)
+        self.assertEqual(up(2, 0, 0), 9)
+
+    def test_update_mem(self):
+        @cache.memcache(keyarg=0, maxage=10)
+        def up(a, b, c):
+            return a+b+c
+
+        self.assertEqual(up(1, 1, 1), 3)
+        up.update(1, 0)
+        up.update(2, 9)
+        self.assertEqual(up(1, 0, 0), 0)
+        self.assertEqual(up(2, 0, 0), 9)
+
+    def test_update_db(self):
+        @cache.cache(keyarg=0, maxage=10)
+        def up(a, b, c):
+            return a+b+c
+
+        self.assertEqual(up(1, 1, 1), 3)
+        up.update(1, 0)
+        up.update(2, 9)
+        self.assertEqual(up(1, 0, 0), 0)
+        self.assertEqual(up(2, 0, 0), 9)
+
+    def test_invalidate_mem_simple(self):
+        @cache.memcache(keyarg=0)
+        def inv(a, b, c):
+            return a+b+c
+
+        self.assertEqual(inv(1, 1, 1), 3)
+        inv.invalidate(1)
+        inv.invalidate(2)
+        self.assertEqual(inv(1, 0, 0), 1)
+        self.assertEqual(inv(2, 0, 0), 2)
+
+    def test_invalidate_mem(self):
+        @cache.memcache(keyarg=0, maxage=10)
+        def inv(a, b, c):
+            return a+b+c
+
+        self.assertEqual(inv(1, 1, 1), 3)
+        inv.invalidate(1)
+        inv.invalidate(2)
+        self.assertEqual(inv(1, 0, 0), 1)
+        self.assertEqual(inv(2, 0, 0), 2)
+
+    def test_invalidate_db(self):
+        @cache.cache(keyarg=0, maxage=10)
+        def inv(a, b, c):
+            return a+b+c
+
+        self.assertEqual(inv(1, 1, 1), 3)
+        inv.invalidate(1)
+        inv.invalidate(2)
+        self.assertEqual(inv(1, 0, 0), 1)
+        self.assertEqual(inv(2, 0, 0), 2)
+
+    def test_database_read(self):
+        @cache.cache(keyarg=0, maxage=10)
+        def db(a, b, c):
+            return a+b+c
+
+        # initialize cache
+        self.assertEqual(db(1, 1, 1), 3)
+        db.update(2, 6)
+
+        # check and clear the in-memory portion of said cache
+        self.assertEqual(db.cache[1][0], 3)
+        self.assertEqual(db.cache[2][0], 6)
+        db.cache.clear()
+        self.assertEqual(db.cache, {})
+
+        # fetch results from database
+        self.assertEqual(db(1, 0, 0), 3)
+        self.assertEqual(db(2, 0, 0), 6)
+
+        # check in-memory cache updates
+        self.assertEqual(db.cache[1][0], 3)
+        self.assertEqual(db.cache[2][0], 6)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_cookies.py b/test/test_cookies.py
index 4f294bf..c39a5e6 100644
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2017-2019 Mike Fährmann
+# Copyright 2017-2020 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -12,7 +12,6 @@ from unittest import mock
 
 import logging
 import tempfile
-import http.cookiejar
 from os.path import join
 
 import gallery_dl.config as config
@@ -34,7 +33,7 @@ class TestCookiejar(unittest.TestCase):
         cls.invalid_cookiefile = join(cls.path.name, "invalid.txt")
         with open(cls.invalid_cookiefile, "w") as file:
             file.write("""# asd
-.example.org\tTRUE\t/\tFALSE\t253402210800\tNAME\tVALUE
+.example.org\tTRUE/FALSE\t253402210800\tNAME\tVALUE
 """)
 
     @classmethod
@@ -55,7 +54,7 @@ class TestCookiejar(unittest.TestCase):
         self.assertEqual(cookie.value , "VALUE")
 
     def test_invalid_cookiefile(self):
-        self._test_warning(self.invalid_cookiefile, http.cookiejar.LoadError)
+        self._test_warning(self.invalid_cookiefile, ValueError)
 
     def test_invalid_filename(self):
         self._test_warning(join(self.path.name, "nothing"), FileNotFoundError)
diff --git a/test/test_results.py b/test/test_results.py
index e87b4b8..538abfa 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -12,6 +12,7 @@ import sys
 import re
 import json
 import hashlib
+import datetime
 import unittest
 from gallery_dl import extractor, util, job, config, exception
 
@@ -21,14 +22,17 @@ TRAVIS_SKIP = {
     "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie", "bobx",
     "archivedmoe", "archiveofsins", "thebarchive", "fireden", "4plebs",
     "sankaku", "idolcomplex", "mangahere", "readcomiconline", "mangadex",
-    "sankakucomplex", "warosu", "fuskator", "patreon",
+    "sankakucomplex", "warosu", "fuskator", "patreon", "komikcast",
 }
 
 # temporary issues, etc.
 BROKEN = {
-    "imxto",
+    "35photo",
     "mangapark",
     "photobucket",
+    "sexcom",
+    "hentaicafe",
+    "worldthree",
 }
 
 
@@ -154,6 +158,9 @@ class TestExtractorResults(unittest.TestCase):
             elif isinstance(test, str):
                 if test.startswith("re:"):
                     self.assertRegex(value, test[3:], msg=key)
+                elif test.startswith("dt:"):
+                    self.assertIsInstance(value, datetime.datetime, msg=key)
+                    self.assertEqual(str(value), test[3:], msg=key)
                 elif test.startswith("type:"):
                     self.assertEqual(type(value).__name__, test[5:], msg=key)
                 else:
@@ -267,7 +274,7 @@ class TestFormatter(util.Formatter):
         return ""
 
     def _apply_simple(self, key, fmt):
-        if key == "extension" or "._format_optional." in repr(fmt):
+        if key == "extension" or "._parse_optional." in repr(fmt):
             return self._noop
 
         def wrap(obj):
@@ -275,7 +282,7 @@ class TestFormatter(util.Formatter):
         return wrap
 
     def _apply(self, key, funcs, fmt):
-        if key == "extension" or "._format_optional." in repr(fmt):
+        if key == "extension" or "._parse_optional." in repr(fmt):
             return self._noop
 
         def wrap(obj):
@@ -301,6 +308,7 @@ def setup_test_config():
     config.set(("extractor", "nijie")     , "username", email)
     config.set(("extractor", "seiga")     , "username", email)
     config.set(("extractor", "danbooru")  , "username", None)
+    config.set(("extractor", "e621")      , "username", None)
     config.set(("extractor", "instagram") , "username", None)
     config.set(("extractor", "twitter")   , "username", None)
 
diff --git a/test/test_util.py b/test/test_util.py
index 5a103cf..ffabd37 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2019 Mike Fährmann
+# Copyright 2015-2020 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,10 @@
 
 import unittest
 import sys
+import io
 import random
 import string
+import http.cookiejar
 
 from gallery_dl import util, text, exception
 
@@ -158,11 +160,106 @@ class TestISO639_1(unittest.TestCase):
             self.assertEqual(func(*args), result)
 
 
+class TestCookiesTxt(unittest.TestCase):
+
+    def test_load_cookiestxt(self):
+
+        def _assert(content, expected):
+            cookies = util.load_cookiestxt(io.StringIO(content, None))
+            for c, e in zip(cookies, expected):
+                self.assertEqual(c.__dict__, e.__dict__)
+
+        _assert("", [])
+        _assert("\n\n\n", [])
+        _assert("$ Comment", [])
+        _assert("# Comment", [])
+        _assert(" # Comment \n\n $ Comment ", [])
+        _assert(
+            ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue",
+            [self._cookie("name", "value", ".example.org")],
+        )
+        _assert(
+            ".example.org\tTRUE\t/\tTRUE\t\tname\t",
+            [self._cookie("name", "", ".example.org")],
+        )
+        _assert(
+            "# Netscape HTTP Cookie File\n"
+            "\n"
+            "# default\n"
+            ".example.org	TRUE	/	FALSE	0	n1	v1\n"
+            ".example.org	TRUE	/	TRUE	2145945600	n2	v2\n"
+            ".example.org	TRUE	/path	FALSE	0		n3\n"
+            "\n"
+            "  # # extra # #  \n"
+            "www.example.org	FALSE	/	FALSE		n4	\n"
+            "www.example.org	FALSE	/path	FALSE	100	n5	v5\n",
+            [
+                self._cookie(
+                    "n1", "v1", ".example.org", True, "/", False),
+                self._cookie(
+                    "n2", "v2", ".example.org", True, "/", True, 2145945600),
+                self._cookie(
+                    "n3", None, ".example.org", True, "/path", False),
+                self._cookie(
+                    "n4", ""  , "www.example.org", False, "/", False),
+                self._cookie(
+                    "n5", "v5", "www.example.org", False, "/path", False, 100),
+            ],
+        )
+
+        with self.assertRaises(ValueError):
+            util.load_cookiestxt("example.org\tTRUE\t/\tTRUE\t0\tname")
+
+    def test_save_cookiestxt(self):
+
+        def _assert(cookies, expected):
+            fp = io.StringIO(newline=None)
+            util.save_cookiestxt(fp, cookies)
+            self.assertMultiLineEqual(fp.getvalue(), expected)
+
+        _assert([], "# Netscape HTTP Cookie File\n\n")
+        _assert(
+            [self._cookie("name", "value", ".example.org")],
+            "# Netscape HTTP Cookie File\n\n"
+            ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n",
+        )
+        _assert(
+            [
+                self._cookie(
+                    "n1", "v1", ".example.org", True, "/", False),
+                self._cookie(
+                    "n2", "v2", ".example.org", True, "/", True, 2145945600),
+                self._cookie(
+                    "n3", None, ".example.org", True, "/path", False),
+                self._cookie(
+                    "n4", ""  , "www.example.org", False, "/", False),
+                self._cookie(
+                    "n5", "v5", "www.example.org", False, "/path", False, 100),
+            ],
+            "# Netscape HTTP Cookie File\n"
+            "\n"
+            ".example.org	TRUE	/	FALSE	0	n1	v1\n"
+            ".example.org	TRUE	/	TRUE	2145945600	n2	v2\n"
+            ".example.org	TRUE	/path	FALSE	0		n3\n"
+            "www.example.org	FALSE	/	FALSE	0	n4	\n"
+            "www.example.org	FALSE	/path	FALSE	100	n5	v5\n",
+        )
+
+    def _cookie(self, name, value, domain, domain_specified=True,
+                path="/", secure=True, expires=None):
+        return http.cookiejar.Cookie(
+            0, name, value, None, False,
+            domain, domain_specified, domain.startswith("."),
+            path, False, secure, expires, False, None, None, {},
+        )
+
+
 class TestFormatter(unittest.TestCase):
 
     kwdict = {
         "a": "hElLo wOrLd",
         "b": "äöü",
+        "d": {"a": "foo", "b": 0, "c": None},
         "l": ["a", "b", "c"],
         "n": None,
         "u": "%27%3C%20/%20%3E%27",
@@ -227,6 +324,26 @@ class TestFormatter(unittest.TestCase):
         self._run_test("{missing[key]}", replacement, default)
         self._run_test("{missing:?a//}", "a" + default, default)
 
+    def test_alternative(self):
+        self._run_test("{a|z}"    , "hElLo wOrLd")
+        self._run_test("{z|a}"    , "hElLo wOrLd")
+        self._run_test("{z|y|a}"  , "hElLo wOrLd")
+        self._run_test("{z|y|x|a}", "hElLo wOrLd")
+        self._run_test("{z|n|a|y}", "hElLo wOrLd")
+
+        self._run_test("{z|a!C}"      , "Hello World")
+        self._run_test("{z|a:Rh/C/}"  , "CElLo wOrLd")
+        self._run_test("{z|a!C:RH/C/}", "Cello World")
+        self._run_test("{z|y|x:?</>/}", "")
+
+        self._run_test("{d[c]|d[b]|d[a]}", "0")
+        self._run_test("{d[a]|d[b]|d[c]}", "foo")
+        self._run_test("{d[z]|d[y]|d[x]}", "None")
+
+    def test_indexing(self):
+        self._run_test("{l[0]}" , "a")
+        self._run_test("{a[6]}" , "w")
+
     def test_slicing(self):
         v = self.kwdict["a"]
         self._run_test("{a[1:10]}"  , v[1:10])
@@ -273,6 +390,18 @@ class TestFormatter(unittest.TestCase):
         self._run_test("{a!l:Rl//}" , "heo word")
         self._run_test("{name:Rame/othing/}", "Nothing")
 
+    def test_chain_special(self):
+        # multiple replacements
+        self._run_test("{a:Rh/C/RE/e/RL/l/}", "Cello wOrld")
+        self._run_test("{d[b]!s:R1/Q/R2/A/R0/Y/}", "Y")
+
+        # join-and-replace
+        self._run_test("{l:J-/Rb/E/}", "a-E-c")
+
+        # optional-and-maxlen
+        self._run_test("{d[a]:?</>/L1/too long/}", "<too long>")
+        self._run_test("{d[c]:?</>/L5/too long/}", "")
+
     def _run_test(self, format_string, result, default=None):
         formatter = util.Formatter(format_string, default)
         output = formatter.format_map(self.kwdict)