New upstream version 1.16.0.upstream/1.16.0

3 files changed, 71 insertions, 61 deletions

diff --git a/test/test_cookies.py b/test/test_cookies.py
index f691980..d103d02 100644
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@@ -88,7 +88,7 @@ class TestCookiedict(unittest.TestCase):
         self.assertEqual(sorted(cookies.values()), sorted(self.cdict.values()))
 
     def test_domain(self):
-        for category in ["exhentai", "nijie", "sankaku", "seiga"]:
+        for category in ["exhentai", "idolcomplex", "nijie", "seiga"]:
             extr = _get_extractor(category)
             cookies = extr.session.cookies
             for key in self.cdict:
@@ -104,10 +104,10 @@ class TestCookieLogin(unittest.TestCase):
 
     def test_cookie_login(self):
         extr_cookies = {
-            "exhentai": ("ipb_member_id", "ipb_pass_hash"),
-            "nijie"   : ("nemail", "nlogin"),
-            "sankaku" : ("login", "pass_hash"),
-            "seiga"   : ("user_session",),
+            "exhentai"   : ("ipb_member_id", "ipb_pass_hash"),
+            "idolcomplex": ("login", "pass_hash"),
+            "nijie"      : ("nemail", "nlogin"),
+            "seiga"      : ("user_session",),
         }
         for category, cookienames in extr_cookies.items():
             cookies = {name: "value" for name in cookienames}
diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py
index 524e501..74e8742 100644
--- a/test/test_postprocessor.py
+++ b/test/test_postprocessor.py
@@ -15,11 +15,12 @@ from unittest.mock import Mock, mock_open, patch
 import logging
 import zipfile
 import tempfile
+import collections
 from datetime import datetime, timezone as tz
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from gallery_dl import extractor, output, util  # noqa E402
-from gallery_dl import postprocessor, util, config  # noqa E402
+from gallery_dl import postprocessor, config  # noqa E402
 from gallery_dl.postprocessor.common import PostProcessor  # noqa E402
 
 
@@ -34,6 +35,7 @@ class FakeJob():
         self.pathfmt = util.PathFormat(self.extractor)
         self.out = output.NullOutput()
         self.get_logger = logging.getLogger
+        self.hooks = collections.defaultdict(list)
 
 
 class TestPostprocessorModule(unittest.TestCase):
@@ -78,6 +80,9 @@ class BasePostprocessorTest(unittest.TestCase):
         cls.dir.cleanup()
         config.clear()
 
+    def tearDown(self):
+        self.job.hooks.clear()
+
     def _create(self, options=None, data=None):
         kwdict = {"category": "test", "filename": "file", "extension": "ext"}
         if options is None:
@@ -92,6 +97,11 @@ class BasePostprocessorTest(unittest.TestCase):
         pp = postprocessor.find(self.__class__.__name__[:-4].lower())
         return pp(self.job, options)
 
+    def _trigger(self, events=None, *args):
+        for event in (events or ("prepare", "file")):
+            for callback in self.job.hooks[event]:
+                callback(self.pathfmt, *args)
+
 
 class ClassifyTest(BasePostprocessorTest):
 
@@ -111,7 +121,7 @@ class ClassifyTest(BasePostprocessorTest):
         self.assertEqual(self.pathfmt.realpath, path + "/file.jpg")
 
         with patch("os.makedirs") as mkdirs:
-            pp.run(self.pathfmt)
+            self._trigger()
             mkdirs.assert_called_once_with(path, exist_ok=True)
 
     def test_classify_noop(self):
@@ -123,7 +133,7 @@ class ClassifyTest(BasePostprocessorTest):
         self.assertEqual(self.pathfmt.realpath, rp)
 
         with patch("os.makedirs") as mkdirs:
-            pp.run(self.pathfmt)
+            self._trigger()
             self.assertEqual(mkdirs.call_count, 0)
 
     def test_classify_custom(self):
@@ -143,7 +153,7 @@ class ClassifyTest(BasePostprocessorTest):
         self.assertEqual(self.pathfmt.realpath, path + "/file.foo")
 
         with patch("os.makedirs") as mkdirs:
-            pp.run(self.pathfmt)
+            self._trigger()
             mkdirs.assert_called_once_with(path, exist_ok=True)
 
 
@@ -175,8 +185,7 @@ class MetadataTest(BasePostprocessorTest):
         self.assertEqual(pp.extension, "JSON")
 
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
 
         path = self.pathfmt.realpath + ".JSON"
         m.assert_called_once_with(path, "w", encoding="utf-8")
@@ -197,41 +206,37 @@ class MetadataTest(BasePostprocessorTest):
         self.assertEqual(pp.extension, "txt")
 
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
 
         path = self.pathfmt.realpath + ".txt"
         m.assert_called_once_with(path, "w", encoding="utf-8")
         self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
 
     def test_metadata_tags_split_1(self):
-        pp = self._create(
+        self._create(
             {"mode": "tags"},
             {"tags": "foo, bar, baz"},
         )
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
         self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
 
     def test_metadata_tags_split_2(self):
-        pp = self._create(
+        self._create(
             {"mode": "tags"},
             {"tags": "foobar1 foobar2 foobarbaz"},
         )
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
         self.assertEqual(self._output(m), "foobar1\nfoobar2\nfoobarbaz\n")
 
     def test_metadata_tags_tagstring(self):
-        pp = self._create(
+        self._create(
             {"mode": "tags"},
             {"tag_string": "foo, bar, baz"},
         )
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
         self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
 
     def test_metadata_custom(self):
@@ -239,12 +244,12 @@ class MetadataTest(BasePostprocessorTest):
             pp = self._create(pp_info, {"foo": "bar"})
             self.assertEqual(pp.write, pp._write_custom)
             self.assertEqual(pp.extension, "txt")
-            self.assertTrue(pp.contentfmt)
+            self.assertTrue(pp._content_fmt)
 
             with patch("builtins.open", mock_open()) as m:
-                pp.prepare(self.pathfmt)
-                pp.run(self.pathfmt)
+                self._trigger()
             self.assertEqual(self._output(m), "bar\nNone\n")
+            self.job.hooks.clear()
 
         test({"mode": "custom", "content-format": "{foo}\n{missing}\n"})
         test({"mode": "custom", "content-format": ["{foo}", "{missing}"]})
@@ -256,53 +261,61 @@ class MetadataTest(BasePostprocessorTest):
             "extension-format": "json",
         })
 
-        self.assertEqual(pp._filename, pp._filename_custom)
+        self.assertEqual(pp._filename, pp._filename_extfmt)
 
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
 
         path = self.pathfmt.realdirectory + "file.json"
         m.assert_called_once_with(path, "w", encoding="utf-8")
 
     def test_metadata_extfmt_2(self):
-        pp = self._create({
+        self._create({
             "extension-format": "{extension!u}-data:{category:Res/ES/}",
         })
 
         self.pathfmt.prefix = "2."
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
 
         path = self.pathfmt.realdirectory + "file.2.EXT-data:tESt"
         m.assert_called_once_with(path, "w", encoding="utf-8")
 
     def test_metadata_directory(self):
-        pp = self._create({
+        self._create({
             "directory": "metadata",
         })
 
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
 
         path = self.pathfmt.realdirectory + "metadata/file.ext.json"
         m.assert_called_once_with(path, "w", encoding="utf-8")
 
     def test_metadata_directory_2(self):
-        pp = self._create({
+        self._create({
             "directory"       : "metadata////",
             "extension-format": "json",
         })
 
         with patch("builtins.open", mock_open()) as m:
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
 
         path = self.pathfmt.realdirectory + "metadata/file.json"
         m.assert_called_once_with(path, "w", encoding="utf-8")
 
+    def test_metadata_filename(self):
+        self._create({
+            "filename"        : "{category}_{filename}_meta.data",
+            "extension-format": "json",
+        })
+
+        with patch("builtins.open", mock_open()) as m:
+            self._trigger()
+
+        path = self.pathfmt.realdirectory + "test_file_meta.data"
+        m.assert_called_once_with(path, "w", encoding="utf-8")
+
     @staticmethod
     def _output(mock):
         return "".join(
@@ -319,21 +332,18 @@ class MtimeTest(BasePostprocessorTest):
         self.assertEqual(pp.key, "date")
 
     def test_mtime_datetime(self):
-        pp = self._create(None, {"date": datetime(1980, 1, 1, tzinfo=tz.utc)})
-        pp.prepare(self.pathfmt)
-        pp.run(self.pathfmt)
+        self._create(None, {"date": datetime(1980, 1, 1, tzinfo=tz.utc)})
+        self._trigger()
         self.assertEqual(self.pathfmt.kwdict["_mtime"], 315532800)
 
     def test_mtime_timestamp(self):
-        pp = self._create(None, {"date": 315532800})
-        pp.prepare(self.pathfmt)
-        pp.run(self.pathfmt)
+        self._create(None, {"date": 315532800})
+        self._trigger()
         self.assertEqual(self.pathfmt.kwdict["_mtime"], 315532800)
 
     def test_mtime_custom(self):
-        pp = self._create({"key": "foo"}, {"foo": 315532800})
-        pp.prepare(self.pathfmt)
-        pp.run(self.pathfmt)
+        self._create({"key": "foo"}, {"foo": 315532800})
+        self._trigger()
         self.assertEqual(self.pathfmt.kwdict["_mtime"], 315532800)
 
 
@@ -341,8 +351,8 @@ class ZipTest(BasePostprocessorTest):
 
     def test_zip_default(self):
         pp = self._create()
+        self.assertEqual(self.job.hooks["file"][0], pp.write)
         self.assertEqual(pp.path, self.pathfmt.realdirectory)
-        self.assertEqual(pp.run, pp._write)
         self.assertEqual(pp.delete, True)
         self.assertEqual(pp.args, (
             pp.path[:-1] + ".zip", "a", zipfile.ZIP_STORED, True,
@@ -351,8 +361,8 @@ class ZipTest(BasePostprocessorTest):
 
     def test_zip_safe(self):
         pp = self._create({"mode": "safe"})
+        self.assertEqual(self.job.hooks["file"][0], pp.write_safe)
         self.assertEqual(pp.path, self.pathfmt.realdirectory)
-        self.assertEqual(pp.run, pp._write_safe)
         self.assertEqual(pp.delete, True)
         self.assertEqual(pp.args, (
             pp.path[:-1] + ".zip", "a", zipfile.ZIP_STORED, True,
@@ -383,8 +393,7 @@ class ZipTest(BasePostprocessorTest):
                 self.pathfmt.temppath = file.name
                 self.pathfmt.filename = name
 
-                pp.prepare(self.pathfmt)
-                pp.run(self.pathfmt)
+                self._trigger()
 
                 nti = pp.zfile.NameToInfo
                 self.assertEqual(len(nti), i+1)
@@ -397,12 +406,11 @@ class ZipTest(BasePostprocessorTest):
             self.assertIn("file2.ext", nti)
 
             # write the last file a second time (will be skipped)
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
             self.assertEqual(len(pp.zfile.NameToInfo), 3)
 
         # close file
-        pp.run_final(self.pathfmt, 0)
+        self._trigger(("finalize",), 0)
 
         # reopen to check persistence
         with zipfile.ZipFile(pp.zfile.filename) as file:
@@ -428,14 +436,13 @@ class ZipTest(BasePostprocessorTest):
         for i in range(3):
             self.pathfmt.temppath = self.pathfmt.realdirectory + "file.ext"
             self.pathfmt.filename = "file{}.ext".format(i)
-            pp.prepare(self.pathfmt)
-            pp.run(self.pathfmt)
+            self._trigger()
 
-        # write the last file a second time (will be skipped)
-        pp.prepare(self.pathfmt)
-        pp.run(self.pathfmt)
+        # write the last file a second time (should be skipped)
+        self._trigger()
 
-        pp.run_final(self.pathfmt, 0)
+        # close file
+        self._trigger(("finalize",), 0)
 
         self.assertEqual(pp.zfile.write.call_count, 3)
         for call in pp.zfile.write.call_args_list:
diff --git a/test/test_results.py b/test/test_results.py
index 759a4b4..4e9f4b2 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -30,7 +30,7 @@ TRAVIS_SKIP = {
 
 # temporary issues, etc.
 BROKEN = {
-    "4plebs",
+    "dokireader",
     "imagevenue",
     "photobucket",
 }
@@ -208,6 +208,9 @@ class ResultJob(job.DownloadJob):
         self._update_kwdict(kwdict, False)
         self.format_directory(kwdict)
 
+    def handle_metadata(self, kwdict):
+        pass
+
     def handle_queue(self, url, kwdict):
         self.queue = True
         self._update_url(url)
@@ -367,7 +370,7 @@ def generate_tests():
     # filter available extractor classes
     extractors = [
         extr for extr in extractor.extractors()
-        if fltr(extr.category, getattr(extr, "basecategory", None))
+        if fltr(extr.category, extr.basecategory)
     ]
 
     # add 'test_...' methods