New upstream version 1.31.1.upstream/1.31.1

10 files changed, 705 insertions, 163 deletions

diff --git a/test/test_downloader.py b/test/test_downloader.py
index f6c3dbe..fb442c4 100644
--- a/test/test_downloader.py
+++ b/test/test_downloader.py
@@ -298,6 +298,15 @@ class TestHTTPDownloader(TestDownloaderBase):
         self.assertTrue(success)
         self.assertEqual(pathfmt.temppath, "")
 
+    def test_http_empty(self):
+        url = f"{self.address}/~NUL"
+        pathfmt = self._prepare_destination(None, extension=None)
+        with self.assertLogs(self.downloader.log, "WARNING") as log_info:
+            success = self.downloader.download(url, pathfmt)
+        self.assertFalse(success)
+        self.assertEqual(log_info.output[0],
+                         "WARNING:downloader.http:Empty file")
+
 
 class TestTextDownloader(TestDownloaderBase):
 
@@ -400,6 +409,7 @@ SAMPLES = {
     ("blend", b"BLENDER-v303RENDH"),
     ("obj" , b"# Blender v3.2.0 OBJ File: 'foo.blend'"),
     ("clip", b"CSFCHUNK\x00\x00\x00\x00"),
+    ("~NUL", b""),
 }
 
 
@@ -428,8 +438,9 @@ def generate_tests():
         return test
 
     for idx, (ext, content) in enumerate(SAMPLES):
-        test = generate_test(idx, ext, content)
-        setattr(TestHTTPDownloader, test.__name__, test)
+        if ext[0].isalnum():
+            test = generate_test(idx, ext, content)
+            setattr(TestHTTPDownloader, test.__name__, test)
 
 
 generate_tests()
diff --git a/test/test_dt.py b/test/test_dt.py
new file mode 100644
index 0000000..02e3ac2
--- /dev/null
+++ b/test/test_dt.py
@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2025 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+import os
+import sys
+import unittest
+
+import datetime
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from gallery_dl import dt # noqa E402
+
+
+class TestDatetime(unittest.TestCase):
+
+    def test_convert(self, f=dt.convert):
+
+        def _assert(value, expected):
+            result = f(value)
+            self.assertIsInstance(result, datetime.datetime)
+            self.assertEqual(result, expected, msg=repr(value))
+
+        d = datetime.datetime(2010, 1, 1)
+        self.assertIs(f(d), d)
+
+        _assert(d             , d)
+        _assert(1262304000    , d)
+        _assert(1262304000.0  , d)
+        _assert(1262304000.123, d)
+        _assert("1262304000"  , d)
+
+        _assert("2010-01-01"                      , d)
+        _assert("2010-01-01 00:00:00"             , d)
+        _assert("2010-01-01T00:00:00"             , d)
+        _assert("2010-01-01T00:00:00.123456"      , d)
+        _assert("2009-12-31T19:00:00-05:00"       , d)
+        _assert("2009-12-31T19:00:00.123456-05:00", d)
+        _assert("2010-01-01T00:00:00Z"            , d)
+        _assert("2010-01-01T00:00:00.123456Z"     , d)
+        _assert("2009-12-31T19:00:00-0500"        , d)
+        _assert("2009-12-31T19:00:00.123456-0500" , d)
+
+        _assert(0    , dt.NONE)
+        _assert(""   , dt.NONE)
+        _assert("foo", dt.NONE)
+        _assert(None , dt.NONE)
+        _assert(()   , dt.NONE)
+        _assert([]   , dt.NONE)
+        _assert({}   , dt.NONE)
+        _assert((1, 2, 3), dt.NONE)
+
+    @unittest.skipIf(sys.hexversion < 0x30b0000,
+                     "extended fromisoformat timezones")
+    def test_convert_tz(self, f=dt.convert):
+
+        def _assert(value, expected):
+            result = f(value)
+            self.assertIsInstance(result, datetime.datetime)
+            self.assertEqual(result, expected, msg=repr(value))
+
+        d = datetime.datetime(2010, 1, 1)
+        _assert("2009-12-31T19:00:00-05"          , d)
+        _assert("2009-12-31T19:00:00.123456-05"   , d)
+
+    def test_to_timestamp(self, f=dt.to_ts):
+        self.assertEqual(f(dt.EPOCH), 0.0)
+        self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0)
+        self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)),
+                         1262304000.128000)
+        with self.assertRaises(TypeError):
+            f(None)
+
+    def test_to_timestamp_string(self, f=dt.to_ts_string):
+        self.assertEqual(f(dt.EPOCH), "0")
+        self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000")
+        self.assertEqual(f(None), "")
+
+    def test_from_timestamp(self, f=dt.from_ts):
+        self.assertEqual(f(0.0), dt.EPOCH)
+        self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1))
+        self.assertEqual(f(1262304000.128000).replace(microsecond=0),
+                         datetime.datetime(2010, 1, 1, 0, 0, 0))
+
+    def test_now(self, f=dt.now):
+        self.assertIsInstance(f(), datetime.datetime)
+
+    def test_parse_timestamp(self, f=dt.parse_ts):
+        null = dt.from_ts(0)
+        value = dt.from_ts(1555816235)
+
+        self.assertEqual(f(0)           , null)
+        self.assertEqual(f("0")         , null)
+        self.assertEqual(f(1555816235)  , value)
+        self.assertEqual(f("1555816235"), value)
+
+        for value in ((), [], {}, None, ""):
+            self.assertEqual(f(value), dt.NONE)
+            self.assertEqual(f(value, "foo"), "foo")
+
+    def test_parse(self, f=dt.parse):
+        self.assertEqual(
+            f("1970.01.01", "%Y.%m.%d"),
+            dt.EPOCH,
+        )
+        self.assertEqual(
+            f("May 7, 2019 9:33 am", "%B %d, %Y %I:%M %p"),
+            datetime.datetime(2019, 5, 7, 9, 33, 0),
+        )
+        self.assertEqual(
+            f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"),
+            datetime.datetime(2019, 5, 7, 12, 25, 2),
+        )
+
+        for value in ((), [], {}, None, 1, 2.3):
+            self.assertEqual(f(value, "%Y"), dt.NONE)
+
+    def test_parse_iso(self, f=dt.parse_iso):
+        self.assertEqual(
+            f("1970-01-01T00:00:00+00:00"),
+            dt.from_ts(0),
+        )
+        self.assertEqual(
+            f("2019-05-07T21:25:02+09:00"),
+            datetime.datetime(2019, 5, 7, 12, 25, 2),
+        )
+        self.assertEqual(
+            f("2019-05-07T12:25:02Z"),
+            datetime.datetime(2019, 5, 7, 12, 25, 2),
+        )
+        self.assertEqual(
+            f("2019-05-07 21:25:02"),
+            datetime.datetime(2019, 5, 7, 21, 25, 2),
+        )
+        self.assertEqual(
+            f("1970-01-01"),
+            dt.EPOCH,
+        )
+        self.assertEqual(
+            f("1970.01.01"),
+            dt.NONE,
+        )
+        self.assertEqual(
+            f("1970-01-01T00:00:00+0000"),
+            dt.EPOCH,
+        )
+        self.assertEqual(
+            f("2019-05-07T21:25:02.753+0900"),
+            datetime.datetime(2019, 5, 7, 12, 25, 2),
+        )
+
+        for value in ((), [], {}, None, 1, 2.3):
+            self.assertEqual(f(value), dt.NONE)
+
+    def test_none(self):
+        self.assertFalse(dt.NONE)
+        self.assertIsInstance(dt.NONE, dt.datetime)
+        self.assertEqual(str(dt.NONE), "[Invalid DateTime]")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_extractor.py b/test/test_extractor.py
index a623e1d..c06b890 100644
--- a/test/test_extractor.py
+++ b/test/test_extractor.py
@@ -14,10 +14,9 @@ from unittest.mock import patch
 
 import time
 import string
-from datetime import datetime, timedelta
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from gallery_dl import extractor, util  # noqa E402
+from gallery_dl import extractor, util, dt, config  # noqa E402
 from gallery_dl.extractor import mastodon  # noqa E402
 from gallery_dl.extractor.common import Extractor, Message  # noqa E402
 from gallery_dl.extractor.directlink import DirectlinkExtractor  # noqa E402
@@ -40,7 +39,7 @@ class FakeExtractor(Extractor):
     pattern = "fake:"
 
     def items(self):
-        yield Message.Version, 1
+        yield Message.Noop
         yield Message.Url, "text:foobar", {}
 
 
@@ -233,8 +232,8 @@ class TestExtractorWait(unittest.TestCase):
 
     def test_wait_until_datetime(self):
         extr = extractor.find("generic:https://example.org/")
-        until = util.datetime_utcnow() + timedelta(seconds=5)
-        until_local = datetime.now() + timedelta(seconds=5)
+        until = dt.now() + dt.timedelta(seconds=5)
+        until_local = dt.datetime.now() + dt.timedelta(seconds=5)
 
         if not until.microsecond:
             until = until.replace(microsecond=until_local.microsecond)
@@ -251,8 +250,8 @@ class TestExtractorWait(unittest.TestCase):
             self._assert_isotime(calls[0][1][1], until_local)
 
     def _assert_isotime(self, output, until):
-        if not isinstance(until, datetime):
-            until = datetime.fromtimestamp(until)
+        if not isinstance(until, dt.datetime):
+            until = dt.datetime.fromtimestamp(until)
         o = self._isotime_to_seconds(output)
         u = self._isotime_to_seconds(until.time().isoformat()[:8])
         self.assertLessEqual(o-u, 1.0)
@@ -262,6 +261,79 @@ class TestExtractorWait(unittest.TestCase):
         return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
 
 
+class TextExtractorCommonDateminmax(unittest.TestCase):
+
+    def setUp(self):
+        config.clear()
+
+    tearDown = setUp
+
+    def test_date_min_max_default(self):
+        extr = extractor.find("generic:https://example.org/")
+
+        dmin, dmax = extr._get_date_min_max()
+        self.assertEqual(dmin, None)
+        self.assertEqual(dmax, None)
+
+        dmin, dmax = extr._get_date_min_max(..., -1)
+        self.assertEqual(dmin, ...)
+        self.assertEqual(dmax, -1)
+
+    def test_date_min_max_timestamp(self):
+        extr = extractor.find("generic:https://example.org/")
+        config.set((), "date-min", 1262304000)
+        config.set((), "date-max", 1262304000.123)
+
+        dmin, dmax = extr._get_date_min_max()
+        self.assertEqual(dmin, 1262304000)
+        self.assertEqual(dmax, 1262304000.123)
+
+    def test_date_min_max_iso(self):
+        extr = extractor.find("generic:https://example.org/")
+        config.set((), "date-min", "2010-01-01")
+        config.set((), "date-max", "2010-01-01T00:01:03")
+
+        dmin, dmax = extr._get_date_min_max()
+        self.assertEqual(dmin, 1262304000)
+        self.assertEqual(dmax, 1262304063)
+
+    def test_date_min_max_iso_invalid(self):
+        extr = extractor.find("generic:https://example.org/")
+        config.set((), "date-min", "2010-01-01")
+        config.set((), "date-max", "2010-01")
+
+        with self.assertLogs() as log_info:
+            dmin, dmax = extr._get_date_min_max()
+        self.assertEqual(dmin, 1262304000)
+        self.assertEqual(dmax, None)
+
+        self.assertEqual(len(log_info.output), 1)
+        self.assertEqual(
+            log_info.output[0],
+            "WARNING:generic:Unable to parse 'date-max': "
+            "Invalid isoformat string '2010-01'")
+
+    def test_date_min_max_fmt(self):
+        extr = extractor.find("generic:https://example.org/")
+        config.set((), "date-format", "%B %d %Y")
+        config.set((), "date-min", "January 01 2010")
+        config.set((), "date-max", "August 18 2022")
+
+        dmin, dmax = extr._get_date_min_max()
+        self.assertEqual(dmin, 1262304000)
+        self.assertEqual(dmax, 1660780800)
+
+    def test_date_min_max_mix(self):
+        extr = extractor.find("generic:https://example.org/")
+        config.set((), "date-format", "%B %d %Y")
+        config.set((), "date-min", "January 01 2010")
+        config.set((), "date-max", 1262304061)
+
+        dmin, dmax = extr._get_date_min_max()
+        self.assertEqual(dmin, 1262304000)
+        self.assertEqual(dmax, 1262304061)
+
+
 class TextExtractorOAuth(unittest.TestCase):
 
     def test_oauth1(self):
diff --git a/test/test_formatter.py b/test/test_formatter.py
index 01e3a88..67df279 100644
--- a/test/test_formatter.py
+++ b/test/test_formatter.py
@@ -15,7 +15,7 @@ import datetime
 import tempfile
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from gallery_dl import formatter, text, util, config  # noqa E402
+from gallery_dl import formatter, text, dt, util, config  # noqa E402
 
 try:
     import jinja2
@@ -154,7 +154,7 @@ class TestFormatter(unittest.TestCase):
         self._run_test("{t}" , self.kwdict["t"] , None, int)
         self._run_test("{t}" , self.kwdict["t"] , None, util.identity)
         self._run_test("{dt}", self.kwdict["dt"], None, util.identity)
-        self._run_test("{ds}", self.kwdict["dt"], None, text.parse_datetime)
+        self._run_test("{ds}", self.kwdict["dt"], None, dt.parse_iso)
         self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"],
                        None, util.identity)
 
@@ -248,6 +248,19 @@ class TestFormatter(unittest.TestCase):
         self._run_test("{a:L50/foo/>51}", "foo")
         self._run_test("{a:Lab/foo/}", "foo")
 
+    def test_specifier_maxlen_bytes(self):
+        v = self.kwdict["a"]
+        self._run_test("{a:Lb5/foo/}" , "foo")
+        self._run_test("{a:Lb50/foo/}", v)
+        self._run_test("{a:Lb50/foo/>50}", " " * 39 + v)
+        self._run_test("{a:Lb50/foo/>51}", "foo")
+        self._run_test("{a:Lbab/foo/}", "foo")
+
+        v = self.kwdict["j"]
+        self._run_test("{j:Lb5/foo/}" , "foo")
+        self._run_test("{j:Lb50/foo/}", v)
+        self._run_test("{j:Lbab/foo/}", "foo")
+
     def test_specifier_join(self):
         self._run_test("{l:J}"       , "abc")
         self._run_test("{l:J,}"      , "a,b,c")
@@ -271,8 +284,8 @@ class TestFormatter(unittest.TestCase):
 
     def test_specifier_datetime(self):
         self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", "2010-01-01 00:00:00")
-        self._run_test("{ds:D%Y}", "2010-01-01T01:00:00+01:00")
-        self._run_test("{l:D%Y}", "None")
+        self._run_test("{ds:D%Y}", "[Invalid DateTime]")
+        self._run_test("{l2:D%Y}", "[Invalid DateTime]")
 
     def test_specifier_offset(self):
         self._run_test("{dt:O 01:00}", "2010-01-01 01:00:00")
@@ -332,6 +345,17 @@ class TestFormatter(unittest.TestCase):
         with self.assertRaises(ValueError):
             self._run_test("{a:Xfoo/ */}", "hello wo *")
 
+    def test_specifier_limit_bytes(self):
+        self._run_test("{a:Xb20/ */}", "hElLo wOrLd")
+        self._run_test("{a:Xb10/ */}", "hElLo wO *")
+
+        self._run_test("{j:Xb50/〜/}", "げんそうきょう")
+        self._run_test("{j:Xb20/〜/}", "げんそうき〜")
+        self._run_test("{j:Xb20/ */}", "げんそうきょ *")
+
+        with self.assertRaises(ValueError):
+            self._run_test("{a:Xbfoo/ */}", "hello wo *")
+
     def test_specifier_map(self):
         self._run_test("{L:Mname/}" ,
                        "['John Doe', 'Jane Smith', 'Max Mustermann']")
@@ -345,6 +369,15 @@ class TestFormatter(unittest.TestCase):
         with self.assertRaises(ValueError):
             self._run_test("{t:Mname", "")
 
+    def test_specifier_identity(self):
+        self._run_test("{a:I}", self.kwdict["a"])
+        self._run_test("{i:I}", self.kwdict["i"])
+        self._run_test("{dt:I}", self.kwdict["dt"])
+
+        self._run_test("{t!D:I}", self.kwdict["dt"])
+        self._run_test("{t!D:I/O+01:30}", self.kwdict["dt"])
+        self._run_test("{i:A+1/I}", self.kwdict["i"]+1)
+
     def test_chain_special(self):
         # multiple replacements
         self._run_test("{a:Rh/C/RE/e/RL/l/}", "Cello wOrld")
diff --git a/test/test_job.py b/test/test_job.py
index 0a533ea..ec86c6c 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -214,6 +214,7 @@ Request interval (default):
     def test_base_category(self):
         extr = TestExtractor.from_url("test:")
         extr.basecategory = "test_basecategory"
+        extr.basesubcategory = "test_basesubcategory"
 
         self.assertEqual(self._capture_stdout(extr), """\
 Category / Subcategory / Basecategory
@@ -376,7 +377,7 @@ class TestExtractor(Extractor):
         root = "https://example.org"
         user = self.user
 
-        yield Message.Directory, {
+        yield Message.Directory, "", {
             "user": user,
             "author": user,
         }
diff --git a/test/test_path.py b/test/test_path.py
new file mode 100644
index 0000000..0639339
--- /dev/null
+++ b/test/test_path.py
@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2025 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from gallery_dl import path, extractor, config  # noqa E402
+
+KWDICT = {
+    "category" : "test",
+    "filename" : "file",
+    "extension": "ext",
+    "name"     : "test-テスト-'&>-/:~",
+    "ext"      : "txt",
+    "foo"      : "bar",
+    "id"       : 123,
+}
+
+
+class TestPath(unittest.TestCase):
+
+    def _pfmt(self, data={}, kwdict=False, extr=extractor.find("noop")):
+        pathfmt = path.PathFormat(extr)
+
+        if kwdict:
+            pathfmt.set_directory({
+                **(kwdict if isinstance(kwdict, dict) else KWDICT),
+                **data,
+            })
+
+        return pathfmt
+
+    def setUp(self):
+        config.clear()
+        path.WINDOWS = False
+
+
+class TestPathObject(TestPath):
+
+    def test_default(self):
+        pfmt = self._pfmt()
+
+        self.assertEqual(pfmt.kwdict, {})
+        self.assertEqual(pfmt.delete, False)
+        self.assertEqual(pfmt.filename, "")
+        self.assertEqual(pfmt.extension, "")
+        self.assertEqual(pfmt.directory, "")
+        self.assertEqual(pfmt.realdirectory, "")
+        self.assertEqual(pfmt.path, "")
+        self.assertEqual(pfmt.realpath, "")
+        self.assertEqual(pfmt.temppath, "")
+        self.assertEqual(pfmt.basedirectory, "./gallery-dl/")
+        self.assertEqual(pfmt.strip, "")
+
+        self.assertIs(pfmt.filename_conditions, None)
+        self.assertIs(pfmt.directory_conditions, None)
+
+        self.assertTrue(callable(pfmt.extension_map))
+        self.assertTrue(callable(pfmt.extension_map))
+        self.assertTrue(callable(pfmt.extension_map))
+        self.assertTrue(callable(pfmt.clean_segment))
+        self.assertTrue(callable(pfmt.clean_path))
+
+        self.assertTrue(callable(pfmt.filename_formatter))
+        for fmt in pfmt.directory_formatters:
+            self.assertTrue(callable(fmt))
+
+    def test_str(self):
+        pfmt = self._pfmt()
+        self.assertEqual(str(pfmt), pfmt.realdirectory)
+        self.assertEqual(str(pfmt), "")
+
+        pfmt = self._pfmt()
+        pfmt.set_directory(KWDICT)
+        pfmt.set_filename(KWDICT)
+        pfmt.build_path()
+        self.assertEqual(str(pfmt), pfmt.realpath)
+        self.assertEqual(str(pfmt), "./gallery-dl/test/file.ext")
+
+
+class TestPathOptions(TestPath):
+
+    def test_option_filename(self):
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname , "file.ext")
+
+        config.set((), "filename", "foo.{foo}")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "foo.bar")
+
+        config.set((), "filename", {
+            "foo == 'baz'": "foo",
+            "id % 2"      : "bar",
+            ""            : "baz",
+        })
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "bar")
+
+    def test_option_directory(self):
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.directory    , "./gallery-dl/test/")
+        self.assertEqual(pfmt.realdirectory, "./gallery-dl/test/")
+
+        config.set((), "directory", ["{foo}", "bar"])
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.directory    , "./gallery-dl/bar/bar/")
+        self.assertEqual(pfmt.realdirectory, "./gallery-dl/bar/bar/")
+
+        config.set((), "directory", {
+            "foo == 'baz'": ["a", "b", "c"],
+            "id % 2"      : ["odd", "{id}"],
+            ""            : ["{foo}", "bar"],
+        })
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.directory    , "./gallery-dl/odd/123/")
+        self.assertEqual(pfmt.realdirectory, "./gallery-dl/odd/123/")
+
+    def test_option_basedirectory(self):
+        config.set((), "base-directory", "{foo}")
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.realdirectory, "{foo}/test/")
+
+        config.set((), "base-directory", {
+            "foo == 'baz'": "bar",
+            "id % 2"      : "./odd",
+            ""            : "./default",
+        })
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.realdirectory, "./odd/test/")
+
+    def test_option_keywordsdefault(self):
+        config.set((), "directory", ["{baz}"])
+        config.set((), "base-directory", "")
+
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.realdirectory, "None/")
+
+        config.set((), "keywords-default", "ND")
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.realdirectory, "ND/")
+
+        config.set((), "keywords-default", "")
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(pfmt.realdirectory, "")
+
+    def test_option_extensionmap_default(self):
+        kwdict = KWDICT.copy()
+        pfmt = self._pfmt()
+        pfmt.set_filename(kwdict)
+        self.assertEqual(pfmt.extension, "ext")
+
+        pfmt.set_extension("jpg")
+        self.assertEqual(pfmt.extension, "jpg")
+        self.assertEqual(kwdict["extension"], "jpg")
+
+        pfmt.set_extension("png")
+        self.assertEqual(pfmt.extension, "png")
+        self.assertEqual(kwdict["extension"], "png")
+
+        pfmt.set_extension("jpeg")
+        self.assertEqual(pfmt.extension, "jpg")
+        self.assertEqual(kwdict["extension"], "jpg")
+
+        for ext, repl in path.EXTENSION_MAP.items():
+            pfmt.set_extension(ext)
+            self.assertEqual(pfmt.extension, repl)
+            self.assertEqual(kwdict["extension"], repl)
+
+    def test_option_extensionmap_custom(self):
+        extmap = {
+            "bitmap": "bmp",
+            "ping"  : "png",
+            "jiff"  : "gif",
+        }
+        config.set((), "extension-map", extmap)
+
+        kwdict = KWDICT.copy()
+        pfmt = self._pfmt()
+        pfmt.set_filename(kwdict)
+
+        pfmt.set_extension("jpg")
+        self.assertEqual(pfmt.extension, "jpg")
+        self.assertEqual(kwdict["extension"], "jpg")
+
+        pfmt.set_extension("ping")
+        self.assertEqual(pfmt.extension, "png")
+        self.assertEqual(kwdict["extension"], "png")
+
+        for ext, repl in extmap.items():
+            pfmt.set_extension(ext)
+            self.assertEqual(pfmt.extension, repl)
+            self.assertEqual(kwdict["extension"], repl)
+
+        for ext, repl in path.EXTENSION_MAP.items():
+            pfmt.set_extension(ext)
+            self.assertNotEqual(pfmt.extension, repl)
+            self.assertNotEqual(kwdict["extension"], repl)
+
+    def test_option_pathrestrict(self):
+        config.set((), "filename", "{name}.{ext}")
+
+        config.set((), "path-restrict", "unix")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "test-テスト-'&>-_:~.txt", "unix")
+
+        config.set((), "path-restrict", "windows")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "test-テスト-'&_-__~.txt", "windows")
+
+        config.set((), "path-restrict", "ascii")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "test____________.txt", "ascii")
+
+        config.set((), "path-restrict", "ascii+")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "test-___-'&_-__~.txt", "ascii+")
+
+    def test_option_pathrestrict_custom(self):
+        config.set((), "filename", "{name}.{ext}")
+
+        config.set((), "path-restrict", "ts-")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "_e___テスト_'&>_/:~._x_", "custom str")
+
+        config.set((), "path-restrict", {
+            "t": "A",
+            "s": "B",
+            "-": "###",
+            "/": "|"
+        })
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "AeBA###テスト###'&>###|:~.AxA", "custom dict")
+
+        config.set((), "path-restrict", {
+            "a-z": "x",
+            "テ": "te",
+            "ス": "su",
+            "ト": "to",
+        })
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "xxxx-tesuto-'&>-/:~.xxx", "custom dict range")
+
+    def test_option_pathreplace(self):
+        config.set((), "filename", "{name}.{ext}")
+
+        config.set((), "path-restrict", "unix")
+        config.set((), "path-replace", "&")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "test-テスト-'&>-&:~.txt", "&")
+
+        config.set((), "path-restrict", "windows")
+        config.set((), "path-replace", "***")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "test-テスト-'&***-******~.txt", "***")
+
+    def test_option_pathremove(self):
+        config.set((), "filename", "{name}.{ext}")
+
+        config.set((), "path-remove", "-&/")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "testテスト'>_:~.txt")
+
+        config.set((), "path-remove", "a-z0-9")
+        fname = self._pfmt().build_filename(KWDICT)
+        self.assertEqual(fname, "-テスト-'&>-_:~.")
+
+    def test_option_pathstrip(self):
+        config.set((), "directory", [" . {name}.{ext} . "])
+        config.set((), "base-directory", "")
+        config.set((), "path-restrict", "unix")
+
+        config.set((), "path-strip", "unix")
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(
+            pfmt.realdirectory, ". test-テスト-'&>-_:~.txt ./", "unix")
+
+        config.set((), "path-strip", "windows")
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(
+            pfmt.realdirectory, ". test-テスト-'&>-_:~.txt/", "windows")
+
+        config.set((), "path-strip", "txt")
+        pfmt = self._pfmt(kwdict=True)
+        self.assertEqual(
+            pfmt.realdirectory, ". test-テスト-'&>-_:~.txt ./", "custom")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py
index 5d52e1d..e4d01c2 100644
--- a/test/test_postprocessor.py
+++ b/test/test_postprocessor.py
@@ -78,6 +78,7 @@ class BasePostprocessorTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.dir = tempfile.TemporaryDirectory()
+        config.clear()
         config.set((), "base-directory", cls.dir.name)
         cls.job = FakeJob()
 
@@ -374,6 +375,40 @@ class ExecTest(BasePostprocessorTest):
         m_aa.assert_called_once_with(self.pathfmt.kwdict)
         m_ac.assert_called_once()
 
+    def test_verbose_string(self):
+        self._create({
+            "command": "echo foo bar",
+            "verbose": False,
+        })
+
+        with patch("gallery_dl.util.Popen") as p, \
+                self.assertLogs(level=10) as log_info:
+            i = Mock()
+            i.wait.return_value = 123
+            p.return_value = i
+            self._trigger(("after",))
+
+        msg = "DEBUG:postprocessor.exec:Running 'echo'"
+        self.assertEqual(log_info.output[0], msg)
+        self.assertIn("'echo' returned with non-zero ", log_info.output[1])
+
+    def test_verbose_list(self):
+        self._create({
+            "command": ["echo", "foo", "bar"],
+            "verbose": False,
+        })
+
+        with patch("gallery_dl.util.Popen") as p, \
+                self.assertLogs(level=10) as log_info:
+            i = Mock()
+            i.wait.return_value = 123
+            p.return_value = i
+            self._trigger(("after",))
+
+        msg = "DEBUG:postprocessor.exec:Running 'echo'"
+        self.assertEqual(log_info.output[0], msg)
+        self.assertIn("'echo' returned with non-zero ", log_info.output[1])
+
 
 class HashTest(BasePostprocessorTest):
 
@@ -453,7 +488,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realpath}.JSON"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
         self.assertEqual(self._output(m), """{
     "category": "test",
@@ -473,6 +508,7 @@ class MetadataTest(BasePostprocessorTest):
             "indent"    : None,
             "open"      : "a",
             "encoding"  : "UTF-8",
+            "newline"   : "\r\n",
             "extension" : "JSON",
         }, {
             "public"    : "hello ワールド",
@@ -487,7 +523,9 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realpath}.JSON"
-        m.assert_called_once_with(path, "a", encoding="UTF-8")
+        m.assert_called_once_with(path, "a", encoding="UTF-8", newline='\r\n')
+        # Since we mocked the call to open,
+        # we don't actually see the effect of setting newline.
         self.assertEqual(self._output(m), """{\
 "_private" : "foo \\u30d0\\u30fc",\
 "category" : "test",\
@@ -508,7 +546,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realpath}.txt"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
         self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
 
     def test_metadata_tags_split_1(self):
@@ -599,7 +637,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realdirectory}file.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_extfmt_2(self):
         self._create({
@@ -611,7 +649,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realdirectory}file.2.EXT-data:tESt"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_directory(self):
         self._create({
@@ -622,7 +660,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realdirectory}metadata/file.ext.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_directory_2(self):
         self._create({
@@ -634,7 +672,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realdirectory}metadata/file.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_directory_format(self):
         self._create(
@@ -646,7 +684,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realdirectory}../json/12500/file.ext.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_directory_empty(self):
         self._create(
@@ -657,7 +695,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realdirectory}./file.ext.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_basedirectory(self):
         self._create({"base-directory": True})
@@ -666,7 +704,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.basedirectory}file.ext.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_basedirectory_custom(self):
         self._create({
@@ -678,7 +716,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = "/home/test/meta/file.ext.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_filename(self):
         self._create({
@@ -690,7 +728,7 @@ class MetadataTest(BasePostprocessorTest):
             self._trigger()
 
         path = f"{self.pathfmt.realdirectory}test_file__meta_.data"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_meta_path(self):
         self._create({
@@ -790,7 +828,7 @@ class MetadataTest(BasePostprocessorTest):
         self.assertGreater(len(self._output(m)), 0)
 
         path = f"{self.pathfmt.realdirectory}file.ext.json"
-        m.assert_called_once_with(path, "w", encoding="utf-8")
+        m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
 
     def test_metadata_option_skip_false(self):
         self._create({"skip": False})
@@ -802,6 +840,28 @@ class MetadataTest(BasePostprocessorTest):
         self.assertTrue(not e.called)
         self.assertTrue(m.called)
 
+    def test_metadata_option_newline(self):
+        self._create({
+            "newline": "\r\n",
+            "filename"      : "data.json",
+            "directory"     : "",
+            "base-directory": self.dir.name,
+        })
+
+        self._trigger()
+
+        path = os.path.join(self.dir.name, "data.json")
+        with open(path, newline="") as fp:
+            content = fp.read()
+
+        self.assertEqual(content, """\
+{\r\n\
+    "category": "test",\r\n\
+    "filename": "file",\r\n\
+    "extension": "ext"\r\n\
+}\r\n\
+""")
+
     def test_metadata_option_include(self):
         self._create(
             {"include": ["_private", "filename", "foo"], "sort": True},
diff --git a/test/test_results.py b/test/test_results.py
index e7fcabf..0865686 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -310,10 +310,13 @@ class TestExtractorResults(unittest.TestCase):
         elif isinstance(test, range):
             self.assertRange(value, test, msg=path)
         elif isinstance(test, set):
-            try:
-                self.assertIn(value, test, msg=path)
-            except AssertionError:
-                self.assertIn(type(value), test, msg=path)
+            for item in test:
+                if isinstance(item, type) and isinstance(value, item) or \
+                        value == item:
+                    break
+            else:
+                v = type(value) if len(str(value)) > 64 else value
+                self.fail(f"{v!r} not in {test}: {path}")
         elif isinstance(test, list):
             subtest = False
             for idx, item in enumerate(test):
@@ -423,8 +426,7 @@ class ResultJob(job.DownloadJob):
 
     def run(self):
         self._init()
-        for msg in self.extractor:
-            self.dispatch(msg)
+        self.dispatch(self.extractor)
 
     def handle_url(self, url, kwdict, fallback=None):
         self._update_url(url)
diff --git a/test/test_text.py b/test/test_text.py
index 0e187d7..eac7906 100644
--- a/test/test_text.py
+++ b/test/test_text.py
@@ -11,8 +11,6 @@ import os
 import sys
 import unittest
 
-import datetime
-
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from gallery_dl import text, util  # noqa E402
 
@@ -203,6 +201,10 @@ class TestText(unittest.TestCase):
         self.assertEqual(f("http://example.org/v2/filename.ext"), result)
         self.assertEqual(
             f("http://example.org/v2/filename.ext?param=value#frag"), result)
+        self.assertEqual(
+            f("http://example.org/v2/foo%202?bar&<>.ext?param=value#frag"),
+            {"filename": "foo 2", "extension": ""},
+        )
 
         # long "extension"
         fn = "httpswww.example.orgpath-path-path-path-path-path-path-path"
@@ -212,6 +214,24 @@ class TestText(unittest.TestCase):
         for value in INVALID:
             self.assertEqual(f(value), empty)
 
+    def test_nameext_from_name(self, f=text.nameext_from_name):
+        self.assertEqual(
+            f(""),
+            {"filename": "", "extension": ""},
+        )
+        self.assertEqual(
+            f("filename.ext"),
+            {"filename": "filename", "extension": "ext"},
+        )
+        self.assertEqual(
+            f("foo%202?bar&<>.ext"),
+            {"filename": "foo%202?bar&<>", "extension": "ext"},
+        )
+
+        # long "extension"
+        fn = "httpswww.example.orgpath-path-path-path-path-path-path-path"
+        self.assertEqual(f(fn), {"filename": fn, "extension": ""})
+
     def test_extract(self, f=text.extract):
         txt = "<a><b>"
         self.assertEqual(f(txt, "<", ">"), ("a" , 3))
@@ -519,51 +539,6 @@ class TestText(unittest.TestCase):
         self.assertEqual(f({"ä&": "あと", "#": "?"}),
                          "%C3%A4%26=%E3%81%82%E3%81%A8&%23=%3F")
 
-    def test_parse_timestamp(self, f=text.parse_timestamp):
-        null = util.datetime_utcfromtimestamp(0)
-        value = util.datetime_utcfromtimestamp(1555816235)
-
-        self.assertEqual(f(0)           , null)
-        self.assertEqual(f("0")         , null)
-        self.assertEqual(f(1555816235)  , value)
-        self.assertEqual(f("1555816235"), value)
-
-        for value in INVALID_ALT:
-            self.assertEqual(f(value), None)
-            self.assertEqual(f(value, "foo"), "foo")
-
-    def test_parse_datetime(self, f=text.parse_datetime):
-        null = util.datetime_utcfromtimestamp(0)
-
-        self.assertEqual(f("1970-01-01T00:00:00+00:00"), null)
-        self.assertEqual(f("1970-01-01T00:00:00+0000") , null)
-        self.assertEqual(f("1970.01.01", "%Y.%m.%d")   , null)
-
-        self.assertEqual(
-            f("2019-05-07T21:25:02+09:00"),
-            datetime.datetime(2019, 5, 7, 12, 25, 2),
-        )
-        self.assertEqual(
-            f("2019-05-07T21:25:02+0900"),
-            datetime.datetime(2019, 5, 7, 12, 25, 2),
-        )
-        self.assertEqual(
-            f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"),
-            datetime.datetime(2019, 5, 7, 12, 25, 2),
-        )
-        self.assertEqual(
-            f("2019-05-07T21:25:02", "%Y-%m-%dT%H:%M:%S", utcoffset=9),
-            datetime.datetime(2019, 5, 7, 12, 25, 2),
-        )
-        self.assertEqual(
-            f("2019-05-07 21:25:02"),
-            "2019-05-07 21:25:02",
-        )
-
-        for value in INVALID:
-            self.assertEqual(f(value), None)
-        self.assertEqual(f("1970.01.01"), "1970.01.01")
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/test_util.py b/test/test_util.py
index bfaab01..6784874 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -40,6 +40,7 @@ class TestRange(unittest.TestCase):
     def test_parse_digit(self):
         f = self.predicate._parse
 
+        self.assertEqual(f(2), [range(2, 3)])
         self.assertEqual(f("2"), [range(2, 3)])
 
         self.assertEqual(
@@ -48,6 +49,12 @@ class TestRange(unittest.TestCase):
              range(3, 4),
              range(4, 5)],
         )
+        self.assertEqual(
+            f(["2", "3", "4"]),
+            [range(2, 3),
+             range(3, 4),
+             range(4, 5)],
+        )
 
     def test_parse_range(self):
         f = self.predicate._parse
@@ -406,89 +413,6 @@ def hash(value):
         self.assertEqual(expr(value), result)
 
 
-class TestDatetime(unittest.TestCase):
-
-    def test_to_datetime(self, f=util.to_datetime):
-
-        def _assert(value, expected):
-            result = f(value)
-            self.assertIsInstance(result, datetime.datetime)
-            self.assertEqual(result, expected, msg=repr(value))
-
-        dt = datetime.datetime(2010, 1, 1)
-        self.assertIs(f(dt), dt)
-
-        _assert(dt            , dt)
-        _assert(1262304000    , dt)
-        _assert(1262304000.0  , dt)
-        _assert(1262304000.123, dt)
-        _assert("1262304000"  , dt)
-
-        _assert("2010-01-01"                      , dt)
-        _assert("2010-01-01 00:00:00"             , dt)
-        _assert("2010-01-01T00:00:00"             , dt)
-        _assert("2010-01-01T00:00:00.123456"      , dt)
-        _assert("2009-12-31T19:00:00-05:00"       , dt)
-        _assert("2009-12-31T19:00:00.123456-05:00", dt)
-        _assert("2010-01-01T00:00:00Z"            , dt)
-        _assert("2010-01-01T00:00:00.123456Z"     , dt)
-
-        _assert(0    , util.EPOCH)
-        _assert(""   , util.EPOCH)
-        _assert("foo", util.EPOCH)
-        _assert(None , util.EPOCH)
-        _assert(()   , util.EPOCH)
-        _assert([]   , util.EPOCH)
-        _assert({}   , util.EPOCH)
-        _assert((1, 2, 3), util.EPOCH)
-
-    @unittest.skipIf(sys.hexversion < 0x30b0000,
-                     "extended fromisoformat timezones")
-    def test_to_datetime_tz(self, f=util.to_datetime):
-
-        def _assert(value, expected):
-            result = f(value)
-            self.assertIsInstance(result, datetime.datetime)
-            self.assertEqual(result, expected, msg=repr(value))
-
-        dt = datetime.datetime(2010, 1, 1)
-
-        _assert("2009-12-31T19:00:00-05"          , dt)
-        _assert("2009-12-31T19:00:00-0500"        , dt)
-        _assert("2009-12-31T19:00:00.123456-05"   , dt)
-        _assert("2009-12-31T19:00:00.123456-0500" , dt)
-
-    def test_datetime_to_timestamp(self, f=util.datetime_to_timestamp):
-        self.assertEqual(f(util.EPOCH), 0.0)
-        self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0)
-        self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)),
-                         1262304000.128000)
-        with self.assertRaises(TypeError):
-            f(None)
-
-    def test_datetime_to_timestamp_string(
-            self, f=util.datetime_to_timestamp_string):
-        self.assertEqual(f(util.EPOCH), "0")
-        self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000")
-        self.assertEqual(f(None), "")
-
-    def test_datetime_from_timestamp(
-            self, f=util.datetime_from_timestamp):
-        self.assertEqual(f(0.0), util.EPOCH)
-        self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1))
-        self.assertEqual(f(1262304000.128000).replace(microsecond=0),
-                         datetime.datetime(2010, 1, 1, 0, 0, 0))
-
-    def test_datetime_utcfromtimestamp(
-            self, f=util.datetime_utcfromtimestamp):
-        self.assertEqual(f(0.0), util.EPOCH)
-        self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1))
-
-    def test_datetime_utcnow(
-            self, f=util.datetime_utcnow):
-        self.assertIsInstance(f(), datetime.datetime)
-
-
 class TestOther(unittest.TestCase):
 
     def test_bencode(self):