From a24ec1647aeac35a63b744ea856011ad6e06be3b Mon Sep 17 00:00:00 2001 From: Unit 193 Date: Sat, 20 Dec 2025 05:49:04 -0500 Subject: New upstream version 1.31.1. --- test/test_downloader.py | 15 ++- test/test_dt.py | 167 +++++++++++++++++++++++++ test/test_extractor.py | 86 +++++++++++-- test/test_formatter.py | 41 ++++++- test/test_job.py | 3 +- test/test_path.py | 297 +++++++++++++++++++++++++++++++++++++++++++++ test/test_postprocessor.py | 86 +++++++++++-- test/test_results.py | 14 ++- test/test_text.py | 69 ++++------- test/test_util.py | 90 ++------------ 10 files changed, 705 insertions(+), 163 deletions(-) create mode 100644 test/test_dt.py create mode 100644 test/test_path.py (limited to 'test') diff --git a/test/test_downloader.py b/test/test_downloader.py index f6c3dbe..fb442c4 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -298,6 +298,15 @@ class TestHTTPDownloader(TestDownloaderBase): self.assertTrue(success) self.assertEqual(pathfmt.temppath, "") + def test_http_empty(self): + url = f"{self.address}/~NUL" + pathfmt = self._prepare_destination(None, extension=None) + with self.assertLogs(self.downloader.log, "WARNING") as log_info: + success = self.downloader.download(url, pathfmt) + self.assertFalse(success) + self.assertEqual(log_info.output[0], + "WARNING:downloader.http:Empty file") + class TestTextDownloader(TestDownloaderBase): @@ -400,6 +409,7 @@ SAMPLES = { ("blend", b"BLENDER-v303RENDH"), ("obj" , b"# Blender v3.2.0 OBJ File: 'foo.blend'"), ("clip", b"CSFCHUNK\x00\x00\x00\x00"), + ("~NUL", b""), } @@ -428,8 +438,9 @@ def generate_tests(): return test for idx, (ext, content) in enumerate(SAMPLES): - test = generate_test(idx, ext, content) - setattr(TestHTTPDownloader, test.__name__, test) + if ext[0].isalnum(): + test = generate_test(idx, ext, content) + setattr(TestHTTPDownloader, test.__name__, test) generate_tests() diff --git a/test/test_dt.py b/test/test_dt.py new file mode 100644 index 0000000..02e3ac2 --- /dev/null +++ b/test/test_dt.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import sys +import unittest + +import datetime + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from gallery_dl import dt # noqa E402 + + +class TestDatetime(unittest.TestCase): + + def test_convert(self, f=dt.convert): + + def _assert(value, expected): + result = f(value) + self.assertIsInstance(result, datetime.datetime) + self.assertEqual(result, expected, msg=repr(value)) + + d = datetime.datetime(2010, 1, 1) + self.assertIs(f(d), d) + + _assert(d , d) + _assert(1262304000 , d) + _assert(1262304000.0 , d) + _assert(1262304000.123, d) + _assert("1262304000" , d) + + _assert("2010-01-01" , d) + _assert("2010-01-01 00:00:00" , d) + _assert("2010-01-01T00:00:00" , d) + _assert("2010-01-01T00:00:00.123456" , d) + _assert("2009-12-31T19:00:00-05:00" , d) + _assert("2009-12-31T19:00:00.123456-05:00", d) + _assert("2010-01-01T00:00:00Z" , d) + _assert("2010-01-01T00:00:00.123456Z" , d) + _assert("2009-12-31T19:00:00-0500" , d) + _assert("2009-12-31T19:00:00.123456-0500" , d) + + _assert(0 , dt.NONE) + _assert("" , dt.NONE) + _assert("foo", dt.NONE) + _assert(None , dt.NONE) + _assert(() , dt.NONE) + _assert([] , dt.NONE) + _assert({} , dt.NONE) + _assert((1, 2, 3), dt.NONE) + + @unittest.skipIf(sys.hexversion < 0x30b0000, + "extended fromisoformat timezones") + def test_convert_tz(self, f=dt.convert): + + def _assert(value, expected): + result = f(value) + self.assertIsInstance(result, datetime.datetime) + self.assertEqual(result, expected, msg=repr(value)) + + d = datetime.datetime(2010, 1, 1) + _assert("2009-12-31T19:00:00-05" , d) + _assert("2009-12-31T19:00:00.123456-05" , d) + + def test_to_timestamp(self, f=dt.to_ts): + self.assertEqual(f(dt.EPOCH), 0.0) + self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) + self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), + 1262304000.128000) + with self.assertRaises(TypeError): + f(None) + + def test_to_timestamp_string(self, f=dt.to_ts_string): + self.assertEqual(f(dt.EPOCH), "0") + self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") + self.assertEqual(f(None), "") + + def test_from_timestamp(self, f=dt.from_ts): + self.assertEqual(f(0.0), dt.EPOCH) + self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) + self.assertEqual(f(1262304000.128000).replace(microsecond=0), + datetime.datetime(2010, 1, 1, 0, 0, 0)) + + def test_now(self, f=dt.now): + self.assertIsInstance(f(), datetime.datetime) + + def test_parse_timestamp(self, f=dt.parse_ts): + null = dt.from_ts(0) + value = dt.from_ts(1555816235) + + self.assertEqual(f(0) , null) + self.assertEqual(f("0") , null) + self.assertEqual(f(1555816235) , value) + self.assertEqual(f("1555816235"), value) + + for value in ((), [], {}, None, ""): + self.assertEqual(f(value), dt.NONE) + self.assertEqual(f(value, "foo"), "foo") + + def test_parse(self, f=dt.parse): + self.assertEqual( + f("1970.01.01", "%Y.%m.%d"), + dt.EPOCH, + ) + self.assertEqual( + f("May 7, 2019 9:33 am", "%B %d, %Y %I:%M %p"), + datetime.datetime(2019, 5, 7, 9, 33, 0), + ) + self.assertEqual( + f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + + for value in ((), [], {}, None, 1, 2.3): + self.assertEqual(f(value, "%Y"), dt.NONE) + + def test_parse_iso(self, f=dt.parse_iso): + self.assertEqual( + f("1970-01-01T00:00:00+00:00"), + dt.from_ts(0), + ) + self.assertEqual( + f("2019-05-07T21:25:02+09:00"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + self.assertEqual( + f("2019-05-07T12:25:02Z"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + self.assertEqual( + f("2019-05-07 21:25:02"), + datetime.datetime(2019, 5, 7, 21, 25, 2), + ) + self.assertEqual( + f("1970-01-01"), + dt.EPOCH, + ) + self.assertEqual( + f("1970.01.01"), + dt.NONE, + ) + self.assertEqual( + f("1970-01-01T00:00:00+0000"), + dt.EPOCH, + ) + self.assertEqual( + f("2019-05-07T21:25:02.753+0900"), + datetime.datetime(2019, 5, 7, 12, 25, 2), + ) + + for value in ((), [], {}, None, 1, 2.3): + self.assertEqual(f(value), dt.NONE) + + def test_none(self): + self.assertFalse(dt.NONE) + self.assertIsInstance(dt.NONE, dt.datetime) + self.assertEqual(str(dt.NONE), "[Invalid DateTime]") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_extractor.py b/test/test_extractor.py index a623e1d..c06b890 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -14,10 +14,9 @@ from unittest.mock import patch import time import string -from datetime import datetime, timedelta sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, util # noqa E402 +from gallery_dl import extractor, util, dt, config # noqa E402 from gallery_dl.extractor import mastodon # noqa E402 from gallery_dl.extractor.common import Extractor, Message # noqa E402 from gallery_dl.extractor.directlink import DirectlinkExtractor # noqa E402 @@ -40,7 +39,7 @@ class FakeExtractor(Extractor): pattern = "fake:" def items(self): - yield Message.Version, 1 + yield Message.Noop yield Message.Url, "text:foobar", {} @@ -233,8 +232,8 @@ class TestExtractorWait(unittest.TestCase): def test_wait_until_datetime(self): extr = extractor.find("generic:https://example.org/") - until = util.datetime_utcnow() + timedelta(seconds=5) - until_local = datetime.now() + timedelta(seconds=5) + until = dt.now() + dt.timedelta(seconds=5) + until_local = dt.datetime.now() + dt.timedelta(seconds=5) if not until.microsecond: until = until.replace(microsecond=until_local.microsecond) @@ -251,8 +250,8 @@ class TestExtractorWait(unittest.TestCase): self._assert_isotime(calls[0][1][1], until_local) def _assert_isotime(self, output, until): - if not isinstance(until, datetime): - until = datetime.fromtimestamp(until) + if not isinstance(until, dt.datetime): + until = dt.datetime.fromtimestamp(until) o = self._isotime_to_seconds(output) u = self._isotime_to_seconds(until.time().isoformat()[:8]) self.assertLessEqual(o-u, 1.0) @@ -262,6 +261,79 @@ class TestExtractorWait(unittest.TestCase): return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) +class TextExtractorCommonDateminmax(unittest.TestCase): + + def setUp(self): + config.clear() + + tearDown = setUp + + def test_date_min_max_default(self): + extr = extractor.find("generic:https://example.org/") + + dmin, dmax = extr._get_date_min_max() + self.assertEqual(dmin, None) + self.assertEqual(dmax, None) + + dmin, dmax = extr._get_date_min_max(..., -1) + self.assertEqual(dmin, ...) + self.assertEqual(dmax, -1) + + def test_date_min_max_timestamp(self): + extr = extractor.find("generic:https://example.org/") + config.set((), "date-min", 1262304000) + config.set((), "date-max", 1262304000.123) + + dmin, dmax = extr._get_date_min_max() + self.assertEqual(dmin, 1262304000) + self.assertEqual(dmax, 1262304000.123) + + def test_date_min_max_iso(self): + extr = extractor.find("generic:https://example.org/") + config.set((), "date-min", "2010-01-01") + config.set((), "date-max", "2010-01-01T00:01:03") + + dmin, dmax = extr._get_date_min_max() + self.assertEqual(dmin, 1262304000) + self.assertEqual(dmax, 1262304063) + + def test_date_min_max_iso_invalid(self): + extr = extractor.find("generic:https://example.org/") + config.set((), "date-min", "2010-01-01") + config.set((), "date-max", "2010-01") + + with self.assertLogs() as log_info: + dmin, dmax = extr._get_date_min_max() + self.assertEqual(dmin, 1262304000) + self.assertEqual(dmax, None) + + self.assertEqual(len(log_info.output), 1) + self.assertEqual( + log_info.output[0], + "WARNING:generic:Unable to parse 'date-max': " + "Invalid isoformat string '2010-01'") + + def test_date_min_max_fmt(self): + extr = extractor.find("generic:https://example.org/") + config.set((), "date-format", "%B %d %Y") + config.set((), "date-min", "January 01 2010") + config.set((), "date-max", "August 18 2022") + + dmin, dmax = extr._get_date_min_max() + self.assertEqual(dmin, 1262304000) + self.assertEqual(dmax, 1660780800) + + def test_date_min_max_mix(self): + extr = extractor.find("generic:https://example.org/") + config.set((), "date-format", "%B %d %Y") + config.set((), "date-min", "January 01 2010") + config.set((), "date-max", 1262304061) + + dmin, dmax = extr._get_date_min_max() + self.assertEqual(dmin, 1262304000) + self.assertEqual(dmax, 1262304061) + + class TextExtractorOAuth(unittest.TestCase): def test_oauth1(self): diff --git a/test/test_formatter.py b/test/test_formatter.py index 01e3a88..67df279 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -15,7 +15,7 @@ import datetime import tempfile sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import formatter, text, util, config # noqa E402 +from gallery_dl import formatter, text, dt, util, config # noqa E402 try: import jinja2 @@ -154,7 +154,7 @@ class TestFormatter(unittest.TestCase): self._run_test("{t}" , self.kwdict["t"] , None, int) self._run_test("{t}" , self.kwdict["t"] , None, util.identity) self._run_test("{dt}", self.kwdict["dt"], None, util.identity) - self._run_test("{ds}", self.kwdict["dt"], None, text.parse_datetime) + self._run_test("{ds}", self.kwdict["dt"], None, dt.parse_iso) self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"], None, util.identity) @@ -248,6 +248,19 @@ class TestFormatter(unittest.TestCase): self._run_test("{a:L50/foo/>51}", "foo") self._run_test("{a:Lab/foo/}", "foo") + def test_specifier_maxlen_bytes(self): + v = self.kwdict["a"] + self._run_test("{a:Lb5/foo/}" , "foo") + self._run_test("{a:Lb50/foo/}", v) + self._run_test("{a:Lb50/foo/>50}", " " * 39 + v) + self._run_test("{a:Lb50/foo/>51}", "foo") + self._run_test("{a:Lbab/foo/}", "foo") + + v = self.kwdict["j"] + self._run_test("{j:Lb5/foo/}" , "foo") + self._run_test("{j:Lb50/foo/}", v) + self._run_test("{j:Lbab/foo/}", "foo") + def test_specifier_join(self): self._run_test("{l:J}" , "abc") self._run_test("{l:J,}" , "a,b,c") @@ -271,8 +284,8 @@ class TestFormatter(unittest.TestCase): def test_specifier_datetime(self): self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", "2010-01-01 00:00:00") - self._run_test("{ds:D%Y}", "2010-01-01T01:00:00+01:00") - self._run_test("{l:D%Y}", "None") + self._run_test("{ds:D%Y}", "[Invalid DateTime]") + self._run_test("{l2:D%Y}", "[Invalid DateTime]") def test_specifier_offset(self): self._run_test("{dt:O 01:00}", "2010-01-01 01:00:00") @@ -332,6 +345,17 @@ class TestFormatter(unittest.TestCase): with self.assertRaises(ValueError): self._run_test("{a:Xfoo/ */}", "hello wo *") + def test_specifier_limit_bytes(self): + self._run_test("{a:Xb20/ */}", "hElLo wOrLd") + self._run_test("{a:Xb10/ */}", "hElLo wO *") + + self._run_test("{j:Xb50/〜/}", "げんそうきょう") + self._run_test("{j:Xb20/〜/}", "げんそうき〜") + self._run_test("{j:Xb20/ */}", "げんそうきょ *") + + with self.assertRaises(ValueError): + self._run_test("{a:Xbfoo/ */}", "hello wo *") + def test_specifier_map(self): self._run_test("{L:Mname/}" , "['John Doe', 'Jane Smith', 'Max Mustermann']") @@ -345,6 +369,15 @@ class TestFormatter(unittest.TestCase): with self.assertRaises(ValueError): self._run_test("{t:Mname", "") + def test_specifier_identity(self): + self._run_test("{a:I}", self.kwdict["a"]) + self._run_test("{i:I}", self.kwdict["i"]) + self._run_test("{dt:I}", self.kwdict["dt"]) + + self._run_test("{t!D:I}", self.kwdict["dt"]) + self._run_test("{t!D:I/O+01:30}", self.kwdict["dt"]) + self._run_test("{i:A+1/I}", self.kwdict["i"]+1) + def test_chain_special(self): # multiple replacements self._run_test("{a:Rh/C/RE/e/RL/l/}", "Cello wOrld") diff --git a/test/test_job.py b/test/test_job.py index 0a533ea..ec86c6c 100644 --- a/test/test_job.py +++ b/test/test_job.py @@ -214,6 +214,7 @@ Request interval (default): def test_base_category(self): extr = TestExtractor.from_url("test:") extr.basecategory = "test_basecategory" + extr.basesubcategory = "test_basesubcategory" self.assertEqual(self._capture_stdout(extr), """\ Category / Subcategory / Basecategory @@ -376,7 +377,7 @@ class TestExtractor(Extractor): root = "https://example.org" user = self.user - yield Message.Directory, { + yield Message.Directory, "", { "user": user, "author": user, } diff --git a/test/test_path.py b/test/test_path.py new file mode 100644 index 0000000..0639339 --- /dev/null +++ b/test/test_path.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2025 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from gallery_dl import path, extractor, config # noqa E402 + +KWDICT = { + "category" : "test", + "filename" : "file", + "extension": "ext", + "name" : "test-テスト-'&>-/:~", + "ext" : "txt", + "foo" : "bar", + "id" : 123, +} + + +class TestPath(unittest.TestCase): + + def _pfmt(self, data={}, kwdict=False, extr=extractor.find("noop")): + pathfmt = path.PathFormat(extr) + + if kwdict: + pathfmt.set_directory({ + **(kwdict if isinstance(kwdict, dict) else KWDICT), + **data, + }) + + return pathfmt + + def setUp(self): + config.clear() + path.WINDOWS = False + + +class TestPathObject(TestPath): + + def test_default(self): + pfmt = self._pfmt() + + self.assertEqual(pfmt.kwdict, {}) + self.assertEqual(pfmt.delete, False) + self.assertEqual(pfmt.filename, "") + self.assertEqual(pfmt.extension, "") + self.assertEqual(pfmt.directory, "") + self.assertEqual(pfmt.realdirectory, "") + self.assertEqual(pfmt.path, "") + self.assertEqual(pfmt.realpath, "") + self.assertEqual(pfmt.temppath, "") + self.assertEqual(pfmt.basedirectory, "./gallery-dl/") + self.assertEqual(pfmt.strip, "") + + self.assertIs(pfmt.filename_conditions, None) + self.assertIs(pfmt.directory_conditions, None) + + self.assertTrue(callable(pfmt.extension_map)) + self.assertTrue(callable(pfmt.extension_map)) + self.assertTrue(callable(pfmt.extension_map)) + self.assertTrue(callable(pfmt.clean_segment)) + self.assertTrue(callable(pfmt.clean_path)) + + self.assertTrue(callable(pfmt.filename_formatter)) + for fmt in pfmt.directory_formatters: + self.assertTrue(callable(fmt)) + + def test_str(self): + pfmt = self._pfmt() + self.assertEqual(str(pfmt), pfmt.realdirectory) + self.assertEqual(str(pfmt), "") + + pfmt = self._pfmt() + pfmt.set_directory(KWDICT) + pfmt.set_filename(KWDICT) + pfmt.build_path() + self.assertEqual(str(pfmt), pfmt.realpath) + self.assertEqual(str(pfmt), "./gallery-dl/test/file.ext") + + +class TestPathOptions(TestPath): + + def test_option_filename(self): + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname , "file.ext") + + config.set((), "filename", "foo.{foo}") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "foo.bar") + + config.set((), "filename", { + "foo == 'baz'": "foo", + "id % 2" : "bar", + "" : "baz", + }) + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "bar") + + def test_option_directory(self): + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.directory , "./gallery-dl/test/") + self.assertEqual(pfmt.realdirectory, "./gallery-dl/test/") + + config.set((), "directory", ["{foo}", "bar"]) + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.directory , "./gallery-dl/bar/bar/") + self.assertEqual(pfmt.realdirectory, "./gallery-dl/bar/bar/") + + config.set((), "directory", { + "foo == 'baz'": ["a", "b", "c"], + "id % 2" : ["odd", "{id}"], + "" : ["{foo}", "bar"], + }) + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.directory , "./gallery-dl/odd/123/") + self.assertEqual(pfmt.realdirectory, "./gallery-dl/odd/123/") + + def test_option_basedirectory(self): + config.set((), "base-directory", "{foo}") + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.realdirectory, "{foo}/test/") + + config.set((), "base-directory", { + "foo == 'baz'": "bar", + "id % 2" : "./odd", + "" : "./default", + }) + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.realdirectory, "./odd/test/") + + def test_option_keywordsdefault(self): + config.set((), "directory", ["{baz}"]) + config.set((), "base-directory", "") + + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.realdirectory, "None/") + + config.set((), "keywords-default", "ND") + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.realdirectory, "ND/") + + config.set((), "keywords-default", "") + pfmt = self._pfmt(kwdict=True) + self.assertEqual(pfmt.realdirectory, "") + + def test_option_extensionmap_default(self): + kwdict = KWDICT.copy() + pfmt = self._pfmt() + pfmt.set_filename(kwdict) + self.assertEqual(pfmt.extension, "ext") + + pfmt.set_extension("jpg") + self.assertEqual(pfmt.extension, "jpg") + self.assertEqual(kwdict["extension"], "jpg") + + pfmt.set_extension("png") + self.assertEqual(pfmt.extension, "png") + self.assertEqual(kwdict["extension"], "png") + + pfmt.set_extension("jpeg") + self.assertEqual(pfmt.extension, "jpg") + self.assertEqual(kwdict["extension"], "jpg") + + for ext, repl in path.EXTENSION_MAP.items(): + pfmt.set_extension(ext) + self.assertEqual(pfmt.extension, repl) + self.assertEqual(kwdict["extension"], repl) + + def test_option_extensionmap_custom(self): + extmap = { + "bitmap": "bmp", + "ping" : "png", + "jiff" : "gif", + } + config.set((), "extension-map", extmap) + + kwdict = KWDICT.copy() + pfmt = self._pfmt() + pfmt.set_filename(kwdict) + + pfmt.set_extension("jpg") + self.assertEqual(pfmt.extension, "jpg") + self.assertEqual(kwdict["extension"], "jpg") + + pfmt.set_extension("ping") + self.assertEqual(pfmt.extension, "png") + self.assertEqual(kwdict["extension"], "png") + + for ext, repl in extmap.items(): + pfmt.set_extension(ext) + self.assertEqual(pfmt.extension, repl) + self.assertEqual(kwdict["extension"], repl) + + for ext, repl in path.EXTENSION_MAP.items(): + pfmt.set_extension(ext) + self.assertNotEqual(pfmt.extension, repl) + self.assertNotEqual(kwdict["extension"], repl) + + def test_option_pathrestrict(self): + config.set((), "filename", "{name}.{ext}") + + config.set((), "path-restrict", "unix") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "test-テスト-'&>-_:~.txt", "unix") + + config.set((), "path-restrict", "windows") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "test-テスト-'&_-__~.txt", "windows") + + config.set((), "path-restrict", "ascii") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "test____________.txt", "ascii") + + config.set((), "path-restrict", "ascii+") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "test-___-'&_-__~.txt", "ascii+") + + def test_option_pathrestrict_custom(self): + config.set((), "filename", "{name}.{ext}") + + config.set((), "path-restrict", "ts-") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "_e___テスト_'&>_/:~._x_", "custom str") + + config.set((), "path-restrict", { + "t": "A", + "s": "B", + "-": "###", + "/": "|" + }) + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "AeBA###テスト###'&>###|:~.AxA", "custom dict") + + config.set((), "path-restrict", { + "a-z": "x", + "テ": "te", + "ス": "su", + "ト": "to", + }) + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "xxxx-tesuto-'&>-/:~.xxx", "custom dict range") + + def test_option_pathreplace(self): + config.set((), "filename", "{name}.{ext}") + + config.set((), "path-restrict", "unix") + config.set((), "path-replace", "&") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "test-テスト-'&>-&:~.txt", "&") + + config.set((), "path-restrict", "windows") + config.set((), "path-replace", "***") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "test-テスト-'&***-******~.txt", "***") + + def test_option_pathremove(self): + config.set((), "filename", "{name}.{ext}") + + config.set((), "path-remove", "-&/") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "testテスト'>_:~.txt") + + config.set((), "path-remove", "a-z0-9") + fname = self._pfmt().build_filename(KWDICT) + self.assertEqual(fname, "-テスト-'&>-_:~.") + + def test_option_pathstrip(self): + config.set((), "directory", [" . {name}.{ext} . "]) + config.set((), "base-directory", "") + config.set((), "path-restrict", "unix") + + config.set((), "path-strip", "unix") + pfmt = self._pfmt(kwdict=True) + self.assertEqual( + pfmt.realdirectory, ". test-テスト-'&>-_:~.txt ./", "unix") + + config.set((), "path-strip", "windows") + pfmt = self._pfmt(kwdict=True) + self.assertEqual( + pfmt.realdirectory, ". test-テスト-'&>-_:~.txt/", "windows") + + config.set((), "path-strip", "txt") + pfmt = self._pfmt(kwdict=True) + self.assertEqual( + pfmt.realdirectory, ". test-テスト-'&>-_:~.txt ./", "custom") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 5d52e1d..e4d01c2 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -78,6 +78,7 @@ class BasePostprocessorTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.dir = tempfile.TemporaryDirectory() + config.clear() config.set((), "base-directory", cls.dir.name) cls.job = FakeJob() @@ -374,6 +375,40 @@ class ExecTest(BasePostprocessorTest): m_aa.assert_called_once_with(self.pathfmt.kwdict) m_ac.assert_called_once() + def test_verbose_string(self): + self._create({ + "command": "echo foo bar", + "verbose": False, + }) + + with patch("gallery_dl.util.Popen") as p, \ + self.assertLogs(level=10) as log_info: + i = Mock() + i.wait.return_value = 123 + p.return_value = i + self._trigger(("after",)) + + msg = "DEBUG:postprocessor.exec:Running 'echo'" + self.assertEqual(log_info.output[0], msg) + self.assertIn("'echo' returned with non-zero ", log_info.output[1]) + + def test_verbose_list(self): + self._create({ + "command": ["echo", "foo", "bar"], + "verbose": False, + }) + + with patch("gallery_dl.util.Popen") as p, \ + self.assertLogs(level=10) as log_info: + i = Mock() + i.wait.return_value = 123 + p.return_value = i + self._trigger(("after",)) + + msg = "DEBUG:postprocessor.exec:Running 'echo'" + self.assertEqual(log_info.output[0], msg) + self.assertIn("'echo' returned with non-zero ", log_info.output[1]) + class HashTest(BasePostprocessorTest): @@ -453,7 +488,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realpath}.JSON" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) self.assertEqual(self._output(m), """{ "category": "test", @@ -473,6 +508,7 @@ class MetadataTest(BasePostprocessorTest): "indent" : None, "open" : "a", "encoding" : "UTF-8", + "newline" : "\r\n", "extension" : "JSON", }, { "public" : "hello ワールド", @@ -487,7 +523,9 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realpath}.JSON" - m.assert_called_once_with(path, "a", encoding="UTF-8") + m.assert_called_once_with(path, "a", encoding="UTF-8", newline='\r\n') + # Since we mocked the call to open, + # we don't actually see the effect of setting newline. self.assertEqual(self._output(m), """{\ "_private" : "foo \\u30d0\\u30fc",\ "category" : "test",\ @@ -508,7 +546,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realpath}.txt" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) self.assertEqual(self._output(m), "foo\nbar\nbaz\n") def test_metadata_tags_split_1(self): @@ -599,7 +637,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}file.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_extfmt_2(self): self._create({ @@ -611,7 +649,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}file.2.EXT-data:tESt" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory(self): self._create({ @@ -622,7 +660,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}metadata/file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory_2(self): self._create({ @@ -634,7 +672,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}metadata/file.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory_format(self): self._create( @@ -646,7 +684,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}../json/12500/file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory_empty(self): self._create( @@ -657,7 +695,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}./file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_basedirectory(self): self._create({"base-directory": True}) @@ -666,7 +704,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.basedirectory}file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_basedirectory_custom(self): self._create({ @@ -678,7 +716,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = "/home/test/meta/file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_filename(self): self._create({ @@ -690,7 +728,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}test_file__meta_.data" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_meta_path(self): self._create({ @@ -790,7 +828,7 @@ class MetadataTest(BasePostprocessorTest): self.assertGreater(len(self._output(m)), 0) path = f"{self.pathfmt.realdirectory}file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_option_skip_false(self): self._create({"skip": False}) @@ -802,6 +840,28 @@ class MetadataTest(BasePostprocessorTest): self.assertTrue(not e.called) self.assertTrue(m.called) + def test_metadata_option_newline(self): + self._create({ + "newline": "\r\n", + "filename" : "data.json", + "directory" : "", + "base-directory": self.dir.name, + }) + + self._trigger() + + path = os.path.join(self.dir.name, "data.json") + with open(path, newline="") as fp: + content = fp.read() + + self.assertEqual(content, """\ +{\r\n\ + "category": "test",\r\n\ + "filename": "file",\r\n\ + "extension": "ext"\r\n\ +}\r\n\ +""") + def test_metadata_option_include(self): self._create( {"include": ["_private", "filename", "foo"], "sort": True}, diff --git a/test/test_results.py b/test/test_results.py index e7fcabf..0865686 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -310,10 +310,13 @@ class TestExtractorResults(unittest.TestCase): elif isinstance(test, range): self.assertRange(value, test, msg=path) elif isinstance(test, set): - try: - self.assertIn(value, test, msg=path) - except AssertionError: - self.assertIn(type(value), test, msg=path) + for item in test: + if isinstance(item, type) and isinstance(value, item) or \ + value == item: + break + else: + v = type(value) if len(str(value)) > 64 else value + self.fail(f"{v!r} not in {test}: {path}") elif isinstance(test, list): subtest = False for idx, item in enumerate(test): @@ -423,8 +426,7 @@ class ResultJob(job.DownloadJob): def run(self): self._init() - for msg in self.extractor: - self.dispatch(msg) + self.dispatch(self.extractor) def handle_url(self, url, kwdict, fallback=None): self._update_url(url) diff --git a/test/test_text.py b/test/test_text.py index 0e187d7..eac7906 100644 --- a/test/test_text.py +++ b/test/test_text.py @@ -11,8 +11,6 @@ import os import sys import unittest -import datetime - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import text, util # noqa E402 @@ -203,6 +201,10 @@ class TestText(unittest.TestCase): self.assertEqual(f("http://example.org/v2/filename.ext"), result) self.assertEqual( f("http://example.org/v2/filename.ext?param=value#frag"), result) + self.assertEqual( + f("http://example.org/v2/foo%202?bar&<>.ext?param=value#frag"), + {"filename": "foo 2", "extension": ""}, + ) # long "extension" fn = "httpswww.example.orgpath-path-path-path-path-path-path-path" @@ -212,6 +214,24 @@ class TestText(unittest.TestCase): for value in INVALID: self.assertEqual(f(value), empty) + def test_nameext_from_name(self, f=text.nameext_from_name): + self.assertEqual( + f(""), + {"filename": "", "extension": ""}, + ) + self.assertEqual( + f("filename.ext"), + {"filename": "filename", "extension": "ext"}, + ) + self.assertEqual( + f("foo%202?bar&<>.ext"), + {"filename": "foo%202?bar&<>", "extension": "ext"}, + ) + + # long "extension" + fn = "httpswww.example.orgpath-path-path-path-path-path-path-path" + self.assertEqual(f(fn), {"filename": fn, "extension": ""}) + def test_extract(self, f=text.extract): txt = "" self.assertEqual(f(txt, "<", ">"), ("a" , 3)) @@ -519,51 +539,6 @@ class TestText(unittest.TestCase): self.assertEqual(f({"ä&": "あと", "#": "?"}), "%C3%A4%26=%E3%81%82%E3%81%A8&%23=%3F") - def test_parse_timestamp(self, f=text.parse_timestamp): - null = util.datetime_utcfromtimestamp(0) - value = util.datetime_utcfromtimestamp(1555816235) - - self.assertEqual(f(0) , null) - self.assertEqual(f("0") , null) - self.assertEqual(f(1555816235) , value) - self.assertEqual(f("1555816235"), value) - - for value in INVALID_ALT: - self.assertEqual(f(value), None) - self.assertEqual(f(value, "foo"), "foo") - - def test_parse_datetime(self, f=text.parse_datetime): - null = util.datetime_utcfromtimestamp(0) - - self.assertEqual(f("1970-01-01T00:00:00+00:00"), null) - self.assertEqual(f("1970-01-01T00:00:00+0000") , null) - self.assertEqual(f("1970.01.01", "%Y.%m.%d") , null) - - self.assertEqual( - f("2019-05-07T21:25:02+09:00"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02+0900"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02.753+0900", "%Y-%m-%dT%H:%M:%S.%f%z"), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07T21:25:02", "%Y-%m-%dT%H:%M:%S", utcoffset=9), - datetime.datetime(2019, 5, 7, 12, 25, 2), - ) - self.assertEqual( - f("2019-05-07 21:25:02"), - "2019-05-07 21:25:02", - ) - - for value in INVALID: - self.assertEqual(f(value), None) - self.assertEqual(f("1970.01.01"), "1970.01.01") - if __name__ == "__main__": unittest.main() diff --git a/test/test_util.py b/test/test_util.py index bfaab01..6784874 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -40,6 +40,7 @@ class TestRange(unittest.TestCase): def test_parse_digit(self): f = self.predicate._parse + self.assertEqual(f(2), [range(2, 3)]) self.assertEqual(f("2"), [range(2, 3)]) self.assertEqual( @@ -48,6 +49,12 @@ class TestRange(unittest.TestCase): range(3, 4), range(4, 5)], ) + self.assertEqual( + f(["2", "3", "4"]), + [range(2, 3), + range(3, 4), + range(4, 5)], + ) def test_parse_range(self): f = self.predicate._parse @@ -406,89 +413,6 @@ def hash(value): self.assertEqual(expr(value), result) -class TestDatetime(unittest.TestCase): - - def test_to_datetime(self, f=util.to_datetime): - - def _assert(value, expected): - result = f(value) - self.assertIsInstance(result, datetime.datetime) - self.assertEqual(result, expected, msg=repr(value)) - - dt = datetime.datetime(2010, 1, 1) - self.assertIs(f(dt), dt) - - _assert(dt , dt) - _assert(1262304000 , dt) - _assert(1262304000.0 , dt) - _assert(1262304000.123, dt) - _assert("1262304000" , dt) - - _assert("2010-01-01" , dt) - _assert("2010-01-01 00:00:00" , dt) - _assert("2010-01-01T00:00:00" , dt) - _assert("2010-01-01T00:00:00.123456" , dt) - _assert("2009-12-31T19:00:00-05:00" , dt) - _assert("2009-12-31T19:00:00.123456-05:00", dt) - _assert("2010-01-01T00:00:00Z" , dt) - _assert("2010-01-01T00:00:00.123456Z" , dt) - - _assert(0 , util.EPOCH) - _assert("" , util.EPOCH) - _assert("foo", util.EPOCH) - _assert(None , util.EPOCH) - _assert(() , util.EPOCH) - _assert([] , util.EPOCH) - _assert({} , util.EPOCH) - _assert((1, 2, 3), util.EPOCH) - - @unittest.skipIf(sys.hexversion < 0x30b0000, - "extended fromisoformat timezones") - def test_to_datetime_tz(self, f=util.to_datetime): - - def _assert(value, expected): - result = f(value) - self.assertIsInstance(result, datetime.datetime) - self.assertEqual(result, expected, msg=repr(value)) - - dt = datetime.datetime(2010, 1, 1) - - _assert("2009-12-31T19:00:00-05" , dt) - _assert("2009-12-31T19:00:00-0500" , dt) - _assert("2009-12-31T19:00:00.123456-05" , dt) - _assert("2009-12-31T19:00:00.123456-0500" , dt) - - def test_datetime_to_timestamp(self, f=util.datetime_to_timestamp): - self.assertEqual(f(util.EPOCH), 0.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), - 1262304000.128000) - with self.assertRaises(TypeError): - f(None) - - def test_datetime_to_timestamp_string( - self, f=util.datetime_to_timestamp_string): - self.assertEqual(f(util.EPOCH), "0") - self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") - self.assertEqual(f(None), "") - - def test_datetime_from_timestamp( - self, f=util.datetime_from_timestamp): - self.assertEqual(f(0.0), util.EPOCH) - self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - self.assertEqual(f(1262304000.128000).replace(microsecond=0), - datetime.datetime(2010, 1, 1, 0, 0, 0)) - - def test_datetime_utcfromtimestamp( - self, f=util.datetime_utcfromtimestamp): - self.assertEqual(f(0.0), util.EPOCH) - self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - - def test_datetime_utcnow( - self, f=util.datetime_utcnow): - self.assertIsInstance(f(), datetime.datetime) - - class TestOther(unittest.TestCase): def test_bencode(self): -- cgit v1.2.3