diff options
Diffstat (limited to 'gallery_dl/util.py')
| -rw-r--r-- | gallery_dl/util.py | 89 |
1 files changed, 59 insertions, 30 deletions
diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 78663a0..fbede3e 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -21,6 +21,7 @@ import sqlite3 import binascii import datetime import operator +import functools import itertools import urllib.parse from http.cookiejar import Cookie @@ -346,8 +347,6 @@ CODES = { "zh": "Chinese", } -SPECIAL_EXTRACTORS = {"oauth", "recursive", "test"} - class UniversalNone(): """None-style object that supports more operations than None itself""" @@ -373,6 +372,20 @@ class UniversalNone(): NONE = UniversalNone() WINDOWS = (os.name == "nt") SENTINEL = object() +SPECIAL_EXTRACTORS = {"oauth", "recursive", "test"} +GLOBALS = { + "parse_int": text.parse_int, + "urlsplit" : urllib.parse.urlsplit, + "datetime" : datetime.datetime, + "abort" : raises(exception.StopExtraction), + "terminate": raises(exception.TerminateExtraction), + "re" : re, +} + + +def compile_expression(expr, name="<expr>", globals=GLOBALS): + code_object = compile(expr, name, "eval") + return functools.partial(eval, code_object, globals) def build_predicate(predicates): @@ -472,20 +485,13 @@ class UniquePredicate(): class FilterPredicate(): """Predicate; True if evaluating the given expression returns True""" - def __init__(self, filterexpr, target="image"): + def __init__(self, expr, target="image"): name = "<{} filter>".format(target) - self.codeobj = compile(filterexpr, name, "eval") - self.globals = { - "parse_int": text.parse_int, - "urlsplit" : urllib.parse.urlsplit, - "datetime" : datetime.datetime, - "abort" : raises(exception.StopExtraction), - "re" : re, - } + self.expr = compile_expression(expr, name) - def __call__(self, url, kwds): + def __call__(self, _, kwdict): try: - return eval(self.codeobj, self.globals, kwds) + return self.expr(kwdict) except exception.GalleryDLException: raise except Exception as exc: @@ -749,25 +755,30 @@ class PathFormat(): } def __init__(self, extractor): - filename_fmt = extractor.config("filename") - if filename_fmt is None: - filename_fmt = extractor.filename_fmt - - directory_fmt = extractor.config("directory") - if directory_fmt is None: - directory_fmt = extractor.directory_fmt - - extension_map = extractor.config("extension-map") - if extension_map is None: - extension_map = self.EXTENSION_MAP - self.extension_map = extension_map.get + config = extractor.config + kwdefault = config("keywords-default") - kwdefault = extractor.config("keywords-default") + filename_fmt = config("filename") try: + if filename_fmt is None: + filename_fmt = extractor.filename_fmt + elif isinstance(filename_fmt, dict): + self.filename_conditions = [ + (compile_expression(expr), + Formatter(fmt, kwdefault).format_map) + for expr, fmt in filename_fmt.items() if expr + ] + self.build_filename = self.build_filename_conditional + filename_fmt = filename_fmt.get("", extractor.filename_fmt) + self.filename_formatter = Formatter( filename_fmt, kwdefault).format_map except Exception as exc: raise exception.FilenameFormatError(exc) + + directory_fmt = config("directory") + if directory_fmt is None: + directory_fmt = extractor.directory_fmt try: self.directory_formatters = [ Formatter(dirfmt, kwdefault).format_map @@ -784,7 +795,7 @@ class PathFormat(): basedir = extractor._parentdir if not basedir: - basedir = extractor.config("base-directory") + basedir = config("base-directory") if basedir is None: basedir = "." + os.sep + "gallery-dl" + os.sep elif basedir: @@ -795,8 +806,13 @@ class PathFormat(): basedir += os.sep self.basedirectory = basedir - restrict = extractor.config("path-restrict", "auto") - replace = extractor.config("path-replace", "_") + extension_map = config("extension-map") + if extension_map is None: + extension_map = self.EXTENSION_MAP + self.extension_map = extension_map.get + + restrict = config("path-restrict", "auto") + replace = config("path-replace", "_") if restrict == "auto": restrict = "\\\\|/<>:\"?*" if WINDOWS else "/" elif restrict == "unix": @@ -807,7 +823,7 @@ class PathFormat(): restrict = "^0-9A-Za-z_." self.clean_segment = self._build_cleanfunc(restrict, replace) - remove = extractor.config("path-remove", "\x00-\x1f\x7f") + remove = config("path-remove", "\x00-\x1f\x7f") self.clean_path = self._build_cleanfunc(remove, "") @staticmethod @@ -927,6 +943,19 @@ class PathFormat(): except Exception as exc: raise exception.FilenameFormatError(exc) + def build_filename_conditional(self): + kwdict = self.kwdict + + try: + for condition, formatter in self.filename_conditions: + if condition(kwdict): + break + else: + formatter = self.filename_formatter + return self.clean_path(self.clean_segment(formatter(kwdict))) + except Exception as exc: + raise exception.FilenameFormatError(exc) + def build_path(self): """Combine directory and filename to full paths""" if self._create_directory: |