New upstream version 1.10.2upstream/1.10.2

1 files changed, 110 insertions, 69 deletions

diff --git a/gallery_dl/util.py b/gallery_dl/util.py
index 02d998d..79fa175 100644
--- a/gallery_dl/util.py
+++ b/gallery_dl/util.py
@@ -391,10 +391,18 @@ class Formatter():
             if field_name:
                 self.fields.append((
                     len(self.result),
-                    self._field_access(field_name, format_spec, conversion)
+                    self._field_access(field_name, format_spec, conversion),
                 ))
                 self.result.append("")
 
+        if len(self.result) == 1:
+            if self.fields:
+                self.format_map = self.fields[0][1]
+            else:
+                self.format_map = lambda _: format_string
+            del self.result
+            del self.fields
+
     def format_map(self, kwargs):
         """Apply 'kwargs' to the initial format_string and return its result"""
         for index, func in self.fields:
@@ -512,48 +520,63 @@ class Formatter():
 class PathFormat():
 
     def __init__(self, extractor):
-        self.filename_fmt = extractor.config(
-            "filename", extractor.filename_fmt)
-        self.directory_fmt = extractor.config(
-            "directory", extractor.directory_fmt)
-        self.kwdefault = extractor.config("keywords-default")
+        filename_fmt = extractor.config("filename", extractor.filename_fmt)
+        directory_fmt = extractor.config("directory", extractor.directory_fmt)
+        kwdefault = extractor.config("keywords-default")
 
         try:
-            self.formatter = Formatter(self.filename_fmt, self.kwdefault)
+            self.filename_formatter = Formatter(
+                filename_fmt, kwdefault).format_map
         except Exception as exc:
             raise exception.FormatError(exc, "filename")
 
-        self.delete = False
-        self.has_extension = False
-        self.keywords = {}
-        self.filename = ""
+        try:
+            self.directory_formatters = [
+                Formatter(dirfmt, kwdefault).format_map
+                for dirfmt in directory_fmt
+            ]
+        except Exception as exc:
+            raise exception.FormatError(exc, "directory")
+
         self.directory = self.realdirectory = ""
+        self.filename = ""
+        self.extension = ""
+        self.prefix = ""
+        self.kwdict = {}
+        self.delete = False
         self.path = self.realpath = self.temppath = ""
 
-        self.basedirectory = expand_path(
+        basedir = expand_path(
             extractor.config("base-directory", (".", "gallery-dl")))
-        if os.altsep and os.altsep in self.basedirectory:
-            self.basedirectory = self.basedirectory.replace(os.altsep, os.sep)
+        if os.altsep and os.altsep in basedir:
+            basedir = basedir.replace(os.altsep, os.sep)
+        if basedir[-1] != os.sep:
+            basedir += os.sep
+        self.basedirectory = basedir
 
-        restrict = extractor.config("restrict-filenames", "auto")
+        restrict = extractor.config("path-restrict", "auto")
         if restrict == "auto":
-            restrict = "<>:\"\\/|?*" if os.name == "nt" else "/"
+            restrict = "\\\\|/<>:\"?*" if os.name == "nt" else "/"
         elif restrict == "unix":
             restrict = "/"
         elif restrict == "windows":
-            restrict = "<>:\"\\/|?*"
-        self.clean_path = self._build_cleanfunc(restrict)
+            restrict = "\\\\|/<>:\"?*"
+
+        remove = extractor.config("path-remove", "\x00-\x1f\x7f")
+
+        self.clean_segment = self._build_cleanfunc(restrict, "_")
+        self.clean_path = self._build_cleanfunc(remove, "")
 
     @staticmethod
-    def _build_cleanfunc(repl):
-        if not repl:
+    def _build_cleanfunc(chars, repl):
+        if not chars:
             return lambda x: x
-        elif len(repl) == 1:
-            def func(x, r=repl):
-                return x.replace(r, "_")
+        elif len(chars) == 1:
+            def func(x, c=chars, r=repl):
+                return x.replace(c, r)
         else:
-            def func(x, sub=re.compile("[" + re.escape(repl) + "]").sub):
-                return sub("_", x)
+            def func(x, sub=re.compile("[" + chars + "]").sub, r=repl):
+                return sub(r, x)
         return func
 
     def open(self, mode="wb"):
@@ -562,68 +585,91 @@ class PathFormat():
 
     def exists(self, archive=None):
         """Return True if the file exists on disk or in 'archive'"""
-        if archive and archive.check(self.keywords):
+        if archive and self.kwdict in archive:
             return self.fix_extension()
-        if self.has_extension and os.path.exists(self.realpath):
-            return True
+        if self.extension and os.path.exists(self.realpath):
+            return self.check_file()
         return False
 
-    def set_directory(self, keywords):
+    @staticmethod
+    def check_file():
+        return True
+
+    def _enum_file(self):
+        num = 1
+        while True:
+            self.prefix = str(num) + "."
+            self.set_extension(self.extension, False)
+            if not os.path.exists(self.realpath):
+                return False
+            num += 1
+
+    def set_directory(self, kwdict):
         """Build directory path and create it if necessary"""
+
+        # Build path segments by applying 'kwdict' to directory format strings
         try:
             segments = [
-                self.clean_path(
-                    Formatter(segment, self.kwdefault)
-                    .format_map(keywords).strip())
-                for segment in self.directory_fmt
+                self.clean_segment(format_map(kwdict).strip())
+                for format_map in self.directory_formatters
             ]
         except Exception as exc:
             raise exception.FormatError(exc, "directory")
 
-        self.directory = os.path.join(
-            self.basedirectory,
-            *segments
-        )
+        # Join path segements
+        sep = os.sep
+        directory = self.clean_path(self.basedirectory + sep.join(segments))
 
-        # remove trailing path separator;
-        # occurs if the last argument to os.path.join() is an empty string
-        if self.directory[-1] == os.sep:
-            self.directory = self.directory[:-1]
+        # Ensure directory ends with a path separator
+        if directory[-1] != sep:
+            directory += sep
+        self.directory = directory
 
-        self.realdirectory = self.adjust_path(self.directory)
+        # Enable longer-than-260-character paths on Windows
+        if os.name == "nt":
+            self.realdirectory = "\\\\?\\" + os.path.abspath(directory) + sep
+        else:
+            self.realdirectory = directory
+
+        # Create directory tree
         os.makedirs(self.realdirectory, exist_ok=True)
 
-    def set_keywords(self, keywords):
-        """Set filename keywords"""
-        self.keywords = keywords
-        self.temppath = ""
-        self.has_extension = bool(keywords.get("extension"))
-        if self.has_extension:
+    def set_filename(self, kwdict):
+        """Set general filename data"""
+        self.kwdict = kwdict
+        self.temppath = self.prefix = ""
+        self.extension = kwdict["extension"]
+
+        if self.extension:
             self.build_path()
 
     def set_extension(self, extension, real=True):
-        """Set the 'extension' keyword"""
-        self.has_extension = real
-        self.keywords["extension"] = extension
+        """Set filename extension"""
+        if real:
+            self.extension = extension
+        self.kwdict["extension"] = self.prefix + extension
         self.build_path()
 
     def fix_extension(self, _=None):
-        if not self.has_extension:
-            self.set_extension("")
+        """Fix filenames without a given filename extension"""
+        if not self.extension:
+            self.set_extension("", False)
             if self.path[-1] == ".":
                 self.path = self.path[:-1]
                 self.temppath = self.realpath = self.realpath[:-1]
         return True
 
     def build_path(self):
-        """Use filename-keywords and directory to build a full path"""
+        """Use filename metadata and directory to build a full path"""
+
+        # Apply 'kwdict' to filename format string
         try:
-            self.filename = self.clean_path(
-                self.formatter.format_map(self.keywords))
+            self.filename = filename = self.clean_path(self.clean_segment(
+                self.filename_formatter(self.kwdict)))
         except Exception as exc:
             raise exception.FormatError(exc, "filename")
 
-        filename = os.sep + self.filename
+        # Combine directory and filename to full paths
         self.path = self.directory + filename
         self.realpath = self.realdirectory + filename
         if not self.temppath:
@@ -631,7 +677,7 @@ class PathFormat():
 
     def part_enable(self, part_directory=None):
         """Enable .part file usage"""
-        if self.has_extension:
+        if self.extension:
             self.temppath += ".part"
         else:
             self.set_extension("part", False)
@@ -657,16 +703,16 @@ class PathFormat():
             return
 
         if self.temppath != self.realpath:
-            # move temp file to its actual location
+            # Move temp file to its actual location
             try:
                 os.replace(self.temppath, self.realpath)
             except OSError:
                 shutil.copyfile(self.temppath, self.realpath)
                 os.unlink(self.temppath)
 
-        if "_mtime" in self.keywords:
-            # set file modification time
-            mtime = self.keywords["_mtime"]
+        if "_mtime" in self.kwdict:
+            # Set file modification time
+            mtime = self.kwdict["_mtime"]
             if mtime:
                 try:
                     if isinstance(mtime, str):
@@ -675,11 +721,6 @@ class PathFormat():
                 except Exception:
                     pass
 
-    @staticmethod
-    def adjust_path(path):
-        """Enable longer-than-260-character paths on windows"""
-        return "\\\\?\\" + os.path.abspath(path) if os.name == "nt" else path
-
 
 class DownloadArchive():
 
@@ -693,8 +734,8 @@ class DownloadArchive():
             "archive-format", extractor.archive_fmt)
         ).format_map
 
-    def check(self, kwdict):
-        """Return True if item described by 'kwdict' exists in archive"""
+    def __contains__(self, kwdict):
+        """Return True if the item described by 'kwdict' exists in archive"""
         key = self.keygen(kwdict)
         self.cursor.execute(
             "SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,))