diff options
Diffstat (limited to 'gallery_dl/util.py')
| -rw-r--r-- | gallery_dl/util.py | 71 |
1 files changed, 65 insertions, 6 deletions
diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 83cf84b..85b871b 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -113,6 +113,57 @@ def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4): fp.write("\n") +def dump_response(response, fp=sys.stdout, + headers=True, content=True, hide_auth=True): + """Write the contents of 'response' into a file-like object""" + + if headers: + request = response.request + req_headers = request.headers.copy() + outfmt = """\ +{request.method} {request.url} +Status: {response.status_code} {response.reason} + +Request Headers +--------------- +{request_headers} + +Response Headers +---------------- +{response_headers} +""" + if hide_auth: + authorization = req_headers.get("Authorization") + if authorization: + atype, sep, _ = authorization.partition(" ") + req_headers["Authorization"] = atype + " ***" if sep else "***" + + cookies = req_headers.get("Cookie") + if cookies: + req_headers["Cookie"] = ";".join( + cookie.partition("=")[0] + "=***" + for cookie in cookies.split(";") + ) + + fp.write(outfmt.format( + request=request, + response=response, + request_headers="\n".join( + name + ": " + value + for name, value in req_headers.items() + ), + response_headers="\n".join( + name + ": " + value + for name, value in response.headers.items() + ), + ).encode()) + + if content: + if headers: + fp.write(b"\nContent\n-------\n") + fp.write(response.content) + + def expand_path(path): """Expand environment variables and tildes (~)""" if not path: @@ -270,6 +321,8 @@ class UniversalNone(): NONE = UniversalNone() +WINDOWS = (os.name == "nt") +SENTINEL = object() def build_predicate(predicates): @@ -672,22 +725,26 @@ class PathFormat(): self.basedirectory = basedir restrict = extractor.config("path-restrict", "auto") + replace = extractor.config("path-replace", "_") + if restrict == "auto": - restrict = "\\\\|/<>:\"?*" if os.name == "nt" else "/" + restrict = "\\\\|/<>:\"?*" if WINDOWS else "/" elif restrict == "unix": restrict = "/" elif restrict == "windows": restrict = "\\\\|/<>:\"?*" + self.clean_segment = self._build_cleanfunc(restrict, replace) remove = extractor.config("path-remove", "\x00-\x1f\x7f") - - self.clean_segment = self._build_cleanfunc(restrict, "_") self.clean_path = self._build_cleanfunc(remove, "") @staticmethod def _build_cleanfunc(chars, repl): if not chars: return lambda x: x + elif isinstance(chars, dict): + def func(x, table=str.maketrans(chars)): + return x.translate(table) elif len(chars) == 1: def func(x, c=chars, r=repl): return x.replace(c, r) @@ -726,7 +783,7 @@ class PathFormat(): def set_directory(self, kwdict): """Build directory path and create it if necessary""" - windows = os.name == "nt" + self.kwdict = kwdict # Build path segments by applying 'kwdict' to directory format strings segments = [] @@ -734,7 +791,7 @@ class PathFormat(): try: for formatter in self.directory_formatters: segment = formatter(kwdict).strip() - if windows: + if WINDOWS: # remove trailing dots and spaces (#647) segment = segment.rstrip(". ") if segment: @@ -751,7 +808,7 @@ class PathFormat(): directory += sep self.directory = directory - if windows: + if WINDOWS: # Enable longer-than-260-character paths on Windows directory = "\\\\?\\" + os.path.abspath(directory) @@ -772,6 +829,8 @@ class PathFormat(): if self.extension: self.build_path() + else: + self.filename = "" def set_extension(self, extension, real=True): """Set filename extension""" |