diff options
Diffstat (limited to 'gallery_dl/util.py')
| -rw-r--r-- | gallery_dl/util.py | 254 |
1 files changed, 176 insertions, 78 deletions
diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 13bf80e..232047c 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -21,6 +21,7 @@ import datetime import operator import itertools import urllib.parse +from http.cookiejar import Cookie from email.utils import mktime_tz, parsedate_tz from . import text, exception @@ -135,6 +136,67 @@ def remove_directory(path): pass +def load_cookiestxt(fp): + """Parse a Netscape cookies.txt file and return a list of its Cookies""" + cookies = [] + + for line in fp: + + line = line.lstrip() + # strip '#HttpOnly_' + if line.startswith("#HttpOnly_"): + line = line[10:] + # ignore empty lines and comments + if not line or line[0] in ("#", "$"): + continue + # strip trailing '\n' + if line[-1] == "\n": + line = line[:-1] + + domain, domain_specified, path, secure, expires, name, value = \ + line.split("\t") + if not name: + name = value + value = None + + cookies.append(Cookie( + 0, name, value, + None, False, + domain, + domain_specified == "TRUE", + domain.startswith("."), + path, False, + secure == "TRUE", + None if expires == "0" or not expires else expires, + False, None, None, {}, + )) + + return cookies + + +def save_cookiestxt(fp, cookies): + """Write 'cookies' in Netscape cookies.txt format to 'fp'""" + fp.write("# Netscape HTTP Cookie File\n\n") + + for cookie in cookies: + if cookie.value is None: + name = "" + value = cookie.name + else: + name = cookie.name + value = cookie.value + + fp.write("\t".join(( + cookie.domain, + "TRUE" if cookie.domain.startswith(".") else "FALSE", + cookie.path, + "TRUE" if cookie.secure else "FALSE", + "0" if cookie.expires is None else str(cookie.expires), + name, + value, + )) + "\n") + + def code_to_language(code, default=None): """Map an ISO 639-1 language code to its actual name""" return CODES.get((code or "").lower(), default) @@ -419,63 +481,85 @@ class Formatter(): self.format_map = self.fields[0][1] else: self.format_map = lambda _: format_string - del self.result - del self.fields + del self.result, self.fields - def format_map(self, kwargs): - """Apply 'kwargs' to the initial format_string and return its result""" + def format_map(self, kwdict): + """Apply 'kwdict' to the initial format_string and return its result""" + result = self.result for index, func in self.fields: - self.result[index] = func(kwargs) - return "".join(self.result) + result[index] = func(kwdict) + return "".join(result) def _field_access(self, field_name, format_spec, conversion): - first, rest = _string.formatter_field_name_split(field_name) + fmt = self._parse_format_spec(format_spec, conversion) + + if "|" in field_name: + return self._apply_list([ + self._parse_field_name(fn) + for fn in field_name.split("|") + ], fmt) + else: + key, funcs = self._parse_field_name(field_name) + if funcs: + return self._apply(key, funcs, fmt) + return self._apply_simple(key, fmt) + @staticmethod + def _parse_field_name(field_name): + first, rest = _string.formatter_field_name_split(field_name) funcs = [] + for is_attr, key in rest: if is_attr: func = operator.attrgetter - elif ":" in key: - func = self._slicegetter else: func = operator.itemgetter + try: + if ":" in key: + start, _, stop = key.partition(":") + stop, _, step = stop.partition(":") + start = int(start) if start else None + stop = int(stop) if stop else None + step = int(step) if step else None + key = slice(start, stop, step) + except TypeError: + pass # key is an integer + funcs.append(func(key)) - if conversion: - funcs.append(self.CONVERSIONS[conversion]) + return first, funcs - if format_spec: - if format_spec[0] == "?": - func = self._format_optional - elif format_spec[0] == "L": - func = self._format_maxlen - elif format_spec[0] == "J": - func = self._format_join - elif format_spec[0] == "R": - func = self._format_replace - else: - func = self._format_default - fmt = func(format_spec) - else: - fmt = str + def _parse_format_spec(self, format_spec, conversion): + fmt = self._build_format_func(format_spec) + if not conversion: + return fmt - if funcs: - return self._apply(first, funcs, fmt) - return self._apply_simple(first, fmt) + conversion = self.CONVERSIONS[conversion] + if fmt is format: + return conversion + else: + def chain(obj): + return fmt(conversion(obj)) + return chain - def _apply_simple(self, key, fmt): - def wrap(obj): - if key in obj: - obj = obj[key] - else: - obj = self.default - return fmt(obj) - return wrap + def _build_format_func(self, format_spec): + if format_spec: + fmt = format_spec[0] + if fmt == "?": + return self._parse_optional(format_spec) + if fmt == "L": + return self._parse_maxlen(format_spec) + if fmt == "J": + return self._parse_join(format_spec) + if fmt == "R": + return self._parse_replace(format_spec) + return self._default_format(format_spec) + return format def _apply(self, key, funcs, fmt): - def wrap(obj): + def wrap(kwdict): try: - obj = obj[key] + obj = kwdict[key] for func in funcs: obj = func(obj) except Exception: @@ -483,54 +567,66 @@ class Formatter(): return fmt(obj) return wrap - @staticmethod - def _slicegetter(key): - start, _, stop = key.partition(":") - stop, _, step = stop.partition(":") - start = int(start) if start else None - stop = int(stop) if stop else None - step = int(step) if step else None - return operator.itemgetter(slice(start, stop, step)) + def _apply_simple(self, key, fmt): + def wrap(kwdict): + return fmt(kwdict[key] if key in kwdict else self.default) + return wrap - @staticmethod - def _format_optional(format_spec): - def wrap(obj): - if not obj: - return "" - return before + format(obj, format_spec) + after + def _apply_list(self, lst, fmt): + def wrap(kwdict): + for key, funcs in lst: + try: + obj = kwdict[key] + for func in funcs: + obj = func(obj) + if obj is not None: + break + except Exception: + pass + else: + obj = self.default + return fmt(obj) + return wrap + + def _parse_optional(self, format_spec): before, after, format_spec = format_spec.split("/", 2) before = before[1:] - return wrap + fmt = self._build_format_func(format_spec) - @staticmethod - def _format_maxlen(format_spec): - def wrap(obj): - obj = format(obj, format_spec) - return obj if len(obj) <= maxlen else replacement + def optional(obj): + return before + fmt(obj) + after if obj else "" + return optional + + def _parse_maxlen(self, format_spec): maxlen, replacement, format_spec = format_spec.split("/", 2) maxlen = text.parse_int(maxlen[1:]) - return wrap + fmt = self._build_format_func(format_spec) - @staticmethod - def _format_join(format_spec): - def wrap(obj): - obj = separator.join(obj) - return format(obj, format_spec) + def mlen(obj): + obj = fmt(obj) + return obj if len(obj) <= maxlen else replacement + return mlen + + def _parse_join(self, format_spec): separator, _, format_spec = format_spec.partition("/") separator = separator[1:] - return wrap + fmt = self._build_format_func(format_spec) - @staticmethod - def _format_replace(format_spec): - def wrap(obj): - obj = obj.replace(old, new) - return format(obj, format_spec) + def join(obj): + return fmt(separator.join(obj)) + return join + + def _parse_replace(self, format_spec): old, new, format_spec = format_spec.split("/", 2) old = old[1:] - return wrap + fmt = self._build_format_func(format_spec) + + def replace(obj): + return fmt(obj.replace(old, new)) + return replace @staticmethod - def _format_default(format_spec): + def _default_format(format_spec): def wrap(obj): return format(obj, format_spec) return wrap @@ -565,12 +661,14 @@ class PathFormat(): self.delete = False self.path = self.realpath = self.temppath = "" - basedir = expand_path( - extractor.config("base-directory", (".", "gallery-dl"))) - if os.altsep and os.altsep in basedir: - basedir = basedir.replace(os.altsep, os.sep) - if basedir[-1] != os.sep: - basedir += os.sep + basedir = extractor._parentdir + if not basedir: + basedir = expand_path( + extractor.config("base-directory", (".", "gallery-dl"))) + if os.altsep and os.altsep in basedir: + basedir = basedir.replace(os.altsep, os.sep) + if basedir[-1] != os.sep: + basedir += os.sep self.basedirectory = basedir restrict = extractor.config("path-restrict", "auto") |