diff options
Diffstat (limited to 'gallery_dl/extractor/common.py')
| -rw-r--r-- | gallery_dl/extractor/common.py | 47 |
1 files changed, 27 insertions, 20 deletions
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 34e65c5..13c7bbe 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -19,11 +19,10 @@ import getpass import logging import requests import threading -from datetime import datetime from xml.etree import ElementTree from requests.adapters import HTTPAdapter from .message import Message -from .. import config, output, text, util, cache, exception +from .. import config, output, text, util, dt, cache, exception urllib3 = requests.packages.urllib3 @@ -32,7 +31,9 @@ class Extractor(): category = "" subcategory = "" basecategory = "" + basesubcategory = "" categorytransfer = False + parent = False directory_fmt = ("{category}",) filename_fmt = "{filename}.{extension}" archive_fmt = "" @@ -64,6 +65,10 @@ class Extractor(): else: self.category = CATEGORY_MAP[self.category] + self.parse_datetime = dt.parse + self.parse_datetime_iso = dt.parse_iso + self.parse_timestamp = dt.parse_ts + self._cfgpath = ("extractor", self.category, self.subcategory) self._parentdir = "" @@ -89,7 +94,8 @@ class Extractor(): pass def items(self): - yield Message.Version, 1 + return + yield def skip(self, num): return 0 @@ -313,9 +319,9 @@ class Extractor(): seconds = float(seconds) until = now + seconds elif until: - if isinstance(until, datetime): + if isinstance(until, dt.datetime): # convert to UTC timestamp - until = util.datetime_to_timestamp(until) + until = dt.to_ts(until) else: until = float(until) seconds = until - now @@ -327,7 +333,7 @@ class Extractor(): return if reason: - t = datetime.fromtimestamp(until).time() + t = dt.datetime.fromtimestamp(until).time() isotime = f"{t.hour:02}:{t.minute:02}:{t.second:02}" self.log.info("Waiting until %s (%s)", isotime, reason) time.sleep(seconds) @@ -652,7 +658,7 @@ class Extractor(): self.log.warning( "cookies: %s/%s expired at %s", cookie.domain.lstrip("."), cookie.name, - datetime.fromtimestamp(cookie.expires)) + dt.datetime.fromtimestamp(cookie.expires)) continue elif diff <= 86400: @@ -693,13 +699,16 @@ class Extractor(): def get(key, default): ts = self.config(key, default) if isinstance(ts, str): - try: - ts = int(datetime.strptime(ts, fmt).timestamp()) - except ValueError as exc: - self.log.warning("Unable to parse '%s': %s", key, exc) + dt_obj = dt.parse_iso(ts) if fmt is None else dt.parse(ts, fmt) + if dt_obj is dt.NONE: + self.log.warning( + "Unable to parse '%s': Invalid %s string '%s'", + key, "isoformat" if fmt is None else "date", ts) ts = default + else: + ts = int(dt.to_ts(dt_obj)) return ts - fmt = self.config("date-format", "%Y-%m-%dT%H:%M:%S") + fmt = self.config("date-format") return get("date-min", dmin), get("date-max", dmax) @classmethod @@ -793,7 +802,7 @@ class GalleryExtractor(Extractor): enum = util.enumerate_reversed images = enum(imgs, 1) - yield Message.Directory, data + yield Message.Directory, "", data enum_key = self.enum if assets: @@ -912,7 +921,7 @@ class Dispatch(): elif isinstance(include, str): include = include.replace(" ", "").split(",") - results = [(Message.Version, 1)] + results = [] for category in include: try: extr, url = extractors[category] @@ -962,18 +971,16 @@ class BaseExtractor(Extractor): def __init__(self, match): if not self.category: - self.groups = match.groups() - self.match = match - self._init_category() + self._init_category(match) Extractor.__init__(self, match) - def _init_category(self): - for index, group in enumerate(self.groups): + def _init_category(self, match): + for index, group in enumerate(match.groups()): if group is not None: if index: self.category, self.root, info = self.instances[index-1] if not self.root: - self.root = text.root_from_url(self.match[0]) + self.root = text.root_from_url(match[0]) self.config_instance = info.get else: self.root = group |