New upstream version 1.31.1.upstream/1.31.1

1 files changed, 27 insertions, 20 deletions

diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index 34e65c5..13c7bbe 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -19,11 +19,10 @@ import getpass
 import logging
 import requests
 import threading
-from datetime import datetime
 from xml.etree import ElementTree
 from requests.adapters import HTTPAdapter
 from .message import Message
-from .. import config, output, text, util, cache, exception
+from .. import config, output, text, util, dt, cache, exception
 urllib3 = requests.packages.urllib3
 
 
@@ -32,7 +31,9 @@ class Extractor():
     category = ""
     subcategory = ""
     basecategory = ""
+    basesubcategory = ""
     categorytransfer = False
+    parent = False
     directory_fmt = ("{category}",)
     filename_fmt = "{filename}.{extension}"
     archive_fmt = ""
@@ -64,6 +65,10 @@ class Extractor():
             else:
                 self.category = CATEGORY_MAP[self.category]
 
+        self.parse_datetime = dt.parse
+        self.parse_datetime_iso = dt.parse_iso
+        self.parse_timestamp = dt.parse_ts
+
         self._cfgpath = ("extractor", self.category, self.subcategory)
         self._parentdir = ""
 
@@ -89,7 +94,8 @@ class Extractor():
         pass
 
     def items(self):
-        yield Message.Version, 1
+        return
+        yield
 
     def skip(self, num):
         return 0
@@ -313,9 +319,9 @@ class Extractor():
             seconds = float(seconds)
             until = now + seconds
         elif until:
-            if isinstance(until, datetime):
+            if isinstance(until, dt.datetime):
                 # convert to UTC timestamp
-                until = util.datetime_to_timestamp(until)
+                until = dt.to_ts(until)
             else:
                 until = float(until)
             seconds = until - now
@@ -327,7 +333,7 @@ class Extractor():
             return
 
         if reason:
-            t = datetime.fromtimestamp(until).time()
+            t = dt.datetime.fromtimestamp(until).time()
             isotime = f"{t.hour:02}:{t.minute:02}:{t.second:02}"
             self.log.info("Waiting until %s (%s)", isotime, reason)
         time.sleep(seconds)
@@ -652,7 +658,7 @@ class Extractor():
                     self.log.warning(
                         "cookies: %s/%s expired at %s",
                         cookie.domain.lstrip("."), cookie.name,
-                        datetime.fromtimestamp(cookie.expires))
+                        dt.datetime.fromtimestamp(cookie.expires))
                     continue
 
                 elif diff <= 86400:
@@ -693,13 +699,16 @@ class Extractor():
         def get(key, default):
             ts = self.config(key, default)
             if isinstance(ts, str):
-                try:
-                    ts = int(datetime.strptime(ts, fmt).timestamp())
-                except ValueError as exc:
-                    self.log.warning("Unable to parse '%s': %s", key, exc)
+                dt_obj = dt.parse_iso(ts) if fmt is None else dt.parse(ts, fmt)
+                if dt_obj is dt.NONE:
+                    self.log.warning(
+                        "Unable to parse '%s': Invalid %s string '%s'",
+                        key, "isoformat" if fmt is None else "date", ts)
                     ts = default
+                else:
+                    ts = int(dt.to_ts(dt_obj))
             return ts
-        fmt = self.config("date-format", "%Y-%m-%dT%H:%M:%S")
+        fmt = self.config("date-format")
         return get("date-min", dmin), get("date-max", dmax)
 
     @classmethod
@@ -793,7 +802,7 @@ class GalleryExtractor(Extractor):
                     enum = util.enumerate_reversed
             images = enum(imgs, 1)
 
-        yield Message.Directory, data
+        yield Message.Directory, "", data
         enum_key = self.enum
 
         if assets:
@@ -912,7 +921,7 @@ class Dispatch():
         elif isinstance(include, str):
             include = include.replace(" ", "").split(",")
 
-        results = [(Message.Version, 1)]
+        results = []
         for category in include:
             try:
                 extr, url = extractors[category]
@@ -962,18 +971,16 @@ class BaseExtractor(Extractor):
 
     def __init__(self, match):
         if not self.category:
-            self.groups = match.groups()
-            self.match = match
-            self._init_category()
+            self._init_category(match)
         Extractor.__init__(self, match)
 
-    def _init_category(self):
-        for index, group in enumerate(self.groups):
+    def _init_category(self, match):
+        for index, group in enumerate(match.groups()):
             if group is not None:
                 if index:
                     self.category, self.root, info = self.instances[index-1]
                     if not self.root:
-                        self.root = text.root_from_url(self.match[0])
+                        self.root = text.root_from_url(match[0])
                     self.config_instance = info.get
                 else:
                     self.root = group