diff options
Diffstat (limited to 'gallery_dl/extractor/common.py')
| -rw-r--r-- | gallery_dl/extractor/common.py | 65 |
1 files changed, 32 insertions, 33 deletions
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 5efea4a..15cc776 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -24,6 +24,7 @@ class Extractor(): category = "" subcategory = "" + basecategory = "" categorytransfer = False directory_fmt = ("{category}",) filename_fmt = "{filename}.{extension}" @@ -31,8 +32,9 @@ class Extractor(): cookiedomain = "" root = "" test = None - _request_last = 0 - _request_interval = 0 + request_interval = 0.0 + request_interval_min = 0.0 + request_timestamp = 0.0 def __init__(self, match): self.session = requests.Session() @@ -48,11 +50,17 @@ class Extractor(): self._retries = self.config("retries", 4) self._timeout = self.config("timeout", 30) self._verify = self.config("verify", True) - self._request_interval = self.config( - "sleep-request", self._request_interval) + self.request_interval = self.config( + "sleep-request", self.request_interval) if self._retries < 0: self._retries = float("inf") + if self.request_interval < self.request_interval_min: + self.request_interval = self.request_interval_min + + if self.basecategory: + self.config = self._config_shared + self.config_accumulate = self._config_shared_accumulate self._init_headers() self._init_cookies() @@ -80,6 +88,19 @@ class Extractor(): def config_accumulate(self, key): return config.accumulate(self._cfgpath, key) + def _config_shared(self, key, default=None): + return config.interpolate_common(("extractor",), ( + (self.category, self.subcategory), + (self.basecategory, self.subcategory), + ), key, default) + + def _config_shared_accumulate(self, key): + values = config.accumulate(self._cfgpath, key) + conf = config.get(("extractor",), self.basecategory) + if conf: + values[:0] = config.accumulate((self.subcategory,), key, conf=conf) + return values + def request(self, url, *, method="GET", session=None, retries=None, encoding=None, fatal=True, notfound=None, **kwargs): tries = 1 @@ -89,10 +110,10 @@ class Extractor(): kwargs.setdefault("verify", self._verify) response = None - if self._request_interval: - seconds = (self._request_interval - - (time.time() - Extractor._request_last)) - if seconds > 0: + if self.request_interval: + seconds = (self.request_interval - + (time.time() - Extractor.request_timestamp)) + if seconds > 0.0: self.log.debug("Sleeping for %.5s seconds", seconds) time.sleep(seconds) @@ -135,12 +156,12 @@ class Extractor(): if code < 500 and code != 429 and code != 430: break finally: - Extractor._request_last = time.time() + Extractor.request_timestamp = time.time() self.log.debug("%s (%s/%s)", msg, tries, retries+1) if tries > retries: break - time.sleep(tries) + time.sleep(max(tries, self.request_interval)) tries += 1 raise exception.HttpError(msg, response) @@ -506,28 +527,6 @@ class AsynchronousMixin(): messages.put(None) -class SharedConfigMixin(): - """Enable sharing of config settings based on 'basecategory'""" - basecategory = "" - - def config(self, key, default=None): - return config.interpolate_common( - ("extractor",), ( - (self.category, self.subcategory), - (self.basecategory, self.subcategory), - ), key, default, - ) - - def config_accumulate(self, key): - values = config.accumulate(self._cfgpath, key) - - conf = config.get(("extractor",), self.basecategory) - if conf: - values[:0] = config.accumulate((self.subcategory,), key, conf=conf) - - return values - - def generate_extractors(extractor_data, symtable, classes): """Dynamically generate Extractor classes""" extractors = config.get(("extractor",), classes[0].basecategory) @@ -539,7 +538,7 @@ def generate_extractors(extractor_data, symtable, classes): for category, info in extractor_data.items(): - if not isinstance(info, dict): + if not isinstance(info, dict) or "root" not in info: continue root = info["root"] |