diff options
Diffstat (limited to 'gallery_dl/extractor/common.py')
| -rw-r--r-- | gallery_dl/extractor/common.py | 23 |
1 files changed, 20 insertions, 3 deletions
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index e6c0968..357deac 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -31,6 +31,8 @@ class Extractor(): cookiedomain = "" root = "" test = None + _request_last = 0 + _request_interval = 0 def __init__(self, match): self.session = requests.Session() @@ -40,10 +42,14 @@ class Extractor(): self._cookiefile = None self._cookiejar = self.session.cookies self._parentdir = "" + + self._cfgpath = ("extractor", self.category, self.subcategory) self._write_pages = self.config("write-pages", False) self._retries = self.config("retries", 4) self._timeout = self.config("timeout", 30) self._verify = self.config("verify", True) + self._request_interval = self.config( + "sleep-request", self._request_interval) if self._retries < 0: self._retries = float("inf") @@ -69,8 +75,10 @@ class Extractor(): return 0 def config(self, key, default=None): - return config.interpolate( - ("extractor", self.category, self.subcategory), key, default) + return config.interpolate(self._cfgpath, key, default) + + def config_accumulate(self, key): + return config.accumulate(self._cfgpath, key) def request(self, url, *, method="GET", session=None, retries=None, encoding=None, fatal=True, notfound=None, **kwargs): @@ -81,6 +89,13 @@ class Extractor(): kwargs.setdefault("verify", self._verify) response = None + if self._request_interval: + seconds = (self._request_interval - + (time.time() - Extractor._request_last)) + if seconds > 0: + self.log.debug("Sleeping for %.5s seconds", seconds) + time.sleep(seconds) + while True: try: response = session.request(method, url, **kwargs) @@ -119,11 +134,13 @@ class Extractor(): msg = "'{} {}' for '{}'".format(code, reason, url) if code < 500 and code != 429 and code != 430: break + finally: + Extractor._request_last = time.time() self.log.debug("%s (%s/%s)", msg, tries, retries+1) if tries > retries: break - time.sleep(min(2 ** (tries-1), 1800)) + time.sleep(tries) tries += 1 raise exception.HttpError(msg, response) |