diff options
Diffstat (limited to 'gallery_dl/extractor/exhentai.py')
| -rw-r--r-- | gallery_dl/extractor/exhentai.py | 93 |
1 files changed, 48 insertions, 45 deletions
diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index a479d00..acad95c 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -26,7 +26,7 @@ class ExhentaiExtractor(Extractor): cookies_domain = ".exhentai.org" cookies_names = ("ipb_member_id", "ipb_pass_hash") root = "https://exhentai.org" - request_interval = 5.0 + request_interval = (3.0, 6.0) ciphers = "DEFAULT:!DH" LIMIT = False @@ -67,14 +67,15 @@ class ExhentaiExtractor(Extractor): if username: return self.cookies_update(self._login_impl(username, password)) - self.log.info("no username given; using e-hentai.org") - self.root = "https://e-hentai.org" - self.cookies_domain = ".e-hentai.org" - self.cookies.set("nw", "1", domain=self.cookies_domain) + if self.version == "ex": + self.log.info("No username or cookies given; using e-hentai.org") + self.root = "https://e-hentai.org" + self.cookies_domain = ".e-hentai.org" + self.cookies.set("nw", "1", domain=self.cookies_domain) self.original = False self.limits = False - @cache(maxage=90*24*3600, keyarg=1) + @cache(maxage=90*86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) @@ -124,6 +125,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): self.key_show = None self.key_next = None self.count = 0 + self.data = None def _init(self): source = self.config("source") @@ -138,11 +140,15 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): self.limits = False self.fallback_retries = self.config("fallback-retries", 2) - if self.fallback_retries < 0: - self.fallback_retries = float("inf") - self.original = self.config("original", True) + def finalize(self): + if self.data: + self.log.info("Use '%s/s/%s/%s-%s' as input URL " + "to continue downloading from the current position", + self.root, self.data["image_token"], + self.gallery_id, self.data["num"]) + def favorite(self, slot="0"): url = self.root + "/gallerypopups.php" params = { @@ -178,32 +184,10 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): self.gallery_token = part.split("/")[1] gpage = self._gallery_page() - data = self.get_metadata(gpage) + self.data = data = self.get_metadata(gpage) self.count = text.parse_int(data["filecount"]) yield Message.Directory, data - def _validate_response(response): - # declared inside 'items()' to be able to access 'data' - if not response.history and response.headers.get( - "content-type", "").startswith("text/html"): - page = response.text - self.log.warning("'%s'", page) - - if " requires GP" in page: - gp = self.config("gp") - if gp == "stop": - raise exception.StopExtraction("Not enough GP") - elif gp == "wait": - input("Press ENTER to continue.") - return response.url - - self.log.info("Falling back to non-original downloads") - self.original = False - return data["_url_1280"] - - self._report_limits(data) - return True - images = itertools.chain( (self.image_from_page(ipage),), self.images_from_api()) for url, image in images: @@ -211,7 +195,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): if self.limits: self._check_limits(data) if "/fullimg" in url: - data["_http_validate"] = _validate_response + data["_http_validate"] = self._validate_response else: data["_http_validate"] = None yield Message.Url, url, data @@ -219,6 +203,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): fav = self.config("fav") if fav is not None: self.favorite(fav) + self.data = None def _items_hitomi(self): if self.config("metadata", False): @@ -332,7 +317,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): data["_nl"] = nl self.key_show = extr('var showkey="', '";') - self._check_509(iurl, data) + self._check_509(iurl) return url, text.nameext_from_url(url, data) def images_from_api(self): @@ -382,33 +367,51 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): data["_url_1280"] = imgurl data["_nl"] = nl - self._check_509(imgurl, data) + self._check_509(imgurl) yield url, text.nameext_from_url(url, data) request["imgkey"] = nextkey - def _report_limits(self, data): + def _validate_response(self, response): + if not response.history and response.headers.get( + "content-type", "").startswith("text/html"): + page = response.text + self.log.warning("'%s'", page) + + if " requires GP" in page: + gp = self.config("gp") + if gp == "stop": + raise exception.StopExtraction("Not enough GP") + elif gp == "wait": + input("Press ENTER to continue.") + return response.url + + self.log.info("Falling back to non-original downloads") + self.original = False + return self.data["_url_1280"] + + self._report_limits() + return True + + def _report_limits(self): ExhentaiExtractor.LIMIT = True - raise exception.StopExtraction( - "Image limit reached! " - "Continue with '%s/s/%s/%s-%s' as URL after resetting it.", - self.root, data["image_token"], self.gallery_id, data["num"]) + raise exception.StopExtraction("Image limit reached!") def _check_limits(self, data): if not self._remaining or data["num"] % 25 == 0: self._update_limits() self._remaining -= data["cost"] if self._remaining <= 0: - self._report_limits(data) + self._report_limits() - def _check_509(self, url, data): + def _check_509(self, url): # full 509.gif URLs # - https://exhentai.org/img/509.gif # - https://ehgt.org/g/509.gif if url.endswith(("hentai.org/img/509.gif", "ehgt.org/g/509.gif")): self.log.debug(url) - self._report_limits(data) + self._report_limits() def _update_limits(self): url = "https://e-hentai.org/home.php" @@ -449,14 +452,14 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): def _fallback_original(self, nl, fullimg): url = "{}?nl={}".format(fullimg, nl) - for _ in range(self.fallback_retries): + for _ in util.repeat(self.fallback_retries): yield url def _fallback_1280(self, nl, num, token=None): if not token: token = self.key_start - for _ in range(self.fallback_retries): + for _ in util.repeat(self.fallback_retries): url = "{}/s/{}/{}-{}?nl={}".format( self.root, token, self.gallery_id, num, nl) |