diff options
Diffstat (limited to 'gallery_dl/extractor/furaffinity.py')
| -rw-r--r-- | gallery_dl/extractor/furaffinity.py | 33 |
1 files changed, 24 insertions, 9 deletions
diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py index 86e1678..8c2887e 100644 --- a/gallery_dl/extractor/furaffinity.py +++ b/gallery_dl/extractor/furaffinity.py @@ -29,9 +29,10 @@ class FuraffinityExtractor(Extractor): self.offset = 0 if self.config("descriptions") == "html": - self._process_description = lambda x: x.strip() + self._process_description = str.strip def items(self): + external = self.config("external", False) metadata = self.metadata() for post_id in util.advance(self.posts(), self.offset): post = self._parse_post(post_id) @@ -41,8 +42,10 @@ class FuraffinityExtractor(Extractor): yield Message.Directory, post yield Message.Url, post["url"], post - def posts(self): - return self._pagination() + if external: + for url in text.extract_iter( + post["_description"], 'href="http', '"'): + yield Message.Queue, "http" + url, post def metadata(self): return None @@ -80,8 +83,7 @@ class FuraffinityExtractor(Extractor): data["tags"] = text.split_html(tags) data["title"] = text.unescape(extr("<h2><p>", "</p></h2>")) data["artist"] = extr("<strong>", "<") - data["description"] = self._process_description(extr( - 'class="section-body">', '</div>')) + data["_description"] = extr('class="section-body">', '</div>') data["views"] = pi(rh(extr('class="views">', '</span>'))) data["favorites"] = pi(rh(extr('class="favorites">', '</span>'))) data["comments"] = pi(rh(extr('class="comments">', '</span>'))) @@ -108,12 +110,12 @@ class FuraffinityExtractor(Extractor): data["tags"] = text.split_html(extr( 'id="keywords">', '</div>'))[::2] data["rating"] = extr('<img alt="', ' ') - data["description"] = self._process_description(extr( - "</table>", "</table>")) + data["_description"] = extr("</table>", "</table>") data["artist_url"] = data["artist"].replace("_", "").lower() data["user"] = self.user or data["artist_url"] data["date"] = text.parse_timestamp(data["filename"].partition(".")[0]) + data["description"] = self._process_description(data["_description"]) return data @@ -121,12 +123,12 @@ class FuraffinityExtractor(Extractor): def _process_description(description): return text.unescape(text.remove_html(description, "", "")) - def _pagination(self): + def _pagination(self, path): num = 1 while True: url = "{}/{}/{}/{}/".format( - self.root, self.subcategory, self.user, num) + self.root, path, self.user, num) page = self.request(url).text post_id = None @@ -191,6 +193,9 @@ class FuraffinityGalleryExtractor(FuraffinityExtractor): "count": 6, }) + def posts(self): + return self._pagination("gallery") + class FuraffinityScrapsExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's scraps""" @@ -203,6 +208,9 @@ class FuraffinityScrapsExtractor(FuraffinityExtractor): "count": ">= 3", }) + def posts(self): + return self._pagination("scraps") + class FuraffinityFavoriteExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's favorites""" @@ -273,6 +281,13 @@ class FuraffinityPostExtractor(FuraffinityExtractor): "height" : 120, }, }), + # 'external' option (#1492) + ("https://www.furaffinity.net/view/42166511/", { + "options": (("external", True),), + "pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/" + r"|http://www\.postybirb\.com", + "count": 2, + }), ("https://furaffinity.net/view/21835115/"), ("https://sfw.furaffinity.net/view/21835115/"), ("https://www.furaffinity.net/full/21835115/"), |