diff options
Diffstat (limited to 'gallery_dl/extractor/sexcom.py')
| -rw-r--r-- | gallery_dl/extractor/sexcom.py | 42 |
1 files changed, 35 insertions, 7 deletions
diff --git a/gallery_dl/extractor/sexcom.py b/gallery_dl/extractor/sexcom.py index aa2b16b..afd4eaa 100644 --- a/gallery_dl/extractor/sexcom.py +++ b/gallery_dl/extractor/sexcom.py @@ -23,9 +23,9 @@ class SexcomExtractor(Extractor): def items(self): yield Message.Version, 1 yield Message.Directory, self.metadata() - for url in self.pins(): - pin = self._parse_pin(url) - yield Message.Url, pin["url"], pin + for pin in map(self._parse_pin, self.pins()): + if pin: + yield Message.Url, pin["url"], pin def metadata(self): return {} @@ -49,8 +49,13 @@ class SexcomExtractor(Extractor): return url = text.urljoin(self.root, url) - def _parse_pin(self, pin_url): - extr = text.extract_from(self.request(pin_url).text) + def _parse_pin(self, url): + response = self.request(url, fatal=False) + if response.status_code >= 400: + self.log.warning('Unable to fetch %s ("%s: %s")', + url, response.status_code, response.reason) + return None + extr = text.extract_from(response.text) data = {} data["thumbnail"] = extr('itemprop="thumbnail" content="', '"') @@ -88,10 +93,10 @@ class SexcomExtractor(Extractor): class SexcomPinExtractor(SexcomExtractor): - """Extractor a pinned image or video on www.sex.com""" + """Extractor for a pinned image or video on www.sex.com""" subcategory = "pin" directory_fmt = ("{category}",) - pattern = r"(?:https?://)?(?:www\.)?sex\.com/pin/(\d+)" + pattern = r"(?:https?://)?(?:www\.)?sex\.com/pin/(\d+)(?!.*#related$)" test = ( # picture ("https://www.sex.com/pin/56714360/", { @@ -124,6 +129,10 @@ class SexcomPinExtractor(SexcomExtractor): ("https://www.sex.com/pin/55847384-very-nicely-animated/", { "pattern": "ytdl:https://www.pornhub.com/embed/ph56ef24b6750f2", }), + # 404 + ("https://www.sex.com/pin/55847385/", { + "count": 0, + }), ) def __init__(self, match): @@ -134,6 +143,25 @@ class SexcomPinExtractor(SexcomExtractor): return ("{}/pin/{}/".format(self.root, self.pin_id),) +class SexcomRelatedPinExtractor(SexcomPinExtractor): + """Extractor for related pins on www.sex.com""" + subcategory = "related-pin" + directory_fmt = ("{category}", "related {original_pin[pin_id]}") + pattern = r"(?:https?://)?(?:www\.)?sex\.com/pin/(\d+).*#related$" + test = ("https://www.sex.com/pin/56714360/#related", { + "count": 24, + }) + + def metadata(self): + pin = self._parse_pin(SexcomPinExtractor.pins(self)[0]) + return {"original_pin": pin} + + def pins(self): + url = "{}/pin/related?pinId={}&limit=24&offset=0".format( + self.root, self.pin_id) + return self._pagination(url) + + class SexcomBoardExtractor(SexcomExtractor): """Extractor for pins from a board on www.sex.com""" subcategory = "board" |