1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://girlsreleased.com/"""
from .common import Extractor, Message
from .. import text
import itertools
BASE_PATTERN = r"(?:https?://)?(?:www\.)?girlsreleased\.com"
class GirlsreleasedExtractor(Extractor):
"""Base class for girlsreleased extractors"""
category = "girlsreleased"
root = "https://girlsreleased.com"
request_interval = 0.5
request_interval_min = 0.2
def items(self):
data = {"_extractor": GirlsreleasedSetExtractor}
base = f"{self.root}/set/"
for set in self._pagination():
yield Message.Queue, f"{base}{set[0]}", data
def _pagination(self):
base = f"{self.root}/api/0.2/sets/{self._path}/{self.groups[0]}/page/"
for pnum in itertools.count():
sets = self.request_json(f"{base}{pnum}")["sets"]
if not sets:
return
yield from sets[1:] if pnum else sets
if len(sets) < 80:
return
class GirlsreleasedSetExtractor(GirlsreleasedExtractor):
"""Extractor for girlsreleased galleries"""
subcategory = "set"
pattern = rf"{BASE_PATTERN}/set/(\d+)"
example = "https://girlsreleased.com/set/12345"
def items(self):
url = f"{self.root}/api/0.2/set/{self.groups[0]}"
json = self.request_json(url)["set"]
data = {
"title": json["name"] or json["id"],
"id": json["id"],
"site": json["site"],
"model": [model for _, model in json["models"]],
"date": self.parse_timestamp(json["date"]),
"count": len(json["images"]),
"url": "https://girlsreleased.com/set/" + json["id"],
}
yield Message.Directory, "", data
for data["num"], image in enumerate(json["images"], 1):
text.nameext_from_url(image[5], data)
yield Message.Queue, image[3], data
class GirlsreleasedModelExtractor(GirlsreleasedExtractor):
"""Extractor for girlsreleased models"""
subcategory = _path = "model"
pattern = rf"{BASE_PATTERN}/model/(\d+(?:/.+)?)"
example = "https://girlsreleased.com/model/12345/MODEL"
class GirlsreleasedSiteExtractor(GirlsreleasedExtractor):
"""Extractor for girlsreleased sites"""
subcategory = _path = "site"
pattern = rf"{BASE_PATTERN}/site/([^/?#]+(?:/model/\d+/?.*)?)"
example = "https://girlsreleased.com/site/SITE"
|
