1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
# -*- coding: utf-8 -*-
# Copyright 2022-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://toyhou.se/"""
from .common import Extractor, Message
from .. import text, util
BASE_PATTERN = r"(?:https?://)?(?:www\.)?toyhou\.se"
class ToyhouseExtractor(Extractor):
"""Base class for toyhouse extractors"""
category = "toyhouse"
root = "https://toyhou.se"
directory_fmt = ("{category}", "{user|artists!S}")
archive_fmt = "{id}"
def __init__(self, match):
Extractor.__init__(self, match)
self.user = match.group(1)
self.offset = 0
def items(self):
metadata = self.metadata()
for post in util.advance(self.posts(), self.offset):
if metadata:
post.update(metadata)
text.nameext_from_url(post["url"], post)
post["id"], _, post["hash"] = post["filename"].partition("_")
yield Message.Directory, post
yield Message.Url, post["url"], post
def posts(self):
return ()
def metadata(self):
return None
def skip(self, num):
self.offset += num
return num
def _parse_post(self, post, needle='<a href="'):
extr = text.extract_from(post)
return {
"url": extr(needle, '"'),
"date": text.parse_datetime(extr(
'Credits\n</h2>\n<div class="mb-1">', '<'),
"%d %b %Y, %I:%M:%S %p"),
"artists": [
text.remove_html(artist)
for artist in extr(
'<div class="artist-credit">', '</div>\n</div>').split(
'<div class="artist-credit">')
],
"characters": text.split_html(extr(
'<div class="image-characters', '</div>\n</div>'))[2:],
}
def _pagination(self, path):
url = self.root + path
params = {"page": 1}
while True:
page = self.request(url, params=params).text
cnt = 0
for post in text.extract_iter(
page, '<li class="gallery-item">', '</li>'):
cnt += 1
yield self._parse_post(post)
if not cnt and params["page"] == 1:
if self._accept_content_warning(page):
continue
return
if cnt < 18:
return
params["page"] += 1
def _accept_content_warning(self, page):
pos = page.find(' name="_token"') + 1
token, pos = text.extract(page, ' value="', '"', pos)
user , pos = text.extract(page, ' value="', '"', pos)
if not token or not user:
return False
data = {"_token": token, "user": user}
self.request(self.root + "/~account/warnings/accept",
method="POST", data=data, allow_redirects=False)
return True
class ToyhouseArtExtractor(ToyhouseExtractor):
"""Extractor for artworks of a toyhouse user"""
subcategory = "art"
pattern = BASE_PATTERN + r"/([^/?#]+)/art"
example = "https://www.toyhou.se/USER/art"
def posts(self):
return self._pagination("/{}/art".format(self.user))
def metadata(self):
return {"user": self.user}
class ToyhouseImageExtractor(ToyhouseExtractor):
"""Extractor for individual toyhouse images"""
subcategory = "image"
pattern = (r"(?:https?://)?(?:"
r"(?:www\.)?toyhou\.se/~images|"
r"f\d+\.toyhou\.se/file/[^/?#]+/(?:image|watermark)s"
r")/(\d+)")
example = "https://toyhou.se/~images/12345"
def posts(self):
url = "{}/~images/{}".format(self.root, self.user)
return (self._parse_post(
self.request(url).text, '<img class="mw-100" src="'),)
|
