1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
# -*- coding: utf-8 -*-
# Copyright 2017-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://warosu.org/"""
from .common import Extractor, Message
from .. import text
class WarosuThreadExtractor(Extractor):
"""Extractor for threads on warosu.org"""
category = "warosu"
subcategory = "thread"
root = "https://warosu.org"
directory_fmt = ("{category}", "{board}", "{thread} - {title}")
filename_fmt = "{tim} {filename}.{extension}"
archive_fmt = "{board}_{thread}_{tim}"
pattern = r"(?:https?://)?(?:www\.)?warosu\.org/([^/]+)/thread/(\d+)"
example = "https://warosu.org/a/thread/12345"
def __init__(self, match):
Extractor.__init__(self, match)
self.board, self.thread = match.groups()
def items(self):
url = f"{self.root}/{self.board}/thread/{self.thread}"
page = self.request(url).text
data = self.metadata(page)
posts = self.posts(page)
if not data["title"]:
data["title"] = text.unescape(text.remove_html(
posts[0]["com"]))[:50]
yield Message.Directory, "", data
for post in posts:
if "image" in post:
for key in ("w", "h", "no", "time", "tim"):
post[key] = text.parse_int(post[key])
dt = self.parse_timestamp(post["time"])
# avoid zero-padding 'day' with %d
post["now"] = dt.strftime(f"%a, %b {dt.day}, %Y %H:%M:%S")
post.update(data)
yield Message.Url, post["image"], post
def metadata(self, page):
boardname = text.extr(page, "<title>", "</title>")
title = text.unescape(text.extr(page, "class=filetitle>", "<"))
return {
"board" : self.board,
"board_name": boardname.split(" - ")[1],
"thread" : self.thread,
"title" : title,
}
def posts(self, page):
"""Build a list of all post objects"""
page = text.extr(page, "<div class=content", "</form>")
needle = "<table>"
return [self.parse(post) for post in page.split(needle)]
def parse(self, post):
"""Build post object by extracting data from an HTML post"""
data = self._extract_post(post)
if '<span class="fileinfo' in post and \
self._extract_image(post, data):
part = data["image"].rpartition("/")[2]
data["tim"], _, data["extension"] = part.partition(".")
data["ext"] = "." + data["extension"]
return data
def _extract_post(self, post):
extr = text.extract_from(post)
return {
"no" : extr("id=p", ">"),
"name": extr("class=postername>", "<").strip(),
"time": extr("class=posttime title=", "000>"),
"com" : text.unescape(text.remove_html(extr(
"<blockquote>", "</blockquote>").strip())),
}
def _extract_image(self, post, data):
extr = text.extract_from(post)
extr('<span class="fileinfo">', "")
data["fsize"] = extr("File: ", ", ")
data["w"] = extr("", "x")
data["h"] = extr("", ", ")
data["filename"] = text.unquote(extr(
"", "<").rstrip().rpartition(".")[0])
extr("<br>", "")
if url := extr("<a href=", ">"):
if url[0] == "/":
data["image"] = self.root + url
elif "warosu." not in url:
return False
else:
data["image"] = url
return True
return False
|
