1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
# -*- coding: utf-8 -*-
# Copyright 2019-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://www.plurk.com/"""
from .common import Extractor, Message
from .. import text, util, exception
import datetime
import re
class PlurkExtractor(Extractor):
"""Base class for plurk extractors"""
category = "plurk"
root = "https://www.plurk.com"
request_interval = (0.5, 1.5)
def items(self):
urls = self._urls_ex if self.config("comments", False) else self._urls
for plurk in self.plurks():
for url in urls(plurk):
yield Message.Queue, url, plurk
def plurks(self):
"""Return an iterable with all relevant 'plurk' objects"""
@staticmethod
def _urls(obj):
"""Extract URLs from a 'plurk' object"""
return text.extract_iter(obj["content"], ' href="', '"')
def _urls_ex(self, plurk):
"""Extract URLs from a 'plurk' and its comments"""
yield from self._urls(plurk)
for comment in self._comments(plurk):
yield from self._urls(comment)
def _comments(self, plurk):
"""Return an iterable with a 'plurk's comments"""
url = "https://www.plurk.com/Responses/get"
data = {"plurk_id": plurk["id"], "count": "200"}
headers = {
"Origin": self.root,
"Referer": self.root,
"X-Requested-With": "XMLHttpRequest",
}
while True:
info = self.request(
url, method="POST", headers=headers, data=data).json()
yield from info["responses"]
if not info["has_newer"]:
return
elif info["has_newer"] < 200:
del data["count"]
data["from_response_id"] = info["responses"][-1]["id"] + 1
@staticmethod
def _load(data):
if not data:
raise exception.NotFoundError("user")
return util.json_loads(re.sub(r"new Date\(([^)]+)\)", r"\1", data))
class PlurkTimelineExtractor(PlurkExtractor):
"""Extractor for URLs from all posts in a Plurk timeline"""
subcategory = "timeline"
pattern = r"(?:https?://)?(?:www\.)?plurk\.com/(?!p/)(\w+)/?(?:$|[?#])"
example = "https://www.plurk.com/USER"
def __init__(self, match):
PlurkExtractor.__init__(self, match)
self.user = match.group(1)
def plurks(self):
url = "{}/{}".format(self.root, self.user)
page = self.request(url).text
user_id, pos = text.extract(page, '"page_user": {"id":', ',')
plurks = self._load(text.extract(page, "_PLURKS = ", ";\n", pos)[0])
headers = {"Referer": url, "X-Requested-With": "XMLHttpRequest"}
data = {"user_id": user_id.strip()}
url = "https://www.plurk.com/TimeLine/getPlurks"
while plurks:
yield from plurks
offset = datetime.datetime.strptime(
plurks[-1]["posted"], "%a, %d %b %Y %H:%M:%S %Z")
data["offset"] = offset.strftime("%Y-%m-%dT%H:%M:%S.000Z")
response = self.request(
url, method="POST", headers=headers, data=data)
plurks = response.json()["plurks"]
class PlurkPostExtractor(PlurkExtractor):
"""Extractor for URLs from a Plurk post"""
subcategory = "post"
pattern = r"(?:https?://)?(?:www\.)?plurk\.com/p/(\w+)"
example = "https://www.plurk.com/p/12345"
def __init__(self, match):
PlurkExtractor.__init__(self, match)
self.plurk_id = match.group(1)
def plurks(self):
url = "{}/p/{}".format(self.root, self.plurk_id)
page = self.request(url).text
user, pos = text.extract(page, " GLOBAL = ", "\n")
data, pos = text.extract(page, "plurk = ", ";\n", pos)
data = self._load(data)
data["user"] = self._load(user)["page_user"]
return (data,)
|
