1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
# -*- coding: utf-8 -*-
# Copyright 2019-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for http://blog.livedoor.jp/"""
from .common import Extractor, Message
from .. import text
class LivedoorExtractor(Extractor):
"""Base class for livedoor extractors"""
category = "livedoor"
root = "http://blog.livedoor.jp"
filename_fmt = "{post[id]}_{post[title]}_{num:>02}.{extension}"
directory_fmt = ("{category}", "{post[user]}")
archive_fmt = "{post[id]}_{hash}"
def __init__(self, match):
Extractor.__init__(self, match)
self.user = match.group(1)
def items(self):
for post in self.posts():
images = self._images(post)
if images:
yield Message.Directory, {"post": post}
for image in images:
yield Message.Url, image["url"], image
def posts(self):
"""Return an iterable with post objects"""
def _load(self, data, body):
extr = text.extract_from(data)
tags = text.extr(body, 'class="article-tags">', '</dl>')
about = extr('rdf:about="', '"')
return {
"id" : text.parse_int(
about.rpartition("/")[2].partition(".")[0]),
"title" : text.unescape(extr('dc:title="', '"')),
"categories" : extr('dc:subject="', '"').partition(",")[::2],
"description": extr('dc:description="', '"'),
"date" : text.parse_datetime(extr('dc:date="', '"')),
"tags" : text.split_html(tags)[1:] if tags else [],
"user" : self.user,
"body" : body,
}
def _images(self, post):
imgs = []
body = post.pop("body")
for num, img in enumerate(text.extract_iter(body, "<img ", ">"), 1):
src = text.extr(img, 'src="', '"')
alt = text.extr(img, 'alt="', '"')
if not src:
continue
if "://livedoor.blogimg.jp/" in src:
url = src.replace("http:", "https:", 1).replace("-s.", ".")
else:
url = text.urljoin(self.root, src)
name, _, ext = url.rpartition("/")[2].rpartition(".")
imgs.append({
"url" : url,
"num" : num,
"hash" : name,
"filename" : alt or name,
"extension": ext,
"post" : post,
})
return imgs
class LivedoorBlogExtractor(LivedoorExtractor):
"""Extractor for a user's blog on blog.livedoor.jp"""
subcategory = "blog"
pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/?(?:$|[?#])"
example = "http://blog.livedoor.jp/USER/"
def posts(self):
url = "{}/{}".format(self.root, self.user)
while url:
extr = text.extract_from(self.request(url).text)
while True:
data = extr('<rdf:RDF', '</rdf:RDF>')
if not data:
break
body = extr('class="article-body-inner">',
'class="article-footer">')
yield self._load(data, body)
url = extr('<a rel="next" href="', '"')
class LivedoorPostExtractor(LivedoorExtractor):
"""Extractor for images from a blog post on blog.livedoor.jp"""
subcategory = "post"
pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/archives/(\d+)"
example = "http://blog.livedoor.jp/USER/archives/12345.html"
def __init__(self, match):
LivedoorExtractor.__init__(self, match)
self.post_id = match.group(2)
def posts(self):
url = "{}/{}/archives/{}.html".format(
self.root, self.user, self.post_id)
extr = text.extract_from(self.request(url).text)
data = extr('<rdf:RDF', '</rdf:RDF>')
body = extr('class="article-body-inner">', 'class="article-footer">')
return (self._load(data, body),)
|
