diff options
Diffstat (limited to 'gallery_dl/extractor/weibo.py')
| -rw-r--r-- | gallery_dl/extractor/weibo.py | 352 |
1 files changed, 252 insertions, 100 deletions
diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 1929f98..a7068c8 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -10,28 +10,52 @@ from .common import Extractor, Message from .. import text, exception +from ..cache import cache import itertools +import random import json +BASE_PATTERN = r"(?:https?://)?(?:www\.|m\.)?weibo\.c(?:om|n)" +USER_PATTERN = BASE_PATTERN + r"/(?:(u|n|p(?:rofile)?)/)?([^/?#]+)(?:/home)?" + class WeiboExtractor(Extractor): category = "weibo" directory_fmt = ("{category}", "{user[screen_name]}") filename_fmt = "{status[id]}_{num:>02}.{extension}" archive_fmt = "{status[id]}_{num}" - root = "https://m.weibo.cn" + root = "https://weibo.com" request_interval = (1.0, 2.0) def __init__(self, match): Extractor.__init__(self, match) + self._prefix, self.user = match.groups() self.retweets = self.config("retweets", True) self.videos = self.config("videos", True) + self.livephoto = self.config("livephoto", True) + + cookies = _cookie_cache() + if cookies is not None: + self.session.cookies.update(cookies) + + def request(self, url, **kwargs): + response = Extractor.request(self, url, **kwargs) + + if response.history and "passport.weibo.com" in response.url: + self._sina_visitor_system(response) + response = Extractor.request(self, url, **kwargs) + + return response def items(self): original_retweets = (self.retweets == "original") for status in self.statuses(): + status["date"] = text.parse_datetime( + status["created_at"], "%a %b %d %H:%M:%S %z %Y") + yield Message.Directory, status + if self.retweets and "retweeted_status" in status: if original_retweets: status = status["retweeted_status"] @@ -45,96 +69,69 @@ class WeiboExtractor(Extractor): files = self._files_from_status(status) for num, file in enumerate(files, 1): - if num == 1: - status["date"] = text.parse_datetime( - status["created_at"], "%a %b %d %H:%M:%S %z %Y") - yield Message.Directory, status + if file["url"].startswith("http:"): + file["url"] = "https:" + file["url"][5:] + if "filename" not in file: + text.nameext_from_url(file["url"], file) file["status"] = status file["num"] = num yield Message.Url, file["url"], file - def statuses(self): - """Returns an iterable containing all relevant 'status' objects""" + def _files_from_status(self, status): + pic_ids = status.get("pic_ids") + if pic_ids: + pics = status["pic_infos"] + for pic_id in pic_ids: + pic = pics[pic_id] + pic_type = pic.get("type") - def _status_by_id(self, status_id): - url = "{}/detail/{}".format(self.root, status_id) - page = self.request(url, fatal=False).text - data = text.extract(page, "var $render_data = [", "][0] || {};")[0] - return json.loads(data)["status"] if data else None + if pic_type == "gif" and self.videos: + yield {"url": pic["video"]} - def _files_from_status(self, status): - page_info = status.pop("page_info", ()) - if "pics" in status: - if len(status["pics"]) < status["pic_num"]: - status = self._status_by_id(status["id"]) or status - for image in status.pop("pics"): - pid = image["pid"] - if "large" in image: - image = image["large"] - geo = image.get("geo") or {} - yield text.nameext_from_url(image["url"], { - "url" : image["url"], - "pid" : pid, - "width" : text.parse_int(geo.get("width")), - "height": text.parse_int(geo.get("height")), - }) - - if self.videos and "media_info" in page_info: - info = page_info["media_info"] - url = info.get("stream_url_hd") or info.get("stream_url") - if url: - data = text.nameext_from_url(url, { - "url" : url, - "pid" : 0, - "width" : 0, - "height": 0, - }) - if data["extension"] == "m3u8": - data["extension"] = "mp4" - data["url"] = "ytdl:" + url - data["_ytdl_extra"] = {"protocol": "m3u8_native"} - yield data + elif pic_type == "livephoto" and self.livephoto: + yield pic["largest"].copy() + file = {"url": pic["video"]} + file["filehame"], _, file["extension"] = \ + pic["video"].rpartition("%2F")[2].rpartition(".") + yield file -class WeiboUserExtractor(WeiboExtractor): - """Extractor for all images of a user on weibo.cn""" - subcategory = "user" - pattern = (r"(?:https?://)?(?:www\.|m\.)?weibo\.c(?:om|n)" - r"/(?:u|p(?:rofile)?)/(\d+)") - test = ( - ("https://m.weibo.cn/u/2314621010", { - "range": "1-30", - }), - # deleted (#2521) - ("https://weibo.com/u/7500315942", { - "count": 0, - }), - ("https://m.weibo.cn/profile/2314621010"), - ("https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO"), - ("https://www.weibo.com/p/1003062314621010/home"), - ) + else: + yield pic["largest"].copy() - def __init__(self, match): - WeiboExtractor.__init__(self, match) - self.user_id = match.group(1)[-10:] + if "page_info" in status: + page_info = status["page_info"] + if "media_info" not in page_info or not self.videos: + return + media = max(page_info["media_info"]["playback_list"], + key=lambda m: m["meta"]["quality_index"]) + yield media["play_info"].copy() - def statuses(self): - url = self.root + "/api/container/getIndex" + def _status_by_id(self, status_id): + url = "{}/ajax/statuses/show?id={}".format(self.root, status_id) + return self.request(url).json() + + def _user_id(self): + if self.user.isdecimal(): + return self.user[-10:] + else: + url = "{}/ajax/profile/info?{}={}".format( + self.root, + "screen_name" if self._prefix == "n" else "custom", + self.user) + return self.request(url).json()["data"]["user"]["idstr"] + + def _pagination(self, endpoint, params): + url = self.root + "/ajax" + endpoint headers = { - "Accept": "application/json, text/plain, */*", "X-Requested-With": "XMLHttpRequest", - "MWeibo-Pwa": "1", "X-XSRF-TOKEN": None, - "Referer": "{}/u/{}".format(self.root, self.user_id), - } - params = { - "type": "uid", - "value": self.user_id, - "containerid": "107603" + self.user_id, + "Referer": "{}/u/{}".format(self.root, params["uid"]), } while True: response = self.request(url, params=params, headers=headers) + headers["Accept"] = "application/json, text/plain, */*" headers["X-XSRF-TOKEN"] = response.cookies.get("XSRF-TOKEN") data = response.json() @@ -145,56 +142,211 @@ class WeiboUserExtractor(WeiboExtractor): '"%s"', data.get("msg") or "unknown error") data = data["data"] - for card in data["cards"]: - if "mblog" in card: - yield card["mblog"] - - info = data.get("cardlistInfo") - if not info: - # occasionally weibo returns an empty response - # repeating the same request usually/eventually yields - # the correct response. - continue - - params["since_id"] = sid = info.get("since_id") - if not sid: + statuses = data["list"] + if not statuses: return + yield from statuses + + if "next_cursor" in data: + params["cursor"] = data["next_cursor"] + elif "page" in params: + params["page"] += 1 + elif data["since_id"]: + params["sinceid"] = data["since_id"] + else: + params["since_id"] = statuses[-1]["id"] - 1 + + def _sina_visitor_system(self, response): + self.log.info("Sina Visitor System") + + passport_url = "https://passport.weibo.com/visitor/genvisitor" + headers = {"Referer": response.url} + data = { + "cb": "gen_callback", + "fp": '{"os":"1","browser":"Gecko91,0,0,0","fonts":"undefined",' + '"screenInfo":"1920*1080*24","plugins":""}', + } + + page = Extractor.request( + self, passport_url, method="POST", headers=headers, data=data).text + data = json.loads(text.extract(page, "(", ");")[0])["data"] + + passport_url = "https://passport.weibo.com/visitor/visitor" + params = { + "a" : "incarnate", + "t" : data["tid"], + "w" : "2", + "c" : "{:>03}".format(data["confidence"]), + "gc" : "", + "cb" : "cross_domain", + "from" : "weibo", + "_rand": random.random(), + } + response = Extractor.request(self, passport_url, params=params) + _cookie_cache.update("", response.cookies) + + +class WeiboUserExtractor(WeiboExtractor): + """Extractor for weibo user profiles""" + subcategory = "user" + pattern = USER_PATTERN + r"(?:$|#)" + test = ( + ("https://weibo.com/1758989602"), + ("https://weibo.com/u/1758989602"), + ("https://weibo.com/p/1758989602"), + ("https://m.weibo.cn/profile/2314621010"), + ("https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO"), + ("https://www.weibo.com/p/1003062314621010/home"), + ) + + def items(self): + base = " {}/u/{}?tabtype=".format(self.root, self._user_id()) + return self._dispatch_extractors(( + (WeiboHomeExtractor , base + "home"), + (WeiboFeedExtractor , base + "feed"), + (WeiboVideosExtractor, base + "newVideo"), + (WeiboAlbumExtractor , base + "album"), + ), ("feed",)) + + +class WeiboHomeExtractor(WeiboExtractor): + """Extractor for weibo 'home' listings""" + subcategory = "home" + pattern = USER_PATTERN + r"\?tabtype=home" + test = ("https://weibo.com/1758989602?tabtype=home", { + "range": "1-30", + "count": 30, + }) + + def statuses(self): + endpoint = "/profile/myhot" + params = {"uid": self._user_id(), "page": 1, "feature": "2"} + return self._pagination(endpoint, params) + + +class WeiboFeedExtractor(WeiboExtractor): + """Extractor for weibo user feeds""" + subcategory = "feed" + pattern = USER_PATTERN + r"\?tabtype=feed" + test = ( + ("https://weibo.com/1758989602?tabtype=feed", { + "range": "1-30", + "count": 30, + }), + ("https://weibo.com/zhouyuxi77?tabtype=feed", { + "keyword": {"status": {"user": {"id": 7488709788}}}, + "range": "1", + }), + ("https://www.weibo.com/n/周于希Sally?tabtype=feed", { + "keyword": {"status": {"user": {"id": 7488709788}}}, + "range": "1", + }), + # deleted (#2521) + ("https://weibo.com/u/7500315942?tabtype=feed", { + "count": 0, + }), + ) + + def statuses(self): + endpoint = "/statuses/mymblog" + params = {"uid": self._user_id(), "feature": "0"} + return self._pagination(endpoint, params) + + +class WeiboVideosExtractor(WeiboExtractor): + """Extractor for weibo 'newVideo' listings""" + subcategory = "videos" + pattern = USER_PATTERN + r"\?tabtype=newVideo" + test = ("https://weibo.com/1758989602?tabtype=newVideo", { + "pattern": r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", + "range": "1-30", + "count": 30, + }) + + def statuses(self): + endpoint = "/profile/getWaterFallContent" + params = {"uid": self._user_id()} + return self._pagination(endpoint, params) + + +class WeiboArticleExtractor(WeiboExtractor): + """Extractor for weibo 'article' listings""" + subcategory = "article" + pattern = USER_PATTERN + r"\?tabtype=article" + test = ("https://weibo.com/1758989602?tabtype=article", { + "count": 0, + }) + + def statuses(self): + endpoint = "/statuses/mymblog" + params = {"uid": self._user_id(), "page": 1, "feature": "10"} + return self._pagination(endpoint, params) + + +class WeiboAlbumExtractor(WeiboExtractor): + """Extractor for weibo 'album' listings""" + subcategory = "album" + pattern = USER_PATTERN + r"\?tabtype=album" + test = ("https://weibo.com/1758989602?tabtype=album", { + "pattern": r"https://wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)", + "range": "1-3", + "count": 3, + }) + + def statuses(self): + endpoint = "/profile/getImageWall" + params = {"uid": self._user_id()} + + seen = set() + for image in self._pagination(endpoint, params): + mid = image["mid"] + if mid not in seen: + seen.add(mid) + yield self._status_by_id(mid) class WeiboStatusExtractor(WeiboExtractor): """Extractor for images from a status on weibo.cn""" subcategory = "status" - pattern = (r"(?:https?://)?(?:www\.|m\.)?weibo\.c(?:om|n)" - r"/(?:detail|status|\d+)/(\w+)") + pattern = BASE_PATTERN + r"/(detail|status|\d+)/(\w+)" test = ( ("https://m.weibo.cn/detail/4323047042991618", { "pattern": r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", "keyword": {"status": {"date": "dt:2018-12-30 13:56:36"}}, }), ("https://m.weibo.cn/detail/4339748116375525", { - "pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_hd", + "pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", }), # unavailable video (#427) ("https://m.weibo.cn/status/4268682979207023", { - "exception": exception.NotFoundError, + "exception": exception.HttpError, }), # non-numeric status ID (#664) ("https://weibo.com/3314883543/Iy7fj4qVg"), # original retweets (#1542) ("https://m.weibo.cn/detail/4600272267522211", { "options": (("retweets", "original"),), - "keyword": {"status": {"id": "4600167083287033"}}, + "keyword": {"status": {"id": 4600167083287033}}, + }), + # type == livephoto (#2146) + ("https://weibo.com/5643044717/KkuDZ4jAA", { + "range": "2,4,6", + "pattern": r"https://video\.weibo\.com/media/play\?livephoto=" + r"https%3A%2F%2Fus.sinaimg.cn%2F\w+\.mov", + }), + # type == gif + ("https://weibo.com/1758989602/LvBhm5DiP", { + "pattern": r"http://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM01041" + r"20005tc0E010\.mp4\?label=gif_mp4", }), ("https://m.weibo.cn/status/4339748116375525"), ("https://m.weibo.cn/5746766133/4339748116375525"), ) - def __init__(self, match): - WeiboExtractor.__init__(self, match) - self.status_id = match.group(1) - def statuses(self): - status = self._status_by_id(self.status_id) - if not status: - raise exception.NotFoundError("status") - return (status,) + return (self._status_by_id(self.user),) + + +@cache(maxage=356*86400) +def _cookie_cache(): + return None |