diff options
Diffstat (limited to 'gallery_dl/extractor/twitter.py')
| -rw-r--r-- | gallery_dl/extractor/twitter.py | 80 |
1 files changed, 65 insertions, 15 deletions
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 7b6bf21..a7d2de5 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -362,6 +362,23 @@ class TwitterListMembersExtractor(TwitterExtractor): yield Message.Queue, url, user +class TwitterFollowingExtractor(TwitterExtractor): + """Extractor for followed users""" + subcategory = "following" + pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/following(?!\w)" + test = ( + ("https://twitter.com/supernaturepics/following"), + ("https://www.twitter.com/id:2976459548/following"), + ) + + def items(self): + self.login() + for user in TwitterAPI(self).user_following(self.user): + user["_extractor"] = TwitterTimelineExtractor + url = "{}/i/user/{}".format(self.root, user["rest_id"]) + yield Message.Queue, url, user + + class TwitterSearchExtractor(TwitterExtractor): """Extractor for all images from a search timeline""" subcategory = "search" @@ -451,6 +468,11 @@ class TwitterTweetExtractor(TwitterExtractor): "date" : "dt:2020-08-20 04:00:28", }, }), + # all Tweets from a conversation (#1319) + ("https://twitter.com/BlankArts_/status/1323314488611872769", { + "options": (("conversations", True),), + "count": ">= 50", + }), ) def __init__(self, match): @@ -458,6 +480,8 @@ class TwitterTweetExtractor(TwitterExtractor): self.tweet_id = match.group(2) def tweets(self): + if self.config("conversations", False): + return TwitterAPI(self).conversation(self.tweet_id) return TwitterAPI(self).tweet(self.tweet_id) @@ -537,6 +561,10 @@ class TwitterAPI(): break return tweets + def conversation(self, conversation_id): + endpoint = "/2/timeline/conversation/{}.json".format(conversation_id) + return self._pagination(endpoint) + def timeline_profile(self, screen_name): user_id = self._user_id_by_screen_name(screen_name) endpoint = "/2/timeline/profile/{}.json".format(user_id) @@ -577,18 +605,8 @@ class TwitterAPI(): params["spelling_corrections"] = "1" return self._pagination(endpoint, params) - def list_members(self, list_id): - endpoint = "/graphql/3pV4YlpljXUTFAa1jVNWQw/ListMembers" - variables = { - "listId": list_id, - "count" : 20, - "withTweetResult": False, - "withUserResult" : False, - } - return self._pagination_members(endpoint, variables) - def list_by_rest_id(self, list_id): - endpoint = "/graphql/EhaI2uiCBJI97e28GN8WjQ/ListByRestId" + endpoint = "/graphql/18MAHTcDU-TdJSjWWmoH7w/ListByRestId" params = {"variables": '{"listId":"' + list_id + '"' ',"withUserResult":false}'} try: @@ -596,8 +614,33 @@ class TwitterAPI(): except KeyError: raise exception.NotFoundError("list") + def list_members(self, list_id): + endpoint = "/graphql/tA7h9hy4U0Yc9COfIOh3qQ/ListMembers" + variables = { + "listId": list_id, + "count" : 100, + "withTweetResult": False, + "withUserResult" : False, + } + return self._pagination_graphql( + endpoint, variables, "list", "members_timeline") + + def user_following(self, screen_name): + endpoint = "/graphql/Q_QTiPvoXwsA13eoA7okIQ/Following" + variables = { + "userId": self._user_id_by_screen_name(screen_name), + "count" : 100, + "withTweetResult": False, + "withUserResult" : False, + "withTweetQuoteCount" : False, + "withHighlightedLabel" : False, + "includePromotedContent": False, + } + return self._pagination_graphql( + endpoint, variables, "user", "following_timeline") + def user_by_screen_name(self, screen_name): - endpoint = "/graphql/ZRnOhhXPwue_JGILb9TNug/UserByScreenName" + endpoint = "/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName" params = {"variables": '{"screen_name":"' + screen_name + '"' ',"withHighlightedLabel":true}'} try: @@ -691,6 +734,13 @@ class TwitterAPI(): tweet = True cursor = cursor["value"] + elif entry_startswith("conversationThread-"): + tweet_ids.extend( + item["entryId"][6:] + for item in entry["content"]["timelineModule"]["items"] + if item["entryId"].startswith("tweet-") + ) + # process tweets for tweet_id in tweet_ids: try: @@ -728,15 +778,15 @@ class TwitterAPI(): return params["cursor"] = cursor - def _pagination_members(self, endpoint, variables): + def _pagination_graphql(self, endpoint, variables, key, timeline): while True: cursor = entry = stop = None params = {"variables": json.dumps(variables)} data = self._call(endpoint, params) try: - instructions = (data["data"]["list"]["members_timeline"] - ["timeline"]["instructions"]) + instructions = \ + data["data"][key][timeline]["timeline"]["instructions"] except KeyError: raise exception.AuthorizationError() |