diff options
Diffstat (limited to 'gallery_dl/extractor/twitter.py')
| -rw-r--r-- | gallery_dl/extractor/twitter.py | 410 |
1 files changed, 269 insertions, 141 deletions
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 4303524..c928507 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -16,6 +16,7 @@ import random BASE_PATTERN = (r"(?:https?://)?(?:www\.|mobile\.)?" r"(?:(?:[fv]x)?twitter|(?:fix(?:up|v))?x)\.com") +USER_PATTERN = rf"{BASE_PATTERN}/([^/?#]+)" class TwitterExtractor(Extractor): @@ -47,8 +48,9 @@ class TwitterExtractor(Extractor): self.cards_blacklist = self.config("cards-blacklist") if not self.config("transform", True): - self._transform_user = util.identity - self._transform_tweet = util.identity + self._transform_community = \ + self._transform_tweet = \ + self._transform_user = util.identity self._cursor = None self._user = None @@ -412,6 +414,11 @@ class TwitterExtractor(Extractor): content = tget("full_text") or tget("text") or "" entities = legacy["entities"] + if "author_community_relationship" in tweet: + tdata["community"] = self._transform_community( + tweet["author_community_relationship"] + ["community_results"]["result"]) + if hashtags := entities.get("hashtags"): tdata["hashtags"] = [t["text"] for t in hashtags] @@ -453,6 +460,36 @@ class TwitterExtractor(Extractor): return tdata + def _transform_community(self, com): + try: + cid = com.get("id_str") or com["rest_id"] + except KeyError: + return {} + + try: + return self._user_cache[f"C#{cid}"] + except KeyError: + pass + + self._user_cache[f"C#{cid}"] = cdata = { + "id": text.parse_int(cid), + "name": com["name"], + "description": com["description"], + "date": text.parse_timestamp(com["created_at"] // 1000), + "nsfw": com["is_nsfw"], + "role": com["role"], + "member_count": com["member_count"], + "rules": [rule["name"] for rule in com["rules"]], + "admin": (admin := com.get("admin_results")) and + admin["result"]["core"]["screen_name"], # noqa: E131 + "creator": (creator := com.get("creator_results")) and + creator["result"]["core"]["screen_name"], # noqa: E131 + "banner": (banner := com.get("custom_banner_media")) and + banner["media_info"]["original_img_url"], # noqa: E131 + } + + return cdata + def _transform_user(self, user): try: uid = user.get("rest_id") or user["id_str"] @@ -465,35 +502,35 @@ class TwitterExtractor(Extractor): except KeyError: pass - if "legacy" in user: - user = user["legacy"] + core = user.get("core") or user + legacy = user.get("legacy") or user + lget = legacy.get - uget = user.get - if uget("withheld_scope"): - self.log.warning("'%s'", uget("description")) + if lget("withheld_scope"): + self.log.warning("'%s'", lget("description")) - entities = user["entities"] + entities = legacy["entities"] self._user_cache[uid] = udata = { "id" : text.parse_int(uid), - "name" : user["screen_name"], - "nick" : user["name"], - "location" : uget("location"), + "name" : core["screen_name"], + "nick" : core["name"], + "location" : user["location"]["location"], "date" : text.parse_datetime( - uget("created_at"), "%a %b %d %H:%M:%S %z %Y"), - "verified" : uget("verified", False), - "protected" : uget("protected", False), - "profile_banner" : uget("profile_banner_url", ""), - "profile_image" : uget( - "profile_image_url_https", "").replace("_normal.", "."), - "favourites_count": uget("favourites_count"), - "followers_count" : uget("followers_count"), - "friends_count" : uget("friends_count"), - "listed_count" : uget("listed_count"), - "media_count" : uget("media_count"), - "statuses_count" : uget("statuses_count"), + core["created_at"], "%a %b %d %H:%M:%S %z %Y"), + "verified" : user["verification"]["verified"], + "protected" : user["privacy"]["protected"], + "profile_banner" : lget("profile_banner_url", ""), + "profile_image" : user["avatar"]["image_url"].replace( + "_normal.", "."), + "favourites_count": lget("favourites_count"), + "followers_count" : lget("followers_count"), + "friends_count" : lget("friends_count"), + "listed_count" : lget("listed_count"), + "media_count" : lget("media_count"), + "statuses_count" : lget("statuses_count"), } - descr = user["description"] + descr = legacy["description"] if urls := entities["description"].get("urls"): for url in urls: try: @@ -604,34 +641,92 @@ class TwitterExtractor(Extractor): return self.cookies_update(_login_impl(self, username, password)) +class TwitterHomeExtractor(TwitterExtractor): + """Extractor for Twitter home timelines""" + subcategory = "home" + pattern = (BASE_PATTERN + + r"/(?:home(?:/fo(?:llowing|r[-_ ]?you()))?|i/timeline)/?$") + example = "https://x.com/home" + + def tweets(self): + if self.groups[0] is None: + return self.api.home_latest_timeline() + return self.api.home_timeline() + + +class TwitterSearchExtractor(TwitterExtractor): + """Extractor for Twitter search results""" + subcategory = "search" + pattern = BASE_PATTERN + r"/search/?\?(?:[^&#]+&)*q=([^&#]+)" + example = "https://x.com/search?q=QUERY" + + def metadata(self): + return {"search": text.unquote(self.user)} + + def tweets(self): + query = text.unquote(self.user.replace("+", " ")) + + user = None + for item in query.split(): + item = item.strip("()") + if item.startswith("from:"): + if user: + user = None + break + else: + user = item[5:] + + if user is not None: + try: + self._assign_user(self.api.user_by_screen_name(user)) + except KeyError: + pass + + return self.api.search_timeline(query) + + +class TwitterHashtagExtractor(TwitterExtractor): + """Extractor for Twitter hashtags""" + subcategory = "hashtag" + pattern = BASE_PATTERN + r"/hashtag/([^/?#]+)" + example = "https://x.com/hashtag/NAME" + + def items(self): + url = f"{self.root}/search?q=%23{self.user}" + data = {"_extractor": TwitterSearchExtractor} + yield Message.Queue, url, data + + class TwitterUserExtractor(Dispatch, TwitterExtractor): """Extractor for a Twitter user""" - pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" + pattern = (BASE_PATTERN + r"/(?:" + r"([^/?#]+)/?(?:$|\?|#)" r"|i(?:/user/|ntent/user\?user_id=)(\d+))") example = "https://x.com/USER" def items(self): user, user_id = self.groups if user_id is not None: - user = "id:" + user_id + user = f"id:{user_id}" base = f"{self.root}/{user}/" return self._dispatch_extractors(( - (TwitterInfoExtractor , base + "info"), - (TwitterAvatarExtractor , base + "photo"), - (TwitterBackgroundExtractor, base + "header_photo"), - (TwitterTimelineExtractor , base + "timeline"), - (TwitterTweetsExtractor , base + "tweets"), - (TwitterMediaExtractor , base + "media"), - (TwitterRepliesExtractor , base + "with_replies"), - (TwitterLikesExtractor , base + "likes"), + (TwitterInfoExtractor , f"{base}info"), + (TwitterAvatarExtractor , f"{base}photo"), + (TwitterBackgroundExtractor, f"{base}header_photo"), + (TwitterTimelineExtractor , f"{base}timeline"), + (TwitterTweetsExtractor , f"{base}tweets"), + (TwitterMediaExtractor , f"{base}media"), + (TwitterRepliesExtractor , f"{base}with_replies"), + (TwitterHighlightsExtractor, f"{base}highlights"), + (TwitterLikesExtractor , f"{base}likes"), ), ("timeline",)) class TwitterTimelineExtractor(TwitterExtractor): """Extractor for a Twitter user timeline""" subcategory = "timeline" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/timeline(?!\w)" + pattern = rf"{USER_PATTERN}/timeline(?!\w)" example = "https://x.com/USER/timeline" def _init_cursor(self): @@ -728,7 +823,7 @@ class TwitterTimelineExtractor(TwitterExtractor): class TwitterTweetsExtractor(TwitterExtractor): """Extractor for Tweets from a user's Tweets timeline""" subcategory = "tweets" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/tweets(?!\w)" + pattern = rf"{USER_PATTERN}/tweets(?!\w)" example = "https://x.com/USER/tweets" def tweets(self): @@ -738,17 +833,27 @@ class TwitterTweetsExtractor(TwitterExtractor): class TwitterRepliesExtractor(TwitterExtractor): """Extractor for Tweets from a user's timeline including replies""" subcategory = "replies" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/with_replies(?!\w)" + pattern = rf"{USER_PATTERN}/with_replies(?!\w)" example = "https://x.com/USER/with_replies" def tweets(self): return self.api.user_tweets_and_replies(self.user) +class TwitterHighlightsExtractor(TwitterExtractor): + """Extractor for Tweets from a user's highlights timeline""" + subcategory = "highlights" + pattern = rf"{USER_PATTERN}/highlights(?!\w)" + example = "https://x.com/USER/highlights" + + def tweets(self): + return self.api.user_highlights(self.user) + + class TwitterMediaExtractor(TwitterExtractor): """Extractor for Tweets from a user's Media timeline""" subcategory = "media" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/media(?!\w)" + pattern = rf"{USER_PATTERN}/media(?!\w)" example = "https://x.com/USER/media" def tweets(self): @@ -758,7 +863,7 @@ class TwitterMediaExtractor(TwitterExtractor): class TwitterLikesExtractor(TwitterExtractor): """Extractor for liked tweets""" subcategory = "likes" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/likes(?!\w)" + pattern = rf"{USER_PATTERN}/likes(?!\w)" example = "https://x.com/USER/likes" def metadata(self): @@ -808,7 +913,7 @@ class TwitterListMembersExtractor(TwitterExtractor): class TwitterFollowingExtractor(TwitterExtractor): """Extractor for followed users""" subcategory = "following" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/following(?!\w)" + pattern = rf"{USER_PATTERN}/following(?!\w)" example = "https://x.com/USER/following" def items(self): @@ -819,7 +924,7 @@ class TwitterFollowingExtractor(TwitterExtractor): class TwitterFollowersExtractor(TwitterExtractor): """Extractor for a user's followers""" subcategory = "followers" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/followers(?!\w)" + pattern = rf"{USER_PATTERN}/followers(?!\w)" example = "https://x.com/USER/followers" def items(self): @@ -827,52 +932,12 @@ class TwitterFollowersExtractor(TwitterExtractor): return self._users_result(TwitterAPI(self).user_followers(self.user)) -class TwitterSearchExtractor(TwitterExtractor): - """Extractor for Twitter search results""" - subcategory = "search" - pattern = BASE_PATTERN + r"/search/?\?(?:[^&#]+&)*q=([^&#]+)" - example = "https://x.com/search?q=QUERY" - - def metadata(self): - return {"search": text.unquote(self.user)} - - def tweets(self): - query = text.unquote(self.user.replace("+", " ")) - - user = None - for item in query.split(): - item = item.strip("()") - if item.startswith("from:"): - if user: - user = None - break - else: - user = item[5:] - - if user is not None: - try: - self._assign_user(self.api.user_by_screen_name(user)) - except KeyError: - pass - - return self.api.search_timeline(query) - - -class TwitterHashtagExtractor(TwitterExtractor): - """Extractor for Twitter hashtags""" - subcategory = "hashtag" - pattern = BASE_PATTERN + r"/hashtag/([^/?#]+)" - example = "https://x.com/hashtag/NAME" - - def items(self): - url = f"{self.root}/search?q=%23{self.user}" - data = {"_extractor": TwitterSearchExtractor} - yield Message.Queue, url, data - - class TwitterCommunityExtractor(TwitterExtractor): """Extractor for a Twitter community""" subcategory = "community" + directory_fmt = ("{category}", "Communities", + "{community[name]} ({community[id]})") + archive_fmt = "C_{community[id]}_{tweet_id}_{num}" pattern = BASE_PATTERN + r"/i/communities/(\d+)" example = "https://x.com/i/communities/12345" @@ -885,6 +950,8 @@ class TwitterCommunityExtractor(TwitterExtractor): class TwitterCommunitiesExtractor(TwitterExtractor): """Extractor for followed Twitter communities""" subcategory = "communities" + directory_fmt = TwitterCommunityExtractor.directory_fmt + archive_fmt = TwitterCommunityExtractor.archive_fmt pattern = BASE_PATTERN + r"/([^/?#]+)/communities/?$" example = "https://x.com/i/communities" @@ -1002,7 +1069,7 @@ class TwitterQuotesExtractor(TwitterExtractor): class TwitterInfoExtractor(TwitterExtractor): """Extractor for a user's profile data""" subcategory = "info" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/info" + pattern = rf"{USER_PATTERN}/info" example = "https://x.com/USER/info" def items(self): @@ -1021,13 +1088,13 @@ class TwitterAvatarExtractor(TwitterExtractor): subcategory = "avatar" filename_fmt = "avatar {date}.{extension}" archive_fmt = "AV_{user[id]}_{date}" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/photo" + pattern = rf"{USER_PATTERN}/photo" example = "https://x.com/USER/photo" def tweets(self): self.api._user_id_by_screen_name(self.user) user = self._user_obj - url = user["legacy"]["profile_image_url_https"] + url = user["avatar"]["image_url"] if url == ("https://abs.twimg.com/sticky" "/default_profile_images/default_profile_normal.png"): @@ -1043,7 +1110,7 @@ class TwitterBackgroundExtractor(TwitterExtractor): subcategory = "background" filename_fmt = "background {date}.{extension}" archive_fmt = "BG_{user[id]}_{date}" - pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/header_photo" + pattern = rf"{USER_PATTERN}/header_photo" example = "https://x.com/USER/header_photo" def tweets(self): @@ -1169,9 +1236,10 @@ class TwitterAPI(): } self.features = { "hidden_profile_subscriptions_enabled": True, + "payments_enabled": False, + "rweb_xchat_enabled": False, "profile_label_improvements_pcf_label_in_post_enabled": True, "rweb_tipjar_consumption_enabled": True, - "responsive_web_graphql_exclude_directive_enabled": True, "verified_phone_label_enabled": False, "highlights_tweets_tab_ui_enabled": True, "responsive_web_twitter_article_notes_tab_enabled": True, @@ -1179,26 +1247,26 @@ class TwitterAPI(): "creator_subscriptions_tweet_preview_api_enabled": True, "responsive_web_graphql_" "skip_user_profile_image_extensions_enabled": False, - "responsive_web_graphql_" - "timeline_navigation_enabled": True, + "responsive_web_graphql_timeline_navigation_enabled": True, } self.features_pagination = { "rweb_video_screen_enabled": False, + "payments_enabled": False, + "rweb_xchat_enabled": False, "profile_label_improvements_pcf_label_in_post_enabled": True, "rweb_tipjar_consumption_enabled": True, - "responsive_web_graphql_exclude_directive_enabled": True, "verified_phone_label_enabled": False, "creator_subscriptions_tweet_preview_api_enabled": True, - "responsive_web_graphql_" - "timeline_navigation_enabled": True, - "responsive_web_graphql_" - "skip_user_profile_image_extensions_enabled": False, + "responsive_web_graphql" + "_timeline_navigation_enabled": True, + "responsive_web_graphql" + "_skip_user_profile_image_extensions_enabled": False, "premium_content_api_read_enabled": False, "communities_web_enable_tweet_community_results_fetch": True, "c9s_tweet_anatomy_moderator_badge_enabled": True, "responsive_web_grok_analyze_button_fetch_trends_enabled": False, "responsive_web_grok_analyze_post_followups_enabled": True, - "responsive_web_jetfuel_frame": False, + "responsive_web_jetfuel_frame": True, "responsive_web_grok_share_attachment_enabled": True, "articles_preview_enabled": True, "responsive_web_edit_tweet_api_enabled": True, @@ -1212,22 +1280,27 @@ class TwitterAPI(): "creator_subscriptions_quote_tweet_preview_enabled": False, "freedom_of_speech_not_reach_fetch_enabled": True, "standardized_nudges_misinfo": True, - "tweet_with_visibility_results_" - "prefer_gql_limited_actions_policy_enabled": True, + "tweet_with_visibility_results" + "_prefer_gql_limited_actions_policy_enabled": True, "longform_notetweets_rich_text_read_enabled": True, "longform_notetweets_inline_media_enabled": True, "responsive_web_grok_image_annotation_enabled": True, + "responsive_web_grok_imagine_annotation_enabled": True, + "responsive_web_grok" + "_community_note_auto_translation_is_enabled": False, "responsive_web_enhance_cards_enabled": False, } def tweet_result_by_rest_id(self, tweet_id): - endpoint = "/graphql/Vg2Akr5FzUmF0sTplA5k6g/TweetResultByRestId" + endpoint = "/graphql/qxWQxcMLiTPcavz9Qy5hwQ/TweetResultByRestId" variables = { "tweetId": tweet_id, "withCommunity": False, "includePromotedContent": False, "withVoice": False, } + features = self.features_pagination.copy() + del features["rweb_video_screen_enabled"] field_toggles = { "withArticleRichContentState": True, "withArticlePlainText": False, @@ -1236,7 +1309,7 @@ class TwitterAPI(): } params = { "variables" : self._json_dumps(variables), - "features" : self._json_dumps(self.features_pagination), + "features" : self._json_dumps(features), "fieldToggles": self._json_dumps(field_toggles), } tweet = self._call(endpoint, params)["data"]["tweetResult"]["result"] @@ -1245,16 +1318,16 @@ class TwitterAPI(): if tweet.get("__typename") == "TweetUnavailable": reason = tweet.get("reason") - if reason == "NsfwLoggedOut": - raise exception.AuthorizationError("NSFW Tweet") + if reason in ("NsfwViewerHasNoStatedAge", "NsfwLoggedOut"): + raise exception.AuthRequired(message="NSFW Tweet") if reason == "Protected": - raise exception.AuthorizationError("Protected Tweet") + raise exception.AuthRequired(message="Protected Tweet") raise exception.AbortExtraction(f"Tweet unavailable ('{reason}')") return tweet def tweet_detail(self, tweet_id): - endpoint = "/graphql/b9Yw90FMr_zUb8DvA8r2ug/TweetDetail" + endpoint = "/graphql/iFEr5AcP121Og4wx9Yqo3w/TweetDetail" variables = { "focalTweetId": tweet_id, "referrer": "profile", @@ -1278,7 +1351,7 @@ class TwitterAPI(): field_toggles=field_toggles) def user_tweets(self, screen_name): - endpoint = "/graphql/M3Hpkrb8pjWkEuGdLeXMOA/UserTweets" + endpoint = "/graphql/E8Wq-_jFSaU7hxVcuOPR9g/UserTweets" variables = { "userId": self._user_id_by_screen_name(screen_name), "count": 100, @@ -1293,7 +1366,7 @@ class TwitterAPI(): endpoint, variables, field_toggles=field_toggles) def user_tweets_and_replies(self, screen_name): - endpoint = "/graphql/pz0IHaV_t7T4HJavqqqcIA/UserTweetsAndReplies" + endpoint = "/graphql/-O3QOHrVn1aOm_cF5wyTCQ/UserTweetsAndReplies" variables = { "userId": self._user_id_by_screen_name(screen_name), "count": 100, @@ -1307,8 +1380,22 @@ class TwitterAPI(): return self._pagination_tweets( endpoint, variables, field_toggles=field_toggles) + def user_highlights(self, screen_name): + endpoint = "/graphql/gmHw9geMTncZ7jeLLUUNOw/UserHighlightsTweets" + variables = { + "userId": self._user_id_by_screen_name(screen_name), + "count": 100, + "includePromotedContent": False, + "withVoice": True, + } + field_toggles = { + "withArticlePlainText": False, + } + return self._pagination_tweets( + endpoint, variables, field_toggles=field_toggles) + def user_media(self, screen_name): - endpoint = "/graphql/8B9DqlaGvYyOvTCzzZWtNA/UserMedia" + endpoint = "/graphql/jCRhbOzdgOHp6u9H4g2tEg/UserMedia" variables = { "userId": self._user_id_by_screen_name(screen_name), "count": 100, @@ -1324,7 +1411,7 @@ class TwitterAPI(): endpoint, variables, field_toggles=field_toggles) def user_likes(self, screen_name): - endpoint = "/graphql/uxjTlmrTI61zreSIV1urbw/Likes" + endpoint = "/graphql/TGEKkJG_meudeaFcqaxM-Q/Likes" variables = { "userId": self._user_id_by_screen_name(screen_name), "count": 100, @@ -1340,7 +1427,7 @@ class TwitterAPI(): endpoint, variables, field_toggles=field_toggles) def user_bookmarks(self): - endpoint = "/graphql/ztCdjqsvvdL0dE8R5ME0hQ/Bookmarks" + endpoint = "/graphql/pLtjrO4ubNh996M_Cubwsg/Bookmarks" variables = { "count": 100, "includePromotedContent": False, @@ -1348,29 +1435,35 @@ class TwitterAPI(): return self._pagination_tweets( endpoint, variables, ("bookmark_timeline_v2", "timeline"), False) - def list_latest_tweets_timeline(self, list_id): - endpoint = "/graphql/LSefrrxhpeX8HITbKfWz9g/ListLatestTweetsTimeline" - variables = { - "listId": list_id, - "count": 100, - } - return self._pagination_tweets( - endpoint, variables, ("list", "tweets_timeline", "timeline")) - def search_timeline(self, query, product="Latest"): - endpoint = "/graphql/fL2MBiqXPk5pSrOS5ACLdA/SearchTimeline" + endpoint = "/graphql/4fpceYZ6-YQCx_JSl_Cn_A/SearchTimeline" variables = { "rawQuery": query, "count": 100, "querySource": "typed_query", "product": product, + "withGrokTranslatedBio": False, } return self._pagination_tweets( endpoint, variables, ("search_by_raw_query", "search_timeline", "timeline")) + def community_query(self, community_id): + endpoint = "/graphql/2W09l7nD7ZbxGQHXvfB22w/CommunityQuery" + params = { + "variables": self._json_dumps({ + "communityId": community_id, + }), + "features": self._json_dumps({ + "c9s_list_members_action_api_enabled": False, + "c9s_superc9s_indication_enabled": False, + }), + } + return (self._call(endpoint, params) + ["data"]["communityResults"]["result"]) + def community_tweets_timeline(self, community_id): - endpoint = "/graphql/awszcpgwaIeqqNfmzjxUow/CommunityTweetsTimeline" + endpoint = "/graphql/Nyt-88UX4-pPCImZNUl9RQ/CommunityTweetsTimeline" variables = { "communityId": community_id, "count": 100, @@ -1384,7 +1477,7 @@ class TwitterAPI(): "timeline")) def community_media_timeline(self, community_id): - endpoint = "/graphql/HfMuDHto2j3NKUeiLjKWHA/CommunityMediaTimeline" + endpoint = "/graphql/ZniZ7AAK_VVu1xtSx1V-gQ/CommunityMediaTimeline" variables = { "communityId": community_id, "count": 100, @@ -1396,7 +1489,7 @@ class TwitterAPI(): "timeline")) def communities_main_page_timeline(self, screen_name): - endpoint = ("/graphql/NbdrKPY_h_nlvZUg7oqH5Q" + endpoint = ("/graphql/p048a9n3hTPppQyK7FQTFw" "/CommunitiesMainPageTimeline") variables = { "count": 100, @@ -1406,6 +1499,27 @@ class TwitterAPI(): endpoint, variables, ("viewer", "communities_timeline", "timeline")) + def home_timeline(self): + endpoint = "/graphql/DXmgQYmIft1oLP6vMkJixw/HomeTimeline" + variables = { + "count": 100, + "includePromotedContent": False, + "latestControlAvailable": True, + "withCommunity": True, + } + return self._pagination_tweets( + endpoint, variables, ("home", "home_timeline_urt")) + + def home_latest_timeline(self): + endpoint = "/graphql/SFxmNKWfN9ySJcXG_tjX8g/HomeLatestTimeline" + variables = { + "count": 100, + "includePromotedContent": False, + "latestControlAvailable": True, + } + return self._pagination_tweets( + endpoint, variables, ("home", "home_timeline_urt")) + def live_event_timeline(self, event_id): endpoint = f"/2/live_event/timeline/{event_id}.json" params = self.params.copy() @@ -1422,8 +1536,17 @@ class TwitterAPI(): return (self._call(endpoint, params) ["twitter_objects"]["live_events"][event_id]) + def list_latest_tweets_timeline(self, list_id): + endpoint = "/graphql/06JtmwM8k_1cthpFZITVVA/ListLatestTweetsTimeline" + variables = { + "listId": list_id, + "count": 100, + } + return self._pagination_tweets( + endpoint, variables, ("list", "tweets_timeline", "timeline")) + def list_members(self, list_id): - endpoint = "/graphql/v97svwb-qcBmzv6QruDuNg/ListMembers" + endpoint = "/graphql/naea_MSad4pOb-D6_oVv_g/ListMembers" variables = { "listId": list_id, "count": 100, @@ -1432,35 +1555,38 @@ class TwitterAPI(): endpoint, variables, ("list", "members_timeline", "timeline")) def user_followers(self, screen_name): - endpoint = "/graphql/jqZ0_HJBA6mnu18iTZYm9w/Followers" + endpoint = "/graphql/i6PPdIMm1MO7CpAqjau7sw/Followers" variables = { "userId": self._user_id_by_screen_name(screen_name), "count": 100, "includePromotedContent": False, + "withGrokTranslatedBio": False, } return self._pagination_users(endpoint, variables) def user_followers_verified(self, screen_name): - endpoint = "/graphql/GHg0X_FjrJoISwwLPWi1LQ/BlueVerifiedFollowers" + endpoint = "/graphql/fxEl9kp1Tgolqkq8_Lo3sg/BlueVerifiedFollowers" variables = { "userId": self._user_id_by_screen_name(screen_name), "count": 100, "includePromotedContent": False, + "withGrokTranslatedBio": False, } return self._pagination_users(endpoint, variables) def user_following(self, screen_name): - endpoint = "/graphql/4QHbs4wmzgtU91f-t96_Eg/Following" + endpoint = "/graphql/SaWqzw0TFAWMx1nXWjXoaQ/Following" variables = { "userId": self._user_id_by_screen_name(screen_name), "count": 100, "includePromotedContent": False, + "withGrokTranslatedBio": False, } return self._pagination_users(endpoint, variables) @memcache(keyarg=1) def user_by_rest_id(self, rest_id): - endpoint = "/graphql/5vdJ5sWkbSRDiiNZvwc2Yg/UserByRestId" + endpoint = "/graphql/8r5oa_2vD0WkhIAOkY4TTA/UserByRestId" features = self.features params = { "variables": self._json_dumps({ @@ -1472,7 +1598,7 @@ class TwitterAPI(): @memcache(keyarg=1) def user_by_screen_name(self, screen_name): - endpoint = "/graphql/32pL5BWe9WKeSK1MoPvFQQ/UserByScreenName" + endpoint = "/graphql/ck5KkZ8t5cOmoLssopN99Q/UserByScreenName" features = self.features.copy() features["subscriptions_verification_info_" "is_identity_verified_enabled"] = True @@ -1481,6 +1607,7 @@ class TwitterAPI(): params = { "variables": self._json_dumps({ "screen_name": screen_name, + "withGrokTranslatedBio": False, }), "features": self._json_dumps(features), "fieldToggles": self._json_dumps({ @@ -1618,7 +1745,8 @@ class TwitterAPI(): return data elif response.status_code in (403, 404) and \ not self.headers["x-twitter-auth-type"]: - raise exception.AuthorizationError("Login required") + raise exception.AuthRequired( + "authenticated cookies", "timeline") elif response.status_code == 429: self._handle_ratelimit(response) continue @@ -1870,19 +1998,16 @@ class TwitterAPI(): continue if "retweeted_status_result" in legacy: - retweet = legacy["retweeted_status_result"]["result"] - if "tweet" in retweet: - retweet = retweet["tweet"] - if original_retweets: - try: + try: + retweet = legacy["retweeted_status_result"]["result"] + if "tweet" in retweet: + retweet = retweet["tweet"] + if original_retweets: retweet["legacy"]["retweeted_status_id_str"] = \ retweet["rest_id"] retweet["_retweet_id_str"] = tweet["rest_id"] tweet = retweet - except KeyError: - continue - else: - try: + else: legacy["retweeted_status_id_str"] = \ retweet["rest_id"] tweet["author"] = \ @@ -1904,8 +2029,11 @@ class TwitterAPI(): rtlegacy["withheld_scope"] legacy["full_text"] = rtlegacy["full_text"] - except KeyError: - pass + except Exception as exc: + extr.log.debug( + "%s: %s: %s", + tweet.get("rest_id"), exc.__class__.__name__, exc) + continue yield tweet |