diff options
Diffstat (limited to 'gallery_dl/extractor/twitter.py')
| -rw-r--r-- | gallery_dl/extractor/twitter.py | 89 |
1 files changed, 63 insertions, 26 deletions
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index f924292..f459fba 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -168,6 +168,11 @@ class TwitterExtractor(Extractor): if key in bvals: value = bvals[key].get("image_value") if value and "url" in value: + base, sep, size = value["url"].rpartition("&name=") + if sep: + base += sep + value["url"] = base + self._size_image + value["_fallback"] = self._image_fallback(base) files.append(value) return elif name == "unified_card": @@ -759,7 +764,10 @@ class TwitterAPI(): "__fs_interactive_text": False, "__fs_dont_mention_me_view_api_enabled": False, } + + self._log_warnings = extractor.config("warnings") self._json_dumps = json.JSONEncoder(separators=(",", ":")).encode + self._user = None cookies = extractor.session.cookies cookiedomain = extractor.cookiedomain @@ -898,6 +906,15 @@ class TwitterAPI(): } return self._pagination_users(endpoint, variables) + def user_by_rest_id(self, rest_id): + endpoint = "/graphql/I5nvpI91ljifos1Y3Lltyg/UserByRestId" + params = {"variables": self._json_dumps({ + "userId": rest_id, + "withSafetyModeUserFields": True, + "withSuperFollowsUserFields": True, + })} + return self._call(endpoint, params)["data"]["user"]["result"] + def user_by_screen_name(self, screen_name): endpoint = "/graphql/7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName" params = {"variables": self._json_dumps({ @@ -909,11 +926,12 @@ class TwitterAPI(): def _user_id_by_screen_name(self, screen_name): if screen_name.startswith("id:"): + self._user = util.SENTINEL return screen_name[3:] user = () try: - user = self.user_by_screen_name(screen_name) + user = self._user = self.user_by_screen_name(screen_name) return user["rest_id"] except KeyError: if "unavailable_message" in user: @@ -929,7 +947,7 @@ class TwitterAPI(): endpoint = "/1.1/guest/activate.json" return str(self._call(endpoint, None, root, "POST")["guest_token"]) - def _call(self, endpoint, params, root=None, method="GET", warning=True): + def _call(self, endpoint, params, root=None, method="GET"): if root is None: root = self.root @@ -954,7 +972,7 @@ class TwitterAPI(): if response.status_code < 400: # success - if errors and warning: + if errors and self._log_warnings: self.extractor.log.warning(errors) return data @@ -965,22 +983,6 @@ class TwitterAPI(): self.extractor.wait(until=until, seconds=seconds) continue - if response.status_code == 401 and \ - "have been blocked from viewing" in errors: - # account blocked - extr = self.extractor - if self.headers["x-twitter-auth-type"] and \ - extr.config("logout"): - guest_token = self._guest_token() - extr.session.cookies.set( - "gt", guest_token, domain=extr.cookiedomain) - extr._cookiefile = None - del extr.session.cookies["auth_token"] - self.headers["x-guest-token"] = guest_token - self.headers["x-twitter-auth-type"] = None - extr.log.info("Retrying API request as guest") - continue - # error raise exception.StopExtraction( "%s %s (%s)", response.status_code, response.reason, errors) @@ -1070,9 +1072,10 @@ class TwitterAPI(): params["cursor"] = cursor def _pagination_tweets(self, endpoint, variables, path=None): + extr = self.extractor variables.update(self.variables) - original_retweets = (self.extractor.retweets == "original") - pinned_tweet = self.extractor.pinned + original_retweets = (extr.retweets == "original") + pinned_tweet = extr.pinned while True: params = {"variables": self._json_dumps(variables)} @@ -1083,13 +1086,47 @@ class TwitterAPI(): instructions = (data["user"]["result"]["timeline"] ["timeline"]["instructions"]) else: + instructions = data for key in path: - data = data[key] - instructions = data["instructions"] + instructions = instructions[key] + instructions = instructions["instructions"] entries = instructions[0]["entries"] except (KeyError, IndexError): - return + extr.log.debug(data) + + if self._user: + user = self._user + if user is util.SENTINEL: + try: + user = self.user_by_rest_id(variables["userId"]) + except KeyError: + raise exception.NotFoundError("user") + user = user.get("legacy") + if not user: + pass + elif user.get("blocked_by"): + if self.headers["x-twitter-auth-type"] and \ + extr.config("logout"): + guest_token = self._guest_token() + extr.session.cookies.set( + "gt", guest_token, domain=extr.cookiedomain) + extr._cookiefile = None + del extr.session.cookies["auth_token"] + self.headers["x-guest-token"] = guest_token + self.headers["x-twitter-auth-type"] = None + extr.log.info("Retrying API request as guest") + continue + raise exception.AuthorizationError( + "{} blocked your account".format( + user["screen_name"])) + elif user.get("protected"): + raise exception.AuthorizationError( + "{}'s Tweets are protected".format( + user["screen_name"])) + + raise exception.StopExtraction( + "Unable to retrieve Tweets from this timeline") tweets = [] tweet = cursor = None @@ -1121,7 +1158,7 @@ class TwitterAPI(): ["itemContent"]["tweet_results"]["result"]) legacy = tweet["legacy"] except KeyError: - self.extractor.log.debug( + extr.log.debug( "Skipping %s (deleted)", (entry.get("entryId") or "").rpartition("-")[2]) continue @@ -1160,7 +1197,7 @@ class TwitterAPI(): quoted["legacy"]["quoted_by_id_str"] = tweet["rest_id"] yield quoted except KeyError: - self.extractor.log.debug( + extr.log.debug( "Skipping quote of %s (deleted)", tweet.get("rest_id")) continue |