diff options
Diffstat (limited to 'gallery_dl/extractor/twitter.py')
| -rw-r--r-- | gallery_dl/extractor/twitter.py | 65 |
1 files changed, 52 insertions, 13 deletions
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index fd0140d..7e78941 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -50,12 +50,17 @@ class TwitterExtractor(Extractor): if not self.retweets and "retweeted_status_id_str" in tweet: self.log.debug("Skipping %s (retweet)", tweet["id_str"]) continue - if not self.replies and "in_reply_to_user_id_str" in tweet: - self.log.debug("Skipping %s (reply)", tweet["id_str"]) - continue if not self.quoted and "quoted" in tweet: self.log.debug("Skipping %s (quoted tweet)", tweet["id_str"]) continue + if "in_reply_to_user_id_str" in tweet and ( + not self.replies or ( + self.replies == "self" and + tweet["in_reply_to_user_id_str"] != tweet["user_id_str"] + ) + ): + self.log.debug("Skipping %s (reply)", tweet["id_str"]) + continue files = [] if "extended_entities" in tweet: @@ -452,6 +457,15 @@ class TwitterTweetExtractor(TwitterExtractor): "options": (("replies", False),), "count": 0, }), + # 'replies' to self (#1254) + ("https://twitter.com/i/web/status/1424882930803908612", { + "options": (("replies", "self"),), + "count": 4, + }), + ("https://twitter.com/i/web/status/1424898916156284928", { + "options": (("replies", "self"),), + "count": 0, + }), # "quoted" option (#854) ("https://twitter.com/StobiesGalaxy/status/1270755918330896395", { "options": (("quoted", True),), @@ -582,8 +596,8 @@ class TwitterAPI(): "ext": "mediaStats,highlightedLabel", } - cookies = self.extractor.session.cookies - cookiedomain = ".twitter.com" + cookies = extractor.session.cookies + cookiedomain = extractor.cookiedomain # CSRF csrf_token = cookies.get("ct0", domain=cookiedomain) @@ -726,21 +740,46 @@ class TwitterAPI(): if csrf_token: self.headers["x-csrf-token"] = csrf_token + data = response.json() + if "errors" in data: + try: + msg = ", ".join( + '"' + error["message"] + '"' + for error in data["errors"] + ) + except Exception: + msg = data["errors"] + if response.status_code < 400: + self.extractor.log.warning(msg) + else: + msg = "" + if response.status_code < 400: - return response.json() + # success + return data + if response.status_code == 429: + # rate limit exceeded until = response.headers.get("x-rate-limit-reset") seconds = None if until else 60 self.extractor.wait(until=until, seconds=seconds) continue - try: - msg = ", ".join( - '"' + error["message"] + '"' - for error in response.json()["errors"] - ) - except Exception: - msg = response.text + if response.status_code == 401 and \ + "have been blocked from viewing" in msg: + # account blocked + extr = extr = self.extractor + if self.headers["x-twitter-auth-type"] and \ + extr.config("logout"): + guest_token = self._guest_token() + extr.session.cookies.set( + "gt", guest_token, domain=extr.cookiedomain) + self.headers["x-guest-token"] = guest_token + self.headers["x-twitter-auth-type"] = None + extr.log.info("Retrying API request as guest") + continue + + # error raise exception.StopExtraction( "%s %s (%s)", response.status_code, response.reason, msg) |