diff options
Diffstat (limited to 'gallery_dl/extractor/twitter.py')
| -rw-r--r-- | gallery_dl/extractor/twitter.py | 71 |
1 files changed, 37 insertions, 34 deletions
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index b769912..4034732 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2020 Mike Fährmann +# Copyright 2016-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -239,7 +239,7 @@ class TwitterExtractor(Extractor): def _login_impl(self, username, password): self.log.info("Logging in as %s", username) - token = util.generate_csrf_token() + token = util.generate_token() self.session.cookies.clear() self.request(self.root + "/login") @@ -272,8 +272,8 @@ class TwitterExtractor(Extractor): class TwitterTimelineExtractor(TwitterExtractor): """Extractor for all images from a user's timeline""" subcategory = "timeline" - pattern = BASE_PATTERN + \ - r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])|intent/user\?user_id=(\d+))" + pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" + r"|i(?:/user/|ntent/user\?user_id=)(\d+))") test = ( ("https://twitter.com/supernaturepics", { "range": "1-40", @@ -281,14 +281,15 @@ class TwitterTimelineExtractor(TwitterExtractor): }), ("https://mobile.twitter.com/supernaturepics?p=i"), ("https://www.twitter.com/id:2976459548"), + ("https://twitter.com/i/user/2976459548"), ("https://twitter.com/intent/user?user_id=2976459548"), ) def __init__(self, match): TwitterExtractor.__init__(self, match) - uid = match.group(2) - if uid: - self.user = "id:" + uid + user_id = match.group(2) + if user_id: + self.user = "id:" + user_id def tweets(self): return TwitterAPI(self).timeline_profile(self.user) @@ -355,8 +356,7 @@ class TwitterListMembersExtractor(TwitterExtractor): self.login() for user in TwitterAPI(self).list_members(self.user): user["_extractor"] = TwitterTimelineExtractor - url = "{}/intent/user?user_id={}".format( - self.root, user["rest_id"]) + url = "{}/i/user/{}".format(self.root, user["rest_id"]) yield Message.Queue, url, user @@ -509,7 +509,7 @@ class TwitterAPI(): # CSRF csrf_token = cookies.get("ct0", domain=cookiedomain) if not csrf_token: - csrf_token = util.generate_csrf_token() + csrf_token = util.generate_token() cookies.set("ct0", csrf_token, domain=cookiedomain) self.headers["x-csrf-token"] = csrf_token @@ -617,31 +617,34 @@ class TwitterAPI(): def _call(self, endpoint, params, root=None, method="GET"): if root is None: root = self.root - response = self.extractor.request( - root + endpoint, method=method, params=params, - headers=self.headers, fatal=None) - - # update 'x-csrf-token' header (#1170) - csrf_token = response.cookies.get("ct0") - if csrf_token: - self.headers["x-csrf-token"] = csrf_token - - if response.status_code < 400: - return response.json() - if response.status_code == 429: - until = response.headers.get("x-rate-limit-reset") - self.extractor.wait(until=until, seconds=(None if until else 60)) - return self._call(endpoint, params, method) - try: - msg = ", ".join( - '"' + error["message"] + '"' - for error in response.json()["errors"] - ) - except Exception: - msg = response.text - raise exception.StopExtraction( - "%s %s (%s)", response.status_code, response.reason, msg) + while True: + response = self.extractor.request( + root + endpoint, method=method, params=params, + headers=self.headers, fatal=None) + + # update 'x-csrf-token' header (#1170) + csrf_token = response.cookies.get("ct0") + if csrf_token: + self.headers["x-csrf-token"] = csrf_token + + if response.status_code < 400: + return response.json() + if response.status_code == 429: + until = response.headers.get("x-rate-limit-reset") + seconds = None if until else 60 + self.extractor.wait(until=until, seconds=seconds) + continue + + try: + msg = ", ".join( + '"' + error["message"] + '"' + for error in response.json()["errors"] + ) + except Exception: + msg = response.text + raise exception.StopExtraction( + "%s %s (%s)", response.status_code, response.reason, msg) def _pagination(self, endpoint, params=None): if params is None: |