summaryrefslogtreecommitdiffstats
path: root/gallery_dl/extractor/twitter.py
diff options
context:
space:
mode:
Diffstat (limited to 'gallery_dl/extractor/twitter.py')
-rw-r--r--gallery_dl/extractor/twitter.py245
1 files changed, 125 insertions, 120 deletions
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
index 896bf28..7252d05 100644
--- a/gallery_dl/extractor/twitter.py
+++ b/gallery_dl/extractor/twitter.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2016-2023 Mike Fährmann
+# Copyright 2016-2025 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -8,12 +8,11 @@
"""Extractors for https://x.com/"""
-from .common import Extractor, Message
+from .common import Extractor, Message, Dispatch
from .. import text, util, exception
from ..cache import cache, memcache
import itertools
import random
-import re
BASE_PATTERN = (r"(?:https?://)?(?:www\.|mobile\.)?"
r"(?:(?:[fv]x)?twitter|(?:fix(?:up|v))?x)\.com")
@@ -32,7 +31,7 @@ class TwitterExtractor(Extractor):
def __init__(self, match):
Extractor.__init__(self, match)
- self.user = match.group(1)
+ self.user = match[1]
def _init(self):
self.unavailable = self.config("unavailable", False)
@@ -72,21 +71,16 @@ class TwitterExtractor(Extractor):
self.login()
self.api = TwitterAPI(self)
metadata = self.metadata()
-
- if self.config("expand"):
- tweets = self._expand_tweets(self.tweets())
- self.tweets = lambda : tweets
-
- if self.config("unique", True):
- seen_tweets = set()
- else:
- seen_tweets = None
+ seen_tweets = set() if self.config("unique", True) else None
if self.twitpic:
- self._find_twitpic = re.compile(
+ self._find_twitpic = util.re(
r"https?(://twitpic\.com/(?!photos/)\w+)").findall
- for tweet in self.tweets():
+ tweets = self.tweets()
+ if self.config("expand"):
+ tweets = self._expand_tweets(tweets)
+ for tweet in tweets:
if "legacy" in tweet:
data = tweet["legacy"]
@@ -129,6 +123,12 @@ class TwitterExtractor(Extractor):
tdata.update(metadata)
tdata["count"] = len(files)
yield Message.Directory, tdata
+
+ del tdata["source_id"]
+ del tdata["sensitive_flags"]
+ if "source_user" in tdata:
+ del tdata["source_user"]
+
for tdata["num"], file in enumerate(files, 1):
file.update(tdata)
url = file.pop("url")
@@ -170,8 +170,29 @@ class TwitterExtractor(Extractor):
return files
def _extract_media(self, tweet, entities, files):
+ flags_tweet = None
+
for media in entities:
+ if "sensitive_media_warning" in media:
+ flags_media = media["sensitive_media_warning"]
+
+ flags = []
+ if "adult_content" in flags_media:
+ flags.append("Nudity")
+ if "other" in flags_media:
+ flags.append("Sensitive")
+ if "graphic_violence" in flags_media:
+ flags.append("Violence")
+
+ if flags_tweet is None:
+ flags_tweet = set(flags)
+ else:
+ flags_tweet.update(flags)
+ flags_media = flags
+ else:
+ flags_media = ()
+
if "ext_media_availability" in media:
ext = media["ext_media_availability"]
if ext.get("status") == "Unavailable":
@@ -180,38 +201,22 @@ class TwitterExtractor(Extractor):
if not self.unavailable:
continue
- mtype = media.get("type")
- descr = media.get("ext_alt_text")
- width = media["original_info"].get("width", 0)
- height = media["original_info"].get("height", 0)
-
if "video_info" in media:
if self.videos == "ytdl":
- files.append({
- "url": "ytdl:{}/i/web/status/{}".format(
- self.root, tweet["id_str"]),
- "type" : mtype,
- "width" : width,
- "height" : height,
- "extension" : None,
- "description": descr,
- })
+ url = f"ytdl:{self.root}/i/web/status/{tweet['id_str']}"
+ file = {"url": url, "extension": "mp4"}
elif self.videos:
video_info = media["video_info"]
variant = max(
video_info["variants"],
key=lambda v: v.get("bitrate", 0),
)
- files.append({
- "url" : variant["url"],
- "type" : mtype,
- "width" : width,
- "height" : height,
- "bitrate" : variant.get("bitrate", 0),
- "duration" : video_info.get(
+ file = {
+ "url" : variant["url"],
+ "bitrate" : variant.get("bitrate", 0),
+ "duration": video_info.get(
"duration_millis", 0) / 1000,
- "description": descr,
- })
+ }
elif "media_url_https" in media:
url = media["media_url_https"]
if url[-4] == ".":
@@ -219,16 +224,37 @@ class TwitterExtractor(Extractor):
base += "?format=" + fmt + "&name="
else:
base = url.rpartition("=")[0] + "="
- files.append(text.nameext_from_url(url, {
- "url" : base + self._size_image,
- "type" : mtype,
- "width" : width,
- "height" : height,
- "_fallback" : self._image_fallback(base),
- "description": descr,
- }))
+ file = text.nameext_from_url(url, {
+ "url" : base + self._size_image,
+ "_fallback": self._image_fallback(base),
+ })
else:
files.append({"url": media["media_url"]})
+ continue
+
+ file["type"] = media.get("type")
+ file["width"] = media["original_info"].get("width", 0)
+ file["height"] = media["original_info"].get("height", 0)
+ file["description"] = media.get("ext_alt_text")
+ file["sensitive_flags"] = flags_media
+ self._extract_media_source(file, media)
+ files.append(file)
+
+ tweet["sensitive_flags"] = \
+ () if flags_tweet is None else sorted(flags_tweet)
+
+ def _extract_media_source(self, dest, media):
+ dest["source_id"] = 0
+
+ if "source_status_id_str" in media:
+ try:
+ dest["source_id"] = text.parse_int(
+ media["source_status_id_str"])
+ dest["source_user"] = self._transform_user(
+ media["additional_media_info"]["source_user"]
+ ["user_results"]["result"])
+ except Exception:
+ pass
def _image_fallback(self, base):
for fmt in self._size_fallback:
@@ -252,8 +278,7 @@ class TwitterExtractor(Extractor):
bvals = {bval["key"]: bval["value"]
for bval in card["binding_values"]}
- cbl = self.cards_blacklist
- if cbl:
+ if cbl := self.cards_blacklist:
if name in cbl:
return
if "vanity_url" in bvals:
@@ -288,7 +313,7 @@ class TwitterExtractor(Extractor):
if self.cards == "ytdl":
tweet_id = tweet.get("rest_id") or tweet["id_str"]
- url = "ytdl:{}/i/web/status/{}".format(self.root, tweet_id)
+ url = f"ytdl:{self.root}/i/web/status/{tweet_id}"
files.append({"url": url})
def _extract_twitpic(self, tweet, files):
@@ -313,8 +338,8 @@ class TwitterExtractor(Extractor):
response = self.request(url, fatal=False)
if response.status_code >= 400:
continue
- url = text.extr(response.text, 'name="twitter:image" value="', '"')
- if url:
+ if url := text.extr(
+ response.text, 'name="twitter:image" value="', '"'):
files.append({"url": url})
def _transform_tweet(self, tweet):
@@ -354,12 +379,14 @@ class TwitterExtractor(Extractor):
tget("in_reply_to_status_id_str")),
"conversation_id": text.parse_int(
tget("conversation_id_str")),
+ "source_id" : 0,
"date" : date,
"author" : author,
"user" : self._user or author,
"lang" : legacy["lang"],
"source" : text.extr(source, ">", "<") if source else "",
"sensitive" : tget("possibly_sensitive"),
+ "sensitive_flags": tget("sensitive_flags"),
"favorite_count": tget("favorite_count"),
"quote_count" : tget("quote_count"),
"reply_count" : tget("reply_count"),
@@ -383,12 +410,10 @@ class TwitterExtractor(Extractor):
content = tget("full_text") or tget("text") or ""
entities = legacy["entities"]
- hashtags = entities.get("hashtags")
- if hashtags:
+ if hashtags := entities.get("hashtags"):
tdata["hashtags"] = [t["text"] for t in hashtags]
- mentions = entities.get("user_mentions")
- if mentions:
+ if mentions := entities.get("user_mentions"):
tdata["mentions"] = [{
"id": text.parse_int(u["id_str"]),
"name": u["screen_name"],
@@ -396,8 +421,7 @@ class TwitterExtractor(Extractor):
} for u in mentions]
content = text.unescape(content)
- urls = entities.get("urls")
- if urls:
+ if urls := entities.get("urls"):
for url in urls:
try:
content = content.replace(url["url"], url["expanded_url"])
@@ -417,9 +441,11 @@ class TwitterExtractor(Extractor):
tdata["reply_to"] = legacy["in_reply_to_screen_name"]
if "quoted_by" in legacy:
tdata["quote_by"] = legacy["quoted_by"]
+ if "extended_entities" in legacy:
+ self._extract_media_source(
+ tdata, legacy["extended_entities"]["media"][0])
if tdata["retweet_id"]:
- tdata["content"] = "RT @{}: {}".format(
- author["name"], tdata["content"])
+ tdata["content"] = f"RT @{author['name']}: {tdata['content']}"
tdata["date_original"] = text.parse_timestamp(
((tdata["retweet_id"] >> 22) + 1288834974657) // 1000)
@@ -466,8 +492,7 @@ class TwitterExtractor(Extractor):
}
descr = user["description"]
- urls = entities["description"].get("urls")
- if urls:
+ if urls := entities["description"].get("urls"):
for url in urls:
try:
descr = descr.replace(url["url"], url["expanded_url"])
@@ -577,27 +602,18 @@ class TwitterExtractor(Extractor):
return self.cookies_update(_login_impl(self, username, password))
-class TwitterUserExtractor(TwitterExtractor):
+class TwitterUserExtractor(Dispatch, TwitterExtractor):
"""Extractor for a Twitter user"""
- subcategory = "user"
pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])"
r"|i(?:/user/|ntent/user\?user_id=)(\d+))")
example = "https://x.com/USER"
- def __init__(self, match):
- TwitterExtractor.__init__(self, match)
- user_id = match.group(2)
- if user_id:
- self.user = "id:" + user_id
-
- def initialize(self):
- pass
-
- def finalize(self):
- pass
-
def items(self):
- base = "{}/{}/".format(self.root, self.user)
+ user, user_id = self.groups
+ if user_id is not None:
+ user = "id:" + user_id
+
+ base = f"{self.root}/{user}/"
return self._dispatch_extractors((
(TwitterInfoExtractor , base + "info"),
(TwitterAvatarExtractor , base + "photo"),
@@ -663,12 +679,12 @@ class TwitterTimelineExtractor(TwitterExtractor):
self.api._user_id_by_screen_name(self.user)
# build search query
- query = "from:{} max_id:{}".format(self._user["name"], tweet_id)
+ query = f"from:{self._user['name']} max_id:{tweet_id}"
if self.retweets:
query += " include:retweets include:nativeretweets"
if state <= 2:
- self._cursor_prefix = "2_{}/".format(tweet_id)
+ self._cursor_prefix = f"2_{tweet_id}/"
if reset:
self._cursor = self._cursor_prefix
@@ -684,7 +700,7 @@ class TwitterTimelineExtractor(TwitterExtractor):
if state <= 3:
# yield unfiltered search results
- self._cursor_prefix = "3_{}/".format(tweet_id)
+ self._cursor_prefix = f"3_{tweet_id}/"
if reset:
self._cursor = self._cursor_prefix
@@ -704,7 +720,7 @@ class TwitterTimelineExtractor(TwitterExtractor):
return self.api.user_media
if strategy == "with_replies":
return self.api.user_tweets_and_replies
- raise exception.StopExtraction("Invalid strategy '%s'", strategy)
+ raise exception.AbortExtraction(f"Invalid strategy '{strategy}'")
class TwitterTweetsExtractor(TwitterExtractor):
@@ -847,7 +863,7 @@ class TwitterHashtagExtractor(TwitterExtractor):
example = "https://x.com/hashtag/NAME"
def items(self):
- url = "{}/search?q=%23{}".format(self.root, self.user)
+ url = f"{self.root}/search?q=%23{self.user}"
data = {"_extractor": TwitterSearchExtractor}
yield Message.Queue, url, data
@@ -898,11 +914,10 @@ class TwitterTweetExtractor(TwitterExtractor):
def __init__(self, match):
TwitterExtractor.__init__(self, match)
- self.tweet_id = match.group(2)
+ self.tweet_id = match[2]
def tweets(self):
- conversations = self.config("conversations")
- if conversations:
+ if conversations := self.config("conversations"):
self._accessible = (conversations == "accessible")
return self._tweets_conversation(self.tweet_id)
@@ -919,8 +934,8 @@ class TwitterTweetExtractor(TwitterExtractor):
try:
self._assign_user(tweet["core"]["user_results"]["result"])
except KeyError:
- raise exception.StopExtraction(
- "'%s'", tweet.get("reason") or "Unavailable")
+ raise exception.AbortExtraction(
+ f"'{tweet.get('reason') or 'Unavailable'}'")
yield tweet
@@ -977,7 +992,7 @@ class TwitterQuotesExtractor(TwitterExtractor):
example = "https://x.com/USER/status/12345/quotes"
def items(self):
- url = "{}/search?q=quoted_tweet_id:{}".format(self.root, self.user)
+ url = f"{self.root}/search?q=quoted_tweet_id:{self.user}"
data = {"_extractor": TwitterSearchExtractor}
yield Message.Queue, url, data
@@ -1055,8 +1070,7 @@ class TwitterImageExtractor(Extractor):
TwitterExtractor._init_sizes(self)
def items(self):
- base = "https://pbs.twimg.com/media/{}?format={}&name=".format(
- self.id, self.fmt)
+ base = f"https://pbs.twimg.com/media/{self.id}?format={self.fmt}&name="
data = {
"filename": self.id,
@@ -1233,7 +1247,7 @@ class TwitterAPI():
raise exception.AuthorizationError("NSFW Tweet")
if reason == "Protected":
raise exception.AuthorizationError("Protected Tweet")
- raise exception.StopExtraction("Tweet unavailable ('%s')", reason)
+ raise exception.AbortExtraction(f"Tweet unavailable ('{reason}')")
return tweet
@@ -1391,7 +1405,7 @@ class TwitterAPI():
("viewer", "communities_timeline", "timeline"))
def live_event_timeline(self, event_id):
- endpoint = "/2/live_event/timeline/{}.json".format(event_id)
+ endpoint = f"/2/live_event/timeline/{event_id}.json"
params = self.params.copy()
params["timeline_id"] = "recap"
params["urt"] = "true"
@@ -1399,7 +1413,7 @@ class TwitterAPI():
return self._pagination_legacy(endpoint, params)
def live_event(self, event_id):
- endpoint = "/1.1/live_event/1/{}/timeline.json".format(event_id)
+ endpoint = f"/1.1/live_event/1/{event_id}/timeline.json"
params = self.params.copy()
params["count"] = "0"
params["urt"] = "true"
@@ -1484,9 +1498,9 @@ class TwitterAPI():
return user["rest_id"]
except KeyError:
if "unavailable_message" in user:
- raise exception.NotFoundError("{} ({})".format(
- user["unavailable_message"].get("text"),
- user.get("reason")), False)
+ raise exception.NotFoundError(
+ f"{user['unavailable_message'].get('text')} "
+ f"({user.get('reason')})", False)
else:
raise exception.NotFoundError("user")
@@ -1543,8 +1557,7 @@ class TwitterAPI():
headers=self.headers, fatal=None)
# update 'x-csrf-token' header (#1170)
- csrf_token = response.cookies.get("ct0")
- if csrf_token:
+ if csrf_token := response.cookies.get("ct0"):
self.headers["x-csrf-token"] = csrf_token
remaining = int(response.headers.get("x-rate-limit-remaining", 6))
@@ -1614,13 +1627,12 @@ class TwitterAPI():
except Exception:
pass
- raise exception.StopExtraction(
- "%s %s (%s)", response.status_code, response.reason, errors)
+ raise exception.AbortExtraction(
+ f"{response.status_code} {response.reason} ({errors})")
def _pagination_legacy(self, endpoint, params):
extr = self.extractor
- cursor = extr._init_cursor()
- if cursor:
+ if cursor := extr._init_cursor():
params["cursor"] = cursor
original_retweets = (extr.retweets == "original")
bottom = ("cursor-bottom-", "sq-cursor-bottom")
@@ -1701,8 +1713,7 @@ class TwitterAPI():
yield tweet
if "quoted_status_id_str" in tweet:
- quoted = tweets.get(tweet["quoted_status_id_str"])
- if quoted:
+ if quoted := tweets.get(tweet["quoted_status_id_str"]):
quoted = quoted.copy()
quoted["author"] = users[quoted["user_id_str"]]
quoted["quoted_by"] = tweet["user"]["screen_name"]
@@ -1722,8 +1733,7 @@ class TwitterAPI():
pinned_tweet = extr.pinned
params = {"variables": None}
- cursor = extr._init_cursor()
- if cursor:
+ if cursor := extr._init_cursor():
variables["cursor"] = cursor
if features is None:
features = self.features_pagination
@@ -1772,8 +1782,7 @@ class TwitterAPI():
except LookupError:
extr.log.debug(data)
- user = extr._user_obj
- if user:
+ if user := extr._user_obj:
user = user["legacy"]
if user.get("blocked_by"):
if self.headers["x-twitter-auth-type"] and \
@@ -1784,14 +1793,12 @@ class TwitterAPI():
extr.log.info("Retrying API request as guest")
continue
raise exception.AuthorizationError(
- "{} blocked your account".format(
- user["screen_name"]))
+ f"{user['screen_name']} blocked your account")
elif user.get("protected"):
raise exception.AuthorizationError(
- "{}'s Tweets are protected".format(
- user["screen_name"]))
+ f"{user['screen_name']}'s Tweets are protected")
- raise exception.StopExtraction(
+ raise exception.AbortExtraction(
"Unable to retrieve Tweets from this timeline")
tweets = []
@@ -1924,8 +1931,7 @@ class TwitterAPI():
def _pagination_users(self, endpoint, variables, path=None):
extr = self.extractor
- cursor = extr._init_cursor()
- if cursor:
+ if cursor := extr._init_cursor():
variables["cursor"] = cursor
params = {
"variables": None,
@@ -1970,7 +1976,7 @@ class TwitterAPI():
def _handle_ratelimit(self, response):
rl = self.extractor.config("ratelimit")
if rl == "abort":
- raise exception.StopExtraction("Rate limit exceeded")
+ raise exception.AbortExtraction("Rate limit exceeded")
elif rl and isinstance(rl, str) and rl.startswith("wait:"):
until = None
seconds = text.parse_float(rl.partition(":")[2]) or 60.0
@@ -2000,8 +2006,7 @@ def _login_impl(extr, username, password):
method="POST", fatal=None)
# update 'x-csrf-token' header (#5945)
- csrf_token = response.cookies.get("ct0")
- if csrf_token:
+ if csrf_token := response.cookies.get("ct0"):
headers["x-csrf-token"] = csrf_token
try:
@@ -2019,7 +2024,7 @@ def _login_impl(extr, username, password):
errors = []
for error in data.get("errors") or ():
msg = error.get("message")
- errors.append('"{}"'.format(msg) if msg else "Unknown error")
+ errors.append(f'"{msg}"' if msg else "Unknown error")
extr.log.debug(response.text)
raise exception.AuthenticationError(", ".join(errors))
@@ -2154,7 +2159,7 @@ def _login_impl(extr, username, password):
raise exception.AuthenticationError(
"No 'auth_token' cookie received")
else:
- raise exception.StopExtraction("Unrecognized subtask %s", subtask)
+ raise exception.AbortExtraction(f"Unrecognized subtask {subtask}")
inputs = {"subtask_id": subtask}
inputs.update(data)
@@ -2163,7 +2168,7 @@ def _login_impl(extr, username, password):
"subtask_inputs": [inputs],
}
- extr.sleep(random.uniform(1.0, 3.0), "login ({})".format(subtask))
+ extr.sleep(random.uniform(1.0, 3.0), f"login ({subtask})")
flow_token, subtask = process(data)
return {
generated by cgit v1.2.3 (git 2.46.0) at 2026-01-26 19:24:11 +0000