New upstream version 1.19.0.upstream/1.19.0

1 files changed, 29 insertions, 8 deletions

diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
index 2dfcb55..4a3f6cd 100644
--- a/gallery_dl/extractor/twitter.py
+++ b/gallery_dl/extractor/twitter.py
@@ -50,7 +50,7 @@ class TwitterExtractor(Extractor):
             if not self.retweets and "retweeted_status_id_str" in tweet:
                 self.log.debug("Skipping %s (retweet)", tweet["id_str"])
                 continue
-            if not self.quoted and "quoted" in tweet:
+            if not self.quoted and "quoted_by_id_str" in tweet:
                 self.log.debug("Skipping %s (quoted tweet)", tweet["id_str"])
                 continue
             if "in_reply_to_user_id_str" in tweet and (
@@ -139,8 +139,10 @@ class TwitterExtractor(Extractor):
                 for size in ("original", "x_large", "large", "small"):
                     key = prefix + size
                     if key in bvals:
-                        files.append(bvals[key]["image_value"])
-                        return
+                        value = bvals[key].get("image_value")
+                        if value and "url" in value:
+                            files.append(value)
+                            return
         elif self.videos:
             url = "ytdl:{}/i/web/status/{}".format(self.root, tweet["id_str"])
             files.append({"url": url})
@@ -199,6 +201,8 @@ class TwitterExtractor(Extractor):
 
         if "in_reply_to_screen_name" in tweet:
             tdata["reply_to"] = tweet["in_reply_to_screen_name"]
+        if "quoted_by_id_str" in tweet:
+            tdata["quote_by"] = text.parse_int(tweet["quoted_by_id_str"])
 
         if "author" in tweet:
             tdata["author"] = self._transform_user(tweet["author"])
@@ -316,7 +320,7 @@ class TwitterExtractor(Extractor):
 
 
 class TwitterTimelineExtractor(TwitterExtractor):
-    """Extractor for all images from a user's timeline"""
+    """Extractor for Tweets from a user's timeline"""
     subcategory = "timeline"
     pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])"
                r"|i(?:/user/|ntent/user\?user_id=)(\d+))")
@@ -341,8 +345,25 @@ class TwitterTimelineExtractor(TwitterExtractor):
         return TwitterAPI(self).timeline_profile(self.user)
 
 
+class TwitterRepliesExtractor(TwitterExtractor):
+    """Extractor for Tweets from a user's timeline including replies"""
+    subcategory = "replies"
+    pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/with_replies(?!\w)"
+    test = (
+        ("https://twitter.com/supernaturepics/with_replies", {
+            "range": "1-40",
+            "url": "c570ac1aae38ed1463be726cc46f31cac3d82a40",
+        }),
+        ("https://mobile.twitter.com/supernaturepics/with_replies#t"),
+        ("https://www.twitter.com/id:2976459548/with_replies"),
+    )
+
+    def tweets(self):
+        return TwitterAPI(self).timeline_profile(self.user, replies=True)
+
+
 class TwitterMediaExtractor(TwitterExtractor):
-    """Extractor for all images from a user's Media Tweets"""
+    """Extractor for Tweets from a user's Media timeline"""
     subcategory = "media"
     pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/media(?!\w)"
     test = (
@@ -652,11 +673,11 @@ class TwitterAPI():
         endpoint = "/2/timeline/conversation/{}.json".format(conversation_id)
         return self._pagination(endpoint)
 
-    def timeline_profile(self, screen_name):
+    def timeline_profile(self, screen_name, replies=False):
         user_id = self._user_id_by_screen_name(screen_name)
         endpoint = "/2/timeline/profile/{}.json".format(user_id)
         params = self.params.copy()
-        params["include_tweet_replies"] = "false"
+        params["include_tweet_replies"] = "true" if replies else "false"
         return self._pagination(endpoint, params)
 
     def timeline_media(self, screen_name):
@@ -886,7 +907,7 @@ class TwitterAPI():
                         quoted = quoted.copy()
                         quoted["author"] = users[quoted["user_id_str"]]
                         quoted["user"] = tweet["user"]
-                        quoted["quoted"] = True
+                        quoted["quoted_by_id_str"] = tweet["id_str"]
                         yield quoted
 
             # update cursor value