New upstream version 1.24.5.upstream/1.24.5

1 files changed, 161 insertions, 24 deletions

diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index aeb2d0a..a3187fa 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -118,11 +118,18 @@ class DeviantartExtractor(Extractor):
             if "flash" in deviation:
                 yield self.commit(deviation, deviation["flash"])
 
-            if "excerpt" in deviation and self.commit_journal:
-                journal = self.api.deviation_content(deviation["deviationid"])
-                if self.extra:
-                    deviation["_journal"] = journal["html"]
-                yield self.commit_journal(deviation, journal)
+            if self.commit_journal:
+                if "excerpt" in deviation:
+                    journal = self.api.deviation_content(
+                        deviation["deviationid"])
+                elif "body" in deviation:
+                    journal = {"html": deviation.pop("body")}
+                else:
+                    journal = None
+                if journal:
+                    if self.extra:
+                        deviation["_journal"] = journal["html"]
+                    yield self.commit_journal(deviation, journal)
 
             if not self.extra:
                 continue
@@ -150,10 +157,19 @@ class DeviantartExtractor(Extractor):
         """Adjust the contents of a Deviation-object"""
         if "index" not in deviation:
             try:
-                deviation["index"] = text.parse_int(
-                    deviation["url"].rpartition("-")[2])
+                if deviation["url"].startswith("https://sta.sh"):
+                    filename = deviation["content"]["src"].split("/")[5]
+                    deviation["index_base36"] = filename.partition("-")[0][1:]
+                    deviation["index"] = id_from_base36(
+                        deviation["index_base36"])
+                else:
+                    deviation["index"] = text.parse_int(
+                        deviation["url"].rpartition("-")[2])
             except KeyError:
                 deviation["index"] = 0
+                deviation["index_base36"] = "0"
+        if "index_base36" not in deviation:
+            deviation["index_base36"] = base36_from_id(deviation["index"])
 
         if self.user:
             deviation["username"] = self.user
@@ -170,13 +186,11 @@ class DeviantartExtractor(Extractor):
 
         if self.comments:
             deviation["comments"] = (
-                self.api.comments_deviation(deviation["deviationid"])
+                self.api.comments(deviation["deviationid"], target="deviation")
                 if deviation["stats"]["comments"] else ()
             )
 
         # filename metadata
-        alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
-        deviation["index_base36"] = util.bencode(deviation["index"], alphabet)
         sub = re.compile(r"\W").sub
         deviation["filename"] = "".join((
             sub("_", deviation["title"].lower()), "_by_",
@@ -253,9 +267,10 @@ class DeviantartExtractor(Extractor):
         html = journal["html"]
         if html.startswith("<style"):
             html = html.partition("</style>")[2]
+        head, _, tail = html.rpartition("<script")
         content = "\n".join(
             text.unescape(text.remove_html(txt))
-            for txt in html.rpartition("<script")[0].split("<br />")
+            for txt in (head or tail).split("<br />")
         )
         txt = JOURNAL_TEMPLATE_TEXT.format(
             title=deviation["title"],
@@ -402,8 +417,9 @@ class DeviantartUserExtractor(DeviantartExtractor):
         }),
         ("https://www.deviantart.com/shimoda7", {
             "options": (("include", "all"),),
-            "pattern": r"/shimoda7/(gallery(/scraps)?|posts|favourites)$",
-            "count": 4,
+            "pattern": r"/shimoda7/"
+                       r"(gallery(/scraps)?|posts(/statuses)?|favourites)$",
+            "count": 5,
         }),
         ("https://shimoda7.deviantart.com/"),
     )
@@ -414,6 +430,7 @@ class DeviantartUserExtractor(DeviantartExtractor):
             (DeviantartGalleryExtractor , base + "gallery"),
             (DeviantartScrapsExtractor  , base + "gallery/scraps"),
             (DeviantartJournalExtractor , base + "posts"),
+            (DeviantartStatusExtractor  , base + "posts/statuses"),
             (DeviantartFavoriteExtractor, base + "favourites"),
         ), ("gallery",))
 
@@ -746,6 +763,97 @@ class DeviantartJournalExtractor(DeviantartExtractor):
         return self.api.browse_user_journals(self.user, self.offset)
 
 
+class DeviantartStatusExtractor(DeviantartExtractor):
+    """Extractor for an artist's status updates"""
+    subcategory = "status"
+    directory_fmt = ("{category}", "{username}", "Status")
+    filename_fmt = "{category}_{index}_{title}_{date}.{extension}"
+    archive_fmt = "S_{_username}_{index}.{extension}"
+    pattern = BASE_PATTERN + r"/posts/statuses"
+    test = (
+        ("https://www.deviantart.com/t1na/posts/statuses", {
+            "count": 0,
+        }),
+        ("https://www.deviantart.com/justgalym/posts/statuses", {
+            "count": 4,
+            "url": "bf4c44c0c60ff2648a880f4c3723464ad3e7d074",
+        }),
+        # shared deviation
+        ("https://www.deviantart.com/justgalym/posts/statuses", {
+            "options": (("journals", "none"),),
+            "count": 1,
+            "pattern": r"https://images-wixmp-\w+\.wixmp\.com/f"
+                       r"/[^/]+/[^.]+\.jpg\?token=",
+        }),
+        # shared sta.sh item
+        ("https://www.deviantart.com/vanillaghosties/posts/statuses", {
+            "options": (("journals", "none"), ("original", False)),
+            "range": "5-",
+            "count": 1,
+            "keyword": {
+                "index"       : int,
+                "index_base36": "re:^[0-9a-z]+$",
+                "url"         : "re:^https://sta.sh",
+            },
+        }),
+        ("https://www.deviantart.com/justgalym/posts/statuses", {
+            "options": (("journals", "text"),),
+            "url": "c8744f7f733a3029116607b826321233c5ca452d",
+        }),
+    )
+
+    def deviations(self):
+        for status in self.api.user_statuses(self.user, self.offset):
+            yield from self.status(status)
+
+    def status(self, status):
+        for item in status.get("items") or ():  # do not trust is_share
+            # shared deviations/statuses
+            if "deviation" in item:
+                yield item["deviation"].copy()
+            if "status" in item:
+                yield from self.status(item["status"].copy())
+        # assume is_deleted == true means necessary fields are missing
+        if status["is_deleted"]:
+            self.log.warning(
+                "Skipping status %s (deleted)", status.get("statusid"))
+            return
+        yield status
+
+    def prepare(self, deviation):
+        if "deviationid" in deviation:
+            return DeviantartExtractor.prepare(self, deviation)
+
+        try:
+            path = deviation["url"].split("/")
+            deviation["index"] = text.parse_int(path[-1] or path[-2])
+        except KeyError:
+            deviation["index"] = 0
+
+        if self.user:
+            deviation["username"] = self.user
+            deviation["_username"] = self.user.lower()
+        else:
+            deviation["username"] = deviation["author"]["username"]
+            deviation["_username"] = deviation["username"].lower()
+
+        deviation["date"] = dt = text.parse_datetime(deviation["ts"])
+        deviation["published_time"] = int(util.datetime_to_timestamp(dt))
+
+        deviation["da_category"] = "Status"
+        deviation["category_path"] = "status"
+        deviation["is_downloadable"] = False
+        deviation["title"] = "Status Update"
+
+        comments_count = deviation.pop("comments_count", 0)
+        deviation["stats"] = {"comments": comments_count}
+        if self.comments:
+            deviation["comments"] = (
+                self.api.comments(deviation["statusid"], target="status")
+                if comments_count else ()
+            )
+
+
 class DeviantartPopularExtractor(DeviantartExtractor):
     """Extractor for popular deviations"""
     subcategory = "popular"
@@ -867,7 +975,9 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
     archive_fmt = "g_{_username}_{index}.{extension}"
     pattern = (BASE_PATTERN + r"/(art|journal)/(?:[^/?#]+-)?(\d+)"
                r"|(?:https?://)?(?:www\.)?deviantart\.com/"
-               r"(?:view/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)(\d+)")
+               r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)"
+               r"(\d+)"  # bare deviation ID without slug
+               r"|(?:https?://)?fav\.me/d([0-9a-z]+)")  # base36
     test = (
         (("https://www.deviantart.com/shimoda7/art/For-the-sake-10073852"), {
             "options": (("original", 0),),
@@ -940,6 +1050,15 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
         ("https://www.deviantart.com/view/1", {
             "exception": exception.NotFoundError,
         }),
+        # /deviation/ (#3558)
+        ("https://www.deviantart.com/deviation/817215762"),
+        # fav.me (#3558)
+        ("https://fav.me/ddijrpu", {
+            "count": 1,
+        }),
+        ("https://fav.me/dddd", {
+            "exception": exception.NotFoundError,
+        }),
         # old-style URLs
         ("https://shimoda7.deviantart.com"
          "/art/For-the-sake-of-a-memory-10073852"),
@@ -956,7 +1075,8 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
     def __init__(self, match):
         DeviantartExtractor.__init__(self, match)
         self.type = match.group(3)
-        self.deviation_id = match.group(4) or match.group(5)
+        self.deviation_id = \
+            match.group(4) or match.group(5) or id_from_base36(match.group(6))
 
     def deviations(self):
         url = "{}/{}/{}/{}".format(
@@ -1149,9 +1269,9 @@ class DeviantartOAuthAPI():
                   "mature_content": self.mature}
         return self._pagination_list(endpoint, params)
 
-    def comments_deviation(self, deviation_id, offset=0):
-        """Fetch comments posted on a deviation"""
-        endpoint = "/comments/deviation/" + deviation_id
+    def comments(self, id, target, offset=0):
+        """Fetch comments posted on a target"""
+        endpoint = "/comments/{}/{}".format(target, id)
         params = {"maxdepth": "5", "offset": offset, "limit": 50,
                   "mature_content": self.mature}
         return self._pagination_list(endpoint, params=params, key="thread")
@@ -1187,8 +1307,6 @@ class DeviantartOAuthAPI():
 
     def deviation_metadata(self, deviations):
         """ Fetch deviation metadata for a set of deviations"""
-        if not deviations:
-            return []
         endpoint = "/deviation/metadata?" + "&".join(
             "deviationids[{}]={}".format(num, deviation["deviationid"])
             for num, deviation in enumerate(deviations)
@@ -1224,6 +1342,12 @@ class DeviantartOAuthAPI():
         endpoint = "/user/profile/" + username
         return self._call(endpoint, fatal=False)
 
+    def user_statuses(self, username, offset=0):
+        """Yield status updates of a specific user"""
+        endpoint = "/user/statuses/"
+        params = {"username": username, "offset": offset, "limit": 50}
+        return self._pagination(endpoint, params)
+
     def user_friends_watch(self, username):
         """Watch a user"""
         endpoint = "/user/friends/watch/" + username
@@ -1350,10 +1474,12 @@ class DeviantartOAuthAPI():
                             "Private deviations detected! Run 'gallery-dl "
                             "oauth:deviantart' and follow the instructions to "
                             "be able to access them.")
-                if self.metadata:
-                    self._metadata(results)
-                if self.folders:
-                    self._folders(results)
+                # "statusid" cannot be used instead
+                if results and "deviationid" in results[0]:
+                    if self.metadata:
+                        self._metadata(results)
+                    if self.folders:
+                        self._folders(results)
             yield from results
 
             if not data["has_more"] and (
@@ -1561,6 +1687,17 @@ def _login_impl(extr, username, password):
     }
 
 
+def id_from_base36(base36):
+    return util.bdecode(base36, _ALPHABET)
+
+
+def base36_from_id(deviation_id):
+    return util.bencode(int(deviation_id), _ALPHABET)
+
+
+_ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+
 ###############################################################################
 # Journal Formats #############################################################