New upstream version 1.17.2.upstream/1.17.2

1 files changed, 56 insertions, 22 deletions

diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index 47286b7..9d1701f 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -20,7 +20,7 @@ import re
 
 BASE_PATTERN = (
     r"(?:https?://)?(?:"
-    r"(?:www\.)?deviantart\.com/([\w-]+)|"
+    r"(?:www\.)?deviantart\.com/(?!watch/)([\w-]+)|"
     r"(?!www\.)([\w-]+)\.deviantart\.com)"
 )
 
@@ -78,10 +78,6 @@ class DeviantartExtractor(Extractor):
             else:
                 self.user = profile["user"]["username"]
 
-        if self.extra:
-            finditer_stash = DeviantartStashExtractor.pattern.finditer
-            finditer_deviation = DeviantartDeviationExtractor.pattern.finditer
-
         yield Message.Version, 1
         for deviation in self.deviations():
             if isinstance(deviation, tuple):
@@ -109,7 +105,8 @@ class DeviantartExtractor(Extractor):
                         intermediary, count = re.subn(
                             r"(/f/[^/]+/[^/]+)/v\d+/.*",
                             r"/intermediary\1", content["src"], 1)
-                        if count and self._check_url(intermediary):
+                        if count:
+                            deviation["_fallback"] = (content["src"],)
                             content["src"] = intermediary
                     if self.quality:
                         content["src"] = re.sub(
@@ -138,14 +135,10 @@ class DeviantartExtractor(Extractor):
             if self.extra:
                 txt = (deviation.get("description", "") +
                        deviation.get("_journal", ""))
-                for match in finditer_stash(txt):
+                for match in DeviantartStashExtractor.pattern.finditer(txt):
                     url = text.ensure_http_scheme(match.group(0))
                     deviation["_extractor"] = DeviantartStashExtractor
                     yield Message.Queue, url, deviation
-                for match in finditer_deviation(txt):
-                    url = text.ensure_http_scheme(match.group(0))
-                    deviation["_extractor"] = DeviantartDeviationExtractor
-                    yield Message.Queue, url, deviation
 
     def deviations(self):
         """Return an iterable containing all relevant Deviation-objects"""
@@ -290,9 +283,6 @@ class DeviantartExtractor(Extractor):
         if mtype and mtype.startswith("image/"):
             content.update(data)
 
-    def _check_url(self, url):
-        return self.request(url, method="HEAD", fatal=False).status_code < 400
-
     def _limited_request(self, url, **kwargs):
         """Limits HTTP requests to one every 2 seconds"""
         kwargs["fatal"] = None
@@ -718,15 +708,16 @@ class DeviantartPopularExtractor(DeviantartExtractor):
         if path:
             self.category_path = path.strip("/")
         if trange:
-            trange = trange[8:] if trange.startswith("popular-") else ""
+            if trange.startswith("popular-"):
+                trange = trange[8:]
             self.time_range = trange.replace("-", "").replace("hours", "hr")
         if query:
             self.search_term = query.get("q")
 
         self.popular = {
             "search": self.search_term or "",
-            "range": trange or "24-hours",
-            "path": self.category_path,
+            "range" : trange or "",
+            "path"  : self.category_path,
         }
 
     def deviations(self):
@@ -738,6 +729,30 @@ class DeviantartPopularExtractor(DeviantartExtractor):
         deviation["popular"] = self.popular
 
 
+class DeviantartWatchExtractor(DeviantartExtractor):
+    """Extractor for Deviations from watched users"""
+    subcategory = "watch"
+    pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com"
+               r"/(?:watch/deviations|notifications/watch)()()")
+    test = (
+        ("https://www.deviantart.com/watch/deviations"),
+        ("https://www.deviantart.com/notifications/watch"),
+    )
+
+    def deviations(self):
+        return self.api.browse_deviantsyouwatch()
+
+
+class DeviantartWatchPostsExtractor(DeviantartExtractor):
+    """Extractor for Posts from watched users"""
+    subcategory = "watch-posts"
+    pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()"
+    test = ("https://www.deviantart.com/watch/posts",)
+
+    def deviations(self):
+        return self.api.browse_posts_deviantsyouwatch()
+
+
 ###############################################################################
 # Eclipse #####################################################################
 
@@ -926,6 +941,20 @@ class DeviantartOAuthAPI():
             self.client_id,
         )
 
+    def browse_deviantsyouwatch(self, offset=0):
+        """Yield deviations from users you watch"""
+        endpoint = "browse/deviantsyouwatch"
+        params = {"limit": "50", "offset": offset,
+                  "mature_content": self.mature}
+        return self._pagination(endpoint, params, public=False)
+
+    def browse_posts_deviantsyouwatch(self, offset=0):
+        """Yield posts from users you watch"""
+        endpoint = "browse/posts/deviantsyouwatch"
+        params = {"limit": "50", "offset": offset,
+                  "mature_content": self.mature}
+        return self._pagination(endpoint, params, public=False, unpack=True)
+
     def browse_popular(self, query=None, timerange=None, offset=0):
         """Yield popular deviations"""
         endpoint = "browse/popular"
@@ -1085,16 +1114,21 @@ class DeviantartOAuthAPI():
                 self.log.error(msg)
                 return data
 
-    def _pagination(self, endpoint, params, extend=True, public=True):
+    def _pagination(self, endpoint, params,
+                    extend=True, public=True, unpack=False):
         warn = True
         while True:
             data = self._call(endpoint, params, public=public)
             if "results" not in data:
                 self.log.error("Unexpected API response: %s", data)
                 return
+            results = data["results"]
 
+            if unpack:
+                results = [item["journal"] for item in results
+                           if "journal" in item]
             if extend:
-                if public and len(data["results"]) < params["limit"]:
+                if public and len(results) < params["limit"]:
                     if self.refresh_token_key:
                         self.log.debug("Switching to private access token")
                         public = False
@@ -1106,10 +1140,10 @@ class DeviantartOAuthAPI():
                             "oauth:deviantart' and follow the instructions to "
                             "be able to access them.")
                 if self.metadata:
-                    self._metadata(data["results"])
+                    self._metadata(results)
                 if self.folders:
-                    self._folders(data["results"])
-            yield from data["results"]
+                    self._folders(results)
+            yield from results
 
             if not data["has_more"]:
                 return