New upstream version 1.26.9.upstream/1.26.9

1 files changed, 82 insertions, 15 deletions

diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py
index b58b3d3..49fde7b 100644
--- a/gallery_dl/extractor/artstation.py
+++ b/gallery_dl/extractor/artstation.py
@@ -29,11 +29,13 @@ class ArtstationExtractor(Extractor):
         self.user = match.group(1) or match.group(2)
 
     def items(self):
-        data = self.metadata()
-
-        projects = self.projects()
+        videos = self.config("videos", True)
+        previews = self.config("previews", False)
         external = self.config("external", False)
         max_posts = self.config("max-posts")
+
+        data = self.metadata()
+        projects = self.projects()
         if max_posts:
             projects = itertools.islice(projects, max_posts)
 
@@ -45,13 +47,29 @@ class ArtstationExtractor(Extractor):
                 asset["num"] = num
                 yield Message.Directory, asset
 
-                if adict["has_embedded_player"] and external:
+                if adict["has_embedded_player"]:
                     player = adict["player_embedded"]
                     url = (text.extr(player, 'src="', '"') or
                            text.extr(player, "src='", "'"))
-                    if url and not url.startswith(self.root):
-                        asset["extension"] = None
-                        yield Message.Url, "ytdl:" + url, asset
+                    if url.startswith(self.root):
+                        # video clip hosted on artstation
+                        if videos:
+                            page = self.request(url).text
+                            url = text.extr(page, ' src="', '"')
+                            text.nameext_from_url(url, asset)
+                            yield Message.Url, url, asset
+                    elif url:
+                        # external URL
+                        if external:
+                            asset["extension"] = "mp4"
+                            yield Message.Url, "ytdl:" + url, asset
+                    else:
+                        self.log.debug(player)
+                        self.log.warning(
+                            "Failed to extract embedded player URL (%s)",
+                            adict.get("id"))
+
+                    if not previews:
                         continue
 
                 if adict["has_image"]:
@@ -59,10 +77,11 @@ class ArtstationExtractor(Extractor):
                     text.nameext_from_url(url, asset)
 
                     url = self._no_cache(url)
-                    lhs, _, rhs = url.partition("/large/")
-                    if rhs:
-                        url = lhs + "/4k/" + rhs
-                        asset["_fallback"] = self._image_fallback(lhs, rhs)
+                    if "/video_clips/" not in url:
+                        lhs, _, rhs = url.partition("/large/")
+                        if rhs:
+                            url = lhs + "/4k/" + rhs
+                            asset["_fallback"] = self._image_fallback(lhs, rhs)
 
                     yield Message.Url, url, asset
 
@@ -175,7 +194,7 @@ class ArtstationUserExtractor(ArtstationExtractor):
     subcategory = "user"
     pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com"
                r"/(?!artwork|projects|search)([^/?#]+)(?:/albums/all)?"
-               r"|((?!www)\w+)\.artstation\.com(?:/projects)?)/?$")
+               r"|((?!www)[\w-]+)\.artstation\.com(?:/projects)?)/?$")
     example = "https://www.artstation.com/USER"
 
     def projects(self):
@@ -192,7 +211,7 @@ class ArtstationAlbumExtractor(ArtstationExtractor):
     archive_fmt = "a_{album[id]}_{asset[id]}"
     pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com"
                r"/(?!artwork|projects|search)([^/?#]+)"
-               r"|((?!www)\w+)\.artstation\.com)/albums/(\d+)")
+               r"|((?!www)[\w-]+)\.artstation\.com)/albums/(\d+)")
     example = "https://www.artstation.com/USER/albums/12345"
 
     def __init__(self, match):
@@ -226,7 +245,7 @@ class ArtstationLikesExtractor(ArtstationExtractor):
     directory_fmt = ("{category}", "{userinfo[username]}", "Likes")
     archive_fmt = "f_{userinfo[id]}_{asset[id]}"
     pattern = (r"(?:https?://)?(?:www\.)?artstation\.com"
-               r"/(?!artwork|projects|search)([^/?#]+)/likes/?")
+               r"/(?!artwork|projects|search)([^/?#]+)/likes")
     example = "https://www.artstation.com/USER/likes"
 
     def projects(self):
@@ -234,6 +253,54 @@ class ArtstationLikesExtractor(ArtstationExtractor):
         return self._pagination(url)
 
 
+class ArtstationCollectionExtractor(ArtstationExtractor):
+    """Extractor for an artstation collection"""
+    subcategory = "collection"
+    directory_fmt = ("{category}", "{user}",
+                     "{collection[id]} {collection[name]}")
+    archive_fmt = "c_{collection[id]}_{asset[id]}"
+    pattern = (r"(?:https?://)?(?:www\.)?artstation\.com"
+               r"/(?!artwork|projects|search)([^/?#]+)/collections/(\d+)")
+    example = "https://www.artstation.com/USER/collections/12345"
+
+    def __init__(self, match):
+        ArtstationExtractor.__init__(self, match)
+        self.collection_id = match.group(2)
+
+    def metadata(self):
+        url = "{}/collections/{}.json".format(
+            self.root, self.collection_id)
+        params = {"username": self.user}
+        collection = self.request(
+            url, params=params, notfound="collection").json()
+        return {"collection": collection, "user": self.user}
+
+    def projects(self):
+        url = "{}/collections/{}/projects.json".format(
+            self.root, self.collection_id)
+        params = {"collection_id": self.collection_id}
+        return self._pagination(url, params)
+
+
+class ArtstationCollectionsExtractor(ArtstationExtractor):
+    """Extractor for an artstation user's collections"""
+    subcategory = "collections"
+    pattern = (r"(?:https?://)?(?:www\.)?artstation\.com"
+               r"/(?!artwork|projects|search)([^/?#]+)/collections/?$")
+    example = "https://www.artstation.com/USER/collections"
+
+    def items(self):
+        url = self.root + "/collections.json"
+        params = {"username": self.user}
+
+        for collection in self.request(
+                url, params=params, notfound="collections").json():
+            url = "{}/{}/collections/{}".format(
+                self.root, self.user, collection["id"])
+            collection["_extractor"] = ArtstationCollectionExtractor
+            yield Message.Queue, url, collection
+
+
 class ArtstationChallengeExtractor(ArtstationExtractor):
     """Extractor for submissions of artstation challenges"""
     subcategory = "challenge"
@@ -355,7 +422,7 @@ class ArtstationImageExtractor(ArtstationExtractor):
     """Extractor for images from a single artstation project"""
     subcategory = "image"
     pattern = (r"(?:https?://)?(?:"
-               r"(?:\w+\.)?artstation\.com/(?:artwork|projects|search)"
+               r"(?:[\w-]+\.)?artstation\.com/(?:artwork|projects|search)"
                r"|artstn\.co/p)/(\w+)")
     example = "https://www.artstation.com/artwork/abcde"