diff options
Diffstat (limited to 'gallery_dl/extractor/vsco.py')
| -rw-r--r-- | gallery_dl/extractor/vsco.py | 176 |
1 files changed, 176 insertions, 0 deletions
diff --git a/gallery_dl/extractor/vsco.py b/gallery_dl/extractor/vsco.py new file mode 100644 index 0000000..639ec82 --- /dev/null +++ b/gallery_dl/extractor/vsco.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://vsco.co/""" + +from .common import Extractor, Message +from .. import text +import json + + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?vsco\.co/([^/]+)" + + +class VscoExtractor(Extractor): + """Base class for vsco extractors""" + category = "vsco" + root = "https://vsco.co" + directory_fmt = ("{category}", "{user}") + filename_fmt = "{id}.{extension}" + archive_fmt = "{id}" + + def __init__(self, match): + Extractor.__init__(self, match) + self.user = match.group(1).lower() + + def items(self): + yield Message.Version, 1 + yield Message.Directory, {"user": self.user} + for img in self.images(): + url = "https://" + (img.get("video_url") or img["responsive_url"]) + data = text.nameext_from_url(url, { + "id" : img["_id"], + "user" : self.user, + "grid" : img["grid_name"], + "meta" : img.get("image_meta") or {}, + "tags" : [tag["text"] for tag in img.get("tags") or ()], + "date" : text.parse_timestamp(img["upload_date"] // 1000), + "video" : img["is_video"], + "width" : img["width"], + "height": img["height"], + "description": img["description"], + }) + yield Message.Url, url, data + + def images(self): + """Return an iterable with all relevant image objects""" + + def _extract_preload_state(self, url): + page = self.request(url, notfound=self.subcategory).text + return json.loads(text.extract(page, "__PRELOADED_STATE__ = ", "<")[0]) + + def _pagination(self, url, params, token, key, extra): + headers = { + "Referer" : "{}/{}".format(self.root, self.user), + "Authorization" : "Bearer " + token, + "X-Client-Platform": "web", + "X-Client-Build" : "1", + } + + yield from map(self._transform_media, extra) + + while True: + data = self.request(url, params=params, headers=headers).json() + if not data.get(key): + return + yield from data[key] + params["page"] += 1 + + @staticmethod + def _transform_media(media): + media["_id"] = media["id"] + media["is_video"] = media["isVideo"] + media["grid_name"] = media["gridName"] + media["upload_date"] = media["uploadDate"] + media["responsive_url"] = media["responsiveUrl"] + media["video_url"] = media.get("videoUrl") + media["image_meta"] = media.get("imageMeta") + return media + + +class VscoUserExtractor(VscoExtractor): + """Extractor for images from a user on vsco.co""" + subcategory = "user" + pattern = BASE_PATTERN + r"/images/" + test = ("https://vsco.co/missuri/images/1", { + "range": "1-80", + "count": 80, + "pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+", + }) + + def images(self): + url = "{}/{}/images/1".format(self.root, self.user) + data = self._extract_preload_state(url) + + tkn = data["users"]["currentUser"]["tkn"] + sid = str(data["sites"]["siteByUsername"][self.user]["site"]["id"]) + + url = "{}/api/2.0/medias".format(self.root) + params = {"page": 2, "size": "30", "site_id": sid} + return self._pagination(url, params, tkn, "media", ( + data["medias"]["byId"][mid]["media"] + for mid in data["medias"]["bySiteId"][sid]["medias"]["1"] + )) + + +class VscoCollectionExtractor(VscoExtractor): + """Extractor for images from a collection on vsco.co""" + subcategory = "collection" + directory_fmt = ("{category}", "{user}", "collection") + archive_fmt = "c_{user}_{id}" + pattern = BASE_PATTERN + r"/collection/" + test = ("https://vsco.co/vsco/collection/1", { + "range": "1-80", + "count": 80, + "pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+", + }) + + def images(self): + url = "{}/{}/collection/1".format(self.root, self.user) + data = self._extract_preload_state(url) + + tkn = data["users"]["currentUser"]["tkn"] + cid = (data["sites"]["siteByUsername"][self.user] + ["site"]["siteCollectionId"]) + + url = "{}/api/2.0/collections/{}/medias".format(self.root, cid) + params = {"page": 2, "size": "20"} + return self._pagination(url, params, tkn, "medias", ( + data["medias"]["byId"][mid]["media"] + for mid in data + ["collections"]["byCollectionId"][cid]["collection"]["1"] + )) + + +class VscoImageExtractor(VscoExtractor): + """Extractor for individual images on vsco.co""" + subcategory = "image" + pattern = BASE_PATTERN + r"/media/([0-9a-fA-F]+)" + test = ( + ("https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", { + "url": "faa214d10f859f374ad91da3f7547d2439f5af08", + "content": "1394d070828d82078035f19a92f404557b56b83f", + "keyword": { + "id" : "5d34b93ef632433030707ce2", + "user" : "erenyildiz", + "grid" : "erenyildiz", + "meta" : dict, + "tags" : list, + "date" : "type:datetime", + "video" : False, + "width" : 1537, + "height": 1537, + "description": "re:Ni seviyorum. #vsco #vscox #vscochallenges", + }, + }), + ("https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", { + "url": "08e7eef3301756ce81206c0b47c1e9373756a74a", + "content": "e739f058d726ee42c51c180a505747972a7dfa47", + "keyword": {"video" : True}, + }), + ) + + def __init__(self, match): + VscoExtractor.__init__(self, match) + self.media_id = match.group(2) + + def images(self): + url = "{}/{}/media/{}".format(self.root, self.user, self.media_id) + data = self._extract_preload_state(url) + media = data["medias"]["byId"].popitem()[1]["media"] + return (self._transform_media(media),) |