diff options
Diffstat (limited to 'gallery_dl/extractor/500px.py')
| -rw-r--r-- | gallery_dl/extractor/500px.py | 171 |
1 files changed, 99 insertions, 72 deletions
diff --git a/gallery_dl/extractor/500px.py b/gallery_dl/extractor/500px.py index 7ecdef7..96cb021 100644 --- a/gallery_dl/extractor/500px.py +++ b/gallery_dl/extractor/500px.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,7 +9,7 @@ """Extractors for https://500px.com/""" from .common import Extractor, Message -from .. import text +import json BASE_PATTERN = r"(?:https?://)?(?:web\.)?500px\.com" @@ -48,7 +48,7 @@ class _500pxExtractor(Extractor): def photos(self): """Returns an iterable containing all relevant photo IDs""" - def _extend(self, photos): + def _extend(self, edges): """Extend photos with additional metadata and higher resolution URLs""" url = "https://api.500px.com/v1/photos" params = { @@ -62,40 +62,42 @@ class _500pxExtractor(Extractor): "liked_by" : "1", "following_sample" : "100", "image_size" : "4096", - "ids" : ",".join(str(p["id"]) for p in photos), + "ids" : ",".join( + str(edge["node"]["legacyId"]) for edge in edges), } - data = self._api_call(url, params)["photos"] - for photo in photos: - pid = str(photo["id"]) - photo.update(data[pid]) - return photos + data = self._request_api(url, params)["photos"] + return [ + data[str(edge["node"]["legacyId"])] + for edge in edges + ] - def _api_call(self, url, params, csrf_token=None): + def _request_api(self, url, params, csrf_token=None): headers = {"Origin": self.root, "X-CSRF-Token": csrf_token} return self.request(url, headers=headers, params=params).json() - def _pagination(self, url, params, csrf): - params["page"] = 1 - while True: - data = self._api_call(url, params, csrf) - yield from self._extend(data["photos"]) - - if params["page"] >= data["total_pages"]: - return - params["page"] += 1 + def _request_graphql(self, opname, variables, query_hash): + url = "https://api.500px.com/graphql" + params = { + "operationName": opname, + "variables" : json.dumps(variables), + "extensions" : '{"persistedQuery":{"version":1' + ',"sha256Hash":"' + query_hash + '"}}', + } + return self.request(url, params=params).json()["data"] class _500pxUserExtractor(_500pxExtractor): """Extractor for photos from a user's photostream on 500px.com""" subcategory = "user" - pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/?(?:$|\?|#)" + pattern = BASE_PATTERN + r"/(?!photo/)(?:p/)?([^/?&#]+)/?(?:$|\?|#)" test = ( - ("https://500px.com/light_expression_photography", { + ("https://500px.com/p/light_expression_photography", { "pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2", "range": "1-99", "count": 99, }), + ("https://500px.com/light_expression_photography"), ("https://web.500px.com/light_expression_photography"), ) @@ -104,72 +106,97 @@ class _500pxUserExtractor(_500pxExtractor): self.user = match.group(1) def photos(self): - # get csrf token and user id from webpage - url = "{}/{}".format(self.root, self.user) - page = self.request(url).text - csrf_token, pos = text.extract(page, 'csrf-token" content="', '"') - user_id , pos = text.extract(page, '/user/', '"', pos) + variables = {"username": self.user, "pageSize": 20} + photos = self._request_graphql( + "OtherPhotosQuery", variables, + "54524abbdc809f8d4e10d37839e8ab2d" + "3035413688cad9c7fbece13b66637e9d", + )["user"]["photos"] - # get user photos - url = "https://api.500px.com/v1/photos" - params = { - "feature" : "user", - "stream" : "photos", - "rpp" : "50", - "user_id" : user_id, - } - return self._pagination(url, params, csrf_token) + while True: + yield from self._extend(photos["edges"]) + + if not photos["pageInfo"]["hasNextPage"]: + return + + variables["cursor"] = photos["pageInfo"]["endCursor"] + photos = self._request_graphql( + "OtherPhotosPaginationContainerQuery", variables, + "6d31e01104456ce642a2c6fc2f936812" + "b0f2a65c442d03e1521d769c20efe507", + )["userByUsername"]["photos"] class _500pxGalleryExtractor(_500pxExtractor): """Extractor for photo galleries on 500px.com""" subcategory = "gallery" directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}") - pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/galleries/([^/?&#]+)" - test = ("https://500px.com/fashvamp/galleries/lera", { - "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", - "count": 3, - "keyword": { - "gallery": dict, - "user": dict, - }, - }) + pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?" + r"([^/?&#]+)/galleries/([^/?&#]+)") + test = ( + ("https://500px.com/p/fashvamp/galleries/lera", { + "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", + "count": 3, + "keyword": { + "gallery": dict, + "user": dict, + }, + }), + ("https://500px.com/fashvamp/galleries/lera"), + ) def __init__(self, match): _500pxExtractor.__init__(self, match) self.user_name, self.gallery_name = match.groups() - self.user_id = self.gallery_id = self.csrf_token = None + self.user_id = self._photos = None def metadata(self): - # get csrf token and user id from webpage - url = "{}/{}/galleries/{}".format( - self.root, self.user_name, self.gallery_name) - page = self.request(url).text - self.csrf_token, pos = text.extract(page, 'csrf-token" content="', '"') - self.user_id , pos = text.extract(page, 'App.CuratorId =', '\n', pos) - self.user_id = self.user_id.strip(" '\";") - - # get gallery metadata; transform gallery name into id - url = "https://api.500px.com/v1/users/{}/galleries/{}".format( - self.user_id, self.gallery_name) - params = { - # "include_user": "true", - "include_cover": "1", - "cover_size": "2048", + user = self._request_graphql( + "ProfileRendererQuery", {"username": self.user_name}, + "db1dba2cb7b7e94916d1005db16fea1a39d6211437b691c4de2f1a606c21c5fb", + )["profile"] + self.user_id = str(user["legacyId"]) + + variables = { + "galleryOwnerLegacyId": self.user_id, + "ownerLegacyId" : self.user_id, + "slug" : self.gallery_name, + "token" : None, + "pageSize" : 20, + } + gallery = self._request_graphql( + "GalleriesDetailQueryRendererQuery", variables, + "1afc7dede86ff73456b4defbc5aeb593e330b990943d114cbef7da5be0d7ce2f", + )["gallery"] + + self._photos = gallery["photos"] + del gallery["photos"] + return { + "gallery": gallery, + "user" : user, } - data = self._api_call(url, params, self.csrf_token) - self.gallery_id = data["gallery"]["id"] - return data def photos(self): - url = "https://api.500px.com/v1/users/{}/galleries/{}/items".format( - self.user_id, self.gallery_id) - params = { - "sort" : "position", - "sort_direction" : "asc", - "rpp" : "50", + photos = self._photos + variables = { + "ownerLegacyId": self.user_id, + "slug" : self.gallery_name, + "token" : None, + "pageSize" : 20, } - return self._pagination(url, params, self.csrf_token) + + while True: + yield from self._extend(photos["edges"]) + + if not photos["pageInfo"]["hasNextPage"]: + return + + variables["cursor"] = photos["pageInfo"]["endCursor"] + photos = self._request_graphql( + "GalleriesDetailPaginationContainerQuery", variables, + "3fcbc9ea1589f31c86fc43a0a02c2163" + "cab070f9d376651f270de9f30f031539", + )["galleryByOwnerIdAndSlugOrToken"]["photos"] class _500pxImageExtractor(_500pxExtractor): @@ -226,5 +253,5 @@ class _500pxImageExtractor(_500pxExtractor): self.photo_id = match.group(1) def photos(self): - photos = ({"id": self.photo_id},) - return self._extend(photos) + edges = ({"node": {"legacyId": self.photo_id}},) + return self._extend(edges) |