diff options
Diffstat (limited to 'gallery_dl/extractor/patreon.py')
| -rw-r--r-- | gallery_dl/extractor/patreon.py | 70 |
1 files changed, 46 insertions, 24 deletions
diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 59c5f15..1f520c3 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -103,7 +103,10 @@ class PatreonExtractor(Extractor): """Return all relevant post objects""" def _pagination(self, url): - headers = {"Referer": self.root} + headers = { + "Referer" : self.root + "/", + "Content-Type": "application/vnd.api+json", + } while url: url = text.ensure_http_scheme(url) @@ -199,23 +202,36 @@ class PatreonExtractor(Extractor): return ( "https://www.patreon.com/api/" + endpoint + - "?include=user,images,attachments,user_defined_tags,campaign,poll." - "choices,poll.current_user_responses.user,poll.current_user_respon" - "ses.choice,poll.current_user_responses.poll,access_rules.tier.nul" - "l" - - "&fields[post]=change_visibility_at,comment_count,content,current_" - "user_can_delete,current_user_can_view,current_user_has_liked,embe" - "d,image,is_paid,like_count,min_cents_pledged_to_view,post_file,pu" - "blished_at,patron_count,patreon_url,post_type,pledge_url,thumbnai" - "l_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_ow" - "ner" + "?include=campaign,access_rules,attachments,audio,images,media," + "native_video_insights,poll.choices," + "poll.current_user_responses.user," + "poll.current_user_responses.choice," + "poll.current_user_responses.poll," + "user,user_defined_tags,ti_checks" + + "&fields[campaign]=currency,show_audio_post_download_links," + "avatar_photo_url,avatar_photo_image_urls,earnings_visibility," + "is_nsfw,is_monthly,name,url" + + "&fields[post]=change_visibility_at,comment_count,commenter_count," + "content,current_user_can_comment,current_user_can_delete," + "current_user_can_view,current_user_has_liked,embed,image," + "insights_last_updated_at,is_paid,like_count,meta_image_url," + "min_cents_pledged_to_view,post_file,post_metadata,published_at," + "patreon_url,post_type,pledge_url,preview_asset_type,thumbnail," + "thumbnail_url,teaser_text,title,upgrade_url,url," + "was_posted_by_campaign_owner,has_ti_violation,moderation_status," + "post_level_suspension_removal_date,pls_one_liners_by_category," + "video_preview,view_count" + + "&fields[post_tag]=tag_type,value" "&fields[user]=image_url,full_name,url" - "&fields[campaign]=avatar_photo_url,earnings_visibility,is_nsfw,is" - "_monthly,name,url" - "&fields[access_rule]=access_rule_type,amount_cents" + query + + "&fields[access_rule]=access_rule_type,amount_cents" + "&fields[media]=id,image_urls,download_url,metadata,file_name" + "&fields[native_video_insights]=average_view_duration," + "average_view_pct,has_preview,id,last_updated_at,num_views," + "preview_views,video_duration" + query + - "&json-api-use-default-includes=false" "&json-api-version=1.0" ) @@ -234,6 +250,10 @@ class PatreonExtractor(Extractor): filetypes = filetypes.split(",") return [genmap[ft] for ft in filetypes] + def _extract_bootstrap(self, page): + return json.loads(text.extr( + page, "window.patreon.bootstrap,", "\n});") + "}") + class PatreonCreatorExtractor(PatreonExtractor): """Extractor for a creator's works""" @@ -282,10 +302,12 @@ class PatreonCreatorExtractor(PatreonExtractor): url = "{}/user/posts?u={}".format(self.root, creator_id) else: url = "{}/{}/posts".format(self.root, self.creator) - page = self.request(url, notfound="creator").text - campaign_id = text.extr(page, "/campaign/", "/") - if not campaign_id: + + try: + data = self._extract_bootstrap(page) + campaign_id = data["creator"]["data"]["id"] + except (KeyError, ValueError): raise exception.NotFoundError("creator") filters = "".join( @@ -295,10 +317,10 @@ class PatreonCreatorExtractor(PatreonExtractor): ) url = self._build_url("posts", ( - "&sort=" + query.get("sort", "-published_at") + - "&filter[is_draft]=false" + "&filter[campaign_id]=" + campaign_id + "&filter[contains_exclusive_posts]=true" - "&filter[campaign_id]=" + campaign_id + filters + "&filter[is_draft]=false" + filters + + "&sort=" + query.get("sort", "-published_at") )) return self._pagination(url) @@ -313,6 +335,7 @@ class PatreonUserExtractor(PatreonExtractor): url = self._build_url("stream", ( "&page[cursor]=null" "&filter[is_following]=true" + "&json-api-use-default-includes=false" )) return self._pagination(url) @@ -347,8 +370,7 @@ class PatreonPostExtractor(PatreonExtractor): def posts(self): url = "{}/posts/{}".format(self.root, self.slug) page = self.request(url, notfound="post").text - data = text.extract(page, "window.patreon.bootstrap,", "\n});")[0] - post = json.loads(data + "}")["post"] + post = self._extract_bootstrap(page)["post"] included = self._transform(post["included"]) return (self._process(post["data"], included),) |