# -*- coding: utf-8 -*- # Copyright 2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.patreon.com/""" from .common import Extractor, Message from .. import text from ..cache import memcache class PatreonExtractor(Extractor): """Base class for patreon extractors""" category = "patreon" root = "https://www.patreon.com" directory_fmt = ("{category}", "{creator[full_name]}") filename_fmt = "{id}_{title}_{num:>02}.{extension}" archive_fmt = "{id}_{num}" _warning = True def items(self): yield Message.Version, 1 if self._warning: if "session_id" not in self.session.cookies: self.log.warning("no 'session_id' cookie set") PatreonExtractor._warning = False for post in self.posts(): yield Message.Directory, post post["num"] = 0 content = post.get("content") postfile = post.get("post_file") for url in text.extract_iter(content or "", 'src="', '"'): post["num"] += 1 yield Message.Url, url, text.nameext_from_url(url, post) if postfile: post["num"] += 1 text.nameext_from_url(postfile["name"], post) yield Message.Url, postfile["url"], post for attachment in post["attachments"]: post["num"] += 1 text.nameext_from_url(attachment["name"], post) yield Message.Url, attachment["url"], post def posts(self): """Return all relevant post objects""" def _pagination(self, url): headers = {"Referer": self.root} empty = [] while url: posts = self.request(url, headers=headers).json() if "included" not in posts: return # collect attachments attachments = {} for inc in posts["included"]: if inc["type"] == "attachment": attachments[inc["id"]] = inc["attributes"] # update posts for post in posts["data"]: attr = post["attributes"] attr["id"] = text.parse_int(post["id"]) attr["date"] = text.parse_datetime( attr["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z") attr["creator"] = self._user( post["relationships"]["user"]["links"]["related"]) # add attachments to post attributes files = post["relationships"].get("attachments") if files: attr["attachments"] = [ attachments[f["id"]] for f in files["data"] ] else: attr["attachments"] = empty yield attr if "links" not in posts: return url = posts["links"].get("next") @memcache(keyarg=1) def _user(self, url): user = self.request(url).json()["data"] attr = user["attributes"] attr["id"] = user["id"] attr["date"] = text.parse_datetime( attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z") return attr @staticmethod def _build_url(endpoint, query): return ( "https://www.patreon.com/api/" + endpoint + "?include=user,attachments,user_defined_tags,campaign,poll.choices" ",poll.current_user_responses.user,poll.current_user_responses.cho" "ice,poll.current_user_responses.poll,access_rules.tier.null" "&fields[post]=change_visibility_at,comment_count,content,current_" "user_can_delete,current_user_can_view,current_user_has_liked,embe" "d,image,is_paid,like_count,min_cents_pledged_to_view,post_file,pu" "blished_at,patron_count,patreon_url,post_type,pledge_url,thumbnai" "l_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_ow" "ner" "&fields[user]=image_url,full_name,url" "&fields[campaign]=avatar_photo_url,earnings_visibility,is_nsfw,is" "_monthly,name,url" "&fields[access_rule]=access_rule_type,amount_cents" + query + "&json-api-use-default-includes=false" "&json-api-version=1.0" ) class PatreonCreatorExtractor(PatreonExtractor): """Extractor for a creator's works""" subcategory = "creator" pattern = (r"(?:https?://)?(?:www\.)?patreon\.com" r"/(?!(?:home|join|login|signup)(?:$|[/?&#]))([^/?&#]+)/?") test = ("https://www.patreon.com/koveliana", { "range": "1-25", "count": ">= 25", "keyword": { "attachments": list, "comment_count": int, "content": str, "creator": dict, "date": "type:datetime", "id": int, "like_count": int, "post_type": str, "published_at": str, "title": str, }, }) def __init__(self, match): PatreonExtractor.__init__(self, match) self.creator = match.group(1).lower() def posts(self): url = "{}/{}".format(self.root, self.creator) page = self.request(url).text campaign_id = text.extract(page, "/campaign/", "/")[0] url = self._build_url("posts", ( "&sort=-published_at" "&filter[is_draft]=false" "&filter[contains_exclusive_posts]=true" "&filter[campaign_id]=" + campaign_id )) return self._pagination(url) class PatreonUserExtractor(PatreonExtractor): """Extractor for media from creators supported by you""" subcategory = "user" pattern = r"(?:https?://)?(?:www\.)?patreon\.com/home$" test = ("https://www.patreon.com/home",) def posts(self): url = self._build_url("stream", ( "&page[cursor]=null" "&filter[is_following]=true" )) return self._pagination(url)