diff options
Diffstat (limited to 'gallery_dl/extractor/patreon.py')
| -rw-r--r-- | gallery_dl/extractor/patreon.py | 183 |
1 files changed, 183 insertions, 0 deletions
diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py new file mode 100644 index 0000000..4884497 --- /dev/null +++ b/gallery_dl/extractor/patreon.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://www.patreon.com/""" + +from .common import Extractor, Message +from .. import text +from ..cache import memcache + + +class PatreonExtractor(Extractor): + """Base class for patreon extractors""" + category = "patreon" + root = "https://www.patreon.com" + directory_fmt = ("{category}", "{creator[full_name]}") + filename_fmt = "{id}_{title}_{num:>02}.{extension}" + archive_fmt = "{id}_{num}" + _warning = True + + def items(self): + yield Message.Version, 1 + + if self._warning: + if "session_id" not in self.session.cookies: + self.log.warning("no 'session_id' cookie set") + PatreonExtractor._warning = False + + for post in self.posts(): + yield Message.Directory, post + + post["num"] = 0 + content = post.get("content") + postfile = post.get("post_file") + + for url in text.extract_iter(content or "", 'src="', '"'): + post["num"] += 1 + yield Message.Url, url, text.nameext_from_url(url, post) + + if postfile: + post["num"] += 1 + text.nameext_from_url(postfile["name"], post) + yield Message.Url, postfile["url"], post + + for attachment in post["attachments"]: + post["num"] += 1 + text.nameext_from_url(attachment["name"], post) + yield Message.Url, attachment["url"], post + + def posts(self): + """Return all relevant post objects""" + + def _pagination(self, url): + headers = {"Referer": self.root} + empty = [] + + while url: + posts = self.request(url, headers=headers).json() + + if "included" not in posts: + return + + # collect attachments + attachments = {} + for inc in posts["included"]: + if inc["type"] == "attachment": + attachments[inc["id"]] = inc["attributes"] + + # update posts + for post in posts["data"]: + attr = post["attributes"] + attr["id"] = text.parse_int(post["id"]) + attr["date"] = text.parse_datetime( + attr["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + attr["creator"] = self._user( + post["relationships"]["user"]["links"]["related"]) + + # add attachments to post attributes + files = post["relationships"].get("attachments") + if files: + attr["attachments"] = [ + attachments[f["id"]] + for f in files["data"] + ] + else: + attr["attachments"] = empty + + yield attr + + if "links" not in posts: + return + url = posts["links"].get("next") + + @memcache(keyarg=1) + def _user(self, url): + user = self.request(url).json()["data"] + attr = user["attributes"] + attr["id"] = user["id"] + attr["date"] = text.parse_datetime( + attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z") + return attr + + @staticmethod + def _build_url(endpoint, query): + return ( + "https://www.patreon.com/api/" + endpoint + + + "?include=user,attachments,user_defined_tags,campaign,poll.choices" + ",poll.current_user_responses.user,poll.current_user_responses.cho" + "ice,poll.current_user_responses.poll,access_rules.tier.null" + + "&fields[post]=change_visibility_at,comment_count,content,current_" + "user_can_delete,current_user_can_view,current_user_has_liked,embe" + "d,image,is_paid,like_count,min_cents_pledged_to_view,post_file,pu" + "blished_at,patron_count,patreon_url,post_type,pledge_url,thumbnai" + "l_url,teaser_text,title,upgrade_url,url,was_posted_by_campaign_ow" + "ner" + "&fields[user]=image_url,full_name,url" + "&fields[campaign]=avatar_photo_url,earnings_visibility,is_nsfw,is" + "_monthly,name,url" + "&fields[access_rule]=access_rule_type,amount_cents" + query + + + "&json-api-use-default-includes=false" + "&json-api-version=1.0" + ) + + +class PatreonCreatorExtractor(PatreonExtractor): + """Extractor for a creator's works""" + subcategory = "creator" + pattern = (r"(?:https?://)?(?:www\.)?patreon\.com" + r"/(?!(?:home|join|login|signup)(?:$|[/?&#]))([^/?&#]+)/?") + test = ("https://www.patreon.com/koveliana", { + "range": "1-25", + "count": ">= 25", + "keyword": { + "attachments": list, + "comment_count": int, + "content": str, + "creator": dict, + "date": "type:datetime", + "id": int, + "like_count": int, + "post_type": str, + "published_at": str, + "title": str, + }, + }) + + def __init__(self, match): + PatreonExtractor.__init__(self, match) + self.creator = match.group(1).lower() + + def posts(self): + url = "{}/{}".format(self.root, self.creator) + page = self.request(url).text + campaign_id = text.extract(page, "/campaign/", "/")[0] + + url = self._build_url("posts", ( + "&sort=-published_at" + "&filter[is_draft]=false" + "&filter[contains_exclusive_posts]=true" + "&filter[campaign_id]=" + campaign_id + )) + return self._pagination(url) + + +class PatreonUserExtractor(PatreonExtractor): + """Extractor for media from creators supported by you""" + subcategory = "user" + pattern = r"(?:https?://)?(?:www\.)?patreon\.com/home$" + test = ("https://www.patreon.com/home",) + + def posts(self): + url = self._build_url("stream", ( + "&page[cursor]=null" + "&filter[is_following]=true" + )) + return self._pagination(url) |