diff options
Diffstat (limited to 'gallery_dl/extractor/mastodon.py')
| -rw-r--r-- | gallery_dl/extractor/mastodon.py | 216 |
1 files changed, 92 insertions, 124 deletions
diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index 0e063d5..daa3d65 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2020 Mike Fährmann +# Copyright 2019-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -8,35 +8,25 @@ """Extractors for mastodon instances""" -from .common import Extractor, Message -from .. import text, util, config, exception -import re +from .common import BaseExtractor, Message +from .. import text, exception +from ..cache import cache -class MastodonExtractor(Extractor): +class MastodonExtractor(BaseExtractor): """Base class for mastodon extractors""" basecategory = "mastodon" directory_fmt = ("mastodon", "{instance}", "{account[username]}") filename_fmt = "{category}_{id}_{media[id]}.{extension}" archive_fmt = "{media[id]}" cookiedomain = None - instance = None - root = None def __init__(self, match): - Extractor.__init__(self, match) - self.api = MastodonAPI(self) - - def config(self, key, default=None): - return config.interpolate_common( - ("extractor",), ( - (self.category, self.subcategory), - (self.basecategory, self.instance, self.subcategory), - ), key, default, - ) + BaseExtractor.__init__(self, match) + self.instance = self.root.partition("://")[2] + self.item = match.group(match.lastindex) def items(self): - yield Message.Version, 1 for status in self.statuses(): attachments = status["media_attachments"] if attachments: @@ -60,34 +50,81 @@ class MastodonExtractor(Extractor): status["created_at"][:19], "%Y-%m-%dT%H:%M:%S") +INSTANCES = { + "mastodon.social": { + "root" : "https://mastodon.social", + "access-token" : "Y06R36SMvuXXN5_wiPKFAEFiQaMSQg0o_hGgc86Jj48", + "client-id" : "dBSHdpsnOUZgxOnjKSQrWEPakO3ctM7HmsyoOd4FcRo", + "client-secret": "DdrODTHs_XoeOsNVXnILTMabtdpWrWOAtrmw91wU1zI", + }, + "pawoo": { + "root" : "https://pawoo.net", + "access-token" : "c12c9d275050bce0dc92169a28db09d7" + "0d62d0a75a8525953098c167eacd3668", + "client-id" : "978a25f843ec01e53d09be2c290cd75c" + "782bc3b7fdbd7ea4164b9f3c3780c8ff", + "client-secret": "9208e3d4a7997032cf4f1b0e12e5df38" + "8428ef1fadb446dcfeb4f5ed6872d97b", + }, + "baraag": { + "root" : "https://baraag.net", + "access-token" : "53P1Mdigf4EJMH-RmeFOOSM9gdSDztmrAYFgabOKKE0", + "client-id" : "czxx2qilLElYHQ_sm-lO8yXuGwOHxLX9RYYaD0-nq1o", + "client-secret": "haMaFdMBgK_-BIxufakmI2gFgkYjqmgXGEO2tB-R2xY", + } +} + +BASE_PATTERN = MastodonExtractor.update(INSTANCES) + + class MastodonUserExtractor(MastodonExtractor): """Extractor for all images of an account/user""" subcategory = "user" - - def __init__(self, match): - MastodonExtractor.__init__(self, match) - self.account_name = match.group(1) + pattern = BASE_PATTERN + r"/@([^/?#]+)(?:/media)?/?$" + test = ( + ("https://mastodon.social/@jk", { + "pattern": r"https://files.mastodon.social/media_attachments" + r"/files/(\d+/){3,}original/\w+", + "range": "1-60", + "count": 60, + }), + ("https://pawoo.net/@yoru_nine/", { + "range": "1-60", + "count": 60, + }), + ("https://baraag.net/@pumpkinnsfw"), + ) def statuses(self): - handle = "@{}@{}".format(self.account_name, self.instance) - for account in self.api.account_search(handle, 1): - if account["username"] == self.account_name: + api = MastodonAPI(self) + username = self.item + handle = "@{}@{}".format(username, self.instance) + for account in api.account_search(handle, 1): + if account["username"] == username: break else: raise exception.NotFoundError("account") - return self.api.account_statuses(account["id"]) + return api.account_statuses(account["id"]) class MastodonStatusExtractor(MastodonExtractor): """Extractor for images from a status""" subcategory = "status" - - def __init__(self, match): - MastodonExtractor.__init__(self, match) - self.status_id = match.group(1) + pattern = BASE_PATTERN + r"/@[^/?#]+/(\d+)" + test = ( + ("https://mastodon.social/@jk/103794036899778366", { + "count": 4, + }), + ("https://pawoo.net/@yoru_nine/105038878897832922", { + "content": "b52e807f8ab548d6f896b09218ece01eba83987a", + }), + ("https://baraag.net/@pumpkinnsfw/104364170556898443", { + "content": "67748c1b828c58ad60d0fe5729b59fb29c872244", + }), + ) def statuses(self): - return (self.api.status(self.status_id),) + return (MastodonAPI(self).status(self.item),) class MastodonAPI(): @@ -97,35 +134,46 @@ class MastodonAPI(): https://github.com/tootsuite/documentation/blob/master/Using-the-API/API.md """ - def __init__(self, extractor, access_token=None): + def __init__(self, extractor): self.root = extractor.root self.extractor = extractor + access_token = extractor.config("access-token") + if access_token is None or access_token == "cache": + access_token = _access_token_cache(extractor.instance) if not access_token: - access_token = extractor.config( - "access-token", extractor.access_token) - self.headers = {"Authorization": "Bearer {}".format(access_token)} + try: + access_token = INSTANCES[extractor.category]["access-token"] + except (KeyError, TypeError): + raise exception.StopExtraction( + "Missing access token.\n" + "Run 'gallery-dl oauth:mastodon:%s' to obtain one.", + extractor.instance) + + self.headers = {"Authorization": "Bearer " + access_token} def account_search(self, query, limit=40): """Search for content""" + endpoint = "/v1/accounts/search" params = {"q": query, "limit": limit} - return self._call("accounts/search", params).json() + return self._call(endpoint, params).json() def account_statuses(self, account_id): """Get an account's statuses""" - endpoint = "accounts/{}/statuses".format(account_id) + endpoint = "/v1/accounts/{}/statuses".format(account_id) params = {"only_media": "1"} return self._pagination(endpoint, params) def status(self, status_id): - """Fetch a Status""" - return self._call("statuses/" + status_id).json() + """Fetch a status""" + endpoint = "/v1/statuses/" + status_id + return self._call(endpoint).json() def _call(self, endpoint, params=None): if endpoint.startswith("http"): url = endpoint else: - url = "{}/api/v1/{}".format(self.root, endpoint) + url = self.root + "/api" + endpoint while True: response = self.extractor.request( @@ -145,7 +193,7 @@ class MastodonAPI(): raise exception.StopExtraction(response.json().get("error")) def _pagination(self, endpoint, params): - url = "{}/api/v1/{}".format(self.root, endpoint) + url = endpoint while url: response = self._call(url, params) yield from response.json() @@ -156,86 +204,6 @@ class MastodonAPI(): url = url["url"] -def generate_extractors(): - """Dynamically generate Extractor classes for Mastodon instances""" - - symtable = globals() - extractors = config.get(("extractor",), "mastodon") - if extractors: - util.combine_dict(EXTRACTORS, extractors) - config.set(("extractor",), "mastodon", EXTRACTORS) - - for instance, info in EXTRACTORS.items(): - - if not isinstance(info, dict): - continue - - category = info.get("category") or instance.replace(".", "") - root = info.get("root") or "https://" + instance - name = (info.get("name") or category).capitalize() - token = info.get("access-token") - pattern = info.get("pattern") or re.escape(instance) - - class Extr(MastodonUserExtractor): - pass - - Extr.__name__ = Extr.__qualname__ = name + "UserExtractor" - Extr.__doc__ = "Extractor for all images of a user on " + instance - Extr.category = category - Extr.instance = instance - Extr.pattern = (r"(?:https?://)?" + pattern + - r"/@([^/?#]+)(?:/media)?/?$") - Extr.test = info.get("test-user") - Extr.root = root - Extr.access_token = token - symtable[Extr.__name__] = Extr - - class Extr(MastodonStatusExtractor): - pass - - Extr.__name__ = Extr.__qualname__ = name + "StatusExtractor" - Extr.__doc__ = "Extractor for images from a status on " + instance - Extr.category = category - Extr.instance = instance - Extr.pattern = r"(?:https?://)?" + pattern + r"/@[^/?#]+/(\d+)" - Extr.test = info.get("test-status") - Extr.root = root - Extr.access_token = token - symtable[Extr.__name__] = Extr - - -EXTRACTORS = { - "mastodon.social": { - "category" : "mastodon.social", - "access-token" : "Y06R36SMvuXXN5_wiPKFAEFiQaMSQg0o_hGgc86Jj48", - "client-id" : "dBSHdpsnOUZgxOnjKSQrWEPakO3ctM7HmsyoOd4FcRo", - "client-secret": "DdrODTHs_XoeOsNVXnILTMabtdpWrWOAtrmw91wU1zI", - "test-user" : ("https://mastodon.social/@jk", { - "pattern": r"https://files.mastodon.social/media_attachments" - r"/files/(\d+/){3,}original/\w+", - "range": "1-60", - "count": 60, - }), - "test-status" : ("https://mastodon.social/@jk/103794036899778366", { - "count": 4, - }), - }, - "pawoo.net": { - "category" : "pawoo", - "access-token" : "c12c9d275050bce0dc92169a28db09d7" - "0d62d0a75a8525953098c167eacd3668", - "client-id" : "978a25f843ec01e53d09be2c290cd75c" - "782bc3b7fdbd7ea4164b9f3c3780c8ff", - "client-secret": "9208e3d4a7997032cf4f1b0e12e5df38" - "8428ef1fadb446dcfeb4f5ed6872d97b", - }, - "baraag.net": { - "category" : "baraag", - "access-token" : "53P1Mdigf4EJMH-RmeFOOSM9gdSDztmrAYFgabOKKE0", - "client-id" : "czxx2qilLElYHQ_sm-lO8yXuGwOHxLX9RYYaD0-nq1o", - "client-secret": "haMaFdMBgK_-BIxufakmI2gFgkYjqmgXGEO2tB-R2xY", - }, -} - - -generate_extractors() +@cache(maxage=100*365*24*3600, keyarg=0) +def _access_token_cache(instance): + return None |