diff options
Diffstat (limited to 'gallery_dl/extractor/tungsten.py')
| -rw-r--r-- | gallery_dl/extractor/tungsten.py | 100 |
1 files changed, 100 insertions, 0 deletions
diff --git a/gallery_dl/extractor/tungsten.py b/gallery_dl/extractor/tungsten.py new file mode 100644 index 0000000..20d5a59 --- /dev/null +++ b/gallery_dl/extractor/tungsten.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://tungsten.run/""" + +from .common import Extractor, Message +from .. import text + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?tungsten\.run" + + +class TungstenExtractor(Extractor): + """Base class for tungsten extractors""" + category = "tungsten" + root = "https://tungsten.run" + directory_fmt = ("{category}", "{user[username]}") + filename_fmt = "{date} {title:?/ /}{uuid}.{extension}" + archive_fmt = "{uuid}" + + def items(self): + for post in self.posts(): + url = post["original_url"] + post["date"] = text.parse_datetime(post["created_at"]) + post["filename"] = url[url.rfind("/")+1:] + post["extension"] = "webp" + yield Message.Directory, post + yield Message.Url, url, post + + def _pagination(self, url, params): + params["page"] = 1 + params["per_page"] = 40 + + headers = { + "Origin": self.root, + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", + } + + while True: + data = self.request_json(url, params=params, headers=headers) + + yield from data + + if len(data) < params["per_page"]: + break + params["page"] += 1 + + +class TungstenPostExtractor(TungstenExtractor): + subcategory = "post" + pattern = rf"{BASE_PATTERN}/post/(\w+)" + example = "https://tungsten.run/post/AbCdEfGhIjKlMnOp" + + def posts(self): + url = f"{self.root}/post/{self.groups[0]}" + page = self.request(url).text + data = self._extract_nextdata(page) + return (data["props"]["pageProps"]["post"],) + + +class TungstenModelExtractor(TungstenExtractor): + subcategory = "model" + pattern = rf"{BASE_PATTERN}/model/(\w+)(?:/?\?model_version=(\w+))?" + example = "https://tungsten.run/model/AbCdEfGhIjKlM" + + def posts(self): + uuid_model, uuid_version = self.groups + + if uuid_version is None: + url = f"{self.root}/model/{uuid_model}/" + page = self.request(url).text + uuid_version = text.extr(page, '"modelVersionUUID":"', '"') + + url = "https://api.tungsten.run/v1/posts" + params = { + "sort" : "top_all_time", + "tweakable_only": "false", + "following" : "false", + "model_version_uuid": uuid_version, + } + return self._pagination(url, params) + + +class TungstenUserExtractor(TungstenExtractor): + subcategory = "user" + pattern = rf"{BASE_PATTERN}/user/([^/?#]+)" + example = "https://tungsten.run/user/USER/posts" + + def posts(self): + url = f"{self.root}/user/{self.groups[0]}" + page = self.request(url).text + uuid_user = text.extr(page, '"user":{"uuid":"', '"') + + url = f"https://api.tungsten.run/v1/users/{uuid_user}/posts" + params = {"sort": "top_all_time"} + return self._pagination(url, params) |
