diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/bash_completion.py | 56 | ||||
| -rwxr-xr-x | scripts/build_testresult_db.py | 56 | ||||
| -rwxr-xr-x | scripts/create_test_data.py | 69 | ||||
| -rw-r--r-- | scripts/hook-gallery_dl.py | 9 | ||||
| -rwxr-xr-x | scripts/man.py | 304 | ||||
| -rwxr-xr-x | scripts/pyinstaller.py | 18 | ||||
| -rwxr-xr-x | scripts/release.sh | 167 | ||||
| -rwxr-xr-x | scripts/run_tests.sh | 24 | ||||
| -rwxr-xr-x | scripts/supportedsites.py | 285 | ||||
| -rw-r--r-- | scripts/util.py | 11 |
10 files changed, 0 insertions, 999 deletions
diff --git a/scripts/bash_completion.py b/scripts/bash_completion.py deleted file mode 100755 index 69e6a79..0000000 --- a/scripts/bash_completion.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Mike Fährmann -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Generate bash completion script from gallery-dl's argument parser""" - -import util -from gallery_dl import option - - -TEMPLATE = """_gallery_dl() -{ - local cur prev - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - - if [[ "${prev}" =~ ^(%(fileopts)s)$ ]]; then - COMPREPLY=( $(compgen -f -- "${cur}") ) - elif [[ "${prev}" =~ ^(%(diropts)s)$ ]]; then - COMPREPLY=( $(compgen -d -- "${cur}") ) - else - COMPREPLY=( $(compgen -W "%(opts)s" -- "${cur}") ) - fi -} - -complete -F _gallery_dl gallery-dl -""" - -opts = [] -diropts = [] -fileopts = [] -for action in option.build_parser()._actions: - - if action.metavar in ("DEST",): - diropts.extend(action.option_strings) - - elif action.metavar in ("FILE", "CFG"): - fileopts.extend(action.option_strings) - - for opt in action.option_strings: - if opt.startswith("--"): - opts.append(opt) - -PATH = util.path("gallery-dl.bash_completion") -with open(PATH, "w", encoding="utf-8") as file: - file.write(TEMPLATE % { - "opts" : " ".join(opts), - "diropts" : "|".join(diropts), - "fileopts": "|".join(fileopts), - }) diff --git a/scripts/build_testresult_db.py b/scripts/build_testresult_db.py deleted file mode 100755 index fda9f64..0000000 --- a/scripts/build_testresult_db.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -"""Collect results of extractor unit tests""" - -import sys -import os.path -import datetime - -import util -from gallery_dl import extractor, job, config -from test.test_results import setup_test_config - - -# filter test cases - -tests = [ - (idx, extr, url, result) - - for extr in extractor.extractors() - if hasattr(extr, "test") and extr.test - if len(sys.argv) <= 1 or extr.category in sys.argv - - for idx, (url, result) in enumerate(extr._get_tests()) - if result -] - - -# setup target directory - -path = util.path("archive", "testdb", str(datetime.date.today())) -os.makedirs(path, exist_ok=True) - - -for idx, extr, url, result in tests: - - # filename - name = "{}-{}-{}.json".format(extr.category, extr.subcategory, idx) - print(name) - - # config values - setup_test_config() - - if "options" in result: - for key, value in result["options"]: - config.set(key.split("."), value) - if "range" in result: - config.set(("image-range",), result["range"]) - config.set(("chapter-range",), result["range"]) - - # write test data - try: - with open(os.path.join(path, name), "w") as outfile: - job.DataJob(url, file=outfile, ensure_ascii=False).run() - except KeyboardInterrupt: - sys.exit() diff --git a/scripts/create_test_data.py b/scripts/create_test_data.py deleted file mode 100755 index 14ab0c0..0000000 --- a/scripts/create_test_data.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2015-2019 Mike Fährmann -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Create testdata for extractor tests""" - -import argparse - -import util # noqa -from gallery_dl import extractor -from test.test_results import ResultJob, setup_test_config - - -TESTDATA_FMT = """ - test = ("{}", {{ - "url": "{}", - "keyword": "{}", - "content": "{}", - }}) -""" - -TESTDATA_EXCEPTION_FMT = """ - test = ("{}", {{ - "exception": exception.{}, - }}) -""" - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--content", action="store_true") - parser.add_argument("--recreate", action="store_true") - parser.add_argument("urls", nargs="*") - args = parser.parse_args() - - if args.recreate: - urls = [ - test[0] - for extr in extractor.extractors() if extr.category in args.urls - for test in extr.test - ] - else: - urls = args.urls - - setup_test_config() - - for url in urls: - tjob = ResultJob(url, content=args.content) - try: - tjob.run() - except Exception as exc: - fmt = TESTDATA_EXCEPTION_FMT - data = (exc.__class__.__name__,) - else: - fmt = TESTDATA_FMT - data = (tjob.hash_url.hexdigest(), - tjob.hash_keyword.hexdigest(), - tjob.hash_content.hexdigest()) - print(tjob.extractor.__class__.__name__) - print(fmt.format(url, *data)) - - -if __name__ == '__main__': - main() diff --git a/scripts/hook-gallery_dl.py b/scripts/hook-gallery_dl.py deleted file mode 100644 index d549019..0000000 --- a/scripts/hook-gallery_dl.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- - -from gallery_dl import extractor, downloader, postprocessor - -hiddenimports = [ - package.__name__ + "." + module - for package in (extractor, downloader, postprocessor) - for module in package.modules -] diff --git a/scripts/man.py b/scripts/man.py deleted file mode 100755 index 91608a3..0000000 --- a/scripts/man.py +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright 2019 Mike Fährmann -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Generate man pages""" - -import re -import datetime - -import util -import gallery_dl.option -import gallery_dl.version - - -def build_gallery_dl_1(path=None): - - OPTS_FMT = """.TP\n.B "{}" {}\n{}""" - - TEMPLATE = r""" -.TH "GALLERY-DL" "1" "%(date)s" "%(version)s" "gallery-dl Manual" -.\" disable hyphenation -.nh - -.SH NAME -gallery-dl \- download image-galleries and -collections - -.SH SYNOPSIS -.B gallery-dl -[OPTION]... URL... - -.SH DESCRIPTION -.B gallery-dl -is a command-line program to download image-galleries and -collections -from several image hosting sites. It is a cross-platform tool -with many configuration options and powerful filenaming capabilities. - -.SH OPTIONS -%(options)s - -.SH EXAMPLES -.TP -gallery-dl \f[I]URL\f[] -Download images from \f[I]URL\f[]. -.TP -gallery-dl -g -u <username> -p <password> \f[I]URL\f[] -Print direct URLs from a site that requires authentication. -.TP -gallery-dl --filter 'type == "ugoira"' --range '2-4' \f[I]URL\f[] -Apply filter and range expressions. This will only download -the second, third, and fourth file where its type value is equal to "ugoira". -.TP -gallery-dl r:\f[I]URL\f[] -Scan \f[I]URL\f[] for other URLs and invoke \f[B]gallery-dl\f[] on them. -.TP -gallery-dl oauth:\f[I]SITE\-NAME\f[] -Gain OAuth authentication tokens for -.IR deviantart , -.IR flickr , -.IR reddit , -.IR smugmug ", and" -.IR tumblr . - -.SH FILES -.TP -.I /etc/gallery-dl.conf -The system wide configuration file. -.TP -.I ~/.config/gallery-dl/config.json -Per user configuration file. -.TP -.I ~/.gallery-dl.conf -Alternate per user configuration file. - -.SH BUGS -https://github.com/mikf/gallery-dl/issues - -.SH AUTHORS -Mike Fährmann <mike_faehrmann@web.de> -.br -and https://github.com/mikf/gallery-dl/graphs/contributors - -.SH "SEE ALSO" -.BR gallery-dl.conf (5) -""" - - options = [] - for action in gallery_dl.option.build_parser()._actions: - if action.help.startswith("=="): - continue - options.append(OPTS_FMT.format( - ", ".join(action.option_strings).replace("-", r"\-"), - r"\f[I]{}\f[]".format(action.metavar) if action.metavar else "", - action.help, - )) - - if not path: - path = util.path("gallery-dl.1") - with open(path, "w", encoding="utf-8") as file: - file.write(TEMPLATE.lstrip() % { - "options": "\n".join(options), - "version": gallery_dl.version.__version__, - "date" : datetime.datetime.now().strftime("%Y-%m-%d"), - }) - - -def build_gallery_dl_conf_5(path=None): - - TEMPLATE = r""" -.TH "GALLERY-DL.CONF" "5" "%(date)s" "%(version)s" "gallery-dl Manual" -.\" disable hyphenation -.nh -.\" disable justification (adjust text to left margin only) -.ad l - -.SH NAME -gallery-dl.conf \- gallery-dl configuration file - -.SH DESCRIPTION -gallery-dl will search for configuration files in the following places -every time it is started, unless -.B --ignore-config -is specified: -.PP -.RS 4 -.nf -.I /etc/gallery-dl.conf -.I $HOME/.config/gallery-dl/config.json -.I $HOME/.gallery-dl.conf -.fi -.RE -.PP -It is also possible to specify additional configuration files with the -.B -c/--config -command-line option or to add further option values with -.B -o/--option -as <key>=<value> pairs, - -Configuration files are JSON-based and therefore don't allow any ordinary -comments, but, since unused keys are simply ignored, it is possible to utilize -those as makeshift comments by settings their values to arbitrary strings. - -.SH EXAMPLE -{ -.RS 4 -"base-directory": "/tmp/", -.br -"extractor": { -.RS 4 -"pixiv": { -.RS 4 -"directory": ["Pixiv", "Works", "{user[id]}"], -.br -"filename": "{id}{num}.{extension}", -.br -"username": "foo", -.br -"password": "bar" -.RE -}, -.br -"flickr": { -.RS 4 -"_comment": "OAuth keys for account 'foobar'", -.br -"access-token": "0123456789-0123456789abcdef", -.br -"access-token-secret": "fedcba9876543210" -.RE -} -.RE -}, -.br -"downloader": { -.RS 4 -"retries": 3, -.br -"timeout": 2.5 -.RE -} -.RE -} - -%(options)s - -.SH BUGS -https://github.com/mikf/gallery-dl/issues - -.SH AUTHORS -Mike Fährmann <mike_faehrmann@web.de> -.br -and https://github.com/mikf/gallery-dl/graphs/contributors - -.SH "SEE ALSO" -.BR gallery-dl (1) -""" - - sections = parse_docs_configuration() - content = [] - - for sec_name, section in sections.items(): - content.append(".SH " + sec_name.upper()) - - for opt_name, option in section.items(): - content.append(".SS " + opt_name) - - for field, text in option.items(): - if field in ("Type", "Default"): - content.append('.IP "{}:" {}'.format(field, len(field)+2)) - content.append(strip_rst(text)) - else: - content.append('.IP "{}:" 4'.format(field)) - content.append(strip_rst(text, field != "Example")) - - if not path: - path = util.path("gallery-dl.conf.5") - with open(path, "w", encoding="utf-8") as file: - file.write(TEMPLATE.lstrip() % { - "options": "\n".join(content), - "version": gallery_dl.version.__version__, - "date" : datetime.datetime.now().strftime("%Y-%m-%d"), - }) - - -def parse_docs_configuration(): - - doc_path = util.path("docs", "configuration.rst") - with open(doc_path, encoding="utf-8") as file: - doc_lines = file.readlines() - - sections = {} - sec_name = None - options = None - opt_name = None - opt_desc = None - name = None - last = last2 = None - for line in doc_lines: - - # start of new section - if re.match(r"^=+$", line): - if sec_name and options: - sections[sec_name] = options - sec_name = last.strip() - options = {} - - elif re.match(r"^=+ =+$", line): - # start of option table - if re.match(r"^-+$", last): - opt_name = last2.strip() - opt_desc = {} - # end of option table - elif opt_desc: - options[opt_name] = opt_desc - opt_name = None - name = None - - # inside option table - elif opt_name: - if line[0].isalpha(): - name, _, line = line.partition(" ") - opt_desc[name] = "" - line = line.strip() - if line.startswith(("* ", "- ")): - line = "\n" + line - elif line.startswith("| "): - line = line[2:] + "\n.br" - opt_desc[name] += line + "\n" - - last2 = last - last = line - sections[sec_name] = options - - return sections - - -def strip_rst(text, extended=True, *, ITALIC=r"\\f[I]\1\\f[]", REGULAR=r"\1"): - - text = text.replace("\\", "\\\\") - - # ``foo`` - repl = ITALIC if extended else REGULAR - text = re.sub(r"``([^`]+)``", repl, text) - # |foo|_ - text = re.sub(r"\|([^|]+)\|_*", ITALIC, text) - # `foo`_ - text = re.sub(r"`([^`]+)`_+", ITALIC, text) - # `foo` - text = re.sub(r"`([^`]+)`", REGULAR, text) - # foo_ - text = re.sub(r"([A-Za-z0-9-]+)_+(?=\s)", ITALIC, text) - # ------- - text = re.sub(r"---+", "", text) - - return text - - -if __name__ == "__main__": - build_gallery_dl_1() - build_gallery_dl_conf_5() diff --git a/scripts/pyinstaller.py b/scripts/pyinstaller.py deleted file mode 100755 index 879ae50..0000000 --- a/scripts/pyinstaller.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -"""Build a standalone executable using PyInstaller""" - -import PyInstaller.__main__ -import util - -PyInstaller.__main__.run([ - "--onefile", - "--console", - "--name", "gallery-dl." + ("exe" if PyInstaller.is_win else "bin"), - "--additional-hooks-dir", util.path("scripts"), - "--distpath", util.path("dist"), - "--workpath", util.path("build"), - "--specpath", util.path("build"), - util.path("gallery_dl", "__main__.py"), -]) diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index ef444e0..0000000 --- a/scripts/release.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash -set -e - -prompt() { - echo "root: ${ROOTDIR} old: ${OLDVERSION} - new: ${NEWVERSION}" - read -n 1 -r -p "Proceed? [Y/n] " P - echo - if [ "$P" == y -o "$P" == Y -o -z "$P" ]; then - return 0 - else - exit 1 - fi -} - -cleanup() { - cd "${ROOTDIR}" - echo Removing old build directory - - if [ -d ./build ]; then - rm -rf ./build - fi -} - -update() { - cd "${ROOTDIR}" - echo Updating version to ${NEWVERSION} - - sed -i "s#\"${PYVERSION}\"#\"${NEWVERSION}\"#" "gallery_dl/version.py" - sed -i "s#v${OLDVERSION}#v${NEWVERSION}#" "${README}" -} - -update-dev() { - cd "${ROOTDIR}" - - IFS="." read MAJOR MINOR BUILD <<< "${NEWVERSION}" - BUILD=$((BUILD+1)) - # update version to -dev - sed -i "s#\"${NEWVERSION}\"#\"${MAJOR}.${MINOR}.${BUILD}-dev\"#" "gallery_dl/version.py" - # add 'unreleased' line to changelog - sed -i "2i\\\n## Unreleased" "${CHANGELOG}" - - git add "gallery_dl/version.py" "${CHANGELOG}" -} - -build-python() { - cd "${ROOTDIR}" - echo Building bdist_wheel and sdist - - python setup.py bdist_wheel sdist -} - -build-linux() { - cd "${ROOTDIR}" - echo Building Linux executable - - make executable -} - -build-windows() { - cd "${ROOTDIR}/dist" - echo Building Windows executable - - # remove old executable - rm -f "gallery-dl.exe" - - # build windows exe in vm - ln -fs "${ROOTDIR}" /tmp/ - vmstart "Windows 7" & - disown - while [ ! -e "gallery-dl.exe" ] ; do - sleep 5 - done - sleep 2 - - # check exe version - OUTPUT="$(wine gallery-dl.exe --version)" - if [[ ! "${OUTPUT%?}" == "${NEWVERSION}" ]]; then - echo "exe version mismatch: ${OUTPUT} != ${NEWVERSION}" - exit 3 - fi -} - -sign() { - cd "${ROOTDIR}/dist" - echo Signing files - - gpg --detach-sign --armor gallery_dl-${NEWVERSION}-py3-none-any.whl - gpg --detach-sign --armor gallery_dl-${NEWVERSION}.tar.gz - gpg --detach-sign --yes gallery-dl.exe - gpg --detach-sign --yes gallery-dl.bin -} - -changelog() { - cd "${ROOTDIR}" - echo Updating "${CHANGELOG}" - - # - replace "#NN" with link to actual issue - # - insert new version and date - sed -i \ - -e "s*\([( ]\)#\([0-9]\+\)*\1[#\2](https://github.com/mikf/gallery-dl/issues/\2)*g" \ - -e "s*^## [Uu]nreleased*## ${NEWVERSION} - $(date +%Y-%m-%d)*" \ - "${CHANGELOG}" -} - -supportedsites() { - cd "${ROOTDIR}" - echo Checking if "${SUPPORTEDSITES}" is up to date - - ./scripts/supportedsites.py - if ! git diff --quiet "${SUPPORTEDSITES}"; then - echo "updated ${SUPPORTEDSITES} contains changes" - exit 4 - fi -} - -git-upload() { - cd "${ROOTDIR}" - echo Pushing changes to github - - git add "gallery_dl/version.py" "${README}" "${CHANGELOG}" - git commit -S -m "release version ${NEWVERSION}" - git tag -s -m "version ${NEWVERSION}" "v${NEWVERSION}" - git push - git push origin "v${NEWVERSION}" -} - -pypi-upload() { - cd "${ROOTDIR}/dist" - echo Uploading to PyPI - - twine upload gallery_dl-${NEWVERSION}* -} - - -ROOTDIR="$(realpath "$(dirname "$0")/..")/" -README="README.rst" -CHANGELOG="CHANGELOG.md" -SUPPORTEDSITES="./docs/supportedsites.rst" - -LASTTAG="$(git describe --abbrev=0 --tags)" -OLDVERSION="${LASTTAG#v}" -PYVERSION="$(python -c "import gallery_dl as g; print(g.__version__)")" - -if [[ "$1" ]]; then - NEWVERSION="$1" -else - NEWVERSION="${PYVERSION%-dev}" -fi - -if [[ ! $NEWVERSION =~ [0-9]+\.[0-9]+\.[0-9]+(-[a-z]+(\.[0-9]+)?)?$ ]]; then - echo "invalid version: $NEWVERSION" - exit 2 -fi - - -prompt -supportedsites -cleanup -update -build-python -build-linux -build-windows -sign -changelog -git-upload -pypi-upload -update-dev diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh deleted file mode 100755 index d8c8a03..0000000 --- a/scripts/run_tests.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -TESTS_CORE=(config cookies downloader extractor oauth postprocessor text util) -TESTS_RESULTS=(results) - - -# select tests -case "${1:-${GALLERYDL_TESTS:-core}}" in - core) TESTS=( ${TESTS_CORE[@]} );; - results) TESTS=( ${TESTS_RESULTS[@]} );; - *) TESTS=( );; -esac - - -# transform each array element to test_###.py -TESTS=( ${TESTS[@]/#/test_} ) -TESTS=( ${TESTS[@]/%/.py} ) - - -# run 'nosetests' with selected tests -# (or all tests if ${TESTS} is empty) -nosetests --verbose -w "${DIR}/../test" ${TESTS[@]} diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py deleted file mode 100755 index 96c11d6..0000000 --- a/scripts/supportedsites.py +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -"""Generate a reStructuredText document with all supported sites""" - -import sys -import collections - -import util -from gallery_dl import extractor - - -CATEGORY_MAP = { - "2chan" : "Futaba Channel", - "35photo" : "35PHOTO", - "adultempire" : "Adult Empire", - "archivedmoe" : "Archived.Moe", - "archiveofsins" : "Archive of Sins", - "artstation" : "ArtStation", - "b4k" : "arch.b4k.co", - "bobx" : "BobX", - "deviantart" : "DeviantArt", - "dokireader" : "Doki Reader", - "dynastyscans" : "Dynasty Reader", - "e621" : "e621", - "erolord" : "EroLord.com", - "e-hentai" : "E-Hentai", - "exhentai" : "ExHentai", - "fallenangels" : "Fallen Angels Scans", - "fashionnova" : "Fashion Nova", - "hbrowse" : "HBrowse", - "hentai2read" : "Hentai2Read", - "hentaicafe" : "Hentai Cafe", - "hentaifoundry" : "Hentai Foundry", - "hentaifox" : "HentaiFox", - "hentaihere" : "HentaiHere", - "hitomi" : "Hitomi.la", - "idolcomplex" : "Idol Complex", - "imagebam" : "ImageBam", - "imagefap" : "ImageFap", - "imgbb" : "ImgBB", - "imgbox" : "imgbox", - "imgth" : "imgth", - "imgur" : "imgur", - "jaiminisbox" : "Jaimini's Box", - "kireicake" : "Kirei Cake", - "kissmanga" : "KissManga", - "lineblog" : "LINE BLOG", - "livedoor" : "livedoor Blog", - "mangadex" : "MangaDex", - "mangafox" : "Manga Fox", - "mangahere" : "Manga Here", - "mangapark" : "MangaPark", - "mangastream" : "Manga Stream", - "myportfolio" : "Adobe Portfolio", - "nhentai" : "nhentai", - "nijie" : "nijie", - "nsfwalbum" : "NSFWalbum.com", - "nyafuu" : "Nyafuu Archive", - "paheal" : "rule #34", - "powermanga" : "PowerManga", - "readcomiconline": "Read Comic Online", - "rbt" : "RebeccaBlackTech", - "rule34" : "Rule 34", - "sankaku" : "Sankaku Channel", - "sankakucomplex" : "Sankaku Complex", - "seaotterscans" : "Sea Otter Scans", - "seiga" : "Niconico Seiga", - "senmanga" : "Sen Manga", - "sensescans" : "Sense-Scans", - "sexcom" : "Sex.com", - "simplyhentai" : "Simply Hentai", - "slickpic" : "SlickPic", - "slideshare" : "SlideShare", - "smugmug" : "SmugMug", - "thebarchive" : "The /b/ Archive", - "vanillarock" : "もえぴりあ", - "vsco" : "VSCO", - "wikiart" : "WikiArt.org", - "worldthree" : "World Three", - "xhamster" : "xHamster", - "xvideos" : "XVideos", - "yaplog" : "yaplog!", - "yuki" : "yuki.la 4chan archive", -} - -SUBCATEGORY_MAP = { - "artwork": "Artwork Listings", - "artists": "", - "doujin" : "Doujin", - "gallery": "Galleries", - "image" : "individual Images", - "issue" : "Comic-Issues", - "manga" : "Manga", - "me" : "pixiv.me Links", - "media" : "Media Timelines", - "path" : "Images from Users and Folders", - "pinit" : "pin.it Links", - "popular": "Popular Images", - "recent" : "Recent Images", - "search" : "Search Results", - "stash" : "Sta.sh", - "status" : "Images from Statuses", - "tag" : "Tag-Searches", - "user" : "User Profiles", - "work" : "Individual Images", - "related-pin" : "related Pins", - "related-board": "", -} - -AUTH_MAP = { - "danbooru" : "Optional", - "deviantart" : "Optional (OAuth)", - "e-hentai" : "Optional", - "exhentai" : "Optional", - "flickr" : "Optional (OAuth)", - "idolcomplex": "Optional", - "imgbb" : "Optional", - "instagram" : "Optional", - "luscious" : "Optional", - "mangoxo" : "Optional", - "nijie" : "Required", - "pixiv" : "Required", - "reddit" : "Optional (OAuth)", - "sankaku" : "Optional", - "seiga" : "Required", - "smugmug" : "Optional (OAuth)", - "tsumino" : "Optional", - "tumblr" : "Optional (OAuth)", - "twitter" : "Optional", - "wallhaven" : ("Optional (`API Key " - "<configuration.rst#extractorwallhavenapi-key>`__)"), -} - -IGNORE_LIST = ( - "directlink", - "oauth", - "recursive", - "test", -) - - -def domain(cls): - """Return the web-domain related to an extractor class""" - url = sys.modules[cls.__module__].__doc__.split()[-1] - if url.startswith("http"): - return url - - if hasattr(cls, "root") and cls.root: - return cls.root + "/" - - if hasattr(cls, "https"): - scheme = "https" if cls.https else "http" - netloc = cls.__doc__.split()[-1] - return "{}://{}/".format(scheme, netloc) - - test = next(cls._get_tests(), None) - if test: - url = test[0] - return url[:url.find("/", 8)+1] - - return "" - - -def category_text(cls): - """Return a human-readable representation of a category""" - c = cls.category - return CATEGORY_MAP.get(c) or c.capitalize() - - -def subcategory_text(cls): - """Return a human-readable representation of a subcategory""" - sc = cls.subcategory - if sc in SUBCATEGORY_MAP: - return SUBCATEGORY_MAP[sc] - sc = sc.capitalize() - return sc if sc.endswith("s") else sc + "s" - - -def category_key(cls): - """Generate sorting keys by category""" - key = category_text(cls).lower() - if cls.__module__.endswith(".imagehosts"): - key = "zz" + key - return key - - -def subcategory_key(cls): - """Generate sorting keys by subcategory""" - if cls.subcategory == "issue": - return "A" - return cls.subcategory - - -def build_extractor_list(): - """Generate a sorted list of lists of extractor classes""" - extractors = collections.defaultdict(list) - - # get lists of extractor classes grouped by category - for extr in extractor.extractors(): - if not extr.category or extr.category in IGNORE_LIST: - continue - extractors[extr.category].append(extr) - - # sort extractor lists with the same category - for extrlist in extractors.values(): - extrlist.sort(key=subcategory_key) - - # ugly hack to add e-hentai.org - eh = [] - for extr in extractors["exhentai"]: - class eh_extr(extr): - category = "e-hentai" - root = "https://e-hentai.org" - eh.append(eh_extr) - extractors["e-hentai"] = eh - - # sort lists by category - return sorted( - extractors.values(), - key=lambda lst: category_key(lst[0]), - ) - - -# define table columns -COLUMNS = ( - ("Site", 20, - lambda x: category_text(x[0])), - ("URL" , 35, - lambda x: domain(x[0])), - ("Capabilities", 50, - lambda x: ", ".join(subcategory_text(extr) for extr in x - if subcategory_text(extr))), - ("Authentication", 16, - lambda x: AUTH_MAP.get(x[0].category, "")), -) - - -def write_output(fobj, columns, extractors): - - def pad(output, col, category=None): - size = col[1] - output = output if isinstance(output, str) else col[2](output) - - if len(output) > size: - sub = "|{}-{}|".format(category, col[0][0]) - subs.append((sub, output)) - output = sub - - return output + " " * (size - len(output)) - - w = fobj.write - subs = [] - - # caption - w("Supported Sites\n") - w("===============\n") - w("Unless otherwise known, assume all sites to be NSFW\n\n") - - # table head - sep = " ".join("=" * c[1] for c in columns) + "\n" - w(sep) - w(" ".join(pad(c[0], c) for c in columns).strip() + "\n") - w(sep) - - # table body - for lst in extractors: - w(" ".join( - pad(col[2](lst), col, lst[0].category) - for col in columns - ).strip()) - w("\n") - - # table bottom - w(sep) - w("\n") - - # substitutions - for sub, value in subs: - w(".. {} replace:: {}\n".format(sub, value)) - - -outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst" -with open(util.path("docs", outfile), "w") as file: - write_output(file, COLUMNS, build_extractor_list()) diff --git a/scripts/util.py b/scripts/util.py deleted file mode 100644 index bfbd6cb..0000000 --- a/scripts/util.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -import os.path - -ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.realpath(ROOTDIR)) - - -def path(*segments, join=os.path.join): - return join(ROOTDIR, *segments) |