From 90e50db2e3c38f523bb5195d295290b06e5cedb0 Mon Sep 17 00:00:00 2001 From: Unit 193 Date: Sun, 3 May 2020 00:06:40 -0400 Subject: New upstream version 1.13.6 --- PKG-INFO | 8 ++--- README.rst | 6 ++-- data/man/gallery-dl.1 | 2 +- data/man/gallery-dl.conf.5 | 34 ++++++++++++++++-- gallery_dl.egg-info/PKG-INFO | 8 ++--- gallery_dl.egg-info/SOURCES.txt | 1 + gallery_dl/cloudflare.py | 24 ++++++++----- gallery_dl/downloader/http.py | 4 +++ gallery_dl/extractor/__init__.py | 1 + gallery_dl/extractor/artstation.py | 5 +-- gallery_dl/extractor/blogger.py | 8 ++--- gallery_dl/extractor/deviantart.py | 3 +- gallery_dl/extractor/newgrounds.py | 10 +++--- gallery_dl/extractor/patreon.py | 46 +++++++++++++++++------- gallery_dl/extractor/realbooru.py | 2 +- gallery_dl/extractor/speakerdeck.py | 70 +++++++++++++++++++++++++++++++++++++ gallery_dl/extractor/twitter.py | 10 +++++- gallery_dl/extractor/vsco.py | 15 +++----- gallery_dl/extractor/weibo.py | 4 ++- gallery_dl/version.py | 2 +- test/test_cache.py | 11 ++++-- test/test_config.py | 31 ++++++++++++++-- test/test_cookies.py | 6 ++-- test/test_downloader.py | 23 ++++++------ test/test_extractor.py | 25 +++++++------ test/test_oauth.py | 7 ++-- test/test_postprocessor.py | 15 ++++---- test/test_results.py | 7 ++-- test/test_text.py | 6 +++- test/test_util.py | 7 ++-- 30 files changed, 294 insertions(+), 107 deletions(-) create mode 100644 gallery_dl/extractor/speakerdeck.py diff --git a/PKG-INFO b/PKG-INFO index b9a9444..085159b 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery_dl -Version: 1.13.5 +Version: 1.13.6 Summary: Command-line program to download image-galleries and -collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Author: Mike Fährmann @@ -94,8 +94,8 @@ Description: ========== put it into your `PATH `__, and run it inside a command prompt (like ``cmd.exe``). - - `Windows `__ - - `Linux `__ + - `Windows `__ + - `Linux `__ These executables include a Python 3.8 interpreter and all required Python packages. @@ -247,7 +247,7 @@ Description: ========== .. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf .. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst .. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst - .. _stable: https://github.com/mikf/gallery-dl/archive/v1.13.5.tar.gz + .. _stable: https://github.com/mikf/gallery-dl/archive/v1.13.6.tar.gz .. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz .. _Python: https://www.python.org/downloads/ diff --git a/README.rst b/README.rst index 50a6fae..0fd597a 100644 --- a/README.rst +++ b/README.rst @@ -83,8 +83,8 @@ Download a standalone executable file, put it into your `PATH `__, and run it inside a command prompt (like ``cmd.exe``). -- `Windows `__ -- `Linux `__ +- `Windows `__ +- `Linux `__ These executables include a Python 3.8 interpreter and all required Python packages. @@ -236,7 +236,7 @@ access to *gallery-dl*. Authorize it and you will be shown one or more .. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf .. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst .. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst -.. _stable: https://github.com/mikf/gallery-dl/archive/v1.13.5.tar.gz +.. _stable: https://github.com/mikf/gallery-dl/archive/v1.13.6.tar.gz .. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz .. _Python: https://www.python.org/downloads/ diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1 index 9ef6a93..c115752 100644 --- a/data/man/gallery-dl.1 +++ b/data/man/gallery-dl.1 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL" "1" "2020-04-27" "1.13.5" "gallery-dl Manual" +.TH "GALLERY-DL" "1" "2020-05-02" "1.13.6" "gallery-dl Manual" .\" disable hyphenation .nh diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5 index 6c48a70..a944167 100644 --- a/data/man/gallery-dl.conf.5 +++ b/data/man/gallery-dl.conf.5 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL.CONF" "5" "2020-04-27" "1.13.5" "gallery-dl Manual" +.TH "GALLERY-DL.CONF" "5" "2020-05-02" "1.13.6" "gallery-dl Manual" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -1338,6 +1338,16 @@ You can use \f[I]"all"\f[] instead of listing all types separately. .IP "Description:" 4 Extract tweet text as \f[I]content\f[] metadata. +.SS extractor.twitter.replies +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Extract media from replies to other Tweets. + .SS extractor.twitter.retweets .IP "Type:" 6 \f[I]bool\f[] @@ -1346,7 +1356,7 @@ Extract tweet text as \f[I]content\f[] metadata. \f[I]true\f[] .IP "Description:" 4 -Extract images from retweets. +Extract media from Retweets. .SS extractor.twitter.twitpic .IP "Type:" 6 @@ -1400,6 +1410,26 @@ your account's browsing settings and default filters when searching. See https://wallhaven.cc/help/api for more information. +.SS extractor.weibo.retweets +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Extract media from retweeted posts. + +.SS extractor.weibo.videos +.IP "Type:" 6 +\f[I]bool\f[] + +.IP "Default:" 9 +\f[I]true\f[] + +.IP "Description:" 4 +Download video files. + .SS extractor.[booru].tags .IP "Type:" 6 \f[I]bool\f[] diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO index a0c6286..a79e69f 100644 --- a/gallery_dl.egg-info/PKG-INFO +++ b/gallery_dl.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery-dl -Version: 1.13.5 +Version: 1.13.6 Summary: Command-line program to download image-galleries and -collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Author: Mike Fährmann @@ -94,8 +94,8 @@ Description: ========== put it into your `PATH `__, and run it inside a command prompt (like ``cmd.exe``). - - `Windows `__ - - `Linux `__ + - `Windows `__ + - `Linux `__ These executables include a Python 3.8 interpreter and all required Python packages. @@ -247,7 +247,7 @@ Description: ========== .. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf .. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst .. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst - .. _stable: https://github.com/mikf/gallery-dl/archive/v1.13.5.tar.gz + .. _stable: https://github.com/mikf/gallery-dl/archive/v1.13.6.tar.gz .. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz .. _Python: https://www.python.org/downloads/ diff --git a/gallery_dl.egg-info/SOURCES.txt b/gallery_dl.egg-info/SOURCES.txt index 1df3675..5d3f07b 100644 --- a/gallery_dl.egg-info/SOURCES.txt +++ b/gallery_dl.egg-info/SOURCES.txt @@ -136,6 +136,7 @@ gallery_dl/extractor/simplyhentai.py gallery_dl/extractor/slickpic.py gallery_dl/extractor/slideshare.py gallery_dl/extractor/smugmug.py +gallery_dl/extractor/speakerdeck.py gallery_dl/extractor/test.py gallery_dl/extractor/tsumino.py gallery_dl/extractor/tumblr.py diff --git a/gallery_dl/cloudflare.py b/gallery_dl/cloudflare.py index e3ebd1a..43ccdeb 100644 --- a/gallery_dl/cloudflare.py +++ b/gallery_dl/cloudflare.py @@ -144,11 +144,15 @@ def evaluate_expression(expr, page, netloc, *, # evaluate them, # and accumulate their values in 'result' result = "" - for subexpr in split_re.findall(expr) or (expr,): - result += str(sum( - VALUES[part] - for part in subexpr.split("[]") - )) + for subexpr in expr.strip("+()").split(")+("): + value = 0 + for part in subexpr.split("+"): + if "-" in part: + p1, _, p2 = part.partition("-") + value += VALUES[p1] - VALUES[p2] + else: + value += VALUES[part] + result += str(value) return int(result) @@ -158,12 +162,14 @@ OPERATORS = { "*": operator.mul, } + VALUES = { "": 0, - "+": 0, - "!+": 1, - "!!": 1, - "+!!": 1, + "!": 1, + "[]": 0, + "!![]": 1, + "(!![]": 1, + "(!![])": 1, } diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 64a2978..021dc16 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -243,6 +243,10 @@ MIMETYPE_MAP = { "image/webp": "webp", "image/svg+xml": "svg", + "image/vnd.adobe.photoshop": "psd", + "image/x-photoshop": "psd", + "application/x-photoshop": "psd", + "video/webm": "webm", "video/ogg": "ogg", "video/mp4": "mp4", diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 2c87eb3..85fbddb 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -105,6 +105,7 @@ modules = [ "slickpic", "slideshare", "smugmug", + "speakerdeck", "tsumino", "tumblr", "twitter", diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py index ceda29c..c504dba 100644 --- a/gallery_dl/extractor/artstation.py +++ b/gallery_dl/extractor/artstation.py @@ -39,8 +39,9 @@ class ArtstationExtractor(Extractor): if adict["has_embedded_player"] and self.external: player = adict["player_embedded"] - url = text.extract(player, 'src="', '"')[0] - if not url.startswith(self.root): + url = text.extract(player, 'src="', '"')[0] or \ + text.extract(player, "src='", "'")[0] + if url and not url.startswith(self.root): asset["extension"] = None yield Message.Url, "ytdl:" + url, asset continue diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index 2657b5d..331cfc2 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -109,7 +109,7 @@ class BloggerPostExtractor(BloggerExtractor): "posts" : int, "published" : "2010-11-21T10:19:42-08:00", "updated" : str, - "url" : "http://www.julianbunker.com/", + "url" : "http://julianbphotography.blogspot.com/", }, "post": { "author" : "Julian Bunker", @@ -128,9 +128,7 @@ class BloggerPostExtractor(BloggerExtractor): "url": str, }, }), - ("blogger:http://www.julianbunker.com/2010/12/moon-rise.html", { - "url": "9928429fb62f712eb4de80f53625eccecc614aae", - }), + ("blogger:http://www.julianbunker.com/2010/12/moon-rise.html"), # video (#587) (("http://cfnmscenesinmovies.blogspot.com/2011/11/" "cfnm-scene-jenna-fischer-in-office.html"), { @@ -156,7 +154,7 @@ class BloggerBlogExtractor(BloggerExtractor): "count": 25, "pattern": r"https://\d\.bp\.blogspot\.com/.*/s0/[^.]+\.jpg", }), - ("blogger:http://www.julianbunker.com/", { + ("blogger:https://www.kefblog.com.ng/", { "range": "1-25", "count": 25, }), diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index ca722b8..2631052 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -1006,7 +1006,8 @@ class DeviantartOAuthAPI(): msg = "API responded with {} {}".format( status, response.reason) if status == 429: - self.delay += 1 + if self.delay < 9: + self.delay += 1 self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay) else: self.log.error(msg) diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index bb87a69..17fe935 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -288,7 +288,7 @@ class NewgroundsMediaExtractor(NewgroundsExtractor): class NewgroundsArtExtractor(NewgroundsExtractor): """Extractor for all images of a newgrounds user""" subcategory = "art" - pattern = r"(?:https?://)?([^.]+)\.newgrounds\.com/art/?$" + pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/art/?$" test = ("https://tomfulp.newgrounds.com/art", { "pattern": NewgroundsImageExtractor.pattern, "count": ">= 3", @@ -298,7 +298,7 @@ class NewgroundsArtExtractor(NewgroundsExtractor): class NewgroundsAudioExtractor(NewgroundsExtractor): """Extractor for all audio submissions of a newgrounds user""" subcategory = "audio" - pattern = r"(?:https?://)?([^.]+)\.newgrounds\.com/audio/?$" + pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/audio/?$" test = ("https://tomfulp.newgrounds.com/audio", { "pattern": r"https://audio.ngfiles.com/\d+/\d+_.+\.mp3", "count": ">= 4", @@ -308,7 +308,7 @@ class NewgroundsAudioExtractor(NewgroundsExtractor): class NewgroundsMoviesExtractor(NewgroundsExtractor): """Extractor for all movies of a newgrounds user""" subcategory = "movies" - pattern = r"(?:https?://)?([^.]+)\.newgrounds\.com/movies/?$" + pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/movies/?$" test = ("https://tomfulp.newgrounds.com/movies", { "pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", "range": "1-10", @@ -319,7 +319,7 @@ class NewgroundsMoviesExtractor(NewgroundsExtractor): class NewgroundsUserExtractor(NewgroundsExtractor): """Extractor for a newgrounds user profile""" subcategory = "user" - pattern = r"(?:https?://)?([^.]+)\.newgrounds\.com/?$" + pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/?$" test = ( ("https://tomfulp.newgrounds.com", { "pattern": "https://tomfulp.newgrounds.com/art$", @@ -414,6 +414,6 @@ class NewgroundsFollowingExtractor(NewgroundsFavoriteExtractor): @staticmethod def _extract_favorites(page): return [ - "https://" + user.rpartition('"')[2] + "https://" + user.rpartition('"')[2].lstrip("/:") for user in text.extract_iter(page, 'class="item-user', '">