diff options
Diffstat (limited to 'gallery_dl/extractor/common.py')
| -rw-r--r-- | gallery_dl/extractor/common.py | 183 |
1 files changed, 104 insertions, 79 deletions
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 52e5199..1d81dfc 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2021 Mike Fährmann +# Copyright 2014-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -38,6 +38,7 @@ class Extractor(): request_interval = 0.0 request_interval_min = 0.0 request_timestamp = 0.0 + tls12 = True def __init__(self, match): self.log = logging.getLogger(self.category) @@ -219,14 +220,7 @@ class Extractor(): self.session = session = requests.Session() headers = session.headers headers.clear() - - source_address = self.config("source-address") - if source_address: - if isinstance(source_address, str): - source_address = (source_address, 0) - else: - source_address = (source_address[0], source_address[1]) - session.mount("http://", SourceAdapter(source_address)) + ssl_options = ssl_ciphers = 0 browser = self.config("browser") or self.browser if browser and isinstance(browser, str): @@ -243,12 +237,21 @@ class Extractor(): platform = "Macintosh; Intel Mac OS X 11.5" if browser == "chrome": - _emulate_browser_chrome(session, platform, source_address) + if platform.startswith("Macintosh"): + platform = platform.replace(".", "_") + "_2" else: - _emulate_browser_firefox(session, platform, source_address) + browser = "firefox" + + for key, value in HTTP_HEADERS[browser]: + if value and "{}" in value: + headers[key] = value.format(platform) + else: + headers[key] = value + + ssl_options |= (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | + ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1) + ssl_ciphers = SSL_CIPHERS[browser] else: - if source_address: - session.mount("https://", SourceAdapter(source_address)) headers["User-Agent"] = self.config("user-agent", ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64; " "rv:91.0) Gecko/20100101 Firefox/91.0")) @@ -260,11 +263,31 @@ class Extractor(): if custom_headers: headers.update(custom_headers) - ciphers = self.config("ciphers") - if ciphers: - if isinstance(ciphers, list): - ciphers = ":".join(ciphers) - session.mount("https://", HTTPSAdapter(ciphers)) + custom_ciphers = self.config("ciphers") + if custom_ciphers: + if isinstance(custom_ciphers, list): + ssl_ciphers = ":".join(custom_ciphers) + else: + ssl_ciphers = custom_ciphers + + source_address = self.config("source-address") + if source_address: + if isinstance(source_address, str): + source_address = (source_address, 0) + else: + source_address = (source_address[0], source_address[1]) + + tls12 = self.config("tls12") + if tls12 is None: + tls12 = self.tls12 + if not tls12: + ssl_options |= ssl.OP_NO_TLSv1_2 + self.log.debug("TLS 1.2 disabled.") + + adapter = _build_requests_adapter( + ssl_options, ssl_ciphers, source_address) + session.mount("https://", adapter) + session.mount("http://", adapter) def _init_proxies(self): """Update the session's proxy map""" @@ -615,29 +638,10 @@ class BaseExtractor(Extractor): ) -class SourceAdapter(HTTPAdapter): +class RequestsAdapter(HTTPAdapter): - def __init__(self, source_address): - self.source_address = source_address - HTTPAdapter.__init__(self) - - def init_poolmanager(self, *args, **kwargs): - kwargs["source_address"] = self.source_address - return HTTPAdapter.init_poolmanager(self, *args, **kwargs) - - def proxy_manager_for(self, *args, **kwargs): - kwargs["source_address"] = self.source_address - return HTTPAdapter.proxy_manager_for(self, *args, **kwargs) - - -class HTTPSAdapter(HTTPAdapter): - - def __init__(self, ciphers, source_address=None): - context = self.ssl_context = ssl.create_default_context() - context.options |= (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | - ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1) - context.set_ecdh_curve("prime256v1") - context.set_ciphers(ciphers) + def __init__(self, ssl_context=None, source_address=None): + self.ssl_context = ssl_context self.source_address = source_address HTTPAdapter.__init__(self) @@ -652,19 +656,59 @@ class HTTPSAdapter(HTTPAdapter): return HTTPAdapter.proxy_manager_for(self, *args, **kwargs) -def _emulate_browser_firefox(session, platform, source_address): - headers = session.headers - headers["User-Agent"] = ("Mozilla/5.0 (" + platform + "; rv:91.0) " - "Gecko/20100101 Firefox/91.0") - headers["Accept"] = ("text/html,application/xhtml+xml," - "application/xml;q=0.9,image/webp,*/*;q=0.8") - headers["Accept-Language"] = "en-US,en;q=0.5" - headers["Accept-Encoding"] = "gzip, deflate" - headers["Referer"] = None - headers["Upgrade-Insecure-Requests"] = "1" - headers["Cookie"] = None +def _build_requests_adapter(ssl_options, ssl_ciphers, source_address): + key = (ssl_options, ssl_ciphers, source_address) + try: + return _adapter_cache[key] + except KeyError: + pass - session.mount("https://", HTTPSAdapter( + if ssl_options or ssl_ciphers: + ssl_context = ssl.create_default_context() + if ssl_options: + ssl_context.options |= ssl_options + if ssl_ciphers: + ssl_context.set_ecdh_curve("prime256v1") + ssl_context.set_ciphers(ssl_ciphers) + else: + ssl_context = None + + adapter = _adapter_cache[key] = RequestsAdapter( + ssl_context, source_address) + return adapter + + +_adapter_cache = {} + + +HTTP_HEADERS = { + "firefox": ( + ("User-Agent", "Mozilla/5.0 ({}; rv:91.0) " + "Gecko/20100101 Firefox/91.0"), + ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,*/*;q=0.8"), + ("Accept-Language", "en-US,en;q=0.5"), + ("Accept-Encoding", "gzip, deflate"), + ("Referer", None), + ("Connection", "keep-alive"), + ("Upgrade-Insecure-Requests", "1"), + ("Cookie", None), + ), + "chrome": ( + ("Upgrade-Insecure-Requests", "1"), + ("User-Agent", "Mozilla/5.0 ({}) AppleWebKit/537.36 (KHTML, " + "like Gecko) Chrome/92.0.4515.131 Safari/537.36"), + ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/webp,image/apng,*/*;q=0.8"), + ("Referer", None), + ("Accept-Encoding", "gzip, deflate"), + ("Accept-Language", "en-US,en;q=0.9"), + ("Cookie", None), + ), +} + +SSL_CIPHERS = { + "firefox": ( "TLS_AES_128_GCM_SHA256:" "TLS_CHACHA20_POLY1305_SHA256:" "TLS_AES_256_GCM_SHA384:" @@ -678,32 +722,13 @@ def _emulate_browser_firefox(session, platform, source_address): "ECDHE-ECDSA-AES128-SHA:" "ECDHE-RSA-AES128-SHA:" "ECDHE-RSA-AES256-SHA:" - "DHE-RSA-AES128-SHA:" - "DHE-RSA-AES256-SHA:" + "AES128-GCM-SHA256:" + "AES256-GCM-SHA384:" "AES128-SHA:" "AES256-SHA:" - "DES-CBC3-SHA", - source_address - )) - - -def _emulate_browser_chrome(session, platform, source_address): - if platform.startswith("Macintosh"): - platform = platform.replace(".", "_") + "_2" - - headers = session.headers - headers["Upgrade-Insecure-Requests"] = "1" - headers["User-Agent"] = ( - "Mozilla/5.0 (" + platform + ") AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36") - headers["Accept"] = ("text/html,application/xhtml+xml,application/xml;" - "q=0.9,image/webp,image/apng,*/*;q=0.8") - headers["Referer"] = None - headers["Accept-Encoding"] = "gzip, deflate" - headers["Accept-Language"] = "en-US,en;q=0.9" - headers["Cookie"] = None - - session.mount("https://", HTTPSAdapter( + "DES-CBC3-SHA" + ), + "chrome": ( "TLS_AES_128_GCM_SHA256:" "TLS_AES_256_GCM_SHA384:" "TLS_CHACHA20_POLY1305_SHA256:" @@ -719,9 +744,9 @@ def _emulate_browser_chrome(session, platform, source_address): "AES256-GCM-SHA384:" "AES128-SHA:" "AES256-SHA:" - "DES-CBC3-SHA", - source_address - )) + "DES-CBC3-SHA" + ), +} # Undo automatic pyOpenSSL injection by requests |