diff options
Diffstat (limited to 'gallery_dl/extractor/common.py')
| -rw-r--r-- | gallery_dl/extractor/common.py | 31 |
1 files changed, 25 insertions, 6 deletions
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 5c5e29e..6ccae7f 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -8,6 +8,7 @@ """Common classes and constants used by extractor modules.""" +import os import re import ssl import time @@ -224,7 +225,9 @@ class Extractor(): headers.clear() ssl_options = ssl_ciphers = 0 - browser = self.config("browser") or self.browser + browser = self.config("browser") + if browser is None: + browser = self.browser if browser and isinstance(browser, str): browser, _, platform = browser.lower().partition(":") @@ -259,6 +262,10 @@ class Extractor(): "rv:102.0) Gecko/20100101 Firefox/102.0")) headers["Accept"] = "*/*" headers["Accept-Language"] = "en-US,en;q=0.5" + + if BROTLI: + headers["Accept-Encoding"] = "gzip, deflate, br" + else: headers["Accept-Encoding"] = "gzip, deflate" custom_headers = self.config("headers") @@ -473,11 +480,16 @@ class Extractor(): fname = "{:>02}_{}".format( Extractor._dump_index, - Extractor._dump_sanitize('_', response.url) - )[:250] + Extractor._dump_sanitize('_', response.url), + ) + + if util.WINDOWS: + path = os.path.abspath(fname)[:255] + else: + path = fname[:251] try: - with open(fname + ".dump", 'wb') as fp: + with open(path + ".txt", 'wb') as fp: util.dump_response( response, fp, headers=(self._write_pages == "all")) except Exception as e: @@ -718,7 +730,7 @@ HTTP_HEADERS = { ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," "image/avif,image/webp,*/*;q=0.8"), ("Accept-Language", "en-US,en;q=0.5"), - ("Accept-Encoding", "gzip, deflate, br"), + ("Accept-Encoding", None), ("Referer", None), ("DNT", "1"), ("Connection", "keep-alive"), @@ -736,7 +748,7 @@ HTTP_HEADERS = { ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," "image/webp,image/apng,*/*;q=0.8"), ("Referer", None), - ("Accept-Encoding", "gzip, deflate"), + ("Accept-Encoding", None), ("Accept-Language", "en-US,en;q=0.9"), ("Cookie", None), ), @@ -783,6 +795,13 @@ SSL_CIPHERS = { } +# detect brotli support +try: + BROTLI = requests.packages.urllib3.response.brotli is not None +except AttributeError: + BROTLI = False + + # Undo automatic pyOpenSSL injection by requests pyopenssl = config.get((), "pyopenssl", False) if not pyopenssl: |