diff options
Diffstat (limited to 'gallery_dl/extractor/common.py')
| -rw-r--r-- | gallery_dl/extractor/common.py | 31 |
1 files changed, 20 insertions, 11 deletions
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 2146fa6..f364124 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -11,7 +11,6 @@ import os import re import ssl -import sys import time import netrc import queue @@ -23,7 +22,7 @@ import requests import threading from requests.adapters import HTTPAdapter from .message import Message -from .. import config, text, util, cache, exception +from .. import config, output, text, util, cache, exception urllib3 = requests.packages.urllib3 @@ -43,6 +42,8 @@ class Extractor(): ciphers = None tls12 = True browser = None + useragent = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; " + "rv:128.0) Gecko/20100101 Firefox/128.0") request_interval = 0.0 request_interval_min = 0.0 request_interval_429 = 60.0 @@ -289,13 +290,8 @@ class Extractor(): def _check_input_allowed(self, prompt=""): input = self.config("input") - if input is None: - try: - input = sys.stdin.isatty() - except Exception: - input = False - + input = output.TTY_STDIN if not input: raise exception.StopExtraction( "User input required (%s)", prompt.strip(" :")) @@ -351,6 +347,9 @@ class Extractor(): headers.clear() ssl_options = ssl_ciphers = 0 + # .netrc Authorization headers are alwsays disabled + session.trust_env = True if self.config("proxy-env", False) else False + browser = self.config("browser") if browser is None: browser = self.browser @@ -384,11 +383,13 @@ class Extractor(): ssl_ciphers = SSL_CIPHERS[browser] else: useragent = self.config("user-agent") - if useragent is None: - useragent = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; " - "rv:128.0) Gecko/20100101 Firefox/128.0") + if useragent is None or useragent == "auto": + useragent = self.useragent elif useragent == "browser": useragent = _browser_useragent() + elif useragent is config.get(("extractor",), "user-agent") and \ + useragent == Extractor.useragent: + useragent = self.useragent headers["User-Agent"] = useragent headers["Accept"] = "*/*" headers["Accept-Language"] = "en-US,en;q=0.5" @@ -660,6 +661,8 @@ class Extractor(): headers=(self._write_pages in ("all", "ALL")), hide_auth=(self._write_pages != "ALL") ) + self.log.info("Writing '%s' response to '%s'", + response.url, path + ".txt") except Exception as e: self.log.warning("Failed to dump HTTP request (%s: %s)", e.__class__.__name__, e) @@ -1008,6 +1011,12 @@ SSL_CIPHERS = { } +# disable Basic Authorization header injection from .netrc data +try: + requests.sessions.get_netrc_auth = lambda _: None +except Exception: + pass + # detect brotli support try: BROTLI = urllib3.response.brotli is not None |