New upstream version 8.3.0.upstream/8.3.0upstream

1 files changed, 32 insertions, 10 deletions

diff --git a/nikola/plugins/command/check.py b/nikola/plugins/command/check.py
index f9b701b..5bcbced 100644
--- a/nikola/plugins/command/check.py
+++ b/nikola/plugins/command/check.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright © 2012-2022 Roberto Alsina and others.
+# Copyright © 2012-2024 Roberto Alsina and others.
 
 # Permission is hereby granted, free of charge, to any
 # person obtaining a copy of this software and associated
@@ -148,6 +148,22 @@ class CommandCheck(Command):
             'default': False,
             'help': 'Check that remote links work.',
         },
+        {
+            'name': 'timeout',
+            'long': 'timeout',
+            'short': 't',
+            'type': int,
+            'default': 30,
+            'help': 'Timeout (in seconds) for HTTP requests in remote checks.',
+        },
+        {
+            'name': 'ignore_query_strings',
+            'long': 'ignore-query-strings',
+            'short': 'q',
+            'type': bool,
+            'default': False,
+            'help': 'Ignore query strings for internal links.',
+        }
     ]
 
     def _execute(self, options, args):
@@ -160,8 +176,9 @@ class CommandCheck(Command):
         else:
             self.logger.level = logging.WARNING
         failure = False
+        self.timeout = options['timeout']
         if options['links']:
-            failure |= self.scan_links(options['find_sources'], options['remote'])
+            failure |= self.scan_links(options['find_sources'], options['remote'], options['ignore_query_strings'])
         if options['files']:
             failure |= self.scan_files()
         if options['clean']:
@@ -171,9 +188,10 @@ class CommandCheck(Command):
 
     existing_targets = set([])
     checked_remote_targets = {}
+    timeout = None
     cache = {}
 
-    def analyze(self, fname, find_sources=False, check_remote=False):
+    def analyze(self, fname, find_sources=False, check_remote=False, ignore_query_strings=False):
         """Analyze links on a page."""
         rv = False
         self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]
@@ -279,19 +297,19 @@ class CommandCheck(Command):
 
                     # Check the remote link works
                     req_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0 (Nikola)'}  # I’m a real boy!
-                    resp = requests.head(target, headers=req_headers, allow_redirects=False)
+                    resp = requests.head(target, headers=req_headers, allow_redirects=False, timeout=self.timeout)
 
                     # Retry client errors (4xx) as GET requests because many servers are broken
                     if resp.status_code >= 400 and resp.status_code <= 499:
                         time.sleep(0.5)
-                        resp = requests.get(target, headers=req_headers, allow_redirects=False)
+                        resp = requests.get(target, headers=req_headers, allow_redirects=False, timeout=self.timeout)
 
                     # Follow redirects and see where they lead, redirects to errors will be reported twice
                     if resp.status_code in [301, 302, 307, 308]:
                         redir_status_code = resp.status_code
                         time.sleep(0.5)
                         # Known redirects are retested using GET because IIS servers otherwise get HEADaches
-                        resp = requests.get(target, headers=req_headers, allow_redirects=True)
+                        resp = requests.get(target, headers=req_headers, allow_redirects=True, timeout=self.timeout)
                         # Permanent redirects should be updated
                         if redir_status_code in [301, 308]:
                             self.logger.warning("Remote link moved PERMANENTLY to \"{0}\" and should be updated in {1}: {2} [HTTP: {3}]".format(resp.url, filename, target, redir_status_code))
@@ -353,6 +371,10 @@ class CommandCheck(Command):
                 else:
                     target_filename_str = target_filename.decode("utf-8", errors="surrogateescape")
 
+                if ignore_query_strings and "?" in target_filename_str:
+                    target_filename, _, _ = target_filename.rpartition("?")
+                    target_filename_str, _, _ = target_filename_str.rpartition("?")
+
                 if any(pattern.search(target_filename_str) for pattern in self.whitelist):
                     continue
 
@@ -371,7 +393,7 @@ class CommandCheck(Command):
             self.logger.error(u"Error with: {0} {1}".format(filename, exc))
         return rv
 
-    def scan_links(self, find_sources=False, check_remote=False):
+    def scan_links(self, find_sources=False, check_remote=False, ignore_query_strings=False):
         """Check links on the site."""
         self.logger.debug("Checking Links:")
         self.logger.debug("===============\n")
@@ -387,13 +409,13 @@ class CommandCheck(Command):
         for fname in _call_nikola_list(self.site, self.cache)[0]:
             if fname.startswith(output_folder):
                 if '.html' == fname[-5:]:
-                    if self.analyze(fname, find_sources, check_remote):
+                    if self.analyze(fname, find_sources, check_remote, ignore_query_strings):
                         failure = True
                 if atom_extension == fname[-len(atom_extension):]:
-                    if self.analyze(fname, find_sources, False):
+                    if self.analyze(fname, find_sources, False, ignore_query_strings):
                         failure = True
                 if fname.endswith('sitemap.xml') or fname.endswith('sitemapindex.xml'):
-                    if self.analyze(fname, find_sources, False):
+                    if self.analyze(fname, find_sources, False, ignore_query_strings):
                         failure = True
         if not failure:
             self.logger.debug("All links checked.")