Imported Upstream version 7.7.3upstream/7.7.3

1 files changed, 4 insertions, 4 deletions

diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py
index 65254b6..7c7f5df 100644
--- a/nikola/plugins/task/robots.py
+++ b/nikola/plugins/task/robots.py
@@ -39,7 +39,6 @@ from nikola import utils
 
 
 class RobotsFile(LateTask):
-
     """Generate a robots.txt file."""
 
     name = "robots_file"
@@ -64,14 +63,15 @@ class RobotsFile(LateTask):
 
             with io.open(robots_path, 'w+', encoding='utf8') as outf:
                 outf.write("Sitemap: {0}\n\n".format(sitemapindex_url))
+                outf.write("User-Agent: *\n")
                 if kw["robots_exclusions"]:
-                    outf.write("User-Agent: *\n")
                     for loc in kw["robots_exclusions"]:
                         outf.write("Disallow: {0}\n".format(loc))
+                outf.write("Host: {0}\n".format(urlparse(kw["base_url"]).netloc))
 
         yield self.group_task()
 
-        if not utils.get_asset_path("robots.txt", [], files_folders=kw["files_folders"]):
+        if not utils.get_asset_path("robots.txt", [], files_folders=kw["files_folders"], output_dir=False):
             yield utils.apply_filters({
                 "basename": self.name,
                 "name": robots_path,
@@ -82,6 +82,6 @@ class RobotsFile(LateTask):
                 "task_dep": ["sitemap"]
             }, kw["filters"])
         elif kw["robots_exclusions"]:
-            utils.LOGGER.warn('Did not generate robots.txt as one already exists in FILES_FOLDERS. ROBOTS_EXCLUSIONS will not have any affect on the copied fie.')
+            utils.LOGGER.warn('Did not generate robots.txt as one already exists in FILES_FOLDERS. ROBOTS_EXCLUSIONS will not have any affect on the copied file.')
         else:
             utils.LOGGER.debug('Did not generate robots.txt as one already exists in FILES_FOLDERS.')