diff options
Diffstat (limited to 'nikola/plugins/task/robots.py')
| -rw-r--r-- | nikola/plugins/task/robots.py | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py index 65254b6..8537fc8 100644 --- a/nikola/plugins/task/robots.py +++ b/nikola/plugins/task/robots.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright © 2012-2015 Roberto Alsina and others. +# Copyright © 2012-2016 Roberto Alsina and others. # Permission is hereby granted, free of charge, to any # person obtaining a copy of this software and associated @@ -39,7 +39,6 @@ from nikola import utils class RobotsFile(LateTask): - """Generate a robots.txt file.""" name = "robots_file" @@ -64,14 +63,15 @@ class RobotsFile(LateTask): with io.open(robots_path, 'w+', encoding='utf8') as outf: outf.write("Sitemap: {0}\n\n".format(sitemapindex_url)) + outf.write("User-Agent: *\n") if kw["robots_exclusions"]: - outf.write("User-Agent: *\n") for loc in kw["robots_exclusions"]: outf.write("Disallow: {0}\n".format(loc)) + outf.write("Host: {0}\n".format(urlparse(kw["base_url"]).netloc)) yield self.group_task() - if not utils.get_asset_path("robots.txt", [], files_folders=kw["files_folders"]): + if not utils.get_asset_path("robots.txt", [], files_folders=kw["files_folders"], output_dir=False): yield utils.apply_filters({ "basename": self.name, "name": robots_path, @@ -82,6 +82,6 @@ class RobotsFile(LateTask): "task_dep": ["sitemap"] }, kw["filters"]) elif kw["robots_exclusions"]: - utils.LOGGER.warn('Did not generate robots.txt as one already exists in FILES_FOLDERS. ROBOTS_EXCLUSIONS will not have any affect on the copied fie.') + utils.LOGGER.warn('Did not generate robots.txt as one already exists in FILES_FOLDERS. ROBOTS_EXCLUSIONS will not have any affect on the copied file.') else: utils.LOGGER.debug('Did not generate robots.txt as one already exists in FILES_FOLDERS.') |