From aec43c7c1740d51fdd5b6100c4bc04e1f00cb09f Mon Sep 17 00:00:00 2001 From: Mikaela Suomalainen Date: Sun, 26 Apr 2015 18:50:25 +0300 Subject: [PATCH] Does the sitemap really need to include 404 error? --- _config.yml | 1 + robots.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/_config.yml b/_config.yml index b063788..624928c 100644 --- a/_config.yml +++ b/_config.yml @@ -29,6 +29,7 @@ sitemap: exclude: # Even if you can see the existence of these pages here, I don't # wish to get search engine attention to them. + - "/404.html" - "/bot.html" - "/channel.html" include: [PGP, PHP, keybase.txt, robots.txt] diff --git a/robots.txt b/robots.txt index 642ca82..66e46d2 100644 --- a/robots.txt +++ b/robots.txt @@ -3,6 +3,7 @@ Allow: / # I prefer search engines to not advertise these pages even if you can find # them here or in `_config.yml`. +Deny: /404.html Deny: /bot.html Deny: /channel.html