From 0a2a8fb126f9a96f4acef1db79e76a1dd8574749 Mon Sep 17 00:00:00 2001 From: Mikaela Suomalainen Date: Sun, 26 Apr 2015 18:45:04 +0300 Subject: [PATCH] block bot & channel in sitemap.xml & robots.txt --- _config.yml | 5 +++++ robots.txt | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/_config.yml b/_config.yml index 941f896..b063788 100644 --- a/_config.yml +++ b/_config.yml @@ -26,4 +26,9 @@ gems: - jekyll-sitemap sitemap: file: "/sitemap.xml" + exclude: + # Even if you can see the existence of these pages here, I don't + # wish to get search engine attention to them. + - "/bot.html" + - "/channel.html" include: [PGP, PHP, keybase.txt, robots.txt] diff --git a/robots.txt b/robots.txt index 6b9d944..642ca82 100644 --- a/robots.txt +++ b/robots.txt @@ -1,4 +1,9 @@ User-agent: * Allow: / +# I prefer search engines to not advertise these pages even if you can find +# them here or in `_config.yml`. +Deny: /bot.html +Deny: /channel.html + Sitemap: https://mikaela.info/sitemap.xml