From 2798fca073505014731b194136afbcdddfd06d8a Mon Sep 17 00:00:00 2001 From: Aminda Suomalainen Date: Sun, 2 Jun 2024 12:49:59 +0300 Subject: [PATCH] robots.txt: even more specific rules --- robots.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/robots.txt b/robots.txt index 8335981..5f535b0 100644 --- a/robots.txt +++ b/robots.txt @@ -1,6 +1,6 @@ User-agent: * -Allow: / Allow: /$ +Allow: /*.html$ Allow: /blog/$ Allow: /blog/feed.xml Allow: /blog/feed.en.xml @@ -8,8 +8,10 @@ Allow: /blog/feed.fi.xml Allow: /irc/$ Allow: /matrix/$ Allow: /n/$ +Allow: /robots.txt Allow: /sitemap.xml Allow: /sitemaps.xml +Disallow: / Disallow: /*.asc$ Disallow: /*.json$ Disallow: /*.sig$