robots.txt: even more specific rules

This commit is contained in:
Aminda Suomalainen 2024-06-02 12:49:59 +03:00
parent b23e622fae
commit 2798fca073
Signed by: Mikaela
SSH Key Fingerprint: SHA256:CXLULpqNBdUKB6E6fLA1b/4SzG0HvKD19PbIePU175Q
1 changed files with 3 additions and 1 deletions

View File

@ -1,6 +1,6 @@
User-agent: *
Allow: /
Allow: /$
Allow: /*.html$
Allow: /blog/$
Allow: /blog/feed.xml
Allow: /blog/feed.en.xml
@ -8,8 +8,10 @@ Allow: /blog/feed.fi.xml
Allow: /irc/$
Allow: /matrix/$
Allow: /n/$
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /sitemaps.xml
Disallow: /
Disallow: /*.asc$
Disallow: /*.json$
Disallow: /*.sig$