From dde3294601de88269dfb9019b0362067c643a963 Mon Sep 17 00:00:00 2001
From: Aminda Suomalainen <suomalainen+git@mikaela.info>
Date: Sun, 2 Jun 2024 12:58:30 +0300
Subject: [PATCH] robots.txt: sync with aminda.eu

while naturally removing irrelevant parts
---
 robots.txt | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 1 deletion(-)

diff --git a/robots.txt b/robots.txt
index 51db127..5ed9544 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,6 +1,77 @@
 User-agent: *
-Allow: /
+Allow: /$
+Allow: /en/$
+Allow: /*.html$
+Allow: /robots.txt
+Allow: /sitemap.xml
+Disallow: /
 Sitemap: https://cv.aminda.eu/sitemap.xml
 
+# https://darkvisitors.com/robots-txt-builder
+# Dark Visitors Robots.txt
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/bytespider
+
+User-agent: Bytespider
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/ccbot
+
+User-agent: CCBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/claudebot
+
+User-agent: ClaudeBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/diffbot
+
+User-agent: Diffbot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/facebookbot
+
+User-agent: FacebookBot
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/google-extended
+
+User-agent: Google-Extended
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/gptbot
+
 User-agent: GPTBot
 Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/omgili
+
+User-agent: omgili
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/anthropic-ai
+
+User-agent: anthropic-ai
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/claude-web
+
+User-agent: Claude-Web
+Disallow: /
+
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/cohere-ai
+
+User-agent: cohere-ai
+Disallow: /