diff --git a/robots.txt b/robots.txt index 1b42ebb..8f6aefe 100644 --- a/robots.txt +++ b/robots.txt @@ -25,26 +25,56 @@ Noindex: /n/ Sitemap: https://aminda.eu/sitemap.xml Sitemap: https://aminda.eu/sitemaps.xml -User-agent: GPTBot -Disallow: / +# https://darkvisitors.com/robots-txt-builder +# Dark Visitors robots.txt + +# AI Assistant +# https://darkvisitors.com/agents/chatgpt-user User-agent: ChatGPT-User Disallow: / -User-agent: Google-Extended +# AI Assistant +# https://darkvisitors.com/agents/cohere-ai + +User-agent: cohere-ai Disallow: / -User-agent: CCBot -Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/anthropic-ai User-agent: anthropic-ai Disallow: / -User-agent: Omgilibot +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot Disallow: / -User-agent: Omgili -Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot User-agent: FacebookBot Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + +User-agent: GPTBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/omgilibot + +User-agent: omgilibot +Disallow: / + +User-agent: omgili +Disallow: /