From 2e70384d1fc7ddbe2ca732e7dcd1ff135c9d5f8c Mon Sep 17 00:00:00 2001 From: Aminda Suomalainen Date: Fri, 29 Dec 2023 18:01:48 +0200 Subject: [PATCH] robots.txt: replace codeberg.org/robots.txt copying with Dark Visitors --- robots.txt | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/robots.txt b/robots.txt index 1b42ebb..8f6aefe 100644 --- a/robots.txt +++ b/robots.txt @@ -25,26 +25,56 @@ Noindex: /n/ Sitemap: https://aminda.eu/sitemap.xml Sitemap: https://aminda.eu/sitemaps.xml -User-agent: GPTBot -Disallow: / +# https://darkvisitors.com/robots-txt-builder +# Dark Visitors robots.txt + +# AI Assistant +# https://darkvisitors.com/agents/chatgpt-user User-agent: ChatGPT-User Disallow: / -User-agent: Google-Extended +# AI Assistant +# https://darkvisitors.com/agents/cohere-ai + +User-agent: cohere-ai Disallow: / -User-agent: CCBot -Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/anthropic-ai User-agent: anthropic-ai Disallow: / -User-agent: Omgilibot +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot Disallow: / -User-agent: Omgili -Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot User-agent: FacebookBot Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + +User-agent: GPTBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/omgilibot + +User-agent: omgilibot +Disallow: / + +User-agent: omgili +Disallow: /