From f8e6d326ba1ee50c8e93fab46d4471a157d95e6c Mon Sep 17 00:00:00 2001 From: teddit Date: Thu, 18 Nov 2021 20:29:24 +0100 Subject: [PATCH] disallow Applebot, crawls way too much --- static/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/static/robots.txt b/static/robots.txt index 23132a0..18a2344 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,2 +1,3 @@ User-agent: MJ12bot +User-agent: Applebot Disallow: /