From 9bda780c400303a1aa9a100f44141120410846fe Mon Sep 17 00:00:00 2001 From: Aminda Suomalainen Date: Fri, 31 May 2024 06:29:21 +0300 Subject: [PATCH] controlled allowing of letting the bots in --- _config.yml | 3 +- blog/_posts/2015-03-25-leaving-bots-life.md | 1 + blog/_posts/2015-05-31-weechat-filter.md | 1 + blog/_posts/2015-06-12-ufw.md | 1 + blog/_posts/2015-06-22-ipv6.md | 1 + ...sahkopostia-lakivaliokunnalle-translaki.md | 1 + blog/_posts/2015-11-29-SSID_optout_nomap.md | 1 + .../2015-12-25-asus-tunnelbroker-net.md | 1 + blog/_posts/2017-04-30-post-trans.md | 1 + ...9-07-11-android-private-dns-in-practice.md | 1 + .../2021-08-03-matrix-perfect-privacy-not.md | 1 + .../2021-11-23-leaving-privacyguides.md | 1 + ...x-community-abuse-security-by-obscurity.md | 1 + ...2-08-22-ssh-signing-verifying-functions.md | 1 + blog/_posts/2024-05-17-https-everywhere.md | 1 + .../2024-05-22-policy-contentblocker.md | 1 + blog/index.md | 1 + index.html | 1 + n/coauthor.md | 3 +- n/dns.md | 3 +- n/essentialsoftware.md | 3 +- n/f-droid.md | 3 +- n/fairbuds.md | 3 +- n/firewalld.md | 3 +- n/gpg.md | 3 +- n/keyoxide.md | 3 +- n/localectl.md | 3 +- n/matrixspoilers.md | 3 +- n/obtainium.md | 3 +- pages/discuss.markdown | 1 + pages/irc/irc.markdown | 1 + pages/matrix.markdown | 1 + robots.txt | 67 +++++++++---------- 33 files changed, 76 insertions(+), 47 deletions(-) diff --git a/_config.yml b/_config.yml index 14a3771..a56e6a0 100644 --- a/_config.yml +++ b/_config.yml @@ -12,7 +12,8 @@ baseurl: "" # the subpath of your site, e.g. /blog/ # At least it's just a static page. #url: "https://www.aminda.eu" # the base hostname & protocol for your site url: "https://aminda.eu" # the base hostname & protocol for your site -# Default meta-robots tags, can be set per-page +# Default meta-robots tags, can be set per-page. SET PER PAGE, DON'T ALLOW +# THEM HERE! Remember /n and less good blog posts etc. robots: noai, nofollow, noindex codeberg_username: Mikaela diff --git a/blog/_posts/2015-03-25-leaving-bots-life.md b/blog/_posts/2015-03-25-leaving-bots-life.md index 8939d1f..ca8096e 100644 --- a/blog/_posts/2015-03-25-leaving-bots-life.md +++ b/blog/_posts/2015-03-25-leaving-bots-life.md @@ -5,6 +5,7 @@ title: "Little on my life currently" category: [english] tags: [irc, english, life] redirect_from: /english/2015/03/25/leaving-bots-life.html +robots: noai --- ## A little on my life currently diff --git a/blog/_posts/2015-05-31-weechat-filter.md b/blog/_posts/2015-05-31-weechat-filter.md index 9384bf1..a5b6dcb 100644 --- a/blog/_posts/2015-05-31-weechat-filter.md +++ b/blog/_posts/2015-05-31-weechat-filter.md @@ -8,6 +8,7 @@ redirect_from: - /filter.html - /weechat-filter.html - /english/2015/05/31/weechat-filter.html +robots: noai --- _Everyone knows the `/ignore` command, but there is also `/filter` which diff --git a/blog/_posts/2015-06-12-ufw.md b/blog/_posts/2015-06-12-ufw.md index f1aae88..424fbc1 100644 --- a/blog/_posts/2015-06-12-ufw.md +++ b/blog/_posts/2015-06-12-ufw.md @@ -7,6 +7,7 @@ tags: [english] redirect_from: - /ufw/ - /english/2015/06/12/ufw.html +robots: noai --- _This post describes my UFW config and is here so I find it from somewhere diff --git a/blog/_posts/2015-06-22-ipv6.md b/blog/_posts/2015-06-22-ipv6.md index d09a60c..11fd983 100644 --- a/blog/_posts/2015-06-22-ipv6.md +++ b/blog/_posts/2015-06-22-ipv6.md @@ -9,6 +9,7 @@ redirect_from: - /IPv6/ - /english/2015/06/22/ipv6.html lang: en +robots: noai --- _There appears to be a lot of confusion on IPv6 and in this post I try to diff --git a/blog/_posts/2015-11-21-sahkopostia-lakivaliokunnalle-translaki.md b/blog/_posts/2015-11-21-sahkopostia-lakivaliokunnalle-translaki.md index f312953..c5b9b4f 100644 --- a/blog/_posts/2015-11-21-sahkopostia-lakivaliokunnalle-translaki.md +++ b/blog/_posts/2015-11-21-sahkopostia-lakivaliokunnalle-translaki.md @@ -6,6 +6,7 @@ category: [finnish] tags: [finnish, translaki] redirect_from: /finnish/2015/11/21/sahkopostia-lakivaliokunnalle-translaki.html lang: fi +robots: noai --- _Toivottavasti tämä viesti on edes jotenkuten järkevä, se olisi ollut diff --git a/blog/_posts/2015-11-29-SSID_optout_nomap.md b/blog/_posts/2015-11-29-SSID_optout_nomap.md index bdac8b7..d27b305 100644 --- a/blog/_posts/2015-11-29-SSID_optout_nomap.md +++ b/blog/_posts/2015-11-29-SSID_optout_nomap.md @@ -10,6 +10,7 @@ redirect_from: - /openwireless_nomap.html - /openwireless5_nomap.html - /english/2015/11/29/SSID_optout_nomap.html +robots: noai --- _Anxiety is now focusing to WLAN mapping & key sharing and how easy it is diff --git a/blog/_posts/2015-12-25-asus-tunnelbroker-net.md b/blog/_posts/2015-12-25-asus-tunnelbroker-net.md index 8826f3f..1199c40 100644 --- a/blog/_posts/2015-12-25-asus-tunnelbroker-net.md +++ b/blog/_posts/2015-12-25-asus-tunnelbroker-net.md @@ -6,6 +6,7 @@ category: [english] tags: [IPv6, english] redirect_from: /english/2015/12/25/asus-tunnelbroker-net.html lang: en +robots: noai --- _I am typing this, because there is a lot of misinformation on this, diff --git a/blog/_posts/2017-04-30-post-trans.md b/blog/_posts/2017-04-30-post-trans.md index b331d6f..73fa6e2 100644 --- a/blog/_posts/2017-04-30-post-trans.md +++ b/blog/_posts/2017-04-30-post-trans.md @@ -5,6 +5,7 @@ title: "Post trans process, orchiectomy" category: [english] tags: [english, life, buddhism] redirect_from: /english/2017/04/30/post-trans.html +robots: noai --- _A little confusing flood of what my fingers brought up on the last past diff --git a/blog/_posts/2019-07-11-android-private-dns-in-practice.md b/blog/_posts/2019-07-11-android-private-dns-in-practice.md index 036b03c..e332e85 100644 --- a/blog/_posts/2019-07-11-android-private-dns-in-practice.md +++ b/blog/_posts/2019-07-11-android-private-dns-in-practice.md @@ -8,6 +8,7 @@ redirect_from: - /dns.html - /dot.html lang: en +robots: noai --- _Since I first heard of Android 9 Private DNS I wondered how it will work diff --git a/blog/_posts/2021-08-03-matrix-perfect-privacy-not.md b/blog/_posts/2021-08-03-matrix-perfect-privacy-not.md index dd7f770..1daf03d 100644 --- a/blog/_posts/2021-08-03-matrix-perfect-privacy-not.md +++ b/blog/_posts/2021-08-03-matrix-perfect-privacy-not.md @@ -7,6 +7,7 @@ redirect_from: - /matrix.html - /element.html lang: en +robots: noai --- _Having used Matrix since 2016 and hearing about its greatness without any diff --git a/blog/_posts/2021-11-23-leaving-privacyguides.md b/blog/_posts/2021-11-23-leaving-privacyguides.md index 94c5cf1..15c0672 100644 --- a/blog/_posts/2021-11-23-leaving-privacyguides.md +++ b/blog/_posts/2021-11-23-leaving-privacyguides.md @@ -4,6 +4,7 @@ title: "Leaving Privacy Guides team" category: [english] tags: [privacyguides] lang: en +robots: noai --- _PrivacyGuides.org was supposed to be the continuation of PrivacyTools.io diff --git a/blog/_posts/2021-12-05-matrix-community-abuse-security-by-obscurity.md b/blog/_posts/2021-12-05-matrix-community-abuse-security-by-obscurity.md index 7e8dfde..9d8ea9e 100644 --- a/blog/_posts/2021-12-05-matrix-community-abuse-security-by-obscurity.md +++ b/blog/_posts/2021-12-05-matrix-community-abuse-security-by-obscurity.md @@ -4,6 +4,7 @@ title: "Matrix abuse protection model for community maintainers: security by obs category: [english] tags: [matrix] lang: en +robots: noai --- _I am administrator or moderator in multiple communities in Matrix, the most sizable diff --git a/blog/_posts/2022-08-22-ssh-signing-verifying-functions.md b/blog/_posts/2022-08-22-ssh-signing-verifying-functions.md index 6526f5b..079aad4 100644 --- a/blog/_posts/2022-08-22-ssh-signing-verifying-functions.md +++ b/blog/_posts/2022-08-22-ssh-signing-verifying-functions.md @@ -3,6 +3,7 @@ layout: post title: "bash/zsh functions for easier SSH signing and verification" category: [english] tags: [ssh] +robots: noai --- _I have been using SSH signed git commits from 8 months and started signing things with my SSH key instead of PGP keys and thought to share how to do that more easily_ diff --git a/blog/_posts/2024-05-17-https-everywhere.md b/blog/_posts/2024-05-17-https-everywhere.md index a7b3fdf..b393254 100644 --- a/blog/_posts/2024-05-17-https-everywhere.md +++ b/blog/_posts/2024-05-17-https-everywhere.md @@ -8,6 +8,7 @@ redirect_from: - /httpseverywhere.html - /https.html lang: en +robots: noai --- _I used to be sad since the EFF discontinued HTTPS Everywhere extension since the setting often didn't sync and it only applied to me as opposed to everyone using a shared computer. However since I have dived into browser policies, this is no longer an issue for me._ diff --git a/blog/_posts/2024-05-22-policy-contentblocker.md b/blog/_posts/2024-05-22-policy-contentblocker.md index 3ddc2fa..f69c8c9 100644 --- a/blog/_posts/2024-05-22-policy-contentblocker.md +++ b/blog/_posts/2024-05-22-policy-contentblocker.md @@ -8,6 +8,7 @@ redirect_from: - /ublockorigin.html - /https2.html lang: en +robots: noai --- _I previously wrote about enforcing HTTPS for all users/profiles through browser policy receiving some positive feedback and I felt like continuing on the subject by instructing with extension installation. This barely scratches the surface of what browser policy can do for you either though._ diff --git a/blog/index.md b/blog/index.md index 16de315..93231ee 100644 --- a/blog/index.md +++ b/blog/index.md @@ -5,6 +5,7 @@ navigation: true permalink: /blog/ excerpt: "Blog index, posts in English and posts in Finnish — Blogin etusivu, postaukset englanniksi ja postaukset suomeksi." lang: en +robots: noai --- diff --git a/index.html b/index.html index 96e0e1b..8787004 100644 --- a/index.html +++ b/index.html @@ -2,6 +2,7 @@ layout: index title: Index excerpt: "I am a Highly Sensitive Autistic Pirate with Linux experience since 2008. I am familiar with git and looking for employment." +robots: noai ---

diff --git a/n/coauthor.md b/n/coauthor.md index e6f28cb..f4b7749 100644 --- a/n/coauthor.md +++ b/n/coauthor.md @@ -1,8 +1,9 @@ --- layout: mini permalink: /n/coauthor.html -sitemap: false +sitemap: true lang: en +robots: noai --- # Git forge private emails diff --git a/n/dns.md b/n/dns.md index 9abadac..84053d5 100644 --- a/n/dns.md +++ b/n/dns.md @@ -6,8 +6,9 @@ redirect_from: - /n/rethink.html - /n/esni.html - /n/ech.html -sitemap: false +sitemap: true lang: en +robots: noai --- # DNS notes diff --git a/n/essentialsoftware.md b/n/essentialsoftware.md index cb0a3f8..7c9ac4d 100644 --- a/n/essentialsoftware.md +++ b/n/essentialsoftware.md @@ -1,8 +1,9 @@ --- layout: mini permalink: /n/essentialsoftware.html -sitemap: false +sitemap: true lang: en +robots: noai --- # Essential software for new systems diff --git a/n/f-droid.md b/n/f-droid.md index 89a085a..463b4d2 100644 --- a/n/f-droid.md +++ b/n/f-droid.md @@ -2,8 +2,9 @@ layout: mini permalink: /n/f-droid.html redirect_from: /n/fdroid.html -sitemap: false +sitemap: true lang: en +robots: noai --- F-Droid is kind of apt for Android with multiple repositories available. This diff --git a/n/fairbuds.md b/n/fairbuds.md index a98a165..9548091 100644 --- a/n/fairbuds.md +++ b/n/fairbuds.md @@ -3,8 +3,9 @@ layout: mini permalink: /n/fairbuds.html redirect_from: - /n/fairbudsxl.html -sitemap: false +sitemap: true lang: en +robots: noai --- # Fairbuds XL configuration diff --git a/n/firewalld.md b/n/firewalld.md index f82cba9..3422205 100644 --- a/n/firewalld.md +++ b/n/firewalld.md @@ -3,8 +3,9 @@ layout: mini permalink: /n/firewalld.html redirect_from: - /n/firewall-cmd.html -sitemap: false +sitemap: true lang: en +robots: noai --- # Quick note on firewalld usage diff --git a/n/gpg.md b/n/gpg.md index d6f3757..702ce92 100644 --- a/n/gpg.md +++ b/n/gpg.md @@ -13,7 +13,8 @@ redirect_from: - /n/wkd.html - n/keybase.html lang: en -sitemap: false +sitemap: true +robots: noai --- # Quick GPG notes where I may find them diff --git a/n/keyoxide.md b/n/keyoxide.md index 76340ba..b65c01f 100644 --- a/n/keyoxide.md +++ b/n/keyoxide.md @@ -2,8 +2,9 @@ layout: mini permalink: /n/keyoxide.html redirect_from: /n/keyoxide-flutter.html -sitemap: false +sitemap: true lang: en +robots: noai --- > Keyoxide is a decentralized tool to create and verify decentralized online identities. diff --git a/n/localectl.md b/n/localectl.md index cde21df..52c2980 100644 --- a/n/localectl.md +++ b/n/localectl.md @@ -6,8 +6,9 @@ redirect_from: - /locales.html - /english/2016/10/16/adjusting-locales.html - /blog/english/2016/10/16/adjusting-locales.html -sitemap: false +sitemap: true lang: en +robots: noai --- # Quick notes on `localectl` settings diff --git a/n/matrixspoilers.md b/n/matrixspoilers.md index 137c509..0116dfe 100644 --- a/n/matrixspoilers.md +++ b/n/matrixspoilers.md @@ -4,8 +4,9 @@ permalink: /n/matrixspoilers.html redirect_from: - n/nheko.html - n/matrixspoiler.html -sitemap: false +sitemap: true lang: en +robots: noai --- The Matrix specification supports spoilers since version 1.1: diff --git a/n/obtainium.md b/n/obtainium.md index af52817..d294150 100644 --- a/n/obtainium.md +++ b/n/obtainium.md @@ -1,8 +1,9 @@ --- layout: mini permalink: /n/obtainium.html -sitemap: false +sitemap: true lang: en +robots: noai --- [Obtainium](https://github.com/ImranR98/Obtainium/) downloads apps directly from forge/whatever without need for app diff --git a/pages/discuss.markdown b/pages/discuss.markdown index 2591b86..05723c7 100644 --- a/pages/discuss.markdown +++ b/pages/discuss.markdown @@ -6,6 +6,7 @@ redirect_from: /discuss/index.html excerpt: "Introduction to my public instant messaging chats" navigation: true lang: en +robots: noai --- I have multiple instant messaging chatrooms. diff --git a/pages/irc/irc.markdown b/pages/irc/irc.markdown index e767897..594eb38 100644 --- a/pages/irc/irc.markdown +++ b/pages/irc/irc.markdown @@ -4,6 +4,7 @@ title: IRC navigation: true permalink: /irc/ excerpt: "Where do you find me at IRC and verify that it's me. Also includes my IRC related posts that are hopefully helpful." +robots: noai --- IRC has been a big part of my life, I discovered it during junior high school diff --git a/pages/matrix.markdown b/pages/matrix.markdown index a6204d5..c22e982 100644 --- a/pages/matrix.markdown +++ b/pages/matrix.markdown @@ -7,6 +7,7 @@ excerpt: "Matrix, a decentralised conversation store" redirect_from: - /matrix.html lang: en +robots: noai --- Just like [IRC](/irc/), _Matrix_ has became a part of my social life online. My diff --git a/robots.txt b/robots.txt index 2243ce1..4e30b9a 100644 --- a/robots.txt +++ b/robots.txt @@ -1,48 +1,15 @@ User-agent: * Allow: / -Allow: /$ -Allow: /index.html -Disallow: /blog/english/2015/ -Disallow: /blog/english/2016/ -Disallow: /blog/finnish/2015/ -Disallow: /blog/finnish/2016/ -Disallow: /blog/finnish/2017/ Disallow: /txt/ Disallow: /php/ Disallow: /r/ Disallow: /or/ -Disallow: /n/ -Noindex: /blog/english/2015/ -Noindex: /blog/english/2016/ -Noindex: /blog/finnish/2015/ -Noindex: /blog/finnish/2016/ -Noindex: /blog/finnish/2017/ -Noindex: /txt/ -Noindex: /php/ -Noindex: /r/ -Noindex: /or/ -Noindex: /n/ +Disallow: /ir/ Sitemap: https://aminda.eu/sitemap.xml Sitemap: https://aminda.eu/sitemaps.xml -# These were previously part of darkvisitors but are not anymore for some reason? -User-agent: ChatGPT-User -Disallow: / - -User-agent: cohere-ai -Disallow: / - -User-agent: omgilibot -Disallow: / - # https://darkvisitors.com/robots-txt-builder -# Dark Visitors robots.txt - -# AI Data Scraper -# https://darkvisitors.com/agents/anthropic-ai - -User-agent: anthropic-ai -Disallow: / +# Dark Visitors Robots.txt # AI Data Scraper # https://darkvisitors.com/agents/bytespider @@ -56,6 +23,18 @@ Disallow: / User-agent: CCBot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/claudebot + +User-agent: ClaudeBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/diffbot + +User-agent: Diffbot +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/facebookbot @@ -79,3 +58,21 @@ Disallow: / User-agent: omgili Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/claude-web + +User-agent: Claude-Web +Disallow: / + +# Undocumented AI Agent +# https://darkvisitors.com/agents/cohere-ai + +User-agent: cohere-ai +Disallow: /