From 4b05dcde431ae7aa71dcd705358fbc2c7439db67 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:16:34 +0200 Subject: [PATCH] [chore] Update robots.txt, give chatgpt the middle finger (#2085) --- internal/web/robots.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/internal/web/robots.go b/internal/web/robots.go index b843203b5..a79c31034 100644 --- a/internal/web/robots.go +++ b/internal/web/robots.go @@ -27,29 +27,43 @@ const ( robotsPath = "/robots.txt" robotsMetaAllowSome = "nofollow, noarchive, nositelinkssearchbox, max-image-preview:standard" // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#robotsmeta robotsTxt = `# GoToSocial robots.txt -- to edit, see internal/web/robots.go -# more info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro +# More info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro + +# Before we commence, a giant fuck you to ChatGPT in particular. +# https://platform.openai.com/docs/gptbot +User-agent: GPTBot +Disallow: / + +# Rules for everything else. User-agent: * Crawl-delay: 500 -# api stuff + +# API endpoints. Disallow: /api/ -# auth/login stuff + +# Auth/login endpoints. Disallow: /auth/ Disallow: /oauth/ Disallow: /check_your_email Disallow: /wait_for_approval Disallow: /account_disabled -# well known stuff + +# Well-known endpoints. Disallow: /.well-known/ -# files + +# Fileserver/media. Disallow: /fileserver/ -# s2s AP stuff + +# Fedi S2S API endpoints. Disallow: /users/ Disallow: /emoji/ -# panels + +# Settings panels. Disallow: /admin Disallow: /user Disallow: /settings/ -# domain blocklist + +# Domain blocklist. Disallow: /about/suspended` )