diff --git a/docker/nginx.conf b/docker/nginx.conf index 3be6ad5..f0a985d 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -36,6 +36,11 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_http_version 1.1; + # Serve robots.txt from the non-collected dir as a special case. + location /robots.txt { + alias /takahe/static/robots.txt; + } + # Serves static files from the collected dir location /static/ { # Files in static have cache-busting hashes in the name, thus can be cached forever diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..d910569 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,5 @@ +User-Agent: * + +# Don't allow any bot to crawl tags. +Disallow: /tags/ +Disallow: /tags/*