takahe/templates/robots.txt
2023-01-29 12:27:07 -07:00

13 lines
242 B
Text

User-Agent: *
# Don't allow any bot to crawl tags.
Disallow: /tags/
Disallow: /tags/*
# Don't allow bots to crawl through the proxy
Disallow: /proxy/*
{% for user_agent in user_agents %}
User-agent: {{user_agent}}
Disallow: /
{% endfor %}