docs: update robots.txt (#56010)

PR Close #56010
This commit is contained in:
Emma Twersky 2024-05-22 12:06:58 -07:00 committed by Dylan Hunn
parent 5d64589e4c
commit e67cb09404

View file

@ -1,2 +1,3 @@
User-agent: Algolia Crawler
Allow: /
# Allow all URLs (see https://www.robotstxt.org/robotstxt.html)
User-agent: *
Disallow: /