diff --git a/robots.txt b/robots.txt index e097a47..0079f66 100644 --- a/robots.txt +++ b/robots.txt @@ -1,3 +1,4 @@ +User-agent: AI User-agent: AI2Bot User-agent: Ai2Bot-Dolma User-agent: Amazonbot @@ -35,4 +36,4 @@ User-agent: iaskspider/2.0 User-agent: img2dataset User-agent: omgili User-agent: omgilibot -Disallow: / \ No newline at end of file +Disallow: /