cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 19:13:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chore: add ai2bot

Browse source
This commit is contained in:
Cory Dransfeldt 2024-08-28 20:07:32 -07:00
parent 6dc900b582
commit 0f8723558f
No known key found for this signature in database
1 changed files with 14 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
robots.json
View file

@ -1,4 +1,18 @@
{
"AI2Bot": {
"description": "Explores 'certain domains' to find web content.",
"frequency": "No information. provided.",
"function": "Content is used to train open language models.",
"operator": "[Ai2](https://allenai.org/crawler)",
"respect": "Yes"
},
"Ai2Bot-Dolma": {
"description": "Explores 'certain domains' to find web content.",
"frequency": "No information. provided.",
"function": "Content is used to train open language models.",
"operator": "[Ai2](https://allenai.org/crawler)",
"respect": "Yes"
},
"Amazonbot": {
"description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses.",
"frequency": "No information. provided.",