cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-12 05:57:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge 8dc36aa2e2 into 5b8650b99b

Browse source
This commit is contained in:
Kyle Buckingham 2025-04-01 15:25:48 -07:00 committed by GitHub
commit 68531dd8e2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 17 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
robots.json
View file

@ -69,13 +69,6 @@
"frequency": "Only when prompted by a user.", "frequency": "Only when prompted by a user.",
"description": "Used by plugins in ChatGPT to answer queries based on user input." "description": "Used by plugins in ChatGPT to answer queries based on user input."
}, },
"Claude-Web": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time.",
"function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
},
"ClaudeBot": { "ClaudeBot": {
"operator": "[Anthropic](https://www.anthropic.com)", "operator": "[Anthropic](https://www.anthropic.com)",
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)", "respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
@ -83,6 +76,20 @@
"frequency": "No information provided.", "frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic." "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
}, },
"Claude-User": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time.",
"function": "Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent.",
"frequency": "No information provided.",
"description": "Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent."
},
"Claude-SearchBot": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time.",
"function": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses.",
"frequency": "No information provided.",
"description": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses."
},
"cohere-ai": { "cohere-ai": {
"operator": "[Cohere](https://cohere.com)", "operator": "[Cohere](https://cohere.com)",
"respect": "Unclear at this time.", "respect": "Unclear at this time.",
@ -328,4 +335,4 @@
"frequency": "No information.", "frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs." "description": "Retrieves data used for You.com web search engine and LLMs."
} }
} }

3
robots.txt
View file

@ -8,8 +8,9 @@ User-agent: Brightbot 1.0
User-agent: Bytespider User-agent: Bytespider
User-agent: CCBot User-agent: CCBot
User-agent: ChatGPT-User User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: cohere-ai User-agent: cohere-ai
User-agent: cohere-training-data-crawler User-agent: cohere-training-data-crawler
User-agent: Crawlspace User-agent: Crawlspace