cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-05-18 08:13:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix(robots.json): remove duplicate entry for Meta-ExternalAgent

Browse source
This commit is contained in:
Cory Dransfeldt 2025-05-15 21:00:23 -07:00
parent 16d1de7094
commit 8a44daa68a
No known key found for this signature in database
1 changed files with 2 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

9
robots.json
View file

@ -244,17 +244,10 @@
"frequency": "Unclear at this time.", "frequency": "Unclear at this time.",
"description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot" "description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot"
}, },
"meta-externalagent": { "Meta-ExternalAgent": {
"operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)", "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)",
"respect": "Yes", "respect": "Yes",
"function": "Used to train models and improve products.", "function": "Used to train models and improve products.",
"frequency": "No information.",
"description": "\"The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.\""
},
"Meta-ExternalAgent": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Data Scrapers",
"frequency": "Unclear at this time.", "frequency": "Unclear at this time.",
"description": "Meta-ExternalAgent is a web crawler used by Meta to download training data for its AI models and improve its products by indexing content directly. More info can be found at https://darkvisitors.com/agents/agents/meta-externalagent" "description": "Meta-ExternalAgent is a web crawler used by Meta to download training data for its AI models and improve its products by indexing content directly. More info can be found at https://darkvisitors.com/agents/agents/meta-externalagent"
}, },