cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-05-17 16:03:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Update from Dark Visitors
Some checks failed
/ run-tests (push) Waiting to run
Details
/ ai-robots-txt (push) Has been cancelled
Details

Browse source
This commit is contained in:
dark-visitors 2025-04-25 00:56:57 +00:00
parent 4654e14e9c
commit 934ac7b318
1 changed files with 15 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

16
robots.json
View file

@ -251,6 +251,13 @@
"frequency": "No information.",
"description": "\"The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.\""
},
"Meta-ExternalAgent": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Data Scrapers",
"frequency": "Unclear at this time.",
"description": "Meta-ExternalAgent is a web crawler used by Meta to download training data for its AI models and improve its products by indexing content directly. More info can be found at https://darkvisitors.com/agents/agents/meta-externalagent"
},
"meta-externalfetcher": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
@ -258,6 +265,13 @@
"frequency": "Unclear at this time.",
"description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
},
"Meta-ExternalFetcher": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Assistants",
"frequency": "Unclear at this time.",
"description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
},
"NovaAct": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
@ -384,4 +398,4 @@
"frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs."
}
}
}