cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 19:13:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Daily update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2024-08-22 01:11:43 +00:00
parent 3580a7096f
commit 7bfc1647a8
1 changed files with 7 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
robots.json
View file

@ -181,6 +181,13 @@
"operator": "[Velen Crawler](https://velen.io)",
"respect": "[Yes](https://velen.io)"
},
"Webzio-Extended": {
"description": "Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended",
"frequency": "Unclear at this time.",
"function": "AI Data Scrapers",
"operator": "Unclear at this time.",
"respect": "Unclear at this time."
},
"YouBot": {
"description": "Retrieves data used for You.com web search engine and LLMs.",
"frequency": "No information.",