cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 19:13:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Daily update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2024-09-08 01:19:31 +00:00
parent b15b8062ce
commit 5963cbf9f7
1 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
robots.json
View file

@ -125,13 +125,6 @@
"operator": "Google",
"respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
},
"iaskspider/2.0": {
"description": "Used to provide answers to user queries.",
"frequency": "Unclear at this time.",
"function": "Crawls sites to provide answers to user queries.",
"operator": "iAsk",
"respect": "No"
},
"ICC-Crawler": {
"description": "Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business.",
"frequency": "No information.",
@ -237,6 +230,13 @@
"operator": "Meta/Facebook",
"respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)"
},
"iaskspider/2.0": {
"description": "Used to provide answers to user queries.",
"frequency": "Unclear at this time.",
"function": "Crawls sites to provide answers to user queries.",
"operator": "iAsk",
"respect": "No"
},
"img2dataset": {
"description": "Downloads large sets of images into datasets for LLM training or other purposes.",
"frequency": "At the discretion of img2dataset users.",