cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 19:13:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chore: drop in additional data

Browse source
This commit is contained in:
Cory Dransfeldt 2024-08-01 15:33:07 -07:00
parent f18f0d99b9
commit b20dfec1e4
No known key found for this signature in database
1 changed files with 14 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
robots.json
View file

@ -76,6 +76,13 @@
"frequency": "Up to 1 page per second",
"description": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically."
},
"facebookexternalhit": {
"operator": "Meta\/Facebook",
"respect": "[Yes](https:\/\/developers.facebook.com\/docs\/sharing\/bot\/)",
"function": "No information.",
"frequency": "Unclear at this time.",
"description": "Unclear at this time."
},
"Google-Extended": {
"operator": "Google",
"respect": "[Yes](https:\/\/developers.google.com\/search\/docs\/crawling-indexing\/overview-google-crawlers)",
@ -153,6 +160,13 @@
"frequency": "Takes action based on user prompts.",
"description": "Operated by Perplexity to obtain results in response to user queries."
},
"PetalBot": {
"operator": "[Huawei](https:\/\/huawei.com\/)",
"respect": "Yes",
"function": "Used to provide recommendations in Hauwei assistant and AI search services.",
"frequency": "No explicit frequency provided.",
"description": "Operated by Huawei to provide search and AI assistant services."
},
"Scrapy": {
"operator": "[Zyte](https:\/\/www.zyte.com)",
"respect": "Unclear at this time.",