cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-12 05:57:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Adding dataprovider.com's spider

Browse source
This commit is contained in:
Gregory Hammond 2024-09-14 23:24:43 +00:00 committed by GitHub
parent 6b8d7f5890
commit 1abf68b107
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
robots.json
View file

@ -69,6 +69,13 @@
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time."
},
"Dataprovider": {
"description": "We’ve summarized the web, so you can find the exact information you need, whether it be technology trends, hosting, security risks, business information, website traffic or content.",
"frequency": "Unclear at this time.",
"function": "Aggregates structured web data for searching and AI model training.",
"operator": "[Dataprovider.com](https://www.dataprovider.com/)",
"respect": "[Yes](https://www.dataprovider.com/spider/)."
},
"Diffbot": {
"description": "Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training.",
"frequency": "Unclear at this time.",
@ -258,4 +265,4 @@
"operator": "[Webz.io](https://webz.io/)",
"respect": "[Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html)"
}
}
}