cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 11:03:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2024-11-24 00:57:05 +00:00
parent 58985737e7
commit 37065f9118
1 changed files with 7 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
robots.json
View file

@ -223,6 +223,13 @@
"operator": "[Webz.io](https://webz.io/)",
"respect": "[Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html)"
},
"PanguBot": {
"operator": "the Chinese company Huawei",
"respect": "Unclear at this time.",
"function": "AI Data Scrapers",
"frequency": "Unclear at this time.",
"description": "PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot"
},
"PerplexityBot": {
"operator": "[Perplexity](https://www.perplexity.ai/)",
"respect": "[No](https://www.macstories.net/stories/wired-confirms-perplexity-is-bypassing-efforts-by-websites-to-block-its-web-crawler/)",