mirror of
https://github.com/ai-robots-txt/ai.robots.txt.git
synced 2025-05-17 16:03:10 +00:00
Merge pull request #122 from ai-robots-txt/qualified-bot
chore(robots.json): adds QualifiedBot crawler
This commit is contained in:
commit
28e69e631b
1 changed files with 8 additions and 1 deletions
|
@ -335,6 +335,13 @@
|
|||
"operator": "[Huawei](https://huawei.com/)",
|
||||
"respect": "Yes"
|
||||
},
|
||||
"QualifiedBot": {
|
||||
"description": "Operated by Qualified as part of their suite of AI product offerings.",
|
||||
"frequency": "No explicit frequency provided.",
|
||||
"function": "Company offers AI agents and other related products; usage can be assumed to support said products.",
|
||||
"operator": "[Qualified](https://www.qualified.com)",
|
||||
"respect": "Unclear at this time.",
|
||||
},
|
||||
"Scrapy": {
|
||||
"description": "\"AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets.\"",
|
||||
"frequency": "No information.",
|
||||
|
@ -398,4 +405,4 @@
|
|||
"frequency": "No information.",
|
||||
"description": "Retrieves data used for You.com web search engine and LLMs."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue