cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 11:03:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Added bing bots (see https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)

Browse source
This commit is contained in:
theRealProHacker 2025-02-28 13:31:02 +01:00 committed by GitHub
parent 33288d3e23
commit 3be8edc5e4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 21 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
robots.json
View file

@ -1,4 +1,11 @@
{
"adidxbot/2.0": {
"operator": "[Bing](bing.com)",
"respect": "[Yes](https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)",
"function": "Crawler",
"frequency": "Unknown",
"description": "AdIdxBot is the crawler used by Bing Ads. AdIdxBot crawls ads and follows the websites from those ads for quality control. Just like Bingbot, AdIdxBot has both “desktop” and “mobile” variants."
},
"AI2Bot": {
"operator": "[Ai2](https://allenai.org/crawler)",
"respect": "Yes",
@ -41,6 +48,13 @@
"frequency": "Unclear at this time.",
"description": "Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools."
},
"bingbot/2.0": {
"operator": "[Bing](bing.com)",
"respect": "[Yes](https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)",
"function": "Crawler",
"frequency": "Days",
"description": "Bingbot is our standard crawler and handles most of our crawling needs each day. Bingbot uses different types of user agent strings."
},
"Brightbot 1.0": {
"operator": "Browsing.ai",
"respect": "Unclear at this time.",
@ -223,6 +237,13 @@
"frequency": "Unclear at this time.",
"description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
},
"MicrosoftPreview/2.0": {
"operator": "[Microsoft](microsoft.com)",
"respect": "[Yes](https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0)",
"function": "Crawler",
"frequency": "Days",
"description": "MicrosoftPreview generates page snapshots for Microsoft products. Note that MicrosoftPreview has \"desktop\" and \"mobile\" variants."
},
"OAI-SearchBot": {
"operator": "[OpenAI](https://openai.com)",
"respect": "[Yes](https://platform.openai.com/docs/bots)",