mirror of
https://github.com/ai-robots-txt/ai.robots.txt.git
synced 2025-04-04 19:13:57 +00:00
Daily update from Dark Visitors
This commit is contained in:
dark-visitors
parent
04f630f7f8
commit
faf81efb12
1 changed files with 137 additions and 137 deletions
274
robots.json
274
robots.json
|
|
@ -14,72 +14,93 @@
|
|||
"respect": "Yes"
|
||||
},
|
||||
"Amazonbot": {
|
||||
"description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses.",
|
||||
"frequency": "No information. provided.",
|
||||
"function": "Service improvement and enabling answers for Alexa users.",
|
||||
"operator": "Amazon",
|
||||
"respect": "Yes"
|
||||
"respect": "Yes",
|
||||
"function": "Service improvement and enabling answers for Alexa users.",
|
||||
"frequency": "No information. provided.",
|
||||
"description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses."
|
||||
},
|
||||
"anthropic-ai": {
|
||||
"operator": "[Anthropic](https://www.anthropic.com)",
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "Scrapes data to train Anthropic's AI products.",
|
||||
"frequency": "No information. provided.",
|
||||
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
|
||||
},
|
||||
"Applebot": {
|
||||
"description": "Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://darkvisitors.com/agents/agents/applebot",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "AI Search Crawlers",
|
||||
"operator": "Unclear at this time.",
|
||||
"respect": "Unclear at this time."
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "AI Search Crawlers",
|
||||
"frequency": "Unclear at this time.",
|
||||
"description": "Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://darkvisitors.com/agents/agents/applebot"
|
||||
},
|
||||
"Applebot-Extended": {
|
||||
"description": "Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others.",
|
||||
"operator": "[Apple](https://support.apple.com/en-us/119829#datausage)",
|
||||
"respect": "Yes"
|
||||
"respect": "Yes",
|
||||
"function": "Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"description": "Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools."
|
||||
},
|
||||
"Bytespider": {
|
||||
"description": "Downloads data to train LLMS, including ChatGPT competitors.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "LLM training.",
|
||||
"operator": "ByteDance",
|
||||
"respect": "No"
|
||||
"respect": "No",
|
||||
"function": "LLM training.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"description": "Downloads data to train LLMS, including ChatGPT competitors."
|
||||
},
|
||||
"CCBot": {
|
||||
"description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers).",
|
||||
"frequency": "Monthly at present.",
|
||||
"function": "Provides open crawl dataset, used for many purposes, including Machine Learning/AI.",
|
||||
"operator": "[Common Crawl Foundation](https://commoncrawl.org)",
|
||||
"respect": "[Yes](https://commoncrawl.org/ccbot)"
|
||||
"respect": "[Yes](https://commoncrawl.org/ccbot)",
|
||||
"function": "Provides open crawl dataset, used for many purposes, including Machine Learning/AI.",
|
||||
"frequency": "Monthly at present.",
|
||||
"description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers)."
|
||||
},
|
||||
"ChatGPT-User": {
|
||||
"description": "Used by plugins in ChatGPT to answer queries based on user input.",
|
||||
"frequency": "Only when prompted by a user.",
|
||||
"function": "Takes action based on user prompts.",
|
||||
"operator": "[OpenAI](https://openai.com)",
|
||||
"respect": "Yes"
|
||||
"respect": "Yes",
|
||||
"function": "Takes action based on user prompts.",
|
||||
"frequency": "Only when prompted by a user.",
|
||||
"description": "Used by plugins in ChatGPT to answer queries based on user input."
|
||||
},
|
||||
"Claude-Web": {
|
||||
"description": "Scrapes data to train LLMs and AI products offered by Anthropic.",
|
||||
"frequency": "No information. provided.",
|
||||
"function": "Scrapes data to train Anthropic's AI products.",
|
||||
"operator": "[Anthropic](https://www.anthropic.com)",
|
||||
"respect": "Unclear at this time."
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "Scrapes data to train Anthropic's AI products.",
|
||||
"frequency": "No information. provided.",
|
||||
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
|
||||
},
|
||||
"ClaudeBot": {
|
||||
"description": "Scrapes data to train LLMs and AI products offered by Anthropic.",
|
||||
"frequency": "No information. provided.",
|
||||
"function": "Scrapes data to train Anthropic's AI products.",
|
||||
"operator": "[Anthropic](https://www.anthropic.com)",
|
||||
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)"
|
||||
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
|
||||
"function": "Scrapes data to train Anthropic's AI products.",
|
||||
"frequency": "No information. provided.",
|
||||
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
|
||||
},
|
||||
"cohere-ai": {
|
||||
"operator": "[Cohere](https://cohere.com)",
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "Retrieves data to provide responses to user-initiated prompts.",
|
||||
"frequency": "Takes action based on user prompts.",
|
||||
"description": "Retrieves data based on user prompts."
|
||||
},
|
||||
"Diffbot": {
|
||||
"description": "Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "Aggregates structured web data for monitoring and AI model training.",
|
||||
"operator": "[Diffbot](https://www.diffbot.com/)",
|
||||
"respect": "At the discretion of Diffbot users."
|
||||
"respect": "At the discretion of Diffbot users.",
|
||||
"function": "Aggregates structured web data for monitoring and AI model training.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"description": "Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training."
|
||||
},
|
||||
"FacebookBot": {
|
||||
"description": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically.",
|
||||
"frequency": "Up to 1 page per second",
|
||||
"operator": "Meta/Facebook",
|
||||
"respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)",
|
||||
"function": "Training language models",
|
||||
"frequency": "Up to 1 page per second",
|
||||
"description": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically."
|
||||
},
|
||||
"facebookexternalhit": {
|
||||
"description": "Unclear at this time.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "No information.",
|
||||
"operator": "Meta/Facebook",
|
||||
"respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)"
|
||||
},
|
||||
|
|
@ -90,19 +111,12 @@
|
|||
"operator": "Unknown",
|
||||
"respect": "[Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler)"
|
||||
},
|
||||
"GPTBot": {
|
||||
"description": "Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies.",
|
||||
"frequency": "No information.",
|
||||
"function": "Scrapes data to train OpenAI's products.",
|
||||
"operator": "[OpenAI](https://openai.com)",
|
||||
"respect": "Yes"
|
||||
},
|
||||
"Google-Extended": {
|
||||
"description": "Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search.",
|
||||
"frequency": "No information.",
|
||||
"function": "LLM training.",
|
||||
"operator": "Google",
|
||||
"respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
|
||||
"respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
|
||||
"function": "LLM training.",
|
||||
"frequency": "No information.",
|
||||
"description": "Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search."
|
||||
},
|
||||
"GoogleOther": {
|
||||
"description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
|
||||
|
|
@ -125,6 +139,20 @@
|
|||
"operator": "Google",
|
||||
"respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
|
||||
},
|
||||
"GPTBot": {
|
||||
"operator": "[OpenAI](https://openai.com)",
|
||||
"respect": "Yes",
|
||||
"function": "Scrapes data to train OpenAI's products.",
|
||||
"frequency": "No information.",
|
||||
"description": "Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies."
|
||||
},
|
||||
"iaskspider/2.0": {
|
||||
"description": "Used to provide answers to user queries.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "Crawls sites to provide answers to user queries.",
|
||||
"operator": "iAsk",
|
||||
"respect": "No"
|
||||
},
|
||||
"ICC-Crawler": {
|
||||
"description": "Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business.",
|
||||
"frequency": "No information.",
|
||||
|
|
@ -132,13 +160,6 @@
|
|||
"operator": "[NICT](https://nict.go.jp)",
|
||||
"respect": "Yes"
|
||||
},
|
||||
"ISSCyberRiskCrawler": {
|
||||
"description": "Used to train machine learning based models to quantify cyber risk.",
|
||||
"frequency": "No information.",
|
||||
"function": "Scrapes data to train machine learning models.",
|
||||
"operator": "[ISS-Corporate](https://iss-cyber.com)",
|
||||
"respect": "No"
|
||||
},
|
||||
"ImagesiftBot": {
|
||||
"description": "Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Our web intelligence products use this index to enable search and retrieval of similar images.",
|
||||
"frequency": "No information.",
|
||||
|
|
@ -146,40 +167,68 @@
|
|||
"operator": "[ImageSift](https://imagesift.com)",
|
||||
"respect": "[Yes](https://imagesift.com/about)"
|
||||
},
|
||||
"Kangaroo Bot": {
|
||||
"description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "AI Data Scrapers",
|
||||
"operator": "Unclear at this time.",
|
||||
"img2dataset": {
|
||||
"description": "Downloads large sets of images into datasets for LLM training or other purposes.",
|
||||
"frequency": "At the discretion of img2dataset users.",
|
||||
"function": "Scrapes images for use in LLMs.",
|
||||
"operator": "[img2dataset](https://github.com/rom1504/img2dataset)",
|
||||
"respect": "Unclear at this time."
|
||||
},
|
||||
"ISSCyberRiskCrawler": {
|
||||
"description": "Used to train machine learning based models to quantify cyber risk.",
|
||||
"frequency": "No information.",
|
||||
"function": "Scrapes data to train machine learning models.",
|
||||
"operator": "[ISS-Corporate](https://iss-cyber.com)",
|
||||
"respect": "No"
|
||||
},
|
||||
"Kangaroo Bot": {
|
||||
"operator": "Unclear at this time.",
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "AI Data Scrapers",
|
||||
"frequency": "Unclear at this time.",
|
||||
"description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot"
|
||||
},
|
||||
"Meta-ExternalAgent": {
|
||||
"description": "\"The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.\"",
|
||||
"frequency": "No information.",
|
||||
"function": "Used to train models and improve products.",
|
||||
"operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)",
|
||||
"respect": "Yes."
|
||||
"respect": "Yes.",
|
||||
"function": "Used to train models and improve products.",
|
||||
"frequency": "No information.",
|
||||
"description": "\"The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.\""
|
||||
},
|
||||
"Meta-ExternalFetcher": {
|
||||
"description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "AI Assistants",
|
||||
"operator": "Unclear at this time.",
|
||||
"respect": "Unclear at this time."
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "AI Assistants",
|
||||
"frequency": "Unclear at this time.",
|
||||
"description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
|
||||
},
|
||||
"OAI-SearchBot": {
|
||||
"description": "Crawls sites to surface as results in SearchGPT.",
|
||||
"frequency": "No information.",
|
||||
"function": "Search result generation.",
|
||||
"operator": "[OpenAI](https://openai.com)",
|
||||
"respect": "[Yes](https://platform.openai.com/docs/bots)"
|
||||
"respect": "[Yes](https://platform.openai.com/docs/bots)",
|
||||
"function": "Search result generation.",
|
||||
"frequency": "No information.",
|
||||
"description": "Crawls sites to surface as results in SearchGPT."
|
||||
},
|
||||
"omgili": {
|
||||
"operator": "[Webz.io](https://webz.io/)",
|
||||
"respect": "[Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/)",
|
||||
"function": "Data is sold.",
|
||||
"frequency": "No information.",
|
||||
"description": "Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training."
|
||||
},
|
||||
"omgilibot": {
|
||||
"description": "Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io.",
|
||||
"frequency": "No information.",
|
||||
"function": "Data is sold.",
|
||||
"operator": "[Webz.io](https://webz.io/)",
|
||||
"respect": "[Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html)"
|
||||
},
|
||||
"PerplexityBot": {
|
||||
"description": "Operated by Perplexity to obtain results in response to user queries.",
|
||||
"frequency": "Takes action based on user prompts.",
|
||||
"function": "Used to answer queries at the request of users.",
|
||||
"operator": "[Perplexity](https://www.perplexity.ai/)",
|
||||
"respect": "[No](https://www.macstories.net/stories/wired-confirms-perplexity-is-bypassing-efforts-by-websites-to-block-its-web-crawler/)"
|
||||
"respect": "[No](https://www.macstories.net/stories/wired-confirms-perplexity-is-bypassing-efforts-by-websites-to-block-its-web-crawler/)",
|
||||
"function": "Used to answer queries at the request of users.",
|
||||
"frequency": "Takes action based on user prompts.",
|
||||
"description": "Operated by Perplexity to obtain results in response to user queries."
|
||||
},
|
||||
"PetalBot": {
|
||||
"description": "Operated by Huawei to provide search and AI assistant services.",
|
||||
|
|
@ -203,11 +252,11 @@
|
|||
"respect": "Unclear at this time."
|
||||
},
|
||||
"Timpibot": {
|
||||
"description": "Makes data available for training AI models.",
|
||||
"frequency": "No information.",
|
||||
"function": "Scrapes data for use in training LLMs.",
|
||||
"operator": "[Timpi](https://timpi.io)",
|
||||
"respect": "Unclear at this time."
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "Scrapes data for use in training LLMs.",
|
||||
"frequency": "No information.",
|
||||
"description": "Makes data available for training AI models."
|
||||
},
|
||||
"VelenPublicWebCrawler": {
|
||||
"description": "\"Our goal with this crawler is to build business datasets and machine learning models to better understand the web.\"",
|
||||
|
|
@ -217,66 +266,17 @@
|
|||
"respect": "[Yes](https://velen.io)"
|
||||
},
|
||||
"Webzio-Extended": {
|
||||
"description": "Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "AI Data Scrapers",
|
||||
"operator": "Unclear at this time.",
|
||||
"respect": "Unclear at this time."
|
||||
"respect": "Unclear at this time.",
|
||||
"function": "AI Data Scrapers",
|
||||
"frequency": "Unclear at this time.",
|
||||
"description": "Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended"
|
||||
},
|
||||
"YouBot": {
|
||||
"description": "Retrieves data used for You.com web search engine and LLMs.",
|
||||
"frequency": "No information.",
|
||||
"function": "Scrapes data for search engine and LLMs.",
|
||||
"operator": "[You](https://about.you.com/youchat/)",
|
||||
"respect": "[Yes](https://about.you.com/youbot/)"
|
||||
},
|
||||
"anthropic-ai": {
|
||||
"description": "Scrapes data to train LLMs and AI products offered by Anthropic.",
|
||||
"frequency": "No information. provided.",
|
||||
"function": "Scrapes data to train Anthropic's AI products.",
|
||||
"operator": "[Anthropic](https://www.anthropic.com)",
|
||||
"respect": "Unclear at this time."
|
||||
},
|
||||
"cohere-ai": {
|
||||
"description": "Retrieves data based on user prompts.",
|
||||
"frequency": "Takes action based on user prompts.",
|
||||
"function": "Retrieves data to provide responses to user-initiated prompts.",
|
||||
"operator": "[Cohere](https://cohere.com)",
|
||||
"respect": "Unclear at this time."
|
||||
},
|
||||
"facebookexternalhit": {
|
||||
"description": "Unclear at this time.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "No information.",
|
||||
"operator": "Meta/Facebook",
|
||||
"respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)"
|
||||
},
|
||||
"iaskspider/2.0": {
|
||||
"description": "Used to provide answers to user queries.",
|
||||
"frequency": "Unclear at this time.",
|
||||
"function": "Crawls sites to provide answers to user queries.",
|
||||
"operator": "iAsk",
|
||||
"respect": "No"
|
||||
},
|
||||
"img2dataset": {
|
||||
"description": "Downloads large sets of images into datasets for LLM training or other purposes.",
|
||||
"frequency": "At the discretion of img2dataset users.",
|
||||
"function": "Scrapes images for use in LLMs.",
|
||||
"operator": "[img2dataset](https://github.com/rom1504/img2dataset)",
|
||||
"respect": "Unclear at this time."
|
||||
},
|
||||
"omgili": {
|
||||
"description": "Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training.",
|
||||
"respect": "[Yes](https://about.you.com/youbot/)",
|
||||
"function": "Scrapes data for search engine and LLMs.",
|
||||
"frequency": "No information.",
|
||||
"function": "Data is sold.",
|
||||
"operator": "[Webz.io](https://webz.io/)",
|
||||
"respect": "[Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/)"
|
||||
},
|
||||
"omgilibot": {
|
||||
"description": "Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io.",
|
||||
"frequency": "No information.",
|
||||
"function": "Data is sold.",
|
||||
"operator": "[Webz.io](https://webz.io/)",
|
||||
"respect": "[Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html)"
|
||||
"description": "Retrieves data used for You.com web search engine and LLMs."
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue