cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-12 05:57:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Fix typo and trigger rerun of main job

Browse source
This commit is contained in:
Glyn Normington 2024-10-19 04:42:27 +01:00
parent 83c8603071
commit 38a388097c
2 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
code/dark_visitors.py
View file

@ -34,7 +34,7 @@ for section in soup.find_all("div", {"class": "agent-links-section"}):
default_values = { default_values = {
"Unclear at this time.", "Unclear at this time.",
"No information. provided.", "No information provided.",
"No information.", "No information.",
"No explicit frequency provided." "No explicit frequency provided."
} }

12
robots.json
View file

@ -1,14 +1,14 @@
{ {
"AI2Bot": { "AI2Bot": {
"description": "Explores 'certain domains' to find web content.", "description": "Explores 'certain domains' to find web content.",
"frequency": "No information. provided.", "frequency": "No information provided.",
"function": "Content is used to train open language models.", "function": "Content is used to train open language models.",
"operator": "[Ai2](https://allenai.org/crawler)", "operator": "[Ai2](https://allenai.org/crawler)",
"respect": "Yes" "respect": "Yes"
}, },
"Ai2Bot-Dolma": { "Ai2Bot-Dolma": {
"description": "Explores 'certain domains' to find web content.", "description": "Explores 'certain domains' to find web content.",
"frequency": "No information. provided.", "frequency": "No information provided.",
"function": "Content is used to train open language models.", "function": "Content is used to train open language models.",
"operator": "[Ai2](https://allenai.org/crawler)", "operator": "[Ai2](https://allenai.org/crawler)",
"respect": "Yes" "respect": "Yes"
@ -17,14 +17,14 @@
"operator": "Amazon", "operator": "Amazon",
"respect": "Yes", "respect": "Yes",
"function": "Service improvement and enabling answers for Alexa users.", "function": "Service improvement and enabling answers for Alexa users.",
"frequency": "No information. provided.", "frequency": "No information provided.",
"description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses." "description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses."
}, },
"anthropic-ai": { "anthropic-ai": {
"operator": "[Anthropic](https://www.anthropic.com)", "operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time.", "respect": "Unclear at this time.",
"function": "Scrapes data to train Anthropic's AI products.", "function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information. provided.", "frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic." "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
}, },
"Applebot": { "Applebot": {
@ -66,14 +66,14 @@
"operator": "[Anthropic](https://www.anthropic.com)", "operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time.", "respect": "Unclear at this time.",
"function": "Scrapes data to train Anthropic's AI products.", "function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information. provided.", "frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic." "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
}, },
"ClaudeBot": { "ClaudeBot": {
"operator": "[Anthropic](https://www.anthropic.com)", "operator": "[Anthropic](https://www.anthropic.com)",
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)", "respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
"function": "Scrapes data to train Anthropic's AI products.", "function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information. provided.", "frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic." "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
}, },
"cohere-ai": { "cohere-ai": {