cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 11:03:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Fix typo and trigger rerun of main job

Browse source
This commit is contained in:
Glyn Normington 2024-10-19 04:42:27 +01:00
parent 83c8603071
commit 38a388097c
2 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
code/dark_visitors.py
View file

@ -34,7 +34,7 @@ for section in soup.find_all("div", {"class": "agent-links-section"}):
default_values = {
"Unclear at this time.",
"No information. provided.",
"No information provided.",
"No information.",
"No explicit frequency provided."
}

12
robots.json
View file

@ -1,14 +1,14 @@
{
"AI2Bot": {
"description": "Explores 'certain domains' to find web content.",
"frequency": "No information. provided.",
"frequency": "No information provided.",
"function": "Content is used to train open language models.",
"operator": "[Ai2](https://allenai.org/crawler)",
"respect": "Yes"
},
"Ai2Bot-Dolma": {
"description": "Explores 'certain domains' to find web content.",
"frequency": "No information. provided.",
"frequency": "No information provided.",
"function": "Content is used to train open language models.",
"operator": "[Ai2](https://allenai.org/crawler)",
"respect": "Yes"
@ -17,14 +17,14 @@
"operator": "Amazon",
"respect": "Yes",
"function": "Service improvement and enabling answers for Alexa users.",
"frequency": "No information. provided.",
"frequency": "No information provided.",
"description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses."
},
"anthropic-ai": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time.",
"function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information. provided.",
"frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
},
"Applebot": {
@ -66,14 +66,14 @@
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "Unclear at this time.",
"function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information. provided.",
"frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
},
"ClaudeBot": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
"function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information. provided.",
"frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
},
"cohere-ai": {