cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 19:13:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #73 from nisbet-hubbard/patch-8

Browse source 
Actually block Semrush’s AI tools
This commit is contained in:
Glyn Normington 2025-02-01 10:50:50 +00:00 committed by GitHub
commit f9e2c5810b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 11 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
robots.json
View file

@ -265,12 +265,19 @@
"operator": "[Zyte](https://www.zyte.com)", "operator": "[Zyte](https://www.zyte.com)",
"respect": "Unclear at this time." "respect": "Unclear at this time."
}, },
"SemrushBot": { "SemrushBot-OCOB": {
"operator": "[Semrush](https://www.semrush.com/)", "operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)", "respect": "[Yes](https://www.semrush.com/bot/)",
"function": "Scrapes data for use in LLM article-writing tool.", "function": "Crawls your site for ContentShake AI tool.",
"frequency": "Roughly once every 10 seconds.", "frequency": "Roughly once every 10 seconds.",
"description": "SemrushBot is a bot which, among other functions, scrapes data for use in ContentShake AI tool reports." "description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
},
"SemrushBot-SWA": {
"operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)",
"function": "Checks URLs on your site for SWA tool.",
"frequency": "Roughly once every 10 seconds.",
"description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
}, },
"Sidetrade indexer bot": { "Sidetrade indexer bot": {
"description": "AI product training.", "description": "AI product training.",
@ -307,4 +314,4 @@
"frequency": "No information.", "frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs." "description": "Retrieves data used for You.com web search engine and LLMs."
} }
} }