cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 11:03:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #67 from Nightfirecat/semrushbot

Browse source 
Block SemrushBot
This commit is contained in:
Cory Dransfeldt 2025-01-06 12:51:43 -08:00 committed by GitHub
commit 565dca3dc0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 7 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
robots.json
View file

@ -258,6 +258,13 @@
"operator": "[Zyte](https://www.zyte.com)",
"respect": "Unclear at this time."
},
"SemrushBot": {
"operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)",
"function": "Scrapes data for use in LLM article-writing tool.",
"frequency": "Roughly once every 10 seconds.",
"description": "SemrushBot is a bot which, among other functions, scrapes data for use in ContentShake AI tool reports."
},
"Sidetrade indexer bot": {
"description": "AI product training.",
"frequency": "No information.",