cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-04 11:03:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Added SemrushBot/7~bl

Browse source
This commit is contained in:
theRealProHacker 2025-02-28 13:20:19 +01:00 committed by GitHub
parent 5e7c3c432f
commit 33288d3e23
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
robots.json
View file

@ -286,6 +286,13 @@
"frequency": "Roughly once every 10 seconds.",
"description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
},
"SemrushBot/7~bl": {
"operator": "[Semrush](https://www.semrush.com/)",
"respect": "Yes",
"function": "Regularly crawls sites",
"frequency": "Every 2 days",
"description": "N/A"
},
"Sidetrade indexer bot": {
"description": "AI product training.",
"frequency": "No information.",
@ -321,4 +328,4 @@
"frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs."
}
}
}