cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-19 00:26:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

chore: add Diffbot and scoopit

Browse source
This commit is contained in:
Cory Dransfeldt 2024-05-05 14:50:04 -07:00
parent 29e0799332
commit dea035365f
No known key found for this signature in database
5 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
edge-functions/block-bots.js
View file

@ -29,6 +29,7 @@ const botUas = [
'Claude-Web',
'cohere-ai',
'DataForSeoBot',
'Diffbot',
'FacebookBot',
'FriendlyCrawler',
'Google-Extended',
@ -44,6 +45,7 @@ const botUas = [
'peer39_crawler/1.0',
'PerplexityBot',
'PiplBot',
'scoop.it',
'Seekr',
'YouBot',
]