cdransf/coryd.dev-eleventy
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: clean up data files

Browse source
This commit is contained in:
Cory Dransfeldt 2024-05-05 14:31:07 -07:00
parent 77e6fe27e3
commit cce53c2656
No known key found for this signature in database
15 changed files with 98 additions and 129 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/_data/robots.js
View file

@ -1,5 +1,34 @@
import robots from './json/robots.js'
export default async function () {
return robots
return [
'AdsBot-Google',
'Amazonbot',
'anthropic-ai',
'Applebot',
'AwarioRssBot',
'AwarioSmartBot',
'Bytespider',
'CCBot',
'ChatGPT-User',
'ClaudeBot',
'Claude-Web',
'cohere-ai',
'DataForSeoBot',
'FacebookBot',
'FriendlyCrawler',
'Google-Extended',
'GoogleOther',
'GPTBot',
'img2dataset',
'ImagesiftBot',
'magpie-crawler',
'Meltwater',
'omgili',
'omgilibot',
'peer39_crawler',
'peer39_crawler/1.0',
'PerplexityBot',
'PiplBot',
'Seekr',
'YouBot',
]
}