mirror of
https://github.com/ai-robots-txt/ai.robots.txt.git
synced 2025-05-19 08:43:11 +00:00
57 lines
No EOL
2.1 KiB
Caddyfile
57 lines
No EOL
2.1 KiB
Caddyfile
@aibots {
|
|
header User-Agent "*AI2Bot*"
|
|
header User-Agent "*Ai2Bot-Dolma*"
|
|
header User-Agent "*aiHitBot*"
|
|
header User-Agent "*Amazonbot*"
|
|
header User-Agent "*anthropic-ai*"
|
|
header User-Agent "*Applebot*"
|
|
header User-Agent "*Applebot-Extended*"
|
|
header User-Agent "*Brightbot 1.0*"
|
|
header User-Agent "*Bytespider*"
|
|
header User-Agent "*CCBot*"
|
|
header User-Agent "*ChatGPT-User*"
|
|
header User-Agent "*Claude-Web*"
|
|
header User-Agent "*ClaudeBot*"
|
|
header User-Agent "*cohere-ai*"
|
|
header User-Agent "*cohere-training-data-crawler*"
|
|
header User-Agent "*Cotoyogi*"
|
|
header User-Agent "*Crawlspace*"
|
|
header User-Agent "*Diffbot*"
|
|
header User-Agent "*DuckAssistBot*"
|
|
header User-Agent "*FacebookBot*"
|
|
header User-Agent "*Factset_spyderbot*"
|
|
header User-Agent "*FirecrawlAgent*"
|
|
header User-Agent "*FriendlyCrawler*"
|
|
header User-Agent "*Google-Extended*"
|
|
header User-Agent "*GoogleOther*"
|
|
header User-Agent "*GoogleOther-Image*"
|
|
header User-Agent "*GoogleOther-Video*"
|
|
header User-Agent "*GPTBot*"
|
|
header User-Agent "*iaskspider/2.0*"
|
|
header User-Agent "*ICC-Crawler*"
|
|
header User-Agent "*ImagesiftBot*"
|
|
header User-Agent "*img2dataset*"
|
|
header User-Agent "*imgproxy*"
|
|
header User-Agent "*ISSCyberRiskCrawler*"
|
|
header User-Agent "*Kangaroo Bot*"
|
|
header User-Agent "*Meta-ExternalAgent*"
|
|
header User-Agent "*Meta-ExternalFetcher*"
|
|
header User-Agent "*NovaAct*"
|
|
header User-Agent "*OAI-SearchBot*"
|
|
header User-Agent "*omgili*"
|
|
header User-Agent "*omgilibot*"
|
|
header User-Agent "*Operator*"
|
|
header User-Agent "*PanguBot*"
|
|
header User-Agent "*Perplexity-User*"
|
|
header User-Agent "*PerplexityBot*"
|
|
header User-Agent "*PetalBot*"
|
|
header User-Agent "*Scrapy*"
|
|
header User-Agent "*SemrushBot-OCOB*"
|
|
header User-Agent "*SemrushBot-SWA*"
|
|
header User-Agent "*Sidetrade indexer bot*"
|
|
header User-Agent "*TikTokSpider*"
|
|
header User-Agent "*Timpibot*"
|
|
header User-Agent "*VelenPublicWebCrawler*"
|
|
header User-Agent "*Webzio-Extended*"
|
|
header User-Agent "*YouBot*"
|
|
} |