diff --git a/src/_data/json/robots.js b/src/_data/json/robots.js index 84e41314..d68f1125 100644 --- a/src/_data/json/robots.js +++ b/src/_data/json/robots.js @@ -30,6 +30,7 @@ export default [ 'Anthill', 'anthropic-ai', 'Apexoo', + 'Applebot', 'Aspiegel', 'Asterias', 'Atomseobot', diff --git a/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md b/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md index 883912c6..b4365ee1 100644 --- a/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md +++ b/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md @@ -41,6 +41,7 @@ Disallow: User-agent: AdsBot-Google User-agent: Amazonbot User-agent: anthropic-ai +User-agent: Applebot User-agent: AwarioRssBot User-agent: AwarioSmartBot User-agent: Bytespider