From 33a67ef5199282332f3a304ed50c3904c37d952e Mon Sep 17 00:00:00 2001 From: Cory Dransfeldt Date: Wed, 27 Mar 2024 11:40:19 -0700 Subject: [PATCH] chore: update robots + posts --- src/_data/json/robots.js | 1 + src/posts/2024/go-ahead-and-block-ai-web-crawlers.md | 1 + 2 files changed, 2 insertions(+) diff --git a/src/_data/json/robots.js b/src/_data/json/robots.js index 84e41314..d68f1125 100644 --- a/src/_data/json/robots.js +++ b/src/_data/json/robots.js @@ -30,6 +30,7 @@ export default [ 'Anthill', 'anthropic-ai', 'Apexoo', + 'Applebot', 'Aspiegel', 'Asterias', 'Atomseobot', diff --git a/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md b/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md index 883912c6..b4365ee1 100644 --- a/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md +++ b/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md @@ -41,6 +41,7 @@ Disallow: User-agent: AdsBot-Google User-agent: Amazonbot User-agent: anthropic-ai +User-agent: Applebot User-agent: AwarioRssBot User-agent: AwarioSmartBot User-agent: Bytespider