diff --git a/src/_data/json/robots.js b/src/_data/json/robots.js index b26da74c..048a407b 100644 --- a/src/_data/json/robots.js +++ b/src/_data/json/robots.js @@ -293,6 +293,7 @@ export default [ 'Meanpathbot', 'Mediatoolkitbot', 'MegaIndex.ru', + 'Meltwater', 'Metauri', 'MicroMessenger', 'Microsoft Data Access', @@ -642,6 +643,7 @@ export default [ 'satoristudio.net', 'scalaj-http', 'scan.lol', + 'Seekr', 'seobility', 'seocompany.store', 'seoscanners', diff --git a/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md b/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md index bfc72b60..74287d61 100644 --- a/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md +++ b/src/posts/2024/go-ahead-and-block-ai-web-crawlers.md @@ -53,14 +53,17 @@ User-agent: cohere-ai User-agent: DataForSeoBot User-agent: FacebookBot User-agent: Google-Extended +User-agent: GoogleOther User-agent: GPTBot User-agent: ImagesiftBot User-agent: magpie-crawler +User-agent: Meltwater User-agent: omgili User-agent: omgilibot User-agent: peer39_crawler User-agent: peer39_crawler/1.0 User-agent: PerplexityBot +User-agent: Seekr User-agent: YouBot Disallow: / ```