diff --git a/robots.txt b/robots.txt index d2ca026..004d5b8 100644 --- a/robots.txt +++ b/robots.txt @@ -2,8 +2,6 @@ User-agent: AdsBot-Google User-agent: Amazonbot User-agent: anthropic-ai User-agent: Applebot-Extended -User-agent: AwarioRssBot -User-agent: AwarioSmartBot User-agent: Bytespider User-agent: CCBot User-agent: ChatGPT-User @@ -17,14 +15,10 @@ User-agent: Google-Extended User-agent: GoogleOther User-agent: GPTBot User-agent: img2dataset -User-agent: magpie-crawler -User-agent: Meltwater User-agent: omgili User-agent: omgilibot User-agent: peer39_crawler User-agent: peer39_crawler/1.0 User-agent: PerplexityBot -User-agent: PiplBot -User-agent: scoop.it User-agent: YouBot Disallow: /