mirror of
https://github.com/ai-robots-txt/ai.robots.txt.git
synced 2025-05-17 16:03:10 +00:00
Tests for HAProxy file added.
This commit is contained in:
parent
66da70905f
commit
a4a9f2ac2b
2 changed files with 58 additions and 1 deletions
47
code/test_files/haproxy-block-ai-bots.txt
Normal file
47
code/test_files/haproxy-block-ai-bots.txt
Normal file
|
@ -0,0 +1,47 @@
|
|||
AI2Bot
|
||||
Ai2Bot-Dolma
|
||||
Amazonbot
|
||||
anthropic-ai
|
||||
Applebot
|
||||
Applebot-Extended
|
||||
Bytespider
|
||||
CCBot
|
||||
ChatGPT-User
|
||||
Claude-Web
|
||||
ClaudeBot
|
||||
cohere-ai
|
||||
Diffbot
|
||||
FacebookBot
|
||||
facebookexternalhit
|
||||
FriendlyCrawler
|
||||
Google-Extended
|
||||
GoogleOther
|
||||
GoogleOther-Image
|
||||
GoogleOther-Video
|
||||
GPTBot
|
||||
iaskspider/2.0
|
||||
ICC-Crawler
|
||||
ImagesiftBot
|
||||
img2dataset
|
||||
ISSCyberRiskCrawler
|
||||
Kangaroo Bot
|
||||
Meta-ExternalAgent
|
||||
Meta-ExternalFetcher
|
||||
OAI-SearchBot
|
||||
omgili
|
||||
omgilibot
|
||||
Perplexity-User
|
||||
PerplexityBot
|
||||
PetalBot
|
||||
Scrapy
|
||||
Sidetrade indexer bot
|
||||
Timpibot
|
||||
VelenPublicWebCrawler
|
||||
Webzio-Extended
|
||||
YouBot
|
||||
crawler.with.dots
|
||||
star***crawler
|
||||
Is this a crawler?
|
||||
a[mazing]{42}(robot)
|
||||
2^32$
|
||||
curl|sudo bash
|
|
@ -4,7 +4,7 @@
|
|||
import json
|
||||
import unittest
|
||||
|
||||
from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx
|
||||
from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx, json_to_haproxy
|
||||
|
||||
class RobotsUnittestExtensions:
|
||||
def loadJson(self, pathname):
|
||||
|
@ -60,6 +60,16 @@ class TestNginxConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
|
|||
robots_nginx = json_to_nginx(self.robots_dict)
|
||||
self.assertEqualsFile("test_files/nginx-block-ai-bots.conf", robots_nginx)
|
||||
|
||||
class TestHaproxyConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
|
||||
maxDiff = 8192
|
||||
|
||||
def setUp(self):
|
||||
self.robots_dict = self.loadJson("test_files/robots.json")
|
||||
|
||||
def test_haproxy_generation(self):
|
||||
robots_haproxy = json_to_haproxy(self.robots_dict)
|
||||
self.assertEqualsFile("test_files/haproxy-block-ai-bots.txt", robots_haproxy)
|
||||
|
||||
class TestRobotsNameCleaning(unittest.TestCase):
|
||||
def test_clean_name(self):
|
||||
from robots import clean_robot_name
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue