mirror of
https://github.com/ai-robots-txt/ai.robots.txt.git
synced 2025-05-17 16:03:10 +00:00
Tests for HAProxy file added.
This commit is contained in:
parent
66da70905f
commit
a4a9f2ac2b
2 changed files with 58 additions and 1 deletions
47
code/test_files/haproxy-block-ai-bots.txt
Normal file
47
code/test_files/haproxy-block-ai-bots.txt
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
AI2Bot
|
||||||
|
Ai2Bot-Dolma
|
||||||
|
Amazonbot
|
||||||
|
anthropic-ai
|
||||||
|
Applebot
|
||||||
|
Applebot-Extended
|
||||||
|
Bytespider
|
||||||
|
CCBot
|
||||||
|
ChatGPT-User
|
||||||
|
Claude-Web
|
||||||
|
ClaudeBot
|
||||||
|
cohere-ai
|
||||||
|
Diffbot
|
||||||
|
FacebookBot
|
||||||
|
facebookexternalhit
|
||||||
|
FriendlyCrawler
|
||||||
|
Google-Extended
|
||||||
|
GoogleOther
|
||||||
|
GoogleOther-Image
|
||||||
|
GoogleOther-Video
|
||||||
|
GPTBot
|
||||||
|
iaskspider/2.0
|
||||||
|
ICC-Crawler
|
||||||
|
ImagesiftBot
|
||||||
|
img2dataset
|
||||||
|
ISSCyberRiskCrawler
|
||||||
|
Kangaroo Bot
|
||||||
|
Meta-ExternalAgent
|
||||||
|
Meta-ExternalFetcher
|
||||||
|
OAI-SearchBot
|
||||||
|
omgili
|
||||||
|
omgilibot
|
||||||
|
Perplexity-User
|
||||||
|
PerplexityBot
|
||||||
|
PetalBot
|
||||||
|
Scrapy
|
||||||
|
Sidetrade indexer bot
|
||||||
|
Timpibot
|
||||||
|
VelenPublicWebCrawler
|
||||||
|
Webzio-Extended
|
||||||
|
YouBot
|
||||||
|
crawler.with.dots
|
||||||
|
star***crawler
|
||||||
|
Is this a crawler?
|
||||||
|
a[mazing]{42}(robot)
|
||||||
|
2^32$
|
||||||
|
curl|sudo bash
|
|
@ -4,7 +4,7 @@
|
||||||
import json
|
import json
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx
|
from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx, json_to_haproxy
|
||||||
|
|
||||||
class RobotsUnittestExtensions:
|
class RobotsUnittestExtensions:
|
||||||
def loadJson(self, pathname):
|
def loadJson(self, pathname):
|
||||||
|
@ -60,6 +60,16 @@ class TestNginxConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
|
||||||
robots_nginx = json_to_nginx(self.robots_dict)
|
robots_nginx = json_to_nginx(self.robots_dict)
|
||||||
self.assertEqualsFile("test_files/nginx-block-ai-bots.conf", robots_nginx)
|
self.assertEqualsFile("test_files/nginx-block-ai-bots.conf", robots_nginx)
|
||||||
|
|
||||||
|
class TestHaproxyConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
|
||||||
|
maxDiff = 8192
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.robots_dict = self.loadJson("test_files/robots.json")
|
||||||
|
|
||||||
|
def test_haproxy_generation(self):
|
||||||
|
robots_haproxy = json_to_haproxy(self.robots_dict)
|
||||||
|
self.assertEqualsFile("test_files/haproxy-block-ai-bots.txt", robots_haproxy)
|
||||||
|
|
||||||
class TestRobotsNameCleaning(unittest.TestCase):
|
class TestRobotsNameCleaning(unittest.TestCase):
|
||||||
def test_clean_name(self):
|
def test_clean_name(self):
|
||||||
from robots import clean_robot_name
|
from robots import clean_robot_name
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue