cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-05-19 16:53:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Added Test and Docu

Browse source
This commit is contained in:
Julian Beittel 2025-04-23 09:41:18 +02:00
parent e519f9f94c
commit 1e92364c13
5 changed files with 19 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

2
code/robots.py
View file

@ -180,7 +180,7 @@ def json_to_nginx(robot_json):
def json_to_caddy(robot_json):
caddyfile = "@aibots {\n "
caddyfile += "\n ".join(f'header User-Agent "*{k}*"' for k in robot_json.keys())
caddyfile += f' header_regexp User-Agent "{list_to_pcre(robot_json.keys())}"'
caddyfile += "\n}"
return caddyfile

3
code/test_files/Caddyfile Normal file
View file

@ -0,0 +1,3 @@
@aibots {
header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)"
}

13
code/tests.py
View file

@ -4,7 +4,7 @@
import json
import unittest
from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx
from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx, json_to_caddy
class RobotsUnittestExtensions:
def loadJson(self, pathname):
@ -66,6 +66,17 @@ class TestRobotsNameCleaning(unittest.TestCase):
self.assertEqual(clean_robot_name("Perplexity‑User"), "Perplexity-User")
class TestCaddyfileGeneration(unittest.TestCase, RobotsUnittestExtensions):
maxDiff = 8192
def setUp(self):
self.robots_dict = self.loadJson("test_files/robots.json")
def test_caddyfile_generation(self):
robots_caddyfile = json_to_caddy(self.robots_dict)
self.assertEqualsFile("test_files/Caddyfile", robots_caddyfile)
if __name__ == "__main__":
import os
os.chdir(os.path.dirname(__file__))