cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-05-19 08:43:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'main' into main

Browse source
This commit is contained in:
Crazyroostereye 2025-04-30 22:13:51 +02:00 • committed by GitHub
commit 6baa7725b3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 160 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

11
code/robots.py
View file

@ -178,12 +178,19 @@ def json_to_nginx(robot_json):
config = f"if ($http_user_agent ~* \"{list_to_pcre(robot_json.keys())}\") {{\n return 403;\n}}"
return config
def json_to_caddy(robot_json):
caddyfile = "@aibots {\n "
caddyfile += f' header_regexp User-Agent "{list_to_pcre(robot_json.keys())}"'
caddyfile += "\n}"
return caddyfile
def json_to_haproxy(robots_json):
# Creates a source file for HAProxy. Follow instructions in the README to implement it.
txt = "\n".join(f"{k}" for k in robots_json.keys())
return txt
def update_file_if_changed(file_name, converter):
"""Update files if newer content is available and log the (in)actions."""
@ -217,6 +224,10 @@ def conversions():
update_file_if_changed(
file_name="./Caddyfile",
converter=json_to_caddy
update_file_if_changed(
file_name="./haproxy-block-ai-bots.txt",
converter=json_to_haproxy,
)