cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-04-19 00:26:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

call main after update

Browse source
This commit is contained in:
Chenghao Mou 2024-08-07 11:31:58 +01:00
parent cebf809391
commit 944bee0f56
4 changed files with 50 additions and 5406 deletions
Download patch file Download diff file Expand all files Collapse all files

2
code/dark_visitors.py
View file

@ -26,6 +26,8 @@ to_include = [
for section in soup.find_all("div", {"class": "agent-links-section"}):
category = section.find("h2").get_text()
if category not in to_include:
continue
for agent in section.find_all("a", href=True):
name = agent.find("div", {"class": "agent-name"}).get_text().strip()
desc = agent.find("p").get_text().strip()