Update from Dark Visitors

chore(robots.json): adds imgproxy crawler

.github/FUNDING.yml

@@ -1,14 +0,0 @@
-# These are supported funding model platforms
-
-github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
-polar: # Replace with a single Polar username
-buy_me_a_coffee: cory
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/ai_robots_update.yml

@@ -0,0 +1,31 @@
+name: Updates for AI robots files
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  dark-visitors:
+    runs-on: ubuntu-latest
+    name: dark-visitors
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - run: |
+          pip install beautifulsoup4 requests
+          git config --global user.name "dark-visitors"
+          git config --global user.email "dark-visitors@users.noreply.github.com"
+          echo "Updating robots.json with data from darkvisitor.com ..."
+          python code/robots.py --update
+          echo "... done."
+          git --no-pager diff
+          git add -A
+          git diff --quiet && git diff --staged --quiet || (git commit -m "Update from Dark Visitors" && git push)
+        shell: bash
+  convert:
+    name: convert
+    needs: dark-visitors
+    uses: ./.github/workflows/main.yml
+    secrets: inherit
+    with:
+      message: "Update from Dark Visitors"

.github/workflows/main.yml

@@ -0,0 +1,48 @@
+on:
+  workflow_call:
+    inputs:
+      message:
+        type: string
+        required: true
+        description: The message to commit
+  push:
+    paths:
+      - 'robots.json'
+      - '.github/workflows/**'
+      - 'code/**'
+    branches:
+      - "main"
+
+jobs:
+  ai-robots-txt:
+    runs-on: ubuntu-latest
+    name: ai-robots-txt
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - run: |
+          pip install beautifulsoup4
+          git config --global user.name "ai.robots.txt"
+          git config --global user.email "ai.robots.txt@users.noreply.github.com"
+          git log -1
+          git status
+          echo "Updating robots.txt and table-of-bot-metrics.md if necessary ..."
+          python code/robots.py --convert
+          echo "... done."
+          git --no-pager diff
+          git add -A
+          if [ -z "$(git diff --staged)" ]; then
+            # To have the action run successfully, if no changes are staged, we
+            # manually skip the later commits because they fail with exit code 1
+            # and this would then display as a failure for the Action.
+            echo "No staged changes to commit. Skipping commit and push."
+            exit 0
+          fi
+          if [ -n "${{ inputs.message }}" ]; then
+            git commit -m "${{ inputs.message }}"
+          else
+            git commit -m "${{ github.event.head_commit.message }}"
+          fi
+          git push
+        shell: bash

.github/workflows/run-tests.yml

@@ -0,0 +1,21 @@
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Install dependencies
+        run: |
+          pip install -U requests beautifulsoup4
+      - name: Run tests
+        run: |
+          code/tests.py

.github/workflows/upload-robots-txt-file-to-release.yml

@@ -0,0 +1,29 @@
+---
+
+name: "Upload robots.txt file to release"
+run-name: "Upload robots.txt file to release"
+
+on:
+  release:
+    types:
+      - published
+
+permissions:
+  contents: write
+
+jobs:
+  upload-robots-txt-file-to-release:
+    name: "Upload robots.txt file to release"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.release.tag_name }}
+
+      - name: "Upload"
+        run: gh --repo "${REPO}" release upload "${TAG}" robots.txt
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          TAG: ${{ github.event.release.tag_name }}

.gitignore

@@ -1 +1,4 @@
-.DS_Store
+.DS_Store
+.venv
+venv
+__pycache__

.htaccess

@@ -0,0 +1,3 @@
+RewriteEngine On
+RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot) [NC]
+RewriteRule !^/?robots\.txt$ - [F,L]

FAQ.md

@@ -0,0 +1,57 @@
+# Frequently asked questions
+
+## Why should we block these crawlers?
+
+They're extractive, confer no benefit to the creators of data they're ingesting and also have wide-ranging negative externalities: particularly copyright abuse and environmental impact.
+
+**[How Tech Giants Cut Corners to Harvest Data for A.I.](https://www.nytimes.com/2024/04/06/technology/tech-giants-harvest-data-artificial-intelligence.html?unlocked_article_code=1.ik0.Ofja.L21c1wyW-0xj&ugrp=m)**
+> OpenAI, Google and Meta ignored corporate policies, altered their own rules and discussed skirting copyright law as they sought online information to train their newest artificial intelligence systems.
+
+**[How AI copyright lawsuits could make the whole industry go extinct](https://www.theverge.com/24062159/ai-copyright-fair-use-lawsuits-new-york-times-openai-chatgpt-decoder-podcast)**
+> The New York Times' lawsuit against OpenAI is part of a broader, industry-shaking copyright challenge that could define the future of AI.
+
+**[Reconciling the contrasting narratives on the environmental impact of large language models](https://www.nature.com/articles/s41598-024-76682-6)**
+> Studies have shown that the training of just one LLM can consume as much energy as five cars do across their lifetimes. The water footprint of AI is also substantial; for example, recent work has highlighted that water consumption associated with AI models involves data centers using millions of gallons of water per day for cooling. Additionally, the energy consumption and carbon emissions of AI are projected to grow quickly in the coming years [...].
+
+**[Scientists Predict AI to Generate Millions of Tons of E-Waste](https://www.sciencealert.com/scientists-predict-ai-to-generate-millions-of-tons-of-e-waste)**
+> we could end up with between 1.2 million and 5 million metric tons of additional electronic waste by the end of this decade [the 2020's].
+
+## How do we know AI companies/bots respect `robots.txt`?
+
+The short answer is that we don't. `robots.txt` is a well-established standard, but compliance is voluntary. There is no enforcement mechanism.
+
+## Why might AI web crawlers respect `robots.txt`?
+
+Larger and/or reputable companies developing AI models probably wouldn't want to damage their reputation by ignoring `robots.txt`.
+
+Also, given the contentious nature of AI and the possibility of legislation limiting its development, companies developing AI models will probably want to be seen to be behaving ethically, and so should (eventually) respect `robots.txt`.
+
+## Can we block crawlers based on user agent strings?
+
+Yes, provided the crawlers identify themselves and your application/hosting supports doing so.
+
+Some crawlers — [such as Perplexity](https://rknight.me/blog/perplexity-ai-is-lying-about-its-user-agent/) — do not identify themselves via their user agent strings and, as such, are difficult to block.
+
+## What can we do if a bot doesn't respect `robots.txt`?
+
+That depends on your stack.
+
+- Nginx
+  - [Blocking Bots with Nginx](https://rknight.me/blog/blocking-bots-with-nginx/) by Robb Knight
+  - [Blocking AI web crawlers](https://underlap.org/blocking-ai-web-crawlers) by Glyn Normington
+- Apache httpd
+  - [Blockin' bots.](https://ethanmarcotte.com/wrote/blockin-bots/) by Ethan Marcotte
+  - [Blocking Bots With 11ty And Apache](https://flamedfury.com/posts/blocking-bots-with-11ty-and-apache/) by fLaMEd fury
+  > [!TIP]
+  > The snippets in these articles all use `mod_rewrite`, which [should be considered a last resort](https://httpd.apache.org/docs/trunk/rewrite/avoid.html). A good alternative that's less resource-intensive is `mod_setenvif`; see [httpd docs](https://httpd.apache.org/docs/trunk/rewrite/access.html#blocking-of-robots) for an example. You should also consider [setting this up in `httpd.conf` instead of `.htaccess`](https://httpd.apache.org/docs/trunk/howto/htaccess.html#when) if it's available to you.
+- Netlify
+  - [Blockin' bots on Netlify](https://www.jeremiak.com/blog/block-bots-netlify-edge-functions/) by Jeremia Kimelman
+- Cloudflare
+  - [Block AI bots, scrapers and crawlers with a single click](https://blog.cloudflare.com/declaring-your-aindependence-block-ai-bots-scrapers-and-crawlers-with-a-single-click) by Cloudflare
+  - [I’m blocking AI crawlers](https://roelant.net/en/2024/im-blocking-ai-crawlers-part-2/) by Roelant
+- Vercel
+  - [Block AI Bots Firewall Rule](https://vercel.com/templates/firewall/block-ai-bots-firewall-rule) by Vercel
+
+## How can I contribute?
+
+Open a pull request. It will be reviewed and acted upon appropriately. **We really appreciate contributions** — this is a community effort.

README.md

@@ -2,25 +2,57 @@
 
 <img src="/assets/images/noai-logo.png" width="100" />
 
-**[Subscribe to updates via RSS/Atom by clicking on this link.](https://github.com/ai-robots-txt/ai.robots.txt/releases.atom)**
-
-_(Or paste the link into your preferred feed reader.)_
-
----
-
-This is an open list of web crawlers associated with AI companies and the training of LLMs to block. We encourage you to contribute to and implement this list on your own site.
+This is an open list of web crawlers associated with AI companies and the training of LLMs to block. We encourage you to contribute to and implement this list on your own site. See [information about the listed crawlers](./table-of-bot-metrics.md) and the [FAQ](https://github.com/ai-robots-txt/ai.robots.txt/blob/main/FAQ.md).
 
 A number of these crawlers have been sourced from [Dark Visitors](https://darkvisitors.com) and we appreciate the ongoing effort they put in to track these crawlers. 
 
 If you'd like to add information about a crawler to the list, please make a pull request with the bot name added to `robots.txt`, `ai.txt`, and any relevant details in `table-of-bot-metrics.md` to help people understand what's crawling.
 
+## Usage
+
+This repository provides the following files:
+- `robots.txt`
+- `.htaccess`
+- `nginx-block-ai-bots.conf`
+
+`robots.txt` implements the Robots Exclusion Protocol ([RFC 9309](https://www.rfc-editor.org/rfc/rfc9309.html)).
+
+`.htaccess` may be used to configure web servers such as [Apache httpd](https://httpd.apache.org/) to return an error page when one of the listed AI crawlers sends a request to the web server.
+Note that, as stated in the [httpd documentation](https://httpd.apache.org/docs/current/howto/htaccess.html), more performant methods than an `.htaccess` file exist.
+
+`nginx-block-ai-bots.conf` implements a Nginx configuration snippet that can be included in any virtual host `server {}` block via the `include` directive.
+
+
+## Contributing
+
+A note about contributing: updates should be added/made to `robots.json`. A GitHub action will then generate the updated `robots.txt`, `table-of-bot-metrics.md`, `.htaccess` and `nginx-block-ai-bots.conf`.
+
+You can run the tests by [installing](https://www.python.org/about/gettingstarted/) Python 3 and issuing:
+```console
+code/tests.py
+```
+
+## Subscribe to updates
+
+You can subscribe to list updates via RSS/Atom with the releases feed:
+
+```
+https://github.com/ai-robots-txt/ai.robots.txt/releases.atom
+```
+
+You can subscribe with [Feedly](https://feedly.com/i/subscription/feed/https://github.com/ai-robots-txt/ai.robots.txt/releases.atom), [Inoreader](https://www.inoreader.com/?add_feed=https://github.com/ai-robots-txt/ai.robots.txt/releases.atom), [The Old Reader](https://theoldreader.com/feeds/subscribe?url=https://github.com/ai-robots-txt/ai.robots.txt/releases.atom), [Feedbin](https://feedbin.me/?subscribe=https://github.com/ai-robots-txt/ai.robots.txt/releases.atom), or any other reader app.
+
+Alternatively, you can also subscribe to new releases with your GitHub account by clicking the ⬇️ on "Watch" button at the top of this page, clicking "Custom" and selecting "Releases".
+
+## Report abusive crawlers
+
+If you use [Cloudflare's hard block](https://blog.cloudflare.com/declaring-your-aindependence-block-ai-bots-scrapers-and-crawlers-with-a-single-click) alongside this list, you can report abusive crawlers that don't respect `robots.txt` [here](https://docs.google.com/forms/d/e/1FAIpQLScbUZ2vlNSdcsb8LyTeSF7uLzQI96s0BKGoJ6wQ6ocUFNOKEg/viewform).
+But even if you don't use Cloudflare's hard block, their list of [verified bots](https://radar.cloudflare.com/traffic/verified-bots) may come in handy.
 ## Additional resources
 
 - [Blocking Bots with Nginx](https://rknight.me/blog/blocking-bots-with-nginx/) by Robb Knight
 - [Blockin' bots.](https://ethanmarcotte.com/wrote/blockin-bots/) by Ethan Marcotte
 - [Blocking Bots With 11ty And Apache](https://flamedfury.com/posts/blocking-bots-with-11ty-and-apache/) by fLaMEd fury
 - [Blockin' bots on Netlify](https://www.jeremiak.com/blog/block-bots-netlify-edge-functions/) by Jeremia Kimelman
-
----
-
-Thank you to [Glyn](https://github.com/glyn) for pushing [me](https://coryd.dev) to set this up after [I posted about blocking these crawlers](https://coryd.dev/posts/2024/go-ahead-and-block-ai-web-crawlers/).
+- [Blocking AI web crawlers](https://underlap.org/blocking-ai-web-crawlers) by Glyn Normington
+- [Block AI Bots from Crawling Websites Using Robots.txt](https://originality.ai/ai-bot-blocking) by Jonathan Gillham, Originality.AI

ai.txt

@@ -1,6 +0,0 @@
-# Spawning AI
-# Prevent datasets from using the following file types
-
-User-Agent: *
-Disallow: /
-Disallow: *

code/robots.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+
+import json
+import re
+import requests
+
+from bs4 import BeautifulSoup
+from pathlib import Path
+
+
+def load_robots_json():
+    """Load the robots.json contents into a dictionary."""
+    return json.loads(Path("./robots.json").read_text(encoding="utf-8"))
+
+
+def get_agent_soup():
+    """Retrieve current known agents from darkvisitors.com"""
+    session = requests.Session()
+    try:
+        response = session.get("https://darkvisitors.com/agents")
+    except requests.exceptions.ConnectionError:
+        print(
+            "ERROR: Could not gather the current agents from https://darkvisitors.com/agents"
+        )
+        return
+    return BeautifulSoup(response.text, "html.parser")
+
+
+def updated_robots_json(soup):
+    """Update AI scraper information with data from darkvisitors."""
+    existing_content = load_robots_json()
+    to_include = [
+        "AI Assistants",
+        "AI Data Scrapers",
+        "AI Search Crawlers",
+        # "Archivers",
+        # "Developer Helpers",
+        # "Fetchers",
+        # "Intelligence Gatherers",
+        # "Scrapers",
+        # "Search Engine Crawlers",
+        # "SEO Crawlers",
+        # "Uncategorized",
+        "Undocumented AI Agents",
+    ]
+
+    for section in soup.find_all("div", {"class": "agent-links-section"}):
+        category = section.find("h2").get_text()
+        if category not in to_include:
+            continue
+        for agent in section.find_all("a", href=True):
+            name = agent.find("div", {"class": "agent-name"}).get_text().strip()
+            name = clean_robot_name(name)
+            desc = agent.find("p").get_text().strip()
+
+            default_values = {
+                "Unclear at this time.",
+                "No information provided.",
+                "No information.",
+                "No explicit frequency provided.",
+            }
+            default_value = "Unclear at this time."
+
+            # Parse the operator information from the description if possible
+            operator = default_value
+            if "operated by " in desc:
+                try:
+                    operator = desc.split("operated by ", 1)[1].split(".", 1)[0].strip()
+                except Exception as e:
+                    print(f"Error: {e}")
+
+            def consolidate(field: str, value: str) -> str:
+                # New entry
+                if name not in existing_content:
+                    return value
+                # New field
+                if field not in existing_content[name]:
+                    return value
+                # Unclear value
+                if (
+                    existing_content[name][field] in default_values
+                    and value not in default_values
+                ):
+                    return value
+                # Existing value
+                return existing_content[name][field]
+
+            existing_content[name] = {
+                "operator": consolidate("operator", operator),
+                "respect": consolidate("respect", default_value),
+                "function": consolidate("function", f"{category}"),
+                "frequency": consolidate("frequency", default_value),
+                "description": consolidate(
+                    "description",
+                    f"{desc} More info can be found at https://darkvisitors.com/agents{agent['href']}",
+                ),
+            }
+
+    print(f"Total: {len(existing_content)}")
+    sorted_keys = sorted(existing_content, key=lambda k: k.lower())
+    sorted_robots = {k: existing_content[k] for k in sorted_keys}
+    return sorted_robots
+
+
+def clean_robot_name(name):
+    """ Clean the robot name by removing some characters that were mangled by html software once. """
+    # This was specifically spotted in "Perplexity-User"
+    # Looks like a non-breaking hyphen introduced by the HTML rendering software
+    # Reading the source page for Perplexity: https://docs.perplexity.ai/guides/bots
+    # You can see the bot is listed several times as "Perplexity-User" with a normal hyphen, 
+    # and it's only the Row-Heading that has the special hyphen
+    # 
+    # Technically, there's no reason there wouldn't someday be a bot that 
+    # actually uses a non-breaking hyphen, but that seems unlikely,
+    # so this solution should be fine for now.
+    result = re.sub(r"\u2011", "-", name)
+    if result != name:
+        print(f"\tCleaned '{name}' to '{result}' - unicode/html mangled chars normalized.")
+    return result
+
+
+def ingest_darkvisitors():
+    old_robots_json = load_robots_json()
+    soup = get_agent_soup()
+    if soup:
+        robots_json = updated_robots_json(soup)
+        print(
+            "robots.json is unchanged."
+            if robots_json == old_robots_json
+            else "robots.json got updates."
+        )
+        Path("./robots.json").write_text(
+            json.dumps(robots_json, indent=4), encoding="utf-8"
+        )
+
+
+def json_to_txt(robots_json):
+    """Compose the robots.txt from the robots.json file."""
+    robots_txt = "\n".join(f"User-agent: {k}" for k in robots_json.keys())
+    robots_txt += "\nDisallow: /\n"
+    return robots_txt
+
+
+def escape_md(s):
+    return re.sub(r"([]*\\|`(){}<>#+-.!_[])", r"\\\1", s)
+
+
+def json_to_table(robots_json):
+    """Compose a markdown table with the information in robots.json"""
+    table = "| Name | Operator | Respects `robots.txt` | Data use | Visit regularity | Description |\n"
+    table += "|------|----------|-----------------------|----------|------------------|-------------|\n"
+
+    for name, robot in robots_json.items():
+        table += f'| {escape_md(name)} | {robot["operator"]} | {robot["respect"]} | {robot["function"]} | {robot["frequency"]} | {robot["description"]} |\n'
+
+    return table
+
+
+def list_to_pcre(lst):
+    # Python re is not 100% identical to PCRE which is used by Apache, but it
+    # should probably be close enough in the real world for re.escape to work.
+    formatted = "|".join(map(re.escape, lst))
+    return f"({formatted})"
+
+
+def json_to_htaccess(robot_json):
+    # Creates a .htaccess filter file. It uses a regular expression to filter out
+    # User agents that contain any of the blocked values.
+    htaccess = "RewriteEngine On\n"
+    htaccess += f"RewriteCond %{{HTTP_USER_AGENT}} {list_to_pcre(robot_json.keys())} [NC]\n"
+    htaccess += "RewriteRule !^/?robots\\.txt$ - [F,L]\n"
+    return htaccess
+
+def json_to_nginx(robot_json):
+    # Creates an Nginx config file. This config snippet can be included in 
+    # nginx server{} blocks to block AI bots.
+    config = f"if ($http_user_agent ~* \"{list_to_pcre(robot_json.keys())}\") {{\n    return 403;\n}}"
+    return config
+
+
+def update_file_if_changed(file_name, converter):
+    """Update files if newer content is available and log the (in)actions."""
+    new_content = converter(load_robots_json())
+    filepath = Path(file_name)
+    # "touch" will create the file if it doesn't exist yet
+    filepath.touch()
+    old_content = filepath.read_text(encoding="utf-8")
+    if old_content == new_content:
+        print(f"{file_name} is already up to date.")
+    else:
+        Path(file_name).write_text(new_content, encoding="utf-8")
+        print(f"{file_name} has been updated.")
+
+
+def conversions():
+    """Triggers the conversions from the json file."""
+    update_file_if_changed(file_name="./robots.txt", converter=json_to_txt)
+    update_file_if_changed(
+        file_name="./table-of-bot-metrics.md",
+        converter=json_to_table,
+    )
+    update_file_if_changed(
+        file_name="./.htaccess",
+        converter=json_to_htaccess,
+    )
+    update_file_if_changed(
+        file_name="./nginx-block-ai-bots.conf",
+        converter=json_to_nginx,
+    )
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(
+        prog="ai-robots",
+        description="Collects and updates information about web scrapers of AI companies.",
+        epilog="One of the flags must be set.\n",
+    )
+    parser.add_argument(
+        "--update",
+        action="store_true",
+        help="Update the robots.json file with data from darkvisitors.com/agents",
+    )
+    parser.add_argument(
+        "--convert",
+        action="store_true",
+        help="Create the robots.txt and markdown table from robots.json",
+    )
+    args = parser.parse_args()
+
+    if not (args.update or args.convert):
+        print("ERROR: please provide one of the possible flags.")
+        parser.print_help()
+
+    if args.update:
+        ingest_darkvisitors()
+    if args.convert:
+        conversions()

code/test_files/.htaccess

@@ -0,0 +1,3 @@
+RewriteEngine On
+RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash) [NC]
+RewriteRule !^/?robots\.txt$ - [F,L]

code/test_files/nginx-block-ai-bots.conf

@@ -0,0 +1,3 @@
+if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)") {
+    return 403;
+}

code/test_files/robots.json

@@ -0,0 +1,331 @@
+{
+    "AI2Bot": {
+        "description": "Explores 'certain domains' to find web content.",
+        "frequency": "No information provided.",
+        "function": "Content is used to train open language models.",
+        "operator": "[Ai2](https://allenai.org/crawler)",
+        "respect": "Yes"
+    },
+    "Ai2Bot-Dolma": {
+        "description": "Explores 'certain domains' to find web content.",
+        "frequency": "No information provided.",
+        "function": "Content is used to train open language models.",
+        "operator": "[Ai2](https://allenai.org/crawler)",
+        "respect": "Yes"
+    },
+    "Amazonbot": {
+        "operator": "Amazon",
+        "respect": "Yes",
+        "function": "Service improvement and enabling answers for Alexa users.",
+        "frequency": "No information provided.",
+        "description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses."
+    },
+    "anthropic-ai": {
+        "operator": "[Anthropic](https://www.anthropic.com)",
+        "respect": "Unclear at this time.",
+        "function": "Scrapes data to train Anthropic's AI products.",
+        "frequency": "No information provided.",
+        "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
+    },
+    "Applebot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Search Crawlers",
+        "frequency": "Unclear at this time.",
+        "description": "Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://darkvisitors.com/agents/agents/applebot"
+    },
+    "Applebot-Extended": {
+        "operator": "[Apple](https://support.apple.com/en-us/119829#datausage)",
+        "respect": "Yes",
+        "function": "Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others.",
+        "frequency": "Unclear at this time.",
+        "description": "Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools."
+    },
+    "Bytespider": {
+        "operator": "ByteDance",
+        "respect": "No",
+        "function": "LLM training.",
+        "frequency": "Unclear at this time.",
+        "description": "Downloads data to train LLMS, including ChatGPT competitors."
+    },
+    "CCBot": {
+        "operator": "[Common Crawl Foundation](https://commoncrawl.org)",
+        "respect": "[Yes](https://commoncrawl.org/ccbot)",
+        "function": "Provides open crawl dataset, used for many purposes, including Machine Learning/AI.",
+        "frequency": "Monthly at present.",
+        "description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers)."
+    },
+    "ChatGPT-User": {
+        "operator": "[OpenAI](https://openai.com)",
+        "respect": "Yes",
+        "function": "Takes action based on user prompts.",
+        "frequency": "Only when prompted by a user.",
+        "description": "Used by plugins in ChatGPT to answer queries based on user input."
+    },
+    "Claude-Web": {
+        "operator": "[Anthropic](https://www.anthropic.com)",
+        "respect": "Unclear at this time.",
+        "function": "Scrapes data to train Anthropic's AI products.",
+        "frequency": "No information provided.",
+        "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
+    },
+    "ClaudeBot": {
+        "operator": "[Anthropic](https://www.anthropic.com)",
+        "respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
+        "function": "Scrapes data to train Anthropic's AI products.",
+        "frequency": "No information provided.",
+        "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
+    },
+    "cohere-ai": {
+        "operator": "[Cohere](https://cohere.com)",
+        "respect": "Unclear at this time.",
+        "function": "Retrieves data to provide responses to user-initiated prompts.",
+        "frequency": "Takes action based on user prompts.",
+        "description": "Retrieves data based on user prompts."
+    },
+    "Diffbot": {
+        "operator": "[Diffbot](https://www.diffbot.com/)",
+        "respect": "At the discretion of Diffbot users.",
+        "function": "Aggregates structured web data for monitoring and AI model training.",
+        "frequency": "Unclear at this time.",
+        "description": "Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training."
+    },
+    "FacebookBot": {
+        "operator": "Meta/Facebook",
+        "respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)",
+        "function": "Training language models",
+        "frequency": "Up to 1 page per second",
+        "description": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically."
+    },
+    "facebookexternalhit": {
+        "description": "Unclear at this time.",
+        "frequency": "Unclear at this time.",
+        "function": "No information.",
+        "operator": "Meta/Facebook",
+        "respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)"
+    },
+    "FriendlyCrawler": {
+        "description": "Unclear who the operator is; but data is used for training/machine learning.",
+        "frequency": "Unclear at this time.",
+        "function": "We are using the data from the crawler to build datasets for machine learning experiments.",
+        "operator": "Unknown",
+        "respect": "[Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler)"
+    },
+    "Google-Extended": {
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
+        "function": "LLM training.",
+        "frequency": "No information.",
+        "description": "Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search."
+    },
+    "GoogleOther": {
+        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data.",
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
+    },
+    "GoogleOther-Image": {
+        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data.",
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
+    },
+    "GoogleOther-Video": {
+        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data.",
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
+    },
+    "GPTBot": {
+        "operator": "[OpenAI](https://openai.com)",
+        "respect": "Yes",
+        "function": "Scrapes data to train OpenAI's products.",
+        "frequency": "No information.",
+        "description": "Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies."
+    },
+    "iaskspider/2.0": {
+        "description": "Used to provide answers to user queries.",
+        "frequency": "Unclear at this time.",
+        "function": "Crawls sites to provide answers to user queries.",
+        "operator": "iAsk",
+        "respect": "No"
+    },
+    "ICC-Crawler": {
+        "description": "Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business.",
+        "frequency": "No information.",
+        "function": "Scrapes data to train and support AI technologies.",
+        "operator": "[NICT](https://nict.go.jp)",
+        "respect": "Yes"
+    },
+    "ImagesiftBot": {
+        "description": "Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Our web intelligence products use this index to enable search and retrieval of similar images.",
+        "frequency": "No information.",
+        "function": "ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support our suite of web intelligence products",
+        "operator": "[ImageSift](https://imagesift.com)",
+        "respect": "[Yes](https://imagesift.com/about)"
+    },
+    "img2dataset": {
+        "description": "Downloads large sets of images into datasets for LLM training or other purposes.",
+        "frequency": "At the discretion of img2dataset users.",
+        "function": "Scrapes images for use in LLMs.",
+        "operator": "[img2dataset](https://github.com/rom1504/img2dataset)",
+        "respect": "Unclear at this time."
+    },
+    "ISSCyberRiskCrawler": {
+        "description": "Used to train machine learning based models to quantify cyber risk.",
+        "frequency": "No information.",
+        "function": "Scrapes data to train machine learning models.",
+        "operator": "[ISS-Corporate](https://iss-cyber.com)",
+        "respect": "No"
+    },
+    "Kangaroo Bot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot"
+    },
+    "Meta-ExternalAgent": {
+        "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)",
+        "respect": "Yes.",
+        "function": "Used to train models and improve products.",
+        "frequency": "No information.",
+        "description": "\"The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.\""
+    },
+    "Meta-ExternalFetcher": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
+    },
+    "OAI-SearchBot": {
+        "operator": "[OpenAI](https://openai.com)",
+        "respect": "[Yes](https://platform.openai.com/docs/bots)",
+        "function": "Search result generation.",
+        "frequency": "No information.",
+        "description": "Crawls sites to surface as results in SearchGPT."
+    },
+    "omgili": {
+        "operator": "[Webz.io](https://webz.io/)",
+        "respect": "[Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/)",
+        "function": "Data is sold.",
+        "frequency": "No information.",
+        "description": "Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training."
+    },
+    "omgilibot": {
+        "description": "Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io.",
+        "frequency": "No information.",
+        "function": "Data is sold.",
+        "operator": "[Webz.io](https://webz.io/)",
+        "respect": "[Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html)"
+    },
+    "Perplexity-User": {
+        "operator": "[Perplexity](https://www.perplexity.ai/)",
+        "respect": "[No](https://docs.perplexity.ai/guides/bots)",
+        "function": "Used to answer queries at the request of users.",
+        "frequency": "Only when prompted by a user.",
+        "description": "Visit web pages to help provide an accurate answer and include links to the page in Perplexity response."
+    },
+    "PerplexityBot": {
+        "operator": "[Perplexity](https://www.perplexity.ai/)",
+        "respect": "[No](https://www.macstories.net/stories/wired-confirms-perplexity-is-bypassing-efforts-by-websites-to-block-its-web-crawler/)",
+        "function": "Used to answer queries at the request of users.",
+        "frequency": "Takes action based on user prompts.",
+        "description": "Operated by Perplexity to obtain results in response to user queries."
+    },
+    "PetalBot": {
+        "description": "Operated by Huawei to provide search and AI assistant services.",
+        "frequency": "No explicit frequency provided.",
+        "function": "Used to provide recommendations in Hauwei assistant and AI search services.",
+        "operator": "[Huawei](https://huawei.com/)",
+        "respect": "Yes"
+    },
+    "Scrapy": {
+        "description": "\"AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data for a variety of uses including training AI.",
+        "operator": "[Zyte](https://www.zyte.com)",
+        "respect": "Unclear at this time."
+    },
+    "Sidetrade indexer bot": {
+        "description": "AI product training.",
+        "frequency": "No information.",
+        "function": "Extracts data for a variety of uses including training AI.",
+        "operator": "[Sidetrade](https://www.sidetrade.com)",
+        "respect": "Unclear at this time."
+    },
+    "Timpibot": {
+        "operator": "[Timpi](https://timpi.io)",
+        "respect": "Unclear at this time.",
+        "function": "Scrapes data for use in training LLMs.",
+        "frequency": "No information.",
+        "description": "Makes data available for training AI models."
+    },
+    "VelenPublicWebCrawler": {
+        "description": "\"Our goal with this crawler is to build business datasets and machine learning models to better understand the web.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data for business data sets and machine learning models.",
+        "operator": "[Velen Crawler](https://velen.io)",
+        "respect": "[Yes](https://velen.io)"
+    },
+    "Webzio-Extended": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended"
+    },
+    "YouBot": {
+        "operator": "[You](https://about.you.com/youchat/)",
+        "respect": "[Yes](https://about.you.com/youbot/)",
+        "function": "Scrapes data for search engine and LLMs.",
+        "frequency": "No information.",
+        "description": "Retrieves data used for You.com web search engine and LLMs."
+    },
+    "crawler.with.dots": {
+        "operator": "Test suite",
+        "respect": "No",
+        "function": "To ensure the code works correctly.",
+        "frequency": "No information.",
+        "description": "When used in the .htaccess regular expression dots need to be escaped."
+    },
+    "star***crawler": {
+        "operator": "Test suite",
+        "respect": "No",
+        "function": "To ensure the code works correctly.",
+        "frequency": "No information.",
+        "description": "When used in the .htaccess regular expression stars need to be escaped."
+    },
+    "Is this a crawler?": {
+        "operator": "Test suite",
+        "respect": "No",
+        "function": "To ensure the code works correctly.",
+        "frequency": "No information.",
+        "description": "When used in the .htaccess regular expression spaces and question marks need to be escaped."
+    },
+    "a[mazing]{42}(robot)": {
+        "operator": "Test suite",
+        "respect": "No",
+        "function": "To ensure the code works correctly.",
+        "frequency": "No information.",
+        "description": "When used in the .htaccess regular expression parantheses, braces, etc. need to be escaped."
+    },
+    "2^32$": {
+        "operator": "Test suite",
+        "respect": "No",
+        "function": "To ensure the code works correctly.",
+        "frequency": "No information.",
+        "description": "When used in the .htaccess regular expression RE anchor characters need to be escaped."
+    },
+    "curl|sudo bash": {
+        "operator": "Test suite",
+        "respect": "No",
+        "function": "To ensure the code works correctly.",
+        "frequency": "No information.",
+        "description": "When used in the .htaccess regular expression pipes need to be escaped."
+    }
+}

code/test_files/robots.txt

@@ -0,0 +1,48 @@
+User-agent: AI2Bot
+User-agent: Ai2Bot-Dolma
+User-agent: Amazonbot
+User-agent: anthropic-ai
+User-agent: Applebot
+User-agent: Applebot-Extended
+User-agent: Bytespider
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: Claude-Web
+User-agent: ClaudeBot
+User-agent: cohere-ai
+User-agent: Diffbot
+User-agent: FacebookBot
+User-agent: facebookexternalhit
+User-agent: FriendlyCrawler
+User-agent: Google-Extended
+User-agent: GoogleOther
+User-agent: GoogleOther-Image
+User-agent: GoogleOther-Video
+User-agent: GPTBot
+User-agent: iaskspider/2.0
+User-agent: ICC-Crawler
+User-agent: ImagesiftBot
+User-agent: img2dataset
+User-agent: ISSCyberRiskCrawler
+User-agent: Kangaroo Bot
+User-agent: Meta-ExternalAgent
+User-agent: Meta-ExternalFetcher
+User-agent: OAI-SearchBot
+User-agent: omgili
+User-agent: omgilibot
+User-agent: Perplexity-User
+User-agent: PerplexityBot
+User-agent: PetalBot
+User-agent: Scrapy
+User-agent: Sidetrade indexer bot
+User-agent: Timpibot
+User-agent: VelenPublicWebCrawler
+User-agent: Webzio-Extended
+User-agent: YouBot
+User-agent: crawler.with.dots
+User-agent: star***crawler
+User-agent: Is this a crawler?
+User-agent: a[mazing]{42}(robot)
+User-agent: 2^32$
+User-agent: curl|sudo bash
+Disallow: /

code/test_files/table-of-bot-metrics.md

@@ -0,0 +1,49 @@
+| Name | Operator | Respects `robots.txt` | Data use | Visit regularity | Description |
+|------|----------|-----------------------|----------|------------------|-------------|
+| AI2Bot | [Ai2](https://allenai.org/crawler) | Yes | Content is used to train open language models. | No information provided. | Explores 'certain domains' to find web content. |
+| Ai2Bot\-Dolma | [Ai2](https://allenai.org/crawler) | Yes | Content is used to train open language models. | No information provided. | Explores 'certain domains' to find web content. |
+| Amazonbot | Amazon | Yes | Service improvement and enabling answers for Alexa users. | No information provided. | Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses. |
+| anthropic\-ai | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| Applebot | Unclear at this time. | Unclear at this time. | AI Search Crawlers | Unclear at this time. | Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://darkvisitors.com/agents/agents/applebot |
+| Applebot\-Extended | [Apple](https://support.apple.com/en-us/119829#datausage) | Yes | Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others. | Unclear at this time. | Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools. |
+| Bytespider | ByteDance | No | LLM training. | Unclear at this time. | Downloads data to train LLMS, including ChatGPT competitors. |
+| CCBot | [Common Crawl Foundation](https://commoncrawl.org) | [Yes](https://commoncrawl.org/ccbot) | Provides open crawl dataset, used for many purposes, including Machine Learning/AI. | Monthly at present. | Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers). |
+| ChatGPT\-User | [OpenAI](https://openai.com) | Yes | Takes action based on user prompts. | Only when prompted by a user. | Used by plugins in ChatGPT to answer queries based on user input. |
+| Claude\-Web | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| ClaudeBot | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| cohere\-ai | [Cohere](https://cohere.com) | Unclear at this time. | Retrieves data to provide responses to user-initiated prompts. | Takes action based on user prompts. | Retrieves data based on user prompts. |
+| Diffbot | [Diffbot](https://www.diffbot.com/) | At the discretion of Diffbot users. | Aggregates structured web data for monitoring and AI model training. | Unclear at this time. | Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training. |
+| FacebookBot | Meta/Facebook | [Yes](https://developers.facebook.com/docs/sharing/bot/) | Training language models | Up to 1 page per second | Officially used for training Meta "speech recognition technology," unknown if used to train Meta AI specifically. |
+| facebookexternalhit | Meta/Facebook | [Yes](https://developers.facebook.com/docs/sharing/bot/) | No information. | Unclear at this time. | Unclear at this time. |
+| FriendlyCrawler | Unknown | [Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler) | We are using the data from the crawler to build datasets for machine learning experiments. | Unclear at this time. | Unclear who the operator is; but data is used for training/machine learning. |
+| Google\-Extended | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | LLM training. | No information. | Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search. |
+| GoogleOther | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
+| GoogleOther\-Image | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
+| GoogleOther\-Video | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
+| GPTBot | [OpenAI](https://openai.com) | Yes | Scrapes data to train OpenAI's products. | No information. | Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies. |
+| iaskspider/2\.0 | iAsk | No | Crawls sites to provide answers to user queries. | Unclear at this time. | Used to provide answers to user queries. |
+| ICC\-Crawler | [NICT](https://nict.go.jp) | Yes | Scrapes data to train and support AI technologies. | No information. | Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business. |
+| ImagesiftBot | [ImageSift](https://imagesift.com) | [Yes](https://imagesift.com/about) | ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support our suite of web intelligence products | No information. | Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Our web intelligence products use this index to enable search and retrieval of similar images. |
+| img2dataset | [img2dataset](https://github.com/rom1504/img2dataset) | Unclear at this time. | Scrapes images for use in LLMs. | At the discretion of img2dataset users. | Downloads large sets of images into datasets for LLM training or other purposes. |
+| ISSCyberRiskCrawler | [ISS-Corporate](https://iss-cyber.com) | No | Scrapes data to train machine learning models. | No information. | Used to train machine learning based models to quantify cyber risk. |
+| Kangaroo Bot | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot |
+| Meta\-ExternalAgent | [Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers) | Yes. | Used to train models and improve products. | No information. | "The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly." |
+| Meta\-ExternalFetcher | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher |
+| OAI\-SearchBot | [OpenAI](https://openai.com) | [Yes](https://platform.openai.com/docs/bots) | Search result generation. | No information. | Crawls sites to surface as results in SearchGPT. |
+| omgili | [Webz.io](https://webz.io/) | [Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/) | Data is sold. | No information. | Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training. |
+| omgilibot | [Webz.io](https://webz.io/) | [Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html) | Data is sold. | No information. | Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io. |
+| Perplexity\-User | [Perplexity](https://www.perplexity.ai/) | [No](https://docs.perplexity.ai/guides/bots) | Used to answer queries at the request of users. | Only when prompted by a user. | Visit web pages to help provide an accurate answer and include links to the page in Perplexity response. |
+| PerplexityBot | [Perplexity](https://www.perplexity.ai/) | [No](https://www.macstories.net/stories/wired-confirms-perplexity-is-bypassing-efforts-by-websites-to-block-its-web-crawler/) | Used to answer queries at the request of users. | Takes action based on user prompts. | Operated by Perplexity to obtain results in response to user queries. |
+| PetalBot | [Huawei](https://huawei.com/) | Yes | Used to provide recommendations in Hauwei assistant and AI search services. | No explicit frequency provided. | Operated by Huawei to provide search and AI assistant services. |
+| Scrapy | [Zyte](https://www.zyte.com) | Unclear at this time. | Scrapes data for a variety of uses including training AI. | No information. | "AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets." |
+| Sidetrade indexer bot | [Sidetrade](https://www.sidetrade.com) | Unclear at this time. | Extracts data for a variety of uses including training AI. | No information. | AI product training. |
+| Timpibot | [Timpi](https://timpi.io) | Unclear at this time. | Scrapes data for use in training LLMs. | No information. | Makes data available for training AI models. |
+| VelenPublicWebCrawler | [Velen Crawler](https://velen.io) | [Yes](https://velen.io) | Scrapes data for business data sets and machine learning models. | No information. | "Our goal with this crawler is to build business datasets and machine learning models to better understand the web." |
+| Webzio\-Extended | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended |
+| YouBot | [You](https://about.you.com/youchat/) | [Yes](https://about.you.com/youbot/) | Scrapes data for search engine and LLMs. | No information. | Retrieves data used for You.com web search engine and LLMs. |
+| crawler\.with\.dots | Test suite | No | To ensure the code works correctly. | No information. | When used in the .htaccess regular expression dots need to be escaped. |
+| star\*\*\*crawler | Test suite | No | To ensure the code works correctly. | No information. | When used in the .htaccess regular expression stars need to be escaped. |
+| Is this a crawler? | Test suite | No | To ensure the code works correctly. | No information. | When used in the .htaccess regular expression spaces and question marks need to be escaped. |
+| a\[mazing\]\{42\}\(robot\) | Test suite | No | To ensure the code works correctly. | No information. | When used in the .htaccess regular expression parantheses, braces, etc. need to be escaped. |
+| 2^32$ | Test suite | No | To ensure the code works correctly. | No information. | When used in the .htaccess regular expression RE anchor characters need to be escaped. |
+| curl\|sudo bash | Test suite | No | To ensure the code works correctly. | No information. | When used in the .htaccess regular expression pipes need to be escaped. |

code/tests.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""To run these tests just execute this script."""
+
+import json
+import unittest
+
+from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx
+
+class RobotsUnittestExtensions:
+    def loadJson(self, pathname):
+        with open(pathname, "rt") as f:
+            return json.load(f)
+
+    def assertEqualsFile(self, f, s):
+        with open(f, "rt") as f:
+            f_contents = f.read()
+
+        return self.assertMultiLineEqual(f_contents, s)
+
+
+class TestRobotsTXTGeneration(unittest.TestCase, RobotsUnittestExtensions):
+    maxDiff = 8192
+
+    def setUp(self):
+        self.robots_dict = self.loadJson("test_files/robots.json")
+
+    def test_robots_txt_generation(self):
+        robots_txt = json_to_txt(self.robots_dict)
+        self.assertEqualsFile("test_files/robots.txt", robots_txt)
+
+
+class TestTableMetricsGeneration(unittest.TestCase, RobotsUnittestExtensions):
+    maxDiff = 32768
+
+    def setUp(self):
+        self.robots_dict = self.loadJson("test_files/robots.json")
+
+    def test_table_generation(self):
+        robots_table = json_to_table(self.robots_dict)
+        self.assertEqualsFile("test_files/table-of-bot-metrics.md", robots_table)
+
+
+class TestHtaccessGeneration(unittest.TestCase, RobotsUnittestExtensions):
+    maxDiff = 8192
+
+    def setUp(self):
+        self.robots_dict = self.loadJson("test_files/robots.json")
+
+    def test_htaccess_generation(self):
+        robots_htaccess = json_to_htaccess(self.robots_dict)
+        self.assertEqualsFile("test_files/.htaccess", robots_htaccess)
+
+class TestNginxConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
+    maxDiff = 8192
+
+    def setUp(self):
+        self.robots_dict = self.loadJson("test_files/robots.json")
+
+    def test_nginx_generation(self):
+        robots_nginx = json_to_nginx(self.robots_dict)
+        self.assertEqualsFile("test_files/nginx-block-ai-bots.conf", robots_nginx)
+
+class TestRobotsNameCleaning(unittest.TestCase):
+    def test_clean_name(self):
+        from robots import clean_robot_name
+
+        self.assertEqual(clean_robot_name("Perplexity‑User"), "Perplexity-User")
+
+if __name__ == "__main__":
+    import os
+    os.chdir(os.path.dirname(__file__))
+
+    unittest.main(verbosity=2)

nginx-block-ai-bots.conf

@@ -0,0 +1,3 @@
+if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot)") {
+    return 403;
+}

robots.json

@@ -0,0 +1,338 @@
+{
+    "AI2Bot": {
+        "operator": "[Ai2](https://allenai.org/crawler)",
+        "respect": "Yes",
+        "function": "Content is used to train open language models.",
+        "frequency": "No information provided.",
+        "description": "Explores 'certain domains' to find web content."
+    },
+    "Ai2Bot-Dolma": {
+        "description": "Explores 'certain domains' to find web content.",
+        "frequency": "No information provided.",
+        "function": "Content is used to train open language models.",
+        "operator": "[Ai2](https://allenai.org/crawler)",
+        "respect": "Yes"
+    },
+    "Amazonbot": {
+        "operator": "Amazon",
+        "respect": "Yes",
+        "function": "Service improvement and enabling answers for Alexa users.",
+        "frequency": "No information provided.",
+        "description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses."
+    },
+    "anthropic-ai": {
+        "operator": "[Anthropic](https://www.anthropic.com)",
+        "respect": "Unclear at this time.",
+        "function": "Scrapes data to train Anthropic's AI products.",
+        "frequency": "No information provided.",
+        "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
+    },
+    "Applebot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Search Crawlers",
+        "frequency": "Unclear at this time.",
+        "description": "Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://darkvisitors.com/agents/agents/applebot"
+    },
+    "Applebot-Extended": {
+        "operator": "[Apple](https://support.apple.com/en-us/119829#datausage)",
+        "respect": "Yes",
+        "function": "Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others.",
+        "frequency": "Unclear at this time.",
+        "description": "Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools."
+    },
+    "Brightbot 1.0": {
+        "operator": "Browsing.ai",
+        "respect": "Unclear at this time.",
+        "function": "LLM/AI training.",
+        "frequency": "Unclear at this time.",
+        "description": "Scrapes data to train LLMs and AI products focused on website customer support."
+    },
+    "Bytespider": {
+        "operator": "ByteDance",
+        "respect": "No",
+        "function": "LLM training.",
+        "frequency": "Unclear at this time.",
+        "description": "Downloads data to train LLMS, including ChatGPT competitors."
+    },
+    "CCBot": {
+        "operator": "[Common Crawl Foundation](https://commoncrawl.org)",
+        "respect": "[Yes](https://commoncrawl.org/ccbot)",
+        "function": "Provides open crawl dataset, used for many purposes, including Machine Learning/AI.",
+        "frequency": "Monthly at present.",
+        "description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers)."
+    },
+    "ChatGPT-User": {
+        "operator": "[OpenAI](https://openai.com)",
+        "respect": "Yes",
+        "function": "Takes action based on user prompts.",
+        "frequency": "Only when prompted by a user.",
+        "description": "Used by plugins in ChatGPT to answer queries based on user input."
+    },
+    "Claude-Web": {
+        "operator": "[Anthropic](https://www.anthropic.com)",
+        "respect": "Unclear at this time.",
+        "function": "Scrapes data to train Anthropic's AI products.",
+        "frequency": "No information provided.",
+        "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
+    },
+    "ClaudeBot": {
+        "operator": "[Anthropic](https://www.anthropic.com)",
+        "respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
+        "function": "Scrapes data to train Anthropic's AI products.",
+        "frequency": "No information provided.",
+        "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
+    },
+    "cohere-ai": {
+        "operator": "[Cohere](https://cohere.com)",
+        "respect": "Unclear at this time.",
+        "function": "Retrieves data to provide responses to user-initiated prompts.",
+        "frequency": "Takes action based on user prompts.",
+        "description": "Retrieves data based on user prompts."
+    },
+    "cohere-training-data-crawler": {
+        "operator": "Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "cohere-training-data-crawler is a web crawler operated by Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products. More info can be found at https://darkvisitors.com/agents/agents/cohere-training-data-crawler"
+    },
+    "Crawlspace": {
+        "operator": "[Crawlspace](https://crawlspace.dev)",
+        "respect": "[Yes](https://news.ycombinator.com/item?id=42756654)",
+        "function": "Scrapes data",
+        "frequency": "Unclear at this time.",
+        "description": "Provides crawling services for any purpose, probably including AI model training."
+    },
+    "Diffbot": {
+        "operator": "[Diffbot](https://www.diffbot.com/)",
+        "respect": "At the discretion of Diffbot users.",
+        "function": "Aggregates structured web data for monitoring and AI model training.",
+        "frequency": "Unclear at this time.",
+        "description": "Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training."
+    },
+    "DuckAssistBot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "DuckAssistBot is used by DuckDuckGo's DuckAssist feature to fetch content and generate realtime AI answers to user searches. More info can be found at https://darkvisitors.com/agents/agents/duckassistbot"
+    },
+    "FacebookBot": {
+        "operator": "Meta/Facebook",
+        "respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)",
+        "function": "Training language models",
+        "frequency": "Up to 1 page per second",
+        "description": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically."
+    },
+    "FriendlyCrawler": {
+        "description": "Unclear who the operator is; but data is used for training/machine learning.",
+        "frequency": "Unclear at this time.",
+        "function": "We are using the data from the crawler to build datasets for machine learning experiments.",
+        "operator": "Unknown",
+        "respect": "[Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler)"
+    },
+    "Google-Extended": {
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
+        "function": "LLM training.",
+        "frequency": "No information.",
+        "description": "Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search."
+    },
+    "GoogleOther": {
+        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data.",
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
+    },
+    "GoogleOther-Image": {
+        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data.",
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
+    },
+    "GoogleOther-Video": {
+        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data.",
+        "operator": "Google",
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
+    },
+    "GPTBot": {
+        "operator": "[OpenAI](https://openai.com)",
+        "respect": "Yes",
+        "function": "Scrapes data to train OpenAI's products.",
+        "frequency": "No information.",
+        "description": "Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies."
+    },
+    "iaskspider/2.0": {
+        "description": "Used to provide answers to user queries.",
+        "frequency": "Unclear at this time.",
+        "function": "Crawls sites to provide answers to user queries.",
+        "operator": "iAsk",
+        "respect": "No"
+    },
+    "ICC-Crawler": {
+        "description": "Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business.",
+        "frequency": "No information.",
+        "function": "Scrapes data to train and support AI technologies.",
+        "operator": "[NICT](https://nict.go.jp)",
+        "respect": "Yes"
+    },
+    "ImagesiftBot": {
+        "description": "Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Our web intelligence products use this index to enable search and retrieval of similar images.",
+        "frequency": "No information.",
+        "function": "ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support our suite of web intelligence products",
+        "operator": "[ImageSift](https://imagesift.com)",
+        "respect": "[Yes](https://imagesift.com/about)"
+    },
+    "img2dataset": {
+        "description": "Downloads large sets of images into datasets for LLM training or other purposes.",
+        "frequency": "At the discretion of img2dataset users.",
+        "function": "Scrapes images for use in LLMs.",
+        "operator": "[img2dataset](https://github.com/rom1504/img2dataset)",
+        "respect": "Unclear at this time."
+    },
+    "imgproxy": {
+        "frequency": "No information.",
+        "function": "Not documented or explained on operator's site.",
+        "operator": "[imgproxy](https://imgproxy.net)",
+        "respect": "Unclear at this time.",
+        "description": "AI-powered image processing."
+    },
+    "ISSCyberRiskCrawler": {
+        "description": "Used to train machine learning based models to quantify cyber risk.",
+        "frequency": "No information.",
+        "function": "Scrapes data to train machine learning models.",
+        "operator": "[ISS-Corporate](https://iss-cyber.com)",
+        "respect": "No"
+    },
+    "Kangaroo Bot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot"
+    },
+    "Meta-ExternalAgent": {
+        "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)",
+        "respect": "Yes.",
+        "function": "Used to train models and improve products.",
+        "frequency": "No information.",
+        "description": "\"The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.\""
+    },
+    "Meta-ExternalFetcher": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
+    },
+    "OAI-SearchBot": {
+        "operator": "[OpenAI](https://openai.com)",
+        "respect": "[Yes](https://platform.openai.com/docs/bots)",
+        "function": "Search result generation.",
+        "frequency": "No information.",
+        "description": "Crawls sites to surface as results in SearchGPT."
+    },
+    "omgili": {
+        "operator": "[Webz.io](https://webz.io/)",
+        "respect": "[Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/)",
+        "function": "Data is sold.",
+        "frequency": "No information.",
+        "description": "Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training."
+    },
+    "omgilibot": {
+        "description": "Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io.",
+        "frequency": "No information.",
+        "function": "Data is sold.",
+        "operator": "[Webz.io](https://webz.io/)",
+        "respect": "[Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html)"
+    },
+    "PanguBot": {
+        "operator": "the Chinese company Huawei",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot"
+    },
+    "Perplexity-User": {
+        "operator": "[Perplexity](https://www.perplexity.ai/)",
+        "respect": "[No](https://docs.perplexity.ai/guides/bots)",
+        "function": "Used to answer queries at the request of users.",
+        "frequency": "Only when prompted by a user.",
+        "description": "Visit web pages to help provide an accurate answer and include links to the page in Perplexity response."
+    },
+    "PerplexityBot": {
+        "operator": "[Perplexity](https://www.perplexity.ai/)",
+        "respect": "[Yes](https://docs.perplexity.ai/guides/bots)",
+        "function": "Search result generation.",
+        "frequency": "No information.",
+        "description": "Crawls sites to surface as results in Perplexity."
+    },
+    "PetalBot": {
+        "description": "Operated by Huawei to provide search and AI assistant services.",
+        "frequency": "No explicit frequency provided.",
+        "function": "Used to provide recommendations in Hauwei assistant and AI search services.",
+        "operator": "[Huawei](https://huawei.com/)",
+        "respect": "Yes"
+    },
+    "Scrapy": {
+        "description": "\"AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data for a variety of uses including training AI.",
+        "operator": "[Zyte](https://www.zyte.com)",
+        "respect": "Unclear at this time."
+    },
+    "SemrushBot-OCOB": {
+        "operator": "[Semrush](https://www.semrush.com/)",
+        "respect": "[Yes](https://www.semrush.com/bot/)",
+        "function": "Crawls your site for ContentShake AI tool.",
+        "frequency": "Roughly once every 10 seconds.",
+        "description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
+    },
+    "SemrushBot-SWA": {
+        "operator": "[Semrush](https://www.semrush.com/)",
+        "respect": "[Yes](https://www.semrush.com/bot/)",
+        "function": "Checks URLs on your site for SWA tool.",
+        "frequency": "Roughly once every 10 seconds.",
+        "description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
+    },
+    "Sidetrade indexer bot": {
+        "description": "AI product training.",
+        "frequency": "No information.",
+        "function": "Extracts data for a variety of uses including training AI.",
+        "operator": "[Sidetrade](https://www.sidetrade.com)",
+        "respect": "Unclear at this time."
+    },
+    "Timpibot": {
+        "operator": "[Timpi](https://timpi.io)",
+        "respect": "Unclear at this time.",
+        "function": "Scrapes data for use in training LLMs.",
+        "frequency": "No information.",
+        "description": "Makes data available for training AI models."
+    },
+    "VelenPublicWebCrawler": {
+        "description": "\"Our goal with this crawler is to build business datasets and machine learning models to better understand the web.\"",
+        "frequency": "No information.",
+        "function": "Scrapes data for business data sets and machine learning models.",
+        "operator": "[Velen Crawler](https://velen.io)",
+        "respect": "[Yes](https://velen.io)"
+    },
+    "Webzio-Extended": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended"
+    },
+    "YouBot": {
+        "operator": "[You](https://about.you.com/youchat/)",
+        "respect": "[Yes](https://about.you.com/youbot/)",
+        "function": "Scrapes data for search engine and LLMs.",
+        "frequency": "No information.",
+        "description": "Retrieves data used for You.com web search engine and LLMs."
+    }
+}

robots.txt

@@ -1,24 +1,49 @@
-User-agent: AdsBot-Google
+User-agent: AI2Bot
+User-agent: Ai2Bot-Dolma
 User-agent: Amazonbot
 User-agent: anthropic-ai
+User-agent: Applebot
 User-agent: Applebot-Extended
+User-agent: Brightbot 1.0
 User-agent: Bytespider
 User-agent: CCBot
 User-agent: ChatGPT-User
-User-agent: ClaudeBot
 User-agent: Claude-Web
+User-agent: ClaudeBot
 User-agent: cohere-ai
+User-agent: cohere-training-data-crawler
+User-agent: Crawlspace
 User-agent: Diffbot
+User-agent: DuckAssistBot
 User-agent: FacebookBot
 User-agent: FriendlyCrawler
 User-agent: Google-Extended
 User-agent: GoogleOther
+User-agent: GoogleOther-Image
+User-agent: GoogleOther-Video
 User-agent: GPTBot
+User-agent: iaskspider/2.0
+User-agent: ICC-Crawler
+User-agent: ImagesiftBot
 User-agent: img2dataset
+User-agent: imgproxy
+User-agent: ISSCyberRiskCrawler
+User-agent: Kangaroo Bot
+User-agent: Meta-ExternalAgent
+User-agent: Meta-ExternalFetcher
+User-agent: OAI-SearchBot
 User-agent: omgili
 User-agent: omgilibot
-User-agent: peer39_crawler
-User-agent: peer39_crawler/1.0
+User-agent: PanguBot
+User-agent: Perplexity-User
 User-agent: PerplexityBot
+User-agent: PetalBot
+User-agent: Scrapy
+User-agent: SemrushBot-OCOB
+User-agent: SemrushBot-SWA
+User-agent: Sidetrade indexer bot
+User-agent: Timpibot
+User-agent: VelenPublicWebCrawler
+User-agent: Webzio-Extended
 User-agent: YouBot
 Disallow: /

table-of-bot-metrics.md

@@ -1,24 +1,50 @@
-|Name            |Operator |Respects `robots.txt`  |Data use  |Visit regularity  |Description  |
-|----------------|---------|-----------------------|----------|------------------|-------------|
-| AdsBot-Google   | Google  | Yes (Exceptions for Dynamic Search Ads) | Analyzes website content for ad relevancy, improves ad serving for Google Ads. Data anonymized according to [Google's Privacy Policy](https://policies.google.com/privacy). Unclear on data retention or use by other products. | Varies depending on campaign activity and website updates. Crawls optimized to minimize impact, specific frequency not public. | Web crawler by Google Ads to analyze websites for ad effectiveness and ensure ad relevancy to webpage content. |
-|Amazonbot      | Amazon | Yes | Service improvement and enabling answers for Alexa users. | No information provided. | Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses. |
-|anthropic-ai  | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
-|Applebot-Extended   |    [Apple](https://support.apple.com/en-us/119829#datausage)    |          Yes           |          |                  |      Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools.       |
-|Bytespider    | ByteDance | No | LLM training. | Unclear at this time. | Downloads data to train LLMS, including ChatGPT competitors. |
-|CCBot         | [Common Crawl](https://commoncrawl.org) | [Yes](https://commoncrawl.org/ccbot) | Provides crawl data for an open source repository that has been used to train LLMs. | Unclear at this time. | Sources data that is made openly available and is used to train AI models. |
-|ChatGPT-User   | [OpenAI](https://openai.com) | Yes | Takes action based on user prompts. | Only when prompted by a user. | Used by plugins in ChatGPT to answer queries based on user input. |
-|ClaudeBot      | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
-|Claude-Web | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
-|cohere-ai | [Cohere](https://cohere.com) | Unclear at this time. | Retrieves data to provide responses to user-initiated prompts. | Takes action based on user prompts. | Retrieves data based on user prompts. |
-|Diffbot | [Diffbot](https://www.diffbot.com/) | At the discretion of Diffbot users. | Aggregates structured web data for monitoring and AI model training. | Unclear at this time. | Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training. |
-|FacebookBot    |         |                       |          |                  |             |
-|Google-Extended|         |                       |          |                  |             |
-|GoogleOther    |         |                       |          |                  |             |
-|GPTBot        | [OpenAI](https://openai.com) | Yes | Scrapes data to train OpenAI's products. | No information provided. | Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies. |
-| img2dataset |         |                       |          |                  |             |
-|omgili        |         |                       |          |                  |             |
-|omgilibot     |         |                       |          |                  |             |
-|peer39_crawler|         |                       |          |                  |             |
-|peer39_crawler/1.0|         |                       |          |                  |             |
-|PerplexityBot | [Perplexity](https://www.perplexity.ai/) | [Yes](https://docs.perplexity.ai/docs/perplexitybot) | Used to answer queries at the request of users. | Takes action based on user prompts.  | Operated by Perplexity to obtain results in response to user queries. |
-|YouBot        |         |                       |          |                  |             |
+| Name | Operator | Respects `robots.txt` | Data use | Visit regularity | Description |
+|------|----------|-----------------------|----------|------------------|-------------|
+| AI2Bot | [Ai2](https://allenai.org/crawler) | Yes | Content is used to train open language models. | No information provided. | Explores 'certain domains' to find web content. |
+| Ai2Bot\-Dolma | [Ai2](https://allenai.org/crawler) | Yes | Content is used to train open language models. | No information provided. | Explores 'certain domains' to find web content. |
+| Amazonbot | Amazon | Yes | Service improvement and enabling answers for Alexa users. | No information provided. | Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses. |
+| anthropic\-ai | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| Applebot | Unclear at this time. | Unclear at this time. | AI Search Crawlers | Unclear at this time. | Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://darkvisitors.com/agents/agents/applebot |
+| Applebot\-Extended | [Apple](https://support.apple.com/en-us/119829#datausage) | Yes | Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others. | Unclear at this time. | Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools. |
+| Brightbot 1\.0 | Browsing.ai | Unclear at this time. | LLM/AI training. | Unclear at this time. | Scrapes data to train LLMs and AI products focused on website customer support. |
+| Bytespider | ByteDance | No | LLM training. | Unclear at this time. | Downloads data to train LLMS, including ChatGPT competitors. |
+| CCBot | [Common Crawl Foundation](https://commoncrawl.org) | [Yes](https://commoncrawl.org/ccbot) | Provides open crawl dataset, used for many purposes, including Machine Learning/AI. | Monthly at present. | Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers). |
+| ChatGPT\-User | [OpenAI](https://openai.com) | Yes | Takes action based on user prompts. | Only when prompted by a user. | Used by plugins in ChatGPT to answer queries based on user input. |
+| Claude\-Web | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| ClaudeBot | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| cohere\-ai | [Cohere](https://cohere.com) | Unclear at this time. | Retrieves data to provide responses to user-initiated prompts. | Takes action based on user prompts. | Retrieves data based on user prompts. |
+| cohere\-training\-data\-crawler | Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products | Unclear at this time. | AI Data Scrapers | Unclear at this time. | cohere-training-data-crawler is a web crawler operated by Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products. More info can be found at https://darkvisitors.com/agents/agents/cohere-training-data-crawler |
+| Crawlspace | [Crawlspace](https://crawlspace.dev) | [Yes](https://news.ycombinator.com/item?id=42756654) | Scrapes data | Unclear at this time. | Provides crawling services for any purpose, probably including AI model training. |
+| Diffbot | [Diffbot](https://www.diffbot.com/) | At the discretion of Diffbot users. | Aggregates structured web data for monitoring and AI model training. | Unclear at this time. | Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training. |
+| DuckAssistBot | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | DuckAssistBot is used by DuckDuckGo's DuckAssist feature to fetch content and generate realtime AI answers to user searches. More info can be found at https://darkvisitors.com/agents/agents/duckassistbot |
+| FacebookBot | Meta/Facebook | [Yes](https://developers.facebook.com/docs/sharing/bot/) | Training language models | Up to 1 page per second | Officially used for training Meta "speech recognition technology," unknown if used to train Meta AI specifically. |
+| FriendlyCrawler | Unknown | [Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler) | We are using the data from the crawler to build datasets for machine learning experiments. | Unclear at this time. | Unclear who the operator is; but data is used for training/machine learning. |
+| Google\-Extended | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | LLM training. | No information. | Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search. |
+| GoogleOther | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
+| GoogleOther\-Image | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
+| GoogleOther\-Video | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
+| GPTBot | [OpenAI](https://openai.com) | Yes | Scrapes data to train OpenAI's products. | No information. | Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies. |
+| iaskspider/2\.0 | iAsk | No | Crawls sites to provide answers to user queries. | Unclear at this time. | Used to provide answers to user queries. |
+| ICC\-Crawler | [NICT](https://nict.go.jp) | Yes | Scrapes data to train and support AI technologies. | No information. | Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business. |
+| ImagesiftBot | [ImageSift](https://imagesift.com) | [Yes](https://imagesift.com/about) | ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support our suite of web intelligence products | No information. | Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Our web intelligence products use this index to enable search and retrieval of similar images. |
+| img2dataset | [img2dataset](https://github.com/rom1504/img2dataset) | Unclear at this time. | Scrapes images for use in LLMs. | At the discretion of img2dataset users. | Downloads large sets of images into datasets for LLM training or other purposes. |
+| imgproxy | [imgproxy](https://imgproxy.net) | Unclear at this time. | Not documented or explained on operator's site. | No information. | AI-powered image processing. |
+| ISSCyberRiskCrawler | [ISS-Corporate](https://iss-cyber.com) | No | Scrapes data to train machine learning models. | No information. | Used to train machine learning based models to quantify cyber risk. |
+| Kangaroo Bot | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot |
+| Meta\-ExternalAgent | [Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers) | Yes. | Used to train models and improve products. | No information. | "The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly." |
+| Meta\-ExternalFetcher | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher |
+| OAI\-SearchBot | [OpenAI](https://openai.com) | [Yes](https://platform.openai.com/docs/bots) | Search result generation. | No information. | Crawls sites to surface as results in SearchGPT. |
+| omgili | [Webz.io](https://webz.io/) | [Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/) | Data is sold. | No information. | Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training. |
+| omgilibot | [Webz.io](https://webz.io/) | [Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html) | Data is sold. | No information. | Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io. |
+| PanguBot | the Chinese company Huawei | Unclear at this time. | AI Data Scrapers | Unclear at this time. | PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot |
+| Perplexity\-User | [Perplexity](https://www.perplexity.ai/) | [No](https://docs.perplexity.ai/guides/bots) | Used to answer queries at the request of users. | Only when prompted by a user. | Visit web pages to help provide an accurate answer and include links to the page in Perplexity response. |
+| PerplexityBot | [Perplexity](https://www.perplexity.ai/) | [Yes](https://docs.perplexity.ai/guides/bots) | Search result generation. | No information. | Crawls sites to surface as results in Perplexity. |
+| PetalBot | [Huawei](https://huawei.com/) | Yes | Used to provide recommendations in Hauwei assistant and AI search services. | No explicit frequency provided. | Operated by Huawei to provide search and AI assistant services. |
+| Scrapy | [Zyte](https://www.zyte.com) | Unclear at this time. | Scrapes data for a variety of uses including training AI. | No information. | "AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets." |
+| SemrushBot\-OCOB | [Semrush](https://www.semrush.com/) | [Yes](https://www.semrush.com/bot/) | Crawls your site for ContentShake AI tool. | Roughly once every 10 seconds. | You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL). |
+| SemrushBot\-SWA | [Semrush](https://www.semrush.com/) | [Yes](https://www.semrush.com/bot/) | Checks URLs on your site for SWA tool. | Roughly once every 10 seconds. | You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL). |
+| Sidetrade indexer bot | [Sidetrade](https://www.sidetrade.com) | Unclear at this time. | Extracts data for a variety of uses including training AI. | No information. | AI product training. |
+| Timpibot | [Timpi](https://timpi.io) | Unclear at this time. | Scrapes data for use in training LLMs. | No information. | Makes data available for training AI models. |
+| VelenPublicWebCrawler | [Velen Crawler](https://velen.io) | [Yes](https://velen.io) | Scrapes data for business data sets and machine learning models. | No information. | "Our goal with this crawler is to build business datasets and machine learning models to better understand the web." |
+| Webzio\-Extended | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended |
+| YouBot | [You](https://about.you.com/youchat/) | [Yes](https://about.you.com/youbot/) | Scrapes data for search engine and LLMs. | No information. | Retrieves data used for You.com web search engine and LLMs. |