make conversions and updates separately triggerable

.github/workflows/ai_robots_update.yml

@@ -18,10 +18,17 @@ jobs:
           pip install beautifulsoup4 requests
           git config --global user.name "dark-visitors"
           git config --global user.email "dark-visitors@users.noreply.github.com"
-          echo "Running update script ..."
+          echo "Updating robots.json with data from darkvisitor.com ..."
-          python code/dark_visitors.py
+          python code/dark_visitors.py --update
           echo "... done."
           git --no-pager diff
           git add -A
-          git diff --quiet && git diff --staged --quiet || (git commit -m "Daily update from Dark Visitors" && git push)
+          git diff --quiet && git diff --staged --quiet || (git commit -m "Update from Dark Visitors" && git push)
+          
+          echo "Updating robots.txt and table-of-bot-metrics.md if necessary ..."
+          python code/dark_visitors.py --convert
+          echo "... done."
+          git --no-pager diff
+          git add -A
+          git diff --quiet && git diff --staged --quiet || (git commit -m "Updated from new robots.json" && git push)
         shell: bash

code/dark_visitors.py

@@ -153,5 +153,31 @@ def conversions():
 
 
 if __name__ == "__main__":
-    ingest_darkvisitors()
+    import argparse
-    conversions()
+
+    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(
+        prog="ai-robots",
+        description="Collects and updates information about web scrapers of AI companies.",
+        epilog="One of the flags must be set.\n",
+    )
+    parser.add_argument(
+        "--update",
+        action="store_true",
+        help="Update the robots.json file with data from darkvisitors.com/agents",
+    )
+    parser.add_argument(
+        "--convert",
+        action="store_true",
+        help="Create the robots.txt and markdown table from robots.json",
+    )
+    args = parser.parse_args()
+
+    if not (args.update or args.convert):
+        print("ERROR: please provide one of the possible flags.")
+        parser.print_help()
+
+    if args.update:
+        ingest_darkvisitors()
+    if args.convert:
+        conversions()