feat: move robots to cms

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "coryd.dev",
-  "version": "20.2.7",
+  "version": "20.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "coryd.dev",
-      "version": "20.2.7",
+      "version": "20.3.0",
       "license": "MIT",
       "dependencies": {
         "@cdransf/api-text": "^1.4.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "coryd.dev",
-  "version": "20.2.7",
+  "version": "20.3.0",
   "description": "The source for my personal site. Built using 11ty.",
   "type": "module",
   "scripts": {

src/_data/robots.js

@@ -1,36 +1,34 @@
-export default async function () {
-  return [
-    'AdsBot-Google',
-    'Amazonbot',
-    'anthropic-ai',
-    'Applebot-Extended',
-    'AwarioRssBot',
-    'AwarioSmartBot',
-    'Bytespider',
-    'CCBot',
-    'ChatGPT-User',
-    'ClaudeBot',
-    'Claude-Web',
-    'cohere-ai',
-    'DataForSeoBot',
-    'Diffbot',
-    'FacebookBot',
-    'FriendlyCrawler',
-    'Google-Extended',
-    'GoogleOther',
-    'GPTBot',
-    'img2dataset',
-    'ImagesiftBot',
-    'magpie-crawler',
-    'Meltwater',
-    'omgili',
-    'omgilibot',
-    'peer39_crawler',
-    'peer39_crawler/1.0',
-    'PerplexityBot',
-    'PiplBot',
-    'scoop.it',
-    'Seekr',
-    'YouBot',
-  ]
+import { createClient } from '@supabase/supabase-js'
+
+const SUPABASE_URL = process.env.SUPABASE_URL
+const SUPABASE_KEY = process.env.SUPABASE_KEY
+const supabase = createClient(SUPABASE_URL, SUPABASE_KEY)
+
+const PAGE_SIZE = 100
+
+const fetchAllRobots = async () => {
+  let robots = []
+  let from = 0
+  let to = PAGE_SIZE - 1
+
+  while (true) {
+    const { data, error } = await supabase
+      .from('robots')
+      .select('user_agent')
+      .range(from, to)
+
+    if (error) {
+      console.error('Error fetching robot data:', error)
+      return null
+    }
+
+    robots = robots.concat(data)
+    if (data.length < PAGE_SIZE) break
+  }
+
+  return robots.map(robot => robot.user_agent)
+}
+
+export default async function () {
+  return await fetchAllRobots()
 }

src/_includes/partials/home/status.liquid

@@ -2,7 +2,7 @@
 {%- assign book = books | bookStatus: 'started' | reverse | first -%}
 {%- assign show = tv.recentlyWatched | first -%}
 <div class="home-status">
-  <p><strong class="highlight-text">I'm a software developer based in Camarillo, California</strong>. I enjoy hanging out with my beautiful family and 3 rescue dogs, technology, automation, <a href="https://coryd.dev/music">music</a>, writing, <a href="https://coryd.dev/books">reading</a>, <a href="https://coryd.dev/watching#tv">tv</a> and <a href="https://coryd.dev/watching#movies">movies</a>. Lately I've been listening to a lot of <strong class="highlight-text">{{ artist.title }}</strong>, reading <strong class="highlight-text">{{ book.title }}</strong> and watching <strong class="highlight-text">{{ show.name }}</strong>.</p>
+  <p><strong class="highlight-text">I'm a software developer based in Camarillo, California</strong>. I enjoy hanging out with my beautiful family and 3 rescue dogs, technology, automation, <a href="/music">music</a>, <a href="/posts">writing</a>, <a href="/books">reading</a>, <a href="/watching#tv">tv</a> and <a href="/watching#movies">movies</a>. Lately I've been listening to a lot of <strong class="highlight-text">{{ artist.title }}</strong>, reading <strong class="highlight-text">{{ book.title }}</strong> and watching <strong class="highlight-text">{{ show.name }}</strong>.</p>
   <p>{{ status.emoji }} {{ status.content }}</p>
   {% render "partials/widgets/now-playing.liquid" %}
 </div>