feat: move robots to cms

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "coryd.dev",
-  "version": "20.2.7",
+  "version": "20.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "coryd.dev",
-      "version": "20.2.7",
+      "version": "20.3.0",
       "license": "MIT",
       "dependencies": {
         "@cdransf/api-text": "^1.4.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "coryd.dev",
-  "version": "20.2.7",
+  "version": "20.3.0",
   "description": "The source for my personal site. Built using 11ty.",
   "type": "module",
   "scripts": {

src/_data/robots.js

@@ -1,36 +1,34 @@
-export default async function () {
+import { createClient } from '@supabase/supabase-js'
-  return [
+
-    'AdsBot-Google',
+const SUPABASE_URL = process.env.SUPABASE_URL
-    'Amazonbot',
+const SUPABASE_KEY = process.env.SUPABASE_KEY
-    'anthropic-ai',
+const supabase = createClient(SUPABASE_URL, SUPABASE_KEY)
-    'Applebot-Extended',
+
-    'AwarioRssBot',
+const PAGE_SIZE = 100
-    'AwarioSmartBot',
+
-    'Bytespider',
+const fetchAllRobots = async () => {
-    'CCBot',
+  let robots = []
-    'ChatGPT-User',
+  let from = 0
-    'ClaudeBot',
+  let to = PAGE_SIZE - 1
-    'Claude-Web',
+
-    'cohere-ai',
+  while (true) {
-    'DataForSeoBot',
+    const { data, error } = await supabase
-    'Diffbot',
+      .from('robots')
-    'FacebookBot',
+      .select('user_agent')
-    'FriendlyCrawler',
+      .range(from, to)
-    'Google-Extended',
+
-    'GoogleOther',
+    if (error) {
-    'GPTBot',
+      console.error('Error fetching robot data:', error)
-    'img2dataset',
+      return null
-    'ImagesiftBot',
+    }
-    'magpie-crawler',
+
-    'Meltwater',
+    robots = robots.concat(data)
-    'omgili',
+    if (data.length < PAGE_SIZE) break
-    'omgilibot',
+  }
-    'peer39_crawler',
+
-    'peer39_crawler/1.0',
+  return robots.map(robot => robot.user_agent)
-    'PerplexityBot',
+}
-    'PiplBot',
+
-    'scoop.it',
+export default async function () {
-    'Seekr',
+  return await fetchAllRobots()
-    'YouBot',
-  ]
 }

src/_includes/partials/home/status.liquid

@@ -2,7 +2,7 @@
 {%- assign book = books | bookStatus: 'started' | reverse | first -%}
 {%- assign show = tv.recentlyWatched | first -%}
 <div class="home-status">
-  <p><strong class="highlight-text">I'm a software developer based in Camarillo, California</strong>. I enjoy hanging out with my beautiful family and 3 rescue dogs, technology, automation, <a href="https://coryd.dev/music">music</a>, writing, <a href="https://coryd.dev/books">reading</a>, <a href="https://coryd.dev/watching#tv">tv</a> and <a href="https://coryd.dev/watching#movies">movies</a>. Lately I've been listening to a lot of <strong class="highlight-text">{{ artist.title }}</strong>, reading <strong class="highlight-text">{{ book.title }}</strong> and watching <strong class="highlight-text">{{ show.name }}</strong>.</p>
+  <p><strong class="highlight-text">I'm a software developer based in Camarillo, California</strong>. I enjoy hanging out with my beautiful family and 3 rescue dogs, technology, automation, <a href="/music">music</a>, <a href="/posts">writing</a>, <a href="/books">reading</a>, <a href="/watching#tv">tv</a> and <a href="/watching#movies">movies</a>. Lately I've been listening to a lot of <strong class="highlight-text">{{ artist.title }}</strong>, reading <strong class="highlight-text">{{ book.title }}</strong> and watching <strong class="highlight-text">{{ show.name }}</strong>.</p>
   <p>{{ status.emoji }} {{ status.content }}</p>
   {% render "partials/widgets/now-playing.liquid" %}
 </div>