feat: move robots to cms

src/_data/robots.js

@@ -1,36 +1,34 @@
-export default async function () {
-  return [
-    'AdsBot-Google',
-    'Amazonbot',
-    'anthropic-ai',
-    'Applebot-Extended',
-    'AwarioRssBot',
-    'AwarioSmartBot',
-    'Bytespider',
-    'CCBot',
-    'ChatGPT-User',
-    'ClaudeBot',
-    'Claude-Web',
-    'cohere-ai',
-    'DataForSeoBot',
-    'Diffbot',
-    'FacebookBot',
-    'FriendlyCrawler',
-    'Google-Extended',
-    'GoogleOther',
-    'GPTBot',
-    'img2dataset',
-    'ImagesiftBot',
-    'magpie-crawler',
-    'Meltwater',
-    'omgili',
-    'omgilibot',
-    'peer39_crawler',
-    'peer39_crawler/1.0',
-    'PerplexityBot',
-    'PiplBot',
-    'scoop.it',
-    'Seekr',
-    'YouBot',
-  ]
+import { createClient } from '@supabase/supabase-js'
+
+const SUPABASE_URL = process.env.SUPABASE_URL
+const SUPABASE_KEY = process.env.SUPABASE_KEY
+const supabase = createClient(SUPABASE_URL, SUPABASE_KEY)
+
+const PAGE_SIZE = 100
+
+const fetchAllRobots = async () => {
+  let robots = []
+  let from = 0
+  let to = PAGE_SIZE - 1
+
+  while (true) {
+    const { data, error } = await supabase
+      .from('robots')
+      .select('user_agent')
+      .range(from, to)
+
+    if (error) {
+      console.error('Error fetching robot data:', error)
+      return null
+    }
+
+    robots = robots.concat(data)
+    if (data.length < PAGE_SIZE) break
+  }
+
+  return robots.map(robot => robot.user_agent)
+}
+
+export default async function () {
+  return await fetchAllRobots()
 }