feat: dynamic pages to sitemap

astro.config.mjs

@@ -1,12 +1,17 @@
 import { defineConfig } from "astro/config";
 import cloudflare from "@astrojs/cloudflare";
 import sitemap from "@astrojs/sitemap";
+import sitemapData from "./src/utils/data/static/sitemapData.json";
 
 export default defineConfig({
   site: "https://coryd.dev",
   output: "server",
   adapter: cloudflare(),
-  integrations: [sitemap()],
+  integrations: [
+    sitemap({
+      customPages: sitemapData.map((entry) => entry.loc),
+    }),
+  ],
   prefetch: true,
   vite: {
     build: {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "coryd.dev",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "coryd.dev",
-      "version": "1.2.0",
+      "version": "1.3.0",
       "dependencies": {
         "@astrojs/check": "0.9.4",
         "@astrojs/cloudflare": "^11.2.0",
@@ -27,6 +27,7 @@
         "youtube-video-element": "^1.1.6"
       },
       "devDependencies": {
+        "dotenv": "16.4.5",
         "dotenv-flow": "4.1.0",
         "fast-xml-parser": "4.5.0",
         "html-to-text": "9.0.5",
@@ -63,9 +64,9 @@
       }
     },
     "node_modules/@astro-community/astro-embed-bluesky": {
-      "version": "0.1.1",
+      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/@astro-community/astro-embed-bluesky/-/astro-embed-bluesky-0.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/@astro-community/astro-embed-bluesky/-/astro-embed-bluesky-0.1.2.tgz",
-      "integrity": "sha512-IOm05SUyEFtXCXpeIxbTp1650s6Q46eXs4n2CBCCDvuRYtmHBGUqwPwoFOD3sO9zg+lTfFY/Va9Le8o8Wp22NQ==",
+      "integrity": "sha512-DBTRxNOJuQO9BWLzML80lL92mA1vpc3TYkvTZPMCTTDF69szWpgzKBGUVbq7NZCbN4oigfj6IxJHLa3tBmbLyg==",
       "license": "MIT",
       "dependencies": {
         "@atproto/api": "^0.13.14",
@@ -2348,9 +2349,9 @@
       }
     },
     "node_modules/@types/node": {
-      "version": "22.9.3",
+      "version": "22.9.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.4.tgz",
-      "integrity": "sha512-F3u1fs/fce3FFk+DAxbxc78DF8x0cY09RRL8GnXLmkJ1jvx3TtPdWoTT5/NiYfI5ASqXBmfqJi9dZ3gxMx4lzw==",
+      "integrity": "sha512-d9RWfoR7JC/87vj7n+PVTzGg9hDyuFjir3RxUHbjFSKNd9mpxbxwMEyaCim/ddCmy4IuW7HjTzF3g9p3EtWEOg==",
       "license": "MIT",
       "dependencies": {
         "undici-types": "~6.19.8"
@@ -2366,9 +2367,9 @@
       }
     },
     "node_modules/@types/phoenix": {
-      "version": "1.6.5",
+      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.5.tgz",
+      "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.6.tgz",
-      "integrity": "sha512-xegpDuR+z0UqG9fwHqNoy3rI7JDlvaPh2TY47Fl80oq6g+hXT+c/LEuE43X48clZ6lOfANl5WrPur9fYO1RJ/w==",
+      "integrity": "sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A==",
       "license": "MIT"
     },
     "node_modules/@types/sax": {
@@ -2755,9 +2756,9 @@
       }
     },
     "node_modules/astro-auto-import/node_modules/@types/node": {
-      "version": "18.19.65",
+      "version": "18.19.66",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.65.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.66.tgz",
-      "integrity": "sha512-Ay5BZuO1UkTmVHzZJNvZKw/E+iB3GQABb6kijEz89w2JrfhNA+M/ebp18pfz9Gqe9ywhMC8AA8yC01lZq48J+Q==",
+      "integrity": "sha512-14HmtUdGxFUalGRfLLn9Gc1oNWvWh5zNbsyOLo5JV6WARSeN1QcEBKRnZm9QqNfrutgsl/hY4eJW63aZ44aBCg==",
       "license": "MIT",
       "dependencies": {
         "undici-types": "~5.26.4"
@@ -3619,9 +3620,9 @@
       "license": "MIT"
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.5.64",
+      "version": "1.5.65",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.64.tgz",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.65.tgz",
-      "integrity": "sha512-IXEuxU+5ClW2IGEYFC2T7szbyVgehupCWQe5GNh+H065CD6U6IFN0s4KeAMFGNmQolRU4IV7zGBWSYMmZ8uuqQ==",
+      "integrity": "sha512-PWVzBjghx7/wop6n22vS2MLU8tKGd4Q91aCEGhG/TYmW6PP5OcSXcdnxTe1NNt0T66N8D6jxh4kC8UsdzOGaIw==",
       "license": "ISC"
     },
     "node_modules/emmet": {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "coryd.dev",
   "type": "module",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "scripts": {
     "dev": "astro dev",
     "start": "astro dev",
@@ -9,6 +9,8 @@
     "preview": "wrangler pages dev ./dist",
     "astro": "astro",
     "clean": "rimraf dist",
+    "sitemap:generate": "node scripts/sitemap-generate.mjs",
+    "update": "npm upgrade && ncu && npx @astrojs/upgrade && npm i && node scripts/sitemap-generate.mjs",
     "update:deps": "npm upgrade && ncu && npx @astrojs/upgrade && npm i",
     "build:worker": "node scripts/worker-build.mjs $WORKER_NAME",
     "deploy:worker": "wrangler deploy --env production --config workers/$npm_config_worker/wrangler.toml"
@@ -33,6 +35,7 @@
     "youtube-video-element": "^1.1.6"
   },
   "devDependencies": {
+    "dotenv": "16.4.5",
     "dotenv-flow": "4.1.0",
     "fast-xml-parser": "4.5.0",
     "html-to-text": "9.0.5",

queries/views/feeds/all-content.psql

@@ -23,49 +23,57 @@ WITH feed_data AS (
         json_build_object('title', CONCAT(l.title, ' via ', l.name), 'url', l.link, 'description', l.description, 'date', l.date) AS feed
     FROM
         optimized_links l
-UNION ALL
+    UNION ALL
-SELECT
+    SELECT
-    b.date_finished AS content_date,
+        b.date_finished AS content_date,
-    b.title,
+        b.title,
-    b.description,
+        b.description,
-    CONCAT('https://coryd.dev', b.url) AS url,
+        CONCAT('https://coryd.dev', b.url) AS url,
-    b.image,
+        b.image,
-    b.rating,
+        b.rating,
-    b.tags,
+        b.tags,
-    CASE WHEN LOWER(b.status) = 'finished' THEN
+        CASE WHEN LOWER(b.status) = 'finished' THEN
-        json_build_object('title', b.title, 'url', CONCAT('https://coryd.dev', b.url), 'description', CASE WHEN b.review IS NOT NULL THEN
+            json_build_object('title', CONCAT(b.title, ' by ', b.author, CASE WHEN b.rating IS NOT NULL THEN
-                b.review
+                        CONCAT(' (', b.rating, ')')
-            ELSE
+                    ELSE
-                b.description
+                        ''
-            END, 'image', b.image, 'rating', b.rating, 'date', b.date_finished)
+                    END), 'url', CONCAT('https://coryd.dev', b.url), 'description', CASE WHEN b.review IS NOT NULL THEN
-    ELSE
+                    b.review
-        NULL
+                ELSE
-    END AS feed
+                    b.description
-FROM
+                END, 'image', b.image, 'rating', b.rating, 'date', b.date_finished)
-    optimized_books b
+        ELSE
-UNION ALL
+            NULL
-SELECT
+        END AS feed
-    m.last_watched AS content_date,
+    FROM
-    m.title,
+        optimized_books b
-    m.description,
+    UNION ALL
-    CONCAT('https://coryd.dev', m.url) AS url,
+    SELECT
-    m.image,
+        m.last_watched AS content_date,
-    m.rating,
+        m.title,
-    m.tags,
+        m.description,
-    CASE WHEN m.last_watched IS NOT NULL THEN
+        CONCAT('https://coryd.dev', m.url) AS url,
-        json_build_object('title', m.title, 'url', CONCAT('https://coryd.dev', m.url), 'description', CASE WHEN m.review IS NOT NULL THEN
+        m.image,
-                m.review
+        m.rating,
-            ELSE
+        m.tags,
-                m.description
+        CASE WHEN m.last_watched IS NOT NULL THEN
-            END, 'image', m.backdrop, 'rating', m.rating, 'date', m.last_watched)
+            json_build_object('title', CONCAT(m.title, CASE WHEN m.rating IS NOT NULL THEN
-    ELSE
+                        CONCAT(' (', m.rating, ')')
-        NULL
+                    ELSE
-    END AS feed
+                        ''
-FROM
+                    END), 'url', CONCAT('https://coryd.dev', m.url), 'description', CASE WHEN m.review IS NOT NULL THEN
-    optimized_movies m
+                    m.review
+                ELSE
+                    m.description
+                END, 'image', m.backdrop, 'rating', m.rating, 'date', m.last_watched)
+        ELSE
+            NULL
+        END AS feed
+    FROM
+        optimized_movies m
 )
 SELECT
-    json_agg(feed_data.* ORDER BY feed_data.content_date DESC) AS feed
+    json_agg(feed.* ORDER BY feed.content_date DESC) AS feed
 FROM (
     SELECT
         *
@@ -75,5 +83,5 @@ FROM (
         feed IS NOT NULL
     ORDER BY
         content_date DESC
-    LIMIT 20) AS feed_data;
+    LIMIT 20) AS feed;
 

queries/views/media/books.psql

@@ -78,7 +78,11 @@ json_build_object('title', CONCAT(b.title, ' by ', b.author), 'image', CONCAT('/
   END) AS grid,
 CASE WHEN LOWER(b.read_status) = 'finished'
   AND b.star_rating IS NOT NULL THEN
-  json_build_object('title', b.title, 'url', CONCAT('https://coryd.dev', b.slug), 'date', b.date_finished, 'description', CASE WHEN b.review IS NOT NULL THEN
+  json_build_object('title', CONCAT(b.title, 'by ', b.author, CASE WHEN b.star_rating IS NOT NULL THEN
+        CONCAT(' (', b.star_rating, ')')
+      ELSE
+        ''
+      END), 'url', CONCAT('https://coryd.dev', b.slug), 'date', b.date_finished, 'description', CASE WHEN b.review IS NOT NULL THEN
       b.review
     ELSE
       b.description

queries/views/media/movies.psql

@@ -83,7 +83,11 @@ WHERE
   r.movies_id = m.id) AS related_movies,
 CASE WHEN m.star_rating IS NOT NULL
   AND m.last_watched IS NOT NULL THEN
-  json_build_object('title', m.title, 'url', CONCAT('https://coryd.dev', m.slug), 'date', m.last_watched, 'description', CASE WHEN m.review IS NOT NULL THEN
+  json_build_object('title', CONCAT(m.title, CASE WHEN m.star_rating IS NOT NULL THEN
+        CONCAT(' (', m.star_rating, ')')
+      ELSE
+        ''
+      END), 'url', CONCAT('https://coryd.dev', m.slug), 'date', m.last_watched, 'description', CASE WHEN m.review IS NOT NULL THEN
       m.review
     ELSE
       m.description

queries/views/media/music/artists.psql

@@ -3,7 +3,7 @@ SELECT
     ar.name_string AS name,
     ar.slug AS url,
     ar.tentative,
-    to_char(ar.total_plays, 'FM999,999,999,999') AS total_plays, -- Format total_plays with commas
+    to_char(ar.total_plays, 'FM999,999,999,999') AS total_plays,
     ar.country,
     ar.description,
     ar.favorite,
@@ -14,12 +14,12 @@ SELECT
     ar.emoji,
     ar.tattoo,
     CONCAT('/', df.filename_disk) AS image,
-    json_build_object('alt', CONCAT(to_char(ar.total_plays, 'FM999,999,999,999'), ' plays of ', ar.name_string), -- Format total_plays in alt text
+    json_build_object('alt', CONCAT(to_char(ar.total_plays, 'FM999,999,999,999'), ' plays of ', ar.name_string),
-        'subtext', CONCAT(to_char(ar.total_plays, 'FM999,999,999,999'), ' plays') -- Format total_plays in subtext
+        'subtext', CONCAT(to_char(ar.total_plays, 'FM999,999,999,999'), ' plays')
 ) AS grid,
 (
         SELECT
-            json_agg(json_build_object('name', a.name, 'release_year', a.release_year, 'total_plays', to_char(a.total_plays, 'FM999,999,999,999'), -- Format total_plays here as well
+            json_agg(json_build_object('name', a.name, 'release_year', a.release_year, 'total_plays', to_char(a.total_plays, 'FM999,999,999,999'),
                     'art', df_album.filename_disk)
             ORDER BY a.release_year)
         FROM

scripts/sitemap-generate.mjs

@@ -0,0 +1,52 @@
+import { createClient } from "@supabase/supabase-js";
+import { writeFileSync } from "fs";
+import dotenv from "dotenv";
+
+dotenv.config({ path: ".env.local" });
+
+const SUPABASE_URL = process.env.SUPABASE_URL;
+const SUPABASE_KEY = process.env.SUPABASE_KEY;
+const supabase = createClient(SUPABASE_URL, SUPABASE_KEY);
+
+const PAGE_SIZE = 1000;
+
+async function fetchPaginatedData() {
+  let allEntries = [];
+  let currentPage = 0;
+
+  while (true) {
+    const { data, error } = await supabase
+      .from("optimized_sitemap")
+      .select("url, title, lastmod, changefreq, priority")
+      .range(currentPage * PAGE_SIZE, (currentPage + 1) * PAGE_SIZE - 1);
+
+    if (error) throw new Error(`Error fetching sitemap data: ${error.message}`);
+
+    if (!data || data.length === 0) break;
+
+    allEntries = allEntries.concat(data);
+    currentPage += 1;
+  }
+
+  return allEntries;
+}
+
+async function generateSitemap() {
+  try {
+    const data = await fetchPaginatedData();
+
+    const sitemapEntries = data.map((entry) => ({ loc: entry.url }));
+
+    writeFileSync(
+      "src/utils/data/static/sitemapData.json",
+      JSON.stringify(sitemapEntries, null, 2),
+      "utf8"
+    );
+
+    console.log("Sitemap data generated successfully!");
+  } catch (error) {
+    console.error("Error generating sitemap:", error);
+  }
+}
+
+generateSitemap();