feat: cache links to b2

.env

@@ -1,3 +1,6 @@
+ACCESS_KEY_B2=
+SECRET_KEY_B2=
+BUCKET_B2=
 API_KEY_LASTFM=
 API_KEY_PLAUSIBLE=
 API_KEY_TRAKT=

package.json

@@ -30,6 +30,7 @@
     "@11ty/eleventy-img": "^4.0.2",
     "@11ty/eleventy-plugin-rss": "^1.2.0",
     "@11ty/eleventy-plugin-syntaxhighlight": "^5.0.0",
+    "@aws-sdk/client-s3": "^3.373.0",
     "@rknightuk/eleventy-plugin-post-graph": "^1.0.6",
     "child_process": "^1.0.2",
     "dotenv-flow": "^4.1.0",

src/_data/links.js

@@ -1,6 +1,27 @@
 import EleventyFetch from '@11ty/eleventy-fetch'
+import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
+
+const getReadableData = (readable) => {
+  return new Promise((resolve, reject) => {
+    const chunks = []
+    readable.once('error', (err) => reject(err))
+    readable.on('data', (chunk) => chunks.push(chunk))
+    readable.once('end', () => resolve(chunks.join('')))
+  })
+}
 
 export default async function () {
+   const client = new S3Client({
+     credentials: {
+       accessKeyId: process.env.ACCESS_KEY_B2,
+       secretAccessKey: process.env.SECRET_KEY_B2,
+     },
+     endpoint: {
+       url: 'https://s3.us-west-001.backblazeb2.com',
+     },
+     region: 'us-west-1',
+   })
+  const BUCKET_B2 = process.env.BUCKET_B2
   const API_TOKEN_READWISE = process.env.API_TOKEN_READWISE
   const formatLinkData = (links) => links.map((link) => {
     return {
@@ -15,9 +36,21 @@ export default async function () {
     }
   })
   const fullData = [];
+  const maxRequests = process.env.ELEVENTY_PRODUCTION ? 10 : 2
   let nextPageCursor = null;
   let requestCount = 0;
 
+  if (process.env.ELEVENTY_PRODUCTION) {
+    const cachedLinksOutput = await client.send(
+        new GetObjectCommand({
+          Bucket: BUCKET_B2,
+          Key: 'links.json',
+       })
+    )
+    const cachedLinksData = getReadableData(cachedLinksOutput.Body)
+    const cachedLinks = await cachedLinksData.then((tracks) => JSON.parse(tracks)).catch()
+  }
+
   while (true) {
     const queryParams = new URLSearchParams()
     if (nextPageCursor) queryParams.append('pageCursor', nextPageCursor)
@@ -34,7 +67,24 @@ export default async function () {
     fullData.push(...data['results']);
     nextPageCursor = data['nextPageCursor']
     requestCount++;
-    if (!nextPageCursor || requestCount === 5) break;
+    if (!nextPageCursor || requestCount === maxRequests) break;
+  }
+
+  if (process.env.ELEVENTY_PRODUCTION) {
+    const mergedData = [...new Set([
+      ...Object.values(cachedLinks),
+      ...formatLinkData(fullData).filter((link) => link.tags.includes('share'))
+    ])]
+
+    await client.send(
+      new PutObjectCommand({
+        Bucket: BUCKET_B2,
+        Key: 'links.json',
+        Body: JSON.stringify(mergedData),
+      })
+     )
+
+    return mergedData
   }
 
   return formatLinkData(fullData).filter((link) => link.tags.includes('share'))