diff --git a/package-lock.json b/package-lock.json index 50c4d649..d715d684 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,18 @@ { "name": "coryd.dev", - "version": "7.2.0", + "version": "7.2.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "coryd.dev", - "version": "7.2.0", + "version": "7.2.3", "license": "MIT", "dependencies": { "@cdransf/api-text": "^1.2.2", "@cdransf/theme-toggle": "^1.2.3", "@daviddarnes/mastodon-post": "^1.1.1", + "@netlify/blobs": "^6.5.0", "@remy/webmention": "^1.5.0", "@zachleat/pagefind-search": "^1.0.3", "@zachleat/webcare-webshare": "^1.0.3" @@ -1768,6 +1769,14 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@netlify/blobs": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz", + "integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==", + "engines": { + "node": "^14.16.0 || >=16.0.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/package.json b/package.json index 7fc31696..ece8e973 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@cdransf/api-text": "^1.2.2", "@cdransf/theme-toggle": "^1.2.3", "@daviddarnes/mastodon-post": "^1.1.1", + "@netlify/blobs": "^6.5.0", "@remy/webmention": "^1.5.0", "@zachleat/pagefind-search": "^1.0.3", "@zachleat/webcare-webshare": "^1.0.3" diff --git a/src/_data/links.js b/src/_data/links.js index 8dbc9166..be35bbf7 100644 --- a/src/_data/links.js +++ b/src/_data/links.js @@ -1,14 +1,15 @@ +import { getStore } from "@netlify/blobs"; import EleventyFetch from '@11ty/eleventy-fetch' -import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3' +// import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3' -const getReadableData = (readable) => { - return new Promise((resolve, reject) => { - const chunks = [] - readable.once('error', (err) => reject(err)) - readable.on('data', (chunk) => chunks.push(chunk)) - readable.once('end', () => resolve(chunks.join(''))) - }) -} +// const getReadableData = (readable) => { +// return new Promise((resolve, reject) => { +// const chunks = [] +// readable.once('error', (err) => reject(err)) +// readable.on('data', (chunk) => chunks.push(chunk)) +// readable.once('end', () => resolve(chunks.join(''))) +// }) +// } const filterDuplicates = array => { const seenIds = new Set(); @@ -16,17 +17,18 @@ const filterDuplicates = array => { }; export default async function () { - const client = new S3Client({ - credentials: { - accessKeyId: process.env.ACCESS_KEY_B2, - secretAccessKey: process.env.SECRET_KEY_B2, - }, - endpoint: { - url: 'https://s3.us-west-001.backblazeb2.com', - }, - region: 'us-west-1', - }) - const BUCKET_B2 = process.env.BUCKET_B2 + // const client = new S3Client({ + // credentials: { + // accessKeyId: process.env.ACCESS_KEY_B2, + // secretAccessKey: process.env.SECRET_KEY_B2, + // }, + // endpoint: { + // url: 'https://s3.us-west-001.backblazeb2.com', + // }, + // region: 'us-west-1', + // }) + // const BUCKET_B2 = process.env.BUCKET_B2 + let READWISE_LINKS_STORE; const API_TOKEN_READWISE = process.env.API_TOKEN_READWISE const formatLinkData = (links) => links.map((link) => { return { @@ -48,16 +50,21 @@ export default async function () { let cachedLinks; if (process.env.ELEVENTY_PRODUCTION) { - const cachedLinksOutput = await client.send( - new GetObjectCommand({ - Bucket: BUCKET_B2, - Key: 'links.json', - }) - ) - const cachedLinksData = getReadableData(cachedLinksOutput.Body) - cachedLinks = await cachedLinksData.then((tracks) => JSON.parse(tracks)).catch() + READWISE_LINKS_STORE = getStore("readwise-links"); + await READWISE_LINKS_STORE.get('cached-links'); } + // if (process.env.ELEVENTY_PRODUCTION) { + // const cachedLinksOutput = await client.send( + // new GetObjectCommand({ + // Bucket: BUCKET_B2, + // Key: 'links.json', + // }) + // ) + // const cachedLinksData = getReadableData(cachedLinksOutput.Body) + // cachedLinks = await cachedLinksData.then((tracks) => JSON.parse(tracks)).catch() + // } + while (true) { const queryParams = new URLSearchParams() if (nextPageCursor) queryParams.append('pageCursor', nextPageCursor) @@ -77,19 +84,27 @@ export default async function () { if (!nextPageCursor || requestCount === maxRequests) break; } + // if (process.env.ELEVENTY_PRODUCTION) { + // const mergedData = filterDuplicates([ + // ...formatLinkData(fullData).filter((link) => link.tags.includes('share')), + // ...Object.values(cachedLinks) + // ]).sort((a,b) => new Date(b.date) - new Date(a.date)); + + // await client.send( + // new PutObjectCommand({ + // Bucket: BUCKET_B2, + // Key: 'links.json', + // Body: JSON.stringify(mergedData), + // }) + // ) + // return mergedData + // } if (process.env.ELEVENTY_PRODUCTION) { const mergedData = filterDuplicates([ ...formatLinkData(fullData).filter((link) => link.tags.includes('share')), ...Object.values(cachedLinks) ]).sort((a,b) => new Date(b.date) - new Date(a.date)); - - await client.send( - new PutObjectCommand({ - Bucket: BUCKET_B2, - Key: 'links.json', - Body: JSON.stringify(mergedData), - }) - ) + await READWISE_LINKS_STORE.setJSON('cached-links', mergedData); return mergedData }