cdransf/coryd.dev-eleventy
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test: netlify blobs

Browse source
This commit is contained in:
Cory Dransfeldt 2024-03-03 15:42:02 -08:00
parent 4dc8f0dcd1
commit a2c2669f28
No known key found for this signature in database
3 changed files with 63 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
package-lock.json generated
View file

@ -1,17 +1,18 @@
{
"name": "coryd.dev",
"version": "7.2.0",
"version": "7.2.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "coryd.dev",
"version": "7.2.0",
"version": "7.2.3",
"license": "MIT",
"dependencies": {
"@cdransf/api-text": "^1.2.2",
"@cdransf/theme-toggle": "^1.2.3",
"@daviddarnes/mastodon-post": "^1.1.1",
"@netlify/blobs": "^6.5.0",
"@remy/webmention": "^1.5.0",
"@zachleat/pagefind-search": "^1.0.3",
"@zachleat/webcare-webshare": "^1.0.3"
@ -1768,6 +1769,14 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@netlify/blobs": {
"version": "6.5.0",
"resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz",
"integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==",
"engines": {
"node": "^14.16.0 || >=16.0.0"
}
},
"node_modules/@nodelib/fs.scandir": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

1
package.json
View file

@ -24,6 +24,7 @@
"@cdransf/api-text": "^1.2.2",
"@cdransf/theme-toggle": "^1.2.3",
"@daviddarnes/mastodon-post": "^1.1.1",
"@netlify/blobs": "^6.5.0",
"@remy/webmention": "^1.5.0",
"@zachleat/pagefind-search": "^1.0.3",
"@zachleat/webcare-webshare": "^1.0.3"

87
src/_data/links.js
View file

@ -1,14 +1,15 @@
import { getStore } from "@netlify/blobs";
import EleventyFetch from '@11ty/eleventy-fetch'
import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
// import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
const getReadableData = (readable) => {
return new Promise((resolve, reject) => {
const chunks = []
readable.once('error', (err) => reject(err))
readable.on('data', (chunk) => chunks.push(chunk))
readable.once('end', () => resolve(chunks.join('')))
})
}
// const getReadableData = (readable) => {
// return new Promise((resolve, reject) => {
// const chunks = []
// readable.once('error', (err) => reject(err))
// readable.on('data', (chunk) => chunks.push(chunk))
// readable.once('end', () => resolve(chunks.join('')))
// })
// }
const filterDuplicates = array => {
const seenIds = new Set();
@ -16,17 +17,18 @@ const filterDuplicates = array => {
};
export default async function () {
const client = new S3Client({
credentials: {
accessKeyId: process.env.ACCESS_KEY_B2,
secretAccessKey: process.env.SECRET_KEY_B2,
},
endpoint: {
url: 'https://s3.us-west-001.backblazeb2.com',
},
region: 'us-west-1',
})
const BUCKET_B2 = process.env.BUCKET_B2
// const client = new S3Client({
// credentials: {
// accessKeyId: process.env.ACCESS_KEY_B2,
// secretAccessKey: process.env.SECRET_KEY_B2,
// },
// endpoint: {
// url: 'https://s3.us-west-001.backblazeb2.com',
// },
// region: 'us-west-1',
// })
// const BUCKET_B2 = process.env.BUCKET_B2
let READWISE_LINKS_STORE;
const API_TOKEN_READWISE = process.env.API_TOKEN_READWISE
const formatLinkData = (links) => links.map((link) => {
return {
@ -48,16 +50,21 @@ export default async function () {
let cachedLinks;
if (process.env.ELEVENTY_PRODUCTION) {
const cachedLinksOutput = await client.send(
new GetObjectCommand({
Bucket: BUCKET_B2,
Key: 'links.json',
})
)
const cachedLinksData = getReadableData(cachedLinksOutput.Body)
cachedLinks = await cachedLinksData.then((tracks) => JSON.parse(tracks)).catch()
READWISE_LINKS_STORE = getStore("readwise-links");
await READWISE_LINKS_STORE.get('cached-links');
}
// if (process.env.ELEVENTY_PRODUCTION) {
// const cachedLinksOutput = await client.send(
// new GetObjectCommand({
// Bucket: BUCKET_B2,
// Key: 'links.json',
// })
// )
// const cachedLinksData = getReadableData(cachedLinksOutput.Body)
// cachedLinks = await cachedLinksData.then((tracks) => JSON.parse(tracks)).catch()
// }
while (true) {
const queryParams = new URLSearchParams()
if (nextPageCursor) queryParams.append('pageCursor', nextPageCursor)
@ -77,19 +84,27 @@ export default async function () {
if (!nextPageCursor || requestCount === maxRequests) break;
}
// if (process.env.ELEVENTY_PRODUCTION) {
// const mergedData = filterDuplicates([
// ...formatLinkData(fullData).filter((link) => link.tags.includes('share')),
// ...Object.values(cachedLinks)
// ]).sort((a,b) => new Date(b.date) - new Date(a.date));
// await client.send(
// new PutObjectCommand({
// Bucket: BUCKET_B2,
// Key: 'links.json',
// Body: JSON.stringify(mergedData),
// })
// )
// return mergedData
// }
if (process.env.ELEVENTY_PRODUCTION) {
const mergedData = filterDuplicates([
...formatLinkData(fullData).filter((link) => link.tags.includes('share')),
...Object.values(cachedLinks)
]).sort((a,b) => new Date(b.date) - new Date(a.date));
await client.send(
new PutObjectCommand({
Bucket: BUCKET_B2,
Key: 'links.json',
Body: JSON.stringify(mergedData),
})
)
await READWISE_LINKS_STORE.setJSON('cached-links', mergedData);
return mergedData
}