parent
7f234cf321
commit
7f5d59c339
3 changed files with 38 additions and 63 deletions
13
package-lock.json
generated
13
package-lock.json
generated
|
@ -1,18 +1,17 @@
|
||||||
{
|
{
|
||||||
"name": "coryd.dev",
|
"name": "coryd.dev",
|
||||||
"version": "7.2.3",
|
"version": "7.2.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "coryd.dev",
|
"name": "coryd.dev",
|
||||||
"version": "7.2.3",
|
"version": "7.2.0",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@cdransf/api-text": "^1.2.2",
|
"@cdransf/api-text": "^1.2.2",
|
||||||
"@cdransf/theme-toggle": "^1.2.3",
|
"@cdransf/theme-toggle": "^1.2.3",
|
||||||
"@daviddarnes/mastodon-post": "^1.1.1",
|
"@daviddarnes/mastodon-post": "^1.1.1",
|
||||||
"@netlify/blobs": "^6.5.0",
|
|
||||||
"@remy/webmention": "^1.5.0",
|
"@remy/webmention": "^1.5.0",
|
||||||
"@zachleat/pagefind-search": "^1.0.3",
|
"@zachleat/pagefind-search": "^1.0.3",
|
||||||
"@zachleat/webcare-webshare": "^1.0.3"
|
"@zachleat/webcare-webshare": "^1.0.3"
|
||||||
|
@ -1769,14 +1768,6 @@
|
||||||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@netlify/blobs": {
|
|
||||||
"version": "6.5.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-6.5.0.tgz",
|
|
||||||
"integrity": "sha512-wRFlNnL/Qv3WNLZd3OT/YYqF1zb6iPSo8T31sl9ccL1ahBxW1fBqKgF4b1XL7Z+6mRIkatvcsVPkWBcO+oJMNA==",
|
|
||||||
"engines": {
|
|
||||||
"node": "^14.16.0 || >=16.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@nodelib/fs.scandir": {
|
"node_modules/@nodelib/fs.scandir": {
|
||||||
"version": "2.1.5",
|
"version": "2.1.5",
|
||||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||||
|
|
|
@ -24,7 +24,6 @@
|
||||||
"@cdransf/api-text": "^1.2.2",
|
"@cdransf/api-text": "^1.2.2",
|
||||||
"@cdransf/theme-toggle": "^1.2.3",
|
"@cdransf/theme-toggle": "^1.2.3",
|
||||||
"@daviddarnes/mastodon-post": "^1.1.1",
|
"@daviddarnes/mastodon-post": "^1.1.1",
|
||||||
"@netlify/blobs": "^6.5.0",
|
|
||||||
"@remy/webmention": "^1.5.0",
|
"@remy/webmention": "^1.5.0",
|
||||||
"@zachleat/pagefind-search": "^1.0.3",
|
"@zachleat/pagefind-search": "^1.0.3",
|
||||||
"@zachleat/webcare-webshare": "^1.0.3"
|
"@zachleat/webcare-webshare": "^1.0.3"
|
||||||
|
|
|
@ -1,15 +1,14 @@
|
||||||
import { getStore } from "@netlify/blobs";
|
|
||||||
import EleventyFetch from '@11ty/eleventy-fetch'
|
import EleventyFetch from '@11ty/eleventy-fetch'
|
||||||
// import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
|
import { S3Client, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
|
||||||
|
|
||||||
// const getReadableData = (readable) => {
|
const getReadableData = (readable) => {
|
||||||
// return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
// const chunks = []
|
const chunks = []
|
||||||
// readable.once('error', (err) => reject(err))
|
readable.once('error', (err) => reject(err))
|
||||||
// readable.on('data', (chunk) => chunks.push(chunk))
|
readable.on('data', (chunk) => chunks.push(chunk))
|
||||||
// readable.once('end', () => resolve(chunks.join('')))
|
readable.once('end', () => resolve(chunks.join('')))
|
||||||
// })
|
})
|
||||||
// }
|
}
|
||||||
|
|
||||||
const filterDuplicates = array => {
|
const filterDuplicates = array => {
|
||||||
const seenIds = new Set();
|
const seenIds = new Set();
|
||||||
|
@ -17,18 +16,17 @@ const filterDuplicates = array => {
|
||||||
};
|
};
|
||||||
|
|
||||||
export default async function () {
|
export default async function () {
|
||||||
// const client = new S3Client({
|
const client = new S3Client({
|
||||||
// credentials: {
|
credentials: {
|
||||||
// accessKeyId: process.env.ACCESS_KEY_B2,
|
accessKeyId: process.env.ACCESS_KEY_B2,
|
||||||
// secretAccessKey: process.env.SECRET_KEY_B2,
|
secretAccessKey: process.env.SECRET_KEY_B2,
|
||||||
// },
|
},
|
||||||
// endpoint: {
|
endpoint: {
|
||||||
// url: 'https://s3.us-west-001.backblazeb2.com',
|
url: 'https://s3.us-west-001.backblazeb2.com',
|
||||||
// },
|
},
|
||||||
// region: 'us-west-1',
|
region: 'us-west-1',
|
||||||
// })
|
})
|
||||||
// const BUCKET_B2 = process.env.BUCKET_B2
|
const BUCKET_B2 = process.env.BUCKET_B2
|
||||||
let READWISE_LINKS_STORE;
|
|
||||||
const API_TOKEN_READWISE = process.env.API_TOKEN_READWISE
|
const API_TOKEN_READWISE = process.env.API_TOKEN_READWISE
|
||||||
const formatLinkData = (links) => links.map((link) => {
|
const formatLinkData = (links) => links.map((link) => {
|
||||||
return {
|
return {
|
||||||
|
@ -50,21 +48,16 @@ export default async function () {
|
||||||
let cachedLinks;
|
let cachedLinks;
|
||||||
|
|
||||||
if (process.env.ELEVENTY_PRODUCTION) {
|
if (process.env.ELEVENTY_PRODUCTION) {
|
||||||
READWISE_LINKS_STORE = getStore("readwise-links");
|
const cachedLinksOutput = await client.send(
|
||||||
await READWISE_LINKS_STORE.get('cached-links');
|
new GetObjectCommand({
|
||||||
|
Bucket: BUCKET_B2,
|
||||||
|
Key: 'links.json',
|
||||||
|
})
|
||||||
|
)
|
||||||
|
const cachedLinksData = getReadableData(cachedLinksOutput.Body)
|
||||||
|
cachedLinks = await cachedLinksData.then((tracks) => JSON.parse(tracks)).catch()
|
||||||
}
|
}
|
||||||
|
|
||||||
// if (process.env.ELEVENTY_PRODUCTION) {
|
|
||||||
// const cachedLinksOutput = await client.send(
|
|
||||||
// new GetObjectCommand({
|
|
||||||
// Bucket: BUCKET_B2,
|
|
||||||
// Key: 'links.json',
|
|
||||||
// })
|
|
||||||
// )
|
|
||||||
// const cachedLinksData = getReadableData(cachedLinksOutput.Body)
|
|
||||||
// cachedLinks = await cachedLinksData.then((tracks) => JSON.parse(tracks)).catch()
|
|
||||||
// }
|
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
const queryParams = new URLSearchParams()
|
const queryParams = new URLSearchParams()
|
||||||
if (nextPageCursor) queryParams.append('pageCursor', nextPageCursor)
|
if (nextPageCursor) queryParams.append('pageCursor', nextPageCursor)
|
||||||
|
@ -84,27 +77,19 @@ export default async function () {
|
||||||
if (!nextPageCursor || requestCount === maxRequests) break;
|
if (!nextPageCursor || requestCount === maxRequests) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// if (process.env.ELEVENTY_PRODUCTION) {
|
|
||||||
// const mergedData = filterDuplicates([
|
|
||||||
// ...formatLinkData(fullData).filter((link) => link.tags.includes('share')),
|
|
||||||
// ...Object.values(cachedLinks)
|
|
||||||
// ]).sort((a,b) => new Date(b.date) - new Date(a.date));
|
|
||||||
|
|
||||||
// await client.send(
|
|
||||||
// new PutObjectCommand({
|
|
||||||
// Bucket: BUCKET_B2,
|
|
||||||
// Key: 'links.json',
|
|
||||||
// Body: JSON.stringify(mergedData),
|
|
||||||
// })
|
|
||||||
// )
|
|
||||||
// return mergedData
|
|
||||||
// }
|
|
||||||
if (process.env.ELEVENTY_PRODUCTION) {
|
if (process.env.ELEVENTY_PRODUCTION) {
|
||||||
const mergedData = filterDuplicates([
|
const mergedData = filterDuplicates([
|
||||||
...formatLinkData(fullData).filter((link) => link.tags.includes('share')),
|
...formatLinkData(fullData).filter((link) => link.tags.includes('share')),
|
||||||
...Object.values(cachedLinks)
|
...Object.values(cachedLinks)
|
||||||
]).sort((a,b) => new Date(b.date) - new Date(a.date));
|
]).sort((a,b) => new Date(b.date) - new Date(a.date));
|
||||||
await READWISE_LINKS_STORE.setJSON('cached-links', mergedData);
|
|
||||||
|
await client.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: BUCKET_B2,
|
||||||
|
Key: 'links.json',
|
||||||
|
Body: JSON.stringify(mergedData),
|
||||||
|
})
|
||||||
|
)
|
||||||
return mergedData
|
return mergedData
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Reference in a new issue