feat: music caching v1.1.0

.env

@@ -1,3 +1,6 @@
+ACCESS_KEY_WASABI=
+SECRET_KEY_WASABI=
+BUCKET_WASABI=
 API_KEY_PLAUSIBLE=
 API_KEY_TRAKT=
 API_KEY_WEBMENTIONS_CORYD_DEV=

package.json

@@ -21,6 +21,7 @@
     "@11ty/eleventy-fetch": "^4.0.0",
     "@11ty/eleventy-img": "^3.1.0",
     "@11ty/eleventy-plugin-syntaxhighlight": "^5.0.0",
+    "@aws-sdk/client-s3": "^3.377.0",
     "@catppuccin/tailwindcss": "^0.1.6",
     "@commitlint/cli": "^17.6.6",
     "@commitlint/config-conventional": "^17.6.6",
@@ -47,6 +48,7 @@
     "ics-to-json-extended": "^1.1.4",
     "lint-staged": "^13.2.3",
     "liquidjs": "^10.8.3",
+    "lodash": "^4.17.21",
     "luxon": "^3.3.0",
     "markdown-it": "^13.0.1",
     "markdown-it-anchor": "^8.4.1",

src/_data/json/mocks/music.json

@@ -0,0 +1,12 @@
+{
+  "i.rXXXdmUa6Nme-1689970612847": {
+    "name": "Sacrificial Blood Oath In The Temple Of K'zadu",
+    "artist": "Gateway",
+    "album": "Galgendood",
+    "art": "https://store-033.blobstore.apple.com/sq-mq-us-033-000002/18/f1/a3/18f1a37a-8c9a-169a-5458-464aea20ce05/image?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20230721T202228Z&X-Amz-SignedHeaders=host&X-Amz-Expires=86400&X-Amz-Credential=MKIAU0HKO2RBEAT0UMZS%2F20230721%2Fstore-033%2Fs3%2Faws4_request&X-Amz-Signature=85790600221880597074559ed3674564f17ca3df6634d6fa15496baf7aca5d56",
+    "url": "https://rateyourmusic.com/search?searchtype=l&searchterm=Galgendood%20Gateway",
+    "id": "i.rXXXdmUa6Nme",
+    "playTime": 1689970612847,
+    "duration": 338808
+  }
+}

src/_data/json/title-case-exceptions.json

@@ -1,5 +1,5 @@
 {
-  "exceptions": [
+  "words": [
     "a",
     "and",
     "but",
@@ -15,5 +15,6 @@
     "so",
     "the",
     "yet"
-  ]
+  ],
+  "artists": ["NoMeansNo"]
 }

src/_data/music.js

@@ -1,8 +1,98 @@
-const { AssetCache } = require('@11ty/eleventy-fetch')
-const { aliasArtist, sanitizeMedia, sortByPlays } = require('../utils/media')
+const { S3Client, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3')
+const _ = require('lodash')
+const mockedMusic = require('./json/mocks/music.json')
+const { getReadableData } = require('../utils/aws')
+const { aliasArtist, sanitizeMedia } = require('../utils/media')
 const { titleCase } = require('../utils/grammar')
 
+const diffTracks = (cache, tracks) => {
+  const trackCompareSet = Object.values(tracks)
+  const cacheCompareSet = _.orderBy(Object.values(cache), ['time'], ['desc'])
+  const diffedTracks = {}
+  const comparedTracks = _.differenceWith(trackCompareSet, cacheCompareSet, (a, b) =>
+    _.isEqual(a.id, b.id)
+  )
+
+  for (let i = 0; i < comparedTracks.length; i++)
+    diffedTracks[`${comparedTracks[i]?.id}-${comparedTracks[i].playTime}`] = comparedTracks[i]
+
+  return diffedTracks
+}
+
+const formatTracks = (tracks) => {
+  let formattedTracks = {}
+  let time = new Date().getTime()
+
+  Object.values(tracks).forEach((track) => {
+    const artistFormatted = titleCase(aliasArtist(track.attributes['artistName']))
+    const albumFormatted = titleCase(sanitizeMedia(track.attributes['albumName']))
+    const trackFormatted = sanitizeMedia(track.attributes['name'])
+    formattedTracks[`${track.id}-${time}`] = {
+      name: trackFormatted,
+      artist: artistFormatted,
+      album: albumFormatted,
+      genre: track['relationships']?.['library'].data[0]?.attributes['genreNames'][0] || '',
+      art: track.attributes.artwork.url.replace('{w}', '300').replace('{h}', '300'),
+      url:
+        track['relationships'] && track['relationships'].albums.data.length > 0
+          ? `https://song.link/${track['relationships'].albums.data.pop().attributes.url}`
+          : `https://rateyourmusic.com/search?searchtype=l&searchterm=${encodeURI(
+              albumFormatted
+            )}%20${encodeURI(artistFormatted)}`,
+      id: track.id,
+      playTime: time - parseInt(track.attributes['durationInMillis']),
+      duration: parseInt(track.attributes['durationInMillis']),
+    }
+  })
+  return formattedTracks
+}
+
+const deriveCharts = (tracks) => {
+  const charts = {
+    artists: {},
+    albums: {},
+  }
+
+  Object.values(tracks).forEach((track) => {
+    if (!charts.artists[track.artist]) {
+      charts.artists[track.artist] = {
+        artist: track.artist,
+        genre: track.genre,
+        url: `https://rateyourmusic.com/search?searchterm=${encodeURI(track.artist)}`,
+        plays: 1,
+      }
+    } else {
+      charts.artists[track.artist].plays++
+    }
+
+    if (!charts.albums[track.album]) {
+      charts.albums[track.album] = {
+        name: track.album,
+        artist: track.artist,
+        art: track.art,
+        url: track.url,
+        plays: 1,
+      }
+    } else {
+      charts.albums[track.album].plays++
+    }
+  })
+
+  return charts
+}
+
 module.exports = async function () {
+  const client = new S3Client({
+    credentials: {
+      accessKeyId: process.env.ACCESS_KEY_WASABI,
+      secretAccessKey: process.env.SECRET_KEY_WASABI,
+    },
+    endpoint: {
+      url: 'https://s3.us-west-1.wasabisys.com',
+    },
+    region: 'us-west-1',
+  })
+  const WASABI_BUCKET = process.env.BUCKET_WASABI
   const APPLE_BEARER = process.env.API_BEARER_APPLE_MUSIC
   const APPLE_MUSIC_TOKEN = process.env.API_TOKEN_APPLE_MUSIC
   const APPLE_TOKEN_RESPONSE = await fetch(process.env.APPLE_RENEW_TOKEN_URL, {
@@ -16,19 +106,20 @@ module.exports = async function () {
   })
     .then((data) => data.json())
     .catch()
+
+  const DATE = new Date()
+  DATE.setDate(DATE.getDate() + ((7 - DATE.getDay()) % 7))
+  const DATE_STAMP = `${DATE.getFullYear()}-${DATE.getDate()}-${DATE.getMonth()}`
+
   const APPLE_TOKEN = APPLE_TOKEN_RESPONSE['music-token']
-  const asset = new AssetCache('recent_tracks_data')
   const PAGE_SIZE = 30
   const PAGES = 10
-  const charts = {
-    artists: {},
-    albums: {},
-  }
-  let CURRENT_PAGE = 0
-  let res = []
-  let hasNextPage = true
 
-  if (asset.isCacheValid('1h')) return await asset.getCachedValue()
+  let charts
+  let CURRENT_PAGE = 0
+  let hasNextPage = true
+  let res = []
+  let cachedTracks = mockedMusic
 
   while (CURRENT_PAGE < PAGES && hasNextPage) {
     const URL = `https://api.music.apple.com/v1/me/recent/played/tracks?limit=${PAGE_SIZE}&offset=${
@@ -47,40 +138,42 @@ module.exports = async function () {
     if (tracks.data.length) res = [...res, ...tracks.data]
     CURRENT_PAGE++
   }
-  res.forEach((track) => {
-    const formattedArtist = titleCase(aliasArtist(track.attributes['artistName']))
-    const formattedAlbum = titleCase(sanitizeMedia(track.attributes['albumName']))
 
-    if (!charts.artists[formattedArtist]) {
-      charts.artists[formattedArtist] = {
-        artist: formattedArtist,
-        url: `https://rateyourmusic.com/search?searchterm=${encodeURI(formattedArtist)}`,
-        genre: track['relationships']?.['library'].data[0]?.attributes['genreNames'][0] || '',
-        plays: 1,
-      }
-    } else {
-      charts.artists[formattedArtist].plays++
+  if (process.env.ELEVENTY_PRODUCTION === 'true') {
+    try {
+      const cachedTracksOutput = await client.send(
+        new GetObjectCommand({
+          Bucket: WASABI_BUCKET,
+          Key: `${DATE_STAMP}-music-history.json`,
+        })
+      )
+      const cachedTracksData = getReadableData(cachedTracksOutput.Body)
+      cachedTracks = await cachedTracksData.then((tracks) => JSON.parse(tracks)).catch()
+    } catch (e) {
+      console.log('No cached tracks')
+      cachedTracks = {}
     }
+  }
 
-    if (!charts.albums[formattedAlbum]) {
-      charts.albums[formattedAlbum] = {
-        name: formattedAlbum,
-        artist: formattedArtist,
-        art: track.attributes.artwork.url.replace('{w}', '300').replace('{h}', '300'),
-        url: track['relationships']
-          ? `https://song.link/${track['relationships'].albums.data.pop().attributes.url}`
-          : `https://rateyourmusic.com/search?searchtype=l&searchterm=${encodeURI(
-              formattedAlbum
-            )}%20${encodeURI(formattedArtist)}`,
-        plays: 1,
-      }
-    } else {
-      charts.albums[formattedAlbum].plays++
-    }
-  })
-  charts.artists = sortByPlays(charts.artists).splice(0, 8)
-  charts.albums = sortByPlays(charts.albums).splice(0, 8)
+  const diffedTracks = diffTracks(cachedTracks, formatTracks(res))
+  const updatedCache = {
+    ...cachedTracks,
+    ...diffedTracks,
+  }
+
+  charts = deriveCharts(updatedCache)
+  charts.artists = _.orderBy(Object.values(charts.artists), ['plays'], ['desc']).splice(0, 8)
+  charts.albums = _.orderBy(Object.values(charts.albums), ['plays'], ['desc']).splice(0, 8)
+
+  if (!_.isEmpty(diffedTracks) && process.env.ELEVENTY_PRODUCTION === 'true') {
+    await client.send(
+      new PutObjectCommand({
+        Bucket: WASABI_BUCKET,
+        Key: `${DATE_STAMP}-music-history.json`,
+        Body: JSON.stringify(updatedCache),
+      })
+    )
+  }
 
-  await asset.save(charts, 'json')
   return charts
 }

src/utils/arrays.js

@@ -0,0 +1,3 @@
+module.exports = {
+  getKeyByValue: (object, value) => Object.keys(object).find((key) => object[key].includes(value)),
+}

src/utils/grammar.js

@@ -10,11 +10,12 @@ module.exports = {
    */
   titleCase: (string) => {
     if (!string) return ''
+    if (titleCaseExceptions.artists.includes(string)) return string
     return string
       .toLowerCase()
       .split(' ')
       .map((word, i) => {
-        return titleCaseExceptions.exceptions.includes(word) && i !== 0
+        return titleCaseExceptions.words.includes(word) && i !== 0
           ? word
           : word.charAt(0).toUpperCase().concat(word.substring(1))
       })