diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 99f0630..c59f549 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,15 +23,17 @@ jobs: uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} - # == Cache frequently built and/or with large dependencies == - # == Eviction policy: not accessed in over 7 days - # == Limit: 10 GB - # cache: 'npm' - # == Path to lock file when using mono repos == - # cache-dependency-path: subdir/package-lock.json - - - name: Restore packages - run: npm ci + registry-url: 'https://npm.pkg.github.com' + scope: '@makerxstudio' + cache: 'npm' + cache-dependency-path: ./package-lock.json + + # run npm ci preventing script access to npm auth token + - run: npm ci --ignore-scripts + env: + NODE_AUTH_TOKEN: ${{ secrets.npm-auth-token || secrets.GITHUB_TOKEN }} + # allow scripts to run without npm auth token + - run: npm rebuild - name: Lint commit messages run: npx commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} --verbose diff --git a/package-lock.json b/package-lock.json index 88b3f29..13c2be2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.1", "license": "MIT", "dependencies": { - "@aws-sdk/client-s3": "^3.391.0", + "@makerxstudio/node-cache": "^1.0.0", "async-retry": "^1.3.3", "blockstore-core": "^4.3.3", "ipfs-unixfs-importer": "^15.1.7", @@ -1924,7 +1924,6 @@ "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dev": true, "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", @@ -1941,7 +1940,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "dev": true, "engines": { "node": ">=12" }, @@ -1953,7 +1951,6 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true, "engines": { "node": ">=12" }, @@ -1964,14 +1961,12 @@ "node_modules/@isaacs/cliui/node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, "node_modules/@isaacs/cliui/node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -1988,7 +1983,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dev": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -2003,7 +1997,6 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -2522,6 +2515,74 @@ "integrity": "sha512-JHvErK0IPjyWkukJ0Bu6uPq0w1/XBCAelNCo4cM/vm/ybNATve8gzJ8dZx4e+1ubnL3PQumwOyuKkLV3AEfYLg==", "dev": true }, + "node_modules/@makerxstudio/node-cache": { + "version": "1.0.0", + "resolved": "https://npm.pkg.github.com/download/@makerxstudio/node-cache/1.0.0/ff8b4069f4ef91d72f0a45aa2e124cc78aebc3fe", + "integrity": "sha512-IdVKhNXNLSdpG58lPd1US8xa0sK/Yc9l7Zb+tJ2F+G+7Frxngrd/jgLIpE2fL/GdOdNm4R22HvzF5Pp8KHfbkg==", + "license": "MIT", + "dependencies": { + "@aws-sdk/client-s3": "^3.391.0", + "glob": "^10.3.10", + "mime": "^3.0.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@makerxstudio/node-cache/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@makerxstudio/node-cache/node_modules/glob": { + "version": "10.3.10", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", + "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.5", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@makerxstudio/node-cache/node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@makerxstudio/node-cache/node_modules/minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@multiformats/multiaddr": { "version": "12.1.11", "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-12.1.11.tgz", @@ -2760,7 +2821,6 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "dev": true, "optional": true, "engines": { "node": ">=14" @@ -4285,7 +4345,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -4300,7 +4359,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -4520,8 +4578,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/base64-js": { "version": "1.5.1", @@ -4944,7 +5001,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -4955,8 +5011,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/compare-func": { "version": "2.0.0", @@ -5141,7 +5196,6 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -5552,8 +5606,7 @@ "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, "node_modules/electron-to-chromium": { "version": "1.4.295", @@ -5576,8 +5629,7 @@ "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/emojilib": { "version": "2.4.0", @@ -6434,7 +6486,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dev": true, "dependencies": { "cross-spawn": "^7.0.0", "signal-exit": "^4.0.1" @@ -6450,7 +6501,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, "engines": { "node": ">=14" }, @@ -7638,7 +7688,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -7902,8 +7951,7 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/issue-parser": { "version": "6.0.0", @@ -8066,7 +8114,6 @@ "version": "2.3.6", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "dev": true, "dependencies": { "@isaacs/cliui": "^8.0.2" }, @@ -9216,7 +9263,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", - "dev": true, "engines": { "node": ">=16 || 14 >=14.17" } @@ -12834,7 +12880,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "engines": { "node": ">=8" } @@ -12849,7 +12894,6 @@ "version": "1.10.1", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz", "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==", - "dev": true, "dependencies": { "lru-cache": "^9.1.1 || ^10.0.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -12865,7 +12909,6 @@ "version": "10.1.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz", "integrity": "sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==", - "dev": true, "engines": { "node": "14 || >=16.14" } @@ -13989,7 +14032,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -14001,7 +14043,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "engines": { "node": ">=8" } @@ -14304,7 +14345,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -14319,7 +14359,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -14395,7 +14434,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -14408,7 +14446,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -16306,7 +16343,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "dependencies": { "isexe": "^2.0.0" }, @@ -16380,7 +16416,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", diff --git a/package.json b/package.json index d796118..12729c1 100644 --- a/package.json +++ b/package.json @@ -45,13 +45,12 @@ "node": ">=18.0" }, "dependencies": { - "@aws-sdk/client-s3": "^3.391.0", + "@makerxstudio/node-cache": "^1.0.0", "async-retry": "^1.3.3", "blockstore-core": "^4.3.3", "ipfs-unixfs-importer": "^15.1.7", "multiformats": "^9.9.0" }, - "peerDependencies": {}, "devDependencies": { "@commitlint/cli": "^18.4.3", "@commitlint/config-conventional": "^18.4.3", diff --git a/src/cache.ts b/src/cache.ts deleted file mode 100644 index 9f40ded..0000000 --- a/src/cache.ts +++ /dev/null @@ -1,177 +0,0 @@ -import type { S3Client } from '@aws-sdk/client-s3' -import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3' -import fs from 'fs/promises' -import path from 'path' - -export interface ObjectCache { - getAndCache( - cacheKey: string, - generator: (existing: T | undefined) => Promise, - staleAfterSeconds?: number, - returnStaleResult?: boolean, - isBinary?: boolean, - ): Promise - - put(cacheKey: string, data: T): Promise -} - -export class S3ObjectCache implements ObjectCache { - private s3Client: S3Client - private bucket: string - private keyPrefix?: string - - constructor(s3Client: S3Client, bucket: string, keyPrefix?: string) { - this.s3Client = s3Client - this.bucket = bucket - this.keyPrefix = keyPrefix - } - - async put(cacheKey: string, data: T): Promise { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const zlib = require('zlib') - const fileName = data instanceof Uint8Array ? `${cacheKey}.gz` : `${cacheKey}.json.gz` - const bucketAndKey = { - Bucket: this.bucket, - Key: this.keyPrefix ? path.join(this.keyPrefix, fileName) : fileName, - } - await this.s3Client.send( - new PutObjectCommand({ - ...bucketAndKey, - Body: zlib.gzipSync(data instanceof Uint8Array ? data : JSON.stringify(data, null, 2)), - }), - ) - } - - async getAndCache( - cacheKey: string, - generator: (existing: T | undefined) => Promise, - staleAfterSeconds?: number, - returnStaleResult?: boolean, - isBinary?: boolean, - ): Promise { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const zlib = require('zlib') - const fileName = isBinary ? `${cacheKey}.gz` : `${cacheKey}.json.gz` - const bucketAndKey = { - Bucket: this.bucket, - Key: this.keyPrefix ? path.join(this.keyPrefix, fileName) : fileName, - } - const existingCache = await this.s3Client - .send(new GetObjectCommand(bucketAndKey)) - .then(async (x) => ({ Body: await x.Body!.transformToByteArray(), LastModified: x.LastModified })) - .catch(() => undefined) - const expired = staleAfterSeconds && existingCache && (+new Date() - +existingCache.LastModified!) / 1000 > staleAfterSeconds - - const existingJson = !!existingCache && !isBinary ? zlib.gunzipSync(existingCache.Body!).toString('utf-8') : undefined - const existing = existingCache ? (isBinary ? (zlib.gunzipSync(existingCache.Body!) as T) : (JSON.parse(existingJson) as T)) : undefined - - let value = existing - if (!existing || expired) { - // eslint-disable-next-line no-console - console.debug( - !existingCache - ? `Cache value '${cacheKey}' empty; getting data for the first time` - : `Cache value '${cacheKey}' expired: ${existingCache.LastModified!.toISOString()}`, - ) - try { - value = await generator(existing) - await this.put(cacheKey, value) - // eslint-disable-next-line no-console - console.log(`Cached value '${bucketAndKey.Key}' written`) - } catch (e: unknown) { - const err = e instanceof Error ? e : new Error(String(e)) - if (existingCache && returnStaleResult) { - // eslint-disable-next-line no-console - console.error(err) - // eslint-disable-next-line no-console - console.warn( - `Received error ${ - err.message || err - } when trying to repopulate cache value '${cacheKey}'; failing gracefully and using the cache`, - ) - } else { - throw e - } - } - } else { - // eslint-disable-next-line no-console - console.debug(`Found cached value '${bucketAndKey.Key}' which is within ${staleAfterSeconds} seconds old so using that`) - } - - return value! - } -} - -export class FileSystemObjectCache implements ObjectCache { - private cacheDirectory: string - constructor(cacheDirectory: string) { - this.cacheDirectory = cacheDirectory - } - - async put(cacheKey: string, data: T): Promise { - const cachePath = path.join(this.cacheDirectory, data instanceof Uint8Array ? cacheKey : `${cacheKey}.json`) - await fs.writeFile(cachePath, data instanceof Uint8Array ? data : JSON.stringify(data, null, 2), { - encoding: data instanceof Uint8Array ? null : 'utf-8', - }) - } - - async getAndCache( - cacheKey: string, - generator: (existing: T | undefined) => Promise, - staleAfterSeconds?: number, - returnStaleResult?: boolean, - isBinary?: boolean, - ): Promise { - const cachePath = path.join(this.cacheDirectory, isBinary ? cacheKey : `${cacheKey}.json`) - const existingCache = await fs.stat(cachePath).catch((_e) => false) - const expired = - staleAfterSeconds && typeof existingCache !== 'boolean' && (+new Date() - +existingCache.mtime) / 1000 > staleAfterSeconds - - if (!existingCache || expired) { - // eslint-disable-next-line no-console - console.debug( - !existingCache - ? `Cache value '${cacheKey}' empty; getting data for the first time` - : `Cache value '${cacheKey}' expired: ${existingCache.mtime.toISOString()}`, - ) - try { - const existingJson = existingCache ? await fs.readFile(cachePath, { encoding: isBinary ? null : 'utf-8' }) : undefined - const existing = existingJson - ? isBinary - ? (Buffer.from(existingJson) as unknown as T) - : (JSON.parse(existingJson as string) as T) - : undefined - const value = await generator(existing) - await this.put(cacheKey, value) - // eslint-disable-next-line no-console - console.log(`Cached value '${cacheKey}' written to ${cachePath}`) - } catch (e: unknown) { - const err = e instanceof Error ? e : new Error(String(e)) - if (existingCache && returnStaleResult) { - // eslint-disable-next-line no-console - console.error(err) - // eslint-disable-next-line no-console - console.warn( - `Received error ${ - err.message || err - } when trying to repopulate cache value '${cacheKey}'; failing gracefully and using the cache`, - ) - } else { - throw e - } - } - } else { - // eslint-disable-next-line no-console - console.debug(`Found cached value '${cacheKey}' at ${cachePath} which is within ${staleAfterSeconds} seconds old so using that`) - } - - if (isBinary) { - const content = await fs.readFile(cachePath, { encoding: null }) - return Buffer.from(content) as unknown as T - } - - const valueJson = await fs.readFile(cachePath, { encoding: 'utf-8' }) - const value = JSON.parse(valueJson) as T - return value - } -} diff --git a/src/index.ts b/src/index.ts index ce4b496..e0293eb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,2 +1,2 @@ export * from './ipfs' -export * from './cache' +export * from '@makerxstudio/node-cache' diff --git a/src/ipfs.spec.ts b/src/ipfs.spec.ts index a47ed99..bb58a2d 100644 --- a/src/ipfs.spec.ts +++ b/src/ipfs.spec.ts @@ -1,5 +1,5 @@ import { PinataStorageWithCache } from './ipfs' -import { ObjectCache } from './cache' +import { ObjectCache } from '@makerxstudio/node-cache' import { mock, Mock } from 'ts-jest-mocker' const testToken = 'DUMMY_TOKEN' diff --git a/src/ipfs.ts b/src/ipfs.ts index ff3197f..1173b7c 100644 --- a/src/ipfs.ts +++ b/src/ipfs.ts @@ -1,7 +1,7 @@ import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' -import type { ObjectCache } from './cache' +import type { BinaryCacheOptions, BinaryWithMetadata, CacheOptions, ObjectCache } from '@makerxstudio/node-cache' import { fetchWithRetry } from './http' export interface IPFS { @@ -86,8 +86,10 @@ export class CacheOnlyIPFS implements IPFS { const json = await response.json() return json as T }, - undefined, - true, + { + staleAfterSeconds: undefined, + returnStaleResultOnError: true, + }, ) } @@ -98,8 +100,10 @@ export class CacheOnlyIPFS implements IPFS { (_e) => { return Promise.resolve(data) }, - 0, - false, + { + staleAfterSeconds: 0, + returnStaleResultOnError: false, + }, ) return { cid: cid.toString() } } @@ -114,9 +118,11 @@ export class CacheOnlyIPFS implements IPFS { const response = await fetchWithRetry(`https://${cid}.ipfs.cf-ipfs.com/`) return Buffer.from(await response.arrayBuffer()) }, - undefined, - true, - true, + { + staleAfterSeconds: undefined, + returnStaleResultOnError: true, + isBinary: true, + }, ) } @@ -127,8 +133,10 @@ export class CacheOnlyIPFS implements IPFS { (_e) => { return Promise.resolve(blob) }, - 0, - false, + { + staleAfterSeconds: 0, + returnStaleResultOnError: false, + }, ) return { cid: cid.toString() } } @@ -175,8 +183,10 @@ export class PinataStorageWithCache implements IPFS { const json = await response.json() return json as T }, - undefined, - true, + { + staleAfterSeconds: undefined, + returnStaleResultOnError: true, + }, ) } @@ -220,9 +230,11 @@ export class PinataStorageWithCache implements IPFS { console.debug(`Cache miss for ${cid}, fetching from IPFS`) return Buffer.from(await response.arrayBuffer()) }, - undefined, - true, - true, + { + staleAfterSeconds: undefined, + returnStaleResultOnError: true, + isBinary: true, + }, ) } @@ -298,16 +310,24 @@ export class PinataStorageWithCache implements IPFS { } class NoOpCache implements ObjectCache { - getAndCache( - _cacheKey: string, - generator: (existing: T | undefined) => Promise, - _staleAfterSeconds?: number | undefined, - _returnStaleResult?: boolean | undefined, - _isBinary?: boolean | undefined, - ): Promise { + getAndCache(cacheKey: string, generator: (existing: T | undefined) => Promise, _options?: CacheOptions | undefined): Promise { return generator(undefined) } - put(_cacheKey: string, _data: T): Promise { + async getAndCacheBinary( + _cacheKey: string, + generator: (existing: Uint8Array | undefined) => Promise, + options?: BinaryCacheOptions | undefined, + ): Promise { + return { + data: await generator(undefined), + mimeType: options?.mimeType ?? 'application/octet-stream', + fileExtension: null, + } + } + put(_cacheKey: string, _data: T, _mimeType?: string | undefined): Promise { + return Promise.resolve() + } + putBinary(_cacheKey: string, _data: Uint8Array, _mimeType?: string | undefined): Promise { return Promise.resolve() } }