diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..06d77ed6 --- /dev/null +++ b/.env.example @@ -0,0 +1,30 @@ +# Your Cloudflare account tag. +# +# Needed for: +# - Directory cache scripts +CLOUDFLARE_ACCOUNT_ID= + +# Cloudflare V4 API token. +# +# Needed for: +# - Directory cache scripts +# +# Required permissions: +# - `Workers KV Storage`: Edit +# - `Workers R2 Storage`: Read +# +# See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/ +CLOUDFLARE_API_TOKEN= + +# S3 credentials for your R2 bucket. +# +# Needed for: +# - Directory listings in the worker. +# - Directory cache scripts +# +# Required permissions: +# - `Object Read Only` +# +# See https://dash.cloudflare.com/?account=/r2/api-tokens +S3_ACCESS_KEY_ID= +S3_ACCESS_KEY_SECRET= diff --git a/.github/workflows/update-links.yml b/.github/workflows/update-links.yml index 2d88aa02..81dc650e 100644 --- a/.github/workflows/update-links.yml +++ b/.github/workflows/update-links.yml @@ -6,6 +6,11 @@ permissions: on: # Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml workflow_dispatch: + inputs: + version: + description: 'Node.js version (ex/ `v20.0.0`)' + required: true + type: string schedule: - cron: '0 0 * * *' @@ -81,6 +86,15 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }} + # Do this last for now so we avoid breaking releases if anything breaks + - name: Update Directory Cache + run: node scripts/update-directory-cache.mjs "$VERSION_INPUT" + env: + VERSION_INPUT: '${{ inputs.version }}' + CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }} + S3_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }} + S3_ACCESS_KEY_SECRET: ${{ secrets.CF_SECRET_ACCESS_KEY }} + - name: Alert on Failure if: failure() && github.repository == 'nodejs/release-cloudflare-worker' uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # 2.3.3 diff --git a/.gitignore b/.gitignore index 1c6a8066..eb628556 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ node_modules/ dist/ .dev.vars .sentryclirc +.env diff --git a/package-lock.json b/package-lock.json index eec8848c..e89c94d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "dependencies": { "@aws-sdk/client-s3": "^3.859.0", + "cloudflare": "^5.2.0", "itty-router": "^5.0.22", "mustache": "^4.2.0", "toucan-js": "^4.1.1" @@ -3589,13 +3590,22 @@ "version": "24.6.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.6.1.tgz", "integrity": "sha512-ljvjjs3DNXummeIaooB4cLBKg2U6SPI6Hjra/9rRIy7CpM0HpLtG9HptkMKAb4HYWy5S7HUvJEuWgr/y0U8SHw==", - "dev": true, "license": "MIT", "peer": true, "dependencies": { "undici-types": "~7.13.0" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, "node_modules/@types/uuid": { "version": "9.0.8", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", @@ -4314,6 +4324,18 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.14.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", @@ -4359,6 +4381,18 @@ "node": ">= 6.0.0" } }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -4418,6 +4452,12 @@ "node": ">=12" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -4485,6 +4525,19 @@ "node": ">=8" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -4568,6 +4621,36 @@ "node": ">= 10.0" } }, + "node_modules/cloudflare": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/cloudflare/-/cloudflare-5.2.0.tgz", + "integrity": "sha512-dVzqDpPFYR9ApEC9e+JJshFJZXcw4HzM8W+3DHzO5oy9+8rLC53G7x6fEf9A7/gSuSCxuvndzui5qJKftfIM9A==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/cloudflare/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/cloudflare/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, "node_modules/color": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", @@ -4611,6 +4694,18 @@ "simple-swizzle": "^0.2.2" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -4688,6 +4783,15 @@ "dev": true, "license": "MIT" }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/detect-libc": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", @@ -4716,6 +4820,20 @@ "tslib": "^2.0.3" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -4751,6 +4869,24 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -4758,6 +4894,33 @@ "dev": true, "license": "MIT" }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.25.4", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.4.tgz", @@ -5054,6 +5217,15 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/exit-hook": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-2.2.1.tgz", @@ -5242,6 +5414,41 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -5256,6 +5463,52 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-tsconfig": { "version": "4.10.0", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.0.tgz", @@ -5339,6 +5592,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", @@ -5354,6 +5619,45 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/html-minifier-terser": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-7.2.0.tgz", @@ -5400,6 +5704,15 @@ "node": ">= 6" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -5635,6 +5948,15 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -5672,6 +5994,27 @@ "node": ">=10.0.0" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/miniflare": { "version": "4.20251011.1", "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20251011.1.tgz", @@ -5724,7 +6067,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/mustache": { @@ -5772,11 +6114,30 @@ "tslib": "^2.0.3" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dev": true, "license": "MIT", "dependencies": { "whatwg-url": "^5.0.0" @@ -6676,7 +7037,6 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true, "license": "MIT" }, "node_modules/ts-api-utils": { @@ -6776,7 +7136,6 @@ "version": "7.13.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.13.0.tgz", "integrity": "sha512-Ov2Rr9Sx+fRgagJ5AX0qvItZG/JKKoBRAVITs1zk7IqZGTJUwgUr7qoYBpWwakpWilTZFM98rG/AFRocu10iIQ==", - "dev": true, "license": "MIT" }, "node_modules/unenv": { @@ -7030,18 +7389,25 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true, "license": "BSD-2-Clause" }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, "license": "MIT", "dependencies": { "tr46": "~0.0.3", diff --git a/package.json b/package.json index 29b39365..d56e67ac 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.859.0", + "cloudflare": "^5.2.0", "itty-router": "^5.0.22", "mustache": "^4.2.0", "toucan-js": "^4.1.1" diff --git a/scripts/build-directory-cache.mjs b/scripts/build-directory-cache.mjs new file mode 100755 index 00000000..8ab4fc2a --- /dev/null +++ b/scripts/build-directory-cache.mjs @@ -0,0 +1,82 @@ +#!/usr/bin/env node +// @ts-check + +/** + * This is probably not the script you want to run. + * + * This script builds the directory cache from scratch, which uses a _lot_ of + * memory and a noteable amount of time when ran against the dist-prod bucket. + * + * This script should only be ran when there isn't already a directory cache, + * for other scenarios try `./update-directory-cache.mjs`. + * + * Usage: build-directory-cache.mjs [--purge] + * --purge: Use with caution. It will delete all elements in the cache and + * then rebuild it. + */ + +import { + DIRECTORY_CACHE_NAMESPACE_ID, + PROD_BUCKET, + RELEASE_DIR, +} from './constants.mjs'; +import { + addSymlinksToDirectoryCache, + createCloudflareClient, + createS3Client, + deleteKvNamespaceKeys, + getLatestVersionMapping, + listKvNamespace, + listR2DirectoryRecursive, + writeKeysToKv, +} from './utils.mjs'; + +// Ensure all necessary environment variables are set +for (const envVar of [ + // 'CLOUDFLARE_API_TOKEN', + 'S3_ACCESS_KEY_ID', + 'S3_ACCESS_KEY_SECRET', +]) { + if (!process.env[envVar]) { + throw new TypeError(`${envVar} missing from process.env`); + } +} + +// Create the clients we need to access the data +const s3Client = createS3Client( + process.env.S3_ACCESS_KEY_ID, + process.env.S3_ACCESS_KEY_SECRET +); + +const cfClient = createCloudflareClient(); + +// List the entire R2 bucket +const directories = await listR2DirectoryRecursive(s3Client, PROD_BUCKET); +console.log(`Listed ${directories.size} directories from ${PROD_BUCKET}`); + +const latestVersions = await getLatestVersionMapping( + s3Client, + directories.get(RELEASE_DIR), + directories +); + +await addSymlinksToDirectoryCache(s3Client, directories, latestVersions); + +// Purge directory cache if wanted +if (process.argv[2] === '--purge') { + const cachedDirectoryKeys = await listKvNamespace( + cfClient, + DIRECTORY_CACHE_NAMESPACE_ID + ); + console.log(`Purging ${cachedDirectoryKeys.size} keys`); + + await deleteKvNamespaceKeys( + cfClient, + DIRECTORY_CACHE_NAMESPACE_ID, + Array.from(cachedDirectoryKeys) + ); +} + +// Update keys in the directory cache +console.log(`Writing ${directories.size} keys to directory cache...`); +await writeKeysToKv(cfClient, DIRECTORY_CACHE_NAMESPACE_ID, directories); diff --git a/scripts/build-r2-symlinks.mjs b/scripts/build-r2-symlinks.mjs index 0c9783d0..1983a5c3 100644 --- a/scripts/build-r2-symlinks.mjs +++ b/scripts/build-r2-symlinks.mjs @@ -9,7 +9,12 @@ import { S3Client, } from '@aws-sdk/client-s3'; import { Linker } from 'nodejs-latest-linker/common.js'; -import { DOCS_DIR, ENDPOINT, PROD_BUCKET, RELEASE_DIR } from './constants.mjs'; +import { + DOCS_DIR, + R2_ENDPOINT, + PROD_BUCKET, + RELEASE_DIR, +} from './constants.mjs'; const DOCS_DIRECTORY_OUT = join( import.meta.dirname, @@ -52,7 +57,7 @@ if (!process.env.CF_SECRET_ACCESS_KEY) { } const client = new S3Client({ - endpoint: ENDPOINT, + endpoint: R2_ENDPOINT, region: 'auto', credentials: { accessKeyId: process.env.CF_ACCESS_KEY_ID, diff --git a/scripts/constants.mjs b/scripts/constants.mjs index 7c5eb4fe..b1eb1b89 100644 --- a/scripts/constants.mjs +++ b/scripts/constants.mjs @@ -1,20 +1,39 @@ -'use strict'; - import { dirname, join } from 'node:path'; import { readdir, readFile, stat } from 'node:fs/promises'; -export const ENDPOINT = - process.env.ENDPOINT ?? - 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'; +export const CLOUDFLARE_ACCOUNT_ID = + process.env.CLOUDFLARE_ACCOUNT_ID ?? '07be8d2fbc940503ca1be344714cb0d1'; -export const PROD_BUCKET = process.env.PROD_BUCKET ?? 'dist-prod'; +export const R2_ENDPOINT = + process.env.R2_ENDPOINT ?? + `https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`; -export const STAGING_BUCKET = process.env.STAGING_BUCKET ?? 'dist-staging'; +/** + * The id of the KV namespace used for caching directories + */ +export const DIRECTORY_CACHE_NAMESPACE_ID = + process.env.DIRECTORY_CACHE_NAMESPACE_ID ?? ''; -export const R2_RETRY_COUNT = 3; +export const R2_RETRY_COUNT = 5; +export const KV_RETRY_COUNT = 5; -export const RELEASE_DIR = 'nodejs/release/'; +export const PROD_BUCKET = process.env.PROD_BUCKET ?? 'dist-prod'; +export const STAGING_BUCKET = process.env.STAGING_BUCKET ?? 'dist-staging'; +export const RELEASE_DIR = 'nodejs/release/'; export const DOCS_DIR = 'nodejs/docs/'; export const DEV_BUCKET_PATH = join(import.meta.dirname, '..', 'dev-bucket'); +export const STATIC_FILE_SYMLINKS_PATH = join( + import.meta.dirname, + '..', + 'src', + 'constants', + 'fileSymlinks.json' +); + +/** + * Paths in the R2 bucket that we should always be updating whenever a new + * version releases. + */ +export const ALWAYS_UPDATED_PATHS = [RELEASE_DIR, DOCS_DIR]; diff --git a/scripts/update-directory-cache.mjs b/scripts/update-directory-cache.mjs new file mode 100644 index 00000000..c2ecc662 --- /dev/null +++ b/scripts/update-directory-cache.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node +// @ts-check + +/** + * Update the most commonly updated paths in the directory cache for a Node.js + * release. + * + * Usage: update-directory-cache.mjs + */ + +import { + ALWAYS_UPDATED_PATHS, + DIRECTORY_CACHE_NAMESPACE_ID, + PROD_BUCKET, + RELEASE_DIR, +} from './constants.mjs'; +import { + createCloudflareClient, + createS3Client, + listR2DirectoryRecursive, + listR2Directory, + writeKeysToKv, + getLatestVersionMapping, + addSymlinksToDirectoryCache, +} from './utils.mjs'; + +const VERSION = process.argv[2].toLowerCase(); + +if (!VERSION) { + throw new TypeError('version missing from args'); +} + +if (!VERSION.startsWith('v')) { + throw new TypeError( + 'provided version not in correct format, expected vX.X.X' + ); +} + +// Ensure all necessary environment variables are set +for (const envVar of [ + // 'CLOUDFLARE_API_TOKEN', + 'S3_ACCESS_KEY_ID', + 'S3_ACCESS_KEY_SECRET', +]) { + if (!process.env[envVar]) { + throw new TypeError(`${envVar} missing from process.env`); + } +} + +// Create the clients that we need for fetching the data +const s3Client = createS3Client( + process.env.S3_ACCESS_KEY_ID, + process.env.S3_ACCESS_KEY_SECRET +); + +const cfClient = createCloudflareClient(); + +// List the directory of the new release (and subdirectories) +const directories = await listR2DirectoryRecursive( + s3Client, + PROD_BUCKET, + `nodejs/release/${VERSION}/` +); + +// Non-recursively list the paths that we always want to update +await Promise.all( + ALWAYS_UPDATED_PATHS.map(path => + listR2Directory(s3Client, PROD_BUCKET, path).then(value => { + directories.set(path, value); + }) + ) +); + +const latestVersions = await getLatestVersionMapping( + s3Client, + directories.get(RELEASE_DIR), + directories +); + +await addSymlinksToDirectoryCache(s3Client, directories, latestVersions); + +// Update keys in the directory cache +console.log(`Writing ${directories.size} keys to directory cache...`); +await writeKeysToKv(cfClient, DIRECTORY_CACHE_NAMESPACE_ID, directories); diff --git a/scripts/utils.mjs b/scripts/utils.mjs new file mode 100644 index 00000000..76da8858 --- /dev/null +++ b/scripts/utils.mjs @@ -0,0 +1,468 @@ +// @ts-check +import { readFile, writeFile } from 'node:fs/promises'; +import { basename, dirname } from 'node:path'; +import { + HeadObjectCommand, + ListObjectsV2Command, + S3Client, +} from '@aws-sdk/client-s3'; +import Cloudflare from 'cloudflare'; +import { Linker } from 'nodejs-latest-linker/common.js'; +import { + CLOUDFLARE_ACCOUNT_ID, + DOCS_DIR, + KV_RETRY_COUNT, + PROD_BUCKET, + R2_ENDPOINT, + R2_RETRY_COUNT, + RELEASE_DIR, + STATIC_FILE_SYMLINKS_PATH, +} from './constants.mjs'; + +/** + * @param {string} accessKeyId + * @param {string} secretAccessKey + * @returns {S3Client} + */ +export function createS3Client(accessKeyId, secretAccessKey) { + return new S3Client({ + endpoint: R2_ENDPOINT, + region: 'auto', + credentials: { + accessKeyId: accessKeyId, + secretAccessKey: secretAccessKey, + }, + }); +} + +export function createCloudflareClient() { + return new Cloudflare({ + maxRetries: KV_RETRY_COUNT, + }); +} + +/** + * Recursively list the contents of an R2 bucket under + * + * @typedef {Map} DirectoryListMapping + * + * @param {S3Client} client + * @param {string} bucket + * @param {string | undefined} [directory] + * @param {DirectoryListMapping} [listings] + * @returns {Promise} + */ +export async function listR2DirectoryRecursive( + client, + bucket, + directory = undefined, + listings = new Map() +) { + // List the current directory + const response = await listR2Directory(client, bucket, directory); + + // Add current directory to the listings object + listings.set(directory ?? '/', response); + + // List the subdirectories + await Promise.all( + response.subdirectories.map(subdirectory => + listR2DirectoryRecursive( + client, + bucket, + `${directory ? directory : ''}${subdirectory}`, + listings + ) + ) + ); + + return listings; +} + +/** + * List the contents of a directory in R2 + * + * @param {S3Client} client + * @param {string} bucket + * @param {string | undefined} directory + * @returns {Promise} + */ +export async function listR2Directory(client, bucket, directory = undefined) { + /** + * @type {Set} + */ + const subdirectories = new Set(); + + /** + * @type {Set} + */ + const files = new Set(); + + let hasIndexHtmlFile = false; + + let truncated = true; + let continuationToken; + let lastModified = new Date(0); + while (truncated) { + /** + * @type {import('@aws-sdk/client-s3').ListObjectsV2Output | undefined} + */ + let data = undefined; + + let retries = R2_RETRY_COUNT; + while (retries) { + try { + data = await client.send( + new ListObjectsV2Command({ + Bucket: bucket, + Delimiter: '/', + Prefix: directory, + ContinuationToken: continuationToken, + MaxKeys: 1000, + }) + ); + + break; + } catch (err) { + console.warn('listR2Directory listing error: ', err); + retries--; + + if (retries === 0) { + console.log('exhausted R2 retries'); + throw err; + } + } + } + + if (!data) { + // Should never reach this + throw new TypeError('expected listing response to be defined'); + } + + if (data.CommonPrefixes) { + data.CommonPrefixes.forEach(value => { + if (value.Prefix) { + subdirectories.add(value.Prefix.substring(directory?.length ?? 0)); + } + }); + } + + if (data.Contents) { + data.Contents.forEach(value => { + if (value.Key) { + if (value.Key.match(/index.htm(?:l)$/)) { + hasIndexHtmlFile = true; + } + + files.add({ + name: value.Key.substring(directory?.length ?? 0), + lastModified: value.LastModified, + size: value.Size, + }); + + if (value.LastModified > lastModified) { + lastModified = value.LastModified; + } + } + }); + } + + truncated = data.IsTruncated; + continuationToken = data.NextContinuationToken; + } + + return { + subdirectories: Array.from(subdirectories), + hasIndexHtmlFile, + files: Array.from(files), + lastModified, + }; +} + +/** + * @param {S3Client} client + * @param {string} bucket + * @param {string} path + * @param {DirectoryListMapping} [cachedDirectories=undefined] + * @returns {Promise} + */ +export async function headR2File( + client, + bucket, + path, + cachedDirectories = undefined +) { + const directoryName = `${dirname(path)}/`; + const cachedDirectory = cachedDirectories?.get(directoryName); + if (cachedDirectory) { + const fileName = path.substring(directoryName.length); + + return cachedDirectory.files.find(file => file.name === fileName); + } + + const data = await client.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: path, + }) + ); + + if (!data.LastModified || !data.ContentLength) { + return undefined; + } + + return { + name: path, + lastModified: data.LastModified, + size: data.ContentLength, + }; +} + +/** + * @typedef {Record} LatestVersionMapping + * + * @param {S3Client} client + * @param {import('../src/providers/provider.js').ReadDirectoryResult} releases + * @param {DirectoryListMapping} cachedDirectories + * @returns {Promise} + */ +export async function getLatestVersionMapping( + client, + releases, + cachedDirectories +) { + const linker = new Linker({ baseDir: RELEASE_DIR, docs: DOCS_DIR }); + + /** + * Creates mappings to the latest versions of Node + * @type {Map} + * @example { 'nodejs/release/latest-v20.x': 'nodejs/release/v20.x.x' } + */ + const links = await linker.getLinks( + [...releases.subdirectories, ...releases.files.map(file => file.name)], + async directory => { + const r2Path = `${directory}/`; + + const listing = + cachedDirectories.get(r2Path) ?? + (await listR2Directory(client, PROD_BUCKET, r2Path)); + + return [ + ...listing.subdirectories, + ...listing.files.map(file => file.name), + ]; + } + ); + + /** + * @type {Record} + * @example {'latest-v20.x': 'v20.x.x'} + */ + const latestVersions = {}; + + for (const [key, value] of links) { + const trimmedKey = key.substring(RELEASE_DIR.length); + const trimmedValue = value.substring(RELEASE_DIR.length); + + latestVersions[trimmedKey] = trimmedValue; + } + + return latestVersions; +} + +/** + * R2 doesn't have symlinks, but we make heavy use of them. This means we need + * to add them into the cache ourselves. + * + * We use symlinks to point to both files and directories. + * + * The file symlinks are more or less static. There are only two that are + * constantly updated with each release: + * - `nodejs/release/node-latest.tar.gz` + * - `node-config-schema.json` + * + * The directory symlinks are dynamic with a few exceptions for a select number + * of directories in `nodejs/docs/` + * + * @param {S3Client} client + * @param {DirectoryListMapping} cachedDirectories + * @param {LatestVersionMapping} latestVersions + */ +export async function addSymlinksToDirectoryCache( + client, + cachedDirectories, + latestVersions +) { + const releaseDirectory = cachedDirectories.get(RELEASE_DIR); + if (!releaseDirectory) { + console.warn(`Release directory not found at path ${RELEASE_DIR}`); + return; + } + + const latestVersion = latestVersions['latest']; + + // Add the `nodejs/release/node-latest.tar.gz` symlink to the directory cache + { + const nodeLatestPath = `nodejs/release/${latestVersions['node-latest.tar.gz'].replaceAll('latest', latestVersion)}`; + + // Stat the file that `node-latest.tar.gz` points to so we can have accurate + // size & last modified info for the directory listing. Also acts as a safety + // check so this doesn't point to an invalid path without us noticing. + const nodeLatest = await headR2File(client, PROD_BUCKET, nodeLatestPath); + if (!nodeLatest) { + throw new TypeError( + `node-latest.tar.gz points to ${latestVersions['node-latest.tar.gz']} which doesn't exist in the prod bucket` + ); + } + + releaseDirectory.files.push({ + name: 'node-latest.tar.gz', + lastModified: nodeLatest.lastModified, + size: nodeLatest.size, + }); + } + + // Add `latest-*` directory symlinks to the directory cache + releaseDirectory.subdirectories.concat( + Object.keys(latestVersions) + .filter(version => version !== 'node-latest.tar.gz') + .map(version => `${version}/`) + ); + + // Manually override the contents of `nodejs/docs` to be a union of what's + // actually in that directory as well as the versions present in + // `nodejs/release`. + const docsDirectory = cachedDirectories.get(DOCS_DIR); + if (docsDirectory) { + // We only want the `vX.X.X/` and `latest-X/` directories + const versionDirectories = releaseDirectory.subdirectories.filter( + subdirectory => + subdirectory.startsWith('v') || subdirectory.startsWith('latest-') + ); + + docsDirectory.subdirectories = Array.from( + new Set([...docsDirectory.subdirectories, ...versionDirectories]) + ).sort(); + } + + // Add the static file symlinks to the directory cache + { + /** + * @type {Record} + */ + const fileSymlinks = JSON.parse( + await readFile(STATIC_FILE_SYMLINKS_PATH, 'utf8') + ); + + // This isn't static, delete it for now + fileSymlinks['node-config-schema.json'] = undefined; + + // Add the symlinks to the directory cache + for (const symlink of Object.keys(fileSymlinks)) { + const sourceFilePath = fileSymlinks[symlink]; + + if (!sourceFilePath) { + // Hit a file we don't want to stat, ignore it + continue; + } + + // Stat the source file so we can get its size, last modified + const sourceFile = await headR2File( + client, + PROD_BUCKET, + fileSymlinks[symlink], + cachedDirectories + ); + + if (!sourceFile) { + throw new TypeError( + `symlink '${symlink}' points to invalid file '${sourceFile}'` + ); + } + + const directoryPath = `${dirname(symlink)}/`; + + let directory = cachedDirectories.get(directoryPath); + if (!directory) { + directory = await listR2Directory(client, PROD_BUCKET, directoryPath); + cachedDirectories.set(directoryPath, directory); + } + + directory.files.push({ + ...sourceFile, + name: basename(symlink), + }); + } + + // Update the node-config-schema.json file symlink to point to the latest + // version + fileSymlinks['node-config-schema.json'] = + `${RELEASE_DIR}${latestVersion}/docs/node-config-schema.json`; + + // Update file so it can be used in the worker + await writeFile( + STATIC_FILE_SYMLINKS_PATH, + JSON.stringify(fileSymlinks, null, 2) + ); + } +} + +/** + * @param {Cloudflare} client + * @param {string} namespace + * @returns {Promise>} + */ +export async function listKvNamespace(client, namespace) { + /** + * @type {Set} + */ + const keys = new Set(); + + for await (const key of client.kv.namespaces.keys.list(namespace, { + account_id: CLOUDFLARE_ACCOUNT_ID, + })) { + keys.add(key.name); + } + + return keys; +} + +/** + * @param {Cloudflare} client + * @param {string} namespace + * @param {Map} values + */ +export async function writeKeysToKv(client, namespace, values) { + const keys = values.keys().toArray(); + + while (keys.length) { + // Can only write 10,000 keys at once + const batch = keys.splice(0, 10_000); + + await client.kv.namespaces.bulkUpdate(namespace, { + account_id: CLOUDFLARE_ACCOUNT_ID, + body: batch.map(key => ({ + key, + value: JSON.stringify(values.get(key)), + })), + }); + } +} + +/** + * Delete keys in a KV namespace + * @param {Cloudflare} client + * @param {string} namespace + * @param {Array} keys + */ +export async function deleteKvNamespaceKeys(client, namespace, keys) { + while (keys.length) { + // Can only delete 10,000 keys at once + const batch = keys.splice(0, 10_000); + + await client.kv.namespaces.bulkDelete(namespace, { + account_id: CLOUDFLARE_ACCOUNT_ID, + body: batch, + }); + } +} diff --git a/src/constants/limits.ts b/src/constants/limits.ts index 5c1f95de..0c53edce 100644 --- a/src/constants/limits.ts +++ b/src/constants/limits.ts @@ -1,8 +1,13 @@ /** - * Max amount of retries for R2 requests + * Max amount of retries for requests to R2 */ export const R2_RETRY_LIMIT = 5; +/** + * Max amount of retries for requests to KV + */ +export const KV_RETRY_LIMIT = 5; + /** * Max amount of keys to be returned in a S3 request */ diff --git a/src/env.ts b/src/env.ts index a929e029..2fb9040b 100644 --- a/src/env.ts +++ b/src/env.ts @@ -16,6 +16,16 @@ export interface Env { */ R2_BUCKET: R2Bucket; + /** + * KV namespace with cached directory listings + */ + DIRECTORIES_NAMESPACE: KVNamespace; + + /** + * Temp flag for whether or not to use KV instead of S3 + */ + USE_KV: boolean; + /** * Endpoint to hit when using the S3 api. */ diff --git a/src/providers/kvProvider.ts b/src/providers/kvProvider.ts new file mode 100644 index 00000000..e14cc996 --- /dev/null +++ b/src/providers/kvProvider.ts @@ -0,0 +1,57 @@ +import { KV_RETRY_LIMIT } from '../constants/limits'; +import type { Context } from '../context'; +import { retryWrapper } from '../utils/provider'; +import type { + GetFileOptions, + GetFileResult, + HeadFileResult, + Provider, + ReadDirectoryOptions, + ReadDirectoryResult, +} from './provider'; + +type KvProviderCtorOptions = { + ctx: Context; +}; + +export class KvProvider implements Provider { + #ctx: Context; + + constructor({ ctx }: KvProviderCtorOptions) { + this.#ctx = ctx; + } + + headFile(_: string): Promise { + throw new Error('Method not implemented.'); + } + + getFile(_: string, _2?: GetFileOptions): Promise { + throw new Error('Method not implemented.'); + } + + async readDirectory( + path: string, + options?: ReadDirectoryOptions + ): Promise { + const result = await retryWrapper( + async () => { + return this.#ctx.env.DIRECTORIES_NAMESPACE.get( + path, + 'json' + ); + }, + KV_RETRY_LIMIT, + this.#ctx.sentry + ); + + if (result === null) { + return undefined; + } + + if (options && !options.listFiles) { + result.files = []; + } + + return result; + } +} diff --git a/src/providers/r2Provider.ts b/src/providers/r2Provider.ts index 226178ac..af24cdb1 100644 --- a/src/providers/r2Provider.ts +++ b/src/providers/r2Provider.ts @@ -16,6 +16,7 @@ import type { ReadDirectoryResult, } from './provider'; import { S3Provider } from './s3Provider'; +import { KvProvider } from './kvProvider'; type CachedFile = { name: string; @@ -111,33 +112,41 @@ export class R2Provider implements Provider { path: string, options?: ReadDirectoryOptions ): Promise { - if (path in CACHED_DIRECTORIES) { - const result: CachedDirectory = - CACHED_DIRECTORIES[path as keyof typeof CACHED_DIRECTORIES]; + if (this.ctx.env.USE_KV) { + const kvProvider = new KvProvider({ + ctx: this.ctx, + }); + + return kvProvider.readDirectory(path, options); + } else { + if (path in CACHED_DIRECTORIES) { + const result: CachedDirectory = + CACHED_DIRECTORIES[path as keyof typeof CACHED_DIRECTORIES]; - if (typeof result.lastModified === 'string') { - result.lastModified = new Date(result.lastModified); + if (typeof result.lastModified === 'string') { + result.lastModified = new Date(result.lastModified); - for (const file of result.files) { - // @ts-expect-error this isn't readonly - file.lastModified = new Date(file.lastModified); + for (const file of result.files) { + // @ts-expect-error this isn't readonly + file.lastModified = new Date(file.lastModified); + } } + + // @ts-expect-error at this point the result is parsed already + return Promise.resolve({ + subdirectories: result.subdirectories, + files: result.files, + hasIndexHtmlFile: result.hasIndexHtmlFile, + lastModified: new Date(result.lastModified), + }); } - // @ts-expect-error at this point the result is parsed already - return Promise.resolve({ - subdirectories: result.subdirectories, - files: result.files, - hasIndexHtmlFile: result.hasIndexHtmlFile, - lastModified: new Date(result.lastModified), + const s3Provider = new S3Provider({ + ctx: this.ctx, }); - } - - const s3Provider = new S3Provider({ - ctx: this.ctx, - }); - return s3Provider.readDirectory(path, options); + return s3Provider.readDirectory(path, options); + } } } diff --git a/src/providers/s3Provider.ts b/src/providers/s3Provider.ts index c4ba7636..7fd9ff3d 100644 --- a/src/providers/s3Provider.ts +++ b/src/providers/s3Provider.ts @@ -26,13 +26,13 @@ type S3ProviderCtorOptions = { * directories. Using the S3 api was the recommended fix from the R2 team. */ export class S3Provider implements Provider { - private ctx: Context; - private client: S3Client; + #ctx: Context; + #client: S3Client; constructor({ ctx }: S3ProviderCtorOptions) { - this.ctx = ctx; + this.#ctx = ctx; - this.client = new S3Client({ + this.#client = new S3Client({ region: 'auto', endpoint: ctx.env.S3_ENDPOINT, credentials: { @@ -73,9 +73,9 @@ export class S3Provider implements Provider { while (isTruncated) { const result = await retryWrapper( async () => { - return this.client.send( + return this.#client.send( new ListObjectsV2Command({ - Bucket: this.ctx.env.BUCKET_NAME, + Bucket: this.#ctx.env.BUCKET_NAME, Prefix: path, Delimiter: '/', MaxKeys: S3_MAX_KEYS, @@ -84,7 +84,7 @@ export class S3Provider implements Provider { ); }, R2_RETRY_LIMIT, - this.ctx.sentry + this.#ctx.sentry ); result.CommonPrefixes?.forEach(directory => {