Skip to content

Commit 4c6ad40

Browse files
trivikrtrivikr
authored
feat(middleware-flexible-checksums): use CRC64NVME JS implementation if CRT is not available (#7595)
1 parent 69196b7 commit 4c6ad40

File tree

6 files changed

+196
-16
lines changed

6 files changed

+196
-16
lines changed

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,6 @@ bindings to be included as a dependency with your application. This functionalit
665665
- [Amazon S3 Multi-Region Access Points](https://docs.aws.amazon.com/AmazonS3/latest/userguide/MultiRegionAccessPoints.html)
666666
- [Amazon S3 Object Integrity](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
667667
- [Amazon CloudFront-KeyValueStore](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/kvs-with-functions-kvp.html)
668-
- [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
669668

670669
If the required AWS Common Runtime components are not installed, you will receive an error like:
671670

@@ -682,6 +681,10 @@ https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime-
682681
indicating that the required dependency is missing to use the associated functionality. To install this dependency, follow
683682
the provided [instructions](#installing-the-aws-common-runtime-crt-dependency).
684683

684+
For [Amazon S3 CRC64-NVME checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html),
685+
we include JavaScript implementation by default, but you can optionally install and include `"@aws-sdk/crc64-nvme-crt"`
686+
on Node.js runtime for improved performance.
687+
685688
#### Installing the AWS Common Runtime (CRT) Dependency
686689

687690
You can install the CRT dependency with different commands depending on the package management tool you are using.

‎packages/middleware-flexible-checksums/src/middleware-flexible-checksums.crt.integ.spec.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
// Testing CRT implementation of CRC64NVME
2+
import "@aws-sdk/crc64-nvme-crt";
3+
4+
import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src";
5+
import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3";
6+
import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http";
7+
import { Readable, Transform } from "stream";
8+
import { describe, expect, test as it } from "vitest";
9+
10+
import { RequestChecksumCalculation, ResponseChecksumValidation } from "./constants";
11+
12+
describe("middleware-flexible-checksums", () => {
13+
const logger = {
14+
trace() {},
15+
debug() {},
16+
info() {},
17+
warn() {},
18+
error() {},
19+
};
20+
21+
const testCases: [string, string][] = [
22+
["", "AAAAAAAAAAA="],
23+
["abc", "BeXKuz/B+us="],
24+
["Hello world", "OOJZ0D8xKts="],
25+
];
26+
27+
describe(S3.name, () => {
28+
describe("putObject", () => {
29+
describe.each([undefined, RequestChecksumCalculation.WHEN_SUPPORTED, RequestChecksumCalculation.WHEN_REQUIRED])(
30+
`when requestChecksumCalculation='%s'`,
31+
(requestChecksumCalculation) => {
32+
testCases.forEach(([body, checksumValue]) => {
33+
const client = new S3({ region: "us-west-2", logger, requestChecksumCalculation });
34+
const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`;
35+
36+
it(`tests ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => {
37+
requireRequestsFrom(client).toMatch({
38+
method: "PUT",
39+
hostname: "s3.us-west-2.amazonaws.com",
40+
protocol: "https:",
41+
path: "/b/k",
42+
headers: {
43+
"content-type": "application/octet-stream",
44+
...(body.length
45+
? {
46+
"content-length": body.length.toString(),
47+
}
48+
: {}),
49+
"x-amz-sdk-checksum-algorithm": ChecksumAlgorithm.CRC64NVME,
50+
[checksumHeader]: checksumValue,
51+
host: "s3.us-west-2.amazonaws.com",
52+
"x-amz-user-agent": /./,
53+
"user-agent": /./,
54+
"amz-sdk-invocation-id": /./,
55+
"amz-sdk-request": /./,
56+
"x-amz-date": /./,
57+
"x-amz-content-sha256": /./,
58+
authorization: /./,
59+
},
60+
query: {
61+
"x-id": "PutObject",
62+
},
63+
});
64+
65+
await client.putObject({
66+
Bucket: "b",
67+
Key: "k",
68+
Body: body,
69+
ChecksumAlgorithm: ChecksumAlgorithm.CRC64NVME,
70+
});
71+
72+
expect.hasAssertions();
73+
});
74+
});
75+
}
76+
);
77+
});
78+
79+
describe("getObject", () => {
80+
describe.each([undefined, ResponseChecksumValidation.WHEN_SUPPORTED, ResponseChecksumValidation.WHEN_REQUIRED])(
81+
`when responseChecksumValidation='%s'`,
82+
(responseChecksumValidation) => {
83+
testCases.forEach(([body, checksumValue]) => {
84+
const checksumHeader = `x-amz-checksum-${ChecksumAlgorithm.CRC64NVME.toLowerCase()}`;
85+
86+
it(`validates ${checksumHeader}="${checksumValue}"" for checksum="${ChecksumAlgorithm.CRC64NVME}"`, async () => {
87+
const client = new S3({
88+
region: "us-west-2",
89+
logger,
90+
requestHandler: new (class implements HttpHandler {
91+
async handle(request: HttpRequest): Promise<any> {
92+
expect(request).toMatchObject({
93+
method: "GET",
94+
hostname: "s3.us-west-2.amazonaws.com",
95+
protocol: "https:",
96+
path: "/b/k",
97+
headers: {
98+
"x-amz-checksum-mode": "ENABLED",
99+
host: "s3.us-west-2.amazonaws.com",
100+
"x-amz-user-agent": /./,
101+
"user-agent": /./,
102+
"amz-sdk-invocation-id": /./,
103+
"amz-sdk-request": /./,
104+
"x-amz-date": /./,
105+
"x-amz-content-sha256": /./,
106+
authorization: /./,
107+
},
108+
query: {
109+
"x-id": "GetObject",
110+
},
111+
});
112+
return {
113+
response: new HttpResponse({
114+
statusCode: 200,
115+
headers: {
116+
"content-type": "application/octet-stream",
117+
"content-length": body.length.toString(),
118+
[checksumHeader]: checksumValue,
119+
},
120+
body: Readable.from([body]),
121+
}),
122+
};
123+
}
124+
updateHttpClientConfig(key: never, value: never): void {}
125+
httpHandlerConfigs() {
126+
return {};
127+
}
128+
})(),
129+
responseChecksumValidation,
130+
});
131+
132+
const response = await client.getObject({
133+
Bucket: "b",
134+
Key: "k",
135+
ChecksumMode: "ENABLED",
136+
});
137+
138+
await expect(response.Body?.transformToString()).resolves.toEqual(body);
139+
});
140+
});
141+
}
142+
);
143+
});
144+
});
145+
});

‎packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { getE2eTestResources } from "@aws-sdk/aws-util-test/src";
2-
import { S3, UploadPartCommandOutput } from "@aws-sdk/client-s3";
2+
import { ChecksumAlgorithm, S3, UploadPartCommandOutput } from "@aws-sdk/client-s3";
33
import { Upload } from "@aws-sdk/lib-storage";
44
import { FetchHttpHandler } from "@smithy/fetch-http-handler";
55
import type { HttpRequest, HttpResponse } from "@smithy/types";
@@ -108,6 +108,7 @@ describe("S3 checksums", () => {
108108
});
109109
expect.hasAssertions();
110110
});
111+
111112
it("should assist user input streams by buffering to the minimum 8kb required by S3", async () => {
112113
await s3.putObject({
113114
Bucket,
@@ -124,6 +125,7 @@ describe("S3 checksums", () => {
124125
});
125126
expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024);
126127
});
128+
127129
it("should be able to write an object with a webstream body (using fetch handler without checksum)", async () => {
128130
const handler = s3_noChecksum.config.requestHandler;
129131
s3_noChecksum.config.requestHandler = new FetchHttpHandler();
@@ -140,6 +142,7 @@ describe("S3 checksums", () => {
140142
});
141143
expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(24 * 1024);
142144
});
145+
143146
it("@aws-sdk/lib-storage Upload should allow webstreams to be used", async () => {
144147
await new Upload({
145148
client: s3,
@@ -155,6 +158,7 @@ describe("S3 checksums", () => {
155158
});
156159
expect((await get.Body?.transformToByteArray())?.byteLength).toEqual(6 * 1024 * 1024);
157160
});
161+
158162
it("should allow streams to be used in a manually orchestrated MPU", async () => {
159163
const cmpu = await s3.createMultipartUpload({
160164
Bucket,
@@ -252,4 +256,40 @@ describe("S3 checksums", () => {
252256
await expect(checksumStream.getReader().closed).resolves.toBe(undefined);
253257
});
254258
});
259+
260+
describe("Checksum computation", () => {
261+
it.each([
262+
["Hello world", ChecksumAlgorithm.CRC32, "i9aeUg=="],
263+
["Hello world", ChecksumAlgorithm.CRC32C, "crUfeA=="],
264+
["Hello world", ChecksumAlgorithm.CRC64NVME, "OOJZ0D8xKts="],
265+
["Hello world", ChecksumAlgorithm.SHA1, "e1AsOh9IyGCa4hLN+2Od7jlnP14="],
266+
["Hello world", ChecksumAlgorithm.SHA256, "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="],
267+
])(
268+
"when body is '%s' with checksum algorithm '%s' the checksum is '%s'",
269+
async (body, checksumAlgorithm, checksumValue) => {
270+
const client = new S3();
271+
client.middlewareStack.addRelativeTo(
272+
(next: any) => async (args: any) => {
273+
const request = args.request as HttpRequest;
274+
expect(request.headers["x-amz-sdk-checksum-algorithm"]).toEqual(checksumAlgorithm);
275+
expect(request.headers[`x-amz-checksum-${checksumAlgorithm.toLowerCase()}`]).toEqual(checksumValue);
276+
return next(args);
277+
},
278+
{
279+
relation: "after",
280+
toMiddleware: "flexibleChecksumsMiddleware",
281+
}
282+
);
283+
284+
await client.putObject({
285+
Bucket,
286+
Key: Key + "-checksum-" + checksumAlgorithm,
287+
Body: body,
288+
ChecksumAlgorithm: checksumAlgorithm,
289+
});
290+
291+
expect.assertions(2);
292+
}
293+
);
294+
});
255295
}, 60_000);

‎packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
// Required for testing CRC64NVME
2-
import "@aws-sdk/crc64-nvme-crt";
3-
41
import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src";
52
import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3";
63
import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http";

‎packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.spec.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { AwsCrc32c } from "@aws-crypto/crc32c";
2-
import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
2+
import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
33
import { describe, expect, test as it, vi } from "vitest";
44

55
import { ChecksumAlgorithm } from "./constants";
@@ -27,8 +27,9 @@ describe(selectChecksumAlgorithmFunction.name, () => {
2727
expect(() => selectChecksumAlgorithmFunction("UNSUPPORTED" as any, mockConfig as any)).toThrow();
2828
});
2929

30-
it("throws error if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => {
31-
expect(() => selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toThrow();
30+
it("returns Crc64Nvme if crc64NvmeCrtContainer.CrtCrc64Nvme is not a function", () => {
31+
crc64NvmeCrtContainer.CrtCrc64Nvme = null;
32+
expect(selectChecksumAlgorithmFunction(ChecksumAlgorithm.CRC64NVME, mockConfig as any)).toBe(Crc64Nvme);
3233
});
3334

3435
it("returns crc64NvmeCrtContainer.CrtCrc64Nvme if available from container", () => {

‎packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { AwsCrc32c } from "@aws-crypto/crc32c";
2-
import { crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
2+
import { Crc64Nvme, crc64NvmeCrtContainer } from "@aws-sdk/crc64-nvme";
33
import { ChecksumConstructor, HashConstructor } from "@smithy/types";
44

55
import { PreviouslyResolved } from "./configuration";
@@ -22,13 +22,7 @@ export const selectChecksumAlgorithmFunction = (
2222
return AwsCrc32c;
2323
case ChecksumAlgorithm.CRC64NVME:
2424
if (typeof crc64NvmeCrtContainer.CrtCrc64Nvme !== "function") {
25-
throw new Error(
26-
`Please check whether you have installed the "@aws-sdk/crc64-nvme-crt" package explicitly. \n` +
27-
`You must also register the package by calling [require("@aws-sdk/crc64-nvme-crt");] ` +
28-
`or an ESM equivalent such as [import "@aws-sdk/crc64-nvme-crt";]. \n` +
29-
"For more information please go to " +
30-
"https://github.com/aws/aws-sdk-js-v3#functionality-requiring-aws-common-runtime-crt"
31-
);
25+
return Crc64Nvme;
3226
}
3327
return crc64NvmeCrtContainer.CrtCrc64Nvme;
3428
case ChecksumAlgorithm.SHA1:

0 commit comments

Comments
 (0)