This commit is contained in:
2025-01-04 00:34:03 +01:00
parent 41829408dc
commit 0ca14bbc19
18111 changed files with 1871397 additions and 0 deletions

View File

@@ -0,0 +1 @@
export {};

View File

@@ -0,0 +1,15 @@
export var ChecksumAlgorithm;
(function (ChecksumAlgorithm) {
ChecksumAlgorithm["MD5"] = "MD5";
ChecksumAlgorithm["CRC32"] = "CRC32";
ChecksumAlgorithm["CRC32C"] = "CRC32C";
ChecksumAlgorithm["SHA1"] = "SHA1";
ChecksumAlgorithm["SHA256"] = "SHA256";
})(ChecksumAlgorithm || (ChecksumAlgorithm = {}));
export var ChecksumLocation;
(function (ChecksumLocation) {
ChecksumLocation["HEADER"] = "header";
ChecksumLocation["TRAILER"] = "trailer";
})(ChecksumLocation || (ChecksumLocation = {}));
export const DEFAULT_CHECKSUM_ALGORITHM = ChecksumAlgorithm.MD5;
export const S3_EXPRESS_DEFAULT_CHECKSUM_ALGORITHM = ChecksumAlgorithm.CRC32;

View File

@@ -0,0 +1,69 @@
import { HttpRequest } from "@smithy/protocol-http";
import { getChecksumAlgorithmForRequest } from "./getChecksumAlgorithmForRequest";
import { getChecksumLocationName } from "./getChecksumLocationName";
import { hasHeader } from "./hasHeader";
import { isStreaming } from "./isStreaming";
import { selectChecksumAlgorithmFunction } from "./selectChecksumAlgorithmFunction";
import { stringHasher } from "./stringHasher";
export const flexibleChecksumsMiddlewareOptions = {
name: "flexibleChecksumsMiddleware",
step: "build",
tags: ["BODY_CHECKSUM"],
override: true,
};
export const flexibleChecksumsMiddleware = (config, middlewareConfig) => (next, context) => async (args) => {
if (!HttpRequest.isInstance(args.request)) {
return next(args);
}
const { request } = args;
const { body: requestBody, headers } = request;
const { base64Encoder, streamHasher } = config;
const { input, requestChecksumRequired, requestAlgorithmMember } = middlewareConfig;
const checksumAlgorithm = getChecksumAlgorithmForRequest(input, {
requestChecksumRequired,
requestAlgorithmMember,
}, !!context.isS3ExpressBucket);
let updatedBody = requestBody;
let updatedHeaders = headers;
if (checksumAlgorithm) {
const checksumLocationName = getChecksumLocationName(checksumAlgorithm);
const checksumAlgorithmFn = selectChecksumAlgorithmFunction(checksumAlgorithm, config);
if (isStreaming(requestBody)) {
const { getAwsChunkedEncodingStream, bodyLengthChecker } = config;
updatedBody = getAwsChunkedEncodingStream(requestBody, {
base64Encoder,
bodyLengthChecker,
checksumLocationName,
checksumAlgorithmFn,
streamHasher,
});
updatedHeaders = {
...headers,
"content-encoding": headers["content-encoding"]
? `${headers["content-encoding"]},aws-chunked`
: "aws-chunked",
"transfer-encoding": "chunked",
"x-amz-decoded-content-length": headers["content-length"],
"x-amz-content-sha256": "STREAMING-UNSIGNED-PAYLOAD-TRAILER",
"x-amz-trailer": checksumLocationName,
};
delete updatedHeaders["content-length"];
}
else if (!hasHeader(checksumLocationName, headers)) {
const rawChecksum = await stringHasher(checksumAlgorithmFn, requestBody);
updatedHeaders = {
...headers,
[checksumLocationName]: base64Encoder(rawChecksum),
};
}
}
const result = await next({
...args,
request: {
...request,
headers: updatedHeaders,
body: updatedBody,
},
});
return result;
};

View File

@@ -0,0 +1,50 @@
import { HttpRequest } from "@smithy/protocol-http";
import { getChecksumAlgorithmListForResponse } from "./getChecksumAlgorithmListForResponse";
import { getChecksumLocationName } from "./getChecksumLocationName";
import { isChecksumWithPartNumber } from "./isChecksumWithPartNumber";
import { isStreaming } from "./isStreaming";
import { createReadStreamOnBuffer } from "./streams/create-read-stream-on-buffer";
import { validateChecksumFromResponse } from "./validateChecksumFromResponse";
export const flexibleChecksumsResponseMiddlewareOptions = {
name: "flexibleChecksumsResponseMiddleware",
toMiddleware: "deserializerMiddleware",
relation: "after",
tags: ["BODY_CHECKSUM"],
override: true,
};
export const flexibleChecksumsResponseMiddleware = (config, middlewareConfig) => (next, context) => async (args) => {
if (!HttpRequest.isInstance(args.request)) {
return next(args);
}
const input = args.input;
const result = await next(args);
const response = result.response;
let collectedStream = undefined;
const { requestValidationModeMember, responseAlgorithms } = middlewareConfig;
if (requestValidationModeMember && input[requestValidationModeMember] === "ENABLED") {
const { clientName, commandName } = context;
const isS3WholeObjectMultipartGetResponseChecksum = clientName === "S3Client" &&
commandName === "GetObjectCommand" &&
getChecksumAlgorithmListForResponse(responseAlgorithms).every((algorithm) => {
const responseHeader = getChecksumLocationName(algorithm);
const checksumFromResponse = response.headers[responseHeader];
return !checksumFromResponse || isChecksumWithPartNumber(checksumFromResponse);
});
if (isS3WholeObjectMultipartGetResponseChecksum) {
return result;
}
const isStreamingBody = isStreaming(response.body);
if (isStreamingBody) {
collectedStream = await config.streamCollector(response.body);
response.body = createReadStreamOnBuffer(collectedStream);
}
await validateChecksumFromResponse(result.response, {
config,
responseAlgorithms,
});
if (isStreamingBody && collectedStream) {
response.body = createReadStreamOnBuffer(collectedStream);
}
}
return result;
};

View File

@@ -0,0 +1,6 @@
import { isStreaming } from "./isStreaming";
import { stringHasher } from "./stringHasher";
export const getChecksum = async (body, { streamHasher, checksumAlgorithmFn, base64Encoder }) => {
const digest = isStreaming(body) ? streamHasher(checksumAlgorithmFn, body) : stringHasher(checksumAlgorithmFn, body);
return base64Encoder(await digest);
};

View File

@@ -0,0 +1,14 @@
import { DEFAULT_CHECKSUM_ALGORITHM, S3_EXPRESS_DEFAULT_CHECKSUM_ALGORITHM } from "./constants";
import { CLIENT_SUPPORTED_ALGORITHMS } from "./types";
export const getChecksumAlgorithmForRequest = (input, { requestChecksumRequired, requestAlgorithmMember }, isS3Express) => {
const defaultAlgorithm = isS3Express ? S3_EXPRESS_DEFAULT_CHECKSUM_ALGORITHM : DEFAULT_CHECKSUM_ALGORITHM;
if (!requestAlgorithmMember || !input[requestAlgorithmMember]) {
return requestChecksumRequired ? defaultAlgorithm : undefined;
}
const checksumAlgorithm = input[requestAlgorithmMember];
if (!CLIENT_SUPPORTED_ALGORITHMS.includes(checksumAlgorithm)) {
throw new Error(`The checksum algorithm "${checksumAlgorithm}" is not supported by the client.` +
` Select one of ${CLIENT_SUPPORTED_ALGORITHMS}.`);
}
return checksumAlgorithm;
};

View File

@@ -0,0 +1,11 @@
import { CLIENT_SUPPORTED_ALGORITHMS, PRIORITY_ORDER_ALGORITHMS } from "./types";
export const getChecksumAlgorithmListForResponse = (responseAlgorithms = []) => {
const validChecksumAlgorithms = [];
for (const algorithm of PRIORITY_ORDER_ALGORITHMS) {
if (!responseAlgorithms.includes(algorithm) || !CLIENT_SUPPORTED_ALGORITHMS.includes(algorithm)) {
continue;
}
validChecksumAlgorithms.push(algorithm);
}
return validChecksumAlgorithms;
};

View File

@@ -0,0 +1,2 @@
import { ChecksumAlgorithm } from "./constants";
export const getChecksumLocationName = (algorithm) => algorithm === ChecksumAlgorithm.MD5 ? "content-md5" : `x-amz-checksum-${algorithm.toLowerCase()}`;

View File

@@ -0,0 +1,8 @@
import { flexibleChecksumsMiddleware, flexibleChecksumsMiddlewareOptions, } from "./flexibleChecksumsMiddleware";
import { flexibleChecksumsResponseMiddleware, flexibleChecksumsResponseMiddlewareOptions, } from "./flexibleChecksumsResponseMiddleware";
export const getFlexibleChecksumsPlugin = (config, middlewareConfig) => ({
applyToStack: (clientStack) => {
clientStack.add(flexibleChecksumsMiddleware(config, middlewareConfig), flexibleChecksumsMiddlewareOptions);
clientStack.addRelativeTo(flexibleChecksumsResponseMiddleware(config, middlewareConfig), flexibleChecksumsResponseMiddlewareOptions);
},
});

View File

@@ -0,0 +1,9 @@
export const hasHeader = (header, headers) => {
const soughtHeader = header.toLowerCase();
for (const headerName of Object.keys(headers)) {
if (soughtHeader === headerName.toLowerCase()) {
return true;
}
}
return false;
};

View File

@@ -0,0 +1,3 @@
export * from "./constants";
export * from "./flexibleChecksumsMiddleware";
export * from "./getFlexibleChecksumsPlugin";

View File

@@ -0,0 +1,13 @@
export const isChecksumWithPartNumber = (checksum) => {
const lastHyphenIndex = checksum.lastIndexOf("-");
if (lastHyphenIndex !== -1) {
const numberPart = checksum.slice(lastHyphenIndex + 1);
if (!numberPart.startsWith("0")) {
const number = parseInt(numberPart, 10);
if (!isNaN(number) && number >= 1 && number <= 10000) {
return true;
}
}
}
return false;
};

View File

@@ -0,0 +1,2 @@
import { isArrayBuffer } from "@smithy/is-array-buffer";
export const isStreaming = (body) => body !== undefined && typeof body !== "string" && !ArrayBuffer.isView(body) && !isArrayBuffer(body);

View File

@@ -0,0 +1,10 @@
import { AwsCrc32 } from "@aws-crypto/crc32";
import { AwsCrc32c } from "@aws-crypto/crc32c";
import { ChecksumAlgorithm } from "./constants";
export const selectChecksumAlgorithmFunction = (checksumAlgorithm, config) => ({
[ChecksumAlgorithm.MD5]: config.md5,
[ChecksumAlgorithm.CRC32]: AwsCrc32,
[ChecksumAlgorithm.CRC32C]: AwsCrc32c,
[ChecksumAlgorithm.SHA1]: config.sha1,
[ChecksumAlgorithm.SHA256]: config.sha256,
}[checksumAlgorithm]);

View File

@@ -0,0 +1,3 @@
export function createReadStreamOnBuffer(buffer) {
return new Blob([buffer]).stream();
}

View File

@@ -0,0 +1,7 @@
import { Transform } from "stream";
export function createReadStreamOnBuffer(buffer) {
const stream = new Transform();
stream.push(buffer);
stream.push(null);
return stream;
}

View File

@@ -0,0 +1,6 @@
import { toUint8Array } from "@smithy/util-utf8";
export const stringHasher = (checksumAlgorithmFn, body) => {
const hash = new checksumAlgorithmFn();
hash.update(toUint8Array(body || ""));
return hash.digest();
};

View File

@@ -0,0 +1,13 @@
import { ChecksumAlgorithm } from "./constants";
export const CLIENT_SUPPORTED_ALGORITHMS = [
ChecksumAlgorithm.CRC32,
ChecksumAlgorithm.CRC32C,
ChecksumAlgorithm.SHA1,
ChecksumAlgorithm.SHA256,
];
export const PRIORITY_ORDER_ALGORITHMS = [
ChecksumAlgorithm.CRC32,
ChecksumAlgorithm.CRC32C,
ChecksumAlgorithm.SHA1,
ChecksumAlgorithm.SHA256,
];

View File

@@ -0,0 +1,22 @@
import { getChecksum } from "./getChecksum";
import { getChecksumAlgorithmListForResponse } from "./getChecksumAlgorithmListForResponse";
import { getChecksumLocationName } from "./getChecksumLocationName";
import { selectChecksumAlgorithmFunction } from "./selectChecksumAlgorithmFunction";
export const validateChecksumFromResponse = async (response, { config, responseAlgorithms }) => {
const checksumAlgorithms = getChecksumAlgorithmListForResponse(responseAlgorithms);
const { body: responseBody, headers: responseHeaders } = response;
for (const algorithm of checksumAlgorithms) {
const responseHeader = getChecksumLocationName(algorithm);
const checksumFromResponse = responseHeaders[responseHeader];
if (checksumFromResponse) {
const checksumAlgorithmFn = selectChecksumAlgorithmFunction(algorithm, config);
const { streamHasher, base64Encoder } = config;
const checksum = await getChecksum(responseBody, { streamHasher, checksumAlgorithmFn, base64Encoder });
if (checksum === checksumFromResponse) {
break;
}
throw new Error(`Checksum mismatch: expected "${checksum}" but received "${checksumFromResponse}"` +
` in response header "${responseHeader}".`);
}
}
};