This commit is contained in:
2025-01-04 00:34:03 +01:00
parent 41829408dc
commit 0ca14bbc19
18111 changed files with 1871397 additions and 0 deletions

View File

@@ -0,0 +1,20 @@
import { DefaultRateLimiter, RETRY_MODES } from "@smithy/util-retry";
import { StandardRetryStrategy } from "./StandardRetryStrategy";
export class AdaptiveRetryStrategy extends StandardRetryStrategy {
constructor(maxAttemptsProvider, options) {
const { rateLimiter, ...superOptions } = options ?? {};
super(maxAttemptsProvider, superOptions);
this.rateLimiter = rateLimiter ?? new DefaultRateLimiter();
this.mode = RETRY_MODES.ADAPTIVE;
}
async retry(next, args) {
return super.retry(next, args, {
beforeRequest: async () => {
return this.rateLimiter.getSendToken();
},
afterRequest: (response) => {
this.rateLimiter.updateClientSendingRate(response);
},
});
}
}

View File

@@ -0,0 +1,90 @@
import { HttpRequest, HttpResponse } from "@smithy/protocol-http";
import { isThrottlingError } from "@smithy/service-error-classification";
import { DEFAULT_MAX_ATTEMPTS, DEFAULT_RETRY_DELAY_BASE, INITIAL_RETRY_TOKENS, INVOCATION_ID_HEADER, REQUEST_HEADER, RETRY_MODES, THROTTLING_RETRY_DELAY_BASE, } from "@smithy/util-retry";
import { v4 } from "uuid";
import { getDefaultRetryQuota } from "./defaultRetryQuota";
import { defaultDelayDecider } from "./delayDecider";
import { defaultRetryDecider } from "./retryDecider";
import { asSdkError } from "./util";
export class StandardRetryStrategy {
constructor(maxAttemptsProvider, options) {
this.maxAttemptsProvider = maxAttemptsProvider;
this.mode = RETRY_MODES.STANDARD;
this.retryDecider = options?.retryDecider ?? defaultRetryDecider;
this.delayDecider = options?.delayDecider ?? defaultDelayDecider;
this.retryQuota = options?.retryQuota ?? getDefaultRetryQuota(INITIAL_RETRY_TOKENS);
}
shouldRetry(error, attempts, maxAttempts) {
return attempts < maxAttempts && this.retryDecider(error) && this.retryQuota.hasRetryTokens(error);
}
async getMaxAttempts() {
let maxAttempts;
try {
maxAttempts = await this.maxAttemptsProvider();
}
catch (error) {
maxAttempts = DEFAULT_MAX_ATTEMPTS;
}
return maxAttempts;
}
async retry(next, args, options) {
let retryTokenAmount;
let attempts = 0;
let totalDelay = 0;
const maxAttempts = await this.getMaxAttempts();
const { request } = args;
if (HttpRequest.isInstance(request)) {
request.headers[INVOCATION_ID_HEADER] = v4();
}
while (true) {
try {
if (HttpRequest.isInstance(request)) {
request.headers[REQUEST_HEADER] = `attempt=${attempts + 1}; max=${maxAttempts}`;
}
if (options?.beforeRequest) {
await options.beforeRequest();
}
const { response, output } = await next(args);
if (options?.afterRequest) {
options.afterRequest(response);
}
this.retryQuota.releaseRetryTokens(retryTokenAmount);
output.$metadata.attempts = attempts + 1;
output.$metadata.totalRetryDelay = totalDelay;
return { response, output };
}
catch (e) {
const err = asSdkError(e);
attempts++;
if (this.shouldRetry(err, attempts, maxAttempts)) {
retryTokenAmount = this.retryQuota.retrieveRetryTokens(err);
const delayFromDecider = this.delayDecider(isThrottlingError(err) ? THROTTLING_RETRY_DELAY_BASE : DEFAULT_RETRY_DELAY_BASE, attempts);
const delayFromResponse = getDelayFromRetryAfterHeader(err.$response);
const delay = Math.max(delayFromResponse || 0, delayFromDecider);
totalDelay += delay;
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
}
if (!err.$metadata) {
err.$metadata = {};
}
err.$metadata.attempts = attempts;
err.$metadata.totalRetryDelay = totalDelay;
throw err;
}
}
}
}
const getDelayFromRetryAfterHeader = (response) => {
if (!HttpResponse.isInstance(response))
return;
const retryAfterHeaderName = Object.keys(response.headers).find((key) => key.toLowerCase() === "retry-after");
if (!retryAfterHeaderName)
return;
const retryAfter = response.headers[retryAfterHeaderName];
const retryAfterSeconds = Number(retryAfter);
if (!Number.isNaN(retryAfterSeconds))
return retryAfterSeconds * 1000;
const retryAfterDate = new Date(retryAfter);
return retryAfterDate.getTime() - Date.now();
};

View File

@@ -0,0 +1,52 @@
import { normalizeProvider } from "@smithy/util-middleware";
import { AdaptiveRetryStrategy, DEFAULT_MAX_ATTEMPTS, DEFAULT_RETRY_MODE, RETRY_MODES, StandardRetryStrategy, } from "@smithy/util-retry";
export const ENV_MAX_ATTEMPTS = "AWS_MAX_ATTEMPTS";
export const CONFIG_MAX_ATTEMPTS = "max_attempts";
export const NODE_MAX_ATTEMPT_CONFIG_OPTIONS = {
environmentVariableSelector: (env) => {
const value = env[ENV_MAX_ATTEMPTS];
if (!value)
return undefined;
const maxAttempt = parseInt(value);
if (Number.isNaN(maxAttempt)) {
throw new Error(`Environment variable ${ENV_MAX_ATTEMPTS} mast be a number, got "${value}"`);
}
return maxAttempt;
},
configFileSelector: (profile) => {
const value = profile[CONFIG_MAX_ATTEMPTS];
if (!value)
return undefined;
const maxAttempt = parseInt(value);
if (Number.isNaN(maxAttempt)) {
throw new Error(`Shared config file entry ${CONFIG_MAX_ATTEMPTS} mast be a number, got "${value}"`);
}
return maxAttempt;
},
default: DEFAULT_MAX_ATTEMPTS,
};
export const resolveRetryConfig = (input) => {
const { retryStrategy } = input;
const maxAttempts = normalizeProvider(input.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
return {
...input,
maxAttempts,
retryStrategy: async () => {
if (retryStrategy) {
return retryStrategy;
}
const retryMode = await normalizeProvider(input.retryMode)();
if (retryMode === RETRY_MODES.ADAPTIVE) {
return new AdaptiveRetryStrategy(maxAttempts);
}
return new StandardRetryStrategy(maxAttempts);
},
};
};
export const ENV_RETRY_MODE = "AWS_RETRY_MODE";
export const CONFIG_RETRY_MODE = "retry_mode";
export const NODE_RETRY_MODE_CONFIG_OPTIONS = {
environmentVariableSelector: (env) => env[ENV_RETRY_MODE],
configFileSelector: (profile) => profile[CONFIG_RETRY_MODE],
default: DEFAULT_RETRY_MODE,
};

View File

@@ -0,0 +1,27 @@
import { NO_RETRY_INCREMENT, RETRY_COST, TIMEOUT_RETRY_COST } from "@smithy/util-retry";
export const getDefaultRetryQuota = (initialRetryTokens, options) => {
const MAX_CAPACITY = initialRetryTokens;
const noRetryIncrement = options?.noRetryIncrement ?? NO_RETRY_INCREMENT;
const retryCost = options?.retryCost ?? RETRY_COST;
const timeoutRetryCost = options?.timeoutRetryCost ?? TIMEOUT_RETRY_COST;
let availableCapacity = initialRetryTokens;
const getCapacityAmount = (error) => (error.name === "TimeoutError" ? timeoutRetryCost : retryCost);
const hasRetryTokens = (error) => getCapacityAmount(error) <= availableCapacity;
const retrieveRetryTokens = (error) => {
if (!hasRetryTokens(error)) {
throw new Error("No retry token available");
}
const capacityAmount = getCapacityAmount(error);
availableCapacity -= capacityAmount;
return capacityAmount;
};
const releaseRetryTokens = (capacityReleaseAmount) => {
availableCapacity += capacityReleaseAmount ?? noRetryIncrement;
availableCapacity = Math.min(availableCapacity, MAX_CAPACITY);
};
return Object.freeze({
hasRetryTokens,
retrieveRetryTokens,
releaseRetryTokens,
});
};

View File

@@ -0,0 +1,2 @@
import { MAXIMUM_RETRY_DELAY } from "@smithy/util-retry";
export const defaultDelayDecider = (delayBase, attempts) => Math.floor(Math.min(MAXIMUM_RETRY_DELAY, Math.random() * 2 ** attempts * delayBase));

View File

@@ -0,0 +1,7 @@
export * from "./AdaptiveRetryStrategy";
export * from "./StandardRetryStrategy";
export * from "./configurations";
export * from "./delayDecider";
export * from "./omitRetryHeadersMiddleware";
export * from "./retryDecider";
export * from "./retryMiddleware";

View File

@@ -0,0 +1 @@
export const isStreamingPayload = (request) => request?.body instanceof ReadableStream;

View File

@@ -0,0 +1,3 @@
import { Readable } from "stream";
export const isStreamingPayload = (request) => request?.body instanceof Readable ||
(typeof ReadableStream !== "undefined" && request?.body instanceof ReadableStream);

View File

@@ -0,0 +1,22 @@
import { HttpRequest } from "@smithy/protocol-http";
import { INVOCATION_ID_HEADER, REQUEST_HEADER } from "@smithy/util-retry";
export const omitRetryHeadersMiddleware = () => (next) => async (args) => {
const { request } = args;
if (HttpRequest.isInstance(request)) {
delete request.headers[INVOCATION_ID_HEADER];
delete request.headers[REQUEST_HEADER];
}
return next(args);
};
export const omitRetryHeadersMiddlewareOptions = {
name: "omitRetryHeadersMiddleware",
tags: ["RETRY", "HEADERS", "OMIT_RETRY_HEADERS"],
relation: "before",
toMiddleware: "awsAuthMiddleware",
override: true,
};
export const getOmitRetryHeadersPlugin = (options) => ({
applyToStack: (clientStack) => {
clientStack.addRelativeTo(omitRetryHeadersMiddleware(), omitRetryHeadersMiddlewareOptions);
},
});

View File

@@ -0,0 +1,7 @@
import { isClockSkewError, isRetryableByTrait, isThrottlingError, isTransientError, } from "@smithy/service-error-classification";
export const defaultRetryDecider = (error) => {
if (!error) {
return false;
}
return isRetryableByTrait(error) || isClockSkewError(error) || isThrottlingError(error) || isTransientError(error);
};

View File

@@ -0,0 +1,112 @@
import { HttpRequest, HttpResponse } from "@smithy/protocol-http";
import { isServerError, isThrottlingError, isTransientError } from "@smithy/service-error-classification";
import { NoOpLogger } from "@smithy/smithy-client";
import { INVOCATION_ID_HEADER, REQUEST_HEADER } from "@smithy/util-retry";
import { v4 } from "uuid";
import { isStreamingPayload } from "./isStreamingPayload/isStreamingPayload";
import { asSdkError } from "./util";
export const retryMiddleware = (options) => (next, context) => async (args) => {
let retryStrategy = await options.retryStrategy();
const maxAttempts = await options.maxAttempts();
if (isRetryStrategyV2(retryStrategy)) {
retryStrategy = retryStrategy;
let retryToken = await retryStrategy.acquireInitialRetryToken(context["partition_id"]);
let lastError = new Error();
let attempts = 0;
let totalRetryDelay = 0;
const { request } = args;
const isRequest = HttpRequest.isInstance(request);
if (isRequest) {
request.headers[INVOCATION_ID_HEADER] = v4();
}
while (true) {
try {
if (isRequest) {
request.headers[REQUEST_HEADER] = `attempt=${attempts + 1}; max=${maxAttempts}`;
}
const { response, output } = await next(args);
retryStrategy.recordSuccess(retryToken);
output.$metadata.attempts = attempts + 1;
output.$metadata.totalRetryDelay = totalRetryDelay;
return { response, output };
}
catch (e) {
const retryErrorInfo = getRetryErrorInfo(e);
lastError = asSdkError(e);
if (isRequest && isStreamingPayload(request)) {
(context.logger instanceof NoOpLogger ? console : context.logger)?.warn("An error was encountered in a non-retryable streaming request.");
throw lastError;
}
try {
retryToken = await retryStrategy.refreshRetryTokenForRetry(retryToken, retryErrorInfo);
}
catch (refreshError) {
if (!lastError.$metadata) {
lastError.$metadata = {};
}
lastError.$metadata.attempts = attempts + 1;
lastError.$metadata.totalRetryDelay = totalRetryDelay;
throw lastError;
}
attempts = retryToken.getRetryCount();
const delay = retryToken.getRetryDelay();
totalRetryDelay += delay;
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
else {
retryStrategy = retryStrategy;
if (retryStrategy?.mode)
context.userAgent = [...(context.userAgent || []), ["cfg/retry-mode", retryStrategy.mode]];
return retryStrategy.retry(next, args);
}
};
const isRetryStrategyV2 = (retryStrategy) => typeof retryStrategy.acquireInitialRetryToken !== "undefined" &&
typeof retryStrategy.refreshRetryTokenForRetry !== "undefined" &&
typeof retryStrategy.recordSuccess !== "undefined";
const getRetryErrorInfo = (error) => {
const errorInfo = {
error,
errorType: getRetryErrorType(error),
};
const retryAfterHint = getRetryAfterHint(error.$response);
if (retryAfterHint) {
errorInfo.retryAfterHint = retryAfterHint;
}
return errorInfo;
};
const getRetryErrorType = (error) => {
if (isThrottlingError(error))
return "THROTTLING";
if (isTransientError(error))
return "TRANSIENT";
if (isServerError(error))
return "SERVER_ERROR";
return "CLIENT_ERROR";
};
export const retryMiddlewareOptions = {
name: "retryMiddleware",
tags: ["RETRY"],
step: "finalizeRequest",
priority: "high",
override: true,
};
export const getRetryPlugin = (options) => ({
applyToStack: (clientStack) => {
clientStack.add(retryMiddleware(options), retryMiddlewareOptions);
},
});
export const getRetryAfterHint = (response) => {
if (!HttpResponse.isInstance(response))
return;
const retryAfterHeaderName = Object.keys(response.headers).find((key) => key.toLowerCase() === "retry-after");
if (!retryAfterHeaderName)
return;
const retryAfter = response.headers[retryAfterHeaderName];
const retryAfterSeconds = Number(retryAfter);
if (!Number.isNaN(retryAfterSeconds))
return new Date(retryAfterSeconds * 1000);
const retryAfterDate = new Date(retryAfter);
return retryAfterDate;
};

View File

@@ -0,0 +1 @@
export {};

View File

@@ -0,0 +1,9 @@
export const asSdkError = (error) => {
if (error instanceof Error)
return error;
if (error instanceof Object)
return Object.assign(new Error(), error);
if (typeof error === "string")
return new Error(error);
return new Error(`AWS SDK error wrapper for ${error}`);
};