This commit is contained in:
2025-01-04 00:34:03 +01:00
parent 41829408dc
commit 0ca14bbc19
18111 changed files with 1871397 additions and 0 deletions

View File

@@ -0,0 +1,62 @@
import { Crc32 } from "@aws-crypto/crc32";
import { HeaderMarshaller } from "./HeaderMarshaller";
import { splitMessage } from "./splitMessage";
export class EventStreamCodec {
constructor(toUtf8, fromUtf8) {
this.headerMarshaller = new HeaderMarshaller(toUtf8, fromUtf8);
this.messageBuffer = [];
this.isEndOfStream = false;
}
feed(message) {
this.messageBuffer.push(this.decode(message));
}
endOfStream() {
this.isEndOfStream = true;
}
getMessage() {
const message = this.messageBuffer.pop();
const isEndOfStream = this.isEndOfStream;
return {
getMessage() {
return message;
},
isEndOfStream() {
return isEndOfStream;
},
};
}
getAvailableMessages() {
const messages = this.messageBuffer;
this.messageBuffer = [];
const isEndOfStream = this.isEndOfStream;
return {
getMessages() {
return messages;
},
isEndOfStream() {
return isEndOfStream;
},
};
}
encode({ headers: rawHeaders, body }) {
const headers = this.headerMarshaller.format(rawHeaders);
const length = headers.byteLength + body.byteLength + 16;
const out = new Uint8Array(length);
const view = new DataView(out.buffer, out.byteOffset, out.byteLength);
const checksum = new Crc32();
view.setUint32(0, length, false);
view.setUint32(4, headers.byteLength, false);
view.setUint32(8, checksum.update(out.subarray(0, 8)).digest(), false);
out.set(headers, 12);
out.set(body, headers.byteLength + 12);
view.setUint32(length - 4, checksum.update(out.subarray(8, length - 4)).digest(), false);
return out;
}
decode(message) {
const { headers, body } = splitMessage(message);
return { headers: this.headerMarshaller.parse(headers), body };
}
formatHeaders(rawHeaders) {
return this.headerMarshaller.format(rawHeaders);
}
}

View File

@@ -0,0 +1,182 @@
import { fromHex, toHex } from "@smithy/util-hex-encoding";
import { Int64 } from "./Int64";
export class HeaderMarshaller {
constructor(toUtf8, fromUtf8) {
this.toUtf8 = toUtf8;
this.fromUtf8 = fromUtf8;
}
format(headers) {
const chunks = [];
for (const headerName of Object.keys(headers)) {
const bytes = this.fromUtf8(headerName);
chunks.push(Uint8Array.from([bytes.byteLength]), bytes, this.formatHeaderValue(headers[headerName]));
}
const out = new Uint8Array(chunks.reduce((carry, bytes) => carry + bytes.byteLength, 0));
let position = 0;
for (const chunk of chunks) {
out.set(chunk, position);
position += chunk.byteLength;
}
return out;
}
formatHeaderValue(header) {
switch (header.type) {
case "boolean":
return Uint8Array.from([header.value ? 0 : 1]);
case "byte":
return Uint8Array.from([2, header.value]);
case "short":
const shortView = new DataView(new ArrayBuffer(3));
shortView.setUint8(0, 3);
shortView.setInt16(1, header.value, false);
return new Uint8Array(shortView.buffer);
case "integer":
const intView = new DataView(new ArrayBuffer(5));
intView.setUint8(0, 4);
intView.setInt32(1, header.value, false);
return new Uint8Array(intView.buffer);
case "long":
const longBytes = new Uint8Array(9);
longBytes[0] = 5;
longBytes.set(header.value.bytes, 1);
return longBytes;
case "binary":
const binView = new DataView(new ArrayBuffer(3 + header.value.byteLength));
binView.setUint8(0, 6);
binView.setUint16(1, header.value.byteLength, false);
const binBytes = new Uint8Array(binView.buffer);
binBytes.set(header.value, 3);
return binBytes;
case "string":
const utf8Bytes = this.fromUtf8(header.value);
const strView = new DataView(new ArrayBuffer(3 + utf8Bytes.byteLength));
strView.setUint8(0, 7);
strView.setUint16(1, utf8Bytes.byteLength, false);
const strBytes = new Uint8Array(strView.buffer);
strBytes.set(utf8Bytes, 3);
return strBytes;
case "timestamp":
const tsBytes = new Uint8Array(9);
tsBytes[0] = 8;
tsBytes.set(Int64.fromNumber(header.value.valueOf()).bytes, 1);
return tsBytes;
case "uuid":
if (!UUID_PATTERN.test(header.value)) {
throw new Error(`Invalid UUID received: ${header.value}`);
}
const uuidBytes = new Uint8Array(17);
uuidBytes[0] = 9;
uuidBytes.set(fromHex(header.value.replace(/\-/g, "")), 1);
return uuidBytes;
}
}
parse(headers) {
const out = {};
let position = 0;
while (position < headers.byteLength) {
const nameLength = headers.getUint8(position++);
const name = this.toUtf8(new Uint8Array(headers.buffer, headers.byteOffset + position, nameLength));
position += nameLength;
switch (headers.getUint8(position++)) {
case 0:
out[name] = {
type: BOOLEAN_TAG,
value: true,
};
break;
case 1:
out[name] = {
type: BOOLEAN_TAG,
value: false,
};
break;
case 2:
out[name] = {
type: BYTE_TAG,
value: headers.getInt8(position++),
};
break;
case 3:
out[name] = {
type: SHORT_TAG,
value: headers.getInt16(position, false),
};
position += 2;
break;
case 4:
out[name] = {
type: INT_TAG,
value: headers.getInt32(position, false),
};
position += 4;
break;
case 5:
out[name] = {
type: LONG_TAG,
value: new Int64(new Uint8Array(headers.buffer, headers.byteOffset + position, 8)),
};
position += 8;
break;
case 6:
const binaryLength = headers.getUint16(position, false);
position += 2;
out[name] = {
type: BINARY_TAG,
value: new Uint8Array(headers.buffer, headers.byteOffset + position, binaryLength),
};
position += binaryLength;
break;
case 7:
const stringLength = headers.getUint16(position, false);
position += 2;
out[name] = {
type: STRING_TAG,
value: this.toUtf8(new Uint8Array(headers.buffer, headers.byteOffset + position, stringLength)),
};
position += stringLength;
break;
case 8:
out[name] = {
type: TIMESTAMP_TAG,
value: new Date(new Int64(new Uint8Array(headers.buffer, headers.byteOffset + position, 8)).valueOf()),
};
position += 8;
break;
case 9:
const uuidBytes = new Uint8Array(headers.buffer, headers.byteOffset + position, 16);
position += 16;
out[name] = {
type: UUID_TAG,
value: `${toHex(uuidBytes.subarray(0, 4))}-${toHex(uuidBytes.subarray(4, 6))}-${toHex(uuidBytes.subarray(6, 8))}-${toHex(uuidBytes.subarray(8, 10))}-${toHex(uuidBytes.subarray(10))}`,
};
break;
default:
throw new Error(`Unrecognized header type tag`);
}
}
return out;
}
}
var HEADER_VALUE_TYPE;
(function (HEADER_VALUE_TYPE) {
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["boolTrue"] = 0] = "boolTrue";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["boolFalse"] = 1] = "boolFalse";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["byte"] = 2] = "byte";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["short"] = 3] = "short";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["integer"] = 4] = "integer";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["long"] = 5] = "long";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["byteArray"] = 6] = "byteArray";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["string"] = 7] = "string";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["timestamp"] = 8] = "timestamp";
HEADER_VALUE_TYPE[HEADER_VALUE_TYPE["uuid"] = 9] = "uuid";
})(HEADER_VALUE_TYPE || (HEADER_VALUE_TYPE = {}));
const BOOLEAN_TAG = "boolean";
const BYTE_TAG = "byte";
const SHORT_TAG = "short";
const INT_TAG = "integer";
const LONG_TAG = "long";
const BINARY_TAG = "binary";
const STRING_TAG = "string";
const TIMESTAMP_TAG = "timestamp";
const UUID_TAG = "uuid";
const UUID_PATTERN = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/;

View File

@@ -0,0 +1,43 @@
import { toHex } from "@smithy/util-hex-encoding";
export class Int64 {
constructor(bytes) {
this.bytes = bytes;
if (bytes.byteLength !== 8) {
throw new Error("Int64 buffers must be exactly 8 bytes");
}
}
static fromNumber(number) {
if (number > 9223372036854776000 || number < -9223372036854776000) {
throw new Error(`${number} is too large (or, if negative, too small) to represent as an Int64`);
}
const bytes = new Uint8Array(8);
for (let i = 7, remaining = Math.abs(Math.round(number)); i > -1 && remaining > 0; i--, remaining /= 256) {
bytes[i] = remaining;
}
if (number < 0) {
negate(bytes);
}
return new Int64(bytes);
}
valueOf() {
const bytes = this.bytes.slice(0);
const negative = bytes[0] & 0b10000000;
if (negative) {
negate(bytes);
}
return parseInt(toHex(bytes), 16) * (negative ? -1 : 1);
}
toString() {
return String(this.valueOf());
}
}
function negate(bytes) {
for (let i = 0; i < 8; i++) {
bytes[i] ^= 0xff;
}
for (let i = 7; i > -1; i--) {
bytes[i]++;
if (bytes[i] !== 0)
break;
}
}

View File

@@ -0,0 +1 @@
export {};

View File

@@ -0,0 +1,14 @@
export class MessageDecoderStream {
constructor(options) {
this.options = options;
}
[Symbol.asyncIterator]() {
return this.asyncIterator();
}
async *asyncIterator() {
for await (const bytes of this.options.inputStream) {
const decoded = this.options.decoder.decode(bytes);
yield decoded;
}
}
}

View File

@@ -0,0 +1,17 @@
export class MessageEncoderStream {
constructor(options) {
this.options = options;
}
[Symbol.asyncIterator]() {
return this.asyncIterator();
}
async *asyncIterator() {
for await (const msg of this.options.messageStream) {
const encoded = this.options.encoder.encode(msg);
yield encoded;
}
if (this.options.includeEndFrame) {
yield new Uint8Array(0);
}
}
}

View File

@@ -0,0 +1,16 @@
export class SmithyMessageDecoderStream {
constructor(options) {
this.options = options;
}
[Symbol.asyncIterator]() {
return this.asyncIterator();
}
async *asyncIterator() {
for await (const message of this.options.messageStream) {
const deserialized = await this.options.deserializer(message);
if (deserialized === undefined)
continue;
yield deserialized;
}
}
}

View File

@@ -0,0 +1,14 @@
export class SmithyMessageEncoderStream {
constructor(options) {
this.options = options;
}
[Symbol.asyncIterator]() {
return this.asyncIterator();
}
async *asyncIterator() {
for await (const chunk of this.options.inputStream) {
const payloadBuf = this.options.serializer(chunk);
yield payloadBuf;
}
}
}

View File

@@ -0,0 +1,146 @@
import { Int64 } from "./Int64";
export const vectors = {
all_headers: {
expectation: "success",
encoded: Uint8Array.from([
0, 0, 0, 204, 0, 0, 0, 175, 15, 174, 100, 202, 10, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 4, 0, 0, 160,
12, 12, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 16, 97, 112, 112, 108, 105, 99, 97, 116,
105, 111, 110, 47, 106, 115, 111, 110, 10, 98, 111, 111, 108, 32, 102, 97, 108, 115, 101, 1, 9, 98, 111, 111, 108,
32, 116, 114, 117, 101, 0, 4, 98, 121, 116, 101, 2, 207, 8, 98, 121, 116, 101, 32, 98, 117, 102, 6, 0, 20, 73, 39,
109, 32, 97, 32, 108, 105, 116, 116, 108, 101, 32, 116, 101, 97, 112, 111, 116, 33, 9, 116, 105, 109, 101, 115,
116, 97, 109, 112, 8, 0, 0, 0, 0, 0, 132, 95, 237, 5, 105, 110, 116, 49, 54, 3, 0, 42, 5, 105, 110, 116, 54, 52,
5, 0, 0, 0, 0, 2, 135, 87, 178, 4, 117, 117, 105, 100, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
123, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125, 171, 165, 241, 12,
]),
decoded: {
headers: {
"event-type": {
type: "integer",
value: 40972,
},
"content-type": {
type: "string",
value: "application/json",
},
"bool false": {
type: "boolean",
value: false,
},
"bool true": {
type: "boolean",
value: true,
},
byte: {
type: "byte",
value: -49,
},
"byte buf": {
type: "binary",
value: Uint8Array.from([
73, 39, 109, 32, 97, 32, 108, 105, 116, 116, 108, 101, 32, 116, 101, 97, 112, 111, 116, 33,
]),
},
timestamp: {
type: "timestamp",
value: new Date(8675309),
},
int16: {
type: "short",
value: 42,
},
int64: {
type: "long",
value: Int64.fromNumber(42424242),
},
uuid: {
type: "uuid",
value: "01020304-0506-0708-090a-0b0c0d0e0f10",
},
},
body: Uint8Array.from([123, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125]),
},
},
empty_message: {
expectation: "success",
encoded: Uint8Array.from([0, 0, 0, 16, 0, 0, 0, 0, 5, 194, 72, 235, 125, 152, 200, 255]),
decoded: {
headers: {},
body: Uint8Array.from([]),
},
},
int32_header: {
expectation: "success",
encoded: Uint8Array.from([
0, 0, 0, 45, 0, 0, 0, 16, 65, 196, 36, 184, 10, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 4, 0, 0, 160, 12,
123, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125, 54, 244, 128, 160,
]),
decoded: {
headers: {
"event-type": {
type: "integer",
value: 40972,
},
},
body: Uint8Array.from([123, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125]),
},
},
payload_no_headers: {
expectation: "success",
encoded: Uint8Array.from([
0, 0, 0, 29, 0, 0, 0, 0, 253, 82, 140, 90, 123, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125, 195, 101, 57,
54,
]),
decoded: {
headers: {},
body: Uint8Array.from([123, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125]),
},
},
payload_one_str_header: {
expectation: "success",
encoded: Uint8Array.from([
0, 0, 0, 61, 0, 0, 0, 32, 7, 253, 131, 150, 12, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0,
16, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 106, 115, 111, 110, 123, 39, 102, 111, 111, 39, 58,
39, 98, 97, 114, 39, 125, 141, 156, 8, 177,
]),
decoded: {
headers: {
"content-type": {
type: "string",
value: "application/json",
},
},
body: Uint8Array.from([123, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125]),
},
},
corrupted_headers: {
expectation: "failure",
encoded: Uint8Array.from([
0, 0, 0, 61, 0, 0, 0, 32, 7, 253, 131, 150, 12, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0,
16, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 106, 115, 111, 110, 123, 97, 102, 111, 111, 39, 58,
39, 98, 97, 114, 39, 125, 141, 156, 8, 177,
]),
},
corrupted_header_len: {
expectation: "failure",
encoded: Uint8Array.from([
0, 0, 0, 61, 0, 0, 0, 33, 7, 253, 131, 150, 12, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0,
16, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 106, 115, 111, 110, 123, 39, 102, 111, 111, 39, 58,
39, 98, 97, 114, 39, 125, 141, 156, 8, 177,
]),
},
corrupted_length: {
expectation: "failure",
encoded: Uint8Array.from([
0, 0, 0, 62, 0, 0, 0, 32, 7, 253, 131, 150, 12, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0,
16, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 106, 115, 111, 110, 123, 39, 102, 111, 111, 39, 58,
39, 98, 97, 114, 39, 125, 141, 156, 8, 177,
]),
},
corrupted_payload: {
expectation: "failure",
encoded: Uint8Array.from([
0, 0, 0, 29, 0, 0, 0, 0, 253, 82, 140, 90, 91, 39, 102, 111, 111, 39, 58, 39, 98, 97, 114, 39, 125, 195, 101, 57,
54,
]),
},
};

View File

@@ -0,0 +1,8 @@
export * from "./EventStreamCodec";
export * from "./HeaderMarshaller";
export * from "./Int64";
export * from "./Message";
export * from "./MessageDecoderStream";
export * from "./MessageEncoderStream";
export * from "./SmithyMessageDecoderStream";
export * from "./SmithyMessageEncoderStream";

View File

@@ -0,0 +1,30 @@
import { Crc32 } from "@aws-crypto/crc32";
const PRELUDE_MEMBER_LENGTH = 4;
const PRELUDE_LENGTH = PRELUDE_MEMBER_LENGTH * 2;
const CHECKSUM_LENGTH = 4;
const MINIMUM_MESSAGE_LENGTH = PRELUDE_LENGTH + CHECKSUM_LENGTH * 2;
export function splitMessage({ byteLength, byteOffset, buffer }) {
if (byteLength < MINIMUM_MESSAGE_LENGTH) {
throw new Error("Provided message too short to accommodate event stream message overhead");
}
const view = new DataView(buffer, byteOffset, byteLength);
const messageLength = view.getUint32(0, false);
if (byteLength !== messageLength) {
throw new Error("Reported message length does not match received message length");
}
const headerLength = view.getUint32(PRELUDE_MEMBER_LENGTH, false);
const expectedPreludeChecksum = view.getUint32(PRELUDE_LENGTH, false);
const expectedMessageChecksum = view.getUint32(byteLength - CHECKSUM_LENGTH, false);
const checksummer = new Crc32().update(new Uint8Array(buffer, byteOffset, PRELUDE_LENGTH));
if (expectedPreludeChecksum !== checksummer.digest()) {
throw new Error(`The prelude checksum specified in the message (${expectedPreludeChecksum}) does not match the calculated CRC32 checksum (${checksummer.digest()})`);
}
checksummer.update(new Uint8Array(buffer, byteOffset + PRELUDE_LENGTH, byteLength - (PRELUDE_LENGTH + CHECKSUM_LENGTH)));
if (expectedMessageChecksum !== checksummer.digest()) {
throw new Error(`The message checksum (${checksummer.digest()}) did not match the expected value of ${expectedMessageChecksum}`);
}
return {
headers: new DataView(buffer, byteOffset + PRELUDE_LENGTH + CHECKSUM_LENGTH, headerLength),
body: new Uint8Array(buffer, byteOffset + PRELUDE_LENGTH + CHECKSUM_LENGTH + headerLength, messageLength - headerLength - (PRELUDE_LENGTH + CHECKSUM_LENGTH + CHECKSUM_LENGTH)),
};
}

View File

@@ -0,0 +1 @@
export {};