SnowLuma v2.4.2 source snapshot
Dev Build / cleanup (push) Waiting to run
Dev Build / typecheck (push) Waiting to run
Publish MCP to npm / Build, test, and publish MCP (push) Waiting to run
Publish SDK to npm / Build, test, and publish SDK (push) Waiting to run
Sync action catalog to docs / sync (push) Waiting to run
Publish UI to npm / Build and publish UI (push) Waiting to run
Build and Release / preflight (push) Waiting to run
Dev Build / cleanup (push) Waiting to run
Dev Build / typecheck (push) Waiting to run
Publish MCP to npm / Build, test, and publish MCP (push) Waiting to run
Publish SDK to npm / Build, test, and publish SDK (push) Waiting to run
Sync action catalog to docs / sync (push) Waiting to run
Publish UI to npm / Build and publish UI (push) Waiting to run
Build and Release / preflight (push) Waiting to run
This commit is contained in:
TanTan committed
commit
fa9c73363d
1367 files changed
+355735
No files matched your search
@@ -0,0 +1,94 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { schemaCodecs } from '../schema';
|
||||
|
||||
const bytesSchema = 'interface Msg { data: pb<1,bytes>; }';
|
||||
|
||||
describe('encoded buffers', () => {
|
||||
it('round-trips messages around allocation and slab boundaries', () => {
|
||||
const codec = schemaCodecs<{ data: Uint8Array }>(bytesSchema);
|
||||
|
||||
for (const length of [
|
||||
0, 1, 61, 62, 63, 64, 65, 125, 127, 128, 4092, 4093, 4094, 4095, 4096, 8192,
|
||||
]) {
|
||||
const data = Uint8Array.from({ length }, (_, i) => (i * 13) & 255);
|
||||
const encoded = codec.encode({ data });
|
||||
expect(codec.decode(encoded)).toEqual({ data });
|
||||
|
||||
if (encoded.length <= 64 || encoded.length > 4096) {
|
||||
expect(encoded.byteOffset).toBe(0);
|
||||
expect(encoded.buffer.byteLength).toBe(encoded.length);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps retained results intact across slab rotation and subsequent mutations', () => {
|
||||
const codec = schemaCodecs<{ data: Uint8Array }>(bytesSchema);
|
||||
const retained = Array.from({ length: 200 }, (_, index) => {
|
||||
const data = new Uint8Array([63, 128, 800, 1300][index % 4]).fill(index);
|
||||
const encoded = codec.encode({ data });
|
||||
return { data, encoded, snapshot: encoded.slice() };
|
||||
});
|
||||
expect(retained.some(({ encoded }) => encoded.byteOffset > 0)).toBe(true);
|
||||
|
||||
for (const { data, encoded, snapshot } of retained) {
|
||||
expect(encoded).toEqual(snapshot);
|
||||
expect(codec.decode(encoded)).toEqual({ data });
|
||||
}
|
||||
|
||||
retained[0].encoded.fill(0);
|
||||
|
||||
for (const { encoded, snapshot } of retained.slice(1)) {
|
||||
expect(encoded).toEqual(snapshot);
|
||||
}
|
||||
});
|
||||
|
||||
it('supports encoding from a getter without reusing the inner result', () => {
|
||||
const codec = schemaCodecs<{ data: Uint8Array }>(bytesSchema);
|
||||
const inner = new Uint8Array(128).fill(7);
|
||||
const outer = new Uint8Array(256).fill(9);
|
||||
let nested: Uint8Array | undefined;
|
||||
const encoded = codec.encode({
|
||||
get data() {
|
||||
nested = codec.encode({ data: inner });
|
||||
return outer;
|
||||
},
|
||||
});
|
||||
expect(codec.decode(encoded)).toEqual({ data: outer });
|
||||
expect(codec.decode(nested!)).toEqual({ data: inner });
|
||||
});
|
||||
|
||||
it('replaces a slab whose backing buffer was transferred', () => {
|
||||
const codec = schemaCodecs<{ data: Uint8Array }>(bytesSchema);
|
||||
const data = new Uint8Array(128).fill(3);
|
||||
const first = codec.encode({ data });
|
||||
const transferred = structuredClone(first, { transfer: [first.buffer as ArrayBuffer] });
|
||||
expect(first.byteLength).toBe(0);
|
||||
expect(codec.decode(transferred)).toEqual({ data });
|
||||
expect(codec.decode(codec.encode({ data }))).toEqual({ data });
|
||||
});
|
||||
|
||||
it('decodes unaligned pooled messages containing strings, fixed values and nested data', () => {
|
||||
const codec = schemaCodecs<{
|
||||
text: string;
|
||||
ratio: number;
|
||||
inner: { value: bigint; data: Uint8Array };
|
||||
}>(`
|
||||
interface Inner {value:pb<1,fixed_64>;data:pb<2,bytes>;}
|
||||
interface Msg {text:pb<1,string>;ratio:pb<2,double>;inner:pb<3,Inner>;}
|
||||
`);
|
||||
let unaligned = false;
|
||||
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const value = {
|
||||
text: '中文🚀'.repeat(8) + i,
|
||||
ratio: Math.PI + i,
|
||||
inner: { value: (1n << 63n) + BigInt(i), data: new Uint8Array(17 + i).fill(i) },
|
||||
};
|
||||
const encoded = codec.encode(value);
|
||||
unaligned ||= encoded.byteOffset % 8 !== 0;
|
||||
expect(codec.decode(encoded)).toEqual(value);
|
||||
}
|
||||
|
||||
expect(unaligned).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,245 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { schemaRegistry } from '../schema';
|
||||
import { generateCode } from '../../../src/codegen/generator';
|
||||
const simple = 'interface SimpleMsg { value: pb<1, uint_32>; }';
|
||||
const nested =
|
||||
'interface Inner { value: pb<1, uint_32>; } interface Outer { inner: pb<1, Inner>; }';
|
||||
const string = 'interface StringMsg { text: pb<1, string>; }';
|
||||
const bool = 'interface BoolMsg { active: pb<1, bool>; }';
|
||||
const repeated =
|
||||
'interface RepeatedMsg { ids: pb_repeated<1, uint_32>; names: pb_repeated<2, string>; }';
|
||||
|
||||
function makeRoundTrip(schema: string, msgName: string) {
|
||||
const gen = generateCode(schemaRegistry(schema, `${msgName}.ts`));
|
||||
return new Function(
|
||||
gen + `\nreturn { enc: protobuf_encode_${msgName}, dec: protobuf_decode_${msgName} };`,
|
||||
)() as {
|
||||
enc: (obj: any) => Uint8Array;
|
||||
dec: (data: Uint8Array) => any;
|
||||
};
|
||||
}
|
||||
|
||||
describe('code generator', () => {
|
||||
it('allocates one output buffer for nested repeated messages', () => {
|
||||
const schema = `interface Leaf { text: pb<1,string>; } interface Msg { children: pb_repeated<1,Leaf>; }`;
|
||||
const registry = schemaRegistry(schema, 'allocation.ts');
|
||||
const code = generateCode(registry, { encodeRoots: ['Msg'], decodeRoots: [] });
|
||||
let allocations = 0;
|
||||
class CountingBytes extends Uint8Array {
|
||||
constructor(size: number) {
|
||||
super(size);
|
||||
allocations++;
|
||||
}
|
||||
static get [Symbol.species]() {
|
||||
return Uint8Array;
|
||||
}
|
||||
}
|
||||
const encode = new Function('Uint8Array', code + '\nreturn protobuf_encode_Msg;')(
|
||||
CountingBytes,
|
||||
);
|
||||
const input = { children: Array.from({ length: 32 }, (_, i) => ({ text: '你好' + i })) };
|
||||
const encoded = encode(input);
|
||||
expect(allocations).toBe(1);
|
||||
expect(makeRoundTrip(schema, 'Msg').dec(encoded)).toEqual(input);
|
||||
});
|
||||
|
||||
it('omits unused helpers and decoder functions from encode-only output', () => {
|
||||
const code = generateCode(
|
||||
schemaRegistry('interface Msg { value: pb<1,uint_32>; }', 'minimal.ts'),
|
||||
{
|
||||
encodeRoots: ['Msg'],
|
||||
decodeRoots: [],
|
||||
},
|
||||
);
|
||||
expect(code).not.toContain('TextDecoder');
|
||||
expect(code).not.toContain('TextEncoder');
|
||||
expect(code).not.toContain('DataView');
|
||||
expect(code).not.toContain('BigInt');
|
||||
expect(code).not.toContain('protobuf_decode_');
|
||||
});
|
||||
|
||||
it('round-trips a deep mixture of strings and nested messages', () => {
|
||||
const schema = Array.from(
|
||||
{ length: 24 },
|
||||
(_, i) =>
|
||||
`interface M${i} { text: pb<1,string>; ${i < 23 ? `next: pb<2,M${i + 1}>;` : ''} }`,
|
||||
).join('\n');
|
||||
let input: any = { text: 'end' };
|
||||
|
||||
for (let i = 22; i >= 0; i--) {
|
||||
input = { text: 'level' + i, next: input };
|
||||
}
|
||||
|
||||
const { enc, dec } = makeRoundTrip(schema, 'M0');
|
||||
expect(dec(enc(input))).toEqual(input);
|
||||
});
|
||||
it('round-trips int_32 and preserves five-byte negative varints', () => {
|
||||
const { enc, dec } = makeRoundTrip(
|
||||
`interface IntMsg {
|
||||
value: pb<1, int_32>;
|
||||
values: pb_repeated<2, int_32>;
|
||||
}`,
|
||||
'IntMsg',
|
||||
);
|
||||
expect([...enc({ value: -1 })]).toEqual([0x08, 0xff, 0xff, 0xff, 0xff, 0x0f]);
|
||||
const input = { value: -2147483648, values: [-1, -2147483648, 0, 2147483647] };
|
||||
expect(dec(enc(input))).toEqual(input);
|
||||
});
|
||||
|
||||
it('preserves fixed_32 unsigned values and sfixed_32 signed values', () => {
|
||||
const { enc, dec } = makeRoundTrip(
|
||||
`interface FixedMsg {
|
||||
unsigned: pb<1, fixed_32>;
|
||||
signed: pb<2, sfixed_32>;
|
||||
values: pb_repeated<3, fixed_32>;
|
||||
}`,
|
||||
'FixedMsg',
|
||||
);
|
||||
const input = {
|
||||
unsigned: 0xffffffff,
|
||||
signed: -2147483648,
|
||||
values: [0x80000000, 0xffffffff],
|
||||
};
|
||||
expect(dec(enc(input))).toEqual(input);
|
||||
});
|
||||
|
||||
it('uses bigint defaults for all 64-bit integer fields', () => {
|
||||
const { enc, dec } = makeRoundTrip(
|
||||
`interface Defaults {
|
||||
unsigned: pb<1, uint_64>;
|
||||
signed: pb<2, int_64>;
|
||||
zigzag: pb<3, sint_64>;
|
||||
fixed: pb<4, fixed_64>;
|
||||
sfixed: pb<5, sfixed_64>;
|
||||
}`,
|
||||
'Defaults',
|
||||
);
|
||||
const result = dec(new Uint8Array());
|
||||
expect(result).toEqual({ unsigned: 0n, signed: 0n, zigzag: 0n, fixed: 0n, sfixed: 0n });
|
||||
expect(enc(result)).toEqual(new Uint8Array());
|
||||
});
|
||||
|
||||
it('decodes tags with the highest valid field number', () => {
|
||||
const { enc, dec } = makeRoundTrip(
|
||||
'interface HighTag { value: pb<536870911, uint_32>; }',
|
||||
'HighTag',
|
||||
);
|
||||
const bytes = new Uint8Array([0xf8, 0xff, 0xff, 0xff, 0x0f, 7]);
|
||||
expect(enc({ value: 7 })).toEqual(bytes);
|
||||
expect(dec(bytes)).toEqual({ value: 7 });
|
||||
});
|
||||
|
||||
it('generates encode/decode for simple uint_32', () => {
|
||||
const gen = generateCode(schemaRegistry(simple, 't.ts'));
|
||||
expect(gen).toContain('function protobuf_encode_SimpleMsg(obj)');
|
||||
expect(gen).toContain('function protobuf_decode_SimpleMsg(data');
|
||||
// Tag for field 1, varint should be pre-computed as 0x08
|
||||
expect(gen).toContain('buf[offset++] = 8;');
|
||||
});
|
||||
|
||||
it('generates nested message code', () => {
|
||||
const gen = generateCode(schemaRegistry(nested, 't.ts'));
|
||||
expect(gen).toContain('protobuf_encode_Inner(');
|
||||
expect(gen).toContain('protobuf_decode_Inner(data, offset, offset + _len)');
|
||||
});
|
||||
|
||||
it('round-trip uint_32', () => {
|
||||
const { enc, dec } = makeRoundTrip(simple, 'SimpleMsg');
|
||||
expect(dec(enc({ value: 42 })).value).toBe(42);
|
||||
});
|
||||
|
||||
it('round-trip nested message', () => {
|
||||
const { enc, dec } = makeRoundTrip(nested, 'Outer');
|
||||
expect(dec(enc({ inner: { value: 999 } })).inner.value).toBe(999);
|
||||
});
|
||||
|
||||
it('round-trip string', () => {
|
||||
const { enc, dec } = makeRoundTrip(string, 'StringMsg');
|
||||
expect(dec(enc({ text: 'hello world' })).text).toBe('hello world');
|
||||
});
|
||||
|
||||
it('round-trip unicode string', () => {
|
||||
const { enc, dec } = makeRoundTrip(string, 'StringMsg');
|
||||
expect(dec(enc({ text: '你好，protobuf🚀' })).text).toBe('你好，protobuf🚀');
|
||||
});
|
||||
|
||||
it('round-trip bool', () => {
|
||||
const { enc, dec } = makeRoundTrip(bool, 'BoolMsg');
|
||||
expect(dec(enc({ active: true })).active).toBe(true);
|
||||
const empty = enc({ active: false });
|
||||
expect(empty.length).toBe(0);
|
||||
expect(dec(empty).active).toBe(false);
|
||||
});
|
||||
|
||||
it('proto3 default encoding (0 → empty)', () => {
|
||||
const { enc } = makeRoundTrip(simple, 'SimpleMsg');
|
||||
expect(enc({ value: 0 }).length).toBe(0);
|
||||
});
|
||||
|
||||
it('round-trip repeated uint_32', () => {
|
||||
const { enc, dec } = makeRoundTrip(repeated, 'RepeatedMsg');
|
||||
const result = dec(enc({ ids: [10, 20, 30], names: [] }));
|
||||
expect(result.ids).toEqual([10, 20, 30]);
|
||||
expect(result.names).toEqual([]);
|
||||
});
|
||||
|
||||
it('round-trip repeated string', () => {
|
||||
const { enc, dec } = makeRoundTrip(repeated, 'RepeatedMsg');
|
||||
const result = dec(enc({ ids: [], names: ['hello', 'world'] }));
|
||||
expect(result.ids).toEqual([]);
|
||||
expect(result.names).toEqual(['hello', 'world']);
|
||||
});
|
||||
|
||||
it('repeated empty arrays round-trip', () => {
|
||||
const { enc, dec } = makeRoundTrip(repeated, 'RepeatedMsg');
|
||||
const result = dec(enc({ ids: [], names: [] }));
|
||||
expect(result.ids).toEqual([]);
|
||||
expect(result.names).toEqual([]);
|
||||
});
|
||||
|
||||
it('round-trip bigint-backed 64-bit integers', () => {
|
||||
const schema = `
|
||||
interface BigIntMsg {
|
||||
unsigned: pb<1, uint_64>;
|
||||
signed: pb<2, int_64>;
|
||||
zigzag: pb<3, sint_64>;
|
||||
fixed: pb<4, fixed_64>;
|
||||
sfixed: pb<5, sfixed_64>;
|
||||
ids: pb_repeated<6, uint_64>;
|
||||
}`;
|
||||
const { enc, dec } = makeRoundTrip(schema, 'BigIntMsg');
|
||||
const input = {
|
||||
unsigned: 18446744073709551615n,
|
||||
signed: -1n,
|
||||
zigzag: -1234567890123456789n,
|
||||
fixed: 0x0102030405060708n,
|
||||
sfixed: -0x0102030405060708n,
|
||||
ids: [1n, 2n, 9007199254740993n],
|
||||
};
|
||||
expect(dec(enc(input))).toEqual(input);
|
||||
});
|
||||
|
||||
it('round-trip sint_32 with negative values', () => {
|
||||
const schema = `
|
||||
interface Sint32Msg {
|
||||
value: pb<1, sint_32>;
|
||||
values: pb_repeated<2, sint_32>;
|
||||
}`;
|
||||
const { enc, dec } = makeRoundTrip(schema, 'Sint32Msg');
|
||||
const input = { value: -789, values: [-1, 0, 1, -789, 2147483647, -2147483648] };
|
||||
expect(dec(enc(input))).toEqual(input);
|
||||
});
|
||||
|
||||
it('round-trip double as ieee754 number', () => {
|
||||
const schema = `
|
||||
interface DoubleMsg {
|
||||
value: pb<1, double>;
|
||||
values: pb_repeated<2, double>;
|
||||
}`;
|
||||
const { enc, dec } = makeRoundTrip(schema, 'DoubleMsg');
|
||||
const result = dec(enc({ value: Math.PI, values: [Math.E, -0.5] }));
|
||||
expect(result.value).toBeCloseTo(Math.PI);
|
||||
expect(result.values[0]).toBeCloseTo(Math.E);
|
||||
expect(result.values[1]).toBeCloseTo(-0.5);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,212 @@
|
||||
import { deepStrictEqual } from 'node:assert';
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { schemaRegistry } from '../schema';
|
||||
import { generateCode } from '../../../src/codegen/generator';
|
||||
|
||||
const code = generateCode(
|
||||
schemaRegistry(`
|
||||
interface Text { value: pb<1,string>; }
|
||||
interface Nested { texts: pb_repeated<1,Text>; }
|
||||
`),
|
||||
);
|
||||
const { encode, decode, encodeNested, decodeNested } = new Function(`${code}
|
||||
return {
|
||||
encode: protobuf_encode_Text, decode: protobuf_decode_Text,
|
||||
encodeNested: protobuf_encode_Nested, decodeNested: protobuf_decode_Nested,
|
||||
};`)() as {
|
||||
decode(bytes: Uint8Array): { value: string };
|
||||
encode(value: { value: string }): Uint8Array;
|
||||
encodeNested(value: { texts: { value: string }[] }): Uint8Array;
|
||||
decodeNested(bytes: Uint8Array): { texts: { value: string }[] };
|
||||
};
|
||||
const encoder = new TextEncoder();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
function wire(payload: Uint8Array): Uint8Array {
|
||||
const prefix = [10];
|
||||
let length = payload.length;
|
||||
|
||||
do {
|
||||
prefix.push((length & 127) | (length >= 128 ? 128 : 0));
|
||||
length >>>= 7;
|
||||
} while (length);
|
||||
|
||||
return Uint8Array.from([...prefix, ...payload]);
|
||||
}
|
||||
|
||||
describe('UTF-8 codecs', () => {
|
||||
it('matches TextEncoder across native encoding boundaries, including unpaired surrogates', () => {
|
||||
for (const length of [31, 32, 33, 42, 43, 63, 64, 65, 127, 128, 129, 4096]) {
|
||||
for (const unit of ['a', 'é', '中文', '🚀', '\ud800', '\udc00', 'a\ud800']) {
|
||||
const value = unit.repeat(length);
|
||||
// Generic iterable equality repeats per-byte work on these large views.
|
||||
deepStrictEqual(encode({ value }), wire(encoder.encode(value)));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves every ASCII byte across short and native decoding boundaries', () => {
|
||||
for (const length of [
|
||||
0, 1, 3, 4, 7, 8, 15, 16, 23, 24, 30, 31, 32, 33, 63, 64, 127, 128, 4096,
|
||||
]) {
|
||||
for (let byte = 0; byte < 128; byte++) {
|
||||
const value = String.fromCharCode(byte).repeat(length);
|
||||
deepStrictEqual(
|
||||
encode({ value }),
|
||||
length ? wire(encoder.encode(value)) : new Uint8Array(),
|
||||
);
|
||||
expect(decode(wire(encoder.encode(value)))).toEqual({ value });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('matches TextDecoder for Unicode after ASCII prefixes and at slice boundaries', () => {
|
||||
const suffixes = [
|
||||
'\u0080',
|
||||
'é',
|
||||
'\u07ff',
|
||||
'\u0800',
|
||||
'中文',
|
||||
'\ud7ff',
|
||||
'\ue000',
|
||||
'\uffff',
|
||||
'🚀',
|
||||
'\u{10000}',
|
||||
'\u{10ffff}',
|
||||
'\ufeff',
|
||||
'\ufeffx\ufeff',
|
||||
'\ud800',
|
||||
'\udc00',
|
||||
];
|
||||
|
||||
for (let length = 0; length <= 64; length++) {
|
||||
for (const suffix of suffixes) {
|
||||
const value = 'a'.repeat(length) + suffix;
|
||||
const payload = encoder.encode(value);
|
||||
const bytes = wire(payload);
|
||||
const padded = Uint8Array.from([255, 255, ...bytes, 128, 128]);
|
||||
expect(decode(padded.subarray(2, 2 + bytes.length)).value).toBe(
|
||||
decoder.decode(payload),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('matches replacement behavior for all one-byte and two-byte sequences', () => {
|
||||
const payload = new Uint8Array(2);
|
||||
|
||||
for (let first = 0; first < 256; first++) {
|
||||
payload[0] = first;
|
||||
const one = payload.subarray(0, 1);
|
||||
expect(decode(wire(one)).value).toBe(decoder.decode(one));
|
||||
|
||||
for (let second = 0; second < 256; second++) {
|
||||
payload[1] = second;
|
||||
expect(decode(wire(payload)).value).toBe(decoder.decode(payload));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('matches TextDecoder across randomized valid Unicode strings', () => {
|
||||
let state = 0x5a17cafe;
|
||||
|
||||
for (let sample = 0; sample < 5000; sample++) {
|
||||
let value = 'a'.repeat(sample % 17);
|
||||
|
||||
for (let index = 0; index < sample % 21; index++) {
|
||||
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
||||
value += String.fromCodePoint(state % 0x110000);
|
||||
}
|
||||
|
||||
const payload = encoder.encode(value);
|
||||
expect(decode(wire(payload)).value).toBe(decoder.decode(payload));
|
||||
}
|
||||
});
|
||||
|
||||
it('matches TextDecoder for invalid and truncated multibyte sequences', () => {
|
||||
const sequences = [
|
||||
[0xe0, 0x80, 0x80],
|
||||
[0xed, 0xa0, 0x80],
|
||||
[0xf0, 0x80, 0x80, 0x80],
|
||||
[0xf4, 0x90, 0x80, 0x80],
|
||||
[0xf5, 0x80, 0x80, 0x80],
|
||||
[0xff],
|
||||
[0xc2],
|
||||
[0xe1, 0x80],
|
||||
[0xf1, 0x80, 0x80],
|
||||
[0xe1, 65, 0x80],
|
||||
[0xf1, 0x80, 65, 0x80],
|
||||
[0xef, 0xbb, 0xbf, 0xff],
|
||||
];
|
||||
|
||||
for (let length = 0; length <= 33; length++) {
|
||||
for (const sequence of sequences) {
|
||||
const payload = Uint8Array.from([...Array(length).fill(65), ...sequence]);
|
||||
expect(decode(wire(payload)).value).toBe(decoder.decode(payload));
|
||||
}
|
||||
}
|
||||
|
||||
let state = 0x42abcdef;
|
||||
|
||||
for (let sample = 0; sample < 5000; sample++) {
|
||||
const payload = Uint8Array.from({ length: sample % 65 }, () => {
|
||||
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
||||
return state >>> 24;
|
||||
});
|
||||
expect(decode(wire(payload)).value).toBe(decoder.decode(payload));
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps nested and repeated strings independent', () => {
|
||||
const texts = [
|
||||
'',
|
||||
'ascii',
|
||||
'a'.repeat(31),
|
||||
'a'.repeat(32),
|
||||
'中文🚀',
|
||||
'a\ufeffb',
|
||||
'\ufeffx',
|
||||
'中文🚀'.repeat(64),
|
||||
].map((value) => ({ value }));
|
||||
expect(decodeNested(encodeNested({ texts }))).toEqual({
|
||||
texts: texts.map(({ value }) => ({ value: decoder.decode(encoder.encode(value)) })),
|
||||
});
|
||||
// A truncated UTF-8 sequence must not consume bytes from the following field.
|
||||
const first = wire(Uint8Array.from([0xf0, 0x9f]));
|
||||
const second = wire(encoder.encode('next'));
|
||||
const nested = Uint8Array.from([10, first.length, ...first, 10, second.length, ...second]);
|
||||
expect(decodeNested(nested)).toEqual({ texts: [{ value: '�' }, { value: 'next' }] });
|
||||
});
|
||||
|
||||
it('measures UTF-8 correctly when a code point crosses a scratch-buffer boundary', () => {
|
||||
for (const length of [4093, 4094, 4095, 4096, 4097, 8191]) {
|
||||
for (const suffix of ['é', '中', '🚀', '\ud800', '\udc00', '\ud800x']) {
|
||||
const value = 'a'.repeat(length) + suffix + '中文🚀'.repeat(4000);
|
||||
deepStrictEqual(encode({ value }), wire(encoder.encode(value)));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves replacement and BOM behavior after long malformed inputs', () => {
|
||||
for (const length of [127, 128, 129, 255, 4095, 4096, 4097]) {
|
||||
for (const sequence of [
|
||||
[0xff],
|
||||
[0xe0, 0x80, 0x80],
|
||||
[0xed, 0xa0, 0x80],
|
||||
[0xf0, 0x9f],
|
||||
[0xef, 0xbb, 0xbf],
|
||||
]) {
|
||||
for (const prefix of [[], [0xef, 0xbb, 0xbf]]) {
|
||||
const payload = Uint8Array.from([
|
||||
...prefix,
|
||||
...Array(length).fill(65),
|
||||
...sequence,
|
||||
]);
|
||||
expect(decode(wire(payload)).value).toBe(decoder.decode(payload));
|
||||
const valid = encoder.encode('中文🚀'.repeat(64));
|
||||
expect(decode(wire(valid)).value).toBe(decoder.decode(valid));
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { schemaCodecs as codecs } from '../schema';
|
||||
|
||||
describe('generated wire format', () => {
|
||||
it('matches TextEncoder at UTF-8 length-prefix boundaries', () => {
|
||||
const codec = codecs<{ value: string }>('interface Msg {value:pb<1,string>;}');
|
||||
|
||||
for (const length of [127, 128, 16383, 16384]) {
|
||||
for (const value of ['a'.repeat(length), '界🚀\ud800'.repeat(length)]) {
|
||||
const payload = new TextEncoder().encode(value);
|
||||
const prefix = [10];
|
||||
let remaining = payload.length;
|
||||
|
||||
do {
|
||||
prefix.push((remaining % 128) | (remaining >= 128 ? 128 : 0));
|
||||
remaining = Math.floor(remaining / 128);
|
||||
} while (remaining);
|
||||
|
||||
const expected = new Uint8Array(prefix.length + payload.length);
|
||||
expected.set(prefix);
|
||||
expected.set(payload, prefix.length);
|
||||
expect(codec.encode({ value })).toEqual(expected);
|
||||
expect(codec.decode(expected)).toEqual({
|
||||
value: new TextDecoder().decode(payload),
|
||||
});
|
||||
}
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('decodes floating-point, fixed-width and byte fields from unaligned subarrays', () => {
|
||||
const codec = codecs<{ f: number; d: number; fixed: bigint; data: Uint8Array }>(
|
||||
`interface Msg {f:pb<1,float>;d:pb<2,double>;fixed:pb<3,fixed_64>;data:pb<4,bytes>;}`,
|
||||
);
|
||||
const input = {
|
||||
f: Math.fround(1 / 3),
|
||||
d: -Number.MAX_VALUE,
|
||||
fixed: (1n << 64n) - 1n,
|
||||
data: new Uint8Array([0, 1, 255]),
|
||||
};
|
||||
const wire = codec.encode(input);
|
||||
|
||||
for (const padding of [1, 3, 7, 31]) {
|
||||
const buffer = new Uint8Array(wire.length + padding + 5).fill(255);
|
||||
buffer.set(wire, padding);
|
||||
expect(codec.decode(buffer.subarray(padding, padding + wire.length))).toEqual(input);
|
||||
}
|
||||
});
|
||||
it('matches an independent bigint varint reference at every byte boundary', () => {
|
||||
const codec = codecs<{ value: bigint }>('interface Msg { value: pb<1, uint_64>; }');
|
||||
const values = new Set<bigint>([0n, 1n, (1n << 64n) - 1n]);
|
||||
|
||||
for (let bit = 1n; bit < 64n; bit++) {
|
||||
values.add((1n << bit) - 1n);
|
||||
values.add(1n << bit);
|
||||
values.add((1n << bit) + 1n);
|
||||
}
|
||||
|
||||
for (const value of values) {
|
||||
let rest = value;
|
||||
const bytes = [];
|
||||
|
||||
do {
|
||||
bytes.push(Number(rest & 127n) | (rest > 127n ? 128 : 0));
|
||||
rest >>= 7n;
|
||||
} while (rest);
|
||||
|
||||
const wire = new Uint8Array([8, ...bytes]);
|
||||
expect(codec.encode({ value })).toEqual(value === 0n ? new Uint8Array() : wire);
|
||||
expect(codec.decode(wire)).toEqual({ value });
|
||||
}
|
||||
});
|
||||
|
||||
it('decodes nonzero booleans whose low 32 bits are zero', () => {
|
||||
const codec = codecs<{ value: boolean }>('interface Msg { value: pb<1, bool>; }');
|
||||
expect(codec.decode(new Uint8Array([8, 128, 128, 128, 128, 16]))).toEqual({ value: true });
|
||||
});
|
||||
const uint32 = codecs<{ value: number }>('interface Msg { value: pb<1, uint_32>; }');
|
||||
const varints: [number, number[]][] = [
|
||||
[1, [0x01]],
|
||||
[127, [0x7f]],
|
||||
[128, [0x80, 0x01]],
|
||||
[300, [0xac, 0x02]],
|
||||
[16384, [0x80, 0x80, 0x01]],
|
||||
[0xffffffff, [0xff, 0xff, 0xff, 0xff, 0x0f]],
|
||||
];
|
||||
|
||||
it.each(varints)('encodes uint32 %i using the expected wire bytes', (value, bytes) => {
|
||||
expect([...uint32.encode({ value })]).toEqual([0x08, ...bytes]);
|
||||
});
|
||||
|
||||
it.each(varints)('decodes uint32 %i from independent wire bytes', (value, bytes) => {
|
||||
expect(uint32.decode(new Uint8Array([0x08, ...bytes]))).toEqual({ value });
|
||||
});
|
||||
|
||||
it('omits scalar zero and decodes an explicit zero', () => {
|
||||
expect(uint32.encode({ value: 0 })).toEqual(new Uint8Array());
|
||||
expect(uint32.decode(new Uint8Array([8, 0]))).toEqual({ value: 0 });
|
||||
});
|
||||
|
||||
it.each(['hello', '', '你好', '🚀', '\ud800', '\udc00'])('encodes UTF-8 %j', (value) => {
|
||||
const codec = codecs<{ value: string }>('interface Msg { value: pb<1, string>; }');
|
||||
const payload = new TextEncoder().encode(value);
|
||||
const bytes = new Uint8Array([0x0a, payload.length, ...payload]);
|
||||
expect(codec.encode({ value })).toEqual(value === '' ? new Uint8Array() : bytes);
|
||||
expect(codec.decode(bytes)).toEqual({ value: new TextDecoder().decode(payload) });
|
||||
});
|
||||
|
||||
it('encodes bytes with a length prefix and copies the decoded data', () => {
|
||||
const codec = codecs<{ value: Uint8Array }>('interface Msg { value: pb<1, bytes>; }');
|
||||
const bytes = new Uint8Array([10, 3, 1, 128, 255]);
|
||||
const value = new Uint8Array([1, 128, 255]);
|
||||
expect(codec.encode({ value })).toEqual(bytes);
|
||||
const result = codec.decode(bytes);
|
||||
bytes[2] = 0;
|
||||
expect(result.value).toEqual(value);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['varint', [0x10, 0xac, 0x02]],
|
||||
['64-bit', [0x11, 1, 2, 3, 4, 5, 6, 7, 8]],
|
||||
['length-delimited', [0x12, 3, 0x41, 0x42, 0x43]],
|
||||
['32-bit', [0x15, 1, 2, 3, 4]],
|
||||
])('skips unknown %s fields and resumes decoding', (_name, bytes) => {
|
||||
expect(uint32.decode(new Uint8Array([...(bytes as number[]), 8, 7]))).toEqual({ value: 7 });
|
||||
});
|
||||
|
||||
const packedCases: [string, (number | bigint | boolean)[], number[]][] = [
|
||||
['uint_32', [1, 300], [1, 0xac, 2]],
|
||||
['int_32', [1, -2], [1, 0xfe, ...Array(8).fill(0xff), 1]],
|
||||
['sint_32', [-1, 1], [1, 2]],
|
||||
['bool', [true, false], [1, 0]],
|
||||
['uint_64', [1n, 300n], [1, 0xac, 2]],
|
||||
['int_64', [1n, -2n], [1, 0xfe, ...Array(8).fill(0xff), 1]],
|
||||
['sint_64', [-1n, 1n], [1, 2]],
|
||||
['fixed_32', [0xffffffff, 1], [255, 255, 255, 255, 1, 0, 0, 0]],
|
||||
['sfixed_32', [-1, 1], [255, 255, 255, 255, 1, 0, 0, 0]],
|
||||
['fixed_64', [1n], [1, 0, 0, 0, 0, 0, 0, 0]],
|
||||
['sfixed_64', [-1n], Array(8).fill(255)],
|
||||
['float', [1], [0, 0, 128, 63]],
|
||||
['double', [1], [0, 0, 0, 0, 0, 0, 240, 63]],
|
||||
];
|
||||
|
||||
it.each(packedCases)(
|
||||
'decodes packed, unpacked and split %s arrays',
|
||||
(type, values, payload) => {
|
||||
const codec = codecs<{ values: typeof values }>(
|
||||
`interface Msg { values: pb_repeated<1, ${type}>; }`,
|
||||
);
|
||||
const packed = [10, payload.length, ...payload];
|
||||
const unpacked = codec.encode({ values });
|
||||
expect(codec.decode(new Uint8Array(packed))).toEqual({ values });
|
||||
expect(codec.decode(new Uint8Array([...packed, ...unpacked, ...packed]))).toEqual({
|
||||
values: [...values, ...values, ...values],
|
||||
});
|
||||
expect(codec.decode(new Uint8Array([10, 0]))).toEqual({ values: [] });
|
||||
},
|
||||
);
|
||||
|
||||
const malformedCodec = codecs(`
|
||||
interface Inner { value: pb<1, uint_32>; }
|
||||
interface Msg {
|
||||
value: pb<1, uint_32>;
|
||||
big: pb<2, uint_64>;
|
||||
text: pb<3, string>;
|
||||
data: pb<4, bytes>;
|
||||
float: pb<5, float>;
|
||||
double: pb<6, double>;
|
||||
inner: pb<7, Inner>;
|
||||
values: pb_repeated<8, uint_32>;
|
||||
fixed: pb_repeated<9, fixed_32>;
|
||||
}
|
||||
`);
|
||||
const invalidCases: [string, number[]][] = [
|
||||
['zero tag', [0]],
|
||||
['invalid wire type', [15]],
|
||||
['truncated tag', [128]],
|
||||
['overflowing tag', [255, 255, 255, 255, 31]],
|
||||
['missing varint', [8]],
|
||||
['unterminated varint', [8, 128]],
|
||||
['overflowing varint', [8, ...Array(10).fill(255)]],
|
||||
['truncated bigint', [16, 128]],
|
||||
['overflowing bigint', [16, ...Array(9).fill(255), 2]],
|
||||
['missing string length', [26]],
|
||||
['truncated string', [26, 2, 65]],
|
||||
['overflowing length', [26, 255, 255, 255, 255, 31]],
|
||||
['truncated bytes', [34, 2, 1]],
|
||||
['truncated float', [45, 0]],
|
||||
['truncated double', [49, 0]],
|
||||
['nested message exceeds parent', [58, 5, 8, 1]],
|
||||
['nested varint exceeds nested boundary', [58, 2, 8, 128, 8, 7]],
|
||||
['packed varint exceeds packed boundary', [66, 1, 128, 8, 7]],
|
||||
['packed fixed32 has partial value', [74, 3, 1, 2, 3, 8, 7]],
|
||||
['unknown truncated varint', [80, 128]],
|
||||
['unknown truncated fixed64', [81, 0]],
|
||||
['unknown truncated bytes', [82, 5, 1]],
|
||||
['unknown truncated fixed32', [85, 0]],
|
||||
];
|
||||
|
||||
it.each(invalidCases)('rejects %s', (_name, bytes) => {
|
||||
expect(() => malformedCodec.decode(new Uint8Array(bytes))).toThrow(/protobuf/);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user