import { EventEmitter } from 'node:events'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { getLogLevel, setLogLevel, subscribeLogs, type LogEntry } from '@snowluma/common/logger'; import type { ReqDataHighwayHead, RespDataHighwayHead } from '@snowluma/proto-defs/highway'; import { protobuf_decode, protobuf_encode } from '@snowluma/proton'; import type { BridgeContext } from '@snowluma/protocol/bridge-context'; import { packHighwayFrame, unpackHighwayFrame, computeMd5 } from '@snowluma/protocol/highway/utils'; import { uploadHighwayHttp, type ChunkSource, type HighwaySession, } from '@snowluma/protocol/highway'; const { connect } = vi.hoisted(() => ({ connect: vi.fn() })); vi.mock('net', () => ({ default: { createConnection: connect } })); const BLOCK = 1024 * 1024; const bridge = { identity: { uin: '10001' } } as unknown as BridgeContext; const session: HighwaySession = { sigSession: new Uint8Array([1]), sessionKey: new Uint8Array(), servers: [ { host: '192.0.2.1', port: 80 }, { host: '192.0.2.2', port: 80 }, ], }; const turn = () => new Promise((resolve) => setImmediate(resolve)); interface Request { socket: TestSocket; head: ReqDataHighwayHead; bytes: Uint8Array; offset: number; respond(override?: RespDataHighwayHead): void; frame(bytes: Uint8Array): void; disconnect(): void; } let requests: Request[]; let sockets: TestSocket[]; let active: Set; let peak: number; let onRequest: (request: Request) => void; class TestSocket extends EventEmitter { destroyed = false; readableEnded = false; writable = true; private buffer = Buffer.alloc(0); setTimeout() { return this; } write(data: Uint8Array, callback: (error?: Error) => void): boolean { if (this.destroyed) { callback(new Error('socket closed')); return false; } this.buffer = Buffer.concat([this.buffer, data]); const end = this.buffer.indexOf('\r\n\r\n'); if (end >= 0) { const length = Number( this.buffer .subarray(0, end) .toString() .match(/Content-Length: (\d+)/i)![1], ); if (this.buffer.length >= end + 4 + length) { const { head, body } = unpackHighwayFrame( this.buffer.subarray(end + 4, end + 4 + length), ); const decoded = protobuf_decode(head); this.buffer = this.buffer.subarray(end + 4 + length); const request: Request = { socket: this, head: decoded, bytes: body, offset: Number(decoded.msgSegHead?.dataOffset ?? 0n), frame: (frame) => { if (this.destroyed) { throw new Error('reply on destroyed socket'); } active.delete(request); this.emit( 'data', Buffer.concat([ Buffer.from( `HTTP/1.1 200 OK\r\nContent-Length: ${frame.length}\r\n\r\n`, ), frame, ]), ); }, respond: (override = {}) => request.frame( packHighwayFrame( protobuf_encode({ msgBaseHead: { seq: decoded.msgBaseHead?.seq }, msgSegHead: { filesize: decoded.msgSegHead?.filesize, dataOffset: decoded.msgSegHead?.dataOffset, dataLength: decoded.msgSegHead?.dataLength, }, ...override, }), new Uint8Array(), ), ), disconnect: () => this.emit('error', new Error('ECONNRESET')), }; expect([...active].some((item) => item.socket === this)).toBe(false); requests.push(request); active.add(request); peak = Math.max(peak, active.size); // Deliberately respond before the write callback. Real fast peers // can reply while socket.write is still completing. onRequest(request); } } callback(); return true; } destroy() { if (!this.destroyed) { this.destroyed = true; this.writable = false; for (const request of active) { if (request.socket === this) { active.delete(request); } } queueMicrotask(() => this.emit('close')); } return this; } } class Source implements ChunkSource { reads: number[] = []; closes = 0; constructor(readonly size: number) {} async read(offset: number, length: number) { this.reads.push(offset); return new Uint8Array(length).fill(offset / BLOCK + 1); } async close() { this.closes += 1; } } function upload(source: ChunkSource, concurrency: number) { return uploadHighwayHttp(bridge, session, 1005, source, new Uint8Array(16), new Uint8Array(), { concurrency, }); } const level = getLogLevel(); beforeEach(() => { requests = []; sockets = []; active = new Set(); peak = 0; onRequest = () => {}; connect.mockReset().mockImplementation((_options, listener) => { const socket = new TestSocket(); sockets.push(socket); setImmediate(listener); return socket; }); }); afterEach(() => { setLogLevel(level); vi.unstubAllEnvs(); vi.useRealTimers(); for (const socket of sockets) { socket.destroy(); } }); describe('bounded parallel highway uploads', () => { it('does not let trickled response bytes extend the whole-file deadline', async () => { vi.useFakeTimers(); vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '100'); vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_IDLE_TIMEOUT_MS', '30'); onRequest = ({ socket }) => { const timer = setInterval(() => socket.emit('data', Buffer.from('x')), 10); socket.once('close', () => clearInterval(timer)); }; const source = new Source(80); const start = Date.now(); const failed = expect(upload(source, 1)).rejects.toThrow('timed out after 100ms'); await Promise.all([failed, vi.runAllTimersAsync()]); expect(Date.now() - start).toBe(100); expect(source.closes).toBe(1); expect(active.size).toBe(0); expect(vi.getTimerCount()).toBe(0); }); it('drains a pending source read after the deadline without sending its bytes', async () => { vi.useFakeTimers(); vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '50'); const source = new Source(80); let reading = false; const read = source.read.bind(source); source.read = async (offset, length) => { reading = true; await new Promise((resolve) => setTimeout(resolve, 100)); reading = false; return read(offset, length); }; const close = vi.spyOn(source, 'close').mockImplementation(async () => { expect(reading).toBe(false); }); const failed = expect(upload(source, 1)).rejects.toThrow('timed out after 50ms'); await Promise.all([failed, vi.runAllTimersAsync()]); expect(connect).not.toHaveBeenCalled(); expect(close).toHaveBeenCalledTimes(1); expect(vi.getTimerCount()).toBe(0); }); it('aborts all parallel requests at one whole-file deadline', async () => { vi.useFakeTimers(); vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '1000'); const source = new Source(6 * BLOCK); const start = Date.now(); const failed = expect(upload(source, 3)).rejects.toThrow('timed out after 1000ms'); await Promise.all([failed, vi.runAllTimersAsync()]); expect(Date.now() - start).toBe(1000); expect(requests).toHaveLength(3); expect(sockets.every((socket) => socket.destroyed)).toBe(true); expect(active.size).toBe(0); expect(source.closes).toBe(1); expect(vi.getTimerCount()).toBe(0); }); it.each([2, 3, 4])( 'uses %i connections, accepts out-of-order acknowledgements and sends the tail last', async (concurrency) => { const source = new Source(6 * BLOCK + 17); const confirmed = new Set(); const checkAndAck = (request: Request) => { if (request.offset === 6 * BLOCK) { expect(confirmed.size).toBe(6); } expect(request.bytes.every((byte) => byte === request.offset / BLOCK + 1)).toBe( true, ); expect(Buffer.from(request.head.msgSegHead!.md5!)).toEqual( Buffer.from(computeMd5(request.bytes)), ); confirmed.add(request.offset); request.respond(); }; const result = upload(source, concurrency); await vi.waitFor(() => expect(requests).toHaveLength(concurrency)); expect(source.closes).toBe(0); onRequest = checkAndAck; for (const request of [...active].reverse()) { checkAndAck(request); } await result; expect(peak).toBe(concurrency); expect(sockets).toHaveLength(concurrency); expect(requests).toHaveLength(7); expect(new Set(requests.map((request) => request.head.msgBaseHead?.seq)).size).toBe(7); expect(source.closes).toBe(1); expect(sockets.every((socket) => socket.destroyed)).toBe(true); }, ); it('logs confirmed bytes separately from the contiguous prefix after out-of-order replies', async () => { const entries: LogEntry[] = []; setLogLevel('trace'); const unsubscribe = subscribeLogs((entry) => entries.push(entry)); const source = new Source(3 * BLOCK + 1); try { const result = upload(source, 3); await vi.waitFor(() => expect(requests).toHaveLength(3)); requests[2].respond(); await vi.waitFor(() => expect( entries.some((entry) => entry.message.includes('acknowledgedBytes=1048576 contiguousBytes=0'), ), ).toBe(true), ); expect(requests).toHaveLength(3); onRequest = (request) => request.respond(); requests[1].respond(); requests[0].respond(); await result; const terminal = entries.filter((entry) => entry.message.startsWith('highway_put_terminal '), ); expect(terminal).toHaveLength(1); expect(terminal[0]?.message).toContain(`uploadedBytes=${source.size}`); } finally { unsubscribe(); } }); it('surfaces repeated backpressure after the retry budget is exhausted', async () => { const source = new Source(4 * BLOCK); onRequest = (request) => { if (request.offset === 0) { request.respond({ errorCode: 102902 }); } else { request.respond(); } }; await expect(upload(source, 3)).rejects.toThrow(/error_code=102902/); expect(requests.filter((request) => request.offset === 0)).toHaveLength(8); expect(requests.some((request) => request.offset === 3 * BLOCK)).toBe(false); expect(source.closes).toBe(1); }); it('limits lookahead behind a slow chunk and recovers an idle connection failure', async () => { const source = new Source(14 * BLOCK + 1); onRequest = (request) => { if (request.offset !== 0) { request.respond(); } }; const result = upload(source, 4); await vi.waitFor(() => expect(requests).toHaveLength(10)); await turn(); expect(Math.max(...requests.map((request) => request.offset))).toBe(9 * BLOCK); const idle = sockets.find( (socket) => ![...active].some((request) => request.socket === socket), )!; idle.emit('error', new Error('idle connection reset')); onRequest = (request) => request.respond(); requests[0].respond(); await result; expect(requests).toHaveLength(15); expect(source.closes).toBe(1); }); it('retries only the unconfirmed range after a transport failure', async () => { const source = new Source(6 * BLOCK); let failed = false; onRequest = (request) => { if (request.offset === BLOCK && !failed) { failed = true; request.disconnect(); } else { request.respond(); } }; await upload(source, 3); const retried = requests.filter((request) => request.offset === BLOCK); expect(retried).toHaveLength(2); expect(retried[1].bytes).toEqual(retried[0].bytes); expect(retried[1].head.msgBaseHead?.retryTimes).toBe(1); expect(retried[1].head.msgBaseHead?.seq).not.toBe(retried[0].head.msgBaseHead?.seq); expect(source.reads).toHaveLength(6); expect(requests.filter((request) => request.offset !== BLOCK)).toHaveLength(5); expect(source.closes).toBe(1); }); it.each([102902, 302902])( 'drains in-flight requests and reduces concurrency on %i', async (code) => { const source = new Source(7 * BLOCK); const entries: LogEntry[] = []; const unsubscribe = subscribeLogs((entry) => entries.push(entry)); try { const result = upload(source, 4); await vi.waitFor(() => expect(requests).toHaveLength(4)); requests[0].respond({ errorCode: code }); await vi.waitFor(() => expect( entries.some((entry) => entry.message.includes('reducing concurrency from 4 to 1'), ), ).toBe(true), ); await turn(); expect(requests).toHaveLength(4); let afterDrainPeak = 0; onRequest = (request) => { afterDrainPeak = Math.max(afterDrainPeak, active.size); request.respond(); }; for (const request of [...active]) { request.respond(); } await result; expect(afterDrainPeak).toBe(1); expect(requests.filter((request) => request.offset === 0)).toHaveLength(2); expect(requests).toHaveLength(8); expect(source.closes).toBe(1); } finally { unsubscribe(); } }, ); it.each([ { msgBaseHead: { seq: 999 } }, { msgSegHead: { dataOffset: 99n } }, { msgSegHead: { dataLength: 99 } }, { msgSegHead: { filesize: 99n } }, ])('rejects mismatched acknowledgements without retrying (case %#)', async (response) => { const source = new Source(4 * BLOCK); onRequest = (request) => request.respond(response); await expect(upload(source, 3)).rejects.toThrow(/acknowledgement mismatch/); expect(requests.every((request) => (request.head.msgBaseHead?.retryTimes ?? 0) === 0)).toBe( true, ); expect(requests.some((request) => request.offset === 3 * BLOCK)).toBe(false); expect(source.closes).toBe(1); expect(sockets.every((socket) => socket.destroyed)).toBe(true); }); it('rejects segment-level errors even when the top-level code is zero', async () => { const source = new Source(4 * BLOCK); onRequest = (request) => request.respond({ msgSegHead: { retCode: 921 } }); await expect(upload(source, 3)).rejects.toThrow(/segRetCode=921/); expect(source.closes).toBe(1); }); it('stops peers on rejection and waits for pending source reads before closing', async () => { let release!: () => void; let reading = false; const source = new Source(5 * BLOCK); const read = source.read.bind(source); source.read = async (offset, length) => { if (offset === BLOCK) { reading = true; await new Promise((resolve) => { release = resolve; }); reading = false; } return read(offset, length); }; const close = vi.spyOn(source, 'close').mockImplementation(async () => { expect(reading).toBe(false); }); const result = upload(source, 3); const rejected = expect(result).rejects.toThrow(/error_code=921/); await vi.waitFor(() => expect(requests).toHaveLength(2)); requests.find((request) => request.offset === 0)!.respond({ errorCode: 921 }); await turn(); expect(close).not.toHaveBeenCalled(); release(); await rejected; expect(close).toHaveBeenCalledTimes(1); expect(requests).toHaveLength(2); expect(sockets.every((socket) => socket.destroyed)).toBe(true); }); it('stops after bounded transport retries without sending the final chunk', async () => { const source = new Source(5 * BLOCK); onRequest = (request) => { if (request.offset === 0) { request.disconnect(); } else { request.respond(); } }; await expect(upload(source, 3)).rejects.toThrow(/transport failed after 3 attempts/); expect(requests.filter((request) => request.offset === 0)).toHaveLength(3); expect(requests.some((request) => request.offset === 4 * BLOCK)).toBe(false); expect(source.closes).toBe(1); }); it.each([1, BLOCK + 1, 2 * BLOCK])('keeps a %i-byte upload on one connection', async (size) => { onRequest = (request) => request.respond(); await upload(new Source(size), 4); expect(peak).toBe(1); expect(sockets).toHaveLength(1); }); it('keeps legacy success responses without echo fields compatible', async () => { onRequest = (request) => request.respond({ msgBaseHead: undefined, msgSegHead: undefined }); await upload(new Source(4 * BLOCK), 3); expect(requests).toHaveLength(4); }); it.each([0, 5, -1, 1.5, NaN])( 'rejects invalid concurrency %s and closes the source', async (concurrency) => { const source = new Source(4 * BLOCK); await expect(upload(source, concurrency)).rejects.toThrow(/integer from 1 to 4/); expect(source.closes).toBe(1); expect(requests).toHaveLength(0); }, ); it('rejects short reads instead of acknowledging bytes that were not uploaded', async () => { const source = new Source(4 * BLOCK); source.read = async () => new Uint8Array(1); await expect(upload(source, 3)).rejects.toThrow(/expected 1048576/); expect(requests).toHaveLength(0); expect(source.closes).toBe(1); }); });