TanTan
fa9c73363d
Dev Build / cleanup (push) Waiting to run
Dev Build / typecheck (push) Waiting to run
Publish MCP to npm / Build, test, and publish MCP (push) Waiting to run
Publish SDK to npm / Build, test, and publish SDK (push) Waiting to run
Sync action catalog to docs / sync (push) Waiting to run
Publish UI to npm / Build and publish UI (push) Waiting to run
Build and Release / preflight (push) Waiting to run
1559 lines
61 KiB
TypeScript
1559 lines
61 KiB
TypeScript
// highway-client.uploadHighwayHttp 的连接生命周期测试。
|
||
//
|
||
// 方案 3（#211）：单连接、按 offset 顺序、流水线上传，跨 chunk 复用同一个
|
||
// TCP 连接（HTTP keep-alive）。要点：
|
||
// - 顺序发送 → 减少 QQ highway 对重叠块的 NeedDelayRetry（error_code=102902）；
|
||
// - 复用连接 → TCP 拥塞窗口全程保持，省掉每块新建连接的 slow-start。
|
||
// 边缘节点可能在响应后 FIN（#118）；此时自适应地丢弃 socket + 重连 + 重发
|
||
// 同一块（重发幂等），最坏退化成“每块一连接”，不比旧实现差，且仍然保序。
|
||
// 102902 / 302902 不是致命拒绝（#352）：官方 BDH 走 NeedDelayRetry，延迟后
|
||
// 换连接重发同一 offset。
|
||
//
|
||
// 本测试用两种 mock 服务端行为锁定这套自适应逻辑：
|
||
// - keepalive：每个 POST 回一个响应，连接保持 → 整个文件 1 个连接；
|
||
// - closeafter：每个响应后半关闭（readableEnded/emit close）→ 退化成每块
|
||
// 一连接，但仍全部成功。
|
||
|
||
import { getLogLevel, setLogLevel, subscribeLogs, type LogEntry } from '@snowluma/common/logger';
|
||
import type { HttpConn0x6FF501Response, ReqDataHighwayHead } from '@snowluma/proto-defs/highway';
|
||
import { protobuf_decode, protobuf_encode } from '@snowluma/proton';
|
||
import { unpackHighwayFrame } from '@snowluma/protocol/highway/utils';
|
||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||
import { EventEmitter } from 'events';
|
||
|
||
// highway-client 解析响应只关心 errorCode === 0 —— 一个完全空的
|
||
// RespDataHighwayHead protobuf 体（0 字节）会被解为 errorCode=undefined，
|
||
// 走 falsy 通过分支。
|
||
// highway frame：0x28 | head_len(BE32) | body_len(BE32) | head | body | 0x29
|
||
function buildEmptyHighwayResponseFrame(): Buffer {
|
||
const frame = Buffer.alloc(10);
|
||
frame[0] = 0x28;
|
||
frame.writeUInt32BE(0, 1); // head_len
|
||
frame.writeUInt32BE(0, 5); // body_len
|
||
frame[9] = 0x29;
|
||
return frame;
|
||
}
|
||
|
||
function encodeVarint(n: number): number[] {
|
||
const bytes: number[] = [];
|
||
let value = n >>> 0;
|
||
|
||
while (value > 0x7f) {
|
||
bytes.push((value & 0x7f) | 0x80);
|
||
value >>>= 7;
|
||
}
|
||
|
||
bytes.push(value);
|
||
return bytes;
|
||
}
|
||
|
||
// errorCode 帧：RespDataHighwayHead.field3 (errorCode)。tag(field3, varint)=0x18。
|
||
// 921 是定义即拒绝；102902 / 302902 是 QQ BDH 的 NeedDelayRetry（#352）。
|
||
function buildErrorCodeFrame(errorCode = 921): Buffer {
|
||
const head = Buffer.from([0x18, ...encodeVarint(errorCode)]);
|
||
const frame = Buffer.alloc(9 + head.length + 1);
|
||
frame[0] = 0x28;
|
||
frame.writeUInt32BE(head.length, 1);
|
||
frame.writeUInt32BE(0, 5);
|
||
head.copy(frame, 9);
|
||
frame[9 + head.length] = 0x29;
|
||
return frame;
|
||
}
|
||
|
||
function buildMalformedHighwayResponseFrame(): Buffer {
|
||
return Buffer.from([0x28, 0, 0, 0, 5, 0, 0, 0, 0, 0x08, 0x01, 0x29]);
|
||
}
|
||
|
||
function httpWrap(frame: Buffer): Buffer {
|
||
const headers = ['HTTP/1.1 200 OK', `Content-Length: ${frame.length}`, '', ''].join('\r\n');
|
||
return Buffer.concat([Buffer.from(headers, 'ascii'), frame]);
|
||
}
|
||
|
||
type ServerMode = 'keepalive' | 'closeafter';
|
||
|
||
// FakeSocket 模拟 net.Socket 的最小子集，支持在一个连接上顺序处理多个 POST。
|
||
// 累积写入的字节，按 `\r\n\r\n` + Content-Length 切出每个完整 POST，逐个回
|
||
// 响应。`mode` 控制响应后是否半关闭；`failOnPost` 用于模拟链路中途断开。
|
||
class FakeSocket extends EventEmitter {
|
||
destroyed = false;
|
||
readableEnded = false;
|
||
writable = true;
|
||
posts = 0;
|
||
headers: string[] = [];
|
||
frames: Buffer[] = [];
|
||
closeBeforeResponse = false;
|
||
idleBeforeResponse = false;
|
||
stallWrites = false;
|
||
responseDelayMs = 0;
|
||
connectFailure?: Error | 'timeout';
|
||
private connectTimer?: ReturnType<typeof setTimeout>;
|
||
private responseTimer?: ReturnType<typeof setTimeout>;
|
||
private buf = Buffer.alloc(0);
|
||
constructor(
|
||
private readonly responseFrame: Buffer,
|
||
private readonly mode: ServerMode,
|
||
private readonly failOnPost = -1,
|
||
private readonly truncateOnPost = -1,
|
||
private readonly errorAfterPartialOnPost = -1,
|
||
private readonly frameOnPost: Readonly<Record<number, Buffer>> = {},
|
||
) {
|
||
super();
|
||
}
|
||
|
||
setTimeout(ms = 0) {
|
||
clearTimeout(this.connectTimer);
|
||
|
||
if (ms > 0 && this.connectFailure === 'timeout') {
|
||
this.connectTimer = setTimeout(() => this.emit('timeout'), ms);
|
||
}
|
||
}
|
||
|
||
write(data: unknown, cb?: (err?: Error) => void): boolean {
|
||
if (this.stallWrites) {
|
||
return false;
|
||
}
|
||
|
||
if (this.destroyed || !this.writable) {
|
||
const err = new Error('write after socket end');
|
||
|
||
if (cb) {
|
||
cb(err);
|
||
}
|
||
|
||
setImmediate(() => this.emit('error', err));
|
||
return false;
|
||
}
|
||
|
||
this.buf = Buffer.concat([this.buf, Buffer.from(data as Uint8Array)]);
|
||
this.drainPosts();
|
||
|
||
if (cb) {
|
||
cb();
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
private drainPosts(): void {
|
||
for (;;) {
|
||
const s = this.buf.toString('latin1');
|
||
const he = s.indexOf('\r\n\r\n');
|
||
|
||
if (he < 0) {
|
||
return;
|
||
}
|
||
|
||
const m = s.slice(0, he).match(/content-length:\s*(\d+)/i);
|
||
const cl = m ? parseInt(m[1], 10) : 0;
|
||
const total = he + 4 + cl;
|
||
|
||
if (this.buf.length < total) {
|
||
return;
|
||
}
|
||
|
||
this.headers.push(s.slice(0, he));
|
||
this.frames.push(Buffer.from(this.buf.subarray(he + 4, total)));
|
||
this.buf = this.buf.subarray(total);
|
||
this.posts += 1;
|
||
const thisPost = this.posts;
|
||
|
||
const respond = () => {
|
||
if (this.destroyed) {
|
||
return;
|
||
}
|
||
|
||
if (this.idleBeforeResponse) {
|
||
return;
|
||
}
|
||
|
||
if (thisPost === this.failOnPost) {
|
||
this.writable = false;
|
||
this.readableEnded = true;
|
||
this.emit('error', new Error('ECONNRESET'));
|
||
this.emit('close');
|
||
return;
|
||
}
|
||
|
||
if (this.closeBeforeResponse) {
|
||
this.writable = false;
|
||
this.readableEnded = true;
|
||
this.emit('end');
|
||
this.emit('close');
|
||
return;
|
||
}
|
||
|
||
if (thisPost === this.errorAfterPartialOnPost) {
|
||
const full = httpWrap(this.responseFrame);
|
||
this.emit('data', full.subarray(0, full.length - 3));
|
||
this.writable = false;
|
||
this.readableEnded = true;
|
||
this.emit('error', new Error('ECONNRESET after partial response'));
|
||
this.emit('close');
|
||
return;
|
||
}
|
||
|
||
if (thisPost === this.truncateOnPost) {
|
||
// Send the HTTP header + only part of the declared body, then FIN —
|
||
// a truncated response. Dropping the last 3 body bytes keeps the
|
||
// header intact so Content-Length parses but is never satisfied.
|
||
const full = httpWrap(this.responseFrame);
|
||
this.emit('data', full.subarray(0, full.length - 3));
|
||
this.writable = false;
|
||
this.readableEnded = true;
|
||
this.emit('close');
|
||
return;
|
||
}
|
||
|
||
this.emit('data', httpWrap(this.frameOnPost[thisPost] ?? this.responseFrame));
|
||
|
||
if (this.mode === 'closeafter') {
|
||
this.readableEnded = true;
|
||
this.writable = false;
|
||
this.emit('end');
|
||
this.emit('close');
|
||
}
|
||
};
|
||
|
||
if (this.responseDelayMs > 0) {
|
||
this.responseTimer = setTimeout(respond, this.responseDelayMs);
|
||
} else {
|
||
setImmediate(respond);
|
||
}
|
||
}
|
||
}
|
||
|
||
destroy() {
|
||
clearTimeout(this.connectTimer);
|
||
clearTimeout(this.responseTimer);
|
||
this.destroyed = true;
|
||
this.writable = false;
|
||
}
|
||
}
|
||
|
||
// vitest 把 vi.mock 工厂 hoist 到文件顶部，所以工厂里不能引用普通的顶层
|
||
// const。用 vi.hoisted 让这些值跟着 mock 一起 hoist。
|
||
const { createdSockets, socketFactoryRef, createConnectionMock } = vi.hoisted(() => ({
|
||
createdSockets: [] as unknown[],
|
||
socketFactoryRef: { make: null as null | ((idx: number) => unknown) },
|
||
createConnectionMock: vi.fn(),
|
||
}));
|
||
|
||
vi.mock('net', () => ({
|
||
default: { createConnection: createConnectionMock },
|
||
createConnection: createConnectionMock,
|
||
}));
|
||
|
||
import {
|
||
fetchHighwaySession,
|
||
uploadHighwayHttp,
|
||
BufferChunkSource,
|
||
type ChunkSource,
|
||
type HighwaySession,
|
||
} from '@snowluma/protocol/highway';
|
||
import type { BridgeContext } from '@snowluma/protocol/bridge-context';
|
||
|
||
const HIGHWAY_BLOCK_SIZE = 1024 * 1024;
|
||
const previousLogLevel = getLogLevel();
|
||
|
||
function highwayTrace(entries: LogEntry[]): LogEntry[] {
|
||
return entries.filter((entry) => entry.level === 'trace' && entry.scope === 'Highway');
|
||
}
|
||
|
||
function makeBridge(uin = '10001'): BridgeContext {
|
||
return { identity: { uin } as unknown } as unknown as BridgeContext;
|
||
}
|
||
|
||
function makeSession(): HighwaySession {
|
||
return {
|
||
sigSession: new Uint8Array([0xaa]),
|
||
sessionKey: new Uint8Array([0xbb]),
|
||
servers: [{ host: '127.0.0.1', port: 80 }],
|
||
};
|
||
}
|
||
|
||
// 安装一个 socket 工厂：createConnection 每次调用按序号创建一个 FakeSocket。
|
||
function installFactory(make: (idx: number) => FakeSocket): void {
|
||
createdSockets.length = 0;
|
||
socketFactoryRef.make = make as (idx: number) => unknown;
|
||
createConnectionMock.mockReset();
|
||
createConnectionMock.mockImplementation((_opts: unknown, listener?: () => void) => {
|
||
const idx = createdSockets.length;
|
||
const sock = socketFactoryRef.make!(idx) as FakeSocket;
|
||
createdSockets.push(sock);
|
||
|
||
if (sock.connectFailure instanceof Error) {
|
||
setImmediate(() => sock.emit('error', sock.connectFailure));
|
||
} else if (!sock.connectFailure && listener) {
|
||
setImmediate(listener);
|
||
} // 握手成功后异步回调 → tcpConnect resolve
|
||
|
||
return sock as unknown as ReturnType<typeof createConnectionMock>;
|
||
});
|
||
}
|
||
|
||
// 顺序读取记录：offset/length + 关闭时机，用于数据完整性与 close 语义。
|
||
class RecordingChunkSource implements ChunkSource {
|
||
reads: Array<{ offset: number; length: number }> = [];
|
||
inFlight = 0;
|
||
closeCount = 0;
|
||
closedWhileReading = false;
|
||
constructor(readonly size: number) {}
|
||
async read(offset: number, length: number): Promise<Uint8Array> {
|
||
this.inFlight += 1;
|
||
this.reads.push({ offset, length });
|
||
await Promise.resolve();
|
||
this.inFlight -= 1;
|
||
return new Uint8Array(length);
|
||
}
|
||
async close(): Promise<void> {
|
||
if (this.inFlight > 0) {
|
||
this.closedWhileReading = true;
|
||
}
|
||
|
||
this.closeCount += 1;
|
||
}
|
||
}
|
||
|
||
describe('fetchHighwaySession trace lifecycle', () => {
|
||
afterEach(() => {
|
||
setLogLevel(previousLogLevel);
|
||
vi.restoreAllMocks();
|
||
});
|
||
|
||
it('records session metadata without duplicating packet control bytes', async () => {
|
||
const responseData = protobuf_encode<HttpConn0x6FF501Response>({
|
||
httpConn: {
|
||
sigSession: new Uint8Array([0xaa, 0xbb]),
|
||
sessionKey: new Uint8Array([0xcc]),
|
||
serverInfos: [
|
||
{
|
||
serviceType: 1,
|
||
serverAddrs: [{ ip: 0x04030201, port: 8080 }],
|
||
},
|
||
],
|
||
},
|
||
});
|
||
const sendRawPacket = vi.fn<
|
||
(
|
||
cmd: string,
|
||
body: Uint8Array,
|
||
) => Promise<{
|
||
success: boolean;
|
||
gotResponse: boolean;
|
||
errorCode: number;
|
||
errorMessage: string;
|
||
responseData: Buffer;
|
||
}>
|
||
>(async () => ({
|
||
success: true,
|
||
gotResponse: true,
|
||
errorCode: 0,
|
||
errorMessage: '',
|
||
responseData: Buffer.from(responseData),
|
||
}));
|
||
const bridge = {
|
||
identity: { uin: '10001' },
|
||
sendRawPacket,
|
||
} as unknown as BridgeContext;
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
const session = await fetchHighwaySession(bridge);
|
||
expect([...session.sigSession]).toEqual([0xaa, 0xbb]);
|
||
expect([...session.sessionKey]).toEqual([0xcc]);
|
||
expect(session.servers).toEqual([{ host: '1.2.3.4', port: 8080 }]);
|
||
const requestBytes = sendRawPacket.mock.calls[0]![1] as Uint8Array;
|
||
const messages = highwayTrace(entries).map((entry) => entry.message);
|
||
expect(messages).toEqual(
|
||
expect.arrayContaining([
|
||
`highway_session_start serviceCmd="HttpConn.0x6ff_501" requestBytes=${requestBytes.byteLength}`,
|
||
expect.stringContaining(
|
||
`branch=control_response success=true gotResponse=true errorCode=0 errorMessage="" responseBytes=${responseData.byteLength}`,
|
||
),
|
||
expect.stringMatching(
|
||
/^highway_session_terminal outcome=completed reason=session_ready servers=\[\{"host":"1\.2\.3\.4","port":8080\}\] sigBytes=2 sessionKeyBytes=1 elapsedMs=\d+$/,
|
||
),
|
||
]),
|
||
);
|
||
expect(messages.join('\n')).not.toContain('requestHex=');
|
||
expect(messages.join('\n')).not.toContain('responseHex=');
|
||
expect(
|
||
messages.filter((message) => message.startsWith('highway_session_terminal ')),
|
||
).toHaveLength(1);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('keeps a server address encoded as a fixed 32-bit field', async () => {
|
||
// Address field 2 is four raw bytes. The fixture is not produced by the
|
||
// schema encoder, so a varint declaration cannot hide itself.
|
||
const responseData = Buffer.from('8a50150a02aabb1201cc1a0c08011208150102030418903f', 'hex');
|
||
const bridge = {
|
||
identity: { uin: '10001' },
|
||
sendRawPacket: vi.fn(async () => ({
|
||
success: true,
|
||
gotResponse: true,
|
||
errorCode: 0,
|
||
errorMessage: '',
|
||
responseData,
|
||
})),
|
||
} as unknown as BridgeContext;
|
||
|
||
const session = await fetchHighwaySession(bridge);
|
||
|
||
expect(session.servers).toEqual([{ host: '1.2.3.4', port: 8080 }]);
|
||
});
|
||
|
||
it('keeps distinct service-1 candidates in response order', async () => {
|
||
const responseData = protobuf_encode<HttpConn0x6FF501Response>({
|
||
httpConn: {
|
||
sigSession: new Uint8Array([0xaa]),
|
||
serverInfos: [
|
||
{ serviceType: 5, serverAddrs: [{ ip: 0x090200c0, port: 80 }] },
|
||
{
|
||
serviceType: 1,
|
||
serverAddrs: [
|
||
{ ip: 0x010200c0, port: 15000 },
|
||
{ ip: 0x010200c0, port: 15000 },
|
||
{ ip: 0, port: 80 },
|
||
{ ip: 0x020200c0, port: 0 },
|
||
{ ip: 0x020200c0, port: 65536 },
|
||
{ ip: 0x020200c0, port: 443 },
|
||
],
|
||
},
|
||
],
|
||
},
|
||
});
|
||
const bridge = {
|
||
sendRawPacket: vi.fn(async () => ({ success: true, gotResponse: true, responseData })),
|
||
} as unknown as BridgeContext;
|
||
const session = await fetchHighwaySession(bridge);
|
||
expect(session).toMatchObject({
|
||
servers: [
|
||
{ host: '192.0.2.1', port: 15000 },
|
||
{ host: '192.0.2.2', port: 443 },
|
||
],
|
||
});
|
||
});
|
||
|
||
it('rejects a session without upload candidates', async () => {
|
||
const responseData = protobuf_encode<HttpConn0x6FF501Response>({
|
||
httpConn: { sigSession: new Uint8Array([0xaa]), serverInfos: [] },
|
||
});
|
||
const bridge = {
|
||
sendRawPacket: vi.fn(async () => ({ success: true, gotResponse: true, responseData })),
|
||
} as unknown as BridgeContext;
|
||
await expect(fetchHighwaySession(bridge)).rejects.toThrow('no upload servers');
|
||
});
|
||
|
||
it('records a thrown control request as one failed terminal', async () => {
|
||
const bridge = {
|
||
identity: { uin: '10001' },
|
||
sendRawPacket: vi.fn(async () => {
|
||
throw new Error('fixture session pipe failure');
|
||
}),
|
||
} as unknown as BridgeContext;
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await expect(fetchHighwaySession(bridge)).rejects.toThrow(
|
||
'fixture session pipe failure',
|
||
);
|
||
const terminals = highwayTrace(entries).filter((entry) =>
|
||
entry.message.startsWith('highway_session_terminal '),
|
||
);
|
||
expect(terminals).toHaveLength(1);
|
||
expect(terminals[0]!.message).toContain('outcome=failed reason=request_failed');
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
});
|
||
|
||
describe('Highway connection selection', () => {
|
||
afterEach(() => {
|
||
vi.useRealTimers();
|
||
vi.restoreAllMocks();
|
||
vi.unstubAllEnvs();
|
||
setLogLevel(previousLogLevel);
|
||
});
|
||
|
||
function sessionWithCandidates() {
|
||
return {
|
||
...makeSession(),
|
||
servers: [
|
||
{ host: '192.0.2.1', port: 15000 },
|
||
{ host: '192.0.2.2', port: 443 },
|
||
],
|
||
};
|
||
}
|
||
|
||
it.each([1003, 1004])(
|
||
'reaches the fourth candidate after three idle peers for command %i (#637)',
|
||
async (commandId) => {
|
||
vi.useFakeTimers();
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_IDLE_TIMEOUT_MS', '100');
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '5000');
|
||
installFactory(() => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
socket.idleBeforeResponse =
|
||
createConnectionMock.mock.lastCall?.[0].host !== '192.0.2.4';
|
||
return socket;
|
||
});
|
||
const session = {
|
||
...makeSession(),
|
||
servers: Array.from({ length: 4 }, (_, i) => ({
|
||
host: `192.0.2.${i + 1}`,
|
||
port: 443,
|
||
})),
|
||
};
|
||
const source = new RecordingChunkSource(80);
|
||
const completed = expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
commandId,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
),
|
||
).resolves.toBeUndefined();
|
||
await Promise.all([completed, vi.runAllTimersAsync()]);
|
||
expect(createConnectionMock.mock.calls.map(([options]) => options)).toEqual(
|
||
session.servers,
|
||
);
|
||
expect(source.reads).toEqual([{ offset: 0, length: 80 }]);
|
||
expect(source.closeCount).toBe(1);
|
||
expect((createdSockets as FakeSocket[]).every((socket) => socket.destroyed)).toBe(true);
|
||
},
|
||
);
|
||
|
||
it('uses recent successful host and port only for the same account and upload type', async () => {
|
||
vi.useFakeTimers();
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_IDLE_TIMEOUT_MS', '100');
|
||
const session = {
|
||
...makeSession(),
|
||
servers: [
|
||
{ host: '192.0.2.1', port: 14000 },
|
||
{ host: '192.0.2.1', port: 80 },
|
||
],
|
||
};
|
||
const bridge = makeBridge();
|
||
|
||
const send = async (owner: BridgeContext, commandId = 1004, candidates = session) => {
|
||
installFactory(() => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
socket.idleBeforeResponse = createConnectionMock.mock.lastCall?.[0].port === 14000;
|
||
return socket;
|
||
});
|
||
await Promise.all([
|
||
expect(
|
||
uploadHighwayHttp(
|
||
owner,
|
||
candidates,
|
||
commandId,
|
||
new RecordingChunkSource(80),
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
),
|
||
).resolves.toBeUndefined(),
|
||
vi.runAllTimersAsync(),
|
||
]);
|
||
return createConnectionMock.mock.calls.map(([options]) => options);
|
||
};
|
||
|
||
expect(await send(bridge)).toEqual(session.servers);
|
||
expect(await send(bridge)).toEqual([session.servers[1]]);
|
||
expect(await send(makeBridge('10002'))).toEqual(session.servers);
|
||
expect(await send(bridge, 1003)).toEqual(session.servers);
|
||
const replacement = { ...session, servers: [{ host: '192.0.2.2', port: 80 }] };
|
||
expect(await send(bridge, 1004, replacement)).toEqual(replacement.servers);
|
||
await vi.advanceTimersByTimeAsync(5 * 60_000);
|
||
expect(await send(bridge)).toEqual(session.servers);
|
||
});
|
||
|
||
it.each(['response', 'write', 'retry'])(
|
||
'stops a pending %s at the shared upload deadline and releases resources',
|
||
async (phase) => {
|
||
vi.useFakeTimers();
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '150');
|
||
vi.stubEnv(
|
||
'SNOWLUMA_HIGHWAY_UPLOAD_IDLE_TIMEOUT_MS',
|
||
phase === 'retry' ? '50' : '1000',
|
||
);
|
||
installFactory(() => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
socket.idleBeforeResponse = true;
|
||
socket.stallWrites = phase === 'write';
|
||
return socket;
|
||
});
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
const source = new RecordingChunkSource(80);
|
||
const start = Date.now();
|
||
|
||
try {
|
||
const failed = expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
sessionWithCandidates(),
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
),
|
||
).rejects.toThrow('timed out after 150ms');
|
||
await Promise.all([failed, vi.runAllTimersAsync()]);
|
||
expect(Date.now() - start).toBe(150);
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(1);
|
||
expect(source.closeCount).toBe(1);
|
||
expect(
|
||
(createdSockets as FakeSocket[]).every(
|
||
(socket) =>
|
||
socket.destroyed &&
|
||
socket.listenerCount('data') === 0 &&
|
||
socket.listenerCount('close') === 0,
|
||
),
|
||
).toBe(true);
|
||
const terminal = highwayTrace(entries).filter((entry) =>
|
||
entry.message.startsWith('highway_put_terminal '),
|
||
);
|
||
expect(terminal).toHaveLength(1);
|
||
expect(terminal[0]!.message).toContain('reason=upload_timeout');
|
||
expect(vi.getTimerCount()).toBe(0);
|
||
|
||
if (phase === 'retry') {
|
||
expect(
|
||
entries.some(
|
||
(entry) =>
|
||
entry.message.includes('phase=read_response') &&
|
||
entry.message.includes('received=0B'),
|
||
),
|
||
).toBe(true);
|
||
}
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
},
|
||
);
|
||
|
||
it('does not renew the total deadline when a chunk succeeds', async () => {
|
||
vi.useFakeTimers();
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '250');
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_IDLE_TIMEOUT_MS', '200');
|
||
installFactory(() => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
socket.responseDelayMs = 100;
|
||
return socket;
|
||
});
|
||
const source = new RecordingChunkSource(2 * HIGHWAY_BLOCK_SIZE + 80);
|
||
const start = Date.now();
|
||
const failed = expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
),
|
||
).rejects.toThrow('acknowledgedBytes=2097152/2097232');
|
||
await Promise.all([failed, vi.runAllTimersAsync()]);
|
||
expect(Date.now() - start).toBe(250);
|
||
expect(source.closeCount).toBe(1);
|
||
expect(vi.getTimerCount()).toBe(0);
|
||
});
|
||
|
||
it('rejects invalid timeout configuration before connecting and closes the source', async () => {
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '0');
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||
const source = new RecordingChunkSource(80);
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
),
|
||
).rejects.toThrow('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS');
|
||
expect(createConnectionMock).not.toHaveBeenCalled();
|
||
expect(source.closeCount).toBe(1);
|
||
});
|
||
|
||
it.each(['timeout', 'ECONNREFUSED', 'ENETUNREACH'])(
|
||
'switches on %s and uploads over the successful connection',
|
||
async (failure) => {
|
||
vi.useFakeTimers();
|
||
installFactory((idx) => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
|
||
if (idx === 0) {
|
||
socket.connectFailure = failure === 'timeout' ? 'timeout' : new Error(failure);
|
||
}
|
||
|
||
return socket;
|
||
});
|
||
const session = sessionWithCandidates();
|
||
const original = structuredClone(session);
|
||
const source = new RecordingChunkSource(2 * HIGHWAY_BLOCK_SIZE + 1);
|
||
const upload = uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
);
|
||
const completed = expect(upload).resolves.toBeUndefined();
|
||
await vi.runAllTimersAsync();
|
||
await completed;
|
||
expect(createConnectionMock.mock.calls.map(([options]) => options)).toEqual(
|
||
session.servers,
|
||
);
|
||
const [failed, connected] = createdSockets as FakeSocket[];
|
||
expect(failed!.destroyed).toBe(true);
|
||
expect(failed!.posts).toBe(0);
|
||
expect(connected!.posts).toBe(3);
|
||
expect(
|
||
connected!.headers.every((header) => header.includes('Host: 192.0.2.2\r\n')),
|
||
).toBe(true);
|
||
expect(connected!.destroyed).toBe(true);
|
||
expect(source.closeCount).toBe(1);
|
||
expect(session).toEqual(original);
|
||
},
|
||
);
|
||
|
||
it('bounds initial selection by the configured total deadline without repeating candidates', async () => {
|
||
vi.useFakeTimers();
|
||
vi.stubEnv('SNOWLUMA_HIGHWAY_UPLOAD_TIMEOUT_MS', '10000');
|
||
installFactory(() => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
socket.connectFailure = 'timeout';
|
||
return socket;
|
||
});
|
||
const session = {
|
||
...makeSession(),
|
||
servers: Array.from({ length: 5 }, (_, i) => ({
|
||
host: `192.0.2.${i + 1}`,
|
||
port: 15000,
|
||
})),
|
||
};
|
||
const source = new RecordingChunkSource(1);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
const start = Date.now();
|
||
const upload = uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
);
|
||
const failed = expect(upload).rejects.toThrow('timed out after 10000ms');
|
||
await vi.runAllTimersAsync();
|
||
await failed;
|
||
expect(Date.now() - start).toBe(10000);
|
||
expect(createConnectionMock.mock.calls.map(([options]) => options)).toEqual(
|
||
session.servers.slice(0, 4),
|
||
);
|
||
expect(
|
||
(createdSockets as FakeSocket[]).every(
|
||
(socket) => socket.destroyed && socket.posts === 0,
|
||
),
|
||
).toBe(true);
|
||
expect(source.closeCount).toBe(1);
|
||
const terminals = highwayTrace(entries).filter((entry) =>
|
||
entry.message.startsWith('highway_put_terminal '),
|
||
);
|
||
expect(terminals).toHaveLength(1);
|
||
expect(terminals[0]!.message).toContain('reason=upload_timeout');
|
||
expect(terminals[0]!.message).toContain('connections=0');
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('fails immediately after all candidates refuse the connection', async () => {
|
||
vi.useFakeTimers();
|
||
installFactory(() => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
socket.connectFailure = new Error('ECONNREFUSED');
|
||
return socket;
|
||
});
|
||
const session = sessionWithCandidates();
|
||
const source = new RecordingChunkSource(1);
|
||
const upload = uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
);
|
||
const failed = expect(upload).rejects.toThrow(
|
||
/192\.0\.2\.1:15000.*ECONNREFUSED.*192\.0\.2\.2:443.*ECONNREFUSED/,
|
||
);
|
||
await vi.runAllTimersAsync();
|
||
await failed;
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
expect(source.closeCount).toBe(1);
|
||
});
|
||
|
||
it.each([0, HIGHWAY_BLOCK_SIZE])(
|
||
'retries only the unconfirmed chunk at offset %i on the next node',
|
||
async (offset) => {
|
||
vi.useFakeTimers();
|
||
installFactory(() => {
|
||
const host = createConnectionMock.mock.lastCall?.[0].host;
|
||
const socket = new FakeSocket(
|
||
buildEmptyHighwayResponseFrame(),
|
||
'keepalive',
|
||
host === '192.0.2.1' && offset > 0 ? 2 : -1,
|
||
);
|
||
socket.closeBeforeResponse = host === '192.0.2.1' && offset === 0;
|
||
return socket;
|
||
});
|
||
const session = sessionWithCandidates();
|
||
const original = structuredClone(session);
|
||
const source = new RecordingChunkSource(offset + 80);
|
||
const extend = new Uint8Array([0x12, 0x34]);
|
||
const upload = uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
71,
|
||
source,
|
||
new Uint8Array(16),
|
||
extend,
|
||
);
|
||
const completed = expect(upload).resolves.toBeUndefined();
|
||
await Promise.all([completed, vi.runAllTimersAsync()]);
|
||
expect(createConnectionMock.mock.calls.map(([options]) => options)).toEqual(
|
||
session.servers,
|
||
);
|
||
const [failed, recovered] = createdSockets as FakeSocket[];
|
||
expect(recovered!.posts).toBe(1);
|
||
const first = unpackHighwayFrame(failed!.frames.at(-1)!);
|
||
const retry = unpackHighwayFrame(recovered!.frames[0]!);
|
||
const firstHead = protobuf_decode<ReqDataHighwayHead>(first.head);
|
||
const retryHead = protobuf_decode<ReqDataHighwayHead>(retry.head);
|
||
expect(retry.body).toEqual(first.body);
|
||
expect(retryHead).toEqual({
|
||
...firstHead,
|
||
msgBaseHead: { ...firstHead.msgBaseHead, retryTimes: 1 },
|
||
});
|
||
expect(retryHead.msgSegHead?.dataOffset ?? 0n).toBe(BigInt(offset));
|
||
expect(retryHead.bytesReqExtendInfo).toEqual(Buffer.from(extend));
|
||
expect(source.reads).toEqual(
|
||
offset === 0
|
||
? [{ offset: 0, length: 80 }]
|
||
: [
|
||
{ offset: 0, length: HIGHWAY_BLOCK_SIZE },
|
||
{ offset, length: 80 },
|
||
],
|
||
);
|
||
expect(source.closeCount).toBe(1);
|
||
expect(session).toEqual(original);
|
||
expect((createdSockets as FakeSocket[]).every((socket) => socket.destroyed)).toBe(true);
|
||
},
|
||
);
|
||
|
||
it('bounds retries across nodes and reports the unconfirmed chunk without advancing progress', async () => {
|
||
vi.useFakeTimers();
|
||
installFactory(() => {
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
socket.closeBeforeResponse = true;
|
||
return socket;
|
||
});
|
||
const session = sessionWithCandidates();
|
||
const source = new RecordingChunkSource(80);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
const upload = uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
71,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
);
|
||
const failed = expect(upload).rejects.toThrow(
|
||
/transport failed after 3 attempts.*acknowledgedBytes=0.*unconfirmed/,
|
||
);
|
||
await Promise.all([failed, vi.runAllTimersAsync()]);
|
||
expect(createConnectionMock.mock.calls.map(([options]) => options)).toEqual([
|
||
session.servers[0],
|
||
session.servers[1],
|
||
session.servers[0],
|
||
]);
|
||
expect(source.reads).toEqual([{ offset: 0, length: 80 }]);
|
||
expect(source.closeCount).toBe(1);
|
||
const terminals = highwayTrace(entries).filter((entry) =>
|
||
entry.message.startsWith('highway_put_terminal '),
|
||
);
|
||
expect(terminals).toHaveLength(1);
|
||
expect(terminals[0]!.message).toContain('outcome=failed reason=transport_exhausted');
|
||
expect(terminals[0]!.message).toContain('uploadedBytes=0');
|
||
expect((createdSockets as FakeSocket[]).every((socket) => socket.destroyed)).toBe(true);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('continues after the actual failed node and skips candidates that cannot reconnect', async () => {
|
||
vi.useFakeTimers();
|
||
installFactory(() => {
|
||
const host = createConnectionMock.mock.lastCall?.[0].host;
|
||
const socket = new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive');
|
||
|
||
if (host === '192.0.2.1') {
|
||
socket.connectFailure = new Error('ECONNREFUSED');
|
||
} else if (host === '192.0.2.2') {
|
||
socket.closeBeforeResponse = true;
|
||
} else if (host === '192.0.2.3') {
|
||
socket.connectFailure = 'timeout';
|
||
}
|
||
|
||
return socket;
|
||
});
|
||
const session = {
|
||
...makeSession(),
|
||
servers: Array.from({ length: 4 }, (_, i) => ({ host: `192.0.2.${i + 1}`, port: 443 })),
|
||
};
|
||
const source = new RecordingChunkSource(80);
|
||
const upload = uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
71,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
);
|
||
await Promise.all([expect(upload).resolves.toBeUndefined(), vi.runAllTimersAsync()]);
|
||
expect(createConnectionMock.mock.calls.map(([options]) => options)).toEqual(
|
||
session.servers,
|
||
);
|
||
expect((createdSockets as FakeSocket[]).map((socket) => socket.posts)).toEqual([
|
||
0, 1, 0, 1,
|
||
]);
|
||
expect(source.reads).toEqual([{ offset: 0, length: 80 }]);
|
||
expect(source.closeCount).toBe(1);
|
||
});
|
||
|
||
it('keeps the successful node when the peer closes after acknowledgement', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'closeafter'));
|
||
const session = sessionWithCandidates();
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
session,
|
||
71,
|
||
new RecordingChunkSource(HIGHWAY_BLOCK_SIZE + 80),
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
);
|
||
expect(createConnectionMock.mock.calls.map(([options]) => options)).toEqual([
|
||
session.servers[0],
|
||
session.servers[0],
|
||
]);
|
||
});
|
||
|
||
it('does not switch nodes on a protocol rejection', async () => {
|
||
installFactory(() => new FakeSocket(buildErrorCodeFrame(), 'keepalive'));
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
sessionWithCandidates(),
|
||
1004,
|
||
new RecordingChunkSource(1),
|
||
new Uint8Array(16),
|
||
new Uint8Array(),
|
||
),
|
||
).rejects.toThrow('error_code=921');
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(1);
|
||
});
|
||
});
|
||
|
||
describe('uploadHighwayHttp single-connection pipelined upload (#211)', () => {
|
||
afterEach(() => {
|
||
setLogLevel(previousLogLevel);
|
||
vi.restoreAllMocks();
|
||
});
|
||
|
||
it('records complete chunk control lifecycle without copying source bytes', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||
const sourceBytes = new Uint8Array([0xde, 0xad, 0xbe, 0xef]);
|
||
const fileMd5 = new Uint8Array([0x00, 0xff, 0x10, 0x20]);
|
||
const extend = new Uint8Array([0xca, 0xfe]);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(sourceBytes),
|
||
fileMd5,
|
||
extend,
|
||
);
|
||
|
||
const trace = highwayTrace(entries);
|
||
expect(
|
||
trace.every((entry) => entry.req !== undefined && entry.req === trace[0]!.req),
|
||
).toBe(true);
|
||
const messages = trace.map((entry) => entry.message);
|
||
expect(messages).toEqual(
|
||
expect.arrayContaining([
|
||
expect.stringContaining('highway_put_start cmdId=1004'),
|
||
expect.stringContaining('totalBytes=4'),
|
||
expect.stringContaining('fileMd5=00ff1020'),
|
||
expect.stringContaining('extendBytes=2'),
|
||
expect.stringContaining('branch=chunk_read offset=0 bytes=4'),
|
||
expect.stringMatching(
|
||
/^highway_put_branch branch=chunk_control_request offset=0 bytes=4 headBytes=\d+ headHex=[0-9a-f]+$/,
|
||
),
|
||
expect.stringContaining('branch=chunk_attempt offset=0 bytes=4 attempt=1'),
|
||
expect.stringContaining('branch=chunk_control_response offset=0'),
|
||
expect.stringContaining('branch=chunk_control_decoded offset=0 errorCode=0'),
|
||
expect.stringMatching(
|
||
/^highway_put_terminal cmdId=1004 outcome=completed reason=upload_complete totalBytes=4 uploadedBytes=4 connections=1 elapsedMs=\d+$/,
|
||
),
|
||
]),
|
||
);
|
||
expect(messages.join('\n')).not.toContain('deadbeef');
|
||
expect(
|
||
messages.filter((message) => message.startsWith('highway_put_terminal ')),
|
||
).toHaveLength(1);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('records reconnect at the same offset and one successful terminal', async () => {
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', 1)
|
||
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
|
||
);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(new Uint8Array([1, 2, 3])),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
const messages = highwayTrace(entries).map((entry) => entry.message);
|
||
expect(messages).toEqual(
|
||
expect.arrayContaining([
|
||
expect.stringContaining('branch=chunk_retry offset=0 bytes=3 attempt=1'),
|
||
expect.stringContaining('branch=reconnect offset=0 attempt=2 connection=2'),
|
||
expect.stringContaining('outcome=completed reason=upload_complete'),
|
||
]),
|
||
);
|
||
expect(
|
||
messages.filter((message) => message.startsWith('highway_put_terminal ')),
|
||
).toHaveLength(1);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('records definitive server rejection as one failed terminal', async () => {
|
||
installFactory(() => new FakeSocket(buildErrorCodeFrame(), 'keepalive'));
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(new Uint8Array([1])),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
),
|
||
).rejects.toThrow(/error_code=921/);
|
||
const messages = highwayTrace(entries).map((entry) => entry.message);
|
||
expect(messages).toEqual(
|
||
expect.arrayContaining([
|
||
expect.stringContaining('branch=chunk_control_response offset=0'),
|
||
expect.stringContaining('errorCode=921'),
|
||
expect.stringContaining('outcome=failed reason=server_rejected'),
|
||
]),
|
||
);
|
||
expect(
|
||
messages.filter((message) => message.startsWith('highway_put_terminal ')),
|
||
).toHaveLength(1);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('records malformed control bytes before the decode failure terminal', async () => {
|
||
const malformed = buildMalformedHighwayResponseFrame();
|
||
installFactory(() => new FakeSocket(malformed, 'keepalive'));
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(new Uint8Array([1])),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
),
|
||
).rejects.toThrow();
|
||
const messages = highwayTrace(entries).map((entry) => entry.message);
|
||
expect(messages).toContainEqual(
|
||
expect.stringContaining(
|
||
`responseBytes=${malformed.byteLength} responseHex=${malformed.toString('hex')}`,
|
||
),
|
||
);
|
||
const terminals = messages.filter((message) =>
|
||
message.startsWith('highway_put_terminal '),
|
||
);
|
||
expect(terminals).toHaveLength(1);
|
||
expect(terminals[0]).toContain('outcome=failed reason=control_decode_failed');
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('records transport exhaustion after exactly three attempts', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', 1));
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(new Uint8Array([1])),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
),
|
||
).rejects.toThrow(/transport failed after 3 attempts/);
|
||
const messages = highwayTrace(entries).map((entry) => entry.message);
|
||
expect(
|
||
messages.filter((message) => message.includes('branch=chunk_attempt offset=0')),
|
||
).toHaveLength(3);
|
||
expect(
|
||
messages.filter((message) => message.includes('branch=chunk_retry offset=0')),
|
||
).toHaveLength(2);
|
||
const terminals = messages.filter((message) =>
|
||
message.startsWith('highway_put_terminal '),
|
||
);
|
||
expect(terminals).toHaveLength(1);
|
||
expect(terminals[0]).toContain('outcome=failed reason=transport_exhausted');
|
||
expect(terminals[0]).toContain('connections=3');
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('records source read and close failures with truthful unique terminals', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
const readFailure: ChunkSource = {
|
||
size: 1,
|
||
read: vi.fn(async () => {
|
||
throw new Error('fixture read failed');
|
||
}),
|
||
close: vi.fn(async () => undefined),
|
||
};
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
readFailure,
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
),
|
||
).rejects.toThrow('fixture read failed');
|
||
let terminals = highwayTrace(entries).filter((entry) =>
|
||
entry.message.startsWith('highway_put_terminal '),
|
||
);
|
||
expect(terminals).toHaveLength(1);
|
||
expect(terminals[0]!.message).toContain('outcome=failed reason=source_read_failed');
|
||
|
||
entries.length = 0;
|
||
const closeFailure: ChunkSource = {
|
||
size: 0,
|
||
read: vi.fn(async () => new Uint8Array(0)),
|
||
close: vi.fn(async () => {
|
||
throw new Error('fixture close failed');
|
||
}),
|
||
};
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
closeFailure,
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
),
|
||
).rejects.toThrow('fixture close failed');
|
||
terminals = highwayTrace(entries).filter((entry) =>
|
||
entry.message.startsWith('highway_put_terminal '),
|
||
);
|
||
expect(terminals).toHaveLength(1);
|
||
expect(terminals[0]!.message).toContain('outcome=failed reason=source_close_failed');
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('reuses ONE connection for a multi-block payload when keep-alive holds', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(bytes),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
// The whole file went over a single warm connection — the #211 win.
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(1);
|
||
});
|
||
|
||
it('single sub-block payload uses one connection', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||
const bytes = new Uint8Array(512 * 1024); // 1 block (PTT / small image)
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1003,
|
||
new BufferChunkSource(bytes),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(1);
|
||
});
|
||
|
||
it('a zero-length source opens ZERO connections (empty-input invariant)', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1003,
|
||
new BufferChunkSource(new Uint8Array(0)),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
expect(createConnectionMock).not.toHaveBeenCalled();
|
||
});
|
||
|
||
it('degrades to one connection per block when the server FINs after every response (#118 regime)', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'closeafter'));
|
||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(bytes),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
// Server closes after each response → adaptively reconnects per block, but
|
||
// the upload still completes. No worse than the old one-conn-per-chunk.
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(3);
|
||
});
|
||
|
||
it('reconnects and re-sends the SAME block on a mid-stream transport error', async () => {
|
||
// First connection dies while handling its first POST; a fresh connection
|
||
// must re-send that same block (offset unchanged, idempotent) and finish.
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', /* failOnPost */ 1)
|
||
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
|
||
);
|
||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(bytes),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
// Connection #0 failed on block 0 → reconnect (#1) resumes and keep-alives
|
||
// the rest: exactly two connections total.
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
});
|
||
|
||
it('reconnects and preserves partial control-response bytes when a body is truncated', async () => {
|
||
// Peer FINs after the HTTP header but before the highway body is complete.
|
||
// That truncation must be treated as a retryable transport failure (reject
|
||
// in readHttpResponseBody), NOT resolved as a partial frame — otherwise
|
||
// unpackHighwayFrame throws OUTSIDE the retry loop and fails the upload.
|
||
const frame = buildEmptyHighwayResponseFrame();
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(frame, 'keepalive', -1, /* truncateOnPost */ 1)
|
||
: new FakeSocket(frame, 'keepalive'),
|
||
);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(bytes),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
// conn #0 truncated block 0 → reconnect (#1) re-sends it and finishes.
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
const partial = frame.subarray(0, frame.length - 3);
|
||
expect(highwayTrace(entries).map((entry) => entry.message)).toEqual(
|
||
expect.arrayContaining([
|
||
expect.stringContaining(
|
||
`branch=chunk_control_response_partial offset=0 attempt=1 responseBytes=${partial.byteLength} responseHex=${partial.toString('hex')}`,
|
||
),
|
||
]),
|
||
);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('reconnects and preserves partial control-response bytes when the socket errors', async () => {
|
||
const frame = buildEmptyHighwayResponseFrame();
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(frame, 'keepalive', -1, -1, 1)
|
||
: new FakeSocket(frame, 'keepalive'),
|
||
);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE))),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
const partial = frame.subarray(0, frame.length - 3);
|
||
expect(highwayTrace(entries).map((entry) => entry.message)).toEqual(
|
||
expect.arrayContaining([
|
||
expect.stringContaining(
|
||
`branch=chunk_control_response_partial offset=0 attempt=1 responseBytes=${partial.byteLength} responseHex=${partial.toString('hex')}`,
|
||
),
|
||
]),
|
||
);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('does not preserve diagnostic response copies while TRACE is disabled', async () => {
|
||
const frame = buildEmptyHighwayResponseFrame();
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(frame, 'keepalive', -1, -1, 1)
|
||
: new FakeSocket(frame, 'keepalive'),
|
||
);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('info');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
new BufferChunkSource(new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE))),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
expect(highwayTrace(entries)).toEqual([]);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('reads every offset exactly once, in order, and closes the source once (data safety)', async () => {
|
||
installFactory(() => new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'));
|
||
const size = 4 * HIGHWAY_BLOCK_SIZE + 4321; // 4 full + 1 ragged
|
||
const source = new RecordingChunkSource(size);
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
const offsets = source.reads.map((r) => r.offset);
|
||
// Strictly increasing, gap-free, exactly-once coverage of [0, size).
|
||
expect(offsets).toEqual([
|
||
0,
|
||
HIGHWAY_BLOCK_SIZE,
|
||
2 * HIGHWAY_BLOCK_SIZE,
|
||
3 * HIGHWAY_BLOCK_SIZE,
|
||
4 * HIGHWAY_BLOCK_SIZE,
|
||
]);
|
||
expect(source.reads[source.reads.length - 1]!.length).toBe(4321); // ragged tail
|
||
expect(source.reads.reduce((n, r) => n + r.length, 0)).toBe(size);
|
||
expect(source.closeCount).toBe(1);
|
||
expect(source.closedWhileReading).toBe(false);
|
||
});
|
||
|
||
it('rejects on a definitive server error_code and closes the source once (no retry)', async () => {
|
||
installFactory(() => new FakeSocket(buildErrorCodeFrame(), 'keepalive'));
|
||
const source = new RecordingChunkSource(3 * HIGHWAY_BLOCK_SIZE);
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
1004,
|
||
source,
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
),
|
||
).rejects.toThrow(/error_code=921/);
|
||
expect(source.closeCount).toBe(1);
|
||
expect(source.closedWhileReading).toBe(false);
|
||
// Definitive reject on block 0 → stop immediately, don't PUT later blocks.
|
||
expect(source.reads.length).toBeLessThan(3);
|
||
});
|
||
|
||
it('delay-retries 102902 on the same offset, then completes (#352)', async () => {
|
||
// QQ BDH NeedDelayRetry: 102902 is backpressure, not a fatal reject.
|
||
// Official client delays, switches sender, and re-PUTs the same piece.
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(buildErrorCodeFrame(102902), 'keepalive')
|
||
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
|
||
);
|
||
const entries: LogEntry[] = [];
|
||
setLogLevel('trace');
|
||
const unsubscribe = subscribeLogs((entry) => entries.push(entry));
|
||
|
||
try {
|
||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE));
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
71,
|
||
new BufferChunkSource(bytes),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
const messages = highwayTrace(entries).map((entry) => entry.message);
|
||
expect(messages).toEqual(
|
||
expect.arrayContaining([
|
||
expect.stringContaining('branch=chunk_delay_retry offset=0'),
|
||
expect.stringContaining('errorCode=102902'),
|
||
expect.stringContaining('outcome=completed reason=upload_complete'),
|
||
]),
|
||
);
|
||
} finally {
|
||
unsubscribe();
|
||
}
|
||
});
|
||
|
||
it('delay-retries 302902 the same way as 102902', async () => {
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(buildErrorCodeFrame(302902), 'keepalive')
|
||
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
|
||
);
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
71,
|
||
new BufferChunkSource(new Uint8Array(512 * 1024)),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
});
|
||
|
||
it('delay-retries 102902 on a later in-order block after keep-alive was already warm (#352 shape)', async () => {
|
||
// Mirrors the report: sequential upload is healthy, then a later offset
|
||
// returns 102902. Recovery must re-PUT that offset, not abort the file.
|
||
installFactory((idx) =>
|
||
idx === 0
|
||
? new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive', -1, -1, -1, {
|
||
3: buildErrorCodeFrame(102902),
|
||
})
|
||
: new FakeSocket(buildEmptyHighwayResponseFrame(), 'keepalive'),
|
||
);
|
||
const bytes = new Uint8Array(Math.floor(2.5 * HIGHWAY_BLOCK_SIZE)); // 3 blocks
|
||
await uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
71,
|
||
new BufferChunkSource(bytes),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
);
|
||
expect(createConnectionMock).toHaveBeenCalledTimes(2);
|
||
});
|
||
|
||
it('exhausts delay-retries when 102902 keeps coming back', async () => {
|
||
installFactory(() => new FakeSocket(buildErrorCodeFrame(102902), 'keepalive'));
|
||
await expect(
|
||
uploadHighwayHttp(
|
||
makeBridge(),
|
||
makeSession(),
|
||
71,
|
||
new BufferChunkSource(new Uint8Array(16)),
|
||
new Uint8Array(16),
|
||
new Uint8Array(0),
|
||
),
|
||
).rejects.toThrow(/error_code=102902/);
|
||
expect(createConnectionMock.mock.calls.length).toBeGreaterThan(1);
|
||
});
|
||
});
|