Repository navigation
Expand file tree
/
Copy pathtest.ts
More file actions
735 lines (698 loc) · 29.2 KB
/
Copy pathtest.ts
File metadata and controls
735 lines (698 loc) · 29.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
import { randomUUID } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import { basename, join } from "node:path";
import {
acquireTestRunLock,
resolveWrappedTestRunLockPath,
TEST_RUN_ID_ENV,
TEST_RUN_LOCK_PATH_ENV,
TEST_RUN_LOCK_TOKEN_ENV,
} from "./test-run-lock";
import {
createContainedTestTemp,
recoverStaleTestTempArtifactsOnce,
removeTestTempTree,
writeTestTempOwner,
} from "./test-temp";
export interface IsolatedTestEnvironment {
root: string;
env: Record<string, string | undefined>;
cleanup(): void;
}
/**
* Credentials of the developer's own OpenCodex install that the sandbox must not inherit. A
* Windows install stores its data-plane token as a user environment variable, so every shell on
* that machine carries it; a test that then builds a service definition or starts a proxy reads
* the live token instead of its fixture and fails only on a developer machine.
*/
export const LIVE_INSTALL_CREDENTIAL_ENV = [
"OPENCODEX_API_AUTH_TOKEN",
"OPENCODEX_ADMIN_AUTH_TOKEN",
"OCX_API_TOKEN_FILE",
] as const;
export function createIsolatedTestEnvironment(
baseEnv: Record<string, string | undefined> = process.env,
): IsolatedTestEnvironment {
const hostTemp = tmpdir();
const recovery = recoverStaleTestTempArtifactsOnce({ tempRoot: hostTemp });
if (recovery && (recovery.removed > 0 || recovery.errors > 0 || recovery.truncated)) {
console.warn(
`[test] stale TEMP recovery removed ${recovery.removed} OpenCodex test root(s)`
+ (recovery.errors > 0 ? `; ${recovery.errors} could not be reclaimed` : "")
+ (recovery.truncated ? "; the bounded scan will continue on a later run" : "")
+ ".",
);
}
const root = mkdtempSync(join(hostTemp, "opencodex-test-"));
const opencodexHome = join(root, ".opencodex");
const codexHome = join(root, ".codex");
const containedTemp = createContainedTestTemp(root);
mkdirSync(opencodexHome, { recursive: true });
mkdirSync(codexHome, { recursive: true });
if (process.platform === "win32") {
// A Windows sandbox has to look like a real profile, because the known-folder APIs
// resolve relative to USERPROFILE and .NET returns an EMPTY STRING — not an error —
// when the folder it computes does not exist. `resolveWindowsRuntimeRoot` asks
// PowerShell for `GetFolderPath(LocalApplicationData)`, so without these directories
// every Codex coordinator lookup refuses with "Windows effective-account lookup
// returned an empty value" and each refusal surfaces as an unrelated assertion in
// whichever suite happened to touch a Codex home.
mkdirSync(join(root, "AppData", "Local"), { recursive: true });
mkdirSync(join(root, "AppData", "Roaming"), { recursive: true });
}
writeTestTempOwner(root, baseEnv[TEST_RUN_ID_ENV]);
const inherited = { ...baseEnv };
for (const name of LIVE_INSTALL_CREDENTIAL_ENV) delete inherited[name];
return {
root,
env: {
...inherited,
// Captured BEFORE HOME is overwritten: once the child starts with a rewritten
// HOME, `homedir()` returns the sandbox, so this hand-off is the only way the
// real-home write guard can still know which path to protect.
// (devlog 260730_codex_rs_upstream_v2_live_handoff/070.)
OCX_REAL_HOME: baseEnv.OCX_REAL_HOME ?? homedir(),
// Pin git's global config to the developer's real one before HOME moves.
//
// git resolves ~/.gitconfig from HOME, so a sandboxed HOME makes it invisible.
// That silently drops `safe.directory`, and on a checkout whose directory owner
// differs from the running account -- ordinary on Windows when a tool or
// installer created the tree -- every `git` call a test makes then fails with
// "detected dubious ownership". The test reads that as "this is not a git
// repository" and asserts against a fallback, which looks like a product bug in
// whichever adapter collected the metadata. Naming the file keeps the sandbox
// (git still writes nothing here) while leaving git's own trust decisions intact.
GIT_CONFIG_GLOBAL: baseEnv.GIT_CONFIG_GLOBAL ?? join(homedir(), ".gitconfig"),
// Pin Bun's runtime transpiler cache for the same reason. Bun keeps it under the home
// directory (macOS: ~/Library/Caches/bun/@t@), so a sandboxed HOME/USERPROFILE handed every
// batch, and every fixture that gives its child its own HOME, an empty cache: the first child
// re-transpiled each large module (73 src files are over the 50 KB cache threshold). On a busy
// Windows shard that first child took 10-45 s where later ones took 2-5 s, failing whichever
// timed case happened to spawn it. The cache holds transpiled source only, so sharing it keeps
// the sandbox. An explicit value, including "" or "0" to disable it, is kept as given.
BUN_RUNTIME_TRANSPILER_CACHE_PATH: baseEnv.BUN_RUNTIME_TRANSPILER_CACHE_PATH
?? join(hostTemp, "ocx-test-bun-transpiler-cache"),
HOME: root,
USERPROFILE: root,
OPENCODEX_HOME: opencodexHome,
CODEX_HOME: codexHome,
TEMP: containedTemp,
TMP: containedTemp,
TMPDIR: containedTemp,
},
cleanup() {
removeTestTempTree(root);
},
};
}
function hasCliFlag(requested: string[], name: string): boolean {
const delimiterIndex = requested.indexOf("--");
const wrapperArgs = delimiterIndex === -1 ? requested : requested.slice(0, delimiterIndex);
return wrapperArgs.some(arg => arg === name || arg.startsWith(`${name}=`));
}
const DEFAULT_TEST_PARALLELISM = 4;
// Bun 1.4.0 builds `bun test` options from its test, runtime, transpiler, and base tables.
// Only required values consume the next argument. Optional values such as `--parallel=2`
// must stay attached so a bare option cannot hide the positional filter that follows it.
const BUN_TEST_OPTIONS_REQUIRING_VALUES = new Set([
// Test options.
"--timeout",
"--rerun-each",
"--retry",
"--seed",
"--coverage-reporter",
"--coverage-dir",
"-t",
"--test-name-pattern",
"--grep",
"--reporter",
"--reporter-outfile",
"--max-concurrency",
"--path-ignore-patterns",
"--parallel-delay",
"--shard",
"--timings",
// Runtime options accepted by `bun test`.
"--watch-kill-signal",
"-r",
"--preload",
"--require",
"--import",
"--cpu-prof-name",
"--cpu-prof-dir",
"--cpu-prof-interval",
"--heap-prof-name",
"--heap-prof-dir",
"--heap-prof-interval",
"--install",
"-e",
"--eval",
"-p",
"--print",
"--port",
"--origin",
"--conditions",
"--fetch-preconnect",
"--max-http-header-size",
"--dns-result-order",
"--redirect-warnings",
"--disable-warning",
"--title",
"--unhandled-rejections",
"--console-depth",
"--user-agent",
"--cron-title",
"--cron-period",
"--trace-event-categories",
"--trace-event-file-pattern",
"--stack-trace-limit",
// Transpiler and base options accepted by `bun test`.
"--main-fields",
"--extension-order",
"--tsconfig-override",
"-d",
"--define",
"--drop",
"--feature",
"-l",
"--loader",
"--jsx-factory",
"--jsx-fragment",
"--jsx-import-source",
"--jsx-runtime",
"--env-file",
"--cwd",
"-c",
"--config",
]);
export interface ChangedRunPreflight {
comparisonRef: string;
comparisonCommit: string;
changedFiles: string[];
}
const changedComparisonRefs = ["upstream/dev", "origin/dev", "dev"] as const;
/** Choose the highest-priority conventional dev ref without assuming which remote is canonical. */
export function selectChangedComparisonRef(refExists: (ref: string) => boolean): string | null {
return changedComparisonRefs.find(refExists) ?? null;
}
function decodeOutput(output: Uint8Array | undefined): string {
return output ? new TextDecoder().decode(output) : "";
}
function changedComparisonRef(requested: string[]): string | null {
const delimiterIndex = requested.indexOf("--");
const wrapperArgs = delimiterIndex === -1 ? requested : requested.slice(0, delimiterIndex);
const changedArg = wrapperArgs.find(arg => arg === "--changed" || arg.startsWith("--changed="));
if (!changedArg) return null;
if (changedArg === "--changed" || changedArg === "--changed=") {
throw new Error(
"[test] changed mode requires an explicit comparison ref; use --changed=<ref> so the selection can be validated.",
);
}
return changedArg.slice("--changed=".length);
}
function gitRefExists(
ref: string,
cwd: string,
env: Record<string, string | undefined>,
): boolean {
const result = Bun.spawnSync(["git", "rev-parse", "--verify", "--quiet", `${ref}^{commit}`], {
cwd,
env,
stdout: "ignore",
stderr: "ignore",
});
return result.exitCode === 0;
}
function gitOutput(
args: string[],
cwd: string,
env: Record<string, string | undefined>,
): string {
const result = Bun.spawnSync(["git", ...args], {
cwd,
env,
stdout: "pipe",
stderr: "pipe",
});
if (result.exitCode !== 0) {
const detail = decodeOutput(result.stderr).trim() || `exit ${result.exitCode ?? "unknown"}`;
throw new Error(`[test] git ${args[0]} failed while validating changed mode: ${detail}`);
}
return decodeOutput(result.stdout);
}
/** Resolve changed mode and inventory the diff against that commit before invoking Bun. */
export function inspectChangedRun(
requested: string[],
cwd: string = process.cwd(),
env: Record<string, string | undefined> = process.env,
): ChangedRunPreflight | null {
const requestedComparisonRef = changedComparisonRef(requested);
if (!requestedComparisonRef) return null;
if (requestedComparisonRef.startsWith("-")) {
throw new Error(
`[test] --changed comparison ref ${JSON.stringify(requestedComparisonRef)} is invalid.`,
);
}
const comparisonRef = requestedComparisonRef === "dev"
? selectChangedComparisonRef(ref => gitRefExists(ref, cwd, env))
: requestedComparisonRef;
if (!comparisonRef) {
throw new Error(
`[test] --changed=dev could not resolve a comparison ref; none of ${changedComparisonRefs.join(", ")} exists.`,
);
}
if (!gitRefExists(comparisonRef, cwd, env)) {
throw new Error(
`[test] --changed comparison ref ${JSON.stringify(comparisonRef)} does not resolve to a commit.`,
);
}
const comparisonCommit = gitOutput(["merge-base", "HEAD", comparisonRef], cwd, env).trim();
if (!comparisonCommit) {
throw new Error(
`[test] --changed comparison ref ${JSON.stringify(comparisonRef)} has no merge base with HEAD.`,
);
}
const diff = gitOutput(["diff", "--name-only", comparisonCommit, "--"], cwd, env);
const changedFiles = [...new Set(diff.split("\n").filter(Boolean))];
return { comparisonRef, comparisonCommit, changedFiles };
}
/** Refuse a successful changed-mode run when Bun silently selected no tests for a real diff. */
export function changedSelectionFailure(
preflight: ChangedRunPreflight,
output: string,
): string | null {
if (preflight.changedFiles.length === 0) return null;
const summary = output
.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, "")
.match(/Ran\s+(\d+)\s+tests?\s+across\s+(\d+)\s+files?\b/i);
if (!summary) {
return `[test] could not validate --changed=${preflight.comparisonCommit} (${preflight.comparisonRef} merge base): Bun did not emit a recognizable selection summary for a diff containing ${preflight.changedFiles.length} changed file(s).`;
}
if (Number(summary[1]) !== 0 || Number(summary[2]) !== 0) return null;
return `[test] --changed=${preflight.comparisonCommit} (${preflight.comparisonRef} merge base) selected 0 tests across 0 files, but the diff contains ${preflight.changedFiles.length} changed file(s). Bun follows only the parsed module graph; run the relevant focused tests for subprocess, read-as-data, or golden-file dependencies, or run the full suite.`;
}
/**
* True for a filter-less `bun run test`: no file arguments and no `--changed`.
* `--timeout` / `--dots` / `--parallel=N` still count as full.
*/
/** True for a filter-less `bun run test`. `--timeout` / `--dots` / `--parallel=N` still count. */
function isFullSuiteRun(requested: string[]): boolean {
const delimiterIndex = requested.indexOf("--");
const wrapperArgs = delimiterIndex === -1 ? requested : requested.slice(0, delimiterIndex);
const passedThrough = delimiterIndex === -1 ? [] : requested.slice(delimiterIndex + 1);
if (passedThrough.length > 0) return false;
if (hasCliFlag(requested, "--changed")) return false;
for (let index = 0; index < wrapperArgs.length; index++) {
const arg = wrapperArgs[index];
if (arg === "-" || !arg.startsWith("-")) return false;
if (!arg.includes("=") && BUN_TEST_OPTIONS_REQUIRING_VALUES.has(arg)) index++;
}
return true;
}
/**
* Default `bun test` argv for this repo.
*
* `--isolate` keeps a fresh global per file. Bounded parallelism is what makes the suite
* finishable: with isolate alone Bun re-evaluates
* the module graph once per file on a single core, so past ~900 files the run stops looking slow
* and starts looking hung — measured here at 1 h 29 m with zero output, ~57 % CPU and 8.5 MB RSS,
* against a few minutes for the identical suite with four workers. Leaving Bun to select all ten
* workers made deadline-sensitive tests fail under load, so the repository default is deterministic.
* A caller-supplied `--parallel` or `--parallel=N` is left alone.
*/
export function resolveBunTestArgs(
requested: string[],
comparisonCommit?: string,
): string[] {
const delimiterIndex = requested.indexOf("--");
const effectiveRequested = comparisonCommit
? requested.map((arg, index) => (
(delimiterIndex === -1 || index < delimiterIndex)
&& (arg === "--changed" || arg.startsWith("--changed="))
? "--changed=" + comparisonCommit
: arg
))
: requested;
const args = ["--isolate"];
if (!hasCliFlag(effectiveRequested, "--parallel")) {
args.push(`--parallel=${DEFAULT_TEST_PARALLELISM}`);
}
args.push(...effectiveRequested);
if (isFullSuiteRun(effectiveRequested)) args.push("./tests/");
return args;
}
// Paths relative to tests/. An entry moves with its file (scripts/test-layout/move.ts rewrites
// it); the lane label, the ignore glob, and the timeout table all key on the basename.
export const SERIAL_FULL_SUITE_FILES = [
"codex-integration/codex-shim.test.ts",
"providers/cursor/cursor-native-exec-shell.test.ts",
"codex-integration/issue-452-empty-503.test.ts",
"adapters/openai/openai-provider-option-e2e.test.ts",
"ci-workflows/release-helper.test.ts",
// The full macOS isolate pool stalled in the structure gate's synchronous Git
// child after earlier files; fresh-process execution retains the same assertions.
"ci-workflows/structure-ssot.test.ts",
// Synchronous injection subprocesses can wedge the long-lived macOS isolate
// parent while reaping a history Worker; contain them in a fresh bounded lane.
"codex-integration/codex-inject-write-lock.test.ts",
// Its management API import stalled the long-lived macOS isolate pool before
// any case ran; the complete file finishes in under a second in a fresh process.
"routing/subagent-roster-retention.test.ts",
// Linux run 36610213506 stalled this file after its WebSocket admission case
// in a multi-file process; all 11 cases completed in the attribution process.
// Keep its real listener lifecycle in a fresh process on every platform.
"codex-integration/active-registry-admission.test.ts",
"update/update-stop-first.test.ts",
// Relays a 50 MiB WebSocket frame end to end against a 15s deadline, so its result is a
// measurement of the whole process, not of the relay. On a healthy 3-CPU macOS runner the
// echo leg alone spends 7.4s of that budget; whichever half of `--shard=N/2` it lands in
// decides whether it finishes. It has been passing by accident: it sat in the lighter half
// until three unrelated test files were added elsewhere in the tree, Bun repartitioned, and
// it went from 7.4s to over 15s twice in a row without anything on the sideband path
// changing. Quarantining it here is what keeps it a test of the relay instead of a test of
// its neighbours.
"server/server-live.test.ts",
// These exercise the default-home service authority, shared by parallel Bun workers.
// A fresh process/home prevents another file's authority from becoming this fixture's input.
"service/service-ownership-state.test.ts",
"service/service-sqlite-home.test.ts",
"service/service.test.ts",
"service/service-claim.test.ts",
"service/service-wsl-home-ownership.test.ts",
"service/launchd-repair.test.ts",
"cli/cli-update-restart-home.test.ts",
"codex-integration/native-codex-toggle.test.ts",
"codex-integration/native-grok-toggle.test.ts",
] as const;
type SerialLaneBasename = (typeof SERIAL_FULL_SUITE_FILES)[number] extends infer P
? P extends `${string}/${infer B}` ? B : P
: never;
const SERIAL_LANE_TIMEOUT_MS: Partial<Record<SerialLaneBasename, number>> = {
// This file intentionally exercises 33 complete release-script subprocess trees.
// It is ~90s on an idle machine and measured at ~170s under unrelated host load.
"release-helper.test.ts": 5 * 60 * 1000,
};
export interface BunTestLane {
label: string;
args: string[];
timeoutMs: number;
}
function withoutParallelOverride(requested: string[]): string[] {
return requested.filter(arg => arg !== "--parallel" && !arg.startsWith("--parallel="));
}
function canUseSerialLanes(requested: string[]): boolean {
if (!isFullSuiteRun(requested)) return false;
return !["--changed", "--shard", "--reporter-outfile", "--update-timings"].some(flag => hasCliFlag(requested, flag));
}
/** Build the default full-suite plan: one bounded main lane plus isolated risky files. */
export function resolveBunTestPlan(
requested: string[], comparisonCommit?: string,
env: Record<string, string | undefined> = process.env,
): BunTestLane[] {
const rawTimeout = env.OCX_TEST_MAIN_TIMEOUT_MS;
const mainTimeout = rawTimeout === undefined ? 900_000 : Number(rawTimeout);
if (rawTimeout !== undefined && (!/^\d+$/.test(rawTimeout)
|| !Number.isSafeInteger(mainTimeout) || mainTimeout < 60_000 || mainTimeout > 3_600_000)) {
throw new Error("OCX_TEST_MAIN_TIMEOUT_MS must be an integer between 60000 and 3600000");
}
if (!canUseSerialLanes(requested)) {
return [{ label: "suite", args: resolveBunTestArgs(requested, comparisonCommit), timeoutMs: mainTimeout }];
}
const mainArgs = resolveBunTestArgs(requested, comparisonCommit);
const rootIndex = mainArgs.lastIndexOf("./tests/");
const ignores = SERIAL_FULL_SUITE_FILES.flatMap(file => ["--path-ignore-patterns", `**/${basename(file)}`]);
mainArgs.splice(rootIndex === -1 ? mainArgs.length : rootIndex, 0, ...ignores);
const serialRequested = withoutParallelOverride(requested);
return [
{ label: "parallel suite", args: mainArgs, timeoutMs: mainTimeout },
...SERIAL_FULL_SUITE_FILES.map(file => ({
label: basename(file),
args: resolveBunTestArgs(["--parallel=1", ...serialRequested, `./tests/${file}`]),
timeoutMs: SERIAL_LANE_TIMEOUT_MS[basename(file) as SerialLaneBasename] ?? 3 * 60 * 1000,
})),
];
}
function waitWithTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => resolve(null), timeoutMs);
promise.then(
value => {
clearTimeout(timer);
resolve(value);
},
error => {
clearTimeout(timer);
reject(error);
},
);
});
}
/** Read continuously so a timeout can still report output received before EOF. */
export function captureTestOutput(
stdout: ReadableStream<Uint8Array>,
stderr: ReadableStream<Uint8Array>,
) {
const collect = (stream: ReadableStream<Uint8Array>) => {
const reader = stream.getReader();
const decoder = new TextDecoder();
let text = "";
let reading = true;
let complete = false;
const done = (async () => {
try {
while (reading) {
const chunk = await reader.read();
if (!reading) break;
if (chunk.done) {
complete = true;
break;
}
text += decoder.decode(chunk.value, { stream: true });
}
} catch {
// Retain the prefix without turning a pipe error into an unhandled rejection.
} finally {
if (reading) text += decoder.decode();
reading = false;
reader.releaseLock();
}
})();
return {
done,
snapshot: () => ({ text, complete }),
cancel() {
if (!reading) return;
reading = false;
text += decoder.decode();
// A descendant may own a pipe, or a stream's cancellation may never settle.
// Cancellation is best effort; neither it nor EOF may extend the drain bound.
void reader.cancel().catch(() => {});
},
};
};
const out = collect(stdout);
const err = collect(stderr);
return {
async finish(timeoutMs: number) {
const drained = await waitWithTimeout(Promise.all([out.done, err.done]), timeoutMs);
if (drained === null) {
out.cancel();
err.cancel();
}
const stdout = out.snapshot();
const stderr = err.snapshot();
return {
stdout: stdout.text,
stderr: stderr.text,
complete: drained !== null && stdout.complete && stderr.complete,
};
},
};
}
export async function runTestLane(
lane: BunTestLane,
runId: string,
inheritedLock: { lockPath: string; ownerToken: string } | undefined,
capture = false,
writers = {
stdout: (value: string) => { process.stdout.write(value); },
stderr: (value: string) => { process.stderr.write(value); },
},
): Promise<{ exitCode: number; output: string }> {
const isolated = createIsolatedTestEnvironment({
...process.env,
[TEST_RUN_ID_ENV]: runId,
[TEST_RUN_LOCK_PATH_ENV]: inheritedLock?.lockPath,
[TEST_RUN_LOCK_TOKEN_ENV]: inheritedLock?.ownerToken,
// Lanes run many files in parallel, so a test that shortened a PRODUCT timing budget
// (not its own test timeout) needs headroom for process startup on a busy machine.
// See tests/helpers/ci-watchdog.ts `isolationBudgetMs`.
OCX_TEST_FULL_SUITE: "1",
});
const startedAt = Date.now();
let interrupted: NodeJS.Signals | null = null;
const child = Bun.spawn([process.execPath, "test", ...lane.args], {
env: isolated.env,
stdin: "inherit",
stdout: capture ? "pipe" : "inherit",
stderr: capture ? "pipe" : "inherit",
});
const captured = capture ? captureTestOutput(child.stdout!, child.stderr!) : undefined;
const forward = (signal: NodeJS.Signals) => {
interrupted = signal;
try { child.kill(signal); } catch { /* child already exited */ }
};
const onInterrupt = () => forward("SIGINT");
const onTerminate = () => forward("SIGTERM");
process.once("SIGINT", onInterrupt);
process.once("SIGTERM", onTerminate);
const exited = child.exited;
try {
let exitCode = await waitWithTimeout(exited, lane.timeoutMs);
if (exitCode === null) {
console.error(`[test] ${lane.label} exceeded ${Math.round(lane.timeoutMs / 1000)}s; terminating pid ${child.pid}.`);
try { child.kill("SIGTERM"); } catch { /* child already exited */ }
const graceful = await waitWithTimeout(exited, 5_000);
if (graceful === null) {
try { child.kill("SIGKILL"); } catch { /* child already exited */ }
await waitWithTimeout(exited, 2_000);
}
}
// Process exit does not guarantee EOF when a descendant inherited the pipe.
const result = await captured?.finish(1_000);
const stdout = result?.stdout ?? "";
const stderr = result?.stderr ?? "";
if (stdout) writers.stdout(stdout);
if (stderr) writers.stderr(stderr);
const output = stdout + "\n" + stderr;
if (result && !result.complete) {
console.error("[test] captured output is incomplete; collected output is shown above.");
if (exitCode === 0) exitCode = 1;
}
if (exitCode === null) return { exitCode: 124, output };
if (interrupted === "SIGINT") return { exitCode: 130, output };
if (interrupted === "SIGTERM") return { exitCode: 143, output };
const seconds = ((Date.now() - startedAt) / 1000).toFixed(1);
console.warn(`[test] ${lane.label} finished in ${seconds}s (exit ${exitCode}).`);
return { exitCode, output };
} finally {
process.off("SIGINT", onInterrupt);
process.off("SIGTERM", onTerminate);
try {
isolated.cleanup();
} catch {
console.error("[test] deferred cleanup of one test root after Windows kept a handle open; a later run will retry it.");
}
}
}
/**
* `gui` is not a workspace of the root package and declares React only in `gui/package.json`, so a
* root `bun install` never creates `gui/node_modules`. Twenty-six files under `tests/` import
* modules from `gui/src`, which makes those tests fail on a fresh clone or worktree with
* `Cannot find package 'react'` — reported as an "Unhandled error between tests" that names no
* test, so the cause is not obvious from the output.
*
* `.github/workflows/ci.yml` already installs them explicitly for exactly this reason; the local
* runner had no equivalent. Install on demand rather than fail, because the tests genuinely
* require the dependency and `gui/node_modules` is a gitignored build artifact, not source.
*/
export function ensureGuiDependencies(io: {
cwd?: string;
exists?: (path: string) => boolean;
install?: (guiDir: string) => { ok: boolean; detail: string };
log?: (message: string) => void;
} = {}): { kind: "present" | "installed" | "absent" | "failed"; detail?: string } {
const cwd = io.cwd ?? process.cwd();
const exists = io.exists ?? existsSync;
const log = io.log ?? (message => console.warn(message));
const guiDir = join(cwd, "gui");
if (!exists(join(guiDir, "package.json"))) return { kind: "absent" };
if (exists(join(guiDir, "node_modules", "react", "package.json"))) return { kind: "present" };
log("[test] gui dependencies are missing or incomplete; installing them so tests importing gui/src can resolve React.");
const install = io.install ?? ((dir: string) => {
const result = Bun.spawnSync(["bun", "install", "--frozen-lockfile"], {
cwd: dir,
stdout: "pipe",
stderr: "pipe",
});
return {
ok: result.exitCode === 0,
detail: decodeOutput(result.stderr) || decodeOutput(result.stdout),
};
});
const outcome = install(guiDir);
if (outcome.ok) return { kind: "installed" };
return { kind: "failed", detail: outcome.detail };
}
if (import.meta.main) {
const requestedTests = process.argv.slice(2);
const guiDependencies = ensureGuiDependencies();
if (guiDependencies.kind === "failed") {
console.error(
"[test] could not install gui/node_modules, which tests importing gui/src need to resolve React.\n"
+ " Run it manually: cd gui && bun install --frozen-lockfile\n"
+ (guiDependencies.detail ? ` ${guiDependencies.detail.trim().split("\n").slice(-3).join("\n ")}` : ""),
);
process.exitCode = 1;
}
let changedRun: ReturnType<typeof inspectChangedRun> = null;
if (process.exitCode !== 1) {
try {
changedRun = inspectChangedRun(requestedTests);
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
}
}
if (process.exitCode !== 1) {
if (changedRun) {
console.warn(
`[test] changed mode comparison ref: ${changedRun.comparisonRef}; merge base: ${changedRun.comparisonCommit}`,
);
}
const runId = randomUUID();
const lockPath = resolveWrappedTestRunLockPath({ env: process.env });
const lock = await acquireTestRunLock({
runId,
lockPath,
validatedRuntimePath: lockPath !== undefined,
onWait: owner => console.warn(
`[test] another Bun test run${owner ? ` (pid ${owner.pid})` : ""} holds the user lock; waiting. `
+ "Set OCX_TEST_NO_QUEUE=1 only for intentional overlap.",
),
onAcquiredAfterWait: elapsedMs => console.warn(`[test] acquired the user lock after ${Math.round(elapsedMs / 1000)}s.`),
});
const startedAt = Date.now();
try {
const inheritedLock = process.platform === "win32" && lockPath && lock.owner
? { lockPath, ownerToken: lock.owner.token }
: undefined;
let exitCode = 0;
let captured = "";
for (const lane of resolveBunTestPlan(requestedTests, changedRun?.comparisonCommit)) {
const result = await runTestLane(lane, runId, inheritedLock, Boolean(changedRun));
captured += result.output;
if (result.exitCode !== 0 && exitCode === 0) exitCode = result.exitCode;
if ([124, 130, 143].includes(result.exitCode)) break;
}
if (exitCode === 0 && changedRun) {
const selectionFailure = changedSelectionFailure(changedRun, captured);
if (selectionFailure) {
console.error(selectionFailure);
exitCode = 1;
}
}
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
if (isFullSuiteRun(requestedTests) && elapsedSeconds > 600) {
console.warn(
`[test] the suite took ${elapsedSeconds}s; with --parallel=${DEFAULT_TEST_PARALLELISM} it should finish in a few minutes on an idle machine. `
+ "Check for another test runner, a busy CPU, or a test that started polling something real.",
);
}
process.exitCode = exitCode;
} finally {
lock.release();
}
}
}