From b8976bf2d837eb0ccae92cbc92248ceff280c0f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 02:05:21 +0000 Subject: [PATCH 1/3] feat(objectql,metadata-protocol): per-row dropped fields on validate and insertMany outcomes engine.validate answers each accepted row's droppedFields, and an ok insertMany outcome carries what the strips took from that row, both recorded at the strips themselves. The batch-level onFieldsDropped union is unchanged. validateData relays the verdict; insertManyData passes the outcomes through. Two spec TSDoc sentences now say what the producer does. Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- packages/metadata-protocol/src/protocol.ts | 15 ++- packages/objectql/src/engine.ts | 138 ++++++++++++++++----- packages/spec/src/api/export.zod.ts | 6 +- packages/spec/src/api/protocol.zod.ts | 12 +- 4 files changed, 135 insertions(+), 36 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index b05f7bce46d..f8ea1bab4c0 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -11842,6 +11842,12 @@ export class ObjectStackProtocolImplementation implements * Same object-existence gate as every other data entry point (#3770), so * an unknown object fails the same way here as it would on the real write * — a preview that 404s differently from its write is a mirror again. + * + * [#20922] The per-row drops ride the same verdict: `engine.validate` + * answers each accepted row's `droppedFields` from its own strips, so this + * relays them as it relays `errors` / `warnings`. No listener is passed + * here: the engine's `onFieldsDropped` events are the batch-level union, + * and this response has no batch-level slot to put them in. */ async validateData(request: { object: string, data: any, mode?: 'insert' | 'update', context?: any }) { this.assertObjectRegistered(request.object); @@ -13615,7 +13621,7 @@ export class ObjectStackProtocolImplementation implements * channel (an `onFieldsDropped` signature that carries the row), never a * reconstruction at this call site. */ - async insertManyData(request: { object: string, records: any[], context?: any }): Promise<{ object: string; outcomes: Array<{ ok: boolean; record?: any; error?: unknown }>; droppedFields?: DroppedFieldsEvent[] }> { + async insertManyData(request: { object: string, records: any[], context?: any }): Promise<{ object: string; outcomes: Array<{ ok: boolean; record?: any; error?: unknown; droppedFields?: DroppedFieldsEvent[] }>; droppedFields?: DroppedFieldsEvent[] }> { this.assertObjectRegistered(request.object); // [#3770] const engineInsertMany = (this.engine as any)?.insertMany; if (typeof engineInsertMany !== 'function') { @@ -13629,7 +13635,12 @@ export class ObjectStackProtocolImplementation implements const dropped: DroppedFieldsEvent[] = []; const opts: any = { onFieldsDropped: (e: DroppedFieldsEvent) => { dropped.push(e); } }; if (request.context !== undefined) opts.context = request.context; - const outcomes: Array<{ ok: boolean; record?: any; error?: unknown }> = await engineInsertMany.call( + // [#20922] Each `ok` outcome carries the ENGINE's per-row report + // (`outcomes[i].droppedFields`), recorded at its strips — a separate + // channel from the batch-level union below, which still names no row. + // Passed through as the engine answers it; ⛔ nothing here derives a + // row's drops from the union. + const outcomes: Array<{ ok: boolean; record?: any; error?: unknown; droppedFields?: DroppedFieldsEvent[] }> = await engineInsertMany.call( this.engine, request.object, request.records, diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 743ff917c84..c6e3621585e 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -186,9 +186,16 @@ import { * Per-row outcome of {@link ObjectQL.insertMany} (framework#3172). One entry * per input row, in input order: written rows carry the after-hook record, * failed rows carry the per-row error (validation / autonumber / encryption). + * + * [#20922] A written row also carries `droppedFields`: the caller-supplied + * fields the engine LEGALLY stripped from THAT row, one `DroppedFieldsEvent` + * per reason — the per-row channel, recorded at the strips. Present only when + * at least one field was taken from the row. The batch-level `onFieldsDropped` + * events are unchanged and still name no row; this key is not a resolution of + * them. A failed row carries none: its write did not complete. */ export type InsertManyRowOutcome = - | { ok: true; record: any } + | { ok: true; record: any; droppedFields?: DroppedFieldsEvent[] } | { ok: false; error: unknown }; import { CoreServiceName, StorageNameMapping, PLATFORM_PROVIDED_OBJECT_NAMES } from '@objectstack/spec/system'; import { IRealtimeService, RealtimeEventPayload } from '@objectstack/spec/contracts'; @@ -1931,23 +1938,28 @@ function undeclaredWriteFieldErrors( * * @param refused the declared-field door's per-row verdicts, index-aligned: a * row that door refused is left exactly as it is and counts toward nothing. - * @returns the rows as the strip leaves them (index-aligned with the input) and + * @returns the rows as the strip leaves them (index-aligned with the input); * the union of the keys it took, in first-seen order — one report per call, - * the shape every insert-side strip already reports in. + * the shape every insert-side strip already reports in; and `droppedPerRow`, + * the keys taken from EACH row (index-aligned, `[]` for a row it left alone). + * The union and the per-row lists are the same strip's one result read two + * ways: the per-row lists are what {@link droppedFieldEvents} attributes + * to a row on `validate` and on `insertMany`'s outcomes. */ function stripComputedWriteFields( schema: { fields?: unknown } | undefined, rows: readonly unknown[], refused?: readonly unknown[], -): { rows: unknown[]; dropped: string[] } { +): { rows: unknown[]; dropped: string[]; droppedPerRow: string[][] } { const out = rows.slice(); const dropped: string[] = []; + const droppedPerRow: string[][] = rows.map(() => []); const fields = schema?.fields; - if (!fields || typeof fields !== 'object' || Array.isArray(fields)) return { rows: out, dropped }; + if (!fields || typeof fields !== 'object' || Array.isArray(fields)) return { rows: out, dropped, droppedPerRow }; const computed = Object.entries(fields as Record) .filter(([, def]) => def?.type === 'formula') .map(([name]) => name); - if (computed.length === 0) return { rows: out, dropped }; + if (computed.length === 0) return { rows: out, dropped, droppedPerRow }; for (let i = 0; i < out.length; i++) { if (refused?.[i] !== undefined) continue; const row = out[i]; @@ -1960,9 +1972,35 @@ function stripComputedWriteFields( delete copy[name]; if (!dropped.includes(name)) dropped.push(name); } + droppedPerRow[i] = taken; out[i] = copy; } - return { rows: out, dropped }; + return { rows: out, dropped, droppedPerRow }; +} + +/** + * The strip report as `DroppedFieldsEvent`s: one event per reason, `computed` + * before `readonly` — the order the strips run — and an empty list when + * nothing was taken. + * + * ONE builder for both channels the create-side strips report through, so + * they cannot disagree on a reason or its order: the batch-level union (the + * `onFieldsDropped` events of `insert` and {@link ObjectQL.validate}, unchanged + * by the per-row channel) and the per-row lists (`results[i].droppedFields` on + * `validate`, `droppedFields` on an `ok` outcome of `insertMany`). The per-row + * lists are recorded at the strips themselves, never reconstructed from the + * union: "which rows supplied N" is not "which rows dropped N" once a hook can + * exempt a key on one row and not another. + */ +function droppedFieldEvents( + object: string, + computed: readonly string[], + readonly: readonly string[], +): DroppedFieldsEvent[] { + const events: DroppedFieldsEvent[] = []; + if (computed.length > 0) events.push({ object, fields: [...computed], reason: 'computed' }); + if (readonly.length > 0) events.push({ object, fields: [...readonly], reason: 'readonly' }); + return events; } /** @@ -12167,6 +12205,11 @@ export class ObjectQL implements IObjectQLEngine { * reported through `options.onFieldsDropped`, the listener the write reports * through. * + * [#20922] And per row: an accepted row's `droppedFields` names what the + * write would take from THAT row, recorded at the strips themselves — the + * same report `insertMany` hangs on that row's `ok` outcome. The listener's + * events stay the batch-level union, one per reason, naming no row. + * * ## ADR-0104 posture — the whole reason B was rejected * * The verdict is resolved against the TARGET DEPLOYMENT'S REAL POSTURE via @@ -12311,12 +12354,21 @@ export class ObjectQL implements IObjectQLEngine { // (#8093, ADDRESSING IS NOT PAYLOAD). `readonlyWhen` and the primary-key // strip are not run: both judge a prior record or a dispatch this // operation does not have (the named limits below). + // + // [#20922] Each strip's result is recorded twice at the strip itself: into + // the batch-level union the listener reports (unchanged), and into the + // taking row's own list, which becomes that row's `droppedFields` below. const readonlyDropped: string[] = []; - const collectTaken = (before: Record, after: Record): void => { - for (const k of Object.keys(before)) { - if (!(k in after) && !readonlyDropped.includes(k)) readonlyDropped.push(k); + const readonlyDroppedPerRow: string[][] = rows.map(() => []); + const recordTaken = (i: number, taken: readonly string[]): void => { + for (const k of taken) { + if (!readonlyDropped.includes(k)) readonlyDropped.push(k); + if (!readonlyDroppedPerRow[i]!.includes(k)) readonlyDroppedPerRow[i]!.push(k); } }; + const collectTaken = (i: number, before: Record, after: Record): void => { + recordTaken(i, Object.keys(before).filter((k) => !(k in after))); + }; if (!options?.context?.isSystem) { const preserveAudit = options?.context?.preserveAudit === true; if (mode === 'insert') { @@ -12325,7 +12377,7 @@ export class ObjectQL implements IObjectQLEngine { schemaForValidation as any, rows[i], rawRows[i] ?? {}, undefined, { preserveAudit }, ) as Record; if (stripped === rows[i]) continue; - collectTaken(rows[i]!, stripped); + collectTaken(i, rows[i]!, stripped); rows[i] = stripped; } const readonlySubject = staticReadonlyInsertSubject(schemaForValidation as any); @@ -12335,7 +12387,7 @@ export class ObjectQL implements IObjectQLEngine { object, readonlySubject, rows[i]!, rawRows[i] ?? {}, options?.context, { logger: undefined, strictReadonlyWrites: false, hookWrittenKeys: undefined, nowSnap: nowSnapshot, permissionResolution }, ); - for (const k of pass.taken) if (!readonlyDropped.includes(k)) readonlyDropped.push(k); + recordTaken(i, pass.taken); // The write fails on this resolution failure outside partial mode; // a preview has no partial mode, so it fails the same way. if (pass.error !== undefined) throw pass.error; @@ -12350,16 +12402,14 @@ export class ObjectQL implements IObjectQLEngine { schemaForValidation as any, rows[i], supplied, undefined, { preserveAudit }, ) as Record; if (stripped === rows[i]) continue; - collectTaken(rows[i]!, stripped); + collectTaken(i, rows[i]!, stripped); rows[i] = stripped; } } } const onFieldsDropped = options?.onFieldsDropped; if (typeof onFieldsDropped === 'function') { - const drops: DroppedFieldsEvent[] = []; - if (computedStrip.dropped.length > 0) drops.push({ object, fields: computedStrip.dropped, reason: 'computed' }); - if (readonlyDropped.length > 0) drops.push({ object, fields: readonlyDropped, reason: 'readonly' }); + const drops = droppedFieldEvents(object, computedStrip.dropped, readonlyDropped); for (const drop of drops) { try { onFieldsDropped(drop); @@ -12420,7 +12470,7 @@ export class ObjectQL implements IObjectQLEngine { // above. A resolution failure rejects the preview, as it fails the write. const previewPermissionsFor = await this.resolveOptionPermissions(schemaForValidation, rows, permissionResolution); - const results: NonNullable = rows.map((row) => { + const results: NonNullable = rows.map((row, i) => { const warnings: ValidateDataIssue[] = []; // Warn-first admissions are the posture signal the caller came for, so // they are reported — into this row's own bucket, never into the @@ -12457,7 +12507,15 @@ export class ObjectQL implements IObjectQLEngine { } throw e; } - return { valid: true, errors: [], warnings }; + // [#20922] THIS row's drops, from the strips' own per-row record above + // — the report `insertMany` hangs on the same row's `ok` outcome, so a + // dry run and its commit answer one row in one vocabulary. Only on a + // row the verdict accepts: a drop is "the write completed without + // them", and the write refuses an invalid row, so it drops nothing + // (`insertMany` answers that row `ok: false`, with no drops). Absent + // when nothing would be taken. + const droppedFields = droppedFieldEvents(object, computedStrip.droppedPerRow[i]!, readonlyDroppedPerRow[i]!); + return { valid: true, errors: [], warnings, ...(droppedFields.length > 0 ? { droppedFields } : {}) }; }); return { @@ -12888,7 +12946,21 @@ export class ObjectQL implements IObjectQLEngine { // hook that RE-ISSUES the record number lost its write to any caller // that had also submitted the key, while the same hook's write survived // on a caller that had not. The update path's twin (#5591). + // + // [#20922] Every key a caller-write strip takes is recorded twice, at + // the strip: into the batch-level union (`insertDropped`, reported once + // per call below, unchanged) and into the taking row's own list, which + // `insertMany` hangs on that row's `ok` outcome. Per row because the + // rows CAN differ (`hookWrittenKeys` is armed per row), so the union + // cannot be resolved back to rows after the fact. const insertDropped: string[] = []; + const insertDroppedPerRow: string[][] = rows.map(() => []); + const recordInsertTaken = (i: number, taken: readonly string[]): void => { + for (const k of taken) { + if (!insertDropped.includes(k)) insertDropped.push(k); + if (!insertDroppedPerRow[i]!.includes(k)) insertDroppedPerRow[i]!.push(k); + } + }; if (!opCtx.context?.isSystem) { const preserveAudit = opCtx.context?.preserveAudit === true; for (let i = 0; i < rows.length; i++) { @@ -12915,9 +12987,7 @@ export class ObjectQL implements IObjectQLEngine { }, ) as Record; if (stripped === rows[i]) continue; - for (const k of Object.keys(rows[i])) { - if (!(k in stripped) && !insertDropped.includes(k)) insertDropped.push(k); - } + recordInsertTaken(i, Object.keys(rows[i]).filter((k) => !(k in stripped))); rows[i] = stripped; rowHookContexts[i].input.data = stripped; } @@ -12976,8 +13046,8 @@ export class ObjectQL implements IObjectQLEngine { }, ); if (pass.row === rows[i] && pass.taken.length === 0) continue; + recordInsertTaken(i, pass.taken); for (const k of pass.taken) { - if (!insertDropped.includes(k)) insertDropped.push(k); if (preserveAudit && !preserveAuditIgnored.includes(k)) preserveAuditIgnored.push(k); } if (pass.error !== undefined) { @@ -13164,9 +13234,7 @@ export class ObjectQL implements IObjectQLEngine { // so `readonly` would lie about it. One event per reason, in the order // the strips ran; a payload with no `formula` key reports exactly what // it did before. - const insertDrops: DroppedFieldsEvent[] = []; - if (computedDropped.length > 0) insertDrops.push({ object, fields: computedDropped, reason: 'computed' }); - if (insertDropped.length > 0) insertDrops.push({ object, fields: insertDropped, reason: 'readonly' }); + const insertDrops = droppedFieldEvents(object, computedDropped, insertDropped); if (insertDrops.length > 0) { if (options?.strictReadonlyWrites === true) { // Before the driver write and before validation — nothing is @@ -13415,11 +13483,25 @@ export class ObjectQL implements IObjectQLEngine { // results for batch, the single record otherwise. In partial mode the // batch return is instead one outcome PER INPUT ROW ({ok,record} / // {ok:false,error}), in input order (framework#3172). + // + // [#20922] An `ok` outcome also carries what the strips took from + // THAT row (`droppedFields`, absent when nothing was taken), from + // their own per-row record — the same report `validate` answers the + // row's dry run with. A failed outcome carries none: its write did + // not complete, and a drop is "completed without them". const written = isBatch ? (partialMode - ? rows.map((_r, i) => (rowErrors[i] === undefined - ? { ok: true as const, record: rowHookContexts[i].result } - : { ok: false as const, error: rowErrors[i] })) + ? rows.map((_r, i): InsertManyRowOutcome => { + if (rowErrors[i] !== undefined) return { ok: false as const, error: rowErrors[i] }; + const droppedFields = droppedFieldEvents( + object, computedStrip.droppedPerRow[i] ?? [], insertDroppedPerRow[i] ?? [], + ); + return { + ok: true as const, + record: rowHookContexts[i].result, + ...(droppedFields.length > 0 ? { droppedFields } : {}), + }; + }) : rowHookContexts.map((rowCtx) => rowCtx.result)) : rowHookContexts[0].result; // Records ARE written; a summary that could not be recomputed after diff --git a/packages/spec/src/api/export.zod.ts b/packages/spec/src/api/export.zod.ts index fa9fdc4c109..0147ec5abc5 100644 --- a/packages/spec/src/api/export.zod.ts +++ b/packages/spec/src/api/export.zod.ts @@ -327,8 +327,10 @@ export type ImportRequestParsed = z.infer; * - `warnings` was SERVED before it was declared: the REST import runner's * dry-run branch (`packages/rest/src/import-runner.ts`) copies the admitted * findings of the engine's validate verdict onto the row, and both the - * synchronous route and the async job's results carry it (pinned at the wire - * by `packages/rest/src/import-dryrun-parity.test.ts`). Undeclared, it was + * synchronous route and the async job's results carry it (the synchronous + * route is pinned at the wire by + * `packages/rest/src/import-dryrun-parity.test.ts`; no test reads it off the + * async job's results). Undeclared, it was * invisible to every reader typed by this schema, and * `ImportRowResultSchema.parse` stripped it. * - `droppedFields` is DECLARED AHEAD of its producer, per row rather than diff --git a/packages/spec/src/api/protocol.zod.ts b/packages/spec/src/api/protocol.zod.ts index 98757f6462a..a6599d36d23 100644 --- a/packages/spec/src/api/protocol.zod.ts +++ b/packages/spec/src/api/protocol.zod.ts @@ -2119,10 +2119,14 @@ export const ValidateDataRequestSchema = lazySchema(() => z.object({ * * One entry per submitted row, in submission order. * - * A row's `droppedFields` is DECLARED AHEAD of its producer: the engine's - * `validate` already runs the write's strips and reports them through the - * write's own listener, but `validateData` does not yet forward that listener - * or attribute its events to rows, so no server sets the key until it does. + * A row's `droppedFields` is set by the engine's `validate`, which runs the + * write's computed-field strip (every context) and its strips of static + * `readonly` and runtime-owned fields (`isSystem`-gated), records what each + * takes from each row, and reports it on a row the verdict accepts; + * `validateData` relays it. An `update`-mode preview does not run the + * `readonlyWhen` or primary-key strips, which judge a prior record and an + * update dispatch the preview does not have, so it can report fewer drops than + * the update it predicts. * The element IS {@link DroppedFieldsEventSchema} — the same shape and `reason` * vocabulary the write reports, ⛔ never a second enum — so a preview and the * write it predicts answer in one vocabulary, exactly as `errors` / `warnings` From 24ca898f82f4dae49f06f84aa6dcb2adc9074998 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 02:06:54 +0000 Subject: [PATCH 2/3] test(objectql,metadata-protocol): pin per-row dropped fields on dry run and commit Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- .../src/protocol.dropped-fields.bulk.test.ts | 28 ++ .../engine-autonumber-runtime-owned.test.ts | 21 +- .../src/engine-per-row-dropped-fields.test.ts | 267 ++++++++++++++++++ 3 files changed, 307 insertions(+), 9 deletions(-) create mode 100644 packages/objectql/src/engine-per-row-dropped-fields.test.ts diff --git a/packages/metadata-protocol/src/protocol.dropped-fields.bulk.test.ts b/packages/metadata-protocol/src/protocol.dropped-fields.bulk.test.ts index 8d2f2d6d64f..f49ad947ada 100644 --- a/packages/metadata-protocol/src/protocol.dropped-fields.bulk.test.ts +++ b/packages/metadata-protocol/src/protocol.dropped-fields.bulk.test.ts @@ -141,6 +141,11 @@ describe('insertManyData — BATCH-LEVEL droppedFields that names no row (#3455) // so the cases are REPLACED rather than amended. The engine double below // models the exemption, which the old one had no concept of; that is why the // old case stayed green through exactly the shape it was written for. + // + // [#20922] Row precision now comes from the ENGINE, on each `ok` outcome's + // own `droppedFields` (pinned against the real engine in objectql's + // `engine-per-row-dropped-fields.test.ts`). The double below attributes no + // row, so these cases pin what THIS seam adds: nothing derived from the union. /** * `hookStamps` names the rows whose `beforeInsert` hook re-assigns @@ -250,6 +255,29 @@ describe('insertManyData — BATCH-LEVEL droppedFields that names no row (#3455) }); expect(res).not.toHaveProperty('droppedFields'); }); + + it('[#20922] an outcome the ENGINE attributed passes through as answered, beside the unchanged union', async () => { + // The per-row channel is the engine's (`InsertManyRowOutcome.droppedFields`, + // recorded at its strips). This seam relays it and derives nothing: the + // cases above, whose engine attributes no row, still get none. + const rowDrop = { object: 'approval_case', fields: ['approval_status'], reason: 'readonly' as const }; + const insertMany = vi.fn(async (object: string, rows: any[], options?: any) => { + options?.onFieldsDropped?.({ object, fields: ['approval_status'], reason: 'readonly' }); + return rows.map((r, i) => (i === 1 + ? { ok: true, record: { id: 'rec-2', title: r.title, approval_status: 'draft' }, droppedFields: [rowDrop] } + : { ok: true, record: { id: 'rec-1', title: r.title, approval_status: 'draft' } })); + }); + const p = makeProtocol(insertMany as any); + + const res: any = await p.insertManyData({ + object: 'approval_case', + records: [{ title: 'A' }, { title: 'B', approval_status: 'approved' }], + }); + + expect(res.outcomes[0]).not.toHaveProperty('droppedFields'); + expect(res.outcomes[1].droppedFields).toEqual([rowDrop]); + expect(res.droppedFields).toEqual([rowDrop]); + }); }); describe('batchData — per-row droppedFields + context threading (#3455)', () => { diff --git a/packages/objectql/src/engine-autonumber-runtime-owned.test.ts b/packages/objectql/src/engine-autonumber-runtime-owned.test.ts index c80fdcf4633..6e5c5b96f75 100644 --- a/packages/objectql/src/engine-autonumber-runtime-owned.test.ts +++ b/packages/objectql/src/engine-autonumber-runtime-owned.test.ts @@ -305,15 +305,15 @@ describe('#5503 — autonumber is runtime-owned: bulk-create surfaces', () => { expect((res.droppedFields ?? []).flatMap((e: DroppedFieldsEvent) => e.fields)).toContain('account_number'); }); - it('insertManyData reports the union at BATCH level and names no row', async () => { + it('insertManyData reports the union at BATCH level, and the row that lost the value on its own outcome', async () => { // The import runner prefers this partial-success surface, so it is the one - // that has to stay honest — and honest here means naming no row: the - // engine's event carries no row index, and the two facts that would let a - // caller resolve it are both unavailable at the protocol seam. The row - // records below are the second one: BOTH come back carrying - // `account_number`, because the strip is followed by `applyAutonumbers`. - // So "is the key still on the row?" answers the same for the row that was - // stripped and the row that was not. + // that has to stay honest. The batch-level union names no row: the + // engine's event carries no row index. Nor can the row records resolve it: + // BOTH come back carrying `account_number`, because the strip is followed + // by `applyAutonumbers`, so "is the key still on the row?" answers the + // same for the row that was stripped and the row that was not. + // [#20922] Row precision comes from the ENGINE instead: the strip records + // what it took per row, and the row's `ok` outcome carries it. const rig = await makeEngine(); const res: any = await rig.protocol.insertManyData({ object: 'an_account', @@ -323,7 +323,10 @@ describe('#5503 — autonumber is runtime-owned: bulk-create surfaces', () => { ], }); expect(res.outcomes.map((o: any) => o.record.account_number)).toEqual(['ACC-0001', 'ACC-0002']); - for (const o of res.outcomes) expect(o).not.toHaveProperty('droppedFields'); + expect(res.outcomes[0]).not.toHaveProperty('droppedFields'); + expect(res.outcomes[1].droppedFields).toEqual([ + { object: 'an_account', fields: ['account_number'], reason: 'readonly' }, + ]); expect((res.droppedFields ?? []).flatMap((e: DroppedFieldsEvent) => e.fields)).toEqual(['account_number']); }); }); diff --git a/packages/objectql/src/engine-per-row-dropped-fields.test.ts b/packages/objectql/src/engine-per-row-dropped-fields.test.ts new file mode 100644 index 00000000000..c9a0f2e578d --- /dev/null +++ b/packages/objectql/src/engine-per-row-dropped-fields.test.ts @@ -0,0 +1,267 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// #20922 — the per-row drop report. The dry run (`validate` → a row's +// `droppedFields`) and the commit (`insertMany` → an `ok` outcome's +// `droppedFields`) name what the strips took from EACH row, recorded at the +// strips themselves; the batch-level `onFieldsDropped` union beside them is +// unchanged and still names no row. +// +// ## What was wrong (measured on `main` 9b0de7de7) +// +// `engine.validate` and `engine.insert` built the strips' result across ALL +// rows and emitted ONE event per reason, so neither the dry run nor the commit +// could say which row lost which field. `ValidateDataResponseSchema.results[]` +// and `ImportRowResultSchema` declare a per-row `droppedFields`, and nothing +// set it: a formula value or a forged `readonly` column was answered +// `valid: true` / `ok: true` on its row with nothing to say it would not land. +// +// ## What this file pins, on a recording driver +// +// A formula column (`doubled`, reason `computed`) and a static `readonly` +// column (`note`, reason `readonly`): dry run and commit answer the same +// per-row drops with the right reason; a clean row carries none (the control); +// the listener's union is the one it always was. Through the protocol too +// (`validateData`, `insertManyData`), which relay what the engine answers. + +import { describe, it, expect } from 'vitest'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { ObjectQL, type InsertManyRowOutcome } from './engine.js'; +import type { DroppedFieldsEvent } from '@objectstack/spec/data'; + +const silentLogger: any = (() => { + const l: any = { + trace() {}, debug() {}, info() {}, warn() {}, error() {}, fatal() {}, + child() { return l; }, + }; + return l; +})(); + +/** Records what reaches the driver — the payload is the verdict. */ +function makeRecordingDriver() { + const created: Array> = []; + let seq = 0; + const driver: any = { + name: 'recording', version: '0.0.0', supports: {}, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, + async find() { return []; }, + async findOne() { return null; }, + async create(_o: string, data: Record) { + created.push({ ...data }); + return { id: (data.id as string | undefined) ?? `gen_${++seq}`, ...data }; + }, + async update(_o: string, id: string, data: Record) { return { id, ...data }; }, + async updateMany() { return 0; }, + async delete() { return true; }, + async deleteMany() { return 0; }, + async count() { return 0; }, + async bulkCreate(o: string, list: Record[]) { + const out = []; + for (const r of list) out.push(await driver.create(o, r)); + return out; + }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, + async commit() {}, async rollback() {}, + }; + return { driver, created }; +} + +/** + * `doubled` is the formula under test; `note` is a static `readonly` column; + * `code` carries a `maxLength`, so a row can be made INVALID without touching + * either strip; `n` / `title` are the writable controls. + */ +async function makeRig() { + const engine = new ObjectQL({ logger: silentLogger }); + const { driver, created } = makeRecordingDriver(); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject({ + name: 'proj', + fields: { + n: { type: 'number' }, + title: { type: 'text' }, + code: { type: 'text', maxLength: 3 }, + doubled: { type: 'formula', expression: 'record.n * 2' }, + note: { type: 'text', readonly: true }, + }, + } as any); + const protocol = new ObjectStackProtocolImplementation(engine as any); + return { engine, protocol, created }; +} + +function listener() { + const events: DroppedFieldsEvent[] = []; + return { events, onFieldsDropped: (e: DroppedFieldsEvent) => { events.push(e); } }; +} + +const COMPUTED = (...fields: string[]): DroppedFieldsEvent => ({ object: 'proj', fields, reason: 'computed' }); +const READONLY = (...fields: string[]): DroppedFieldsEvent => ({ object: 'proj', fields, reason: 'readonly' }); + +/** + * Four rows, one per cell: a formula value, a forged `readonly` value, a clean + * row (the control), and both at once. + */ +const ROWS: Array> = [ + { n: 1, doubled: 5 }, + { n: 2, note: 'forged' }, + { n: 3, title: 'clean' }, + { n: 4, doubled: 9, note: 'forged' }, +]; +/** What each row of {@link ROWS} loses, in the strips' order. */ +const PER_ROW: Array = [ + [COMPUTED('doubled')], + [READONLY('note')], + undefined, + [COMPUTED('doubled'), READONLY('note')], +]; +/** The batch-level union over {@link ROWS}: one event per reason, naming no row. */ +const UNION: DroppedFieldsEvent[] = [COMPUTED('doubled'), READONLY('note')]; + +function okOutcome(o: InsertManyRowOutcome): Extract { + if (!o.ok) throw new Error(`expected an ok outcome, got ${JSON.stringify(o)}`); + return o; +} + +describe('[#20922] the dry run answers per-row drops — engine.validate', () => { + it('each row names what the write would take from IT, with the right reason; the clean row carries none', async () => { + const { engine, created } = await makeRig(); + const res = await engine.validate('proj', ROWS); + + expect(res.valid).toBe(true); + expect(res.results).toHaveLength(ROWS.length); + res.results.forEach((r, i) => { + expect(r.valid).toBe(true); + if (PER_ROW[i] === undefined) expect(r, `row ${i} took nothing`).not.toHaveProperty('droppedFields'); + else expect(r.droppedFields, `row ${i}`).toEqual(PER_ROW[i]); + // A strip is not a finding. + expect(r.errors).toEqual([]); + }); + expect(created).toHaveLength(0); + }); + + it('the listener still reports the batch-level union, one event per reason, naming no row', async () => { + const { engine } = await makeRig(); + const l = listener(); + await engine.validate('proj', ROWS, { onFieldsDropped: l.onFieldsDropped }); + expect(l.events).toEqual(UNION); + }); + + it('a row the verdict refuses carries no drops — the write would not complete it', async () => { + const { engine } = await makeRig(); + const res = await engine.validate('proj', [{ n: 1, code: 'too-long', doubled: 5 }, { n: 2, doubled: 5 }]); + + expect(res.results[0]!.valid).toBe(false); + expect(res.results[0]).not.toHaveProperty('droppedFields'); + expect(res.results[1]!.droppedFields).toEqual([COMPUTED('doubled')]); + }); + + it('isSystem: the readonly strip stands down per row as it does on the write, the computed strip does not', async () => { + const { engine } = await makeRig(); + const res = await engine.validate('proj', ROWS, { context: { isSystem: true } as any }); + expect(res.results.map((r) => r.droppedFields)).toEqual([ + [COMPUTED('doubled')], undefined, undefined, [COMPUTED('doubled')], + ]); + }); + + it('update mode: the same per-row report from the update-side strips', async () => { + const { engine } = await makeRig(); + const res = await engine.validate('proj', ROWS, { mode: 'update' }); + expect(res.results.map((r) => r.droppedFields)).toEqual(PER_ROW); + }); +}); + +describe('[#20922] the commit answers per-row drops — engine.insertMany', () => { + it('each ok outcome names what was taken from THAT row; the clean row carries none', async () => { + const { engine, created } = await makeRig(); + const outcomes = await engine.insertMany('proj', ROWS); + + expect(outcomes).toHaveLength(ROWS.length); + outcomes.forEach((o, i) => { + const ok = okOutcome(o); + if (PER_ROW[i] === undefined) expect(ok, `row ${i} took nothing`).not.toHaveProperty('droppedFields'); + else expect(ok.droppedFields, `row ${i}`).toEqual(PER_ROW[i]); + }); + // The report describes the stored payload: neither key reached the driver. + expect(created).toHaveLength(ROWS.length); + for (const row of created) { + expect(row).not.toHaveProperty('doubled'); + expect(row).not.toHaveProperty('note'); + } + }); + + it('the dry run and the commit answer the same drops, row for row', async () => { + const { engine } = await makeRig(); + const dry = await engine.validate('proj', ROWS); + const commit = await engine.insertMany('proj', ROWS); + expect(commit.map((o) => okOutcome(o).droppedFields)).toEqual(dry.results.map((r) => r.droppedFields)); + }); + + it('the listener still reports the batch-level union, one event per reason, naming no row', async () => { + const { engine } = await makeRig(); + const l = listener(); + await engine.insertMany('proj', ROWS, { onFieldsDropped: l.onFieldsDropped }); + expect(l.events).toEqual(UNION); + }); + + it('a failed outcome carries no drops — its write did not complete', async () => { + const { engine } = await makeRig(); + const outcomes = await engine.insertMany('proj', [{ n: 1, code: 'too-long', note: 'forged' }, { n: 2, note: 'forged' }]); + + expect(outcomes[0]!.ok).toBe(false); + expect(outcomes[0]).not.toHaveProperty('droppedFields'); + expect(okOutcome(outcomes[1]!).droppedFields).toEqual([READONLY('note')]); + }); + + it('a hook that writes the readonly key on ONE row keeps it there: only the other row is named', async () => { + // The case a reconstruction from the union cannot get right: both rows + // SUPPLIED `note`, so "which rows supplied it" names both. Row 1's + // `beforeInsert` hook assigns it, which exempts it on that row alone. + const { engine, created } = await makeRig(); + engine.registerHook('beforeInsert', async (ctx: any) => { + if (ctx.input.data.n === 2) ctx.input.data.note = 'hook-stamped'; + }, { object: 'proj' }); + const l = listener(); + const outcomes = await engine.insertMany( + 'proj', + [{ n: 1, note: 'forged' }, { n: 2, note: 'forged' }], + { onFieldsDropped: l.onFieldsDropped }, + ); + + expect(okOutcome(outcomes[0]!).droppedFields).toEqual([READONLY('note')]); + expect(okOutcome(outcomes[1]!), 'the hook wrote it, so it was not dropped').not.toHaveProperty('droppedFields'); + expect(created[1]).toMatchObject({ note: 'hook-stamped' }); + expect(l.events).toEqual([READONLY('note')]); + }); + + it('the non-partial batch insert is unchanged: it returns the records, with no per-row slot', async () => { + const { engine } = await makeRig(); + const records = await engine.insert('proj', ROWS); + expect(Array.isArray(records)).toBe(true); + for (const r of records as Array>) expect(r).not.toHaveProperty('droppedFields'); + }); +}); + +describe('[#20922] through the protocol — validateData and insertManyData relay the engine\'s per-row report', () => { + it('validateData answers each row\'s drops in results[].droppedFields', async () => { + const { protocol } = await makeRig(); + const res: any = await protocol.validateData({ object: 'proj', data: ROWS }); + expect(res.results.map((r: any) => r.droppedFields)).toEqual(PER_ROW); + }); + + it('insertManyData passes the per-outcome report through and keeps the batch-level union', async () => { + const { protocol } = await makeRig(); + const res = await protocol.insertManyData({ object: 'proj', records: ROWS }); + expect(res.outcomes.map((o) => o.droppedFields)).toEqual(PER_ROW); + expect(res.droppedFields).toEqual(UNION); + }); + + it('control: nothing taken anywhere ⇒ no per-row key and no batch-level key', async () => { + const { protocol } = await makeRig(); + const dry: any = await protocol.validateData({ object: 'proj', data: [{ n: 1 }, { n: 2, title: 'x' }] }); + for (const r of dry.results) expect(r).not.toHaveProperty('droppedFields'); + const commit = await protocol.insertManyData({ object: 'proj', records: [{ n: 1 }, { n: 2, title: 'x' }] }); + for (const o of commit.outcomes) expect(o).not.toHaveProperty('droppedFields'); + expect(commit).not.toHaveProperty('droppedFields'); + }); +}); From 99a7547af74551f99e2e49ca34fcab988919bc5d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 02:21:45 +0000 Subject: [PATCH 3/3] chore(changeset): per-row dropped fields on validate and insertMany Claude-Session: https://claude.ai/code/session_01Ujdtvqs7ree7WyQmEDwEnG Co-authored-by: Claude --- .changeset/20922-per-row-dropped-fields.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .changeset/20922-per-row-dropped-fields.md diff --git a/.changeset/20922-per-row-dropped-fields.md b/.changeset/20922-per-row-dropped-fields.md new file mode 100644 index 00000000000..cfad23cff04 --- /dev/null +++ b/.changeset/20922-per-row-dropped-fields.md @@ -0,0 +1,13 @@ +--- +'@objectstack/objectql': minor +'@objectstack/metadata-protocol': minor +--- + +The dry run and the partial-success batch insert now say which row lost which field. `ObjectQL.validate` (and `validateData`, which relays it) answers `droppedFields` on each accepted row of `results`, and `ObjectQL.insertMany` (and `insertManyData`, which passes it through) answers `droppedFields` on each `ok` outcome: the caller-supplied fields the engine legally strips from that row, one `DroppedFieldsEvent` per reason, in the engine's own reason vocabulary (`computed` for a `formula` value, `readonly` for a static `readonly` or runtime-owned field). The key is absent when nothing was taken from the row. + +- **Recorded at the strips, never inferred from the union.** Each strip records what it takes from each row as it runs. A `beforeInsert` hook that assigns a protected key on one row keeps it there, so that row is not named, while a sibling row that supplied the same key and lost it is. +- **A row the write does not complete carries none.** A preview row the verdict refuses, and an `ok: false` outcome, carry no `droppedFields`: a drop means the write completed without the field. +- **The dry run and the commit agree.** On `insert` mode the preview runs the same strips the write runs, so a row's preview drops and its outcome drops are the same list. One gap is unchanged: the preview runs no hooks, so a key a `beforeInsert` hook assigns is reported by the preview and kept by the write. An `update`-mode preview does not run the `readonlyWhen` or primary-key strips, which judge a prior record the preview does not read. +- **Unchanged:** the `onFieldsDropped` listener on `insert`, `insertMany` and `validate` still reports the batch-level union, one event per reason, naming no row. So does `insertManyData`'s top-level `droppedFields`. `insert(object, rows[])` still returns the records, with no per-row slot. `strictReadonlyWrites` still refuses the whole batch before any outcome is built. + +Graded `minor` in both packages: each widens a published method's declared answer with a new optional key (`InsertManyRowOutcome` gains `droppedFields`, and so does each outcome of `insertManyData`'s return type), which is an additive widening of the public surface. Nothing is removed, renamed or refused. The keys on the wire, `ValidateDataResponseSchema.results[].droppedFields` and `ImportRowResultSchema.droppedFields`, were already declared in `@objectstack/spec`. The REST import route does not copy the per-row report onto its row results yet.