Skip to content

Commit 7d2e215

Browse files
authored
Analytics: say whether tokens per day count every model call or only runs (#316)
The analytics API now puts the per-call meter's daily totals on the page (every model call: runs, including work on runs started earlier, reviews, follow-ups, chat) instead of what the run records add up to. The chart's title says which one it shows, and the API types carry the metered cost, the run records' cost and what the metered cost was for.
1 parent dbfbd17 commit 7d2e215

3 files changed

Lines changed: 20 additions & 3 deletions

File tree

‎dashboard/e2e/analytics.spec.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ function days(n: number) {
1818

1919
const SUMMARY = {
2020
runs: 42, prs_opened: 30, prs_merged: 21, prs_closed: 3, prs_open: 6, merge_rate: 0.7, failed_runs: 4, failure_rate: 0.095,
21-
cost_usd: 96.4, cost_per_run: 2.3, cost_per_merged_pr: 4.59, tokens_in: 12_000_000, tokens_out: 900_000,
21+
cost_usd: 96.4, run_cost_usd: 41.2, cost_basis: "metered", cost_by_feature: [{ feature: "review", cost_usd: 50 }, { feature: "run", cost_usd: 41.2 }, { feature: "feedback", cost_usd: 5.2 }],
22+
cost_per_run: 2.3, cost_per_merged_pr: 4.59, tokens_in: 12_000_000, tokens_out: 900_000,
2223
median_run_minutes: 11.5, median_hours_to_merge: 5.2, time_to_merge_known: 12, merged_untouched: 9, merged_with_edits: 3, edits_known: 12, reviews: 55,
2324
};
2425

@@ -205,6 +206,8 @@ test("the scorecard answers whether it is worth it, against the previous period"
205206
await expect(page.getByTestId("tile-untouched")).toContainText("75%");
206207
await expect(page.getByTestId("tile-ci")).toContainText("6 of 8 runs went green");
207208
await expect(page.getByTestId("tile-findings")).toContainText("24 fixed in code, 8 settled by a person");
209+
// The tokens per day are the meter's (every model call), and say so.
210+
await expect(page.getByText("Tokens per day, every model call")).toBeVisible();
208211
const views = page.getByRole("navigation", { name: "Analytics views" });
209212
await views.getByRole("link", { name: "Reliability" }).click();
210213
await expect(page.getByRole("table", { name: "Failures by reason" })).toContainText("Timed out after #s");

‎dashboard/src/app/analytics/page.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ function Body({ data, loading, stats, tab }: { data: Analytics; loading: boolean
276276
</div>
277277
</ChartCard>
278278
<ChartCard
279-
title="Tokens per day"
279+
title={s.cost_basis === "metered" ? "Tokens per day, every model call" : "Tokens per day, runs only"}
280280
table={<Table label="Tokens per day" head={["Day", "Tokens", "Runs"]} rows={data.daily.map((r) => [shortDay(r.day), tokens(r.tokens ?? 0), r.runs])} empty="No days." />}
281281
>
282282
{data.daily.every((r) => !r.tokens) ? (

‎dashboard/src/lib/api.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3222,8 +3222,21 @@ export interface AnalyticsSummary {
32223222
failure_rate: number | null;
32233223
/** Runs a person closed on purpose: not failures. Older backends leave it out. */
32243224
closed_runs?: number;
3225+
/**
3226+
* What the team spent on model calls in the period. With `cost_basis`
3227+
* "metered": every call the usage meter counted (runs, including work on
3228+
* runs started earlier, reviews, follow-ups, chat). With "runs" (one
3229+
* repository picked, or an older server): what the run records add up to.
3230+
*/
32253231
cost_usd: number;
3232+
/** What the run records add up to (runs only, on the day each started). */
3233+
run_cost_usd?: number;
3234+
cost_basis?: "metered" | "runs";
3235+
/** The metered cost by what the calls were for, largest first. */
3236+
cost_by_feature?: { feature: string; cost_usd: number }[];
3237+
/** From the run records. */
32263238
cost_per_run: number | null;
3239+
/** From the run records. */
32273240
cost_per_merged_pr: number | null;
32283241
/** Tokens in plus out. */
32293242
tokens?: number;
@@ -3250,7 +3263,8 @@ export interface Analytics {
32503263
previous_period: { start: string; end: string };
32513264
summary: AnalyticsSummary;
32523265
previous: AnalyticsSummary;
3253-
daily: { day: string; runs: number; prs_opened: number; merged: number; failed: number; cost_usd: number; tokens?: number; reviews: number }[];
3266+
/** `cost_usd` and `tokens` are metered when the summary's `cost_basis` is "metered"; `run_*` are the run records'. */
3267+
daily: { day: string; runs: number; prs_opened: number; merged: number; failed: number; cost_usd: number; tokens?: number; run_cost_usd?: number; run_tokens?: number; reviews: number }[];
32543268
sources: { source: string; runs: number; prs: number; merged: number; merge_rate: number | null }[];
32553269
repos: { repo: string; runs: number; prs: number; merged: number; failed: number; cost_usd: number; tokens?: number; merge_rate: number | null; reviews: number }[];
32563270
/** One row per person (however Jira, GitHub or the dashboard named them), Coderhelm's own work, and runs that recorded nobody. */

0 commit comments

Comments
 (0)