forked from EduCraft/curriculum-project-hub
feat(hub): usage fact ledger for run-scoped cost attribution (ADR-0026) (#4)
Co-authored-by: Hong Jiarong <me@jrhim.com> Co-committed-by: Hong Jiarong <me@jrhim.com>
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { prisma, resetDb, seedTestOrganization, DEFAULT_ORG_ID } from "./helpers.js";
|
||||
import { getOrgUsage, getProjectUsage } from "../../src/org/usage.js";
|
||||
|
||||
/**
|
||||
* ADR-0026: org/project usage rolls up from the UsageFact ledger, not from the
|
||||
* AgentRun scalar cache. These tests pin the fact-based aggregation contract:
|
||||
* - a run with a cost-bearing fact → runsWithCost, costUsd summed
|
||||
* - a run whose facts all have null costUsd → runsWithoutCost (≠ zero)
|
||||
* - non-token meter (quantity/unit) is stored but does not corrupt token sums
|
||||
* - multiple facts per run (main loop + external capability) are summed together
|
||||
* - the AgentRun rollup cache is NOT consulted by getOrgUsage
|
||||
*/
|
||||
describe("usage rollup from UsageFact (ADR-0026)", () => {
|
||||
beforeEach(async () => {
|
||||
await resetDb();
|
||||
await seedTestOrganization();
|
||||
});
|
||||
|
||||
async function seedRun(opts: {
|
||||
readonly projectId: string;
|
||||
readonly projectName: string;
|
||||
readonly folderId?: string;
|
||||
readonly facts: ReadonlyArray<{
|
||||
readonly kind?: string;
|
||||
readonly provider?: string;
|
||||
readonly model?: string;
|
||||
readonly inputTokens?: number;
|
||||
readonly outputTokens?: number;
|
||||
readonly costUsd?: number | null;
|
||||
readonly costSource?: string;
|
||||
readonly capabilityId?: string;
|
||||
readonly quantity?: number | null;
|
||||
readonly unit?: string | null;
|
||||
}>;
|
||||
}): Promise<void> {
|
||||
const project = await prisma.project.create({
|
||||
data: {
|
||||
id: opts.projectId,
|
||||
organizationId: DEFAULT_ORG_ID,
|
||||
name: opts.projectName,
|
||||
workspaceDir: `/tmp/test-${opts.projectId}`,
|
||||
...(opts.folderId !== undefined ? { folderId: opts.folderId } : {}),
|
||||
},
|
||||
});
|
||||
const startedAt = new Date("2026-07-18T10:00:00Z");
|
||||
const finishedAt = new Date("2026-07-18T10:05:00Z");
|
||||
const run = await prisma.agentRun.create({
|
||||
data: {
|
||||
projectId: project.id,
|
||||
entrypoint: "FEISHU",
|
||||
provider: "openrouter",
|
||||
model: "mock-model",
|
||||
metadata: {},
|
||||
status: "COMPLETED",
|
||||
prompt: "test",
|
||||
startedAt,
|
||||
finishedAt,
|
||||
// Deliberately set the rollup cache to values that differ from the
|
||||
// facts, to prove getOrgUsage reads facts, not the cache.
|
||||
inputTokens: 999,
|
||||
outputTokens: 999,
|
||||
costUsd: 999,
|
||||
costSource: "provider_reported",
|
||||
},
|
||||
});
|
||||
for (const [i, f] of opts.facts.entries()) {
|
||||
await prisma.usageFact.create({
|
||||
data: {
|
||||
runId: run.id,
|
||||
occurredAt: new Date(startedAt.getTime() + i * 1000),
|
||||
kind: f.kind ?? "model_completion",
|
||||
provider: f.provider ?? "openrouter",
|
||||
model: f.model ?? "mock-model",
|
||||
inputTokens: f.inputTokens ?? null,
|
||||
outputTokens: f.outputTokens ?? null,
|
||||
costUsd: f.costUsd ?? null,
|
||||
costSource: f.costSource ?? (f.costUsd !== undefined && f.costUsd !== null ? "provider_reported" : "unknown"),
|
||||
capabilityId: f.capabilityId ?? null,
|
||||
quantity: f.quantity ?? null,
|
||||
unit: f.unit ?? null,
|
||||
metadata: {},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
it("sums cost and tokens from facts, ignoring the AgentRun rollup cache", async () => {
|
||||
await seedRun({
|
||||
projectId: "proj-a",
|
||||
projectName: "A",
|
||||
facts: [{ inputTokens: 10, outputTokens: 5, costUsd: 0.0023 }],
|
||||
});
|
||||
await seedRun({
|
||||
projectId: "proj-b",
|
||||
projectName: "B",
|
||||
facts: [{ inputTokens: 20, outputTokens: 10, costUsd: 0.01 }],
|
||||
});
|
||||
|
||||
const report = await getOrgUsage(prisma, { organizationId: DEFAULT_ORG_ID });
|
||||
|
||||
expect(report.totals.runCount).toBe(2);
|
||||
expect(report.totals.runsWithCost).toBe(2);
|
||||
expect(report.totals.runsWithoutCost).toBe(0);
|
||||
expect(report.totals.inputTokens).toBe(30);
|
||||
expect(report.totals.outputTokens).toBe(15);
|
||||
expect(report.totals.costUsd).toBeCloseTo(0.0123, 8);
|
||||
});
|
||||
|
||||
it("treats a run with only null-cost facts as runsWithoutCost, never zero", async () => {
|
||||
await seedRun({
|
||||
projectId: "proj-unknown",
|
||||
projectName: "Unknown",
|
||||
facts: [{ inputTokens: 7, outputTokens: 3, costUsd: null, costSource: "unknown" }],
|
||||
});
|
||||
await seedRun({
|
||||
projectId: "proj-known",
|
||||
projectName: "Known",
|
||||
facts: [{ inputTokens: 4, outputTokens: 2, costUsd: 0.05 }],
|
||||
});
|
||||
|
||||
const report = await getOrgUsage(prisma, { organizationId: DEFAULT_ORG_ID });
|
||||
|
||||
expect(report.totals.runCount).toBe(2);
|
||||
expect(report.totals.runsWithCost).toBe(1);
|
||||
expect(report.totals.runsWithoutCost).toBe(1);
|
||||
// The unknown-cost run still contributes its tokens.
|
||||
expect(report.totals.inputTokens).toBe(11);
|
||||
expect(report.totals.outputTokens).toBe(5);
|
||||
// And its cost is NOT counted as zero — only the known cost is summed.
|
||||
expect(report.totals.costUsd).toBeCloseTo(0.05, 8);
|
||||
});
|
||||
|
||||
it("sums multiple facts per run (main loop + external capability)", async () => {
|
||||
await seedRun({
|
||||
projectId: "proj-multi",
|
||||
projectName: "Multi",
|
||||
facts: [
|
||||
{ kind: "model_completion", inputTokens: 100, outputTokens: 50, costUsd: 0.02 },
|
||||
{
|
||||
kind: "external_capability",
|
||||
provider: "mineru",
|
||||
model: null,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
costUsd: 0.015,
|
||||
costSource: "provider_reported",
|
||||
capabilityId: "pdf_to_md_bundle",
|
||||
quantity: 12,
|
||||
unit: "pages",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const report = await getOrgUsage(prisma, { organizationId: DEFAULT_ORG_ID });
|
||||
expect(report.totals.runCount).toBe(1);
|
||||
expect(report.totals.runsWithCost).toBe(1);
|
||||
expect(report.totals.inputTokens).toBe(100);
|
||||
expect(report.totals.outputTokens).toBe(50);
|
||||
expect(report.totals.costUsd).toBeCloseTo(0.035, 8);
|
||||
});
|
||||
|
||||
it("a run with no facts at all is runsWithoutCost and contributes nothing", async () => {
|
||||
await seedRun({ projectId: "proj-empty", projectName: "Empty", facts: [] });
|
||||
|
||||
const report = await getOrgUsage(prisma, { organizationId: DEFAULT_ORG_ID });
|
||||
expect(report.totals.runCount).toBe(1);
|
||||
expect(report.totals.runsWithCost).toBe(0);
|
||||
expect(report.totals.runsWithoutCost).toBe(1);
|
||||
expect(report.totals.inputTokens).toBe(0);
|
||||
expect(report.totals.outputTokens).toBe(0);
|
||||
expect(report.totals.costUsd).toBeNull();
|
||||
});
|
||||
|
||||
it("getProjectUsage returns just that project's row", async () => {
|
||||
await seedRun({ projectId: "proj-x", projectName: "X", facts: [{ costUsd: 0.1, inputTokens: 1, outputTokens: 1 }] });
|
||||
await seedRun({ projectId: "proj-y", projectName: "Y", facts: [{ costUsd: 0.2, inputTokens: 2, outputTokens: 2 }] });
|
||||
|
||||
const row = await getProjectUsage(prisma, { organizationId: DEFAULT_ORG_ID, projectId: "proj-x" });
|
||||
expect(row.projectId).toBe("proj-x");
|
||||
expect(row.runCount).toBe(1);
|
||||
expect(row.costUsd).toBeCloseTo(0.1, 8);
|
||||
});
|
||||
|
||||
it("returns an empty report for an org with no projects", async () => {
|
||||
const report = await getOrgUsage(prisma, { organizationId: DEFAULT_ORG_ID });
|
||||
expect(report.projects).toHaveLength(0);
|
||||
expect(report.totals.runCount).toBe(0);
|
||||
expect(report.totals.costUsd).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -290,6 +290,9 @@ function mockPrisma(): PrismaClient {
|
||||
findUnique: vi.fn(async () => null),
|
||||
update: vi.fn(async () => ({ id: "run-1" })),
|
||||
},
|
||||
usageFact: {
|
||||
create: vi.fn(async () => ({ id: "fact-1" })),
|
||||
},
|
||||
projectAgentLock: {
|
||||
findUnique: vi.fn(async () => (lock === null ? null : { runId: lock.runId })),
|
||||
create: vi.fn(async (args: { data: { projectId: string; runId: string } }) => {
|
||||
|
||||
Reference in New Issue
Block a user