/** * Real-model integration test — exercises the full agent tool-calls loop against * a live OpenRouter model + real cph binary + real workspace. * * Unlike MockProvider tests, this verifies: * - generateText's tool-calls loop with a real model (GLM/Claude/…) * - model returns tool_calls → runner dispatches → tool_result back → continue * - cph_check / read_file / list_files against a real engineering-file tree * - transcript JSONL read/append on a real filesystem * * Skipped automatically when OPENROUTER_API_KEY is not set (CI without secrets, * or local dev without a key). To run locally: * cp .env.example .env # fill in OPENROUTER_API_KEY * npx vitest run test/integration/real-model.test.ts */ import { describe, it, expect } from "vitest"; import { cp, rm, readFile } from "node:fs/promises"; import { join } from "node:path"; import "dotenv/config"; import { createModelFactory } from "../../src/agent/provider.js"; import { ToolRegistry, feishuContextTool } from "../../src/agent/tools.js"; import { readFileTool, writeFileTool, listFilesTool } from "../../src/agent/workspace.js"; import { cphCheckTool, cphBuildTool } from "../../src/agent/cph.js"; import { runAgent } from "../../src/agent/runner.js"; import type { FeishuRuntime } from "../../src/feishu/client.js"; import type { PrismaClient } from "@prisma/client"; const API_KEY = process.env["OPENROUTER_API_KEY"]; const HAS_KEY = API_KEY !== undefined && API_KEY !== ""; const describeOrSkip = HAS_KEY ? describe : describe.skip; const EXAMPLES_DIR = join(process.cwd(), "..", "examples", "TH-141"); const MODEL = process.env["SMOKE_MODEL"] ?? "z-ai/glm-4.6"; // Stub prisma: runner touches it for lock heartbeat + message persistence, // neither of which need a real DB for this test. const stubPrisma = { projectAgentLock: { update: async () => ({}), deleteMany: async () => ({}) }, agentMessage: { createMany: async () => ({}) }, agentFileChange: { createMany: async () => ({}) }, } as unknown as PrismaClient; const stubFeishuRt: FeishuRuntime = { client: { im: { v1: { message: { create: async () => ({}), get: async () => ({ data: {} }), list: async () => ({ data: {} }) } } }, } as never, logger: { info() {}, warn() {}, error() {}, debug() {}, fatal() {}, child() { return this; }, level: "silent" } as never, }; describeOrSkip("real model integration (OpenRouter)", () => { let ws: string; async function setupWorkspace(): Promise { ws = join(process.cwd(), ".test-ws-real"); await rm(ws, { recursive: true, force: true }); await cp(EXAMPLES_DIR, ws, { recursive: true }); return ws; } async function teardownWorkspace(): Promise { await rm(ws, { recursive: true, force: true }); } it("runs a multi-step tool-calls loop: list files → cph check → answer", async () => { const workspace = await setupWorkspace(); try { const tools = new ToolRegistry(); tools.register("feishu_read_context", feishuContextTool(async () => JSON.stringify({ stub: true }), stubFeishuRt)); tools.register("read_file", readFileTool); tools.register("write_file", writeFileTool); tools.register("list_files", listFilesTool); tools.register("cph_check", cphCheckTool); tools.register("cph_build", cphBuildTool); const modelFactory = createModelFactory(); const transcript = join(workspace, ".cph", "sessions", "real-test.jsonl"); const result = await runAgent(modelFactory, tools, { runId: "real-test-run", projectId: "real-test-project", prompt: "列出工程文件的目录结构,然后跑 cph check,用中文告诉我结果。", model: MODEL, project: { projectId: "real-test-project", boundChatId: "chat-test", workspaceDir: workspace }, prisma: stubPrisma, systemPrompt: "你是一个教研工程文件的助手。你可以读写工程文件、跑 cph check 校验、跑 cph build 渲染。请用中文回复。", transcriptPath: transcript, maxIterations: 10, }); // The model should complete (not hit the iteration cap or fail). expect(result.status).toBe("completed"); // It should have produced messages beyond just the initial prompt. expect(result.messages.length).toBeGreaterThan(1); // The transcript file should have been written. const transcriptContent = await readFile(transcript, "utf8"); expect(transcriptContent.length).toBeGreaterThan(0); const lines = transcriptContent.trim().split("\n"); expect(lines.length).toBeGreaterThan(1); // Each line should be valid JSON. for (const line of lines) { expect(() => JSON.parse(line)).not.toThrow(); } } finally { await teardownWorkspace(); } }, 60_000); // 60s timeout — real model calls take time. it("can write a file and read it back through the agent loop", async () => { const workspace = await setupWorkspace(); try { const tools = new ToolRegistry(); tools.register("read_file", readFileTool); tools.register("write_file", writeFileTool); tools.register("list_files", listFilesTool); tools.register("cph_check", cphCheckTool); tools.register("cph_build", cphBuildTool); const modelFactory = createModelFactory(); const result = await runAgent(modelFactory, tools, { runId: "real-test-run-2", projectId: "real-test-project", prompt: "在工程文件根目录创建一个文件 note.txt,内容写'测试笔记',然后用 read_file 读回来,告诉我内容。", model: MODEL, project: { projectId: "real-test-project", boundChatId: "chat-test", workspaceDir: workspace }, prisma: stubPrisma, systemPrompt: "你是一个教研工程文件的助手。请用中文回复。", maxIterations: 10, }); expect(result.status).toBe("completed"); // The file should actually exist on disk (the agent called write_file). const { readFile } = await import("node:fs/promises"); const note = await readFile(join(workspace, "note.txt"), "utf8"); expect(note).toContain("测试笔记"); } finally { await teardownWorkspace(); } }, 60_000); });