forked from EduCraft/curriculum-project-hub
feat(hub): concurrent multi-PDF convert_pdf_to_md + readable skills (v0.0.40)
Teachers convert many PDFs in one tool call with bounded Docmind concurrency. Each item keeps its own output_dir/document.md and UsageFact; failures are per-file. Mirror role skills to .cph/runtime-skills and CPH_RUNTIME_SKILLS_DIR so agents can Read SKILL.md instead of dead .claude/sandbox stubs.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { chmod, lstat, mkdir, realpath } from "node:fs/promises";
|
||||
import { chmod, cp, lstat, mkdir, readdir, realpath, rm } from "node:fs/promises";
|
||||
import { homedir } from "node:os";
|
||||
import { isAbsolute, join, relative, resolve } from "node:path";
|
||||
import type { RoleSkillEntry } from "./models.js";
|
||||
@@ -142,6 +142,25 @@ export async function createAgentSecurityPolicy(input: AgentSecurityInput): Prom
|
||||
runId: input.runId,
|
||||
skills: selectedSkills,
|
||||
});
|
||||
// Mirror selected skills under the workspace so the agent can Read SKILL.md
|
||||
// without guessing the opaque host plugin UUID path. Workspace `.claude/` and
|
||||
// `.mcp.json` are Claude sandbox stubs — not skill/MCP source of truth.
|
||||
const runtimeSkillsRel = join(".cph", "runtime-skills");
|
||||
const runtimeSkillsAbs = join(workspaceDir, runtimeSkillsRel);
|
||||
await rm(runtimeSkillsAbs, { recursive: true, force: true });
|
||||
await mkdir(runtimeSkillsAbs, { recursive: true, mode: 0o700 });
|
||||
if (skillPlugin !== null) {
|
||||
const pluginSkillsRoot = join(skillPlugin.root, "skills");
|
||||
const skillNames = await readdir(pluginSkillsRoot);
|
||||
for (const skillName of skillNames) {
|
||||
await cp(join(pluginSkillsRoot, skillName), join(runtimeSkillsAbs, skillName), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
env.CPH_RUNTIME_SKILLS_DIR = runtimeSkillsAbs;
|
||||
env.CPH_RUNTIME_SKILLS_REL = runtimeSkillsRel;
|
||||
}
|
||||
return {
|
||||
cwd: workspaceDir,
|
||||
workspaceRoot,
|
||||
|
||||
@@ -63,6 +63,135 @@ export interface PdfToMdBundleDeps {
|
||||
readonly prisma: PrismaClient;
|
||||
}
|
||||
|
||||
/** Default max concurrent Docmind jobs for one convert_pdf_to_md batch call. */
|
||||
export const DEFAULT_PDF_TO_MD_CONCURRENCY = 3;
|
||||
/** Hard ceiling for agent-requested concurrency (also clamps env). */
|
||||
export const MAX_PDF_TO_MD_CONCURRENCY = 8;
|
||||
/** Max PDFs accepted in one batch tool call. */
|
||||
export const MAX_PDF_TO_MD_BATCH_ITEMS = 32;
|
||||
|
||||
export function clampPdfToMdConcurrency(value: number): number {
|
||||
if (!Number.isFinite(value)) return DEFAULT_PDF_TO_MD_CONCURRENCY;
|
||||
const n = Math.trunc(value);
|
||||
if (n < 1) return 1;
|
||||
if (n > MAX_PDF_TO_MD_CONCURRENCY) return MAX_PDF_TO_MD_CONCURRENCY;
|
||||
return n;
|
||||
}
|
||||
|
||||
/** Read HUB_PDF_TO_MD_MAX_CONCURRENT (default 3, max 8). */
|
||||
export function readPdfToMdConcurrency(
|
||||
env: Readonly<Record<string, string | undefined>> = process.env,
|
||||
): number {
|
||||
const raw = env["HUB_PDF_TO_MD_MAX_CONCURRENT"]?.trim();
|
||||
if (raw === undefined || raw === "") return DEFAULT_PDF_TO_MD_CONCURRENCY;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
throw new Error(`HUB_PDF_TO_MD_MAX_CONCURRENT must be a positive integer, got ${raw}`);
|
||||
}
|
||||
return clampPdfToMdConcurrency(parsed);
|
||||
}
|
||||
|
||||
export interface PdfToMdBatchItem {
|
||||
readonly inputPath: string;
|
||||
readonly outputDir: string;
|
||||
}
|
||||
|
||||
export type PdfToMdBatchItemResult =
|
||||
| {
|
||||
readonly ok: true;
|
||||
readonly inputPath: string;
|
||||
readonly outputDir: string;
|
||||
readonly result: CapabilityInvocationResult;
|
||||
}
|
||||
| {
|
||||
readonly ok: false;
|
||||
readonly inputPath: string;
|
||||
readonly outputDir: string;
|
||||
readonly error: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Run worker over items with bounded parallelism. Order of results matches
|
||||
* input order. Rejects in worker are not swallowed — caller should catch.
|
||||
*/
|
||||
export async function mapPool<T, R>(
|
||||
items: readonly T[],
|
||||
concurrency: number,
|
||||
worker: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
if (items.length === 0) return [];
|
||||
const limit = Math.max(1, Math.min(Math.trunc(concurrency), items.length));
|
||||
const results = new Array<R>(items.length);
|
||||
let next = 0;
|
||||
async function runWorker(): Promise<void> {
|
||||
for (;;) {
|
||||
const index = next;
|
||||
next += 1;
|
||||
if (index >= items.length) return;
|
||||
results[index] = await worker(items[index]!, index);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: limit }, () => runWorker()));
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert multiple PDFs with bounded concurrency. Each item is attribute-
|
||||
* independent (own paths + own UsageFact). Failures are per-item and do not
|
||||
* cancel siblings; order matches `items`.
|
||||
*/
|
||||
export async function invokePdfToMdBatch(
|
||||
adapter: CapabilityAdapter,
|
||||
base: Omit<CapabilityInvocationInput, "inputPath" | "outputDir">,
|
||||
items: readonly PdfToMdBatchItem[],
|
||||
concurrency: number = DEFAULT_PDF_TO_MD_CONCURRENCY,
|
||||
): Promise<PdfToMdBatchItemResult[]> {
|
||||
if (items.length === 0) {
|
||||
throw new Error("pdf_to_md batch requires at least one item");
|
||||
}
|
||||
if (items.length > MAX_PDF_TO_MD_BATCH_ITEMS) {
|
||||
throw new Error(
|
||||
`pdf_to_md batch supports at most ${MAX_PDF_TO_MD_BATCH_ITEMS} items per call (got ${items.length})`,
|
||||
);
|
||||
}
|
||||
const seenOutputDirs = new Set<string>();
|
||||
for (const item of items) {
|
||||
if (item.inputPath.trim() === "" || item.outputDir.trim() === "") {
|
||||
throw new Error("pdf_to_md batch items require non-empty inputPath and outputDir");
|
||||
}
|
||||
const key = item.outputDir.replace(/\\/g, "/").replace(/\/+$/, "");
|
||||
if (seenOutputDirs.has(key)) {
|
||||
throw new Error(
|
||||
`pdf_to_md batch items must use distinct output_dir values; duplicate: ${item.outputDir}`,
|
||||
);
|
||||
}
|
||||
seenOutputDirs.add(key);
|
||||
}
|
||||
const limit = clampPdfToMdConcurrency(concurrency);
|
||||
return mapPool(items, limit, async (item) => {
|
||||
try {
|
||||
const result = await adapter.invoke({
|
||||
...base,
|
||||
inputPath: item.inputPath,
|
||||
outputDir: item.outputDir,
|
||||
});
|
||||
return {
|
||||
ok: true as const,
|
||||
inputPath: item.inputPath,
|
||||
outputDir: item.outputDir,
|
||||
result,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false as const,
|
||||
inputPath: item.inputPath,
|
||||
outputDir: item.outputDir,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Build the pdf_to_md_bundle adapter. The client is injectable for testing. */
|
||||
export function createPdfToMdBundleAdapter(deps: PdfToMdBundleDeps): CapabilityAdapter {
|
||||
return {
|
||||
|
||||
@@ -9,7 +9,13 @@ import { CPH_HUB_MCP_TOOL_IDS, type CphHubMcpToolId } from "../agent/roleTools.j
|
||||
import { WorkspaceFileBoundaryError } from "../security/workspaceFiles.js";
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import type { LocalSecretEnvelope } from "../security/secretEnvelope.js";
|
||||
import { createPdfToMdBundleAdapter } from "../capability/pdfToMdBundle.js";
|
||||
import {
|
||||
createPdfToMdBundleAdapter,
|
||||
invokePdfToMdBatch,
|
||||
readPdfToMdConcurrency,
|
||||
MAX_PDF_TO_MD_BATCH_ITEMS,
|
||||
type PdfToMdBatchItemResult,
|
||||
} from "../capability/pdfToMdBundle.js";
|
||||
import { AliyunDocmindClient } from "../capability/docmindClient.js";
|
||||
|
||||
export interface FileDeliveryToolOptions {
|
||||
@@ -246,21 +252,54 @@ export function createFileDeliveryMcpServer(options: FileDeliveryToolOptions): M
|
||||
tools.push(
|
||||
tool(
|
||||
"convert_pdf_to_md",
|
||||
"Convert a PDF file in the workspace to a Markdown bundle (markdown + extracted images) using Alibaba Cloud Document Mind. The PDF must already be in the workspace (use feishu_download_resource first if it came from Feishu). Returns the path to the generated markdown file and the list of extracted image paths. Mathematical formulas are converted to LaTeX.",
|
||||
"Convert one or more PDF files in the workspace to Markdown bundles (markdown + extracted images) via Alibaba Cloud Document Mind. PDFs must already be in the workspace (use feishu_download_resource first for Feishu attachments). Prefer a single call with `items` for multiple PDFs — Hub converts them concurrently (bounded). Each item needs its own output_dir because the tool always writes document.md inside that directory. Formulas become LaTeX.",
|
||||
{
|
||||
input_path: z.string().describe("Relative path to the input PDF within the workspace."),
|
||||
output_dir: z.string().describe("Relative directory within the workspace to write the markdown and images into. Will be created if it does not exist."),
|
||||
input_path: z.string().optional().describe("Single-file mode: workspace-relative path to the input PDF. Required when `items` is omitted."),
|
||||
output_dir: z.string().optional().describe("Single-file mode: workspace-relative directory for document.md + images. Required when `items` is omitted."),
|
||||
items: z.array(z.object({
|
||||
input_path: z.string().describe("Workspace-relative path to one input PDF."),
|
||||
output_dir: z.string().describe("Workspace-relative output directory for this PDF (must be unique per item)."),
|
||||
})).min(1).max(MAX_PDF_TO_MD_BATCH_ITEMS).optional().describe(`Batch mode: multiple PDFs converted concurrently. Max ${MAX_PDF_TO_MD_BATCH_ITEMS} items. Do not reuse output_dir across items.`),
|
||||
concurrency: z.number().int().min(1).max(8).optional().describe("Optional parallel job limit for batch mode (1-8). Defaults to HUB_PDF_TO_MD_MAX_CONCURRENT (usually 3)."),
|
||||
},
|
||||
async (args) => {
|
||||
const base = {
|
||||
runId: options.runId,
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
workspaceDir: options.workspaceDir,
|
||||
prisma: options.prisma,
|
||||
};
|
||||
try {
|
||||
if (args.items !== undefined && args.items.length > 0) {
|
||||
const batchResults = await invokePdfToMdBatch(
|
||||
adapter,
|
||||
base,
|
||||
args.items.map((item) => ({
|
||||
inputPath: item.input_path,
|
||||
outputDir: item.output_dir,
|
||||
})),
|
||||
args.concurrency ?? readPdfToMdConcurrency(),
|
||||
);
|
||||
return {
|
||||
content: [{ type: "text", text: formatPdfToMdBatchResult(batchResults) }],
|
||||
...(batchResults.every((item) => item.ok) ? {} : { isError: true }),
|
||||
};
|
||||
}
|
||||
if (args.input_path === undefined || args.input_path.trim() === ""
|
||||
|| args.output_dir === undefined || args.output_dir.trim() === "") {
|
||||
return {
|
||||
isError: true,
|
||||
content: [{
|
||||
type: "text",
|
||||
text: "convert_pdf_to_md requires either items[{input_path,output_dir},...] or both input_path and output_dir.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
const result = await adapter.invoke({
|
||||
runId: options.runId,
|
||||
organizationId: options.organizationId,
|
||||
projectId: options.projectId,
|
||||
workspaceDir: options.workspaceDir,
|
||||
...base,
|
||||
inputPath: args.input_path,
|
||||
outputDir: args.output_dir,
|
||||
prisma: options.prisma,
|
||||
});
|
||||
const lines = [`Converted PDF to markdown. ${result.artifacts.length} files written:`];
|
||||
for (const artifact of result.artifacts) {
|
||||
@@ -292,6 +331,32 @@ export function createFileDeliveryMcpServer(options: FileDeliveryToolOptions): M
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
function formatPdfToMdBatchResult(results: readonly PdfToMdBatchItemResult[]): string {
|
||||
const ok = results.filter((item) => item.ok);
|
||||
const failed = results.filter((item) => !item.ok);
|
||||
const lines = [
|
||||
`Batch PDF→Markdown finished: ${ok.length} succeeded, ${failed.length} failed (of ${results.length}).`,
|
||||
];
|
||||
for (const item of results) {
|
||||
if (item.ok) {
|
||||
const md = item.result.artifacts.find((artifact) => artifact.kind === "markdown")?.path;
|
||||
lines.push(
|
||||
`OK ${item.inputPath} → ${item.outputDir}`
|
||||
+ (md !== undefined ? ` (${md})` : "")
|
||||
+ `; pages=${item.result.consumption.quantity}`
|
||||
+ `; cost=$${(item.result.consumption.costUsd ?? 0).toFixed(4)}`,
|
||||
);
|
||||
for (const artifact of item.result.artifacts) {
|
||||
lines.push(` - ${artifact.path} (${artifact.kind})`);
|
||||
}
|
||||
} else {
|
||||
lines.push(`FAIL ${item.inputPath} → ${item.outputDir}: ${item.error}`);
|
||||
}
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function mcpInstructions(enabledTools: ReadonlySet<CphHubMcpToolId>): string {
|
||||
const instructions: string[] = [];
|
||||
if (enabledTools.has("send_file")) {
|
||||
@@ -315,10 +380,14 @@ function mcpInstructions(enabledTools: ReadonlySet<CphHubMcpToolId>): string {
|
||||
}
|
||||
if (enabledTools.has("convert_pdf_to_md")) {
|
||||
instructions.push(
|
||||
"Use convert_pdf_to_md when the user asks to convert a PDF to Markdown.",
|
||||
"If the PDF came from a Feishu message, first use feishu_download_resource to save it to the workspace, then call convert_pdf_to_md.",
|
||||
"Do NOT attempt to parse PDFs yourself with Read or Bash — always use convert_pdf_to_md for accurate text, formula, and image extraction.",
|
||||
"Use convert_pdf_to_md when the user asks to convert a PDF (or several PDFs) to Markdown.",
|
||||
"If PDFs came from Feishu, download each with feishu_download_resource first, then convert.",
|
||||
"For multiple PDFs, call convert_pdf_to_md once with items=[{input_path,output_dir},...] so Hub converts them concurrently; give each file its own output_dir (the tool writes document.md inside it).",
|
||||
"Do NOT attempt to parse PDFs yourself with Read or Bash — always use convert_pdf_to_md.",
|
||||
);
|
||||
}
|
||||
instructions.push(
|
||||
"Role skill docs (when bound) are readable at .cph/runtime-skills/<skill-name>/SKILL.md or $CPH_RUNTIME_SKILLS_DIR/<skill-name>/SKILL.md. Prefer the Skill tool when available. Workspace .claude/ and .mcp.json are sandbox stubs — not skill or MCP source.",
|
||||
);
|
||||
return instructions.join(" ");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user