feat(hub): concurrent multi-PDF convert_pdf_to_md + readable skills (v0.0.40) (#20)

This commit is contained in:
2026-07-21 13:56:10 +08:00
10 changed files with 475 additions and 36 deletions
+2
View File
@@ -26,6 +26,8 @@ HUB_AGENT_MAX_RUN_SECONDS="1800"
HUB_HTTP_BODY_LIMIT_BYTES="1048576" HUB_HTTP_BODY_LIMIT_BYTES="1048576"
HUB_MAX_FILES_PER_MESSAGE="20" HUB_MAX_FILES_PER_MESSAGE="20"
HUB_MAX_FILE_BYTES="26214400" HUB_MAX_FILE_BYTES="26214400"
# Max concurrent Alibaba Docmind jobs for one convert_pdf_to_md batch (1-8).
HUB_PDF_TO_MD_MAX_CONCURRENT="3"
HUB_HTTP_REQUESTS_PER_MINUTE="120" HUB_HTTP_REQUESTS_PER_MINUTE="120"
HUB_FEISHU_EVENTS_PER_MINUTE="120" HUB_FEISHU_EVENTS_PER_MINUTE="120"
+1
View File
@@ -175,6 +175,7 @@ HUB_AGENT_MAX_RUN_SECONDS=
HUB_HTTP_BODY_LIMIT_BYTES= HUB_HTTP_BODY_LIMIT_BYTES=
HUB_MAX_FILES_PER_MESSAGE= HUB_MAX_FILES_PER_MESSAGE=
HUB_MAX_FILE_BYTES= HUB_MAX_FILE_BYTES=
HUB_PDF_TO_MD_MAX_CONCURRENT=3
HUB_HTTP_REQUESTS_PER_MINUTE= HUB_HTTP_REQUESTS_PER_MINUTE=
HUB_FEISHU_EVENTS_PER_MINUTE= HUB_FEISHU_EVENTS_PER_MINUTE=
HUB_FEISHU_LISTENER_ENABLED=true HUB_FEISHU_LISTENER_ENABLED=true
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "@paradigm/hub", "name": "@paradigm/hub",
"version": "0.0.39", "version": "0.0.40",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@paradigm/hub", "name": "@paradigm/hub",
"version": "0.0.39", "version": "0.0.40",
"dependencies": { "dependencies": {
"@alicloud/credentials": "^2.4.5", "@alicloud/credentials": "^2.4.5",
"@alicloud/docmind-api20220711": "^1.4.15", "@alicloud/docmind-api20220711": "^1.4.15",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@paradigm/hub", "name": "@paradigm/hub",
"version": "0.0.39", "version": "0.0.40",
"private": true, "private": true,
"type": "module", "type": "module",
"engines": { "engines": {
+52 -19
View File
@@ -2,62 +2,95 @@
name: pdf-to-md name: pdf-to-md
description: > description: >
Convert PDF documents to Markdown bundles using the convert_pdf_to_md tool. Convert PDF documents to Markdown bundles using the convert_pdf_to_md tool.
Handles PDFs from Feishu messages, local workspace files, and produces Handles single or multiple PDFs (concurrent batch), Feishu attachments, and
high-quality Markdown with LaTeX formulas and extracted images. local workspace files. Produces high-quality Markdown with LaTeX formulas
and extracted images.
--- ---
# PDF to Markdown Conversion # PDF to Markdown Conversion
## When to use ## When to use
Use this skill when the user asks to convert a PDF to Markdown, extract text Use this skill when the user asks to convert a PDF (or several PDFs) to
from a PDF, or turn a PDF document into an editable format. Markdown, extract text from a PDF, or turn PDF documents into an editable
format.
## How it works ## How it works
The `convert_pdf_to_md` tool (provided by the `cph_hub` MCP server) calls The `convert_pdf_to_md` tool (provided by the in-process `cph_hub` MCP server)
Alibaba Cloud Document Mind to parse the PDF. It: calls Alibaba Cloud Document Mind to parse each PDF. It:
- Extracts text in reading order (handles multi-column, scanned, and - Extracts text in reading order (handles multi-column, scanned, and
multi-language documents) multi-language documents)
- Converts mathematical formulas to **LaTeX** (`$...$` inline, `$$...$$` block) - Converts mathematical formulas to **LaTeX** (`$...$` inline, `$$...$$` block)
- Extracts tables as Markdown tables - Extracts tables as Markdown tables
- Downloads embedded images into the output directory - Downloads embedded images into the output directory
- Writes a single `document.md` file plus image files - Writes a single `document.md` file plus image files **per** `output_dir`
There is **no** workspace `.mcp.json` source file. MCP tools are injected by
Hub at run start. Do not look for MCP or skill source under workspace
`.claude/` — those paths are sandbox stubs (often character devices) and are
not readable constitution.
## Where this skill text lives
Prefer the Skill tool when the runtime offers it. If you need to re-read these
instructions with Read:
- Workspace copy (always under cwd): `.cph/runtime-skills/pdf-to-md/SKILL.md`
- Absolute path env: `$CPH_RUNTIME_SKILLS_DIR/pdf-to-md/SKILL.md`
## Workflow ## Workflow
### PDF from a Feishu message ### One PDF from a Feishu message
1. Use `feishu_read_context` to find the `file_key` of the PDF attachment. 1. Use `feishu_read_context` to find the `file_key` of the PDF attachment.
2. Use `feishu_download_resource` to download it into the workspace. 2. Use `feishu_download_resource` to download it into the workspace.
3. Use `convert_pdf_to_md` with the downloaded file path and an output directory. 3. Use `convert_pdf_to_md` with `input_path` + `output_dir`.
### PDF already in the workspace ### PDF already in the workspace
1. Use `convert_pdf_to_md` directly with the file path and an output directory. 1. Use `convert_pdf_to_md` with `input_path` and `output_dir`.
### Multiple PDFs (concurrent)
1. Download or locate every PDF in the workspace first.
2. Call **`convert_pdf_to_md` once** with:
```json
{
"items": [
{ "input_path": "sources/a.pdf", "output_dir": "md/a" },
{ "input_path": "sources/b.pdf", "output_dir": "md/b" }
]
}
```
3. Hub submits Docmind jobs with bounded concurrency (default 3, max 8;
optional `concurrency` argument). Prefer this over N sequential tool calls.
4. **Each item must use a distinct `output_dir`** — the tool always writes
`document.md` inside that directory; shared dirs overwrite each other.
5. Partial failure returns per-file OK/FAIL lines; re-run only failed items.
## Important rules ## Important rules
- **Always** use `convert_pdf_to_md` for PDF→Markdown. Do NOT attempt to parse - **Always** use `convert_pdf_to_md` for PDF→Markdown. Do NOT attempt to parse
PDFs yourself with Read, Bash, Python, or any other method. The tool provides PDFs yourself with Read, Bash, Python, or any other method.
accurate formula, table, and image extraction that manual methods cannot
match.
- If `convert_pdf_to_md` fails because no capability connection is configured, - If `convert_pdf_to_md` fails because no capability connection is configured,
tell the user to ask their organization admin to configure the Aliyun tell the user to ask their organization admin to configure the Aliyun
docmind credential in the admin web UI (组织后台 → 能力). docmind credential in the admin web UI (组织后台 → 能力).
- The output directory will be created if it does not exist. - The output directory will be created if it does not exist.
- After conversion, use `send_file` to send the generated markdown back to the - After conversion, use `send_file` to send generated markdown (or a zip you
user if they requested it. assemble) back to the user if they requested delivery.
## Output ## Output
The tool returns a list of generated files: Per `output_dir`:
- `document.md` — the main markdown file - `document.md` — the main markdown file
- `*.jpg` / `*.png` — extracted images, referenced from the markdown - `*.jpg` / `*.png` — extracted images, referenced from the markdown
## Cost ## Cost
The conversion is billed per page (0.04 CNY/page ≈ $0.0056/page for the Billed per page (0.04 CNY/page ≈ $0.0056/page for enhanced formula mode).
enhanced formula mode). The cost is automatically recorded on the run's Each successful file records its own usage fact on the run ledger.
usage ledger.
+20 -1
View File
@@ -1,4 +1,4 @@
import { chmod, lstat, mkdir, realpath } from "node:fs/promises"; import { chmod, cp, lstat, mkdir, readdir, realpath, rm } from "node:fs/promises";
import { homedir } from "node:os"; import { homedir } from "node:os";
import { isAbsolute, join, relative, resolve } from "node:path"; import { isAbsolute, join, relative, resolve } from "node:path";
import type { RoleSkillEntry } from "./models.js"; import type { RoleSkillEntry } from "./models.js";
@@ -142,6 +142,25 @@ export async function createAgentSecurityPolicy(input: AgentSecurityInput): Prom
runId: input.runId, runId: input.runId,
skills: selectedSkills, skills: selectedSkills,
}); });
// Mirror selected skills under the workspace so the agent can Read SKILL.md
// without guessing the opaque host plugin UUID path. Workspace `.claude/` and
// `.mcp.json` are Claude sandbox stubs — not skill/MCP source of truth.
const runtimeSkillsRel = join(".cph", "runtime-skills");
const runtimeSkillsAbs = join(workspaceDir, runtimeSkillsRel);
await rm(runtimeSkillsAbs, { recursive: true, force: true });
await mkdir(runtimeSkillsAbs, { recursive: true, mode: 0o700 });
if (skillPlugin !== null) {
const pluginSkillsRoot = join(skillPlugin.root, "skills");
const skillNames = await readdir(pluginSkillsRoot);
for (const skillName of skillNames) {
await cp(join(pluginSkillsRoot, skillName), join(runtimeSkillsAbs, skillName), {
recursive: true,
force: true,
});
}
env.CPH_RUNTIME_SKILLS_DIR = runtimeSkillsAbs;
env.CPH_RUNTIME_SKILLS_REL = runtimeSkillsRel;
}
return { return {
cwd: workspaceDir, cwd: workspaceDir,
workspaceRoot, workspaceRoot,
+129
View File
@@ -63,6 +63,135 @@ export interface PdfToMdBundleDeps {
readonly prisma: PrismaClient; readonly prisma: PrismaClient;
} }
/** Default max concurrent Docmind jobs for one convert_pdf_to_md batch call. */
export const DEFAULT_PDF_TO_MD_CONCURRENCY = 3;
/** Hard ceiling for agent-requested concurrency (also clamps env). */
export const MAX_PDF_TO_MD_CONCURRENCY = 8;
/** Max PDFs accepted in one batch tool call. */
export const MAX_PDF_TO_MD_BATCH_ITEMS = 32;
export function clampPdfToMdConcurrency(value: number): number {
if (!Number.isFinite(value)) return DEFAULT_PDF_TO_MD_CONCURRENCY;
const n = Math.trunc(value);
if (n < 1) return 1;
if (n > MAX_PDF_TO_MD_CONCURRENCY) return MAX_PDF_TO_MD_CONCURRENCY;
return n;
}
/** Read HUB_PDF_TO_MD_MAX_CONCURRENT (default 3, max 8). */
export function readPdfToMdConcurrency(
env: Readonly<Record<string, string | undefined>> = process.env,
): number {
const raw = env["HUB_PDF_TO_MD_MAX_CONCURRENT"]?.trim();
if (raw === undefined || raw === "") return DEFAULT_PDF_TO_MD_CONCURRENCY;
const parsed = Number(raw);
if (!Number.isFinite(parsed) || parsed < 1) {
throw new Error(`HUB_PDF_TO_MD_MAX_CONCURRENT must be a positive integer, got ${raw}`);
}
return clampPdfToMdConcurrency(parsed);
}
export interface PdfToMdBatchItem {
readonly inputPath: string;
readonly outputDir: string;
}
export type PdfToMdBatchItemResult =
| {
readonly ok: true;
readonly inputPath: string;
readonly outputDir: string;
readonly result: CapabilityInvocationResult;
}
| {
readonly ok: false;
readonly inputPath: string;
readonly outputDir: string;
readonly error: string;
};
/**
* Run worker over items with bounded parallelism. Order of results matches
* input order. Rejects in worker are not swallowed — caller should catch.
*/
export async function mapPool<T, R>(
items: readonly T[],
concurrency: number,
worker: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
if (items.length === 0) return [];
const limit = Math.max(1, Math.min(Math.trunc(concurrency), items.length));
const results = new Array<R>(items.length);
let next = 0;
async function runWorker(): Promise<void> {
for (;;) {
const index = next;
next += 1;
if (index >= items.length) return;
results[index] = await worker(items[index]!, index);
}
}
await Promise.all(Array.from({ length: limit }, () => runWorker()));
return results;
}
/**
* Convert multiple PDFs with bounded concurrency. Each item is attribute-
* independent (own paths + own UsageFact). Failures are per-item and do not
* cancel siblings; order matches `items`.
*/
export async function invokePdfToMdBatch(
adapter: CapabilityAdapter,
base: Omit<CapabilityInvocationInput, "inputPath" | "outputDir">,
items: readonly PdfToMdBatchItem[],
concurrency: number = DEFAULT_PDF_TO_MD_CONCURRENCY,
): Promise<PdfToMdBatchItemResult[]> {
if (items.length === 0) {
throw new Error("pdf_to_md batch requires at least one item");
}
if (items.length > MAX_PDF_TO_MD_BATCH_ITEMS) {
throw new Error(
`pdf_to_md batch supports at most ${MAX_PDF_TO_MD_BATCH_ITEMS} items per call (got ${items.length})`,
);
}
const seenOutputDirs = new Set<string>();
for (const item of items) {
if (item.inputPath.trim() === "" || item.outputDir.trim() === "") {
throw new Error("pdf_to_md batch items require non-empty inputPath and outputDir");
}
const key = item.outputDir.replace(/\\/g, "/").replace(/\/+$/, "");
if (seenOutputDirs.has(key)) {
throw new Error(
`pdf_to_md batch items must use distinct output_dir values; duplicate: ${item.outputDir}`,
);
}
seenOutputDirs.add(key);
}
const limit = clampPdfToMdConcurrency(concurrency);
return mapPool(items, limit, async (item) => {
try {
const result = await adapter.invoke({
...base,
inputPath: item.inputPath,
outputDir: item.outputDir,
});
return {
ok: true as const,
inputPath: item.inputPath,
outputDir: item.outputDir,
result,
};
} catch (error) {
return {
ok: false as const,
inputPath: item.inputPath,
outputDir: item.outputDir,
error: error instanceof Error ? error.message : String(error),
};
}
});
}
/** Build the pdf_to_md_bundle adapter. The client is injectable for testing. */ /** Build the pdf_to_md_bundle adapter. The client is injectable for testing. */
export function createPdfToMdBundleAdapter(deps: PdfToMdBundleDeps): CapabilityAdapter { export function createPdfToMdBundleAdapter(deps: PdfToMdBundleDeps): CapabilityAdapter {
return { return {
+81 -12
View File
@@ -9,7 +9,13 @@ import { CPH_HUB_MCP_TOOL_IDS, type CphHubMcpToolId } from "../agent/roleTools.j
import { WorkspaceFileBoundaryError } from "../security/workspaceFiles.js"; import { WorkspaceFileBoundaryError } from "../security/workspaceFiles.js";
import type { PrismaClient } from "@prisma/client"; import type { PrismaClient } from "@prisma/client";
import type { LocalSecretEnvelope } from "../security/secretEnvelope.js"; import type { LocalSecretEnvelope } from "../security/secretEnvelope.js";
import { createPdfToMdBundleAdapter } from "../capability/pdfToMdBundle.js"; import {
createPdfToMdBundleAdapter,
invokePdfToMdBatch,
readPdfToMdConcurrency,
MAX_PDF_TO_MD_BATCH_ITEMS,
type PdfToMdBatchItemResult,
} from "../capability/pdfToMdBundle.js";
import { AliyunDocmindClient } from "../capability/docmindClient.js"; import { AliyunDocmindClient } from "../capability/docmindClient.js";
export interface FileDeliveryToolOptions { export interface FileDeliveryToolOptions {
@@ -246,21 +252,54 @@ export function createFileDeliveryMcpServer(options: FileDeliveryToolOptions): M
tools.push( tools.push(
tool( tool(
"convert_pdf_to_md", "convert_pdf_to_md",
"Convert a PDF file in the workspace to a Markdown bundle (markdown + extracted images) using Alibaba Cloud Document Mind. The PDF must already be in the workspace (use feishu_download_resource first if it came from Feishu). Returns the path to the generated markdown file and the list of extracted image paths. Mathematical formulas are converted to LaTeX.", "Convert one or more PDF files in the workspace to Markdown bundles (markdown + extracted images) via Alibaba Cloud Document Mind. PDFs must already be in the workspace (use feishu_download_resource first for Feishu attachments). Prefer a single call with `items` for multiple PDFs — Hub converts them concurrently (bounded). Each item needs its own output_dir because the tool always writes document.md inside that directory. Formulas become LaTeX.",
{ {
input_path: z.string().describe("Relative path to the input PDF within the workspace."), input_path: z.string().optional().describe("Single-file mode: workspace-relative path to the input PDF. Required when `items` is omitted."),
output_dir: z.string().describe("Relative directory within the workspace to write the markdown and images into. Will be created if it does not exist."), output_dir: z.string().optional().describe("Single-file mode: workspace-relative directory for document.md + images. Required when `items` is omitted."),
items: z.array(z.object({
input_path: z.string().describe("Workspace-relative path to one input PDF."),
output_dir: z.string().describe("Workspace-relative output directory for this PDF (must be unique per item)."),
})).min(1).max(MAX_PDF_TO_MD_BATCH_ITEMS).optional().describe(`Batch mode: multiple PDFs converted concurrently. Max ${MAX_PDF_TO_MD_BATCH_ITEMS} items. Do not reuse output_dir across items.`),
concurrency: z.number().int().min(1).max(8).optional().describe("Optional parallel job limit for batch mode (1-8). Defaults to HUB_PDF_TO_MD_MAX_CONCURRENT (usually 3)."),
}, },
async (args) => { async (args) => {
const base = {
runId: options.runId,
organizationId: options.organizationId,
projectId: options.projectId,
workspaceDir: options.workspaceDir,
prisma: options.prisma,
};
try { try {
if (args.items !== undefined && args.items.length > 0) {
const batchResults = await invokePdfToMdBatch(
adapter,
base,
args.items.map((item) => ({
inputPath: item.input_path,
outputDir: item.output_dir,
})),
args.concurrency ?? readPdfToMdConcurrency(),
);
return {
content: [{ type: "text", text: formatPdfToMdBatchResult(batchResults) }],
...(batchResults.every((item) => item.ok) ? {} : { isError: true }),
};
}
if (args.input_path === undefined || args.input_path.trim() === ""
|| args.output_dir === undefined || args.output_dir.trim() === "") {
return {
isError: true,
content: [{
type: "text",
text: "convert_pdf_to_md requires either items[{input_path,output_dir},...] or both input_path and output_dir.",
}],
};
}
const result = await adapter.invoke({ const result = await adapter.invoke({
runId: options.runId, ...base,
organizationId: options.organizationId,
projectId: options.projectId,
workspaceDir: options.workspaceDir,
inputPath: args.input_path, inputPath: args.input_path,
outputDir: args.output_dir, outputDir: args.output_dir,
prisma: options.prisma,
}); });
const lines = [`Converted PDF to markdown. ${result.artifacts.length} files written:`]; const lines = [`Converted PDF to markdown. ${result.artifacts.length} files written:`];
for (const artifact of result.artifacts) { for (const artifact of result.artifacts) {
@@ -292,6 +331,32 @@ export function createFileDeliveryMcpServer(options: FileDeliveryToolOptions): M
}); });
} }
function formatPdfToMdBatchResult(results: readonly PdfToMdBatchItemResult[]): string {
const ok = results.filter((item) => item.ok);
const failed = results.filter((item) => !item.ok);
const lines = [
`Batch PDF→Markdown finished: ${ok.length} succeeded, ${failed.length} failed (of ${results.length}).`,
];
for (const item of results) {
if (item.ok) {
const md = item.result.artifacts.find((artifact) => artifact.kind === "markdown")?.path;
lines.push(
`OK ${item.inputPath}${item.outputDir}`
+ (md !== undefined ? ` (${md})` : "")
+ `; pages=${item.result.consumption.quantity}`
+ `; cost=$${(item.result.consumption.costUsd ?? 0).toFixed(4)}`,
);
for (const artifact of item.result.artifacts) {
lines.push(` - ${artifact.path} (${artifact.kind})`);
}
} else {
lines.push(`FAIL ${item.inputPath}${item.outputDir}: ${item.error}`);
}
}
return lines.join("\n");
}
function mcpInstructions(enabledTools: ReadonlySet<CphHubMcpToolId>): string { function mcpInstructions(enabledTools: ReadonlySet<CphHubMcpToolId>): string {
const instructions: string[] = []; const instructions: string[] = [];
if (enabledTools.has("send_file")) { if (enabledTools.has("send_file")) {
@@ -315,10 +380,14 @@ function mcpInstructions(enabledTools: ReadonlySet<CphHubMcpToolId>): string {
} }
if (enabledTools.has("convert_pdf_to_md")) { if (enabledTools.has("convert_pdf_to_md")) {
instructions.push( instructions.push(
"Use convert_pdf_to_md when the user asks to convert a PDF to Markdown.", "Use convert_pdf_to_md when the user asks to convert a PDF (or several PDFs) to Markdown.",
"If the PDF came from a Feishu message, first use feishu_download_resource to save it to the workspace, then call convert_pdf_to_md.", "If PDFs came from Feishu, download each with feishu_download_resource first, then convert.",
"Do NOT attempt to parse PDFs yourself with Read or Bash — always use convert_pdf_to_md for accurate text, formula, and image extraction.", "For multiple PDFs, call convert_pdf_to_md once with items=[{input_path,output_dir},...] so Hub converts them concurrently; give each file its own output_dir (the tool writes document.md inside it).",
"Do NOT attempt to parse PDFs yourself with Read or Bash — always use convert_pdf_to_md.",
); );
} }
instructions.push(
"Role skill docs (when bound) are readable at .cph/runtime-skills/<skill-name>/SKILL.md or $CPH_RUNTIME_SKILLS_DIR/<skill-name>/SKILL.md. Prefer the Skill tool when available. Workspace .claude/ and .mcp.json are sandbox stubs — not skill or MCP source.",
);
return instructions.join(" "); return instructions.join(" ");
} }
+39 -1
View File
@@ -1,8 +1,9 @@
import { mkdir, mkdtemp, realpath, rm, symlink } from "node:fs/promises"; import { mkdir, mkdtemp, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest"; import { afterEach, describe, expect, it } from "vitest";
import { createAgentSecurityPolicy } from "../../src/agent/security.js"; import { createAgentSecurityPolicy } from "../../src/agent/security.js";
import { importSkillDirectory } from "../../src/agent/skillStore.js";
describe("agent subprocess security policy", () => { describe("agent subprocess security policy", () => {
const roots: string[] = []; const roots: string[] = [];
@@ -135,6 +136,43 @@ describe("agent subprocess security policy", () => {
})).rejects.toThrow("Agent temp path is too long for sandbox bridge sockets"); })).rejects.toThrow("Agent temp path is too long for sandbox bridge sockets");
}); });
it("mirrors selected skills under .cph/runtime-skills and exposes CPH_RUNTIME_SKILLS_DIR", async () => {
const { root, workspaceRoot, workspace } = await makeWorkspace();
const storeRoot = join(root, "skills-store");
const skillSource = join(root, "skill-src", "pdf-to-md");
await mkdir(skillSource, { recursive: true });
await writeFile(
join(skillSource, "SKILL.md"),
"---\nname: pdf-to-md\ndescription: convert\n---\n# pdf-to-md\nbatch items\n",
);
const installed = await importSkillDirectory({ sourceDir: skillSource, storeRoot });
const policy = await createAgentSecurityPolicy({
runId: "run-skills",
workspaceRoot,
workspaceDir: workspace,
skills: [{
name: "pdf-to-md",
version: "1",
contentDigest: installed.contentDigest,
}],
hostEnv: {
PATH: "/usr/bin:/bin",
HUB_SKILL_STORE_ROOT: storeRoot,
},
});
const canonicalWorkspace = await realpath(workspace);
const mirrored = join(canonicalWorkspace, ".cph", "runtime-skills", "pdf-to-md", "SKILL.md");
await expect(readFile(mirrored, "utf8")).resolves.toContain("batch items");
expect(policy.env.CPH_RUNTIME_SKILLS_DIR).toBe(join(canonicalWorkspace, ".cph", "runtime-skills"));
expect(policy.env.CPH_RUNTIME_SKILLS_REL).toBe(join(".cph", "runtime-skills"));
expect(policy.skillIds).toEqual(["cph-runtime:pdf-to-md"]);
expect(policy.sandbox.filesystem.allowRead).toEqual(
expect.arrayContaining([canonicalWorkspace, policy.skillPluginRoot]),
);
await policy.cleanup();
});
it("rejects a project workspace whose real path escapes the configured workspace root", async () => { it("rejects a project workspace whose real path escapes the configured workspace root", async () => {
const { root, workspaceRoot } = await makeWorkspace(); const { root, workspaceRoot } = await makeWorkspace();
const outside = join(root, "outside"); const outside = join(root, "outside");
+148
View File
@@ -0,0 +1,148 @@
import { describe, expect, it, vi } from "vitest";
import {
clampPdfToMdConcurrency,
invokePdfToMdBatch,
mapPool,
readPdfToMdConcurrency,
} from "../../src/capability/pdfToMdBundle.js";
import type { CapabilityAdapter, CapabilityInvocationResult } from "../../src/capability/types.js";
function okResult(label: string, pages: number): CapabilityInvocationResult {
return {
artifacts: [{ path: `out/${label}/document.md`, kind: "markdown" }],
consumption: {
provider: "aliyun_docmind",
model: null,
inputTokens: null,
outputTokens: null,
quantity: pages,
unit: "pages",
costUsd: pages * 0.0056,
correlationId: `job-${label}`,
},
};
}
describe("pdf_to_md batch concurrency helpers", () => {
it("mapPool caps in-flight workers", async () => {
let inFlight = 0;
let maxInFlight = 0;
const values = await mapPool([1, 2, 3, 4, 5], 2, async (item) => {
inFlight += 1;
maxInFlight = Math.max(maxInFlight, inFlight);
await new Promise((resolve) => setTimeout(resolve, 40));
inFlight -= 1;
return item * 10;
});
expect(values).toEqual([10, 20, 30, 40, 50]);
expect(maxInFlight).toBe(2);
});
it("clamps concurrency and reads env", () => {
expect(clampPdfToMdConcurrency(99)).toBe(8);
expect(clampPdfToMdConcurrency(0)).toBe(1);
expect(readPdfToMdConcurrency({})).toBe(3);
expect(readPdfToMdConcurrency({ HUB_PDF_TO_MD_MAX_CONCURRENT: "5" })).toBe(5);
expect(() => readPdfToMdConcurrency({ HUB_PDF_TO_MD_MAX_CONCURRENT: "nope" })).toThrow(/positive integer/);
});
it("runs batch items concurrently and preserves order", async () => {
let inFlight = 0;
let maxInFlight = 0;
const adapter: CapabilityAdapter = {
capabilityId: "pdf_to_md_bundle",
invoke: vi.fn(async (input) => {
inFlight += 1;
maxInFlight = Math.max(maxInFlight, inFlight);
await new Promise((resolve) => setTimeout(resolve, 60));
inFlight -= 1;
const label = input.inputPath.includes("b") ? "b" : input.inputPath.includes("c") ? "c" : "a";
return okResult(label, label === "a" ? 1 : label === "b" ? 2 : 3);
}),
};
const batch = await invokePdfToMdBatch(
adapter,
{
runId: "run-1",
organizationId: "org-1",
projectId: "proj-1",
workspaceDir: "/tmp/ws",
prisma: {} as never,
},
[
{ inputPath: "a.pdf", outputDir: "out/a" },
{ inputPath: "b.pdf", outputDir: "out/b" },
{ inputPath: "c.pdf", outputDir: "out/c" },
],
3,
);
expect(batch.map((item) => item.ok)).toEqual([true, true, true]);
expect(maxInFlight).toBe(3);
expect(adapter.invoke).toHaveBeenCalledTimes(3);
if (batch[0]?.ok && batch[1]?.ok && batch[2]?.ok) {
expect(batch[0].result.consumption.correlationId).toBe("job-a");
expect(batch[1].result.consumption.correlationId).toBe("job-b");
expect(batch[2].result.consumption.correlationId).toBe("job-c");
}
});
it("isolates per-item failures without canceling siblings", async () => {
const adapter: CapabilityAdapter = {
capabilityId: "pdf_to_md_bundle",
invoke: vi.fn(async (input) => {
if (input.inputPath.includes("bad")) {
throw new Error("boom");
}
return okResult("ok", 1);
}),
};
const batch = await invokePdfToMdBatch(
adapter,
{
runId: "run-1",
organizationId: "org-1",
projectId: "proj-1",
workspaceDir: "/tmp/ws",
prisma: {} as never,
},
[
{ inputPath: "ok.pdf", outputDir: "out/ok" },
{ inputPath: "bad.pdf", outputDir: "out/bad" },
],
2,
);
expect(batch[0]?.ok).toBe(true);
expect(batch[1]?.ok).toBe(false);
if (batch[1]?.ok === false) {
expect(batch[1].error).toMatch(/boom/);
}
});
it("rejects duplicate output dirs before starting work", async () => {
const invoke = vi.fn();
const adapter: CapabilityAdapter = {
capabilityId: "pdf_to_md_bundle",
invoke,
};
await expect(invokePdfToMdBatch(
adapter,
{
runId: "run-1",
organizationId: "org-1",
projectId: "proj-1",
workspaceDir: "/tmp/ws",
prisma: {} as never,
},
[
{ inputPath: "a.pdf", outputDir: "out/same" },
{ inputPath: "b.pdf", outputDir: "out/same/" },
],
2,
)).rejects.toThrow(/distinct output_dir/);
expect(invoke).not.toHaveBeenCalled();
});
});