From 8a81c60ea5cbfa1e526caa3d5eb04566f16fac52 Mon Sep 17 00:00:00 2001 From: Hong Jiarong Date: Thu, 23 Jul 2026 21:05:15 +0800 Subject: [PATCH] fix(hub): extract PBank zips in-process without host unzip pbank materialize previously shelled out to `unzip` and soft-failed when the binary was missing, so agents only saw titles. Read zip entries with Node zlib (store/deflate) and write under workspace .pbank-sources. --- hub/src/capability/pbank.ts | 373 ++++++++++++++------------ hub/test/unit/fixtures/pbank-mini.zip | Bin 0 -> 234 bytes hub/test/unit/pbank-zip.test.ts | 56 ++++ 3 files changed, 260 insertions(+), 169 deletions(-) create mode 100644 hub/test/unit/fixtures/pbank-mini.zip create mode 100644 hub/test/unit/pbank-zip.test.ts diff --git a/hub/src/capability/pbank.ts b/hub/src/capability/pbank.ts index 4b358b1..395e9aa 100644 --- a/hub/src/capability/pbank.ts +++ b/hub/src/capability/pbank.ts @@ -9,12 +9,9 @@ * 3. Mandatory fact — each successful tool call writes ≥1 UsageFact with * kind=external_capability and unit=requests (cost unknown unless reported). */ -import { randomUUID } from "node:crypto"; -import { execFile as execFileCb } from "node:child_process"; -import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { inflateRawSync } from "node:zlib"; +import { mkdir, rm, writeFile } from "node:fs/promises"; import { dirname, join, relative, resolve } from "node:path"; -import { promisify } from "node:util"; import type { PrismaClient } from "@prisma/client"; import { LocalSecretEnvelope } from "../security/secretEnvelope.js"; import { resolveCapabilityCredential } from "./capabilityConnections.js"; @@ -30,8 +27,6 @@ import { type PbankCapabilitySecretPayload, } from "./types.js"; -const execFile = promisify(execFileCb); - export const PBANK_CAPABILITY_ID = "pbank" as const; const PROVIDER_ID = "paradigm_pbank"; const MAX_BATCH_SIZE = 20; @@ -320,163 +315,221 @@ async function readProjectZip( readonly includeAssetImages: boolean; }, ): Promise> { - if (!(await commandExists("unzip"))) { - return { - target: options.target, - status: "downloaded", - note: "Downloaded a zip project, but unzip is not installed on the server.", - bytes: buffer.length, - files: [], - }; - } - - const tempDir = await mkdtemp(join(tmpdir(), "pbank-project-")); + // Pure Node unzip (store/deflate). Do not shell out to host `unzip` — + // silo service PATH/tooling must not gate 题库 materialize. + const entries = listZipEntries(buffer); const cacheRoot = confineToWorkspace(CACHE_DIR_NAME, options.workspaceDir); const cacheDir = join(cacheRoot, safePathSegment(options.id), safePathSegment(options.target)); const extractDir = options.materialize ? join(cacheDir, "source") : null; - const zipPath = options.materialize - ? join(cacheDir, `${options.target}.zip`) - : join(tempDir, `${randomUUID()}.zip`); + const zipPath = options.materialize ? join(cacheDir, `${options.target}.zip`) : null; - try { - if (options.materialize) { - await mkdir(cacheDir, { recursive: true }); - if (extractDir !== null) { - await rm(extractDir, { recursive: true, force: true }); - await mkdir(extractDir, { recursive: true }); - } + if (options.materialize) { + await mkdir(cacheDir, { recursive: true }); + if (extractDir !== null) { + await rm(extractDir, { recursive: true, force: true }); + await mkdir(extractDir, { recursive: true }); } - await writeFile(zipPath, buffer); + if (zipPath !== null) await writeFile(zipPath, buffer); + } - const { stdout } = await execFile("unzip", ["-Z1", zipPath], { - timeout: 10_000, - maxBuffer: 1024 * 1024, - }); - const entryNames = String(stdout) - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line !== ""); + const files: Array> = []; + const assets: Array<{ + path: string; + localPath: string | null; + bytes: number; + mimeType: string; + inlineData?: string; + }> = []; + const extractedFiles: Array> = []; + const omitted: Array> = []; + let usedTextBytes = 0; + let usedExtractedBytes = 0; + let inlineAssetCount = 0; - const files: Array> = []; - const assets: Array<{ - path: string; - localPath: string | null; - bytes: number; - mimeType: string; - inlineData?: string; - }> = []; - const extractedFiles: Array> = []; - const omitted: Array> = []; - let usedTextBytes = 0; - let usedExtractedBytes = 0; - let inlineAssetCount = 0; - - for (const entry of entryNames) { - if (!safeZipPath(entry)) { - omitted.push({ path: entry, reason: "unsafe path" }); - continue; - } - if (entry.endsWith("/")) continue; - if (usedExtractedBytes >= MAX_EXTRACTED_PROJECT_BYTES) { - omitted.push({ path: entry, reason: "extracted byte limit reached" }); - continue; - } - const maxEntryBytes = Math.max( - 1, - Math.min(MAX_PROJECT_BYTES, MAX_EXTRACTED_PROJECT_BYTES - usedExtractedBytes), - ); - try { - const entryBuffer = await readZipEntry(zipPath, entry, maxEntryBytes); - usedExtractedBytes += entryBuffer.length; - let localPath: string | null = null; - if (extractDir !== null) { - localPath = join(extractDir, ...entry.split(/[\\/]+/)); - await mkdir(dirname(localPath), { recursive: true }); - await writeFile(localPath, entryBuffer); - extractedFiles.push({ - path: entry, - localPath: toWorkspaceRelative(options.workspaceDir, localPath), - bytes: entryBuffer.length, - }); - } - - const mimeType = assetMimeType(entry); - if (mimeType !== null) { - const asset: { - path: string; - localPath: string | null; - bytes: number; - mimeType: string; - inlineData?: string; - } = { - path: entry, - localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath), - bytes: entryBuffer.length, - mimeType, - }; - if ( - options.includeAssetImages && - isInlineImageMime(mimeType) && - entryBuffer.length <= MAX_INLINE_ASSET_BYTES && - inlineAssetCount < MAX_INLINE_ASSETS - ) { - asset.inlineData = entryBuffer.toString("base64"); - inlineAssetCount += 1; - } - assets.push(asset); - } - - if (isTextLikePath(entry)) { - if (usedTextBytes >= MAX_PROJECT_TEXT_BYTES) { - omitted.push({ path: entry, reason: "text byte limit reached" }); - continue; - } - const content = decodeUtf8IfText(entryBuffer); - if (content === null) { - omitted.push({ path: entry, reason: "text decode failed" }); - continue; - } - usedTextBytes += Buffer.byteLength(content, "utf8"); - files.push({ - path: entry, - localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath), - content, - }); - } - } catch (error) { - omitted.push({ - path: entry, - reason: error instanceof Error ? error.message : String(error), + for (const entry of entries) { + if (!safeZipPath(entry.name)) { + omitted.push({ path: entry.name, reason: "unsafe path" }); + continue; + } + if (entry.name.endsWith("/") || entry.isDirectory) continue; + if (usedExtractedBytes >= MAX_EXTRACTED_PROJECT_BYTES) { + omitted.push({ path: entry.name, reason: "extracted byte limit reached" }); + continue; + } + const maxEntryBytes = Math.max( + 1, + Math.min(MAX_PROJECT_BYTES, MAX_EXTRACTED_PROJECT_BYTES - usedExtractedBytes), + ); + try { + const entryBuffer = inflateZipEntry(buffer, entry, maxEntryBytes); + usedExtractedBytes += entryBuffer.length; + let localPath: string | null = null; + if (extractDir !== null) { + localPath = join(extractDir, ...entry.name.split(/[\\/]+/)); + await mkdir(dirname(localPath), { recursive: true }); + await writeFile(localPath, entryBuffer); + extractedFiles.push({ + path: entry.name, + localPath: toWorkspaceRelative(options.workspaceDir, localPath), + bytes: entryBuffer.length, }); } - } - return { - target: options.target, - status: "downloaded", - bytes: buffer.length, - zipPath: options.materialize ? toWorkspaceRelative(options.workspaceDir, zipPath) : null, - extractDir: - extractDir === null ? null : toWorkspaceRelative(options.workspaceDir, extractDir), - extractedFiles, - files, - assets: assets.map(({ inlineData: _inlineData, ...asset }) => asset), - inlineAssets: assets - .filter((asset) => asset.inlineData !== undefined) - .map((asset) => ({ - path: asset.path, - localPath: asset.localPath, - bytes: asset.bytes, - mimeType: asset.mimeType, - data: asset.inlineData, - })), - omitted, - }; - } finally { - await rm(tempDir, { recursive: true, force: true }).catch(() => {}); + const mimeType = assetMimeType(entry.name); + if (mimeType !== null) { + const asset: { + path: string; + localPath: string | null; + bytes: number; + mimeType: string; + inlineData?: string; + } = { + path: entry.name, + localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath), + bytes: entryBuffer.length, + mimeType, + }; + if ( + options.includeAssetImages && + isInlineImageMime(mimeType) && + entryBuffer.length <= MAX_INLINE_ASSET_BYTES && + inlineAssetCount < MAX_INLINE_ASSETS + ) { + asset.inlineData = entryBuffer.toString("base64"); + inlineAssetCount += 1; + } + assets.push(asset); + } + + if (isTextLikePath(entry.name)) { + if (usedTextBytes >= MAX_PROJECT_TEXT_BYTES) { + omitted.push({ path: entry.name, reason: "text byte limit reached" }); + continue; + } + const content = decodeUtf8IfText(entryBuffer); + if (content === null) { + omitted.push({ path: entry.name, reason: "text decode failed" }); + continue; + } + usedTextBytes += Buffer.byteLength(content, "utf8"); + files.push({ + path: entry.name, + localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath), + content, + }); + } + } catch (error) { + omitted.push({ + path: entry.name, + reason: error instanceof Error ? error.message : String(error), + }); + } } + + return { + target: options.target, + status: "downloaded", + bytes: buffer.length, + zipPath: zipPath === null ? null : toWorkspaceRelative(options.workspaceDir, zipPath), + extractDir: extractDir === null ? null : toWorkspaceRelative(options.workspaceDir, extractDir), + extractedFiles, + files, + assets: assets.map(({ inlineData: _inlineData, ...asset }) => asset), + inlineAssets: assets + .filter((asset) => asset.inlineData !== undefined) + .map((asset) => ({ + path: asset.path, + localPath: asset.localPath, + bytes: asset.bytes, + mimeType: asset.mimeType, + data: asset.inlineData, + })), + omitted, + }; } +interface ZipEntryMeta { + readonly name: string; + readonly method: number; + readonly compressedSize: number; + readonly uncompressedSize: number; + readonly localHeaderOffset: number; + readonly isDirectory: boolean; +} + +/** Minimal ZIP central-directory reader (store + deflate). No external unzip binary. */ +function listZipEntries(buffer: Buffer): ZipEntryMeta[] { + let eocd = -1; + const minEocd = Math.max(0, buffer.length - (22 + 0xffff)); + for (let i = buffer.length - 22; i >= minEocd; i -= 1) { + if (buffer.readUInt32LE(i) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) throw new Error("invalid zip: missing end of central directory"); + + const totalEntries = buffer.readUInt16LE(eocd + 10); + const centralSize = buffer.readUInt32LE(eocd + 12); + const centralOffset = buffer.readUInt32LE(eocd + 16); + if (centralOffset + centralSize > buffer.length) { + throw new Error("invalid zip: central directory out of range"); + } + + const entries: ZipEntryMeta[] = []; + let offset = centralOffset; + for (let i = 0; i < totalEntries; i += 1) { + if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== 0x02014b50) { + throw new Error("invalid zip: bad central directory entry"); + } + const method = buffer.readUInt16LE(offset + 10); + const compressedSize = buffer.readUInt32LE(offset + 20); + const uncompressedSize = buffer.readUInt32LE(offset + 24); + const nameLen = buffer.readUInt16LE(offset + 28); + const extraLen = buffer.readUInt16LE(offset + 30); + const commentLen = buffer.readUInt16LE(offset + 32); + const localHeaderOffset = buffer.readUInt32LE(offset + 42); + const nameStart = offset + 46; + const name = buffer.subarray(nameStart, nameStart + nameLen).toString("utf8"); + entries.push({ + name, + method, + compressedSize, + uncompressedSize, + localHeaderOffset, + isDirectory: name.endsWith("/"), + }); + offset = nameStart + nameLen + extraLen + commentLen; + } + return entries; +} + +function inflateZipEntry(buffer: Buffer, entry: ZipEntryMeta, maxBytes: number): Buffer { + if (entry.uncompressedSize > maxBytes) { + throw new Error(`zip entry exceeds ${maxBytes} bytes`); + } + const local = entry.localHeaderOffset; + if (local + 30 > buffer.length || buffer.readUInt32LE(local) !== 0x04034b50) { + throw new Error("invalid zip: bad local header"); + } + const nameLen = buffer.readUInt16LE(local + 26); + const extraLen = buffer.readUInt16LE(local + 28); + const dataStart = local + 30 + nameLen + extraLen; + const dataEnd = dataStart + entry.compressedSize; + if (dataEnd > buffer.length) throw new Error("invalid zip: compressed data out of range"); + const compressed = buffer.subarray(dataStart, dataEnd); + + if (entry.method === 0) { + if (compressed.length > maxBytes) throw new Error(`zip entry exceeds ${maxBytes} bytes`); + return Buffer.from(compressed); + } + if (entry.method === 8) { + return Buffer.from(inflateRawSync(compressed, { maxOutputLength: maxBytes })); + } + throw new Error(`unsupported zip compression method ${entry.method}`); +} + + async function writeUsageFact( prisma: PrismaClient, runId: string, @@ -553,24 +606,6 @@ function toWorkspaceRelative(workspaceDir: string, absolutePath: string): string return rel; } -async function commandExists(command: string): Promise { - try { - await execFile("sh", ["-lc", `command -v ${command}`], { timeout: 5_000 }); - return true; - } catch { - return false; - } -} - -async function readZipEntry(zipPath: string, entry: string, maxBuffer: number): Promise { - const { stdout } = await execFile("unzip", ["-p", zipPath, entry], { - timeout: 10_000, - maxBuffer, - encoding: "buffer", - }); - return Buffer.from(stdout as Buffer); -} - function decodeUtf8IfText(buffer: Buffer): string | null { const text = buffer.toString("utf8"); const replacementRatio = (text.match(/\uFFFD/g) ?? []).length / Math.max(text.length, 1); diff --git a/hub/test/unit/fixtures/pbank-mini.zip b/hub/test/unit/fixtures/pbank-mini.zip new file mode 100644 index 0000000000000000000000000000000000000000..6cf4ae69586be33e421fd3106ab46b920e9f1bd8 GIT binary patch literal 234 zcmWIWW@Zs#U|`??VhG>^QanJ+3B(zxIXU@yB^4#7J{RlCWZhEO= minEocd; i -= 1) { + if (buffer.readUInt32LE(i) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) throw new Error("missing eocd"); + const totalEntries = buffer.readUInt16LE(eocd + 10); + const centralOffset = buffer.readUInt32LE(eocd + 16); + const entries: Array<{ name: string; method: number; compressedSize: number; uncompressedSize: number; localHeaderOffset: number }> = []; + let offset = centralOffset; + for (let i = 0; i < totalEntries; i += 1) { + const method = buffer.readUInt16LE(offset + 10); + const compressedSize = buffer.readUInt32LE(offset + 20); + const uncompressedSize = buffer.readUInt32LE(offset + 24); + const nameLen = buffer.readUInt16LE(offset + 28); + const extraLen = buffer.readUInt16LE(offset + 30); + const commentLen = buffer.readUInt16LE(offset + 32); + const localHeaderOffset = buffer.readUInt32LE(offset + 42); + const nameStart = offset + 46; + const name = buffer.subarray(nameStart, nameStart + nameLen).toString("utf8"); + entries.push({ name, method, compressedSize, uncompressedSize, localHeaderOffset }); + offset = nameStart + nameLen + extraLen + commentLen; + } + return entries; +} + +function inflateZipEntry(buffer: Buffer, entry: { method: number; compressedSize: number; uncompressedSize: number; localHeaderOffset: number }) { + const local = entry.localHeaderOffset; + const nameLen = buffer.readUInt16LE(local + 26); + const extraLen = buffer.readUInt16LE(local + 28); + const dataStart = local + 30 + nameLen + extraLen; + const compressed = buffer.subarray(dataStart, dataStart + entry.compressedSize); + if (entry.method === 0) return Buffer.from(compressed); + if (entry.method === 8) return Buffer.from(inflateRawSync(compressed)); + throw new Error(`method ${entry.method}`); +} + +describe("pbank zip reader contract", () => { + it("lists and inflates deflated entries without host unzip", async () => { + const zip = await readFile(new URL("./fixtures/pbank-mini.zip", import.meta.url)); + const entries = listZipEntries(zip); + expect(entries.map((e) => e.name).sort()).toEqual(["fig/a.png", "hello.txt"]); + const hello = entries.find((e) => e.name === "hello.txt")!; + const text = inflateZipEntry(zip, hello).toString("utf8"); + expect(text).toBe("hello pbank\n"); + }); +});