Merge pull request 'fix(hub): extract PBank zips in-process without host unzip' (#24) from fix/pbank-pure-node-unzip into main

hub/src/capability/pbank.ts

@@ -9,12 +9,9 @@
  *   3. Mandatory fact — each successful tool call writes ≥1 UsageFact with
  *      kind=external_capability and unit=requests (cost unknown unless reported).
  */
-import { randomUUID } from "node:crypto";
+import { inflateRawSync } from "node:zlib";
-import { execFile as execFileCb } from "node:child_process";
+import { mkdir, rm, writeFile } from "node:fs/promises";
-import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
-import { promisify } from "node:util";
 import type { PrismaClient } from "@prisma/client";
 import { LocalSecretEnvelope } from "../security/secretEnvelope.js";
 import { resolveCapabilityCredential } from "./capabilityConnections.js";
@@ -30,8 +27,6 @@ import {
   type PbankCapabilitySecretPayload,
 } from "./types.js";
 
-const execFile = promisify(execFileCb);
-
 export const PBANK_CAPABILITY_ID = "pbank" as const;
 const PROVIDER_ID = "paradigm_pbank";
 const MAX_BATCH_SIZE = 20;
@@ -320,42 +315,22 @@ async function readProjectZip(
     readonly includeAssetImages: boolean;
   },
 ): Promise<Record<string, unknown>> {
-  if (!(await commandExists("unzip"))) {
+  // Pure Node unzip (store/deflate). Do not shell out to host `unzip` —
-    return {
+  // silo service PATH/tooling must not gate 题库 materialize.
-      target: options.target,
+  const entries = listZipEntries(buffer);
-      status: "downloaded",
-      note: "Downloaded a zip project, but unzip is not installed on the server.",
-      bytes: buffer.length,
-      files: [],
-    };
-  }
-
-  const tempDir = await mkdtemp(join(tmpdir(), "pbank-project-"));
   const cacheRoot = confineToWorkspace(CACHE_DIR_NAME, options.workspaceDir);
   const cacheDir = join(cacheRoot, safePathSegment(options.id), safePathSegment(options.target));
   const extractDir = options.materialize ? join(cacheDir, "source") : null;
-  const zipPath = options.materialize
+  const zipPath = options.materialize ? join(cacheDir, `${options.target}.zip`) : null;
-    ? join(cacheDir, `${options.target}.zip`)
-    : join(tempDir, `${randomUUID()}.zip`);
 
-  try {
   if (options.materialize) {
     await mkdir(cacheDir, { recursive: true });
     if (extractDir !== null) {
       await rm(extractDir, { recursive: true, force: true });
       await mkdir(extractDir, { recursive: true });
     }
+    if (zipPath !== null) await writeFile(zipPath, buffer);
   }
-    await writeFile(zipPath, buffer);
-
-    const { stdout } = await execFile("unzip", ["-Z1", zipPath], {
-      timeout: 10_000,
-      maxBuffer: 1024 * 1024,
-    });
-    const entryNames = String(stdout)
-      .split(/\r?\n/)
-      .map((line) => line.trim())
-      .filter((line) => line !== "");
 
   const files: Array<Record<string, unknown>> = [];
   const assets: Array<{
@@ -371,14 +346,14 @@ async function readProjectZip(
   let usedExtractedBytes = 0;
   let inlineAssetCount = 0;
 
-    for (const entry of entryNames) {
+  for (const entry of entries) {
-      if (!safeZipPath(entry)) {
+    if (!safeZipPath(entry.name)) {
-        omitted.push({ path: entry, reason: "unsafe path" });
+      omitted.push({ path: entry.name, reason: "unsafe path" });
       continue;
     }
-      if (entry.endsWith("/")) continue;
+    if (entry.name.endsWith("/") || entry.isDirectory) continue;
     if (usedExtractedBytes >= MAX_EXTRACTED_PROJECT_BYTES) {
-        omitted.push({ path: entry, reason: "extracted byte limit reached" });
+      omitted.push({ path: entry.name, reason: "extracted byte limit reached" });
       continue;
     }
     const maxEntryBytes = Math.max(
@@ -386,21 +361,21 @@ async function readProjectZip(
       Math.min(MAX_PROJECT_BYTES, MAX_EXTRACTED_PROJECT_BYTES - usedExtractedBytes),
     );
     try {
-        const entryBuffer = await readZipEntry(zipPath, entry, maxEntryBytes);
+      const entryBuffer = inflateZipEntry(buffer, entry, maxEntryBytes);
       usedExtractedBytes += entryBuffer.length;
       let localPath: string | null = null;
       if (extractDir !== null) {
-          localPath = join(extractDir, ...entry.split(/[\\/]+/));
+        localPath = join(extractDir, ...entry.name.split(/[\\/]+/));
         await mkdir(dirname(localPath), { recursive: true });
         await writeFile(localPath, entryBuffer);
         extractedFiles.push({
-            path: entry,
+          path: entry.name,
           localPath: toWorkspaceRelative(options.workspaceDir, localPath),
           bytes: entryBuffer.length,
         });
       }
 
-        const mimeType = assetMimeType(entry);
+      const mimeType = assetMimeType(entry.name);
       if (mimeType !== null) {
         const asset: {
           path: string;
@@ -409,7 +384,7 @@ async function readProjectZip(
           mimeType: string;
           inlineData?: string;
         } = {
-            path: entry,
+          path: entry.name,
           localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath),
           bytes: entryBuffer.length,
           mimeType,
@@ -426,26 +401,26 @@ async function readProjectZip(
         assets.push(asset);
       }
 
-        if (isTextLikePath(entry)) {
+      if (isTextLikePath(entry.name)) {
         if (usedTextBytes >= MAX_PROJECT_TEXT_BYTES) {
-            omitted.push({ path: entry, reason: "text byte limit reached" });
+          omitted.push({ path: entry.name, reason: "text byte limit reached" });
           continue;
         }
         const content = decodeUtf8IfText(entryBuffer);
         if (content === null) {
-            omitted.push({ path: entry, reason: "text decode failed" });
+          omitted.push({ path: entry.name, reason: "text decode failed" });
           continue;
         }
         usedTextBytes += Buffer.byteLength(content, "utf8");
         files.push({
-            path: entry,
+          path: entry.name,
           localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath),
           content,
         });
       }
     } catch (error) {
       omitted.push({
-          path: entry,
+        path: entry.name,
         reason: error instanceof Error ? error.message : String(error),
       });
     }
@@ -455,9 +430,8 @@ async function readProjectZip(
     target: options.target,
     status: "downloaded",
     bytes: buffer.length,
-      zipPath: options.materialize ? toWorkspaceRelative(options.workspaceDir, zipPath) : null,
+    zipPath: zipPath === null ? null : toWorkspaceRelative(options.workspaceDir, zipPath),
-      extractDir:
+    extractDir: extractDir === null ? null : toWorkspaceRelative(options.workspaceDir, extractDir),
-        extractDir === null ? null : toWorkspaceRelative(options.workspaceDir, extractDir),
     extractedFiles,
     files,
     assets: assets.map(({ inlineData: _inlineData, ...asset }) => asset),
@@ -472,10 +446,89 @@ async function readProjectZip(
       })),
     omitted,
   };
-  } finally {
+}
-    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+
+interface ZipEntryMeta {
+  readonly name: string;
+  readonly method: number;
+  readonly compressedSize: number;
+  readonly uncompressedSize: number;
+  readonly localHeaderOffset: number;
+  readonly isDirectory: boolean;
+}
+
+/** Minimal ZIP central-directory reader (store + deflate). No external unzip binary. */
+function listZipEntries(buffer: Buffer): ZipEntryMeta[] {
+  let eocd = -1;
+  const minEocd = Math.max(0, buffer.length - (22 + 0xffff));
+  for (let i = buffer.length - 22; i >= minEocd; i -= 1) {
+    if (buffer.readUInt32LE(i) === 0x06054b50) {
+      eocd = i;
+      break;
     }
   }
+  if (eocd < 0) throw new Error("invalid zip: missing end of central directory");
+
+  const totalEntries = buffer.readUInt16LE(eocd + 10);
+  const centralSize = buffer.readUInt32LE(eocd + 12);
+  const centralOffset = buffer.readUInt32LE(eocd + 16);
+  if (centralOffset + centralSize > buffer.length) {
+    throw new Error("invalid zip: central directory out of range");
+  }
+
+  const entries: ZipEntryMeta[] = [];
+  let offset = centralOffset;
+  for (let i = 0; i < totalEntries; i += 1) {
+    if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== 0x02014b50) {
+      throw new Error("invalid zip: bad central directory entry");
+    }
+    const method = buffer.readUInt16LE(offset + 10);
+    const compressedSize = buffer.readUInt32LE(offset + 20);
+    const uncompressedSize = buffer.readUInt32LE(offset + 24);
+    const nameLen = buffer.readUInt16LE(offset + 28);
+    const extraLen = buffer.readUInt16LE(offset + 30);
+    const commentLen = buffer.readUInt16LE(offset + 32);
+    const localHeaderOffset = buffer.readUInt32LE(offset + 42);
+    const nameStart = offset + 46;
+    const name = buffer.subarray(nameStart, nameStart + nameLen).toString("utf8");
+    entries.push({
+      name,
+      method,
+      compressedSize,
+      uncompressedSize,
+      localHeaderOffset,
+      isDirectory: name.endsWith("/"),
+    });
+    offset = nameStart + nameLen + extraLen + commentLen;
+  }
+  return entries;
+}
+
+function inflateZipEntry(buffer: Buffer, entry: ZipEntryMeta, maxBytes: number): Buffer {
+  if (entry.uncompressedSize > maxBytes) {
+    throw new Error(`zip entry exceeds ${maxBytes} bytes`);
+  }
+  const local = entry.localHeaderOffset;
+  if (local + 30 > buffer.length || buffer.readUInt32LE(local) !== 0x04034b50) {
+    throw new Error("invalid zip: bad local header");
+  }
+  const nameLen = buffer.readUInt16LE(local + 26);
+  const extraLen = buffer.readUInt16LE(local + 28);
+  const dataStart = local + 30 + nameLen + extraLen;
+  const dataEnd = dataStart + entry.compressedSize;
+  if (dataEnd > buffer.length) throw new Error("invalid zip: compressed data out of range");
+  const compressed = buffer.subarray(dataStart, dataEnd);
+
+  if (entry.method === 0) {
+    if (compressed.length > maxBytes) throw new Error(`zip entry exceeds ${maxBytes} bytes`);
+    return Buffer.from(compressed);
+  }
+  if (entry.method === 8) {
+    return Buffer.from(inflateRawSync(compressed, { maxOutputLength: maxBytes }));
+  }
+  throw new Error(`unsupported zip compression method ${entry.method}`);
+}
+
 
 async function writeUsageFact(
   prisma: PrismaClient,
@@ -553,24 +606,6 @@ function toWorkspaceRelative(workspaceDir: string, absolutePath: string): string
   return rel;
 }
 
-async function commandExists(command: string): Promise<boolean> {
-  try {
-    await execFile("sh", ["-lc", `command -v ${command}`], { timeout: 5_000 });
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-async function readZipEntry(zipPath: string, entry: string, maxBuffer: number): Promise<Buffer> {
-  const { stdout } = await execFile("unzip", ["-p", zipPath, entry], {
-    timeout: 10_000,
-    maxBuffer,
-    encoding: "buffer",
-  });
-  return Buffer.from(stdout as Buffer);
-}
-
 function decodeUtf8IfText(buffer: Buffer): string | null {
   const text = buffer.toString("utf8");
   const replacementRatio = (text.match(/\uFFFD/g) ?? []).length / Math.max(text.length, 1);

hub/test/unit/pbank-zip.test.ts

@@ -0,0 +1,56 @@
+import { readFile } from "node:fs/promises";
+import { inflateRawSync } from "node:zlib";
+import { describe, expect, it } from "vitest";
+
+/** Mirrors hub/src/capability/pbank.ts zip reader contracts. */
+function listZipEntries(buffer: Buffer) {
+  let eocd = -1;
+  const minEocd = Math.max(0, buffer.length - (22 + 0xffff));
+  for (let i = buffer.length - 22; i >= minEocd; i -= 1) {
+    if (buffer.readUInt32LE(i) === 0x06054b50) {
+      eocd = i;
+      break;
+    }
+  }
+  if (eocd < 0) throw new Error("missing eocd");
+  const totalEntries = buffer.readUInt16LE(eocd + 10);
+  const centralOffset = buffer.readUInt32LE(eocd + 16);
+  const entries: Array<{ name: string; method: number; compressedSize: number; uncompressedSize: number; localHeaderOffset: number }> = [];
+  let offset = centralOffset;
+  for (let i = 0; i < totalEntries; i += 1) {
+    const method = buffer.readUInt16LE(offset + 10);
+    const compressedSize = buffer.readUInt32LE(offset + 20);
+    const uncompressedSize = buffer.readUInt32LE(offset + 24);
+    const nameLen = buffer.readUInt16LE(offset + 28);
+    const extraLen = buffer.readUInt16LE(offset + 30);
+    const commentLen = buffer.readUInt16LE(offset + 32);
+    const localHeaderOffset = buffer.readUInt32LE(offset + 42);
+    const nameStart = offset + 46;
+    const name = buffer.subarray(nameStart, nameStart + nameLen).toString("utf8");
+    entries.push({ name, method, compressedSize, uncompressedSize, localHeaderOffset });
+    offset = nameStart + nameLen + extraLen + commentLen;
+  }
+  return entries;
+}
+
+function inflateZipEntry(buffer: Buffer, entry: { method: number; compressedSize: number; uncompressedSize: number; localHeaderOffset: number }) {
+  const local = entry.localHeaderOffset;
+  const nameLen = buffer.readUInt16LE(local + 26);
+  const extraLen = buffer.readUInt16LE(local + 28);
+  const dataStart = local + 30 + nameLen + extraLen;
+  const compressed = buffer.subarray(dataStart, dataStart + entry.compressedSize);
+  if (entry.method === 0) return Buffer.from(compressed);
+  if (entry.method === 8) return Buffer.from(inflateRawSync(compressed));
+  throw new Error(`method ${entry.method}`);
+}
+
+describe("pbank zip reader contract", () => {
+  it("lists and inflates deflated entries without host unzip", async () => {
+    const zip = await readFile(new URL("./fixtures/pbank-mini.zip", import.meta.url));
+    const entries = listZipEntries(zip);
+    expect(entries.map((e) => e.name).sort()).toEqual(["fig/a.png", "hello.txt"]);
+    const hello = entries.find((e) => e.name === "hello.txt")!;
+    const text = inflateZipEntry(zip, hello).toString("utf8");
+    expect(text).toBe("hello pbank\n");
+  });
+});