forked from EduCraft/curriculum-project-hub
fix(hub): extract PBank zips in-process without host unzip
pbank materialize previously shelled out to `unzip` and soft-failed when the binary was missing, so agents only saw titles. Read zip entries with Node zlib (store/deflate) and write under workspace .pbank-sources.
This commit is contained in:
+105
-70
@@ -9,12 +9,9 @@
|
||||
* 3. Mandatory fact — each successful tool call writes ≥1 UsageFact with
|
||||
* kind=external_capability and unit=requests (cost unknown unless reported).
|
||||
*/
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { execFile as execFileCb } from "node:child_process";
|
||||
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { inflateRawSync } from "node:zlib";
|
||||
import { mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, relative, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import { LocalSecretEnvelope } from "../security/secretEnvelope.js";
|
||||
import { resolveCapabilityCredential } from "./capabilityConnections.js";
|
||||
@@ -30,8 +27,6 @@ import {
|
||||
type PbankCapabilitySecretPayload,
|
||||
} from "./types.js";
|
||||
|
||||
const execFile = promisify(execFileCb);
|
||||
|
||||
export const PBANK_CAPABILITY_ID = "pbank" as const;
|
||||
const PROVIDER_ID = "paradigm_pbank";
|
||||
const MAX_BATCH_SIZE = 20;
|
||||
@@ -320,42 +315,22 @@ async function readProjectZip(
|
||||
readonly includeAssetImages: boolean;
|
||||
},
|
||||
): Promise<Record<string, unknown>> {
|
||||
if (!(await commandExists("unzip"))) {
|
||||
return {
|
||||
target: options.target,
|
||||
status: "downloaded",
|
||||
note: "Downloaded a zip project, but unzip is not installed on the server.",
|
||||
bytes: buffer.length,
|
||||
files: [],
|
||||
};
|
||||
}
|
||||
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "pbank-project-"));
|
||||
// Pure Node unzip (store/deflate). Do not shell out to host `unzip` —
|
||||
// silo service PATH/tooling must not gate 题库 materialize.
|
||||
const entries = listZipEntries(buffer);
|
||||
const cacheRoot = confineToWorkspace(CACHE_DIR_NAME, options.workspaceDir);
|
||||
const cacheDir = join(cacheRoot, safePathSegment(options.id), safePathSegment(options.target));
|
||||
const extractDir = options.materialize ? join(cacheDir, "source") : null;
|
||||
const zipPath = options.materialize
|
||||
? join(cacheDir, `${options.target}.zip`)
|
||||
: join(tempDir, `${randomUUID()}.zip`);
|
||||
const zipPath = options.materialize ? join(cacheDir, `${options.target}.zip`) : null;
|
||||
|
||||
try {
|
||||
if (options.materialize) {
|
||||
await mkdir(cacheDir, { recursive: true });
|
||||
if (extractDir !== null) {
|
||||
await rm(extractDir, { recursive: true, force: true });
|
||||
await mkdir(extractDir, { recursive: true });
|
||||
}
|
||||
if (zipPath !== null) await writeFile(zipPath, buffer);
|
||||
}
|
||||
await writeFile(zipPath, buffer);
|
||||
|
||||
const { stdout } = await execFile("unzip", ["-Z1", zipPath], {
|
||||
timeout: 10_000,
|
||||
maxBuffer: 1024 * 1024,
|
||||
});
|
||||
const entryNames = String(stdout)
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line !== "");
|
||||
|
||||
const files: Array<Record<string, unknown>> = [];
|
||||
const assets: Array<{
|
||||
@@ -371,14 +346,14 @@ async function readProjectZip(
|
||||
let usedExtractedBytes = 0;
|
||||
let inlineAssetCount = 0;
|
||||
|
||||
for (const entry of entryNames) {
|
||||
if (!safeZipPath(entry)) {
|
||||
omitted.push({ path: entry, reason: "unsafe path" });
|
||||
for (const entry of entries) {
|
||||
if (!safeZipPath(entry.name)) {
|
||||
omitted.push({ path: entry.name, reason: "unsafe path" });
|
||||
continue;
|
||||
}
|
||||
if (entry.endsWith("/")) continue;
|
||||
if (entry.name.endsWith("/") || entry.isDirectory) continue;
|
||||
if (usedExtractedBytes >= MAX_EXTRACTED_PROJECT_BYTES) {
|
||||
omitted.push({ path: entry, reason: "extracted byte limit reached" });
|
||||
omitted.push({ path: entry.name, reason: "extracted byte limit reached" });
|
||||
continue;
|
||||
}
|
||||
const maxEntryBytes = Math.max(
|
||||
@@ -386,21 +361,21 @@ async function readProjectZip(
|
||||
Math.min(MAX_PROJECT_BYTES, MAX_EXTRACTED_PROJECT_BYTES - usedExtractedBytes),
|
||||
);
|
||||
try {
|
||||
const entryBuffer = await readZipEntry(zipPath, entry, maxEntryBytes);
|
||||
const entryBuffer = inflateZipEntry(buffer, entry, maxEntryBytes);
|
||||
usedExtractedBytes += entryBuffer.length;
|
||||
let localPath: string | null = null;
|
||||
if (extractDir !== null) {
|
||||
localPath = join(extractDir, ...entry.split(/[\\/]+/));
|
||||
localPath = join(extractDir, ...entry.name.split(/[\\/]+/));
|
||||
await mkdir(dirname(localPath), { recursive: true });
|
||||
await writeFile(localPath, entryBuffer);
|
||||
extractedFiles.push({
|
||||
path: entry,
|
||||
path: entry.name,
|
||||
localPath: toWorkspaceRelative(options.workspaceDir, localPath),
|
||||
bytes: entryBuffer.length,
|
||||
});
|
||||
}
|
||||
|
||||
const mimeType = assetMimeType(entry);
|
||||
const mimeType = assetMimeType(entry.name);
|
||||
if (mimeType !== null) {
|
||||
const asset: {
|
||||
path: string;
|
||||
@@ -409,7 +384,7 @@ async function readProjectZip(
|
||||
mimeType: string;
|
||||
inlineData?: string;
|
||||
} = {
|
||||
path: entry,
|
||||
path: entry.name,
|
||||
localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath),
|
||||
bytes: entryBuffer.length,
|
||||
mimeType,
|
||||
@@ -426,26 +401,26 @@ async function readProjectZip(
|
||||
assets.push(asset);
|
||||
}
|
||||
|
||||
if (isTextLikePath(entry)) {
|
||||
if (isTextLikePath(entry.name)) {
|
||||
if (usedTextBytes >= MAX_PROJECT_TEXT_BYTES) {
|
||||
omitted.push({ path: entry, reason: "text byte limit reached" });
|
||||
omitted.push({ path: entry.name, reason: "text byte limit reached" });
|
||||
continue;
|
||||
}
|
||||
const content = decodeUtf8IfText(entryBuffer);
|
||||
if (content === null) {
|
||||
omitted.push({ path: entry, reason: "text decode failed" });
|
||||
omitted.push({ path: entry.name, reason: "text decode failed" });
|
||||
continue;
|
||||
}
|
||||
usedTextBytes += Buffer.byteLength(content, "utf8");
|
||||
files.push({
|
||||
path: entry,
|
||||
path: entry.name,
|
||||
localPath: localPath === null ? null : toWorkspaceRelative(options.workspaceDir, localPath),
|
||||
content,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
omitted.push({
|
||||
path: entry,
|
||||
path: entry.name,
|
||||
reason: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
@@ -455,9 +430,8 @@ async function readProjectZip(
|
||||
target: options.target,
|
||||
status: "downloaded",
|
||||
bytes: buffer.length,
|
||||
zipPath: options.materialize ? toWorkspaceRelative(options.workspaceDir, zipPath) : null,
|
||||
extractDir:
|
||||
extractDir === null ? null : toWorkspaceRelative(options.workspaceDir, extractDir),
|
||||
zipPath: zipPath === null ? null : toWorkspaceRelative(options.workspaceDir, zipPath),
|
||||
extractDir: extractDir === null ? null : toWorkspaceRelative(options.workspaceDir, extractDir),
|
||||
extractedFiles,
|
||||
files,
|
||||
assets: assets.map(({ inlineData: _inlineData, ...asset }) => asset),
|
||||
@@ -472,10 +446,89 @@ async function readProjectZip(
|
||||
})),
|
||||
omitted,
|
||||
};
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
|
||||
interface ZipEntryMeta {
|
||||
readonly name: string;
|
||||
readonly method: number;
|
||||
readonly compressedSize: number;
|
||||
readonly uncompressedSize: number;
|
||||
readonly localHeaderOffset: number;
|
||||
readonly isDirectory: boolean;
|
||||
}
|
||||
|
||||
/** Minimal ZIP central-directory reader (store + deflate). No external unzip binary. */
|
||||
function listZipEntries(buffer: Buffer): ZipEntryMeta[] {
|
||||
let eocd = -1;
|
||||
const minEocd = Math.max(0, buffer.length - (22 + 0xffff));
|
||||
for (let i = buffer.length - 22; i >= minEocd; i -= 1) {
|
||||
if (buffer.readUInt32LE(i) === 0x06054b50) {
|
||||
eocd = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (eocd < 0) throw new Error("invalid zip: missing end of central directory");
|
||||
|
||||
const totalEntries = buffer.readUInt16LE(eocd + 10);
|
||||
const centralSize = buffer.readUInt32LE(eocd + 12);
|
||||
const centralOffset = buffer.readUInt32LE(eocd + 16);
|
||||
if (centralOffset + centralSize > buffer.length) {
|
||||
throw new Error("invalid zip: central directory out of range");
|
||||
}
|
||||
|
||||
const entries: ZipEntryMeta[] = [];
|
||||
let offset = centralOffset;
|
||||
for (let i = 0; i < totalEntries; i += 1) {
|
||||
if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== 0x02014b50) {
|
||||
throw new Error("invalid zip: bad central directory entry");
|
||||
}
|
||||
const method = buffer.readUInt16LE(offset + 10);
|
||||
const compressedSize = buffer.readUInt32LE(offset + 20);
|
||||
const uncompressedSize = buffer.readUInt32LE(offset + 24);
|
||||
const nameLen = buffer.readUInt16LE(offset + 28);
|
||||
const extraLen = buffer.readUInt16LE(offset + 30);
|
||||
const commentLen = buffer.readUInt16LE(offset + 32);
|
||||
const localHeaderOffset = buffer.readUInt32LE(offset + 42);
|
||||
const nameStart = offset + 46;
|
||||
const name = buffer.subarray(nameStart, nameStart + nameLen).toString("utf8");
|
||||
entries.push({
|
||||
name,
|
||||
method,
|
||||
compressedSize,
|
||||
uncompressedSize,
|
||||
localHeaderOffset,
|
||||
isDirectory: name.endsWith("/"),
|
||||
});
|
||||
offset = nameStart + nameLen + extraLen + commentLen;
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function inflateZipEntry(buffer: Buffer, entry: ZipEntryMeta, maxBytes: number): Buffer {
|
||||
if (entry.uncompressedSize > maxBytes) {
|
||||
throw new Error(`zip entry exceeds ${maxBytes} bytes`);
|
||||
}
|
||||
const local = entry.localHeaderOffset;
|
||||
if (local + 30 > buffer.length || buffer.readUInt32LE(local) !== 0x04034b50) {
|
||||
throw new Error("invalid zip: bad local header");
|
||||
}
|
||||
const nameLen = buffer.readUInt16LE(local + 26);
|
||||
const extraLen = buffer.readUInt16LE(local + 28);
|
||||
const dataStart = local + 30 + nameLen + extraLen;
|
||||
const dataEnd = dataStart + entry.compressedSize;
|
||||
if (dataEnd > buffer.length) throw new Error("invalid zip: compressed data out of range");
|
||||
const compressed = buffer.subarray(dataStart, dataEnd);
|
||||
|
||||
if (entry.method === 0) {
|
||||
if (compressed.length > maxBytes) throw new Error(`zip entry exceeds ${maxBytes} bytes`);
|
||||
return Buffer.from(compressed);
|
||||
}
|
||||
if (entry.method === 8) {
|
||||
return Buffer.from(inflateRawSync(compressed, { maxOutputLength: maxBytes }));
|
||||
}
|
||||
throw new Error(`unsupported zip compression method ${entry.method}`);
|
||||
}
|
||||
|
||||
|
||||
async function writeUsageFact(
|
||||
prisma: PrismaClient,
|
||||
@@ -553,24 +606,6 @@ function toWorkspaceRelative(workspaceDir: string, absolutePath: string): string
|
||||
return rel;
|
||||
}
|
||||
|
||||
async function commandExists(command: string): Promise<boolean> {
|
||||
try {
|
||||
await execFile("sh", ["-lc", `command -v ${command}`], { timeout: 5_000 });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function readZipEntry(zipPath: string, entry: string, maxBuffer: number): Promise<Buffer> {
|
||||
const { stdout } = await execFile("unzip", ["-p", zipPath, entry], {
|
||||
timeout: 10_000,
|
||||
maxBuffer,
|
||||
encoding: "buffer",
|
||||
});
|
||||
return Buffer.from(stdout as Buffer);
|
||||
}
|
||||
|
||||
function decodeUtf8IfText(buffer: Buffer): string | null {
|
||||
const text = buffer.toString("utf8");
|
||||
const replacementRatio = (text.match(/\uFFFD/g) ?? []).length / Math.max(text.length, 1);
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,56 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { inflateRawSync } from "node:zlib";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
/** Mirrors hub/src/capability/pbank.ts zip reader contracts. */
|
||||
function listZipEntries(buffer: Buffer) {
|
||||
let eocd = -1;
|
||||
const minEocd = Math.max(0, buffer.length - (22 + 0xffff));
|
||||
for (let i = buffer.length - 22; i >= minEocd; i -= 1) {
|
||||
if (buffer.readUInt32LE(i) === 0x06054b50) {
|
||||
eocd = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (eocd < 0) throw new Error("missing eocd");
|
||||
const totalEntries = buffer.readUInt16LE(eocd + 10);
|
||||
const centralOffset = buffer.readUInt32LE(eocd + 16);
|
||||
const entries: Array<{ name: string; method: number; compressedSize: number; uncompressedSize: number; localHeaderOffset: number }> = [];
|
||||
let offset = centralOffset;
|
||||
for (let i = 0; i < totalEntries; i += 1) {
|
||||
const method = buffer.readUInt16LE(offset + 10);
|
||||
const compressedSize = buffer.readUInt32LE(offset + 20);
|
||||
const uncompressedSize = buffer.readUInt32LE(offset + 24);
|
||||
const nameLen = buffer.readUInt16LE(offset + 28);
|
||||
const extraLen = buffer.readUInt16LE(offset + 30);
|
||||
const commentLen = buffer.readUInt16LE(offset + 32);
|
||||
const localHeaderOffset = buffer.readUInt32LE(offset + 42);
|
||||
const nameStart = offset + 46;
|
||||
const name = buffer.subarray(nameStart, nameStart + nameLen).toString("utf8");
|
||||
entries.push({ name, method, compressedSize, uncompressedSize, localHeaderOffset });
|
||||
offset = nameStart + nameLen + extraLen + commentLen;
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function inflateZipEntry(buffer: Buffer, entry: { method: number; compressedSize: number; uncompressedSize: number; localHeaderOffset: number }) {
|
||||
const local = entry.localHeaderOffset;
|
||||
const nameLen = buffer.readUInt16LE(local + 26);
|
||||
const extraLen = buffer.readUInt16LE(local + 28);
|
||||
const dataStart = local + 30 + nameLen + extraLen;
|
||||
const compressed = buffer.subarray(dataStart, dataStart + entry.compressedSize);
|
||||
if (entry.method === 0) return Buffer.from(compressed);
|
||||
if (entry.method === 8) return Buffer.from(inflateRawSync(compressed));
|
||||
throw new Error(`method ${entry.method}`);
|
||||
}
|
||||
|
||||
describe("pbank zip reader contract", () => {
|
||||
it("lists and inflates deflated entries without host unzip", async () => {
|
||||
const zip = await readFile(new URL("./fixtures/pbank-mini.zip", import.meta.url));
|
||||
const entries = listZipEntries(zip);
|
||||
expect(entries.map((e) => e.name).sort()).toEqual(["fig/a.png", "hello.txt"]);
|
||||
const hello = entries.find((e) => e.name === "hello.txt")!;
|
||||
const text = inflateZipEntry(zip, hello).toString("utf8");
|
||||
expect(text).toBe("hello pbank\n");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user