diff --git a/bayarea-ai-server/Dockerfile.cuda b/bayarea-ai-server/Dockerfile.cuda index 8283eca..1ffd741 100644 --- a/bayarea-ai-server/Dockerfile.cuda +++ b/bayarea-ai-server/Dockerfile.cuda @@ -19,6 +19,7 @@ RUN git clone --depth 1 https://github.com/ggml-org/llama.cpp /src RUN cmake /src -B /build \ -DGGML_CUDA=ON \ -DCMAKE_CUDA_ARCHITECTURES=120 \ + -DGGML_CUDA_FORCE_CUBLAS=OFF \ -DBUILD_SHARED_LIBS=OFF \ -DLLAMA_CURL=ON \ && cmake --build /build --config Release -j --target llama-server diff --git a/vision_runner.ts b/vision_runner.ts index 74e87fc..e349e78 100644 --- a/vision_runner.ts +++ b/vision_runner.ts @@ -42,7 +42,6 @@ const execFileP = promisify(execFile); // --------------------------------------------------------------------------- interface Args { - input: string; pdfRoot: string; api: string; limit: number; @@ -60,15 +59,14 @@ function parseArgs(): Args { const i = a.indexOf(flag); return i >= 0 && a[i + 1] !== undefined ? a[i + 1] : def; }; - const input = get("--input", "out/buyers.json")!; const pdfRoot = get("--pdf-root"); const api = (get("--api", "http://localhost:8000/v1") || "").replace(/\/+$/, ""); if (!pdfRoot) { console.error("Fehler: --pdf-root ist erforderlich."); process.exit(1); + throw new Error("unreachable"); // hilft dem TS-Narrowing } return { - input, pdfRoot: pdfRoot.replace(/^~(?=\/)/, os.homedir()), api, limit: parseInt(get("--limit", "0")!, 10) || 0, @@ -77,9 +75,10 @@ function parseArgs(): Args { const v = get("--dpi", "auto")!; return v === "auto" ? "auto" : parseInt(v, 10) || 150; })(), - maxPages: parseInt(get("--max-pages", "8")!, 10) || 8, + // Nur PDFs mit HOECHSTENS so vielen Seiten werden per Vision gescannt. + maxPages: parseInt(get("--max-pages", "10")!, 10) || 10, force: a.includes("--force"), - outDir: get("--out-dir", path.dirname(input))!, + outDir: get("--out-dir", "out")!, timeoutMs: (parseInt(get("--timeout", "180")!, 10) || 180) * 1000, }; } @@ -223,6 +222,17 @@ async function buildPdfIndex(root: string): Promise> { return index; } +/** Seitenzahl via pdfinfo (poppler-utils). -1 bei Fehler/beschaedigtem PDF. */ +async function pdfPageCount(pdfPath: string): Promise { + try { + const { stdout } = await execFileP("pdfinfo", [pdfPath], { timeout: 30_000 }); + const m = stdout.match(/^Pages:\s+(\d+)/m); + return m ? parseInt(m[1], 10) : -1; + } catch { + return -1; + } +} + async function renderPdf(pdfPath: string, dpi: number, maxPages: number, tmpDir: string): Promise { const prefix = path.join(tmpDir, "page"); await execFileP("pdftoppm", ["-png", "-r", String(dpi), "-l", String(maxPages), pdfPath, prefix], { @@ -465,10 +475,10 @@ async function main(): Promise { process.exit(1); } - const all: BuyerRecord[] = JSON.parse(await fsp.readFile(args.input, "utf8")); - const visionPath = path.join(args.outDir, "buyers_vision.json"); - const mergedPath = path.join(args.outDir, "buyers_merged.json"); + await fsp.mkdir(args.outDir, { recursive: true }); + + // Resume-Stand laden (bereits verarbeitete Dokumente) const done = new Map(); try { const prev: BuyerRecord[] = JSON.parse(await fsp.readFile(visionPath, "utf8")); @@ -477,21 +487,64 @@ async function main(): Promise { /* kein Resume-Stand */ } - let candidates = all.filter((r) => r.is_buyer_sheet === false); + // --------------------------------------------------------------------- + // PHASE 1 — INDEX: alle PDFs scannen, file_name + _pages_total sofort + // ins buyers_vision.json eintragen (auch die zu grossen, dann markiert). + // --------------------------------------------------------------------- + console.error(`Phase 1: Indexiere PDFs unter ${args.pdfRoot} ...`); + const pdfIndex = await buildPdfIndex(args.pdfRoot); + const names = [...pdfIndex.keys()].sort(); + console.error(`${names.length} PDFs gefunden. Ermittle Seitenzahlen ...`); + + let idx = 0; + for (const name of names) { + idx++; + // schon indexiert (mit gueltiger Seitenzahl)? dann nicht neu zaehlen + const existing = done.get(name); + if (existing && typeof existing["_pages_total"] === "number" && existing["_pages_total"]! >= 0 && !args.force) { + if (idx % 50 === 0) progress(`Index ${idx}/${names.length}`); + continue; + } + const pdfPath = pdfIndex.get(name)!; + const pages = await pdfPageCount(pdfPath); + const tooMany = pages < 0 ? false : pages > args.maxPages; + const prev = done.get(name) ?? { file_name: name, is_buyer_sheet: false }; + done.set(name, { + ...prev, + file_name: name, + _pages_total: pages, + ...(pages < 0 ? { _index_error: "pdfinfo fehlgeschlagen (beschaedigt?)" } : {}), + ...(tooMany ? { _skipped_too_many_pages: true } : {}), + }); + if (idx % 25 === 0 || idx === names.length) progress(`Index ${idx}/${names.length}`); + } + // Index sofort persistieren, bevor die teure Phase 2 startet + await fsp.writeFile(visionPath, JSON.stringify([...done.values()], null, 2)); + process.stderr.write("\n"); + + // --------------------------------------------------------------------- + // PHASE 2 — INHALT: nur PDFs <= maxPages, noch nicht (fehlerfrei) erledigt. + // --------------------------------------------------------------------- + let candidates = [...done.values()].filter((r) => { + const pages = r["_pages_total"] as number | undefined; + return typeof pages === "number" && pages > 0 && pages <= args.maxPages; + }); if (args.only) candidates = candidates.filter((r) => r.file_name === args.only); - // Resume-Skip VOR dem Limit: bereits fehlerfrei Verarbeitete zählen nicht mit + // Resume-Skip VOR dem Limit: fehlerfrei mit echtem Vision-Ergebnis = fertig const pending = candidates.filter((r) => { const prev = done.get(r.file_name); - return !(prev && !prev["_vision_error"] && !args.force); + const hasResult = prev && prev["_parser"] === "vision" && !prev["_vision_error"]; + return !(hasResult && !args.force); }); const skipped = candidates.length - pending.length; const targets = args.limit > 0 ? pending.slice(0, args.limit) : pending; - console.error(`Indexiere PDFs unter ${args.pdfRoot} ...`); - const pdfIndex = await buildPdfIndex(args.pdfRoot); + const tooManyCount = [...done.values()].filter((r) => r["_skipped_too_many_pages"]).length; console.error( - `${pdfIndex.size} PDFs gefunden. ${candidates.length} Kandidaten, ${skipped} bereits verarbeitet, ${targets.length} in diesem Lauf.` + `Phase 2: ${candidates.length} Kandidaten (<=${args.maxPages} Seiten), ` + + `${tooManyCount} zu gross (uebersprungen), ${skipped} bereits verarbeitet, ` + + `${targets.length} in diesem Lauf.` ); const model = await fetchModelId(args.api); @@ -560,21 +613,17 @@ async function main(): Promise { await fsp.rm(tmpDir, { recursive: true, force: true }); } - // Inkrementell sichern (Resume-fähig) — merged gleich mit - await fsp.mkdir(args.outDir, { recursive: true }); + // Inkrementell sichern (Resume-fähig) await fsp.writeFile(visionPath, JSON.stringify([...done.values()], null, 2)); - await fsp.writeFile(mergedPath, JSON.stringify(all.map((r) => done.get(r.file_name) ?? r), null, 2)); } - // Merge: Gleis A + Gleis B - const merged = all.map((r) => done.get(r.file_name) ?? r); - await fsp.writeFile(mergedPath, JSON.stringify(merged, null, 2)); + await fsp.writeFile(visionPath, JSON.stringify([...done.values()], null, 2)); process.stderr.write("\n"); console.error( `Fertig. OK: ${ok}, CA only: ${caOnly}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}` ); - console.error(`→ ${visionPath}\n→ ${mergedPath}`); + console.error(`→ ${visionPath}`); } main().catch((e) => {