update

2026-07-14 12:34:44 -05:00
parent 6c3786f200
commit 96eb7ad9b9
2 changed files with 71 additions and 21 deletions
--- a/bayarea-ai-server/Dockerfile.cuda
+++ b/bayarea-ai-server/Dockerfile.cuda
@@ -19,6 +19,7 @@ RUN git clone --depth 1 https://github.com/ggml-org/llama.cpp /src
 RUN cmake /src -B /build \
      -DGGML_CUDA=ON \
      -DCMAKE_CUDA_ARCHITECTURES=120 \
+      -DGGML_CUDA_FORCE_CUBLAS=OFF \
      -DBUILD_SHARED_LIBS=OFF \
      -DLLAMA_CURL=ON \
    && cmake --build /build --config Release -j --target llama-server
--- a/vision_runner.ts
+++ b/vision_runner.ts
@@ -42,7 +42,6 @@ const execFileP = promisify(execFile);
 // ---------------------------------------------------------------------------

 interface Args {
-  input: string;
  pdfRoot: string;
  api: string;
  limit: number;
@@ -60,15 +59,14 @@ function parseArgs(): Args {
    const i = a.indexOf(flag);
    return i >= 0 && a[i + 1] !== undefined ? a[i + 1] : def;
  };
-  const input = get("--input", "out/buyers.json")!;
  const pdfRoot = get("--pdf-root");
  const api = (get("--api", "http://localhost:8000/v1") || "").replace(/\/+$/, "");
  if (!pdfRoot) {
    console.error("Fehler: --pdf-root <Verzeichnis> ist erforderlich.");
    process.exit(1);
+    throw new Error("unreachable"); // hilft dem TS-Narrowing
  }
  return {
-    input,
    pdfRoot: pdfRoot.replace(/^~(?=\/)/, os.homedir()),
    api,
    limit: parseInt(get("--limit", "0")!, 10) || 0,
@@ -77,9 +75,10 @@ function parseArgs(): Args {
      const v = get("--dpi", "auto")!;
      return v === "auto" ? "auto" : parseInt(v, 10) || 150;
    })(),
-    maxPages: parseInt(get("--max-pages", "8")!, 10) || 8,
+    // Nur PDFs mit HOECHSTENS so vielen Seiten werden per Vision gescannt.
+    maxPages: parseInt(get("--max-pages", "10")!, 10) || 10,
    force: a.includes("--force"),
-    outDir: get("--out-dir", path.dirname(input))!,
+    outDir: get("--out-dir", "out")!,
    timeoutMs: (parseInt(get("--timeout", "180")!, 10) || 180) * 1000,
  };
 }
@@ -223,6 +222,17 @@ async function buildPdfIndex(root: string): Promise<Map<string, string>> {
  return index;
 }

+/** Seitenzahl via pdfinfo (poppler-utils). -1 bei Fehler/beschaedigtem PDF. */
+async function pdfPageCount(pdfPath: string): Promise<number> {
+  try {
+    const { stdout } = await execFileP("pdfinfo", [pdfPath], { timeout: 30_000 });
+    const m = stdout.match(/^Pages:\s+(\d+)/m);
+    return m ? parseInt(m[1], 10) : -1;
+  } catch {
+    return -1;
+  }
+}
+
 async function renderPdf(pdfPath: string, dpi: number, maxPages: number, tmpDir: string): Promise<string[]> {
  const prefix = path.join(tmpDir, "page");
  await execFileP("pdftoppm", ["-png", "-r", String(dpi), "-l", String(maxPages), pdfPath, prefix], {
@@ -465,10 +475,10 @@ async function main(): Promise<void> {
    process.exit(1);
  }

-  const all: BuyerRecord[] = JSON.parse(await fsp.readFile(args.input, "utf8"));
-
  const visionPath = path.join(args.outDir, "buyers_vision.json");
-  const mergedPath = path.join(args.outDir, "buyers_merged.json");
+  await fsp.mkdir(args.outDir, { recursive: true });
+
+  // Resume-Stand laden (bereits verarbeitete Dokumente)
  const done = new Map<string, BuyerRecord>();
  try {
    const prev: BuyerRecord[] = JSON.parse(await fsp.readFile(visionPath, "utf8"));
@@ -477,21 +487,64 @@ async function main(): Promise<void> {
    /* kein Resume-Stand */
  }

-  let candidates = all.filter((r) => r.is_buyer_sheet === false);
+  // ---------------------------------------------------------------------
+  // PHASE 1 — INDEX: alle PDFs scannen, file_name + _pages_total sofort
+  // ins buyers_vision.json eintragen (auch die zu grossen, dann markiert).
+  // ---------------------------------------------------------------------
+  console.error(`Phase 1: Indexiere PDFs unter ${args.pdfRoot} ...`);
+  const pdfIndex = await buildPdfIndex(args.pdfRoot);
+  const names = [...pdfIndex.keys()].sort();
+  console.error(`${names.length} PDFs gefunden. Ermittle Seitenzahlen ...`);
+
+  let idx = 0;
+  for (const name of names) {
+    idx++;
+    // schon indexiert (mit gueltiger Seitenzahl)? dann nicht neu zaehlen
+    const existing = done.get(name);
+    if (existing && typeof existing["_pages_total"] === "number" && existing["_pages_total"]! >= 0 && !args.force) {
+      if (idx % 50 === 0) progress(`Index ${idx}/${names.length}`);
+      continue;
+    }
+    const pdfPath = pdfIndex.get(name)!;
+    const pages = await pdfPageCount(pdfPath);
+    const tooMany = pages < 0 ? false : pages > args.maxPages;
+    const prev = done.get(name) ?? { file_name: name, is_buyer_sheet: false };
+    done.set(name, {
+      ...prev,
+      file_name: name,
+      _pages_total: pages,
+      ...(pages < 0 ? { _index_error: "pdfinfo fehlgeschlagen (beschaedigt?)" } : {}),
+      ...(tooMany ? { _skipped_too_many_pages: true } : {}),
+    });
+    if (idx % 25 === 0 || idx === names.length) progress(`Index ${idx}/${names.length}`);
+  }
+  // Index sofort persistieren, bevor die teure Phase 2 startet
+  await fsp.writeFile(visionPath, JSON.stringify([...done.values()], null, 2));
+  process.stderr.write("\n");
+
+  // ---------------------------------------------------------------------
+  // PHASE 2 — INHALT: nur PDFs <= maxPages, noch nicht (fehlerfrei) erledigt.
+  // ---------------------------------------------------------------------
+  let candidates = [...done.values()].filter((r) => {
+    const pages = r["_pages_total"] as number | undefined;
+    return typeof pages === "number" && pages > 0 && pages <= args.maxPages;
+  });
  if (args.only) candidates = candidates.filter((r) => r.file_name === args.only);

-  // Resume-Skip VOR dem Limit: bereits fehlerfrei Verarbeitete zählen nicht mit
+  // Resume-Skip VOR dem Limit: fehlerfrei mit echtem Vision-Ergebnis = fertig
  const pending = candidates.filter((r) => {
    const prev = done.get(r.file_name);
-    return !(prev && !prev["_vision_error"] && !args.force);
+    const hasResult = prev && prev["_parser"] === "vision" && !prev["_vision_error"];
+    return !(hasResult && !args.force);
  });
  const skipped = candidates.length - pending.length;
  const targets = args.limit > 0 ? pending.slice(0, args.limit) : pending;

-  console.error(`Indexiere PDFs unter ${args.pdfRoot} ...`);
-  const pdfIndex = await buildPdfIndex(args.pdfRoot);
+  const tooManyCount = [...done.values()].filter((r) => r["_skipped_too_many_pages"]).length;
  console.error(
-    `${pdfIndex.size} PDFs gefunden. ${candidates.length} Kandidaten, ${skipped} bereits verarbeitet, ${targets.length} in diesem Lauf.`
+    `Phase 2: ${candidates.length} Kandidaten (<=${args.maxPages} Seiten), ` +
+    `${tooManyCount} zu gross (uebersprungen), ${skipped} bereits verarbeitet, ` +
+    `${targets.length} in diesem Lauf.`
  );

  const model = await fetchModelId(args.api);
@@ -560,21 +613,17 @@ async function main(): Promise<void> {
      await fsp.rm(tmpDir, { recursive: true, force: true });
    }

-    // Inkrementell sichern (Resume-fähig) — merged gleich mit
-    await fsp.mkdir(args.outDir, { recursive: true });
+    // Inkrementell sichern (Resume-fähig)
    await fsp.writeFile(visionPath, JSON.stringify([...done.values()], null, 2));
-    await fsp.writeFile(mergedPath, JSON.stringify(all.map((r) => done.get(r.file_name) ?? r), null, 2));
  }

-  // Merge: Gleis A + Gleis B
-  const merged = all.map((r) => done.get(r.file_name) ?? r);
-  await fsp.writeFile(mergedPath, JSON.stringify(merged, null, 2));
+  await fsp.writeFile(visionPath, JSON.stringify([...done.values()], null, 2));

  process.stderr.write("\n");
  console.error(
    `Fertig. OK: ${ok}, CA only: ${caOnly}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}`
  );
-  console.error(`→ ${visionPath}\n→ ${mergedPath}`);
+  console.error(`→ ${visionPath}`);
 }

 main().catch((e) => {