asdasd

.gitignore

@@ -1,4 +1,6 @@
 poc_out
 node_modules
 package-lock.json
-*.jsonl
+*.json
+*.log
+*.txt

bayarea-ai-server/docker-compose.yml

@@ -107,7 +107,7 @@ services:
       - -fa
       - "on"
       - -c
-      - "32768"
+      - "40960"
       - --parallel
       - "1"
       # KV-Cache quantisieren — 32GB VRAM, 35B-A3B + mmproj + Bildkontext:
@@ -121,7 +121,7 @@ services:
       - --image-min-tokens
       - "1024"
       - --image-max-tokens
-      - "4096"
+      - "6144"
       - --temp
       - "0.1"
       - --top-p

notes_filter.ts

@@ -0,0 +1,138 @@
+#!/usr/bin/env npx tsx
+/**
+ * notes_filter.ts — Notes-PDFs aussortieren und Seed-JSON erzeugen
+ *
+ * 1. Scannt alle PDFs unter --pdf-root rekursiv.
+ * 2. "Notes"-Dateien: prüft, ob eine Hauptdatei desselben Käufers existiert
+ *    (Schlüssel: "Nachname, Vorname" — Datum wird ignoriert, da es abweicht).
+ *    → mit Gegenstück: überspringen. Ohne Gegenstück: Report (out/notes_orphans.txt).
+ * 3. Schreibt Seed-JSON mit ALLEN Nicht-Notes-PDFs. Vorhandene Einträge aus
+ *    --buyers (Gleis A / Vision) werden unverändert übernommen, neue Dateien
+ *    bekommen einen Eintrag mit is_buyer_sheet:false (→ vision_runner nimmt sie).
+ *
+ * Aufruf:
+ *   npx tsx notes_filter.ts \
+ *     --pdf-root "/mnt/bizmatch-nas/AA Buyers NDA's/Buyers NDA's A-Z" \
+ *     --buyers out/buyers.json \
+ *     --out out/buyers_seed.json
+ */
+
+import * as fsp from "node:fs/promises";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+
+interface BuyerRecord { file_name: string; is_buyer_sheet: boolean; [k: string]: unknown }
+
+function parseArgs() {
+  const a = process.argv.slice(2);
+  const get = (f: string, d: string | null = null) => {
+    const i = a.indexOf(f);
+    return i >= 0 && a[i + 1] !== undefined ? a[i + 1] : d;
+  };
+  const pdfRoot = get("--pdf-root");
+  if (!pdfRoot) { console.error("Fehler: --pdf-root fehlt."); process.exit(1); }
+  return {
+    pdfRoot: pdfRoot.replace(/^~(?=\/)/, os.homedir()),
+    buyers: get("--buyers", "out/buyers.json")!,
+    out: get("--out", "out/buyers_seed.json")!,
+  };
+}
+
+/** Erkennt "Notes", "Note" und verklebte Varianten ("Notesb102312").
+ *  Es wird nur der Teil NACH dem ersten Komma durchsucht, damit Nachnamen
+ *  wie "Note, John" nicht fälschlich als Notes-Datei gelten. */
+function isNotes(base: string): boolean {
+  const comma = base.indexOf(",");
+  const scope = comma >= 0 ? base.slice(comma + 1) : base;
+  return /\bnotes?[a-z]?\d*\b/i.test(scope);
+}
+
+/** "Stone, Mike via Anna Stone 031220 Notes.pdf" → "stone,mike" */
+function nameKey(base: string): string {
+  let s = base.replace(/\.pdf$/i, "");
+  s = s.replace(/\bnotes?[a-z]?\d*\b/gi, " ");   // Note/Notes/Notesb102312 raus
+  s = s.replace(/\b\d{4,8}\b/g, " ");            // Datums-Tokens raus
+  s = s.replace(/\s+/g, " ").trim();
+  const comma = s.indexOf(",");
+  if (comma < 0) return s.toLowerCase();          // Fallback: ganzer Rest
+  const last = s.slice(0, comma).trim();
+  const first = (s.slice(comma + 1).trim().split(" ")[0] ?? "");
+  return `${last},${first}`.toLowerCase();
+}
+
+async function collectPdfs(root: string): Promise<string[]> {
+  const result: string[] = [];
+  const stack = [root];
+  while (stack.length) {
+    const dir = stack.pop()!;
+    let entries: fs.Dirent[];
+    try { entries = await fsp.readdir(dir, { withFileTypes: true }); } catch { continue; }
+    for (const e of entries) {
+      const p = path.join(dir, e.name);
+      if (e.isDirectory()) stack.push(p);
+      else if (e.isFile() && e.name.toLowerCase().endsWith(".pdf")) result.push(e.name);
+    }
+  }
+  return result;
+}
+
+const EMPTY_FIELDS = {
+  name_company: null, prospective_buyer: null, company: null, phone: null,
+  cell: null, email: null, address: null, state: null, how_did_you_hear: null,
+  interested_in_updates: null, types_of_business_raw: null,
+  background_experience: null, total_purchase_price: null, down_payment: null,
+  down_payment_raw: null, date_of_introduction: null,
+};
+
+async function main() {
+  const args = parseArgs();
+
+  console.error(`Scanne ${args.pdfRoot} ...`);
+  const all = await collectPdfs(args.pdfRoot);
+  const notesFiles = all.filter(isNotes);
+  const mainFiles = all.filter((f) => !isNotes(f));
+  console.error(`${all.length} PDFs: ${mainFiles.length} Hauptdateien, ${notesFiles.length} Notes-Dateien`);
+
+  // Notes-Abgleich
+  const mainKeys = new Set(mainFiles.map(nameKey));
+  const skippable: string[] = [];
+  const orphans: string[] = [];
+  for (const n of notesFiles) (mainKeys.has(nameKey(n)) ? skippable : orphans).push(n);
+  console.error(`Notes mit Gegenstück (werden ignoriert): ${skippable.length}`);
+  console.error(`Notes OHNE Gegenstück (bitte prüfen):    ${orphans.length}`);
+  if (orphans.length) {
+    const orphanPath = path.join(path.dirname(args.out), "notes_orphans.txt");
+    await fsp.mkdir(path.dirname(orphanPath), { recursive: true });
+    await fsp.writeFile(orphanPath, orphans.sort().join("\n") + "\n");
+    console.error(`→ ${orphanPath}`);
+  }
+
+  // Vorhandene Ergebnisse laden
+  const existing = new Map<string, BuyerRecord>();
+  try {
+    const prev: BuyerRecord[] = JSON.parse(await fsp.readFile(args.buyers, "utf8"));
+    for (const r of prev) existing.set(r.file_name, r);
+  } catch {
+    console.error(`Hinweis: ${args.buyers} nicht gefunden — starte mit leerem Bestand.`);
+  }
+
+  // In die Extraktion gehen: alle Hauptdateien + Notes OHNE Gegenstück.
+  // Notes MIT Gegenstück fallen weg (auch deren alte Einträge).
+  const includeFiles = [...mainFiles, ...orphans];
+  let seeded = 0, carried = 0;
+  const out: BuyerRecord[] = includeFiles.sort().map((f) => {
+    const prev = existing.get(f);
+    if (prev) { carried++; return prev; }
+    seeded++;
+    return { file_name: f, is_buyer_sheet: false, ...EMPTY_FIELDS, _parser: "seed" };
+  });
+
+  await fsp.mkdir(path.dirname(args.out), { recursive: true });
+  await fsp.writeFile(args.out, JSON.stringify(out, null, 2));
+  console.error(
+    `Seed geschrieben: ${out.length} Einträge (${carried} übernommen, ${seeded} neu, davon ${orphans.length} Orphan-Notes) → ${args.out}`
+  );
+}
+
+main().catch((e) => { console.error("Abbruch:", e instanceof Error ? e.message : e); process.exit(1); });

vision_runner.ts

@@ -93,7 +93,8 @@ interface BuyerRecord {
 
 /** Rohantwort des VLM — alles verbatim, Normalisierung erfolgt in TS. */
 interface VisionRaw {
-  is_buyer_sheet: boolean;
+  doc_type: "buyer_sheet" | "ca_only" | "other";
+  info_sheet_count: number;
   info_page: number | null;
   ca_page: number | null;
   name_company: string | null;
@@ -244,13 +245,15 @@ const RESPONSE_SCHEMA = {
   type: "object",
   additionalProperties: false,
   required: [
-    "is_buyer_sheet", "info_page", "ca_page", "name_company", "prospective_buyer",
-    "company", "phone", "cell", "email", "address", "state", "how_did_you_hear",
-    "interested_in_updates", "types_of_business_raw", "background_experience",
+    "doc_type", "info_sheet_count", "info_page", "ca_page", "name_company",
+    "prospective_buyer", "company", "phone", "cell", "email", "address",
+    "state", "how_did_you_hear", "interested_in_updates",
+    "types_of_business_raw", "background_experience",
     "total_purchase_price", "down_payment_raw", "date_of_introduction_raw",
   ],
   properties: {
-    is_buyer_sheet: { type: "boolean" },
+    doc_type: { type: "string", enum: ["buyer_sheet", "ca_only", "other"] },
+    info_sheet_count: { type: "integer" },
     info_page: nullableInt,
     ca_page: nullableInt,
     name_company: nullableString,
@@ -279,12 +282,16 @@ const SYSTEM_PROMPT =
 
 const USER_PROMPT = `You see all pages of one PDF, in order (image 1 = page 1).
 
-Task: Decide whether this document contains a business brokerage "BUYER INFORMATION SHEET" form, and if so, transcribe its fields.
+Task: Classify the document and transcribe form fields from a business brokerage "BUYER INFORMATION SHEET" package.
 
-1. is_buyer_sheet: true only if a page with the heading "BUYER INFORMATION SHEET" exists. If the document is something else (notes, listing, letter, other form), return is_buyer_sheet=false and null for every field.
-2. info_page: page number (1-based) of the BUYER INFORMATION SHEET page, else null.
-3. ca_page: page number of the confidentiality agreement page containing text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL", else null.
-4. From the info page, transcribe VERBATIM (exactly as written, do not normalize, do not expand abbreviations):
+1. doc_type:
+   - "buyer_sheet": at least one page with the heading "BUYER INFORMATION SHEET" exists.
+   - "ca_only": no such page, but a confidentiality agreement page exists (text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL").
+   - "other": neither (notes, listing, letter, other form) — then return null for every field and 0 for info_sheet_count.
+2. info_sheet_count: how many separate BUYER INFORMATION SHEET pages the document contains (some files contain two filled sheets). 0 if none. If more than one, transcribe the fields from the FIRST sheet only.
+3. info_page: page number (1-based) of the (first) BUYER INFORMATION SHEET page, else null.
+4. ca_page: page number of the (first) confidentiality agreement page, else null.
+5. If doc_type is "buyer_sheet", transcribe from the info page VERBATIM (exactly as written, do not normalize, do not expand abbreviations):
    - name_company: value of the "Name/Company" line
    - prospective_buyer: value of the "Prospective Buyer" line
    - company: value of a separate "Company" line if present
@@ -295,9 +302,12 @@ Task: Decide whether this document contains a business brokerage "BUYER INFORMAT
    - background_experience: "Background/Experience" (may span multiple lines)
    - total_purchase_price: "Total Purchase Price" as written
    - down_payment_raw: "Down Payment" as written (e.g. "$350,000", "1.5M", "TBD")
-5. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null.
+6. If doc_type is "ca_only": transcribe what the confidentiality agreement page offers — the prospective buyer's printed or signed name into prospective_buyer, plus address/phone/email if they appear on that page. Everything not present stays null.
+7. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null.
 
-A blank field, "N/A", "n", or an empty line = transcribe it as written; if truly empty, use null. Return only the JSON object.`;
+A blank field, "N/A", "n", or an empty line = transcribe it as written; if truly empty, use null.
+Handwriting rules: Transcribe handwritten values letter by letter — do NOT complete them from context or from other fields. Email addresses are the most reliable spelling source on the page: read the email character by character, and if a handwritten name is ambiguous (e.g. B vs D), prefer the spelling that appears in the email address. Never alter the email itself to match your reading of the name.
+Return only the JSON object.`;
 
 async function fetchModelId(api: string): Promise<string> {
   try {
@@ -363,14 +373,17 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco
   };
   delete (base as Record<string, unknown>)["_vision_error"];
 
-  if (!vis.is_buyer_sheet) {
-    return { ...base, is_buyer_sheet: false, _info_page: null, _ca_page: null };
+  if (vis.doc_type === "other") {
+    return { ...base, is_buyer_sheet: false, _doc_type: "other", _info_page: null, _ca_page: null };
   }
 
+  // buyer_sheet ODER ca_only: alles übernehmen, was das Dokument hergibt
   const dp = normDownPayment(vis.down_payment_raw);
   return {
     ...base,
-    is_buyer_sheet: true,
+    is_buyer_sheet: vis.doc_type === "buyer_sheet",
+    _doc_type: vis.doc_type,
+    _info_sheet_count: vis.info_sheet_count ?? (vis.doc_type === "buyer_sheet" ? 1 : 0),
     name_company: cleanStr(vis.name_company),
     prospective_buyer: cleanStr(vis.prospective_buyer),
     company: cleanStr(vis.company),
@@ -400,6 +413,11 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco
 const RETRIES = 3;
 
 function progress(line: string): void {
+  // Im Log-Modus (Pipe/tee): vollständige Zeilen mit Timestamp statt \r-Überschreiben
+  if (!process.stderr.isTTY) {
+    process.stderr.write(`[${new Date().toISOString()}] ${line}\n`);
+    return;
+  }
   const cols = process.stderr.columns ?? 120;
   process.stderr.write("\r" + line.slice(0, cols - 1).padEnd(cols - 1));
 }
@@ -416,9 +434,6 @@ async function main(): Promise<void> {
   }
 
   const all: BuyerRecord[] = JSON.parse(await fsp.readFile(args.input, "utf8"));
-  let targets = all.filter((r) => r.is_buyer_sheet === false);
-  if (args.only) targets = targets.filter((r) => r.file_name === args.only);
-  if (args.limit > 0) targets = targets.slice(0, args.limit);
 
   const visionPath = path.join(args.outDir, "buyers_vision.json");
   const mergedPath = path.join(args.outDir, "buyers_merged.json");
@@ -430,26 +445,32 @@ async function main(): Promise<void> {
     /* kein Resume-Stand */
   }
 
+  let candidates = all.filter((r) => r.is_buyer_sheet === false);
+  if (args.only) candidates = candidates.filter((r) => r.file_name === args.only);
+
+  // Resume-Skip VOR dem Limit: bereits fehlerfrei Verarbeitete zählen nicht mit
+  const pending = candidates.filter((r) => {
+    const prev = done.get(r.file_name);
+    return !(prev && !prev["_vision_error"] && !args.force);
+  });
+  const skipped = candidates.length - pending.length;
+  const targets = args.limit > 0 ? pending.slice(0, args.limit) : pending;
+
   console.error(`Indexiere PDFs unter ${args.pdfRoot} ...`);
   const pdfIndex = await buildPdfIndex(args.pdfRoot);
-  console.error(`${pdfIndex.size} PDFs gefunden. ${targets.length} Einträge zu verarbeiten.`);
+  console.error(
+    `${pdfIndex.size} PDFs gefunden. ${candidates.length} Kandidaten, ${skipped} bereits verarbeitet, ${targets.length} in diesem Lauf.`
+  );
 
   const model = await fetchModelId(args.api);
   console.error(`Modell: ${model} @ ${args.api}`);
 
-  let ok = 0, notSheet = 0, errors = 0, skipped = 0;
+  let ok = 0, caOnly = 0, notSheet = 0, errors = 0;
 
   for (let i = 0; i < targets.length; i++) {
     const det = targets[i];
     const tag = `[${i + 1}/${targets.length}] ${det.file_name}`;
 
-    const prev = done.get(det.file_name);
-    if (prev && !prev["_vision_error"] && !args.force) {
-      skipped++;
-      progress(`${tag} … übersprungen (bereits verarbeitet)`);
-      continue;
-    }
-
     const pdfPath = pdfIndex.get(det.file_name);
     if (!pdfPath) {
       done.set(det.file_name, { ...det, _vision_error: "PDF nicht gefunden", _vision_ts: new Date().toISOString() });
@@ -483,7 +504,8 @@ async function main(): Promise<void> {
       } else {
         const merged = mergeRecord(det, vis, model);
         done.set(det.file_name, merged);
-        if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); }
+        if (merged["_doc_type"] === "ca_only") { caOnly++; progress(`${tag} … CA only`); }
+        else if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); }
         else { notSheet++; progress(`${tag} … kein Buyer Sheet`); }
       }
     } catch (e) {
@@ -506,7 +528,7 @@ async function main(): Promise<void> {
 
   process.stderr.write("\n");
   console.error(
-    `Fertig. OK: ${ok}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}`
+    `Fertig. OK: ${ok}, CA only: ${caOnly}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}`
   );
   console.error(`→ ${visionPath}\n→ ${mergedPath}`);
 }