asdsa

.gitignore

@@ -1,4 +1,6 @@
 poc_out
 node_modules
 package-lock.json
-*.jsonl
+*.json
+*.log
+*.txt

notes_filter.ts

@@ -39,12 +39,19 @@ function parseArgs() {
   };
 }
 
-const isNotes = (base: string) => /\bnotes\b/i.test(base);
+/** Erkennt "Notes", "Note" und verklebte Varianten ("Notesb102312").
+ *  Es wird nur der Teil NACH dem ersten Komma durchsucht, damit Nachnamen
+ *  wie "Note, John" nicht fälschlich als Notes-Datei gelten. */
+function isNotes(base: string): boolean {
+  const comma = base.indexOf(",");
+  const scope = comma >= 0 ? base.slice(comma + 1) : base;
+  return /\bnotes?[a-z]?\d*\b/i.test(scope);
+}
 
 /** "Stone, Mike via Anna Stone 031220 Notes.pdf" → "stone,mike" */
 function nameKey(base: string): string {
   let s = base.replace(/\.pdf$/i, "");
-  s = s.replace(/\bnotes\b/gi, " ");
+  s = s.replace(/\bnotes?[a-z]?\d*\b/gi, " ");   // Note/Notes/Notesb102312 raus
   s = s.replace(/\b\d{4,8}\b/g, " ");            // Datums-Tokens raus
   s = s.replace(/\s+/g, " ").trim();
   const comma = s.indexOf(",");
@@ -101,27 +108,31 @@ async function main() {
     console.error(`→ ${orphanPath}`);
   }
 
-  // Vorhandene Ergebnisse übernehmen (Notes-Einträge dabei verwerfen)
+  // Vorhandene Ergebnisse laden
   const existing = new Map<string, BuyerRecord>();
   try {
     const prev: BuyerRecord[] = JSON.parse(await fsp.readFile(args.buyers, "utf8"));
-    for (const r of prev) if (!isNotes(r.file_name)) existing.set(r.file_name, r);
+    for (const r of prev) existing.set(r.file_name, r);
   } catch {
     console.error(`Hinweis: ${args.buyers} nicht gefunden — starte mit leerem Bestand.`);
   }
 
-  // Seed: alle Hauptdateien, vorhandene Daten bleiben erhalten
+  // In die Extraktion gehen: alle Hauptdateien + Notes OHNE Gegenstück.
-  let seeded = 0;
+  // Notes MIT Gegenstück fallen weg (auch deren alte Einträge).
-  const out: BuyerRecord[] = mainFiles.sort().map((f) => {
+  const includeFiles = [...mainFiles, ...orphans];
+  let seeded = 0, carried = 0;
+  const out: BuyerRecord[] = includeFiles.sort().map((f) => {
     const prev = existing.get(f);
-    if (prev) return prev;
+    if (prev) { carried++; return prev; }
     seeded++;
     return { file_name: f, is_buyer_sheet: false, ...EMPTY_FIELDS, _parser: "seed" };
   });
 
   await fsp.mkdir(path.dirname(args.out), { recursive: true });
   await fsp.writeFile(args.out, JSON.stringify(out, null, 2));
-  console.error(`Seed geschrieben: ${out.length} Einträge (${existing.size} übernommen, ${seeded} neu) → ${args.out}`);
+  console.error(
+    `Seed geschrieben: ${out.length} Einträge (${carried} übernommen, ${seeded} neu, davon ${orphans.length} Orphan-Notes) → ${args.out}`
+  );
 }
 
 main().catch((e) => { console.error("Abbruch:", e instanceof Error ? e.message : e); process.exit(1); });

vision_runner.ts

@@ -93,7 +93,8 @@ interface BuyerRecord {
 
 /** Rohantwort des VLM — alles verbatim, Normalisierung erfolgt in TS. */
 interface VisionRaw {
-  is_buyer_sheet: boolean;
+  doc_type: "buyer_sheet" | "ca_only" | "other";
+  info_sheet_count: number;
   info_page: number | null;
   ca_page: number | null;
   name_company: string | null;
@@ -244,13 +245,15 @@ const RESPONSE_SCHEMA = {
   type: "object",
   additionalProperties: false,
   required: [
-    "is_buyer_sheet", "info_page", "ca_page", "name_company", "prospective_buyer",
+    "doc_type", "info_sheet_count", "info_page", "ca_page", "name_company",
-    "company", "phone", "cell", "email", "address", "state", "how_did_you_hear",
+    "prospective_buyer", "company", "phone", "cell", "email", "address",
-    "interested_in_updates", "types_of_business_raw", "background_experience",
+    "state", "how_did_you_hear", "interested_in_updates",
+    "types_of_business_raw", "background_experience",
     "total_purchase_price", "down_payment_raw", "date_of_introduction_raw",
   ],
   properties: {
-    is_buyer_sheet: { type: "boolean" },
+    doc_type: { type: "string", enum: ["buyer_sheet", "ca_only", "other"] },
+    info_sheet_count: { type: "integer" },
     info_page: nullableInt,
     ca_page: nullableInt,
     name_company: nullableString,
@@ -279,12 +282,16 @@ const SYSTEM_PROMPT =
 
 const USER_PROMPT = `You see all pages of one PDF, in order (image 1 = page 1).
 
-Task: Decide whether this document contains a business brokerage "BUYER INFORMATION SHEET" form, and if so, transcribe its fields.
+Task: Classify the document and transcribe form fields from a business brokerage "BUYER INFORMATION SHEET" package.
 
-1. is_buyer_sheet: true only if a page with the heading "BUYER INFORMATION SHEET" exists. If the document is something else (notes, listing, letter, other form), return is_buyer_sheet=false and null for every field.
+1. doc_type:
-2. info_page: page number (1-based) of the BUYER INFORMATION SHEET page, else null.
+   - "buyer_sheet": at least one page with the heading "BUYER INFORMATION SHEET" exists.
-3. ca_page: page number of the confidentiality agreement page containing text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL", else null.
+   - "ca_only": no such page, but a confidentiality agreement page exists (text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL").
-4. From the info page, transcribe VERBATIM (exactly as written, do not normalize, do not expand abbreviations):
+   - "other": neither (notes, listing, letter, other form) — then return null for every field and 0 for info_sheet_count.
+2. info_sheet_count: how many separate BUYER INFORMATION SHEET pages the document contains (some files contain two filled sheets). 0 if none. If more than one, transcribe the fields from the FIRST sheet only.
+3. info_page: page number (1-based) of the (first) BUYER INFORMATION SHEET page, else null.
+4. ca_page: page number of the (first) confidentiality agreement page, else null.
+5. If doc_type is "buyer_sheet", transcribe from the info page VERBATIM (exactly as written, do not normalize, do not expand abbreviations):
    - name_company: value of the "Name/Company" line
    - prospective_buyer: value of the "Prospective Buyer" line
    - company: value of a separate "Company" line if present
@@ -295,7 +302,8 @@ Task: Decide whether this document contains a business brokerage "BUYER INFORMAT
    - background_experience: "Background/Experience" (may span multiple lines)
    - total_purchase_price: "Total Purchase Price" as written
    - down_payment_raw: "Down Payment" as written (e.g. "$350,000", "1.5M", "TBD")
-5. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null.
+6. If doc_type is "ca_only": transcribe what the confidentiality agreement page offers — the prospective buyer's printed or signed name into prospective_buyer, plus address/phone/email if they appear on that page. Everything not present stays null.
+7. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null.
 
 A blank field, "N/A", "n", or an empty line = transcribe it as written; if truly empty, use null. Return only the JSON object.`;
 
@@ -363,14 +371,17 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco
   };
   delete (base as Record<string, unknown>)["_vision_error"];
 
-  if (!vis.is_buyer_sheet) {
+  if (vis.doc_type === "other") {
-    return { ...base, is_buyer_sheet: false, _info_page: null, _ca_page: null };
+    return { ...base, is_buyer_sheet: false, _doc_type: "other", _info_page: null, _ca_page: null };
   }
 
+  // buyer_sheet ODER ca_only: alles übernehmen, was das Dokument hergibt
   const dp = normDownPayment(vis.down_payment_raw);
   return {
     ...base,
-    is_buyer_sheet: true,
+    is_buyer_sheet: vis.doc_type === "buyer_sheet",
+    _doc_type: vis.doc_type,
+    _info_sheet_count: vis.info_sheet_count ?? (vis.doc_type === "buyer_sheet" ? 1 : 0),
     name_company: cleanStr(vis.name_company),
     prospective_buyer: cleanStr(vis.prospective_buyer),
     company: cleanStr(vis.company),
@@ -400,6 +411,11 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco
 const RETRIES = 3;
 
 function progress(line: string): void {
+  // Im Log-Modus (Pipe/tee): vollständige Zeilen mit Timestamp statt \r-Überschreiben
+  if (!process.stderr.isTTY) {
+    process.stderr.write(`[${new Date().toISOString()}] ${line}\n`);
+    return;
+  }
   const cols = process.stderr.columns ?? 120;
   process.stderr.write("\r" + line.slice(0, cols - 1).padEnd(cols - 1));
 }
@@ -416,9 +432,6 @@ async function main(): Promise<void> {
   }
 
   const all: BuyerRecord[] = JSON.parse(await fsp.readFile(args.input, "utf8"));
-  let targets = all.filter((r) => r.is_buyer_sheet === false);
-  if (args.only) targets = targets.filter((r) => r.file_name === args.only);
-  if (args.limit > 0) targets = targets.slice(0, args.limit);
 
   const visionPath = path.join(args.outDir, "buyers_vision.json");
   const mergedPath = path.join(args.outDir, "buyers_merged.json");
@@ -430,26 +443,32 @@ async function main(): Promise<void> {
     /* kein Resume-Stand */
   }
 
+  let candidates = all.filter((r) => r.is_buyer_sheet === false);
+  if (args.only) candidates = candidates.filter((r) => r.file_name === args.only);
+
+  // Resume-Skip VOR dem Limit: bereits fehlerfrei Verarbeitete zählen nicht mit
+  const pending = candidates.filter((r) => {
+    const prev = done.get(r.file_name);
+    return !(prev && !prev["_vision_error"] && !args.force);
+  });
+  const skipped = candidates.length - pending.length;
+  const targets = args.limit > 0 ? pending.slice(0, args.limit) : pending;
+
   console.error(`Indexiere PDFs unter ${args.pdfRoot} ...`);
   const pdfIndex = await buildPdfIndex(args.pdfRoot);
-  console.error(`${pdfIndex.size} PDFs gefunden. ${targets.length} Einträge zu verarbeiten.`);
+  console.error(
+    `${pdfIndex.size} PDFs gefunden. ${candidates.length} Kandidaten, ${skipped} bereits verarbeitet, ${targets.length} in diesem Lauf.`
+  );
 
   const model = await fetchModelId(args.api);
   console.error(`Modell: ${model} @ ${args.api}`);
 
-  let ok = 0, notSheet = 0, errors = 0, skipped = 0;
+  let ok = 0, caOnly = 0, notSheet = 0, errors = 0;
 
   for (let i = 0; i < targets.length; i++) {
     const det = targets[i];
     const tag = `[${i + 1}/${targets.length}] ${det.file_name}`;
 
-    const prev = done.get(det.file_name);
-    if (prev && !prev["_vision_error"] && !args.force) {
-      skipped++;
-      progress(`${tag} … übersprungen (bereits verarbeitet)`);
-      continue;
-    }
-
     const pdfPath = pdfIndex.get(det.file_name);
     if (!pdfPath) {
       done.set(det.file_name, { ...det, _vision_error: "PDF nicht gefunden", _vision_ts: new Date().toISOString() });
@@ -483,7 +502,8 @@ async function main(): Promise<void> {
       } else {
         const merged = mergeRecord(det, vis, model);
         done.set(det.file_name, merged);
-        if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); }
+        if (merged["_doc_type"] === "ca_only") { caOnly++; progress(`${tag} … CA only`); }
+        else if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); }
         else { notSheet++; progress(`${tag} … kein Buyer Sheet`); }
       }
     } catch (e) {
@@ -506,7 +526,7 @@ async function main(): Promise<void> {
 
   process.stderr.write("\n");
   console.error(
-    `Fertig. OK: ${ok}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}`
+    `Fertig. OK: ${ok}, CA only: ${caOnly}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}`
   );
   console.error(`→ ${visionPath}\n→ ${mergedPath}`);
 }