diff --git a/.gitignore b/.gitignore index d82c806..d181542 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ poc_out node_modules package-lock.json -*.jsonl \ No newline at end of file +*.json +*.log +*.txt \ No newline at end of file diff --git a/notes_filter.ts b/notes_filter.ts index 2fe1e79..1968d56 100644 --- a/notes_filter.ts +++ b/notes_filter.ts @@ -39,12 +39,19 @@ function parseArgs() { }; } -const isNotes = (base: string) => /\bnotes\b/i.test(base); +/** Erkennt "Notes", "Note" und verklebte Varianten ("Notesb102312"). + * Es wird nur der Teil NACH dem ersten Komma durchsucht, damit Nachnamen + * wie "Note, John" nicht fälschlich als Notes-Datei gelten. */ +function isNotes(base: string): boolean { + const comma = base.indexOf(","); + const scope = comma >= 0 ? base.slice(comma + 1) : base; + return /\bnotes?[a-z]?\d*\b/i.test(scope); +} /** "Stone, Mike via Anna Stone 031220 Notes.pdf" → "stone,mike" */ function nameKey(base: string): string { let s = base.replace(/\.pdf$/i, ""); - s = s.replace(/\bnotes\b/gi, " "); + s = s.replace(/\bnotes?[a-z]?\d*\b/gi, " "); // Note/Notes/Notesb102312 raus s = s.replace(/\b\d{4,8}\b/g, " "); // Datums-Tokens raus s = s.replace(/\s+/g, " ").trim(); const comma = s.indexOf(","); @@ -101,27 +108,31 @@ async function main() { console.error(`→ ${orphanPath}`); } - // Vorhandene Ergebnisse übernehmen (Notes-Einträge dabei verwerfen) + // Vorhandene Ergebnisse laden const existing = new Map(); try { const prev: BuyerRecord[] = JSON.parse(await fsp.readFile(args.buyers, "utf8")); - for (const r of prev) if (!isNotes(r.file_name)) existing.set(r.file_name, r); + for (const r of prev) existing.set(r.file_name, r); } catch { console.error(`Hinweis: ${args.buyers} nicht gefunden — starte mit leerem Bestand.`); } - // Seed: alle Hauptdateien, vorhandene Daten bleiben erhalten - let seeded = 0; - const out: BuyerRecord[] = mainFiles.sort().map((f) => { + // In die Extraktion gehen: alle Hauptdateien + Notes OHNE Gegenstück. + // Notes MIT Gegenstück fallen weg (auch deren alte Einträge). + const includeFiles = [...mainFiles, ...orphans]; + let seeded = 0, carried = 0; + const out: BuyerRecord[] = includeFiles.sort().map((f) => { const prev = existing.get(f); - if (prev) return prev; + if (prev) { carried++; return prev; } seeded++; return { file_name: f, is_buyer_sheet: false, ...EMPTY_FIELDS, _parser: "seed" }; }); await fsp.mkdir(path.dirname(args.out), { recursive: true }); await fsp.writeFile(args.out, JSON.stringify(out, null, 2)); - console.error(`Seed geschrieben: ${out.length} Einträge (${existing.size} übernommen, ${seeded} neu) → ${args.out}`); + console.error( + `Seed geschrieben: ${out.length} Einträge (${carried} übernommen, ${seeded} neu, davon ${orphans.length} Orphan-Notes) → ${args.out}` + ); } main().catch((e) => { console.error("Abbruch:", e instanceof Error ? e.message : e); process.exit(1); }); \ No newline at end of file diff --git a/vision_runner.ts b/vision_runner.ts index 1f5f53d..26e9387 100644 --- a/vision_runner.ts +++ b/vision_runner.ts @@ -93,7 +93,8 @@ interface BuyerRecord { /** Rohantwort des VLM — alles verbatim, Normalisierung erfolgt in TS. */ interface VisionRaw { - is_buyer_sheet: boolean; + doc_type: "buyer_sheet" | "ca_only" | "other"; + info_sheet_count: number; info_page: number | null; ca_page: number | null; name_company: string | null; @@ -244,13 +245,15 @@ const RESPONSE_SCHEMA = { type: "object", additionalProperties: false, required: [ - "is_buyer_sheet", "info_page", "ca_page", "name_company", "prospective_buyer", - "company", "phone", "cell", "email", "address", "state", "how_did_you_hear", - "interested_in_updates", "types_of_business_raw", "background_experience", + "doc_type", "info_sheet_count", "info_page", "ca_page", "name_company", + "prospective_buyer", "company", "phone", "cell", "email", "address", + "state", "how_did_you_hear", "interested_in_updates", + "types_of_business_raw", "background_experience", "total_purchase_price", "down_payment_raw", "date_of_introduction_raw", ], properties: { - is_buyer_sheet: { type: "boolean" }, + doc_type: { type: "string", enum: ["buyer_sheet", "ca_only", "other"] }, + info_sheet_count: { type: "integer" }, info_page: nullableInt, ca_page: nullableInt, name_company: nullableString, @@ -279,12 +282,16 @@ const SYSTEM_PROMPT = const USER_PROMPT = `You see all pages of one PDF, in order (image 1 = page 1). -Task: Decide whether this document contains a business brokerage "BUYER INFORMATION SHEET" form, and if so, transcribe its fields. +Task: Classify the document and transcribe form fields from a business brokerage "BUYER INFORMATION SHEET" package. -1. is_buyer_sheet: true only if a page with the heading "BUYER INFORMATION SHEET" exists. If the document is something else (notes, listing, letter, other form), return is_buyer_sheet=false and null for every field. -2. info_page: page number (1-based) of the BUYER INFORMATION SHEET page, else null. -3. ca_page: page number of the confidentiality agreement page containing text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL", else null. -4. From the info page, transcribe VERBATIM (exactly as written, do not normalize, do not expand abbreviations): +1. doc_type: + - "buyer_sheet": at least one page with the heading "BUYER INFORMATION SHEET" exists. + - "ca_only": no such page, but a confidentiality agreement page exists (text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL"). + - "other": neither (notes, listing, letter, other form) — then return null for every field and 0 for info_sheet_count. +2. info_sheet_count: how many separate BUYER INFORMATION SHEET pages the document contains (some files contain two filled sheets). 0 if none. If more than one, transcribe the fields from the FIRST sheet only. +3. info_page: page number (1-based) of the (first) BUYER INFORMATION SHEET page, else null. +4. ca_page: page number of the (first) confidentiality agreement page, else null. +5. If doc_type is "buyer_sheet", transcribe from the info page VERBATIM (exactly as written, do not normalize, do not expand abbreviations): - name_company: value of the "Name/Company" line - prospective_buyer: value of the "Prospective Buyer" line - company: value of a separate "Company" line if present @@ -295,7 +302,8 @@ Task: Decide whether this document contains a business brokerage "BUYER INFORMAT - background_experience: "Background/Experience" (may span multiple lines) - total_purchase_price: "Total Purchase Price" as written - down_payment_raw: "Down Payment" as written (e.g. "$350,000", "1.5M", "TBD") -5. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null. +6. If doc_type is "ca_only": transcribe what the confidentiality agreement page offers — the prospective buyer's printed or signed name into prospective_buyer, plus address/phone/email if they appear on that page. Everything not present stays null. +7. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null. A blank field, "N/A", "n", or an empty line = transcribe it as written; if truly empty, use null. Return only the JSON object.`; @@ -363,14 +371,17 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco }; delete (base as Record)["_vision_error"]; - if (!vis.is_buyer_sheet) { - return { ...base, is_buyer_sheet: false, _info_page: null, _ca_page: null }; + if (vis.doc_type === "other") { + return { ...base, is_buyer_sheet: false, _doc_type: "other", _info_page: null, _ca_page: null }; } + // buyer_sheet ODER ca_only: alles übernehmen, was das Dokument hergibt const dp = normDownPayment(vis.down_payment_raw); return { ...base, - is_buyer_sheet: true, + is_buyer_sheet: vis.doc_type === "buyer_sheet", + _doc_type: vis.doc_type, + _info_sheet_count: vis.info_sheet_count ?? (vis.doc_type === "buyer_sheet" ? 1 : 0), name_company: cleanStr(vis.name_company), prospective_buyer: cleanStr(vis.prospective_buyer), company: cleanStr(vis.company), @@ -400,6 +411,11 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco const RETRIES = 3; function progress(line: string): void { + // Im Log-Modus (Pipe/tee): vollständige Zeilen mit Timestamp statt \r-Überschreiben + if (!process.stderr.isTTY) { + process.stderr.write(`[${new Date().toISOString()}] ${line}\n`); + return; + } const cols = process.stderr.columns ?? 120; process.stderr.write("\r" + line.slice(0, cols - 1).padEnd(cols - 1)); } @@ -416,9 +432,6 @@ async function main(): Promise { } const all: BuyerRecord[] = JSON.parse(await fsp.readFile(args.input, "utf8")); - let targets = all.filter((r) => r.is_buyer_sheet === false); - if (args.only) targets = targets.filter((r) => r.file_name === args.only); - if (args.limit > 0) targets = targets.slice(0, args.limit); const visionPath = path.join(args.outDir, "buyers_vision.json"); const mergedPath = path.join(args.outDir, "buyers_merged.json"); @@ -430,26 +443,32 @@ async function main(): Promise { /* kein Resume-Stand */ } + let candidates = all.filter((r) => r.is_buyer_sheet === false); + if (args.only) candidates = candidates.filter((r) => r.file_name === args.only); + + // Resume-Skip VOR dem Limit: bereits fehlerfrei Verarbeitete zählen nicht mit + const pending = candidates.filter((r) => { + const prev = done.get(r.file_name); + return !(prev && !prev["_vision_error"] && !args.force); + }); + const skipped = candidates.length - pending.length; + const targets = args.limit > 0 ? pending.slice(0, args.limit) : pending; + console.error(`Indexiere PDFs unter ${args.pdfRoot} ...`); const pdfIndex = await buildPdfIndex(args.pdfRoot); - console.error(`${pdfIndex.size} PDFs gefunden. ${targets.length} Einträge zu verarbeiten.`); + console.error( + `${pdfIndex.size} PDFs gefunden. ${candidates.length} Kandidaten, ${skipped} bereits verarbeitet, ${targets.length} in diesem Lauf.` + ); const model = await fetchModelId(args.api); console.error(`Modell: ${model} @ ${args.api}`); - let ok = 0, notSheet = 0, errors = 0, skipped = 0; + let ok = 0, caOnly = 0, notSheet = 0, errors = 0; for (let i = 0; i < targets.length; i++) { const det = targets[i]; const tag = `[${i + 1}/${targets.length}] ${det.file_name}`; - const prev = done.get(det.file_name); - if (prev && !prev["_vision_error"] && !args.force) { - skipped++; - progress(`${tag} … übersprungen (bereits verarbeitet)`); - continue; - } - const pdfPath = pdfIndex.get(det.file_name); if (!pdfPath) { done.set(det.file_name, { ...det, _vision_error: "PDF nicht gefunden", _vision_ts: new Date().toISOString() }); @@ -483,7 +502,8 @@ async function main(): Promise { } else { const merged = mergeRecord(det, vis, model); done.set(det.file_name, merged); - if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); } + if (merged["_doc_type"] === "ca_only") { caOnly++; progress(`${tag} … CA only`); } + else if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); } else { notSheet++; progress(`${tag} … kein Buyer Sheet`); } } } catch (e) { @@ -506,7 +526,7 @@ async function main(): Promise { process.stderr.write("\n"); console.error( - `Fertig. OK: ${ok}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}` + `Fertig. OK: ${ok}, CA only: ${caOnly}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}` ); console.error(`→ ${visionPath}\n→ ${mergedPath}`); }