aknuth/ai-bayarea
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

asdsa

Browse Source
This commit is contained in:
Andreas Knuth
2026-07-12 17:34:29 -05:00
parent f44d998a7f
commit 82cca38f29
3 changed files with 71 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -1,4 +1,6 @@
poc_out
node_modules
package-lock.json
*.jsonl
*.json
*.log
*.txt

29
notes_filter.ts
View File

@@ -39,12 +39,19 @@ function parseArgs() {
};
}
const isNotes = (base: string) => /\bnotes\b/i.test(base);
/** Erkennt "Notes", "Note" und verklebte Varianten ("Notesb102312").
* Es wird nur der Teil NACH dem ersten Komma durchsucht, damit Nachnamen
* wie "Note, John" nicht fälschlich als Notes-Datei gelten. */
function isNotes(base: string): boolean {
const comma = base.indexOf(",");
const scope = comma >= 0 ? base.slice(comma + 1) : base;
return /\bnotes?[a-z]?\d*\b/i.test(scope);
}
/** "Stone, Mike via Anna Stone 031220 Notes.pdf" → "stone,mike" */
function nameKey(base: string): string {
let s = base.replace(/\.pdf$/i, "");
s = s.replace(/\bnotes\b/gi, " ");
s = s.replace(/\bnotes?[a-z]?\d*\b/gi, " "); // Note/Notes/Notesb102312 raus
s = s.replace(/\b\d{4,8}\b/g, " "); // Datums-Tokens raus
s = s.replace(/\s+/g, " ").trim();
const comma = s.indexOf(",");
@@ -101,27 +108,31 @@ async function main() {
console.error(`→ ${orphanPath}`);
}
// Vorhandene Ergebnisse übernehmen (Notes-Einträge dabei verwerfen)
// Vorhandene Ergebnisse laden
const existing = new Map<string, BuyerRecord>();
try {
const prev: BuyerRecord[] = JSON.parse(await fsp.readFile(args.buyers, "utf8"));
for (const r of prev) if (!isNotes(r.file_name)) existing.set(r.file_name, r);
for (const r of prev) existing.set(r.file_name, r);
} catch {
console.error(`Hinweis: ${args.buyers} nicht gefunden — starte mit leerem Bestand.`);
}
// Seed: alle Hauptdateien, vorhandene Daten bleiben erhalten
let seeded = 0;
const out: BuyerRecord[] = mainFiles.sort().map((f) => {
// In die Extraktion gehen: alle Hauptdateien + Notes OHNE Gegenstück.
// Notes MIT Gegenstück fallen weg (auch deren alte Einträge).
const includeFiles = [...mainFiles, ...orphans];
let seeded = 0, carried = 0;
const out: BuyerRecord[] = includeFiles.sort().map((f) => {
const prev = existing.get(f);
if (prev) return prev;
if (prev) { carried++; return prev; }
seeded++;
return { file_name: f, is_buyer_sheet: false, ...EMPTY_FIELDS, _parser: "seed" };
});
await fsp.mkdir(path.dirname(args.out), { recursive: true });
await fsp.writeFile(args.out, JSON.stringify(out, null, 2));
console.error(`Seed geschrieben: ${out.length} Einträge (${existing.size} übernommen, ${seeded} neu) → ${args.out}`);
console.error(
`Seed geschrieben: ${out.length} Einträge (${carried} übernommen, ${seeded} neu, davon ${orphans.length} Orphan-Notes) → ${args.out}`
);
}
main().catch((e) => { console.error("Abbruch:", e instanceof Error ? e.message : e); process.exit(1); });

76
vision_runner.ts
View File

@@ -93,7 +93,8 @@ interface BuyerRecord {
/** Rohantwort des VLM — alles verbatim, Normalisierung erfolgt in TS. */
interface VisionRaw {
is_buyer_sheet: boolean;
doc_type: "buyer_sheet" | "ca_only" | "other";
info_sheet_count: number;
info_page: number | null;
ca_page: number | null;
name_company: string | null;
@@ -244,13 +245,15 @@ const RESPONSE_SCHEMA = {
type: "object",
additionalProperties: false,
required: [
"is_buyer_sheet", "info_page", "ca_page", "name_company", "prospective_buyer",
"company", "phone", "cell", "email", "address", "state", "how_did_you_hear",
"interested_in_updates", "types_of_business_raw", "background_experience",
"doc_type", "info_sheet_count", "info_page", "ca_page", "name_company",
"prospective_buyer", "company", "phone", "cell", "email", "address",
"state", "how_did_you_hear", "interested_in_updates",
"types_of_business_raw", "background_experience",
"total_purchase_price", "down_payment_raw", "date_of_introduction_raw",
],
properties: {
is_buyer_sheet: { type: "boolean" },
doc_type: { type: "string", enum: ["buyer_sheet", "ca_only", "other"] },
info_sheet_count: { type: "integer" },
info_page: nullableInt,
ca_page: nullableInt,
name_company: nullableString,
@@ -279,12 +282,16 @@ const SYSTEM_PROMPT =
const USER_PROMPT = `You see all pages of one PDF, in order (image 1 = page 1).
Task: Decide whether this document contains a business brokerage "BUYER INFORMATION SHEET" form, and if so, transcribe its fields.
Task: Classify the document and transcribe form fields from a business brokerage "BUYER INFORMATION SHEET" package.
1. is_buyer_sheet: true only if a page with the heading "BUYER INFORMATION SHEET" exists. If the document is something else (notes, listing, letter, other form), return is_buyer_sheet=false and null for every field.
2. info_page: page number (1-based) of the BUYER INFORMATION SHEET page, else null.
3. ca_page: page number of the confidentiality agreement page containing text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL", else null.
4. From the info page, transcribe VERBATIM (exactly as written, do not normalize, do not expand abbreviations):
1. doc_type:
- "buyer_sheet": at least one page with the heading "BUYER INFORMATION SHEET" exists.
- "ca_only": no such page, but a confidentiality agreement page exists (text like "PROSPECTIVE BUYER AGREES TO KEEP AND HOLD CONFIDENTIAL").
- "other": neither (notes, listing, letter, other form) — then return null for every field and 0 for info_sheet_count.
2. info_sheet_count: how many separate BUYER INFORMATION SHEET pages the document contains (some files contain two filled sheets). 0 if none. If more than one, transcribe the fields from the FIRST sheet only.
3. info_page: page number (1-based) of the (first) BUYER INFORMATION SHEET page, else null.
4. ca_page: page number of the (first) confidentiality agreement page, else null.
5. If doc_type is "buyer_sheet", transcribe from the info page VERBATIM (exactly as written, do not normalize, do not expand abbreviations):
- name_company: value of the "Name/Company" line
- prospective_buyer: value of the "Prospective Buyer" line
- company: value of a separate "Company" line if present
@@ -295,7 +302,8 @@ Task: Decide whether this document contains a business brokerage "BUYER INFORMAT
- background_experience: "Background/Experience" (may span multiple lines)
- total_purchase_price: "Total Purchase Price" as written
- down_payment_raw: "Down Payment" as written (e.g. "$350,000", "1.5M", "TBD")
5. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null.
6. If doc_type is "ca_only": transcribe what the confidentiality agreement page offers — the prospective buyer's printed or signed name into prospective_buyer, plus address/phone/email if they appear on that page. Everything not present stays null.
7. date_of_introduction_raw: the date written next to the buyer's signature on the confidentiality agreement page, verbatim (e.g. "6/25/26"), else null.
A blank field, "N/A", "n", or an empty line = transcribe it as written; if truly empty, use null. Return only the JSON object.`;
@@ -363,14 +371,17 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco
};
delete (base as Record<string, unknown>)["_vision_error"];
if (!vis.is_buyer_sheet) {
return { ...base, is_buyer_sheet: false, _info_page: null, _ca_page: null };
if (vis.doc_type === "other") {
return { ...base, is_buyer_sheet: false, _doc_type: "other", _info_page: null, _ca_page: null };
}
// buyer_sheet ODER ca_only: alles übernehmen, was das Dokument hergibt
const dp = normDownPayment(vis.down_payment_raw);
return {
...base,
is_buyer_sheet: true,
is_buyer_sheet: vis.doc_type === "buyer_sheet",
_doc_type: vis.doc_type,
_info_sheet_count: vis.info_sheet_count ?? (vis.doc_type === "buyer_sheet" ? 1 : 0),
name_company: cleanStr(vis.name_company),
prospective_buyer: cleanStr(vis.prospective_buyer),
company: cleanStr(vis.company),
@@ -400,6 +411,11 @@ function mergeRecord(det: BuyerRecord, vis: VisionRaw, model: string): BuyerReco
const RETRIES = 3;
function progress(line: string): void {
// Im Log-Modus (Pipe/tee): vollständige Zeilen mit Timestamp statt \r-Überschreiben
if (!process.stderr.isTTY) {
process.stderr.write(`[${new Date().toISOString()}] ${line}\n`);
return;
}
const cols = process.stderr.columns ?? 120;
process.stderr.write("\r" + line.slice(0, cols - 1).padEnd(cols - 1));
}
@@ -416,9 +432,6 @@ async function main(): Promise<void> {
}
const all: BuyerRecord[] = JSON.parse(await fsp.readFile(args.input, "utf8"));
let targets = all.filter((r) => r.is_buyer_sheet === false);
if (args.only) targets = targets.filter((r) => r.file_name === args.only);
if (args.limit > 0) targets = targets.slice(0, args.limit);
const visionPath = path.join(args.outDir, "buyers_vision.json");
const mergedPath = path.join(args.outDir, "buyers_merged.json");
@@ -430,26 +443,32 @@ async function main(): Promise<void> {
/* kein Resume-Stand */
}
let candidates = all.filter((r) => r.is_buyer_sheet === false);
if (args.only) candidates = candidates.filter((r) => r.file_name === args.only);
// Resume-Skip VOR dem Limit: bereits fehlerfrei Verarbeitete zählen nicht mit
const pending = candidates.filter((r) => {
const prev = done.get(r.file_name);
return !(prev && !prev["_vision_error"] && !args.force);
});
const skipped = candidates.length - pending.length;
const targets = args.limit > 0 ? pending.slice(0, args.limit) : pending;
console.error(`Indexiere PDFs unter ${args.pdfRoot} ...`);
const pdfIndex = await buildPdfIndex(args.pdfRoot);
console.error(`${pdfIndex.size} PDFs gefunden. ${targets.length} Einträge zu verarbeiten.`);
console.error(
`${pdfIndex.size} PDFs gefunden. ${candidates.length} Kandidaten, ${skipped} bereits verarbeitet, ${targets.length} in diesem Lauf.`
);
const model = await fetchModelId(args.api);
console.error(`Modell: ${model} @ ${args.api}`);
let ok = 0, notSheet = 0, errors = 0, skipped = 0;
let ok = 0, caOnly = 0, notSheet = 0, errors = 0;
for (let i = 0; i < targets.length; i++) {
const det = targets[i];
const tag = `[${i + 1}/${targets.length}] ${det.file_name}`;
const prev = done.get(det.file_name);
if (prev && !prev["_vision_error"] && !args.force) {
skipped++;
progress(`${tag} … übersprungen (bereits verarbeitet)`);
continue;
}
const pdfPath = pdfIndex.get(det.file_name);
if (!pdfPath) {
done.set(det.file_name, { ...det, _vision_error: "PDF nicht gefunden", _vision_ts: new Date().toISOString() });
@@ -483,7 +502,8 @@ async function main(): Promise<void> {
} else {
const merged = mergeRecord(det, vis, model);
done.set(det.file_name, merged);
if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); }
if (merged["_doc_type"] === "ca_only") { caOnly++; progress(`${tag} … CA only`); }
else if (merged.is_buyer_sheet) { ok++; progress(`${tag} … OK`); }
else { notSheet++; progress(`${tag} … kein Buyer Sheet`); }
}
} catch (e) {
@@ -506,7 +526,7 @@ async function main(): Promise<void> {
process.stderr.write("\n");
console.error(
`Fertig. OK: ${ok}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}`
`Fertig. OK: ${ok}, CA only: ${caOnly}, kein Buyer Sheet: ${notSheet}, Fehler: ${errors}, übersprungen: ${skipped}`
);
console.error(`→ ${visionPath}\n→ ${mergedPath}`);
}