From b8c862885e6cf46a5857adac27c1220416b7d0b4 Mon Sep 17 00:00:00 2001 From: Andreas Knuth Date: Sat, 11 Jul 2026 15:56:14 -0500 Subject: [PATCH] typescript ansatz --- .gitignore | 5 +- BuyerSheetParser.ts | 184 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 13 ++++ run_parser.ts | 28 +++++++ 4 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 BuyerSheetParser.ts create mode 100644 package.json create mode 100644 run_parser.ts diff --git a/.gitignore b/.gitignore index 5797656..d82c806 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -poc_out \ No newline at end of file +poc_out +node_modules +package-lock.json +*.jsonl \ No newline at end of file diff --git a/BuyerSheetParser.ts b/BuyerSheetParser.ts new file mode 100644 index 0000000..627a7ac --- /dev/null +++ b/BuyerSheetParser.ts @@ -0,0 +1,184 @@ +import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.js'; +import * as fs from 'fs'; + +export interface BuyerSheetData { + is_buyer_sheet: boolean; + name_company: string | null; + prospective_buyer: string | null; + company: string | null; + phone: string | null; + cell: string | null; + email: string | null; + address: string | null; + state: string | null; + how_did_you_hear: string | null; + interested_in_updates: boolean | null; + types_of_business_raw: string | null; + background_experience: string | null; + total_purchase_price: string | null; + down_payment: string | null; + down_payment_raw: string | null; + date_of_introduction: string | null; + _checkbox_pending: boolean; + _parser: string; + _info_page: number; + _ca_page: number; +} + +const LABELS = [ + "NAME / COMPANY", "PHONE", "ADDRESS", "EMAIL ADDRESS", + "HOW DID YOU HEAR", "ARE YOU INTERESTED", "TYPES OF BUSINESSES", + "BACKGROUND", "TOTAL PURCHASE PRICE", "DOWN PAYMENT", + "INCOME REQUIREMENTS", "ACCOUNTANT", "ATTORNEY", "BANK", +]; + +export class DeterministicParser { + + public async parsePdf(filePath: string): Promise { + const dataBuffer = fs.readFileSync(filePath); + const loadingTask = pdfjsLib.getDocument({ data: new Uint8Array(dataBuffer) }); + const pdfDocument = await loadingTask.promise; + const page = await pdfDocument.getPage(1); + + const textContent = await page.getTextContent(); + + // 1. Textfragmente mit X, Y und Breite (Width) auslesen + const mappedItems = textContent.items + .filter((item: any) => item.str.trim() !== '') + .map((item: any) => ({ + text: item.str, + x: item.transform[4], + y: item.transform[5], + width: item.width + })) + .sort((a, b) => b.y - a.y || a.x - b.x); // Y absteigend (oben nach unten) + + // 2. Zeilen bilden (Y-Toleranz) + const linesGrouped: {y: number, items: any[]}[] = []; + let currentY: number | null = null; + let currentItems = []; + + for (const item of mappedItems) { + if (currentY === null || Math.abs(item.y - currentY) <= 3) { + currentItems.push(item); + currentY = currentY === null ? item.y : currentY; + } else { + linesGrouped.push({ y: currentY, items: currentItems }); + currentItems = [item]; + currentY = item.y; + } + } + if (currentItems.length > 0 && currentY !== null) { + linesGrouped.push({ y: currentY, items: currentItems }); + } + + // 3. Zerrissene Wörter reparieren (Kerning) + const lines: { y: number, text: string }[] = []; + for (const group of linesGrouped) { + group.items.sort((a, b) => a.x - b.x); + let lineStr = ""; + let prevEnd = -1; + + for (const item of group.items) { + if (prevEnd !== -1) { + const gap = item.x - prevEnd; + // Wenn die Lücke größer als ~4 Pixel ist, ist es ein echtes Leerzeichen + if (gap > 4) { + lineStr += " "; + } + } + lineStr += item.text; + prevEnd = item.x + item.width; + } + lines.push({ y: group.y, text: lineStr }); + } + + // 4. Y-Intervall Parsing (Die "Nutzer-Idee") + const rawResults: Record = {}; + LABELS.forEach(lbl => rawResults[lbl] = []); + + let currentLabel: string | null = null; + // Längste Labels zuerst suchen, damit "EMAIL ADDRESS" vor "ADDRESS" gefunden wird + const sortedLabels = [...LABELS].sort((a, b) => b.length - a.length); + + for (const line of lines) { + const textUpper = line.text.toUpperCase().replace(/\s+/g, ''); + + // Boilerplate ignorieren, der Labels enthält ("VERIFICATION OF DOWN PAYMENT") + if (textUpper.includes("SELLERMAYREQUIREVERIFICATION")) continue; + + let foundLabel: string | null = null; + for (const lbl of sortedLabels) { + const lblClean = lbl.toUpperCase().replace(/\s+/g, ''); + if (textUpper.includes(lblClean)) { + foundLabel = lbl; + break; + } + } + + if (foundLabel) { + currentLabel = foundLabel; + + // Falls der Wert direkt auf derselben Zeile steht (z.B. "NAME/COMPANY: Gunnar Schultz") + if (line.text.includes(':')) { + const parts = line.text.split(':'); + const val = parts.slice(1).join(':').replace(/_+/g, '').trim(); // Unterstriche entfernen + if (val.length > 0) { + rawResults[currentLabel].push(val); + } + } + } else if (currentLabel) { + // Diese Zeile ist kein Label, gehört also zum aktuellen Bereich! + const cleanVal = line.text.replace(/_+/g, '').trim(); + + // Footer und Artefakte ignorieren + if (cleanVal.length > 0 && !cleanVal.includes("Doc ID") && !cleanVal.includes("Bizmatch")) { + rawResults[currentLabel].push(cleanVal); + } + } + } + + return this.mapToTargetStructure(rawResults, filePath); + } + + private mapToTargetStructure(raw: Record, filePath: string): BuyerSheetData { + const getVal = (label: string) => raw[label] && raw[label].length > 0 ? raw[label].join(' ') : null; + + const nameCompany = getVal("NAME / COMPANY"); + + // Down Payment normalisieren + const downPaymentRaw = getVal("DOWN PAYMENT")?.replace(/[^0-9,]/g, '') || null; + const downPaymentClean = downPaymentRaw ? downPaymentRaw.replace(/,/g, '') : null; + + // Datum aus Dateinamen extrahieren + let dateOfIntro = null; + const dateMatch = filePath.match(/(\d{2})(\d{2})(\d{2})\.pdf$/); + if (dateMatch) { + dateOfIntro = `20${dateMatch[3]}-${dateMatch[1]}-${dateMatch[2]}`; + } + + return { + is_buyer_sheet: true, + name_company: nameCompany, + prospective_buyer: nameCompany ? nameCompany.split('/')[0].trim() : null, + company: null, + phone: getVal("PHONE"), + cell: null, + email: getVal("EMAIL ADDRESS"), + address: getVal("ADDRESS"), + state: null, + how_did_you_hear: getVal("HOW DID YOU HEAR"), + interested_in_updates: null, + types_of_business_raw: getVal("TYPES OF BUSINESSES"), + background_experience: getVal("BACKGROUND"), + total_purchase_price: getVal("TOTAL PURCHASE PRICE"), + down_payment: downPaymentClean, + down_payment_raw: downPaymentRaw, + date_of_introduction: dateOfIntro, + _checkbox_pending: true, + _parser: "deterministic", + _info_page: 1, + _ca_page: 2 + }; + } +} \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..15f9527 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "dependencies": { + "canvas": "^3.2.3", + "pdf-parse": "^1.1.1", + "pdfjs-dist": "^3.11.174" + }, + "devDependencies": { + "@types/node": "^26.1.1", + "@types/pdf-parse": "^1.1.5", + "ts-node": "^10.9.2", + "typescript": "^7.0.2" + } +} diff --git a/run_parser.ts b/run_parser.ts new file mode 100644 index 0000000..a28b6bf --- /dev/null +++ b/run_parser.ts @@ -0,0 +1,28 @@ +// run_parser.ts +import { DeterministicParser } from './BuyerSheetParser.ts'; + +async function main() { + // Einfaches Auslesen der Kommandozeilenargumente + const args = process.argv.slice(2); + const pdfArgIndex = args.indexOf('--pdf'); + + if (pdfArgIndex === -1 || !args[pdfArgIndex + 1]) { + console.error("Usage: npx ts-node run_parser.ts --pdf "); + process.exit(1); + } + + const pdfPath = args[pdfArgIndex + 1]; + const parser = new DeterministicParser(); + + try { + console.log(`Lese PDF: ${pdfPath} ...\n`); + const result = await parser.parsePdf(pdfPath); + + // JSON formatiert und farbig (optional) ausgeben + console.log(JSON.stringify(result, null, 2)); + } catch (error) { + console.error("Fehler beim Parsen des PDFs:", error); + } +} + +main(); \ No newline at end of file