333 lines
13 KiB
TypeScript
333 lines
13 KiB
TypeScript
import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.js';
|
|
import * as fs from 'fs';
|
|
|
|
export interface BuyerSheetData {
|
|
file_name: string; // NEU
|
|
is_buyer_sheet: boolean;
|
|
name_company: string | null;
|
|
prospective_buyer: string | null;
|
|
company: string | null;
|
|
phone: string | null;
|
|
cell: string | null;
|
|
email: string | null;
|
|
address: string | null;
|
|
state: string | null;
|
|
how_did_you_hear: string | null;
|
|
interested_in_updates: boolean | null;
|
|
types_of_business_raw: string | null;
|
|
background_experience: string | null;
|
|
total_purchase_price: string | null;
|
|
down_payment: string | null;
|
|
down_payment_raw: string | null;
|
|
date_of_introduction: string | null;
|
|
_checkbox_pending: boolean;
|
|
_parser: string;
|
|
_info_page: number | null;
|
|
_ca_page: number | null;
|
|
}
|
|
|
|
const LABELS = [
|
|
"NAME / COMPANY", "PHONE", "ADDRESS", "EMAIL ADDRESS",
|
|
"HOW DID YOU HEAR", "ARE YOU INTERESTED", "TYPES OF BUSINESSES",
|
|
"BACKGROUND", "TOTAL PURCHASE PRICE", "DOWN PAYMENT",
|
|
"INCOME REQUIREMENTS", "ACCOUNTANT", "ATTORNEY", "BANK",
|
|
];
|
|
|
|
export class DeterministicParser {
|
|
|
|
public async parsePdf(filePath: string, fileName: string): Promise<BuyerSheetData | null> {
|
|
const dataBuffer = fs.readFileSync(filePath);
|
|
const loadingTask = pdfjsLib.getDocument({ data: new Uint8Array(dataBuffer) });
|
|
const pdfDocument = await loadingTask.promise;
|
|
|
|
// Regel: Alles über 10 Seiten wird radikal ignoriert
|
|
if (pdfDocument.numPages > 10) {
|
|
return null;
|
|
}
|
|
|
|
let infoPageNum: number | null = null;
|
|
let caPageNum: number | null = null;
|
|
let infoPageObj: any = null;
|
|
let infoPageTextContent: any = null;
|
|
|
|
// ==========================================
|
|
// 1. Dynamische Seitensuche (Visuell sortiert & kugelsicher)
|
|
// ==========================================
|
|
for (let i = 1; i <= pdfDocument.numPages; i++) {
|
|
const page = await pdfDocument.getPage(i);
|
|
const textContent = await page.getTextContent();
|
|
|
|
// Elemente mit Koordinaten versehen und wie ein Mensch lesen (von oben nach unten, links nach rechts)
|
|
const sortedItems = textContent.items
|
|
.map((item: any) => ({
|
|
text: item.str,
|
|
x: item.transform[4],
|
|
y: item.transform[5]
|
|
}))
|
|
.sort((a: any, b: any) => {
|
|
// Y-Toleranz für Buchstaben auf derselben Zeile
|
|
if (Math.abs(b.y - a.y) > 5) {
|
|
return b.y - a.y;
|
|
}
|
|
return a.x - b.x;
|
|
});
|
|
|
|
// Wir werfen ALLE Leerzeichen, Striche, Punkte und unsichtbare Artefakte weg.
|
|
// Übrig bleibt eine reine, unverwüstliche Buchstabenkette.
|
|
const textRaw = sortedItems.map((i: any) => i.text).join('').toUpperCase().replace(/[^A-Z]/g, '');
|
|
|
|
// Anker-Suche in der sauberen Zeichenkette
|
|
if (textRaw.includes('BUYERINFORMATIONSHEET')) {
|
|
infoPageNum = i;
|
|
infoPageObj = page;
|
|
infoPageTextContent = textContent;
|
|
}
|
|
if (textRaw.includes('PROSPECTIVEBUYERAGREESTOKEEPANDHOLDCONFIDENTIAL')) {
|
|
caPageNum = i;
|
|
}
|
|
}
|
|
|
|
// ==========================================
|
|
// 2. Fallback für Bild/Scan PDFs
|
|
// ==========================================
|
|
if (!infoPageNum || !infoPageObj || !infoPageTextContent) {
|
|
return this.createEmptyFallback(fileName, filePath, infoPageNum, caPageNum);
|
|
}
|
|
|
|
// ==========================================
|
|
// 3. Werte-Extraktion (nur auf der Info-Seite!)
|
|
// ==========================================
|
|
const mappedItems = infoPageTextContent.items
|
|
.filter((item: any) => item.str.trim() !== '')
|
|
.map((item: any) => ({
|
|
text: item.str,
|
|
x: item.transform[4],
|
|
y: item.transform[5],
|
|
width: item.width
|
|
}))
|
|
.sort((a: any, b: any) => b.y - a.y || a.x - b.x);
|
|
|
|
const linesGrouped: { y: number, items: any[] }[] = [];
|
|
let currentY: number | null = null;
|
|
let currentItems = [];
|
|
|
|
for (const item of mappedItems) {
|
|
if (currentY === null || Math.abs(item.y - currentY) <= 3) {
|
|
currentItems.push(item);
|
|
currentY = currentY === null ? item.y : currentY;
|
|
} else {
|
|
linesGrouped.push({ y: currentY, items: currentItems });
|
|
currentItems = [item];
|
|
currentY = item.y;
|
|
}
|
|
}
|
|
if (currentItems.length > 0 && currentY !== null) {
|
|
linesGrouped.push({ y: currentY, items: currentItems });
|
|
}
|
|
|
|
const lines: { y: number, text: string }[] = [];
|
|
for (const group of linesGrouped) {
|
|
group.items.sort((a, b) => a.x - b.x);
|
|
let lineStr = "";
|
|
let prevEnd = -1;
|
|
|
|
for (const item of group.items) {
|
|
if (prevEnd !== -1) {
|
|
const gap = item.x - prevEnd;
|
|
if (gap > 4) lineStr += " ";
|
|
}
|
|
lineStr += item.text;
|
|
prevEnd = item.x + item.width;
|
|
}
|
|
lines.push({ y: group.y, text: lineStr });
|
|
}
|
|
|
|
const rawResults: Record<string, string[]> = {};
|
|
LABELS.forEach(lbl => rawResults[lbl] = []);
|
|
let currentLabel: string | null = null;
|
|
const sortedLabels = [...LABELS].sort((a, b) => b.length - a.length);
|
|
|
|
for (const line of lines) {
|
|
const textUpper = line.text.toUpperCase().replace(/\s+/g, '');
|
|
if (textUpper.includes("SELLERMAYREQUIREVERIFICATION")) continue;
|
|
|
|
let foundLabel: string | null = null;
|
|
for (const lbl of sortedLabels) {
|
|
const lblClean = lbl.toUpperCase().replace(/\s+/g, '');
|
|
if (textUpper.includes(lblClean)) {
|
|
foundLabel = lbl;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (foundLabel) {
|
|
currentLabel = foundLabel;
|
|
if (line.text.includes(':')) {
|
|
const parts = line.text.split(':');
|
|
const val = parts.slice(1).join(':').replace(/_+/g, '').trim();
|
|
if (val.length > 0) rawResults[currentLabel].push(val);
|
|
}
|
|
} else if (currentLabel) {
|
|
const cleanVal = line.text.replace(/_+/g, '').trim();
|
|
if (cleanVal.length > 0 && !cleanVal.includes("Doc ID") && !cleanVal.includes("Bizmatch")) {
|
|
rawResults[currentLabel].push(cleanVal);
|
|
}
|
|
}
|
|
}
|
|
|
|
const visualCheckboxResult = await this.detectGraphicCheckbox(infoPageObj, mappedItems);
|
|
return this.mapToTargetStructure(rawResults, filePath, fileName, visualCheckboxResult, infoPageNum, caPageNum);
|
|
}
|
|
|
|
private createEmptyFallback(fileName: string, filePath: string, infoPage: number | null, caPage: number | null): BuyerSheetData {
|
|
let dateOfIntro = null;
|
|
const dateMatch = filePath.match(/(\d{2})(\d{2})(\d{2})\.pdf$/);
|
|
if (dateMatch) dateOfIntro = `20${dateMatch[3]}-${dateMatch[1]}-${dateMatch[2]}`;
|
|
|
|
return {
|
|
file_name: fileName,
|
|
is_buyer_sheet: false,
|
|
name_company: null,
|
|
prospective_buyer: null,
|
|
company: null,
|
|
phone: null,
|
|
cell: null,
|
|
email: null,
|
|
address: null,
|
|
state: null,
|
|
how_did_you_hear: null,
|
|
interested_in_updates: null,
|
|
types_of_business_raw: null,
|
|
background_experience: null,
|
|
total_purchase_price: null,
|
|
down_payment: null,
|
|
down_payment_raw: null,
|
|
date_of_introduction: dateOfIntro,
|
|
_checkbox_pending: true,
|
|
_parser: "deterministic", // Signal für den AI-Pass
|
|
_info_page: infoPage,
|
|
_ca_page: caPage
|
|
};
|
|
}
|
|
|
|
private mapToTargetStructure(
|
|
raw: Record<string, string[]>,
|
|
filePath: string,
|
|
fileName: string,
|
|
visualCheckboxResult: boolean | null,
|
|
infoPageNum: number | null,
|
|
caPageNum: number | null
|
|
): BuyerSheetData {
|
|
const getVal = (label: string) => raw[label] && raw[label].length > 0 ? raw[label].join(' ') : null;
|
|
const nameCompany = getVal("NAME / COMPANY");
|
|
|
|
let phoneRaw = getVal("PHONE");
|
|
let phoneClean = null;
|
|
let cellClean = null;
|
|
|
|
if (phoneRaw) {
|
|
const phoneMatch = phoneRaw.split(/FAX:|CELL:/)[0];
|
|
phoneClean = phoneMatch ? phoneMatch.trim() : null;
|
|
const cellMatch = phoneRaw.match(/CELL:\s*(.*)/);
|
|
cellClean = cellMatch && cellMatch[1] ? cellMatch[1].trim() : null;
|
|
}
|
|
|
|
let addressClean = getVal("ADDRESS");
|
|
if (addressClean) {
|
|
addressClean = addressClean.replace(/PO BOX \/ STREET\s+CITY \/ STATE \/ ZIP/g, '').trim();
|
|
if (addressClean === '') addressClean = null;
|
|
}
|
|
|
|
const interestedRaw = getVal("ARE YOU INTERESTED");
|
|
let interestedInUpdates: boolean | null = null;
|
|
let checkboxPending = true;
|
|
|
|
if (interestedRaw) {
|
|
if (/(✔|☑|X|✓)\s*YES/i.test(interestedRaw) || /YES\s*(✔|☑|X|✓)/i.test(interestedRaw)) {
|
|
interestedInUpdates = true;
|
|
checkboxPending = false;
|
|
} else if (/(✔|☑|X|✓)\s*NO/i.test(interestedRaw) || /NO\s*(✔|☑|X|✓)/i.test(interestedRaw)) {
|
|
interestedInUpdates = false;
|
|
checkboxPending = false;
|
|
}
|
|
}
|
|
if (checkboxPending && visualCheckboxResult !== null) {
|
|
interestedInUpdates = visualCheckboxResult;
|
|
checkboxPending = false;
|
|
}
|
|
|
|
const downPaymentRaw = getVal("DOWN PAYMENT")?.replace(/[^0-9,]/g, '') || null;
|
|
const downPaymentClean = downPaymentRaw ? downPaymentRaw.replace(/,/g, '') : null;
|
|
|
|
let dateOfIntro = null;
|
|
const dateMatch = filePath.match(/(\d{2})(\d{2})(\d{2})\.pdf$/);
|
|
if (dateMatch) dateOfIntro = `20${dateMatch[3]}-${dateMatch[1]}-${dateMatch[2]}`;
|
|
|
|
return {
|
|
file_name: fileName,
|
|
is_buyer_sheet: true,
|
|
name_company: nameCompany,
|
|
prospective_buyer: nameCompany ? nameCompany.split('/')[0].trim() : null,
|
|
company: null,
|
|
phone: phoneClean,
|
|
cell: cellClean,
|
|
email: getVal("EMAIL ADDRESS"),
|
|
address: addressClean,
|
|
state: null,
|
|
how_did_you_hear: getVal("HOW DID YOU HEAR"),
|
|
interested_in_updates: interestedInUpdates,
|
|
types_of_business_raw: getVal("TYPES OF BUSINESSES"),
|
|
background_experience: getVal("BACKGROUND"),
|
|
total_purchase_price: getVal("TOTAL PURCHASE PRICE"),
|
|
down_payment: downPaymentClean,
|
|
down_payment_raw: downPaymentRaw,
|
|
date_of_introduction: dateOfIntro,
|
|
_checkbox_pending: checkboxPending,
|
|
_parser: "deterministic",
|
|
_info_page: infoPageNum,
|
|
_ca_page: caPageNum
|
|
};
|
|
}
|
|
|
|
private async detectGraphicCheckbox(page: any, textItems: any[]): Promise<boolean | null> {
|
|
let yesX = null, noX = null, targetY = null;
|
|
|
|
for (const item of textItems) {
|
|
const textUpper = item.text.toUpperCase().trim();
|
|
if (textUpper.includes("YES")) { yesX = item.x; targetY = item.y; }
|
|
}
|
|
for (const item of textItems) {
|
|
const textUpper = item.text.toUpperCase().trim();
|
|
if (textUpper.includes("NO") && targetY !== null && Math.abs(item.y - targetY) < 5) noX = item.x;
|
|
}
|
|
|
|
if (yesX === null || noX === null || targetY === null) return null;
|
|
|
|
const opList = await page.getOperatorList();
|
|
let currentTransform = [1, 0, 0, 1, 0, 0];
|
|
|
|
for (let i = 0; i < opList.fnArray.length; i++) {
|
|
const fn = opList.fnArray[i];
|
|
const args = opList.argsArray[i];
|
|
|
|
if (fn === pdfjsLib.OPS.transform) currentTransform = args;
|
|
else if (
|
|
fn === pdfjsLib.OPS.paintImageXObject ||
|
|
fn === pdfjsLib.OPS.paintInlineImageXObject ||
|
|
fn === pdfjsLib.OPS.paintJpegXObject
|
|
) {
|
|
const width = Math.abs(currentTransform[0]);
|
|
const height = Math.abs(currentTransform[3]);
|
|
const imgX = currentTransform[4];
|
|
const imgY = currentTransform[5];
|
|
|
|
if (width < 40 && height < 40 && Math.abs(imgY - targetY) < 50) {
|
|
const distToYes = Math.abs(imgX - yesX);
|
|
const distToNo = Math.abs(imgX - noX);
|
|
return distToYes < distToNo;
|
|
}
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
} |