diff --git a/enrich_and_anonymize.ts b/enrich_and_anonymize.ts new file mode 100644 index 0000000..5cc85ad --- /dev/null +++ b/enrich_and_anonymize.ts @@ -0,0 +1,119 @@ +#!/usr/bin/env npx tsx +/** + * enrich_and_anonymize.ts + * + * Zwei Aufgaben in einem Durchlauf auf einer buyers_vision.json: + * 1) Ergaenzt "name_from_filename" (falls fehlt) UND "_letter" + * (der A-Z-Unterordner, in dem das PDF liegt) fuer die PDF-Pfad-Aufloesung. + * 2) Optional (--anonymize): ersetzt PII-Felder (phone, cell, email, address) + * durch konsistente Faker-Dummydaten, damit das JSON weitergegeben werden + * kann. Leere Felder bleiben leer. Gleicher Originalwert -> gleicher Dummy. + * + * Der _letter wird aus dem Nachnamen (erstes Zeichen von name_from_filename) + * abgeleitet - das entspricht der Ordnerstruktur "...Buyers NDA's A-Z//". + * + * Aufruf: + * npx tsx enrich_and_anonymize.ts buyers_vision.json + * npx tsx enrich_and_anonymize.ts buyers_vision.json --anonymize + * npx tsx enrich_and_anonymize.ts buyers_vision.json --anonymize --overwrite + */ +import { promises as fsp } from "node:fs"; +import { faker } from "@faker-js/faker"; + +faker.seed(1234); // reproduzierbar + +// IDENTISCH zur Funktion im vision_runner.ts +function nameFromFilename(fileName: string): string | null { + let s = fileName.replace(/\.pdf$/i, ""); + s = s.replace(/\([^)]*\)/g, " ").replace(/\s+/g, " ").trim(); + const comma = s.indexOf(","); + if (comma < 0) { + const first = s.replace(/\b\d{4,8}\b.*$/, "").trim(); + return first.length >= 2 ? first : null; + } + const last = s.slice(0, comma).trim(); + const rest = s.slice(comma + 1).trim(); + const firstName = (rest.match(/^[A-Za-zÀ-ÿ.'-]+/) || [""])[0]; + if (!last || !firstName) return null; + return `${last}, ${firstName}`; +} + +/** A-Z-Unterordner aus dem Nachnamen. Fallback: erstes Zeichen des Dateinamens. */ +function letterFromName(fileName: string, nameFF: string | null): string | null { + const src = (nameFF ?? fileName).trim(); + const ch = src.charAt(0).toUpperCase(); + return /[A-Z]/.test(ch) ? ch : null; +} + +const isEmpty = (v: unknown) => v === null || v === undefined || v === ""; + +// Konsistenz-Caches: gleicher Originalwert -> gleicher Dummy +const caches: Record> = { + phone: new Map(), cell: new Map(), email: new Map(), address: new Map(), +}; +function fakePII(field: string, value: string): string { + const key = value.trim().toLowerCase(); + const cache = caches[field]; + if (cache.has(key)) return cache.get(key)!; + let dummy: string; + switch (field) { + case "phone": + case "cell": dummy = faker.phone.number(); break; + case "email": dummy = faker.internet.email(); break; + case "address": dummy = faker.location.streetAddress({ useFullAddress: true }); break; + default: dummy = faker.lorem.word(); + } + cache.set(key, dummy); + return dummy; +} + +async function main() { + const args = process.argv.slice(2); + const file = args.find((a) => !a.startsWith("--")); + const overwrite = args.includes("--overwrite"); + const anonymize = args.includes("--anonymize"); + if (!file) { + console.error("Aufruf: npx tsx enrich_and_anonymize.ts [--anonymize] [--overwrite]"); + process.exit(1); + } + + const raw = await fsp.readFile(file, "utf8"); + const records: Array> = JSON.parse(raw); + await fsp.writeFile(file + ".bak", raw); + + let nameSet = 0, letterSet = 0, anonFields = 0; + const PII = ["phone", "cell", "email", "address"]; + + for (const r of records) { + const fn = r["file_name"] as string | undefined; + if (!fn) continue; + + // name_from_filename + const hasName = !isEmpty(r["name_from_filename"]); + if (!hasName || overwrite) { + const val = nameFromFilename(fn); + if (r["name_from_filename"] !== val) { r["name_from_filename"] = val; nameSet++; } + } + // _letter (A-Z-Unterordner) + const hasLetter = !isEmpty(r["_letter"]); + if (!hasLetter || overwrite) { + const lv = letterFromName(fn, (r["name_from_filename"] as string | null) ?? null); + if (r["_letter"] !== lv) { r["_letter"] = lv; letterSet++; } + } + // Anonymisierung + if (anonymize) { + for (const f of PII) { + if (!isEmpty(r[f])) { r[f] = fakePII(f, String(r[f])); anonFields++; } + } + } + } + + await fsp.writeFile(file, JSON.stringify(records, null, 2)); + console.log(`Datensaetze: ${records.length}`); + console.log(`name_from_filename: ${nameSet} gesetzt`); + console.log(`_letter: ${letterSet} gesetzt`); + if (anonymize) console.log(`PII anonymisiert: ${anonFields} Felder`); + console.log(`Backup: ${file}.bak`); +} + +main().catch((e) => { console.error(e); process.exit(1); }); \ No newline at end of file diff --git a/package.json b/package.json index 15f9527..305e1a6 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@faker-js/faker": "^10.5.0", "canvas": "^3.2.3", "pdf-parse": "^1.1.1", "pdfjs-dist": "^3.11.174"