119 lines
4.5 KiB
TypeScript
119 lines
4.5 KiB
TypeScript
#!/usr/bin/env npx tsx
|
|
/**
|
|
* enrich_and_anonymize.ts
|
|
*
|
|
* Zwei Aufgaben in einem Durchlauf auf einer buyers_vision.json:
|
|
* 1) Ergaenzt "name_from_filename" (falls fehlt) UND "_letter"
|
|
* (der A-Z-Unterordner, in dem das PDF liegt) fuer die PDF-Pfad-Aufloesung.
|
|
* 2) Optional (--anonymize): ersetzt PII-Felder (phone, cell, email, address)
|
|
* durch konsistente Faker-Dummydaten, damit das JSON weitergegeben werden
|
|
* kann. Leere Felder bleiben leer. Gleicher Originalwert -> gleicher Dummy.
|
|
*
|
|
* Der _letter wird aus dem Nachnamen (erstes Zeichen von name_from_filename)
|
|
* abgeleitet - das entspricht der Ordnerstruktur "...Buyers NDA's A-Z/<Letter>/".
|
|
*
|
|
* Aufruf:
|
|
* npx tsx enrich_and_anonymize.ts buyers_vision.json
|
|
* npx tsx enrich_and_anonymize.ts buyers_vision.json --anonymize
|
|
* npx tsx enrich_and_anonymize.ts buyers_vision.json --anonymize --overwrite
|
|
*/
|
|
import { promises as fsp } from "node:fs";
|
|
import { faker } from "@faker-js/faker";
|
|
|
|
faker.seed(1234); // reproduzierbar
|
|
|
|
// IDENTISCH zur Funktion im vision_runner.ts
|
|
function nameFromFilename(fileName: string): string | null {
|
|
let s = fileName.replace(/\.pdf$/i, "");
|
|
s = s.replace(/\([^)]*\)/g, " ").replace(/\s+/g, " ").trim();
|
|
const comma = s.indexOf(",");
|
|
if (comma < 0) {
|
|
const first = s.replace(/\b\d{4,8}\b.*$/, "").trim();
|
|
return first.length >= 2 ? first : null;
|
|
}
|
|
const last = s.slice(0, comma).trim();
|
|
const rest = s.slice(comma + 1).trim();
|
|
const firstName = (rest.match(/^[A-Za-zÀ-ÿ.'-]+/) || [""])[0];
|
|
if (!last || !firstName) return null;
|
|
return `${last}, ${firstName}`;
|
|
}
|
|
|
|
/** A-Z-Unterordner aus dem Nachnamen. Fallback: erstes Zeichen des Dateinamens. */
|
|
function letterFromName(fileName: string, nameFF: string | null): string | null {
|
|
const src = (nameFF ?? fileName).trim();
|
|
const ch = src.charAt(0).toUpperCase();
|
|
return /[A-Z]/.test(ch) ? ch : null;
|
|
}
|
|
|
|
const isEmpty = (v: unknown) => v === null || v === undefined || v === "";
|
|
|
|
// Konsistenz-Caches: gleicher Originalwert -> gleicher Dummy
|
|
const caches: Record<string, Map<string, string>> = {
|
|
phone: new Map(), cell: new Map(), email: new Map(), address: new Map(),
|
|
};
|
|
function fakePII(field: string, value: string): string {
|
|
const key = value.trim().toLowerCase();
|
|
const cache = caches[field];
|
|
if (cache.has(key)) return cache.get(key)!;
|
|
let dummy: string;
|
|
switch (field) {
|
|
case "phone":
|
|
case "cell": dummy = faker.phone.number(); break;
|
|
case "email": dummy = faker.internet.email(); break;
|
|
case "address": dummy = faker.location.streetAddress({ useFullAddress: true }); break;
|
|
default: dummy = faker.lorem.word();
|
|
}
|
|
cache.set(key, dummy);
|
|
return dummy;
|
|
}
|
|
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
const file = args.find((a) => !a.startsWith("--"));
|
|
const overwrite = args.includes("--overwrite");
|
|
const anonymize = args.includes("--anonymize");
|
|
if (!file) {
|
|
console.error("Aufruf: npx tsx enrich_and_anonymize.ts <buyers_vision.json> [--anonymize] [--overwrite]");
|
|
process.exit(1);
|
|
}
|
|
|
|
const raw = await fsp.readFile(file, "utf8");
|
|
const records: Array<Record<string, unknown>> = JSON.parse(raw);
|
|
await fsp.writeFile(file + ".bak", raw);
|
|
|
|
let nameSet = 0, letterSet = 0, anonFields = 0;
|
|
const PII = ["phone", "cell", "email", "address"];
|
|
|
|
for (const r of records) {
|
|
const fn = r["file_name"] as string | undefined;
|
|
if (!fn) continue;
|
|
|
|
// name_from_filename
|
|
const hasName = !isEmpty(r["name_from_filename"]);
|
|
if (!hasName || overwrite) {
|
|
const val = nameFromFilename(fn);
|
|
if (r["name_from_filename"] !== val) { r["name_from_filename"] = val; nameSet++; }
|
|
}
|
|
// _letter (A-Z-Unterordner)
|
|
const hasLetter = !isEmpty(r["_letter"]);
|
|
if (!hasLetter || overwrite) {
|
|
const lv = letterFromName(fn, (r["name_from_filename"] as string | null) ?? null);
|
|
if (r["_letter"] !== lv) { r["_letter"] = lv; letterSet++; }
|
|
}
|
|
// Anonymisierung
|
|
if (anonymize) {
|
|
for (const f of PII) {
|
|
if (!isEmpty(r[f])) { r[f] = fakePII(f, String(r[f])); anonFields++; }
|
|
}
|
|
}
|
|
}
|
|
|
|
await fsp.writeFile(file, JSON.stringify(records, null, 2));
|
|
console.log(`Datensaetze: ${records.length}`);
|
|
console.log(`name_from_filename: ${nameSet} gesetzt`);
|
|
console.log(`_letter: ${letterSet} gesetzt`);
|
|
if (anonymize) console.log(`PII anonymisiert: ${anonFields} Felder`);
|
|
console.log(`Backup: ${file}.bak`);
|
|
}
|
|
|
|
main().catch((e) => { console.error(e); process.exit(1); }); |