gfdfg

python/inspect_poc.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+inspect_poc.py - Wertet buyers_raw.jsonl nach dem POC-Lauf aus.
+
+Zeigt:
+  - Feld-Fuellquoten (wie oft ist welches Feld nicht null?)
+  - Text- vs Vision-Qualitaet im Vergleich
+  - die Multi-Sheet-Gruppierung: welche Person hat mehrere Sheets?
+  - Datums-Parsing-Quote (fuer die "letzte 5 Jahre"-Abfragen kritisch)
+
+Aufruf: python inspect_poc.py ./poc_out/buyers_raw.jsonl
+"""
+
+import sys
+import json
+import re
+from collections import defaultdict, Counter
+
+
+def person_key(rec):
+    """Gruppierungsschluessel: bevorzugt echten Namen, sonst name_company."""
+    name = (rec.get("prospective_buyer") or rec.get("name_company") or "").strip().lower()
+    # "Nachname, Vorname" und "Vorname Nachname" grob angleichen
+    name = re.sub(r"\s+", " ", name)
+    return name or "(unbekannt)"
+
+
+def main():
+    path = sys.argv[1] if len(sys.argv) > 1 else "./poc_out/buyers_raw.jsonl"
+    records = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+
+    ok = [r for r in records if "_error" not in r]
+    err = [r for r in records if "_error" in r]
+
+    print(f"Datensaetze: {len(records)}  (ok: {len(ok)}, Fehler: {len(err)})\n")
+
+    # Feld-Fuellquoten
+    fields = ["name_company", "prospective_buyer", "company", "phone", "cell",
+              "email", "address", "state", "how_did_you_hear",
+              "interested_in_updates", "types_of_business", "background_experience",
+              "date_of_introduction"]
+    print("=== Feld-Fuellquoten (nicht-null / erfolgreiche) ===")
+    for fld in fields:
+        filled = 0
+        for r in ok:
+            v = r.get(fld)
+            if v not in (None, "", [], {}):
+                filled += 1
+        pct = 100 * filled / len(ok) if ok else 0
+        print(f"  {fld:<24} {filled:>4}/{len(ok)}  ({pct:4.0f}%)")
+
+    # Text vs Vision
+    print("\n=== Text- vs Vision-Pfad ===")
+    for mode in ("text", "vision"):
+        sub = [r for r in ok if r.get("extraction_mode") == mode]
+        if not sub:
+            continue
+        # als grobe Qualitaetsmetrik: durchschnittliche Zahl gefuellter Felder
+        avg_filled = sum(
+            sum(1 for f in fields if r.get(f) not in (None, "", [], {})) for r in sub
+        ) / len(sub)
+        print(f"  {mode:<7} {len(sub):>4} Docs, im Schnitt {avg_filled:.1f}/{len(fields)} Felder gefuellt")
+
+    # Datums-Parsing
+    print("\n=== Date of Introduction ===")
+    iso = sum(1 for r in ok if isinstance(r.get("date_of_introduction"), str)
+              and re.match(r"\d{4}-\d{2}-\d{2}", r["date_of_introduction"]))
+    nonnull = sum(1 for r in ok if r.get("date_of_introduction"))
+    print(f"  vorhanden: {nonnull}/{len(ok)}   davon sauber ISO (YYYY-MM-DD): {iso}")
+
+    # Multi-Sheet-Gruppierung
+    print("\n=== Personen mit mehreren Sheets ===")
+    groups = defaultdict(list)
+    for r in ok:
+        groups[person_key(r)].append(r)
+    multi = {k: v for k, v in groups.items() if len(v) > 1}
+    print(f"  distinkte Personen: {len(groups)}, davon mit >1 Sheet: {len(multi)}")
+    for name, recs in sorted(multi.items(), key=lambda x: -len(x[1]))[:10]:
+        cats = set()
+        dates = []
+        for r in recs:
+            cats.update(r.get("types_of_business") or [])
+            if r.get("date_of_introduction"):
+                dates.append(r["date_of_introduction"])
+        print(f"    {name!r}: {len(recs)} Sheets | Kategorien: {sorted(cats)} | Daten: {sorted(dates)}")
+
+    # Haeufigste Kategorien
+    print("\n=== Top 20 Kategorien (roh) ===")
+    cats = Counter()
+    for r in ok:
+        for c in (r.get("types_of_business") or []):
+            cats[c.strip().lower()] += 1
+    for c, n in cats.most_common(20):
+        print(f"  {n:>4}  {c}")
+
+
+if __name__ == "__main__":
+    main()