103 lines
3.8 KiB
Python
103 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
inspect_poc.py - Wertet buyers_raw.jsonl nach dem POC-Lauf aus.
|
|
|
|
Zeigt:
|
|
- Feld-Fuellquoten (wie oft ist welches Feld nicht null?)
|
|
- Text- vs Vision-Qualitaet im Vergleich
|
|
- die Multi-Sheet-Gruppierung: welche Person hat mehrere Sheets?
|
|
- Datums-Parsing-Quote (fuer die "letzte 5 Jahre"-Abfragen kritisch)
|
|
|
|
Aufruf: python inspect_poc.py ./poc_out/buyers_raw.jsonl
|
|
"""
|
|
|
|
import sys
|
|
import json
|
|
import re
|
|
from collections import defaultdict, Counter
|
|
|
|
|
|
def person_key(rec):
|
|
"""Gruppierungsschluessel: bevorzugt echten Namen, sonst name_company."""
|
|
name = (rec.get("prospective_buyer") or rec.get("name_company") or "").strip().lower()
|
|
# "Nachname, Vorname" und "Vorname Nachname" grob angleichen
|
|
name = re.sub(r"\s+", " ", name)
|
|
return name or "(unbekannt)"
|
|
|
|
|
|
def main():
|
|
path = sys.argv[1] if len(sys.argv) > 1 else "./poc_out/buyers_raw.jsonl"
|
|
records = []
|
|
with open(path, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
records.append(json.loads(line))
|
|
|
|
ok = [r for r in records if "_error" not in r]
|
|
err = [r for r in records if "_error" in r]
|
|
|
|
print(f"Datensaetze: {len(records)} (ok: {len(ok)}, Fehler: {len(err)})\n")
|
|
|
|
# Feld-Fuellquoten
|
|
fields = ["name_company", "prospective_buyer", "company", "phone", "cell",
|
|
"email", "address", "state", "how_did_you_hear",
|
|
"interested_in_updates", "types_of_business", "background_experience",
|
|
"date_of_introduction"]
|
|
print("=== Feld-Fuellquoten (nicht-null / erfolgreiche) ===")
|
|
for fld in fields:
|
|
filled = 0
|
|
for r in ok:
|
|
v = r.get(fld)
|
|
if v not in (None, "", [], {}):
|
|
filled += 1
|
|
pct = 100 * filled / len(ok) if ok else 0
|
|
print(f" {fld:<24} {filled:>4}/{len(ok)} ({pct:4.0f}%)")
|
|
|
|
# Text vs Vision
|
|
print("\n=== Text- vs Vision-Pfad ===")
|
|
for mode in ("text", "vision"):
|
|
sub = [r for r in ok if r.get("extraction_mode") == mode]
|
|
if not sub:
|
|
continue
|
|
# als grobe Qualitaetsmetrik: durchschnittliche Zahl gefuellter Felder
|
|
avg_filled = sum(
|
|
sum(1 for f in fields if r.get(f) not in (None, "", [], {})) for r in sub
|
|
) / len(sub)
|
|
print(f" {mode:<7} {len(sub):>4} Docs, im Schnitt {avg_filled:.1f}/{len(fields)} Felder gefuellt")
|
|
|
|
# Datums-Parsing
|
|
print("\n=== Date of Introduction ===")
|
|
iso = sum(1 for r in ok if isinstance(r.get("date_of_introduction"), str)
|
|
and re.match(r"\d{4}-\d{2}-\d{2}", r["date_of_introduction"]))
|
|
nonnull = sum(1 for r in ok if r.get("date_of_introduction"))
|
|
print(f" vorhanden: {nonnull}/{len(ok)} davon sauber ISO (YYYY-MM-DD): {iso}")
|
|
|
|
# Multi-Sheet-Gruppierung
|
|
print("\n=== Personen mit mehreren Sheets ===")
|
|
groups = defaultdict(list)
|
|
for r in ok:
|
|
groups[person_key(r)].append(r)
|
|
multi = {k: v for k, v in groups.items() if len(v) > 1}
|
|
print(f" distinkte Personen: {len(groups)}, davon mit >1 Sheet: {len(multi)}")
|
|
for name, recs in sorted(multi.items(), key=lambda x: -len(x[1]))[:10]:
|
|
cats = set()
|
|
dates = []
|
|
for r in recs:
|
|
cats.update(r.get("types_of_business") or [])
|
|
if r.get("date_of_introduction"):
|
|
dates.append(r["date_of_introduction"])
|
|
print(f" {name!r}: {len(recs)} Sheets | Kategorien: {sorted(cats)} | Daten: {sorted(dates)}")
|
|
|
|
# Haeufigste Kategorien
|
|
print("\n=== Top 20 Kategorien (roh) ===")
|
|
cats = Counter()
|
|
for r in ok:
|
|
for c in (r.get("types_of_business") or []):
|
|
cats[c.strip().lower()] += 1
|
|
for c, n in cats.most_common(20):
|
|
print(f" {n:>4} {c}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |