update

2026-07-10 16:34:27 -05:00
parent 6881d932b8
commit 741d82c06b
3 changed files with 235 additions and 19 deletions
--- a/debug_one_pdf.py
+++ b/debug_one_pdf.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+debug_one_pdf.py - Verarbeitet EIN PDF und zeigt die ROHE Modell-Antwort,
+egal ob sie als JSON parst oder nicht. Zum Debuggen einzelner Problemfaelle.
+
+Aufruf:
+  python3 debug_one_pdf.py \
+    --api http://localhost:8000/v1 --model "gemma-4-12b" \
+    --pdf ~/data/S/"Scott, Dexter Notes 031226.pdf"
+"""
+import argparse, io, base64, json, re
+from openai import OpenAI
+from pypdf import PdfReader
+from pdf2image import convert_from_path
+
+# --- dieselbe Konfiguration wie im Hauptscript ---
+JSON_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "is_buyer_sheet": {"type": "boolean"},
+        "name_company": {"type": ["string", "null"]},
+        "prospective_buyer": {"type": ["string", "null"]},
+        "company": {"type": ["string", "null"]},
+        "phone": {"type": ["string", "null"]},
+        "cell": {"type": ["string", "null"]},
+        "email": {"type": ["string", "null"]},
+        "address": {"type": ["string", "null"]},
+        "state": {"type": ["string", "null"]},
+        "how_did_you_hear": {"type": ["string", "null"]},
+        "interested_in_updates": {"type": ["boolean", "null"]},
+        "types_of_business_raw": {"type": ["string", "null"]},
+        "types_of_business": {"type": "array", "items": {"type": "string"}},
+        "background_experience": {"type": ["string", "null"]},
+        "date_of_introduction": {"type": ["string", "null"]},
+    },
+    "required": ["is_buyer_sheet", "name_company", "prospective_buyer", "company",
+                 "phone", "cell", "email", "address", "state", "how_did_you_hear",
+                 "interested_in_updates", "types_of_business_raw", "types_of_business",
+                 "background_experience", "date_of_introduction"],
+}
+
+SYSTEM_PROMPT = """Du bist ein praezises Datenextraktions-System fuer Formulare der Firma "BizMatch Business Brokerage". Extrahiere die verlangten Felder als JSON. Fehlende Felder -> null. ERFINDE NICHTS."""
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--api", default="http://localhost:8000/v1")
+ap.add_argument("--model", default="gemma-4-12b")
+ap.add_argument("--pdf", required=True)
+ap.add_argument("--max-tokens", type=int, default=512)
+args = ap.parse_args()
+
+client = OpenAI(base_url=args.api, api_key="x", timeout=120, max_retries=0)
+
+n_pages = len(PdfReader(args.pdf).pages)
+print(f"PDF: {args.pdf}")
+print(f"Seiten: {n_pages}")
+
+# Vision-Pfad (2 Seiten, wie im Hauptscript)
+imgs = convert_from_path(args.pdf, first_page=1, last_page=2, dpi=150)
+payload = [{"type":"text","text":"Extrahiere die Felder aus diesem Buyer Information Sheet:"}]
+for img in imgs:
+    w,h = img.size
+    scale = min(1.0, 1600/max(w,h))
+    if scale < 1.0:
+        img = img.resize((int(w*scale), int(h*scale)))
+    buf = io.BytesIO(); img.convert("RGB").save(buf, format="JPEG", quality=80)
+    b64 = base64.b64encode(buf.getvalue()).decode()
+    payload.append({"type":"image_url","image_url":{"url":f"data:image/jpeg;base64,{b64}"}})
+
+print(f"\nSende {len(imgs)} Bild(er) an das Modell, max_tokens={args.max_tokens}...\n")
+
+resp = client.chat.completions.create(
+    model=args.model,
+    messages=[{"role":"system","content":SYSTEM_PROMPT},
+              {"role":"user","content":payload}],
+    temperature=0.0,
+    max_tokens=args.max_tokens,
+    response_format={"type":"json_schema","json_schema":{"name":"buyer","schema":JSON_SCHEMA}},
+    extra_body={"chat_template_kwargs":{"enable_thinking":False}},
+)
+
+choice = resp.choices[0]
+raw = choice.message.content
+print("=== finish_reason ===")
+print(choice.finish_reason)
+print(f"\n=== ROHE ANTWORT ({len(raw or '')} Zeichen) ===")
+print(repr(raw))
+print("\n=== ANTWORT LESBAR ===")
+print(raw)
+
+print("\n=== PARSE-VERSUCH ===")
+try:
+    parsed = json.loads(raw)
+    print("OK, parst sauber:")
+    print(json.dumps(parsed, indent=2, ensure_ascii=False))
+except Exception as e:
+    print(f"FEHLER: {e}")
--- a/extract_buyers_llamacpp.py
+++ b/extract_buyers_llamacpp.py
@@ -174,15 +174,16 @@ def extract_text(pdf_path, max_pages=4):
        return ""


-def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=1600, quality=80, dpi=150):
+def pdf_to_base64_images(pdf_path, first_page=1, max_pages=2, max_dim=1600, quality=80, dpi=150):
    """
-    Erste max_pages Seiten -> Base64-JPEG.
-    Moderate Aufloesung (dpi=150, max_dim=1600): reicht fuer Checkbox/Handschrift
-    (im Test bei dpi=150 erkannt), haelt aber die Vision-Payload klein genug,
-    dass der Prompt-Cache nicht ueberlaeuft (die "making room ... MiB"-Warnungen).
+    Seiten [first_page .. first_page+max_pages-1] -> Base64-JPEG.
+    Moderate Aufloesung (dpi=150, max_dim=1600): reicht fuer Checkbox/Handschrift.
+    first_page erlaubt das Ueberspringen einer fuehrenden Seite (z.B. die
+    ueberfluessige handschriftliche Notiz-Seite bei "Notes"-Dateien).
    """
+    last_page = first_page + max_pages - 1
    try:
-        images = convert_from_path(pdf_path, first_page=1, last_page=max_pages, dpi=dpi)
+        images = convert_from_path(pdf_path, first_page=first_page, last_page=last_page, dpi=dpi)
    except Exception as e:
        print(f"   [Bildkonvertierung-Fehler] {e}")
        return []
@@ -202,11 +203,12 @@ def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=1600, quality=80, dpi=15
 # LLM-Call (mit Guided Decoding + Vision-Budget)
 # ---------------------------------------------------------------------------

-def call_model(client, model, content_payload, vision_budget=None):
+def call_model(client, model, content_payload, vision_budget=None, max_tokens=768):
    # llama.cpp erzwingt JSON ueber response_format mit json_schema.
    # WICHTIG gegen Gemma-4 "<unused49>-Flood"/Runaway:
    #  - enable_thinking:false per Request (zuverlaessiger als nur Server-Flag)
-    #  - kleines max_tokens (ein Datensatz braucht ~300-400 Tokens)
+    #  - max_tokens moderat (768: genug fuer volle Datensaetze mit langem
+    #    Background/vielen Kategorien, aber begrenzt genug gegen Runaway)
    #  - repeat_penalty gegen Wiederholschleifen
    resp = client.chat.completions.create(
        model=model,
@@ -215,7 +217,7 @@ def call_model(client, model, content_payload, vision_budget=None):
            {"role": "user", "content": content_payload},
        ],
        temperature=0.0,
-        max_tokens=512,
+        max_tokens=max_tokens,
        response_format={
            "type": "json_schema",
            "json_schema": {"name": "buyer_sheet", "schema": JSON_SCHEMA},
@@ -305,8 +307,8 @@ def _extract_via_text(client, model, text):
    return data


-def _extract_via_vision(client, model, pdf_path, max_pages):
-    imgs = pdf_to_base64_images(pdf_path, max_pages=max_pages)
+def _extract_via_vision(client, model, pdf_path, max_pages, first_page=1):
+    imgs = pdf_to_base64_images(pdf_path, first_page=first_page, max_pages=max_pages)
    if not imgs:
        return None
    payload = [{"type": "text", "text": USER_IMG_INTRO}]
@@ -322,33 +324,41 @@ def _extract_via_vision(client, model, pdf_path, max_pages):

 def process_pdf(client, model, pdf_path, vision_budget=None):
    """
-    Hybrid-Strategie:
+    Hybrid-Strategie mit Notes-Seitenlogik:
+      - "Notes"-Dateien: Seite 1 ist eine ueberfluessige handschriftliche Notiz.
+        Die Buyer-Felder stehen auf Seite 2 (Info Sheet) + 3 (CA mit Datum/Name).
+        Darum bei Notes-Dateien first_page=2.
+      - Normale Dateien: Info Sheet = Seite 1, CA = Seite 2. first_page=1.
      - reiner Scan (keine Textebene): nur Vision
-      - grosser Scan (> MAX_SCAN_PAGES_TOTAL): nur erste Seite Vision
+      - grosser Scan (> MAX_SCAN_PAGES_TOTAL): nur erste relevante Seite pruefen
      - getipptes PDF mit Textebene: Text UND Vision, dann mergen
-        (Text bringt getippten Background, Vision die handschriftlichen Felder)
    """
    n_pages = pdf_page_count(pdf_path)
    text = extract_text(pdf_path)
    has_text = len(text.strip()) >= TEXT_SCAN_THRESHOLD

+    # "Notes" im Dateinamen -> fuehrende Notiz-Seite ueberspringen.
+    # Aber nur, wenn genug Seiten da sind (sonst normal ab Seite 1).
+    is_notes = "notes" in os.path.basename(pdf_path).lower()
+    vpage = 2 if (is_notes and n_pages >= 3) else 1
+
    truncated_note = None
    try:
        if not has_text and n_pages > MAX_SCAN_PAGES_TOTAL:
-            # grosser reiner Scan: nur erste Seite pruefen
-            truncated_note = f"grosser Scan ({n_pages} Seiten), nur erste Seite geprueft"
-            data = _extract_via_vision(client, model, pdf_path, max_pages=1)
+            # grosser reiner Scan: nur die erste relevante Seite pruefen
+            truncated_note = f"grosser Scan ({n_pages} Seiten), begrenzt geprueft"
+            data = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage)
            mode = "vision"
        elif not has_text:
            # reiner Scan
-            data = _extract_via_vision(client, model, pdf_path, max_pages=2)
+            data = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage)
            mode = "vision"
        else:
            # Textebene vorhanden -> HYBRID: beide Pfade, dann mergen
            text_rec = _extract_via_text(client, model, text)
            vision_rec = None
            if n_pages <= MAX_SCAN_PAGES_TOTAL:
-                vision_rec = _extract_via_vision(client, model, pdf_path, max_pages=2)
+                vision_rec = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage)
            data = merge_records(text_rec, vision_rec)
            mode = "hybrid" if vision_rec is not None else "text"
    except Exception as e:
@@ -357,6 +367,8 @@ def process_pdf(client, model, pdf_path, vision_budget=None):
    if data is None:
        return {"_error": "JSON-Parse fehlgeschlagen", "n_pages": n_pages}, mode
    data["n_pages"] = n_pages
+    if is_notes:
+        data["_notes_file"] = True
    if truncated_note:
        data["_note"] = truncated_note
    return data, mode
@@ -411,6 +423,11 @@ def main():
            print(f"[{i}/{len(pdfs)}] {fdate or '????-??-??'}  {rel}")
            try:
                data, mode = process_pdf(client, args.model, pdf, args.vision_budget)
+                # Bei Fehler EINMAL erneut versuchen (faengt seltene
+                # nicht-deterministische Parse-Ausrutscher ab).
+                if "_error" in data:
+                    print(f"   (Fehler, ein Wiederholversuch...)")
+                    data, mode = process_pdf(client, args.model, pdf, args.vision_budget)
            except KeyboardInterrupt:
                print("\nAbbruch durch Nutzer. Bisheriges ist gesichert.")
                break
--- a/inspect_poc.py
+++ b/inspect_poc.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+inspect_poc.py - Wertet buyers_raw.jsonl nach dem POC-Lauf aus.
+
+Zeigt:
+  - Feld-Fuellquoten (wie oft ist welches Feld nicht null?)
+  - Text- vs Vision-Qualitaet im Vergleich
+  - die Multi-Sheet-Gruppierung: welche Person hat mehrere Sheets?
+  - Datums-Parsing-Quote (fuer die "letzte 5 Jahre"-Abfragen kritisch)
+
+Aufruf: python inspect_poc.py ./poc_out/buyers_raw.jsonl
+"""
+
+import sys
+import json
+import re
+from collections import defaultdict, Counter
+
+
+def person_key(rec):
+    """Gruppierungsschluessel: bevorzugt echten Namen, sonst name_company."""
+    name = (rec.get("prospective_buyer") or rec.get("name_company") or "").strip().lower()
+    # "Nachname, Vorname" und "Vorname Nachname" grob angleichen
+    name = re.sub(r"\s+", " ", name)
+    return name or "(unbekannt)"
+
+
+def main():
+    path = sys.argv[1] if len(sys.argv) > 1 else "./poc_out/buyers_raw.jsonl"
+    records = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+
+    ok = [r for r in records if "_error" not in r]
+    err = [r for r in records if "_error" in r]
+
+    print(f"Datensaetze: {len(records)}  (ok: {len(ok)}, Fehler: {len(err)})\n")
+
+    # Feld-Fuellquoten
+    fields = ["name_company", "prospective_buyer", "company", "phone", "cell",
+              "email", "address", "state", "how_did_you_hear",
+              "interested_in_updates", "types_of_business", "background_experience",
+              "date_of_introduction"]
+    print("=== Feld-Fuellquoten (nicht-null / erfolgreiche) ===")
+    for fld in fields:
+        filled = 0
+        for r in ok:
+            v = r.get(fld)
+            if v not in (None, "", [], {}):
+                filled += 1
+        pct = 100 * filled / len(ok) if ok else 0
+        print(f"  {fld:<24} {filled:>4}/{len(ok)}  ({pct:4.0f}%)")
+
+    # Text vs Vision
+    print("\n=== Text- vs Vision-Pfad ===")
+    for mode in ("text", "vision"):
+        sub = [r for r in ok if r.get("extraction_mode") == mode]
+        if not sub:
+            continue
+        # als grobe Qualitaetsmetrik: durchschnittliche Zahl gefuellter Felder
+        avg_filled = sum(
+            sum(1 for f in fields if r.get(f) not in (None, "", [], {})) for r in sub
+        ) / len(sub)
+        print(f"  {mode:<7} {len(sub):>4} Docs, im Schnitt {avg_filled:.1f}/{len(fields)} Felder gefuellt")
+
+    # Datums-Parsing
+    print("\n=== Date of Introduction ===")
+    iso = sum(1 for r in ok if isinstance(r.get("date_of_introduction"), str)
+              and re.match(r"\d{4}-\d{2}-\d{2}", r["date_of_introduction"]))
+    nonnull = sum(1 for r in ok if r.get("date_of_introduction"))
+    print(f"  vorhanden: {nonnull}/{len(ok)}   davon sauber ISO (YYYY-MM-DD): {iso}")
+
+    # Multi-Sheet-Gruppierung
+    print("\n=== Personen mit mehreren Sheets ===")
+    groups = defaultdict(list)
+    for r in ok:
+        groups[person_key(r)].append(r)
+    multi = {k: v for k, v in groups.items() if len(v) > 1}
+    print(f"  distinkte Personen: {len(groups)}, davon mit >1 Sheet: {len(multi)}")
+    for name, recs in sorted(multi.items(), key=lambda x: -len(x[1]))[:10]:
+        cats = set()
+        dates = []
+        for r in recs:
+            cats.update(r.get("types_of_business") or [])
+            if r.get("date_of_introduction"):
+                dates.append(r["date_of_introduction"])
+        print(f"    {name!r}: {len(recs)} Sheets | Kategorien: {sorted(cats)} | Daten: {sorted(dates)}")
+
+    # Haeufigste Kategorien
+    print("\n=== Top 20 Kategorien (roh) ===")
+    cats = Counter()
+    for r in ok:
+        for c in (r.get("types_of_business") or []):
+            cats[c.strip().lower()] += 1
+    for c, n in cats.most_common(20):
+        print(f"  {n:>4}  {c}")
+
+
+if __name__ == "__main__":
+    main()