diff --git a/debug_one_pdf.py b/debug_one_pdf.py new file mode 100644 index 0000000..b90ba18 --- /dev/null +++ b/debug_one_pdf.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +debug_one_pdf.py - Verarbeitet EIN PDF und zeigt die ROHE Modell-Antwort, +egal ob sie als JSON parst oder nicht. Zum Debuggen einzelner Problemfaelle. + +Aufruf: + python3 debug_one_pdf.py \ + --api http://localhost:8000/v1 --model "gemma-4-12b" \ + --pdf ~/data/S/"Scott, Dexter Notes 031226.pdf" +""" +import argparse, io, base64, json, re +from openai import OpenAI +from pypdf import PdfReader +from pdf2image import convert_from_path + +# --- dieselbe Konfiguration wie im Hauptscript --- +JSON_SCHEMA = { + "type": "object", + "properties": { + "is_buyer_sheet": {"type": "boolean"}, + "name_company": {"type": ["string", "null"]}, + "prospective_buyer": {"type": ["string", "null"]}, + "company": {"type": ["string", "null"]}, + "phone": {"type": ["string", "null"]}, + "cell": {"type": ["string", "null"]}, + "email": {"type": ["string", "null"]}, + "address": {"type": ["string", "null"]}, + "state": {"type": ["string", "null"]}, + "how_did_you_hear": {"type": ["string", "null"]}, + "interested_in_updates": {"type": ["boolean", "null"]}, + "types_of_business_raw": {"type": ["string", "null"]}, + "types_of_business": {"type": "array", "items": {"type": "string"}}, + "background_experience": {"type": ["string", "null"]}, + "date_of_introduction": {"type": ["string", "null"]}, + }, + "required": ["is_buyer_sheet", "name_company", "prospective_buyer", "company", + "phone", "cell", "email", "address", "state", "how_did_you_hear", + "interested_in_updates", "types_of_business_raw", "types_of_business", + "background_experience", "date_of_introduction"], +} + +SYSTEM_PROMPT = """Du bist ein praezises Datenextraktions-System fuer Formulare der Firma "BizMatch Business Brokerage". Extrahiere die verlangten Felder als JSON. Fehlende Felder -> null. ERFINDE NICHTS.""" + +ap = argparse.ArgumentParser() +ap.add_argument("--api", default="http://localhost:8000/v1") +ap.add_argument("--model", default="gemma-4-12b") +ap.add_argument("--pdf", required=True) +ap.add_argument("--max-tokens", type=int, default=512) +args = ap.parse_args() + +client = OpenAI(base_url=args.api, api_key="x", timeout=120, max_retries=0) + +n_pages = len(PdfReader(args.pdf).pages) +print(f"PDF: {args.pdf}") +print(f"Seiten: {n_pages}") + +# Vision-Pfad (2 Seiten, wie im Hauptscript) +imgs = convert_from_path(args.pdf, first_page=1, last_page=2, dpi=150) +payload = [{"type":"text","text":"Extrahiere die Felder aus diesem Buyer Information Sheet:"}] +for img in imgs: + w,h = img.size + scale = min(1.0, 1600/max(w,h)) + if scale < 1.0: + img = img.resize((int(w*scale), int(h*scale))) + buf = io.BytesIO(); img.convert("RGB").save(buf, format="JPEG", quality=80) + b64 = base64.b64encode(buf.getvalue()).decode() + payload.append({"type":"image_url","image_url":{"url":f"data:image/jpeg;base64,{b64}"}}) + +print(f"\nSende {len(imgs)} Bild(er) an das Modell, max_tokens={args.max_tokens}...\n") + +resp = client.chat.completions.create( + model=args.model, + messages=[{"role":"system","content":SYSTEM_PROMPT}, + {"role":"user","content":payload}], + temperature=0.0, + max_tokens=args.max_tokens, + response_format={"type":"json_schema","json_schema":{"name":"buyer","schema":JSON_SCHEMA}}, + extra_body={"chat_template_kwargs":{"enable_thinking":False}}, +) + +choice = resp.choices[0] +raw = choice.message.content +print("=== finish_reason ===") +print(choice.finish_reason) +print(f"\n=== ROHE ANTWORT ({len(raw or '')} Zeichen) ===") +print(repr(raw)) +print("\n=== ANTWORT LESBAR ===") +print(raw) + +print("\n=== PARSE-VERSUCH ===") +try: + parsed = json.loads(raw) + print("OK, parst sauber:") + print(json.dumps(parsed, indent=2, ensure_ascii=False)) +except Exception as e: + print(f"FEHLER: {e}") \ No newline at end of file diff --git a/extract_buyers_llamacpp.py b/extract_buyers_llamacpp.py index 038c08c..e7812c3 100644 --- a/extract_buyers_llamacpp.py +++ b/extract_buyers_llamacpp.py @@ -174,15 +174,16 @@ def extract_text(pdf_path, max_pages=4): return "" -def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=1600, quality=80, dpi=150): +def pdf_to_base64_images(pdf_path, first_page=1, max_pages=2, max_dim=1600, quality=80, dpi=150): """ - Erste max_pages Seiten -> Base64-JPEG. - Moderate Aufloesung (dpi=150, max_dim=1600): reicht fuer Checkbox/Handschrift - (im Test bei dpi=150 erkannt), haelt aber die Vision-Payload klein genug, - dass der Prompt-Cache nicht ueberlaeuft (die "making room ... MiB"-Warnungen). + Seiten [first_page .. first_page+max_pages-1] -> Base64-JPEG. + Moderate Aufloesung (dpi=150, max_dim=1600): reicht fuer Checkbox/Handschrift. + first_page erlaubt das Ueberspringen einer fuehrenden Seite (z.B. die + ueberfluessige handschriftliche Notiz-Seite bei "Notes"-Dateien). """ + last_page = first_page + max_pages - 1 try: - images = convert_from_path(pdf_path, first_page=1, last_page=max_pages, dpi=dpi) + images = convert_from_path(pdf_path, first_page=first_page, last_page=last_page, dpi=dpi) except Exception as e: print(f" [Bildkonvertierung-Fehler] {e}") return [] @@ -202,11 +203,12 @@ def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=1600, quality=80, dpi=15 # LLM-Call (mit Guided Decoding + Vision-Budget) # --------------------------------------------------------------------------- -def call_model(client, model, content_payload, vision_budget=None): +def call_model(client, model, content_payload, vision_budget=None, max_tokens=768): # llama.cpp erzwingt JSON ueber response_format mit json_schema. # WICHTIG gegen Gemma-4 "-Flood"/Runaway: # - enable_thinking:false per Request (zuverlaessiger als nur Server-Flag) - # - kleines max_tokens (ein Datensatz braucht ~300-400 Tokens) + # - max_tokens moderat (768: genug fuer volle Datensaetze mit langem + # Background/vielen Kategorien, aber begrenzt genug gegen Runaway) # - repeat_penalty gegen Wiederholschleifen resp = client.chat.completions.create( model=model, @@ -215,7 +217,7 @@ def call_model(client, model, content_payload, vision_budget=None): {"role": "user", "content": content_payload}, ], temperature=0.0, - max_tokens=512, + max_tokens=max_tokens, response_format={ "type": "json_schema", "json_schema": {"name": "buyer_sheet", "schema": JSON_SCHEMA}, @@ -305,8 +307,8 @@ def _extract_via_text(client, model, text): return data -def _extract_via_vision(client, model, pdf_path, max_pages): - imgs = pdf_to_base64_images(pdf_path, max_pages=max_pages) +def _extract_via_vision(client, model, pdf_path, max_pages, first_page=1): + imgs = pdf_to_base64_images(pdf_path, first_page=first_page, max_pages=max_pages) if not imgs: return None payload = [{"type": "text", "text": USER_IMG_INTRO}] @@ -322,33 +324,41 @@ def _extract_via_vision(client, model, pdf_path, max_pages): def process_pdf(client, model, pdf_path, vision_budget=None): """ - Hybrid-Strategie: + Hybrid-Strategie mit Notes-Seitenlogik: + - "Notes"-Dateien: Seite 1 ist eine ueberfluessige handschriftliche Notiz. + Die Buyer-Felder stehen auf Seite 2 (Info Sheet) + 3 (CA mit Datum/Name). + Darum bei Notes-Dateien first_page=2. + - Normale Dateien: Info Sheet = Seite 1, CA = Seite 2. first_page=1. - reiner Scan (keine Textebene): nur Vision - - grosser Scan (> MAX_SCAN_PAGES_TOTAL): nur erste Seite Vision + - grosser Scan (> MAX_SCAN_PAGES_TOTAL): nur erste relevante Seite pruefen - getipptes PDF mit Textebene: Text UND Vision, dann mergen - (Text bringt getippten Background, Vision die handschriftlichen Felder) """ n_pages = pdf_page_count(pdf_path) text = extract_text(pdf_path) has_text = len(text.strip()) >= TEXT_SCAN_THRESHOLD + # "Notes" im Dateinamen -> fuehrende Notiz-Seite ueberspringen. + # Aber nur, wenn genug Seiten da sind (sonst normal ab Seite 1). + is_notes = "notes" in os.path.basename(pdf_path).lower() + vpage = 2 if (is_notes and n_pages >= 3) else 1 + truncated_note = None try: if not has_text and n_pages > MAX_SCAN_PAGES_TOTAL: - # grosser reiner Scan: nur erste Seite pruefen - truncated_note = f"grosser Scan ({n_pages} Seiten), nur erste Seite geprueft" - data = _extract_via_vision(client, model, pdf_path, max_pages=1) + # grosser reiner Scan: nur die erste relevante Seite pruefen + truncated_note = f"grosser Scan ({n_pages} Seiten), begrenzt geprueft" + data = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage) mode = "vision" elif not has_text: # reiner Scan - data = _extract_via_vision(client, model, pdf_path, max_pages=2) + data = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage) mode = "vision" else: # Textebene vorhanden -> HYBRID: beide Pfade, dann mergen text_rec = _extract_via_text(client, model, text) vision_rec = None if n_pages <= MAX_SCAN_PAGES_TOTAL: - vision_rec = _extract_via_vision(client, model, pdf_path, max_pages=2) + vision_rec = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage) data = merge_records(text_rec, vision_rec) mode = "hybrid" if vision_rec is not None else "text" except Exception as e: @@ -357,6 +367,8 @@ def process_pdf(client, model, pdf_path, vision_budget=None): if data is None: return {"_error": "JSON-Parse fehlgeschlagen", "n_pages": n_pages}, mode data["n_pages"] = n_pages + if is_notes: + data["_notes_file"] = True if truncated_note: data["_note"] = truncated_note return data, mode @@ -411,6 +423,11 @@ def main(): print(f"[{i}/{len(pdfs)}] {fdate or '????-??-??'} {rel}") try: data, mode = process_pdf(client, args.model, pdf, args.vision_budget) + # Bei Fehler EINMAL erneut versuchen (faengt seltene + # nicht-deterministische Parse-Ausrutscher ab). + if "_error" in data: + print(f" (Fehler, ein Wiederholversuch...)") + data, mode = process_pdf(client, args.model, pdf, args.vision_budget) except KeyboardInterrupt: print("\nAbbruch durch Nutzer. Bisheriges ist gesichert.") break diff --git a/inspect_poc.py b/inspect_poc.py new file mode 100644 index 0000000..64afe84 --- /dev/null +++ b/inspect_poc.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +""" +inspect_poc.py - Wertet buyers_raw.jsonl nach dem POC-Lauf aus. + +Zeigt: + - Feld-Fuellquoten (wie oft ist welches Feld nicht null?) + - Text- vs Vision-Qualitaet im Vergleich + - die Multi-Sheet-Gruppierung: welche Person hat mehrere Sheets? + - Datums-Parsing-Quote (fuer die "letzte 5 Jahre"-Abfragen kritisch) + +Aufruf: python inspect_poc.py ./poc_out/buyers_raw.jsonl +""" + +import sys +import json +import re +from collections import defaultdict, Counter + + +def person_key(rec): + """Gruppierungsschluessel: bevorzugt echten Namen, sonst name_company.""" + name = (rec.get("prospective_buyer") or rec.get("name_company") or "").strip().lower() + # "Nachname, Vorname" und "Vorname Nachname" grob angleichen + name = re.sub(r"\s+", " ", name) + return name or "(unbekannt)" + + +def main(): + path = sys.argv[1] if len(sys.argv) > 1 else "./poc_out/buyers_raw.jsonl" + records = [] + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + records.append(json.loads(line)) + + ok = [r for r in records if "_error" not in r] + err = [r for r in records if "_error" in r] + + print(f"Datensaetze: {len(records)} (ok: {len(ok)}, Fehler: {len(err)})\n") + + # Feld-Fuellquoten + fields = ["name_company", "prospective_buyer", "company", "phone", "cell", + "email", "address", "state", "how_did_you_hear", + "interested_in_updates", "types_of_business", "background_experience", + "date_of_introduction"] + print("=== Feld-Fuellquoten (nicht-null / erfolgreiche) ===") + for fld in fields: + filled = 0 + for r in ok: + v = r.get(fld) + if v not in (None, "", [], {}): + filled += 1 + pct = 100 * filled / len(ok) if ok else 0 + print(f" {fld:<24} {filled:>4}/{len(ok)} ({pct:4.0f}%)") + + # Text vs Vision + print("\n=== Text- vs Vision-Pfad ===") + for mode in ("text", "vision"): + sub = [r for r in ok if r.get("extraction_mode") == mode] + if not sub: + continue + # als grobe Qualitaetsmetrik: durchschnittliche Zahl gefuellter Felder + avg_filled = sum( + sum(1 for f in fields if r.get(f) not in (None, "", [], {})) for r in sub + ) / len(sub) + print(f" {mode:<7} {len(sub):>4} Docs, im Schnitt {avg_filled:.1f}/{len(fields)} Felder gefuellt") + + # Datums-Parsing + print("\n=== Date of Introduction ===") + iso = sum(1 for r in ok if isinstance(r.get("date_of_introduction"), str) + and re.match(r"\d{4}-\d{2}-\d{2}", r["date_of_introduction"])) + nonnull = sum(1 for r in ok if r.get("date_of_introduction")) + print(f" vorhanden: {nonnull}/{len(ok)} davon sauber ISO (YYYY-MM-DD): {iso}") + + # Multi-Sheet-Gruppierung + print("\n=== Personen mit mehreren Sheets ===") + groups = defaultdict(list) + for r in ok: + groups[person_key(r)].append(r) + multi = {k: v for k, v in groups.items() if len(v) > 1} + print(f" distinkte Personen: {len(groups)}, davon mit >1 Sheet: {len(multi)}") + for name, recs in sorted(multi.items(), key=lambda x: -len(x[1]))[:10]: + cats = set() + dates = [] + for r in recs: + cats.update(r.get("types_of_business") or []) + if r.get("date_of_introduction"): + dates.append(r["date_of_introduction"]) + print(f" {name!r}: {len(recs)} Sheets | Kategorien: {sorted(cats)} | Daten: {sorted(dates)}") + + # Haeufigste Kategorien + print("\n=== Top 20 Kategorien (roh) ===") + cats = Counter() + for r in ok: + for c in (r.get("types_of_business") or []): + cats[c.strip().lower()] += 1 + for c, n in cats.most_common(20): + print(f" {n:>4} {c}") + + +if __name__ == "__main__": + main() \ No newline at end of file