aknuth/ai-bayarea
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Andreas Knuth
2026-07-10 16:34:27 -05:00
parent 6881d932b8
commit 741d82c06b
3 changed files with 235 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
debug_one_pdf.py Normal file
View File

@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""
debug_one_pdf.py - Verarbeitet EIN PDF und zeigt die ROHE Modell-Antwort,
egal ob sie als JSON parst oder nicht. Zum Debuggen einzelner Problemfaelle.
Aufruf:
python3 debug_one_pdf.py \
--api http://localhost:8000/v1 --model "gemma-4-12b" \
--pdf ~/data/S/"Scott, Dexter Notes 031226.pdf"
"""
import argparse, io, base64, json, re
from openai import OpenAI
from pypdf import PdfReader
from pdf2image import convert_from_path
# --- dieselbe Konfiguration wie im Hauptscript ---
JSON_SCHEMA = {
"type": "object",
"properties": {
"is_buyer_sheet": {"type": "boolean"},
"name_company": {"type": ["string", "null"]},
"prospective_buyer": {"type": ["string", "null"]},
"company": {"type": ["string", "null"]},
"phone": {"type": ["string", "null"]},
"cell": {"type": ["string", "null"]},
"email": {"type": ["string", "null"]},
"address": {"type": ["string", "null"]},
"state": {"type": ["string", "null"]},
"how_did_you_hear": {"type": ["string", "null"]},
"interested_in_updates": {"type": ["boolean", "null"]},
"types_of_business_raw": {"type": ["string", "null"]},
"types_of_business": {"type": "array", "items": {"type": "string"}},
"background_experience": {"type": ["string", "null"]},
"date_of_introduction": {"type": ["string", "null"]},
},
"required": ["is_buyer_sheet", "name_company", "prospective_buyer", "company",
"phone", "cell", "email", "address", "state", "how_did_you_hear",
"interested_in_updates", "types_of_business_raw", "types_of_business",
"background_experience", "date_of_introduction"],
}
SYSTEM_PROMPT = """Du bist ein praezises Datenextraktions-System fuer Formulare der Firma "BizMatch Business Brokerage". Extrahiere die verlangten Felder als JSON. Fehlende Felder -> null. ERFINDE NICHTS."""
ap = argparse.ArgumentParser()
ap.add_argument("--api", default="http://localhost:8000/v1")
ap.add_argument("--model", default="gemma-4-12b")
ap.add_argument("--pdf", required=True)
ap.add_argument("--max-tokens", type=int, default=512)
args = ap.parse_args()
client = OpenAI(base_url=args.api, api_key="x", timeout=120, max_retries=0)
n_pages = len(PdfReader(args.pdf).pages)
print(f"PDF: {args.pdf}")
print(f"Seiten: {n_pages}")
# Vision-Pfad (2 Seiten, wie im Hauptscript)
imgs = convert_from_path(args.pdf, first_page=1, last_page=2, dpi=150)
payload = [{"type":"text","text":"Extrahiere die Felder aus diesem Buyer Information Sheet:"}]
for img in imgs:
w,h = img.size
scale = min(1.0, 1600/max(w,h))
if scale < 1.0:
img = img.resize((int(w*scale), int(h*scale)))
buf = io.BytesIO(); img.convert("RGB").save(buf, format="JPEG", quality=80)
b64 = base64.b64encode(buf.getvalue()).decode()
payload.append({"type":"image_url","image_url":{"url":f"data:image/jpeg;base64,{b64}"}})
print(f"\nSende {len(imgs)} Bild(er) an das Modell, max_tokens={args.max_tokens}...\n")
resp = client.chat.completions.create(
model=args.model,
messages=[{"role":"system","content":SYSTEM_PROMPT},
{"role":"user","content":payload}],
temperature=0.0,
max_tokens=args.max_tokens,
response_format={"type":"json_schema","json_schema":{"name":"buyer","schema":JSON_SCHEMA}},
extra_body={"chat_template_kwargs":{"enable_thinking":False}},
)
choice = resp.choices[0]
raw = choice.message.content
print("=== finish_reason ===")
print(choice.finish_reason)
print(f"\n=== ROHE ANTWORT ({len(raw or '')} Zeichen) ===")
print(repr(raw))
print("\n=== ANTWORT LESBAR ===")
print(raw)
print("\n=== PARSE-VERSUCH ===")
try:
parsed = json.loads(raw)
print("OK, parst sauber:")
print(json.dumps(parsed, indent=2, ensure_ascii=False))
except Exception as e:
print(f"FEHLER: {e}")

55
extract_buyers_llamacpp.py
View File

@@ -174,15 +174,16 @@ def extract_text(pdf_path, max_pages=4):
return ""
def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=1600, quality=80, dpi=150):
def pdf_to_base64_images(pdf_path, first_page=1, max_pages=2, max_dim=1600, quality=80, dpi=150):
"""
Erste max_pages Seiten -> Base64-JPEG.
Moderate Aufloesung (dpi=150, max_dim=1600): reicht fuer Checkbox/Handschrift
(im Test bei dpi=150 erkannt), haelt aber die Vision-Payload klein genug,
dass der Prompt-Cache nicht ueberlaeuft (die "making room ... MiB"-Warnungen).
Seiten [first_page .. first_page+max_pages-1] -> Base64-JPEG.
Moderate Aufloesung (dpi=150, max_dim=1600): reicht fuer Checkbox/Handschrift.
first_page erlaubt das Ueberspringen einer fuehrenden Seite (z.B. die
ueberfluessige handschriftliche Notiz-Seite bei "Notes"-Dateien).
"""
last_page = first_page + max_pages - 1
try:
images = convert_from_path(pdf_path, first_page=1, last_page=max_pages, dpi=dpi)
images = convert_from_path(pdf_path, first_page=first_page, last_page=last_page, dpi=dpi)
except Exception as e:
print(f" [Bildkonvertierung-Fehler] {e}")
return []
@@ -202,11 +203,12 @@ def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=1600, quality=80, dpi=15
# LLM-Call (mit Guided Decoding + Vision-Budget)
# ---------------------------------------------------------------------------
def call_model(client, model, content_payload, vision_budget=None):
def call_model(client, model, content_payload, vision_budget=None, max_tokens=768):
# llama.cpp erzwingt JSON ueber response_format mit json_schema.
# WICHTIG gegen Gemma-4 "<unused49>-Flood"/Runaway:
# - enable_thinking:false per Request (zuverlaessiger als nur Server-Flag)
# - kleines max_tokens (ein Datensatz braucht ~300-400 Tokens)
# - max_tokens moderat (768: genug fuer volle Datensaetze mit langem
# Background/vielen Kategorien, aber begrenzt genug gegen Runaway)
# - repeat_penalty gegen Wiederholschleifen
resp = client.chat.completions.create(
model=model,
@@ -215,7 +217,7 @@ def call_model(client, model, content_payload, vision_budget=None):
{"role": "user", "content": content_payload},
],
temperature=0.0,
max_tokens=512,
max_tokens=max_tokens,
response_format={
"type": "json_schema",
"json_schema": {"name": "buyer_sheet", "schema": JSON_SCHEMA},
@@ -305,8 +307,8 @@ def _extract_via_text(client, model, text):
return data
def _extract_via_vision(client, model, pdf_path, max_pages):
imgs = pdf_to_base64_images(pdf_path, max_pages=max_pages)
def _extract_via_vision(client, model, pdf_path, max_pages, first_page=1):
imgs = pdf_to_base64_images(pdf_path, first_page=first_page, max_pages=max_pages)
if not imgs:
return None
payload = [{"type": "text", "text": USER_IMG_INTRO}]
@@ -322,33 +324,41 @@ def _extract_via_vision(client, model, pdf_path, max_pages):
def process_pdf(client, model, pdf_path, vision_budget=None):
"""
Hybrid-Strategie:
Hybrid-Strategie mit Notes-Seitenlogik:
- "Notes"-Dateien: Seite 1 ist eine ueberfluessige handschriftliche Notiz.
Die Buyer-Felder stehen auf Seite 2 (Info Sheet) + 3 (CA mit Datum/Name).
Darum bei Notes-Dateien first_page=2.
- Normale Dateien: Info Sheet = Seite 1, CA = Seite 2. first_page=1.
- reiner Scan (keine Textebene): nur Vision
- grosser Scan (> MAX_SCAN_PAGES_TOTAL): nur erste Seite Vision
- grosser Scan (> MAX_SCAN_PAGES_TOTAL): nur erste relevante Seite pruefen
- getipptes PDF mit Textebene: Text UND Vision, dann mergen
(Text bringt getippten Background, Vision die handschriftlichen Felder)
"""
n_pages = pdf_page_count(pdf_path)
text = extract_text(pdf_path)
has_text = len(text.strip()) >= TEXT_SCAN_THRESHOLD
# "Notes" im Dateinamen -> fuehrende Notiz-Seite ueberspringen.
# Aber nur, wenn genug Seiten da sind (sonst normal ab Seite 1).
is_notes = "notes" in os.path.basename(pdf_path).lower()
vpage = 2 if (is_notes and n_pages >= 3) else 1
truncated_note = None
try:
if not has_text and n_pages > MAX_SCAN_PAGES_TOTAL:
# grosser reiner Scan: nur erste Seite pruefen
truncated_note = f"grosser Scan ({n_pages} Seiten), nur erste Seite geprueft"
data = _extract_via_vision(client, model, pdf_path, max_pages=1)
# grosser reiner Scan: nur die erste relevante Seite pruefen
truncated_note = f"grosser Scan ({n_pages} Seiten), begrenzt geprueft"
data = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage)
mode = "vision"
elif not has_text:
# reiner Scan
data = _extract_via_vision(client, model, pdf_path, max_pages=2)
data = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage)
mode = "vision"
else:
# Textebene vorhanden -> HYBRID: beide Pfade, dann mergen
text_rec = _extract_via_text(client, model, text)
vision_rec = None
if n_pages <= MAX_SCAN_PAGES_TOTAL:
vision_rec = _extract_via_vision(client, model, pdf_path, max_pages=2)
vision_rec = _extract_via_vision(client, model, pdf_path, max_pages=2, first_page=vpage)
data = merge_records(text_rec, vision_rec)
mode = "hybrid" if vision_rec is not None else "text"
except Exception as e:
@@ -357,6 +367,8 @@ def process_pdf(client, model, pdf_path, vision_budget=None):
if data is None:
return {"_error": "JSON-Parse fehlgeschlagen", "n_pages": n_pages}, mode
data["n_pages"] = n_pages
if is_notes:
data["_notes_file"] = True
if truncated_note:
data["_note"] = truncated_note
return data, mode
@@ -411,6 +423,11 @@ def main():
print(f"[{i}/{len(pdfs)}] {fdate or '????-??-??'} {rel}")
try:
data, mode = process_pdf(client, args.model, pdf, args.vision_budget)
# Bei Fehler EINMAL erneut versuchen (faengt seltene
# nicht-deterministische Parse-Ausrutscher ab).
if "_error" in data:
print(f" (Fehler, ein Wiederholversuch...)")
data, mode = process_pdf(client, args.model, pdf, args.vision_budget)
except KeyboardInterrupt:
print("\nAbbruch durch Nutzer. Bisheriges ist gesichert.")
break

103
inspect_poc.py Normal file
View File

@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""
inspect_poc.py - Wertet buyers_raw.jsonl nach dem POC-Lauf aus.
Zeigt:
- Feld-Fuellquoten (wie oft ist welches Feld nicht null?)
- Text- vs Vision-Qualitaet im Vergleich
- die Multi-Sheet-Gruppierung: welche Person hat mehrere Sheets?
- Datums-Parsing-Quote (fuer die "letzte 5 Jahre"-Abfragen kritisch)
Aufruf: python inspect_poc.py ./poc_out/buyers_raw.jsonl
"""
import sys
import json
import re
from collections import defaultdict, Counter
def person_key(rec):
"""Gruppierungsschluessel: bevorzugt echten Namen, sonst name_company."""
name = (rec.get("prospective_buyer") or rec.get("name_company") or "").strip().lower()
# "Nachname, Vorname" und "Vorname Nachname" grob angleichen
name = re.sub(r"\s+", " ", name)
return name or "(unbekannt)"
def main():
path = sys.argv[1] if len(sys.argv) > 1 else "./poc_out/buyers_raw.jsonl"
records = []
with open(path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
records.append(json.loads(line))
ok = [r for r in records if "_error" not in r]
err = [r for r in records if "_error" in r]
print(f"Datensaetze: {len(records)} (ok: {len(ok)}, Fehler: {len(err)})\n")
# Feld-Fuellquoten
fields = ["name_company", "prospective_buyer", "company", "phone", "cell",
"email", "address", "state", "how_did_you_hear",
"interested_in_updates", "types_of_business", "background_experience",
"date_of_introduction"]
print("=== Feld-Fuellquoten (nicht-null / erfolgreiche) ===")
for fld in fields:
filled = 0
for r in ok:
v = r.get(fld)
if v not in (None, "", [], {}):
filled += 1
pct = 100 * filled / len(ok) if ok else 0
print(f" {fld:<24} {filled:>4}/{len(ok)} ({pct:4.0f}%)")
# Text vs Vision
print("\n=== Text- vs Vision-Pfad ===")
for mode in ("text", "vision"):
sub = [r for r in ok if r.get("extraction_mode") == mode]
if not sub:
continue
# als grobe Qualitaetsmetrik: durchschnittliche Zahl gefuellter Felder
avg_filled = sum(
sum(1 for f in fields if r.get(f) not in (None, "", [], {})) for r in sub
) / len(sub)
print(f" {mode:<7} {len(sub):>4} Docs, im Schnitt {avg_filled:.1f}/{len(fields)} Felder gefuellt")
# Datums-Parsing
print("\n=== Date of Introduction ===")
iso = sum(1 for r in ok if isinstance(r.get("date_of_introduction"), str)
and re.match(r"\d{4}-\d{2}-\d{2}", r["date_of_introduction"]))
nonnull = sum(1 for r in ok if r.get("date_of_introduction"))
print(f" vorhanden: {nonnull}/{len(ok)} davon sauber ISO (YYYY-MM-DD): {iso}")
# Multi-Sheet-Gruppierung
print("\n=== Personen mit mehreren Sheets ===")
groups = defaultdict(list)
for r in ok:
groups[person_key(r)].append(r)
multi = {k: v for k, v in groups.items() if len(v) > 1}
print(f" distinkte Personen: {len(groups)}, davon mit >1 Sheet: {len(multi)}")
for name, recs in sorted(multi.items(), key=lambda x: -len(x[1]))[:10]:
cats = set()
dates = []
for r in recs:
cats.update(r.get("types_of_business") or [])
if r.get("date_of_introduction"):
dates.append(r["date_of_introduction"])
print(f" {name!r}: {len(recs)} Sheets | Kategorien: {sorted(cats)} | Daten: {sorted(dates)}")
# Haeufigste Kategorien
print("\n=== Top 20 Kategorien (roh) ===")
cats = Counter()
for r in ok:
for c in (r.get("types_of_business") or []):
cats[c.strip().lower()] += 1
for c, n in cats.most_common(20):
print(f" {n:>4} {c}")
if __name__ == "__main__":
main()