aknuth/ai-bayarea
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes

Browse Source
This commit is contained in:
Andreas Knuth
2026-07-10 10:34:50 -05:00
parent 158d2a16e5
commit 5650b7b3fa
5 changed files with 498 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
extract_buyers_llamacpp.py
View File

@@ -67,7 +67,12 @@ JSON_SCHEMA = {
"background_experience": {"type": ["string", "null"]},
"date_of_introduction": {"type": ["string", "null"]},
},
"required": ["is_buyer_sheet", "types_of_business"],
"required": [
"is_buyer_sheet", "name_company", "prospective_buyer", "company",
"phone", "cell", "email", "address", "state", "how_did_you_hear",
"interested_in_updates", "types_of_business_raw", "types_of_business",
"background_experience", "date_of_introduction"
],
}
SYSTEM_PROMPT = """Du bist ein praezises Datenextraktions-System fuer Formulare der Firma "BizMatch Business Brokerage".
@@ -169,12 +174,12 @@ def extract_text(pdf_path, max_pages=4):
return ""
def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=2200, quality=85, dpi=200):
def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=1600, quality=80, dpi=150):
"""
Erste max_pages Seiten -> Base64-JPEG.
Hoehere Aufloesung als zuvor (dpi=200, max_dim=2200), damit die kleinen
YES/NO-Kaestchen erkennbar bleiben. max_pages=2 reicht: Buyer-Felder
stehen auf den ersten Seiten, und es haelt die Vision-Token im Rahmen.
Moderate Aufloesung (dpi=150, max_dim=1600): reicht fuer Checkbox/Handschrift
(im Test bei dpi=150 erkannt), haelt aber die Vision-Payload klein genug,
dass der Prompt-Cache nicht ueberlaeuft (die "making room ... MiB"-Warnungen).
"""
try:
images = convert_from_path(pdf_path, first_page=1, last_page=max_pages, dpi=dpi)
@@ -199,7 +204,10 @@ def pdf_to_base64_images(pdf_path, max_pages=2, max_dim=2200, quality=85, dpi=20
def call_model(client, model, content_payload, vision_budget=None):
# llama.cpp erzwingt JSON ueber response_format mit json_schema.
# (vLLM-spezifisches guided_json / mm_processor_kwargs gibt es hier nicht.)
# WICHTIG gegen Gemma-4 "<unused49>-Flood"/Runaway:
# - enable_thinking:false per Request (zuverlaessiger als nur Server-Flag)
# - kleines max_tokens (ein Datensatz braucht ~300-400 Tokens)
# - repeat_penalty gegen Wiederholschleifen
resp = client.chat.completions.create(
model=model,
messages=[
@@ -207,13 +215,22 @@ def call_model(client, model, content_payload, vision_budget=None):
{"role": "user", "content": content_payload},
],
temperature=0.0,
max_tokens=1200,
max_tokens=512,
response_format={
"type": "json_schema",
"json_schema": {"name": "buyer_sheet", "schema": JSON_SCHEMA},
},
extra_body={
"chat_template_kwargs": {"enable_thinking": False},
"repeat_penalty": 1.05,
},
)
return resp.choices[0].message.content
choice = resp.choices[0]
content = choice.message.content
# finish_reason "length" = Runaway (max_tokens erreicht) -> als Warnung markieren
if getattr(choice, "finish_reason", None) == "length":
return content, "length"
return content, "stop"
def parse_json_loose(txt):
@@ -234,45 +251,111 @@ def parse_json_loose(txt):
TEXT_SCAN_THRESHOLD = 120 # < so viele Textzeichen -> als Scan behandeln
MAX_SCAN_PAGES_TOTAL = 12 # PDFs mit mehr Seiten gelten als "kein reines Sheet"
# Felder, bei denen wir Vision bevorzugen (handschriftlich/visuell auf dem Formular)
_PREFER_VISION = {
"prospective_buyer", "name_company", "company", "phone", "cell", "email",
"address", "state", "how_did_you_hear", "interested_in_updates",
"date_of_introduction",
}
# Felder, bei denen Text meist die sauberere (getippte) Quelle ist
_PREFER_TEXT = {"background_experience", "types_of_business", "types_of_business_raw"}
def process_pdf(client, model, pdf_path, vision_budget):
def _empty(v):
return v in (None, "", [], {})
def merge_records(text_rec, vision_rec):
"""
Fuehrt Text- und Vision-Extraktion desselben PDF zusammen.
Grundregel: nicht-leerer Wert schlaegt leeren; bei Konflikt entscheidet
die bevorzugte Quelle je Feld (Vision fuer Handschrift, Text fuer Getipptes).
"""
if text_rec is None:
return vision_rec
if vision_rec is None:
return text_rec
merged = {}
keys = set(text_rec) | set(vision_rec)
for k in keys:
tv, vv = text_rec.get(k), vision_rec.get(k)
if _empty(tv) and _empty(vv):
merged[k] = tv if k in text_rec else vv
elif _empty(tv):
merged[k] = vv
elif _empty(vv):
merged[k] = tv
else:
# beide gefuellt -> bevorzugte Quelle
if k in _PREFER_VISION:
merged[k] = vv
elif k in _PREFER_TEXT:
merged[k] = tv
else:
merged[k] = vv # Default: Vision
return merged
def _extract_via_text(client, model, text):
payload = USER_TEXT_INTRO + text[:12000]
raw, finish = call_model(client, model, payload)
data = parse_json_loose(raw)
if data is not None and finish == "length":
data["_runaway"] = True # Antwort war abgeschnitten -> unvollstaendig moeglich
return data
def _extract_via_vision(client, model, pdf_path, max_pages):
imgs = pdf_to_base64_images(pdf_path, max_pages=max_pages)
if not imgs:
return None
payload = [{"type": "text", "text": USER_IMG_INTRO}]
for b64 in imgs:
payload.append({"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64}"}})
raw, finish = call_model(client, model, payload)
data = parse_json_loose(raw)
if data is not None and finish == "length":
data["_runaway"] = True
return data
def process_pdf(client, model, pdf_path, vision_budget=None):
"""
Hybrid-Strategie:
- reiner Scan (keine Textebene): nur Vision
- grosser Scan (> MAX_SCAN_PAGES_TOTAL): nur erste Seite Vision
- getipptes PDF mit Textebene: Text UND Vision, dann mergen
(Text bringt getippten Background, Vision die handschriftlichen Felder)
"""
n_pages = pdf_page_count(pdf_path)
text = extract_text(pdf_path)
is_scan = len(text.strip()) < TEXT_SCAN_THRESHOLD
has_text = len(text.strip()) >= TEXT_SCAN_THRESHOLD
# Riesige, voll gescannte Stapel: nicht komplett verarbeiten.
# Wir schauen nur auf die erste Seite, ob ueberhaupt ein Sheet vorliegt.
truncated_note = None
if is_scan and n_pages > MAX_SCAN_PAGES_TOTAL:
truncated_note = f"grosser Scan ({n_pages} Seiten), nur erste Seite geprueft"
imgs = pdf_to_base64_images(pdf_path, max_pages=1)
elif is_scan:
imgs = pdf_to_base64_images(pdf_path, max_pages=2)
else:
imgs = None
if is_scan:
if not imgs:
return {"_error": "Scan, aber Bildkonvertierung fehlgeschlagen",
"n_pages": n_pages}, "vision"
payload = [{"type": "text", "text": USER_IMG_INTRO}]
for b64 in imgs:
payload.append({"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64}"}})
mode = "vision"
else:
payload = USER_TEXT_INTRO + text[:12000]
mode = "text"
try:
raw = call_model(client, model, payload, vision_budget if mode == "vision" else None)
if not has_text and n_pages > MAX_SCAN_PAGES_TOTAL:
# grosser reiner Scan: nur erste Seite pruefen
truncated_note = f"grosser Scan ({n_pages} Seiten), nur erste Seite geprueft"
data = _extract_via_vision(client, model, pdf_path, max_pages=1)
mode = "vision"
elif not has_text:
# reiner Scan
data = _extract_via_vision(client, model, pdf_path, max_pages=2)
mode = "vision"
else:
# Textebene vorhanden -> HYBRID: beide Pfade, dann mergen
text_rec = _extract_via_text(client, model, text)
vision_rec = None
if n_pages <= MAX_SCAN_PAGES_TOTAL:
vision_rec = _extract_via_vision(client, model, pdf_path, max_pages=2)
data = merge_records(text_rec, vision_rec)
mode = "hybrid" if vision_rec is not None else "text"
except Exception as e:
return {"_error": f"API: {type(e).__name__}: {e}", "n_pages": n_pages}, mode
return {"_error": f"API: {type(e).__name__}: {e}", "n_pages": n_pages}, "?"
data = parse_json_loose(raw)
if data is None:
return {"_error": "JSON-Parse fehlgeschlagen", "_raw": (raw or "")[:400],
"n_pages": n_pages}, mode
return {"_error": "JSON-Parse fehlgeschlagen", "n_pages": n_pages}, mode
data["n_pages"] = n_pages
if truncated_note:
data["_note"] = truncated_note
@@ -292,7 +375,7 @@ def main():
ap.add_argument("--limit", type=int, default=150)
ap.add_argument("--recursive", action="store_true",
help="auch Unterordner durchsuchen (Default: nur oberste Ebene)")
ap.add_argument("--timeout", type=float, default=120,
ap.add_argument("--timeout", type=float, default=60,
help="HTTP-Timeout je Anfrage in Sekunden")
ap.add_argument("--vision-budget", type=int, default=0,
help="Gemma Vision-Token-Budget (0=aus/Default 280; sonst 70/140/280/560/1120). "
@@ -319,7 +402,7 @@ def main():
f"{'rekursiv' if args.recursive else 'nur oberste Ebene'}).\n")
all_categories = {}
n_ok = n_err = n_text = n_vision = n_nosheet = 0
n_ok = n_err = n_text = n_vision = n_hybrid = n_nosheet = 0
with open(jsonl_path, "w", encoding="utf-8") as jf:
for i, pdf in enumerate(pdfs, 1):
@@ -336,6 +419,7 @@ def main():
n_text += (mode == "text")
n_vision += (mode == "vision")
n_hybrid += (mode == "hybrid")
record = {"source_file": rel,
"file_date": fdate.isoformat() if fdate else None,
"extraction_mode": mode, **data}
@@ -364,6 +448,7 @@ def main():
print(f" Fehler: {n_err}")
print(f" Text-Pfad: {n_text}")
print(f" Vision-Pfad: {n_vision}")
print(f" Hybrid-Pfad: {n_hybrid}")
print(f" kein Buyer-Sheet: {n_nosheet}")
print(f" distinkte Kategorien (roh): {len(all_categories)}")
print(f"\n JSONL: {jsonl_path}")
@@ -371,4 +456,4 @@ def main():
if __name__ == "__main__":
main()
main()