142 lines
3.6 KiB
YAML
142 lines
3.6 KiB
YAML
# llama-server für Qwen3.6 Vision — zwei Profile:
|
|
# docker compose --profile cuda up -d --build (EC2, RTX PRO 6000 Blackwell)
|
|
# docker compose --profile vulkan up -d --build (inhouse, Radeon AI Pro R9700)
|
|
#
|
|
# Modelle nach ./models legen (Haupt-GGUF + zugehöriger mmproj aus demselben Repo!):
|
|
# CUDA: unsloth/Qwen3.6-27B-GGUF → Qwen3.6-27B-UD-Q4_K_XL.gguf + mmproj-BF16.gguf
|
|
# Vulkan: unsloth/Qwen3.6-35B-A3B-GGUF → Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf + mmproj-BF16.gguf
|
|
#
|
|
# Bewusst KEIN MTP: --mmproj + MTP ist laut Unsloth nicht unterstützt und hat
|
|
# offene OOM-/Hänger-Bugs (llama.cpp #23371, #23430). Für Batch-Extraktion
|
|
# irrelevant, da die Zeit im Image-Encoding steckt, nicht in der Generierung.
|
|
|
|
services:
|
|
llama-cuda:
|
|
profiles: ["cuda"]
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.cuda
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./models:/models
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
command:
|
|
- -m
|
|
- /models/Qwen3.6-27B-UD-Q4_K_XL.gguf
|
|
- --mmproj
|
|
- /models/mmproj-BF16.gguf
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8000"
|
|
- --alias
|
|
- qwen3.6
|
|
- -ngl
|
|
- "99"
|
|
- -fa
|
|
- "on"
|
|
- -c
|
|
- "32768"
|
|
- --parallel
|
|
- "1"
|
|
- --jinja
|
|
# Erlernte Stabilitäts-Settings (Vision + Checkpoints = OOM-Bug):
|
|
- --ctx-checkpoints
|
|
- "0"
|
|
# Qwen-VL braucht min. 1024 Image-Tokens für korrektes Grounding:
|
|
- --image-min-tokens
|
|
- "1024"
|
|
- --image-max-tokens
|
|
- "4096"
|
|
# Extraktion: niedrige Temperatur, kein Repeat-Penalty
|
|
- --temp
|
|
- "0.1"
|
|
- --top-p
|
|
- "0.95"
|
|
- --top-k
|
|
- "20"
|
|
- --repeat-penalty
|
|
- "1.0"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-sf", "http://localhost:8000/health"]
|
|
interval: 15s
|
|
timeout: 5s
|
|
retries: 40
|
|
start_period: 60s
|
|
restart: unless-stopped
|
|
|
|
llama-vulkan:
|
|
profiles: ["vulkan"]
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.vulkan
|
|
ports:
|
|
- "8000:8000"
|
|
volumes:
|
|
- ./models:/models
|
|
devices:
|
|
- /dev/dri:/dev/dri
|
|
- /dev/kfd:/dev/kfd
|
|
# Numerische HOST-GIDs verwenden — Namen wie "render" existieren im
|
|
# Container-Image nicht. GIDs prüfen mit: getent group video render
|
|
group_add:
|
|
- "44" # video (Host-GID ggf. anpassen)
|
|
- "991" # render (Host-GID ggf. anpassen)
|
|
security_opt:
|
|
- seccomp:unconfined
|
|
command:
|
|
- -m
|
|
- /models/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf
|
|
- --mmproj
|
|
- /models/mmproj-BF16.gguf
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8000"
|
|
- --alias
|
|
- qwen3.6
|
|
- -ngl
|
|
- "99"
|
|
- -fa
|
|
- "on"
|
|
- -c
|
|
- "32768"
|
|
- --parallel
|
|
- "1"
|
|
# KV-Cache quantisieren — 32GB VRAM, 35B-A3B + mmproj + Bildkontext:
|
|
- -ctk
|
|
- q8_0
|
|
- -ctv
|
|
- q8_0
|
|
- --jinja
|
|
- --ctx-checkpoints
|
|
- "0"
|
|
- --image-min-tokens
|
|
- "1024"
|
|
- --image-max-tokens
|
|
- "4096"
|
|
- --temp
|
|
- "0.1"
|
|
- --top-p
|
|
- "0.95"
|
|
- --top-k
|
|
- "20"
|
|
- --repeat-penalty
|
|
- "1.0"
|
|
# NOTFALL-Fallback bei Vision-Hängern/OOM unter RADV — mmproj auf CPU,
|
|
# Bildverarbeitung wird DEUTLICH langsamer. Nur aktivieren wenn nötig:
|
|
# - --no-mmproj-offload
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-sf", "http://localhost:8000/health"]
|
|
interval: 15s
|
|
timeout: 5s
|
|
retries: 40
|
|
start_period: 120s
|
|
restart: unless-stopped |