diff --git a/DMS/docker-data/dms/config/rspamd_report.sh b/DMS/docker-data/dms/config/rspamd_report.sh new file mode 100755 index 0000000..e802697 --- /dev/null +++ b/DMS/docker-data/dms/config/rspamd_report.sh @@ -0,0 +1,179 @@ +#!/bin/bash +# =========================================================================== +# rspamd_report.sh - lesbare Auswertung von /var/log/mail/rspamd.log +# +# Die Logzeilen von rspamd enthalten alles, sind aber unlesbar. Dieses Skript +# macht daraus zwei Ansichten: +# +# Uebersicht: eine Zeile je Mail mit Score, Aktion, Scanzeit, Von, An +# Detail: die Symbolaufschluesselung einer Mail, nach Gewicht sortiert, +# mit Summe - also warum genau diese Mail diesen Score hat +# +# AUSFUEHRUNG - immer IM Container: +# +# docker exec mailserver bash /tmp/docker-mailserver/rspamd_report.sh +# docker exec mailserver bash /tmp/docker-mailserver/rspamd_report.sh -n 40 +# docker exec mailserver bash /tmp/docker-mailserver/rspamd_report.sh --detail +# docker exec mailserver bash /tmp/docker-mailserver/rspamd_report.sh --detail C0D8A320066 +# +# Optionen: +# -n N Anzahl Mails in der Uebersicht (Default 15) +# --detail [ID] Symbolaufschluesselung. ID ist ein beliebiges Fragment aus +# Queue-ID oder Message-ID. Ohne ID die letzte Mail. +# --slow MS Nur Mails, deren Scan laenger als MS Millisekunden dauerte +# --log PFAD Anderes Logfile (Default /var/log/mail/rspamd.log) +# +# Hinweis: Das Log enthaelt nur Mails, die seit dem Umstellen auf +# level = "info" (rspamd/local.d/logging.inc) verarbeitet wurden. +# =========================================================================== + +set -uo pipefail + +LOGFILE="/var/log/mail/rspamd.log" +COUNT=15 +DETAIL=0 +DETAIL_ID="" +SLOW_MS=0 + +usage() { sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' ; } + +while [[ $# -gt 0 ]]; do + case "$1" in + -n) COUNT="${2:-15}"; shift 2 ;; + --detail) DETAIL=1 + if [[ ${2:-} != "" && ${2:-} != -* ]]; then DETAIL_ID="$2"; shift 2; else shift; fi ;; + --slow) SLOW_MS="${2:-0}"; shift 2 ;; + --log) LOGFILE="${2:-}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unbekannte Option: $1" >&2; exit 1 ;; + esac +done + +if [[ ! -r ${LOGFILE} ]]; then + echo "FEHLER: ${LOGFILE} nicht lesbar. Laeuft das Skript wirklich IM Container?" >&2 + exit 1 +fi + +TASKS=$(grep 'rspamd_task_write_log' "${LOGFILE}" 2>/dev/null) +if [[ -z ${TASKS} ]]; then + echo "Keine ausgewerteten Mails im Log gefunden." + echo "Steht in rspamd/local.d/logging.inc wirklich level = \"info\"?" + exit 0 +fi + +# --------------------------------------------------------------------------- +# Detailansicht einer einzelnen Mail +# --------------------------------------------------------------------------- +if (( DETAIL )); then + if [[ -n ${DETAIL_ID} ]]; then + LINE=$(grep -F "${DETAIL_ID}" <<<"${TASKS}" | tail -1) + [[ -z ${LINE} ]] && { echo "Keine Mail mit '${DETAIL_ID}' im Log gefunden." >&2; exit 1; } + else + LINE=$(tail -1 <<<"${TASKS}") + fi + + awk ' + { + # Kopfdaten + ts = substr($0, 1, 19) + match($0, /qid: <[^>]*>/); qid = extract() + match($0, /from: <[^>]*>/); from = extract() + match($0, /rcpts: <[^>]*>/); rcpt = extract() + match($0, /settings_id: [^,]*/); sid = substr($0, RSTART+13, RLENGTH-13) + match($0, /time: [0-9.]+ms/); tm = substr($0, RSTART+6, RLENGTH-8) + match($0, /dns req: [0-9]+/); dns = substr($0, RSTART+9, RLENGTH-9) + + # Aktion und Score: (default: F (no action): [4.74/500.00] + if (match($0, /\([a-z ]+\): \[-?[0-9.]+\//)) { + chunk = substr($0, RSTART, RLENGTH) + action = substr(chunk, 2, index(chunk, ")") - 2) + score = substr(chunk, index(chunk, "[") + 1, length(chunk) - index(chunk, "[") - 1) + } + + printf "Zeit: %s\n", ts + printf "Queue-ID: %s\n", qid + printf "Von: %s\n", from + printf "An: %s\n", rcpt + printf "Regel: %s\n", sid + printf "Aktion: %s Score: %s Scanzeit: %s ms DNS: %s\n\n", action, score, tm, dns + + # Symbolblock zwischen "] [" und "]), len:" + i = index($0, "] [") + if (i == 0) { print " (keine Symbole im Log)"; exit } + rest = substr($0, i + 3) + j = index(rest, "]), len:") + if (j == 0) j = length(rest) + syms = substr(rest, 1, j - 1) + + # auf Kommas der obersten Ebene splitten (Klammern koennen Kommas enthalten) + depth = 0; cur = ""; n = 0 + for (k = 1; k <= length(syms); k++) { + c = substr(syms, k, 1) + if (c == "{") depth++ + else if (c == "}") depth-- + if (c == "," && depth == 0) { if (cur != "") entries[++n] = cur; cur = ""; continue } + cur = cur c + } + if (cur != "") entries[++n] = cur + + printf " %8s %-38s %s\n", "GEWICHT", "SYMBOL", "DETAIL" + printf " %8s %-38s %s\n", "-------", "--------------------------------------", "------" + + total = 0 + # zwei Durchlaeufe: erst die mit Gewicht, dann die neutralen + for (pass = 1; pass <= 2; pass++) { + for (e = 1; e <= n; e++) { + entry = entries[e] + p1 = index(entry, "(") + if (p1 == 0) continue + name = substr(entry, 1, p1 - 1) + p2 = index(entry, ")") + w = substr(entry, p1 + 1, p2 - p1 - 1) + payload = "" + b1 = index(entry, "{") + if (b1 > 0) payload = substr(entry, b1 + 1, length(entry) - b1 - 1) + gsub(/;$/, "", payload) + + if (pass == 1 && w + 0 == 0) continue + if (pass == 2 && w + 0 != 0) continue + if (pass == 1) total += w + + printf " %8.2f %-38s %s\n", w, name, substr(payload, 1, 60) + } + if (pass == 1) printf " %8s %s\n", "-------", "" + if (pass == 1) printf " %8.2f %s\n\n neutrale Symbole (Gewicht 0):\n", total, "SUMME" + } + } + function extract( s) { s = substr($0, RSTART, RLENGTH); return substr(s, index(s, "<") + 1, length(s) - index(s, "<") - 1) } + ' <<<"${LINE}" + exit 0 +fi + +# --------------------------------------------------------------------------- +# Uebersicht +# --------------------------------------------------------------------------- +printf '%-19s %7s %-12s %7s %-30s %-30s\n' "ZEIT" "SCORE" "AKTION" "MS" "VON" "AN" +printf -- '--------------------------------------------------------------------------------------------------------------------\n' + +awk -v slow="${SLOW_MS}" ' +{ + ts = substr($0, 1, 19) + match($0, /from: <[^>]*>/); from = extract() + match($0, /rcpts: <[^>]*>/); rcpt = extract() + match($0, /time: [0-9.]+ms/); tm = substr($0, RSTART+6, RLENGTH-8) + 0 + + if (match($0, /\([a-z ]+\): \[-?[0-9.]+\//)) { + chunk = substr($0, RSTART, RLENGTH) + action = substr(chunk, 2, index(chunk, ")") - 2) + score = substr(chunk, index(chunk, "[") + 1, length(chunk) - index(chunk, "[") - 1) + 0 + } else { action = "?"; score = 0 } + + if (tm < slow) next + printf "%-19s %7.2f %-12s %7.0f %-30.30s %-30.30s\n", ts, score, action, tm, from, rcpt +} +function extract( s) { s = substr($0, RSTART, RLENGTH); return substr(s, index(s, "<") + 1, length(s) - index(s, "<") - 1) } +' <<<"${TASKS}" | tail -n "${COUNT}" + +echo +echo "Detail zu einer Mail: $0 --detail " +echo "Nur langsame Scans: $0 --slow 1000" diff --git a/DMS/docker-data/dms/config/train_bayes.sh b/DMS/docker-data/dms/config/train_bayes.sh index 6642a4d..cd9ff0c 100755 --- a/DMS/docker-data/dms/config/train_bayes.sh +++ b/DMS/docker-data/dms/config/train_bayes.sh @@ -22,7 +22,11 @@ # --apply Wirklich lernen (ohne das passiert nichts) # --max N Max. Nachrichten je Postfach und Klasse (Default 400) # --include-sent Gesendete Mails zusaetzlich als Ham lernen -# --only MUSTER Nur Postfaecher, deren Adresse MUSTER enthaelt +# --only MUSTER Nur Postfaecher, deren Adresse eines der Muster +# enthaelt. Mehrfach angebbar oder kommasepariert: +# --only support@,andreas.knuth@ +# WICHTIG: Ohne diese Option laeuft das Skript ueber +# ALLE Postfaecher aller Domains unter /var/mail. # --host HOST:PORT rspamd-Controller (Default 127.0.0.1:11334) # # Passwort: Falls der Controller eines verlangt (bei euch in @@ -48,7 +52,7 @@ MAX_BYTES=5242880 # 5 MB - groessere Mails bringen fuers Lernen nichts BATCH=50 # Dateien pro rspamc-Aufruf APPLY=0 INCLUDE_SENT=0 -ONLY="" +ONLY_PATTERNS=() usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//' ; } @@ -57,7 +61,8 @@ while [[ $# -gt 0 ]]; do --apply) APPLY=1; shift ;; --max) MAX_PER_CLASS="${2:-}"; shift 2 ;; --include-sent) INCLUDE_SENT=1; shift ;; - --only) ONLY="${2:-}"; shift 2 ;; + --only) IFS=',' read -ra _pat <<<"${2:-}" + ONLY_PATTERNS+=("${_pat[@]}"); shift 2 ;; --host) RSPAMC_HOST="${2:-}"; shift 2 ;; -h|--help) usage; exit 0 ;; *) echo "Unbekannte Option: $1" >&2; echo "Hilfe: $0 --help" >&2; exit 1 ;; @@ -126,7 +131,11 @@ else echo "=== Bayes-Training (Trockenlauf - es wird nichts gelernt) ===" fi echo "Basis: ${MAILBASE} | max ${MAX_PER_CLASS} je Postfach und Klasse" -[[ -n ${ONLY} ]] && echo "Filter: nur Postfaecher mit '${ONLY}'" +if (( ${#ONLY_PATTERNS[@]} > 0 )); then + echo "Filter: nur Postfaecher mit '${ONLY_PATTERNS[*]}'" +else + echo "Filter: KEINER - es werden ALLE Postfaecher unter ${MAILBASE} erfasst" +fi printf '\n%-42s %8s %8s\n' "POSTFACH" "SPAM" "HAM" printf -- '---------------------------------------------------------------\n' @@ -145,7 +154,14 @@ for domain_dir in "${MAILBASE}"/*/; do [[ -d ${user_dir}/cur ]] || continue # keine Maildir -> ueberspringen mailbox="$(basename "${user_dir}")@${domain}" - [[ -n ${ONLY} && ${mailbox} != *"${ONLY}"* ]] && continue + + if (( ${#ONLY_PATTERNS[@]} > 0 )); then + matched=0 + for pat in "${ONLY_PATTERNS[@]}"; do + [[ ${mailbox} == *"${pat}"* ]] && { matched=1; break; } + done + (( matched )) || continue + fi mapfile -t spam_files < <(gather "${MAX_PER_CLASS}" \ "${user_dir}/.Junk/cur" "${user_dir}/.Junk/new" \