Compare commits

26 Commits

Author SHA1 Message Date
4422fba707 moving 2026-05-21 14:37:33 -05:00
649a55eecd s3-retention 2026-05-21 10:50:35 -05:00
1936646b20 Bucket Tagging 2026-05-21 10:28:47 -05:00
528497094b SKIP_PROTOCOL_DNS 2026-05-20 17:52:35 -05:00
fedd2d4f34 7 days 2026-05-20 17:32:38 -05:00
119f8b1152 NODE_HOSTNAME muss gesetzt sein 2026-04-29 18:56:31 -05:00
5b0a087a78 Lambda Skip 2026-04-27 21:17:41 -05:00
29a34ce4e5 MX falls nicht vorhanden 2026-04-27 21:08:16 -05:00
0540b1083c mailadmin_network 2026-04-26 14:42:14 -05:00
0ecaa2ce11 Mailadmin Block 2026-04-26 14:35:09 -05:00
25c26e4f79 executable 2026-04-20 17:37:16 -05:00
b2dfb76a7e check 2026-04-20 17:35:36 -05:00
6312d0f563 fix autoconfig 2026-04-19 19:34:38 -05:00
d959b5ec86 use $DOMAIN_NAME instead of $DOMAIN 2026-04-19 18:42:00 -05:00
4b6db1e23d dynamic hostname 2026-04-19 16:29:44 -05:00
e6a81e6d8d fix 2026-04-19 14:45:10 -05:00
faa8d9cc57 mail-betwork 2026-04-19 14:37:51 -05:00
a93f44600c bind port 25 to localhost 2026-04-18 13:16:50 -05:00
ae75afc69a Merge branch 'contabo' of git.bizmatch.net:aknuth/email-amazon into contabo 2026-04-17 14:59:50 -05:00
95baab8e06 update dynamic LoginName 2026-04-17 14:59:43 -05:00
947740232c logo 2026-04-15 23:12:23 -05:00
081a0fad4b fix 2026-04-15 14:28:28 -05:00
1e1265ef1b batch imapsync 2026-04-15 13:49:31 -05:00
9862689c0c no markAsBlocked, 2026-04-12 20:43:37 -05:00
bed6c2a398 fix 2026-04-03 17:03:14 -05:00
27c2be664a standby mode, sns or sqs 2026-04-03 16:54:51 -05:00
27 changed files with 781 additions and 194 deletions

58
DMS/batch_imapsync.sh Normal file
View File

@@ -0,0 +1,58 @@
#!/bin/bash
# batch_imapsync.sh - Führt IMAP-Sync für alle User im Hintergrund aus
# Format der CSV: email@domain.com,SecretPassword123
HOST1=$1
HOST2=$2
CSV_FILE=$3
if [ -z "$HOST1" ] || [ -z "$HOST2" ] || [ -z "$CSV_FILE" ]; then
echo "Usage: $0 <source-host> <target-host> <users.csv>"
echo "Beispiel: $0 secure.emailsrvr.com 147.93.132.244 stxmaterials.csv"
exit 1
fi
# ======================================================================
# Die eigentliche Sync-Funktion (wird in den Hintergrund geschickt)
# ======================================================================
run_sync_jobs() {
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
LOG_DIR="sync_logs_$TIMESTAMP"
mkdir -p "$LOG_DIR"
echo "Beginne Sync-Lauf am $(date)" > "batch_master_${TIMESTAMP}.log"
while IFS=, read -r email password; do
email=$(echo "$email" | tr -d '\r' | xargs)
password=$(echo "$password" | tr -d '\r' | xargs)
[ -z "$email" ] && continue
LOGFILE="$LOG_DIR/imapsync_${email}.log"
echo "[$(date)] Syncing $email -> $LOGFILE" >> "batch_master_${TIMESTAMP}.log"
# Führe Docker imapsync für den aktuellen User aus
docker run --rm gilleslamiral/imapsync imapsync \
--host1 "$HOST1" --user1 "$email" --password1 "$password" --ssl1 \
--host2 "$HOST2" --user2 "$email" --password2 "$password" --ssl2 \
--automap > "$LOGFILE" 2>&1 < /dev/null
done < "$CSV_FILE"
echo "Alle Sync-Jobs beendet am $(date)" >> "batch_master_${TIMESTAMP}.log"
}
# ======================================================================
# Skript-Start: Entkopplung vom Terminal
# ======================================================================
echo "🚀 Starte Batch-IMAP-Sync im Hintergrund..."
# Rufe die Funktion auf, leite alle restlichen Ausgaben ins Nichts und schicke sie in den Hintergrund (&)
run_sync_jobs </dev/null >/dev/null 2>&1 &
echo "✅ Der Job läuft jetzt autark im Hintergrund (sequenziell)."
echo "Du kannst das SSH-Terminal jetzt bedenkenlos schließen!"
echo "Überwache den Gesamtfortschritt mit:"
echo " tail -f batch_master_*.log"
echo "Oder die Details eines einzelnen Postfachs mit:"
echo " tail -f sync_logs_*/imapsync_<email>.log"

View File

@@ -9,10 +9,10 @@ services:
# Node-spezifischer Hostname - A-Record zeigt auf DIESEN Server. # Node-spezifischer Hostname - A-Record zeigt auf DIESEN Server.
# email-srvr.com selbst zeigt auf einen anderen Server und wird hier NICHT verwendet. # email-srvr.com selbst zeigt auf einen anderen Server und wird hier NICHT verwendet.
hostname: node1.email-srvr.com hostname: ${NODE_HOSTNAME}
ports: ports:
- "25:25" - "127.0.0.1:25:25"
- "587:587" - "587:587"
- "465:465" - "465:465"
- "143:143" - "143:143"
@@ -61,8 +61,8 @@ services:
# Kundendomain-SNI wird über postfix-main.cf + dovecot-sni.cf gesteuert. # Kundendomain-SNI wird über postfix-main.cf + dovecot-sni.cf gesteuert.
# ------------------------------------------------------- # -------------------------------------------------------
- SSL_TYPE=manual - SSL_TYPE=manual
- SSL_CERT_PATH=/etc/mail/certs/node1.email-srvr.com/node1.email-srvr.com.crt - SSL_CERT_PATH=/etc/mail/certs/${NODE_HOSTNAME}/${NODE_HOSTNAME}.crt
- SSL_KEY_PATH=/etc/mail/certs/node1.email-srvr.com/node1.email-srvr.com.key - SSL_KEY_PATH=/etc/mail/certs/${NODE_HOSTNAME}/${NODE_HOSTNAME}.key
# SPAM / Rspamd # SPAM / Rspamd
- ENABLE_OPENDKIM=1 - ENABLE_OPENDKIM=1
@@ -107,7 +107,7 @@ services:
# Postfix # Postfix
# POSTFIX_OVERRIDE_HOSTNAME: Was Postfix im EHLO/HELO Banner sendet. # POSTFIX_OVERRIDE_HOSTNAME: Was Postfix im EHLO/HELO Banner sendet.
# node1.email-srvr.com passt zum TLS-Cert und ist der echte Hostname. # node1.email-srvr.com passt zum TLS-Cert und ist der echte Hostname.
- POSTFIX_OVERRIDE_HOSTNAME=node1.email-srvr.com - POSTFIX_OVERRIDE_HOSTNAME=${NODE_HOSTNAME}
- POSTFIX_MYNETWORKS=172.16.0.0/12 172.17.0.0/12 172.18.0.0/12 [::1]/128 [fe80::]/64 - POSTFIX_MYNETWORKS=172.16.0.0/12 172.17.0.0/12 172.18.0.0/12 [::1]/128 [fe80::]/64
- POSTFIX_MAILBOX_SIZE_LIMIT=0 - POSTFIX_MAILBOX_SIZE_LIMIT=0
- POSTFIX_MESSAGE_SIZE_LIMIT=0 - POSTFIX_MESSAGE_SIZE_LIMIT=0
@@ -120,7 +120,7 @@ services:
mail_network: mail_network:
aliases: aliases:
- mailserver - mailserver
- node1.email-srvr.com - ${NODE_HOSTNAME}
roundcube: roundcube:
image: roundcube/roundcubemail:latest image: roundcube/roundcubemail:latest
@@ -136,10 +136,10 @@ services:
- ROUNDCUBEMAIL_DB_USER=roundcube - ROUNDCUBEMAIL_DB_USER=roundcube
- ROUNDCUBEMAIL_DB_PASSWORD=${ROUNDCUBE_DB_PASSWORD} - ROUNDCUBEMAIL_DB_PASSWORD=${ROUNDCUBE_DB_PASSWORD}
# Roundcube verbindet intern über den Docker-Alias # Roundcube verbindet intern über den Docker-Alias
- ROUNDCUBEMAIL_DEFAULT_HOST=ssl://node1.email-srvr.com - ROUNDCUBEMAIL_DEFAULT_HOST=ssl://${NODE_HOSTNAME}
- ROUNDCUBEMAIL_DEFAULT_PORT=993 - ROUNDCUBEMAIL_DEFAULT_PORT=993
# Interner Traffic ohne TLS # Interner Traffic ohne TLS
- ROUNDCUBEMAIL_SMTP_SERVER=ssl://node1.email-srvr.com - ROUNDCUBEMAIL_SMTP_SERVER=ssl://${NODE_HOSTNAME}
- ROUNDCUBEMAIL_SMTP_PORT=465 - ROUNDCUBEMAIL_SMTP_PORT=465
# WICHTIG: Variablen LEER lassen, damit Roundcube keine Authentifizierung versucht! # WICHTIG: Variablen LEER lassen, damit Roundcube keine Authentifizierung versucht!

View File

@@ -27,7 +27,12 @@ CERTS_BASE_PATH=${CERTS_BASE_PATH:-"/etc/mail/certs"}
# Node-Hostname: Fallback-Cert für DMS (kein Wildcard, direktes Cert) # Node-Hostname: Fallback-Cert für DMS (kein Wildcard, direktes Cert)
# Muss mit dem 'hostname' in docker-compose.yml übereinstimmen. # Muss mit dem 'hostname' in docker-compose.yml übereinstimmen.
NODE_HOSTNAME=${NODE_HOSTNAME:-"node1.email-srvr.com"} if [ -z "$NODE_HOSTNAME" ]; then
echo "❌ NODE_HOSTNAME ist nicht gesetzt!"
echo "Beispiel:"
echo " DMS_CONTAINER=mailserver NODE_HOSTNAME=node2.email-srvr.com ./setup-dms-tls.sh"
exit 1
fi
echo "============================================================" echo "============================================================"
echo " 🔐 DMS TLS SNI Setup (Multi-Domain)" echo " 🔐 DMS TLS SNI Setup (Multi-Domain)"

View File

@@ -34,7 +34,7 @@ aws s3api put-public-access-block \
--public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true"
# Lebenszyklus-Konfiguration hinzufügen # Lebenszyklus-Konfiguration hinzufügen
echo "Lebenszyklus-Konfiguration hinzufügen (E-Mails werden nach 90 Tagen gelöscht)..." echo "Lebenszyklus-Konfiguration hinzufügen (E-Mails werden nach 7 Tagen gelöscht)..."
aws s3api put-bucket-lifecycle-configuration \ aws s3api put-bucket-lifecycle-configuration \
--bucket ${S3_BUCKET_NAME} \ --bucket ${S3_BUCKET_NAME} \
--lifecycle-configuration '{ --lifecycle-configuration '{
@@ -43,7 +43,7 @@ aws s3api put-bucket-lifecycle-configuration \
"ID": "DeleteOldEmails", "ID": "DeleteOldEmails",
"Status": "Enabled", "Status": "Enabled",
"Expiration": { "Expiration": {
"Days": 14 "Days": 7
}, },
"Filter": { "Filter": {
"Prefix": "" "Prefix": ""
@@ -76,6 +76,14 @@ aws s3api put-bucket-policy \
] ]
}' }'
# ------------------------
# Cost Allocation Tags setzen
# ------------------------
echo "Setze Cost Allocation Tag (BucketName)..."
aws s3api put-bucket-tagging \
--bucket ${S3_BUCKET_NAME} \
--tagging "TagSet=[{Key=BucketName,Value=${S3_BUCKET_NAME}}]"
echo "S3 Bucket $S3_BUCKET_NAME wurde erfolgreich erstellt und konfiguriert." echo "S3 Bucket $S3_BUCKET_NAME wurde erfolgreich erstellt und konfiguriert."
echo "Bucket-ARN: arn:aws:s3:::$S3_BUCKET_NAME" echo "Bucket-ARN: arn:aws:s3:::$S3_BUCKET_NAME"

View File

@@ -7,9 +7,14 @@
# Standard: mail.${DOMAIN_NAME} # Standard: mail.${DOMAIN_NAME}
# Override: export MAIL_FROM_SUBDOMAIN="mailfrom" (nur der Prefix, ohne Domain) # Override: export MAIL_FROM_SUBDOMAIN="mailfrom" (nur der Prefix, ohne Domain)
# #
# Lambda Deployment:
# Standard: Überspringt den Deploy (SKIP_LAMBDA_DEPLOY=true)
# Override: export SKIP_LAMBDA_DEPLOY="false" (um Lambda neu auszurollen)
#
# Beispiel: # Beispiel:
# export DOMAIN_NAME="buddelectric.net" # export DOMAIN_NAME="buddelectric.net"
# export MAIL_FROM_SUBDOMAIN="mailfrom" # → mailfrom.buddelectric.net # export MAIL_FROM_SUBDOMAIN="mailfrom" # → mailfrom.buddelectric.net
# export SKIP_LAMBDA_DEPLOY="false" # → Lambda wird aktualisiert
# ./awsses.sh # ./awsses.sh
set -e set -e
@@ -18,17 +23,20 @@ set -e
if ! command -v jq &> /dev/null; then echo "Fehler: 'jq' fehlt."; exit 1; fi if ! command -v jq &> /dev/null; then echo "Fehler: 'jq' fehlt."; exit 1; fi
if [ -z "$DOMAIN_NAME" ]; then echo "Fehler: DOMAIN_NAME ist nicht gesetzt."; exit 1; fi if [ -z "$DOMAIN_NAME" ]; then echo "Fehler: DOMAIN_NAME ist nicht gesetzt."; exit 1; fi
# Prüfen ob Python Code da ist
PYTHON_FILE="ses_sns_shim_global.py"
if [ ! -f "$PYTHON_FILE" ]; then
echo "Fehler: $PYTHON_FILE nicht gefunden!"
exit 1
fi
# --- VARIABLEN --- # --- VARIABLEN ---
AWS_REGION=${AWS_REGION:-"us-east-2"} AWS_REGION=${AWS_REGION:-"us-east-2"}
EMAIL_PREFIX=${EMAIL_PREFIX:-""} EMAIL_PREFIX=${EMAIL_PREFIX:-""}
CONFIGURATION_SET_NAME="relay-outbound" CONFIGURATION_SET_NAME="relay-outbound"
SKIP_LAMBDA_DEPLOY=${SKIP_LAMBDA_DEPLOY:-"true"}
# Prüfen ob Python Code da ist (nur wenn auch deployt werden soll)
PYTHON_FILE="ses_sns_shim_global.py"
if [ "$SKIP_LAMBDA_DEPLOY" != "true" ]; then
if [ ! -f "$PYTHON_FILE" ]; then
echo "Fehler: $PYTHON_FILE nicht gefunden!"
exit 1
fi
fi
# MAIL FROM Subdomain (konfigurierbar) # MAIL FROM Subdomain (konfigurierbar)
MAIL_FROM_SUBDOMAIN=${MAIL_FROM_SUBDOMAIN:-"mail"} MAIL_FROM_SUBDOMAIN=${MAIL_FROM_SUBDOMAIN:-"mail"}
@@ -49,6 +57,7 @@ LAMBDA_ROLE_NAME="SesShimGlobalRole"
echo "==========================================================" echo "=========================================================="
echo " SES Setup (S3 -> Global Lambda Shim -> SQS) für $DOMAIN_NAME" echo " SES Setup (S3 -> Global Lambda Shim -> SQS) für $DOMAIN_NAME"
echo " MAIL FROM: $MAIL_FROM_DOMAIN" echo " MAIL FROM: $MAIL_FROM_DOMAIN"
echo " SKIP_LAMBDA: $SKIP_LAMBDA_DEPLOY"
echo "==========================================================" echo "=========================================================="
# --------------------------------------------------------- # ---------------------------------------------------------
@@ -113,41 +122,43 @@ echo " -> Permissions aktualisiert."
sleep 5 sleep 5
# --------------------------------------------------------- # ---------------------------------------------------------
# 4. Lambda Funktion erstellen/updaten (Global!) # 4 & 5. Lambda Funktion + SES Permissions (optional)
# --------------------------------------------------------- # ---------------------------------------------------------
echo "[4/6] Global Lambda Shim deployen..." if [ "$SKIP_LAMBDA_DEPLOY" = "true" ]; then
# Zip erstellen echo "[4/6] Global Lambda Shim deployen... (ÜBERSPRUNGEN)"
cp "$PYTHON_FILE" lambda_function.py echo "[5/6] SES Permission für Lambda... (ÜBERSPRUNGEN)"
zip -q lambda.zip lambda_function.py
# Keine Env-Vars nötig, da dynamisch
ROLE_ARN=$(aws iam get-role --role-name "$LAMBDA_ROLE_NAME" --query 'Role.Arn' --output text)
if ! aws lambda get-function --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null 2>&1; then
echo " -> Erstelle neue Lambda-Funktion..."
aws lambda create-function --function-name "$LAMBDA_NAME" \
--runtime python3.11 --handler lambda_function.lambda_handler \
--role "$ROLE_ARN" --zip-file fileb://lambda.zip \
--region "$AWS_REGION" >/dev/null
else else
echo " -> Aktualisiere existierende Lambda-Funktion..." echo "[4/6] Global Lambda Shim deployen..."
aws lambda update-function-code --function-name "$LAMBDA_NAME" --zip-file fileb://lambda.zip --region "$AWS_REGION" >/dev/null # Zip erstellen
cp "$PYTHON_FILE" lambda_function.py
# Warte kurz zip -q lambda.zip lambda_function.py
sleep 2 # Keine Env-Vars nötig, da dynamisch
ROLE_ARN=$(aws iam get-role --role-name "$LAMBDA_ROLE_NAME" --query 'Role.Arn' --output text)
aws lambda update-function-configuration --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null if ! aws lambda get-function --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null 2>&1; then
fi echo " -> Erstelle neue Lambda-Funktion..."
# Aufräumen aws lambda create-function --function-name "$LAMBDA_NAME" \
rm lambda.zip lambda_function.py --runtime python3.11 --handler lambda_function.lambda_handler \
--role "$ROLE_ARN" --zip-file fileb://lambda.zip \
--region "$AWS_REGION" >/dev/null
else
echo " -> Aktualisiere existierende Lambda-Funktion..."
aws lambda update-function-code --function-name "$LAMBDA_NAME" --zip-file fileb://lambda.zip --region "$AWS_REGION" >/dev/null
# Warte kurz
sleep 2
aws lambda update-function-configuration --function-name "$LAMBDA_NAME" --region "$AWS_REGION" >/dev/null
fi
# Aufräumen
rm lambda.zip lambda_function.py
# --------------------------------------------------------- echo "[5/6] SES Permission für Lambda..."
# 5. Permission: SES darf Lambda aufrufen (Global, einmalig) aws lambda add-permission --function-name "$LAMBDA_NAME" \
# --------------------------------------------------------- --statement-id "AllowSESInvoke-Global" \
echo "[5/6] SES Permission für Lambda..." --action "lambda:InvokeFunction" \
aws lambda add-permission --function-name "$LAMBDA_NAME" \ --principal "ses.amazonaws.com" \
--statement-id "AllowSESInvoke-Global" \ --region "$AWS_REGION" 2>/dev/null || true
--action "lambda:InvokeFunction" \ fi
--principal "ses.amazonaws.com" \
--region "$AWS_REGION" 2>/dev/null || true
# --------------------------------------------------------- # ---------------------------------------------------------
# 6. SES Rule (S3 + Global Lambda) # 6. SES Rule (S3 + Global Lambda)

90
basic_setup/check_logins.py Executable file
View File

@@ -0,0 +1,90 @@
#!/usr/bin/env python3
import csv
import imaplib
import sys
import time
# Konfiguration
IMAP_SERVER = "secure.emailsrvr.com"
IMAP_PORT = 993
DELAY_SECONDS = 1 # Kurze Pause, um Rate-Limiting oder Fail2Ban zu vermeiden
def check_imap_login(email, password):
try:
# Verbindung zum IMAP-Server via SSL herstellen
mail = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
mail.login(email, password)
mail.logout()
return True
except imaplib.IMAP4.error:
# Login fehlgeschlagen (falsches Passwort/User)
return False
except Exception as e:
print(f" [!] Netzwerk- oder Serverfehler bei {email}: {e}")
return False
def main(csv_filepath):
erfolgreich = []
fehlgeschlagen = []
uebersprungen = []
try:
with open(csv_filepath, mode='r', encoding='utf-8') as f:
# Nutze csv.reader für sauberes Parsing der Kommas
reader = csv.reader(f)
for row_num, row in enumerate(reader, start=1):
if not row:
continue
email = row[0].strip()
# Prüfen, ob ein zweites Feld (Passwort) existiert und nicht leer ist
password = row[1].strip() if len(row) > 1 else ""
if not password:
print(f"[{row_num}] Überspringe {email} (Kein Passwort)")
uebersprungen.append(email)
continue
print(f"[{row_num}] Prüfe {email}... ", end="", flush=True)
if check_imap_login(email, password):
print("OK")
erfolgreich.append(email)
else:
print("FEHLGESCHLAGEN")
fehlgeschlagen.append(email)
# Kurze Pause einlegen, um den Mailserver nicht zu fluten
time.sleep(DELAY_SECONDS)
except FileNotFoundError:
print(f"\nFehler: Die Datei '{csv_filepath}' wurde nicht gefunden.")
sys.exit(1)
except Exception as e:
print(f"\nEin unerwarteter Fehler ist aufgetreten: {e}")
sys.exit(1)
# Ausgabe der Zusammenfassung
print("\n" + "="*40)
print("ZUSAMMENFASSUNG DER PRÜFUNG")
print("="*40)
print(f"\nErfolgreich ({len(erfolgreich)}):")
for e in erfolgreich:
print(f" - {e}")
print(f"\nFehlgeschlagen ({len(fehlgeschlagen)}):")
for e in fehlgeschlagen:
print(f" - {e}")
print(f"\nÜbersprungen (kein Passwort) ({len(uebersprungen)}):")
for e in uebersprungen:
print(f" - {e}")
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Verwendung: ./check_logins.py <pfad_zur_datei.csv>")
sys.exit(1)
csv_file = sys.argv[1]
main(csv_file)

View File

@@ -5,15 +5,22 @@
# Setzt mail/imap/smtp/pop Subdomains für domain-spezifischen Mailserver-Zugang # Setzt mail/imap/smtp/pop Subdomains für domain-spezifischen Mailserver-Zugang
# #
# MIGRATIONS-FLAGS: # MIGRATIONS-FLAGS:
# SKIP_CLIENT_DNS=true → Abschnitt 8 (imap/smtp/pop/webmail) + 10 (SRV) überspringen # SKIP_CLIENT_DNS=true → Abschnitt 8 (mail/imap/smtp/pop/webmail) überspringen
# Nutzen: Client-Subdomains bleiben beim alten Provider # Nutzen: Client-Subdomains bleiben komplett beim alten Provider
# SKIP_DMARC=true → Abschnitt 7 (DMARC) überspringen # SKIP_PROTOCOL_DNS=true → nur imap/smtp/pop überspringen, aber mail + webmail setzen
# Nutzen: Bestehenden DMARC-Record nicht anfassen # Nutzen: Webmail/Autodiscover vorbereiten, Mailclients bleiben beim alten Provider
# SKIP_DMARC=true → Abschnitt 7 (DMARC) überspringen
# Nutzen: Bestehenden DMARC-Record nicht anfassen
# #
# Typischer Migrations-Ablauf: # Typischer Migrations-Ablauf:
# Phase 0 (Vorbereitung): SKIP_CLIENT_DNS=true SKIP_DMARC=true → nur SES + SPF # Phase 0a (Vorbereitung, Client-Records bleiben alt):
# Phase 1 (MX Cutover): MX umstellen (manuell) # SKIP_CLIENT_DNS=true SKIP_DMARC=true → nur SES + SPF/DKIM/MailFrom
# Phase 2 (Client Switch): ohne SKIP Flags → alle Records setzen # Phase 0b (Webmail vorbereiten, imap/smtp/pop bleiben alt):
# SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=true SKIP_DMARC=false
# Phase 1 (MX Cutover):
# MX manuell umstellen
# Phase 2 (Client Switch):
# SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=false → alle Client-Records setzen
set -e set -e
@@ -23,6 +30,7 @@ DRY_RUN=${DRY_RUN:-"false"}
# Migrations-Flags (NEU) # Migrations-Flags (NEU)
SKIP_CLIENT_DNS=${SKIP_CLIENT_DNS:-"false"} SKIP_CLIENT_DNS=${SKIP_CLIENT_DNS:-"false"}
SKIP_PROTOCOL_DNS=${SKIP_PROTOCOL_DNS:-"false"}
SKIP_DMARC=${SKIP_DMARC:-"false"} SKIP_DMARC=${SKIP_DMARC:-"false"}
# IP des Mailservers - PFLICHT wenn keine CNAME-Kette gewünscht # IP des Mailservers - PFLICHT wenn keine CNAME-Kette gewünscht
@@ -50,7 +58,8 @@ echo " 🌍 Region: $AWS_REGION"
echo " 📬 Mail-Server Target: $TARGET_MAIL_SERVER" echo " 📬 Mail-Server Target: $TARGET_MAIL_SERVER"
[ -n "$MAIL_SERVER_IP" ] && echo " 🖥️ Server IP: $MAIL_SERVER_IP" [ -n "$MAIL_SERVER_IP" ] && echo " 🖥️ Server IP: $MAIL_SERVER_IP"
[ "$DRY_RUN" = "true" ] && echo " ⚠️ DRY RUN MODE - Keine Änderungen!" [ "$DRY_RUN" = "true" ] && echo " ⚠️ DRY RUN MODE - Keine Änderungen!"
[ "$SKIP_CLIENT_DNS" = "true" ] && echo " ⏭️ SKIP: Client-Subdomains (imap/smtp/pop/webmail/SRV)" [ "$SKIP_CLIENT_DNS" = "true" ] && echo " ⏭️ SKIP: alle Client-Subdomains (mail/imap/smtp/pop/webmail)"
[ "$SKIP_PROTOCOL_DNS" = "true" ] && echo " ⏭️ SKIP: Protokoll-Subdomains imap/smtp/pop bleiben unverändert"
[ "$SKIP_DMARC" = "true" ] && echo " ⏭️ SKIP: DMARC Record" [ "$SKIP_DMARC" = "true" ] && echo " ⏭️ SKIP: DMARC Record"
echo "============================================================" echo "============================================================"
@@ -255,14 +264,23 @@ else
fi fi
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# SCHRITT 6: Root Domain MX (nur Info, wird nicht geändert) # SCHRITT 6: Root Domain MX
# ------------------------------------------------------------------ # ------------------------------------------------------------------
echo "" echo ""
echo "--- 6. Root Domain MX (nur Info, wird nicht geändert) ---" echo "--- 6. Root Domain MX ---"
CURRENT_MX=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?type=MX&name=$DOMAIN_NAME" \ # Prüfen, ob bereits MX-Records für die Root-Domain existieren
MX_COUNT=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?type=MX&name=$DOMAIN_NAME" \
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" \ -H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" \
| jq -r '.result[0].content // "keiner"') | jq -r '.result | length')
echo " MX vorhanden: $CURRENT_MX (wird nicht geändert)"
if [ "$MX_COUNT" -eq 0 ]; then
echo " Kein MX-Record vorhanden. Setze initialen SES Inbound MX..."
ensure_record "MX" "$DOMAIN_NAME" "inbound-smtp.${AWS_REGION}.amazonaws.com" false 10
else
echo " ⛔ MX-Record(s) für @ bereits vorhanden ($MX_COUNT Eintrag/Einträge)."
echo " → Wird zum Schutz der alten Postfächer AUF KEINEN FALL automatisch geändert!"
echo " → Muss nach der Migration manuell umgestellt werden."
fi
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# SCHRITT 7: DMARC # SCHRITT 7: DMARC
@@ -283,11 +301,12 @@ echo ""
echo "--- 8. Mailclient Subdomains (A + CNAME) ---" echo "--- 8. Mailclient Subdomains (A + CNAME) ---"
if [ "$SKIP_CLIENT_DNS" = "true" ]; then if [ "$SKIP_CLIENT_DNS" = "true" ]; then
echo " ⏭️ Übersprungen (SKIP_CLIENT_DNS=true)" echo " ⏭️ Übersprungen (SKIP_CLIENT_DNS=true)"
echo " imap/smtp/pop/webmail bleiben beim alten Provider." echo " mail/imap/smtp/pop/webmail bleiben beim alten Provider."
echo " Setze SKIP_CLIENT_DNS=false nach MX-Cutover + Client-Umstellung." echo " Setze SKIP_CLIENT_DNS=false nach MX-Cutover + Client-Umstellung."
else else
if [ -n "$MAIL_SERVER_IP" ]; then if [ -n "$MAIL_SERVER_IP" ]; then
# A-Record für mail.<domain> direkt auf Server-IP # A-Record für mail.<domain> direkt auf Server-IP
# Wichtig: mail muss DNS-only bleiben; Cloudflare Proxy funktioniert nicht für SMTP/IMAP/POP.
ensure_record "A" "mail.$DOMAIN_NAME" "$MAIL_SERVER_IP" false ensure_record "A" "mail.$DOMAIN_NAME" "$MAIL_SERVER_IP" false
else else
# CNAME auf externen Ziel-Host (nur wenn verschieden) # CNAME auf externen Ziel-Host (nur wenn verschieden)
@@ -296,11 +315,19 @@ else
fi fi
fi fi
# imap, smtp, pop, webmail → CNAME auf mail.<domain> # Webmail kann bereits vorbereitet werden, auch wenn imap/smtp/pop noch beim alten Provider bleiben.
ensure_record "CNAME" "imap.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
ensure_record "CNAME" "smtp.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
ensure_record "CNAME" "pop.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
ensure_record "CNAME" "webmail.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false ensure_record "CNAME" "webmail.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
if [ "$SKIP_PROTOCOL_DNS" = "true" ]; then
echo " ⏭️ Überspringe imap/smtp/pop (SKIP_PROTOCOL_DNS=true)"
echo " imap/smtp/pop bleiben unverändert beim alten Provider."
else
# imap/smtp/pop → CNAME auf mail.<domain>
# Wichtig: diese Records müssen DNS-only bleiben; Cloudflare Proxy funktioniert nicht für Mail-Protokolle.
ensure_record "CNAME" "imap.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
ensure_record "CNAME" "smtp.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
ensure_record "CNAME" "pop.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
fi
fi fi
# ------------------------------------------------------------------ # ------------------------------------------------------------------
@@ -318,7 +345,12 @@ if [ "$SKIP_CLIENT_DNS" = "true" ]; then
echo "" echo ""
echo " ⚠️ Client-Subdomains wurden NICHT geändert." echo " ⚠️ Client-Subdomains wurden NICHT geändert."
echo " Nach MX-Cutover + Worker-Validierung erneut ausführen mit:" echo " Nach MX-Cutover + Worker-Validierung erneut ausführen mit:"
echo " SKIP_CLIENT_DNS=false SKIP_DMARC=false ./cloudflareMigrationDns.sh" echo " SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=false SKIP_DMARC=false ./cloudflareMigrationDns.sh"
elif [ "$SKIP_PROTOCOL_DNS" = "true" ]; then
echo ""
echo " ⚠️ imap/smtp/pop wurden NICHT geändert."
echo " Für den finalen Client-Cutover erneut ausführen mit:"
echo " SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=false SKIP_DMARC=false ./cloudflareMigrationDns.sh"
fi fi
echo "" echo ""
echo " Mailclient-Konfiguration für Kunden:" echo " Mailclient-Konfiguration für Kunden:"

View File

@@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
# create-queue.sh (v2 — mit SNS Fan-Out + Standby Queue) # create-queue.sh (v2 — mit SNS Fan-Out + Standby Queue)
# Usage: DOMAIN=andreasknuth.de ./create-queue.sh # Usage: DOMAIN_NAME=andreasknuth.de ./create-queue.sh
# #
# Erstellt pro Domain: # Erstellt pro Domain:
# - Primary Queue + DLQ (wie bisher, für Contabo) # - Primary Queue + DLQ (wie bisher, für Contabo)
@@ -12,13 +12,13 @@ set -e
AWS_REGION="us-east-2" AWS_REGION="us-east-2"
if [ -z "$DOMAIN" ]; then if [ -z "$DOMAIN_NAME" ]; then
echo "Error: DOMAIN environment variable not set" echo "Error: DOMAIN_NAME environment variable not set"
echo "Usage: DOMAIN=andreasknuth.de $0" echo "Usage: DOMAIN_NAME=andreasknuth.de $0"
exit 1 exit 1
fi fi
DOMAIN_SLUG="${DOMAIN//./-}" DOMAIN_SLUG="${DOMAIN_NAME//./-}"
QUEUE_NAME="${DOMAIN_SLUG}-queue" QUEUE_NAME="${DOMAIN_SLUG}-queue"
DLQ_NAME="${QUEUE_NAME}-dlq" DLQ_NAME="${QUEUE_NAME}-dlq"
STANDBY_QUEUE_NAME="${DOMAIN_SLUG}-standby-queue" STANDBY_QUEUE_NAME="${DOMAIN_SLUG}-standby-queue"
@@ -31,9 +31,9 @@ echo "========================================"
echo "Creating SQS + SNS for Email Delivery" echo "Creating SQS + SNS for Email Delivery"
echo "========================================" echo "========================================"
echo "" echo ""
echo "📧 Domain: $DOMAIN" echo "📧 Domain: $DOMAIN_NAME"
echo " Region: $AWS_REGION" echo " Region: $AWS_REGION"
echo " Account: $ACCOUNT_ID" echo " Account: $ACCOUNT_ID"
echo "" echo ""
# ============================================================ # ============================================================
@@ -184,7 +184,7 @@ echo ""
# Zusammenfassung # Zusammenfassung
# ============================================================ # ============================================================
echo "========================================" echo "========================================"
echo "✅ Setup complete for $DOMAIN" echo "✅ Setup complete for $DOMAIN_NAME"
echo "========================================" echo "========================================"
echo "" echo ""
echo "Primary (Contabo):" echo "Primary (Contabo):"

View File

@@ -0,0 +1,32 @@
#!/bin/bash
echo "Aktualisiere Lifecycle-Regeln (7 Tage) für alle E-Mail-Buckets..."
for BUCKET in $(aws s3api list-buckets --query 'Buckets[].Name' --output text); do
# Prüfen, ob der Name auf '-emails' endet
if [[ "$BUCKET" == *-emails ]]; then
echo "⚙️ Setze 7-Tage-Regel für: $BUCKET"
aws s3api put-bucket-lifecycle-configuration \
--bucket "$BUCKET" \
--lifecycle-configuration '{
"Rules": [
{
"ID": "DeleteOldEmails",
"Status": "Enabled",
"Expiration": {
"Days": 7
},
"Filter": {
"Prefix": ""
}
}
]
}'
else
echo "⏭️ Überspringe (kein E-Mail-Bucket): $BUCKET"
fi
done
echo "Fertig! Alle E-Mail-Buckets löschen jetzt Objekte nach 7 Tagen."

View File

@@ -0,0 +1,31 @@
#!/bin/bash
# sync-s3-tags.sh - Synchronisiert Bucket Tags welche fuer die Abrechung verwendet werden
echo "Passe Bucket-Tags an..."
for BUCKET in $(aws s3api list-buckets --query 'Buckets[].Name' --output text); do
# Aktuellen Tag abfragen (Fehler unterdrücken)
TAG=$(aws s3api get-bucket-tagging --bucket "$BUCKET" --query 'TagSet[?Key==`BucketName`].Value' --output text 2>/dev/null)
# Prüfen, ob der Name auf '-emails' endet
if [[ "$BUCKET" == *-emails ]]; then
# Soll getaggt sein
if [ -z "$TAG" ] || [ "$TAG" == "None" ]; then
echo " Setze fehlendes Tag für: $BUCKET"
aws s3api put-bucket-tagging \
--bucket "$BUCKET" \
--tagging "TagSet=[{Key=BucketName,Value=$BUCKET}]"
else
echo "✅ OK (bereits getaggt): $BUCKET"
fi
else
# Soll NICHT getaggt sein
if [ -n "$TAG" ] && [ "$TAG" != "None" ]; then
echo "🗑️ Entferne Tag von: $BUCKET"
aws s3api delete-bucket-tagging --bucket "$BUCKET"
else
echo "✅ OK (ohne Tag): $BUCKET"
fi
fi
done
echo "Fertig! Alle '-emails' Buckets sind getaggt, bei allen anderen wurden die Tags entfernt."

158
basic_setup/mailadminDns.sh Executable file
View File

@@ -0,0 +1,158 @@
#!/bin/bash
# mailadminDns.sh
# ------------------------------------------------------------------
# Setzt AUSSCHLIESSLICH den DNS-Record fuer den mailadmin-Zugang.
# Faesst NICHTS anderes an (kein SES, SPF, DKIM, MX, keine anderen
# Subdomains). Kann gefahrlos auf bereits migrierten Domains laufen.
#
# Verwendet dieselbe ensure_record-Logik wie cloudflareMigrationDns.sh,
# damit das Verhalten (Create/Update/Skip-bei-identisch) identisch ist.
#
# ------------------------------------------------------------------
# PFLICHT-VARIABLEN:
# DOMAIN_NAME Kundendomain, z.B. innungsapp.com
# CF_API_TOKEN Cloudflare API Token
#
# OPTIONALE VARIABLEN:
# MAILADMIN_HOST Subdomain-Label, Default: "mailadmin"
# -> ergibt mailadmin.<DOMAIN_NAME>
# MAILADMIN_TARGET CNAME-Ziel, Default: "mail.<DOMAIN_NAME>"
# Beispiele:
# mail.innungsapp.com (pro-Domain Modell)
# mailadmin.bayarea-cc.com (zentrales Modell)
# DRY_RUN "true" -> zeigt nur an, aendert nichts
#
# ------------------------------------------------------------------
# BEISPIELE:
#
# # Standard: mailadmin.innungsapp.com -> mail.innungsapp.com
# DOMAIN_NAME=innungsapp.com CF_API_TOKEN=xxx ./mailadminDns.sh
#
# # Zentrales Modell: mailadmin.innungsapp.com -> mailadmin.bayarea-cc.com
# DOMAIN_NAME=innungsapp.com \
# MAILADMIN_TARGET=mailadmin.bayarea-cc.com \
# CF_API_TOKEN=xxx ./mailadminDns.sh
#
# # Erst testen ohne zu aendern
# DOMAIN_NAME=innungsapp.com CF_API_TOKEN=xxx DRY_RUN=true ./mailadminDns.sh
#
# ------------------------------------------------------------------
set -e
# --- KONFIGURATION ---
DRY_RUN=${DRY_RUN:-"false"}
MAILADMIN_HOST=${MAILADMIN_HOST:-"mailadmin"}
MAILADMIN_TARGET=${MAILADMIN_TARGET:-"mail.${DOMAIN_NAME}"}
# --- CHECKS ---
if [ -z "$DOMAIN_NAME" ]; then echo "❌ Fehler: DOMAIN_NAME fehlt."; exit 1; fi
if [ -z "$CF_API_TOKEN" ]; then echo "❌ Fehler: CF_API_TOKEN fehlt."; exit 1; fi
if ! command -v jq &> /dev/null; then echo "❌ Fehler: 'jq' fehlt."; exit 1; fi
if ! command -v curl &> /dev/null; then echo "❌ Fehler: 'curl' fehlt."; exit 1; fi
RECORD_NAME="${MAILADMIN_HOST}.${DOMAIN_NAME}"
echo "============================================================"
echo " 🔧 mailadmin DNS Setup"
echo " 🌐 Domain: $DOMAIN_NAME"
echo " 📍 Record: CNAME $RECORD_NAME$MAILADMIN_TARGET"
[ "$DRY_RUN" = "true" ] && echo " ⚠️ DRY RUN MODE - Keine Änderungen!"
echo "============================================================"
# Schutz: CNAME auf sich selbst macht keinen Sinn
if [ "$RECORD_NAME" == "$MAILADMIN_TARGET" ]; then
echo "❌ Fehler: CNAME-Ziel ist identisch mit dem Record-Namen."
echo " $RECORD_NAME kann nicht auf sich selbst zeigen."
echo " Setze MAILADMIN_TARGET auf einen anderen Host."
exit 1
fi
# 1. ZONE ID HOLEN
echo "🔍 Suche Cloudflare Zone ID..."
ZONE_ID=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN_NAME" \
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" | jq -r '.result[0].id')
if [ "$ZONE_ID" == "null" ] || [ -z "$ZONE_ID" ]; then
echo "❌ Zone nicht gefunden für $DOMAIN_NAME."
exit 1
fi
echo " ✅ Zone ID: $ZONE_ID"
# ------------------------------------------------------------------
# FUNKTION: ensure_record
# Identisch zur Logik in cloudflareMigrationDns.sh, reduziert auf
# die Typen die wir hier brauchen (CNAME).
# ------------------------------------------------------------------
ensure_record() {
local type=$1
local name=$2
local content=$3
local proxied=${4:-false}
echo " ⚙️ Prüfe $type $name..."
local search_res=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?type=$type&name=$name" \
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json")
local rec_id=$(echo "$search_res" | jq -r '.result[0].id')
local rec_content=$(echo "$search_res" | jq -r '.result[0].content')
[ -z "$rec_id" ] && rec_id="null"
[ -z "$rec_content" ] && rec_content="null"
local json_data=$(jq -n --arg t "$type" --arg n "$name" --arg c "$content" --argjson p "$proxied" \
'{type: $t, name: $n, content: $c, ttl: 3600, proxied: $p}')
if [ "$rec_id" == "null" ]; then
if [ "$DRY_RUN" = "true" ]; then
echo " [DRY] Würde ERSTELLEN: $type $name$content"
else
local res=$(curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" \
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" --data "$json_data")
if [ "$(echo $res | jq -r .success)" == "true" ]; then
echo " ✅ Erstellt: $type $name$content"
else
echo " ❌ Fehler beim Erstellen: $(echo $res | jq -r '.errors[0].message')"
exit 1
fi
fi
else
if [ "$rec_content" == "$content" ]; then
echo " 🆗 Identisch ($rec_content). Überspringe."
else
if [ "$DRY_RUN" = "true" ]; then
echo " [DRY] Würde UPDATEN: '$rec_content' → '$content'"
else
local res=$(curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$rec_id" \
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" --data "$json_data")
if [ "$(echo $res | jq -r .success)" == "true" ]; then
echo " ✅ Aktualisiert: '$rec_content' → '$content'"
else
echo " ❌ Fehler beim Updaten: $(echo $res | jq -r '.errors[0].message')"
exit 1
fi
fi
fi
fi
}
# ------------------------------------------------------------------
# Der eigentliche Record
# proxied=false ist hier wichtig: der mailadmin laeuft hinter Caddy
# mit eigenem TLS-Zertifikat. Cloudflare-Proxy davor wuerde die
# Zertifikatskette stoeren bzw. doppeltes TLS-Terminieren.
# ------------------------------------------------------------------
echo ""
echo "--- mailadmin CNAME ---"
ensure_record "CNAME" "$RECORD_NAME" "$MAILADMIN_TARGET" false
echo ""
echo "============================================================"
echo "✅ Fertig."
echo ""
echo " mailadmin erreichbar unter: https://$RECORD_NAME"
echo ""
echo " ⚠️ Wichtig: Caddy muss ein Zertifikat für $RECORD_NAME"
echo " ausstellen. Stelle sicher, dass der Hostname in der"
echo " Caddy-Konfiguration enthalten ist."
echo "============================================================"

View File

@@ -8,24 +8,38 @@ from botocore.exceptions import ClientError
import time import time
import random import random
# Logging konfigurieren
logger = logging.getLogger() logger = logging.getLogger()
logger.setLevel(logging.INFO) logger.setLevel(logging.INFO)
sqs = boto3.client('sqs') sqs = boto3.client('sqs')
sns = boto3.client('sns')
sts_account_id = None
# Retry-Konfiguration
MAX_RETRIES = 3 MAX_RETRIES = 3
BASE_BACKOFF = 1 # Sekunden BASE_BACKOFF = 1
def exponential_backoff(attempt): def exponential_backoff(attempt):
"""Exponential Backoff mit Jitter"""
return BASE_BACKOFF * (2 ** attempt) + random.uniform(0, 1) return BASE_BACKOFF * (2 ** attempt) + random.uniform(0, 1)
def get_account_id():
global sts_account_id
if sts_account_id is None:
sts_account_id = boto3.client('sts').get_caller_identity()['Account']
return sts_account_id
def get_topic_arn(domain):
"""
Generiert Topic-ARN aus Domain.
Konvention: domain.tld -> domain-tld-topic
"""
topic_name = domain.replace('.', '-') + '-topic'
region = os.environ.get('AWS_REGION', 'us-east-2')
account_id = get_account_id()
return f"arn:aws:sns:{region}:{account_id}:{topic_name}"
def get_queue_url(domain): def get_queue_url(domain):
""" """
Generiert Queue-Namen aus Domain und holt URL. Fallback: Direkter SQS-Send für Domains ohne SNS-Topic.
Konvention: domain.tld -> domain-tld-queue
""" """
queue_name = domain.replace('.', '-') + '-queue' queue_name = domain.replace('.', '-') + '-queue'
try: try:
@@ -38,54 +52,91 @@ def get_queue_url(domain):
else: else:
raise raise
def publish_to_sns(topic_arn, message_body, msg_id):
attempt = 0
while attempt < MAX_RETRIES:
try:
sns.publish(
TopicArn=topic_arn,
Message=message_body
)
logger.info(f"✅ Published {msg_id} to SNS: {topic_arn}")
return True
except ClientError as e:
error_code = e.response['Error']['Code']
# Fallback auf SQS bei Topic-nicht-gefunden ODER fehlender Berechtigung
if error_code in ('NotFound', 'NotFoundException', 'AuthorizationError'):
logger.info(f" SNS unavailable for {topic_arn} ({error_code}) — falling back to SQS")
return False
attempt += 1
logger.warning(f"Retry {attempt}/{MAX_RETRIES} SNS: {error_code}")
if attempt == MAX_RETRIES:
raise
time.sleep(exponential_backoff(attempt))
return False
def send_to_sqs(queue_url, message_body, msg_id):
"""Fallback: Direkter SQS-Send (wie bisher)."""
attempt = 0
while attempt < MAX_RETRIES:
try:
sqs.send_message(
QueueUrl=queue_url,
MessageBody=message_body
)
logger.info(f"✅ Sent {msg_id} to SQS: {queue_url}")
return
except ClientError as e:
attempt += 1
error_code = e.response['Error']['Code']
logger.warning(f"Retry {attempt}/{MAX_RETRIES} SQS: {error_code}")
if attempt == MAX_RETRIES:
raise
time.sleep(exponential_backoff(attempt))
def lambda_handler(event, context): def lambda_handler(event, context):
""" """
Nimmt SES Event entgegen, extrahiert Domain dynamisch, Nimmt SES Event entgegen, extrahiert Domain dynamisch.
verpackt Metadaten als 'Fake SNS' und sendet an die domain-spezifische SQS. Strategie: SNS Publish (Fan-Out an Primary + Standby Queue).
Mit integrierter Retry-Logik für SQS-Send. Fallback: Direkter SQS-Send falls kein SNS-Topic existiert.
""" """
try: try:
records = event.get('Records', []) records = event.get('Records', [])
logger.info(f"Received event with {len(records)} records.") logger.info(f"Received event with {len(records)} records.")
for record in records: for record in records:
ses_data = record.get('ses', {}) ses_data = record.get('ses', {})
if not ses_data: if not ses_data:
logger.warning(f"Invalid SES event: Missing 'ses' in record: {record}") logger.warning(f"Invalid SES event: Missing 'ses' in record")
continue continue
mail = ses_data.get('mail', {}) mail = ses_data.get('mail', {})
receipt = ses_data.get('receipt', {}) receipt = ses_data.get('receipt', {})
# Domain extrahieren (aus erstem Recipient)
recipients = receipt.get('recipients', []) or mail.get('destination', []) recipients = receipt.get('recipients', []) or mail.get('destination', [])
if not recipients: if not recipients:
logger.warning("No recipients in event - skipping") logger.warning("No recipients in event - skipping")
continue continue
first_recipient = recipients[0] first_recipient = recipients[0]
domain = first_recipient.split('@')[-1].lower() domain = first_recipient.split('@')[-1].lower()
if not domain: if not domain:
logger.error("Could not extract domain from recipient") logger.error("Could not extract domain from recipient")
continue continue
# Wichtige Metadaten loggen
msg_id = mail.get('messageId', 'unknown') msg_id = mail.get('messageId', 'unknown')
source = mail.get('source', 'unknown') source = mail.get('source', 'unknown')
logger.info(f"Processing Message-ID: {msg_id} for domain: {domain}") logger.info(f"Processing Message-ID: {msg_id} for domain: {domain}")
logger.info(f" From: {source}") logger.info(f" From: {source}")
logger.info(f" To: {recipients}") logger.info(f" To: {recipients}")
# SES JSON als String serialisieren
ses_json_string = json.dumps(ses_data) ses_json_string = json.dumps(ses_data)
# Payload Größe loggen und checken (Safeguard)
payload_size = len(ses_json_string.encode('utf-8')) payload_size = len(ses_json_string.encode('utf-8'))
logger.info(f" Metadata Payload Size: {payload_size} bytes") logger.info(f" Metadata Payload Size: {payload_size} bytes")
if payload_size > 200000: # Arbitrary Limit < SQS 256KB if payload_size > 200000:
raise ValueError("Payload too large for SQS") raise ValueError("Payload too large")
# Fake SNS Payload
fake_sns_payload = { fake_sns_payload = {
"Type": "Notification", "Type": "Notification",
"MessageId": str(uuid.uuid4()), "MessageId": str(uuid.uuid4()),
@@ -94,30 +145,20 @@ def lambda_handler(event, context):
"Message": ses_json_string, "Message": ses_json_string,
"Timestamp": datetime.utcnow().isoformat() + "Z" "Timestamp": datetime.utcnow().isoformat() + "Z"
} }
# Queue URL dynamisch holen message_body = json.dumps(fake_sns_payload)
queue_url = get_queue_url(domain)
# Strategie: SNS zuerst, SQS als Fallback
# SQS Send mit Retries topic_arn = get_topic_arn(domain)
attempt = 0 sns_success = publish_to_sns(topic_arn, message_body, msg_id)
while attempt < MAX_RETRIES:
try: if not sns_success:
sqs.send_message( # Kein SNS-Topic für diese Domain → direkt in SQS (wie bisher)
QueueUrl=queue_url, queue_url = get_queue_url(domain)
MessageBody=json.dumps(fake_sns_payload) send_to_sqs(queue_url, message_body, msg_id)
)
logger.info(f"✅ Successfully forwarded {msg_id} to SQS: {queue_url}")
break
except ClientError as e:
attempt += 1
error_code = e.response['Error']['Code']
logger.warning(f"Retry {attempt}/{MAX_RETRIES} for SQS send: {error_code} - {str(e)}")
if attempt == MAX_RETRIES:
raise
time.sleep(exponential_backoff(attempt))
return {'status': 'ok'} return {'status': 'ok'}
except Exception as e: except Exception as e:
logger.error(f"❌ Critical Error in Lambda Shim: {str(e)}", exc_info=True) logger.error(f"❌ Critical Error in Lambda Shim: {str(e)}", exc_info=True)
raise e raise e

View File

@@ -7,7 +7,16 @@ RUN xcaddy build ${CADDY_VERSION} \
--with github.com/caddy-dns/cloudflare \ --with github.com/caddy-dns/cloudflare \
--with github.com/caddyserver/replace-response --with github.com/caddyserver/replace-response
# Autodiscover Handler in Go bauen (Go ist im Builder-Image bereits verfügbar)
COPY autodiscover-handler.go /src/autodiscover-handler.go
WORKDIR /src
RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/bin/autodiscover-handler autodiscover-handler.go
FROM caddy:${CADDY_VERSION} FROM caddy:${CADDY_VERSION}
COPY --from=builder /usr/bin/caddy /usr/bin/caddy COPY --from=builder /usr/bin/caddy /usr/bin/caddy
RUN mkdir -p /var/log/caddy COPY --from=builder /usr/bin/autodiscover-handler /usr/local/bin/autodiscover-handler
COPY start.sh /usr/local/bin/start.sh
RUN chmod +x /usr/local/bin/start.sh /usr/local/bin/autodiscover-handler \
&& mkdir -p /var/log/caddy
CMD ["/usr/local/bin/start.sh"]

View File

@@ -0,0 +1,109 @@
package main
import (
"fmt"
"io"
"log"
"net/http"
"regexp"
"strings"
)
const port = "8280"
var emailRegex = regexp.MustCompile(`(?i)<EMailAddress>([^<]+)</EMailAddress>`)
func main() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, "OK")
})
http.HandleFunc("/autodiscover/autodiscover.xml", handleAutodiscover)
// Outlook sendet manchmal mit Großbuchstaben
http.HandleFunc("/Autodiscover/Autodiscover.xml", handleAutodiscover)
http.HandleFunc("/AutoDiscover/AutoDiscover.xml", handleAutodiscover)
log.Printf("[autodiscover] Listening on port %s", port)
if err := http.ListenAndServe(":"+port, nil); err != nil {
log.Fatal(err)
}
}
func handleAutodiscover(w http.ResponseWriter, r *http.Request) {
var email string
if r.Method == http.MethodPost {
body, err := io.ReadAll(r.Body)
if err == nil {
if match := emailRegex.FindStringSubmatch(string(body)); len(match) > 1 {
email = strings.TrimSpace(match[1])
}
}
r.Body.Close()
}
var domain string
if email != "" {
parts := strings.SplitN(email, "@", 2)
if len(parts) == 2 {
domain = parts[1]
}
}
if domain == "" {
domain = extractDomainFromHost(r.Host)
}
log.Printf("[autodiscover] %s from %s - email=%q domain=%s", r.Method, r.RemoteAddr, email, domain)
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, buildResponse(domain, email))
}
func extractDomainFromHost(host string) string {
// Strip port
if idx := strings.Index(host, ":"); idx >= 0 {
host = host[:idx]
}
parts := strings.Split(host, ".")
if len(parts) >= 3 && strings.EqualFold(parts[0], "autodiscover") {
return strings.Join(parts[1:], ".")
}
if len(parts) >= 2 {
return strings.Join(parts[len(parts)-2:], ".")
}
return host
}
func buildResponse(domain, loginName string) string {
return fmt.Sprintf(`<?xml version="1.0" encoding="utf-8"?>
<Autodiscover xmlns="http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006">
<Response xmlns="http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a">
<Account>
<AccountType>email</AccountType>
<Action>settings</Action>
<Protocol>
<Type>IMAP</Type>
<Server>imap.%s</Server>
<Port>993</Port>
<DomainRequired>off</DomainRequired>
<LoginName>%s</LoginName>
<SPA>off</SPA>
<SSL>on</SSL>
<AuthRequired>on</AuthRequired>
</Protocol>
<Protocol>
<Type>SMTP</Type>
<Server>smtp.%s</Server>
<Port>465</Port>
<DomainRequired>off</DomainRequired>
<LoginName>%s</LoginName>
<SPA>off</SPA>
<SSL>on</SSL>
<AuthRequired>on</AuthRequired>
</Protocol>
</Account>
</Response>
</Autodiscover>`, domain, loginName, domain, loginName)
}

View File

@@ -13,6 +13,7 @@ services:
- 'host.docker.internal:host-gateway' - 'host.docker.internal:host-gateway'
networks: networks:
- mail_network - mail_network
- mailadmin_network
volumes: volumes:
- $PWD/Caddyfile:/etc/caddy/Caddyfile - $PWD/Caddyfile:/etc/caddy/Caddyfile
- $PWD/mail_certs:/etc/caddy/mail_certs - $PWD/mail_certs:/etc/caddy/mail_certs
@@ -29,3 +30,5 @@ services:
networks: networks:
mail_network: mail_network:
external: true external: true
mailadmin_network:
external: true

View File

@@ -35,7 +35,7 @@
<body> <body>
<div class="card"> <div class="card">
<img src="/logo.png" alt="Logo" class="logo"> <img src="/email-setup/logo.png" alt="Logo" class="logo">
<div id="input-section"> <div id="input-section">
<h1>Email Setup</h1> <h1>Email Setup</h1>

8
caddy/start.sh Normal file
View File

@@ -0,0 +1,8 @@
#!/bin/sh
set -e
# Autodiscover handler im Hintergrund starten
/usr/local/bin/autodiscover-handler &
# Caddy im Vordergrund
exec caddy run --config /etc/caddy/Caddyfile --adapter caddyfile

View File

@@ -78,36 +78,7 @@ OUTPUT="${OUTPUT}(email_settings) {\n"
# --- 1. Outlook Classic Autodiscover (POST + GET XML) --- # --- 1. Outlook Classic Autodiscover (POST + GET XML) ---
OUTPUT="${OUTPUT} # Outlook Autodiscover (XML) - POST und GET\n" OUTPUT="${OUTPUT} # Outlook Autodiscover (XML) - POST und GET\n"
OUTPUT="${OUTPUT} route /autodiscover/autodiscover.xml {\n" OUTPUT="${OUTPUT} route /autodiscover/autodiscover.xml {\n"
OUTPUT="${OUTPUT} header Content-Type \"application/xml\"\n" OUTPUT="${OUTPUT} reverse_proxy localhost:8280\n"
OUTPUT="${OUTPUT} respond \`<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
OUTPUT="${OUTPUT}<Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">\n"
OUTPUT="${OUTPUT} <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">\n"
OUTPUT="${OUTPUT} <Account>\n"
OUTPUT="${OUTPUT} <AccountType>email</AccountType>\n"
OUTPUT="${OUTPUT} <Action>settings</Action>\n"
OUTPUT="${OUTPUT} <Protocol>\n"
OUTPUT="${OUTPUT} <Type>IMAP</Type>\n"
OUTPUT="${OUTPUT} <Server>imap.{labels.1}.{labels.0}</Server>\n"
OUTPUT="${OUTPUT} <Port>993</Port>\n"
OUTPUT="${OUTPUT} <DomainRequired>off</DomainRequired>\n"
OUTPUT="${OUTPUT} <LoginName></LoginName>\n"
OUTPUT="${OUTPUT} <SPA>off</SPA>\n"
OUTPUT="${OUTPUT} <SSL>on</SSL>\n"
OUTPUT="${OUTPUT} <AuthRequired>on</AuthRequired>\n"
OUTPUT="${OUTPUT} </Protocol>\n"
OUTPUT="${OUTPUT} <Protocol>\n"
OUTPUT="${OUTPUT} <Type>SMTP</Type>\n"
OUTPUT="${OUTPUT} <Server>smtp.{labels.1}.{labels.0}</Server>\n"
OUTPUT="${OUTPUT} <Port>465</Port>\n"
OUTPUT="${OUTPUT} <DomainRequired>off</DomainRequired>\n"
OUTPUT="${OUTPUT} <LoginName></LoginName>\n"
OUTPUT="${OUTPUT} <SPA>off</SPA>\n"
OUTPUT="${OUTPUT} <SSL>on</SSL>\n"
OUTPUT="${OUTPUT} <AuthRequired>on</AuthRequired>\n"
OUTPUT="${OUTPUT} </Protocol>\n"
OUTPUT="${OUTPUT} </Account>\n"
OUTPUT="${OUTPUT} </Response>\n"
OUTPUT="${OUTPUT}</Autodiscover>\` 200\n"
OUTPUT="${OUTPUT} }\n" OUTPUT="${OUTPUT} }\n"
OUTPUT="${OUTPUT}\n" OUTPUT="${OUTPUT}\n"
@@ -135,14 +106,14 @@ OUTPUT="${OUTPUT} <hostname>imap.{labels.1}.{labels.0}</hostname>\n"
OUTPUT="${OUTPUT} <port>993</port>\n" OUTPUT="${OUTPUT} <port>993</port>\n"
OUTPUT="${OUTPUT} <socketType>SSL</socketType>\n" OUTPUT="${OUTPUT} <socketType>SSL</socketType>\n"
OUTPUT="${OUTPUT} <authentication>password-cleartext</authentication>\n" OUTPUT="${OUTPUT} <authentication>password-cleartext</authentication>\n"
OUTPUT="${OUTPUT} <username>%%EMAILADDRESS%%</username>\n" OUTPUT="${OUTPUT} <username>%EMAILADDRESS%</username>\n"
OUTPUT="${OUTPUT} </incomingServer>\n" OUTPUT="${OUTPUT} </incomingServer>\n"
OUTPUT="${OUTPUT} <outgoingServer type=\"smtp\">\n" OUTPUT="${OUTPUT} <outgoingServer type=\"smtp\">\n"
OUTPUT="${OUTPUT} <hostname>smtp.{labels.1}.{labels.0}</hostname>\n" OUTPUT="${OUTPUT} <hostname>smtp.{labels.1}.{labels.0}</hostname>\n"
OUTPUT="${OUTPUT} <port>465</port>\n" OUTPUT="${OUTPUT} <port>465</port>\n"
OUTPUT="${OUTPUT} <socketType>SSL</socketType>\n" OUTPUT="${OUTPUT} <socketType>SSL</socketType>\n"
OUTPUT="${OUTPUT} <authentication>password-cleartext</authentication>\n" OUTPUT="${OUTPUT} <authentication>password-cleartext</authentication>\n"
OUTPUT="${OUTPUT} <username>%%EMAILADDRESS%%</username>\n" OUTPUT="${OUTPUT} <username>%EMAILADDRESS%</username>\n"
OUTPUT="${OUTPUT} </outgoingServer>\n" OUTPUT="${OUTPUT} </outgoingServer>\n"
OUTPUT="${OUTPUT} </emailProvider>\n" OUTPUT="${OUTPUT} </emailProvider>\n"
OUTPUT="${OUTPUT}</clientConfig>\` 200\n" OUTPUT="${OUTPUT}</clientConfig>\` 200\n"
@@ -289,6 +260,13 @@ for domain in $DOMAINS; do
OUTPUT="${OUTPUT} format console\n" OUTPUT="${OUTPUT} format console\n"
OUTPUT="${OUTPUT} }\n" OUTPUT="${OUTPUT} }\n"
OUTPUT="${OUTPUT}}\n\n" OUTPUT="${OUTPUT}}\n\n"
# Mailadmin Block
OUTPUT="${OUTPUT}# MailAdmin UI\n"
OUTPUT="${OUTPUT}mailadmin.${domain} {\n"
OUTPUT="${OUTPUT} encode gzip\n"
OUTPUT="${OUTPUT} reverse_proxy mailadmin:3000\n"
OUTPUT="${OUTPUT}}\n\n"
# Autodiscover / Autoconfig Block # Autodiscover / Autoconfig Block
OUTPUT="${OUTPUT}# Autodiscover/Autoconfig für $domain\n" OUTPUT="${OUTPUT}# Autodiscover/Autoconfig für $domain\n"

View File

@@ -10,12 +10,14 @@ services:
ports: ports:
- "9000:8000" # Prometheus metrics (Host:Container) - "9000:8000" # Prometheus metrics (Host:Container)
- "9090:8080" # Health check (Host:Container) - "9090:8080" # Health check (Host:Container)
# Connect to DMS on the host or Docker network
extra_hosts:
- "host.docker.internal:host-gateway"
environment: environment:
- SMTP_HOST=host.docker.internal - SMTP_HOST=mailserver
- SMTP_PORT=25 - SMTP_PORT=25
networks:
- mail_network
volumes: volumes:
worker-logs: worker-logs:
networks:
mail_network:
external: true

View File

@@ -48,6 +48,9 @@ export const config = {
// Monitoring // Monitoring
metricsPort: parseInt(process.env.METRICS_PORT ?? '8000', 10), metricsPort: parseInt(process.env.METRICS_PORT ?? '8000', 10),
healthPort: parseInt(process.env.HEALTH_PORT ?? '8080', 10), healthPort: parseInt(process.env.HEALTH_PORT ?? '8080', 10),
queueSuffix: process.env.QUEUE_SUFFIX ?? '-queue',
standbyMode: (process.env.STANDBY_MODE ?? 'false').toLowerCase() === 'true',
} as const; } as const;
export type Config = typeof config; export type Config = typeof config;
@@ -106,7 +109,7 @@ export function isInternalAddress(email: string): boolean {
/** Convert domain to SQS queue name: bizmatch.net → bizmatch-net-queue */ /** Convert domain to SQS queue name: bizmatch.net → bizmatch-net-queue */
export function domainToQueueName(domain: string): string { export function domainToQueueName(domain: string): string {
return domain.replace(/\./g, '-') + '-queue'; return domain.replace(/\./g, '-') + config.queueSuffix;
} }
/** Convert domain to S3 bucket name: bizmatch.net → bizmatch-net-emails */ /** Convert domain to S3 bucket name: bizmatch.net → bizmatch-net-emails */

View File

@@ -36,6 +36,9 @@ export class RulesProcessor {
workerName: string, workerName: string,
metricsCallback?: MetricsCallback, metricsCallback?: MetricsCallback,
): Promise<boolean> { ): Promise<boolean> {
if (config.standbyMode) {
return false;
}
const rule = await this.dynamodb.getEmailRules(recipient.toLowerCase()); const rule = await this.dynamodb.getEmailRules(recipient.toLowerCase());
if (!rule) return false; if (!rule) return false;

View File

@@ -26,7 +26,7 @@ import { BlocklistChecker } from '../email/blocklist.js';
import { BounceHandler } from '../email/bounce-handler.js'; import { BounceHandler } from '../email/bounce-handler.js';
import { parseEmail, isProcessedByWorker } from '../email/parser.js'; import { parseEmail, isProcessedByWorker } from '../email/parser.js';
import { RulesProcessor } from '../email/rules-processor.js'; import { RulesProcessor } from '../email/rules-processor.js';
import { config } from '../config.js';
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Processor // Processor
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -258,34 +258,40 @@ export class MessageProcessor {
if (totalHandled === recipients.length) { if (totalHandled === recipients.length) {
if (blockedRecipients.length === recipients.length) { if (blockedRecipients.length === recipients.length) {
// All blocked // All blocked — im Standby kein S3 anfassen
try { if (!config.standbyMode) {
await this.s3.markAsBlocked( try {
domain, await this.s3.markAsBlocked(
messageId, domain,
blockedRecipients, messageId,
fromAddrFinal, blockedRecipients,
workerName, fromAddrFinal,
); workerName,
await this.s3.deleteBlockedEmail(domain, messageId, workerName); );
} catch (err: any) { await this.s3.deleteBlockedEmail(domain, messageId, workerName);
log(`⚠ Failed to handle blocked email: ${err.message ?? err}`, 'ERROR', workerName); } catch (err: any) {
return false; log(`⚠ Failed to handle blocked email: ${err.message ?? err}`, 'ERROR', workerName);
return false;
}
} }
} else if (successful.length > 0) { } else if (successful.length > 0) {
await this.s3.markAsProcessed( if (!config.standbyMode) {
domain, await this.s3.markAsProcessed(
messageId, domain,
workerName, messageId,
failedPermanent.length > 0 ? failedPermanent : undefined, workerName,
); failedPermanent.length > 0 ? failedPermanent : undefined,
);
}
} else if (failedPermanent.length > 0) { } else if (failedPermanent.length > 0) {
await this.s3.markAsAllInvalid( if (!config.standbyMode) {
domain, await this.s3.markAsAllInvalid(
messageId, domain,
failedPermanent, messageId,
workerName, failedPermanent,
); workerName,
);
}
} }
// Summary // Summary