Compare commits
26 Commits
7aed24bfff
...
contabo
| Author | SHA1 | Date | |
|---|---|---|---|
| 4422fba707 | |||
| 649a55eecd | |||
| 1936646b20 | |||
| 528497094b | |||
| fedd2d4f34 | |||
| 119f8b1152 | |||
| 5b0a087a78 | |||
| 29a34ce4e5 | |||
| 0540b1083c | |||
| 0ecaa2ce11 | |||
| 25c26e4f79 | |||
| b2dfb76a7e | |||
| 6312d0f563 | |||
| d959b5ec86 | |||
| 4b6db1e23d | |||
| e6a81e6d8d | |||
| faa8d9cc57 | |||
| a93f44600c | |||
| ae75afc69a | |||
| 95baab8e06 | |||
| 947740232c | |||
| 081a0fad4b | |||
| 1e1265ef1b | |||
| 9862689c0c | |||
| bed6c2a398 | |||
| 27c2be664a |
58
DMS/batch_imapsync.sh
Normal file
58
DMS/batch_imapsync.sh
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# batch_imapsync.sh - Führt IMAP-Sync für alle User im Hintergrund aus
|
||||
# Format der CSV: email@domain.com,SecretPassword123
|
||||
|
||||
HOST1=$1
|
||||
HOST2=$2
|
||||
CSV_FILE=$3
|
||||
|
||||
if [ -z "$HOST1" ] || [ -z "$HOST2" ] || [ -z "$CSV_FILE" ]; then
|
||||
echo "Usage: $0 <source-host> <target-host> <users.csv>"
|
||||
echo "Beispiel: $0 secure.emailsrvr.com 147.93.132.244 stxmaterials.csv"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ======================================================================
|
||||
# Die eigentliche Sync-Funktion (wird in den Hintergrund geschickt)
|
||||
# ======================================================================
|
||||
run_sync_jobs() {
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
LOG_DIR="sync_logs_$TIMESTAMP"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
echo "Beginne Sync-Lauf am $(date)" > "batch_master_${TIMESTAMP}.log"
|
||||
|
||||
while IFS=, read -r email password; do
|
||||
email=$(echo "$email" | tr -d '\r' | xargs)
|
||||
password=$(echo "$password" | tr -d '\r' | xargs)
|
||||
|
||||
[ -z "$email" ] && continue
|
||||
|
||||
LOGFILE="$LOG_DIR/imapsync_${email}.log"
|
||||
echo "[$(date)] Syncing $email -> $LOGFILE" >> "batch_master_${TIMESTAMP}.log"
|
||||
|
||||
# Führe Docker imapsync für den aktuellen User aus
|
||||
docker run --rm gilleslamiral/imapsync imapsync \
|
||||
--host1 "$HOST1" --user1 "$email" --password1 "$password" --ssl1 \
|
||||
--host2 "$HOST2" --user2 "$email" --password2 "$password" --ssl2 \
|
||||
--automap > "$LOGFILE" 2>&1 < /dev/null
|
||||
|
||||
done < "$CSV_FILE"
|
||||
|
||||
echo "Alle Sync-Jobs beendet am $(date)" >> "batch_master_${TIMESTAMP}.log"
|
||||
}
|
||||
|
||||
# ======================================================================
|
||||
# Skript-Start: Entkopplung vom Terminal
|
||||
# ======================================================================
|
||||
echo "🚀 Starte Batch-IMAP-Sync im Hintergrund..."
|
||||
|
||||
# Rufe die Funktion auf, leite alle restlichen Ausgaben ins Nichts und schicke sie in den Hintergrund (&)
|
||||
run_sync_jobs </dev/null >/dev/null 2>&1 &
|
||||
|
||||
echo "✅ Der Job läuft jetzt autark im Hintergrund (sequenziell)."
|
||||
echo "Du kannst das SSH-Terminal jetzt bedenkenlos schließen!"
|
||||
echo "Überwache den Gesamtfortschritt mit:"
|
||||
echo " tail -f batch_master_*.log"
|
||||
echo "Oder die Details eines einzelnen Postfachs mit:"
|
||||
echo " tail -f sync_logs_*/imapsync_<email>.log"
|
||||
@@ -9,10 +9,10 @@ services:
|
||||
|
||||
# Node-spezifischer Hostname - A-Record zeigt auf DIESEN Server.
|
||||
# email-srvr.com selbst zeigt auf einen anderen Server und wird hier NICHT verwendet.
|
||||
hostname: node1.email-srvr.com
|
||||
hostname: ${NODE_HOSTNAME}
|
||||
|
||||
ports:
|
||||
- "25:25"
|
||||
- "127.0.0.1:25:25"
|
||||
- "587:587"
|
||||
- "465:465"
|
||||
- "143:143"
|
||||
@@ -61,8 +61,8 @@ services:
|
||||
# Kundendomain-SNI wird über postfix-main.cf + dovecot-sni.cf gesteuert.
|
||||
# -------------------------------------------------------
|
||||
- SSL_TYPE=manual
|
||||
- SSL_CERT_PATH=/etc/mail/certs/node1.email-srvr.com/node1.email-srvr.com.crt
|
||||
- SSL_KEY_PATH=/etc/mail/certs/node1.email-srvr.com/node1.email-srvr.com.key
|
||||
- SSL_CERT_PATH=/etc/mail/certs/${NODE_HOSTNAME}/${NODE_HOSTNAME}.crt
|
||||
- SSL_KEY_PATH=/etc/mail/certs/${NODE_HOSTNAME}/${NODE_HOSTNAME}.key
|
||||
|
||||
# SPAM / Rspamd
|
||||
- ENABLE_OPENDKIM=1
|
||||
@@ -107,7 +107,7 @@ services:
|
||||
# Postfix
|
||||
# POSTFIX_OVERRIDE_HOSTNAME: Was Postfix im EHLO/HELO Banner sendet.
|
||||
# node1.email-srvr.com passt zum TLS-Cert und ist der echte Hostname.
|
||||
- POSTFIX_OVERRIDE_HOSTNAME=node1.email-srvr.com
|
||||
- POSTFIX_OVERRIDE_HOSTNAME=${NODE_HOSTNAME}
|
||||
- POSTFIX_MYNETWORKS=172.16.0.0/12 172.17.0.0/12 172.18.0.0/12 [::1]/128 [fe80::]/64
|
||||
- POSTFIX_MAILBOX_SIZE_LIMIT=0
|
||||
- POSTFIX_MESSAGE_SIZE_LIMIT=0
|
||||
@@ -120,7 +120,7 @@ services:
|
||||
mail_network:
|
||||
aliases:
|
||||
- mailserver
|
||||
- node1.email-srvr.com
|
||||
- ${NODE_HOSTNAME}
|
||||
|
||||
roundcube:
|
||||
image: roundcube/roundcubemail:latest
|
||||
@@ -136,10 +136,10 @@ services:
|
||||
- ROUNDCUBEMAIL_DB_USER=roundcube
|
||||
- ROUNDCUBEMAIL_DB_PASSWORD=${ROUNDCUBE_DB_PASSWORD}
|
||||
# Roundcube verbindet intern über den Docker-Alias
|
||||
- ROUNDCUBEMAIL_DEFAULT_HOST=ssl://node1.email-srvr.com
|
||||
- ROUNDCUBEMAIL_DEFAULT_HOST=ssl://${NODE_HOSTNAME}
|
||||
- ROUNDCUBEMAIL_DEFAULT_PORT=993
|
||||
# Interner Traffic ohne TLS
|
||||
- ROUNDCUBEMAIL_SMTP_SERVER=ssl://node1.email-srvr.com
|
||||
- ROUNDCUBEMAIL_SMTP_SERVER=ssl://${NODE_HOSTNAME}
|
||||
- ROUNDCUBEMAIL_SMTP_PORT=465
|
||||
|
||||
# WICHTIG: Variablen LEER lassen, damit Roundcube keine Authentifizierung versucht!
|
||||
|
||||
@@ -27,7 +27,12 @@ CERTS_BASE_PATH=${CERTS_BASE_PATH:-"/etc/mail/certs"}
|
||||
|
||||
# Node-Hostname: Fallback-Cert für DMS (kein Wildcard, direktes Cert)
|
||||
# Muss mit dem 'hostname' in docker-compose.yml übereinstimmen.
|
||||
NODE_HOSTNAME=${NODE_HOSTNAME:-"node1.email-srvr.com"}
|
||||
if [ -z "$NODE_HOSTNAME" ]; then
|
||||
echo "❌ NODE_HOSTNAME ist nicht gesetzt!"
|
||||
echo "Beispiel:"
|
||||
echo " DMS_CONTAINER=mailserver NODE_HOSTNAME=node2.email-srvr.com ./setup-dms-tls.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "============================================================"
|
||||
echo " 🔐 DMS TLS SNI Setup (Multi-Domain)"
|
||||
|
||||
@@ -34,7 +34,7 @@ aws s3api put-public-access-block \
|
||||
--public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true"
|
||||
|
||||
# Lebenszyklus-Konfiguration hinzufügen
|
||||
echo "Lebenszyklus-Konfiguration hinzufügen (E-Mails werden nach 90 Tagen gelöscht)..."
|
||||
echo "Lebenszyklus-Konfiguration hinzufügen (E-Mails werden nach 7 Tagen gelöscht)..."
|
||||
aws s3api put-bucket-lifecycle-configuration \
|
||||
--bucket ${S3_BUCKET_NAME} \
|
||||
--lifecycle-configuration '{
|
||||
@@ -43,7 +43,7 @@ aws s3api put-bucket-lifecycle-configuration \
|
||||
"ID": "DeleteOldEmails",
|
||||
"Status": "Enabled",
|
||||
"Expiration": {
|
||||
"Days": 14
|
||||
"Days": 7
|
||||
},
|
||||
"Filter": {
|
||||
"Prefix": ""
|
||||
@@ -76,6 +76,14 @@ aws s3api put-bucket-policy \
|
||||
]
|
||||
}'
|
||||
|
||||
# ------------------------
|
||||
# Cost Allocation Tags setzen
|
||||
# ------------------------
|
||||
echo "Setze Cost Allocation Tag (BucketName)..."
|
||||
aws s3api put-bucket-tagging \
|
||||
--bucket ${S3_BUCKET_NAME} \
|
||||
--tagging "TagSet=[{Key=BucketName,Value=${S3_BUCKET_NAME}}]"
|
||||
|
||||
echo "S3 Bucket $S3_BUCKET_NAME wurde erfolgreich erstellt und konfiguriert."
|
||||
echo "Bucket-ARN: arn:aws:s3:::$S3_BUCKET_NAME"
|
||||
|
||||
|
||||
@@ -7,9 +7,14 @@
|
||||
# Standard: mail.${DOMAIN_NAME}
|
||||
# Override: export MAIL_FROM_SUBDOMAIN="mailfrom" (nur der Prefix, ohne Domain)
|
||||
#
|
||||
# Lambda Deployment:
|
||||
# Standard: Überspringt den Deploy (SKIP_LAMBDA_DEPLOY=true)
|
||||
# Override: export SKIP_LAMBDA_DEPLOY="false" (um Lambda neu auszurollen)
|
||||
#
|
||||
# Beispiel:
|
||||
# export DOMAIN_NAME="buddelectric.net"
|
||||
# export MAIL_FROM_SUBDOMAIN="mailfrom" # → mailfrom.buddelectric.net
|
||||
# export SKIP_LAMBDA_DEPLOY="false" # → Lambda wird aktualisiert
|
||||
# ./awsses.sh
|
||||
|
||||
set -e
|
||||
@@ -18,17 +23,20 @@ set -e
|
||||
if ! command -v jq &> /dev/null; then echo "Fehler: 'jq' fehlt."; exit 1; fi
|
||||
if [ -z "$DOMAIN_NAME" ]; then echo "Fehler: DOMAIN_NAME ist nicht gesetzt."; exit 1; fi
|
||||
|
||||
# Prüfen ob Python Code da ist
|
||||
PYTHON_FILE="ses_sns_shim_global.py"
|
||||
if [ ! -f "$PYTHON_FILE" ]; then
|
||||
echo "Fehler: $PYTHON_FILE nicht gefunden!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- VARIABLEN ---
|
||||
AWS_REGION=${AWS_REGION:-"us-east-2"}
|
||||
EMAIL_PREFIX=${EMAIL_PREFIX:-""}
|
||||
CONFIGURATION_SET_NAME="relay-outbound"
|
||||
SKIP_LAMBDA_DEPLOY=${SKIP_LAMBDA_DEPLOY:-"true"}
|
||||
|
||||
# Prüfen ob Python Code da ist (nur wenn auch deployt werden soll)
|
||||
PYTHON_FILE="ses_sns_shim_global.py"
|
||||
if [ "$SKIP_LAMBDA_DEPLOY" != "true" ]; then
|
||||
if [ ! -f "$PYTHON_FILE" ]; then
|
||||
echo "Fehler: $PYTHON_FILE nicht gefunden!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# MAIL FROM Subdomain (konfigurierbar)
|
||||
MAIL_FROM_SUBDOMAIN=${MAIL_FROM_SUBDOMAIN:-"mail"}
|
||||
@@ -49,6 +57,7 @@ LAMBDA_ROLE_NAME="SesShimGlobalRole"
|
||||
echo "=========================================================="
|
||||
echo " SES Setup (S3 -> Global Lambda Shim -> SQS) für $DOMAIN_NAME"
|
||||
echo " MAIL FROM: $MAIL_FROM_DOMAIN"
|
||||
echo " SKIP_LAMBDA: $SKIP_LAMBDA_DEPLOY"
|
||||
echo "=========================================================="
|
||||
|
||||
# ---------------------------------------------------------
|
||||
@@ -113,8 +122,12 @@ echo " -> Permissions aktualisiert."
|
||||
sleep 5
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# 4. Lambda Funktion erstellen/updaten (Global!)
|
||||
# 4 & 5. Lambda Funktion + SES Permissions (optional)
|
||||
# ---------------------------------------------------------
|
||||
if [ "$SKIP_LAMBDA_DEPLOY" = "true" ]; then
|
||||
echo "[4/6] Global Lambda Shim deployen... (ÜBERSPRUNGEN)"
|
||||
echo "[5/6] SES Permission für Lambda... (ÜBERSPRUNGEN)"
|
||||
else
|
||||
echo "[4/6] Global Lambda Shim deployen..."
|
||||
# Zip erstellen
|
||||
cp "$PYTHON_FILE" lambda_function.py
|
||||
@@ -139,15 +152,13 @@ fi
|
||||
# Aufräumen
|
||||
rm lambda.zip lambda_function.py
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# 5. Permission: SES darf Lambda aufrufen (Global, einmalig)
|
||||
# ---------------------------------------------------------
|
||||
echo "[5/6] SES Permission für Lambda..."
|
||||
aws lambda add-permission --function-name "$LAMBDA_NAME" \
|
||||
--statement-id "AllowSESInvoke-Global" \
|
||||
--action "lambda:InvokeFunction" \
|
||||
--principal "ses.amazonaws.com" \
|
||||
--region "$AWS_REGION" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# 6. SES Rule (S3 + Global Lambda)
|
||||
|
||||
90
basic_setup/check_logins.py
Executable file
90
basic_setup/check_logins.py
Executable file
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
import csv
|
||||
import imaplib
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Konfiguration
|
||||
IMAP_SERVER = "secure.emailsrvr.com"
|
||||
IMAP_PORT = 993
|
||||
DELAY_SECONDS = 1 # Kurze Pause, um Rate-Limiting oder Fail2Ban zu vermeiden
|
||||
|
||||
def check_imap_login(email, password):
|
||||
try:
|
||||
# Verbindung zum IMAP-Server via SSL herstellen
|
||||
mail = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
|
||||
mail.login(email, password)
|
||||
mail.logout()
|
||||
return True
|
||||
except imaplib.IMAP4.error:
|
||||
# Login fehlgeschlagen (falsches Passwort/User)
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" [!] Netzwerk- oder Serverfehler bei {email}: {e}")
|
||||
return False
|
||||
|
||||
def main(csv_filepath):
|
||||
erfolgreich = []
|
||||
fehlgeschlagen = []
|
||||
uebersprungen = []
|
||||
|
||||
try:
|
||||
with open(csv_filepath, mode='r', encoding='utf-8') as f:
|
||||
# Nutze csv.reader für sauberes Parsing der Kommas
|
||||
reader = csv.reader(f)
|
||||
for row_num, row in enumerate(reader, start=1):
|
||||
if not row:
|
||||
continue
|
||||
|
||||
email = row[0].strip()
|
||||
# Prüfen, ob ein zweites Feld (Passwort) existiert und nicht leer ist
|
||||
password = row[1].strip() if len(row) > 1 else ""
|
||||
|
||||
if not password:
|
||||
print(f"[{row_num}] Überspringe {email} (Kein Passwort)")
|
||||
uebersprungen.append(email)
|
||||
continue
|
||||
|
||||
print(f"[{row_num}] Prüfe {email}... ", end="", flush=True)
|
||||
|
||||
if check_imap_login(email, password):
|
||||
print("OK")
|
||||
erfolgreich.append(email)
|
||||
else:
|
||||
print("FEHLGESCHLAGEN")
|
||||
fehlgeschlagen.append(email)
|
||||
|
||||
# Kurze Pause einlegen, um den Mailserver nicht zu fluten
|
||||
time.sleep(DELAY_SECONDS)
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"\nFehler: Die Datei '{csv_filepath}' wurde nicht gefunden.")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\nEin unerwarteter Fehler ist aufgetreten: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Ausgabe der Zusammenfassung
|
||||
print("\n" + "="*40)
|
||||
print("ZUSAMMENFASSUNG DER PRÜFUNG")
|
||||
print("="*40)
|
||||
|
||||
print(f"\nErfolgreich ({len(erfolgreich)}):")
|
||||
for e in erfolgreich:
|
||||
print(f" - {e}")
|
||||
|
||||
print(f"\nFehlgeschlagen ({len(fehlgeschlagen)}):")
|
||||
for e in fehlgeschlagen:
|
||||
print(f" - {e}")
|
||||
|
||||
print(f"\nÜbersprungen (kein Passwort) ({len(uebersprungen)}):")
|
||||
for e in uebersprungen:
|
||||
print(f" - {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print("Verwendung: ./check_logins.py <pfad_zur_datei.csv>")
|
||||
sys.exit(1)
|
||||
|
||||
csv_file = sys.argv[1]
|
||||
main(csv_file)
|
||||
@@ -5,15 +5,22 @@
|
||||
# Setzt mail/imap/smtp/pop Subdomains für domain-spezifischen Mailserver-Zugang
|
||||
#
|
||||
# MIGRATIONS-FLAGS:
|
||||
# SKIP_CLIENT_DNS=true → Abschnitt 8 (imap/smtp/pop/webmail) + 10 (SRV) überspringen
|
||||
# Nutzen: Client-Subdomains bleiben beim alten Provider
|
||||
# SKIP_CLIENT_DNS=true → Abschnitt 8 (mail/imap/smtp/pop/webmail) überspringen
|
||||
# Nutzen: Client-Subdomains bleiben komplett beim alten Provider
|
||||
# SKIP_PROTOCOL_DNS=true → nur imap/smtp/pop überspringen, aber mail + webmail setzen
|
||||
# Nutzen: Webmail/Autodiscover vorbereiten, Mailclients bleiben beim alten Provider
|
||||
# SKIP_DMARC=true → Abschnitt 7 (DMARC) überspringen
|
||||
# Nutzen: Bestehenden DMARC-Record nicht anfassen
|
||||
#
|
||||
# Typischer Migrations-Ablauf:
|
||||
# Phase 0 (Vorbereitung): SKIP_CLIENT_DNS=true SKIP_DMARC=true → nur SES + SPF
|
||||
# Phase 1 (MX Cutover): MX umstellen (manuell)
|
||||
# Phase 2 (Client Switch): ohne SKIP Flags → alle Records setzen
|
||||
# Phase 0a (Vorbereitung, Client-Records bleiben alt):
|
||||
# SKIP_CLIENT_DNS=true SKIP_DMARC=true → nur SES + SPF/DKIM/MailFrom
|
||||
# Phase 0b (Webmail vorbereiten, imap/smtp/pop bleiben alt):
|
||||
# SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=true SKIP_DMARC=false
|
||||
# Phase 1 (MX Cutover):
|
||||
# MX manuell umstellen
|
||||
# Phase 2 (Client Switch):
|
||||
# SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=false → alle Client-Records setzen
|
||||
|
||||
set -e
|
||||
|
||||
@@ -23,6 +30,7 @@ DRY_RUN=${DRY_RUN:-"false"}
|
||||
|
||||
# Migrations-Flags (NEU)
|
||||
SKIP_CLIENT_DNS=${SKIP_CLIENT_DNS:-"false"}
|
||||
SKIP_PROTOCOL_DNS=${SKIP_PROTOCOL_DNS:-"false"}
|
||||
SKIP_DMARC=${SKIP_DMARC:-"false"}
|
||||
|
||||
# IP des Mailservers - PFLICHT wenn keine CNAME-Kette gewünscht
|
||||
@@ -50,7 +58,8 @@ echo " 🌍 Region: $AWS_REGION"
|
||||
echo " 📬 Mail-Server Target: $TARGET_MAIL_SERVER"
|
||||
[ -n "$MAIL_SERVER_IP" ] && echo " 🖥️ Server IP: $MAIL_SERVER_IP"
|
||||
[ "$DRY_RUN" = "true" ] && echo " ⚠️ DRY RUN MODE - Keine Änderungen!"
|
||||
[ "$SKIP_CLIENT_DNS" = "true" ] && echo " ⏭️ SKIP: Client-Subdomains (imap/smtp/pop/webmail/SRV)"
|
||||
[ "$SKIP_CLIENT_DNS" = "true" ] && echo " ⏭️ SKIP: alle Client-Subdomains (mail/imap/smtp/pop/webmail)"
|
||||
[ "$SKIP_PROTOCOL_DNS" = "true" ] && echo " ⏭️ SKIP: Protokoll-Subdomains imap/smtp/pop bleiben unverändert"
|
||||
[ "$SKIP_DMARC" = "true" ] && echo " ⏭️ SKIP: DMARC Record"
|
||||
echo "============================================================"
|
||||
|
||||
@@ -255,14 +264,23 @@ else
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SCHRITT 6: Root Domain MX (nur Info, wird nicht geändert)
|
||||
# SCHRITT 6: Root Domain MX
|
||||
# ------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- 6. Root Domain MX (nur Info, wird nicht geändert) ---"
|
||||
CURRENT_MX=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?type=MX&name=$DOMAIN_NAME" \
|
||||
echo "--- 6. Root Domain MX ---"
|
||||
# Prüfen, ob bereits MX-Records für die Root-Domain existieren
|
||||
MX_COUNT=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?type=MX&name=$DOMAIN_NAME" \
|
||||
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" \
|
||||
| jq -r '.result[0].content // "keiner"')
|
||||
echo " ℹ️ MX vorhanden: $CURRENT_MX (wird nicht geändert)"
|
||||
| jq -r '.result | length')
|
||||
|
||||
if [ "$MX_COUNT" -eq 0 ]; then
|
||||
echo " ℹ️ Kein MX-Record vorhanden. Setze initialen SES Inbound MX..."
|
||||
ensure_record "MX" "$DOMAIN_NAME" "inbound-smtp.${AWS_REGION}.amazonaws.com" false 10
|
||||
else
|
||||
echo " ⛔ MX-Record(s) für @ bereits vorhanden ($MX_COUNT Eintrag/Einträge)."
|
||||
echo " → Wird zum Schutz der alten Postfächer AUF KEINEN FALL automatisch geändert!"
|
||||
echo " → Muss nach der Migration manuell umgestellt werden."
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SCHRITT 7: DMARC
|
||||
@@ -283,11 +301,12 @@ echo ""
|
||||
echo "--- 8. Mailclient Subdomains (A + CNAME) ---"
|
||||
if [ "$SKIP_CLIENT_DNS" = "true" ]; then
|
||||
echo " ⏭️ Übersprungen (SKIP_CLIENT_DNS=true)"
|
||||
echo " ℹ️ imap/smtp/pop/webmail bleiben beim alten Provider."
|
||||
echo " ℹ️ mail/imap/smtp/pop/webmail bleiben beim alten Provider."
|
||||
echo " ℹ️ Setze SKIP_CLIENT_DNS=false nach MX-Cutover + Client-Umstellung."
|
||||
else
|
||||
if [ -n "$MAIL_SERVER_IP" ]; then
|
||||
# A-Record für mail.<domain> direkt auf Server-IP
|
||||
# Wichtig: mail muss DNS-only bleiben; Cloudflare Proxy funktioniert nicht für SMTP/IMAP/POP.
|
||||
ensure_record "A" "mail.$DOMAIN_NAME" "$MAIL_SERVER_IP" false
|
||||
else
|
||||
# CNAME auf externen Ziel-Host (nur wenn verschieden)
|
||||
@@ -296,11 +315,19 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
# imap, smtp, pop, webmail → CNAME auf mail.<domain>
|
||||
# Webmail kann bereits vorbereitet werden, auch wenn imap/smtp/pop noch beim alten Provider bleiben.
|
||||
ensure_record "CNAME" "webmail.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
|
||||
|
||||
if [ "$SKIP_PROTOCOL_DNS" = "true" ]; then
|
||||
echo " ⏭️ Überspringe imap/smtp/pop (SKIP_PROTOCOL_DNS=true)"
|
||||
echo " ℹ️ imap/smtp/pop bleiben unverändert beim alten Provider."
|
||||
else
|
||||
# imap/smtp/pop → CNAME auf mail.<domain>
|
||||
# Wichtig: diese Records müssen DNS-only bleiben; Cloudflare Proxy funktioniert nicht für Mail-Protokolle.
|
||||
ensure_record "CNAME" "imap.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
|
||||
ensure_record "CNAME" "smtp.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
|
||||
ensure_record "CNAME" "pop.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
|
||||
ensure_record "CNAME" "webmail.$DOMAIN_NAME" "mail.$DOMAIN_NAME" false
|
||||
fi
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -318,7 +345,12 @@ if [ "$SKIP_CLIENT_DNS" = "true" ]; then
|
||||
echo ""
|
||||
echo " ⚠️ Client-Subdomains wurden NICHT geändert."
|
||||
echo " Nach MX-Cutover + Worker-Validierung erneut ausführen mit:"
|
||||
echo " SKIP_CLIENT_DNS=false SKIP_DMARC=false ./cloudflareMigrationDns.sh"
|
||||
echo " SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=false SKIP_DMARC=false ./cloudflareMigrationDns.sh"
|
||||
elif [ "$SKIP_PROTOCOL_DNS" = "true" ]; then
|
||||
echo ""
|
||||
echo " ⚠️ imap/smtp/pop wurden NICHT geändert."
|
||||
echo " Für den finalen Client-Cutover erneut ausführen mit:"
|
||||
echo " SKIP_CLIENT_DNS=false SKIP_PROTOCOL_DNS=false SKIP_DMARC=false ./cloudflareMigrationDns.sh"
|
||||
fi
|
||||
echo ""
|
||||
echo " Mailclient-Konfiguration für Kunden:"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# create-queue.sh (v2 — mit SNS Fan-Out + Standby Queue)
|
||||
# Usage: DOMAIN=andreasknuth.de ./create-queue.sh
|
||||
# Usage: DOMAIN_NAME=andreasknuth.de ./create-queue.sh
|
||||
#
|
||||
# Erstellt pro Domain:
|
||||
# - Primary Queue + DLQ (wie bisher, für Contabo)
|
||||
@@ -12,13 +12,13 @@ set -e
|
||||
|
||||
AWS_REGION="us-east-2"
|
||||
|
||||
if [ -z "$DOMAIN" ]; then
|
||||
echo "Error: DOMAIN environment variable not set"
|
||||
echo "Usage: DOMAIN=andreasknuth.de $0"
|
||||
if [ -z "$DOMAIN_NAME" ]; then
|
||||
echo "Error: DOMAIN_NAME environment variable not set"
|
||||
echo "Usage: DOMAIN_NAME=andreasknuth.de $0"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DOMAIN_SLUG="${DOMAIN//./-}"
|
||||
DOMAIN_SLUG="${DOMAIN_NAME//./-}"
|
||||
QUEUE_NAME="${DOMAIN_SLUG}-queue"
|
||||
DLQ_NAME="${QUEUE_NAME}-dlq"
|
||||
STANDBY_QUEUE_NAME="${DOMAIN_SLUG}-standby-queue"
|
||||
@@ -31,7 +31,7 @@ echo "========================================"
|
||||
echo "Creating SQS + SNS for Email Delivery"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "📧 Domain: $DOMAIN"
|
||||
echo "📧 Domain: $DOMAIN_NAME"
|
||||
echo " Region: $AWS_REGION"
|
||||
echo " Account: $ACCOUNT_ID"
|
||||
echo ""
|
||||
@@ -184,7 +184,7 @@ echo ""
|
||||
# Zusammenfassung
|
||||
# ============================================================
|
||||
echo "========================================"
|
||||
echo "✅ Setup complete for $DOMAIN"
|
||||
echo "✅ Setup complete for $DOMAIN_NAME"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
echo "Primary (Contabo):"
|
||||
|
||||
32
basic_setup/legacy/s3-retention.sh
Executable file
32
basic_setup/legacy/s3-retention.sh
Executable file
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
echo "Aktualisiere Lifecycle-Regeln (7 Tage) für alle E-Mail-Buckets..."
|
||||
|
||||
for BUCKET in $(aws s3api list-buckets --query 'Buckets[].Name' --output text); do
|
||||
|
||||
# Prüfen, ob der Name auf '-emails' endet
|
||||
if [[ "$BUCKET" == *-emails ]]; then
|
||||
echo "⚙️ Setze 7-Tage-Regel für: $BUCKET"
|
||||
|
||||
aws s3api put-bucket-lifecycle-configuration \
|
||||
--bucket "$BUCKET" \
|
||||
--lifecycle-configuration '{
|
||||
"Rules": [
|
||||
{
|
||||
"ID": "DeleteOldEmails",
|
||||
"Status": "Enabled",
|
||||
"Expiration": {
|
||||
"Days": 7
|
||||
},
|
||||
"Filter": {
|
||||
"Prefix": ""
|
||||
}
|
||||
}
|
||||
]
|
||||
}'
|
||||
else
|
||||
echo "⏭️ Überspringe (kein E-Mail-Bucket): $BUCKET"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Fertig! Alle E-Mail-Buckets löschen jetzt Objekte nach 7 Tagen."
|
||||
31
basic_setup/legacy/sync-s3-tags.sh
Executable file
31
basic_setup/legacy/sync-s3-tags.sh
Executable file
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# sync-s3-tags.sh - Synchronisiert Bucket Tags welche fuer die Abrechung verwendet werden
|
||||
echo "Passe Bucket-Tags an..."
|
||||
|
||||
for BUCKET in $(aws s3api list-buckets --query 'Buckets[].Name' --output text); do
|
||||
# Aktuellen Tag abfragen (Fehler unterdrücken)
|
||||
TAG=$(aws s3api get-bucket-tagging --bucket "$BUCKET" --query 'TagSet[?Key==`BucketName`].Value' --output text 2>/dev/null)
|
||||
|
||||
# Prüfen, ob der Name auf '-emails' endet
|
||||
if [[ "$BUCKET" == *-emails ]]; then
|
||||
# Soll getaggt sein
|
||||
if [ -z "$TAG" ] || [ "$TAG" == "None" ]; then
|
||||
echo "➕ Setze fehlendes Tag für: $BUCKET"
|
||||
aws s3api put-bucket-tagging \
|
||||
--bucket "$BUCKET" \
|
||||
--tagging "TagSet=[{Key=BucketName,Value=$BUCKET}]"
|
||||
else
|
||||
echo "✅ OK (bereits getaggt): $BUCKET"
|
||||
fi
|
||||
else
|
||||
# Soll NICHT getaggt sein
|
||||
if [ -n "$TAG" ] && [ "$TAG" != "None" ]; then
|
||||
echo "🗑️ Entferne Tag von: $BUCKET"
|
||||
aws s3api delete-bucket-tagging --bucket "$BUCKET"
|
||||
else
|
||||
echo "✅ OK (ohne Tag): $BUCKET"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Fertig! Alle '-emails' Buckets sind getaggt, bei allen anderen wurden die Tags entfernt."
|
||||
158
basic_setup/mailadminDns.sh
Executable file
158
basic_setup/mailadminDns.sh
Executable file
@@ -0,0 +1,158 @@
|
||||
#!/bin/bash
|
||||
# mailadminDns.sh
|
||||
# ------------------------------------------------------------------
|
||||
# Setzt AUSSCHLIESSLICH den DNS-Record fuer den mailadmin-Zugang.
|
||||
# Faesst NICHTS anderes an (kein SES, SPF, DKIM, MX, keine anderen
|
||||
# Subdomains). Kann gefahrlos auf bereits migrierten Domains laufen.
|
||||
#
|
||||
# Verwendet dieselbe ensure_record-Logik wie cloudflareMigrationDns.sh,
|
||||
# damit das Verhalten (Create/Update/Skip-bei-identisch) identisch ist.
|
||||
#
|
||||
# ------------------------------------------------------------------
|
||||
# PFLICHT-VARIABLEN:
|
||||
# DOMAIN_NAME Kundendomain, z.B. innungsapp.com
|
||||
# CF_API_TOKEN Cloudflare API Token
|
||||
#
|
||||
# OPTIONALE VARIABLEN:
|
||||
# MAILADMIN_HOST Subdomain-Label, Default: "mailadmin"
|
||||
# -> ergibt mailadmin.<DOMAIN_NAME>
|
||||
# MAILADMIN_TARGET CNAME-Ziel, Default: "mail.<DOMAIN_NAME>"
|
||||
# Beispiele:
|
||||
# mail.innungsapp.com (pro-Domain Modell)
|
||||
# mailadmin.bayarea-cc.com (zentrales Modell)
|
||||
# DRY_RUN "true" -> zeigt nur an, aendert nichts
|
||||
#
|
||||
# ------------------------------------------------------------------
|
||||
# BEISPIELE:
|
||||
#
|
||||
# # Standard: mailadmin.innungsapp.com -> mail.innungsapp.com
|
||||
# DOMAIN_NAME=innungsapp.com CF_API_TOKEN=xxx ./mailadminDns.sh
|
||||
#
|
||||
# # Zentrales Modell: mailadmin.innungsapp.com -> mailadmin.bayarea-cc.com
|
||||
# DOMAIN_NAME=innungsapp.com \
|
||||
# MAILADMIN_TARGET=mailadmin.bayarea-cc.com \
|
||||
# CF_API_TOKEN=xxx ./mailadminDns.sh
|
||||
#
|
||||
# # Erst testen ohne zu aendern
|
||||
# DOMAIN_NAME=innungsapp.com CF_API_TOKEN=xxx DRY_RUN=true ./mailadminDns.sh
|
||||
#
|
||||
# ------------------------------------------------------------------
|
||||
set -e
|
||||
|
||||
# --- KONFIGURATION ---
|
||||
DRY_RUN=${DRY_RUN:-"false"}
|
||||
MAILADMIN_HOST=${MAILADMIN_HOST:-"mailadmin"}
|
||||
MAILADMIN_TARGET=${MAILADMIN_TARGET:-"mail.${DOMAIN_NAME}"}
|
||||
|
||||
# --- CHECKS ---
|
||||
if [ -z "$DOMAIN_NAME" ]; then echo "❌ Fehler: DOMAIN_NAME fehlt."; exit 1; fi
|
||||
if [ -z "$CF_API_TOKEN" ]; then echo "❌ Fehler: CF_API_TOKEN fehlt."; exit 1; fi
|
||||
if ! command -v jq &> /dev/null; then echo "❌ Fehler: 'jq' fehlt."; exit 1; fi
|
||||
if ! command -v curl &> /dev/null; then echo "❌ Fehler: 'curl' fehlt."; exit 1; fi
|
||||
|
||||
RECORD_NAME="${MAILADMIN_HOST}.${DOMAIN_NAME}"
|
||||
|
||||
echo "============================================================"
|
||||
echo " 🔧 mailadmin DNS Setup"
|
||||
echo " 🌐 Domain: $DOMAIN_NAME"
|
||||
echo " 📍 Record: CNAME $RECORD_NAME → $MAILADMIN_TARGET"
|
||||
[ "$DRY_RUN" = "true" ] && echo " ⚠️ DRY RUN MODE - Keine Änderungen!"
|
||||
echo "============================================================"
|
||||
|
||||
# Schutz: CNAME auf sich selbst macht keinen Sinn
|
||||
if [ "$RECORD_NAME" == "$MAILADMIN_TARGET" ]; then
|
||||
echo "❌ Fehler: CNAME-Ziel ist identisch mit dem Record-Namen."
|
||||
echo " $RECORD_NAME kann nicht auf sich selbst zeigen."
|
||||
echo " Setze MAILADMIN_TARGET auf einen anderen Host."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 1. ZONE ID HOLEN
|
||||
echo "🔍 Suche Cloudflare Zone ID..."
|
||||
ZONE_ID=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN_NAME" \
|
||||
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" | jq -r '.result[0].id')
|
||||
|
||||
if [ "$ZONE_ID" == "null" ] || [ -z "$ZONE_ID" ]; then
|
||||
echo "❌ Zone nicht gefunden für $DOMAIN_NAME."
|
||||
exit 1
|
||||
fi
|
||||
echo " ✅ Zone ID: $ZONE_ID"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# FUNKTION: ensure_record
|
||||
# Identisch zur Logik in cloudflareMigrationDns.sh, reduziert auf
|
||||
# die Typen die wir hier brauchen (CNAME).
|
||||
# ------------------------------------------------------------------
|
||||
ensure_record() {
|
||||
local type=$1
|
||||
local name=$2
|
||||
local content=$3
|
||||
local proxied=${4:-false}
|
||||
|
||||
echo " ⚙️ Prüfe $type $name..."
|
||||
|
||||
local search_res=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records?type=$type&name=$name" \
|
||||
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json")
|
||||
|
||||
local rec_id=$(echo "$search_res" | jq -r '.result[0].id')
|
||||
local rec_content=$(echo "$search_res" | jq -r '.result[0].content')
|
||||
|
||||
[ -z "$rec_id" ] && rec_id="null"
|
||||
[ -z "$rec_content" ] && rec_content="null"
|
||||
|
||||
local json_data=$(jq -n --arg t "$type" --arg n "$name" --arg c "$content" --argjson p "$proxied" \
|
||||
'{type: $t, name: $n, content: $c, ttl: 3600, proxied: $p}')
|
||||
|
||||
if [ "$rec_id" == "null" ]; then
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
echo " [DRY] Würde ERSTELLEN: $type $name → $content"
|
||||
else
|
||||
local res=$(curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" \
|
||||
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" --data "$json_data")
|
||||
if [ "$(echo $res | jq -r .success)" == "true" ]; then
|
||||
echo " ✅ Erstellt: $type $name → $content"
|
||||
else
|
||||
echo " ❌ Fehler beim Erstellen: $(echo $res | jq -r '.errors[0].message')"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "$rec_content" == "$content" ]; then
|
||||
echo " 🆗 Identisch ($rec_content). Überspringe."
|
||||
else
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
echo " [DRY] Würde UPDATEN: '$rec_content' → '$content'"
|
||||
else
|
||||
local res=$(curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$rec_id" \
|
||||
-H "Authorization: Bearer $CF_API_TOKEN" -H "Content-Type: application/json" --data "$json_data")
|
||||
if [ "$(echo $res | jq -r .success)" == "true" ]; then
|
||||
echo " ✅ Aktualisiert: '$rec_content' → '$content'"
|
||||
else
|
||||
echo " ❌ Fehler beim Updaten: $(echo $res | jq -r '.errors[0].message')"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Der eigentliche Record
|
||||
# proxied=false ist hier wichtig: der mailadmin laeuft hinter Caddy
|
||||
# mit eigenem TLS-Zertifikat. Cloudflare-Proxy davor wuerde die
|
||||
# Zertifikatskette stoeren bzw. doppeltes TLS-Terminieren.
|
||||
# ------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "--- mailadmin CNAME ---"
|
||||
ensure_record "CNAME" "$RECORD_NAME" "$MAILADMIN_TARGET" false
|
||||
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo "✅ Fertig."
|
||||
echo ""
|
||||
echo " mailadmin erreichbar unter: https://$RECORD_NAME"
|
||||
echo ""
|
||||
echo " ⚠️ Wichtig: Caddy muss ein Zertifikat für $RECORD_NAME"
|
||||
echo " ausstellen. Stelle sicher, dass der Hostname in der"
|
||||
echo " Caddy-Konfiguration enthalten ist."
|
||||
echo "============================================================"
|
||||
@@ -8,24 +8,38 @@ from botocore.exceptions import ClientError
|
||||
import time
|
||||
import random
|
||||
|
||||
# Logging konfigurieren
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
sqs = boto3.client('sqs')
|
||||
sns = boto3.client('sns')
|
||||
sts_account_id = None
|
||||
|
||||
# Retry-Konfiguration
|
||||
MAX_RETRIES = 3
|
||||
BASE_BACKOFF = 1 # Sekunden
|
||||
BASE_BACKOFF = 1
|
||||
|
||||
def exponential_backoff(attempt):
|
||||
"""Exponential Backoff mit Jitter"""
|
||||
return BASE_BACKOFF * (2 ** attempt) + random.uniform(0, 1)
|
||||
|
||||
def get_account_id():
|
||||
global sts_account_id
|
||||
if sts_account_id is None:
|
||||
sts_account_id = boto3.client('sts').get_caller_identity()['Account']
|
||||
return sts_account_id
|
||||
|
||||
def get_topic_arn(domain):
|
||||
"""
|
||||
Generiert Topic-ARN aus Domain.
|
||||
Konvention: domain.tld -> domain-tld-topic
|
||||
"""
|
||||
topic_name = domain.replace('.', '-') + '-topic'
|
||||
region = os.environ.get('AWS_REGION', 'us-east-2')
|
||||
account_id = get_account_id()
|
||||
return f"arn:aws:sns:{region}:{account_id}:{topic_name}"
|
||||
|
||||
def get_queue_url(domain):
|
||||
"""
|
||||
Generiert Queue-Namen aus Domain und holt URL.
|
||||
Konvention: domain.tld -> domain-tld-queue
|
||||
Fallback: Direkter SQS-Send für Domains ohne SNS-Topic.
|
||||
"""
|
||||
queue_name = domain.replace('.', '-') + '-queue'
|
||||
try:
|
||||
@@ -38,11 +52,53 @@ def get_queue_url(domain):
|
||||
else:
|
||||
raise
|
||||
|
||||
def publish_to_sns(topic_arn, message_body, msg_id):
|
||||
attempt = 0
|
||||
while attempt < MAX_RETRIES:
|
||||
try:
|
||||
sns.publish(
|
||||
TopicArn=topic_arn,
|
||||
Message=message_body
|
||||
)
|
||||
logger.info(f"✅ Published {msg_id} to SNS: {topic_arn}")
|
||||
return True
|
||||
except ClientError as e:
|
||||
error_code = e.response['Error']['Code']
|
||||
# Fallback auf SQS bei Topic-nicht-gefunden ODER fehlender Berechtigung
|
||||
if error_code in ('NotFound', 'NotFoundException', 'AuthorizationError'):
|
||||
logger.info(f"ℹ️ SNS unavailable for {topic_arn} ({error_code}) — falling back to SQS")
|
||||
return False
|
||||
attempt += 1
|
||||
logger.warning(f"Retry {attempt}/{MAX_RETRIES} SNS: {error_code}")
|
||||
if attempt == MAX_RETRIES:
|
||||
raise
|
||||
time.sleep(exponential_backoff(attempt))
|
||||
return False
|
||||
|
||||
def send_to_sqs(queue_url, message_body, msg_id):
|
||||
"""Fallback: Direkter SQS-Send (wie bisher)."""
|
||||
attempt = 0
|
||||
while attempt < MAX_RETRIES:
|
||||
try:
|
||||
sqs.send_message(
|
||||
QueueUrl=queue_url,
|
||||
MessageBody=message_body
|
||||
)
|
||||
logger.info(f"✅ Sent {msg_id} to SQS: {queue_url}")
|
||||
return
|
||||
except ClientError as e:
|
||||
attempt += 1
|
||||
error_code = e.response['Error']['Code']
|
||||
logger.warning(f"Retry {attempt}/{MAX_RETRIES} SQS: {error_code}")
|
||||
if attempt == MAX_RETRIES:
|
||||
raise
|
||||
time.sleep(exponential_backoff(attempt))
|
||||
|
||||
def lambda_handler(event, context):
|
||||
"""
|
||||
Nimmt SES Event entgegen, extrahiert Domain dynamisch,
|
||||
verpackt Metadaten als 'Fake SNS' und sendet an die domain-spezifische SQS.
|
||||
Mit integrierter Retry-Logik für SQS-Send.
|
||||
Nimmt SES Event entgegen, extrahiert Domain dynamisch.
|
||||
Strategie: SNS Publish (Fan-Out an Primary + Standby Queue).
|
||||
Fallback: Direkter SQS-Send falls kein SNS-Topic existiert.
|
||||
"""
|
||||
try:
|
||||
records = event.get('Records', [])
|
||||
@@ -51,13 +107,12 @@ def lambda_handler(event, context):
|
||||
for record in records:
|
||||
ses_data = record.get('ses', {})
|
||||
if not ses_data:
|
||||
logger.warning(f"Invalid SES event: Missing 'ses' in record: {record}")
|
||||
logger.warning(f"Invalid SES event: Missing 'ses' in record")
|
||||
continue
|
||||
|
||||
mail = ses_data.get('mail', {})
|
||||
receipt = ses_data.get('receipt', {})
|
||||
|
||||
# Domain extrahieren (aus erstem Recipient)
|
||||
recipients = receipt.get('recipients', []) or mail.get('destination', [])
|
||||
if not recipients:
|
||||
logger.warning("No recipients in event - skipping")
|
||||
@@ -69,23 +124,19 @@ def lambda_handler(event, context):
|
||||
logger.error("Could not extract domain from recipient")
|
||||
continue
|
||||
|
||||
# Wichtige Metadaten loggen
|
||||
msg_id = mail.get('messageId', 'unknown')
|
||||
source = mail.get('source', 'unknown')
|
||||
logger.info(f"Processing Message-ID: {msg_id} for domain: {domain}")
|
||||
logger.info(f" From: {source}")
|
||||
logger.info(f" To: {recipients}")
|
||||
|
||||
# SES JSON als String serialisieren
|
||||
ses_json_string = json.dumps(ses_data)
|
||||
|
||||
# Payload Größe loggen und checken (Safeguard)
|
||||
payload_size = len(ses_json_string.encode('utf-8'))
|
||||
logger.info(f" Metadata Payload Size: {payload_size} bytes")
|
||||
if payload_size > 200000: # Arbitrary Limit < SQS 256KB
|
||||
raise ValueError("Payload too large for SQS")
|
||||
if payload_size > 200000:
|
||||
raise ValueError("Payload too large")
|
||||
|
||||
# Fake SNS Payload
|
||||
fake_sns_payload = {
|
||||
"Type": "Notification",
|
||||
"MessageId": str(uuid.uuid4()),
|
||||
@@ -95,26 +146,16 @@ def lambda_handler(event, context):
|
||||
"Timestamp": datetime.utcnow().isoformat() + "Z"
|
||||
}
|
||||
|
||||
# Queue URL dynamisch holen
|
||||
queue_url = get_queue_url(domain)
|
||||
message_body = json.dumps(fake_sns_payload)
|
||||
|
||||
# SQS Send mit Retries
|
||||
attempt = 0
|
||||
while attempt < MAX_RETRIES:
|
||||
try:
|
||||
sqs.send_message(
|
||||
QueueUrl=queue_url,
|
||||
MessageBody=json.dumps(fake_sns_payload)
|
||||
)
|
||||
logger.info(f"✅ Successfully forwarded {msg_id} to SQS: {queue_url}")
|
||||
break
|
||||
except ClientError as e:
|
||||
attempt += 1
|
||||
error_code = e.response['Error']['Code']
|
||||
logger.warning(f"Retry {attempt}/{MAX_RETRIES} for SQS send: {error_code} - {str(e)}")
|
||||
if attempt == MAX_RETRIES:
|
||||
raise
|
||||
time.sleep(exponential_backoff(attempt))
|
||||
# Strategie: SNS zuerst, SQS als Fallback
|
||||
topic_arn = get_topic_arn(domain)
|
||||
sns_success = publish_to_sns(topic_arn, message_body, msg_id)
|
||||
|
||||
if not sns_success:
|
||||
# Kein SNS-Topic für diese Domain → direkt in SQS (wie bisher)
|
||||
queue_url = get_queue_url(domain)
|
||||
send_to_sqs(queue_url, message_body, msg_id)
|
||||
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
@@ -7,7 +7,16 @@ RUN xcaddy build ${CADDY_VERSION} \
|
||||
--with github.com/caddy-dns/cloudflare \
|
||||
--with github.com/caddyserver/replace-response
|
||||
|
||||
# Autodiscover Handler in Go bauen (Go ist im Builder-Image bereits verfügbar)
|
||||
COPY autodiscover-handler.go /src/autodiscover-handler.go
|
||||
WORKDIR /src
|
||||
RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/bin/autodiscover-handler autodiscover-handler.go
|
||||
|
||||
FROM caddy:${CADDY_VERSION}
|
||||
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
|
||||
RUN mkdir -p /var/log/caddy
|
||||
COPY --from=builder /usr/bin/autodiscover-handler /usr/local/bin/autodiscover-handler
|
||||
COPY start.sh /usr/local/bin/start.sh
|
||||
RUN chmod +x /usr/local/bin/start.sh /usr/local/bin/autodiscover-handler \
|
||||
&& mkdir -p /var/log/caddy
|
||||
|
||||
CMD ["/usr/local/bin/start.sh"]
|
||||
109
caddy/autodiscover-handler.go
Normal file
109
caddy/autodiscover-handler.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const port = "8280"
|
||||
|
||||
var emailRegex = regexp.MustCompile(`(?i)<EMailAddress>([^<]+)</EMailAddress>`)
|
||||
|
||||
func main() {
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, "OK")
|
||||
})
|
||||
|
||||
http.HandleFunc("/autodiscover/autodiscover.xml", handleAutodiscover)
|
||||
// Outlook sendet manchmal mit Großbuchstaben
|
||||
http.HandleFunc("/Autodiscover/Autodiscover.xml", handleAutodiscover)
|
||||
http.HandleFunc("/AutoDiscover/AutoDiscover.xml", handleAutodiscover)
|
||||
|
||||
log.Printf("[autodiscover] Listening on port %s", port)
|
||||
if err := http.ListenAndServe(":"+port, nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func handleAutodiscover(w http.ResponseWriter, r *http.Request) {
|
||||
var email string
|
||||
|
||||
if r.Method == http.MethodPost {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err == nil {
|
||||
if match := emailRegex.FindStringSubmatch(string(body)); len(match) > 1 {
|
||||
email = strings.TrimSpace(match[1])
|
||||
}
|
||||
}
|
||||
r.Body.Close()
|
||||
}
|
||||
|
||||
var domain string
|
||||
if email != "" {
|
||||
parts := strings.SplitN(email, "@", 2)
|
||||
if len(parts) == 2 {
|
||||
domain = parts[1]
|
||||
}
|
||||
}
|
||||
if domain == "" {
|
||||
domain = extractDomainFromHost(r.Host)
|
||||
}
|
||||
|
||||
log.Printf("[autodiscover] %s from %s - email=%q domain=%s", r.Method, r.RemoteAddr, email, domain)
|
||||
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, buildResponse(domain, email))
|
||||
}
|
||||
|
||||
func extractDomainFromHost(host string) string {
|
||||
// Strip port
|
||||
if idx := strings.Index(host, ":"); idx >= 0 {
|
||||
host = host[:idx]
|
||||
}
|
||||
parts := strings.Split(host, ".")
|
||||
if len(parts) >= 3 && strings.EqualFold(parts[0], "autodiscover") {
|
||||
return strings.Join(parts[1:], ".")
|
||||
}
|
||||
if len(parts) >= 2 {
|
||||
return strings.Join(parts[len(parts)-2:], ".")
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func buildResponse(domain, loginName string) string {
|
||||
return fmt.Sprintf(`<?xml version="1.0" encoding="utf-8"?>
|
||||
<Autodiscover xmlns="http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006">
|
||||
<Response xmlns="http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a">
|
||||
<Account>
|
||||
<AccountType>email</AccountType>
|
||||
<Action>settings</Action>
|
||||
<Protocol>
|
||||
<Type>IMAP</Type>
|
||||
<Server>imap.%s</Server>
|
||||
<Port>993</Port>
|
||||
<DomainRequired>off</DomainRequired>
|
||||
<LoginName>%s</LoginName>
|
||||
<SPA>off</SPA>
|
||||
<SSL>on</SSL>
|
||||
<AuthRequired>on</AuthRequired>
|
||||
</Protocol>
|
||||
<Protocol>
|
||||
<Type>SMTP</Type>
|
||||
<Server>smtp.%s</Server>
|
||||
<Port>465</Port>
|
||||
<DomainRequired>off</DomainRequired>
|
||||
<LoginName>%s</LoginName>
|
||||
<SPA>off</SPA>
|
||||
<SSL>on</SSL>
|
||||
<AuthRequired>on</AuthRequired>
|
||||
</Protocol>
|
||||
</Account>
|
||||
</Response>
|
||||
</Autodiscover>`, domain, loginName, domain, loginName)
|
||||
}
|
||||
@@ -13,6 +13,7 @@ services:
|
||||
- 'host.docker.internal:host-gateway'
|
||||
networks:
|
||||
- mail_network
|
||||
- mailadmin_network
|
||||
volumes:
|
||||
- $PWD/Caddyfile:/etc/caddy/Caddyfile
|
||||
- $PWD/mail_certs:/etc/caddy/mail_certs
|
||||
@@ -29,3 +30,5 @@ services:
|
||||
networks:
|
||||
mail_network:
|
||||
external: true
|
||||
mailadmin_network:
|
||||
external: true
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
<body>
|
||||
|
||||
<div class="card">
|
||||
<img src="/logo.png" alt="Logo" class="logo">
|
||||
<img src="/email-setup/logo.png" alt="Logo" class="logo">
|
||||
|
||||
<div id="input-section">
|
||||
<h1>Email Setup</h1>
|
||||
|
||||
8
caddy/start.sh
Normal file
8
caddy/start.sh
Normal file
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# Autodiscover handler im Hintergrund starten
|
||||
/usr/local/bin/autodiscover-handler &
|
||||
|
||||
# Caddy im Vordergrund
|
||||
exec caddy run --config /etc/caddy/Caddyfile --adapter caddyfile
|
||||
@@ -78,36 +78,7 @@ OUTPUT="${OUTPUT}(email_settings) {\n"
|
||||
# --- 1. Outlook Classic Autodiscover (POST + GET XML) ---
|
||||
OUTPUT="${OUTPUT} # Outlook Autodiscover (XML) - POST und GET\n"
|
||||
OUTPUT="${OUTPUT} route /autodiscover/autodiscover.xml {\n"
|
||||
OUTPUT="${OUTPUT} header Content-Type \"application/xml\"\n"
|
||||
OUTPUT="${OUTPUT} respond \`<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
|
||||
OUTPUT="${OUTPUT}<Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">\n"
|
||||
OUTPUT="${OUTPUT} <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">\n"
|
||||
OUTPUT="${OUTPUT} <Account>\n"
|
||||
OUTPUT="${OUTPUT} <AccountType>email</AccountType>\n"
|
||||
OUTPUT="${OUTPUT} <Action>settings</Action>\n"
|
||||
OUTPUT="${OUTPUT} <Protocol>\n"
|
||||
OUTPUT="${OUTPUT} <Type>IMAP</Type>\n"
|
||||
OUTPUT="${OUTPUT} <Server>imap.{labels.1}.{labels.0}</Server>\n"
|
||||
OUTPUT="${OUTPUT} <Port>993</Port>\n"
|
||||
OUTPUT="${OUTPUT} <DomainRequired>off</DomainRequired>\n"
|
||||
OUTPUT="${OUTPUT} <LoginName></LoginName>\n"
|
||||
OUTPUT="${OUTPUT} <SPA>off</SPA>\n"
|
||||
OUTPUT="${OUTPUT} <SSL>on</SSL>\n"
|
||||
OUTPUT="${OUTPUT} <AuthRequired>on</AuthRequired>\n"
|
||||
OUTPUT="${OUTPUT} </Protocol>\n"
|
||||
OUTPUT="${OUTPUT} <Protocol>\n"
|
||||
OUTPUT="${OUTPUT} <Type>SMTP</Type>\n"
|
||||
OUTPUT="${OUTPUT} <Server>smtp.{labels.1}.{labels.0}</Server>\n"
|
||||
OUTPUT="${OUTPUT} <Port>465</Port>\n"
|
||||
OUTPUT="${OUTPUT} <DomainRequired>off</DomainRequired>\n"
|
||||
OUTPUT="${OUTPUT} <LoginName></LoginName>\n"
|
||||
OUTPUT="${OUTPUT} <SPA>off</SPA>\n"
|
||||
OUTPUT="${OUTPUT} <SSL>on</SSL>\n"
|
||||
OUTPUT="${OUTPUT} <AuthRequired>on</AuthRequired>\n"
|
||||
OUTPUT="${OUTPUT} </Protocol>\n"
|
||||
OUTPUT="${OUTPUT} </Account>\n"
|
||||
OUTPUT="${OUTPUT} </Response>\n"
|
||||
OUTPUT="${OUTPUT}</Autodiscover>\` 200\n"
|
||||
OUTPUT="${OUTPUT} reverse_proxy localhost:8280\n"
|
||||
OUTPUT="${OUTPUT} }\n"
|
||||
OUTPUT="${OUTPUT}\n"
|
||||
|
||||
@@ -135,14 +106,14 @@ OUTPUT="${OUTPUT} <hostname>imap.{labels.1}.{labels.0}</hostname>\n"
|
||||
OUTPUT="${OUTPUT} <port>993</port>\n"
|
||||
OUTPUT="${OUTPUT} <socketType>SSL</socketType>\n"
|
||||
OUTPUT="${OUTPUT} <authentication>password-cleartext</authentication>\n"
|
||||
OUTPUT="${OUTPUT} <username>%%EMAILADDRESS%%</username>\n"
|
||||
OUTPUT="${OUTPUT} <username>%EMAILADDRESS%</username>\n"
|
||||
OUTPUT="${OUTPUT} </incomingServer>\n"
|
||||
OUTPUT="${OUTPUT} <outgoingServer type=\"smtp\">\n"
|
||||
OUTPUT="${OUTPUT} <hostname>smtp.{labels.1}.{labels.0}</hostname>\n"
|
||||
OUTPUT="${OUTPUT} <port>465</port>\n"
|
||||
OUTPUT="${OUTPUT} <socketType>SSL</socketType>\n"
|
||||
OUTPUT="${OUTPUT} <authentication>password-cleartext</authentication>\n"
|
||||
OUTPUT="${OUTPUT} <username>%%EMAILADDRESS%%</username>\n"
|
||||
OUTPUT="${OUTPUT} <username>%EMAILADDRESS%</username>\n"
|
||||
OUTPUT="${OUTPUT} </outgoingServer>\n"
|
||||
OUTPUT="${OUTPUT} </emailProvider>\n"
|
||||
OUTPUT="${OUTPUT}</clientConfig>\` 200\n"
|
||||
@@ -290,6 +261,13 @@ for domain in $DOMAINS; do
|
||||
OUTPUT="${OUTPUT} }\n"
|
||||
OUTPUT="${OUTPUT}}\n\n"
|
||||
|
||||
# Mailadmin Block
|
||||
OUTPUT="${OUTPUT}# MailAdmin UI\n"
|
||||
OUTPUT="${OUTPUT}mailadmin.${domain} {\n"
|
||||
OUTPUT="${OUTPUT} encode gzip\n"
|
||||
OUTPUT="${OUTPUT} reverse_proxy mailadmin:3000\n"
|
||||
OUTPUT="${OUTPUT}}\n\n"
|
||||
|
||||
# Autodiscover / Autoconfig Block
|
||||
OUTPUT="${OUTPUT}# Autodiscover/Autoconfig für $domain\n"
|
||||
OUTPUT="${OUTPUT}autodiscover.${domain}, autoconfig.${domain} {\n"
|
||||
|
||||
@@ -10,12 +10,14 @@ services:
|
||||
ports:
|
||||
- "9000:8000" # Prometheus metrics (Host:Container)
|
||||
- "9090:8080" # Health check (Host:Container)
|
||||
# Connect to DMS on the host or Docker network
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
environment:
|
||||
- SMTP_HOST=host.docker.internal
|
||||
- SMTP_HOST=mailserver
|
||||
- SMTP_PORT=25
|
||||
networks:
|
||||
- mail_network
|
||||
|
||||
volumes:
|
||||
worker-logs:
|
||||
networks:
|
||||
mail_network:
|
||||
external: true
|
||||
@@ -48,6 +48,9 @@ export const config = {
|
||||
// Monitoring
|
||||
metricsPort: parseInt(process.env.METRICS_PORT ?? '8000', 10),
|
||||
healthPort: parseInt(process.env.HEALTH_PORT ?? '8080', 10),
|
||||
|
||||
queueSuffix: process.env.QUEUE_SUFFIX ?? '-queue',
|
||||
standbyMode: (process.env.STANDBY_MODE ?? 'false').toLowerCase() === 'true',
|
||||
} as const;
|
||||
|
||||
export type Config = typeof config;
|
||||
@@ -106,7 +109,7 @@ export function isInternalAddress(email: string): boolean {
|
||||
|
||||
/** Convert domain to SQS queue name: bizmatch.net → bizmatch-net-queue */
|
||||
export function domainToQueueName(domain: string): string {
|
||||
return domain.replace(/\./g, '-') + '-queue';
|
||||
return domain.replace(/\./g, '-') + config.queueSuffix;
|
||||
}
|
||||
|
||||
/** Convert domain to S3 bucket name: bizmatch.net → bizmatch-net-emails */
|
||||
|
||||
@@ -36,6 +36,9 @@ export class RulesProcessor {
|
||||
workerName: string,
|
||||
metricsCallback?: MetricsCallback,
|
||||
): Promise<boolean> {
|
||||
if (config.standbyMode) {
|
||||
return false;
|
||||
}
|
||||
const rule = await this.dynamodb.getEmailRules(recipient.toLowerCase());
|
||||
if (!rule) return false;
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import { BlocklistChecker } from '../email/blocklist.js';
|
||||
import { BounceHandler } from '../email/bounce-handler.js';
|
||||
import { parseEmail, isProcessedByWorker } from '../email/parser.js';
|
||||
import { RulesProcessor } from '../email/rules-processor.js';
|
||||
|
||||
import { config } from '../config.js';
|
||||
// ---------------------------------------------------------------------------
|
||||
// Processor
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -258,7 +258,8 @@ export class MessageProcessor {
|
||||
|
||||
if (totalHandled === recipients.length) {
|
||||
if (blockedRecipients.length === recipients.length) {
|
||||
// All blocked
|
||||
// All blocked — im Standby kein S3 anfassen
|
||||
if (!config.standbyMode) {
|
||||
try {
|
||||
await this.s3.markAsBlocked(
|
||||
domain,
|
||||
@@ -272,14 +273,18 @@ export class MessageProcessor {
|
||||
log(`⚠ Failed to handle blocked email: ${err.message ?? err}`, 'ERROR', workerName);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} else if (successful.length > 0) {
|
||||
if (!config.standbyMode) {
|
||||
await this.s3.markAsProcessed(
|
||||
domain,
|
||||
messageId,
|
||||
workerName,
|
||||
failedPermanent.length > 0 ? failedPermanent : undefined,
|
||||
);
|
||||
}
|
||||
} else if (failedPermanent.length > 0) {
|
||||
if (!config.standbyMode) {
|
||||
await this.s3.markAsAllInvalid(
|
||||
domain,
|
||||
messageId,
|
||||
@@ -287,6 +292,7 @@ export class MessageProcessor {
|
||||
workerName,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
const parts: string[] = [];
|
||||
|
||||
Reference in New Issue
Block a user