Merge branch 'contabo' of git.bizmatch.net:aknuth/email-amazon into contabo

DMS/batch_imapsync.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# batch_imapsync.sh - Führt IMAP-Sync für alle User im Hintergrund aus
+# Format der CSV: email@domain.com,SecretPassword123
+
+HOST1=$1
+HOST2=$2
+CSV_FILE=$3
+
+if [ -z "$HOST1" ] || [ -z "$HOST2" ] || [ -z "$CSV_FILE" ]; then
+    echo "Usage: $0 <source-host> <target-host> <users.csv>"
+    echo "Beispiel: $0 secure.emailsrvr.com 147.93.132.244 stxmaterials.csv"
+    exit 1
+fi
+
+# ======================================================================
+# Die eigentliche Sync-Funktion (wird in den Hintergrund geschickt)
+# ======================================================================
+run_sync_jobs() {
+    TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+    LOG_DIR="sync_logs_$TIMESTAMP"
+    mkdir -p "$LOG_DIR"
+    
+    echo "Beginne Sync-Lauf am $(date)" > "batch_master_${TIMESTAMP}.log"
+
+    while IFS=, read -r email password; do
+        email=$(echo "$email" | tr -d '\r' | xargs)
+        password=$(echo "$password" | tr -d '\r' | xargs)
+        
+        [ -z "$email" ] && continue
+        
+        LOGFILE="$LOG_DIR/imapsync_${email}.log"
+        echo "[$(date)] Syncing $email -> $LOGFILE" >> "batch_master_${TIMESTAMP}.log"
+        
+        # Führe Docker imapsync für den aktuellen User aus
+        docker run --rm gilleslamiral/imapsync imapsync \
+            --host1 "$HOST1" --user1 "$email" --password1 "$password" --ssl1 \
+            --host2 "$HOST2" --user2 "$email" --password2 "$password" --ssl2 \
+            --automap > "$LOGFILE" 2>&1 < /dev/null
+            
+    done < "$CSV_FILE"
+    
+    echo "Alle Sync-Jobs beendet am $(date)" >> "batch_master_${TIMESTAMP}.log"
+}
+
+# ======================================================================
+# Skript-Start: Entkopplung vom Terminal
+# ======================================================================
+echo "🚀 Starte Batch-IMAP-Sync im Hintergrund..."
+
+# Rufe die Funktion auf, leite alle restlichen Ausgaben ins Nichts und schicke sie in den Hintergrund (&)
+run_sync_jobs </dev/null >/dev/null 2>&1 &
+
+echo "✅ Der Job läuft jetzt autark im Hintergrund (sequenziell)."
+echo "Du kannst das SSH-Terminal jetzt bedenkenlos schließen!"
+echo "Überwache den Gesamtfortschritt mit:"
+echo "  tail -f batch_master_*.log"
+echo "Oder die Details eines einzelnen Postfachs mit:"
+echo "  tail -f sync_logs_*/imapsync_<email>.log"

basic_setup/ses_sns_shim_global.py

@@ -8,24 +8,38 @@ from botocore.exceptions import ClientError
 import time
 import random
 
-# Logging konfigurieren
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
 sqs = boto3.client('sqs')
+sns = boto3.client('sns')
+sts_account_id = None
 
-# Retry-Konfiguration
 MAX_RETRIES = 3
-BASE_BACKOFF = 1  # Sekunden
+BASE_BACKOFF = 1
 
 def exponential_backoff(attempt):
-    """Exponential Backoff mit Jitter"""
     return BASE_BACKOFF * (2 ** attempt) + random.uniform(0, 1)
 
+def get_account_id():
+    global sts_account_id
+    if sts_account_id is None:
+        sts_account_id = boto3.client('sts').get_caller_identity()['Account']
+    return sts_account_id
+
+def get_topic_arn(domain):
+    """
+    Generiert Topic-ARN aus Domain.
+    Konvention: domain.tld -> domain-tld-topic
+    """
+    topic_name = domain.replace('.', '-') + '-topic'
+    region = os.environ.get('AWS_REGION', 'us-east-2')
+    account_id = get_account_id()
+    return f"arn:aws:sns:{region}:{account_id}:{topic_name}"
+
 def get_queue_url(domain):
     """
-    Generiert Queue-Namen aus Domain und holt URL.
-    Konvention: domain.tld -> domain-tld-queue
+    Fallback: Direkter SQS-Send für Domains ohne SNS-Topic.
     """
     queue_name = domain.replace('.', '-') + '-queue'
     try:
@@ -38,54 +52,91 @@ def get_queue_url(domain):
         else:
             raise
 
+def publish_to_sns(topic_arn, message_body, msg_id):
+    attempt = 0
+    while attempt < MAX_RETRIES:
+        try:
+            sns.publish(
+                TopicArn=topic_arn,
+                Message=message_body
+            )
+            logger.info(f"✅ Published {msg_id} to SNS: {topic_arn}")
+            return True
+        except ClientError as e:
+            error_code = e.response['Error']['Code']
+            # Fallback auf SQS bei Topic-nicht-gefunden ODER fehlender Berechtigung
+            if error_code in ('NotFound', 'NotFoundException', 'AuthorizationError'):
+                logger.info(f"ℹ️ SNS unavailable for {topic_arn} ({error_code}) — falling back to SQS")
+                return False
+            attempt += 1
+            logger.warning(f"Retry {attempt}/{MAX_RETRIES} SNS: {error_code}")
+            if attempt == MAX_RETRIES:
+                raise
+            time.sleep(exponential_backoff(attempt))
+    return False
+
+def send_to_sqs(queue_url, message_body, msg_id):
+    """Fallback: Direkter SQS-Send (wie bisher)."""
+    attempt = 0
+    while attempt < MAX_RETRIES:
+        try:
+            sqs.send_message(
+                QueueUrl=queue_url,
+                MessageBody=message_body
+            )
+            logger.info(f"✅ Sent {msg_id} to SQS: {queue_url}")
+            return
+        except ClientError as e:
+            attempt += 1
+            error_code = e.response['Error']['Code']
+            logger.warning(f"Retry {attempt}/{MAX_RETRIES} SQS: {error_code}")
+            if attempt == MAX_RETRIES:
+                raise
+            time.sleep(exponential_backoff(attempt))
+
 def lambda_handler(event, context):
     """
-    Nimmt SES Event entgegen, extrahiert Domain dynamisch,
-    verpackt Metadaten als 'Fake SNS' und sendet an die domain-spezifische SQS.
-    Mit integrierter Retry-Logik für SQS-Send.
+    Nimmt SES Event entgegen, extrahiert Domain dynamisch.
+    Strategie: SNS Publish (Fan-Out an Primary + Standby Queue).
+    Fallback: Direkter SQS-Send falls kein SNS-Topic existiert.
     """
     try:
         records = event.get('Records', [])
         logger.info(f"Received event with {len(records)} records.")
-        
+
         for record in records:
             ses_data = record.get('ses', {})
             if not ses_data:
-                logger.warning(f"Invalid SES event: Missing 'ses' in record: {record}")
+                logger.warning(f"Invalid SES event: Missing 'ses' in record")
                 continue
-            
+
             mail = ses_data.get('mail', {})
             receipt = ses_data.get('receipt', {})
-            
-            # Domain extrahieren (aus erstem Recipient)
+
             recipients = receipt.get('recipients', []) or mail.get('destination', [])
             if not recipients:
                 logger.warning("No recipients in event - skipping")
                 continue
-            
+
             first_recipient = recipients[0]
             domain = first_recipient.split('@')[-1].lower()
             if not domain:
                 logger.error("Could not extract domain from recipient")
                 continue
-            
-            # Wichtige Metadaten loggen
+
             msg_id = mail.get('messageId', 'unknown')
             source = mail.get('source', 'unknown')
             logger.info(f"Processing Message-ID: {msg_id} for domain: {domain}")
-            logger.info(f" From: {source}")
-            logger.info(f" To: {recipients}")
-            
-            # SES JSON als String serialisieren
+            logger.info(f"  From: {source}")
+            logger.info(f"  To: {recipients}")
+
             ses_json_string = json.dumps(ses_data)
-            
-            # Payload Größe loggen und checken (Safeguard)
+
             payload_size = len(ses_json_string.encode('utf-8'))
-            logger.info(f" Metadata Payload Size: {payload_size} bytes")
-            if payload_size > 200000:  # Arbitrary Limit < SQS 256KB
-                raise ValueError("Payload too large for SQS")
-            
-            # Fake SNS Payload
+            logger.info(f"  Metadata Payload Size: {payload_size} bytes")
+            if payload_size > 200000:
+                raise ValueError("Payload too large")
+
             fake_sns_payload = {
                 "Type": "Notification",
                 "MessageId": str(uuid.uuid4()),
@@ -94,30 +145,20 @@ def lambda_handler(event, context):
                 "Message": ses_json_string,
                 "Timestamp": datetime.utcnow().isoformat() + "Z"
             }
-            
-            # Queue URL dynamisch holen
-            queue_url = get_queue_url(domain)
-            
-            # SQS Send mit Retries
-            attempt = 0
-            while attempt < MAX_RETRIES:
-                try:
-                    sqs.send_message(
-                        QueueUrl=queue_url,
-                        MessageBody=json.dumps(fake_sns_payload)
-                    )
-                    logger.info(f"✅ Successfully forwarded {msg_id} to SQS: {queue_url}")
-                    break
-                except ClientError as e:
-                    attempt += 1
-                    error_code = e.response['Error']['Code']
-                    logger.warning(f"Retry {attempt}/{MAX_RETRIES} for SQS send: {error_code} - {str(e)}")
-                    if attempt == MAX_RETRIES:
-                        raise
-                    time.sleep(exponential_backoff(attempt))
-        
+
+            message_body = json.dumps(fake_sns_payload)
+
+            # Strategie: SNS zuerst, SQS als Fallback
+            topic_arn = get_topic_arn(domain)
+            sns_success = publish_to_sns(topic_arn, message_body, msg_id)
+
+            if not sns_success:
+                # Kein SNS-Topic für diese Domain → direkt in SQS (wie bisher)
+                queue_url = get_queue_url(domain)
+                send_to_sqs(queue_url, message_body, msg_id)
+
         return {'status': 'ok'}
-    
+
     except Exception as e:
         logger.error(f"❌ Critical Error in Lambda Shim: {str(e)}", exc_info=True)
         raise e

caddy/Dockerfile.caddy

@@ -7,7 +7,16 @@ RUN xcaddy build ${CADDY_VERSION} \
   --with github.com/caddy-dns/cloudflare \
   --with github.com/caddyserver/replace-response
 
+# Autodiscover Handler in Go bauen (Go ist im Builder-Image bereits verfügbar)
+COPY autodiscover-handler.go /src/autodiscover-handler.go
+WORKDIR /src
+RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/bin/autodiscover-handler autodiscover-handler.go
+
 FROM caddy:${CADDY_VERSION}
 COPY --from=builder /usr/bin/caddy /usr/bin/caddy
-RUN mkdir -p /var/log/caddy
+COPY --from=builder /usr/bin/autodiscover-handler /usr/local/bin/autodiscover-handler
+COPY start.sh /usr/local/bin/start.sh
+RUN chmod +x /usr/local/bin/start.sh /usr/local/bin/autodiscover-handler \
+    && mkdir -p /var/log/caddy
 
+CMD ["/usr/local/bin/start.sh"]

caddy/autodiscover-handler.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+const port = "8280"
+
+var emailRegex = regexp.MustCompile(`(?i)<EMailAddress>([^<]+)</EMailAddress>`)
+
+func main() {
+	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "OK")
+	})
+
+	http.HandleFunc("/autodiscover/autodiscover.xml", handleAutodiscover)
+	// Outlook sendet manchmal mit Großbuchstaben
+	http.HandleFunc("/Autodiscover/Autodiscover.xml", handleAutodiscover)
+	http.HandleFunc("/AutoDiscover/AutoDiscover.xml", handleAutodiscover)
+
+	log.Printf("[autodiscover] Listening on port %s", port)
+	if err := http.ListenAndServe(":"+port, nil); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func handleAutodiscover(w http.ResponseWriter, r *http.Request) {
+	var email string
+
+	if r.Method == http.MethodPost {
+		body, err := io.ReadAll(r.Body)
+		if err == nil {
+			if match := emailRegex.FindStringSubmatch(string(body)); len(match) > 1 {
+				email = strings.TrimSpace(match[1])
+			}
+		}
+		r.Body.Close()
+	}
+
+	var domain string
+	if email != "" {
+		parts := strings.SplitN(email, "@", 2)
+		if len(parts) == 2 {
+			domain = parts[1]
+		}
+	}
+	if domain == "" {
+		domain = extractDomainFromHost(r.Host)
+	}
+
+	log.Printf("[autodiscover] %s from %s - email=%q domain=%s", r.Method, r.RemoteAddr, email, domain)
+
+	w.Header().Set("Content-Type", "application/xml")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, buildResponse(domain, email))
+}
+
+func extractDomainFromHost(host string) string {
+	// Strip port
+	if idx := strings.Index(host, ":"); idx >= 0 {
+		host = host[:idx]
+	}
+	parts := strings.Split(host, ".")
+	if len(parts) >= 3 && strings.EqualFold(parts[0], "autodiscover") {
+		return strings.Join(parts[1:], ".")
+	}
+	if len(parts) >= 2 {
+		return strings.Join(parts[len(parts)-2:], ".")
+	}
+	return host
+}
+
+func buildResponse(domain, loginName string) string {
+	return fmt.Sprintf(`<?xml version="1.0" encoding="utf-8"?>
+<Autodiscover xmlns="http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006">
+  <Response xmlns="http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a">
+    <Account>
+      <AccountType>email</AccountType>
+      <Action>settings</Action>
+      <Protocol>
+        <Type>IMAP</Type>
+        <Server>imap.%s</Server>
+        <Port>993</Port>
+        <DomainRequired>off</DomainRequired>
+        <LoginName>%s</LoginName>
+        <SPA>off</SPA>
+        <SSL>on</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+      <Protocol>
+        <Type>SMTP</Type>
+        <Server>smtp.%s</Server>
+        <Port>465</Port>
+        <DomainRequired>off</DomainRequired>
+        <LoginName>%s</LoginName>
+        <SPA>off</SPA>
+        <SSL>on</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+    </Account>
+  </Response>
+</Autodiscover>`, domain, loginName, domain, loginName)
+}

caddy/email-setup/setup.html

@@ -35,7 +35,7 @@
 <body>
 
 <div class="card">
-    <img src="/logo.png" alt="Logo" class="logo">
+    <img src="/email-setup/logo.png" alt="Logo" class="logo">
     
     <div id="input-section">
         <h1>Email Setup</h1>

caddy/start.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+# Autodiscover handler im Hintergrund starten
+/usr/local/bin/autodiscover-handler &
+
+# Caddy im Vordergrund
+exec caddy run --config /etc/caddy/Caddyfile --adapter caddyfile

caddy/update-caddy-certs.sh

@@ -78,36 +78,7 @@ OUTPUT="${OUTPUT}(email_settings) {\n"
 # --- 1. Outlook Classic Autodiscover (POST + GET XML) ---
 OUTPUT="${OUTPUT}    # Outlook Autodiscover (XML) - POST und GET\n"
 OUTPUT="${OUTPUT}    route /autodiscover/autodiscover.xml {\n"
-OUTPUT="${OUTPUT}        header Content-Type \"application/xml\"\n"
-OUTPUT="${OUTPUT}        respond \`<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
-OUTPUT="${OUTPUT}<Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">\n"
-OUTPUT="${OUTPUT}  <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">\n"
-OUTPUT="${OUTPUT}    <Account>\n"
-OUTPUT="${OUTPUT}      <AccountType>email</AccountType>\n"
-OUTPUT="${OUTPUT}      <Action>settings</Action>\n"
-OUTPUT="${OUTPUT}      <Protocol>\n"
-OUTPUT="${OUTPUT}        <Type>IMAP</Type>\n"
-OUTPUT="${OUTPUT}        <Server>imap.{labels.1}.{labels.0}</Server>\n"
-OUTPUT="${OUTPUT}        <Port>993</Port>\n"
-OUTPUT="${OUTPUT}        <DomainRequired>off</DomainRequired>\n"
-OUTPUT="${OUTPUT}        <LoginName></LoginName>\n"
-OUTPUT="${OUTPUT}        <SPA>off</SPA>\n"
-OUTPUT="${OUTPUT}        <SSL>on</SSL>\n"
-OUTPUT="${OUTPUT}        <AuthRequired>on</AuthRequired>\n"
-OUTPUT="${OUTPUT}      </Protocol>\n"
-OUTPUT="${OUTPUT}      <Protocol>\n"
-OUTPUT="${OUTPUT}        <Type>SMTP</Type>\n"
-OUTPUT="${OUTPUT}        <Server>smtp.{labels.1}.{labels.0}</Server>\n"
-OUTPUT="${OUTPUT}        <Port>465</Port>\n"
-OUTPUT="${OUTPUT}        <DomainRequired>off</DomainRequired>\n"
-OUTPUT="${OUTPUT}        <LoginName></LoginName>\n"
-OUTPUT="${OUTPUT}        <SPA>off</SPA>\n"
-OUTPUT="${OUTPUT}        <SSL>on</SSL>\n"
-OUTPUT="${OUTPUT}        <AuthRequired>on</AuthRequired>\n"
-OUTPUT="${OUTPUT}      </Protocol>\n"
-OUTPUT="${OUTPUT}    </Account>\n"
-OUTPUT="${OUTPUT}  </Response>\n"
-OUTPUT="${OUTPUT}</Autodiscover>\` 200\n"
+OUTPUT="${OUTPUT}      reverse_proxy localhost:8280\n"
 OUTPUT="${OUTPUT}    }\n"
 OUTPUT="${OUTPUT}\n"
 

email-worker-nodejs/src/config.ts

@@ -48,6 +48,9 @@ export const config = {
   // Monitoring
   metricsPort: parseInt(process.env.METRICS_PORT ?? '8000', 10),
   healthPort: parseInt(process.env.HEALTH_PORT ?? '8080', 10),
+
+  queueSuffix: process.env.QUEUE_SUFFIX ?? '-queue',
+  standbyMode: (process.env.STANDBY_MODE ?? 'false').toLowerCase() === 'true',
 } as const;
 
 export type Config = typeof config;
@@ -106,7 +109,7 @@ export function isInternalAddress(email: string): boolean {
 
 /** Convert domain to SQS queue name: bizmatch.net → bizmatch-net-queue */
 export function domainToQueueName(domain: string): string {
-  return domain.replace(/\./g, '-') + '-queue';
+  return domain.replace(/\./g, '-') + config.queueSuffix;
 }
 
 /** Convert domain to S3 bucket name: bizmatch.net → bizmatch-net-emails */

email-worker-nodejs/src/email/rules-processor.ts

@@ -36,6 +36,9 @@ export class RulesProcessor {
     workerName: string,
     metricsCallback?: MetricsCallback,
   ): Promise<boolean> {
+    if (config.standbyMode) {
+        return false;
+    }    
     const rule = await this.dynamodb.getEmailRules(recipient.toLowerCase());
     if (!rule) return false;
 

email-worker-nodejs/src/worker/message-processor.ts

@@ -26,7 +26,7 @@ import { BlocklistChecker } from '../email/blocklist.js';
 import { BounceHandler } from '../email/bounce-handler.js';
 import { parseEmail, isProcessedByWorker } from '../email/parser.js';
 import { RulesProcessor } from '../email/rules-processor.js';
-
+import { config } from '../config.js';
 // ---------------------------------------------------------------------------
 // Processor
 // ---------------------------------------------------------------------------
@@ -258,34 +258,40 @@ export class MessageProcessor {
 
       if (totalHandled === recipients.length) {
         if (blockedRecipients.length === recipients.length) {
-          // All blocked
-          try {
-            await this.s3.markAsBlocked(
-              domain,
-              messageId,
-              blockedRecipients,
-              fromAddrFinal,
-              workerName,
-            );
-            await this.s3.deleteBlockedEmail(domain, messageId, workerName);
-          } catch (err: any) {
-            log(`⚠ Failed to handle blocked email: ${err.message ?? err}`, 'ERROR', workerName);
-            return false;
+          // All blocked — im Standby kein S3 anfassen
+          if (!config.standbyMode) {
+            try {
+              await this.s3.markAsBlocked(
+                domain,
+                messageId,
+                blockedRecipients,
+                fromAddrFinal,
+                workerName,
+              );
+              await this.s3.deleteBlockedEmail(domain, messageId, workerName);
+            } catch (err: any) {
+              log(`⚠ Failed to handle blocked email: ${err.message ?? err}`, 'ERROR', workerName);
+              return false;
+            }
           }
         } else if (successful.length > 0) {
-          await this.s3.markAsProcessed(
-            domain,
-            messageId,
-            workerName,
-            failedPermanent.length > 0 ? failedPermanent : undefined,
-          );
+          if (!config.standbyMode) {
+            await this.s3.markAsProcessed(
+              domain,
+              messageId,
+              workerName,
+              failedPermanent.length > 0 ? failedPermanent : undefined,
+            );
+          }
         } else if (failedPermanent.length > 0) {
-          await this.s3.markAsAllInvalid(
-            domain,
-            messageId,
-            failedPermanent,
-            workerName,
-          );
+          if (!config.standbyMode) {
+            await this.s3.markAsAllInvalid(
+              domain,
+              messageId,
+              failedPermanent,
+              workerName,
+            );
+          }
         }
 
         // Summary