moved

email-worker/email_processing/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+"""
+Email processing components
+"""
+
+from .parser import EmailParser
+from .bounce_handler import BounceHandler
+from .rules_processor import RulesProcessor
+from .blocklist import BlocklistChecker
+
+__all__ = ['EmailParser', 'BounceHandler', 'RulesProcessor', 'BlocklistChecker']

email-worker/email_processing/blocklist.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+Sender blocklist checking with wildcard support
+"""
+
+import fnmatch
+from typing import List, Dict
+from email.utils import parseaddr
+
+from logger import log
+from aws.dynamodb_handler import DynamoDBHandler
+
+
+class BlocklistChecker:
+    """Checks if senders are blocked"""
+    
+    def __init__(self, dynamodb: DynamoDBHandler):
+        self.dynamodb = dynamodb
+    
+    def is_sender_blocked(
+        self, 
+        recipient: str, 
+        sender: str, 
+        worker_name: str
+    ) -> bool:
+        """
+        Check if sender is blocked for this recipient
+        
+        Args:
+            recipient: Recipient email address
+            sender: Sender email address (may include name)
+            worker_name: Worker name for logging
+            
+        Returns:
+            True if sender is blocked
+        """
+        patterns = self.dynamodb.get_blocked_patterns(recipient)
+        
+        if not patterns:
+            return False
+        
+        sender_clean = parseaddr(sender)[1].lower()
+        
+        for pattern in patterns:
+            if fnmatch.fnmatch(sender_clean, pattern.lower()):
+                log(
+                    f"⛔ BLOCKED: Sender {sender_clean} matches pattern '{pattern}' "
+                    f"for inbox {recipient}",
+                    'WARNING',
+                    worker_name
+                )
+                return True
+        
+        return False
+    
+    def batch_check_blocked_senders(
+        self,
+        recipients: List[str],
+        sender: str,
+        worker_name: str
+    ) -> Dict[str, bool]:
+        """
+        Batch check if sender is blocked for multiple recipients (more efficient)
+        
+        Args:
+            recipients: List of recipient email addresses
+            sender: Sender email address
+            worker_name: Worker name for logging
+            
+        Returns:
+            Dictionary mapping recipient -> is_blocked (bool)
+        """
+        # Get all blocked patterns in one batch call
+        patterns_by_recipient = self.dynamodb.batch_get_blocked_patterns(recipients)
+        
+        sender_clean = parseaddr(sender)[1].lower()
+        result = {}
+        
+        for recipient in recipients:
+            patterns = patterns_by_recipient.get(recipient, [])
+            
+            is_blocked = False
+            for pattern in patterns:
+                if fnmatch.fnmatch(sender_clean, pattern.lower()):
+                    log(
+                        f"⛔ BLOCKED: Sender {sender_clean} matches pattern '{pattern}' "
+                        f"for inbox {recipient}",
+                        'WARNING',
+                        worker_name
+                    )
+                    is_blocked = True
+                    break
+            
+            result[recipient] = is_blocked
+        
+        return result

email-worker/email_processing/bounce_handler.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Bounce detection and header rewriting
+"""
+
+from typing import Tuple, Any
+
+from logger import log
+from aws.dynamodb_handler import DynamoDBHandler
+
+
+class BounceHandler:
+    """Handles bounce detection and header rewriting"""
+    
+    def __init__(self, dynamodb: DynamoDBHandler):
+        self.dynamodb = dynamodb
+    
+    @staticmethod
+    def is_ses_bounce_notification(parsed_email) -> bool:
+        """Check if email is from SES MAILER-DAEMON"""
+        from_header = (parsed_email.get('From') or '').lower()
+        return 'mailer-daemon@' in from_header and 'amazonses.com' in from_header
+    
+    def apply_bounce_logic(
+        self, 
+        parsed, 
+        subject: str, 
+        worker_name: str = 'unified'
+    ) -> Tuple[Any, bool]:
+        """
+        Check for SES Bounce, lookup in DynamoDB and rewrite headers
+        
+        Args:
+            parsed: Parsed email message object
+            subject: Email subject
+            worker_name: Worker name for logging
+            
+        Returns:
+            Tuple of (parsed_email_object, was_modified_bool)
+        """
+        if not self.is_ses_bounce_notification(parsed):
+            return parsed, False
+
+        log("🔍 Detected SES MAILER-DAEMON bounce notification", 'INFO', worker_name)
+        
+        # Extract Message-ID from header
+        message_id = (parsed.get('Message-ID') or '').strip('<>').split('@')[0]
+        
+        if not message_id:
+            log("⚠ Could not extract Message-ID from bounce notification", 'WARNING', worker_name)
+            return parsed, False
+
+        log(f"  Looking up Message-ID: {message_id}", 'INFO', worker_name)
+        
+        # Lookup in DynamoDB
+        bounce_info = self.dynamodb.get_bounce_info(message_id, worker_name)
+        
+        if not bounce_info:
+            return parsed, False
+
+        # Bounce Info ausgeben
+        original_source = bounce_info['original_source']
+        bounced_recipients = bounce_info['bouncedRecipients']
+        bounce_type = bounce_info['bounceType']
+        bounce_subtype = bounce_info['bounceSubType']
+        
+        log(f"✓ Found bounce info:", 'INFO', worker_name)
+        log(f"  Original sender: {original_source}", 'INFO', worker_name)
+        log(f"  Bounce type: {bounce_type}/{bounce_subtype}", 'INFO', worker_name)
+        log(f"  Bounced recipients: {bounced_recipients}", 'INFO', worker_name)
+        
+        if bounced_recipients:
+            new_from = bounced_recipients[0]
+            
+            # Rewrite Headers
+            parsed['X-Original-SES-From'] = parsed.get('From', '')
+            parsed['X-Bounce-Type'] = f"{bounce_type}/{bounce_subtype}"
+            parsed.replace_header('From', new_from)
+            
+            if not parsed.get('Reply-To'):
+                parsed['Reply-To'] = new_from
+            
+            # Subject anpassen
+            if 'delivery status notification' in subject.lower() or 'thanks for your submission' in subject.lower():
+                parsed.replace_header('Subject', f"Delivery Status: {new_from}")
+            
+            log(f"✓ Rewritten FROM: {new_from}", 'SUCCESS', worker_name)
+            return parsed, True
+        
+        log("⚠ No bounced recipients found in bounce info", 'WARNING', worker_name)
+        return parsed, False

email-worker/email_processing/parser.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""
+Email parsing utilities
+"""
+
+from typing import Tuple, Optional
+from email.parser import BytesParser
+from email.policy import SMTP as SMTPPolicy
+
+
+class EmailParser:
+    """Email parsing utilities"""
+    
+    @staticmethod
+    def parse_bytes(raw_bytes: bytes):
+        """Parse raw email bytes into email.message object"""
+        return BytesParser(policy=SMTPPolicy).parsebytes(raw_bytes)
+    
+    @staticmethod
+    def extract_body_parts(parsed) -> Tuple[str, Optional[str]]:
+        """
+        Extract both text/plain and text/html body parts
+        
+        Args:
+            parsed: Parsed email message object
+            
+        Returns:
+            Tuple of (text_body, html_body or None)
+        """
+        text_body = ''
+        html_body = None
+        
+        if parsed.is_multipart():
+            for part in parsed.walk():
+                content_type = part.get_content_type()
+                
+                if content_type == 'text/plain':
+                    try:
+                        text_body += part.get_payload(decode=True).decode('utf-8', errors='ignore')
+                    except Exception:
+                        pass
+                
+                elif content_type == 'text/html':
+                    try:
+                        html_body = part.get_payload(decode=True).decode('utf-8', errors='ignore')
+                    except Exception:
+                        pass
+        else:
+            try:
+                payload = parsed.get_payload(decode=True)
+                if payload:
+                    decoded = payload.decode('utf-8', errors='ignore')
+                    if parsed.get_content_type() == 'text/html':
+                        html_body = decoded
+                    else:
+                        text_body = decoded
+            except Exception:
+                text_body = str(parsed.get_payload())
+        
+        return text_body.strip() if text_body else '(No body content)', html_body
+    
+    @staticmethod
+    def is_processed_by_worker(parsed) -> bool:
+        """
+        Check if email was already processed by our worker (loop detection)
+        
+        Args:
+            parsed: Parsed email message object
+            
+        Returns:
+            True if already processed
+        """
+        x_worker_processed = parsed.get('X-SES-Worker-Processed', '')
+        auto_submitted = parsed.get('Auto-Submitted', '')
+        
+        # Only skip if OUR header is present
+        is_processed_by_us = bool(x_worker_processed)
+        is_our_auto_reply = auto_submitted == 'auto-replied' and x_worker_processed
+        
+        return is_processed_by_us or is_our_auto_reply

email-worker/email_processing/rules_processor.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""
+Email rules processing (Auto-Reply/OOO and Forwarding)
+"""
+
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.utils import parseaddr, formatdate, make_msgid
+from botocore.exceptions import ClientError
+
+from logger import log
+from config import config, is_internal_address
+from aws.dynamodb_handler import DynamoDBHandler
+from aws.ses_handler import SESHandler
+from email_processing.parser import EmailParser
+
+
+class RulesProcessor:
+    """Processes email rules (OOO, Forwarding)"""
+    
+    def __init__(self, dynamodb: DynamoDBHandler, ses: SESHandler):
+        self.dynamodb = dynamodb
+        self.ses = ses
+    
+    def process_rules_for_recipient(
+        self,
+        recipient: str,
+        parsed,
+        domain: str,
+        worker_name: str,
+        metrics_callback=None
+    ):
+        """
+        Process OOO and Forward rules for a recipient
+        
+        Args:
+            recipient: Recipient email address
+            parsed: Parsed email message object
+            domain: Email domain
+            worker_name: Worker name for logging
+            metrics_callback: Optional callback to increment metrics
+        """
+        rule = self.dynamodb.get_email_rules(recipient)
+        
+        if not rule:
+            return
+        
+        original_from = parsed.get('From', '')
+        sender_name, sender_addr = parseaddr(original_from)
+        if not sender_addr:
+            sender_addr = original_from
+        
+        # ============================================
+        # OOO / Auto-Reply handling
+        # ============================================
+        if rule.get('ooo_active', False):
+            self._handle_ooo(
+                recipient, 
+                parsed, 
+                sender_addr, 
+                rule, 
+                domain, 
+                worker_name,
+                metrics_callback
+            )
+        
+        # ============================================
+        # Forward handling
+        # ============================================
+        forwards = rule.get('forwards', [])
+        if forwards:
+            self._handle_forwards(
+                recipient,
+                parsed,
+                original_from,
+                forwards,
+                domain,
+                worker_name,
+                metrics_callback
+            )
+    
+    def _handle_ooo(
+        self,
+        recipient: str,
+        parsed,
+        sender_addr: str,
+        rule: dict,
+        domain: str,
+        worker_name: str,
+        metrics_callback=None
+    ):
+        """Handle Out-of-Office auto-reply"""
+        # Don't reply to automatic messages
+        auto_submitted = parsed.get('Auto-Submitted', '')
+        precedence = (parsed.get('Precedence') or '').lower()
+        
+        if auto_submitted and auto_submitted != 'no':
+            log(f"  ⏭ Skipping OOO for auto-submitted message", 'INFO', worker_name)
+            return
+        
+        if precedence in ['bulk', 'junk', 'list']:
+            log(f"  ⏭ Skipping OOO for {precedence} message", 'INFO', worker_name)
+            return
+        
+        if any(x in sender_addr.lower() for x in ['noreply', 'no-reply', 'mailer-daemon']):
+            log(f"  ⏭ Skipping OOO for noreply address", 'INFO', worker_name)
+            return
+        
+        try:
+            ooo_msg = rule.get('ooo_message', 'I am out of office.')
+            content_type = rule.get('ooo_content_type', 'text')
+            ooo_reply = self._create_ooo_reply(parsed, recipient, ooo_msg, content_type)
+            ooo_bytes = ooo_reply.as_bytes()
+            
+            # Distinguish: Internal (Port 2525) vs External (SES)
+            if is_internal_address(sender_addr):
+                # Internal address → direct via Port 2525
+                success = self._send_internal_email(recipient, sender_addr, ooo_bytes, worker_name)
+                if success:
+                    log(f"✓ Sent OOO reply internally to {sender_addr}", 'SUCCESS', worker_name)
+                else:
+                    log(f"⚠ Internal OOO reply failed to {sender_addr}", 'WARNING', worker_name)
+            else:
+                # External address → via SES
+                success = self.ses.send_raw_email(recipient, sender_addr, ooo_bytes, worker_name)
+                if success:
+                    log(f"✓ Sent OOO reply externally to {sender_addr} via SES", 'SUCCESS', worker_name)
+            
+            if metrics_callback:
+                metrics_callback('autoreply', domain)
+        
+        except Exception as e:
+            log(f"⚠ OOO reply failed to {sender_addr}: {e}", 'ERROR', worker_name)
+    
+    def _handle_forwards(
+        self,
+        recipient: str,
+        parsed,
+        original_from: str,
+        forwards: list,
+        domain: str,
+        worker_name: str,
+        metrics_callback=None
+    ):
+        """Handle email forwarding"""
+        for forward_to in forwards:
+            try:
+                fwd_msg = self._create_forward_message(parsed, recipient, forward_to, original_from)
+                fwd_bytes = fwd_msg.as_bytes()
+                
+                # Distinguish: Internal (Port 2525) vs External (SES)
+                if is_internal_address(forward_to):
+                    # Internal address → direct via Port 2525 (no loop!)
+                    success = self._send_internal_email(recipient, forward_to, fwd_bytes, worker_name)
+                    if success:
+                        log(f"✓ Forwarded internally to {forward_to}", 'SUCCESS', worker_name)
+                    else:
+                        log(f"⚠ Internal forward failed to {forward_to}", 'WARNING', worker_name)
+                else:
+                    # External address → via SES
+                    success = self.ses.send_raw_email(recipient, forward_to, fwd_bytes, worker_name)
+                    if success:
+                        log(f"✓ Forwarded externally to {forward_to} via SES", 'SUCCESS', worker_name)
+                
+                if metrics_callback:
+                    metrics_callback('forward', domain)
+            
+            except Exception as e:
+                log(f"⚠ Forward failed to {forward_to}: {e}", 'ERROR', worker_name)
+    
+    @staticmethod
+    def _send_internal_email(from_addr: str, to_addr: str, raw_message: bytes, worker_name: str) -> bool:
+        """
+        Send email via internal SMTP port (bypasses transport_maps)
+        
+        Args:
+            from_addr: From address
+            to_addr: To address
+            raw_message: Raw MIME message bytes
+            worker_name: Worker name for logging
+            
+        Returns:
+            True on success, False on failure
+        """
+        try:
+            with smtplib.SMTP(config.smtp_host, config.internal_smtp_port, timeout=30) as conn:
+                conn.ehlo()
+                conn.sendmail(from_addr, [to_addr], raw_message)
+                return True
+        except Exception as e:
+            log(f"  ✗ Internal delivery failed to {to_addr}: {e}", 'ERROR', worker_name)
+            return False
+    
+    @staticmethod
+    def _create_ooo_reply(original_parsed, recipient: str, ooo_msg: str, content_type: str = 'text'):
+        """Create Out-of-Office reply as complete MIME message"""
+        text_body, html_body = EmailParser.extract_body_parts(original_parsed)
+        original_subject = original_parsed.get('Subject', '(no subject)')
+        original_from = original_parsed.get('From', 'unknown')
+
+        msg = MIMEMultipart('mixed')
+        msg['From'] = recipient
+        msg['To'] = original_from
+        msg['Subject'] = f"Out of Office: {original_subject}"
+        msg['Date'] = formatdate(localtime=True)
+        msg['Message-ID'] = make_msgid(domain=recipient.split('@')[1])
+        msg['In-Reply-To'] = original_parsed.get('Message-ID', '')
+        msg['References'] = original_parsed.get('Message-ID', '')
+        msg['Auto-Submitted'] = 'auto-replied'
+        msg['X-SES-Worker-Processed'] = 'ooo-reply'
+
+        body_part = MIMEMultipart('alternative')
+        
+        # Text version
+        text_content = f"{ooo_msg}\n\n--- Original Message ---\n"
+        text_content += f"From: {original_from}\n"
+        text_content += f"Subject: {original_subject}\n\n"
+        text_content += text_body
+        body_part.attach(MIMEText(text_content, 'plain', 'utf-8'))
+
+        # HTML version (if desired and original available)
+        if content_type == 'html' or html_body:
+            html_content = f"<div>{ooo_msg}</div><br><hr><br>"
+            html_content += "<strong>Original Message</strong><br>"
+            html_content += f"<strong>From:</strong> {original_from}<br>"
+            html_content += f"<strong>Subject:</strong> {original_subject}<br><br>"
+            html_content += (html_body if html_body else text_body.replace('\n', '<br>'))
+            body_part.attach(MIMEText(html_content, 'html', 'utf-8'))
+
+        msg.attach(body_part)
+        return msg
+    
+    @staticmethod
+    def _create_forward_message(original_parsed, recipient: str, forward_to: str, original_from: str):
+        """Create Forward message as complete MIME message"""
+        original_subject = original_parsed.get('Subject', '(no subject)')
+        original_date = original_parsed.get('Date', 'unknown')
+
+        msg = MIMEMultipart('mixed')
+        msg['From'] = recipient
+        msg['To'] = forward_to
+        msg['Subject'] = f"FWD: {original_subject}"
+        msg['Date'] = formatdate(localtime=True)
+        msg['Message-ID'] = make_msgid(domain=recipient.split('@')[1])
+        msg['Reply-To'] = original_from
+        msg['X-SES-Worker-Processed'] = 'forwarded'
+
+        text_body, html_body = EmailParser.extract_body_parts(original_parsed)
+        body_part = MIMEMultipart('alternative')
+        
+        # Text version
+        fwd_text = "---------- Forwarded message ---------\n"
+        fwd_text += f"From: {original_from}\n"
+        fwd_text += f"Date: {original_date}\n"
+        fwd_text += f"Subject: {original_subject}\n"
+        fwd_text += f"To: {recipient}\n\n"
+        fwd_text += text_body
+        body_part.attach(MIMEText(fwd_text, 'plain', 'utf-8'))
+
+        # HTML version
+        if html_body:
+            fwd_html = "<div style='border-left:3px solid #ccc;padding-left:10px;'>"
+            fwd_html += "<strong>---------- Forwarded message ---------</strong><br>"
+            fwd_html += f"<strong>From:</strong> {original_from}<br>"
+            fwd_html += f"<strong>Date:</strong> {original_date}<br>"
+            fwd_html += f"<strong>Subject:</strong> {original_subject}<br>"
+            fwd_html += f"<strong>To:</strong> {recipient}<br><br>"
+            fwd_html += html_body
+            fwd_html += "</div>"
+            body_part.attach(MIMEText(fwd_html, 'html', 'utf-8'))
+
+        msg.attach(body_part)
+
+        # Copy attachments
+        if original_parsed.is_multipart():
+            for part in original_parsed.walk():
+                if part.get_content_maintype() == 'multipart':
+                    continue
+                if part.get_content_type() in ['text/plain', 'text/html']:
+                    continue
+                msg.attach(part)
+
+        return msg