changes
This commit is contained in:
214
ses-lambda/lambda-function.py
Normal file
214
ses-lambda/lambda-function.py
Normal file
@@ -0,0 +1,214 @@
|
||||
# Optimiertes Lambda mit Verbesserungen
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import logging
|
||||
import ssl
|
||||
import boto3
|
||||
import base64
|
||||
import hashlib
|
||||
import time
|
||||
import gzip
|
||||
|
||||
# Konfiguration
|
||||
API_BASE_URL = os.environ['API_BASE_URL']
|
||||
API_TOKEN = os.environ['API_TOKEN']
|
||||
DEBUG_MODE = os.environ.get('DEBUG_MODE', 'false').lower() == 'true'
|
||||
MAX_EMAIL_SIZE = int(os.environ.get('MAX_EMAIL_SIZE', '10485760')) # 10MB Standard
|
||||
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
def bucket_name_to_domain(bucket_name):
|
||||
"""Konvertiert S3-Bucket-Namen zu Domain-Namen"""
|
||||
# Beispiel: andreasknuth-de-emails -> andreasknuth.de
|
||||
if bucket_name.endswith('-emails'):
|
||||
# Entferne das "-emails" Suffix
|
||||
domain_part = bucket_name[:-7] # Entferne die letzten 7 Zeichen ("-emails")
|
||||
# Ersetze alle verbleibenden "-" durch "."
|
||||
domain = domain_part.replace('-', '.')
|
||||
return domain
|
||||
else:
|
||||
# Fallback: Bucket-Name entspricht nicht dem erwarteten Schema
|
||||
logger.warning(f"Bucket-Name {bucket_name} entspricht nicht dem erwarteten Schema")
|
||||
return None
|
||||
|
||||
def lambda_handler(event, context):
|
||||
"""Optimierter Lambda-Handler mit verbesserter Fehlerbehandlung"""
|
||||
|
||||
# Eindeutige Request-ID für Tracking
|
||||
request_id = context.aws_request_id
|
||||
logger.info(f"[{request_id}] S3-Ereignis empfangen")
|
||||
|
||||
try:
|
||||
# S3-Event-Details extrahieren
|
||||
s3_event = event['Records'][0]['s3']
|
||||
bucket_name = s3_event['bucket']['name']
|
||||
object_key = urllib.parse.unquote_plus(s3_event['object']['key'])
|
||||
|
||||
logger.info(f"[{request_id}] Verarbeite: {bucket_name}/{object_key}")
|
||||
|
||||
# Domain automatisch aus Bucket-Namen ermitteln
|
||||
domain = bucket_name_to_domain(bucket_name)
|
||||
if not domain:
|
||||
logger.error(f"[{request_id}] Konnte Domain nicht aus Bucket-Namen {bucket_name} ermitteln")
|
||||
return {'statusCode': 400, 'body': json.dumps('Ungültiger Bucket-Name')}
|
||||
|
||||
logger.info(f"[{request_id}] Ermittelte Domain: {domain}")
|
||||
|
||||
# Duplikat-Check mit S3-ETag
|
||||
s3_client = boto3.client('s3')
|
||||
head_response = s3_client.head_object(Bucket=bucket_name, Key=object_key)
|
||||
etag = head_response['ETag'].strip('"')
|
||||
content_length = head_response['ContentLength']
|
||||
|
||||
# E-Mail-Größe prüfen
|
||||
if content_length > MAX_EMAIL_SIZE:
|
||||
logger.warning(f"[{request_id}] E-Mail zu groß: {content_length} Bytes (max: {MAX_EMAIL_SIZE})")
|
||||
# Große E-Mails markieren statt löschen
|
||||
mark_large_email(s3_client, bucket_name, object_key, content_length)
|
||||
return {'statusCode': 413, 'body': json.dumps('E-Mail zu groß')}
|
||||
|
||||
# E-Mail laden
|
||||
try:
|
||||
response = s3_client.get_object(Bucket=bucket_name, Key=object_key)
|
||||
email_content = response['Body'].read()
|
||||
logger.info(f"[{request_id}] E-Mail geladen: {len(email_content)} Bytes")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[{request_id}] Fehler beim Laden der E-Mail: {str(e)}")
|
||||
return {'statusCode': 500, 'body': json.dumps(f'S3-Fehler: {str(e)}')}
|
||||
|
||||
# E-Mail-Content komprimieren für Übertragung
|
||||
compressed_content = gzip.compress(email_content)
|
||||
email_base64 = base64.b64encode(compressed_content).decode('utf-8')
|
||||
|
||||
# Payload erstellen
|
||||
payload = {
|
||||
's3_bucket': bucket_name, # S3-Bucket für Marker-Funktionalität
|
||||
's3_key': object_key, # S3-Key für Marker-Funktionalität
|
||||
'domain': domain,
|
||||
'email_content': email_base64,
|
||||
'compressed': True, # Flag für API
|
||||
'etag': etag,
|
||||
'request_id': request_id,
|
||||
'original_size': len(email_content),
|
||||
'compressed_size': len(compressed_content)
|
||||
}
|
||||
|
||||
# API aufrufen mit Retry-Logik
|
||||
success = call_api_with_retry(payload, domain, request_id)
|
||||
|
||||
if success:
|
||||
# Bei Erfolg: E-Mail aus S3 löschen
|
||||
try:
|
||||
s3_client.delete_object(Bucket=bucket_name, Key=object_key)
|
||||
logger.info(f"[{request_id}] E-Mail erfolgreich verarbeitet und gelöscht")
|
||||
return {
|
||||
'statusCode': 200,
|
||||
'body': json.dumps({
|
||||
'message': 'E-Mail erfolgreich verarbeitet',
|
||||
'request_id': request_id,
|
||||
'domain': domain
|
||||
})
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"[{request_id}] Konnte E-Mail nicht löschen: {str(e)}")
|
||||
# Trotzdem Erfolg melden, da E-Mail verarbeitet wurde
|
||||
return {'statusCode': 200, 'body': json.dumps('Verarbeitet, aber nicht gelöscht')}
|
||||
else:
|
||||
# Bei Fehler: E-Mail in S3 belassen für späteren Retry
|
||||
logger.error(f"[{request_id}] E-Mail bleibt in S3 für Retry: {bucket_name}/{object_key}")
|
||||
return {'statusCode': 500, 'body': json.dumps('API-Fehler')}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[{request_id}] Unerwarteter Fehler: {str(e)}", exc_info=True)
|
||||
return {'statusCode': 500, 'body': json.dumps(f'Lambda-Fehler: {str(e)}')}
|
||||
|
||||
def call_api_with_retry(payload, domain, request_id, max_retries=3):
|
||||
"""API-Aufruf mit Retry-Logik"""
|
||||
|
||||
sync_url = f"{API_BASE_URL}/process/{domain}"
|
||||
payload_json = json.dumps(payload).encode('utf-8')
|
||||
|
||||
for attempt in range(max_retries):
|
||||
logger.info(f"[{request_id}] API-Aufruf Versuch {attempt + 1}/{max_retries}")
|
||||
|
||||
try:
|
||||
# Request erstellen
|
||||
req = urllib.request.Request(sync_url, data=payload_json, method="POST")
|
||||
req.add_header('Authorization', f'Bearer {API_TOKEN}')
|
||||
req.add_header('Content-Type', 'application/json')
|
||||
req.add_header('User-Agent', f'AWS-Lambda-EmailProcessor/2.0-{request_id}')
|
||||
req.add_header('X-Request-ID', request_id)
|
||||
|
||||
# SSL-Kontext
|
||||
ssl_context = ssl._create_unverified_context() if DEBUG_MODE else None
|
||||
|
||||
# API-Request mit Timeout
|
||||
timeout = 25 # Lambda hat 30s Timeout, also etwas weniger
|
||||
|
||||
if DEBUG_MODE and ssl_context:
|
||||
with urllib.request.urlopen(req, context=ssl_context, timeout=timeout) as response:
|
||||
response_body = response.read().decode('utf-8')
|
||||
response_code = response.getcode()
|
||||
else:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
response_body = response.read().decode('utf-8')
|
||||
response_code = response.getcode()
|
||||
|
||||
# Erfolgreiche Antwort
|
||||
if response_code == 200:
|
||||
logger.info(f"[{request_id}] API-Erfolg nach Versuch {attempt + 1}")
|
||||
if DEBUG_MODE:
|
||||
logger.info(f"[{request_id}] API-Antwort: {response_body}")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"[{request_id}] API-Antwort {response_code}: {response_body}")
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
error_body = ""
|
||||
try:
|
||||
error_body = e.read().decode('utf-8')
|
||||
except:
|
||||
pass
|
||||
|
||||
logger.error(f"[{request_id}] HTTP-Fehler {e.code} (Versuch {attempt + 1}): {e.reason}")
|
||||
logger.error(f"[{request_id}] Fehlerdetails: {error_body}")
|
||||
|
||||
# Bei 4xx-Fehlern nicht retry (Client-Fehler)
|
||||
if 400 <= e.code < 500:
|
||||
logger.error(f"[{request_id}] Client-Fehler - kein Retry")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[{request_id}] Netzwerk-Fehler (Versuch {attempt + 1}): {str(e)}")
|
||||
|
||||
# Warten vor nächstem Versuch (exponential backoff)
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2 ** attempt # 1s, 2s, 4s
|
||||
logger.info(f"[{request_id}] Warte {wait_time}s vor nächstem Versuch")
|
||||
time.sleep(wait_time)
|
||||
|
||||
logger.error(f"[{request_id}] API-Aufruf nach {max_retries} Versuchen fehlgeschlagen")
|
||||
return False
|
||||
|
||||
def mark_large_email(s3_client, bucket, key, size):
|
||||
"""Markiert große E-Mails mit Metadaten statt sie zu löschen"""
|
||||
try:
|
||||
s3_client.copy_object(
|
||||
Bucket=bucket,
|
||||
Key=key,
|
||||
CopySource={'Bucket': bucket, 'Key': key},
|
||||
Metadata={
|
||||
'status': 'too_large',
|
||||
'size': str(size),
|
||||
'marked_at': str(int(time.time()))
|
||||
},
|
||||
MetadataDirective='REPLACE'
|
||||
)
|
||||
logger.info(f"E-Mail als zu groß markiert: {key} ({size} Bytes)")
|
||||
except Exception as e:
|
||||
logger.error(f"Konnte E-Mail nicht markieren: {str(e)}")
|
||||
Reference in New Issue
Block a user