move and imports changed

This commit is contained in:
2026-03-07 14:59:41 -06:00
parent 908bb76c3a
commit c826d4c299
19 changed files with 3209 additions and 16 deletions

View File

@@ -0,0 +1,62 @@
/**
* Sender blocklist checking with wildcard / glob support
*
* Uses picomatch for pattern matching (equivalent to Python's fnmatch).
* Patterns are stored per-recipient in DynamoDB.
*/
import picomatch from 'picomatch';
import type { DynamoDBHandler } from '../aws/dynamodb.js';
import { log } from '../logger.js';
/**
* Extract the bare email address from a From header value.
* "John Doe <john@example.com>" → "john@example.com"
*/
function extractAddress(sender: string): string {
const match = sender.match(/<([^>]+)>/);
const addr = match ? match[1] : sender;
return addr.trim().toLowerCase();
}
export class BlocklistChecker {
constructor(private dynamodb: DynamoDBHandler) {}
/**
* Batch-check whether a sender is blocked for each recipient.
* Uses a single batch DynamoDB call for efficiency.
*/
async batchCheckBlockedSenders(
recipients: string[],
sender: string,
workerName: string,
): Promise<Record<string, boolean>> {
const patternsByRecipient =
await this.dynamodb.batchGetBlockedPatterns(recipients);
const senderClean = extractAddress(sender);
const result: Record<string, boolean> = {};
for (const recipient of recipients) {
const patterns = patternsByRecipient[recipient] ?? [];
let isBlocked = false;
for (const pattern of patterns) {
if (picomatch.isMatch(senderClean, pattern.toLowerCase())) {
log(
`⛔ BLOCKED: Sender ${senderClean} matches pattern '${pattern}' ` +
`for inbox ${recipient}`,
'WARNING',
workerName,
);
isBlocked = true;
break;
}
}
result[recipient] = isBlocked;
}
return result;
}
}

View File

@@ -0,0 +1,190 @@
/**
* Bounce detection and header rewriting
*
* When Amazon SES returns a bounce, the From header is
* mailer-daemon@amazonses.com. We look up the original sender
* in DynamoDB and rewrite the headers so the bounce appears
* to come from the actual bounced recipient.
*/
import type { ParsedMail } from 'mailparser';
import type { DynamoDBHandler } from '../aws/dynamodb.js';
import { isSesBounceNotification, getHeader } from './parser.js';
import { log } from '../logger.js';
export interface BounceResult {
/** Updated raw bytes (headers rewritten if bounce was detected) */
rawBytes: Buffer;
/** Whether bounce was detected and headers were modified */
modified: boolean;
/** Whether this email is a bounce notification at all */
isBounce: boolean;
/** The effective From address (rewritten or original) */
fromAddr: string;
}
export class BounceHandler {
constructor(private dynamodb: DynamoDBHandler) {}
/**
* Detect SES bounce, look up original sender in DynamoDB,
* and rewrite headers in the raw buffer.
*
* We operate on the raw Buffer because we need to preserve
* the original MIME structure exactly, only swapping specific
* header lines. mailparser's ParsedMail is read-only.
*/
async applyBounceLogic(
parsed: ParsedMail,
rawBytes: Buffer,
subject: string,
workerName = 'unified',
): Promise<BounceResult> {
if (!isSesBounceNotification(parsed)) {
return {
rawBytes,
modified: false,
isBounce: false,
fromAddr: parsed.from?.text ?? '',
};
}
log('🔍 Detected SES MAILER-DAEMON bounce notification', 'INFO', workerName);
// Extract Message-ID from the bounce notification header
const rawMessageId = getHeader(parsed, 'message-id')
.replace(/^</, '')
.replace(/>$/, '')
.split('@')[0];
if (!rawMessageId) {
log('⚠ Could not extract Message-ID from bounce notification', 'WARNING', workerName);
return {
rawBytes,
modified: false,
isBounce: true,
fromAddr: parsed.from?.text ?? '',
};
}
log(` Looking up Message-ID: ${rawMessageId}`, 'INFO', workerName);
const bounceInfo = await this.dynamodb.getBounceInfo(rawMessageId, workerName);
if (!bounceInfo) {
return {
rawBytes,
modified: false,
isBounce: true,
fromAddr: parsed.from?.text ?? '',
};
}
// Log bounce details
log(`✓ Found bounce info:`, 'INFO', workerName);
log(` Original sender: ${bounceInfo.original_source}`, 'INFO', workerName);
log(` Bounce type: ${bounceInfo.bounceType}/${bounceInfo.bounceSubType}`, 'INFO', workerName);
log(` Bounced recipients: ${bounceInfo.bouncedRecipients}`, 'INFO', workerName);
if (!bounceInfo.bouncedRecipients.length) {
log('⚠ No bounced recipients found in bounce info', 'WARNING', workerName);
return {
rawBytes,
modified: false,
isBounce: true,
fromAddr: parsed.from?.text ?? '',
};
}
const newFrom = bounceInfo.bouncedRecipients[0];
// Rewrite headers in raw bytes
let modifiedBytes = rawBytes;
const originalFrom = getHeader(parsed, 'from');
// Replace From header
modifiedBytes = replaceHeader(modifiedBytes, 'From', newFrom);
// Add diagnostic headers
modifiedBytes = addHeader(modifiedBytes, 'X-Original-SES-From', originalFrom);
modifiedBytes = addHeader(
modifiedBytes,
'X-Bounce-Type',
`${bounceInfo.bounceType}/${bounceInfo.bounceSubType}`,
);
// Add Reply-To if not present
if (!getHeader(parsed, 'reply-to')) {
modifiedBytes = addHeader(modifiedBytes, 'Reply-To', newFrom);
}
// Adjust subject for generic delivery status notifications
const subjectLower = subject.toLowerCase();
if (
subjectLower.includes('delivery status notification') ||
subjectLower.includes('thanks for your submission')
) {
modifiedBytes = replaceHeader(
modifiedBytes,
'Subject',
`Delivery Status: ${newFrom}`,
);
}
log(`✓ Rewritten FROM: ${newFrom}`, 'SUCCESS', workerName);
return {
rawBytes: modifiedBytes,
modified: true,
isBounce: true,
fromAddr: newFrom,
};
}
}
// ---------------------------------------------------------------------------
// Raw header manipulation helpers
// ---------------------------------------------------------------------------
/**
* Replace a header value in raw MIME bytes.
* Handles multi-line (folded) headers.
*/
function replaceHeader(raw: Buffer, name: string, newValue: string): Buffer {
const str = raw.toString('utf-8');
// Match header including potential folded continuation lines
const regex = new RegExp(
`^(${escapeRegex(name)}:\\s*).*?(\\r?\\n(?=[^ \\t])|\\r?\\n$)`,
'im',
);
// Also need to consume folded lines
const foldedRegex = new RegExp(
`^${escapeRegex(name)}:[ \\t]*[^\\r\\n]*(?:\\r?\\n[ \\t]+[^\\r\\n]*)*`,
'im',
);
const match = foldedRegex.exec(str);
if (!match) return raw;
const before = str.slice(0, match.index);
const after = str.slice(match.index + match[0].length);
const replaced = `${before}${name}: ${newValue}${after}`;
return Buffer.from(replaced, 'utf-8');
}
/**
* Add a new header line right before the header/body separator.
*/
function addHeader(raw: Buffer, name: string, value: string): Buffer {
const str = raw.toString('utf-8');
// Find the header/body boundary (first blank line)
const sep = str.match(/\r?\n\r?\n/);
if (!sep || sep.index === undefined) return raw;
const before = str.slice(0, sep.index);
const after = str.slice(sep.index);
return Buffer.from(`${before}\r\n${name}: ${value}${after}`, 'utf-8');
}
function escapeRegex(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

View File

@@ -0,0 +1,120 @@
/**
* Email parsing utilities
*
* Wraps `mailparser` for parsing raw MIME bytes and provides
* header sanitization (e.g. Microsoft's malformed Message-IDs).
*/
import { simpleParser, type ParsedMail } from 'mailparser';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export interface BodyParts {
text: string;
html: string | null;
}
// ---------------------------------------------------------------------------
// Parser
// ---------------------------------------------------------------------------
/**
* Parse raw email bytes into a ParsedMail object.
* Applies pre-sanitization for known malformed headers before parsing.
*/
export async function parseEmail(raw: Buffer): Promise<ParsedMail> {
// Pre-sanitize: fix Microsoft's [uuid]@domain Message-IDs
const sanitized = sanitizeRawHeaders(raw);
return simpleParser(sanitized);
}
/**
* Extract text and HTML body parts from a parsed email.
*/
export function extractBodyParts(parsed: ParsedMail): BodyParts {
const text = parsed.text?.trim() || '(No body content)';
const html = parsed.html || null;
return { text, html };
}
/**
* Check if email was already processed by our worker (loop detection).
*/
export function isProcessedByWorker(parsed: ParsedMail): boolean {
const headers = parsed.headers;
const xWorker = headers.get('x-ses-worker-processed');
const autoSubmitted = headers.get('auto-submitted');
const isProcessedByUs = !!xWorker;
const isOurAutoReply = autoSubmitted === 'auto-replied' && !!xWorker;
return isProcessedByUs || isOurAutoReply;
}
/**
* Check if email is a SES MAILER-DAEMON bounce notification.
*/
export function isSesBounceNotification(parsed: ParsedMail): boolean {
const from = (parsed.from?.text ?? '').toLowerCase();
return from.includes('mailer-daemon@') && from.includes('amazonses.com');
}
/**
* Get a header value as string. Handles mailparser's headerlines Map.
*/
export function getHeader(parsed: ParsedMail, name: string): string {
const val = parsed.headers.get(name.toLowerCase());
if (val === undefined || val === null) return '';
if (typeof val === 'string') return val;
if (typeof val === 'object' && 'text' in val) return (val as any).text ?? '';
return String(val);
}
// ---------------------------------------------------------------------------
// Raw header sanitization
// ---------------------------------------------------------------------------
/**
* Fix known problematic patterns in raw MIME headers BEFORE parsing.
*
* Specifically targets Microsoft's `Message-ID: <[uuid]@domain>` which
* causes strict parsers to crash.
*/
function sanitizeRawHeaders(raw: Buffer): Buffer {
// We only need to check/fix the header section (before first blank line).
// For efficiency we work on the first ~8KB where headers live.
const headerEnd = findDoubleNewline(raw);
const headerLen = headerEnd === -1 ? Math.min(raw.length, 8192) : headerEnd;
const headerStr = raw.subarray(0, headerLen).toString('utf-8');
// Fix: Message-ID with square brackets <[...]@...>
if (headerStr.includes('[') || headerStr.includes(']')) {
const fixed = headerStr.replace(
/^(Message-ID:\s*<?)(\[.*?\])(@[^>]*>?\s*)$/im,
(_match, prefix, bracketed, suffix) =>
prefix + bracketed.replace(/\[/g, '').replace(/\]/g, '') + suffix,
);
if (fixed !== headerStr) {
return Buffer.concat([
Buffer.from(fixed, 'utf-8'),
raw.subarray(headerLen),
]);
}
}
return raw;
}
function findDoubleNewline(buf: Buffer): number {
// Look for \r\n\r\n or \n\n
for (let i = 0; i < buf.length - 3; i++) {
if (buf[i] === 0x0d && buf[i + 1] === 0x0a && buf[i + 2] === 0x0d && buf[i + 3] === 0x0a) {
return i;
}
if (buf[i] === 0x0a && buf[i + 1] === 0x0a) {
return i;
}
}
return -1;
}

View File

@@ -0,0 +1,306 @@
/**
* Email rules processing (Auto-Reply / OOO and Forwarding)
* * CLEANED UP & FIXED:
* - Uses MailComposer for ALL message generation (safer MIME handling)
* - Fixes broken attachment forwarding
* - Removed legacy SMTP forwarding
* - Removed manual string concatenation for MIME boundaries
*/
import { createTransport } from 'nodemailer';
import type { ParsedMail } from 'mailparser';
import type { SESHandler } from '../aws/ses.js';
import { extractBodyParts } from './parser.js';
import { log } from '../logger.js';
// Wir nutzen MailComposer direkt für das Erstellen der Raw Bytes
import MailComposer from 'nodemailer/lib/mail-composer/index.js';
import { DynamoDBHandler, EmailRule } from '../aws/dynamodb.js';
import { config, isInternalAddress } from '../config.js';
export type MetricsCallback = (action: 'autoreply' | 'forward', domain: string) => void;
export class RulesProcessor {
constructor(
private dynamodb: DynamoDBHandler,
private ses: SESHandler,
) {}
/**
* Process OOO and Forward rules for a single recipient.
*/
async processRulesForRecipient(
recipient: string,
parsed: ParsedMail,
rawBytes: Buffer,
domain: string,
workerName: string,
metricsCallback?: MetricsCallback,
): Promise<boolean> {
const rule = await this.dynamodb.getEmailRules(recipient.toLowerCase());
if (!rule) return false;
const originalFrom = parsed.from?.text ?? '';
const senderAddr = extractSenderAddress(originalFrom);
// OOO / Auto-Reply
if (rule.ooo_active) {
await this.handleOoo(
recipient,
parsed,
senderAddr,
rule,
domain,
workerName,
metricsCallback,
);
}
// Forwarding
const forwards = rule.forwards ?? [];
if (forwards.length > 0) {
await this.handleForwards(
recipient,
parsed,
originalFrom,
forwards,
domain,
workerName,
metricsCallback,
);
}
return false; // never skip local delivery
}
// -----------------------------------------------------------------------
// OOO
// -----------------------------------------------------------------------
private async handleOoo(
recipient: string,
parsed: ParsedMail,
senderAddr: string,
rule: EmailRule,
domain: string,
workerName: string,
metricsCallback?: MetricsCallback,
): Promise<void> {
// Don't reply to automatic messages
const autoSubmitted = parsed.headers.get('auto-submitted');
const precedence = String(parsed.headers.get('precedence') ?? '').toLowerCase();
if (autoSubmitted && autoSubmitted !== 'no') {
log(' ⏭ Skipping OOO for auto-submitted message', 'INFO', workerName);
return;
}
if (['bulk', 'junk', 'list'].includes(precedence)) {
log(` ⏭ Skipping OOO for ${precedence} message`, 'INFO', workerName);
return;
}
if (/noreply|no-reply|mailer-daemon/i.test(senderAddr)) {
log(' ⏭ Skipping OOO for noreply address', 'INFO', workerName);
return;
}
try {
const oooMsg = (rule.ooo_message as string) ?? 'I am out of office.';
const contentType = (rule.ooo_content_type as string) ?? 'text';
// FIX: Use MailComposer via await
const oooBuffer = await buildOooReply(parsed, recipient, oooMsg, contentType);
if (isInternalAddress(senderAddr)) {
const ok = await sendInternalEmail(recipient, senderAddr, oooBuffer, workerName);
if (ok) log(`✓ Sent OOO reply internally to ${senderAddr}`, 'SUCCESS', workerName);
else log(`⚠ Internal OOO reply failed to ${senderAddr}`, 'WARNING', workerName);
} else {
const ok = await this.ses.sendRawEmail(recipient, senderAddr, oooBuffer, workerName);
if (ok) log(`✓ Sent OOO reply externally to ${senderAddr} via SES`, 'SUCCESS', workerName);
}
metricsCallback?.('autoreply', domain);
} catch (err: any) {
log(`⚠ OOO reply failed to ${senderAddr}: ${err.message ?? err}`, 'ERROR', workerName);
}
}
// -----------------------------------------------------------------------
// Forwarding
// -----------------------------------------------------------------------
private async handleForwards(
recipient: string,
parsed: ParsedMail,
originalFrom: string,
forwards: string[],
domain: string,
workerName: string,
metricsCallback?: MetricsCallback,
): Promise<void> {
for (const forwardTo of forwards) {
try {
// FIX: Correctly await the composer result
const fwdBuffer = await buildForwardMessage(parsed, recipient, forwardTo, originalFrom);
if (isInternalAddress(forwardTo)) {
const ok = await sendInternalEmail(recipient, forwardTo, fwdBuffer, workerName);
if (ok) log(`✓ Forwarded internally to ${forwardTo}`, 'SUCCESS', workerName);
else log(`⚠ Internal forward failed to ${forwardTo}`, 'WARNING', workerName);
} else {
const ok = await this.ses.sendRawEmail(recipient, forwardTo, fwdBuffer, workerName);
if (ok) log(`✓ Forwarded externally to ${forwardTo} via SES`, 'SUCCESS', workerName);
}
metricsCallback?.('forward', domain);
} catch (err: any) {
log(`⚠ Forward failed to ${forwardTo}: ${err.message ?? err}`, 'ERROR', workerName);
}
}
}
}
// ---------------------------------------------------------------------------
// Message building (Using Nodemailer MailComposer for Safety)
// ---------------------------------------------------------------------------
async function buildOooReply(
original: ParsedMail,
recipient: string,
oooMsg: string,
contentType: string,
): Promise<Buffer> {
const { text: textBody, html: htmlBody } = extractBodyParts(original);
const originalSubject = original.subject ?? '(no subject)';
const originalFrom = original.from?.text ?? 'unknown';
const originalMsgId = original.messageId ?? '';
const recipientDomain = recipient.split('@')[1];
// Text version
let textContent = `${oooMsg}\n\n--- Original Message ---\n`;
textContent += `From: ${originalFrom}\n`;
textContent += `Subject: ${originalSubject}\n\n`;
textContent += textBody;
// HTML version
let htmlContent = `<div>${oooMsg}</div><br><hr><br>`;
htmlContent += '<strong>Original Message</strong><br>';
htmlContent += `<strong>From:</strong> ${originalFrom}<br>`;
htmlContent += `<strong>Subject:</strong> ${originalSubject}<br><br>`;
htmlContent += htmlBody ? htmlBody : textBody.replace(/\n/g, '<br>');
const includeHtml = contentType === 'html' || !!htmlBody;
const composer = new MailComposer({
from: recipient,
to: originalFrom,
subject: `Out of Office: ${originalSubject}`,
inReplyTo: originalMsgId,
references: [originalMsgId], // Nodemailer wants array
text: textContent,
html: includeHtml ? htmlContent : undefined,
headers: {
'Auto-Submitted': 'auto-replied',
'X-SES-Worker-Processed': 'ooo-reply',
},
messageId: `<${Date.now()}.${Math.random().toString(36).slice(2)}@${recipientDomain}>`
});
return composer.compile().build();
}
async function buildForwardMessage(
original: ParsedMail,
recipient: string,
forwardTo: string,
originalFrom: string,
): Promise<Buffer> {
const { text: textBody, html: htmlBody } = extractBodyParts(original);
const originalSubject = original.subject ?? '(no subject)';
const originalDate = original.date?.toUTCString() ?? 'unknown';
// Text version
let fwdText = '---------- Forwarded message ---------\n';
fwdText += `From: ${originalFrom}\n`;
fwdText += `Date: ${originalDate}\n`;
fwdText += `Subject: ${originalSubject}\n`;
fwdText += `To: ${recipient}\n\n`;
fwdText += textBody;
// HTML version
let fwdHtml: string | undefined;
if (htmlBody) {
fwdHtml = "<div style='border-left:3px solid #ccc;padding-left:10px;'>";
fwdHtml += '<strong>---------- Forwarded message ---------</strong><br>';
fwdHtml += `<strong>From:</strong> ${originalFrom}<br>`;
fwdHtml += `<strong>Date:</strong> ${originalDate}<br>`;
fwdHtml += `<strong>Subject:</strong> ${originalSubject}<br>`;
fwdHtml += `<strong>To:</strong> ${recipient}<br><br>`;
fwdHtml += htmlBody;
fwdHtml += '</div>';
}
// Config object for MailComposer
const mailOptions: any = {
from: recipient,
to: forwardTo,
subject: `FWD: ${originalSubject}`,
replyTo: originalFrom,
text: fwdText,
html: fwdHtml,
headers: {
'X-SES-Worker-Processed': 'forwarded',
},
};
// Attachments
if (original.attachments && original.attachments.length > 0) {
mailOptions.attachments = original.attachments.map((att) => ({
filename: att.filename ?? 'attachment',
content: att.content,
contentType: att.contentType,
cid: att.cid ?? undefined,
contentDisposition: att.contentDisposition || 'attachment'
}));
}
const composer = new MailComposer(mailOptions);
return composer.compile().build();
}
// ---------------------------------------------------------------------------
// Internal SMTP delivery (port 25, bypasses transport_maps)
// ---------------------------------------------------------------------------
async function sendInternalEmail(
from: string,
to: string,
rawMessage: Buffer,
workerName: string,
): Promise<boolean> {
try {
const transport = createTransport({
host: config.smtpHost,
port: config.internalSmtpPort,
secure: false,
tls: { rejectUnauthorized: false },
});
await transport.sendMail({
envelope: { from, to: [to] },
raw: rawMessage,
});
transport.close();
return true;
} catch (err: any) {
log(` ✗ Internal delivery failed to ${to}: ${err.message ?? err}`, 'ERROR', workerName);
return false;
}
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function extractSenderAddress(fromHeader: string): string {
const match = fromHeader.match(/<([^>]+)>/);
return match ? match[1] : fromHeader;
}