/** * Milestone 1 (R1): Ingestion & Batch Upload Adversarial Stress Suite * Empirical Challenger Verification * * Verifies: * - Empty (0-byte) and truncated files * - Boundary file sizes (10MB boundary, 10MB+1, 500MB) * - 50-file massive batch queue load & strict concurrency throttle * - Mid-flight item removal and worker recovery * - Multi-attempt failure and retry cycles * - Exact vs collision duplicate filtering * - Server failure error boundaries & memory cleanup */ import { describe, test, it, expect, beforeEach } from "./runner"; import { ProcessedReceipt } from "../../src/lib/schema/receipt"; import { BatchQueueItem } from "../../src/components/dashboard/BatchUploadDrawer"; function createMockFile(name: string, sizeBytes: number, mimeType: string): File { const buffer = new Uint8Array(Math.min(sizeBytes, 1024)); // avoid allocating 500MB buffer in memory const blob = new Blob([buffer], { type: mimeType }); const file = new File([blob], name, { type: mimeType, lastModified: Date.now() }); // Explicitly override size property for boundary testing Object.defineProperty(file, "size", { value: sizeBytes }); return file; } class AdversarialBatchQueueManager { items: BatchQueueItem[] = []; activeWorkers = new Set(); maxConcurrent = 2; maxFileSize = 10 * 1024 * 1024; processedReceipts: ProcessedReceipt[] = []; maxSimultaneousObserved = 0; addFiles(files: File[]) { for (const file of files) { const isPdf = file.type === "application/pdf" || file.name.toLowerCase().endsWith(".pdf"); const previewThumbnailUrl = !isPdf ? `blob:http://localhost/mock-thumb-${file.name}` : undefined; const isDuplicate = this.items.some( (i) => i.fileName === file.name && i.fileSizeBytes === file.size ); if (isDuplicate) continue; this.items.push({ id: `upload_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`, file, fileName: file.name || "unnamed_file", fileSizeBytes: file.size || 0, mimeType: file.type || (isPdf ? "application/pdf" : "image/jpeg"), previewThumbnailUrl, isPdf, status: "queued", progressPercent: 0, stageMessage: "In Warteschlange...", }); } } getStats() { const total = this.items.length; const inProgress = this.items.filter((i) => ["queued", "preprocessing", "uploading", "extracting"].includes(i.status) ).length; const success = this.items.filter( (i) => i.status === "success" || i.status === "duplicate_suspected" ).length; const failed = this.items.filter((i) => i.status === "error").length; const aggregatePercent = total === 0 ? 0 : Math.round( this.items.reduce((acc, curr) => acc + curr.progressPercent, 0) / total ); return { total, inProgress, success, failed, aggregatePercent }; } async processItem( itemId: string, handler?: (file: File) => Promise<{ success: boolean; receipts?: any[]; error?: string; duplicateScore?: number }> ) { const item = this.items.find((i) => i.id === itemId); if (!item) return; this.activeWorkers.add(itemId); this.maxSimultaneousObserved = Math.max( this.maxSimultaneousObserved, this.activeWorkers.size ); // 1. Pre-flight Validation if (item.fileSizeBytes > this.maxFileSize) { item.status = "error"; item.progressPercent = 100; item.stageMessage = "Datei zu groß"; item.error = `Die Datei überschreitet das Limit von 10 MB.`; this.activeWorkers.delete(itemId); return; } if (item.fileSizeBytes === 0) { // Empty file handling item.status = "error"; item.progressPercent = 100; item.stageMessage = "Leere Datei"; item.error = "Die Datei enthält keine Daten (0 Bytes)."; this.activeWorkers.delete(itemId); return; } // 2. Preprocessing item.status = "preprocessing"; item.progressPercent = 25; // 3. Uploading item.status = "uploading"; item.progressPercent = 50; // 4. Extracting item.status = "extracting"; item.progressPercent = 75; try { if (handler) { const res = await handler(item.file); if (!res.success) { throw new Error(res.error || "Serverfehler bei der Extraktion"); } const receipts = res.receipts || []; if (receipts.length === 0) { throw new Error("Keine Belegdaten aus dieser Datei extrahiert"); } const isDuplicate = (res.duplicateScore || 0) >= 0.7; item.status = isDuplicate ? "duplicate_suspected" : "success"; item.progressPercent = 100; item.stageMessage = isDuplicate ? "Duplikat erkannt" : "Erfolgreich erfasst"; item.receiptResult = receipts[0]; this.processedReceipts.push(...receipts); } else { item.status = "success"; item.progressPercent = 100; item.stageMessage = "Erfolgreich erfasst"; } } catch (err: any) { item.status = "error"; item.progressPercent = 100; item.stageMessage = "Fehlgeschlagen"; item.error = err.message || "Unbekannter Verarbeitungsfehler"; } finally { this.activeWorkers.delete(itemId); } } async runQueue( handler?: (file: File) => Promise<{ success: boolean; receipts?: any[]; error?: string; duplicateScore?: number }> ) { while (true) { const queued = this.items.filter((i) => i.status === "queued"); if (queued.length === 0) break; const availableSlots = this.maxConcurrent - this.activeWorkers.size; if (availableSlots <= 0) { await new Promise((res) => setTimeout(res, 5)); continue; } const nextBatch = queued.slice(0, availableSlots); await Promise.all(nextBatch.map((item) => this.processItem(item.id, handler))); } } removeItem(itemId: string) { this.activeWorkers.delete(itemId); this.items = this.items.filter((i) => i.id !== itemId); } retryItem(itemId: string) { const item = this.items.find((i) => i.id === itemId); if (item) { item.status = "queued"; item.progressPercent = 0; item.error = undefined; item.stageMessage = "In Warteschlange..."; } } } describe("Milestone 1: Ingestion & Batch Upload — Adversarial Stress Testing", () => { let qm: AdversarialBatchQueueManager; beforeEach(() => { qm = new AdversarialBatchQueueManager(); }); test("ADV-1: Empty (0-byte) files are caught and marked as error without halting the batch", async () => { const files = [ createMockFile("empty_receipt.pdf", 0, "application/pdf"), createMockFile("valid_receipt.jpg", 150000, "image/jpeg"), ]; qm.addFiles(files); expect(qm.items).toHaveLength(2); await qm.runQueue(async (file) => { return { success: true, receipts: [{ id: `rcpt-${file.name}`, merchant: { name: "Aral" } } as any], }; }); const emptyItem = qm.items.find((i) => i.fileName === "empty_receipt.pdf"); expect(emptyItem?.status).toBe("error"); expect(emptyItem?.error).toContain("0 Bytes"); const validItem = qm.items.find((i) => i.fileName === "valid_receipt.jpg"); expect(validItem?.status).toBe("success"); const stats = qm.getStats(); expect(stats.total).toBe(2); expect(stats.failed).toBe(1); expect(stats.success).toBe(1); }); test("ADV-2: Exact Boundary Testing — 10MB limit passes, 10MB+1 byte fails, 500MB fails instantly", async () => { const limitBytes = 10 * 1024 * 1024; const exactLimitFile = createMockFile("exact_10mb.pdf", limitBytes, "application/pdf"); const overLimitFile = createMockFile("over_10mb.pdf", limitBytes + 1, "application/pdf"); const massiveFile = createMockFile("huge_500mb.pdf", 500 * 1024 * 1024, "application/pdf"); qm.addFiles([exactLimitFile, overLimitFile, massiveFile]); expect(qm.items).toHaveLength(3); await qm.runQueue(async (file) => { return { success: true, receipts: [{ id: `rcpt-${file.name}` } as any], }; }); const exactItem = qm.items.find((i) => i.fileName === "exact_10mb.pdf"); expect(exactItem?.status).toBe("success"); const overItem = qm.items.find((i) => i.fileName === "over_10mb.pdf"); expect(overItem?.status).toBe("error"); expect(overItem?.error).toContain("10 MB"); const massiveItem = qm.items.find((i) => i.fileName === "huge_500mb.pdf"); expect(massiveItem?.status).toBe("error"); expect(massiveItem?.error).toContain("10 MB"); }); test("ADV-3: Extreme Multi-File Burst (50 files) strictly preserves <=2 concurrency and calculates 100% aggregate progress", async () => { const files: File[] = []; for (let i = 1; i <= 50; i++) { files.push(createMockFile(`batch_file_${i}.jpg`, 10000 + i, "image/jpeg")); } qm.addFiles(files); expect(qm.items).toHaveLength(50); // Verify all 50 IDs are distinct const ids = new Set(qm.items.map((i) => i.id)); expect(ids.size).toBe(50); await qm.runQueue(async (file) => { // Simulate async processing await new Promise((res) => setTimeout(res, 1)); return { success: true, receipts: [{ id: `rcpt-${file.name}`, merchant: { name: `Store ${file.name}` } } as any], }; }); expect(qm.maxSimultaneousObserved).toBeLessThanOrEqual(2); const stats = qm.getStats(); expect(stats.total).toBe(50); expect(stats.success).toBe(50); expect(stats.failed).toBe(0); expect(stats.aggregatePercent).toBe(100); }); test("ADV-4: Mid-flight item removal recovers active worker slots immediately for subsequent queued items", async () => { const files = [ createMockFile("active_item.jpg", 200000, "image/jpeg"), createMockFile("queued_item1.jpg", 150000, "image/jpeg"), createMockFile("queued_item2.jpg", 180000, "image/jpeg"), ]; qm.addFiles(files); const activeItem = qm.items[0]; // Simulate item starting qm.activeWorkers.add(activeItem.id); activeItem.status = "uploading"; expect(qm.activeWorkers.has(activeItem.id)).toBe(true); // User removes active item mid-flight qm.removeItem(activeItem.id); expect(qm.items).toHaveLength(2); expect(qm.activeWorkers.has(activeItem.id)).toBe(false); expect(qm.items.find((i) => i.id === activeItem.id)).toBeUndefined(); // Now run queue to ensure remaining items process smoothly await qm.runQueue(async (file) => { return { success: true, receipts: [{ id: `rcpt-${file.name}` } as any], }; }); const stats = qm.getStats(); expect(stats.total).toBe(2); expect(stats.success).toBe(2); }); test("ADV-5: Flaky upload — 3 consecutive failures followed by retry success", async () => { const file = createMockFile("flaky_connection.pdf", 120000, "application/pdf"); qm.addFiles([file]); const item = qm.items[0]; let attempts = 0; const flakyHandler = async () => { attempts++; if (attempts < 4) { return { success: false, error: `Netzwerkfehler (Versuch ${attempts})` }; } return { success: true, receipts: [{ id: "rcpt-flaky-ok", merchant: { name: "Telekom" } } as any], }; }; // Attempt 1: Fail await qm.processItem(item.id, flakyHandler); expect(item.status).toBe("error"); expect(item.error).toContain("Versuch 1"); // Retry 1 (Attempt 2): Fail qm.retryItem(item.id); expect(item.status).toBe("queued"); expect(item.error).toBeUndefined(); await qm.processItem(item.id, flakyHandler); expect(item.status).toBe("error"); expect(item.error).toContain("Versuch 2"); // Retry 2 (Attempt 3): Fail qm.retryItem(item.id); await qm.processItem(item.id, flakyHandler); expect(item.status).toBe("error"); expect(item.error).toContain("Versuch 3"); // Retry 3 (Attempt 4): Succeed! qm.retryItem(item.id); await qm.processItem(item.id, flakyHandler); expect(item.status).toBe("success"); expect(item.error).toBeUndefined(); expect(item.receiptResult?.merchant?.name).toBe("Telekom"); }); test("ADV-6: Duplicate Ingestion Handling — Deduplicates exact file additions but allows distinct sizes", () => { const file1 = createMockFile("invoice_2026.pdf", 100000, "application/pdf"); const file1Duplicate = createMockFile("invoice_2026.pdf", 100000, "application/pdf"); const file1DifferentSize = createMockFile("invoice_2026.pdf", 250000, "application/pdf"); qm.addFiles([file1]); expect(qm.items).toHaveLength(1); // Exact duplicate drop -> rejected from adding new queue item qm.addFiles([file1Duplicate]); expect(qm.items).toHaveLength(1); // Same filename but different size -> permitted qm.addFiles([file1DifferentSize]); expect(qm.items).toHaveLength(2); }); test("ADV-7: AI Duplicate Flagging — Flags suspected duplicates with score >= 0.7 as 'duplicate_suspected'", async () => { const file = createMockFile("potential_duplicate.jpg", 140000, "image/jpeg"); qm.addFiles([file]); await qm.processItem(qm.items[0].id, async () => { return { success: true, duplicateScore: 0.95, receipts: [{ id: "rcpt-dup-1", merchant: { name: "Shell" } } as any], }; }); const item = qm.items[0]; expect(item.status).toBe("duplicate_suspected"); expect(item.stageMessage).toBe("Duplikat erkannt"); }); });