scan-receipts/tests/e2e/m1_adversarial.test.ts

/**
 * Milestone 1 (R1): Ingestion & Batch Upload Adversarial Stress Suite
 * Empirical Challenger Verification
 * 
 * Verifies:
 * - Empty (0-byte) and truncated files
 * - Boundary file sizes (10MB boundary, 10MB+1, 500MB)
 * - 50-file massive batch queue load & strict concurrency throttle
 * - Mid-flight item removal and worker recovery
 * - Multi-attempt failure and retry cycles
 * - Exact vs collision duplicate filtering
 * - Server failure error boundaries & memory cleanup
 */

import { describe, test, it, expect, beforeEach } from "./runner";
import { ProcessedReceipt } from "../../src/lib/schema/receipt";
import { BatchQueueItem } from "../../src/components/dashboard/BatchUploadDrawer";

function createMockFile(name: string, sizeBytes: number, mimeType: string): File {
  const buffer = new Uint8Array(Math.min(sizeBytes, 1024)); // avoid allocating 500MB buffer in memory
  const blob = new Blob([buffer], { type: mimeType });
  const file = new File([blob], name, { type: mimeType, lastModified: Date.now() });
  // Explicitly override size property for boundary testing
  Object.defineProperty(file, "size", { value: sizeBytes });
  return file;
}

class AdversarialBatchQueueManager {
  items: BatchQueueItem[] = [];
  activeWorkers = new Set<string>();
  maxConcurrent = 2;
  maxFileSize = 10 * 1024 * 1024;
  processedReceipts: ProcessedReceipt[] = [];
  maxSimultaneousObserved = 0;

  addFiles(files: File[]) {
    for (const file of files) {
      const isPdf =
        file.type === "application/pdf" || file.name.toLowerCase().endsWith(".pdf");
      const previewThumbnailUrl = !isPdf
        ? `blob:http://localhost/mock-thumb-${file.name}`
        : undefined;

      const isDuplicate = this.items.some(
        (i) => i.fileName === file.name && i.fileSizeBytes === file.size
      );
      if (isDuplicate) continue;

      this.items.push({
        id: `upload_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`,
        file,
        fileName: file.name || "unnamed_file",
        fileSizeBytes: file.size || 0,
        mimeType: file.type || (isPdf ? "application/pdf" : "image/jpeg"),
        previewThumbnailUrl,
        isPdf,
        status: "queued",
        progressPercent: 0,
        stageMessage: "In Warteschlange...",
      });
    }
  }

  getStats() {
    const total = this.items.length;
    const inProgress = this.items.filter((i) =>
      ["queued", "preprocessing", "uploading", "extracting"].includes(i.status)
    ).length;
    const success = this.items.filter(
      (i) => i.status === "success" || i.status === "duplicate_suspected"
    ).length;
    const failed = this.items.filter((i) => i.status === "error").length;
    const aggregatePercent =
      total === 0
        ? 0
        : Math.round(
            this.items.reduce((acc, curr) => acc + curr.progressPercent, 0) / total
          );

    return { total, inProgress, success, failed, aggregatePercent };
  }

  async processItem(
    itemId: string,
    handler?: (file: File) => Promise<{ success: boolean; receipts?: any[]; error?: string; duplicateScore?: number }>
  ) {
    const item = this.items.find((i) => i.id === itemId);
    if (!item) return;

    this.activeWorkers.add(itemId);
    this.maxSimultaneousObserved = Math.max(
      this.maxSimultaneousObserved,
      this.activeWorkers.size
    );

    // 1. Pre-flight Validation
    if (item.fileSizeBytes > this.maxFileSize) {
      item.status = "error";
      item.progressPercent = 100;
      item.stageMessage = "Datei zu groß";
      item.error = `Die Datei überschreitet das Limit von 10 MB.`;
      this.activeWorkers.delete(itemId);
      return;
    }

    if (item.fileSizeBytes === 0) {
      // Empty file handling
      item.status = "error";
      item.progressPercent = 100;
      item.stageMessage = "Leere Datei";
      item.error = "Die Datei enthält keine Daten (0 Bytes).";
      this.activeWorkers.delete(itemId);
      return;
    }

    // 2. Preprocessing
    item.status = "preprocessing";
    item.progressPercent = 25;

    // 3. Uploading
    item.status = "uploading";
    item.progressPercent = 50;

    // 4. Extracting
    item.status = "extracting";
    item.progressPercent = 75;

    try {
      if (handler) {
        const res = await handler(item.file);
        if (!res.success) {
          throw new Error(res.error || "Serverfehler bei der Extraktion");
        }

        const receipts = res.receipts || [];
        if (receipts.length === 0) {
          throw new Error("Keine Belegdaten aus dieser Datei extrahiert");
        }

        const isDuplicate = (res.duplicateScore || 0) >= 0.7;
        item.status = isDuplicate ? "duplicate_suspected" : "success";
        item.progressPercent = 100;
        item.stageMessage = isDuplicate ? "Duplikat erkannt" : "Erfolgreich erfasst";
        item.receiptResult = receipts[0];
        this.processedReceipts.push(...receipts);
      } else {
        item.status = "success";
        item.progressPercent = 100;
        item.stageMessage = "Erfolgreich erfasst";
      }
    } catch (err: any) {
      item.status = "error";
      item.progressPercent = 100;
      item.stageMessage = "Fehlgeschlagen";
      item.error = err.message || "Unbekannter Verarbeitungsfehler";
    } finally {
      this.activeWorkers.delete(itemId);
    }
  }

  async runQueue(
    handler?: (file: File) => Promise<{ success: boolean; receipts?: any[]; error?: string; duplicateScore?: number }>
  ) {
    while (true) {
      const queued = this.items.filter((i) => i.status === "queued");
      if (queued.length === 0) break;

      const availableSlots = this.maxConcurrent - this.activeWorkers.size;
      if (availableSlots <= 0) {
        await new Promise((res) => setTimeout(res, 5));
        continue;
      }

      const nextBatch = queued.slice(0, availableSlots);
      await Promise.all(nextBatch.map((item) => this.processItem(item.id, handler)));
    }
  }

  removeItem(itemId: string) {
    this.activeWorkers.delete(itemId);
    this.items = this.items.filter((i) => i.id !== itemId);
  }

  retryItem(itemId: string) {
    const item = this.items.find((i) => i.id === itemId);
    if (item) {
      item.status = "queued";
      item.progressPercent = 0;
      item.error = undefined;
      item.stageMessage = "In Warteschlange...";
    }
  }
}

describe("Milestone 1: Ingestion & Batch Upload — Adversarial Stress Testing", () => {
  let qm: AdversarialBatchQueueManager;

  beforeEach(() => {
    qm = new AdversarialBatchQueueManager();
  });

  test("ADV-1: Empty (0-byte) files are caught and marked as error without halting the batch", async () => {
    const files = [
      createMockFile("empty_receipt.pdf", 0, "application/pdf"),
      createMockFile("valid_receipt.jpg", 150000, "image/jpeg"),
    ];

    qm.addFiles(files);
    expect(qm.items).toHaveLength(2);

    await qm.runQueue(async (file) => {
      return {
        success: true,
        receipts: [{ id: `rcpt-${file.name}`, merchant: { name: "Aral" } } as any],
      };
    });

    const emptyItem = qm.items.find((i) => i.fileName === "empty_receipt.pdf");
    expect(emptyItem?.status).toBe("error");
    expect(emptyItem?.error).toContain("0 Bytes");

    const validItem = qm.items.find((i) => i.fileName === "valid_receipt.jpg");
    expect(validItem?.status).toBe("success");

    const stats = qm.getStats();
    expect(stats.total).toBe(2);
    expect(stats.failed).toBe(1);
    expect(stats.success).toBe(1);
  });

  test("ADV-2: Exact Boundary Testing — 10MB limit passes, 10MB+1 byte fails, 500MB fails instantly", async () => {
    const limitBytes = 10 * 1024 * 1024;
    const exactLimitFile = createMockFile("exact_10mb.pdf", limitBytes, "application/pdf");
    const overLimitFile = createMockFile("over_10mb.pdf", limitBytes + 1, "application/pdf");
    const massiveFile = createMockFile("huge_500mb.pdf", 500 * 1024 * 1024, "application/pdf");

    qm.addFiles([exactLimitFile, overLimitFile, massiveFile]);
    expect(qm.items).toHaveLength(3);

    await qm.runQueue(async (file) => {
      return {
        success: true,
        receipts: [{ id: `rcpt-${file.name}` } as any],
      };
    });

    const exactItem = qm.items.find((i) => i.fileName === "exact_10mb.pdf");
    expect(exactItem?.status).toBe("success");

    const overItem = qm.items.find((i) => i.fileName === "over_10mb.pdf");
    expect(overItem?.status).toBe("error");
    expect(overItem?.error).toContain("10 MB");

    const massiveItem = qm.items.find((i) => i.fileName === "huge_500mb.pdf");
    expect(massiveItem?.status).toBe("error");
    expect(massiveItem?.error).toContain("10 MB");
  });

  test("ADV-3: Extreme Multi-File Burst (50 files) strictly preserves <=2 concurrency and calculates 100% aggregate progress", async () => {
    const files: File[] = [];
    for (let i = 1; i <= 50; i++) {
      files.push(createMockFile(`batch_file_${i}.jpg`, 10000 + i, "image/jpeg"));
    }

    qm.addFiles(files);
    expect(qm.items).toHaveLength(50);

    // Verify all 50 IDs are distinct
    const ids = new Set(qm.items.map((i) => i.id));
    expect(ids.size).toBe(50);

    await qm.runQueue(async (file) => {
      // Simulate async processing
      await new Promise((res) => setTimeout(res, 1));
      return {
        success: true,
        receipts: [{ id: `rcpt-${file.name}`, merchant: { name: `Store ${file.name}` } } as any],
      };
    });

    expect(qm.maxSimultaneousObserved).toBeLessThanOrEqual(2);
    const stats = qm.getStats();
    expect(stats.total).toBe(50);
    expect(stats.success).toBe(50);
    expect(stats.failed).toBe(0);
    expect(stats.aggregatePercent).toBe(100);
  });

  test("ADV-4: Mid-flight item removal recovers active worker slots immediately for subsequent queued items", async () => {
    const files = [
      createMockFile("active_item.jpg", 200000, "image/jpeg"),
      createMockFile("queued_item1.jpg", 150000, "image/jpeg"),
      createMockFile("queued_item2.jpg", 180000, "image/jpeg"),
    ];

    qm.addFiles(files);
    const activeItem = qm.items[0];

    // Simulate item starting
    qm.activeWorkers.add(activeItem.id);
    activeItem.status = "uploading";

    expect(qm.activeWorkers.has(activeItem.id)).toBe(true);

    // User removes active item mid-flight
    qm.removeItem(activeItem.id);

    expect(qm.items).toHaveLength(2);
    expect(qm.activeWorkers.has(activeItem.id)).toBe(false);
    expect(qm.items.find((i) => i.id === activeItem.id)).toBeUndefined();

    // Now run queue to ensure remaining items process smoothly
    await qm.runQueue(async (file) => {
      return {
        success: true,
        receipts: [{ id: `rcpt-${file.name}` } as any],
      };
    });

    const stats = qm.getStats();
    expect(stats.total).toBe(2);
    expect(stats.success).toBe(2);
  });

  test("ADV-5: Flaky upload — 3 consecutive failures followed by retry success", async () => {
    const file = createMockFile("flaky_connection.pdf", 120000, "application/pdf");
    qm.addFiles([file]);
    const item = qm.items[0];

    let attempts = 0;
    const flakyHandler = async () => {
      attempts++;
      if (attempts < 4) {
        return { success: false, error: `Netzwerkfehler (Versuch ${attempts})` };
      }
      return {
        success: true,
        receipts: [{ id: "rcpt-flaky-ok", merchant: { name: "Telekom" } } as any],
      };
    };

    // Attempt 1: Fail
    await qm.processItem(item.id, flakyHandler);
    expect(item.status).toBe("error");
    expect(item.error).toContain("Versuch 1");

    // Retry 1 (Attempt 2): Fail
    qm.retryItem(item.id);
    expect(item.status).toBe("queued");
    expect(item.error).toBeUndefined();
    await qm.processItem(item.id, flakyHandler);
    expect(item.status).toBe("error");
    expect(item.error).toContain("Versuch 2");

    // Retry 2 (Attempt 3): Fail
    qm.retryItem(item.id);
    await qm.processItem(item.id, flakyHandler);
    expect(item.status).toBe("error");
    expect(item.error).toContain("Versuch 3");

    // Retry 3 (Attempt 4): Succeed!
    qm.retryItem(item.id);
    await qm.processItem(item.id, flakyHandler);
    expect(item.status).toBe("success");
    expect(item.error).toBeUndefined();
    expect(item.receiptResult?.merchant?.name).toBe("Telekom");
  });

  test("ADV-6: Duplicate Ingestion Handling — Deduplicates exact file additions but allows distinct sizes", () => {
    const file1 = createMockFile("invoice_2026.pdf", 100000, "application/pdf");
    const file1Duplicate = createMockFile("invoice_2026.pdf", 100000, "application/pdf");
    const file1DifferentSize = createMockFile("invoice_2026.pdf", 250000, "application/pdf");

    qm.addFiles([file1]);
    expect(qm.items).toHaveLength(1);

    // Exact duplicate drop -> rejected from adding new queue item
    qm.addFiles([file1Duplicate]);
    expect(qm.items).toHaveLength(1);

    // Same filename but different size -> permitted
    qm.addFiles([file1DifferentSize]);
    expect(qm.items).toHaveLength(2);
  });

  test("ADV-7: AI Duplicate Flagging — Flags suspected duplicates with score >= 0.7 as 'duplicate_suspected'", async () => {
    const file = createMockFile("potential_duplicate.jpg", 140000, "image/jpeg");
    qm.addFiles([file]);

    await qm.processItem(qm.items[0].id, async () => {
      return {
        success: true,
        duplicateScore: 0.95,
        receipts: [{ id: "rcpt-dup-1", merchant: { name: "Shell" } } as any],
      };
    });

    const item = qm.items[0];
    expect(item.status).toBe("duplicate_suspected");
    expect(item.stageMessage).toBe("Duplikat erkannt");
  });
});