bizmatch-app/scripts/acceptance-nda-refresh.mjs

#!/usr/bin/env node
/**
 * Acceptance for the NDA refresh: incremental mode, the meta-key hygiene of
 * the two background jobs, and the coverage marker.
 *
 * It runs the real server against `scripts/ds-stub.mjs` (a Dropbox Sign whose
 * search index deliberately lags its list) and the real development database,
 * per the README convention:
 *
 *  * every stubbed request id starts with `req-`, and they are deleted again;
 *  * the `ds_*` keys in `app_meta` are snapshotted before the run and written
 *    back afterwards, so a test run cannot leave the mirror looking refreshed
 *    when it is not.
 *
 * Nothing else in the database is touched: the stub only ever hands out `req-`
 * requests, and the ids of the real mirror are unknown to it.
 *
 *   node scripts/acceptance-nda-refresh.mjs
 */
import { spawn } from 'node:child_process';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import pg from 'pg';
import { createStub } from './ds-stub.mjs';

const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const APP_PORT = 8099;
const NDA_PREFIX = 'Buyer Forms - NDA';
/** Before RENAME_DATE, so the full reload runs its pre-rename leg too. */
const BOUNDARY_DAY = '2026-05-01';
const LEGACY_DAY = '2026-05-02';
const META_KEYS = [
  'ds_refresh_state',
  'ds_last_refresh_at',
  'ds_last_refresh_result',
  'ds_mirror_covers_from',
  'ds_sync_state',
  'ds_last_sync_at',
  'ds_last_sync_result',
];

// --------------------------------------------------------------- helpers
let failures = 0;
function check(name, ok, detail = '') {
  console.log(`${ok ? '  ok  ' : ' FAIL '} ${name}${detail ? ` — ${detail}` : ''}`);
  if (!ok) failures += 1;
}

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

/** `npm run dev` sources .env by hand, so the script does the same. */
async function loadEnv() {
  const text = await readFile(path.join(ROOT, '.env'), 'utf8').catch(() => '');
  for (const line of text.split('\n')) {
    const match = /^\s*([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line);
    if (!match) continue;
    const value = match[2].trim().replace(/^["'](.*)["']$/, '$1');
    if (process.env[match[1]] === undefined) process.env[match[1]] = value;
  }
}

const seconds = (date) => Math.floor(date.getTime() / 1000);
const daysAgo = (n) => seconds(new Date(Date.now() - n * 86_400_000));

function request(id, title, createdAt, { signedAt = null, name, email } = {}) {
  return {
    signature_request_id: id,
    title,
    created_at: createdAt,
    is_complete: signedAt !== null,
    is_declined: false,
    files_url: '',
    signatures: [
      {
        signer_name: name,
        signer_email_address: email,
        status_code: signedAt ? 'signed' : 'awaiting_signature',
        signed_at: signedAt,
      },
    ],
    response_data: [],
  };
}

/**
 * The fixture, in the order the assertions reason about:
 *
 * | # | request | why |
 * | 0 | created now, **not in the index** | the lag: only the plain list has it |
 * | 1 | created 40 days ago, signed yesterday, **stale in the index** | activity order puts it on page 1 |
 * | 2.. | 200 unrelated requests | fill pages 1-2, so anything below is out of the incremental pass' reach |
 * | 202 | created on the boundary day | only an inclusive `[day TO *]` returns it |
 * | 203 | pre-rename title | exercises the second leg of the full reload |
 */
function buildFixture(stamp) {
  const fresh = request(`req-${stamp}-fresh`, `${NDA_PREFIX} Nora Fresh - Acme Diner`, daysAgo(0), {
    name: 'Nora Fresh',
    email: `nora.fresh@stubtest.invalid`,
  });
  const oldSigned = request(
    `req-${stamp}-old-signed`,
    `${NDA_PREFIX} Otto Older - Old Deli`,
    daysAgo(40),
    { signedAt: daysAgo(1), name: 'Otto Older', email: 'otto.older@stubtest.invalid' },
  );
  const boundary = request(
    `req-${stamp}-boundary`,
    `${NDA_PREFIX} Bea Boundary - Border Cafe`,
    seconds(new Date(`${BOUNDARY_DAY}T09:00:00Z`)),
    { name: 'Bea Boundary', email: 'bea.boundary@stubtest.invalid' },
  );
  const legacy = request(
    `req-${stamp}-legacy`,
    'Buyer Forms Regina Legacy - Old Bakery',
    seconds(new Date(`${LEGACY_DAY}T11:00:00Z`)),
    { name: 'Regina Legacy', email: 'regina.legacy@stubtest.invalid' },
  );

  // Everything the account is full of and the refresh is not interested in.
  const noise = Array.from({ length: 200 }, (_, i) =>
    request(`req-${stamp}-noise-${i}`, `Other Form ${i} - Someone`, daysAgo(2 + i * 0.05), {
      name: `Noise ${i}`,
      email: `noise${i}@stubtest.invalid`,
    }),
  );

  /** The index copy of the freshly signed one: still pending, as it was. */
  const staleIndexCopy = request(
    oldSigned.signature_request_id,
    oldSigned.title,
    oldSigned.created_at,
    { name: 'Otto Older', email: 'otto.older@stubtest.invalid' },
  );

  return {
    ids: {
      fresh: fresh.signature_request_id,
      oldSigned: oldSigned.signature_request_id,
      boundary: boundary.signature_request_id,
      legacy: legacy.signature_request_id,
    },
    fixture: [
      { request: fresh, indexed: null }, // the lag: not searchable yet at all
      { request: oldSigned, indexed: staleIndexCopy }, // the lag: stale copy
      ...noise.map((r) => ({ request: r, indexed: r })),
      { request: boundary, indexed: boundary },
      { request: legacy, indexed: legacy },
    ],
  };
}

// ------------------------------------------------------------------ main
await loadEnv();
if (!process.env.DATABASE_URL) {
  console.error('DATABASE_URL is missing — run this from the project root with .env in place');
  process.exit(2);
}

const db = new pg.Pool({ connectionString: process.env.DATABASE_URL });
const stamp = String(Date.now());
const { ids, fixture } = buildFixture(stamp);
const stub = createStub(fixture);
let app = null;
let scratch = null;
let metaSnapshot = [];
const appLog = [];

const cleanup = async () => {
  if (app && !app.killed) {
    app.kill('SIGTERM');
    await sleep(300);
  }
  await stub.close().catch(() => {});
  await db
    .query(`DELETE FROM ds_request WHERE signature_request_id LIKE 'req-%'`)
    .catch((err) => console.error(`cleanup of ds_request failed: ${err.message}`));
  // The meta keys go back exactly as they were, updated_at included: a test
  // run must not leave "last refreshed" pointing at itself.
  await db
    .query('DELETE FROM app_meta WHERE key = ANY($1)', [META_KEYS])
    .catch((err) => console.error(`cleanup of app_meta failed: ${err.message}`));
  for (const row of metaSnapshot) {
    await db
      .query('INSERT INTO app_meta (key, value, updated_at) VALUES ($1, $2, $3)', [
        row.key,
        row.value,
        row.updated_at,
      ])
      .catch((err) => console.error(`restoring ${row.key} failed: ${err.message}`));
  }
  if (scratch) await rm(scratch, { recursive: true, force: true }).catch(() => {});
  await db.end().catch(() => {});
};
process.on('SIGINT', async () => {
  await cleanup();
  process.exit(130);
});

const meta = async (key) => {
  const { rows } = await db.query('SELECT value, updated_at FROM app_meta WHERE key = $1', [key]);
  return rows[0] ?? null;
};
const setMeta = (key, value) =>
  db.query(
    `INSERT INTO app_meta (key, value) VALUES ($1, $2)
     ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`,
    [key, value],
  );
const mirrored = async (id) => {
  const { rows } = await db.query(
    'SELECT status, signed_at, created_at FROM ds_request WHERE signature_request_id = $1',
    [id],
  );
  return rows[0] ?? null;
};

try {
  metaSnapshot = (
    await db.query('SELECT key, value, updated_at FROM app_meta WHERE key = ANY($1)', [META_KEYS])
  ).rows;
  await db.query(`DELETE FROM ds_request WHERE signature_request_id LIKE 'req-%'`);

  const stubUrl = await stub.listen(0);
  scratch = await mkdtemp(path.join(tmpdir(), 'bizmatch-nda-'));

  // ------------------------------------------------------------ the app
  app = spawn('npx', ['tsx', 'src/server.ts'], {
    cwd: ROOT,
    env: {
      ...process.env,
      PORT: String(APP_PORT),
      DROPBOX_SIGN_API_KEY: 'stub-key',
      DROPBOX_SIGN_BASE_URL: stubUrl,
      NDA_ROOT: scratch,
      // Keeps the sync's per-id re-check off the real mirror: this run is
      // about which meta keys it writes, not about re-reading 800 rows.
      SYNC_PENDING_MAX_AGE_DAYS: '0',
    },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  for (const stream of [app.stdout, app.stderr]) {
    stream.setEncoding('utf8');
    stream.on('data', (chunk) => {
      for (const line of chunk.split('\n').filter(Boolean)) appLog.push(line);
    });
  }

  const base = `http://127.0.0.1:${APP_PORT}`;
  const { rows: staff } = await db.query('SELECT id FROM staff ORDER BY created_at LIMIT 1');
  if (!staff[0]) throw new Error('no staff row to sign in as');
  const headers = { cookie: `bizmatch_staff=${staff[0].id}` };

  for (let attempt = 0; ; attempt += 1) {
    if (attempt > 100) throw new Error('the app did not come up');
    const ok = await fetch(`${base}/api/health`)
      .then((r) => r.ok)
      .catch(() => false);
    if (ok) break;
    await sleep(300);
  }

  const inbox = async () =>
    (await fetch(`${base}/api/nda-inbox?since=2026-01-01`, { headers })).json();
  const settle = async (field) => {
    for (let attempt = 0; attempt < 300; attempt += 1) {
      const body = await inbox();
      if (body[field] !== 'running') return body;
      await sleep(500);
    }
    throw new Error(`${field} never settled`);
  };

  // ---------------------------------------- 1 · the incremental pass
  console.log('\n1 · incremental refresh over a lagging search index');
  await setMeta('ds_last_refresh_at', new Date(Date.now() - 3_600_000).toISOString());
  await setMeta('ds_refresh_state', 'idle');
  // Deliberately narrow, so the coverage assertions have something to widen.
  await setMeta('ds_mirror_covers_from', '2026-12-31T00:00:00.000Z');

  const searchable = stub.search(`title:"${NDA_PREFIX}" AND created:[${BOUNDARY_DAY} TO *]`);
  check(
    'the stub search index really lags the list',
    !searchable.some((r) => r.signature_request_id === ids.fresh) &&
      searchable.some((r) => r.signature_request_id === ids.boundary),
    'the fresh request is in no search result, the older ones are',
  );

  const started = await fetch(`${base}/api/nda-inbox/refresh`, { method: 'POST', headers });
  check('POST /refresh answers 202', started.status === 202, `got ${started.status}`);
  const afterIncremental = await settle('refresh_state');
  const incremental = afterIncremental.last_refresh_result;

  check('it ran in incremental mode', incremental?.mode === 'incremental', JSON.stringify(incremental));
  check(
    'it read the unfiltered list, not the index',
    stub.calls.filter((c) => c.endpoint === 'list' && c.query).length === 0,
    `queries sent: ${stub.calls.filter((c) => c.query).length}`,
  );
  check(
    `it walked ${2} pages of 100 unfiltered rows`,
    incremental?.pages === 2 && incremental?.seen === 200,
    `pages=${incremental?.pages} seen=${incremental?.seen}`,
  );

  const fresh = await mirrored(ids.fresh);
  check(
    'a request created now lands although the index does not have it',
    fresh?.status === 'pending',
    `mirror row: ${JSON.stringify(fresh)}`,
  );
  const oldSigned = await mirrored(ids.oldSigned);
  check(
    'an old pending signed yesterday is picked up without a sync',
    oldSigned?.status === 'signed' && oldSigned?.signed_at !== null,
    `mirror row: ${JSON.stringify(oldSigned)}`,
  );
  check(
    'and it stored nothing beyond those two',
    incremental?.stored === 2,
    `stored=${incremental?.stored}`,
  );
  check(
    'rows below the first two pages are out of its reach',
    (await mirrored(ids.boundary)) === null && (await mirrored(ids.legacy)) === null,
    'boundary and pre-rename rows are not mirrored yet',
  );

  // ---------------------------------------- 2 · the full reload
  console.log('\n2 · full reload with inclusive bounds');
  const callsBefore = stub.calls.length;
  const reload = await fetch(`${base}/api/nda-inbox/refresh?since=${BOUNDARY_DAY}`, {
    method: 'POST',
    headers,
  });
  check('POST /refresh?since= answers 202', reload.status === 202, `got ${reload.status}`);
  const afterFull = await settle('refresh_state');
  const full = afterFull.last_refresh_result;
  const queries = stub.calls.slice(callsBefore).filter((c) => c.query).map((c) => c.query);

  check('it ran in full mode', full?.mode === 'full', JSON.stringify(full));
  check(
    'every query uses the inclusive range form',
    queries.length > 0 && queries.every((q) => q.includes('created:[')) &&
      !queries.some((q) => q.includes('created:{')),
    queries[0] ?? 'no query sent',
  );
  check(
    'a request created on the boundary day is included',
    (await mirrored(ids.boundary)) !== null,
    `${BOUNDARY_DAY}T09:00Z with since=${BOUNDARY_DAY}`,
  );
  check(
    'the pre-rename leg ran and stored its request',
    (await mirrored(ids.legacy)) !== null && full?.legacy_stored >= 1,
    `legacy_stored=${full?.legacy_stored}`,
  );
  check(
    'covers_from widened to the reloaded window',
    (await meta('ds_mirror_covers_from'))?.value === `${BOUNDARY_DAY}T00:00:00.000Z`,
    `covers_from=${(await meta('ds_mirror_covers_from'))?.value}`,
  );

  // ---------------------------------------- 3 · coverage never narrows
  console.log('\n3 · covers_from only ever widens');
  await setMeta('ds_mirror_covers_from', '2020-01-01T00:00:00.000Z');
  await fetch(`${base}/api/nda-inbox/refresh?since=${BOUNDARY_DAY}`, { method: 'POST', headers });
  await settle('refresh_state');
  check(
    'a narrower window does not overwrite a wider coverage',
    (await meta('ds_mirror_covers_from'))?.value === '2020-01-01T00:00:00.000Z',
    `covers_from=${(await meta('ds_mirror_covers_from'))?.value}`,
  );

  // ---------------------------------------- 4 · meta-key hygiene
  console.log('\n4 · the sync never touches the refresh cursor');
  const refreshBefore = await meta('ds_last_refresh_at');
  const synced = await fetch(`${base}/api/nda-inbox/sync`, { method: 'POST', headers });
  check('POST /sync answers 202', synced.status === 202, `got ${synced.status}`);
  await settle('sync_state');
  const refreshAfter = await meta('ds_last_refresh_at');
  check(
    'ds_last_refresh_at is untouched by a sync run',
    refreshBefore?.value === refreshAfter?.value &&
      String(refreshBefore?.updated_at) === String(refreshAfter?.updated_at),
    `${refreshBefore?.value} @ ${refreshBefore?.updated_at} -> ${refreshAfter?.value} @ ${refreshAfter?.updated_at}`,
  );
  check(
    'the sync stamps its own key instead',
    (await meta('ds_last_sync_at'))?.value != null,
    `ds_last_sync_at=${(await meta('ds_last_sync_at'))?.value}`,
  );

  // ---------------------------------------- the log line
  console.log('\nwalk log lines the run produced:');
  for (const line of appLog) {
    let msg = line;
    try {
      msg = JSON.parse(line).msg ?? line;
    } catch {
      /* not JSON — a tsx or npm line */
    }
    if (typeof msg === 'string' && msg.includes('[nda-refresh] walk')) console.log(`  ${msg}`);
  }
} catch (err) {
  failures += 1;
  console.error(`\nrun failed: ${err.stack ?? err.message}`);
} finally {
  await cleanup();
}

console.log(failures === 0 ? '\nall assertions passed' : `\n${failures} assertion(s) failed`);
process.exit(failures === 0 ? 0 : 1);