fix nda

scripts/acceptance-nda-refresh.mjs

@@ -0,0 +1,414 @@
+#!/usr/bin/env node
+/**
+ * Acceptance for the NDA refresh: incremental mode, the meta-key hygiene of
+ * the two background jobs, and the coverage marker.
+ *
+ * It runs the real server against `scripts/ds-stub.mjs` (a Dropbox Sign whose
+ * search index deliberately lags its list) and the real development database,
+ * per the README convention:
+ *
+ *  * every stubbed request id starts with `req-`, and they are deleted again;
+ *  * the `ds_*` keys in `app_meta` are snapshotted before the run and written
+ *    back afterwards, so a test run cannot leave the mirror looking refreshed
+ *    when it is not.
+ *
+ * Nothing else in the database is touched: the stub only ever hands out `req-`
+ * requests, and the ids of the real mirror are unknown to it.
+ *
+ *   node scripts/acceptance-nda-refresh.mjs
+ */
+import { spawn } from 'node:child_process';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import pg from 'pg';
+import { createStub } from './ds-stub.mjs';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const APP_PORT = 8099;
+const NDA_PREFIX = 'Buyer Forms - NDA';
+/** Before RENAME_DATE, so the full reload runs its pre-rename leg too. */
+const BOUNDARY_DAY = '2026-05-01';
+const LEGACY_DAY = '2026-05-02';
+const META_KEYS = [
+  'ds_refresh_state',
+  'ds_last_refresh_at',
+  'ds_last_refresh_result',
+  'ds_mirror_covers_from',
+  'ds_sync_state',
+  'ds_last_sync_at',
+  'ds_last_sync_result',
+];
+
+// --------------------------------------------------------------- helpers
+let failures = 0;
+function check(name, ok, detail = '') {
+  console.log(`${ok ? '  ok  ' : ' FAIL '} ${name}${detail ? ` — ${detail}` : ''}`);
+  if (!ok) failures += 1;
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/** `npm run dev` sources .env by hand, so the script does the same. */
+async function loadEnv() {
+  const text = await readFile(path.join(ROOT, '.env'), 'utf8').catch(() => '');
+  for (const line of text.split('\n')) {
+    const match = /^\s*([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line);
+    if (!match) continue;
+    const value = match[2].trim().replace(/^["'](.*)["']$/, '$1');
+    if (process.env[match[1]] === undefined) process.env[match[1]] = value;
+  }
+}
+
+const seconds = (date) => Math.floor(date.getTime() / 1000);
+const daysAgo = (n) => seconds(new Date(Date.now() - n * 86_400_000));
+
+function request(id, title, createdAt, { signedAt = null, name, email } = {}) {
+  return {
+    signature_request_id: id,
+    title,
+    created_at: createdAt,
+    is_complete: signedAt !== null,
+    is_declined: false,
+    files_url: '',
+    signatures: [
+      {
+        signer_name: name,
+        signer_email_address: email,
+        status_code: signedAt ? 'signed' : 'awaiting_signature',
+        signed_at: signedAt,
+      },
+    ],
+    response_data: [],
+  };
+}
+
+/**
+ * The fixture, in the order the assertions reason about:
+ *
+ * | # | request | why |
+ * | 0 | created now, **not in the index** | the lag: only the plain list has it |
+ * | 1 | created 40 days ago, signed yesterday, **stale in the index** | activity order puts it on page 1 |
+ * | 2.. | 200 unrelated requests | fill pages 1-2, so anything below is out of the incremental pass' reach |
+ * | 202 | created on the boundary day | only an inclusive `[day TO *]` returns it |
+ * | 203 | pre-rename title | exercises the second leg of the full reload |
+ */
+function buildFixture(stamp) {
+  const fresh = request(`req-${stamp}-fresh`, `${NDA_PREFIX} Nora Fresh - Acme Diner`, daysAgo(0), {
+    name: 'Nora Fresh',
+    email: `nora.fresh@stubtest.invalid`,
+  });
+  const oldSigned = request(
+    `req-${stamp}-old-signed`,
+    `${NDA_PREFIX} Otto Older - Old Deli`,
+    daysAgo(40),
+    { signedAt: daysAgo(1), name: 'Otto Older', email: 'otto.older@stubtest.invalid' },
+  );
+  const boundary = request(
+    `req-${stamp}-boundary`,
+    `${NDA_PREFIX} Bea Boundary - Border Cafe`,
+    seconds(new Date(`${BOUNDARY_DAY}T09:00:00Z`)),
+    { name: 'Bea Boundary', email: 'bea.boundary@stubtest.invalid' },
+  );
+  const legacy = request(
+    `req-${stamp}-legacy`,
+    'Buyer Forms Regina Legacy - Old Bakery',
+    seconds(new Date(`${LEGACY_DAY}T11:00:00Z`)),
+    { name: 'Regina Legacy', email: 'regina.legacy@stubtest.invalid' },
+  );
+
+  // Everything the account is full of and the refresh is not interested in.
+  const noise = Array.from({ length: 200 }, (_, i) =>
+    request(`req-${stamp}-noise-${i}`, `Other Form ${i} - Someone`, daysAgo(2 + i * 0.05), {
+      name: `Noise ${i}`,
+      email: `noise${i}@stubtest.invalid`,
+    }),
+  );
+
+  /** The index copy of the freshly signed one: still pending, as it was. */
+  const staleIndexCopy = request(
+    oldSigned.signature_request_id,
+    oldSigned.title,
+    oldSigned.created_at,
+    { name: 'Otto Older', email: 'otto.older@stubtest.invalid' },
+  );
+
+  return {
+    ids: {
+      fresh: fresh.signature_request_id,
+      oldSigned: oldSigned.signature_request_id,
+      boundary: boundary.signature_request_id,
+      legacy: legacy.signature_request_id,
+    },
+    fixture: [
+      { request: fresh, indexed: null }, // the lag: not searchable yet at all
+      { request: oldSigned, indexed: staleIndexCopy }, // the lag: stale copy
+      ...noise.map((r) => ({ request: r, indexed: r })),
+      { request: boundary, indexed: boundary },
+      { request: legacy, indexed: legacy },
+    ],
+  };
+}
+
+// ------------------------------------------------------------------ main
+await loadEnv();
+if (!process.env.DATABASE_URL) {
+  console.error('DATABASE_URL is missing — run this from the project root with .env in place');
+  process.exit(2);
+}
+
+const db = new pg.Pool({ connectionString: process.env.DATABASE_URL });
+const stamp = String(Date.now());
+const { ids, fixture } = buildFixture(stamp);
+const stub = createStub(fixture);
+let app = null;
+let scratch = null;
+let metaSnapshot = [];
+const appLog = [];
+
+const cleanup = async () => {
+  if (app && !app.killed) {
+    app.kill('SIGTERM');
+    await sleep(300);
+  }
+  await stub.close().catch(() => {});
+  await db
+    .query(`DELETE FROM ds_request WHERE signature_request_id LIKE 'req-%'`)
+    .catch((err) => console.error(`cleanup of ds_request failed: ${err.message}`));
+  // The meta keys go back exactly as they were, updated_at included: a test
+  // run must not leave "last refreshed" pointing at itself.
+  await db
+    .query('DELETE FROM app_meta WHERE key = ANY($1)', [META_KEYS])
+    .catch((err) => console.error(`cleanup of app_meta failed: ${err.message}`));
+  for (const row of metaSnapshot) {
+    await db
+      .query('INSERT INTO app_meta (key, value, updated_at) VALUES ($1, $2, $3)', [
+        row.key,
+        row.value,
+        row.updated_at,
+      ])
+      .catch((err) => console.error(`restoring ${row.key} failed: ${err.message}`));
+  }
+  if (scratch) await rm(scratch, { recursive: true, force: true }).catch(() => {});
+  await db.end().catch(() => {});
+};
+process.on('SIGINT', async () => {
+  await cleanup();
+  process.exit(130);
+});
+
+const meta = async (key) => {
+  const { rows } = await db.query('SELECT value, updated_at FROM app_meta WHERE key = $1', [key]);
+  return rows[0] ?? null;
+};
+const setMeta = (key, value) =>
+  db.query(
+    `INSERT INTO app_meta (key, value) VALUES ($1, $2)
+     ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`,
+    [key, value],
+  );
+const mirrored = async (id) => {
+  const { rows } = await db.query(
+    'SELECT status, signed_at, created_at FROM ds_request WHERE signature_request_id = $1',
+    [id],
+  );
+  return rows[0] ?? null;
+};
+
+try {
+  metaSnapshot = (
+    await db.query('SELECT key, value, updated_at FROM app_meta WHERE key = ANY($1)', [META_KEYS])
+  ).rows;
+  await db.query(`DELETE FROM ds_request WHERE signature_request_id LIKE 'req-%'`);
+
+  const stubUrl = await stub.listen(0);
+  scratch = await mkdtemp(path.join(tmpdir(), 'bizmatch-nda-'));
+
+  // ------------------------------------------------------------ the app
+  app = spawn('npx', ['tsx', 'src/server.ts'], {
+    cwd: ROOT,
+    env: {
+      ...process.env,
+      PORT: String(APP_PORT),
+      DROPBOX_SIGN_API_KEY: 'stub-key',
+      DROPBOX_SIGN_BASE_URL: stubUrl,
+      NDA_ROOT: scratch,
+      // Keeps the sync's per-id re-check off the real mirror: this run is
+      // about which meta keys it writes, not about re-reading 800 rows.
+      SYNC_PENDING_MAX_AGE_DAYS: '0',
+    },
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  for (const stream of [app.stdout, app.stderr]) {
+    stream.setEncoding('utf8');
+    stream.on('data', (chunk) => {
+      for (const line of chunk.split('\n').filter(Boolean)) appLog.push(line);
+    });
+  }
+
+  const base = `http://127.0.0.1:${APP_PORT}`;
+  const { rows: staff } = await db.query('SELECT id FROM staff ORDER BY created_at LIMIT 1');
+  if (!staff[0]) throw new Error('no staff row to sign in as');
+  const headers = { cookie: `bizmatch_staff=${staff[0].id}` };
+
+  for (let attempt = 0; ; attempt += 1) {
+    if (attempt > 100) throw new Error('the app did not come up');
+    const ok = await fetch(`${base}/api/health`)
+      .then((r) => r.ok)
+      .catch(() => false);
+    if (ok) break;
+    await sleep(300);
+  }
+
+  const inbox = async () =>
+    (await fetch(`${base}/api/nda-inbox?since=2026-01-01`, { headers })).json();
+  const settle = async (field) => {
+    for (let attempt = 0; attempt < 300; attempt += 1) {
+      const body = await inbox();
+      if (body[field] !== 'running') return body;
+      await sleep(500);
+    }
+    throw new Error(`${field} never settled`);
+  };
+
+  // ---------------------------------------- 1 · the incremental pass
+  console.log('\n1 · incremental refresh over a lagging search index');
+  await setMeta('ds_last_refresh_at', new Date(Date.now() - 3_600_000).toISOString());
+  await setMeta('ds_refresh_state', 'idle');
+  // Deliberately narrow, so the coverage assertions have something to widen.
+  await setMeta('ds_mirror_covers_from', '2026-12-31T00:00:00.000Z');
+
+  const searchable = stub.search(`title:"${NDA_PREFIX}" AND created:[${BOUNDARY_DAY} TO *]`);
+  check(
+    'the stub search index really lags the list',
+    !searchable.some((r) => r.signature_request_id === ids.fresh) &&
+      searchable.some((r) => r.signature_request_id === ids.boundary),
+    'the fresh request is in no search result, the older ones are',
+  );
+
+  const started = await fetch(`${base}/api/nda-inbox/refresh`, { method: 'POST', headers });
+  check('POST /refresh answers 202', started.status === 202, `got ${started.status}`);
+  const afterIncremental = await settle('refresh_state');
+  const incremental = afterIncremental.last_refresh_result;
+
+  check('it ran in incremental mode', incremental?.mode === 'incremental', JSON.stringify(incremental));
+  check(
+    'it read the unfiltered list, not the index',
+    stub.calls.filter((c) => c.endpoint === 'list' && c.query).length === 0,
+    `queries sent: ${stub.calls.filter((c) => c.query).length}`,
+  );
+  check(
+    `it walked ${2} pages of 100 unfiltered rows`,
+    incremental?.pages === 2 && incremental?.seen === 200,
+    `pages=${incremental?.pages} seen=${incremental?.seen}`,
+  );
+
+  const fresh = await mirrored(ids.fresh);
+  check(
+    'a request created now lands although the index does not have it',
+    fresh?.status === 'pending',
+    `mirror row: ${JSON.stringify(fresh)}`,
+  );
+  const oldSigned = await mirrored(ids.oldSigned);
+  check(
+    'an old pending signed yesterday is picked up without a sync',
+    oldSigned?.status === 'signed' && oldSigned?.signed_at !== null,
+    `mirror row: ${JSON.stringify(oldSigned)}`,
+  );
+  check(
+    'and it stored nothing beyond those two',
+    incremental?.stored === 2,
+    `stored=${incremental?.stored}`,
+  );
+  check(
+    'rows below the first two pages are out of its reach',
+    (await mirrored(ids.boundary)) === null && (await mirrored(ids.legacy)) === null,
+    'boundary and pre-rename rows are not mirrored yet',
+  );
+
+  // ---------------------------------------- 2 · the full reload
+  console.log('\n2 · full reload with inclusive bounds');
+  const callsBefore = stub.calls.length;
+  const reload = await fetch(`${base}/api/nda-inbox/refresh?since=${BOUNDARY_DAY}`, {
+    method: 'POST',
+    headers,
+  });
+  check('POST /refresh?since= answers 202', reload.status === 202, `got ${reload.status}`);
+  const afterFull = await settle('refresh_state');
+  const full = afterFull.last_refresh_result;
+  const queries = stub.calls.slice(callsBefore).filter((c) => c.query).map((c) => c.query);
+
+  check('it ran in full mode', full?.mode === 'full', JSON.stringify(full));
+  check(
+    'every query uses the inclusive range form',
+    queries.length > 0 && queries.every((q) => q.includes('created:[')) &&
+      !queries.some((q) => q.includes('created:{')),
+    queries[0] ?? 'no query sent',
+  );
+  check(
+    'a request created on the boundary day is included',
+    (await mirrored(ids.boundary)) !== null,
+    `${BOUNDARY_DAY}T09:00Z with since=${BOUNDARY_DAY}`,
+  );
+  check(
+    'the pre-rename leg ran and stored its request',
+    (await mirrored(ids.legacy)) !== null && full?.legacy_stored >= 1,
+    `legacy_stored=${full?.legacy_stored}`,
+  );
+  check(
+    'covers_from widened to the reloaded window',
+    (await meta('ds_mirror_covers_from'))?.value === `${BOUNDARY_DAY}T00:00:00.000Z`,
+    `covers_from=${(await meta('ds_mirror_covers_from'))?.value}`,
+  );
+
+  // ---------------------------------------- 3 · coverage never narrows
+  console.log('\n3 · covers_from only ever widens');
+  await setMeta('ds_mirror_covers_from', '2020-01-01T00:00:00.000Z');
+  await fetch(`${base}/api/nda-inbox/refresh?since=${BOUNDARY_DAY}`, { method: 'POST', headers });
+  await settle('refresh_state');
+  check(
+    'a narrower window does not overwrite a wider coverage',
+    (await meta('ds_mirror_covers_from'))?.value === '2020-01-01T00:00:00.000Z',
+    `covers_from=${(await meta('ds_mirror_covers_from'))?.value}`,
+  );
+
+  // ---------------------------------------- 4 · meta-key hygiene
+  console.log('\n4 · the sync never touches the refresh cursor');
+  const refreshBefore = await meta('ds_last_refresh_at');
+  const synced = await fetch(`${base}/api/nda-inbox/sync`, { method: 'POST', headers });
+  check('POST /sync answers 202', synced.status === 202, `got ${synced.status}`);
+  await settle('sync_state');
+  const refreshAfter = await meta('ds_last_refresh_at');
+  check(
+    'ds_last_refresh_at is untouched by a sync run',
+    refreshBefore?.value === refreshAfter?.value &&
+      String(refreshBefore?.updated_at) === String(refreshAfter?.updated_at),
+    `${refreshBefore?.value} @ ${refreshBefore?.updated_at} -> ${refreshAfter?.value} @ ${refreshAfter?.updated_at}`,
+  );
+  check(
+    'the sync stamps its own key instead',
+    (await meta('ds_last_sync_at'))?.value != null,
+    `ds_last_sync_at=${(await meta('ds_last_sync_at'))?.value}`,
+  );
+
+  // ---------------------------------------- the log line
+  console.log('\nwalk log lines the run produced:');
+  for (const line of appLog) {
+    let msg = line;
+    try {
+      msg = JSON.parse(line).msg ?? line;
+    } catch {
+      /* not JSON — a tsx or npm line */
+    }
+    if (typeof msg === 'string' && msg.includes('[nda-refresh] walk')) console.log(`  ${msg}`);
+  }
+} catch (err) {
+  failures += 1;
+  console.error(`\nrun failed: ${err.stack ?? err.message}`);
+} finally {
+  await cleanup();
+}
+
+console.log(failures === 0 ? '\nall assertions passed' : `\n${failures} assertion(s) failed`);
+process.exit(failures === 0 ? 0 : 1);

scripts/ds-stub.mjs

@@ -0,0 +1,139 @@
+/**
+ * A stand-in for the two Dropbox Sign endpoints the NDA inbox uses, with the
+ * one property that broke the refresh in production: **the search index lags
+ * behind the list**.
+ *
+ * `GET /v3/signature_request/list` answers from two different sources:
+ *
+ * * without `query=` — the live list, ordered by last activity (signed_at,
+ *   falling back to created_at), exactly like the real endpoint;
+ * * with `query=` — the *index*, which only holds the snapshot a fixture entry
+ *   declares in `indexed`. A brand-new request is missing from it entirely,
+ *   and a request signed a minute ago is still in it as pending. That is what
+ *   an incremental refresh must not depend on.
+ *
+ * The range syntax is parsed for real, brackets and all, so `[a TO b]`
+ * (inclusive) and `{a TO b}` (exclusive) actually behave differently — which
+ * is what makes the boundary-day assertion meaningful.
+ *
+ * Run standalone for manual pokes:
+ *   node scripts/ds-stub.mjs --port 8099 --fixture some.json
+ */
+import { createServer } from 'node:http';
+import { readFile } from 'node:fs/promises';
+
+const dayOf = (unixSeconds) => new Date(unixSeconds * 1000).toISOString().slice(0, 10);
+
+/** signed_at when there is one, created_at otherwise — the list's sort key. */
+const activityOf = (request) => request.signatures?.[0]?.signed_at ?? request.created_at;
+
+/**
+ * The subset of the query language the refresh sends:
+ * `title:"<phrase>" AND created:[<day> TO <day|*>]`, either bracket style.
+ */
+export function matchesQuery(request, queryString) {
+  const phrase = /title:"([^"]*)"/.exec(queryString)?.[1];
+  if (phrase && !(request.title ?? '').includes(phrase)) return false;
+
+  const range = /created:([[{])\s*(\S+)\s+TO\s+(\S+?)\s*([\]}])/.exec(queryString);
+  if (range) {
+    const [, open, from, to, close] = range;
+    const day = dayOf(request.created_at);
+    // '[' includes the boundary day, '{' excludes it — the whole point.
+    if (from !== '*' && (open === '[' ? day < from : day <= from)) return false;
+    if (to !== '*' && (close === ']' ? day > to : day >= to)) return false;
+  }
+  return true;
+}
+
+/**
+ * `fixture` is a list of `{request, indexed}`: the truth the list and the
+ * per-id endpoint serve, and the possibly stale (or missing) copy the search
+ * index has of it.
+ */
+export function createStub(fixture) {
+  /** Every call the app made — the test asserts on what it asked for. */
+  const calls = [];
+
+  const byActivity = (a, b) => activityOf(b) - activityOf(a);
+
+  const server = createServer((req, res) => {
+    const url = new URL(req.url, 'http://stub');
+    const send = (status, body) => {
+      const payload = JSON.stringify(body);
+      res.writeHead(status, { 'content-type': 'application/json' });
+      res.end(payload);
+    };
+
+    if (url.pathname === '/v3/signature_request/list') {
+      const page = Number(url.searchParams.get('page') ?? '1');
+      const pageSize = Number(url.searchParams.get('page_size') ?? '20');
+      const queryString = url.searchParams.get('query');
+      calls.push({ endpoint: 'list', page, query: queryString });
+
+      const source = queryString
+        ? fixture
+            .map((entry) => entry.indexed)
+            .filter(Boolean)
+            .filter((request) => matchesQuery(request, queryString))
+        : fixture.map((entry) => entry.request);
+      const ordered = [...source].sort(byActivity);
+      const numPages = Math.max(1, Math.ceil(ordered.length / pageSize));
+      const slice = ordered.slice((page - 1) * pageSize, page * pageSize);
+      return send(200, {
+        signature_requests: slice,
+        list_info: { page, num_pages: numPages, num_results: ordered.length },
+      });
+    }
+
+    const single = /^\/v3\/signature_request\/([^/]+)$/.exec(url.pathname);
+    if (single) {
+      const id = decodeURIComponent(single[1]);
+      calls.push({ endpoint: 'get', id });
+      const entry = fixture.find((e) => e.request.signature_request_id === id);
+      // Unknown ids 404 like the real API — the sync walks over real rows the
+      // stub knows nothing about and must simply report them as failures.
+      if (!entry) return send(404, { error: { error_msg: `no such request: ${id}` } });
+      return send(200, { signature_request: entry.request });
+    }
+
+    return send(404, { error: { error_msg: `unhandled path: ${url.pathname}` } });
+  });
+
+  return {
+    calls,
+    /** Resolves to the base URL once it is listening. */
+    listen(port = 0) {
+      return new Promise((resolve) => {
+        server.listen(port, '127.0.0.1', () =>
+          resolve(`http://127.0.0.1:${server.address().port}`),
+        );
+      });
+    },
+    close: () => new Promise((resolve) => server.close(resolve)),
+    /** What a searching walk would have found — the lag, made visible. */
+    search: (queryString) =>
+      fixture
+        .map((entry) => entry.indexed)
+        .filter(Boolean)
+        .filter((request) => matchesQuery(request, queryString)),
+  };
+}
+
+// -------------------------------------------------------- standalone mode
+if (process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/^.*\//, ''))) {
+  const args = process.argv.slice(2);
+  const arg = (name, fallback) => {
+    const at = args.indexOf(`--${name}`);
+    return at === -1 ? fallback : args[at + 1];
+  };
+  const file = arg('fixture');
+  if (file) {
+    const fixture = JSON.parse(await readFile(file, 'utf8'));
+    const stub = createStub(fixture);
+    console.log(`[ds-stub] listening on ${await stub.listen(Number(arg('port', '8099')))}`);
+  } else {
+    console.error('usage: node scripts/ds-stub.mjs --fixture <file.json> [--port 8099]');
+    process.exit(2);
+  }
+}