diff --git a/.claude/settings.local.json b/.claude/settings.local.json index d1c7fd7..75edc26 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -61,7 +61,9 @@ "Bash(curl -s http://127.0.0.1:8099/__reset-stats)", "Bash(bash -n acceptance6.sh)", "Bash(bash -n acceptance6b.sh)", - "Bash(curl -s -m 3 http://127.0.0.1:8099/__stats)" + "Bash(curl -s -m 3 http://127.0.0.1:8099/__stats)", + "Bash(node *)", + "Bash(bash /tmp/claude-1000/-home-aknuth-git-bizmatch-app/0d38fdce-4b18-4159-86e7-135dadf9386d/scratchpad/acceptance6c.sh)" ] } } diff --git a/README.md b/README.md index 958f88d..17749b8 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ Module 4: the buyer side — buyers, contacts, NDA rounds, deals, the guided Module 5: notes, todos and the "Today" view with the follow-up workflow. Module 6a: Dropbox Sign as the source of incoming NDAs — inbox, one-click import and PDF filing on the NAS. +Module 6b: form-field extraction from the signed requests — no AI, no review +step, the answers come straight out of `response_data`. The UI and all domain constants are English. @@ -155,7 +157,7 @@ directory aborts the scan with an error naming the path. 3. Copy the project folder to the AI machine, run `docker compose up -d --build` 4. Restore the dump: `docker compose exec -T db psql -U bizmatch bizmatch < backup.sql` -## API (as of module 6a) +## API (as of module 6b) | Method | Path | Purpose | Session | | ------ | --------------------------- | ------------------------------------------------ | ------- | @@ -198,6 +200,7 @@ directory aborts the scan with an error naming the path. | POST | /api/nda-inbox/refresh | start the background walk (202 / 409 if running) | yes | | POST | /api/nda-inbox/import | import one request into buyer/contact/nda(/deal) | yes | | POST | /api/nda-inbox/sync | refresh the status of every pending imported NDA | yes | +| POST | /api/nda-inbox/backfill-fields | retrofit form fields onto signed rounds | yes | | GET | /api/ndas/:id/file | stream the filed NDA PDF (Range + ETag) | yes | Everything except health, staff (GET+POST) and login requires the session @@ -335,6 +338,100 @@ time. Nothing re-fetches a request per id unnecessarily: sync's candidate set is `SENT AND NOT declined`, so signed and declined rounds are never fetched again. +The inbox is ordered and dated by **`coalesce(signed_at, created_at)`** — the +moment that matters is when the request was signed, falling back to when it +arrived while it still is not. 158 of the 566 mirrored requests were signed on +a different day than they were created, so sorting by `created_at` put them in +the wrong place. + +### Form fields (module 6b) + +A signed request carries every answer of the NDA form in `response_data`, so +importing one needs **no extraction model and no review step** — the values are +read off directly. Verified against the live account, an entry looks like: + +```json +{ "name": "Textbox1", "type": "text", "required": true, + "api_id": "9fb8331d-…", "value": "kkm Foods", "signature_id": "…" } +``` + +`name` is always present, so the mapping is by name; the `api_id`s are stable +per template slot and are indexed as a fallback key. Two traps the real payload +contains: **checkbox values are the strings `"true"`/`"false"`**, not booleans, +and signature/initials slots carry the literal string `"null"`. Both are +handled in `readValues()`, which also drops empty answers. + +| Field | Goes to | +| --- | --- | +| `Textbox1` | `buyer.company_name` | +| `Textbox2` / `Textbox4` | `contact.phone` / `contact.cell` | +| `Textbox5` | `buyer.address` + `buyer.state`, falling back to `Textbox18` — see below | +| `Textbox6` | compared with the signer e-mail — see below | +| `Textbox7` | `buyer.how_heard` | +| `Checkbox1` / `Checkbox2` | `buyer.interested_in_updates` = true / false, neither = null | +| `Textbox8` | `nda.preferred_businesses_text` | +| `Textbox9` | `buyer.background_experience` | +| `Textbox10` / `Textbox11` | `nda.total_purchase_price` / `nda.down_payment` | +| `Textbox12`…`Textbox15` | `nda.income_requirements` / `accountant` / `attorney` / `bank` | +| `Textbox16` | compared with the signer name — see below | +| `DateSigned1` | `nda.intro_date`, parsed from `"07 / 27 / 2026"` | + +The template also sends `Textbox3`, `Textbox17`, `Textbox19`, `Textbox20` (a +signature block repeating company, phone and e-mail) and a second +`DateSigned2`. Those are **not mapped** — `Textbox18` from the same block *is*, +as the address fallback described below. The whole array is stored verbatim in +`nda.raw_form_data`, so a mapping mistake can be corrected later without going +back to Dropbox for every round. + +Two values are never written, only reported, because they would corrupt the +identity the deduplication relies on: + +* `Textbox6` differing from the signer's e-mail → a note on the buyer, + `NDA form lists different email: `. The signer address stays the + contact's e-mail. +* `Textbox16` differing from the signer's name → `Form names prospective + buyer: `. `nda.signer_name` stays the Dropbox signer. + +Everything is written with `coalesce(nullif(btrim(col), ''), )`, i.e. +**fill only what is empty**. On a freshly created buyer every column is NULL so +that fills all of them; on a reused buyer it can never overwrite curated data. +The same guard covers the round, which is what makes the retrofit safe to run +over NDAs people have already edited by hand. + +`POST /api/nda-inbox/backfill-fields` is that retrofit: it walks every round +with a `dropbox_sign_id`, status `SIGNED` and `raw_form_data IS NULL`, fetches +it with a 500 ms pause between calls, and answers +`{candidates, filled, empty, failed, warnings}`. It doubles as the initial load +for the first three months and is safe to re-run — a filled round is no longer +a candidate. + +Two normalisations run over the extracted values before they are stored. Both +only affect the mapped columns — `raw_form_data` always keeps the verbatim +answers, so nothing is lost. + +**Null markers.** A value that is nothing but a "does not apply" marker becomes +NULL: `/^(n|na|n\/a|none|nil|x+|-+|\.+)$/i` after trimming — so `n`, `na`, +`N/A`, `none`, `nil`, `x`/`xx`/`xxx`, any run of dashes and any run of dots. +It is deliberately anchored, which is what keeps `"NASA"`, `"Nancy"`, +`"x-ray"`, `"N. Smith"` and `"none of the above"`. Junk that is not a marker +(`"open"`, `"enough"`) is kept too: that is what the signer wrote, and only an +exact marker is safe to discard. + +> ⚠️ **Two-track normalisation — keep both in sync.** The legacy vision +> pipeline in the QC repo carries its own `NULL_MARKERS` for the scanned-PDF +> route. The `x` → `x+` widening above has **not** been applied there. Whoever +> next touches that pipeline must make the same change, otherwise the same NDA +> yields `"xx"` from the scan route and `NULL` from the Dropbox Sign route, and +> the two sources silently disagree about what "no answer" means. + +**Address fallback.** The form asks for the address twice — `Textbox5` in the +body and `Textbox18` in the signature block — and signers routinely type only a +street in the first and the complete address in the second. When `Textbox5` +carries no state, `Textbox18` is tried; if *it* has one, its address **and** +state are taken together, since combining a street from one field with a state +from the other would invent an address. If neither has a state, `Textbox5`'s +street is kept as-is and the state stays NULL. + `POST /api/nda-inbox/import` is one transaction: buyer + contact (the same reuse rules as the guided inquiry — see `ensureBuyerAndContact`), the NDA round with `dropbox_sign_id` (UNIQUE, so a second import is a `409`), and optionally @@ -468,6 +565,7 @@ migrations/ numbered SQL migrations 004_reset_….sql one-time reset of that column to NULL 005_nda_dropbox_….sql dropbox_sign_id, signer_name, declined on nda 006_ds_request_….sql ds_request mirror + the app_meta key/value table + 007_nda_form_….sql income/accountant/attorney/bank + raw_form_data on nda src/ config.ts env configuration db.ts pg pool, query helpers, withTransaction @@ -478,6 +576,7 @@ src/ dropbox-sign.ts thin Dropbox Sign REST client (list, get, download) nda-files.ts naming, filing and active/inactive moves of NDA PDFs nda-refresh.ts the background walk that mirrors requests into ds_request + nda-fields.ts response_data -> buyer/contact/nda, fill-only-what-is-empty server.ts Fastify app (health, staff, login, businesses, file, static) buyer-routes.ts buyers, contacts, NDA rounds, deals, the inquiry flow workflow-routes.ts notes, todos, documents, the Today view, follow-ups diff --git a/migrations/007_nda_form_fields.sql b/migrations/007_nda_form_fields.sql new file mode 100644 index 0000000..d77f4a7 --- /dev/null +++ b/migrations/007_nda_form_fields.sql @@ -0,0 +1,18 @@ +-- BizMatch Phase 2 — module 6b: the NDA form fields Dropbox Sign returns +-- +-- A signed request carries every answer in response_data, so an import needs +-- no extraction model and no review step. These are the four answers that had +-- no column yet; the rest map onto existing buyer/contact/nda fields. + +ALTER TABLE nda + ADD COLUMN income_requirements text, + ADD COLUMN accountant text, + ADD COLUMN attorney text, + ADD COLUMN bank text, + -- The untouched response_data array. Keeping it means a mapping mistake can + -- be corrected later without going back to Dropbox for every round. + ADD COLUMN raw_form_data jsonb; + +-- The backfill endpoint selects exactly this set. +CREATE INDEX nda_needs_form_data_idx ON nda (dropbox_sign_id) + WHERE dropbox_sign_id IS NOT NULL AND status = 'SIGNED' AND raw_form_data IS NULL; diff --git a/src/buyer-routes.ts b/src/buyer-routes.ts index 4c1a006..36e8839 100644 --- a/src/buyer-routes.ts +++ b/src/buyer-routes.ts @@ -87,7 +87,13 @@ export async function ensureBuyerAndContact( contact: ContactInput; staffId: string; }, -): Promise<{ buyerId: string; createdBuyer: boolean; createdContact: boolean }> { +): Promise<{ + buyerId: string; + /** The contact this inquiry belongs to, reused or freshly created. */ + contactId: string; + createdBuyer: boolean; + createdContact: boolean; +}> { const name = contact.name.trim(); const email = (contact.email ?? '').trim(); const values = [name, email || null, contact.phone?.trim() || null, contact.cell?.trim() || null]; @@ -102,24 +108,27 @@ export async function ensureBuyerAndContact( ORDER BY is_primary DESC, created_at LIMIT 1`, [buyerId, email.toLowerCase(), name.toLowerCase()], ); - if (existing) return { buyerId, createdBuyer: false, createdContact: false }; - await tx.query( - 'INSERT INTO contact (buyer_id, name, email, phone, cell) VALUES ($1, $2, $3, $4, $5)', + if (existing) { + return { buyerId, contactId: existing.id, createdBuyer: false, createdContact: false }; + } + const added = await tx.queryRow<{ id: string }>( + `INSERT INTO contact (buyer_id, name, email, phone, cell) + VALUES ($1, $2, $3, $4, $5) RETURNING id`, [buyerId, ...values], ); - return { buyerId, createdBuyer: false, createdContact: true }; + return { buyerId, contactId: added.id, createdBuyer: false, createdContact: true }; } const buyer = await tx.queryRow<{ id: string }>( 'INSERT INTO buyer (company_name, created_by) VALUES ($1, $2) RETURNING id', [companyName, staffId], ); - await tx.query( + const added = await tx.queryRow<{ id: string }>( `INSERT INTO contact (buyer_id, name, email, phone, cell, is_primary) - VALUES ($1, $2, $3, $4, $5, true)`, + VALUES ($1, $2, $3, $4, $5, true) RETURNING id`, [buyer.id, ...values], ); - return { buyerId: buyer.id, createdBuyer: true, createdContact: true }; + return { buyerId: buyer.id, contactId: added.id, createdBuyer: true, createdContact: true }; } /** `is_primary` is a single-winner flag per buyer. */ @@ -439,6 +448,7 @@ export function registerBuyerRoutes(app: FastifyInstance): void { const ndas = await query<{ id: string }>( `SELECT id, status, nas_path, sent_at, signed_at, intro_date, preferred_businesses_text, total_purchase_price, down_payment, + income_requirements, accountant, attorney, bank, dropbox_sign_id, signer_name, declined, created_at FROM nda WHERE buyer_id = $1 ORDER BY created_at DESC`, [id], @@ -639,6 +649,10 @@ export function registerBuyerRoutes(app: FastifyInstance): void { total_purchase_price: asText, down_payment: asText, intro_date: asDate, + income_requirements: asText, + accountant: asText, + attorney: asText, + bank: asText, }); } catch (err) { if (err instanceof InputError) return reply.code(400).send({ error: err.message }); @@ -657,6 +671,7 @@ export function registerBuyerRoutes(app: FastifyInstance): void { return tx.queryOne( `SELECT id, buyer_id, status, nas_path, sent_at, signed_at, intro_date, preferred_businesses_text, total_purchase_price, down_payment, + income_requirements, accountant, attorney, bank, dropbox_sign_id, signer_name, declined, created_at FROM nda WHERE id = $1`, [row.id], diff --git a/src/dropbox-sign.ts b/src/dropbox-sign.ts index 388c8ba..1e2e7e6 100644 --- a/src/dropbox-sign.ts +++ b/src/dropbox-sign.ts @@ -27,6 +27,8 @@ export interface SignatureRequest { is_declined: boolean; files_url: string; signatures: SignatureSummary[]; + /** The filled-in form fields; only populated once the request is signed. */ + response_data?: unknown; } export class DropboxSignError extends Error { diff --git a/src/nda-fields.ts b/src/nda-fields.ts new file mode 100644 index 0000000..a45f4b2 --- /dev/null +++ b/src/nda-fields.ts @@ -0,0 +1,308 @@ +import type { Tx } from './db.js'; +import type { SignatureRequest } from './dropbox-sign.js'; + +/** + * The NDA form in Dropbox Sign is a fixed template, and a signed request + * returns every answer in `response_data`. So importing a signed NDA needs no + * extraction model and no review step — the values are simply read off. + * + * Entry shape (verified against the live account): + * { name: 'Textbox1', type: 'text', required: true, api_id: '9fb8…', value: 'kkm Foods', signature_id: '…' } + * + * Two traps the real payload contains: checkbox values arrive as the *strings* + * "true"/"false", and signature/initials fields carry the literal string + * "null". Both are handled in readValues(). + */ + +export interface ResponseDataEntry { + name?: string; + api_id?: string; + type?: string; + value?: unknown; +} + +/** Field name -> answer, empty and placeholder answers dropped. */ +export type FormValues = Map; + +/** + * `name` is present on every entry in the live payload, so the mapping is by + * name. The api_id is kept as a fallback key because it is stable per template + * slot and would survive a renaming of the fields. + */ +export function readValues(responseData: unknown): FormValues { + const values: FormValues = new Map(); + if (!Array.isArray(responseData)) return values; + for (const entry of responseData as ResponseDataEntry[]) { + const raw = entry?.value; + if (raw === null || raw === undefined) continue; + const text = typeof raw === 'string' ? raw.trim() : String(raw).trim(); + // "null" is what the signature and initials slots carry when unset. + if (text === '' || text === 'null') continue; + if (entry.name) values.set(entry.name, text); + if (entry.api_id) values.set(`api_id:${entry.api_id}`, text); + } + return values; +} + +/** Checkboxes come back as "true"/"false" strings, not booleans. */ +const isChecked = (value: string | undefined): boolean => value?.toLowerCase() === 'true'; + +/** + * What signers write when a field does not apply to them. Anchored on purpose: + * only a value that is *nothing but* a marker is discarded, so "NASA", + * "x-ray" and "none of the above" survive. The verbatim text stays in + * nda.raw_form_data either way. + * + * Kept in sync with NULL_MARKERS in the legacy vision pipeline (QC repo) — + * see the README: both tracks must normalise identically. + */ +const NULL_MARKER = /^(n|na|n\/a|none|nil|x+|-+|\.+)$/i; + +const withoutNullMarkers = (value: string | null): string | null => + value === null || NULL_MARKER.test(value) ? null : value; + +const US_STATES = new Set([ + 'AL','AK','AZ','AR','CA','CO','CT','DE','FL','GA','HI','ID','IL','IN','IA','KS','KY','LA','ME', + 'MD','MA','MI','MN','MS','MO','MT','NE','NV','NH','NJ','NM','NY','NC','ND','OH','OK','OR','PA', + 'RI','SC','SD','TN','TX','UT','VT','VA','WA','WV','WI','WY','DC', +]); + +/** + * Pulls the state out of a free-text address, e.g. + * "5408 barry Blvd,flower mound,Tx 75022" -> "TX". + * + * Scans from the end and only accepts a code that is last or followed by a ZIP, + * so a stray "in" or "or" in the middle of a street name cannot masquerade as + * Indiana or Oregon. Returns null when the address carries no state at all, + * which is common — plenty of signers type only the street. + */ +export function extractState(address: string | null): string | null { + if (!address) return null; + const tokens = address.split(/[\s,]+/).filter(Boolean); + for (let i = tokens.length - 1; i >= 0; i -= 1) { + const token = tokens[i] ?? ''; + if (!/^[A-Za-z]{2}$/.test(token)) continue; + const code = token.toUpperCase(); + if (!US_STATES.has(code)) continue; + const next = tokens[i + 1]; + if (next === undefined || /^\d{5}(-\d{4})?$/.test(next)) return code; + } + return null; +} + +/** + * The DateSigned slots render as "07 / 27 / 2026" — US month/day/year with + * spaces around the separators. Anything that does not parse cleanly stays + * null rather than becoming a wrong date. + */ +export function normalizeFormDate(value: string | undefined): string | null { + if (!value) return null; + const compact = value.replace(/\s+/g, ''); + const match = /^(\d{1,2})[/.-](\d{1,2})[/.-](\d{2,4})$/.exec(compact); + if (!match) return null; + const [, rawMonth, rawDay, rawYear] = match; + const month = Number(rawMonth); + const day = Number(rawDay); + let year = Number(rawYear); + if (rawYear && rawYear.length === 2) year += year < 70 ? 2000 : 1900; + if (month < 1 || month > 12 || day < 1 || day > 31 || year < 1900 || year > 2200) return null; + // Rejects the impossible days that the range check above lets through. + const asDate = new Date(Date.UTC(year, month - 1, day)); + if (asDate.getUTCMonth() !== month - 1 || asDate.getUTCDate() !== day) return null; + return `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`; +} + +export interface ExtractedFields { + /** buyer */ + company_name: string | null; + address: string | null; + state: string | null; + how_heard: string | null; + background_experience: string | null; + interested_in_updates: boolean | null; + /** contact */ + phone: string | null; + cell: string | null; + /** nda round */ + preferred_businesses_text: string | null; + total_purchase_price: string | null; + down_payment: string | null; + income_requirements: string | null; + accountant: string | null; + attorney: string | null; + bank: string | null; + intro_date: string | null; + /** only used to decide whether a note is warranted */ + form_email: string | null; + prospective_buyer: string | null; +} + +/** Every text field goes through the null-marker rule on the way out. */ +const get = (values: FormValues, name: string): string | null => + withoutNullMarkers(values.get(name) ?? null); + +/** + * The form asks for the address twice: Textbox5 in the body and Textbox18 in + * the signature block. Signers routinely type only a street in the first and + * the complete address in the second, so when Textbox5 carries no state the + * fuller value wins — address and state together, since mixing a street from + * one field with a state from the other would invent an address. + */ +function pickAddress(values: FormValues): { address: string | null; state: string | null } { + const primary = get(values, 'Textbox5'); + const primaryState = extractState(primary); + if (primaryState) return { address: primary, state: primaryState }; + + const fallback = get(values, 'Textbox18'); + const fallbackState = extractState(fallback); + if (fallbackState) return { address: fallback, state: fallbackState }; + + return { address: primary, state: null }; +} + +export function extractFields(responseData: unknown): ExtractedFields { + const values = readValues(responseData); + const { address, state } = pickAddress(values); + return { + company_name: get(values, 'Textbox1'), + address, + state, + how_heard: get(values, 'Textbox7'), + background_experience: get(values, 'Textbox9'), + // Two mutually exclusive boxes; neither ticked means "not answered". + interested_in_updates: isChecked(values.get('Checkbox1')) + ? true + : isChecked(values.get('Checkbox2')) + ? false + : null, + phone: get(values, 'Textbox2'), + cell: get(values, 'Textbox4'), + preferred_businesses_text: get(values, 'Textbox8'), + total_purchase_price: get(values, 'Textbox10'), + down_payment: get(values, 'Textbox11'), + income_requirements: get(values, 'Textbox12'), + accountant: get(values, 'Textbox13'), + attorney: get(values, 'Textbox14'), + bank: get(values, 'Textbox15'), + intro_date: normalizeFormDate(values.get('DateSigned1')), + form_email: get(values, 'Textbox6'), + prospective_buyer: get(values, 'Textbox16'), + }; +} + +const sameish = (a: string | null, b: string | null): boolean => + (a ?? '').trim().toLowerCase() === (b ?? '').trim().toLowerCase(); + +export interface ApplyResult { + notes: string[]; +} + +/** + * Writes the extracted values onto the buyer, its contact and the round. + * + * Everything uses `coalesce(nullif(btrim(col), ''), $new)`, i.e. **fill only + * what is empty**. On a freshly created buyer every column is still NULL, so + * that fills all of them; on a reused buyer it can never overwrite what + * somebody curated by hand. The same guard protects the round, which matters + * for the retrofit endpoint running over NDAs that people have already edited. + */ +export async function applyFormData( + tx: Tx, + { + ndaId, + buyerId, + contactId, + signerName, + signerEmail, + responseData, + staffId, + }: { + ndaId: string; + buyerId: string; + contactId: string | null; + signerName: string; + signerEmail: string; + responseData: unknown; + staffId: string; + }, +): Promise { + const fields = extractFields(responseData); + + await tx.query( + `UPDATE buyer SET + company_name = coalesce(nullif(btrim(company_name), ''), $2), + address = coalesce(nullif(btrim(address), ''), $3), + state = coalesce(nullif(btrim(state), ''), $4), + how_heard = coalesce(nullif(btrim(how_heard), ''), $5), + background_experience = coalesce(nullif(btrim(background_experience), ''), $6), + interested_in_updates = coalesce(interested_in_updates, $7) + WHERE id = $1`, + [ + buyerId, + fields.company_name, + fields.address, + fields.state, + fields.how_heard, + fields.background_experience, + fields.interested_in_updates, + ], + ); + + if (contactId) { + await tx.query( + `UPDATE contact SET + phone = coalesce(nullif(btrim(phone), ''), $2), + cell = coalesce(nullif(btrim(cell), ''), $3) + WHERE id = $1`, + [contactId, fields.phone, fields.cell], + ); + } + + await tx.query( + `UPDATE nda SET + preferred_businesses_text = coalesce(nullif(btrim(preferred_businesses_text), ''), $2), + total_purchase_price = coalesce(nullif(btrim(total_purchase_price), ''), $3), + down_payment = coalesce(nullif(btrim(down_payment), ''), $4), + income_requirements = coalesce(nullif(btrim(income_requirements), ''), $5), + accountant = coalesce(nullif(btrim(accountant), ''), $6), + attorney = coalesce(nullif(btrim(attorney), ''), $7), + bank = coalesce(nullif(btrim(bank), ''), $8), + intro_date = coalesce(intro_date, $9::date), + raw_form_data = $10::jsonb + WHERE id = $1`, + [ + ndaId, + fields.preferred_businesses_text, + fields.total_purchase_price, + fields.down_payment, + fields.income_requirements, + fields.accountant, + fields.attorney, + fields.bank, + fields.intro_date, + JSON.stringify(responseData ?? null), + ], + ); + + // Discrepancies are recorded rather than resolved: the signer identity is + // the dedup anchor and must not drift, but the team still wants to know. + const notes: string[] = []; + if (fields.form_email && !sameish(fields.form_email, signerEmail)) { + notes.push(`NDA form lists different email: ${fields.form_email}`); + } + if (fields.prospective_buyer && !sameish(fields.prospective_buyer, signerName)) { + notes.push(`Form names prospective buyer: ${fields.prospective_buyer}`); + } + for (const text of notes) { + await tx.query( + 'INSERT INTO note (text, highlight, buyer_id, created_by) VALUES ($1, false, $2, $3)', + [text, buyerId, staffId], + ); + } + return { notes }; +} + +/** Convenience for the routes: does this request carry form answers at all? */ +export const hasResponseData = (request: SignatureRequest): boolean => + Array.isArray((request as { response_data?: unknown }).response_data) && + ((request as { response_data: unknown[] }).response_data.length > 0); diff --git a/src/nda-inbox-routes.ts b/src/nda-inbox-routes.ts index 4f2d43d..088ae2f 100644 --- a/src/nda-inbox-routes.ts +++ b/src/nda-inbox-routes.ts @@ -20,11 +20,15 @@ import { startRefresh, } from './nda-refresh.js'; import { NdaFileError, resolveNdaFile, writeNdaFile } from './nda-files.js'; +import { applyFormData } from './nda-fields.js'; import { UUID_RE, badId, sendFile, trimmed } from './http.js'; /** How far back the inbox looks when the caller does not say. */ const DEFAULT_WINDOW_DAYS = 90; +/** Between per-request fetches in the retrofit, same courtesy as the refresh. */ +const FETCH_PAUSE_MS = 500; + /** Words too common in a business name to say anything about a match. */ const STOPWORDS = new Set([ 'the', @@ -134,6 +138,22 @@ async function noteDeclined( ); } +/** + * The contact the form answers belong to. Sync and the retrofit work off an + * NDA rather than an inquiry, so the signer's own contact row is the primary + * one of that buyer. + */ +async function primaryContactId( + tx: { queryOne: (t: string, p?: unknown[]) => Promise }, + buyerId: string, +): Promise { + const row = await tx.queryOne<{ id: string }>( + 'SELECT id FROM contact WHERE buyer_id = $1 ORDER BY is_primary DESC, created_at LIMIT 1', + [buyerId], + ); + return row?.id ?? null; +} + /** * Files the signed PDF for one NDA. Never throws: a NAS that is full, * read-only or unreachable must not undo an import that already succeeded, so @@ -197,20 +217,26 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { signer_email: string | null; /** Also a full timestamp — Dropbox reports the second it was signed. */ signed_at: Date | null; + /** signed_at once signed, created_at until then — what the UI shows. */ + relevant_at: Date | null; nda_id: string | null; buyer_id: string | null; }>( // timestamptz is left to the driver: it comes back as a Date and // serialises to a full ISO-8601 string, offset and all. + // + // The moment that matters is when the request was *signed*, falling back + // to when it was created while it still is not — that is what the list + // is sorted and dated by. The id only breaks ties, so the order is + // deterministic across requests. `SELECT r.signature_request_id, r.title, r.status, r.signer_name, r.signer_email, r.created_at, r.signed_at, + coalesce(r.signed_at, r.created_at) AS relevant_at, n.id AS nda_id, n.buyer_id FROM ds_request r LEFT JOIN nda n ON n.dropbox_sign_id = r.signature_request_id WHERE r.created_at >= $1 - -- Newest first, to the second, exactly as Dropbox lists them. The id - -- only breaks ties so the order is deterministic across requests. - ORDER BY r.created_at DESC, r.signature_request_id DESC`, + ORDER BY coalesce(r.signed_at, r.created_at) DESC, r.signature_request_id DESC`, [since.toISOString()], ); @@ -244,6 +270,7 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { status: row.status, signer: { name: row.signer_name ?? '', email }, signed_at: row.signed_at, + relevant_at: row.relevant_at, imported: row.nda_id ? { nda_id: row.nda_id, buyer_id: row.buyer_id } : null, known_buyer: match ? { @@ -335,15 +362,17 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { const signed = status === 'signed'; const result = await withTransaction(async (tx) => { - const { buyerId: targetBuyer, createdBuyer, createdContact } = await ensureBuyerAndContact( - tx, - { - buyerId, - companyName: null, - contact: { name: signer.name || signer.email, email: signer.email }, - staffId, - }, - ); + const { + buyerId: targetBuyer, + contactId, + createdBuyer, + createdContact, + } = await ensureBuyerAndContact(tx, { + buyerId, + companyName: null, + contact: { name: signer.name || signer.email, email: signer.email }, + staffId, + }); // A declined round was never signed, so it stays SENT and is only // flagged — the team still wants to see that it came back. @@ -368,6 +397,20 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { await noteDeclined(tx, targetBuyer, dayOf(request.created_at), staffId); } + // A signed request carries every form answer, so the round is filled + // in the same transaction — no extraction model, no review step. + const applied = signed + ? await applyFormData(tx, { + ndaId: nda.id, + buyerId: targetBuyer, + contactId, + signerName: signer.name, + signerEmail: signer.email, + responseData: request.response_data, + staffId, + }) + : { notes: [] }; + const deal = businessId ? await tx.queryRow<{ id: string }>( `INSERT INTO deal (buyer_id, business_id, nda_id, created_by) @@ -381,6 +424,7 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { nda_id: nda.id, deal_id: deal?.id ?? null, created: { buyer: createdBuyer, contact: createdContact }, + form_notes: applied.notes, }; }); @@ -404,8 +448,8 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { // Declined rounds are excluded: they stay SENT forever but will never // change state on the Dropbox side again, so re-fetching them is waste. - const pending = await query<{ id: string; dropbox_sign_id: string }>( - `SELECT id, dropbox_sign_id FROM nda + const pending = await query<{ id: string; dropbox_sign_id: string; buyer_id: string }>( + `SELECT id, dropbox_sign_id, buyer_id FROM nda WHERE dropbox_sign_id IS NOT NULL AND status = 'SENT' AND NOT declined ORDER BY sent_at`, ); @@ -427,6 +471,16 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { signer.signed_at, ]); await applySignedRule(tx, nda.id); + // The signature brought the form answers with it. + await applyFormData(tx, { + ndaId: nda.id, + buyerId: nda.buyer_id, + contactId: await primaryContactId(tx, nda.buyer_id), + signerName: signer.name, + signerEmail: signer.email, + responseData: request.response_data, + staffId, + }); }); const filed = await fileSignedPdf(app, nda.id, request); if (filed.warning) warnings.push(filed.warning); @@ -456,6 +510,61 @@ export function registerNdaInboxRoutes(app: FastifyInstance): void { return { checked: pending.length, signed, declined, failed, warnings }; }); + /** + * Retrofit: fills the form fields of signed rounds that were imported before + * extraction existed — and doubles as the initial load for the first three + * months. Paced like the refresh walk, and it only ever touches rounds whose + * raw_form_data is still missing, so it is safe to run repeatedly. + */ + app.post('/api/nda-inbox/backfill-fields', async (req, reply) => { + if (!isConfigured()) return notConfigured(reply); + const staffId = staffIdFromRequest(req); + if (!staffId) return reply.code(401).send({ error: 'not signed in' }); + + const candidates = await query<{ id: string; dropbox_sign_id: string; buyer_id: string }>( + `SELECT id, dropbox_sign_id, buyer_id FROM nda + WHERE dropbox_sign_id IS NOT NULL AND status = 'SIGNED' AND raw_form_data IS NULL + ORDER BY signed_at DESC NULLS LAST`, + ); + + let filled = 0; + let empty = 0; + let failed = 0; + const warnings: string[] = []; + + for (const [index, nda] of candidates.entries()) { + if (index > 0) await new Promise((resolve) => setTimeout(resolve, FETCH_PAUSE_MS)); + try { + const request = await getSignatureRequest(nda.dropbox_sign_id); + if (!Array.isArray(request.response_data) || request.response_data.length === 0) { + // Signed but no answers — nothing to map, and nothing to retry either. + empty += 1; + continue; + } + const signer = signerOf(request); + await withTransaction(async (tx) => { + await applyFormData(tx, { + ndaId: nda.id, + buyerId: nda.buyer_id, + contactId: await primaryContactId(tx, nda.buyer_id), + signerName: signer.name, + signerEmail: signer.email, + responseData: request.response_data, + staffId, + }); + }); + filled += 1; + } catch (err) { + const message = (err as Error).message; + app.log.warn(`[nda-inbox] backfilling ${nda.id} failed: ${message}`); + warnings.push(`${nda.dropbox_sign_id}: ${message}`); + failed += 1; + } + } + + return { candidates: candidates.length, filled, empty, failed, warnings }; + }); + /** The filed PDF of one round, with the same mechanics as business files. */ app.get<{ Params: { id: string } }>('/api/ndas/:id/file', async (req, reply) => { const { id } = req.params; diff --git a/web/src/api.ts b/web/src/api.ts index fcca045..b981a0a 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -110,6 +110,11 @@ export interface NdaRound { signer_name: string | null; /** A declined round stays SENT — it was never signed. */ declined: boolean; + /** Read straight off the signed Dropbox Sign form. */ + income_requirements: string | null; + accountant: string | null; + attorney: string | null; + bank: string | null; deals: Deal[]; } @@ -269,6 +274,8 @@ export interface InboxRow { signer: { name: string; email: string }; /** Full ISO timestamp as well — the second the signature came in. */ signed_at: string | null; + /** signed_at once signed, created_at until then: the moment that matters. */ + relevant_at: string; imported: { nda_id: string; buyer_id: string } | null; known_buyer: { buyer_id: string; company_name: string | null; contact_name: string } | null; business_suggestions: { id: string; name: string; status: BusinessStatus }[]; @@ -294,6 +301,14 @@ export interface ImportResult { warning: string | null; } +export interface BackfillResult { + candidates: number; + filled: number; + empty: number; + failed: number; + warnings: string[]; +} + export interface SyncResult { checked: number; signed: number; @@ -417,6 +432,7 @@ export const api = { business_id: businessId, }), syncInbox: () => post('/api/nda-inbox/sync'), + backfillFields: () => post('/api/nda-inbox/backfill-fields'), }; /** Same-origin streaming URL of the PDF filed for one NDA round. */ diff --git a/web/src/views/BuyerDetail.tsx b/web/src/views/BuyerDetail.tsx index e725630..1cc6082 100644 --- a/web/src/views/BuyerDetail.tsx +++ b/web/src/views/BuyerDetail.tsx @@ -422,6 +422,27 @@ function Round({ value={nda.down_payment} onSave={(next) => guard(() => api.updateNda(nda.id, { down_payment: next }))} /> + {/* Read off the signed Dropbox Sign form, editable like everything else. */} + guard(() => api.updateNda(nda.id, { income_requirements: next }))} + /> + guard(() => api.updateNda(nda.id, { accountant: next }))} + /> + guard(() => api.updateNda(nda.id, { attorney: next }))} + /> + guard(() => api.updateNda(nda.id, { bank: next }))} + />
{rows?.map((row) => ( + {/* Signed → when it was signed; still pending → when it arrived. */} - + {row.signer.name || '—'} {row.signer.email || '—'}