/** * audit-verify: checks an export of an audit trail, or a package for an * agency, offline, without contacting the Platform (spec 08, sections 7 and * 8; AUD-09, AUD-17; S-6b). * * node audit-verify.ts export.json --key PUBLIC_KEY * * Runs on Node.js 22.18 or later, and needs nothing but the file. It is an * independent reading of the published procedure: it shares no code with * the Platform, so that a reader can check it line by line. It checks: * * 0. the export key's signature over the manifest, and that the key is the * one the verification page publishes, given with `--key`: without it, * nothing verifies, since anyone can sign a file with a key of their own; * 1. each entry's hash, from its body and salt (RFC 8785 canonical form), * and that it is the hash the manifest signed; that the file holds each * signed entry once, and nothing the manifest did not sign; * 2. each entry's path inside its chain's subtree, to the chain's root; * 3. each chain's salted commitment, and its path in the batch tree, to the * batch's root; * 4. each batch's digest, worked out again. Nothing is anchored outside the * Platform: every entry rests on the export key and the batch digests; * 5. each chain's links: every chain hash recomputed, the first entry from * the chain's genesis, each run's head leaf rebuilt from its last chain * hash, and which seq numbers the export holds with no gap; * 6. each batch's link to the one before it; * 7. for a package: its summary, rebuilt from the entries, and that every * provider call has a decision that allowed it and a certified purpose, * recorded before it. * * It exits 0 when every check passes and 1 when any fails, and never prints * an entry's content: only services, seq numbers and batch numbers, with every * control character a file carries replaced, so that no file rewrites the screen. */ import { createHash, createPublicKey, verify } from "node:crypto"; import { readFileSync, realpathSync } from "node:fs"; import { fileURLToPath } from "node:url"; type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; type Obj = Record; /** RFC 8785: keys sorted by UTF-16 code units, no whitespace, strings and numbers as ECMAScript writes them. */ export function canonical(value: unknown): string { if (value === null || typeof value === "boolean" || typeof value === "string") { return JSON.stringify(value); } if (typeof value === "number") { if (!Number.isFinite(value)) { throw new Error("a number that is not finite"); } return JSON.stringify(value); } if (Array.isArray(value)) { return `[${value.map(canonical).join(",")}]`; } if (typeof value === "object") { const keys = Object.keys(value as Obj) .filter((key) => (value as Obj)[key] !== undefined) .sort(); return `{${keys.map((key) => `${JSON.stringify(key)}:${canonical((value as Obj)[key])}`).join(",")}}`; } throw new Error("a value JSON cannot hold"); } const sha = (...parts: Buffer[]): Buffer => { const hash = createHash("sha256"); for (const part of parts) { hash.update(part); } return hash.digest(); }; const ascii = (text: string): Buffer => Buffer.from(text, "ascii"); const HEX32 = /^[0-9a-f]{64}$/; const hex32 = (text: unknown): Buffer => { if (typeof text !== "string" || !HEX32.test(text)) { throw new Error("not a 32-byte hash"); } return Buffer.from(text, "hex"); }; const u64 = (n: unknown): Buffer => { if (typeof n !== "number" || !Number.isSafeInteger(n) || n < 0) { throw new Error("not a whole number"); } const out = Buffer.alloc(8); out.writeBigUInt64BE(BigInt(n)); return out; }; const uuid16 = (id: unknown): Buffer => { if (typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id)) { throw new Error("not an id"); } return Buffer.from(id.replaceAll("-", ""), "hex"); }; /** RFC 9162, section 2.1.3.2: the root an audit path leads to from a leaf at `index` of a tree of `size`, or null when it cannot. */ export function rootOf(leaf: Buffer, index: number, size: number, path: readonly unknown[]): string | null { if (!Number.isInteger(index) || !Number.isInteger(size) || index < 0 || index >= size) { return null; } let fn = index; let sn = size - 1; let r = sha(Buffer.from([0]), leaf); for (const node of path) { if (sn === 0 || typeof node !== "string" || !HEX32.test(node)) { return null; } const p = Buffer.from(node, "hex"); if (fn % 2 === 1 || fn === sn) { r = sha(Buffer.from([1]), p, r); while (fn % 2 === 0 && fn !== 0) { fn >>= 1; sn >>= 1; } } else { r = sha(Buffer.from([1]), r, p); } fn >>= 1; sn >>= 1; } return sn === 0 ? r.toString("hex") : null; } /** Each service's tag prefix, as the published procedure gives them: a chain's tags come from its service, never from the file. */ export const PREFIXES: Readonly> = { profile: "PROFILE", search: "PLAT", abuse: "ABUSE", suppression: "SUPPRESSION", cases: "CASES", billing: "BILLING", field: "FIELD", portal: "PORTAL", notify: "NOTIFY", metering: "METERING", audit: "AUDIT" }; /** The Platform's internal tenant, whose chains record provider calls and other entries no tenant reads. */ const INTERNAL_TENANT = "00000000-0000-7000-8000-000000000000"; /** The first batch's previous digest: 32 zero bytes. */ const ZERO = Buffer.alloc(32); /** What the tool found: each check's problems and notes, by service, seq and batch only. */ export interface Report { ok: boolean; problems: string[]; notes: string[]; /** How many entries are not yet in a sealed batch: for them the file rests on the export key alone. */ unsealed: number; } const obj = (value: unknown): Obj => (typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Obj) : {}); const list = (value: unknown): unknown[] => (Array.isArray(value) ? value : []); const str = (value: unknown): string | null => (typeof value === "string" ? value : null); /** Checks an export file, as parsed. `pinnedKey` is the export key's whole public key, as the verification page publishes it. */ export function verifyExport(file: unknown, pinnedKey: string | null = null): Report { const problems: string[] = []; const notes: string[] = []; const f = obj(file); const manifest = obj(f["manifest"]); const signature = obj(f["signature"]); let unsealed = 0; const report = (): Report => ({ ok: problems.length === 0, problems, notes, unsealed }); if (f["format"] !== "audit-export-v2" || manifest["format"] !== "audit-export-v2") { problems.push("the file is not an audit export of format audit-export-v2"); return report(); } // 0. The signature over the manifest, by the export key. try { const raw = Buffer.from(String(signature["public_key"]), "hex"); const keyId = sha(raw).toString("hex").slice(0, 16); if (raw.length !== 32 || signature["alg"] !== "Ed25519" || signature["key_id"] !== keyId) { problems.push("signature: the key is not an Ed25519 key of the id it names"); } else { const key = createPublicKey({ key: Buffer.concat([Buffer.from("302a300506032b6570032100", "hex"), raw]), format: "der", type: "spki" }); const signed = Buffer.concat([ascii("PLAT-EXPORT-v1"), Buffer.from(canonical(manifest), "utf8")]); if (!verify(null, signed, key, Buffer.from(String(signature["value"]), "hex"))) { problems.push("signature: does not match the manifest"); } // Only the whole public key pins it: its 16-digit id tells keys apart, but is too short to prove one. if (pinnedKey === null || !/^[0-9a-f]{64}$/.test(pinnedKey)) { problems.push(`signature: no public key given to check it against: run again with --key and the whole public key the verification page publishes (this file names key ${keyId})`); } else if (pinnedKey !== raw.toString("hex")) { problems.push(`signature: made with key ${keyId}, not the key given`); } } } catch { problems.push("signature: not in its form"); } // The batch records: each digest worked out again (step 4), and each linked to the one before (step 6). const batches = new Map(); for (const one of list(manifest["batches"])) { const batch = obj(one); const seq = batch["seq"]; if (typeof seq !== "number" || batches.has(seq)) { problems.push("batches: a batch named twice, or with no number"); continue; } batches.set(seq, batch); } for (const [seq, batch] of batches) { try { const prev = batch["prev_anchor_digest"] === null ? ZERO : hex32(batch["prev_anchor_digest"]); const end = Date.parse(String(batch["window_end"])); if (Number.isNaN(end)) { throw new Error("no time"); } const digest = sha(ascii("PLAT-ANCHOR-v1"), u64(seq), u64(end), prev, hex32(batch["merkle_root"])).toString("hex"); if (digest !== batch["anchor_digest"]) { problems.push(`batch ${String(seq)}: its digest does not match the batch`); } } catch { problems.push(`batch ${String(seq)}: not in its form`); } if ((seq === 1) !== (batch["prev_anchor_digest"] === null)) { problems.push(`batch ${String(seq)}: ${seq === 1 ? "names a batch before it" : "names no batch before it"}`); } const before = batches.get(seq - 1); if (before && before["anchor_digest"] !== batch["prev_anchor_digest"]) { problems.push(`batch ${String(seq)}: does not follow batch ${String(seq - 1)}`); } } const seqs = [...batches.keys()].sort((a, b) => a - b); for (let i = 1; i < seqs.length; i += 1) { if ((seqs[i] as number) !== (seqs[i - 1] as number) + 1) { notes.push(`batches ${String(seqs[i - 1])} and ${String(seqs[i])}: the batches between them are not in the export, so they are not linked here`); } } // Each chain. The file holds entries only of the chains the manifest signed, each chain once, and each entry once. const bodies = list(f["bodies"]).map(obj); const signedChains = list(manifest["chains"]).map(obj); const seenBodies = new Set(); for (const body of bodies) { const key = `${String(body["service"])}:${String(body["tenant_id"])}`; if (seenBodies.has(key) || !signedChains.some((chain) => chain["service"] === body["service"] && chain["tenant_id"] === body["tenant_id"])) { problems.push(`entries of ${String(body["service"])}: a chain the manifest did not sign, or named twice`); } seenBodies.add(key); } if (!Array.isArray(f["summary"]) || (manifest["kind"] !== "AGENCY" && f["summary"].length > 0)) { problems.push("summary: not a list, or in an export of a firm's own trail, which holds none"); } const allEntries: { service: string; internal: boolean; row: Obj }[] = []; for (const one of list(manifest["chains"])) { const chain = obj(one); const service = String(chain["service"]); const tenantId = chain["tenant_id"]; const internal = tenantId !== manifest["tenant_id"]; const name = `${service}${internal ? " (internal chain)" : ""}`; // The tags are the service's own, whatever the file says; the only internal chain a file may hold is `search`'s. const prefix = Object.hasOwn(PREFIXES, service) ? PREFIXES[service] : undefined; const tags = { entry: `${String(prefix)}-ENTRY-v1`, chain: `${String(prefix)}-CHAIN-v1`, genesis: `${String(prefix)}-GENESIS-v1`, head: `${String(prefix)}-HEAD-v1` }; if (prefix === undefined || canonical(chain["tags"] ?? null) !== canonical(tags)) { problems.push(`${name}: its tags are not its service's`); } if (internal && (tenantId !== INTERNAL_TENANT || service !== "search" || manifest["kind"] !== "AGENCY")) { problems.push(`${name}: a chain of another tenant, or an internal chain other than search's in a package`); } const held = bodies.find((body) => body["service"] === service && body["tenant_id"] === tenantId); const given = list(held?.["entries"]); const rows = new Map(given.map((row) => [obj(row)["seq"], obj(row)])); const entries = list(chain["entries"]).map(obj); // The signed seq numbers are whole numbers from 1, each greater than the one before, so that each matches one row. if (!entries.every((entry, i) => Number.isSafeInteger(entry["seq"]) && (entry["seq"] as number) >= 1 && (i === 0 || (entry["seq"] as number) > (entries[i - 1]?.["seq"] as number)))) { problems.push(`${name}: its entries' numbers are not in order, or repeat`); } if (rows.size !== entries.length || given.length !== rows.size) { problems.push(`${name}: the entries in the file are not the entries signed`); } const runs = list(chain["runs"]).map(obj); const genesis = (() => { try { return sha(ascii(String(tags["genesis"])), uuid16(tenantId)).toString("hex"); } catch { return null; } })(); let last: Obj | null = null; for (const entry of entries) { const seq = entry["seq"] as number; const row = rows.get(seq); const where = `${name} entry ${String(seq)}`; if (!row) { problems.push(`${where}: missing from the file`); continue; } allEntries.push({ service, internal, row }); // Step 1: the entry's hash from its body and salt. try { if (row["tenant_id"] !== tenantId || row["seq"] !== seq) { problems.push(`${where}: belongs to another chain or place`); } const body = { tenant_id: row["tenant_id"], seq: row["seq"], occurred_at: row["occurred_at"], actor_type: row["actor_type"], actor_id: row["actor_id"], event_type: row["event_type"], refs: row["refs"], payload: row["payload"], ip: row["ip"], session_id: row["session_id"], }; // An entry with no salt cannot be checked: none is redacted yet (spec 08, section 10), so it is a problem, not a note. if (row["salt"] === null || row["salt"] === undefined) { problems.push(`${where}: has no salt, so its body cannot be checked`); } else if (sha(ascii(String(tags["entry"])), hex32(row["salt"]), Buffer.from(canonical(body), "utf8")).toString("hex") !== entry["entry_hash"]) { problems.push(`${where}: its hash does not match the entry`); } if (row["entry_hash"] !== entry["entry_hash"] || row["chain_hash"] !== entry["chain_hash"] || row["prev_chain_hash"] !== entry["prev_chain_hash"]) { problems.push(`${where}: its hashes are not the ones signed`); } // Step 5: its link to the entry before it. if (sha(ascii(String(tags["chain"])), hex32(entry["prev_chain_hash"]), hex32(entry["entry_hash"])).toString("hex") !== entry["chain_hash"]) { problems.push(`${where}: its chain hash does not match`); } if (seq === 1 && entry["prev_chain_hash"] !== genesis) { problems.push(`${where}: does not start from the chain's genesis`); } if (last && (last["seq"] as number) + 1 === seq && last["chain_hash"] !== entry["prev_chain_hash"]) { problems.push(`${where}: does not link to the entry before it`); } } catch { problems.push(`${where}: not in its form`); } // Step 2: its path inside its chain's subtree. // An entry the manifest says waits for a batch must lie past every sealed one: otherwise a batch's run would hide it. if (entry["batch_seq"] === null) { unsealed += 1; if (seq <= (chain["sealed_seq"] as number) || runs.some((one) => (one["first_seq"] as number) <= seq && seq <= (one["last_seq"] as number))) { problems.push(`${where}: named as waiting for a batch, though its chain is sealed past it`); } else { notes.push(`${where}: not sealed in a batch yet when exported`); } } else { const run = runs.find((one) => one["batch_seq"] === entry["batch_seq"] && (one["first_seq"] as number) <= seq && seq <= (one["last_seq"] as number)); try { if (!run || rootOf(hex32(entry["entry_hash"]), seq - (run["first_seq"] as number), (run["last_seq"] as number) - (run["first_seq"] as number) + 2, list(entry["path"])) !== run["chain_root"]) { problems.push(`${where}: does not lead to its chain's root in batch ${String(entry["batch_seq"])}`); } } catch { problems.push(`${where}: its proof is not in its form`); } } last = entry; } // Which seq numbers the export holds, and whether any are missing between the first and the last. const held2 = entries.map((entry) => entry["seq"] as number).sort((a, b) => a - b); if (held2.length > 0) { const ranges: string[] = []; let start = held2[0] as number; let prev = start; for (const seq of [...held2.slice(1), Number.NaN]) { if (seq !== prev + 1) { ranges.push(start === prev ? String(start) : `${String(start)} to ${String(prev)}`); start = seq; } prev = seq; } notes.push(`${name}: entries ${ranges.join(", ")}${ranges.length > 1 ? ", the others left out of the export" : ""}; ${String(chain["received_seq"])} received by the audit service when exported`); } // Steps 3 and 5: each run's head leaf, its commitment, and its path in the batch tree. for (const run of runs) { const where = `${name} batch ${String(run["batch_seq"])}`; const batch = batches.get(run["batch_seq"] as number); try { const first = run["first_seq"] as number; const lastSeq = run["last_seq"] as number; const head = sha(ascii(String(tags["head"])), uuid16(tenantId), u64(first), u64(lastSeq), hex32(run["chain_hash"])); if (rootOf(head, lastSeq - first + 1, lastSeq - first + 2, list(run["head_path"])) !== run["chain_root"]) { problems.push(`${where}: the head leaf does not lead to the chain's root`); } const lastEntry = entries.find((entry) => entry["seq"] === lastSeq); if (lastEntry && lastEntry["chain_hash"] !== run["chain_hash"]) { problems.push(`${where}: entry ${String(lastSeq)} is not the one the batch sealed`); } if (!batch) { problems.push(`${where}: the batch is not in the export`); continue; } const commit = sha(ascii("PLAT-CHAINROOT-v1"), hex32(run["leaf_salt"]), hex32(run["chain_root"])); if (rootOf(commit, run["leaf_index"] as number, batch["tree_size"] as number, list(run["batch_path"])) !== batch["merkle_root"]) { problems.push(`${where}: the chain does not lead to the batch's root`); } } catch { problems.push(`${where}: its proof is not in its form`); } } } // Step 7: a package's summary and its provider calls. if (manifest["kind"] === "AGENCY") { const tenantId = String(manifest["tenant_id"]); const own = allEntries.filter((one) => one.service === "search" && !one.internal).map((one) => one.row); const internal = allEntries.filter((one) => one.service === "search" && one.internal).map((one) => one.row); if (canonical(summaryFrom(own, tenantId)) !== canonical(f["summary"] ?? null)) { problems.push("summary: not what the entries say"); } // The internal entries are the tenant's provider calls, and nothing else of the internal chain. for (const row of internal) { if ((row["event_type"] !== "provider.call.sent" && row["event_type"] !== "provider.call.completed") || obj(row["refs"])["tenant_id"] !== tenantId) { problems.push(`internal entry ${String(row["seq"])}: not a provider call made for this tenant`); } } for (const sent of internal.filter((row) => row["event_type"] === "provider.call.sent")) { const refs = obj(sent["refs"]); const where = `provider call in internal entry ${String(sent["seq"])}`; // A call names its decision and its search by id; one that names neither matches nothing. const decisionId = str(refs["decision_id"]); const searchId = str(refs["search_request_id"]); // The decision taken just before the call, for the same search: never a read's decision, nor another search's. const decision = own.find( (row) => decisionId !== null && row["event_type"] === "policy.decision" && obj(row["refs"])["decision_id"] === decisionId && obj(row["refs"])["search_request_id"] === searchId && obj(row["payload"])["phase"] === "PRE_CALL", ); const certified = own.find((row) => searchId !== null && row["event_type"] === "purpose.certified" && obj(row["refs"])["search_request_id"] === searchId); if (!decision || obj(decision["payload"])["outcome"] !== "ALLOW" || String(decision["occurred_at"]) > String(sent["occurred_at"])) { problems.push(`${where}: no decision in the export allowed it before it was sent`); } if (!certified || String(certified["occurred_at"]) > String(sent["occurred_at"])) { problems.push(`${where}: no purpose in the export was certified for it before it was sent`); } const callId = str(refs["provider_call_id"]); const done = internal.find((row) => callId !== null && row["event_type"] === "provider.call.completed" && obj(row["refs"])["provider_call_id"] === callId); if (done && (done["seq"] as number) < (sent["seq"] as number)) { problems.push(`${where}: completed before it was recorded as sent`); } } // A call completed with no record of its being sent was made without the record that must come first (ARC-15). const sentIds = new Set(internal.filter((row) => row["event_type"] === "provider.call.sent").map((row) => obj(row["refs"])["provider_call_id"])); for (const done of internal.filter((row) => row["event_type"] === "provider.call.completed" && !sentIds.has(obj(row["refs"])["provider_call_id"]))) { problems.push(`provider call in internal entry ${String(done["seq"])}: completed, with no record in the export of its being sent`); } // The measures name each provider the package's calls went to, once, and no other (AUD-17). const called = new Set(internal.filter((row) => row["event_type"] === "provider.call.sent").map((row) => String(obj(row["payload"])["provider"]))); const stated = list(manifest["measures"]).map((one) => String(obj(one)["provider"])); for (const provider of [...called].filter((one) => !stated.includes(one)).sort()) { problems.push(`measures: none stated for provider ${provider}, whose calls are in the package`); } if (stated.some((one, i) => !called.has(one) || stated.indexOf(one) !== i)) { problems.push("measures: stated for a provider with no call in the package, or twice"); } for (const measure of list(manifest["measures"]).map(obj)) { notes.push(`provider ${String(measure["provider"])}: dedicated credentials ${String(measure["dedicated_credentials"])}, address lock ${String(measure["address_lock"])}, reconciliation ${String(measure["reconciliation"])}`); } } return report(); } /** The summary, one line per search, rebuilt from the tenant's entries in `search` as spec 08, section 8 sets it out. */ function summaryFrom(rows: readonly Obj[], tenantId: string): Json[] { const sorted = [...rows].sort((a, b) => (a["seq"] as number) - (b["seq"] as number)); const lines = new Map>(); for (const row of sorted) { const refs = obj(row["refs"]); const payload = obj(row["payload"]); const id = str(refs["search_request_id"]); if (id === null) { continue; } if (row["event_type"] === "search.submitted" && !lines.has(id)) { lines.set(id, { search_request_id: id, occurred_at: String(row["occurred_at"]), tenant_id: tenantId, user_id: str(row["actor_id"]), operation: str(payload["operation"]), relation: str(payload["relation"]), matter_id: str(refs["matter_id"]), matter_type: null, dppa_basis: null, dppa_8: null, glba_basis: null, categories_shown: [], record_count: null, }); } } for (const row of sorted) { const line = lines.get(String(obj(row["refs"])["search_request_id"])); const payload = obj(row["payload"]); if (!line) { continue; } if (row["event_type"] === "purpose.certified") { line["matter_type"] = str(payload["matter_type"]); line["dppa_basis"] = str(payload["dppa"]); line["dppa_8"] = typeof payload["dppa_8"] === "boolean" ? payload["dppa_8"] : null; line["glba_basis"] = str(payload["glba"]); } else if (row["event_type"] === "result.viewed") { const shown = list(payload["categories_shown"]).filter((one): one is string => typeof one === "string"); line["categories_shown"] = [...new Set([...(line["categories_shown"] as string[]), ...shown])].sort(); } else if (row["event_type"] === "search.completed") { line["record_count"] = typeof payload["record_count"] === "number" ? payload["record_count"] : null; } } return [...lines.values()].sort((a, b) => String(a["occurred_at"]).localeCompare(String(b["occurred_at"])) || String(a["search_request_id"]).localeCompare(String(b["search_request_id"]))); } /** A line with every control or direction character the file could carry replaced, so that no file can rewrite what is printed. */ export function printable(line: string): string { return line.replace(/[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/g, "?"); } /** Prints a report, as the tool does: each line made printable first. */ export function printReport(report: Report, print: (line: string) => void): void { const write = (line: string): void => print(printable(line)); write(report.ok ? "VERIFIED: every check passed." : `FAILED: ${String(report.problems.length)} problem${report.problems.length === 1 ? "" : "s"}.`); if (report.ok && report.unsealed > 0) { write(` ${String(report.unsealed)} entr${report.unsealed === 1 ? "y is" : "ies are"} not yet in a sealed batch: for ${report.unsealed === 1 ? "it" : "them"}, the file rests on the export key alone. Export again once their batches are sealed.`); } for (const problem of report.problems) { write(` problem: ${problem}`); } for (const note of report.notes) { write(` note: ${note}`); } } /** The command line. */ function main(argv: readonly string[]): number { const args = [...argv]; const keyAt = args.indexOf("--key"); const pinned = keyAt >= 0 ? (args[keyAt + 1] ?? null) : null; if (keyAt >= 0) { args.splice(keyAt, 2); } if (args.length !== 1 || (keyAt >= 0 && pinned === null)) { process.stderr.write("usage: node audit-verify.ts EXPORT.json --key PUBLIC_KEY\n"); return 2; } let file: unknown; try { file = JSON.parse(readFileSync(args[0] as string, "utf8")); } catch { process.stderr.write("The file cannot be read as JSON.\n"); return 2; } const report = verifyExport(file, pinned); printReport(report, (line) => process.stdout.write(`${line}\n`)); return report.ok ? 0 : 1; } /** Whether this file is the program being run, by whatever path it was started: a folder linked elsewhere, as /tmp is on macOS, too. */ function isMain(): boolean { const started = process.argv[1]; if (!started) { return false; } try { return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(started); } catch { return false; } } if (isMain()) { process.exitCode = main(process.argv.slice(2)); }