Study/server/ai/verify.ts
Elijah fe94dbf876
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
Keep rationales on what the question tests
Med math rationales were padded with unrelated drug facts (TSH and heart
rate for a levothyroxine tablet calculation) because the math prompt
asked for "one safety check" and the rationale format asked for at
least two bullets.
- Rationales take 1 to 4 bullets, each about what the question tests,
  with fewer bullets preferred to padding.
- Med math rationales explain only the calculation: the conversion, the
  method and an arithmetic check.
- The second review rewrites a rationale or "why" that brings in facts
  the question does not test.
- The outside-chatbot instructions say the same.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 15:33:38 -07:00

519 lines
30 KiB
TypeScript

/**
* Python-backed checking of generated quizzes, and the `run_python` tool the assistant uses.
*
* After a quiz is generated, a short agent loop gives the model the draft (as Python data and
* as text) plus the original request and asks it to verify everything mechanically with the
* sandbox: question count, the select-all / single / numeric mix, options and correct answers
* per type, duplicates, and every calculation recomputed from its stem. It then submits fixes,
* which are applied here only if the question still validates afterwards.
*/
import { z } from 'zod';
import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../../shared/content.ts';
import type { AppContext } from '../http.ts';
import { runPython } from './sandbox.ts';
import { loadMcpTools, mcpPromptNote, type LoadedMcp } from './mcp.ts';
import { outputBudget, resolveModel, runChat } from './service.ts';
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
/* ------------------------------ data handed to Python ------------------------------ */
/** Compact, 1-based view of a quiz used both in prompts and as the `draft` variable in Python. */
export function quizForPython(quiz: { title: string; questions: Pick<DraftQuestion, 'type' | 'stem' | 'options' | 'rationale' | 'topic'>[] }) {
return {
title: quiz.title,
question_count: quiz.questions.length,
questions: quiz.questions.map((q, i) => ({
number: i + 1,
type: q.type,
topic: q.topic,
stem: q.stem,
options: q.options.map((o, k) => ({ number: k + 1, text: o.text, correct: Boolean(o.correct), why: o.why ?? '' })),
rationale: q.rationale ?? '',
})),
};
}
/** Python source that defines the given variables from JSON (a JSON string is a valid Python string literal). */
export function pythonPrelude(vars: Record<string, unknown>): string {
const lines = ['import json as _json'];
for (const [name, value] of Object.entries(vars)) {
if (!/^[a-z_][a-z0-9_]*$/i.test(name)) continue;
lines.push(`${name} = _json.loads(${JSON.stringify(JSON.stringify(value))})`);
}
return `${lines.join('\n')}\n`;
}
/** Formats a sandbox result for a model: short, and never an exception. */
export function pythonOutput(r: Awaited<ReturnType<typeof runPython>>) {
return {
ok: r.ok,
...(r.stdout ? { stdout: r.stdout } : {}),
...(r.result !== null ? { result: r.result } : {}),
...(r.stderr ? { stderr: r.stderr.slice(0, 2000) } : {}),
...(r.error ? { error: r.error } : {}),
};
}
export const RUN_PYTHON_DESCRIPTION =
'Run Python 3 in a sandbox and get stdout and the value of the last expression. Standard library only (math, fractions, decimal, statistics, re, json, collections, itertools, datetime); no files, network or packages; 10 second limit. Use it whenever exactness matters: recompute dosage, rate and conversion answers, count questions, options and correct answers, check lengths, tally topics. Print what you need to see.';
export const RUN_PYTHON_TOOL: ToolDef = {
name: 'run_python',
description: RUN_PYTHON_DESCRIPTION,
parameters: {
type: 'object',
properties: { code: { type: 'string', description: 'Python source. Variables listed in the instructions are already defined.' } },
required: ['code'],
},
};
/* ------------------------------ verification of a generated quiz ------------------------------ */
export interface QuizRequestFacts {
count: number;
style: string;
sataShare: number;
difficulty?: string;
/** Most calculation (numeric) questions allowed; 0 when the material has no calculations. */
numericCap?: number;
/** The user asked for exactly `numericCap` medication math questions (not from the material). */
numericRequested?: boolean;
}
/** Long text is shortened rather than rejected: a verbose but correct review must not be lost. */
const text = (max: number) => z.coerce.string().transform((v) => v.slice(0, max));
const int = z.coerce.number().int();
const rewriteSchema = z.object({
type: z.enum(['single', 'multi', 'numeric']),
stem: text(4000),
options: z.array(z.object({ text: text(1000), correct: z.coerce.boolean(), why: text(600).optional().default('') })).min(1).max(8),
rationale: text(4000).optional().default(''),
});
const fixSchema = z.object({
question: int.min(1),
action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'rewrite', 'remove']),
correct_options: z.array(int.min(1)).max(12).optional(),
topic: text(80).optional(),
answer: text(200).optional(),
type: z.enum(['single', 'multi', 'numeric']).optional(),
/** rewrite: the whole corrected question. */
question_text: rewriteSchema.optional(),
reason: text(600).optional().default(''),
});
const reviewSchema = z.object({
summary: text(1500).optional().default(''),
checks: z
.array(z.object({ name: text(160), passed: z.coerce.boolean(), detail: text(600).optional().default('') }))
.transform((v) => v.slice(0, 40))
.optional()
.default([]),
fixes: z.array(z.unknown()).optional().default([]).transform((list) => list.flatMap((f) => { const r = fixSchema.safeParse(f); return r.success ? [r.data] : []; })),
});
export type QuizReview = z.infer<typeof reviewSchema>;
export type QuizFix = z.infer<typeof fixSchema>;
const SUBMIT_REVIEW_QUESTION_TEXT = {
type: 'object',
description: 'For rewrite only: the complete corrected question, following the rules for its type.',
properties: {
type: { type: 'string', enum: ['single', 'multi', 'numeric'] },
stem: { type: 'string' },
options: { type: 'array', items: { type: 'object', properties: { text: { type: 'string' }, correct: { type: 'boolean' }, why: { type: 'string' } }, required: ['text', 'correct'] } },
rationale: { type: 'string', description: 'Markdown: one bold key sentence, then 2 to 4 short bullet points.' },
},
required: ['type', 'stem', 'options', 'rationale'],
};
const SUBMIT_REVIEW: ToolDef = {
name: 'submit_review',
description: 'Finish the check: report every check you ran (with its result) and the fixes to apply. Call this exactly once, at the end.',
parameters: {
type: 'object',
properties: {
summary: { type: 'string', description: 'One or two sentences for the student.' },
checks: {
type: 'array',
items: { type: 'object', properties: { name: { type: 'string' }, passed: { type: 'boolean' }, detail: { type: 'string' } }, required: ['name', 'passed'] },
},
fixes: {
type: 'array',
description: 'Only for definite errors. Question and option numbers are 1-based.',
items: {
type: 'object',
properties: {
question: { type: 'integer' },
action: {
type: 'string',
enum: ['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'rewrite', 'remove'],
description:
'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. set_topic: replace a narrow topic label with a broad one. rewrite: replace the whole question (give question_text) when the stem, options or rationale are wrong, contradictory or garbled. remove: drop a duplicate or a question that cannot be saved; removed questions are replaced with new ones automatically.',
},
question_text: SUBMIT_REVIEW_QUESTION_TEXT,
correct_options: { type: 'array', items: { type: 'integer' } },
topic: { type: 'string' },
answer: { type: 'string' },
type: { type: 'string', enum: ['single', 'multi', 'numeric'] },
reason: { type: 'string' },
},
required: ['question', 'action', 'reason'],
},
},
},
required: ['summary', 'checks', 'fixes'],
},
};
function rulesFor(req: QuizRequestFacts): string {
const expectedMulti = req.sataShare <= 0 ? 'none' : req.sataShare >= 1 ? 'all non-numeric questions' : `about ${Math.round(req.sataShare * 100)}% of the non-numeric questions`;
return [
`Requested: ${req.count} NCLEX-style questions, difficulty "${req.difficulty ?? 'mixed'}", select-all (multi) share: ${expectedMulti}, calculation (numeric) questions: ${req.numericRequested ? (req.numericCap ? `exactly ${req.numericCap} medication math questions, requested by the user (they need not come from the material)` : 'none') : req.numericCap ? `at most ${req.numericCap}` : 'none (the material has no calculations)'}.`,
'Rules each question must meet:',
'- single: exactly one correct option and 4 options.',
'- multi (select all that apply): at least 2 correct options; for NCLEX style 6 options with 2 to 5 correct and a stem ending with "Select all that apply."',
'- numeric: the student types a number; options holds the accepted answer(s), all marked correct. The answer must equal the value computed from the stem, rounded as the stem says.',
'- No duplicate or near-duplicate stems; no identical options within a question; no "all of the above" / "none of the above".',
'- The keyed answer must be the clinically correct one (check against the rationale and standard practice).',
'- Topic labels are broad (the main condition, drug class or body system, e.g. "Stroke"), not subtypes or aspects ("Hemorrhagic stroke", "Heparin client teaching").',
'Ignore: which position the correct option is in (answer order is shuffled every time a quiz is taken), quotation marks, capitalisation and other formatting.',
].join('\n');
}
const VERIFY_SYSTEM = `You are the quality checker for quiz questions written for a nursing student. Accuracy is critical: they are used to prepare for licensing exams.
Check the draft mechanically with the run_python tool. The variables \`draft\` (the quiz) and \`request\` (what was asked for) are already defined in every run.
Be efficient: write ONE Python script that does all the mechanical checks at once and prints a compact report, then at most one more run for the calculations. Do not use more than 3 runs. The script should check:
1. The number of questions against request["count"].
2. The mix of types (single / multi / numeric) against the requested select-all share and the calculation limit.
3. For every question: option count, number of correct options, and that it fits its type's rules.
4. Duplicate stems or options.
5. Every numeric question: recompute the answer from the numbers in the stem, apply the rounding the stem asks for, and compare with the keyed answer numerically (ignore units).
6. How many distinct topics there are, and whether any are narrow variants of a broader one.
Also read each question and judge whether the keyed answer is clinically correct.
Then call submit_review once. List only real checks. A check passes when the draft is right after your fixes: if you give fixes for every problem a check found, mark it passed and say what you fixed in its detail. Mark a check failed only for a problem that remains. Never list informational or cosmetic remarks as failed checks. Give a fix for each definite error. Do not "fix" style.
Every problem in a question can be fixed: correct the key with set_correct, or use rewrite to give the whole corrected question (for a garbled or contradictory stem, wrong numbers, a single-answer item with two defensible answers, a select-all item with too few options). Remove duplicates and anything you cannot repair: removed questions are replaced with new ones after your review and the quiz is checked again, so a count that is short only because of your removals is not a failure. Report a count or type mix that is off for any other reason as a failed check.`;
/** Applies validated fixes; returns notes describing what changed or was refused. */
/** A long rationale shortened for a prompt, marked so the model never takes the cut for a fault. */
function clipRationale(text: string, max: number): string {
return text.length > max ? `${text.slice(0, max)} [shortened here; the full rationale is longer]` : text;
}
export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer<typeof fixSchema>[]): { questions: DraftQuestion[]; applied: string[]; refused: string[] } {
const out = questions.map((q) => ({ ...q, options: q.options.map((o) => ({ ...o })) }));
const remove = new Set<number>();
const applied: string[] = [];
const refused: string[] = [];
for (const fix of fixes) {
const i = fix.question - 1;
const q = out[i];
if (!q) {
refused.push(`Question ${fix.question}: does not exist.`);
continue;
}
const before = { ...q, options: q.options.map((o) => ({ ...o })) };
const why = fix.reason ? ` (${fix.reason.replace(/\s+/g, ' ').slice(0, 200)})` : '';
if (fix.action === 'remove') {
remove.add(i);
applied.push(`Question ${fix.question} removed${why}.`);
continue;
}
if (fix.action === 'set_correct' && fix.correct_options?.length) {
const set = new Set(fix.correct_options.map((n) => n - 1));
// The same key again is no fix at all.
if (q.options.every((o, k) => Boolean(o.correct) === set.has(k))) continue;
q.options = q.options.map((o, k) => ({ ...o, correct: set.has(k) }));
if (q.type === 'single' && set.size > 1) q.type = 'multi';
} else if (fix.action === 'set_numeric_answer' && fix.answer?.trim() && q.type === 'numeric') {
q.options = [{ text: fix.answer.trim(), correct: true, why: q.options[0]?.why ?? '' }];
} else if (fix.action === 'set_type' && fix.type && fix.type !== 'numeric') {
q.type = fix.type;
} else if (fix.action === 'set_topic' && fix.topic?.trim()) {
q.topic = fix.topic.trim().slice(0, 80);
} else if (fix.action === 'rewrite' && fix.question_text?.stem.trim()) {
const r = fix.question_text;
q.type = r.type;
q.stem = r.stem.trim();
q.options = r.options.map((o) => ({ text: o.text.trim(), correct: r.type === 'numeric' ? true : o.correct, why: o.why ?? '' }));
if (r.rationale.trim()) q.rationale = r.rationale.trim();
} else {
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} was incomplete.`);
continue;
}
if (validateDraftQuestion(q, i).some((p) => p.level === 'error')) {
out[i] = before;
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} would have made it invalid, so it was not applied${why}.`);
continue;
}
applied.push(`Question ${fix.question}: ${fix.action === 'set_numeric_answer' ? `answer corrected to ${fix.answer!.trim()}` : fix.action === 'set_type' ? `type changed to ${fix.type}` : fix.action === 'set_topic' ? `topic changed to ${q.topic}` : fix.action === 'rewrite' ? 'rewritten' : `answer key corrected`}${fix.action === 'set_topic' ? '' : why}.`);
}
return { questions: out.filter((_, i) => !remove.has(i)), applied, refused };
}
export interface QuizCheck {
draft: QuizDraft;
notes: string[];
review: QuizReview | null;
pythonRuns: number;
/** Checks still failing after the fixes. */
failed: string[];
applied: string[];
refused: string[];
removed: number;
/** Questions replaced by a rewrite: new content that needs checking again. */
rewritten: number;
/** The check itself did not complete (no tools, error, no report). */
incomplete: boolean;
}
/**
* Checks a generated quiz with the model and the Python sandbox, then applies the fixes.
* Never throws: a failed check leaves the draft as it was and says so in the notes.
*/
export async function verifyQuizDraft(
app: AppContext,
draft: QuizDraft,
req: QuizRequestFacts,
opts: { modelId?: string | null; signal?: AbortSignal; onStep?: (label: string) => void } = {},
): Promise<QuizCheck> {
const resolved = resolveModel(app, 'check', opts.modelId);
if (!resolved.model.capabilities.tools) {
return { draft, notes: ['The Python answer check was skipped: the generation model cannot use tools.'], review: null, pythonRuns: 0, failed: [], applied: [], refused: [], removed: 0, rewritten: 0, incomplete: true };
}
const data = quizForPython(draft);
const prelude = pythonPrelude({ draft: data, request: { count: req.count, style: req.style, sata_share: req.sataShare, difficulty: req.difficulty ?? 'mixed' } });
const listing = data.questions
.map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${clipRationale(q.rationale, 600)}` : ''}`)
.join('\n\n');
const messages: ChatMessage[] = [{ role: 'user', content: [{ type: 'text', text: `${rulesFor(req)}\n\nTHE DRAFT (${data.question_count} questions):\n\n${listing}` }] }];
let review: QuizReview | null = null;
let pythonRuns = 0;
let mcpCalls = 0;
const MCP_BUDGET = 4;
const mcp = await loadMcpTools(app, 'check', opts.signal).catch(() => ({ tools: [], servers: [], unavailable: [] }) as LoadedMcp);
const system = VERIFY_SYSTEM + mcpPromptNote(mcp);
// Up to WORK_STEPS turns with Python, then only submit_review is offered until it arrives.
const WORK_STEPS = 4;
const PYTHON_BUDGET = 4;
const MAX_STEPS = 7;
try {
for (let step = 0; step < MAX_STEPS && !review; step += 1) {
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
const wrapUp = step >= WORK_STEPS || pythonRuns >= PYTHON_BUDGET;
if (wrapUp && !messages.some((m) => m.content.some((p) => p.type === 'text' && p.text.startsWith('You have run enough checks')))) {
const nudge: ContentPart = { type: 'text', text: 'You have run enough checks. Call submit_review now with your results and fixes.' };
const lastMsg = messages[messages.length - 1];
// Ride along with the tool results so user and assistant turns keep alternating.
if (lastMsg?.role === 'user') lastMsg.content.push(nudge);
else messages.push({ role: 'user', content: [nudge] });
}
const res = await runChat(app, {
task: 'check',
purpose: 'quiz check',
modelId: opts.modelId,
system,
messages,
// MCP tools stay listed at wrap-up (the transcript may hold their calls) but submit_review is forced.
tools: wrapUp ? [SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)],
toolChoice: wrapUp ? { name: 'submit_review' } : 'auto',
maxOutputTokens: outputBudget(resolved, 16_000),
signal: opts.signal,
});
messages.push(res.message);
const calls = res.message.content.filter((p): p is Extract<ContentPart, { type: 'tool_call' }> => p.type === 'tool_call');
if (calls.length === 0) {
messages.push({ role: 'user', content: [{ type: 'text', text: 'Run your checks with run_python, then call submit_review.' }] });
continue;
}
const results: ContentPart[] = [];
for (const call of calls) {
if (call.name === 'submit_review') {
const parsed = reviewSchema.safeParse(call.input);
if (parsed.success) review = parsed.data;
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: parsed.success ? '{"ok":true}' : JSON.stringify({ error: parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') }), isError: !parsed.success });
} else if (call.name === 'run_python' && pythonRuns >= PYTHON_BUDGET) {
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"Python budget used up. Call submit_review now."}', isError: true });
} else if (call.name === 'run_python') {
pythonRuns += 1;
opts.onStep?.(`Checking the questions with Python (run ${pythonRuns})`);
const r = await runPython(prelude + String(call.input.code ?? ''));
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify(pythonOutput(r)), isError: !r.ok });
} else if (mcp.tools.some((t) => t.def.name === call.name)) {
const binding = mcp.tools.find((t) => t.def.name === call.name)!;
if (mcpCalls >= MCP_BUDGET) {
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"External tool budget used up. Finish the check."}', isError: true });
continue;
}
mcpCalls += 1;
opts.onStep?.(binding.kind === 'mcp' ? `Checking with ${binding.serverName}` : 'Checking on the web');
try {
const { text } = await binding.call(call.input, opts.signal);
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: text.slice(0, 24_000), isError: false });
} catch (err) {
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify({ error: err instanceof Error ? err.message : 'Tool failed' }), isError: true });
}
} else {
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"unknown tool"}', isError: true });
}
}
// The last user message carries the tool results; keep wrap-up instructions in front of the model.
messages.push({ role: 'user', content: results });
}
} catch (err) {
if (err instanceof AIError && err.code === 'aborted') throw err;
const message = err instanceof Error ? err.message : String(err);
return { draft, notes: [`The Python answer check could not finish (${message.slice(0, 200)}). Review the answers yourself.`], review: null, pythonRuns, failed: [], applied: [], refused: [], removed: 0, rewritten: 0, incomplete: true };
}
if (!review) return { draft, notes: ['The Python answer check did not report back. Review the answers yourself.'], review: null, pythonRuns, failed: [], applied: [], refused: [], removed: 0, rewritten: 0, incomplete: true };
const { questions, applied, refused } = applyQuizFixes(draft.questions, review.fixes);
// Remarks the model files as informational or cosmetic are not checks.
const real = review.checks.filter((c) => !/informational|cosmetic|position bias/i.test(`${c.name} ${c.detail}`));
// A problem the model fixed is not a failure: the applied fix is listed on its own.
const fixedHere = (c: { detail: string }) => applied.length > 0 && /\b(fixed|corrected|fix applied)\b/i.test(c.detail ?? '');
// Removals are refilled by the generator, so a count short only by them is not a failure.
const removed = draft.questions.length - questions.length;
const failed = real.filter((c) => !c.passed && !fixedHere(c) && !(removed > 0 && /\bcount\b/i.test(c.name) && questions.length + removed >= req.count));
const notes = [
`Checked with Python (${pythonRuns} run${pythonRuns === 1 ? '' : 's'}): ${real.length - failed.length} of ${real.length} checks passed${applied.length ? `, ${applied.length} fix${applied.length === 1 ? '' : 'es'} applied` : ''}.`,
...failed.map((c) => `Check failed: ${c.name}${c.detail ? `: ${c.detail}` : ''}`),
...applied,
...refused,
];
return {
draft: { ...draft, questions },
notes,
review,
pythonRuns,
failed: failed.map((c) => `${c.name}${c.detail ? `: ${c.detail}` : ''}`),
applied,
refused,
removed,
rewritten: review.fixes.filter((f) => f.action === 'rewrite').length - refused.filter((r) => /rewrite/.test(r)).length,
incomplete: false,
};
}
/* ------------------------------ independent second review ------------------------------ */
/**
* Wording that suggests the writer changed its mind mid-question or left a draft note in:
* "A 78 kg? No — an 80 kg client", "Actually,", "wait", "let me"... Hints for the reviewer,
* which still decides.
*/
const LEFTOVER_RE = /\?\s*no\b|\b(wait|actually|oops|scratch that|let me|i mean|correction|hmm|on second thought)\b[,.:\s—-]/i;
export function suspiciousWording(questions: Pick<DraftQuestion, 'stem' | 'options' | 'rationale'>[]): string[] {
const out: string[] = [];
questions.forEach((q, i) => {
const parts = [['stem', q.stem], ...q.options.map((o, k) => [`option ${k + 1}`, o.text] as const), ['rationale', q.rationale ?? '']] as const;
for (const [where, value] of parts) {
const m = LEFTOVER_RE.exec(value ?? '');
if (m) {
const at = Math.max(0, m.index - 30);
out.push(`Q${i + 1} ${where}: "…${value.slice(at, m.index + 50).replace(/\s+/g, ' ')}…"`);
break;
}
}
});
return out;
}
const SUBMIT_PROOFREAD: ToolDef = {
name: 'submit_proofread',
description: 'Report the questions that need repair, each with its complete rewritten version (or removal). An empty list means every question is sound.',
parameters: {
type: 'object',
properties: {
issues: {
type: 'array',
items: {
type: 'object',
properties: {
question: { type: 'integer', description: '1-based question number.' },
problem: { type: 'string', description: 'What is wrong, in one sentence.' },
action: { type: 'string', enum: ['rewrite', 'remove'] },
question_text: SUBMIT_REVIEW_QUESTION_TEXT,
},
required: ['question', 'problem', 'action'],
},
},
},
required: ['issues'],
},
};
const proofreadSchema = z.object({
issues: z
.array(z.unknown())
.optional()
.default([])
.transform((list) =>
list.flatMap((raw) => {
const r = z.object({ question: int.min(1), problem: text(600).optional().default(''), action: z.enum(['rewrite', 'remove']), question_text: rewriteSchema.optional() }).safeParse(raw);
return r.success ? [r.data] : [];
}),
),
});
const PROOFREAD_SYSTEM = `You are a second, independent reviewer of NCLEX-style quiz questions for a nursing student, reading the finished quiz the way a careful student and an experienced item writer would. Another checker has already verified counts, types and arithmetic; you look at how each question reads and whether it is sound.
Flag a question only for a real problem:
- the writer changed its mind or left a note in the text (for example "A 78 kg? No — an 80 kg client", "Actually,", "wait"), or the wording is garbled, truncated or repeated;
- data needed to answer is missing, or the stem contradicts itself, the options or the rationale;
- two options are defensible, or none is correct; the rationale disagrees with the keyed answer;
- an option gives the answer away, or the content is clinically wrong or outdated;
- the rationale or a choice's "why" brings in facts the question does not test (for example lab monitoring or dose limits in a plain dosage calculation): rewrite it to explain only what the question asks.
For each problem give the complete corrected question with rewrite (same type, the same clinical point, following the rules for its type: single = 4 options with exactly one correct; multi = 6 options with 2 to 5 correct and a stem ending "Select all that apply."; a short plain stem; choices concise and parallel, with no explanation tacked on; numeric = one option, the correct answer with its unit, and "why" showing the working), or remove it if it cannot be saved. Do not flag style preferences or answer order. Call submit_proofread once; an empty list is the right answer when every question is sound.`;
/**
* An independent read of the whole quiz by a fresh model call, for problems the mechanical
* check misses (a writer changing its mind mid-stem, contradictions, two defensible answers).
* Never throws except on cancel.
*/
export async function proofreadQuiz(
app: AppContext,
draft: QuizDraft,
opts: { modelId?: string | null; signal?: AbortSignal } = {},
): Promise<{ draft: QuizDraft; flagged: number; fixes: QuizFix[]; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> {
const none = { draft, flagged: 0, fixes: [], applied: [], refused: [], rewritten: 0, removed: 0 };
const resolved = resolveModel(app, 'review', opts.modelId);
if (!resolved.model.capabilities.tools) return { ...none, incomplete: true };
const data = quizForPython(draft);
const listing = data.questions
.map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${q.rationale}` : ''}`)
.join('\n\n');
const hints = suspiciousWording(draft.questions);
const prompt = `${hints.length ? `Wording that looks like a leftover or a change of mind (confirm and fix each):\n${hints.map((h) => `- ${h}`).join('\n')}\n\n` : ''}THE QUIZ (${data.question_count} questions):\n\n${listing}`;
try {
const res = await runChat(app, {
task: 'review',
purpose: 'quiz second review',
modelId: opts.modelId,
system: PROOFREAD_SYSTEM,
messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
tools: [SUBMIT_PROOFREAD],
toolChoice: { name: 'submit_proofread' },
maxOutputTokens: outputBudget(resolved, 16_000),
signal: opts.signal,
});
const call = res.message.content.find((p) => p.type === 'tool_call');
const parsed = call && call.type === 'tool_call' ? proofreadSchema.safeParse(call.input) : null;
if (!parsed?.success) return { ...none, incomplete: true };
const issues = parsed.data.issues;
const fixes = issues.map((i) => ({ question: i.question, action: i.action, question_text: i.question_text, reason: i.problem }) as z.infer<typeof fixSchema>);
const { questions, applied, refused } = applyQuizFixes(draft.questions, fixes);
return {
draft: { ...draft, questions },
flagged: issues.length,
fixes,
applied,
refused,
rewritten: applied.filter((a) => /: rewritten/.test(a)).length,
removed: draft.questions.length - questions.length,
incomplete: false,
};
} catch (err) {
if (err instanceof AIError && err.code === 'aborted') throw err;
app.log('warn', 'The second review of a quiz failed', err instanceof Error ? err.message : err);
return { ...none, incomplete: true };
}
}