From 1a212bb8ba43000e5e2e122b2386f169202f7525 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 16 Nov 2026 10:07:32 -0700 Subject: [PATCH] Quiz check: fix before handing over, with an independent second review - The check can rewrite a whole question (garbled or contradictory stems, missing data, two defensible answers); a key fix that changes nothing is skipped. - Removed questions are replaced with new ones, so a 50-question quiz stays 50. - A second, independent model call reads the whole quiz for badly written items, helped by a scan for leftovers such as "A 78 kg? No - an 80 kg". - The quiz is checked again after any rewrite, replacement or open problem, up to three rounds, before it is handed over. - Reviewers see full rationales (a shortened one is marked as shortened). Co-Authored-By: Claude Opus 5.5 --- PROGRESS.md | 7 +- docs/AI.md | 20 +++- server/ai/generate.ts | 60 +++++++++- server/ai/verify.ts | 211 ++++++++++++++++++++++++++++++++++-- tests/ai.test.ts | 2 +- tests/mockAi.ts | 27 ++++- tests/revision5.test.ts | 105 ++++++++++++++++++ web/src/pages/DraftPage.tsx | 6 +- 8 files changed, 412 insertions(+), 26 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 6dd73f7..f802fab 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -173,6 +173,11 @@ has no information); MCP and web search tools load automatically; share-link AI week, can be limited to the US or Arizona (`server/services/geo.ts`, DB-IP Lite) and is locked by abuse protection (migration 8). +Follow-up the same day: the quiz check no longer hands over a quiz with known problems. It can +rewrite a whole question, removed questions are replaced, a second independent review reads the +whole quiz for badly written items (a writer changing its mind mid-stem), and the quiz is +checked again until it passes (at most three rounds). + Tests: `tests/revision5.test.ts`. Live check on the rev1 copy with DeepSeek flash: a visitor's 10-card set was written, named ("Heparin, Furosemide, and Dosage Calculations"), saved and listed once with the New tag; cards were single facts with no hints. The rev1 copy's 401 was its @@ -181,7 +186,7 @@ stored DeepSeek connection still holding the old key; it was updated from `.env` ## Verification status Baseline: 162 tests passed, TypeScript passed and production web/server builds passed. -Added six regression tests. Full `npm run check` passes: **274 tests** (fifth revision, 2026-11-16; 261 after share-link generation, 2026-11-15; 257 after the fourth revision; 247 after the third, 208 after the second, 181 on 2026-11-12), TypeScript +Added six regression tests. Full `npm run check` passes: **280 tests** (check loop and second review, 2026-11-16; 274 after the fifth revision; 261 after share-link generation, 2026-11-15; 257 after the fourth revision; 247 after the third, 208 after the second, 181 on 2026-11-12), TypeScript and both production builds. The Docker `test` stage runs the same 181 tests on Node 24. Tests were run with loopback access for the mock provider. The earlier agent manually exercised quiz completion/results, flashcards, streak effects, diff --git a/docs/AI.md b/docs/AI.md index dcbc785..af82501 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -78,9 +78,23 @@ After a quiz is generated (unless "Check answers" is off), the generation model as Python data plus the request (count, select-all share, calculation allowance). In at most three Python runs it checks the count, question types, number of correct answers per type, numeric answers, the calculation allowance and topic breadth, then reports with `submit_review`. -Fixes are applied only when the question stays valid. Remarks the model files as cosmetic or -informational (answer position, quotes) are not counted as failed checks; answer order is -shuffled whenever a quiz is taken. +Fixes are applied only when the question stays valid: a new key (`set_correct`, skipped when it +changes nothing), a numeric answer, a type or topic, a complete `rewrite` of the question (for a +garbled or contradictory stem, missing data, two defensible answers), or `remove`. Remarks the +model files as cosmetic or informational (answer position, quotes) are not counted as failed +checks; answer order is shuffled whenever a quiz is taken. + +Then a **second review**: a fresh model call with no Python reads the whole quiz as a careful +student and item writer would, and rewrites or removes questions that are badly written (a +writer changing its mind mid-stem such as "A 78 kg? No — an 80 kg client", contradictions, +missing data, two defensible answers, a rationale that disagrees with the key). A text scan +for leftovers ("? No", "Actually,", "wait") is passed to it as hints (`suspiciousWording`). + +The quiz is not handed over while anything is outstanding: removed questions are replaced with +new ones, and after any rewrite, replacement, failed check or refused fix the whole quiz is +checked again (both reviews), up to three rounds (`VERIFY_ROUNDS` in `server/ai/generate.ts`). +A count left short after the last round is still refilled. The notes say how many rounds ran +and list every fix by round; anything still failing after three rounds is listed as failed. ## Quiz generation rules diff --git a/server/ai/generate.ts b/server/ai/generate.ts index a7c7a61..7f6a923 100644 --- a/server/ai/generate.ts +++ b/server/ai/generate.ts @@ -42,7 +42,10 @@ import { weakTopics } from '../services/stats.ts'; import { getSetting, setSetting } from '../settings.ts'; import { newId, now, parseJson } from '../util.ts'; import { generateStructured, resolveModel, runChat, runChatWithMcp } from './service.ts'; -import { verifyQuizDraft } from './verify.ts'; +import { proofreadQuiz, verifyQuizDraft, type QuizCheck } from './verify.ts'; + +/** Most quiz checks per generation: the first, then re-checks after fixes and replacements. */ +const VERIFY_ROUNDS = 3; import { AIError } from './types.ts'; /* ------------------------------ drafts ------------------------------ */ @@ -1029,12 +1032,57 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ ); } - // Check counts, the type mix, answer keys and calculations with Python before review. + // Check counts, the type mix, answer keys and calculations with Python, and keep fixing until + // the quiz passes: fixes are applied, removed questions are replaced with new ones, and the + // whole quiz is checked again (at most VERIFY_ROUNDS checks). if (draft.kind === 'quiz' && req.verify !== false) { - setProgress(db, draftId, 0.96, 'Checking the questions with Python', draft); - const checked = await verifyQuizDraft(app, draft, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, label) }); - draft.questions = checked.draft.questions; - notes.push(...checked.notes); + const quiz = draft; + const history: string[] = []; + let last: QuizCheck | null = null; + let rounds = 0; + let secondReview: Awaited> | null = null; + for (let round = 1; round <= VERIFY_ROUNDS; round += 1) { + rounds = round; + if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.'); + setProgress(db, draftId, 0.96, round === 1 ? 'Checking the questions with Python' : `Checking the questions again (round ${round})`, quiz); + last = await verifyQuizDraft(app, quiz, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, round === 1 ? label : `${label} · round ${round}`) }); + quiz.questions = last.draft.questions; + if (last.incomplete) break; + history.push(...last.applied.map((a) => (round > 1 ? `Round ${round}: ${a}` : a))); + // A second, independent read of the whole quiz for badly written questions. + setProgress(db, draftId, 0.98, round === 1 ? 'Second review of the questions' : `Second review · round ${round}`, quiz); + const second = await proofreadQuiz(app, quiz, { modelId: req.modelId, signal }); + quiz.questions = second.draft.questions; + secondReview = second; + history.push(...second.applied.map((a) => `${round > 1 ? `Round ${round}: ` : ''}Second review: ${a}`)); + if (second.refused.length) history.push(...second.refused.map((r) => `Second review: ${r}`)); + // Replace what the check removed (duplicates, unfixable items) with new questions. + const missing = count - quiz.questions.length; + if (missing > 0) { + setProgress(db, draftId, 0.96, `Replacing ${missing} question${missing === 1 ? '' : 's'}`, quiz); + for (let attempt = 0; attempt < 3 && quiz.questions.length < count; attempt += 1) { + const wantMath = forcedMath ? Math.min(forcedMath - numericCount(), count - quiz.questions.length) : 0; + try { + if (wantMath > 0) await runBatch(batches + 30 + round * 3 + attempt, wantMath, true); + else await runBatch(batches + 30 + round * 3 + attempt, Math.min(perBatch, count - quiz.questions.length)); + } catch (err) { + if (err instanceof AIError && err.code === 'aborted') throw err; + } + } + if (forcedMath) quiz.questions = spreadEvenly(quiz.questions, (q) => q.type === 'numeric'); + history.push(`Round ${round}: ${missing} removed question${missing === 1 ? ' was' : 's were'} replaced with new ones${round === VERIFY_ROUNDS ? ' (after the last check)' : ''}.`); + } + // Done when nothing is left to fix and nothing new (rewritten or replacement questions) needs checking. + const done = missing <= 0 && last.rewritten === 0 && second.applied.length === 0 && last.failed.length === 0 && last.refused.length === 0; + if (done || round === VERIFY_ROUNDS) break; + } + if (last) { + const summary = last.notes[0] ?? ''; + notes.push( + ...(last.incomplete ? last.notes : [ + rounds > 1 ? summary.replace(/^Checked with Python/, `Checked with Python in ${rounds} rounds, last round`) : summary, ...last.failed.map((f) => `Check failed: ${f}`), ...(secondReview ? [secondReview.incomplete ? 'The second review could not finish.' : secondReview.flagged === 0 ? 'Second review: no problems found.' : `Second review: ${secondReview.flagged} question${secondReview.flagged === 1 ? '' : 's'} fixed in the last round.`] : []), ...history, ...last.refused]), + ); + } } // A short name and description of what this particular set covers. diff --git a/server/ai/verify.ts b/server/ai/verify.ts index 8f3173b..e95b125 100644 --- a/server/ai/verify.ts +++ b/server/ai/verify.ts @@ -83,13 +83,21 @@ export interface QuizRequestFacts { /** Long text is shortened rather than rejected: a verbose but correct review must not be lost. */ const text = (max: number) => z.coerce.string().transform((v) => v.slice(0, max)); const int = z.coerce.number().int(); +const rewriteSchema = z.object({ + type: z.enum(['single', 'multi', 'numeric']), + stem: text(4000), + options: z.array(z.object({ text: text(1000), correct: z.coerce.boolean(), why: text(600).optional().default('') })).min(1).max(8), + rationale: text(4000).optional().default(''), +}); const fixSchema = z.object({ question: int.min(1), - action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'remove']), + action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'rewrite', 'remove']), correct_options: z.array(int.min(1)).max(12).optional(), topic: text(80).optional(), answer: text(200).optional(), type: z.enum(['single', 'multi', 'numeric']).optional(), + /** rewrite: the whole corrected question. */ + question_text: rewriteSchema.optional(), reason: text(600).optional().default(''), }); const reviewSchema = z.object({ @@ -103,6 +111,18 @@ const reviewSchema = z.object({ }); export type QuizReview = z.infer; +const SUBMIT_REVIEW_QUESTION_TEXT = { + type: 'object', + description: 'For rewrite only: the complete corrected question, following the rules for its type.', + properties: { + type: { type: 'string', enum: ['single', 'multi', 'numeric'] }, + stem: { type: 'string' }, + options: { type: 'array', items: { type: 'object', properties: { text: { type: 'string' }, correct: { type: 'boolean' }, why: { type: 'string' } }, required: ['text', 'correct'] } }, + rationale: { type: 'string' }, + }, + required: ['type', 'stem', 'options', 'rationale'], +}; + const SUBMIT_REVIEW: ToolDef = { name: 'submit_review', description: 'Finish the check: report every check you ran (with its result) and the fixes to apply. Call this exactly once, at the end.', @@ -123,10 +143,11 @@ const SUBMIT_REVIEW: ToolDef = { question: { type: 'integer' }, action: { type: 'string', - enum: ['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'remove'], + enum: ['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'rewrite', 'remove'], description: - 'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. set_topic: replace a narrow topic label with a broad one. remove: drop an unfixable question.', + 'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. set_topic: replace a narrow topic label with a broad one. rewrite: replace the whole question (give question_text) when the stem, options or rationale are wrong, contradictory or garbled. remove: drop a duplicate or a question that cannot be saved; removed questions are replaced with new ones automatically.', }, + question_text: SUBMIT_REVIEW_QUESTION_TEXT, correct_options: { type: 'array', items: { type: 'integer' } }, topic: { type: 'string' }, answer: { type: 'string' }, @@ -166,9 +187,15 @@ Be efficient: write ONE Python script that does all the mechanical checks at onc 5. Every numeric question: recompute the answer from the numbers in the stem, apply the rounding the stem asks for, and compare with the keyed answer numerically (ignore units). 6. How many distinct topics there are, and whether any are narrow variants of a broader one. Also read each question and judge whether the keyed answer is clinically correct. -Then call submit_review once. List only real checks. A check passes when the draft is right after your fixes: if you give fixes for every problem a check found, mark it passed and say what you fixed in its detail. Mark a check failed only for a problem that remains. Never list informational or cosmetic remarks as failed checks. Give a fix for each definite error. Do not "fix" style. A count or mix that is off cannot be fixed here; report it as a failed check.`; +Then call submit_review once. List only real checks. A check passes when the draft is right after your fixes: if you give fixes for every problem a check found, mark it passed and say what you fixed in its detail. Mark a check failed only for a problem that remains. Never list informational or cosmetic remarks as failed checks. Give a fix for each definite error. Do not "fix" style. +Every problem in a question can be fixed: correct the key with set_correct, or use rewrite to give the whole corrected question (for a garbled or contradictory stem, wrong numbers, a single-answer item with two defensible answers, a select-all item with too few options). Remove duplicates and anything you cannot repair: removed questions are replaced with new ones after your review and the quiz is checked again, so a count that is short only because of your removals is not a failure. Report a count or type mix that is off for any other reason as a failed check.`; /** Applies validated fixes; returns notes describing what changed or was refused. */ +/** A long rationale shortened for a prompt, marked so the model never takes the cut for a fault. */ +function clipRationale(text: string, max: number): string { + return text.length > max ? `${text.slice(0, max)} [shortened here; the full rationale is longer]` : text; +} + export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer[]): { questions: DraftQuestion[]; applied: string[]; refused: string[] } { const out = questions.map((q) => ({ ...q, options: q.options.map((o) => ({ ...o })) })); const remove = new Set(); @@ -190,6 +217,8 @@ export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer n - 1)); + // The same key again is no fix at all. + if (q.options.every((o, k) => Boolean(o.correct) === set.has(k))) continue; q.options = q.options.map((o, k) => ({ ...o, correct: set.has(k) })); if (q.type === 'single' && set.size > 1) q.type = 'multi'; } else if (fix.action === 'set_numeric_answer' && fix.answer?.trim() && q.type === 'numeric') { @@ -198,6 +227,12 @@ export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer ({ text: o.text.trim(), correct: r.type === 'numeric' ? true : o.correct, why: o.why ?? '' })); + if (r.rationale.trim()) q.rationale = r.rationale.trim(); } else { refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} was incomplete.`); continue; @@ -207,11 +242,27 @@ export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer !remove.has(i)), applied, refused }; } +export interface QuizCheck { + draft: QuizDraft; + notes: string[]; + review: QuizReview | null; + pythonRuns: number; + /** Checks still failing after the fixes. */ + failed: string[]; + applied: string[]; + refused: string[]; + removed: number; + /** Questions replaced by a rewrite: new content that needs checking again. */ + rewritten: number; + /** The check itself did not complete (no tools, error, no report). */ + incomplete: boolean; +} + /** * Checks a generated quiz with the model and the Python sandbox, then applies the fixes. * Never throws: a failed check leaves the draft as it was and says so in the notes. @@ -221,15 +272,15 @@ export async function verifyQuizDraft( draft: QuizDraft, req: QuizRequestFacts, opts: { modelId?: string | null; signal?: AbortSignal; onStep?: (label: string) => void } = {}, -): Promise<{ draft: QuizDraft; notes: string[]; review: QuizReview | null; pythonRuns: number }> { +): Promise { const resolved = resolveModel(app, 'generate', opts.modelId); if (!resolved.model.capabilities.tools) { - return { draft, notes: ['The Python answer check was skipped: the generation model cannot use tools.'], review: null, pythonRuns: 0 }; + return { draft, notes: ['The Python answer check was skipped: the generation model cannot use tools.'], review: null, pythonRuns: 0, failed: [], applied: [], refused: [], removed: 0, rewritten: 0, incomplete: true }; } const data = quizForPython(draft); const prelude = pythonPrelude({ draft: data, request: { count: req.count, style: req.style, sata_share: req.sataShare, difficulty: req.difficulty ?? 'mixed' } }); const listing = data.questions - .map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${q.rationale.slice(0, 600)}` : ''}`) + .map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${clipRationale(q.rationale, 600)}` : ''}`) .join('\n\n'); const messages: ChatMessage[] = [{ role: 'user', content: [{ type: 'text', text: `${rulesFor(req)}\n\nTHE DRAFT (${data.question_count} questions):\n\n${listing}` }] }]; let review: QuizReview | null = null; @@ -309,21 +360,157 @@ export async function verifyQuizDraft( } catch (err) { if (err instanceof AIError && err.code === 'aborted') throw err; const message = err instanceof Error ? err.message : String(err); - return { draft, notes: [`The Python answer check could not finish (${message.slice(0, 200)}). Review the answers yourself.`], review: null, pythonRuns }; + return { draft, notes: [`The Python answer check could not finish (${message.slice(0, 200)}). Review the answers yourself.`], review: null, pythonRuns, failed: [], applied: [], refused: [], removed: 0, rewritten: 0, incomplete: true }; } - if (!review) return { draft, notes: ['The Python answer check did not report back. Review the answers yourself.'], review: null, pythonRuns }; + if (!review) return { draft, notes: ['The Python answer check did not report back. Review the answers yourself.'], review: null, pythonRuns, failed: [], applied: [], refused: [], removed: 0, rewritten: 0, incomplete: true }; const { questions, applied, refused } = applyQuizFixes(draft.questions, review.fixes); // Remarks the model files as informational or cosmetic are not checks. const real = review.checks.filter((c) => !/informational|cosmetic|position bias/i.test(`${c.name} ${c.detail}`)); // A problem the model fixed is not a failure: the applied fix is listed on its own. const fixedHere = (c: { detail: string }) => applied.length > 0 && /\b(fixed|corrected|fix applied)\b/i.test(c.detail ?? ''); - const failed = real.filter((c) => !c.passed && !fixedHere(c)); + // Removals are refilled by the generator, so a count short only by them is not a failure. + const removed = draft.questions.length - questions.length; + const failed = real.filter((c) => !c.passed && !fixedHere(c) && !(removed > 0 && /\bcount\b/i.test(c.name) && questions.length + removed >= req.count)); const notes = [ `Checked with Python (${pythonRuns} run${pythonRuns === 1 ? '' : 's'}): ${real.length - failed.length} of ${real.length} checks passed${applied.length ? `, ${applied.length} fix${applied.length === 1 ? '' : 'es'} applied` : ''}.`, ...failed.map((c) => `Check failed: ${c.name}${c.detail ? `: ${c.detail}` : ''}`), ...applied, ...refused, ]; - return { draft: { ...draft, questions }, notes, review, pythonRuns }; + return { + draft: { ...draft, questions }, + notes, + review, + pythonRuns, + failed: failed.map((c) => `${c.name}${c.detail ? `: ${c.detail}` : ''}`), + applied, + refused, + removed, + rewritten: review.fixes.filter((f) => f.action === 'rewrite').length - refused.filter((r) => /rewrite/.test(r)).length, + incomplete: false, + }; +} + +/* ------------------------------ independent second review ------------------------------ */ + +/** + * Wording that suggests the writer changed its mind mid-question or left a draft note in: + * "A 78 kg? No — an 80 kg client", "Actually,", "wait", "let me"... Hints for the reviewer, + * which still decides. + */ +const LEFTOVER_RE = /\?\s*no\b|\b(wait|actually|oops|scratch that|let me|i mean|correction|hmm|on second thought)\b[,.:\s—-]/i; + +export function suspiciousWording(questions: Pick[]): string[] { + const out: string[] = []; + questions.forEach((q, i) => { + const parts = [['stem', q.stem], ...q.options.map((o, k) => [`option ${k + 1}`, o.text] as const), ['rationale', q.rationale ?? '']] as const; + for (const [where, value] of parts) { + const m = LEFTOVER_RE.exec(value ?? ''); + if (m) { + const at = Math.max(0, m.index - 30); + out.push(`Q${i + 1} ${where}: "…${value.slice(at, m.index + 50).replace(/\s+/g, ' ')}…"`); + break; + } + } + }); + return out; +} + +const SUBMIT_PROOFREAD: ToolDef = { + name: 'submit_proofread', + description: 'Report the questions that need repair, each with its complete rewritten version (or removal). An empty list means every question is sound.', + parameters: { + type: 'object', + properties: { + issues: { + type: 'array', + items: { + type: 'object', + properties: { + question: { type: 'integer', description: '1-based question number.' }, + problem: { type: 'string', description: 'What is wrong, in one sentence.' }, + action: { type: 'string', enum: ['rewrite', 'remove'] }, + question_text: SUBMIT_REVIEW_QUESTION_TEXT, + }, + required: ['question', 'problem', 'action'], + }, + }, + }, + required: ['issues'], + }, +}; + +const proofreadSchema = z.object({ + issues: z + .array(z.unknown()) + .optional() + .default([]) + .transform((list) => + list.flatMap((raw) => { + const r = z.object({ question: int.min(1), problem: text(600).optional().default(''), action: z.enum(['rewrite', 'remove']), question_text: rewriteSchema.optional() }).safeParse(raw); + return r.success ? [r.data] : []; + }), + ), +}); + +const PROOFREAD_SYSTEM = `You are a second, independent reviewer of NCLEX-style quiz questions for a nursing student, reading the finished quiz the way a careful student and an experienced item writer would. Another checker has already verified counts, types and arithmetic; you look at how each question reads and whether it is sound. +Flag a question only for a real problem: +- the writer changed its mind or left a note in the text (for example "A 78 kg? No — an 80 kg client", "Actually,", "wait"), or the wording is garbled, truncated or repeated; +- data needed to answer is missing, or the stem contradicts itself, the options or the rationale; +- two options are defensible, or none is correct; the rationale disagrees with the keyed answer; +- an option gives the answer away, or the content is clinically wrong or outdated. +For each problem give the complete corrected question with rewrite (same type, the same clinical point, following the rules for its type: single = 4 options with exactly one correct; multi = 5 or 6 options with 2 to 5 correct and a stem ending "Select all that apply."; numeric = one option, the correct answer with its unit, and "why" showing the working), or remove it if it cannot be saved. Do not flag style preferences or answer order. Call submit_proofread once; an empty list is the right answer when every question is sound.`; + +/** + * An independent read of the whole quiz by a fresh model call, for problems the mechanical + * check misses (a writer changing its mind mid-stem, contradictions, two defensible answers). + * Never throws except on cancel. + */ +export async function proofreadQuiz( + app: AppContext, + draft: QuizDraft, + opts: { modelId?: string | null; signal?: AbortSignal } = {}, +): Promise<{ draft: QuizDraft; flagged: number; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> { + const none = { draft, flagged: 0, applied: [], refused: [], rewritten: 0, removed: 0 }; + const resolved = resolveModel(app, 'generate', opts.modelId); + if (!resolved.model.capabilities.tools) return { ...none, incomplete: true }; + const data = quizForPython(draft); + const listing = data.questions + .map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${q.rationale}` : ''}`) + .join('\n\n'); + const hints = suspiciousWording(draft.questions); + const prompt = `${hints.length ? `Wording that looks like a leftover or a change of mind (confirm and fix each):\n${hints.map((h) => `- ${h}`).join('\n')}\n\n` : ''}THE QUIZ (${data.question_count} questions):\n\n${listing}`; + try { + const res = await runChat(app, { + task: 'generate', + purpose: 'quiz second review', + modelId: opts.modelId, + system: PROOFREAD_SYSTEM, + messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }], + tools: [SUBMIT_PROOFREAD], + toolChoice: { name: 'submit_proofread' }, + maxOutputTokens: 16_000, + signal: opts.signal, + }); + const call = res.message.content.find((p) => p.type === 'tool_call'); + const parsed = call && call.type === 'tool_call' ? proofreadSchema.safeParse(call.input) : null; + if (!parsed?.success) return { ...none, incomplete: true }; + const issues = parsed.data.issues; + const fixes = issues.map((i) => ({ question: i.question, action: i.action, question_text: i.question_text, reason: i.problem }) as z.infer); + const { questions, applied, refused } = applyQuizFixes(draft.questions, fixes); + return { + draft: { ...draft, questions }, + flagged: issues.length, + applied, + refused, + rewritten: applied.filter((a) => /: rewritten/.test(a)).length, + removed: draft.questions.length - questions.length, + incomplete: false, + }; + } catch (err) { + if (err instanceof AIError && err.code === 'aborted') throw err; + app.log('warn', 'The second review of a quiz failed', err instanceof Error ? err.message : err); + return { ...none, incomplete: true }; + } } diff --git a/tests/ai.test.ts b/tests/ai.test.ts index 6849034..275ee7d 100644 --- a/tests/ai.test.ts +++ b/tests/ai.test.ts @@ -54,7 +54,7 @@ async function sse(url: string, body: unknown): Promise { const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review'); /** The quick call that groups topic labels after a quiz is written. */ const isGrouping = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_topics'); -const isNaming = (r: { body: any }) => r.body?.tool_choice?.function?.name === 'name_set'; +const isNaming = (r: { body: any }) => ['name_set', 'submit_proofread'].includes(r.body?.tool_choice?.function?.name); async function waitForDraft(id: string) { for (let i = 0; i < 200; i += 1) { diff --git a/tests/mockAi.ts b/tests/mockAi.ts index ee620e3..12e55df 100644 --- a/tests/mockAi.ts +++ b/tests/mockAi.ts @@ -20,6 +20,8 @@ export interface MockAi { delayChat: (ms: number) => void; /** The next N calls of this tool send arguments that are not valid JSON (beyond jsonrepair). */ badJsonNext: (tool: string, times?: number) => void; + /** Arguments for the next submit_review calls, in order (then the default review). */ + scriptReviews: (reviews: Record[]) => void; close: () => Promise; } @@ -73,7 +75,9 @@ function toolArgs(name: string, userText: string, all: string): Record m[1]))]; // Continue numbering after anything already written, so later batches are not duplicates. - const offset = (userText.split('Already written')[1]?.match(/^- /gm) ?? []).length; + const written = userText.split('Already written')[1] ?? ''; + // Continue after the highest number written, so a refill after a removal is never a repeat. + const offset = Math.max((written.match(/^- /gm) ?? []).length, ...[...written.matchAll(/Mock (?:question|math|term) (\d+)/g)].map((m) => Number(m[1]))); void all; switch (name) { case 'submit_questions': @@ -108,6 +112,21 @@ function toolArgs(name: string, userText: string, all: string): Record m[1].trim()); @@ -151,6 +170,7 @@ export function startMockAi(port = 0): Promise { let fail = { status: 0, times: 0 }; let delayMs = 0; let badJson: { tool: string; times: number } | null = null; + let reviews: Record[] = []; const server = http.createServer((req, res) => { const chunks: Buffer[] = []; @@ -203,7 +223,7 @@ export function startMockAi(port = 0): Promise { res.end(); }; const callTool = (name: string) => { - let args = JSON.stringify(toolArgs(name, userText, all)); + let args = JSON.stringify(name === 'submit_review' && reviews.length ? reviews.shift() : toolArgs(name, userText, all)); if (badJson && badJson.tool === name && badJson.times > 0) { badJson.times -= 1; args = '{"cards": [{"front": "Broken" "back": ]]]} :: not json'; @@ -277,6 +297,9 @@ export function startMockAi(port = 0): Promise { badJsonNext: (tool, times = 1) => { badJson = { tool, times }; }, + scriptReviews: (list) => { + reviews = [...list]; + }, close: () => new Promise((done) => server.close(() => done())), }); }); diff --git a/tests/revision5.test.ts b/tests/revision5.test.ts index 876800d..562815c 100644 --- a/tests/revision5.test.ts +++ b/tests/revision5.test.ts @@ -7,6 +7,7 @@ */ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts'; +import { applyQuizFixes, suspiciousWording } from '../server/ai/verify.ts'; import { MIGRATIONS } from '../server/migrations.ts'; import { queueIdle } from '../server/services/documents.ts'; import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts'; @@ -27,6 +28,7 @@ beforeEach(async () => { mock.requests.length = 0; mock.delayChat(0); mock.badJsonNext('', 0); + mock.scriptReviews([]); const { resetShareAiLimits } = await import('../server/routes/shares.ts'); resetShareAiLimits(); resetGeoForTests(); @@ -87,6 +89,109 @@ describe('malformed model output', () => { }); }); +describe('the quiz check', () => { + const isReview = (r: { body: any }) => (r.body?.tools ?? []).some((x: any) => x.function?.name === 'submit_review'); + const pass = { summary: 'All good.', checks: [{ name: 'Question count', passed: true }, { name: 'Answer keys', passed: true }], fixes: [] }; + + it('rewrites broken questions, replaces removed ones and checks again until the quiz passes', async () => { + await connectAi(); + const { cls } = await classWithMaterial(); + mock.scriptReviews([ + { + summary: 'Two problems.', + checks: [ + { name: 'Question count vs request', passed: false, detail: 'Q3 duplicates Q2; removing it leaves 5.' }, + { name: 'Stem integrity', passed: true, detail: 'Q1 rewritten.' }, + ], + fixes: [ + { question: 3, action: 'remove', reason: 'duplicate of Q2' }, + { + question: 1, + action: 'rewrite', + reason: 'garbled stem', + question_text: { type: 'single', stem: 'An 80 kg client in septic shock needs norepinephrine. Which finding is the priority?', options: [{ text: 'MAP 58', correct: true }, { text: 'Temp 38.1', correct: false }, { text: 'HR 104', correct: false }, { text: 'Urine 40 mL/h', correct: false }], rationale: 'Perfusion first.' }, + }, + ], + }, + pass, + ]); + const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 6 })).body.draftId); + expect(draft.status).toBe('ready'); + expect(draft.payload.questions).toHaveLength(6); + expect(draft.payload.questions.some((q: any) => q.stem.startsWith('An 80 kg client'))).toBe(true); + // Two full checks: the first, then the re-check after the fixes and the replacement. + expect(chatCalls().filter((r) => isReview(r) && r.body.messages.filter((m: any) => m.role !== 'system').length === 1)).toHaveLength(2); + const notes = draft.meta.notes.join(' | '); + expect(notes).toMatch(/Checked with Python in 2 rounds/); + expect(notes).toMatch(/1 removed question was replaced/); + expect(notes).not.toMatch(/Check failed/); + }); + + it('checks rewritten questions again', async () => { + await connectAi(); + const { cls } = await classWithMaterial(); + mock.scriptReviews([ + { + summary: 'One rewrite.', + checks: [{ name: 'Stem integrity', passed: true, detail: 'Q2 fixed.' }], + fixes: [ + { question: 2, action: 'rewrite', reason: 'garbled', question_text: { type: 'single', stem: 'Which lab value is monitored for heparin?', options: [{ text: 'aPTT', correct: true }, { text: 'INR', correct: false }, { text: 'BNP', correct: false }, { text: 'TSH', correct: false }], rationale: 'aPTT guides heparin.' } }, + ], + }, + pass, + ]); + const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); + const notes = draft.meta.notes.join(' | '); + expect(notes).toMatch(/in 2 rounds/); + expect(notes).toMatch(/Question 2: rewritten/); + }); + + it('ignores a key fix that changes nothing', () => { + const q = { type: 'single' as const, stem: 'S', options: [{ text: 'a', correct: true }, { text: 'b', correct: false }, { text: 'c', correct: false }, { text: 'd', correct: false }], rationale: '', topic: 'T', difficulty: 'medium' as const, tags: [] }; + const out = applyQuizFixes([q], [{ question: 1, action: 'set_correct', correct_options: [1], reason: 'Informational only' }]); + expect(out.applied).toEqual([]); + expect(out.refused).toEqual([]); + }); + + it('has a second reviewer rewrite a question whose writer changed its mind', async () => { + await connectAi(); + const { cls } = await classWithMaterial(); + // The first check leaves a self-correction in Q1 (as a model once did); the second review catches it. + mock.scriptReviews([ + { + summary: 'Fine.', + checks: [{ name: 'Answer keys', passed: true }], + fixes: [{ question: 1, action: 'rewrite', reason: 'test setup', question_text: { type: 'numeric', stem: 'A 78 kg? No — an 80 kg client receives 4 mg in 250 mL at 15 mL/h. Dose in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000/250*15/60/80' }], rationale: 'Convert, then divide.' } }], + }, + pass, + ]); + const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); + const proofreads = chatCalls().filter((r) => r.body.tool_choice?.function?.name === 'submit_proofread'); + expect(proofreads.length).toBeGreaterThanOrEqual(2); + // The leftover was pointed out to the reviewer, and the question no longer has it. + expect(JSON.stringify(proofreads[0].body.messages)).toMatch(/looks like a leftover/); + expect(draft.payload.questions.some((q: any) => /\?\s*No\b/.test(q.stem))).toBe(false); + expect(draft.meta.notes.join(' | ')).toMatch(/Second review: Question \d+: rewritten/); + }); + + it('spots leftover wording', () => { + const q = (stem: string) => ({ stem, options: [{ text: 'a', correct: true }], rationale: '' }); + expect(suspiciousWording([q('A 78 kg? No — an 80 kg client'), q('A client with heart failure asks what to eat.'), q('Actually, the dose is 5 mg.')])).toHaveLength(2); + }); + + it('stops after three rounds when problems remain', async () => { + await connectAi(); + const { cls } = await classWithMaterial(); + const failing = { summary: 'Still wrong.', checks: [{ name: 'Answer keys', passed: false, detail: 'Q1 is disputed.' }], fixes: [] }; + mock.scriptReviews([failing, failing, failing, failing]); + const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); + expect(draft.status).toBe('ready'); + const notes = draft.meta.notes.join(' | '); + expect(notes).toMatch(/in 3 rounds/); + expect(notes).toMatch(/Check failed: Answer keys/); + }); +}); + describe('generated sets', () => { it('get their own name and description, and flashcards are atomic without hints', async () => { await connectAi(); diff --git a/web/src/pages/DraftPage.tsx b/web/src/pages/DraftPage.tsx index 04edc7d..1e42411 100644 --- a/web/src/pages/DraftPage.tsx +++ b/web/src/pages/DraftPage.tsx @@ -522,6 +522,10 @@ export default function DraftPage() { function noteTone(note: string): 'good' | 'neutral' | 'warn' { const checked = /^Checked with Python[^:]*: (\d+) of (\d+) checks passed/.exec(note); if (checked) return checked[1] === checked[2] ? 'good' : 'warn'; - if (/^Question \d+( removed|: (answer|type|topic))/.test(note) && !/not applied|incomplete|does not exist/.test(note)) return 'neutral'; + // Fixes the check applied (in any round), and removed questions it replaced, are routine. + if (note === 'Second review: no problems found.') return 'good'; + const fix = note.replace(/^Round \d+: /, '').replace(/^Second review: /, ''); + if (/^Question \d+( removed|: (answer|type|topic|rewritten))/.test(fix) && !/not applied|incomplete|does not exist/.test(fix)) return 'neutral'; + if (/^\d+ removed questions? (was|were) replaced/.test(fix)) return 'neutral'; return 'warn'; }