Quiz check: fix before handing over, with an independent second review
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s

- The check can rewrite a whole question (garbled or contradictory stems,
  missing data, two defensible answers); a key fix that changes nothing is
  skipped.
- Removed questions are replaced with new ones, so a 50-question quiz stays 50.
- A second, independent model call reads the whole quiz for badly written
  items, helped by a scan for leftovers such as "A 78 kg? No - an 80 kg".
- The quiz is checked again after any rewrite, replacement or open problem,
  up to three rounds, before it is handed over.
- Reviewers see full rationales (a shortened one is marked as shortened).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 10:07:32 -07:00
parent 85d5c9186d
commit 1a212bb8ba
8 changed files with 412 additions and 26 deletions

View file

@ -54,7 +54,7 @@ async function sse(url: string, body: unknown): Promise<any[]> {
const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review');
/** The quick call that groups topic labels after a quiz is written. */
const isGrouping = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_topics');
const isNaming = (r: { body: any }) => r.body?.tool_choice?.function?.name === 'name_set';
const isNaming = (r: { body: any }) => ['name_set', 'submit_proofread'].includes(r.body?.tool_choice?.function?.name);
async function waitForDraft(id: string) {
for (let i = 0; i < 200; i += 1) {

View file

@ -20,6 +20,8 @@ export interface MockAi {
delayChat: (ms: number) => void;
/** The next N calls of this tool send arguments that are not valid JSON (beyond jsonrepair). */
badJsonNext: (tool: string, times?: number) => void;
/** Arguments for the next submit_review calls, in order (then the default review). */
scriptReviews: (reviews: Record<string, unknown>[]) => void;
close: () => Promise<void>;
}
@ -73,7 +75,9 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
// Only refs from the user's turn count: the system prompt contains an example id.
const refs = [...new Set([...userText.matchAll(/\[(c\d+)\]/g)].map((m) => m[1]))];
// Continue numbering after anything already written, so later batches are not duplicates.
const offset = (userText.split('Already written')[1]?.match(/^- /gm) ?? []).length;
const written = userText.split('Already written')[1] ?? '';
// Continue after the highest number written, so a refill after a removal is never a repeat.
const offset = Math.max((written.match(/^- /gm) ?? []).length, ...[...written.matchAll(/Mock (?:question|math|term) (\d+)/g)].map((m) => Number(m[1])));
void all;
switch (name) {
case 'submit_questions':
@ -108,6 +112,21 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
const fix = /Q2 \[single\]/.test(all) ? [{ question: 2, action: 'set_correct', correct_options: [2], reason: 'mock: option 2 is the right answer' }] : [];
return { summary: 'Checked by the mock.', checks: [{ name: 'Question count', passed: true, detail: 'matches the request' }, { name: 'Correct answers per type', passed: true }], fixes: fix };
}
case 'submit_proofread': {
// Rewrites a question whose stem has a writer's change of mind in it ("? No —").
const m = /^Q(\d+) \[\w+\] .*\?\s*No\b/m.exec(userText);
if (!m) return { issues: [] };
return {
issues: [
{
question: Number(m[1]),
problem: 'The stem gives two weights.',
action: 'rewrite',
question_text: { type: 'numeric', stem: 'An 80 kg client receives norepinephrine 4 mg in 250 mL at 15 mL/h. What dose is the client receiving in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000 mcg / 250 mL x 15 mL/h / 60 / 80 kg = 0.05' }], rationale: 'Convert mg to mcg, find mcg/h, then per minute and per kg.' },
},
],
};
}
case 'submit_topics': {
// Labels of three or more words are broadened to their first word, like a real model would.
const labels = [...(userText.split('Labels:')[1] ?? '').matchAll(/^- (.+)$/gm)].map((m) => m[1].trim());
@ -151,6 +170,7 @@ export function startMockAi(port = 0): Promise<MockAi> {
let fail = { status: 0, times: 0 };
let delayMs = 0;
let badJson: { tool: string; times: number } | null = null;
let reviews: Record<string, unknown>[] = [];
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
@ -203,7 +223,7 @@ export function startMockAi(port = 0): Promise<MockAi> {
res.end();
};
const callTool = (name: string) => {
let args = JSON.stringify(toolArgs(name, userText, all));
let args = JSON.stringify(name === 'submit_review' && reviews.length ? reviews.shift() : toolArgs(name, userText, all));
if (badJson && badJson.tool === name && badJson.times > 0) {
badJson.times -= 1;
args = '{"cards": [{"front": "Broken" "back": ]]]} :: not json';
@ -277,6 +297,9 @@ export function startMockAi(port = 0): Promise<MockAi> {
badJsonNext: (tool, times = 1) => {
badJson = { tool, times };
},
scriptReviews: (list) => {
reviews = [...list];
},
close: () => new Promise((done) => server.close(() => done())),
});
});

View file

@ -7,6 +7,7 @@
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts';
import { applyQuizFixes, suspiciousWording } from '../server/ai/verify.ts';
import { MIGRATIONS } from '../server/migrations.ts';
import { queueIdle } from '../server/services/documents.ts';
import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts';
@ -27,6 +28,7 @@ beforeEach(async () => {
mock.requests.length = 0;
mock.delayChat(0);
mock.badJsonNext('', 0);
mock.scriptReviews([]);
const { resetShareAiLimits } = await import('../server/routes/shares.ts');
resetShareAiLimits();
resetGeoForTests();
@ -87,6 +89,109 @@ describe('malformed model output', () => {
});
});
describe('the quiz check', () => {
const isReview = (r: { body: any }) => (r.body?.tools ?? []).some((x: any) => x.function?.name === 'submit_review');
const pass = { summary: 'All good.', checks: [{ name: 'Question count', passed: true }, { name: 'Answer keys', passed: true }], fixes: [] };
it('rewrites broken questions, replaces removed ones and checks again until the quiz passes', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.scriptReviews([
{
summary: 'Two problems.',
checks: [
{ name: 'Question count vs request', passed: false, detail: 'Q3 duplicates Q2; removing it leaves 5.' },
{ name: 'Stem integrity', passed: true, detail: 'Q1 rewritten.' },
],
fixes: [
{ question: 3, action: 'remove', reason: 'duplicate of Q2' },
{
question: 1,
action: 'rewrite',
reason: 'garbled stem',
question_text: { type: 'single', stem: 'An 80 kg client in septic shock needs norepinephrine. Which finding is the priority?', options: [{ text: 'MAP 58', correct: true }, { text: 'Temp 38.1', correct: false }, { text: 'HR 104', correct: false }, { text: 'Urine 40 mL/h', correct: false }], rationale: 'Perfusion first.' },
},
],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 6 })).body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.questions).toHaveLength(6);
expect(draft.payload.questions.some((q: any) => q.stem.startsWith('An 80 kg client'))).toBe(true);
// Two full checks: the first, then the re-check after the fixes and the replacement.
expect(chatCalls().filter((r) => isReview(r) && r.body.messages.filter((m: any) => m.role !== 'system').length === 1)).toHaveLength(2);
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/Checked with Python in 2 rounds/);
expect(notes).toMatch(/1 removed question was replaced/);
expect(notes).not.toMatch(/Check failed/);
});
it('checks rewritten questions again', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.scriptReviews([
{
summary: 'One rewrite.',
checks: [{ name: 'Stem integrity', passed: true, detail: 'Q2 fixed.' }],
fixes: [
{ question: 2, action: 'rewrite', reason: 'garbled', question_text: { type: 'single', stem: 'Which lab value is monitored for heparin?', options: [{ text: 'aPTT', correct: true }, { text: 'INR', correct: false }, { text: 'BNP', correct: false }, { text: 'TSH', correct: false }], rationale: 'aPTT guides heparin.' } },
],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/in 2 rounds/);
expect(notes).toMatch(/Question 2: rewritten/);
});
it('ignores a key fix that changes nothing', () => {
const q = { type: 'single' as const, stem: 'S', options: [{ text: 'a', correct: true }, { text: 'b', correct: false }, { text: 'c', correct: false }, { text: 'd', correct: false }], rationale: '', topic: 'T', difficulty: 'medium' as const, tags: [] };
const out = applyQuizFixes([q], [{ question: 1, action: 'set_correct', correct_options: [1], reason: 'Informational only' }]);
expect(out.applied).toEqual([]);
expect(out.refused).toEqual([]);
});
it('has a second reviewer rewrite a question whose writer changed its mind', async () => {
await connectAi();
const { cls } = await classWithMaterial();
// The first check leaves a self-correction in Q1 (as a model once did); the second review catches it.
mock.scriptReviews([
{
summary: 'Fine.',
checks: [{ name: 'Answer keys', passed: true }],
fixes: [{ question: 1, action: 'rewrite', reason: 'test setup', question_text: { type: 'numeric', stem: 'A 78 kg? No — an 80 kg client receives 4 mg in 250 mL at 15 mL/h. Dose in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000/250*15/60/80' }], rationale: 'Convert, then divide.' } }],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
const proofreads = chatCalls().filter((r) => r.body.tool_choice?.function?.name === 'submit_proofread');
expect(proofreads.length).toBeGreaterThanOrEqual(2);
// The leftover was pointed out to the reviewer, and the question no longer has it.
expect(JSON.stringify(proofreads[0].body.messages)).toMatch(/looks like a leftover/);
expect(draft.payload.questions.some((q: any) => /\?\s*No\b/.test(q.stem))).toBe(false);
expect(draft.meta.notes.join(' | ')).toMatch(/Second review: Question \d+: rewritten/);
});
it('spots leftover wording', () => {
const q = (stem: string) => ({ stem, options: [{ text: 'a', correct: true }], rationale: '' });
expect(suspiciousWording([q('A 78 kg? No — an 80 kg client'), q('A client with heart failure asks what to eat.'), q('Actually, the dose is 5 mg.')])).toHaveLength(2);
});
it('stops after three rounds when problems remain', async () => {
await connectAi();
const { cls } = await classWithMaterial();
const failing = { summary: 'Still wrong.', checks: [{ name: 'Answer keys', passed: false, detail: 'Q1 is disputed.' }], fixes: [] };
mock.scriptReviews([failing, failing, failing, failing]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
expect(draft.status).toBe('ready');
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/in 3 rounds/);
expect(notes).toMatch(/Check failed: Answer keys/);
});
});
describe('generated sets', () => {
it('get their own name and description, and flashcards are atomic without hints', async () => {
await connectAi();