Quiz check: fix before handing over, with an independent second review
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s

- The check can rewrite a whole question (garbled or contradictory stems,
  missing data, two defensible answers); a key fix that changes nothing is
  skipped.
- Removed questions are replaced with new ones, so a 50-question quiz stays 50.
- A second, independent model call reads the whole quiz for badly written
  items, helped by a scan for leftovers such as "A 78 kg? No - an 80 kg".
- The quiz is checked again after any rewrite, replacement or open problem,
  up to three rounds, before it is handed over.
- Reviewers see full rationales (a shortened one is marked as shortened).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 10:07:32 -07:00
parent 85d5c9186d
commit 1a212bb8ba
8 changed files with 412 additions and 26 deletions

View file

@ -7,6 +7,7 @@
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts';
import { applyQuizFixes, suspiciousWording } from '../server/ai/verify.ts';
import { MIGRATIONS } from '../server/migrations.ts';
import { queueIdle } from '../server/services/documents.ts';
import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts';
@ -27,6 +28,7 @@ beforeEach(async () => {
mock.requests.length = 0;
mock.delayChat(0);
mock.badJsonNext('', 0);
mock.scriptReviews([]);
const { resetShareAiLimits } = await import('../server/routes/shares.ts');
resetShareAiLimits();
resetGeoForTests();
@ -87,6 +89,109 @@ describe('malformed model output', () => {
});
});
describe('the quiz check', () => {
const isReview = (r: { body: any }) => (r.body?.tools ?? []).some((x: any) => x.function?.name === 'submit_review');
const pass = { summary: 'All good.', checks: [{ name: 'Question count', passed: true }, { name: 'Answer keys', passed: true }], fixes: [] };
it('rewrites broken questions, replaces removed ones and checks again until the quiz passes', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.scriptReviews([
{
summary: 'Two problems.',
checks: [
{ name: 'Question count vs request', passed: false, detail: 'Q3 duplicates Q2; removing it leaves 5.' },
{ name: 'Stem integrity', passed: true, detail: 'Q1 rewritten.' },
],
fixes: [
{ question: 3, action: 'remove', reason: 'duplicate of Q2' },
{
question: 1,
action: 'rewrite',
reason: 'garbled stem',
question_text: { type: 'single', stem: 'An 80 kg client in septic shock needs norepinephrine. Which finding is the priority?', options: [{ text: 'MAP 58', correct: true }, { text: 'Temp 38.1', correct: false }, { text: 'HR 104', correct: false }, { text: 'Urine 40 mL/h', correct: false }], rationale: 'Perfusion first.' },
},
],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 6 })).body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.questions).toHaveLength(6);
expect(draft.payload.questions.some((q: any) => q.stem.startsWith('An 80 kg client'))).toBe(true);
// Two full checks: the first, then the re-check after the fixes and the replacement.
expect(chatCalls().filter((r) => isReview(r) && r.body.messages.filter((m: any) => m.role !== 'system').length === 1)).toHaveLength(2);
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/Checked with Python in 2 rounds/);
expect(notes).toMatch(/1 removed question was replaced/);
expect(notes).not.toMatch(/Check failed/);
});
it('checks rewritten questions again', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.scriptReviews([
{
summary: 'One rewrite.',
checks: [{ name: 'Stem integrity', passed: true, detail: 'Q2 fixed.' }],
fixes: [
{ question: 2, action: 'rewrite', reason: 'garbled', question_text: { type: 'single', stem: 'Which lab value is monitored for heparin?', options: [{ text: 'aPTT', correct: true }, { text: 'INR', correct: false }, { text: 'BNP', correct: false }, { text: 'TSH', correct: false }], rationale: 'aPTT guides heparin.' } },
],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/in 2 rounds/);
expect(notes).toMatch(/Question 2: rewritten/);
});
it('ignores a key fix that changes nothing', () => {
const q = { type: 'single' as const, stem: 'S', options: [{ text: 'a', correct: true }, { text: 'b', correct: false }, { text: 'c', correct: false }, { text: 'd', correct: false }], rationale: '', topic: 'T', difficulty: 'medium' as const, tags: [] };
const out = applyQuizFixes([q], [{ question: 1, action: 'set_correct', correct_options: [1], reason: 'Informational only' }]);
expect(out.applied).toEqual([]);
expect(out.refused).toEqual([]);
});
it('has a second reviewer rewrite a question whose writer changed its mind', async () => {
await connectAi();
const { cls } = await classWithMaterial();
// The first check leaves a self-correction in Q1 (as a model once did); the second review catches it.
mock.scriptReviews([
{
summary: 'Fine.',
checks: [{ name: 'Answer keys', passed: true }],
fixes: [{ question: 1, action: 'rewrite', reason: 'test setup', question_text: { type: 'numeric', stem: 'A 78 kg? No — an 80 kg client receives 4 mg in 250 mL at 15 mL/h. Dose in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000/250*15/60/80' }], rationale: 'Convert, then divide.' } }],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
const proofreads = chatCalls().filter((r) => r.body.tool_choice?.function?.name === 'submit_proofread');
expect(proofreads.length).toBeGreaterThanOrEqual(2);
// The leftover was pointed out to the reviewer, and the question no longer has it.
expect(JSON.stringify(proofreads[0].body.messages)).toMatch(/looks like a leftover/);
expect(draft.payload.questions.some((q: any) => /\?\s*No\b/.test(q.stem))).toBe(false);
expect(draft.meta.notes.join(' | ')).toMatch(/Second review: Question \d+: rewritten/);
});
it('spots leftover wording', () => {
const q = (stem: string) => ({ stem, options: [{ text: 'a', correct: true }], rationale: '' });
expect(suspiciousWording([q('A 78 kg? No — an 80 kg client'), q('A client with heart failure asks what to eat.'), q('Actually, the dose is 5 mg.')])).toHaveLength(2);
});
it('stops after three rounds when problems remain', async () => {
await connectAi();
const { cls } = await classWithMaterial();
const failing = { summary: 'Still wrong.', checks: [{ name: 'Answer keys', passed: false, detail: 'Q1 is disputed.' }], fixes: [] };
mock.scriptReviews([failing, failing, failing, failing]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
expect(draft.status).toBe('ready');
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/in 3 rounds/);
expect(notes).toMatch(/Check failed: Answer keys/);
});
});
describe('generated sets', () => {
it('get their own name and description, and flashcards are atomic without hints', async () => {
await connectAi();