Quiz check: fix before handing over, with an independent second review
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s

- The check can rewrite a whole question (garbled or contradictory stems,
  missing data, two defensible answers); a key fix that changes nothing is
  skipped.
- Removed questions are replaced with new ones, so a 50-question quiz stays 50.
- A second, independent model call reads the whole quiz for badly written
  items, helped by a scan for leftovers such as "A 78 kg? No - an 80 kg".
- The quiz is checked again after any rewrite, replacement or open problem,
  up to three rounds, before it is handed over.
- Reviewers see full rationales (a shortened one is marked as shortened).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 10:07:32 -07:00
parent 85d5c9186d
commit 1a212bb8ba
8 changed files with 412 additions and 26 deletions

View file

@ -42,7 +42,10 @@ import { weakTopics } from '../services/stats.ts';
import { getSetting, setSetting } from '../settings.ts';
import { newId, now, parseJson } from '../util.ts';
import { generateStructured, resolveModel, runChat, runChatWithMcp } from './service.ts';
import { verifyQuizDraft } from './verify.ts';
import { proofreadQuiz, verifyQuizDraft, type QuizCheck } from './verify.ts';
/** Most quiz checks per generation: the first, then re-checks after fixes and replacements. */
const VERIFY_ROUNDS = 3;
import { AIError } from './types.ts';
/* ------------------------------ drafts ------------------------------ */
@ -1029,12 +1032,57 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
);
}
// Check counts, the type mix, answer keys and calculations with Python before review.
// Check counts, the type mix, answer keys and calculations with Python, and keep fixing until
// the quiz passes: fixes are applied, removed questions are replaced with new ones, and the
// whole quiz is checked again (at most VERIFY_ROUNDS checks).
if (draft.kind === 'quiz' && req.verify !== false) {
setProgress(db, draftId, 0.96, 'Checking the questions with Python', draft);
const checked = await verifyQuizDraft(app, draft, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, label) });
draft.questions = checked.draft.questions;
notes.push(...checked.notes);
const quiz = draft;
const history: string[] = [];
let last: QuizCheck | null = null;
let rounds = 0;
let secondReview: Awaited<ReturnType<typeof proofreadQuiz>> | null = null;
for (let round = 1; round <= VERIFY_ROUNDS; round += 1) {
rounds = round;
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
setProgress(db, draftId, 0.96, round === 1 ? 'Checking the questions with Python' : `Checking the questions again (round ${round})`, quiz);
last = await verifyQuizDraft(app, quiz, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, round === 1 ? label : `${label} · round ${round}`) });
quiz.questions = last.draft.questions;
if (last.incomplete) break;
history.push(...last.applied.map((a) => (round > 1 ? `Round ${round}: ${a}` : a)));
// A second, independent read of the whole quiz for badly written questions.
setProgress(db, draftId, 0.98, round === 1 ? 'Second review of the questions' : `Second review · round ${round}`, quiz);
const second = await proofreadQuiz(app, quiz, { modelId: req.modelId, signal });
quiz.questions = second.draft.questions;
secondReview = second;
history.push(...second.applied.map((a) => `${round > 1 ? `Round ${round}: ` : ''}Second review: ${a}`));
if (second.refused.length) history.push(...second.refused.map((r) => `Second review: ${r}`));
// Replace what the check removed (duplicates, unfixable items) with new questions.
const missing = count - quiz.questions.length;
if (missing > 0) {
setProgress(db, draftId, 0.96, `Replacing ${missing} question${missing === 1 ? '' : 's'}`, quiz);
for (let attempt = 0; attempt < 3 && quiz.questions.length < count; attempt += 1) {
const wantMath = forcedMath ? Math.min(forcedMath - numericCount(), count - quiz.questions.length) : 0;
try {
if (wantMath > 0) await runBatch(batches + 30 + round * 3 + attempt, wantMath, true);
else await runBatch(batches + 30 + round * 3 + attempt, Math.min(perBatch, count - quiz.questions.length));
} catch (err) {
if (err instanceof AIError && err.code === 'aborted') throw err;
}
}
if (forcedMath) quiz.questions = spreadEvenly(quiz.questions, (q) => q.type === 'numeric');
history.push(`Round ${round}: ${missing} removed question${missing === 1 ? ' was' : 's were'} replaced with new ones${round === VERIFY_ROUNDS ? ' (after the last check)' : ''}.`);
}
// Done when nothing is left to fix and nothing new (rewritten or replacement questions) needs checking.
const done = missing <= 0 && last.rewritten === 0 && second.applied.length === 0 && last.failed.length === 0 && last.refused.length === 0;
if (done || round === VERIFY_ROUNDS) break;
}
if (last) {
const summary = last.notes[0] ?? '';
notes.push(
...(last.incomplete ? last.notes : [
rounds > 1 ? summary.replace(/^Checked with Python/, `Checked with Python in ${rounds} rounds, last round`) : summary, ...last.failed.map((f) => `Check failed: ${f}`), ...(secondReview ? [secondReview.incomplete ? 'The second review could not finish.' : secondReview.flagged === 0 ? 'Second review: no problems found.' : `Second review: ${secondReview.flagged} question${secondReview.flagged === 1 ? '' : 's'} fixed in the last round.`] : []), ...history, ...last.refused]),
);
}
}
// A short name and description of what this particular set covers.