Quiz check: fix before handing over, with an independent second review
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s
- The check can rewrite a whole question (garbled or contradictory stems, missing data, two defensible answers); a key fix that changes nothing is skipped. - Removed questions are replaced with new ones, so a 50-question quiz stays 50. - A second, independent model call reads the whole quiz for badly written items, helped by a scan for leftovers such as "A 78 kg? No - an 80 kg". - The quiz is checked again after any rewrite, replacement or open problem, up to three rounds, before it is handed over. - Reviewers see full rationales (a shortened one is marked as shortened). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
85d5c9186d
commit
1a212bb8ba
8 changed files with 412 additions and 26 deletions
|
|
@ -42,7 +42,10 @@ import { weakTopics } from '../services/stats.ts';
|
|||
import { getSetting, setSetting } from '../settings.ts';
|
||||
import { newId, now, parseJson } from '../util.ts';
|
||||
import { generateStructured, resolveModel, runChat, runChatWithMcp } from './service.ts';
|
||||
import { verifyQuizDraft } from './verify.ts';
|
||||
import { proofreadQuiz, verifyQuizDraft, type QuizCheck } from './verify.ts';
|
||||
|
||||
/** Most quiz checks per generation: the first, then re-checks after fixes and replacements. */
|
||||
const VERIFY_ROUNDS = 3;
|
||||
import { AIError } from './types.ts';
|
||||
|
||||
/* ------------------------------ drafts ------------------------------ */
|
||||
|
|
@ -1029,12 +1032,57 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
|
|||
);
|
||||
}
|
||||
|
||||
// Check counts, the type mix, answer keys and calculations with Python before review.
|
||||
// Check counts, the type mix, answer keys and calculations with Python, and keep fixing until
|
||||
// the quiz passes: fixes are applied, removed questions are replaced with new ones, and the
|
||||
// whole quiz is checked again (at most VERIFY_ROUNDS checks).
|
||||
if (draft.kind === 'quiz' && req.verify !== false) {
|
||||
setProgress(db, draftId, 0.96, 'Checking the questions with Python', draft);
|
||||
const checked = await verifyQuizDraft(app, draft, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, label) });
|
||||
draft.questions = checked.draft.questions;
|
||||
notes.push(...checked.notes);
|
||||
const quiz = draft;
|
||||
const history: string[] = [];
|
||||
let last: QuizCheck | null = null;
|
||||
let rounds = 0;
|
||||
let secondReview: Awaited<ReturnType<typeof proofreadQuiz>> | null = null;
|
||||
for (let round = 1; round <= VERIFY_ROUNDS; round += 1) {
|
||||
rounds = round;
|
||||
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
|
||||
setProgress(db, draftId, 0.96, round === 1 ? 'Checking the questions with Python' : `Checking the questions again (round ${round})`, quiz);
|
||||
last = await verifyQuizDraft(app, quiz, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, round === 1 ? label : `${label} · round ${round}`) });
|
||||
quiz.questions = last.draft.questions;
|
||||
if (last.incomplete) break;
|
||||
history.push(...last.applied.map((a) => (round > 1 ? `Round ${round}: ${a}` : a)));
|
||||
// A second, independent read of the whole quiz for badly written questions.
|
||||
setProgress(db, draftId, 0.98, round === 1 ? 'Second review of the questions' : `Second review · round ${round}`, quiz);
|
||||
const second = await proofreadQuiz(app, quiz, { modelId: req.modelId, signal });
|
||||
quiz.questions = second.draft.questions;
|
||||
secondReview = second;
|
||||
history.push(...second.applied.map((a) => `${round > 1 ? `Round ${round}: ` : ''}Second review: ${a}`));
|
||||
if (second.refused.length) history.push(...second.refused.map((r) => `Second review: ${r}`));
|
||||
// Replace what the check removed (duplicates, unfixable items) with new questions.
|
||||
const missing = count - quiz.questions.length;
|
||||
if (missing > 0) {
|
||||
setProgress(db, draftId, 0.96, `Replacing ${missing} question${missing === 1 ? '' : 's'}`, quiz);
|
||||
for (let attempt = 0; attempt < 3 && quiz.questions.length < count; attempt += 1) {
|
||||
const wantMath = forcedMath ? Math.min(forcedMath - numericCount(), count - quiz.questions.length) : 0;
|
||||
try {
|
||||
if (wantMath > 0) await runBatch(batches + 30 + round * 3 + attempt, wantMath, true);
|
||||
else await runBatch(batches + 30 + round * 3 + attempt, Math.min(perBatch, count - quiz.questions.length));
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && err.code === 'aborted') throw err;
|
||||
}
|
||||
}
|
||||
if (forcedMath) quiz.questions = spreadEvenly(quiz.questions, (q) => q.type === 'numeric');
|
||||
history.push(`Round ${round}: ${missing} removed question${missing === 1 ? ' was' : 's were'} replaced with new ones${round === VERIFY_ROUNDS ? ' (after the last check)' : ''}.`);
|
||||
}
|
||||
// Done when nothing is left to fix and nothing new (rewritten or replacement questions) needs checking.
|
||||
const done = missing <= 0 && last.rewritten === 0 && second.applied.length === 0 && last.failed.length === 0 && last.refused.length === 0;
|
||||
if (done || round === VERIFY_ROUNDS) break;
|
||||
}
|
||||
if (last) {
|
||||
const summary = last.notes[0] ?? '';
|
||||
notes.push(
|
||||
...(last.incomplete ? last.notes : [
|
||||
rounds > 1 ? summary.replace(/^Checked with Python/, `Checked with Python in ${rounds} rounds, last round`) : summary, ...last.failed.map((f) => `Check failed: ${f}`), ...(secondReview ? [secondReview.incomplete ? 'The second review could not finish.' : secondReview.flagged === 0 ? 'Second review: no problems found.' : `Second review: ${secondReview.flagged} question${secondReview.flagged === 1 ? '' : 's'} fixed in the last round.`] : []), ...history, ...last.refused]),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// A short name and description of what this particular set covers.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue