Quiz check: fix before handing over, with an independent second review
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s

- The check can rewrite a whole question (garbled or contradictory stems,
  missing data, two defensible answers); a key fix that changes nothing is
  skipped.
- Removed questions are replaced with new ones, so a 50-question quiz stays 50.
- A second, independent model call reads the whole quiz for badly written
  items, helped by a scan for leftovers such as "A 78 kg? No - an 80 kg".
- The quiz is checked again after any rewrite, replacement or open problem,
  up to three rounds, before it is handed over.
- Reviewers see full rationales (a shortened one is marked as shortened).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 10:07:32 -07:00
parent 85d5c9186d
commit 1a212bb8ba
8 changed files with 412 additions and 26 deletions

View file

@ -20,6 +20,8 @@ export interface MockAi {
delayChat: (ms: number) => void;
/** The next N calls of this tool send arguments that are not valid JSON (beyond jsonrepair). */
badJsonNext: (tool: string, times?: number) => void;
/** Arguments for the next submit_review calls, in order (then the default review). */
scriptReviews: (reviews: Record<string, unknown>[]) => void;
close: () => Promise<void>;
}
@ -73,7 +75,9 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
// Only refs from the user's turn count: the system prompt contains an example id.
const refs = [...new Set([...userText.matchAll(/\[(c\d+)\]/g)].map((m) => m[1]))];
// Continue numbering after anything already written, so later batches are not duplicates.
const offset = (userText.split('Already written')[1]?.match(/^- /gm) ?? []).length;
const written = userText.split('Already written')[1] ?? '';
// Continue after the highest number written, so a refill after a removal is never a repeat.
const offset = Math.max((written.match(/^- /gm) ?? []).length, ...[...written.matchAll(/Mock (?:question|math|term) (\d+)/g)].map((m) => Number(m[1])));
void all;
switch (name) {
case 'submit_questions':
@ -108,6 +112,21 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
const fix = /Q2 \[single\]/.test(all) ? [{ question: 2, action: 'set_correct', correct_options: [2], reason: 'mock: option 2 is the right answer' }] : [];
return { summary: 'Checked by the mock.', checks: [{ name: 'Question count', passed: true, detail: 'matches the request' }, { name: 'Correct answers per type', passed: true }], fixes: fix };
}
case 'submit_proofread': {
// Rewrites a question whose stem has a writer's change of mind in it ("? No —").
const m = /^Q(\d+) \[\w+\] .*\?\s*No\b/m.exec(userText);
if (!m) return { issues: [] };
return {
issues: [
{
question: Number(m[1]),
problem: 'The stem gives two weights.',
action: 'rewrite',
question_text: { type: 'numeric', stem: 'An 80 kg client receives norepinephrine 4 mg in 250 mL at 15 mL/h. What dose is the client receiving in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000 mcg / 250 mL x 15 mL/h / 60 / 80 kg = 0.05' }], rationale: 'Convert mg to mcg, find mcg/h, then per minute and per kg.' },
},
],
};
}
case 'submit_topics': {
// Labels of three or more words are broadened to their first word, like a real model would.
const labels = [...(userText.split('Labels:')[1] ?? '').matchAll(/^- (.+)$/gm)].map((m) => m[1].trim());
@ -151,6 +170,7 @@ export function startMockAi(port = 0): Promise<MockAi> {
let fail = { status: 0, times: 0 };
let delayMs = 0;
let badJson: { tool: string; times: number } | null = null;
let reviews: Record<string, unknown>[] = [];
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
@ -203,7 +223,7 @@ export function startMockAi(port = 0): Promise<MockAi> {
res.end();
};
const callTool = (name: string) => {
let args = JSON.stringify(toolArgs(name, userText, all));
let args = JSON.stringify(name === 'submit_review' && reviews.length ? reviews.shift() : toolArgs(name, userText, all));
if (badJson && badJson.tool === name && badJson.times > 0) {
badJson.times -= 1;
args = '{"cards": [{"front": "Broken" "back": ]]]} :: not json';
@ -277,6 +297,9 @@ export function startMockAi(port = 0): Promise<MockAi> {
badJsonNext: (tool, times = 1) => {
badJson = { tool, times };
},
scriptReviews: (list) => {
reviews = [...list];
},
close: () => new Promise((done) => server.close(() => done())),
});
});