All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
Generation - Every quiz is NCLEX style; "Practice exam" writes 50 questions. - Calculation questions only when the material involves calculations (max 5 per exam). - Broad topics: the class's topics are listed in the prompt, a quick grouping call maps narrow labels onto broad ones, and leftover variants are merged. - Exact select-all count per batch; option order shuffled. - Python check: at most 3 runs, fixed problems count as passed, cosmetic remarks ignored. Assistant - Text, tool steps and cards appear in the order they happen, with paragraph breaks between steps. - Optional thinking box: fixed height, follows the newest text, can be scrolled back. - "+" opens a material picker listing recent items; items can be dragged in from class pages; shorter placeholder. MCP servers - Streamable HTTP client; encrypted headers. - Per-server instructions and switches for chat, Explain, generation and the quiz check. - Settings UI. Share links - Optional "Explain this" and "Ask about this material" per link, off by default. - Items are loaded on the server and checked against the share; no tools; rate-limited. - Migration 4 stores the switches. Study and documents - Standard flashcard mode (optionally repeat missed cards until known) next to spaced repetition. - Streak visuals restyled in the app palette. - Side by side / Original document views, with sandboxed previews of Word and PowerPoint files. Other - Built-in embedding model baked into the image; sharp stubbed if it cannot load; real load error shown in Settings. - Python runner blocks network and process modules in JavaScript too. - "View questions and answers" removed from the quiz start screen. - Docs updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
327 lines
19 KiB
TypeScript
327 lines
19 KiB
TypeScript
/**
|
|
* Python-backed checking of generated quizzes, and the `run_python` tool the assistant uses.
|
|
*
|
|
* After a quiz is generated, a short agent loop gives the model the draft (as Python data and
|
|
* as text) plus the original request and asks it to verify everything mechanically with the
|
|
* sandbox: question count, the select-all / single / numeric mix, options and correct answers
|
|
* per type, duplicates, and every calculation recomputed from its stem. It then submits fixes,
|
|
* which are applied here only if the question still validates afterwards.
|
|
*/
|
|
import { z } from 'zod';
|
|
import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../../shared/content.ts';
|
|
import type { AppContext } from '../http.ts';
|
|
import { runPython } from './sandbox.ts';
|
|
import { loadMcpTools, mcpPromptNote, type LoadedMcp } from './mcp.ts';
|
|
import { resolveModel, runChat } from './service.ts';
|
|
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
|
|
|
|
/* ------------------------------ data handed to Python ------------------------------ */
|
|
|
|
/** Compact, 1-based view of a quiz used both in prompts and as the `draft` variable in Python. */
|
|
export function quizForPython(quiz: { title: string; questions: Pick<DraftQuestion, 'type' | 'stem' | 'options' | 'rationale' | 'topic'>[] }) {
|
|
return {
|
|
title: quiz.title,
|
|
question_count: quiz.questions.length,
|
|
questions: quiz.questions.map((q, i) => ({
|
|
number: i + 1,
|
|
type: q.type,
|
|
topic: q.topic,
|
|
stem: q.stem,
|
|
options: q.options.map((o, k) => ({ number: k + 1, text: o.text, correct: Boolean(o.correct), why: o.why ?? '' })),
|
|
rationale: q.rationale ?? '',
|
|
})),
|
|
};
|
|
}
|
|
|
|
/** Python source that defines the given variables from JSON (a JSON string is a valid Python string literal). */
|
|
export function pythonPrelude(vars: Record<string, unknown>): string {
|
|
const lines = ['import json as _json'];
|
|
for (const [name, value] of Object.entries(vars)) {
|
|
if (!/^[a-z_][a-z0-9_]*$/i.test(name)) continue;
|
|
lines.push(`${name} = _json.loads(${JSON.stringify(JSON.stringify(value))})`);
|
|
}
|
|
return `${lines.join('\n')}\n`;
|
|
}
|
|
|
|
/** Formats a sandbox result for a model: short, and never an exception. */
|
|
export function pythonOutput(r: Awaited<ReturnType<typeof runPython>>) {
|
|
return {
|
|
ok: r.ok,
|
|
...(r.stdout ? { stdout: r.stdout } : {}),
|
|
...(r.result !== null ? { result: r.result } : {}),
|
|
...(r.stderr ? { stderr: r.stderr.slice(0, 2000) } : {}),
|
|
...(r.error ? { error: r.error } : {}),
|
|
};
|
|
}
|
|
|
|
export const RUN_PYTHON_DESCRIPTION =
|
|
'Run Python 3 in a sandbox and get stdout and the value of the last expression. Standard library only (math, fractions, decimal, statistics, re, json, collections, itertools, datetime); no files, network or packages; 10 second limit. Use it whenever exactness matters: recompute dosage, rate and conversion answers, count questions, options and correct answers, check lengths, tally topics. Print what you need to see.';
|
|
|
|
export const RUN_PYTHON_TOOL: ToolDef = {
|
|
name: 'run_python',
|
|
description: RUN_PYTHON_DESCRIPTION,
|
|
parameters: {
|
|
type: 'object',
|
|
properties: { code: { type: 'string', description: 'Python source. Variables listed in the instructions are already defined.' } },
|
|
required: ['code'],
|
|
},
|
|
};
|
|
|
|
/* ------------------------------ verification of a generated quiz ------------------------------ */
|
|
|
|
export interface QuizRequestFacts {
|
|
count: number;
|
|
style: string;
|
|
sataShare: number;
|
|
difficulty?: string;
|
|
/** Most calculation (numeric) questions allowed; 0 when the material has no calculations. */
|
|
numericCap?: number;
|
|
}
|
|
|
|
/** Long text is shortened rather than rejected: a verbose but correct review must not be lost. */
|
|
const text = (max: number) => z.coerce.string().transform((v) => v.slice(0, max));
|
|
const int = z.coerce.number().int();
|
|
const fixSchema = z.object({
|
|
question: int.min(1),
|
|
action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'remove']),
|
|
correct_options: z.array(int.min(1)).max(12).optional(),
|
|
topic: text(80).optional(),
|
|
answer: text(200).optional(),
|
|
type: z.enum(['single', 'multi', 'numeric']).optional(),
|
|
reason: text(600).optional().default(''),
|
|
});
|
|
const reviewSchema = z.object({
|
|
summary: text(1500).optional().default(''),
|
|
checks: z
|
|
.array(z.object({ name: text(160), passed: z.coerce.boolean(), detail: text(600).optional().default('') }))
|
|
.transform((v) => v.slice(0, 40))
|
|
.optional()
|
|
.default([]),
|
|
fixes: z.array(z.unknown()).optional().default([]).transform((list) => list.flatMap((f) => { const r = fixSchema.safeParse(f); return r.success ? [r.data] : []; })),
|
|
});
|
|
export type QuizReview = z.infer<typeof reviewSchema>;
|
|
|
|
const SUBMIT_REVIEW: ToolDef = {
|
|
name: 'submit_review',
|
|
description: 'Finish the check: report every check you ran (with its result) and the fixes to apply. Call this exactly once, at the end.',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
summary: { type: 'string', description: 'One or two sentences for the student.' },
|
|
checks: {
|
|
type: 'array',
|
|
items: { type: 'object', properties: { name: { type: 'string' }, passed: { type: 'boolean' }, detail: { type: 'string' } }, required: ['name', 'passed'] },
|
|
},
|
|
fixes: {
|
|
type: 'array',
|
|
description: 'Only for definite errors. Question and option numbers are 1-based.',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
question: { type: 'integer' },
|
|
action: {
|
|
type: 'string',
|
|
enum: ['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'remove'],
|
|
description:
|
|
'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. set_topic: replace a narrow topic label with a broad one. remove: drop an unfixable question.',
|
|
},
|
|
correct_options: { type: 'array', items: { type: 'integer' } },
|
|
topic: { type: 'string' },
|
|
answer: { type: 'string' },
|
|
type: { type: 'string', enum: ['single', 'multi', 'numeric'] },
|
|
reason: { type: 'string' },
|
|
},
|
|
required: ['question', 'action', 'reason'],
|
|
},
|
|
},
|
|
},
|
|
required: ['summary', 'checks', 'fixes'],
|
|
},
|
|
};
|
|
|
|
function rulesFor(req: QuizRequestFacts): string {
|
|
const expectedMulti = req.sataShare <= 0 ? 'none' : req.sataShare >= 1 ? 'all non-numeric questions' : `about ${Math.round(req.sataShare * 100)}% of the non-numeric questions`;
|
|
return [
|
|
`Requested: ${req.count} NCLEX-style questions, difficulty "${req.difficulty ?? 'mixed'}", select-all (multi) share: ${expectedMulti}, calculation (numeric) questions: ${req.numericCap ? `at most ${req.numericCap}` : 'none (the material has no calculations)'}.`,
|
|
'Rules each question must meet:',
|
|
'- single: exactly one correct option and 4 options.',
|
|
'- multi (select all that apply): at least 2 correct options; for NCLEX style 5 or 6 options with 2 to 5 correct and a stem ending with "Select all that apply."',
|
|
'- numeric: the student types a number; options holds the accepted answer(s), all marked correct. The answer must equal the value computed from the stem, rounded as the stem says.',
|
|
'- No duplicate or near-duplicate stems; no identical options within a question; no "all of the above" / "none of the above".',
|
|
'- The keyed answer must be the clinically correct one (check against the rationale and standard practice).',
|
|
'- Topic labels are broad (the main condition, drug class or body system, e.g. "Stroke"), not subtypes or aspects ("Hemorrhagic stroke", "Heparin client teaching").',
|
|
'Ignore: which position the correct option is in (answer order is shuffled every time a quiz is taken), quotation marks, capitalisation and other formatting.',
|
|
].join('\n');
|
|
}
|
|
|
|
const VERIFY_SYSTEM = `You are the quality checker for quiz questions written for a nursing student. Accuracy is critical: they are used to prepare for licensing exams.
|
|
Check the draft mechanically with the run_python tool. The variables \`draft\` (the quiz) and \`request\` (what was asked for) are already defined in every run.
|
|
Be efficient: write ONE Python script that does all the mechanical checks at once and prints a compact report, then at most one more run for the calculations. Do not use more than 3 runs. The script should check:
|
|
1. The number of questions against request["count"].
|
|
2. The mix of types (single / multi / numeric) against the requested select-all share and the calculation limit.
|
|
3. For every question: option count, number of correct options, and that it fits its type's rules.
|
|
4. Duplicate stems or options.
|
|
5. Every numeric question: recompute the answer from the numbers in the stem, apply the rounding the stem asks for, and compare with the keyed answer numerically (ignore units).
|
|
6. How many distinct topics there are, and whether any are narrow variants of a broader one.
|
|
Also read each question and judge whether the keyed answer is clinically correct.
|
|
Then call submit_review once. List only real checks. A check passes when the draft is right after your fixes: if you give fixes for every problem a check found, mark it passed and say what you fixed in its detail. Mark a check failed only for a problem that remains. Never list informational or cosmetic remarks as failed checks. Give a fix for each definite error. Do not "fix" style. A count or mix that is off cannot be fixed here; report it as a failed check.`;
|
|
|
|
/** Applies validated fixes; returns notes describing what changed or was refused. */
|
|
export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer<typeof fixSchema>[]): { questions: DraftQuestion[]; applied: string[]; refused: string[] } {
|
|
const out = questions.map((q) => ({ ...q, options: q.options.map((o) => ({ ...o })) }));
|
|
const remove = new Set<number>();
|
|
const applied: string[] = [];
|
|
const refused: string[] = [];
|
|
for (const fix of fixes) {
|
|
const i = fix.question - 1;
|
|
const q = out[i];
|
|
if (!q) {
|
|
refused.push(`Question ${fix.question}: does not exist.`);
|
|
continue;
|
|
}
|
|
const before = { ...q, options: q.options.map((o) => ({ ...o })) };
|
|
const why = fix.reason ? ` (${fix.reason.replace(/\s+/g, ' ').slice(0, 200)})` : '';
|
|
if (fix.action === 'remove') {
|
|
remove.add(i);
|
|
applied.push(`Question ${fix.question} removed${why}.`);
|
|
continue;
|
|
}
|
|
if (fix.action === 'set_correct' && fix.correct_options?.length) {
|
|
const set = new Set(fix.correct_options.map((n) => n - 1));
|
|
q.options = q.options.map((o, k) => ({ ...o, correct: set.has(k) }));
|
|
if (q.type === 'single' && set.size > 1) q.type = 'multi';
|
|
} else if (fix.action === 'set_numeric_answer' && fix.answer?.trim() && q.type === 'numeric') {
|
|
q.options = [{ text: fix.answer.trim(), correct: true, why: q.options[0]?.why ?? '' }];
|
|
} else if (fix.action === 'set_type' && fix.type && fix.type !== 'numeric') {
|
|
q.type = fix.type;
|
|
} else if (fix.action === 'set_topic' && fix.topic?.trim()) {
|
|
q.topic = fix.topic.trim().slice(0, 80);
|
|
} else {
|
|
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} was incomplete.`);
|
|
continue;
|
|
}
|
|
if (validateDraftQuestion(q, i).some((p) => p.level === 'error')) {
|
|
out[i] = before;
|
|
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} would have made it invalid, so it was not applied${why}.`);
|
|
continue;
|
|
}
|
|
applied.push(`Question ${fix.question}: ${fix.action === 'set_numeric_answer' ? `answer corrected to ${fix.answer!.trim()}` : fix.action === 'set_type' ? `type changed to ${fix.type}` : fix.action === 'set_topic' ? `topic changed to ${q.topic}` : `answer key corrected`}${fix.action === 'set_topic' ? '' : why}.`);
|
|
}
|
|
return { questions: out.filter((_, i) => !remove.has(i)), applied, refused };
|
|
}
|
|
|
|
/**
|
|
* Checks a generated quiz with the model and the Python sandbox, then applies the fixes.
|
|
* Never throws: a failed check leaves the draft as it was and says so in the notes.
|
|
*/
|
|
export async function verifyQuizDraft(
|
|
app: AppContext,
|
|
draft: QuizDraft,
|
|
req: QuizRequestFacts,
|
|
opts: { modelId?: string | null; signal?: AbortSignal; onStep?: (label: string) => void } = {},
|
|
): Promise<{ draft: QuizDraft; notes: string[]; review: QuizReview | null; pythonRuns: number }> {
|
|
const resolved = resolveModel(app, 'generate', opts.modelId);
|
|
if (!resolved.model.capabilities.tools) {
|
|
return { draft, notes: ['The Python answer check was skipped: the generation model cannot use tools.'], review: null, pythonRuns: 0 };
|
|
}
|
|
const data = quizForPython(draft);
|
|
const prelude = pythonPrelude({ draft: data, request: { count: req.count, style: req.style, sata_share: req.sataShare, difficulty: req.difficulty ?? 'mixed' } });
|
|
const listing = data.questions
|
|
.map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${q.rationale.slice(0, 600)}` : ''}`)
|
|
.join('\n\n');
|
|
const messages: ChatMessage[] = [{ role: 'user', content: [{ type: 'text', text: `${rulesFor(req)}\n\nTHE DRAFT (${data.question_count} questions):\n\n${listing}` }] }];
|
|
let review: QuizReview | null = null;
|
|
let pythonRuns = 0;
|
|
let mcpCalls = 0;
|
|
const MCP_BUDGET = 4;
|
|
const mcp = await loadMcpTools(app, 'check', opts.signal).catch(() => ({ tools: [], servers: [], unavailable: [] }) as LoadedMcp);
|
|
const system = VERIFY_SYSTEM + mcpPromptNote(mcp);
|
|
// Up to WORK_STEPS turns with Python, then only submit_review is offered until it arrives.
|
|
const WORK_STEPS = 4;
|
|
const PYTHON_BUDGET = 4;
|
|
const MAX_STEPS = 7;
|
|
try {
|
|
for (let step = 0; step < MAX_STEPS && !review; step += 1) {
|
|
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
|
|
const wrapUp = step >= WORK_STEPS || pythonRuns >= PYTHON_BUDGET;
|
|
if (wrapUp && !messages.some((m) => m.content.some((p) => p.type === 'text' && p.text.startsWith('You have run enough checks')))) {
|
|
const nudge: ContentPart = { type: 'text', text: 'You have run enough checks. Call submit_review now with your results and fixes.' };
|
|
const lastMsg = messages[messages.length - 1];
|
|
// Ride along with the tool results so user and assistant turns keep alternating.
|
|
if (lastMsg?.role === 'user') lastMsg.content.push(nudge);
|
|
else messages.push({ role: 'user', content: [nudge] });
|
|
}
|
|
const res = await runChat(app, {
|
|
task: 'generate',
|
|
purpose: 'quiz check',
|
|
modelId: opts.modelId,
|
|
system,
|
|
messages,
|
|
// MCP tools stay listed at wrap-up (the transcript may hold their calls) but submit_review is forced.
|
|
tools: wrapUp ? [SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)],
|
|
toolChoice: wrapUp ? { name: 'submit_review' } : 'auto',
|
|
maxOutputTokens: 16_000,
|
|
signal: opts.signal,
|
|
});
|
|
messages.push(res.message);
|
|
const calls = res.message.content.filter((p): p is Extract<ContentPart, { type: 'tool_call' }> => p.type === 'tool_call');
|
|
if (calls.length === 0) {
|
|
messages.push({ role: 'user', content: [{ type: 'text', text: 'Run your checks with run_python, then call submit_review.' }] });
|
|
continue;
|
|
}
|
|
const results: ContentPart[] = [];
|
|
for (const call of calls) {
|
|
if (call.name === 'submit_review') {
|
|
const parsed = reviewSchema.safeParse(call.input);
|
|
if (parsed.success) review = parsed.data;
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: parsed.success ? '{"ok":true}' : JSON.stringify({ error: parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') }), isError: !parsed.success });
|
|
} else if (call.name === 'run_python' && pythonRuns >= PYTHON_BUDGET) {
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"Python budget used up. Call submit_review now."}', isError: true });
|
|
} else if (call.name === 'run_python') {
|
|
pythonRuns += 1;
|
|
opts.onStep?.(`Checking the questions with Python (run ${pythonRuns})`);
|
|
const r = await runPython(prelude + String(call.input.code ?? ''));
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify(pythonOutput(r)), isError: !r.ok });
|
|
} else if (mcp.tools.some((t) => t.def.name === call.name)) {
|
|
const binding = mcp.tools.find((t) => t.def.name === call.name)!;
|
|
if (mcpCalls >= MCP_BUDGET) {
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"External tool budget used up. Finish the check."}', isError: true });
|
|
continue;
|
|
}
|
|
mcpCalls += 1;
|
|
opts.onStep?.(`Checking with ${binding.serverName}`);
|
|
try {
|
|
const text = await binding.call(call.input, opts.signal);
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: text.slice(0, 24_000), isError: false });
|
|
} catch (err) {
|
|
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify({ error: err instanceof Error ? err.message : 'Tool failed' }), isError: true });
|
|
}
|
|
} else {
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"unknown tool"}', isError: true });
|
|
}
|
|
}
|
|
// The last user message carries the tool results; keep wrap-up instructions in front of the model.
|
|
messages.push({ role: 'user', content: results });
|
|
}
|
|
} catch (err) {
|
|
if (err instanceof AIError && err.code === 'aborted') throw err;
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
return { draft, notes: [`The Python answer check could not finish (${message.slice(0, 200)}). Review the answers yourself.`], review: null, pythonRuns };
|
|
}
|
|
if (!review) return { draft, notes: ['The Python answer check did not report back. Review the answers yourself.'], review: null, pythonRuns };
|
|
|
|
const { questions, applied, refused } = applyQuizFixes(draft.questions, review.fixes);
|
|
// Remarks the model files as informational or cosmetic are not checks.
|
|
const real = review.checks.filter((c) => !/informational|cosmetic|position bias/i.test(`${c.name} ${c.detail}`));
|
|
// A problem the model fixed is not a failure: the applied fix is listed on its own.
|
|
const fixedHere = (c: { detail: string }) => applied.length > 0 && /\b(fixed|corrected|fix applied)\b/i.test(c.detail ?? '');
|
|
const failed = real.filter((c) => !c.passed && !fixedHere(c));
|
|
const notes = [
|
|
`Checked with Python (${pythonRuns} run${pythonRuns === 1 ? '' : 's'}): ${real.length - failed.length} of ${real.length} checks passed${applied.length ? `, ${applied.length} fix${applied.length === 1 ? '' : 'es'} applied` : ''}.`,
|
|
...failed.map((c) => `Check failed: ${c.name}${c.detail ? `: ${c.detail}` : ''}`),
|
|
...applied,
|
|
...refused,
|
|
];
|
|
return { draft: { ...draft, questions }, notes, review, pythonRuns };
|
|
}
|