Second revision: MCP servers, shared-link AI, flashcard modes, document previews
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
Generation - Every quiz is NCLEX style; "Practice exam" writes 50 questions. - Calculation questions only when the material involves calculations (max 5 per exam). - Broad topics: the class's topics are listed in the prompt, a quick grouping call maps narrow labels onto broad ones, and leftover variants are merged. - Exact select-all count per batch; option order shuffled. - Python check: at most 3 runs, fixed problems count as passed, cosmetic remarks ignored. Assistant - Text, tool steps and cards appear in the order they happen, with paragraph breaks between steps. - Optional thinking box: fixed height, follows the newest text, can be scrolled back. - "+" opens a material picker listing recent items; items can be dragged in from class pages; shorter placeholder. MCP servers - Streamable HTTP client; encrypted headers. - Per-server instructions and switches for chat, Explain, generation and the quiz check. - Settings UI. Share links - Optional "Explain this" and "Ask about this material" per link, off by default. - Items are loaded on the server and checked against the share; no tools; rate-limited. - Migration 4 stores the switches. Study and documents - Standard flashcard mode (optionally repeat missed cards until known) next to spaced repetition. - Streak visuals restyled in the app palette. - Side by side / Original document views, with sandboxed previews of Word and PowerPoint files. Other - Built-in embedding model baked into the image; sharp stubbed if it cannot load; real load error shown in Settings. - Python runner blocks network and process modules in JavaScript too. - "View questions and answers" removed from the quiz start screen. - Docs updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
5ea61c882c
commit
031f7bec31
45 changed files with 2918 additions and 397 deletions
|
|
@ -11,6 +11,7 @@ import { z } from 'zod';
|
|||
import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../../shared/content.ts';
|
||||
import type { AppContext } from '../http.ts';
|
||||
import { runPython } from './sandbox.ts';
|
||||
import { loadMcpTools, mcpPromptNote, type LoadedMcp } from './mcp.ts';
|
||||
import { resolveModel, runChat } from './service.ts';
|
||||
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
|
||||
|
||||
|
|
@ -73,6 +74,8 @@ export interface QuizRequestFacts {
|
|||
style: string;
|
||||
sataShare: number;
|
||||
difficulty?: string;
|
||||
/** Most calculation (numeric) questions allowed; 0 when the material has no calculations. */
|
||||
numericCap?: number;
|
||||
}
|
||||
|
||||
/** Long text is shortened rather than rejected: a verbose but correct review must not be lost. */
|
||||
|
|
@ -80,8 +83,9 @@ const text = (max: number) => z.coerce.string().transform((v) => v.slice(0, max)
|
|||
const int = z.coerce.number().int();
|
||||
const fixSchema = z.object({
|
||||
question: int.min(1),
|
||||
action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'remove']),
|
||||
action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'remove']),
|
||||
correct_options: z.array(int.min(1)).max(12).optional(),
|
||||
topic: text(80).optional(),
|
||||
answer: text(200).optional(),
|
||||
type: z.enum(['single', 'multi', 'numeric']).optional(),
|
||||
reason: text(600).optional().default(''),
|
||||
|
|
@ -117,10 +121,12 @@ const SUBMIT_REVIEW: ToolDef = {
|
|||
question: { type: 'integer' },
|
||||
action: {
|
||||
type: 'string',
|
||||
enum: ['set_correct', 'set_numeric_answer', 'set_type', 'remove'],
|
||||
description: 'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. remove: drop an unfixable question.',
|
||||
enum: ['set_correct', 'set_numeric_answer', 'set_type', 'set_topic', 'remove'],
|
||||
description:
|
||||
'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. set_topic: replace a narrow topic label with a broad one. remove: drop an unfixable question.',
|
||||
},
|
||||
correct_options: { type: 'array', items: { type: 'integer' } },
|
||||
topic: { type: 'string' },
|
||||
answer: { type: 'string' },
|
||||
type: { type: 'string', enum: ['single', 'multi', 'numeric'] },
|
||||
reason: { type: 'string' },
|
||||
|
|
@ -136,25 +142,29 @@ const SUBMIT_REVIEW: ToolDef = {
|
|||
function rulesFor(req: QuizRequestFacts): string {
|
||||
const expectedMulti = req.sataShare <= 0 ? 'none' : req.sataShare >= 1 ? 'all non-numeric questions' : `about ${Math.round(req.sataShare * 100)}% of the non-numeric questions`;
|
||||
return [
|
||||
`Requested: ${req.count} questions, style "${req.style}", difficulty "${req.difficulty ?? 'mixed'}", select-all (multi) share: ${expectedMulti}.`,
|
||||
`Requested: ${req.count} NCLEX-style questions, difficulty "${req.difficulty ?? 'mixed'}", select-all (multi) share: ${expectedMulti}, calculation (numeric) questions: ${req.numericCap ? `at most ${req.numericCap}` : 'none (the material has no calculations)'}.`,
|
||||
'Rules each question must meet:',
|
||||
'- single: exactly one correct option; 4 options for NCLEX/standard/recall styles.',
|
||||
'- single: exactly one correct option and 4 options.',
|
||||
'- multi (select all that apply): at least 2 correct options; for NCLEX style 5 or 6 options with 2 to 5 correct and a stem ending with "Select all that apply."',
|
||||
'- numeric: the student types a number; options holds the accepted answer(s), all marked correct. The answer must equal the value computed from the stem, rounded as the stem says.',
|
||||
'- No duplicate or near-duplicate stems; no identical options within a question; no "all of the above" / "none of the above".',
|
||||
'- The keyed answer must be the clinically correct one (check against the rationale and standard practice).',
|
||||
'- Topic labels are broad (the main condition, drug class or body system, e.g. "Stroke"), not subtypes or aspects ("Hemorrhagic stroke", "Heparin client teaching").',
|
||||
'Ignore: which position the correct option is in (answer order is shuffled every time a quiz is taken), quotation marks, capitalisation and other formatting.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
const VERIFY_SYSTEM = `You are the quality checker for quiz questions written for a nursing student. Accuracy is critical: they are used to prepare for licensing exams.
|
||||
Check the draft mechanically with the run_python tool. The variables \`draft\` (the quiz) and \`request\` (what was asked for) are already defined in every run. Write Python that counts and verifies instead of eyeballing:
|
||||
Check the draft mechanically with the run_python tool. The variables \`draft\` (the quiz) and \`request\` (what was asked for) are already defined in every run.
|
||||
Be efficient: write ONE Python script that does all the mechanical checks at once and prints a compact report, then at most one more run for the calculations. Do not use more than 3 runs. The script should check:
|
||||
1. The number of questions against request["count"].
|
||||
2. The mix of types (single / multi / numeric) against the requested select-all share.
|
||||
2. The mix of types (single / multi / numeric) against the requested select-all share and the calculation limit.
|
||||
3. For every question: option count, number of correct options, and that it fits its type's rules.
|
||||
4. Duplicate stems or options.
|
||||
5. Every numeric question: recompute the answer from the numbers in the stem with Python, apply the rounding the stem asks for, and compare with the keyed answer numerically (ignore units).
|
||||
5. Every numeric question: recompute the answer from the numbers in the stem, apply the rounding the stem asks for, and compare with the keyed answer numerically (ignore units).
|
||||
6. How many distinct topics there are, and whether any are narrow variants of a broader one.
|
||||
Also read each question and judge whether the keyed answer is clinically correct.
|
||||
Then call submit_review once with every check you ran and a fix for each definite error. Do not "fix" things that are only stylistic. A count or mix that is off cannot be fixed here; report it as a failed check.`;
|
||||
Then call submit_review once. List only real checks. A check passes when the draft is right after your fixes: if you give fixes for every problem a check found, mark it passed and say what you fixed in its detail. Mark a check failed only for a problem that remains. Never list informational or cosmetic remarks as failed checks. Give a fix for each definite error. Do not "fix" style. A count or mix that is off cannot be fixed here; report it as a failed check.`;
|
||||
|
||||
/** Applies validated fixes; returns notes describing what changed or was refused. */
|
||||
export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer<typeof fixSchema>[]): { questions: DraftQuestion[]; applied: string[]; refused: string[] } {
|
||||
|
|
@ -184,6 +194,8 @@ export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer<typeof
|
|||
q.options = [{ text: fix.answer.trim(), correct: true, why: q.options[0]?.why ?? '' }];
|
||||
} else if (fix.action === 'set_type' && fix.type && fix.type !== 'numeric') {
|
||||
q.type = fix.type;
|
||||
} else if (fix.action === 'set_topic' && fix.topic?.trim()) {
|
||||
q.topic = fix.topic.trim().slice(0, 80);
|
||||
} else {
|
||||
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} was incomplete.`);
|
||||
continue;
|
||||
|
|
@ -193,7 +205,7 @@ export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer<typeof
|
|||
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} would have made it invalid, so it was not applied${why}.`);
|
||||
continue;
|
||||
}
|
||||
applied.push(`Question ${fix.question}: ${fix.action === 'set_numeric_answer' ? `answer corrected to ${fix.answer!.trim()}` : fix.action === 'set_type' ? `type changed to ${fix.type}` : `answer key corrected`}${why}.`);
|
||||
applied.push(`Question ${fix.question}: ${fix.action === 'set_numeric_answer' ? `answer corrected to ${fix.answer!.trim()}` : fix.action === 'set_type' ? `type changed to ${fix.type}` : fix.action === 'set_topic' ? `topic changed to ${q.topic}` : `answer key corrected`}${fix.action === 'set_topic' ? '' : why}.`);
|
||||
}
|
||||
return { questions: out.filter((_, i) => !remove.has(i)), applied, refused };
|
||||
}
|
||||
|
|
@ -220,13 +232,18 @@ export async function verifyQuizDraft(
|
|||
const messages: ChatMessage[] = [{ role: 'user', content: [{ type: 'text', text: `${rulesFor(req)}\n\nTHE DRAFT (${data.question_count} questions):\n\n${listing}` }] }];
|
||||
let review: QuizReview | null = null;
|
||||
let pythonRuns = 0;
|
||||
let mcpCalls = 0;
|
||||
const MCP_BUDGET = 4;
|
||||
const mcp = await loadMcpTools(app, 'check', opts.signal).catch(() => ({ tools: [], servers: [], unavailable: [] }) as LoadedMcp);
|
||||
const system = VERIFY_SYSTEM + mcpPromptNote(mcp);
|
||||
// Up to WORK_STEPS turns with Python, then only submit_review is offered until it arrives.
|
||||
const WORK_STEPS = 8;
|
||||
const MAX_STEPS = 12;
|
||||
const WORK_STEPS = 4;
|
||||
const PYTHON_BUDGET = 4;
|
||||
const MAX_STEPS = 7;
|
||||
try {
|
||||
for (let step = 0; step < MAX_STEPS && !review; step += 1) {
|
||||
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
|
||||
const wrapUp = step >= WORK_STEPS || pythonRuns >= 16;
|
||||
const wrapUp = step >= WORK_STEPS || pythonRuns >= PYTHON_BUDGET;
|
||||
if (wrapUp && !messages.some((m) => m.content.some((p) => p.type === 'text' && p.text.startsWith('You have run enough checks')))) {
|
||||
const nudge: ContentPart = { type: 'text', text: 'You have run enough checks. Call submit_review now with your results and fixes.' };
|
||||
const lastMsg = messages[messages.length - 1];
|
||||
|
|
@ -238,9 +255,10 @@ export async function verifyQuizDraft(
|
|||
task: 'generate',
|
||||
purpose: 'quiz check',
|
||||
modelId: opts.modelId,
|
||||
system: VERIFY_SYSTEM,
|
||||
system,
|
||||
messages,
|
||||
tools: wrapUp ? [SUBMIT_REVIEW] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW],
|
||||
// MCP tools stay listed at wrap-up (the transcript may hold their calls) but submit_review is forced.
|
||||
tools: wrapUp ? [SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)],
|
||||
toolChoice: wrapUp ? { name: 'submit_review' } : 'auto',
|
||||
maxOutputTokens: 16_000,
|
||||
signal: opts.signal,
|
||||
|
|
@ -257,13 +275,28 @@ export async function verifyQuizDraft(
|
|||
const parsed = reviewSchema.safeParse(call.input);
|
||||
if (parsed.success) review = parsed.data;
|
||||
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: parsed.success ? '{"ok":true}' : JSON.stringify({ error: parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') }), isError: !parsed.success });
|
||||
} else if (call.name === 'run_python' && pythonRuns >= 16) {
|
||||
} else if (call.name === 'run_python' && pythonRuns >= PYTHON_BUDGET) {
|
||||
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"Python budget used up. Call submit_review now."}', isError: true });
|
||||
} else if (call.name === 'run_python') {
|
||||
pythonRuns += 1;
|
||||
opts.onStep?.(`Checking the questions with Python (run ${pythonRuns})`);
|
||||
const r = await runPython(prelude + String(call.input.code ?? ''));
|
||||
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify(pythonOutput(r)), isError: !r.ok });
|
||||
} else if (mcp.tools.some((t) => t.def.name === call.name)) {
|
||||
const binding = mcp.tools.find((t) => t.def.name === call.name)!;
|
||||
if (mcpCalls >= MCP_BUDGET) {
|
||||
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"External tool budget used up. Finish the check."}', isError: true });
|
||||
continue;
|
||||
}
|
||||
mcpCalls += 1;
|
||||
opts.onStep?.(`Checking with ${binding.serverName}`);
|
||||
try {
|
||||
const text = await binding.call(call.input, opts.signal);
|
||||
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: text.slice(0, 24_000), isError: false });
|
||||
} catch (err) {
|
||||
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
|
||||
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify({ error: err instanceof Error ? err.message : 'Tool failed' }), isError: true });
|
||||
}
|
||||
} else {
|
||||
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"unknown tool"}', isError: true });
|
||||
}
|
||||
|
|
@ -279,9 +312,13 @@ export async function verifyQuizDraft(
|
|||
if (!review) return { draft, notes: ['The Python answer check did not report back. Review the answers yourself.'], review: null, pythonRuns };
|
||||
|
||||
const { questions, applied, refused } = applyQuizFixes(draft.questions, review.fixes);
|
||||
const failed = review.checks.filter((c) => !c.passed);
|
||||
// Remarks the model files as informational or cosmetic are not checks.
|
||||
const real = review.checks.filter((c) => !/informational|cosmetic|position bias/i.test(`${c.name} ${c.detail}`));
|
||||
// A problem the model fixed is not a failure: the applied fix is listed on its own.
|
||||
const fixedHere = (c: { detail: string }) => applied.length > 0 && /\b(fixed|corrected|fix applied)\b/i.test(c.detail ?? '');
|
||||
const failed = real.filter((c) => !c.passed && !fixedHere(c));
|
||||
const notes = [
|
||||
`Checked with Python (${pythonRuns} run${pythonRuns === 1 ? '' : 's'}): ${review.checks.length - failed.length} of ${review.checks.length} checks passed${applied.length ? `, ${applied.length} fix${applied.length === 1 ? '' : 'es'} applied` : ''}.`,
|
||||
`Checked with Python (${pythonRuns} run${pythonRuns === 1 ? '' : 's'}): ${real.length - failed.length} of ${real.length} checks passed${applied.length ? `, ${applied.length} fix${applied.length === 1 ? '' : 'es'} applied` : ''}.`,
|
||||
...failed.map((c) => `Check failed: ${c.name}${c.detail ? `: ${c.detail}` : ''}`),
|
||||
...applied,
|
||||
...refused,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue