diff --git a/package-lock.json b/package-lock.json index 64c1d24..aa8d0a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "lucide-react": "^1.71.0", "mammoth": "^1.13.0", "pdf-lib": "^1.17.1", + "pyodide": "^314.0.7", "radix-ui": "^1.8.1", "react": "^19.3.1", "react-dom": "^19.3.1", @@ -3510,6 +3511,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.6", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.6.tgz", + "integrity": "sha512-uN+9i8bFT5CUcZfyIEYDrSueACEyKGbUs5kC/72DGlZZoinh84sJfVV0i8UOJD1asdzkvLPBRrKs41kZ8MdEXg==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", @@ -6398,6 +6405,19 @@ "node": ">=12.0.0" } }, + "node_modules/pyodide": { + "version": "314.0.7", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-314.0.7.tgz", + "integrity": "sha512-0YvXxEhfEdpLfb/XkM2BFAeMROq0iMUX2bzzH9pOttyMcWkwq+HbE5uyuGD82LN7y2q+SNvi/6V5JEsOlD2R1A==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/radix-ui": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.8.1.tgz", @@ -7554,6 +7574,27 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/ws": { + "version": "8.23.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.23.0.tgz", + "integrity": "sha512-wCkmX8QMOrKjgO8HW2WyDbTSGeqkocxgDzS9iR79rEAexty6/2qzilx3Ag8Tqf9ykop3irLKle0MtVn5qjVnoQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xml-naming": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", diff --git a/package.json b/package.json index b1f830b..cb9fa9b 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "lucide-react": "^1.71.0", "mammoth": "^1.13.0", "pdf-lib": "^1.17.1", + "pyodide": "^314.0.7", "radix-ui": "^1.8.1", "react": "^19.3.1", "react-dom": "^19.3.1", diff --git a/scripts/build-server.mjs b/scripts/build-server.mjs index 2f41883..f01b7cb 100644 --- a/scripts/build-server.mjs +++ b/scripts/build-server.mjs @@ -1,4 +1,5 @@ // Bundles the server into one file. Dependencies stay external and are installed in the image. +import fs from 'node:fs'; import { build } from 'esbuild'; await build({ @@ -12,3 +13,6 @@ await build({ sourcemap: true, logLevel: 'info', }); + +// The Python sandbox runs as its own (permission-restricted) process, so it ships as a file. +fs.copyFileSync('server/python-runner.mjs', 'dist/server/python-runner.mjs'); diff --git a/server/ai/assistant.ts b/server/ai/assistant.ts index 7980874..32f269b 100644 --- a/server/ai/assistant.ts +++ b/server/ai/assistant.ts @@ -21,7 +21,9 @@ import { dueCounts } from '../services/srs.ts'; import { topicStats, totals } from '../services/stats.ts'; import { newId, now, parseJson } from '../util.ts'; import { cancelGeneration, createDraft, getDraft, startGeneration, startImprove } from './generate.ts'; +import { runPython } from './sandbox.ts'; import { resolveModel, runChat } from './service.ts'; +import { pythonOutput, pythonPrelude, quizForPython, RUN_PYTHON_DESCRIPTION } from './verify.ts'; import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts'; /* ------------------------------ UI event model ------------------------------ */ @@ -605,6 +607,36 @@ const TOOLS: ToolSpec[] = [ return { output: { folder_id: folder.id, name: folder.name }, summary: folder.name }; }, }, + { + def: { + name: 'run_python', + description: `${RUN_PYTHON_DESCRIPTION} Pass draft_id or quiz_id to load that draft or quiz as the variable \`draft\` / \`quiz\` (dict with "questions": [{"number", "type", "stem", "options": [{"number", "text", "correct"}], "rationale"}]); decks load as {"cards": [{"front", "back"}]}.`, + parameters: obj( + { + code: { ...S, description: 'Python 3 source. Print the results you need.' }, + draft_id: { ...S, description: 'Optional: load this draft as `draft`.' }, + quiz_id: { ...S, description: 'Optional: load this saved quiz as `quiz`.' }, + }, + ['code'], + ), + }, + label: () => 'Running Python', + run: async (input, env) => { + const { db } = env.app; + const vars: Record = {}; + if (typeof input.draft_id === 'string' && input.draft_id) { + const d = getDraft(db, input.draft_id); + vars.draft = d.payload.kind === 'quiz' ? { ...quizForPython(d.payload), request: d.meta.request ?? {} } : d.payload.kind === 'deck' ? { title: d.payload.title, cards: d.payload.cards.map((c) => ({ front: c.front, back: c.back })) } : { title: d.payload.title, content: d.payload.content }; + } + if (typeof input.quiz_id === 'string' && input.quiz_id) { + const quiz = getQuiz(db, input.quiz_id); + vars.quiz = quizForPython({ title: quiz.title, questions: listQuestions(db, quiz.id).map((q) => ({ ...q, options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why })) })) }); + } + const r = await runPython((Object.keys(vars).length ? pythonPrelude(vars) : '') + String(input.code ?? '')); + const firstLine = (r.error ?? '').trim().split('\n').pop() ?? ''; + return { output: pythonOutput(r), summary: r.ok ? (r.stdout.trim().split('\n').pop() ?? r.result ?? 'done').slice(0, 120) : firstLine.slice(0, 160) }; + }, + }, { def: { name: 'propose_changes', @@ -896,7 +928,8 @@ function systemPrompt(app: AppContext, context: AssistantContext): string { - For "what should I study" questions, call get_performance and give specific, prioritised advice. - When a request is clear, act on it without asking for permission. Ask one short question only when a missing detail would change the result (for example which class, when it is ambiguous). - Be concise and concrete. Use Markdown. Teach like a good clinical instructor: explain the why, connect to patient care, and prefer current standard practice. -- After a tool produces a draft or a session link, reply with one or two sentences; the card already shows the details.`, +- After a tool produces a draft or a session link, reply with one or two sentences; the card already shows the details. +- Use run_python whenever exactness matters: dosage and rate calculations, unit conversions, counting questions, options or correct answers, checking that a quiz meets what was asked for. Quizzes from generate_quiz are already checked this way; when you write questions yourself with create_draft, check the draft with run_python (pass its draft_id) before telling the user it is ready, and fix problems you find.`, ); return lines.join('\n\n'); } diff --git a/server/ai/generate.ts b/server/ai/generate.ts index c2bb2c3..11524d4 100644 --- a/server/ai/generate.ts +++ b/server/ai/generate.ts @@ -42,6 +42,7 @@ import { weakTopics } from '../services/stats.ts'; import { getSetting, setSetting } from '../settings.ts'; import { newId, now, parseJson } from '../util.ts'; import { generateStructured, resolveModel, runChat } from './service.ts'; +import { verifyQuizDraft } from './verify.ts'; import { AIError } from './types.ts'; /* ------------------------------ drafts ------------------------------ */ @@ -285,6 +286,8 @@ export const generateRequestSchema = z.object({ cardStyle: z.enum(['concise', 'detailed', 'medication', 'definition']).optional(), noteStyle: z.enum(['study_guide', 'summary', 'outline', 'cheat_sheet']).optional(), modelId: z.string().nullish(), + /** Quizzes: check the finished draft with the model and the Python sandbox (default on). */ + verify: z.boolean().optional(), }); export type GenerateRequest = z.infer; @@ -757,6 +760,14 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ if (items().length === 0) throw new AIError('invalid_response', 'The model did not produce any usable items. Try again, or choose a different model for content generation.'); if (items().length < count) notes.push(`Generated ${items().length} of the ${count} requested.`); + // Check counts, the type mix, answer keys and calculations with Python before review. + if (draft.kind === 'quiz' && req.verify !== false) { + setProgress(db, draftId, 0.96, 'Checking the questions with Python', draft); + const checked = await verifyQuizDraft(app, draft, { count, style, sataShare: sata, difficulty }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, label) }); + draft.questions = checked.draft.questions; + notes.push(...checked.notes); + } + db.run( `UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`, draft.title, diff --git a/server/ai/sandbox.ts b/server/ai/sandbox.ts new file mode 100644 index 0000000..92fadfa --- /dev/null +++ b/server/ai/sandbox.ts @@ -0,0 +1,163 @@ +/** + * Sandboxed Python for the assistant and generation checks. + * + * Python runs in Pyodide (CPython compiled to WebAssembly) inside a separate Node process + * (`server/python-runner.mjs`) started with Node's permission model: it may read only + * Pyodide's own files and its own script, cannot write files, start processes, use workers + * or load native addons, and gets an empty environment. Inside Python the bridges back to + * JavaScript are removed, so there is no network or host file access, only the standard + * library (math, fractions, decimal, statistics, re, json, datetime, itertools...). + * + * One warm process serves runs one at a time. A run that exceeds its time limit kills the + * process; the next run starts a fresh one. The process is also recycled every so often so + * module-level state cannot pile up across runs. + */ +import { fork, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export interface PythonResult { + ok: boolean; + stdout: string; + stderr: string; + /** repr-like text of the last expression's value, when there is one. */ + result: string | null; + error: string | null; + ms: number; + timedOut?: boolean; +} + +export const PYTHON_TIMEOUT_MS = 10_000; +const MAX_TIMEOUT_MS = 30_000; +const START_TIMEOUT_MS = 60_000; +const RECYCLE_AFTER = 50; +const MAX_CODE = 50_000; + +function runnerPath(): string { + const here = path.dirname(fileURLToPath(import.meta.url)); + // Bundled: dist/server/main.js next to python-runner.mjs. Source: server/ai/sandbox.ts. + for (const candidate of [path.join(here, 'python-runner.mjs'), path.join(here, '..', 'python-runner.mjs')]) { + if (fs.existsSync(candidate)) return candidate; + } + throw new Error('python-runner.mjs is missing from this installation'); +} + +function pyodideEntry(): string { + const require = createRequire(import.meta.url); + return path.join(path.dirname(require.resolve('pyodide/package.json')), 'pyodide.mjs'); +} + +interface Pending { + resolve: (r: PythonResult) => void; + timer: ReturnType; +} + +let child: ChildProcess | null = null; +let ready: Promise | null = null; +let runs = 0; +let nextId = 1; +const pending = new Map(); +/** Runs are serialised: Pyodide is single-threaded and a shared process keeps memory flat. */ +let queue: Promise = Promise.resolve(); + +function kill(reason: string): void { + const c = child; + child = null; + ready = null; + runs = 0; + if (c && c.exitCode === null) c.kill('SIGKILL'); + for (const [id, p] of pending) { + clearTimeout(p.timer); + p.resolve({ ok: false, stdout: '', stderr: '', result: null, error: reason, ms: 0 }); + pending.delete(id); + } +} + +function start(): Promise { + if (ready) return ready; + ready = new Promise((resolve, reject) => { + const entry = pyodideEntry(); + const script = runnerPath(); + const proc = fork(script, [entry], { + // An empty environment: nothing secret to find even if the sandbox were escaped. + env: {}, + execArgv: ['--permission', `--allow-fs-read=${path.dirname(entry)}`, `--allow-fs-read=${script}`, '--max-old-space-size=256'], + stdio: ['ignore', 'ignore', 'pipe', 'ipc'], + serialization: 'json', + }); + let errText = ''; + proc.stderr?.on('data', (d) => { + if (errText.length < 4000) errText += String(d); + }); + const startTimer = setTimeout(() => { + proc.kill('SIGKILL'); + reject(new Error('The Python sandbox took too long to start.')); + }, START_TIMEOUT_MS); + let started = false; + proc.on('message', (msg: any) => { + if (msg?.ready) { + started = true; + clearTimeout(startTimer); + resolve(proc); + return; + } + const p = pending.get(msg?.id); + if (!p) return; + clearTimeout(p.timer); + pending.delete(msg.id); + const { id: _id, ...rest } = msg; + p.resolve(rest as PythonResult); + }); + proc.on('exit', (code) => { + clearTimeout(startTimer); + if (!started) reject(new Error(`The Python sandbox could not start (exit ${code}). ${errText.trim().split('\n').slice(-3).join(' ')}`.trim())); + if (child !== proc) return; // already replaced (killed after a timeout) + child = null; + ready = null; + for (const [id, p] of pending) { + clearTimeout(p.timer); + p.resolve({ ok: false, stdout: '', stderr: '', result: null, error: 'The Python sandbox stopped unexpectedly.', ms: 0 }); + pending.delete(id); + } + }); + child = proc; + }).catch((err) => { + child = null; + ready = null; + throw err; + }); + return ready; +} + +async function runOnce(code: string, timeoutMs: number): Promise { + const proc = await start(); + const id = nextId++; + runs += 1; + const result = await new Promise((resolve) => { + const timer = setTimeout(() => { + pending.delete(id); + resolve({ ok: false, stdout: '', stderr: '', result: null, error: `Stopped after ${Math.round(timeoutMs / 1000)} s (time limit). Avoid long loops; compute directly.`, ms: timeoutMs, timedOut: true }); + kill('The previous run timed out.'); + }, timeoutMs); + pending.set(id, { resolve, timer }); + proc.send({ id, code }); + }); + if (runs >= RECYCLE_AFTER) kill('recycled'); + return result; +} + +/** Runs Python 3 code in the sandbox. Never throws for problems in the code itself. */ +export function runPython(code: string, opts: { timeoutMs?: number } = {}): Promise { + if (code.length > MAX_CODE) return Promise.resolve({ ok: false, stdout: '', stderr: '', result: null, error: `The code is too long (limit ${MAX_CODE} characters).`, ms: 0 }); + const timeoutMs = Math.min(Math.max(opts.timeoutMs ?? PYTHON_TIMEOUT_MS, 1000), MAX_TIMEOUT_MS); + const job = queue.then(() => runOnce(code, timeoutMs)); + queue = job.catch(() => undefined); + return job; +} + +/** Stops the sandbox process (shutdown, tests). */ +export function stopPython(): void { + kill('The Python sandbox was stopped.'); +} diff --git a/server/ai/verify.ts b/server/ai/verify.ts new file mode 100644 index 0000000..7d35c52 --- /dev/null +++ b/server/ai/verify.ts @@ -0,0 +1,290 @@ +/** + * Python-backed checking of generated quizzes, and the `run_python` tool the assistant uses. + * + * After a quiz is generated, a short agent loop gives the model the draft (as Python data and + * as text) plus the original request and asks it to verify everything mechanically with the + * sandbox: question count, the select-all / single / numeric mix, options and correct answers + * per type, duplicates, and every calculation recomputed from its stem. It then submits fixes, + * which are applied here only if the question still validates afterwards. + */ +import { z } from 'zod'; +import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../../shared/content.ts'; +import type { AppContext } from '../http.ts'; +import { runPython } from './sandbox.ts'; +import { resolveModel, runChat } from './service.ts'; +import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts'; + +/* ------------------------------ data handed to Python ------------------------------ */ + +/** Compact, 1-based view of a quiz used both in prompts and as the `draft` variable in Python. */ +export function quizForPython(quiz: { title: string; questions: Pick[] }) { + return { + title: quiz.title, + question_count: quiz.questions.length, + questions: quiz.questions.map((q, i) => ({ + number: i + 1, + type: q.type, + topic: q.topic, + stem: q.stem, + options: q.options.map((o, k) => ({ number: k + 1, text: o.text, correct: Boolean(o.correct), why: o.why ?? '' })), + rationale: q.rationale ?? '', + })), + }; +} + +/** Python source that defines the given variables from JSON (a JSON string is a valid Python string literal). */ +export function pythonPrelude(vars: Record): string { + const lines = ['import json as _json']; + for (const [name, value] of Object.entries(vars)) { + if (!/^[a-z_][a-z0-9_]*$/i.test(name)) continue; + lines.push(`${name} = _json.loads(${JSON.stringify(JSON.stringify(value))})`); + } + return `${lines.join('\n')}\n`; +} + +/** Formats a sandbox result for a model: short, and never an exception. */ +export function pythonOutput(r: Awaited>) { + return { + ok: r.ok, + ...(r.stdout ? { stdout: r.stdout } : {}), + ...(r.result !== null ? { result: r.result } : {}), + ...(r.stderr ? { stderr: r.stderr.slice(0, 2000) } : {}), + ...(r.error ? { error: r.error } : {}), + }; +} + +export const RUN_PYTHON_DESCRIPTION = + 'Run Python 3 in a sandbox and get stdout and the value of the last expression. Standard library only (math, fractions, decimal, statistics, re, json, collections, itertools, datetime); no files, network or packages; 10 second limit. Use it whenever exactness matters: recompute dosage, rate and conversion answers, count questions, options and correct answers, check lengths, tally topics. Print what you need to see.'; + +export const RUN_PYTHON_TOOL: ToolDef = { + name: 'run_python', + description: RUN_PYTHON_DESCRIPTION, + parameters: { + type: 'object', + properties: { code: { type: 'string', description: 'Python source. Variables listed in the instructions are already defined.' } }, + required: ['code'], + }, +}; + +/* ------------------------------ verification of a generated quiz ------------------------------ */ + +export interface QuizRequestFacts { + count: number; + style: string; + sataShare: number; + difficulty?: string; +} + +/** Long text is shortened rather than rejected: a verbose but correct review must not be lost. */ +const text = (max: number) => z.coerce.string().transform((v) => v.slice(0, max)); +const int = z.coerce.number().int(); +const fixSchema = z.object({ + question: int.min(1), + action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'remove']), + correct_options: z.array(int.min(1)).max(12).optional(), + answer: text(200).optional(), + type: z.enum(['single', 'multi', 'numeric']).optional(), + reason: text(600).optional().default(''), +}); +const reviewSchema = z.object({ + summary: text(1500).optional().default(''), + checks: z + .array(z.object({ name: text(160), passed: z.coerce.boolean(), detail: text(600).optional().default('') })) + .transform((v) => v.slice(0, 40)) + .optional() + .default([]), + fixes: z.array(z.unknown()).optional().default([]).transform((list) => list.flatMap((f) => { const r = fixSchema.safeParse(f); return r.success ? [r.data] : []; })), +}); +export type QuizReview = z.infer; + +const SUBMIT_REVIEW: ToolDef = { + name: 'submit_review', + description: 'Finish the check: report every check you ran (with its result) and the fixes to apply. Call this exactly once, at the end.', + parameters: { + type: 'object', + properties: { + summary: { type: 'string', description: 'One or two sentences for the student.' }, + checks: { + type: 'array', + items: { type: 'object', properties: { name: { type: 'string' }, passed: { type: 'boolean' }, detail: { type: 'string' } }, required: ['name', 'passed'] }, + }, + fixes: { + type: 'array', + description: 'Only for definite errors. Question and option numbers are 1-based.', + items: { + type: 'object', + properties: { + question: { type: 'integer' }, + action: { + type: 'string', + enum: ['set_correct', 'set_numeric_answer', 'set_type', 'remove'], + description: 'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. remove: drop an unfixable question.', + }, + correct_options: { type: 'array', items: { type: 'integer' } }, + answer: { type: 'string' }, + type: { type: 'string', enum: ['single', 'multi', 'numeric'] }, + reason: { type: 'string' }, + }, + required: ['question', 'action', 'reason'], + }, + }, + }, + required: ['summary', 'checks', 'fixes'], + }, +}; + +function rulesFor(req: QuizRequestFacts): string { + const expectedMulti = req.sataShare <= 0 ? 'none' : req.sataShare >= 1 ? 'all non-numeric questions' : `about ${Math.round(req.sataShare * 100)}% of the non-numeric questions`; + return [ + `Requested: ${req.count} questions, style "${req.style}", difficulty "${req.difficulty ?? 'mixed'}", select-all (multi) share: ${expectedMulti}.`, + 'Rules each question must meet:', + '- single: exactly one correct option; 4 options for NCLEX/standard/recall styles.', + '- multi (select all that apply): at least 2 correct options; for NCLEX style 5 or 6 options with 2 to 5 correct and a stem ending with "Select all that apply."', + '- numeric: the student types a number; options holds the accepted answer(s), all marked correct. The answer must equal the value computed from the stem, rounded as the stem says.', + '- No duplicate or near-duplicate stems; no identical options within a question; no "all of the above" / "none of the above".', + '- The keyed answer must be the clinically correct one (check against the rationale and standard practice).', + ].join('\n'); +} + +const VERIFY_SYSTEM = `You are the quality checker for quiz questions written for a nursing student. Accuracy is critical: they are used to prepare for licensing exams. +Check the draft mechanically with the run_python tool. The variables \`draft\` (the quiz) and \`request\` (what was asked for) are already defined in every run. Write Python that counts and verifies instead of eyeballing: +1. The number of questions against request["count"]. +2. The mix of types (single / multi / numeric) against the requested select-all share. +3. For every question: option count, number of correct options, and that it fits its type's rules. +4. Duplicate stems or options. +5. Every numeric question: recompute the answer from the numbers in the stem with Python, apply the rounding the stem asks for, and compare with the keyed answer numerically (ignore units). +Also read each question and judge whether the keyed answer is clinically correct. +Then call submit_review once with every check you ran and a fix for each definite error. Do not "fix" things that are only stylistic. A count or mix that is off cannot be fixed here; report it as a failed check.`; + +/** Applies validated fixes; returns notes describing what changed or was refused. */ +export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer[]): { questions: DraftQuestion[]; applied: string[]; refused: string[] } { + const out = questions.map((q) => ({ ...q, options: q.options.map((o) => ({ ...o })) })); + const remove = new Set(); + const applied: string[] = []; + const refused: string[] = []; + for (const fix of fixes) { + const i = fix.question - 1; + const q = out[i]; + if (!q) { + refused.push(`Question ${fix.question}: does not exist.`); + continue; + } + const before = { ...q, options: q.options.map((o) => ({ ...o })) }; + const why = fix.reason ? ` (${fix.reason.replace(/\s+/g, ' ').slice(0, 200)})` : ''; + if (fix.action === 'remove') { + remove.add(i); + applied.push(`Question ${fix.question} removed${why}.`); + continue; + } + if (fix.action === 'set_correct' && fix.correct_options?.length) { + const set = new Set(fix.correct_options.map((n) => n - 1)); + q.options = q.options.map((o, k) => ({ ...o, correct: set.has(k) })); + if (q.type === 'single' && set.size > 1) q.type = 'multi'; + } else if (fix.action === 'set_numeric_answer' && fix.answer?.trim() && q.type === 'numeric') { + q.options = [{ text: fix.answer.trim(), correct: true, why: q.options[0]?.why ?? '' }]; + } else if (fix.action === 'set_type' && fix.type && fix.type !== 'numeric') { + q.type = fix.type; + } else { + refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} was incomplete.`); + continue; + } + if (validateDraftQuestion(q, i).some((p) => p.level === 'error')) { + out[i] = before; + refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} would have made it invalid, so it was not applied${why}.`); + continue; + } + applied.push(`Question ${fix.question}: ${fix.action === 'set_numeric_answer' ? `answer corrected to ${fix.answer!.trim()}` : fix.action === 'set_type' ? `type changed to ${fix.type}` : `answer key corrected`}${why}.`); + } + return { questions: out.filter((_, i) => !remove.has(i)), applied, refused }; +} + +/** + * Checks a generated quiz with the model and the Python sandbox, then applies the fixes. + * Never throws: a failed check leaves the draft as it was and says so in the notes. + */ +export async function verifyQuizDraft( + app: AppContext, + draft: QuizDraft, + req: QuizRequestFacts, + opts: { modelId?: string | null; signal?: AbortSignal; onStep?: (label: string) => void } = {}, +): Promise<{ draft: QuizDraft; notes: string[]; review: QuizReview | null; pythonRuns: number }> { + const resolved = resolveModel(app, 'generate', opts.modelId); + if (!resolved.model.capabilities.tools) { + return { draft, notes: ['The Python answer check was skipped: the generation model cannot use tools.'], review: null, pythonRuns: 0 }; + } + const data = quizForPython(draft); + const prelude = pythonPrelude({ draft: data, request: { count: req.count, style: req.style, sata_share: req.sataShare, difficulty: req.difficulty ?? 'mixed' } }); + const listing = data.questions + .map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${q.rationale.slice(0, 600)}` : ''}`) + .join('\n\n'); + const messages: ChatMessage[] = [{ role: 'user', content: [{ type: 'text', text: `${rulesFor(req)}\n\nTHE DRAFT (${data.question_count} questions):\n\n${listing}` }] }]; + let review: QuizReview | null = null; + let pythonRuns = 0; + // Up to WORK_STEPS turns with Python, then only submit_review is offered until it arrives. + const WORK_STEPS = 8; + const MAX_STEPS = 12; + try { + for (let step = 0; step < MAX_STEPS && !review; step += 1) { + if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.'); + const wrapUp = step >= WORK_STEPS || pythonRuns >= 16; + if (wrapUp && !messages.some((m) => m.content.some((p) => p.type === 'text' && p.text.startsWith('You have run enough checks')))) { + const nudge: ContentPart = { type: 'text', text: 'You have run enough checks. Call submit_review now with your results and fixes.' }; + const lastMsg = messages[messages.length - 1]; + // Ride along with the tool results so user and assistant turns keep alternating. + if (lastMsg?.role === 'user') lastMsg.content.push(nudge); + else messages.push({ role: 'user', content: [nudge] }); + } + const res = await runChat(app, { + task: 'generate', + purpose: 'quiz check', + modelId: opts.modelId, + system: VERIFY_SYSTEM, + messages, + tools: wrapUp ? [SUBMIT_REVIEW] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW], + toolChoice: wrapUp ? { name: 'submit_review' } : 'auto', + maxOutputTokens: 16_000, + signal: opts.signal, + }); + messages.push(res.message); + const calls = res.message.content.filter((p): p is Extract => p.type === 'tool_call'); + if (calls.length === 0) { + messages.push({ role: 'user', content: [{ type: 'text', text: 'Run your checks with run_python, then call submit_review.' }] }); + continue; + } + const results: ContentPart[] = []; + for (const call of calls) { + if (call.name === 'submit_review') { + const parsed = reviewSchema.safeParse(call.input); + if (parsed.success) review = parsed.data; + results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: parsed.success ? '{"ok":true}' : JSON.stringify({ error: parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') }), isError: !parsed.success }); + } else if (call.name === 'run_python' && pythonRuns >= 16) { + results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"Python budget used up. Call submit_review now."}', isError: true }); + } else if (call.name === 'run_python') { + pythonRuns += 1; + opts.onStep?.(`Checking the questions with Python (run ${pythonRuns})`); + const r = await runPython(prelude + String(call.input.code ?? '')); + results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify(pythonOutput(r)), isError: !r.ok }); + } else { + results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"unknown tool"}', isError: true }); + } + } + // The last user message carries the tool results; keep wrap-up instructions in front of the model. + messages.push({ role: 'user', content: results }); + } + } catch (err) { + if (err instanceof AIError && err.code === 'aborted') throw err; + const message = err instanceof Error ? err.message : String(err); + return { draft, notes: [`The Python answer check could not finish (${message.slice(0, 200)}). Review the answers yourself.`], review: null, pythonRuns }; + } + if (!review) return { draft, notes: ['The Python answer check did not report back. Review the answers yourself.'], review: null, pythonRuns }; + + const { questions, applied, refused } = applyQuizFixes(draft.questions, review.fixes); + const failed = review.checks.filter((c) => !c.passed); + const notes = [ + `Checked with Python (${pythonRuns} run${pythonRuns === 1 ? '' : 's'}): ${review.checks.length - failed.length} of ${review.checks.length} checks passed${applied.length ? `, ${applied.length} fix${applied.length === 1 ? '' : 'es'} applied` : ''}.`, + ...failed.map((c) => `Check failed: ${c.name}${c.detail ? `: ${c.detail}` : ''}`), + ...applied, + ...refused, + ]; + return { draft: { ...draft, questions }, notes, review, pythonRuns }; +} diff --git a/server/main.ts b/server/main.ts index a1928b8..85bcfd8 100644 --- a/server/main.ts +++ b/server/main.ts @@ -10,6 +10,7 @@ import { startBackupSchedule } from './services/backup.ts'; import { resumeQueue } from './services/documents.ts'; import { scheduleIndexing } from './services/semantic.ts'; import { seedTemplates } from './services/templates.ts'; +import { stopPython } from './ai/sandbox.ts'; function log(level: 'info' | 'warn' | 'error', message: string, extra?: unknown): void { const line = `${new Date().toISOString()} ${level.toUpperCase().padEnd(5)} ${message}`; @@ -46,6 +47,7 @@ async function main(): Promise { let closing = false; const shutdown = (signal: string) => { + stopPython(); if (closing) return; closing = true; log('info', `${signal} received, shutting down`); diff --git a/server/python-runner.mjs b/server/python-runner.mjs new file mode 100644 index 0000000..a51e896 --- /dev/null +++ b/server/python-runner.mjs @@ -0,0 +1,96 @@ +/** + * Sandboxed Python runner (Pyodide / WebAssembly). Started by server/ai/sandbox.ts as a + * separate Node process with the permission model on: + * --permission --allow-fs-read= --allow-fs-read= + * so even code that escaped the WebAssembly sandbox could not read the data folder, write + * anything, start processes or load native addons. It also gets an empty environment, so + * there are no secrets in `process.env`. + * + * Inside Python, `js` is an empty object (jsglobals: {}) and the bridges back to JavaScript + * (pyodide_js, pyodide.http, micropip) are removed, so code cannot reach fetch, require or + * the host file system. Only the standard library is available. + * + * Protocol (IPC): { id, code, timeoutMs } -> { id, ok, stdout, stderr, result, error, ms }. + * The parent kills this process when a run exceeds its time limit. + */ +import { constants as fsConstants } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +// Emscripten's NODEFS reads open-flag constants through process.binding('constants'), which +// the permission model forbids. Answer that one call from the public fs.constants instead. +process.binding = (name) => { + if (name === 'constants') return { fs: fsConstants }; + throw new Error(`process.binding('${name}') is not available in the Python sandbox`); +}; + +const pyodideEntry = process.argv[2]; +const OUTPUT_LIMIT = 20_000; + +const { loadPyodide } = await import(pathToFileURL(pyodideEntry).href); + +let stdout = ''; +let stderr = ''; +const append = (which, text) => { + if (which === 'out') { + if (stdout.length < OUTPUT_LIMIT) stdout += `${text}\n`; + } else if (stderr.length < OUTPUT_LIMIT) stderr += `${text}\n`; +}; + +const pyodide = await loadPyodide({ + jsglobals: {}, + stdout: (t) => append('out', t), + stderr: (t) => append('err', t), + stdin: () => null, +}); + +pyodide.runPython(` +import sys, importlib.abc + +_BLOCKED = ('js', 'pyodide_js', 'micropip', 'pyodide.http', 'pyodide.webloop', 'pyodide_http') +for _name in list(sys.modules): + if _name in _BLOCKED or _name.split('.')[0] in ('js', 'pyodide_js', 'micropip'): + del sys.modules[_name] + +class _Sandbox(importlib.abc.MetaPathFinder): + blocked = _BLOCKED + roots = ('js', 'pyodide_js', 'micropip') + + def find_spec(self, fullname, path=None, target=None): + if fullname in self.blocked or fullname.split('.')[0] in self.roots: + raise ImportError(f"{fullname} is not available in this sandbox (no network, files or JavaScript)") + return None + +sys.meta_path.insert(0, _Sandbox()) +del _Sandbox, _BLOCKED +`); + +/** Runs one snippet in a fresh namespace. The value of a final expression is returned as `result`. */ +async function run(code) { + stdout = ''; + stderr = ''; + const ns = pyodide.globals.get('dict')(); + const started = Date.now(); + try { + const value = await pyodide.runPythonAsync(code, { globals: ns }); + let result = null; + if (value !== undefined && value !== null) { + result = typeof value === 'object' && typeof value.toString === 'function' ? String(value) : String(value); + if (typeof value?.destroy === 'function') value.destroy(); + } + return { ok: true, stdout: stdout.slice(0, OUTPUT_LIMIT), stderr: stderr.slice(0, OUTPUT_LIMIT), result: result && result.slice(0, OUTPUT_LIMIT), error: null, ms: Date.now() - started }; + } catch (err) { + const message = String(err?.message ?? err); + // Keep the Python traceback, drop the Pyodide/JavaScript frames above it. + const tb = message.includes('Traceback') ? message.slice(message.indexOf('Traceback')) : message; + return { ok: false, stdout: stdout.slice(0, OUTPUT_LIMIT), stderr: stderr.slice(0, OUTPUT_LIMIT), result: null, error: tb.slice(-4000), ms: Date.now() - started }; + } finally { + ns.destroy(); + } +} + +process.on('message', async (msg) => { + if (!msg || typeof msg.code !== 'string') return; + const reply = await run(msg.code); + process.send?.({ id: msg.id, ...reply }); +}); +process.send?.({ ready: true }); diff --git a/tests/ai.test.ts b/tests/ai.test.ts index c3beb6c..ca8796f 100644 --- a/tests/ai.test.ts +++ b/tests/ai.test.ts @@ -50,6 +50,9 @@ async function sse(url: string, body: unknown): Promise { .map((d) => JSON.parse(d)); } +/** A request from the Python quiz check rather than from generation itself. */ +const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review'); + async function waitForDraft(id: string) { for (let i = 0; i < 200; i += 1) { const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft; @@ -195,8 +198,16 @@ describe('generation', () => { // Citations were mapped from excerpt ids to readable labels. expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/); // 12 questions at 8 per batch means two model calls, each carrying the source excerpts. - const calls = mock.requests.filter((r) => r.path.endsWith('/chat/completions')); + const all = mock.requests.filter((r) => r.path.endsWith('/chat/completions')); + const calls = all.filter((r) => !isCheck(r)); expect(calls).toHaveLength(2); + // Then the Python check: one run, a review, and its fix applied to question 2. + const checks = all.filter(isCheck); + expect(checks.length).toBeGreaterThanOrEqual(2); + const pythonResult = JSON.parse(checks[1].body.messages.find((m: any) => m.role === 'tool').content); + expect(pythonResult).toMatchObject({ ok: true, stdout: '12 12 4 [1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2]\n' }); + expect(draft.payload.questions[1].options.map((o: any) => o.correct)).toEqual([false, true, false, false]); + expect(draft.meta.notes.join(' ')).toMatch(/Checked with Python \(1 run\): 2 of 2 checks passed, 1 fix applied/); expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/); expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/); expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } }); @@ -237,7 +248,7 @@ describe('generation', () => { // Deck -> quiz conversion uses the cards as the source. const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId); expect(converted.status).toBe('ready'); - expect(JSON.stringify(mock.requests.at(-1)!.body.messages)).toMatch(/Flashcards from the deck/); + expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/); // Append more cards to the existing deck. const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId); @@ -265,6 +276,7 @@ describe('generation', () => { const draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId); expect(draft.status).toBe('ready'); expect(draft.payload.questions).toHaveLength(3); + expect(draft.meta.notes.join(' ')).toMatch(/check was skipped/); const call = mock.requests.at(-1)!; expect(call.body.tools).toBeUndefined(); expect(call.body.response_format).toEqual({ type: 'json_object' }); diff --git a/tests/mockAi.ts b/tests/mockAi.ts index 220464b..3e5e745 100644 --- a/tests/mockAi.ts +++ b/tests/mockAi.ts @@ -84,6 +84,13 @@ function toolArgs(name: string, userText: string, all: string): Record w.length > 3).slice(0, 6).join(' ') }; + case 'run_python': + return { code: 'qs = draft["questions"]\nprint(len(qs), request["count"], sum(1 for q in qs if q["type"] == "multi"), [sum(o["correct"] for o in q["options"]) for q in qs])' }; + case 'submit_review': { + // Second question (if any) gets a "corrected" key so tests can see fixes applied. + const fix = /Q2 \[single\]/.test(all) ? [{ question: 2, action: 'set_correct', correct_options: [2], reason: 'mock: option 2 is the right answer' }] : []; + return { summary: 'Checked by the mock.', checks: [{ name: 'Question count', passed: true, detail: 'matches the request' }, { name: 'Correct answers per type', passed: true }], fixes: fix }; + } case 'get_performance': case 'list_classes': case 'list_materials': @@ -187,6 +194,11 @@ export function startMockAi(port = 0): Promise { } const forced = body?.tool_choice?.function?.name as string | undefined; + // The quiz checker: run Python once, then submit the review. + if (names.includes('submit_review')) { + const ranPython = messages.some((m) => m.role === 'tool'); + return callTool(forced === 'submit_review' || ranPython ? 'submit_review' : 'run_python'); + } if (forced) return callTool(forced); if (last.role === 'tool') { const ref = /"ref":"(c\d+)"/.exec(text(last.content))?.[1]; diff --git a/tests/python.test.ts b/tests/python.test.ts new file mode 100644 index 0000000..e3e18bb --- /dev/null +++ b/tests/python.test.ts @@ -0,0 +1,125 @@ +/** + * The Python sandbox (Pyodide in a permission-restricted Node process), the assistant's + * run_python tool and the fixes the quiz check may apply. + */ +import { afterAll, afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { DraftQuestion } from '../shared/content.ts'; +import { runToolForTest } from '../server/ai/assistant.ts'; +import { createDraft } from '../server/ai/generate.ts'; +import { runPython, stopPython } from '../server/ai/sandbox.ts'; +import { applyQuizFixes, pythonPrelude } from '../server/ai/verify.ts'; +import { createTestApp, type TestApp } from './helpers.ts'; + +afterAll(() => stopPython()); + +describe('python sandbox', () => { + it('runs standard-library code and returns output and the last value', async () => { + const r = await runPython('from fractions import Fraction\nimport statistics\nprint(5000 / 10000)\nround(73 * 1.2, 1)'); + expect(r).toMatchObject({ ok: true, stdout: '0.5\n', result: '87.6' }); + expect((await runPython('statistics.mean([1, 2])')).error).toMatch(/NameError/); // each run starts clean + }, 60_000); + + it('reports Python errors without the JavaScript frames', async () => { + const r = await runPython('1 / 0'); + expect(r.ok).toBe(false); + expect(r.error).toMatch(/^Traceback/); + expect(r.error).toMatch(/ZeroDivisionError/); + }, 60_000); + + it('has no way to the host: no JavaScript bridge, files, environment, processes or network', async () => { + for (const code of ['import js', 'import pyodide_js', 'from pyodide.code import run_js\nrun_js("process.env")', 'import micropip']) { + const r = await runPython(code); + expect(r.ok, code).toBe(false); + expect(r.error, code).toMatch(/not available in this sandbox/); + } + expect((await runPython('open("/etc/passwd").read()')).error).toMatch(/FileNotFoundError/); + expect((await runPython(`open(${JSON.stringify(process.cwd() + '/package.json')}).read()`)).error).toMatch(/FileNotFoundError/); + expect((await runPython('import os\nsorted(os.environ)')).result).not.toMatch(/APP_SECRET|PATH.*Windows|DATA_DIR/); + expect((await runPython('import subprocess\nsubprocess.run(["ls"])')).error).toMatch(/does not support processes/); + expect((await runPython('import urllib.request\nurllib.request.urlopen("http://127.0.0.1:1")')).ok).toBe(false); + }, 60_000); + + it('stops runaway code at the time limit and recovers for the next run', async () => { + const r = await runPython('while True:\n pass', { timeoutMs: 1500 }); + expect(r).toMatchObject({ ok: false, timedOut: true }); + expect(r.error).toMatch(/time limit/); + expect((await runPython('sum(range(10))')).result).toBe('45'); + }, 60_000); + + it('serialises concurrent runs', async () => { + const results = await Promise.all([1, 2, 3, 4].map((n) => runPython(`${n} * 10`))); + expect(results.map((r) => r.result)).toEqual(['10', '20', '30', '40']); + }, 60_000); + + it('hands data to Python through JSON', async () => { + const r = await runPython(`${pythonPrelude({ draft: { questions: [{ stem: 'Quote "this" \\ and ü' }] } })}print(draft["questions"][0]["stem"])`); + expect(r.stdout).toBe('Quote "this" \\ and ü\n'); + }, 60_000); +}); + +describe('quiz check fixes', () => { + const q = (type: DraftQuestion['type'], correct: number[], n = 4): DraftQuestion => ({ + type, + stem: `Stem ${type}`, + options: Array.from({ length: n }, (_, i) => ({ text: `Option ${i + 1}`, correct: correct.includes(i), why: '' })), + rationale: 'r', + topic: '', + difficulty: 'medium', + tags: [], + }); + + it('applies valid fixes and refuses ones that would break a question', () => { + const questions = [q('single', [0, 1, 2, 3]), q('numeric', [0], 1), q('multi', [0, 1], 5), q('single', [0])]; + questions[1].options[0].text = '12 mL'; + const { questions: out, applied, refused } = applyQuizFixes(questions, [ + { question: 1, action: 'set_correct', correct_options: [2], reason: 'B is right' }, + { question: 2, action: 'set_numeric_answer', answer: '12.5 mL', reason: 'recomputed' }, + { question: 3, action: 'set_correct', correct_options: [], reason: 'empty' }, + { question: 4, action: 'set_numeric_answer', answer: '3', reason: 'not numeric' }, + { question: 9, action: 'remove', reason: 'missing' }, + ]); + expect(out[0].options.map((o) => o.correct)).toEqual([false, true, false, false]); + expect(out[1].options).toEqual([{ text: '12.5 mL', correct: true, why: '' }]); + expect(out[2].options.filter((o) => o.correct)).toHaveLength(2); + expect(applied).toHaveLength(2); + expect(refused).toHaveLength(3); + }); + + it('removes questions and turns a several-answer single into select-all', () => { + const { questions: out } = applyQuizFixes([q('single', [0]), q('single', [0])], [ + { question: 1, action: 'remove', reason: 'duplicate' }, + { question: 2, action: 'set_correct', correct_options: [1, 3], reason: 'two are right' }, + ]); + expect(out).toHaveLength(1); + expect(out[0].type).toBe('multi'); + }); +}); + +describe('assistant run_python tool', () => { + let t: TestApp; + beforeEach(async () => { + t = await createTestApp(); + }); + afterEach(() => t.close()); + + it('loads a draft or a saved quiz as Python data', async () => { + const draftId = createDraft(t.ctx.db, { + kind: 'quiz', + title: 'Calc', + payload: { kind: 'quiz', title: 'Calc', description: '', questions: [{ type: 'numeric', stem: '500 mL over 4 h?', options: [{ text: '125 mL/hr', correct: true }], rationale: '', topic: '', difficulty: 'medium', tags: [] }] }, + meta: { request: { count: 1 } }, + }); + const run = await runToolForTest(t.ctx, 'run_python', { draft_id: draftId, code: 'q = draft["questions"][0]\nprint(len(draft["questions"]), q["type"], 500 / 4 == float(q["options"][0]["text"].split()[0]), draft["request"]["count"])' }); + expect(run.output).toMatchObject({ ok: true, stdout: '1 numeric True 1\n' }); + expect(run.summary).toBe('1 numeric True 1'); + + const cls = (await t.api('POST', '/api/classes', { name: 'C' })).body.class; + const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Q', questions: [{ type: 'multi', stem: 'Pick', options: [{ text: 'a', correct: true }, { text: 'b', correct: true }, { text: 'c', correct: false }] }] })).body.quiz; + const counted = await runToolForTest(t.ctx, 'run_python', { quiz_id: quiz.id, code: '[sum(o["correct"] for o in q["options"]) for q in quiz["questions"]]' }); + expect(counted.output).toMatchObject({ ok: true, result: '[2]' }); + + const failing = await runToolForTest(t.ctx, 'run_python', { code: 'undefined_name' }); + expect(failing.output).toMatchObject({ ok: false }); + expect(failing.summary).toMatch(/NameError/); + }, 60_000); +}); diff --git a/web/src/features/GenerateDialog.tsx b/web/src/features/GenerateDialog.tsx index c26b877..8e97f01 100644 --- a/web/src/features/GenerateDialog.tsx +++ b/web/src/features/GenerateDialog.tsx @@ -6,8 +6,8 @@ import { toast } from 'sonner'; import { api, errorMessage } from '@/lib/api'; import { qk, useAiModels, useAiStatus, useClass } from '@/lib/queries'; import type { DocumentItem, Template } from '@/lib/types'; -import { cx, formatTokens } from '@/lib/util'; -import { Button, Callout, Dialog, Field, Input, Segmented, Select, Textarea } from '@/ui'; +import { cx, formatTokens, useLocalStorage } from '@/lib/util'; +import { Button, Callout, Dialog, Field, Input, Segmented, Select, Switch, Textarea } from '@/ui'; export interface GeneratePreset { kind?: 'quiz' | 'deck' | 'note'; @@ -44,6 +44,7 @@ export function GenerateDialog({ open, onClose, classId, folderId, preset }: { o const [instructions, setInstructions] = useState(''); const [modelId, setModelId] = useState(''); const [templateId, setTemplateId] = useState(''); + const [verify, setVerify] = useLocalStorage('sd.generate.verify', true); const [busy, setBusy] = useState(false); const docs = useMemo(() => (cls.data?.items.filter((i): i is DocumentItem => i.kind === 'document') ?? []).filter((d) => d.status === 'ready' && d.tokenCount > 0), [cls.data]); @@ -103,7 +104,7 @@ export function GenerateDialog({ open, onClose, classId, folderId, preset }: { o else if (mode === 'text') source.text = text; else if (mode === 'weak') source.weakAreas = true; const body: Record = { kind, classId, folderId: folderId ?? null, source, focus: focus.trim() || undefined, instructions: instructions.trim() || undefined, modelId: modelId || undefined }; - if (kind === 'quiz') Object.assign(body, { count, style, difficulty, sataShare: sata / 100 }); + if (kind === 'quiz') Object.assign(body, { count, style, difficulty, sataShare: sata / 100, verify }); else if (kind === 'deck') Object.assign(body, { count, cardStyle }); else Object.assign(body, { noteStyle }); if (fixedSource && kind !== 'note') delete body.count; @@ -264,6 +265,9 @@ export function GenerateDialog({ open, onClose, classId, folderId, preset }: { o setSata(Number(e.target.value))} className="mt-2.5 w-full accent-[var(--accent)]" /> +
+ +
)} {kind === 'deck' && (