Add sandboxed Python for the assistant and a Python check of generated quizzes
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 51s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 51s
- Pyodide (Python 3.14, WebAssembly) in a separate Node process under the permission model: read-only access to Pyodide's files, no writes, processes or network, empty environment, JS bridges removed, time limit with kill and restart. - Assistant tool run_python, which can load a draft or quiz as Python data. - After quiz generation the model checks question count, type mix, correct answers per type, duplicates and recomputed calculations with Python, then submits fixes that are applied only if the question still validates. Switch in the Generate dialog. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f0aabe4d5d
commit
5ea61c882c
13 changed files with 800 additions and 6 deletions
41
package-lock.json
generated
41
package-lock.json
generated
|
|
@ -22,6 +22,7 @@
|
||||||
"lucide-react": "^1.71.0",
|
"lucide-react": "^1.71.0",
|
||||||
"mammoth": "^1.13.0",
|
"mammoth": "^1.13.0",
|
||||||
"pdf-lib": "^1.17.1",
|
"pdf-lib": "^1.17.1",
|
||||||
|
"pyodide": "^314.0.7",
|
||||||
"radix-ui": "^1.8.1",
|
"radix-ui": "^1.8.1",
|
||||||
"react": "^19.3.1",
|
"react": "^19.3.1",
|
||||||
"react-dom": "^19.3.1",
|
"react-dom": "^19.3.1",
|
||||||
|
|
@ -3510,6 +3511,12 @@
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/emscripten": {
|
||||||
|
"version": "1.41.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.6.tgz",
|
||||||
|
"integrity": "sha512-uN+9i8bFT5CUcZfyIEYDrSueACEyKGbUs5kC/72DGlZZoinh84sJfVV0i8UOJD1asdzkvLPBRrKs41kZ8MdEXg==",
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/@types/estree": {
|
"node_modules/@types/estree": {
|
||||||
"version": "1.0.10",
|
"version": "1.0.10",
|
||||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz",
|
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz",
|
||||||
|
|
@ -6398,6 +6405,19 @@
|
||||||
"node": ">=12.0.0"
|
"node": ">=12.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/pyodide": {
|
||||||
|
"version": "314.0.7",
|
||||||
|
"resolved": "https://registry.npmjs.org/pyodide/-/pyodide-314.0.7.tgz",
|
||||||
|
"integrity": "sha512-0YvXxEhfEdpLfb/XkM2BFAeMROq0iMUX2bzzH9pOttyMcWkwq+HbE5uyuGD82LN7y2q+SNvi/6V5JEsOlD2R1A==",
|
||||||
|
"license": "MPL-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@types/emscripten": "^1.41.4",
|
||||||
|
"ws": "^8.5.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/radix-ui": {
|
"node_modules/radix-ui": {
|
||||||
"version": "1.8.1",
|
"version": "1.8.1",
|
||||||
"resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.8.1.tgz",
|
"resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.8.1.tgz",
|
||||||
|
|
@ -7554,6 +7574,27 @@
|
||||||
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
|
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/ws": {
|
||||||
|
"version": "8.23.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/ws/-/ws-8.23.0.tgz",
|
||||||
|
"integrity": "sha512-wCkmX8QMOrKjgO8HW2WyDbTSGeqkocxgDzS9iR79rEAexty6/2qzilx3Ag8Tqf9ykop3irLKle0MtVn5qjVnoQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10.0.0"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"bufferutil": "^4.0.1",
|
||||||
|
"utf-8-validate": ">=5.0.2"
|
||||||
|
},
|
||||||
|
"peerDependenciesMeta": {
|
||||||
|
"bufferutil": {
|
||||||
|
"optional": true
|
||||||
|
},
|
||||||
|
"utf-8-validate": {
|
||||||
|
"optional": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/xml-naming": {
|
"node_modules/xml-naming": {
|
||||||
"version": "0.3.0",
|
"version": "0.3.0",
|
||||||
"resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz",
|
"resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz",
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@
|
||||||
"lucide-react": "^1.71.0",
|
"lucide-react": "^1.71.0",
|
||||||
"mammoth": "^1.13.0",
|
"mammoth": "^1.13.0",
|
||||||
"pdf-lib": "^1.17.1",
|
"pdf-lib": "^1.17.1",
|
||||||
|
"pyodide": "^314.0.7",
|
||||||
"radix-ui": "^1.8.1",
|
"radix-ui": "^1.8.1",
|
||||||
"react": "^19.3.1",
|
"react": "^19.3.1",
|
||||||
"react-dom": "^19.3.1",
|
"react-dom": "^19.3.1",
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
// Bundles the server into one file. Dependencies stay external and are installed in the image.
|
// Bundles the server into one file. Dependencies stay external and are installed in the image.
|
||||||
|
import fs from 'node:fs';
|
||||||
import { build } from 'esbuild';
|
import { build } from 'esbuild';
|
||||||
|
|
||||||
await build({
|
await build({
|
||||||
|
|
@ -12,3 +13,6 @@ await build({
|
||||||
sourcemap: true,
|
sourcemap: true,
|
||||||
logLevel: 'info',
|
logLevel: 'info',
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The Python sandbox runs as its own (permission-restricted) process, so it ships as a file.
|
||||||
|
fs.copyFileSync('server/python-runner.mjs', 'dist/server/python-runner.mjs');
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,9 @@ import { dueCounts } from '../services/srs.ts';
|
||||||
import { topicStats, totals } from '../services/stats.ts';
|
import { topicStats, totals } from '../services/stats.ts';
|
||||||
import { newId, now, parseJson } from '../util.ts';
|
import { newId, now, parseJson } from '../util.ts';
|
||||||
import { cancelGeneration, createDraft, getDraft, startGeneration, startImprove } from './generate.ts';
|
import { cancelGeneration, createDraft, getDraft, startGeneration, startImprove } from './generate.ts';
|
||||||
|
import { runPython } from './sandbox.ts';
|
||||||
import { resolveModel, runChat } from './service.ts';
|
import { resolveModel, runChat } from './service.ts';
|
||||||
|
import { pythonOutput, pythonPrelude, quizForPython, RUN_PYTHON_DESCRIPTION } from './verify.ts';
|
||||||
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
|
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
|
||||||
|
|
||||||
/* ------------------------------ UI event model ------------------------------ */
|
/* ------------------------------ UI event model ------------------------------ */
|
||||||
|
|
@ -605,6 +607,36 @@ const TOOLS: ToolSpec[] = [
|
||||||
return { output: { folder_id: folder.id, name: folder.name }, summary: folder.name };
|
return { output: { folder_id: folder.id, name: folder.name }, summary: folder.name };
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
def: {
|
||||||
|
name: 'run_python',
|
||||||
|
description: `${RUN_PYTHON_DESCRIPTION} Pass draft_id or quiz_id to load that draft or quiz as the variable \`draft\` / \`quiz\` (dict with "questions": [{"number", "type", "stem", "options": [{"number", "text", "correct"}], "rationale"}]); decks load as {"cards": [{"front", "back"}]}.`,
|
||||||
|
parameters: obj(
|
||||||
|
{
|
||||||
|
code: { ...S, description: 'Python 3 source. Print the results you need.' },
|
||||||
|
draft_id: { ...S, description: 'Optional: load this draft as `draft`.' },
|
||||||
|
quiz_id: { ...S, description: 'Optional: load this saved quiz as `quiz`.' },
|
||||||
|
},
|
||||||
|
['code'],
|
||||||
|
),
|
||||||
|
},
|
||||||
|
label: () => 'Running Python',
|
||||||
|
run: async (input, env) => {
|
||||||
|
const { db } = env.app;
|
||||||
|
const vars: Record<string, unknown> = {};
|
||||||
|
if (typeof input.draft_id === 'string' && input.draft_id) {
|
||||||
|
const d = getDraft(db, input.draft_id);
|
||||||
|
vars.draft = d.payload.kind === 'quiz' ? { ...quizForPython(d.payload), request: d.meta.request ?? {} } : d.payload.kind === 'deck' ? { title: d.payload.title, cards: d.payload.cards.map((c) => ({ front: c.front, back: c.back })) } : { title: d.payload.title, content: d.payload.content };
|
||||||
|
}
|
||||||
|
if (typeof input.quiz_id === 'string' && input.quiz_id) {
|
||||||
|
const quiz = getQuiz(db, input.quiz_id);
|
||||||
|
vars.quiz = quizForPython({ title: quiz.title, questions: listQuestions(db, quiz.id).map((q) => ({ ...q, options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why })) })) });
|
||||||
|
}
|
||||||
|
const r = await runPython((Object.keys(vars).length ? pythonPrelude(vars) : '') + String(input.code ?? ''));
|
||||||
|
const firstLine = (r.error ?? '').trim().split('\n').pop() ?? '';
|
||||||
|
return { output: pythonOutput(r), summary: r.ok ? (r.stdout.trim().split('\n').pop() ?? r.result ?? 'done').slice(0, 120) : firstLine.slice(0, 160) };
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
def: {
|
def: {
|
||||||
name: 'propose_changes',
|
name: 'propose_changes',
|
||||||
|
|
@ -896,7 +928,8 @@ function systemPrompt(app: AppContext, context: AssistantContext): string {
|
||||||
- For "what should I study" questions, call get_performance and give specific, prioritised advice.
|
- For "what should I study" questions, call get_performance and give specific, prioritised advice.
|
||||||
- When a request is clear, act on it without asking for permission. Ask one short question only when a missing detail would change the result (for example which class, when it is ambiguous).
|
- When a request is clear, act on it without asking for permission. Ask one short question only when a missing detail would change the result (for example which class, when it is ambiguous).
|
||||||
- Be concise and concrete. Use Markdown. Teach like a good clinical instructor: explain the why, connect to patient care, and prefer current standard practice.
|
- Be concise and concrete. Use Markdown. Teach like a good clinical instructor: explain the why, connect to patient care, and prefer current standard practice.
|
||||||
- After a tool produces a draft or a session link, reply with one or two sentences; the card already shows the details.`,
|
- After a tool produces a draft or a session link, reply with one or two sentences; the card already shows the details.
|
||||||
|
- Use run_python whenever exactness matters: dosage and rate calculations, unit conversions, counting questions, options or correct answers, checking that a quiz meets what was asked for. Quizzes from generate_quiz are already checked this way; when you write questions yourself with create_draft, check the draft with run_python (pass its draft_id) before telling the user it is ready, and fix problems you find.`,
|
||||||
);
|
);
|
||||||
return lines.join('\n\n');
|
return lines.join('\n\n');
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ import { weakTopics } from '../services/stats.ts';
|
||||||
import { getSetting, setSetting } from '../settings.ts';
|
import { getSetting, setSetting } from '../settings.ts';
|
||||||
import { newId, now, parseJson } from '../util.ts';
|
import { newId, now, parseJson } from '../util.ts';
|
||||||
import { generateStructured, resolveModel, runChat } from './service.ts';
|
import { generateStructured, resolveModel, runChat } from './service.ts';
|
||||||
|
import { verifyQuizDraft } from './verify.ts';
|
||||||
import { AIError } from './types.ts';
|
import { AIError } from './types.ts';
|
||||||
|
|
||||||
/* ------------------------------ drafts ------------------------------ */
|
/* ------------------------------ drafts ------------------------------ */
|
||||||
|
|
@ -285,6 +286,8 @@ export const generateRequestSchema = z.object({
|
||||||
cardStyle: z.enum(['concise', 'detailed', 'medication', 'definition']).optional(),
|
cardStyle: z.enum(['concise', 'detailed', 'medication', 'definition']).optional(),
|
||||||
noteStyle: z.enum(['study_guide', 'summary', 'outline', 'cheat_sheet']).optional(),
|
noteStyle: z.enum(['study_guide', 'summary', 'outline', 'cheat_sheet']).optional(),
|
||||||
modelId: z.string().nullish(),
|
modelId: z.string().nullish(),
|
||||||
|
/** Quizzes: check the finished draft with the model and the Python sandbox (default on). */
|
||||||
|
verify: z.boolean().optional(),
|
||||||
});
|
});
|
||||||
export type GenerateRequest = z.infer<typeof generateRequestSchema>;
|
export type GenerateRequest = z.infer<typeof generateRequestSchema>;
|
||||||
|
|
||||||
|
|
@ -757,6 +760,14 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
|
||||||
if (items().length === 0) throw new AIError('invalid_response', 'The model did not produce any usable items. Try again, or choose a different model for content generation.');
|
if (items().length === 0) throw new AIError('invalid_response', 'The model did not produce any usable items. Try again, or choose a different model for content generation.');
|
||||||
if (items().length < count) notes.push(`Generated ${items().length} of the ${count} requested.`);
|
if (items().length < count) notes.push(`Generated ${items().length} of the ${count} requested.`);
|
||||||
|
|
||||||
|
// Check counts, the type mix, answer keys and calculations with Python before review.
|
||||||
|
if (draft.kind === 'quiz' && req.verify !== false) {
|
||||||
|
setProgress(db, draftId, 0.96, 'Checking the questions with Python', draft);
|
||||||
|
const checked = await verifyQuizDraft(app, draft, { count, style, sataShare: sata, difficulty }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, label) });
|
||||||
|
draft.questions = checked.draft.questions;
|
||||||
|
notes.push(...checked.notes);
|
||||||
|
}
|
||||||
|
|
||||||
db.run(
|
db.run(
|
||||||
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
|
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
|
||||||
draft.title,
|
draft.title,
|
||||||
|
|
|
||||||
163
server/ai/sandbox.ts
Normal file
163
server/ai/sandbox.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
||||||
|
/**
|
||||||
|
* Sandboxed Python for the assistant and generation checks.
|
||||||
|
*
|
||||||
|
* Python runs in Pyodide (CPython compiled to WebAssembly) inside a separate Node process
|
||||||
|
* (`server/python-runner.mjs`) started with Node's permission model: it may read only
|
||||||
|
* Pyodide's own files and its own script, cannot write files, start processes, use workers
|
||||||
|
* or load native addons, and gets an empty environment. Inside Python the bridges back to
|
||||||
|
* JavaScript are removed, so there is no network or host file access, only the standard
|
||||||
|
* library (math, fractions, decimal, statistics, re, json, datetime, itertools...).
|
||||||
|
*
|
||||||
|
* One warm process serves runs one at a time. A run that exceeds its time limit kills the
|
||||||
|
* process; the next run starts a fresh one. The process is also recycled every so often so
|
||||||
|
* module-level state cannot pile up across runs.
|
||||||
|
*/
|
||||||
|
import { fork, type ChildProcess } from 'node:child_process';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
export interface PythonResult {
|
||||||
|
ok: boolean;
|
||||||
|
stdout: string;
|
||||||
|
stderr: string;
|
||||||
|
/** repr-like text of the last expression's value, when there is one. */
|
||||||
|
result: string | null;
|
||||||
|
error: string | null;
|
||||||
|
ms: number;
|
||||||
|
timedOut?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const PYTHON_TIMEOUT_MS = 10_000;
|
||||||
|
const MAX_TIMEOUT_MS = 30_000;
|
||||||
|
const START_TIMEOUT_MS = 60_000;
|
||||||
|
const RECYCLE_AFTER = 50;
|
||||||
|
const MAX_CODE = 50_000;
|
||||||
|
|
||||||
|
function runnerPath(): string {
|
||||||
|
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||||
|
// Bundled: dist/server/main.js next to python-runner.mjs. Source: server/ai/sandbox.ts.
|
||||||
|
for (const candidate of [path.join(here, 'python-runner.mjs'), path.join(here, '..', 'python-runner.mjs')]) {
|
||||||
|
if (fs.existsSync(candidate)) return candidate;
|
||||||
|
}
|
||||||
|
throw new Error('python-runner.mjs is missing from this installation');
|
||||||
|
}
|
||||||
|
|
||||||
|
function pyodideEntry(): string {
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
return path.join(path.dirname(require.resolve('pyodide/package.json')), 'pyodide.mjs');
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Pending {
|
||||||
|
resolve: (r: PythonResult) => void;
|
||||||
|
timer: ReturnType<typeof setTimeout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
let child: ChildProcess | null = null;
|
||||||
|
let ready: Promise<ChildProcess> | null = null;
|
||||||
|
let runs = 0;
|
||||||
|
let nextId = 1;
|
||||||
|
const pending = new Map<number, Pending>();
|
||||||
|
/** Runs are serialised: Pyodide is single-threaded and a shared process keeps memory flat. */
|
||||||
|
let queue: Promise<unknown> = Promise.resolve();
|
||||||
|
|
||||||
|
function kill(reason: string): void {
|
||||||
|
const c = child;
|
||||||
|
child = null;
|
||||||
|
ready = null;
|
||||||
|
runs = 0;
|
||||||
|
if (c && c.exitCode === null) c.kill('SIGKILL');
|
||||||
|
for (const [id, p] of pending) {
|
||||||
|
clearTimeout(p.timer);
|
||||||
|
p.resolve({ ok: false, stdout: '', stderr: '', result: null, error: reason, ms: 0 });
|
||||||
|
pending.delete(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function start(): Promise<ChildProcess> {
|
||||||
|
if (ready) return ready;
|
||||||
|
ready = new Promise<ChildProcess>((resolve, reject) => {
|
||||||
|
const entry = pyodideEntry();
|
||||||
|
const script = runnerPath();
|
||||||
|
const proc = fork(script, [entry], {
|
||||||
|
// An empty environment: nothing secret to find even if the sandbox were escaped.
|
||||||
|
env: {},
|
||||||
|
execArgv: ['--permission', `--allow-fs-read=${path.dirname(entry)}`, `--allow-fs-read=${script}`, '--max-old-space-size=256'],
|
||||||
|
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||||
|
serialization: 'json',
|
||||||
|
});
|
||||||
|
let errText = '';
|
||||||
|
proc.stderr?.on('data', (d) => {
|
||||||
|
if (errText.length < 4000) errText += String(d);
|
||||||
|
});
|
||||||
|
const startTimer = setTimeout(() => {
|
||||||
|
proc.kill('SIGKILL');
|
||||||
|
reject(new Error('The Python sandbox took too long to start.'));
|
||||||
|
}, START_TIMEOUT_MS);
|
||||||
|
let started = false;
|
||||||
|
proc.on('message', (msg: any) => {
|
||||||
|
if (msg?.ready) {
|
||||||
|
started = true;
|
||||||
|
clearTimeout(startTimer);
|
||||||
|
resolve(proc);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const p = pending.get(msg?.id);
|
||||||
|
if (!p) return;
|
||||||
|
clearTimeout(p.timer);
|
||||||
|
pending.delete(msg.id);
|
||||||
|
const { id: _id, ...rest } = msg;
|
||||||
|
p.resolve(rest as PythonResult);
|
||||||
|
});
|
||||||
|
proc.on('exit', (code) => {
|
||||||
|
clearTimeout(startTimer);
|
||||||
|
if (!started) reject(new Error(`The Python sandbox could not start (exit ${code}). ${errText.trim().split('\n').slice(-3).join(' ')}`.trim()));
|
||||||
|
if (child !== proc) return; // already replaced (killed after a timeout)
|
||||||
|
child = null;
|
||||||
|
ready = null;
|
||||||
|
for (const [id, p] of pending) {
|
||||||
|
clearTimeout(p.timer);
|
||||||
|
p.resolve({ ok: false, stdout: '', stderr: '', result: null, error: 'The Python sandbox stopped unexpectedly.', ms: 0 });
|
||||||
|
pending.delete(id);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
child = proc;
|
||||||
|
}).catch((err) => {
|
||||||
|
child = null;
|
||||||
|
ready = null;
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
return ready;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runOnce(code: string, timeoutMs: number): Promise<PythonResult> {
|
||||||
|
const proc = await start();
|
||||||
|
const id = nextId++;
|
||||||
|
runs += 1;
|
||||||
|
const result = await new Promise<PythonResult>((resolve) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
pending.delete(id);
|
||||||
|
resolve({ ok: false, stdout: '', stderr: '', result: null, error: `Stopped after ${Math.round(timeoutMs / 1000)} s (time limit). Avoid long loops; compute directly.`, ms: timeoutMs, timedOut: true });
|
||||||
|
kill('The previous run timed out.');
|
||||||
|
}, timeoutMs);
|
||||||
|
pending.set(id, { resolve, timer });
|
||||||
|
proc.send({ id, code });
|
||||||
|
});
|
||||||
|
if (runs >= RECYCLE_AFTER) kill('recycled');
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Runs Python 3 code in the sandbox. Never throws for problems in the code itself. */
|
||||||
|
export function runPython(code: string, opts: { timeoutMs?: number } = {}): Promise<PythonResult> {
|
||||||
|
if (code.length > MAX_CODE) return Promise.resolve({ ok: false, stdout: '', stderr: '', result: null, error: `The code is too long (limit ${MAX_CODE} characters).`, ms: 0 });
|
||||||
|
const timeoutMs = Math.min(Math.max(opts.timeoutMs ?? PYTHON_TIMEOUT_MS, 1000), MAX_TIMEOUT_MS);
|
||||||
|
const job = queue.then(() => runOnce(code, timeoutMs));
|
||||||
|
queue = job.catch(() => undefined);
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stops the sandbox process (shutdown, tests). */
|
||||||
|
export function stopPython(): void {
|
||||||
|
kill('The Python sandbox was stopped.');
|
||||||
|
}
|
||||||
290
server/ai/verify.ts
Normal file
290
server/ai/verify.ts
Normal file
|
|
@ -0,0 +1,290 @@
|
||||||
|
/**
|
||||||
|
* Python-backed checking of generated quizzes, and the `run_python` tool the assistant uses.
|
||||||
|
*
|
||||||
|
* After a quiz is generated, a short agent loop gives the model the draft (as Python data and
|
||||||
|
* as text) plus the original request and asks it to verify everything mechanically with the
|
||||||
|
* sandbox: question count, the select-all / single / numeric mix, options and correct answers
|
||||||
|
* per type, duplicates, and every calculation recomputed from its stem. It then submits fixes,
|
||||||
|
* which are applied here only if the question still validates afterwards.
|
||||||
|
*/
|
||||||
|
import { z } from 'zod';
|
||||||
|
import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../../shared/content.ts';
|
||||||
|
import type { AppContext } from '../http.ts';
|
||||||
|
import { runPython } from './sandbox.ts';
|
||||||
|
import { resolveModel, runChat } from './service.ts';
|
||||||
|
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
|
||||||
|
|
||||||
|
/* ------------------------------ data handed to Python ------------------------------ */
|
||||||
|
|
||||||
|
/** Compact, 1-based view of a quiz used both in prompts and as the `draft` variable in Python. */
|
||||||
|
export function quizForPython(quiz: { title: string; questions: Pick<DraftQuestion, 'type' | 'stem' | 'options' | 'rationale' | 'topic'>[] }) {
|
||||||
|
return {
|
||||||
|
title: quiz.title,
|
||||||
|
question_count: quiz.questions.length,
|
||||||
|
questions: quiz.questions.map((q, i) => ({
|
||||||
|
number: i + 1,
|
||||||
|
type: q.type,
|
||||||
|
topic: q.topic,
|
||||||
|
stem: q.stem,
|
||||||
|
options: q.options.map((o, k) => ({ number: k + 1, text: o.text, correct: Boolean(o.correct), why: o.why ?? '' })),
|
||||||
|
rationale: q.rationale ?? '',
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Python source that defines the given variables from JSON (a JSON string is a valid Python string literal). */
|
||||||
|
export function pythonPrelude(vars: Record<string, unknown>): string {
|
||||||
|
const lines = ['import json as _json'];
|
||||||
|
for (const [name, value] of Object.entries(vars)) {
|
||||||
|
if (!/^[a-z_][a-z0-9_]*$/i.test(name)) continue;
|
||||||
|
lines.push(`${name} = _json.loads(${JSON.stringify(JSON.stringify(value))})`);
|
||||||
|
}
|
||||||
|
return `${lines.join('\n')}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Formats a sandbox result for a model: short, and never an exception. */
|
||||||
|
export function pythonOutput(r: Awaited<ReturnType<typeof runPython>>) {
|
||||||
|
return {
|
||||||
|
ok: r.ok,
|
||||||
|
...(r.stdout ? { stdout: r.stdout } : {}),
|
||||||
|
...(r.result !== null ? { result: r.result } : {}),
|
||||||
|
...(r.stderr ? { stderr: r.stderr.slice(0, 2000) } : {}),
|
||||||
|
...(r.error ? { error: r.error } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const RUN_PYTHON_DESCRIPTION =
|
||||||
|
'Run Python 3 in a sandbox and get stdout and the value of the last expression. Standard library only (math, fractions, decimal, statistics, re, json, collections, itertools, datetime); no files, network or packages; 10 second limit. Use it whenever exactness matters: recompute dosage, rate and conversion answers, count questions, options and correct answers, check lengths, tally topics. Print what you need to see.';
|
||||||
|
|
||||||
|
export const RUN_PYTHON_TOOL: ToolDef = {
|
||||||
|
name: 'run_python',
|
||||||
|
description: RUN_PYTHON_DESCRIPTION,
|
||||||
|
parameters: {
|
||||||
|
type: 'object',
|
||||||
|
properties: { code: { type: 'string', description: 'Python source. Variables listed in the instructions are already defined.' } },
|
||||||
|
required: ['code'],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
/* ------------------------------ verification of a generated quiz ------------------------------ */
|
||||||
|
|
||||||
|
export interface QuizRequestFacts {
|
||||||
|
count: number;
|
||||||
|
style: string;
|
||||||
|
sataShare: number;
|
||||||
|
difficulty?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Long text is shortened rather than rejected: a verbose but correct review must not be lost. */
|
||||||
|
const text = (max: number) => z.coerce.string().transform((v) => v.slice(0, max));
|
||||||
|
const int = z.coerce.number().int();
|
||||||
|
const fixSchema = z.object({
|
||||||
|
question: int.min(1),
|
||||||
|
action: z.enum(['set_correct', 'set_numeric_answer', 'set_type', 'remove']),
|
||||||
|
correct_options: z.array(int.min(1)).max(12).optional(),
|
||||||
|
answer: text(200).optional(),
|
||||||
|
type: z.enum(['single', 'multi', 'numeric']).optional(),
|
||||||
|
reason: text(600).optional().default(''),
|
||||||
|
});
|
||||||
|
const reviewSchema = z.object({
|
||||||
|
summary: text(1500).optional().default(''),
|
||||||
|
checks: z
|
||||||
|
.array(z.object({ name: text(160), passed: z.coerce.boolean(), detail: text(600).optional().default('') }))
|
||||||
|
.transform((v) => v.slice(0, 40))
|
||||||
|
.optional()
|
||||||
|
.default([]),
|
||||||
|
fixes: z.array(z.unknown()).optional().default([]).transform((list) => list.flatMap((f) => { const r = fixSchema.safeParse(f); return r.success ? [r.data] : []; })),
|
||||||
|
});
|
||||||
|
export type QuizReview = z.infer<typeof reviewSchema>;
|
||||||
|
|
||||||
|
const SUBMIT_REVIEW: ToolDef = {
|
||||||
|
name: 'submit_review',
|
||||||
|
description: 'Finish the check: report every check you ran (with its result) and the fixes to apply. Call this exactly once, at the end.',
|
||||||
|
parameters: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
summary: { type: 'string', description: 'One or two sentences for the student.' },
|
||||||
|
checks: {
|
||||||
|
type: 'array',
|
||||||
|
items: { type: 'object', properties: { name: { type: 'string' }, passed: { type: 'boolean' }, detail: { type: 'string' } }, required: ['name', 'passed'] },
|
||||||
|
},
|
||||||
|
fixes: {
|
||||||
|
type: 'array',
|
||||||
|
description: 'Only for definite errors. Question and option numbers are 1-based.',
|
||||||
|
items: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
question: { type: 'integer' },
|
||||||
|
action: {
|
||||||
|
type: 'string',
|
||||||
|
enum: ['set_correct', 'set_numeric_answer', 'set_type', 'remove'],
|
||||||
|
description: 'set_correct: replace which options are correct. set_numeric_answer: replace the answer of a numeric question (number with unit). set_type: change single/multi to match the correct answers. remove: drop an unfixable question.',
|
||||||
|
},
|
||||||
|
correct_options: { type: 'array', items: { type: 'integer' } },
|
||||||
|
answer: { type: 'string' },
|
||||||
|
type: { type: 'string', enum: ['single', 'multi', 'numeric'] },
|
||||||
|
reason: { type: 'string' },
|
||||||
|
},
|
||||||
|
required: ['question', 'action', 'reason'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ['summary', 'checks', 'fixes'],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function rulesFor(req: QuizRequestFacts): string {
|
||||||
|
const expectedMulti = req.sataShare <= 0 ? 'none' : req.sataShare >= 1 ? 'all non-numeric questions' : `about ${Math.round(req.sataShare * 100)}% of the non-numeric questions`;
|
||||||
|
return [
|
||||||
|
`Requested: ${req.count} questions, style "${req.style}", difficulty "${req.difficulty ?? 'mixed'}", select-all (multi) share: ${expectedMulti}.`,
|
||||||
|
'Rules each question must meet:',
|
||||||
|
'- single: exactly one correct option; 4 options for NCLEX/standard/recall styles.',
|
||||||
|
'- multi (select all that apply): at least 2 correct options; for NCLEX style 5 or 6 options with 2 to 5 correct and a stem ending with "Select all that apply."',
|
||||||
|
'- numeric: the student types a number; options holds the accepted answer(s), all marked correct. The answer must equal the value computed from the stem, rounded as the stem says.',
|
||||||
|
'- No duplicate or near-duplicate stems; no identical options within a question; no "all of the above" / "none of the above".',
|
||||||
|
'- The keyed answer must be the clinically correct one (check against the rationale and standard practice).',
|
||||||
|
].join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
const VERIFY_SYSTEM = `You are the quality checker for quiz questions written for a nursing student. Accuracy is critical: they are used to prepare for licensing exams.
|
||||||
|
Check the draft mechanically with the run_python tool. The variables \`draft\` (the quiz) and \`request\` (what was asked for) are already defined in every run. Write Python that counts and verifies instead of eyeballing:
|
||||||
|
1. The number of questions against request["count"].
|
||||||
|
2. The mix of types (single / multi / numeric) against the requested select-all share.
|
||||||
|
3. For every question: option count, number of correct options, and that it fits its type's rules.
|
||||||
|
4. Duplicate stems or options.
|
||||||
|
5. Every numeric question: recompute the answer from the numbers in the stem with Python, apply the rounding the stem asks for, and compare with the keyed answer numerically (ignore units).
|
||||||
|
Also read each question and judge whether the keyed answer is clinically correct.
|
||||||
|
Then call submit_review once with every check you ran and a fix for each definite error. Do not "fix" things that are only stylistic. A count or mix that is off cannot be fixed here; report it as a failed check.`;
|
||||||
|
|
||||||
|
/** Applies validated fixes; returns notes describing what changed or was refused. */
|
||||||
|
export function applyQuizFixes(questions: DraftQuestion[], fixes: z.infer<typeof fixSchema>[]): { questions: DraftQuestion[]; applied: string[]; refused: string[] } {
|
||||||
|
const out = questions.map((q) => ({ ...q, options: q.options.map((o) => ({ ...o })) }));
|
||||||
|
const remove = new Set<number>();
|
||||||
|
const applied: string[] = [];
|
||||||
|
const refused: string[] = [];
|
||||||
|
for (const fix of fixes) {
|
||||||
|
const i = fix.question - 1;
|
||||||
|
const q = out[i];
|
||||||
|
if (!q) {
|
||||||
|
refused.push(`Question ${fix.question}: does not exist.`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const before = { ...q, options: q.options.map((o) => ({ ...o })) };
|
||||||
|
const why = fix.reason ? ` (${fix.reason.replace(/\s+/g, ' ').slice(0, 200)})` : '';
|
||||||
|
if (fix.action === 'remove') {
|
||||||
|
remove.add(i);
|
||||||
|
applied.push(`Question ${fix.question} removed${why}.`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (fix.action === 'set_correct' && fix.correct_options?.length) {
|
||||||
|
const set = new Set(fix.correct_options.map((n) => n - 1));
|
||||||
|
q.options = q.options.map((o, k) => ({ ...o, correct: set.has(k) }));
|
||||||
|
if (q.type === 'single' && set.size > 1) q.type = 'multi';
|
||||||
|
} else if (fix.action === 'set_numeric_answer' && fix.answer?.trim() && q.type === 'numeric') {
|
||||||
|
q.options = [{ text: fix.answer.trim(), correct: true, why: q.options[0]?.why ?? '' }];
|
||||||
|
} else if (fix.action === 'set_type' && fix.type && fix.type !== 'numeric') {
|
||||||
|
q.type = fix.type;
|
||||||
|
} else {
|
||||||
|
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} was incomplete.`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (validateDraftQuestion(q, i).some((p) => p.level === 'error')) {
|
||||||
|
out[i] = before;
|
||||||
|
refused.push(`Question ${fix.question}: the suggested ${fix.action.replace(/_/g, ' ')} would have made it invalid, so it was not applied${why}.`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
applied.push(`Question ${fix.question}: ${fix.action === 'set_numeric_answer' ? `answer corrected to ${fix.answer!.trim()}` : fix.action === 'set_type' ? `type changed to ${fix.type}` : `answer key corrected`}${why}.`);
|
||||||
|
}
|
||||||
|
return { questions: out.filter((_, i) => !remove.has(i)), applied, refused };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks a generated quiz with the model and the Python sandbox, then applies the fixes.
|
||||||
|
* Never throws: a failed check leaves the draft as it was and says so in the notes.
|
||||||
|
*/
|
||||||
|
export async function verifyQuizDraft(
|
||||||
|
app: AppContext,
|
||||||
|
draft: QuizDraft,
|
||||||
|
req: QuizRequestFacts,
|
||||||
|
opts: { modelId?: string | null; signal?: AbortSignal; onStep?: (label: string) => void } = {},
|
||||||
|
): Promise<{ draft: QuizDraft; notes: string[]; review: QuizReview | null; pythonRuns: number }> {
|
||||||
|
const resolved = resolveModel(app, 'generate', opts.modelId);
|
||||||
|
if (!resolved.model.capabilities.tools) {
|
||||||
|
return { draft, notes: ['The Python answer check was skipped: the generation model cannot use tools.'], review: null, pythonRuns: 0 };
|
||||||
|
}
|
||||||
|
const data = quizForPython(draft);
|
||||||
|
const prelude = pythonPrelude({ draft: data, request: { count: req.count, style: req.style, sata_share: req.sataShare, difficulty: req.difficulty ?? 'mixed' } });
|
||||||
|
const listing = data.questions
|
||||||
|
.map((q) => `Q${q.number} [${q.type}] ${q.stem}\n${q.options.map((o) => ` ${o.number}. ${o.correct ? '(correct) ' : ''}${o.text}`).join('\n')}${q.rationale ? `\n Rationale: ${q.rationale.slice(0, 600)}` : ''}`)
|
||||||
|
.join('\n\n');
|
||||||
|
const messages: ChatMessage[] = [{ role: 'user', content: [{ type: 'text', text: `${rulesFor(req)}\n\nTHE DRAFT (${data.question_count} questions):\n\n${listing}` }] }];
|
||||||
|
let review: QuizReview | null = null;
|
||||||
|
let pythonRuns = 0;
|
||||||
|
// Up to WORK_STEPS turns with Python, then only submit_review is offered until it arrives.
|
||||||
|
const WORK_STEPS = 8;
|
||||||
|
const MAX_STEPS = 12;
|
||||||
|
try {
|
||||||
|
for (let step = 0; step < MAX_STEPS && !review; step += 1) {
|
||||||
|
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
|
||||||
|
const wrapUp = step >= WORK_STEPS || pythonRuns >= 16;
|
||||||
|
if (wrapUp && !messages.some((m) => m.content.some((p) => p.type === 'text' && p.text.startsWith('You have run enough checks')))) {
|
||||||
|
const nudge: ContentPart = { type: 'text', text: 'You have run enough checks. Call submit_review now with your results and fixes.' };
|
||||||
|
const lastMsg = messages[messages.length - 1];
|
||||||
|
// Ride along with the tool results so user and assistant turns keep alternating.
|
||||||
|
if (lastMsg?.role === 'user') lastMsg.content.push(nudge);
|
||||||
|
else messages.push({ role: 'user', content: [nudge] });
|
||||||
|
}
|
||||||
|
const res = await runChat(app, {
|
||||||
|
task: 'generate',
|
||||||
|
purpose: 'quiz check',
|
||||||
|
modelId: opts.modelId,
|
||||||
|
system: VERIFY_SYSTEM,
|
||||||
|
messages,
|
||||||
|
tools: wrapUp ? [SUBMIT_REVIEW] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW],
|
||||||
|
toolChoice: wrapUp ? { name: 'submit_review' } : 'auto',
|
||||||
|
maxOutputTokens: 16_000,
|
||||||
|
signal: opts.signal,
|
||||||
|
});
|
||||||
|
messages.push(res.message);
|
||||||
|
const calls = res.message.content.filter((p): p is Extract<ContentPart, { type: 'tool_call' }> => p.type === 'tool_call');
|
||||||
|
if (calls.length === 0) {
|
||||||
|
messages.push({ role: 'user', content: [{ type: 'text', text: 'Run your checks with run_python, then call submit_review.' }] });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const results: ContentPart[] = [];
|
||||||
|
for (const call of calls) {
|
||||||
|
if (call.name === 'submit_review') {
|
||||||
|
const parsed = reviewSchema.safeParse(call.input);
|
||||||
|
if (parsed.success) review = parsed.data;
|
||||||
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: parsed.success ? '{"ok":true}' : JSON.stringify({ error: parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') }), isError: !parsed.success });
|
||||||
|
} else if (call.name === 'run_python' && pythonRuns >= 16) {
|
||||||
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"Python budget used up. Call submit_review now."}', isError: true });
|
||||||
|
} else if (call.name === 'run_python') {
|
||||||
|
pythonRuns += 1;
|
||||||
|
opts.onStep?.(`Checking the questions with Python (run ${pythonRuns})`);
|
||||||
|
const r = await runPython(prelude + String(call.input.code ?? ''));
|
||||||
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: JSON.stringify(pythonOutput(r)), isError: !r.ok });
|
||||||
|
} else {
|
||||||
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output: '{"error":"unknown tool"}', isError: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The last user message carries the tool results; keep wrap-up instructions in front of the model.
|
||||||
|
messages.push({ role: 'user', content: results });
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof AIError && err.code === 'aborted') throw err;
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
return { draft, notes: [`The Python answer check could not finish (${message.slice(0, 200)}). Review the answers yourself.`], review: null, pythonRuns };
|
||||||
|
}
|
||||||
|
if (!review) return { draft, notes: ['The Python answer check did not report back. Review the answers yourself.'], review: null, pythonRuns };
|
||||||
|
|
||||||
|
const { questions, applied, refused } = applyQuizFixes(draft.questions, review.fixes);
|
||||||
|
const failed = review.checks.filter((c) => !c.passed);
|
||||||
|
const notes = [
|
||||||
|
`Checked with Python (${pythonRuns} run${pythonRuns === 1 ? '' : 's'}): ${review.checks.length - failed.length} of ${review.checks.length} checks passed${applied.length ? `, ${applied.length} fix${applied.length === 1 ? '' : 'es'} applied` : ''}.`,
|
||||||
|
...failed.map((c) => `Check failed: ${c.name}${c.detail ? `: ${c.detail}` : ''}`),
|
||||||
|
...applied,
|
||||||
|
...refused,
|
||||||
|
];
|
||||||
|
return { draft: { ...draft, questions }, notes, review, pythonRuns };
|
||||||
|
}
|
||||||
|
|
@ -10,6 +10,7 @@ import { startBackupSchedule } from './services/backup.ts';
|
||||||
import { resumeQueue } from './services/documents.ts';
|
import { resumeQueue } from './services/documents.ts';
|
||||||
import { scheduleIndexing } from './services/semantic.ts';
|
import { scheduleIndexing } from './services/semantic.ts';
|
||||||
import { seedTemplates } from './services/templates.ts';
|
import { seedTemplates } from './services/templates.ts';
|
||||||
|
import { stopPython } from './ai/sandbox.ts';
|
||||||
|
|
||||||
function log(level: 'info' | 'warn' | 'error', message: string, extra?: unknown): void {
|
function log(level: 'info' | 'warn' | 'error', message: string, extra?: unknown): void {
|
||||||
const line = `${new Date().toISOString()} ${level.toUpperCase().padEnd(5)} ${message}`;
|
const line = `${new Date().toISOString()} ${level.toUpperCase().padEnd(5)} ${message}`;
|
||||||
|
|
@ -46,6 +47,7 @@ async function main(): Promise<void> {
|
||||||
|
|
||||||
let closing = false;
|
let closing = false;
|
||||||
const shutdown = (signal: string) => {
|
const shutdown = (signal: string) => {
|
||||||
|
stopPython();
|
||||||
if (closing) return;
|
if (closing) return;
|
||||||
closing = true;
|
closing = true;
|
||||||
log('info', `${signal} received, shutting down`);
|
log('info', `${signal} received, shutting down`);
|
||||||
|
|
|
||||||
96
server/python-runner.mjs
Normal file
96
server/python-runner.mjs
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
/**
|
||||||
|
* Sandboxed Python runner (Pyodide / WebAssembly). Started by server/ai/sandbox.ts as a
|
||||||
|
* separate Node process with the permission model on:
|
||||||
|
* --permission --allow-fs-read=<pyodide dir> --allow-fs-read=<this file>
|
||||||
|
* so even code that escaped the WebAssembly sandbox could not read the data folder, write
|
||||||
|
* anything, start processes or load native addons. It also gets an empty environment, so
|
||||||
|
* there are no secrets in `process.env`.
|
||||||
|
*
|
||||||
|
* Inside Python, `js` is an empty object (jsglobals: {}) and the bridges back to JavaScript
|
||||||
|
* (pyodide_js, pyodide.http, micropip) are removed, so code cannot reach fetch, require or
|
||||||
|
* the host file system. Only the standard library is available.
|
||||||
|
*
|
||||||
|
* Protocol (IPC): { id, code, timeoutMs } -> { id, ok, stdout, stderr, result, error, ms }.
|
||||||
|
* The parent kills this process when a run exceeds its time limit.
|
||||||
|
*/
|
||||||
|
import { constants as fsConstants } from 'node:fs';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
|
||||||
|
// Emscripten's NODEFS reads open-flag constants through process.binding('constants'), which
|
||||||
|
// the permission model forbids. Answer that one call from the public fs.constants instead.
|
||||||
|
process.binding = (name) => {
|
||||||
|
if (name === 'constants') return { fs: fsConstants };
|
||||||
|
throw new Error(`process.binding('${name}') is not available in the Python sandbox`);
|
||||||
|
};
|
||||||
|
|
||||||
|
const pyodideEntry = process.argv[2];
|
||||||
|
const OUTPUT_LIMIT = 20_000;
|
||||||
|
|
||||||
|
const { loadPyodide } = await import(pathToFileURL(pyodideEntry).href);
|
||||||
|
|
||||||
|
let stdout = '';
|
||||||
|
let stderr = '';
|
||||||
|
const append = (which, text) => {
|
||||||
|
if (which === 'out') {
|
||||||
|
if (stdout.length < OUTPUT_LIMIT) stdout += `${text}\n`;
|
||||||
|
} else if (stderr.length < OUTPUT_LIMIT) stderr += `${text}\n`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const pyodide = await loadPyodide({
|
||||||
|
jsglobals: {},
|
||||||
|
stdout: (t) => append('out', t),
|
||||||
|
stderr: (t) => append('err', t),
|
||||||
|
stdin: () => null,
|
||||||
|
});
|
||||||
|
|
||||||
|
pyodide.runPython(`
|
||||||
|
import sys, importlib.abc
|
||||||
|
|
||||||
|
_BLOCKED = ('js', 'pyodide_js', 'micropip', 'pyodide.http', 'pyodide.webloop', 'pyodide_http')
|
||||||
|
for _name in list(sys.modules):
|
||||||
|
if _name in _BLOCKED or _name.split('.')[0] in ('js', 'pyodide_js', 'micropip'):
|
||||||
|
del sys.modules[_name]
|
||||||
|
|
||||||
|
class _Sandbox(importlib.abc.MetaPathFinder):
|
||||||
|
blocked = _BLOCKED
|
||||||
|
roots = ('js', 'pyodide_js', 'micropip')
|
||||||
|
|
||||||
|
def find_spec(self, fullname, path=None, target=None):
|
||||||
|
if fullname in self.blocked or fullname.split('.')[0] in self.roots:
|
||||||
|
raise ImportError(f"{fullname} is not available in this sandbox (no network, files or JavaScript)")
|
||||||
|
return None
|
||||||
|
|
||||||
|
sys.meta_path.insert(0, _Sandbox())
|
||||||
|
del _Sandbox, _BLOCKED
|
||||||
|
`);
|
||||||
|
|
||||||
|
/** Runs one snippet in a fresh namespace. The value of a final expression is returned as `result`. */
|
||||||
|
async function run(code) {
|
||||||
|
stdout = '';
|
||||||
|
stderr = '';
|
||||||
|
const ns = pyodide.globals.get('dict')();
|
||||||
|
const started = Date.now();
|
||||||
|
try {
|
||||||
|
const value = await pyodide.runPythonAsync(code, { globals: ns });
|
||||||
|
let result = null;
|
||||||
|
if (value !== undefined && value !== null) {
|
||||||
|
result = typeof value === 'object' && typeof value.toString === 'function' ? String(value) : String(value);
|
||||||
|
if (typeof value?.destroy === 'function') value.destroy();
|
||||||
|
}
|
||||||
|
return { ok: true, stdout: stdout.slice(0, OUTPUT_LIMIT), stderr: stderr.slice(0, OUTPUT_LIMIT), result: result && result.slice(0, OUTPUT_LIMIT), error: null, ms: Date.now() - started };
|
||||||
|
} catch (err) {
|
||||||
|
const message = String(err?.message ?? err);
|
||||||
|
// Keep the Python traceback, drop the Pyodide/JavaScript frames above it.
|
||||||
|
const tb = message.includes('Traceback') ? message.slice(message.indexOf('Traceback')) : message;
|
||||||
|
return { ok: false, stdout: stdout.slice(0, OUTPUT_LIMIT), stderr: stderr.slice(0, OUTPUT_LIMIT), result: null, error: tb.slice(-4000), ms: Date.now() - started };
|
||||||
|
} finally {
|
||||||
|
ns.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
process.on('message', async (msg) => {
|
||||||
|
if (!msg || typeof msg.code !== 'string') return;
|
||||||
|
const reply = await run(msg.code);
|
||||||
|
process.send?.({ id: msg.id, ...reply });
|
||||||
|
});
|
||||||
|
process.send?.({ ready: true });
|
||||||
|
|
@ -50,6 +50,9 @@ async function sse(url: string, body: unknown): Promise<any[]> {
|
||||||
.map((d) => JSON.parse(d));
|
.map((d) => JSON.parse(d));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A request from the Python quiz check rather than from generation itself. */
|
||||||
|
const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review');
|
||||||
|
|
||||||
async function waitForDraft(id: string) {
|
async function waitForDraft(id: string) {
|
||||||
for (let i = 0; i < 200; i += 1) {
|
for (let i = 0; i < 200; i += 1) {
|
||||||
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
|
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
|
||||||
|
|
@ -195,8 +198,16 @@ describe('generation', () => {
|
||||||
// Citations were mapped from excerpt ids to readable labels.
|
// Citations were mapped from excerpt ids to readable labels.
|
||||||
expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/);
|
expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/);
|
||||||
// 12 questions at 8 per batch means two model calls, each carrying the source excerpts.
|
// 12 questions at 8 per batch means two model calls, each carrying the source excerpts.
|
||||||
const calls = mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
|
const all = mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
|
||||||
|
const calls = all.filter((r) => !isCheck(r));
|
||||||
expect(calls).toHaveLength(2);
|
expect(calls).toHaveLength(2);
|
||||||
|
// Then the Python check: one run, a review, and its fix applied to question 2.
|
||||||
|
const checks = all.filter(isCheck);
|
||||||
|
expect(checks.length).toBeGreaterThanOrEqual(2);
|
||||||
|
const pythonResult = JSON.parse(checks[1].body.messages.find((m: any) => m.role === 'tool').content);
|
||||||
|
expect(pythonResult).toMatchObject({ ok: true, stdout: '12 12 4 [1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2]\n' });
|
||||||
|
expect(draft.payload.questions[1].options.map((o: any) => o.correct)).toEqual([false, true, false, false]);
|
||||||
|
expect(draft.meta.notes.join(' ')).toMatch(/Checked with Python \(1 run\): 2 of 2 checks passed, 1 fix applied/);
|
||||||
expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/);
|
expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/);
|
||||||
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/);
|
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/);
|
||||||
expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } });
|
expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } });
|
||||||
|
|
@ -237,7 +248,7 @@ describe('generation', () => {
|
||||||
// Deck -> quiz conversion uses the cards as the source.
|
// Deck -> quiz conversion uses the cards as the source.
|
||||||
const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId);
|
const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId);
|
||||||
expect(converted.status).toBe('ready');
|
expect(converted.status).toBe('ready');
|
||||||
expect(JSON.stringify(mock.requests.at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
|
expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
|
||||||
|
|
||||||
// Append more cards to the existing deck.
|
// Append more cards to the existing deck.
|
||||||
const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId);
|
const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId);
|
||||||
|
|
@ -265,6 +276,7 @@ describe('generation', () => {
|
||||||
const draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId);
|
const draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId);
|
||||||
expect(draft.status).toBe('ready');
|
expect(draft.status).toBe('ready');
|
||||||
expect(draft.payload.questions).toHaveLength(3);
|
expect(draft.payload.questions).toHaveLength(3);
|
||||||
|
expect(draft.meta.notes.join(' ')).toMatch(/check was skipped/);
|
||||||
const call = mock.requests.at(-1)!;
|
const call = mock.requests.at(-1)!;
|
||||||
expect(call.body.tools).toBeUndefined();
|
expect(call.body.tools).toBeUndefined();
|
||||||
expect(call.body.response_format).toEqual({ type: 'json_object' });
|
expect(call.body.response_format).toEqual({ type: 'json_object' });
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,13 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
|
||||||
return { count: Math.min(count, 6), topic: 'medications', use_all_class_documents: /document/i.test(userText) };
|
return { count: Math.min(count, 6), topic: 'medications', use_all_class_documents: /document/i.test(userText) };
|
||||||
case 'search_documents':
|
case 'search_documents':
|
||||||
return { query: userText.replace(/[^\w\s]/g, ' ').split(/\s+/).filter((w) => w.length > 3).slice(0, 6).join(' ') };
|
return { query: userText.replace(/[^\w\s]/g, ' ').split(/\s+/).filter((w) => w.length > 3).slice(0, 6).join(' ') };
|
||||||
|
case 'run_python':
|
||||||
|
return { code: 'qs = draft["questions"]\nprint(len(qs), request["count"], sum(1 for q in qs if q["type"] == "multi"), [sum(o["correct"] for o in q["options"]) for q in qs])' };
|
||||||
|
case 'submit_review': {
|
||||||
|
// Second question (if any) gets a "corrected" key so tests can see fixes applied.
|
||||||
|
const fix = /Q2 \[single\]/.test(all) ? [{ question: 2, action: 'set_correct', correct_options: [2], reason: 'mock: option 2 is the right answer' }] : [];
|
||||||
|
return { summary: 'Checked by the mock.', checks: [{ name: 'Question count', passed: true, detail: 'matches the request' }, { name: 'Correct answers per type', passed: true }], fixes: fix };
|
||||||
|
}
|
||||||
case 'get_performance':
|
case 'get_performance':
|
||||||
case 'list_classes':
|
case 'list_classes':
|
||||||
case 'list_materials':
|
case 'list_materials':
|
||||||
|
|
@ -187,6 +194,11 @@ export function startMockAi(port = 0): Promise<MockAi> {
|
||||||
}
|
}
|
||||||
|
|
||||||
const forced = body?.tool_choice?.function?.name as string | undefined;
|
const forced = body?.tool_choice?.function?.name as string | undefined;
|
||||||
|
// The quiz checker: run Python once, then submit the review.
|
||||||
|
if (names.includes('submit_review')) {
|
||||||
|
const ranPython = messages.some((m) => m.role === 'tool');
|
||||||
|
return callTool(forced === 'submit_review' || ranPython ? 'submit_review' : 'run_python');
|
||||||
|
}
|
||||||
if (forced) return callTool(forced);
|
if (forced) return callTool(forced);
|
||||||
if (last.role === 'tool') {
|
if (last.role === 'tool') {
|
||||||
const ref = /"ref":"(c\d+)"/.exec(text(last.content))?.[1];
|
const ref = /"ref":"(c\d+)"/.exec(text(last.content))?.[1];
|
||||||
|
|
|
||||||
125
tests/python.test.ts
Normal file
125
tests/python.test.ts
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
/**
|
||||||
|
* The Python sandbox (Pyodide in a permission-restricted Node process), the assistant's
|
||||||
|
* run_python tool and the fixes the quiz check may apply.
|
||||||
|
*/
|
||||||
|
import { afterAll, afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||||
|
import type { DraftQuestion } from '../shared/content.ts';
|
||||||
|
import { runToolForTest } from '../server/ai/assistant.ts';
|
||||||
|
import { createDraft } from '../server/ai/generate.ts';
|
||||||
|
import { runPython, stopPython } from '../server/ai/sandbox.ts';
|
||||||
|
import { applyQuizFixes, pythonPrelude } from '../server/ai/verify.ts';
|
||||||
|
import { createTestApp, type TestApp } from './helpers.ts';
|
||||||
|
|
||||||
|
afterAll(() => stopPython());
|
||||||
|
|
||||||
|
describe('python sandbox', () => {
|
||||||
|
it('runs standard-library code and returns output and the last value', async () => {
|
||||||
|
const r = await runPython('from fractions import Fraction\nimport statistics\nprint(5000 / 10000)\nround(73 * 1.2, 1)');
|
||||||
|
expect(r).toMatchObject({ ok: true, stdout: '0.5\n', result: '87.6' });
|
||||||
|
expect((await runPython('statistics.mean([1, 2])')).error).toMatch(/NameError/); // each run starts clean
|
||||||
|
}, 60_000);
|
||||||
|
|
||||||
|
it('reports Python errors without the JavaScript frames', async () => {
|
||||||
|
const r = await runPython('1 / 0');
|
||||||
|
expect(r.ok).toBe(false);
|
||||||
|
expect(r.error).toMatch(/^Traceback/);
|
||||||
|
expect(r.error).toMatch(/ZeroDivisionError/);
|
||||||
|
}, 60_000);
|
||||||
|
|
||||||
|
it('has no way to the host: no JavaScript bridge, files, environment, processes or network', async () => {
|
||||||
|
for (const code of ['import js', 'import pyodide_js', 'from pyodide.code import run_js\nrun_js("process.env")', 'import micropip']) {
|
||||||
|
const r = await runPython(code);
|
||||||
|
expect(r.ok, code).toBe(false);
|
||||||
|
expect(r.error, code).toMatch(/not available in this sandbox/);
|
||||||
|
}
|
||||||
|
expect((await runPython('open("/etc/passwd").read()')).error).toMatch(/FileNotFoundError/);
|
||||||
|
expect((await runPython(`open(${JSON.stringify(process.cwd() + '/package.json')}).read()`)).error).toMatch(/FileNotFoundError/);
|
||||||
|
expect((await runPython('import os\nsorted(os.environ)')).result).not.toMatch(/APP_SECRET|PATH.*Windows|DATA_DIR/);
|
||||||
|
expect((await runPython('import subprocess\nsubprocess.run(["ls"])')).error).toMatch(/does not support processes/);
|
||||||
|
expect((await runPython('import urllib.request\nurllib.request.urlopen("http://127.0.0.1:1")')).ok).toBe(false);
|
||||||
|
}, 60_000);
|
||||||
|
|
||||||
|
it('stops runaway code at the time limit and recovers for the next run', async () => {
|
||||||
|
const r = await runPython('while True:\n pass', { timeoutMs: 1500 });
|
||||||
|
expect(r).toMatchObject({ ok: false, timedOut: true });
|
||||||
|
expect(r.error).toMatch(/time limit/);
|
||||||
|
expect((await runPython('sum(range(10))')).result).toBe('45');
|
||||||
|
}, 60_000);
|
||||||
|
|
||||||
|
it('serialises concurrent runs', async () => {
|
||||||
|
const results = await Promise.all([1, 2, 3, 4].map((n) => runPython(`${n} * 10`)));
|
||||||
|
expect(results.map((r) => r.result)).toEqual(['10', '20', '30', '40']);
|
||||||
|
}, 60_000);
|
||||||
|
|
||||||
|
it('hands data to Python through JSON', async () => {
|
||||||
|
const r = await runPython(`${pythonPrelude({ draft: { questions: [{ stem: 'Quote "this" \\ and ü' }] } })}print(draft["questions"][0]["stem"])`);
|
||||||
|
expect(r.stdout).toBe('Quote "this" \\ and ü\n');
|
||||||
|
}, 60_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('quiz check fixes', () => {
|
||||||
|
const q = (type: DraftQuestion['type'], correct: number[], n = 4): DraftQuestion => ({
|
||||||
|
type,
|
||||||
|
stem: `Stem ${type}`,
|
||||||
|
options: Array.from({ length: n }, (_, i) => ({ text: `Option ${i + 1}`, correct: correct.includes(i), why: '' })),
|
||||||
|
rationale: 'r',
|
||||||
|
topic: '',
|
||||||
|
difficulty: 'medium',
|
||||||
|
tags: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
it('applies valid fixes and refuses ones that would break a question', () => {
|
||||||
|
const questions = [q('single', [0, 1, 2, 3]), q('numeric', [0], 1), q('multi', [0, 1], 5), q('single', [0])];
|
||||||
|
questions[1].options[0].text = '12 mL';
|
||||||
|
const { questions: out, applied, refused } = applyQuizFixes(questions, [
|
||||||
|
{ question: 1, action: 'set_correct', correct_options: [2], reason: 'B is right' },
|
||||||
|
{ question: 2, action: 'set_numeric_answer', answer: '12.5 mL', reason: 'recomputed' },
|
||||||
|
{ question: 3, action: 'set_correct', correct_options: [], reason: 'empty' },
|
||||||
|
{ question: 4, action: 'set_numeric_answer', answer: '3', reason: 'not numeric' },
|
||||||
|
{ question: 9, action: 'remove', reason: 'missing' },
|
||||||
|
]);
|
||||||
|
expect(out[0].options.map((o) => o.correct)).toEqual([false, true, false, false]);
|
||||||
|
expect(out[1].options).toEqual([{ text: '12.5 mL', correct: true, why: '' }]);
|
||||||
|
expect(out[2].options.filter((o) => o.correct)).toHaveLength(2);
|
||||||
|
expect(applied).toHaveLength(2);
|
||||||
|
expect(refused).toHaveLength(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('removes questions and turns a several-answer single into select-all', () => {
|
||||||
|
const { questions: out } = applyQuizFixes([q('single', [0]), q('single', [0])], [
|
||||||
|
{ question: 1, action: 'remove', reason: 'duplicate' },
|
||||||
|
{ question: 2, action: 'set_correct', correct_options: [1, 3], reason: 'two are right' },
|
||||||
|
]);
|
||||||
|
expect(out).toHaveLength(1);
|
||||||
|
expect(out[0].type).toBe('multi');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('assistant run_python tool', () => {
|
||||||
|
let t: TestApp;
|
||||||
|
beforeEach(async () => {
|
||||||
|
t = await createTestApp();
|
||||||
|
});
|
||||||
|
afterEach(() => t.close());
|
||||||
|
|
||||||
|
it('loads a draft or a saved quiz as Python data', async () => {
|
||||||
|
const draftId = createDraft(t.ctx.db, {
|
||||||
|
kind: 'quiz',
|
||||||
|
title: 'Calc',
|
||||||
|
payload: { kind: 'quiz', title: 'Calc', description: '', questions: [{ type: 'numeric', stem: '500 mL over 4 h?', options: [{ text: '125 mL/hr', correct: true }], rationale: '', topic: '', difficulty: 'medium', tags: [] }] },
|
||||||
|
meta: { request: { count: 1 } },
|
||||||
|
});
|
||||||
|
const run = await runToolForTest(t.ctx, 'run_python', { draft_id: draftId, code: 'q = draft["questions"][0]\nprint(len(draft["questions"]), q["type"], 500 / 4 == float(q["options"][0]["text"].split()[0]), draft["request"]["count"])' });
|
||||||
|
expect(run.output).toMatchObject({ ok: true, stdout: '1 numeric True 1\n' });
|
||||||
|
expect(run.summary).toBe('1 numeric True 1');
|
||||||
|
|
||||||
|
const cls = (await t.api('POST', '/api/classes', { name: 'C' })).body.class;
|
||||||
|
const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Q', questions: [{ type: 'multi', stem: 'Pick', options: [{ text: 'a', correct: true }, { text: 'b', correct: true }, { text: 'c', correct: false }] }] })).body.quiz;
|
||||||
|
const counted = await runToolForTest(t.ctx, 'run_python', { quiz_id: quiz.id, code: '[sum(o["correct"] for o in q["options"]) for q in quiz["questions"]]' });
|
||||||
|
expect(counted.output).toMatchObject({ ok: true, result: '[2]' });
|
||||||
|
|
||||||
|
const failing = await runToolForTest(t.ctx, 'run_python', { code: 'undefined_name' });
|
||||||
|
expect(failing.output).toMatchObject({ ok: false });
|
||||||
|
expect(failing.summary).toMatch(/NameError/);
|
||||||
|
}, 60_000);
|
||||||
|
});
|
||||||
|
|
@ -6,8 +6,8 @@ import { toast } from 'sonner';
|
||||||
import { api, errorMessage } from '@/lib/api';
|
import { api, errorMessage } from '@/lib/api';
|
||||||
import { qk, useAiModels, useAiStatus, useClass } from '@/lib/queries';
|
import { qk, useAiModels, useAiStatus, useClass } from '@/lib/queries';
|
||||||
import type { DocumentItem, Template } from '@/lib/types';
|
import type { DocumentItem, Template } from '@/lib/types';
|
||||||
import { cx, formatTokens } from '@/lib/util';
|
import { cx, formatTokens, useLocalStorage } from '@/lib/util';
|
||||||
import { Button, Callout, Dialog, Field, Input, Segmented, Select, Textarea } from '@/ui';
|
import { Button, Callout, Dialog, Field, Input, Segmented, Select, Switch, Textarea } from '@/ui';
|
||||||
|
|
||||||
export interface GeneratePreset {
|
export interface GeneratePreset {
|
||||||
kind?: 'quiz' | 'deck' | 'note';
|
kind?: 'quiz' | 'deck' | 'note';
|
||||||
|
|
@ -44,6 +44,7 @@ export function GenerateDialog({ open, onClose, classId, folderId, preset }: { o
|
||||||
const [instructions, setInstructions] = useState('');
|
const [instructions, setInstructions] = useState('');
|
||||||
const [modelId, setModelId] = useState('');
|
const [modelId, setModelId] = useState('');
|
||||||
const [templateId, setTemplateId] = useState('');
|
const [templateId, setTemplateId] = useState('');
|
||||||
|
const [verify, setVerify] = useLocalStorage('sd.generate.verify', true);
|
||||||
const [busy, setBusy] = useState(false);
|
const [busy, setBusy] = useState(false);
|
||||||
|
|
||||||
const docs = useMemo(() => (cls.data?.items.filter((i): i is DocumentItem => i.kind === 'document') ?? []).filter((d) => d.status === 'ready' && d.tokenCount > 0), [cls.data]);
|
const docs = useMemo(() => (cls.data?.items.filter((i): i is DocumentItem => i.kind === 'document') ?? []).filter((d) => d.status === 'ready' && d.tokenCount > 0), [cls.data]);
|
||||||
|
|
@ -103,7 +104,7 @@ export function GenerateDialog({ open, onClose, classId, folderId, preset }: { o
|
||||||
else if (mode === 'text') source.text = text;
|
else if (mode === 'text') source.text = text;
|
||||||
else if (mode === 'weak') source.weakAreas = true;
|
else if (mode === 'weak') source.weakAreas = true;
|
||||||
const body: Record<string, unknown> = { kind, classId, folderId: folderId ?? null, source, focus: focus.trim() || undefined, instructions: instructions.trim() || undefined, modelId: modelId || undefined };
|
const body: Record<string, unknown> = { kind, classId, folderId: folderId ?? null, source, focus: focus.trim() || undefined, instructions: instructions.trim() || undefined, modelId: modelId || undefined };
|
||||||
if (kind === 'quiz') Object.assign(body, { count, style, difficulty, sataShare: sata / 100 });
|
if (kind === 'quiz') Object.assign(body, { count, style, difficulty, sataShare: sata / 100, verify });
|
||||||
else if (kind === 'deck') Object.assign(body, { count, cardStyle });
|
else if (kind === 'deck') Object.assign(body, { count, cardStyle });
|
||||||
else Object.assign(body, { noteStyle });
|
else Object.assign(body, { noteStyle });
|
||||||
if (fixedSource && kind !== 'note') delete body.count;
|
if (fixedSource && kind !== 'note') delete body.count;
|
||||||
|
|
@ -264,6 +265,9 @@ export function GenerateDialog({ open, onClose, classId, folderId, preset }: { o
|
||||||
<Field label={`Select-all-that-apply: ${sata}%`}>
|
<Field label={`Select-all-that-apply: ${sata}%`}>
|
||||||
<input type="range" min={0} max={100} step={10} value={sata} onChange={(e) => setSata(Number(e.target.value))} className="mt-2.5 w-full accent-[var(--accent)]" />
|
<input type="range" min={0} max={100} step={10} value={sata} onChange={(e) => setSata(Number(e.target.value))} className="mt-2.5 w-full accent-[var(--accent)]" />
|
||||||
</Field>
|
</Field>
|
||||||
|
<div className="sm:col-span-full">
|
||||||
|
<Switch checked={verify} onChange={setVerify} label="Double-check with Python" description="After writing, the model counts questions, select-all items and correct answers and recomputes calculations in a sandbox, then fixes what is wrong." />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{kind === 'deck' && (
|
{kind === 'deck' && (
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue