Add sandboxed Python for the assistant and a Python check of generated quizzes
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 51s

- Pyodide (Python 3.14, WebAssembly) in a separate Node process under the permission
  model: read-only access to Pyodide's files, no writes, processes or network, empty
  environment, JS bridges removed, time limit with kill and restart.
- Assistant tool run_python, which can load a draft or quiz as Python data.
- After quiz generation the model checks question count, type mix, correct answers per
  type, duplicates and recomputed calculations with Python, then submits fixes that are
  applied only if the question still validates. Switch in the Generate dialog.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-15 13:03:24 -07:00
parent f0aabe4d5d
commit 5ea61c882c
13 changed files with 800 additions and 6 deletions

View file

@ -50,6 +50,9 @@ async function sse(url: string, body: unknown): Promise<any[]> {
.map((d) => JSON.parse(d));
}
/** A request from the Python quiz check rather than from generation itself. */
const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review');
async function waitForDraft(id: string) {
for (let i = 0; i < 200; i += 1) {
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
@ -195,8 +198,16 @@ describe('generation', () => {
// Citations were mapped from excerpt ids to readable labels.
expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/);
// 12 questions at 8 per batch means two model calls, each carrying the source excerpts.
const calls = mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
const all = mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
const calls = all.filter((r) => !isCheck(r));
expect(calls).toHaveLength(2);
// Then the Python check: one run, a review, and its fix applied to question 2.
const checks = all.filter(isCheck);
expect(checks.length).toBeGreaterThanOrEqual(2);
const pythonResult = JSON.parse(checks[1].body.messages.find((m: any) => m.role === 'tool').content);
expect(pythonResult).toMatchObject({ ok: true, stdout: '12 12 4 [1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2]\n' });
expect(draft.payload.questions[1].options.map((o: any) => o.correct)).toEqual([false, true, false, false]);
expect(draft.meta.notes.join(' ')).toMatch(/Checked with Python \(1 run\): 2 of 2 checks passed, 1 fix applied/);
expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/);
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/);
expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } });
@ -237,7 +248,7 @@ describe('generation', () => {
// Deck -> quiz conversion uses the cards as the source.
const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId);
expect(converted.status).toBe('ready');
expect(JSON.stringify(mock.requests.at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
// Append more cards to the existing deck.
const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId);
@ -265,6 +276,7 @@ describe('generation', () => {
const draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.questions).toHaveLength(3);
expect(draft.meta.notes.join(' ')).toMatch(/check was skipped/);
const call = mock.requests.at(-1)!;
expect(call.body.tools).toBeUndefined();
expect(call.body.response_format).toEqual({ type: 'json_object' });