Add sandboxed Python for the assistant and a Python check of generated quizzes
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 51s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 51s
- Pyodide (Python 3.14, WebAssembly) in a separate Node process under the permission model: read-only access to Pyodide's files, no writes, processes or network, empty environment, JS bridges removed, time limit with kill and restart. - Assistant tool run_python, which can load a draft or quiz as Python data. - After quiz generation the model checks question count, type mix, correct answers per type, duplicates and recomputed calculations with Python, then submits fixes that are applied only if the question still validates. Switch in the Generate dialog. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f0aabe4d5d
commit
5ea61c882c
13 changed files with 800 additions and 6 deletions
125
tests/python.test.ts
Normal file
125
tests/python.test.ts
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
/**
|
||||
* The Python sandbox (Pyodide in a permission-restricted Node process), the assistant's
|
||||
* run_python tool and the fixes the quiz check may apply.
|
||||
*/
|
||||
import { afterAll, afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { DraftQuestion } from '../shared/content.ts';
|
||||
import { runToolForTest } from '../server/ai/assistant.ts';
|
||||
import { createDraft } from '../server/ai/generate.ts';
|
||||
import { runPython, stopPython } from '../server/ai/sandbox.ts';
|
||||
import { applyQuizFixes, pythonPrelude } from '../server/ai/verify.ts';
|
||||
import { createTestApp, type TestApp } from './helpers.ts';
|
||||
|
||||
afterAll(() => stopPython());
|
||||
|
||||
describe('python sandbox', () => {
|
||||
it('runs standard-library code and returns output and the last value', async () => {
|
||||
const r = await runPython('from fractions import Fraction\nimport statistics\nprint(5000 / 10000)\nround(73 * 1.2, 1)');
|
||||
expect(r).toMatchObject({ ok: true, stdout: '0.5\n', result: '87.6' });
|
||||
expect((await runPython('statistics.mean([1, 2])')).error).toMatch(/NameError/); // each run starts clean
|
||||
}, 60_000);
|
||||
|
||||
it('reports Python errors without the JavaScript frames', async () => {
|
||||
const r = await runPython('1 / 0');
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.error).toMatch(/^Traceback/);
|
||||
expect(r.error).toMatch(/ZeroDivisionError/);
|
||||
}, 60_000);
|
||||
|
||||
it('has no way to the host: no JavaScript bridge, files, environment, processes or network', async () => {
|
||||
for (const code of ['import js', 'import pyodide_js', 'from pyodide.code import run_js\nrun_js("process.env")', 'import micropip']) {
|
||||
const r = await runPython(code);
|
||||
expect(r.ok, code).toBe(false);
|
||||
expect(r.error, code).toMatch(/not available in this sandbox/);
|
||||
}
|
||||
expect((await runPython('open("/etc/passwd").read()')).error).toMatch(/FileNotFoundError/);
|
||||
expect((await runPython(`open(${JSON.stringify(process.cwd() + '/package.json')}).read()`)).error).toMatch(/FileNotFoundError/);
|
||||
expect((await runPython('import os\nsorted(os.environ)')).result).not.toMatch(/APP_SECRET|PATH.*Windows|DATA_DIR/);
|
||||
expect((await runPython('import subprocess\nsubprocess.run(["ls"])')).error).toMatch(/does not support processes/);
|
||||
expect((await runPython('import urllib.request\nurllib.request.urlopen("http://127.0.0.1:1")')).ok).toBe(false);
|
||||
}, 60_000);
|
||||
|
||||
it('stops runaway code at the time limit and recovers for the next run', async () => {
|
||||
const r = await runPython('while True:\n pass', { timeoutMs: 1500 });
|
||||
expect(r).toMatchObject({ ok: false, timedOut: true });
|
||||
expect(r.error).toMatch(/time limit/);
|
||||
expect((await runPython('sum(range(10))')).result).toBe('45');
|
||||
}, 60_000);
|
||||
|
||||
it('serialises concurrent runs', async () => {
|
||||
const results = await Promise.all([1, 2, 3, 4].map((n) => runPython(`${n} * 10`)));
|
||||
expect(results.map((r) => r.result)).toEqual(['10', '20', '30', '40']);
|
||||
}, 60_000);
|
||||
|
||||
it('hands data to Python through JSON', async () => {
|
||||
const r = await runPython(`${pythonPrelude({ draft: { questions: [{ stem: 'Quote "this" \\ and ü' }] } })}print(draft["questions"][0]["stem"])`);
|
||||
expect(r.stdout).toBe('Quote "this" \\ and ü\n');
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('quiz check fixes', () => {
|
||||
const q = (type: DraftQuestion['type'], correct: number[], n = 4): DraftQuestion => ({
|
||||
type,
|
||||
stem: `Stem ${type}`,
|
||||
options: Array.from({ length: n }, (_, i) => ({ text: `Option ${i + 1}`, correct: correct.includes(i), why: '' })),
|
||||
rationale: 'r',
|
||||
topic: '',
|
||||
difficulty: 'medium',
|
||||
tags: [],
|
||||
});
|
||||
|
||||
it('applies valid fixes and refuses ones that would break a question', () => {
|
||||
const questions = [q('single', [0, 1, 2, 3]), q('numeric', [0], 1), q('multi', [0, 1], 5), q('single', [0])];
|
||||
questions[1].options[0].text = '12 mL';
|
||||
const { questions: out, applied, refused } = applyQuizFixes(questions, [
|
||||
{ question: 1, action: 'set_correct', correct_options: [2], reason: 'B is right' },
|
||||
{ question: 2, action: 'set_numeric_answer', answer: '12.5 mL', reason: 'recomputed' },
|
||||
{ question: 3, action: 'set_correct', correct_options: [], reason: 'empty' },
|
||||
{ question: 4, action: 'set_numeric_answer', answer: '3', reason: 'not numeric' },
|
||||
{ question: 9, action: 'remove', reason: 'missing' },
|
||||
]);
|
||||
expect(out[0].options.map((o) => o.correct)).toEqual([false, true, false, false]);
|
||||
expect(out[1].options).toEqual([{ text: '12.5 mL', correct: true, why: '' }]);
|
||||
expect(out[2].options.filter((o) => o.correct)).toHaveLength(2);
|
||||
expect(applied).toHaveLength(2);
|
||||
expect(refused).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('removes questions and turns a several-answer single into select-all', () => {
|
||||
const { questions: out } = applyQuizFixes([q('single', [0]), q('single', [0])], [
|
||||
{ question: 1, action: 'remove', reason: 'duplicate' },
|
||||
{ question: 2, action: 'set_correct', correct_options: [1, 3], reason: 'two are right' },
|
||||
]);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0].type).toBe('multi');
|
||||
});
|
||||
});
|
||||
|
||||
describe('assistant run_python tool', () => {
|
||||
let t: TestApp;
|
||||
beforeEach(async () => {
|
||||
t = await createTestApp();
|
||||
});
|
||||
afterEach(() => t.close());
|
||||
|
||||
it('loads a draft or a saved quiz as Python data', async () => {
|
||||
const draftId = createDraft(t.ctx.db, {
|
||||
kind: 'quiz',
|
||||
title: 'Calc',
|
||||
payload: { kind: 'quiz', title: 'Calc', description: '', questions: [{ type: 'numeric', stem: '500 mL over 4 h?', options: [{ text: '125 mL/hr', correct: true }], rationale: '', topic: '', difficulty: 'medium', tags: [] }] },
|
||||
meta: { request: { count: 1 } },
|
||||
});
|
||||
const run = await runToolForTest(t.ctx, 'run_python', { draft_id: draftId, code: 'q = draft["questions"][0]\nprint(len(draft["questions"]), q["type"], 500 / 4 == float(q["options"][0]["text"].split()[0]), draft["request"]["count"])' });
|
||||
expect(run.output).toMatchObject({ ok: true, stdout: '1 numeric True 1\n' });
|
||||
expect(run.summary).toBe('1 numeric True 1');
|
||||
|
||||
const cls = (await t.api('POST', '/api/classes', { name: 'C' })).body.class;
|
||||
const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Q', questions: [{ type: 'multi', stem: 'Pick', options: [{ text: 'a', correct: true }, { text: 'b', correct: true }, { text: 'c', correct: false }] }] })).body.quiz;
|
||||
const counted = await runToolForTest(t.ctx, 'run_python', { quiz_id: quiz.id, code: '[sum(o["correct"] for o in q["options"]) for q in quiz["questions"]]' });
|
||||
expect(counted.output).toMatchObject({ ok: true, result: '[2]' });
|
||||
|
||||
const failing = await runToolForTest(t.ctx, 'run_python', { code: 'undefined_name' });
|
||||
expect(failing.output).toMatchObject({ ok: false });
|
||||
expect(failing.summary).toMatch(/NameError/);
|
||||
}, 60_000);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue