/** * The Python sandbox (Pyodide in a permission-restricted Node process), the assistant's * run_python tool and the fixes the quiz check may apply. */ import { afterAll, afterEach, beforeEach, describe, expect, it } from 'vitest'; import type { DraftQuestion } from '../shared/content.ts'; import { runToolForTest } from '../server/ai/assistant.ts'; import { createDraft } from '../server/ai/generate.ts'; import { runPython, stopPython } from '../server/ai/sandbox.ts'; import { applyQuizFixes, pythonPrelude } from '../server/ai/verify.ts'; import { createTestApp, type TestApp } from './helpers.ts'; afterAll(() => stopPython()); describe('python sandbox', () => { it('runs standard-library code and returns output and the last value', async () => { const r = await runPython('from fractions import Fraction\nimport statistics\nprint(5000 / 10000)\nround(73 * 1.2, 1)'); expect(r).toMatchObject({ ok: true, stdout: '0.5\n', result: '87.6' }); expect((await runPython('statistics.mean([1, 2])')).error).toMatch(/NameError/); // each run starts clean }, 60_000); it('reports Python errors without the JavaScript frames', async () => { const r = await runPython('1 / 0'); expect(r.ok).toBe(false); expect(r.error).toMatch(/^Traceback/); expect(r.error).toMatch(/ZeroDivisionError/); }, 60_000); it('has no way to the host: no JavaScript bridge, files, environment, processes or network', async () => { for (const code of ['import js', 'import pyodide_js', 'from pyodide.code import run_js\nrun_js("process.env")', 'import micropip']) { const r = await runPython(code); expect(r.ok, code).toBe(false); expect(r.error, code).toMatch(/not available in this sandbox/); } expect((await runPython('open("/etc/passwd").read()')).error).toMatch(/FileNotFoundError/); expect((await runPython(`open(${JSON.stringify(process.cwd() + '/package.json')}).read()`)).error).toMatch(/FileNotFoundError/); expect((await runPython('import os\nsorted(os.environ)')).result).not.toMatch(/APP_SECRET|PATH.*Windows|DATA_DIR/); expect((await runPython('import subprocess\nsubprocess.run(["ls"])')).error).toMatch(/does not support processes/); expect((await runPython('import urllib.request\nurllib.request.urlopen("http://127.0.0.1:1")')).ok).toBe(false); }, 60_000); it('stops runaway code at the time limit and recovers for the next run', async () => { const r = await runPython('while True:\n pass', { timeoutMs: 1500 }); expect(r).toMatchObject({ ok: false, timedOut: true }); expect(r.error).toMatch(/time limit/); expect((await runPython('sum(range(10))')).result).toBe('45'); }, 60_000); it('serialises concurrent runs', async () => { const results = await Promise.all([1, 2, 3, 4].map((n) => runPython(`${n} * 10`))); expect(results.map((r) => r.result)).toEqual(['10', '20', '30', '40']); }, 60_000); it('hands data to Python through JSON', async () => { const r = await runPython(`${pythonPrelude({ draft: { questions: [{ stem: 'Quote "this" \\ and ü' }] } })}print(draft["questions"][0]["stem"])`); expect(r.stdout).toBe('Quote "this" \\ and ü\n'); }, 60_000); }); describe('quiz check fixes', () => { const q = (type: DraftQuestion['type'], correct: number[], n = 4): DraftQuestion => ({ type, stem: `Stem ${type}`, options: Array.from({ length: n }, (_, i) => ({ text: `Option ${i + 1}`, correct: correct.includes(i), why: '' })), rationale: 'r', topic: '', difficulty: 'medium', tags: [], }); it('applies valid fixes and refuses ones that would break a question', () => { const questions = [q('single', [0, 1, 2, 3]), q('numeric', [0], 1), q('multi', [0, 1], 5), q('single', [0])]; questions[1].options[0].text = '12 mL'; const { questions: out, applied, refused } = applyQuizFixes(questions, [ { question: 1, action: 'set_correct', correct_options: [2], reason: 'B is right' }, { question: 2, action: 'set_numeric_answer', answer: '12.5 mL', reason: 'recomputed' }, { question: 3, action: 'set_correct', correct_options: [], reason: 'empty' }, { question: 4, action: 'set_numeric_answer', answer: '3', reason: 'not numeric' }, { question: 9, action: 'remove', reason: 'missing' }, ]); expect(out[0].options.map((o) => o.correct)).toEqual([false, true, false, false]); expect(out[1].options).toEqual([{ text: '12.5 mL', correct: true, why: '' }]); expect(out[2].options.filter((o) => o.correct)).toHaveLength(2); expect(applied).toHaveLength(2); expect(refused).toHaveLength(3); }); it('removes questions and turns a several-answer single into select-all', () => { const { questions: out } = applyQuizFixes([q('single', [0]), q('single', [0])], [ { question: 1, action: 'remove', reason: 'duplicate' }, { question: 2, action: 'set_correct', correct_options: [1, 3], reason: 'two are right' }, ]); expect(out).toHaveLength(1); expect(out[0].type).toBe('multi'); }); }); describe('assistant run_python tool', () => { let t: TestApp; beforeEach(async () => { t = await createTestApp(); }); afterEach(() => t.close()); it('loads a draft or a saved quiz as Python data', async () => { const draftId = createDraft(t.ctx.db, { kind: 'quiz', title: 'Calc', payload: { kind: 'quiz', title: 'Calc', description: '', questions: [{ type: 'numeric', stem: '500 mL over 4 h?', options: [{ text: '125 mL/hr', correct: true }], rationale: '', topic: '', difficulty: 'medium', tags: [] }] }, meta: { request: { count: 1 } }, }); const run = await runToolForTest(t.ctx, 'run_python', { draft_id: draftId, code: 'q = draft["questions"][0]\nprint(len(draft["questions"]), q["type"], 500 / 4 == float(q["options"][0]["text"].split()[0]), draft["request"]["count"])' }); expect(run.output).toMatchObject({ ok: true, stdout: '1 numeric True 1\n' }); expect(run.summary).toBe('1 numeric True 1'); const cls = (await t.api('POST', '/api/classes', { name: 'C' })).body.class; const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Q', questions: [{ type: 'multi', stem: 'Pick', options: [{ text: 'a', correct: true }, { text: 'b', correct: true }, { text: 'c', correct: false }] }] })).body.quiz; const counted = await runToolForTest(t.ctx, 'run_python', { quiz_id: quiz.id, code: '[sum(o["correct"] for o in q["options"]) for q in quiz["questions"]]' }); expect(counted.output).toMatchObject({ ok: true, result: '[2]' }); const failing = await runToolForTest(t.ctx, 'run_python', { code: 'undefined_name' }); expect(failing.output).toMatchObject({ ok: false }); expect(failing.summary).toMatch(/NameError/); }, 60_000); });