Study/tests/revision2.test.ts
Elijah 031f7bec31
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
Second revision: MCP servers, shared-link AI, flashcard modes, document previews
Generation
- Every quiz is NCLEX style; "Practice exam" writes 50 questions.
- Calculation questions only when the material involves calculations (max 5 per exam).
- Broad topics: the class's topics are listed in the prompt, a quick grouping call maps
  narrow labels onto broad ones, and leftover variants are merged.
- Exact select-all count per batch; option order shuffled.
- Python check: at most 3 runs, fixed problems count as passed, cosmetic remarks ignored.

Assistant
- Text, tool steps and cards appear in the order they happen, with paragraph breaks
  between steps.
- Optional thinking box: fixed height, follows the newest text, can be scrolled back.
- "+" opens a material picker listing recent items; items can be dragged in from class
  pages; shorter placeholder.

MCP servers
- Streamable HTTP client; encrypted headers.
- Per-server instructions and switches for chat, Explain, generation and the quiz check.
- Settings UI.

Share links
- Optional "Explain this" and "Ask about this material" per link, off by default.
- Items are loaded on the server and checked against the share; no tools; rate-limited.
- Migration 4 stores the switches.

Study and documents
- Standard flashcard mode (optionally repeat missed cards until known) next to spaced
  repetition.
- Streak visuals restyled in the app palette.
- Side by side / Original document views, with sandboxed previews of Word and PowerPoint
  files.

Other
- Built-in embedding model baked into the image; sharp stubbed if it cannot load; real
  load error shown in Settings.
- Python runner blocks network and process modules in JavaScript too.
- "View questions and answers" removed from the quiz start screen.
- Docs updated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 15:48:54 -07:00

391 lines
22 KiB
TypeScript

/**
* Regression tests for the second round of V2 revisions: MCP servers (with per-area use and
* instructions), broad topics, the calculation allowance, exam-length quizzes and the
* assistant's interleaved message parts.
*/
import http from 'node:http';
import type { AddressInfo } from 'node:net';
import { strToU8, unzipSync, zipSync } from 'fflate';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { buildDocx, buildPptx, TINY_PNG } from '../server/documents/testFixtures.ts';
import { queueIdle } from '../server/services/documents.ts';
import { resetShareAiLimits } from '../server/routes/shares.ts';
import { consolidateTopics, mentionsCalculations } from '../server/ai/generate.ts';
import { mcpResultText, mcpToolName } from '../server/ai/mcp.ts';
import { createTestApp, type TestApp } from './helpers.ts';
import { startMockAi, type MockAi } from './mockAi.ts';
/* ------------------------------ a tiny Streamable HTTP MCP server ------------------------------ */
interface MockMcp {
url: string;
calls: { method: string; session: string | undefined; auth: string | undefined; params: any }[];
close: () => Promise<void>;
}
function startMockMcp(): Promise<MockMcp> {
const calls: MockMcp['calls'] = [];
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on('data', (c) => chunks.push(c));
req.on('end', () => {
const msg = JSON.parse(Buffer.concat(chunks).toString('utf8') || 'null');
const session = req.headers['mcp-session-id'] as string | undefined;
calls.push({ method: msg?.method, session, auth: req.headers.authorization, params: msg?.params });
if (req.headers.authorization !== 'Bearer secret-token') {
res.writeHead(401).end('unauthorized');
return;
}
if (msg.method === 'initialize') {
res.writeHead(200, { 'content-type': 'application/json', 'mcp-session-id': 'sess-1' });
res.end(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: '2025-06-18', capabilities: { tools: {} }, serverInfo: { name: 'Mock drug ref', version: '1' } } }));
return;
}
if (session !== 'sess-1') {
res.writeHead(400).end('missing session');
return;
}
if (!('id' in msg)) {
res.writeHead(202).end();
return;
}
if (msg.method === 'tools/list') {
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: { tools: [{ name: 'lookup_drug', description: 'Look up a drug monograph.', inputSchema: { type: 'object', properties: { drug: { type: 'string' } }, required: ['drug'] } }] } }));
return;
}
if (msg.method === 'tools/call') {
// Answer as an SSE stream, with an unrelated notification first.
res.writeHead(200, { 'content-type': 'text/event-stream' });
res.write(`event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', method: 'notifications/progress', params: { progress: 1 } })}\n\n`);
res.end(`event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: { content: [{ type: 'text', text: `Monograph for ${msg.params.arguments.drug}: loop diuretic, monitor potassium.` }] } })}\n\n`);
return;
}
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({ jsonrpc: '2.0', id: msg.id, error: { code: -32601, message: 'Method not found' } }));
});
});
return new Promise((resolve) =>
server.listen(0, '127.0.0.1', () => {
const { port } = server.address() as AddressInfo;
resolve({ url: `http://127.0.0.1:${port}/mcp`, calls, close: () => new Promise((done) => server.close(() => done())) });
}),
);
}
let mock: MockAi;
let mcpServer: MockMcp;
let t: TestApp;
beforeAll(async () => {
mock = await startMockAi();
mcpServer = await startMockMcp();
});
afterAll(async () => {
await mock.close();
await mcpServer.close();
});
beforeEach(async () => {
t = await createTestApp();
mock.requests.length = 0;
mcpServer.calls.length = 0;
mock.delayChat(0);
});
afterEach(() => t.close());
async function connectAi() {
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[];
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } });
}
function parseEvents(raw: string): any[] {
return raw
.split(/\n\n/)
.map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join(''))
.filter(Boolean)
.map((d) => JSON.parse(d));
}
async function sse(url: string, body: unknown): Promise<any[]> {
const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
expect(res.status).toBe(200);
return parseEvents(await res.text());
}
const toolNames = (r: { body: any }): string[] => (r.body?.tools ?? []).map((x: any) => x.function?.name);
const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
async function addServer(extra: Record<string, unknown> = {}) {
const res = await t.api('POST', '/api/ai/mcp', { name: 'Drug ref', url: mcpServer.url, headers: { Authorization: 'Bearer secret-token' }, ...extra });
expect(res.status).toBe(201);
return res.body.server;
}
/* ------------------------------ MCP ------------------------------ */
describe('MCP servers', () => {
it('stores servers without ever returning their headers, and tests the connection', async () => {
const server = await addServer({ instructions: 'Use for drug monographs only.' });
expect(server).toMatchObject({ name: 'Drug ref', enabled: true, headerNames: ['Authorization'], instructions: 'Use for drug monographs only.', areas: { chat: true, explain: true, generate: false, check: false } });
const listed = JSON.stringify((await t.api('GET', '/api/ai/mcp')).body);
expect(listed).not.toContain('secret-token');
// Stored encrypted, not in plain text.
expect(JSON.stringify(t.ctx.db.all('SELECT value FROM settings'))).not.toContain('secret-token');
const test = (await t.api('POST', `/api/ai/mcp/${server.id}/test`)).body.result;
expect(test.ok).toBe(true);
expect(test.tools.map((x: any) => x.name)).toEqual(['lookup_drug']);
expect(mcpServer.calls.map((c) => c.method)).toEqual(['initialize', 'notifications/initialized', 'tools/list']);
// A wrong token is reported, not thrown.
await t.api('PATCH', `/api/ai/mcp/${server.id}`, { headers: { Authorization: 'Bearer wrong' } });
const bad = (await t.api('POST', `/api/ai/mcp/${server.id}/test`)).body.result;
expect(bad.ok).toBe(false);
expect(bad.message).toMatch(/401/);
expect((await t.api('GET', '/api/ai/mcp')).body.servers[0].lastTestOk).toBe(false);
expect((await t.api('POST', '/api/ai/mcp', { name: 'x', url: 'ftp://example.com' })).status).toBe(422);
expect((await t.api('DELETE', `/api/ai/mcp/${server.id}`)).status).toBe(200);
expect((await t.api('GET', '/api/ai/mcp')).body.servers).toEqual([]);
});
it('lets the assistant call an MCP tool, with the user instructions in its prompt', async () => {
await connectAi();
await addServer({ instructions: 'Use for drug monographs only.' });
const events = await sse('/api/ai/chat', { message: 'Use the mcp drug reference for furosemide' });
const tools = events.filter((e) => e.type === 'tool').map((e) => e.tool);
expect(tools.some((x) => x.label === 'Drug ref: lookup_drug' && x.status === 'done')).toBe(true);
const call = mcpServer.calls.find((c) => c.method === 'tools/call')!;
expect(call.params).toEqual({ name: 'lookup_drug', arguments: { drug: 'furosemide' } });
expect(call.session).toBe('sess-1');
const first = chatCalls()[0]!;
expect(toolNames(first)).toContain('mcp_drug_ref__lookup_drug');
expect(JSON.stringify(first.body.messages[0])).toContain('Use for drug monographs only.');
// The tool output reached the model.
expect(JSON.stringify(chatCalls().at(-1)!.body.messages)).toContain('Monograph for furosemide');
});
it('offers a server only in the areas it is allowed in', async () => {
await connectAi();
const server = await addServer();
await t.api('PATCH', `/api/ai/mcp/${server.id}`, { areas: { chat: false } });
await sse('/api/ai/chat', { message: 'Use the mcp drug reference for furosemide' });
expect(chatCalls().every((r) => !toolNames(r).some((n) => n.startsWith('mcp_')))).toBe(true);
expect(mcpServer.calls.some((c) => c.method === 'tools/call')).toBe(false);
// Explain is on by default: the explanation may look things up first.
mock.requests.length = 0;
const events = await sse('/api/ai/explain', { kind: 'card', front: 'Furosemide', back: 'Loop diuretic', ask: 'Check the mcp reference too' });
expect(events.some((e) => e.type === 'text')).toBe(true);
expect(events.at(-1).type).toBe('done');
expect(mcpServer.calls.some((c) => c.method === 'tools/call')).toBe(true);
// Generation is off by default, so its requests never list MCP tools…
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
mock.requests.length = 0;
let draftId = (await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'mcp diuretics', count: 2 })).body.draftId;
let draft = await waitForDraft(draftId);
expect(draft.status).toBe('ready');
expect(chatCalls().every((r) => !toolNames(r).some((n) => n.startsWith('mcp_')))).toBe(true);
// …until it is switched on: then a lookup can come before the cards are submitted.
await t.api('PATCH', `/api/ai/mcp/${server.id}`, { areas: { generate: true } });
mcpServer.calls.length = 0;
draftId = (await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'mcp diuretics', count: 2 })).body.draftId;
draft = await waitForDraft(draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.cards.length).toBeGreaterThan(0);
expect(mcpServer.calls.some((c) => c.method === 'tools/call')).toBe(true);
});
it('builds safe tool names and readable results', () => {
const taken = new Set<string>();
expect(mcpToolName('Web Search!', 'search.web', taken)).toBe('mcp_web_search__search_web');
expect(mcpToolName('Web Search!', 'search.web', taken)).toBe('mcp_web_search__search_web_2');
expect(mcpToolName('x'.repeat(80), 'y'.repeat(80), new Set()).length).toBeLessThanOrEqual(64);
expect(mcpResultText({ content: [{ type: 'text', text: 'a' }, { type: 'image', data: '...' }] })).toBe('a\n\n[image omitted]');
expect(mcpResultText({ content: [], structuredContent: { ok: 1 } })).toBe('{"ok":1}');
});
});
async function waitForDraft(id: string) {
for (let i = 0; i < 400; i += 1) {
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
if (d.status !== 'generating') return d;
await new Promise((r) => setTimeout(r, 25));
}
throw new Error('draft never finished');
}
/* ------------------------------ generation ------------------------------ */
describe('quiz generation rules', () => {
it('merges narrow topics into a broader one that exists', () => {
const qs = [{ topic: 'Stroke' }, { topic: 'Hemorrhagic stroke' }, { topic: 'ischemic stroke' }, { topic: 'stroke' }, { topic: 'Heart failure' }];
expect(consolidateTopics(qs).map((q) => q.topic)).toEqual(['Stroke', 'Stroke', 'Stroke', 'Stroke', 'Heart failure']);
// A class's existing topic wins over new spellings.
expect(consolidateTopics([{ topic: 'Right hemisphere stroke' }], ['Stroke']).map((q) => q.topic)).toEqual(['Stroke']);
// Unrelated topics that share a word are left alone.
expect(consolidateTopics([{ topic: 'Heart failure' }, { topic: 'Renal failure' }]).map((q) => q.topic)).toEqual(['Heart failure', 'Renal failure']);
});
it('allows calculation questions only when the material involves calculations', () => {
expect(mentionsCalculations('Calculate the infusion rate in mL/hr.')).toBe(true);
expect(mentionsCalculations('Give 5 mg/kg IV.')).toBe(true);
expect(mentionsCalculations('Signs of right-sided heart failure include edema.')).toBe(false);
});
it('writes 50 questions for a practice exam, with no calculations when the material has none', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Med-surg' })).body.class;
const { draftId } = (await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'heart failure', format: 'exam', verify: false })).body;
const draft = await waitForDraft(draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.questions.length).toBe(50);
const prompts = chatCalls().map((r) => JSON.stringify(r.body.messages));
expect(prompts.some((p) => /calculation/i.test(p) && /none|0/.test(p))).toBe(true);
// Topic labels go through one grouping call that asks for broad topics.
const grouping = chatCalls().filter((r) => toolNames(r).includes('submit_topics'));
expect(grouping.length).toBe(1);
expect(JSON.stringify(grouping[0]!.body.messages)).toContain('broad exam topic');
});
});
/* ------------------------------ original document preview ------------------------------ */
/** A deck whose second slide has a picture and an EMF the browser can't show. */
function deckWithPicture(): Uint8Array {
const files = unzipSync(buildPptx([{ file: 1, title: 'Fluid balance', bullets: ['Daily weights'] }, { file: 2, title: 'Edema grading', bullets: ['Pitting scale 1+ to 4+'] }]));
const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships';
files['ppt/slides/_rels/slide2.xml.rels'] = strToU8(
`<?xml version="1.0" encoding="UTF-8"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId2" Type="${REL}/image" Target="../media/image1.png"/><Relationship Id="rId3" Type="${REL}/image" Target="../media/image2.emf"/></Relationships>`,
);
files['ppt/media/image1.png'] = TINY_PNG;
files['ppt/media/image2.emf'] = new Uint8Array([1, 2, 3]);
return zipSync(files);
}
async function upload(classId: string, filename: string, bytes: Uint8Array) {
const form = new FormData();
form.append('classId', classId);
form.append('file', new File([Buffer.from(bytes)], filename));
const res = await t.raw('POST', '/api/documents', { body: form });
const body = (await res.json()) as any;
await queueIdle(t.ctx);
return body.documents[0];
}
describe('original document view', () => {
it('shows a PowerPoint deck with its slide pictures in a locked-down page', async () => {
const cls = (await t.api('POST', '/api/classes', { name: 'Fundamentals' })).body.class;
const doc = await upload(cls.id, 'fluids.pptx', deckWithPicture());
const res = await t.raw('GET', `/api/documents/${doc.id}/preview`);
expect(res.status).toBe(200);
expect(res.headers.get('content-security-policy')).toContain("default-src 'none'");
expect(res.headers.get('content-security-policy')).toContain('sandbox');
const html = await res.text();
expect(html).toContain('Slide 2');
expect(html).toContain('Edema grading');
expect(html).toContain('data:image/png;base64,');
expect(html).toContain("1 picture on this slide can't be shown here.");
expect(html).not.toMatch(/<script/i);
});
it('shows a Word document, keeps text escaped, and refuses other kinds', async () => {
const cls = (await t.api('POST', '/api/classes', { name: 'Ethics' })).body.class;
const doc = await upload(cls.id, 'consent.docx', buildDocx([{ type: 'heading', level: 1, text: 'Informed consent' }, { type: 'p', text: 'Use <b>teach-back</b> & check understanding.' }]));
const html = await (await t.raw('GET', `/api/documents/${doc.id}/preview`)).text();
expect(html).toContain('Informed consent');
expect(html).toContain('&lt;b&gt;teach-back&lt;/b&gt;');
const text = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Notes', text: 'Plain text.' })).body.document;
expect((await t.raw('GET', `/api/documents/${text.id}/preview`)).status).toBe(404);
expect((await t.raw('GET', `/api/documents/${doc.id}/preview`, { cookie: null })).status).toBe(401);
});
});
/* ------------------------------ AI on share links ------------------------------ */
describe('AI on share links', () => {
async function sharedQuiz() {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Cardiac' })).body.class;
const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Digoxin' })).body.quiz;
const questions = (await t.api('POST', `/api/quizzes/${quiz.id}/questions`, { questions: [{ type: 'single', stem: 'Which finding suggests digoxin toxicity?', options: [{ text: 'Yellow-green halos', correct: true }, { text: 'Hypertension', correct: false }], rationale: 'Visual changes are classic.' }] })).body.questions as any[];
const other = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Private quiz' })).body.quiz;
const otherQ = (await t.api('POST', `/api/quizzes/${other.id}/questions`, { questions: [{ type: 'single', stem: 'Secret private stem', options: [{ text: 'A', correct: true }, { text: 'B', correct: false }] }] })).body.questions[0];
const share = (await t.api('POST', '/api/shares', { kind: 'quiz', targetId: quiz.id })).body.share;
return { quiz, question: questions[0], other, otherQ, share };
}
const visitor = (method: string, url: string, body?: unknown) => t.raw(method, url, { cookie: null, headers: { 'content-type': 'application/json' }, body: body === undefined ? undefined : JSON.stringify(body) });
it('is off until the owner turns it on, and then only explains what was shared', async () => {
resetShareAiLimits();
const { quiz, question, other, otherQ, share } = await sharedQuiz();
expect(share.ai).toEqual({ explain: false, assistant: false });
expect((await t.api('GET', `/api/public/${share.token}`, undefined, { cookie: null })).body.ai).toEqual({ explain: false, assistant: false });
const ask = { kind: 'question', quizId: quiz.id, questionId: question.id, selected: ['Hypertension'] };
expect((await visitor('POST', `/api/public/${share.token}/explain`, ask)).status).toBe(403);
const updated = (await t.api('PATCH', `/api/shares/${share.id}`, { ai: { explain: true } })).body.share;
expect(updated).toMatchObject({ enabled: true, ai: { explain: true, assistant: false } });
expect((await t.api('GET', `/api/public/${share.token}`, undefined, { cookie: null })).body.ai.explain).toBe(true);
mock.requests.length = 0;
const res = await visitor('POST', `/api/public/${share.token}/explain`, ask);
expect(res.status).toBe(200);
const events = parseEvents(await res.text());
expect(events.some((e) => e.type === 'text')).toBe(true);
const sent = JSON.stringify(chatCalls()[0]!.body.messages);
// The question is loaded on the server, with the visitor's choice checked against its options.
expect(sent).toContain('Which finding suggests digoxin toxicity?');
expect(sent).toContain('The student chose: Hypertension');
expect(chatCalls()[0]!.body.tools).toBeUndefined();
// Something outside the share is refused, and so is the chat while it is off.
expect((await visitor('POST', `/api/public/${share.token}/explain`, { kind: 'question', quizId: other.id, questionId: otherQ.id })).status).toBe(404);
expect((await visitor('POST', `/api/public/${share.token}/chat`, { messages: [{ role: 'user', text: 'hi' }] })).status).toBe(403);
});
it('answers questions from the shared material only, without tools, and limits each visitor', async () => {
resetShareAiLimits();
const { share } = await sharedQuiz();
await t.api('PATCH', `/api/shares/${share.id}`, { ai: { assistant: true } });
mock.requests.length = 0;
const res = await visitor('POST', `/api/public/${share.token}/chat`, { messages: [{ role: 'user', text: 'Why are halos a sign of toxicity?' }] });
expect(res.status).toBe(200);
expect(parseEvents(await res.text()).some((e) => e.type === 'text')).toBe(true);
const call = chatCalls()[0]!;
const system = JSON.stringify(call.body.messages[0]);
expect(system).toContain('Which finding suggests digoxin toxicity?');
expect(system).not.toContain('Secret private stem');
expect(call.body.tools).toBeUndefined();
// The last message must come from the visitor.
expect((await visitor('POST', `/api/public/${share.token}/chat`, { messages: [{ role: 'assistant', text: 'x' }] })).status).toBe(422);
let limited = 0;
for (let i = 0; i < 22; i += 1) {
const r = await visitor('POST', `/api/public/${share.token}/chat`, { messages: [{ role: 'user', text: `Question ${i}` }] });
await r.text();
if (r.status === 429) limited += 1;
}
expect(limited).toBeGreaterThan(0);
});
});
/* ------------------------------ assistant message parts ------------------------------ */
describe('assistant transcript', () => {
it('keeps text, tool steps and reasoning in the order they happened', async () => {
await connectAi();
await addServer();
const events = await sse('/api/ai/chat', { message: 'Use the mcp drug reference for furosemide' });
const conversationId = events.find((e) => e.type === 'start').conversationId;
expect(events.some((e) => e.type === 'step')).toBe(true);
const convo = (await t.api('GET', `/api/ai/conversations/${conversationId}`)).body;
const reply = convo.messages.find((m: any) => m.role === 'assistant');
const kinds = reply.parts.map((p: any) => p.kind);
// The tool step comes before the answer that used it.
expect(kinds.indexOf('tool')).toBeLessThan(kinds.lastIndexOf('text'));
});
});