/** * Regression tests for the second round of V2 revisions: MCP servers (with per-area use and * instructions), broad topics, the calculation allowance, exam-length quizzes and the * assistant's interleaved message parts. */ import http from 'node:http'; import type { AddressInfo } from 'node:net'; import { strToU8, unzipSync, zipSync } from 'fflate'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { buildDocx, buildPptx, TINY_PNG } from '../server/documents/testFixtures.ts'; import { queueIdle } from '../server/services/documents.ts'; import { resetShareAiLimits } from '../server/routes/shares.ts'; import { consolidateTopics, mentionsCalculations } from '../server/ai/generate.ts'; import { mcpResultText, mcpToolName } from '../server/ai/mcp.ts'; import { createTestApp, type TestApp } from './helpers.ts'; import { startMockAi, type MockAi } from './mockAi.ts'; /* ------------------------------ a tiny Streamable HTTP MCP server ------------------------------ */ interface MockMcp { url: string; calls: { method: string; session: string | undefined; auth: string | undefined; params: any }[]; close: () => Promise; } function startMockMcp(): Promise { const calls: MockMcp['calls'] = []; const server = http.createServer((req, res) => { const chunks: Buffer[] = []; req.on('data', (c) => chunks.push(c)); req.on('end', () => { const msg = JSON.parse(Buffer.concat(chunks).toString('utf8') || 'null'); const session = req.headers['mcp-session-id'] as string | undefined; calls.push({ method: msg?.method, session, auth: req.headers.authorization, params: msg?.params }); if (req.headers.authorization !== 'Bearer secret-token') { res.writeHead(401).end('unauthorized'); return; } if (msg.method === 'initialize') { res.writeHead(200, { 'content-type': 'application/json', 'mcp-session-id': 'sess-1' }); res.end(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: '2025-06-18', capabilities: { tools: {} }, serverInfo: { name: 'Mock drug ref', version: '1' } } })); return; } if (session !== 'sess-1') { res.writeHead(400).end('missing session'); return; } if (!('id' in msg)) { res.writeHead(202).end(); return; } if (msg.method === 'tools/list') { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: { tools: [{ name: 'lookup_drug', description: 'Look up a drug monograph.', inputSchema: { type: 'object', properties: { drug: { type: 'string' } }, required: ['drug'] } }] } })); return; } if (msg.method === 'tools/call') { // Answer as an SSE stream, with an unrelated notification first. res.writeHead(200, { 'content-type': 'text/event-stream' }); res.write(`event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', method: 'notifications/progress', params: { progress: 1 } })}\n\n`); res.end(`event: message\ndata: ${JSON.stringify({ jsonrpc: '2.0', id: msg.id, result: { content: [{ type: 'text', text: `Monograph for ${msg.params.arguments.drug}: loop diuretic, monitor potassium.` }] } })}\n\n`); return; } res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ jsonrpc: '2.0', id: msg.id, error: { code: -32601, message: 'Method not found' } })); }); }); return new Promise((resolve) => server.listen(0, '127.0.0.1', () => { const { port } = server.address() as AddressInfo; resolve({ url: `http://127.0.0.1:${port}/mcp`, calls, close: () => new Promise((done) => server.close(() => done())) }); }), ); } let mock: MockAi; let mcpServer: MockMcp; let t: TestApp; beforeAll(async () => { mock = await startMockAi(); mcpServer = await startMockMcp(); }); afterAll(async () => { await mock.close(); await mcpServer.close(); }); beforeEach(async () => { t = await createTestApp(); mock.requests.length = 0; mcpServer.calls.length = 0; mock.delayChat(0); }); afterEach(() => t.close()); async function connectAi() { const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider; const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[]; for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } }); } function parseEvents(raw: string): any[] { return raw .split(/\n\n/) .map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join('')) .filter(Boolean) .map((d) => JSON.parse(d)); } async function sse(url: string, body: unknown): Promise { const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); expect(res.status).toBe(200); return parseEvents(await res.text()); } const toolNames = (r: { body: any }): string[] => (r.body?.tools ?? []).map((x: any) => x.function?.name); const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions')); async function addServer(extra: Record = {}) { const res = await t.api('POST', '/api/ai/mcp', { name: 'Drug ref', url: mcpServer.url, headers: { Authorization: 'Bearer secret-token' }, ...extra }); expect(res.status).toBe(201); return res.body.server; } /* ------------------------------ MCP ------------------------------ */ describe('MCP servers', () => { it('stores servers without ever returning their headers, and tests the connection', async () => { const server = await addServer({ instructions: 'Use for drug monographs only.' }); expect(server).toMatchObject({ name: 'Drug ref', enabled: true, headerNames: ['Authorization'], instructions: 'Use for drug monographs only.', areas: { chat: true, explain: true, generate: false, check: false } }); const listed = JSON.stringify((await t.api('GET', '/api/ai/mcp')).body); expect(listed).not.toContain('secret-token'); // Stored encrypted, not in plain text. expect(JSON.stringify(t.ctx.db.all('SELECT value FROM settings'))).not.toContain('secret-token'); const test = (await t.api('POST', `/api/ai/mcp/${server.id}/test`)).body.result; expect(test.ok).toBe(true); expect(test.tools.map((x: any) => x.name)).toEqual(['lookup_drug']); expect(mcpServer.calls.map((c) => c.method)).toEqual(['initialize', 'notifications/initialized', 'tools/list']); // A wrong token is reported, not thrown. await t.api('PATCH', `/api/ai/mcp/${server.id}`, { headers: { Authorization: 'Bearer wrong' } }); const bad = (await t.api('POST', `/api/ai/mcp/${server.id}/test`)).body.result; expect(bad.ok).toBe(false); expect(bad.message).toMatch(/did not accept the sign-in or token/); expect((await t.api('GET', '/api/ai/mcp')).body.servers[0].lastTestOk).toBe(false); expect((await t.api('POST', '/api/ai/mcp', { name: 'x', url: 'ftp://example.com' })).status).toBe(422); expect((await t.api('DELETE', `/api/ai/mcp/${server.id}`)).status).toBe(200); expect((await t.api('GET', '/api/ai/mcp')).body.servers).toEqual([]); }); it('lets the assistant call an MCP tool, with the user instructions in its prompt', async () => { await connectAi(); await addServer({ instructions: 'Use for drug monographs only.' }); const events = await sse('/api/ai/chat', { message: 'Use the mcp drug reference for furosemide' }); const tools = events.filter((e) => e.type === 'tool').map((e) => e.tool); expect(tools.some((x) => x.label === 'Drug ref: Lookup drug' && x.status === 'done')).toBe(true); const call = mcpServer.calls.find((c) => c.method === 'tools/call')!; expect(call.params).toEqual({ name: 'lookup_drug', arguments: { drug: 'furosemide' } }); expect(call.session).toBe('sess-1'); const first = chatCalls()[0]!; expect(toolNames(first)).toContain('mcp_drug_ref__lookup_drug'); expect(JSON.stringify(first.body.messages[0])).toContain('Use for drug monographs only.'); // The tool output reached the model. expect(JSON.stringify(chatCalls().at(-1)!.body.messages)).toContain('Monograph for furosemide'); }); it('offers a server only in the areas it is allowed in', async () => { await connectAi(); const server = await addServer(); await t.api('PATCH', `/api/ai/mcp/${server.id}`, { areas: { chat: false } }); await sse('/api/ai/chat', { message: 'Use the mcp drug reference for furosemide' }); expect(chatCalls().every((r) => !toolNames(r).some((n) => n.startsWith('mcp_')))).toBe(true); expect(mcpServer.calls.some((c) => c.method === 'tools/call')).toBe(false); // Explain is on by default: the explanation may look things up first. mock.requests.length = 0; const events = await sse('/api/ai/explain', { kind: 'card', front: 'Furosemide', back: 'Loop diuretic', ask: 'Check the mcp reference too' }); expect(events.some((e) => e.type === 'text')).toBe(true); expect(events.at(-1).type).toBe('done'); expect(mcpServer.calls.some((c) => c.method === 'tools/call')).toBe(true); // Generation is off by default, so its requests never list MCP tools… const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; mock.requests.length = 0; let draftId = (await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'mcp diuretics', count: 2 })).body.draftId; let draft = await waitForDraft(draftId); expect(draft.status).toBe('ready'); expect(chatCalls().every((r) => !toolNames(r).some((n) => n.startsWith('mcp_')))).toBe(true); // …until it is switched on: then a lookup can come before the cards are submitted. await t.api('PATCH', `/api/ai/mcp/${server.id}`, { areas: { generate: true } }); mcpServer.calls.length = 0; draftId = (await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'mcp diuretics', count: 2 })).body.draftId; draft = await waitForDraft(draftId); expect(draft.status).toBe('ready'); expect(draft.payload.cards.length).toBeGreaterThan(0); expect(mcpServer.calls.some((c) => c.method === 'tools/call')).toBe(true); }); it('builds safe tool names and readable results', () => { const taken = new Set(); expect(mcpToolName('Web Search!', 'search.web', taken)).toBe('mcp_web_search__search_web'); expect(mcpToolName('Web Search!', 'search.web', taken)).toBe('mcp_web_search__search_web_2'); expect(mcpToolName('x'.repeat(80), 'y'.repeat(80), new Set()).length).toBeLessThanOrEqual(64); expect(mcpResultText({ content: [{ type: 'text', text: 'a' }, { type: 'image', data: '...' }] })).toBe('a\n\n[image omitted]'); expect(mcpResultText({ content: [], structuredContent: { ok: 1 } })).toBe('{"ok":1}'); }); }); async function waitForDraft(id: string) { for (let i = 0; i < 400; i += 1) { const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft; if (d.status !== 'generating') return d; await new Promise((r) => setTimeout(r, 25)); } throw new Error('draft never finished'); } /* ------------------------------ generation ------------------------------ */ describe('quiz generation rules', () => { it('merges narrow topics into a broader one that exists', () => { const qs = [{ topic: 'Stroke' }, { topic: 'Hemorrhagic stroke' }, { topic: 'ischemic stroke' }, { topic: 'stroke' }, { topic: 'Heart failure' }]; expect(consolidateTopics(qs).map((q) => q.topic)).toEqual(['Stroke', 'Stroke', 'Stroke', 'Stroke', 'Heart failure']); // A class's existing topic wins over new spellings. expect(consolidateTopics([{ topic: 'Right hemisphere stroke' }], ['Stroke']).map((q) => q.topic)).toEqual(['Stroke']); // Unrelated topics that share a word are left alone. expect(consolidateTopics([{ topic: 'Heart failure' }, { topic: 'Renal failure' }]).map((q) => q.topic)).toEqual(['Heart failure', 'Renal failure']); }); it('allows calculation questions only when the material involves calculations', () => { expect(mentionsCalculations('Calculate the infusion rate in mL/hr.')).toBe(true); expect(mentionsCalculations('Give 5 mg/kg IV.')).toBe(true); expect(mentionsCalculations('Signs of right-sided heart failure include edema.')).toBe(false); }); it('writes 50 questions for a practice exam, with no calculations when the material has none', async () => { await connectAi(); const cls = (await t.api('POST', '/api/classes', { name: 'Med-surg' })).body.class; const { draftId } = (await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'heart failure', format: 'exam', verify: false })).body; const draft = await waitForDraft(draftId); expect(draft.status).toBe('ready'); expect(draft.payload.questions.length).toBe(50); const prompts = chatCalls().map((r) => JSON.stringify(r.body.messages)); expect(prompts.some((p) => /calculation/i.test(p) && /none|0/.test(p))).toBe(true); // Topic labels go through one grouping call that asks for broad topics. const grouping = chatCalls().filter((r) => toolNames(r).includes('submit_topics')); expect(grouping.length).toBe(1); expect(JSON.stringify(grouping[0]!.body.messages)).toContain('broad exam topic'); }); }); /* ------------------------------ original document preview ------------------------------ */ /** A deck whose second slide has a picture and an EMF the browser can't show. */ function deckWithPicture(): Uint8Array { const files = unzipSync(buildPptx([{ file: 1, title: 'Fluid balance', bullets: ['Daily weights'] }, { file: 2, title: 'Edema grading', bullets: ['Pitting scale 1+ to 4+'] }])); const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships'; files['ppt/slides/_rels/slide2.xml.rels'] = strToU8( ``, ); files['ppt/media/image1.png'] = TINY_PNG; files['ppt/media/image2.emf'] = new Uint8Array([1, 2, 3]); return zipSync(files); } async function upload(classId: string, filename: string, bytes: Uint8Array) { const form = new FormData(); form.append('classId', classId); form.append('file', new File([Buffer.from(bytes)], filename)); const res = await t.raw('POST', '/api/documents', { body: form }); const body = (await res.json()) as any; await queueIdle(t.ctx); return body.documents[0]; } describe('original document view', () => { it('shows a PowerPoint deck with its slide pictures in a locked-down page', async () => { const cls = (await t.api('POST', '/api/classes', { name: 'Fundamentals' })).body.class; const doc = await upload(cls.id, 'fluids.pptx', deckWithPicture()); const res = await t.raw('GET', `/api/documents/${doc.id}/preview`); expect(res.status).toBe(200); expect(res.headers.get('content-security-policy')).toContain("default-src 'none'"); expect(res.headers.get('content-security-policy')).toContain('sandbox'); const html = await res.text(); expect(html).toContain('Slide 2'); expect(html).toContain('Edema grading'); expect(html).toContain('data:image/png;base64,'); expect(html).toContain("1 picture on this slide can't be shown here."); expect(html).not.toMatch(/