/** * Regression tests for the first round of V2 revisions: numeric questions, the 7-point grading * scale, editable instructions and presets, documents in collection shares, background * assistant turns with stop/re-attach, @-tagged materials, assistant model switching, cached * pricing and the built-in embedding default. */ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { isPassing, letterGrade, numericMatches, parseNumericAnswer, scoreQuestion } from '../shared/content.ts'; import { parseImport } from '../shared/importer.ts'; import { estimateCost } from '../server/ai/service.ts'; import { queueIdle } from '../server/services/documents.ts'; import { getEmbeddingSettings, setLocalEmbeddingDefault } from '../server/services/semantic.ts'; import { seedTemplates } from '../server/services/templates.ts'; import { createTestApp, type TestApp } from './helpers.ts'; import { startMockAi, type MockAi } from './mockAi.ts'; let mock: MockAi; let t: TestApp; beforeAll(async () => { mock = await startMockAi(); }); afterAll(() => mock.close()); beforeEach(async () => { t = await createTestApp(); mock.requests.length = 0; mock.delayChat(0); }); afterEach(() => t.close()); async function connectAi() { const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider; const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[]; for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } }); return models; } function parseEvents(raw: string): any[] { return raw .split(/\n\n/) .map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join('')) .filter(Boolean) .map((d) => JSON.parse(d)); } async function sse(url: string, body?: unknown): Promise { const res = await t.raw(body === undefined ? 'GET' : 'POST', url, body === undefined ? {} : { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); expect(res.status).toBe(200); return parseEvents(await res.text()); } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); async function waitIdle(conversationId: string) { for (let i = 0; i < 400; i += 1) { const c = (await t.api('GET', `/api/ai/conversations/${conversationId}`)).body; if (!c.conversation.running) return c; await sleep(25); } throw new Error('turn never finished'); } /* ------------------------------ numeric answers + grading ------------------------------ */ describe('numeric answers', () => { it('reads the number and ignores units', () => { expect(parseNumericAnswer('12.5 mL/hr')).toEqual({ value: 12.5, tolerance: 0 }); expect(parseNumericAnswer('1,250 units')?.value).toBe(1250); expect(parseNumericAnswer('.5')?.value).toBe(0.5); expect(parseNumericAnswer('1/4 tab')?.value).toBe(0.25); expect(parseNumericAnswer('12.5 ± 0.1 mL')).toEqual({ value: 12.5, tolerance: 0.1 }); expect(parseNumericAnswer('no number')).toBeNull(); expect(numericMatches('12.50ml', [{ text: '12.5 mL', correct: true }])).toBe(true); expect(numericMatches('12.4', [{ text: '12.5 mL', correct: true }])).toBe(false); expect(numericMatches('12.45', [{ text: '12.5 ± 0.1', correct: true }])).toBe(true); expect(scoreQuestion({ type: 'numeric', options: [{ id: 'a', text: '0.5 mL', correct: true }] }, ['0.5'])).toBe(1); expect(scoreQuestion({ type: 'numeric', options: [{ id: 'a', text: '0.5 mL', correct: true }] }, [])).toBe(0); }); it('imports calculation questions from JSON and from text blocks', () => { const json = parseImport(JSON.stringify({ kind: 'quiz', title: 'Math', questions: [{ type: 'numeric', stem: 'Rate?', answer: '125 mL/hr', rationale: 'r' }, { question: 'Dose?', correctAnswer: 2 }] })); const qs = json.items[0].kind === 'quiz' ? json.items[0].questions : []; expect(qs.map((q) => q.type)).toEqual(['numeric', 'numeric']); expect(qs[0].options).toEqual([{ text: '125 mL/hr', correct: true, why: '' }]); const text = parseImport('1. Infuse 500 mL over 4 hours. Rate in mL/hr?\nAnswer: 125 mL/hr\nRationale: 500/4'); expect(text.items[0]).toMatchObject({ kind: 'quiz', questions: [{ type: 'numeric', options: [{ text: '125 mL/hr', correct: true }] }] }); }); it('stores, validates, takes and scores numeric questions through the API', async () => { const cls = (await t.api('POST', '/api/classes', { name: 'Dosage' })).body.class; const bad = await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Bad', questions: [{ type: 'numeric', stem: 'How much?', options: [{ text: 'half a tablet', correct: true }] }] }); expect(bad.status).toBe(422); const quiz = ( await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Calc', questions: [ { type: 'numeric', stem: 'Give how many mL?', options: [{ text: '0.5 mL', correct: true, why: '5000/10000' }], rationale: 'Desired over have' }, { type: 'numeric', stem: 'Rate?', options: [{ text: '125 mL/hr', correct: true }] }, ], }) ).body.quiz; const attempt = (await t.api('POST', '/api/attempts', { mode: 'practice', source: { type: 'quiz', quizId: quiz.id }, shuffleQuestions: false })).body.attempt; const [q1, q2] = attempt.questions; const patched = (await t.api('PATCH', `/api/attempts/${attempt.id}`, { answers: { [q1.id]: { selected: ['0.50 ml'], checked: true }, [q2.id]: { selected: ['120'], checked: true } } })).body.attempt; expect(patched.answers[q1.id].selected).toEqual(['0.50 ml']); const done = (await t.api('POST', `/api/attempts/${attempt.id}/complete`, {})).body.attempt; expect(done.score).toBe(1); expect(done.maxScore).toBe(2); expect(done.results[q1.id].correct).toBe(true); expect(done.results[q2.id].correct).toBe(false); }); }); describe('7-point grading scale', () => { it('maps percentages to letters with C as the passing grade', () => { expect([100, 93, 92.6, 92, 85, 84, 77, 76, 70, 69, 0].map(letterGrade)).toEqual(['A', 'A', 'A', 'B', 'B', 'C', 'C', 'D', 'D', 'F', 'F']); expect(isPassing(77)).toBe(true); expect(isPassing(76)).toBe(false); }); }); /* ------------------------------ pricing ------------------------------ */ describe('cost estimates', () => { it('bills cached input at the cached price and applies DeepSeek off-peak pricing', () => { const offPeak = Date.UTC(2026, 10, 16, 12, 0); const flash = { model: 'deepseek-flash', providerKind: 'deepseek' as const, priceIn: 0.3, priceOut: 1.2, priceCachedIn: 0.006 }; const cost = estimateCost(flash, { inputTokens: 1_000_000, cacheReadTokens: 800_000, outputTokens: 100_000 }, offPeak)!; // (200k * 0.3 + 800k * 0.006 + 100k * 1.2) / 1M, halved off-peak expect(cost).toBeCloseTo(((200_000 * 0.3 + 800_000 * 0.006 + 100_000 * 1.2) / 1_000_000) * 0.5, 8); const claude = { model: 'claude-sonnet-5-5', providerKind: 'anthropic' as const, priceIn: 2, priceOut: 10, priceCachedIn: null }; expect(estimateCost(claude, { inputTokens: 1_000_000, cacheReadTokens: 500_000, cacheWriteTokens: 100_000, outputTokens: 0 })!).toBeCloseTo((400_000 * 2 + 500_000 * 0.2 + 100_000 * 2.5) / 1_000_000, 8); expect(estimateCost({ model: 'x', providerKind: 'openai_compatible', priceIn: null, priceOut: null }, { inputTokens: 5, outputTokens: 5 })).toBeNull(); }); it('stores an editable cached price per model', async () => { const models = await connectAi(); const updated = (await t.api('PATCH', `/api/ai/models/${models[0].id}`, { priceIn: 1, priceOut: 2, priceCachedIn: 0.05 })).body.model; expect(updated.priceCachedIn).toBe(0.05); const cleared = (await t.api('PATCH', `/api/ai/models/${models[0].id}`, { priceCachedIn: null })).body.model; expect(cleared.priceCachedIn).toBeNull(); }); it('accepts the full range of reasoning efforts in task settings', async () => { for (const effort of ['minimal', 'xhigh', 'max']) { const res = await t.api('PUT', '/api/ai/tasks', { chat: { modelId: null, temperature: null, reasoningEffort: effort, maxOutputTokens: null } }); expect(res.status).toBe(200); expect(res.body.tasks.chat.reasoningEffort).toBe(effort); } }); }); /* ------------------------------ instructions + presets ------------------------------ */ describe('editable generation instructions and presets', () => { it('saves and resets generator instructions, which generation then uses', async () => { const list = (await t.api('GET', '/api/ai/instructions')).body.instructions as any[]; expect(list.find((i) => i.key === 'quiz.nclex').text).toMatch(/numeric/); const custom = 'Write every question about pediatric dosing. Calculation items use type numeric.'; const saved = (await t.api('PUT', '/api/ai/instructions', { key: 'quiz.nclex', text: custom })).body.instructions as any[]; expect(saved.find((i) => i.key === 'quiz.nclex')).toMatchObject({ text: custom, customized: true }); await connectAi(); const cls = (await t.api('POST', '/api/classes', { name: 'Peds' })).body.class; const { draftId } = (await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'fever', count: 2 })).body; for (let i = 0; i < 200 && (await t.api('GET', `/api/drafts/${draftId}`)).body.draft.status === 'generating'; i += 1) await sleep(25); const call = mock.requests.find((r) => r.path.endsWith('/chat/completions'))!; expect(JSON.stringify(call.body.messages[0])).toContain('pediatric dosing'); const reset = (await t.api('PUT', '/api/ai/instructions', { key: 'quiz.nclex', text: null })).body.instructions as any[]; expect(reset.find((i) => i.key === 'quiz.nclex').customized).toBe(false); }); it('keeps the chatbot import instructions separate and editable', async () => { const before = (await t.api('GET', '/api/llm-instructions')).body; expect(before.builtin.quiz).toContain('"numeric"'); await t.api('PUT', '/api/llm-instructions', { kind: 'quiz', text: 'My own chatbot instructions, long enough.' }); const after = (await t.api('GET', '/api/llm-instructions')).body; expect(after.quiz).toBe('My own chatbot instructions, long enough.'); expect(after.customized.quiz).toBe(true); // The built-in generator's instructions are untouched. expect((await t.api('GET', '/api/ai/instructions')).body.instructions.every((i: any) => !i.customized)).toBe(true); }); it('lets built-in presets be edited, keeps the edit across restarts and resets it', async () => { const preset = (await t.api('GET', '/api/templates?kind=generation')).body.templates.find((x: any) => x.id === 'builtin-gen-nclex'); const edited = (await t.api('PATCH', `/api/templates/${preset.id}`, { name: 'My NCLEX set', payload: { ...preset.payload, count: 40 } })).body.template; expect(edited).toMatchObject({ name: 'My NCLEX set', builtin: true, edited: true }); seedTemplates(t.ctx.db); const afterSeed = (await t.api('GET', '/api/templates?kind=generation')).body.templates.find((x: any) => x.id === preset.id); expect(afterSeed).toMatchObject({ name: 'My NCLEX set', edited: true }); expect(afterSeed.payload.count).toBe(40); const reset = (await t.api('POST', `/api/templates/${preset.id}/reset`)).body.template; expect(reset).toMatchObject({ name: preset.name, edited: false }); expect(reset.payload.count).toBe(preset.payload.count); // Built-ins still cannot be deleted. expect((await t.api('DELETE', `/api/templates/${preset.id}`)).status).toBe(404); }); }); /* ------------------------------ sharing ------------------------------ */ describe('class shares include documents', () => { it('lists processed documents and serves them only through the share', async () => { const cls = (await t.api('POST', '/api/classes', { name: 'Shared' })).body.class; const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture notes', text: 'Furosemide causes potassium loss.' })).body.document; await queueIdle(t.ctx); const other = (await t.api('POST', '/api/classes', { name: 'Private' })).body.class; const hidden = (await t.api('POST', '/api/documents/text', { classId: other.id, title: 'Private notes', text: 'secret' })).body.document; await queueIdle(t.ctx); const share = (await t.api('POST', '/api/shares', { kind: 'class', targetId: cls.id })).body.share; const anon = { cookie: null }; const pub = (await t.api('GET', `/api/public/${share.token}`, undefined, anon)).body; expect(pub.items).toEqual([expect.objectContaining({ kind: 'document', id: doc.id, title: 'Lecture notes' })]); const meta = (await t.api('GET', `/api/public/${share.token}/items/document/${doc.id}`, undefined, anon)).body; expect(meta).toMatchObject({ kind: 'document', id: doc.id }); const file = await t.raw('GET', `/api/public/${share.token}/items/document/${doc.id}/file`, { cookie: null }); expect(file.status).toBe(200); expect(await file.text()).toContain('potassium'); expect((await t.raw('GET', `/api/public/${share.token}/items/document/${hidden.id}/file`, { cookie: null })).status).toBe(404); // Turning the share off closes the file route too. await t.api('PATCH', `/api/shares/${share.id}`, { enabled: false }); expect((await t.raw('GET', `/api/public/${share.token}/items/document/${doc.id}/file`, { cookie: null })).status).toBe(404); }); }); /* ------------------------------ assistant ------------------------------ */ describe('assistant turns', () => { it('keeps answering after the browser disconnects, and can be followed again', async () => { await connectAi(); mock.delayChat(150); const res = await t.raw('POST', '/api/ai/chat', { headers: { 'content-type': 'application/json' }, body: JSON.stringify({ message: 'Explain the difference between left-sided and right-sided heart failure' }) }); const reader = res.body!.getReader(); const first = new TextDecoder().decode((await reader.read()).value); const conversationId = parseEvents(first).find((e) => e.type === 'start').conversationId; // Leave the page. await reader.cancel(); expect((await t.api('GET', `/api/ai/conversations/${conversationId}`)).body.conversation.running).toBe(true); expect((await t.api('GET', '/api/ai/conversations')).body.conversations.find((c: any) => c.id === conversationId).running).toBe(true); // Come back: the stream replays from the start of the turn and runs to the end. const replay = await sse(`/api/ai/chat/${conversationId}/stream`); expect(replay[0].type).toBe('start'); expect(replay.at(-1).type).toBe('done'); const convo = await waitIdle(conversationId); expect(convo.messages.map((m: any) => m.role)).toEqual(['user', 'assistant']); expect(convo.messages[1].text.length).toBeGreaterThan(10); // Following a finished conversation ends at once. expect((await sse(`/api/ai/chat/${conversationId}/stream`)).map((e) => e.type)).toEqual(['done']); }); it('stops only when asked, and refuses a second turn while one is running', async () => { await connectAi(); mock.delayChat(300); const res = await t.raw('POST', '/api/ai/chat', { headers: { 'content-type': 'application/json' }, body: JSON.stringify({ message: 'Tell me about digoxin' }) }); const reader = res.body!.getReader(); const conversationId = parseEvents(new TextDecoder().decode((await reader.read()).value)).find((e) => e.type === 'start').conversationId; await reader.cancel(); const busy = await sse('/api/ai/chat', { conversationId, message: 'And another thing' }); expect(busy.find((e) => e.type === 'error').code).toBe('busy'); expect((await t.api('POST', `/api/ai/chat/${conversationId}/stop`)).body.stopped).toBe(true); const convo = await waitIdle(conversationId); expect(convo.messages.at(-1).text).toContain('Stopped'); expect((await t.api('POST', `/api/ai/chat/${conversationId}/stop`)).body.stopped).toBe(false); }); it('passes tagged materials to the model and remembers them on the message', async () => { await connectAi(); const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Cardiac drugs quiz' })).body.quiz; const deck = (await t.api('POST', '/api/decks', { classId: cls.id, title: 'Antibiotics deck', cards: [{ front: 'a', back: 'b' }] })).body.deck; const items = (await t.api('GET', `/api/ai/mentionables?classId=${cls.id}`)).body.items as any[]; expect(items.map((i) => i.title).sort()).toEqual(['Antibiotics deck', 'Cardiac drugs quiz']); expect((await t.api('GET', '/api/ai/mentionables?q=antib')).body.items.map((i: any) => i.id)).toEqual([deck.id]); const events = await sse('/api/ai/chat', { message: 'Make flashcards from this', context: { classId: cls.id, items: [{ kind: 'quiz', id: quiz.id }, { kind: 'deck', id: 'no-such-deck-xyz' }] } }); const call = mock.requests.find((r) => r.path.endsWith('/chat/completions'))!; const system = JSON.stringify(call.body.messages[0]); expect(system).toContain('tagged these materials'); expect(system).toContain(`quiz \\"Cardiac drugs quiz\\" — id ${quiz.id}`); expect(system).not.toContain('no-such-deck-xyz'); const convo = await waitIdle(events[0].conversationId); expect(convo.messages[0].tags).toEqual([{ kind: 'quiz', id: quiz.id, title: 'Cardiac drugs quiz' }]); }); it('uses the model chosen for the chat', async () => { const models = await connectAi(); const small = models.find((m) => m.model === 'mock-small'); const events = await sse('/api/ai/chat', { message: 'Explain the difference between left-sided and right-sided heart failure', modelId: small.id }); expect(events[0]).toMatchObject({ type: 'start', model: small.label }); expect(mock.requests.filter((r) => r.path.endsWith('/chat/completions')).every((r) => r.body.model === 'mock-small')).toBe(true); }); }); /* ------------------------------ semantic search default ------------------------------ */ describe('built-in embeddings', () => { it('is the default until another choice is saved, and can be turned off', async () => { setLocalEmbeddingDefault(true); try { expect(getEmbeddingSettings(t.ctx.db)).toMatchObject({ providerId: 'local', model: 'Xenova/bge-small-en-v1.5' }); const status = (await t.api('GET', '/api/ai/embeddings')).body; expect(status.local).toMatchObject({ model: 'Xenova/bge-small-en-v1.5', dimensions: 384 }); await t.api('PUT', '/api/ai/embeddings', { providerId: null, model: '', dimensions: null }); expect(getEmbeddingSettings(t.ctx.db).providerId).toBeNull(); await t.api('PUT', '/api/ai/embeddings', { providerId: 'local', model: '', dimensions: null }); expect(getEmbeddingSettings(t.ctx.db)).toMatchObject({ providerId: 'local', model: 'Xenova/bge-small-en-v1.5' }); } finally { setLocalEmbeddingDefault(false); } }); });