/** * Fourth revision: live stages on the assistant's tool steps, Explain without practice * questions, interactive blocks by default, asking before searching the web, exams offered in * clarifying questions, and OpenRouter replacing the Ollama connection type. */ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { pastTense } from '../server/ai/assistant.ts'; import { genUiPrompt } from '../server/ai/genui.ts'; import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts'; import { MIGRATIONS } from '../server/migrations.ts'; import { queueIdle } from '../server/services/documents.ts'; import { newId, now } from '../server/util.ts'; import { createTestApp, type TestApp } from './helpers.ts'; import { startMockAi, type MockAi } from './mockAi.ts'; let mock: MockAi; let t: TestApp; beforeAll(async () => { mock = await startMockAi(); }); afterAll(async () => { await mock.close(); }); beforeEach(async () => { t = await createTestApp(); mock.requests.length = 0; mock.delayChat(0); }); afterEach(() => t.close()); async function connectAi() { const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider; const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[]; for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } }); } async function sse(url: string, body: unknown): Promise { const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); expect(res.status).toBe(200); return (await res.text()) .split(/\n\n/) .map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join('')) .filter(Boolean) .map((d) => JSON.parse(d)); } const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions')); describe('assistant tool steps', () => { it('report each stage of a generation while it runs, then a past-tense line', async () => { await connectAi(); const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' }); await queueIdle(t.ctx); const events = await sse('/api/ai/chat', { message: 'Write 3 quiz questions from my documents', context: { classId: cls.id } }); const steps = events.filter((e) => e.type === 'tool' && e.tool.name === 'generate_quiz').map((e) => e.tool); const stages = steps.filter((s) => s.status === 'running' && s.stage).map((s) => s.stage); expect(stages).toContain('Reading your material'); expect(stages.some((s: string) => /^Writing questions/.test(s))).toBe(true); const last = steps.at(-1); expect(last.status).toBe('done'); expect(last.stage).toBeUndefined(); expect(last.label).toMatch(/^Wrote \d+ quiz questions?$/); }); it('turn labels into the past tense once done', () => { expect(pastTense('Searching documents for "sepsis"')).toBe('Searched documents for "sepsis"'); expect(pastTense('Setting up a practice session')).toBe('Set up a practice session'); expect(pastTense('Running Python')).toBe('Ran Python'); expect(pastTense('Something else')).toBe('Something else'); }); }); describe('interactive blocks', () => { it('are encouraged in chats, and Explain never quizzes', () => { const chat = genUiPrompt('chat'); expect(chat).toContain('Reach for a block whenever'); expect(chat).toContain('Question(stem: string'); const explain = genUiPrompt('explain'); expect(explain).not.toContain('Question(stem: string'); expect(explain).not.toContain('Flashcards('); expect(explain).not.toContain('self-check Question'); expect(explain).toContain('never include a Question, Flashcards'); }); it('are part of the Explain request', async () => { await connectAi(); await sse('/api/ai/explain', { kind: 'text', text: 'Loop diuretics waste potassium.' }); const system = chatCalls()[0].body.messages[0].content as string; expect(system).toContain('Give information only'); expect(system).not.toContain('Question(stem: string'); }); }); describe('the assistant prompt', () => { it('lists every class with its exams and keeps quizzes NCLEX style', async () => { await connectAi(); const cls = (await t.api('POST', '/api/classes', { name: 'Med-Surg' })).body.class; const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 3' })).body.folder; await sse('/api/ai/chat', { message: 'hello there' }); const system = chatCalls()[0].body.messages[0].content as string; expect(system).toContain(`- Med-Surg — id ${cls.id}; exams: Exam 3 (exam_id ${exam.id})`); expect(system).toContain('Never ask about the question format or style'); expect(system).toContain('interactive without being asked'); expect(system).toContain('class materials'); }); it('asks before searching the web when the materials do not cover a question', () => { const mcp: LoadedMcp = { tools: [], unavailable: [], servers: [{ id: 'web-search', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web' }] }; const chat = mcpPromptNote(mcp, { askBeforeWeb: true }); expect(chat).toContain('Should I look it up on the web?'); expect(chat).toContain('do not answer from your own knowledge'); const elsewhere = mcpPromptNote(mcp); expect(elsewhere).not.toContain('Should I look it up on the web?'); expect(elsewhere).toContain("anything the student's own material does not cover"); }); }); describe('OpenRouter replaces the Ollama type', () => { it('is offered as a provider, and Ollama is not', async () => { const kinds = (await t.api('GET', '/api/ai/meta')).body.adapters.map((a: any) => a.kind); expect(kinds).toContain('openrouter'); expect(kinds).not.toContain('ollama'); const refused = await t.api('POST', '/api/ai/providers', { name: 'Old', kind: 'ollama' }); expect(refused.status).toBe(422); }); it('moves existing Ollama connections to OpenAI-compatible at /v1', () => { const insert = (base: string | null) => { const id = newId(); t.ctx.db.run('INSERT INTO ai_providers (id, name, kind, base_url, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)', id, 'Ollama', 'ollama', base, now(), now()); return id; }; const ids = [insert(null), insert('http://box:11434'), insert('http://box:11434/v1/'), insert('http://box:11434/api')]; t.ctx.db.exec(MIGRATIONS.find((m) => m.id === 6)!.sql); const rows = ids.map((id) => t.ctx.db.get<{ kind: string; base_url: string }>('SELECT kind, base_url FROM ai_providers WHERE id = ?', id)!); expect(rows.map((r) => r.kind)).toEqual(['openai_compatible', 'openai_compatible', 'openai_compatible', 'openai_compatible']); expect(rows.map((r) => r.base_url)).toEqual(['http://localhost:11434/v1', 'http://box:11434/v1', 'http://box:11434/v1', 'http://box:11434/v1']); }); }); /* ------------------------------ med math and share-link generation ------------------------------ */ async function waitDraft(id: string): Promise { for (let i = 0; i < 400; i += 1) { const draft = (await t.api('GET', `/api/drafts/${id}`)).body.draft; if (draft.status !== 'generating') return draft; await new Promise((r) => setTimeout(r, 25)); } throw new Error('generation did not finish'); } async function classWithMaterial() { const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' })).body.document; await queueIdle(t.ctx); return { cls, doc }; } describe('med math on request', () => { it('writes exactly that many calculations, spread through the quiz, even without calculations in the material', async () => { await connectAi(); const { cls } = await classWithMaterial(); const started = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 8, mathQuestions: 2, verify: false }); expect(started.status).toBe(202); const draft = await waitDraft(started.body.draftId); expect(draft.status).toBe('ready'); const types = draft.payload.questions.map((q: any) => q.type); expect(types).toHaveLength(8); expect(types.filter((x: string) => x === 'numeric')).toHaveLength(2); // Not bunched at the end. expect(types.slice(-2)).not.toEqual(['numeric', 'numeric']); const sent = JSON.stringify(chatCalls().map((r) => r.body.messages)); expect(sent).toContain('medication math'); }); it('spreads items evenly', async () => { const { spreadEvenly } = await import('../server/ai/generate.ts'); const list = [...Array.from({ length: 8 }, (_, i) => `r${i}`), 'm0', 'm1']; expect(spreadEvenly(list, (x) => x.startsWith('m'))).toEqual(['r0', 'r1', 'm0', 'r2', 'r3', 'r4', 'r5', 'm1', 'r6', 'r7']); }); }); describe('share-link visitors can generate', () => { beforeEach(async () => { const { resetShareAiLimits } = await import('../server/routes/shares.ts'); resetShareAiLimits(); }); async function waitSets(token: string): Promise { for (let i = 0; i < 400; i += 1) { const res = await t.api('GET', `/api/public/${token}/generated`); if (!res.body.sets.some((s: any) => s.status === 'generating')) return res.body.sets; await new Promise((r) => setTimeout(r, 25)); } throw new Error('generation did not finish'); } it('is off until the owner allows it, then adds quizzes and flashcards to the class within daily limits', async () => { await connectAi(); const { cls, doc } = await classWithMaterial(); const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder; await t.api('PATCH', `/api/documents/${doc.id}`, { folderId: exam.id }); const share = (await t.api('POST', '/api/shares', { kind: 'folder', targetId: exam.id })).body.share; const token = share.token; expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 4 })).status).toBe(403); await t.api('PATCH', `/api/shares/${share.id}`, { ai: { generate: true, explain: true } }); expect((await t.api('GET', `/api/public/${token}`)).body.ai.generate).toBe(true); // At most 50 questions or 75 cards, materials from the link only. expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 51 })).status).toBe(400); expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 76 })).status).toBe(422); expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', documentIds: ['not-shared'] })).status).toBe(404); mock.requests.length = 0; mock.delayChat(150); const first = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 8, difficulty: 'hard', documentIds: [doc.id], math: true }); expect(first.status).toBe(202); expect(first.body.usage.quiz).toEqual({ used: 1, limit: 4 }); // In the owner's drafts while it is being written. expect((await t.api('GET', '/api/drafts')).body.drafts.find((d: any) => d.id === first.body.set.id)?.status).toBe('generating'); mock.delayChat(0); const [set] = await waitSets(token); expect(set).toMatchObject({ kind: 'quiz', status: 'ready', count: 8 }); expect(set.itemId).toBeTruthy(); // Saved into the exam as if the owner made it, and gone from Drafts. expect((await t.api('GET', '/api/drafts')).body.drafts.some((d: any) => d.id === set.id)).toBe(false); const saved = (await t.api('GET', `/api/quizzes/${set.itemId}`)).body; expect(saved.quiz.folderId).toBe(exam.id); expect(saved.quiz.description).toBe('Made with AI on a share link.'); const shown = (await t.api('GET', `/api/public/${token}`)).body.items.map((i: any) => i.id); expect(shown).toContain(set.itemId); const quiz = (await t.api('GET', `/api/public/${token}/items/quiz/${set.itemId}`)).body; expect(quiz.questions.filter((q: any) => q.type === 'numeric')).toHaveLength(5); const sent = JSON.stringify(chatCalls().map((r) => r.body.messages)); expect(sent).toContain('Difficulty: hard'); expect(sent).toMatch(/Exactly 1 of them must be select-all-that-apply/); const explained = await t.raw('POST', `/api/public/${token}/explain`, { headers: { 'content-type': 'application/json' }, body: JSON.stringify({ kind: 'question', quizId: set.itemId, questionId: quiz.questions[0].id }) }); expect(explained.status).toBe(200); const listed = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id); expect(listed.generated.sets.map((s: any) => s.itemId)).toEqual([set.itemId]); // Four quizzes and two flashcard sets a day. for (let i = 0; i < 3; i += 1) expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 2 })).status).toBe(202); const over = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 2 }); expect(over.status).toBe(429); expect(over.body.error.message).toContain('4 quizzes'); for (let i = 0; i < 2; i += 1) expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3 })).status).toBe(202); expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3 })).status).toBe(429); await waitSets(token); // The owner resets the allowance. const reset = await t.api('POST', `/api/shares/${share.id}/generate/reset`); expect(reset.body.generated.usage.quiz.used).toBe(0); // Deleting the link keeps everything made on it, including a set still being written. mock.delayChat(150); const running = (await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3 })).body.set; expect((await t.api('DELETE', `/api/shares/${share.id}`)).status).toBe(200); mock.delayChat(0); const finished = await waitDraft(running.id); expect(finished.status).toBe('saved'); expect((await t.api('GET', `/api/decks/${finished.savedId}`)).status).toBe(200); expect((await t.api('GET', `/api/quizzes/${set.itemId}`)).status).toBe(200); }); it('defaults to a 50-question quiz', async () => { await connectAi(); const { cls } = await classWithMaterial(); const share = (await t.api('POST', '/api/shares', { kind: 'class', targetId: cls.id })).body.share; await t.api('PATCH', `/api/shares/${share.id}`, { ai: { generate: true } }); const res = await t.api('POST', `/api/public/${share.token}/generate`, { kind: 'quiz' }); expect(res.status).toBe(202); const draft = await waitDraft(res.body.set.id); expect(draft.meta.request).toMatchObject({ count: 50, sataShare: 0.25, format: 'exam' }); }); });