/** * Fifth revision: malformed model output is repaired or retried (and a whole job restarted * once), failed share-link sets can be dismissed by visitors, generated sets get their own * names, flashcards are atomic without hints, the assistant never answers from general * knowledge, and share-link AI expires after a week, is limited to a region and is locked by * abuse protection. */ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts'; import { applyQuizFixes, suspiciousWording } from '../server/ai/verify.ts'; import { MIGRATIONS } from '../server/migrations.ts'; import { queueIdle } from '../server/services/documents.ts'; import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts'; import { createTestApp, type TestApp } from './helpers.ts'; import { startMockAi, type MockAi } from './mockAi.ts'; let mock: MockAi; let t: TestApp; beforeAll(async () => { mock = await startMockAi(); }); afterAll(async () => { await mock.close(); }); beforeEach(async () => { t = await createTestApp(); mock.requests.length = 0; mock.delayChat(0); mock.badJsonNext('', 0); mock.scriptReviews([]); const { resetShareAiLimits } = await import('../server/routes/shares.ts'); resetShareAiLimits(); resetGeoForTests(); }); afterEach(() => t.close()); async function connectAi() { const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider; const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[]; for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } }); } async function classWithMaterial() { const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' })).body.document; await queueIdle(t.ctx); return { cls, doc }; } async function waitDraft(id: string) { for (let i = 0; i < 400; i += 1) { const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft; if (d.status !== 'generating') return d; await new Promise((r) => setTimeout(r, 25)); } throw new Error('generation did not finish'); } const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions')); const isRepair = (r: { body: any }) => JSON.stringify(r.body.messages ?? []).includes('You repair malformed JSON'); describe('malformed model output', () => { it('is sent back to the model for repair instead of failing', async () => { await connectAi(); const { cls } = await classWithMaterial(); mock.badJsonNext('submit_flashcards', 1); const started = await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 }); const draft = await waitDraft(started.body.draftId); expect(draft.status).toBe('ready'); expect(draft.payload.cards).toHaveLength(5); expect(chatCalls().filter(isRepair)).toHaveLength(1); }); it('starts the whole job over once when repair and a retry both fail, then fails for good', async () => { await connectAi(); const { cls } = await classWithMaterial(); // Original, repair and retry all broken: the job starts over and succeeds. mock.badJsonNext('submit_flashcards', 3); let draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId); expect(draft.status).toBe('ready'); expect(draft.payload.cards).toHaveLength(5); // Broken on the second run too: failed, with a readable error. mock.badJsonNext('submit_flashcards', 6); draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId); expect(draft.status).toBe('failed'); expect(draft.error).toMatch(/not valid JSON/); }); }); describe('the quiz check', () => { const isReview = (r: { body: any }) => (r.body?.tools ?? []).some((x: any) => x.function?.name === 'submit_review'); const pass = { summary: 'All good.', checks: [{ name: 'Question count', passed: true }, { name: 'Answer keys', passed: true }], fixes: [] }; it('rewrites broken questions, replaces removed ones and checks again until the quiz passes', async () => { await connectAi(); const { cls } = await classWithMaterial(); mock.scriptReviews([ { summary: 'Two problems.', checks: [ { name: 'Question count vs request', passed: false, detail: 'Q3 duplicates Q2; removing it leaves 5.' }, { name: 'Stem integrity', passed: true, detail: 'Q1 rewritten.' }, ], fixes: [ { question: 3, action: 'remove', reason: 'duplicate of Q2' }, { question: 1, action: 'rewrite', reason: 'garbled stem', question_text: { type: 'single', stem: 'An 80 kg client in septic shock needs norepinephrine. Which finding is the priority?', options: [{ text: 'MAP 58', correct: true }, { text: 'Temp 38.1', correct: false }, { text: 'HR 104', correct: false }, { text: 'Urine 40 mL/h', correct: false }], rationale: 'Perfusion first.' }, }, ], }, pass, ]); const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 6 })).body.draftId); expect(draft.status).toBe('ready'); expect(draft.payload.questions).toHaveLength(6); expect(draft.payload.questions.some((q: any) => q.stem.startsWith('An 80 kg client'))).toBe(true); // Two full checks: the first, then the re-check after the fixes and the replacement. expect(chatCalls().filter((r) => isReview(r) && r.body.messages.filter((m: any) => m.role !== 'system').length === 1)).toHaveLength(2); const notes = draft.meta.notes.join(' | '); expect(notes).toMatch(/Checked with Python in 2 rounds/); expect(notes).toMatch(/1 removed question was replaced/); expect(notes).not.toMatch(/Check failed/); }); it('checks rewritten questions again', async () => { await connectAi(); const { cls } = await classWithMaterial(); mock.scriptReviews([ { summary: 'One rewrite.', checks: [{ name: 'Stem integrity', passed: true, detail: 'Q2 fixed.' }], fixes: [ { question: 2, action: 'rewrite', reason: 'garbled', question_text: { type: 'single', stem: 'Which lab value is monitored for heparin?', options: [{ text: 'aPTT', correct: true }, { text: 'INR', correct: false }, { text: 'BNP', correct: false }, { text: 'TSH', correct: false }], rationale: 'aPTT guides heparin.' } }, ], }, pass, ]); const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); const notes = draft.meta.notes.join(' | '); expect(notes).toMatch(/in 2 rounds/); expect(notes).toMatch(/Question 2: rewritten/); }); it('ignores a key fix that changes nothing', () => { const q = { type: 'single' as const, stem: 'S', options: [{ text: 'a', correct: true }, { text: 'b', correct: false }, { text: 'c', correct: false }, { text: 'd', correct: false }], rationale: '', topic: 'T', difficulty: 'medium' as const, tags: [] }; const out = applyQuizFixes([q], [{ question: 1, action: 'set_correct', correct_options: [1], reason: 'Informational only' }]); expect(out.applied).toEqual([]); expect(out.refused).toEqual([]); }); it('has a second reviewer rewrite a question whose writer changed its mind', async () => { await connectAi(); const { cls } = await classWithMaterial(); // The first check leaves a self-correction in Q1 (as a model once did); the second review catches it. mock.scriptReviews([ { summary: 'Fine.', checks: [{ name: 'Answer keys', passed: true }], fixes: [{ question: 1, action: 'rewrite', reason: 'test setup', question_text: { type: 'numeric', stem: 'A 78 kg? No — an 80 kg client receives 4 mg in 250 mL at 15 mL/h. Dose in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000/250*15/60/80' }], rationale: 'Convert, then divide.' } }], }, pass, ]); const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); const proofreads = chatCalls().filter((r) => r.body.tool_choice?.function?.name === 'submit_proofread'); expect(proofreads.length).toBeGreaterThanOrEqual(2); // The leftover was pointed out to the reviewer, and the question no longer has it. // (The first second-review runs alongside the check that planted it, so a later one sees it.) expect(proofreads.some((p) => /looks like a leftover/.test(JSON.stringify(p.body.messages)))).toBe(true); expect(draft.payload.questions.some((q: any) => /\?\s*No\b/.test(q.stem))).toBe(false); expect(draft.meta.notes.join(' | ')).toMatch(/Second review: Question \d+: rewritten/); }); it('spots leftover wording', () => { const q = (stem: string) => ({ stem, options: [{ text: 'a', correct: true }], rationale: '' }); expect(suspiciousWording([q('A 78 kg? No — an 80 kg client'), q('A client with heart failure asks what to eat.'), q('Actually, the dose is 5 mg.')])).toHaveLength(2); }); it('stops after three rounds when problems remain', async () => { await connectAi(); const { cls } = await classWithMaterial(); const failing = { summary: 'Still wrong.', checks: [{ name: 'Answer keys', passed: false, detail: 'Q1 is disputed.' }], fixes: [] }; mock.scriptReviews([failing, failing, failing, failing]); const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); expect(draft.status).toBe('ready'); const notes = draft.meta.notes.join(' | '); expect(notes).toMatch(/in 3 rounds/); expect(notes).toMatch(/Check failed: Answer keys/); }); }); describe('generation steps in Settings', () => { it('use Content generation settings until a step has its own', async () => { await connectAi(); const { resolveModel } = await import('../server/ai/service.ts'); const models = (await t.api('GET', '/api/ai/models')).body.models.filter((m: any) => m.enabled); for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { capabilities: { reasoning: true } }); const [a, b] = models; const none = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null }; await t.api('PUT', '/api/ai/tasks', { default: { ...none, modelId: a.id }, generate: { ...none, reasoningEffort: 'high' } }); // Nothing set on the steps: they follow Content generation. for (const step of ['plan', 'write', 'check', 'review'] as const) expect(resolveModel(t.ctx, step).params.reasoningEffort).toBe('high'); // A step's own effort wins; the others keep following. await t.api('PUT', '/api/ai/tasks', { plan: { ...none, reasoningEffort: 'low' } }); expect(resolveModel(t.ctx, 'plan').params.reasoningEffort).toBe('low'); expect(resolveModel(t.ctx, 'write').params.reasoningEffort).toBe('high'); // A step with its own model does not take Content generation's parameters for it. if (b) { await t.api('PUT', '/api/ai/tasks', { review: { ...none, modelId: b.id } }); const review = resolveModel(t.ctx, 'review'); expect(review.model.id).toBe(b.id); expect(review.params.reasoningEffort).toBeUndefined(); } const meta = (await t.api('GET', '/api/ai/meta')).body.tasks; expect(meta.plan).toMatchObject({ label: 'Planning', parent: 'generate' }); }); }); describe('generated sets', () => { it('get their own name and description, and flashcards are atomic without hints', async () => { await connectAi(); const { cls } = await classWithMaterial(); const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); expect(draft.payload.title).toBe('Mock Terms'); expect(draft.payload.description).toBe('Covers the mock terms.'); const writing = chatCalls().find((r) => r.body.tool_choice?.function?.name === 'submit_flashcards')!; const tool = writing.body.tools.find((x: any) => x.function.name === 'submit_flashcards'); expect(Object.keys(tool.function.parameters.properties.cards.items.properties)).not.toContain('hint'); expect(writing.body.messages[0].content).toMatch(/Atomic cards/); // The same name twice in a class gets a number. await t.api('POST', `/api/drafts/${draft.id}/save`, {}); const second = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); expect(second.payload.title).toBe('Mock Terms 2'); }); }); describe('the assistant', () => { it('never offers to answer from general knowledge', () => { const mcp: LoadedMcp = { tools: [], servers: [{ id: 'web', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web', research: false }], unavailable: [] }; const note = mcpPromptNote(mcp, { askBeforeWeb: true }); expect(note).not.toMatch(/Answer from general knowledge/); expect(note).toMatch(/never answer from your own knowledge/); expect(note).toMatch(/do not have information/); }); }); /* ------------------------------ share links ------------------------------ */ async function sharedExam(ai: Record) { const { cls, doc } = await classWithMaterial(); const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder; await t.api('PATCH', `/api/documents/${doc.id}`, { folderId: exam.id }); const share = (await t.api('POST', '/api/shares', { kind: 'folder', targetId: exam.id })).body.share; const updated = (await t.api('PATCH', `/api/shares/${share.id}`, { ai })).body.share; return { cls, doc, exam, share: updated, token: share.token as string }; } const from = (ip: string, country?: string) => ({ headers: { 'x-forwarded-for': ip, ...(country ? { 'cf-ipcountry': country } : {}) } }); const explain = (token: string, ip: string, country?: string) => t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'none', cardId: 'none' }, from(ip, country)); describe('failed share-link sets', () => { it('can be dismissed by the visitor and stay in the owner drafts', async () => { await connectAi(); const { token } = await sharedExam({ generate: true }); mock.badJsonNext('submit_flashcards', 6); const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 5 }); expect(started.status).toBe(202); const id = started.body.set.id; await waitDraft(id); const sets = (await t.api('GET', `/api/public/${token}/generated`)).body.sets; expect(sets.find((s: any) => s.id === id)?.status).toBe('failed'); expect((await t.api('POST', `/api/public/${token}/generated/${id}/dismiss`)).status).toBe(200); expect((await t.api('GET', `/api/public/${token}/generated`)).body.sets.some((s: any) => s.id === id)).toBe(false); expect((await t.api('GET', '/api/drafts')).body.drafts.find((d: any) => d.id === id)?.status).toBe('failed'); }); it('show visitors plain progress steps', async () => { await connectAi(); const { token } = await sharedExam({ generate: true }); mock.delayChat(300); const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 4 }); t.ctx.db.run('UPDATE drafts SET progress_label = ? WHERE id = ?', 'Checking the questions with Python (run 1)', started.body.set.id); const set = (await t.api('GET', `/api/public/${token}/generated`)).body.sets.find((s: any) => s.id === started.body.set.id); expect(set.progressLabel).toBe('Verifying questions'); mock.delayChat(0); await waitDraft(started.body.set.id); }); }); describe('share-link AI', () => { it('turns itself off after a week and can be given another week', async () => { await connectAi(); const { share, token } = await sharedExam({ explain: true, generate: true }); expect(share.aiActive).toBe(true); expect(share.aiExpiresAt - Date.now()).toBeGreaterThan(6.9 * 86_400_000); expect((await t.api('GET', `/api/public/${token}`)).body.ai.explain).toBe(true); t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id); expect((await t.api('GET', `/api/public/${token}`)).body.ai).toEqual({ explain: false, assistant: false, generate: false }); expect((await explain(token, '10.0.0.2')).status).toBe(403); const lapsed = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id); expect(lapsed.aiActive).toBe(false); const renewed = (await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share; expect(renewed.aiActive).toBe(true); expect(renewed.ai).toMatchObject({ explain: true, generate: true }); expect((await explain(token, '10.0.0.2')).status).toBe(404); // allowed through; the card does not exist // Switching a feature on after a lapse turns on only that feature, for a fresh week. t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id); const fresh = (await t.api('PATCH', `/api/shares/${share.id}`, { ai: { assistant: true } })).body.share; expect(fresh.ai).toEqual({ explain: false, assistant: true, generate: false }); expect(fresh.aiActive).toBe(true); }); it('works only from the allowed region, failing closed when a visitor cannot be placed', async () => { await connectAi(); const { token } = await sharedExam({ explain: true }); expect((await t.api('PUT', '/api/share-settings/ai', { region: 'us' })).status).toBe(200); // Outside the US: the page works without AI, and the API refuses. const page = await t.api('GET', `/api/public/${token}`, undefined, from('81.2.69.142', 'GB')); expect(page.status).toBe(200); expect(page.body.ai.explain).toBe(false); const refused = await explain(token, '81.2.69.142', 'GB'); expect(refused.status).toBe(403); expect(refused.body.error.code).toBe('ai_region'); expect((await explain(token, '8.8.8.8', 'US')).status).toBe(404); // The local network counts as home; an unplaceable public address does not. expect((await explain(token, '192.168.1.20')).status).toBe(404); expect((await explain(token, '8.8.4.4')).status).toBe(403); // Arizona: the state has to match too. await t.api('PUT', '/api/share-settings/ai', { region: 'arizona' }); expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'CA' } })).status).toBe(403); expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'AZ' } })).status).toBe(404); const status = (await t.api('GET', '/api/share-settings/ai')).body; expect(status.geo.region).toBe('arizona'); expect(status.geo.lastDay.blocked).toBeGreaterThan(0); }); it('is locked by abuse protection when dozens of networks use one link', async () => { await connectAi(); const { share, token } = await sharedExam({ explain: true }); for (let i = 1; i < 25; i += 1) expect((await explain(token, `9.9.9.${i}`)).status).toBe(404); expect((await explain(token, '9.9.9.200')).status).toBe(403); // Stays off for everyone, and the owner sees why. expect((await explain(token, '9.9.9.1')).status).toBe(403); const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id); expect(owner.aiActive).toBe(false); expect(owner.aiBlocked.reason).toMatch(/25 different networks/); // Turning it back on lifts the lock. expect((await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share.aiActive).toBe(true); expect((await explain(token, '9.9.9.1')).status).toBe(404); }); it('is locked by a flood of requests or repeated requests from outside the region', async () => { await connectAi(); const { share, token } = await sharedExam({ explain: true }); await t.api('PUT', '/api/share-settings/ai', { region: 'us' }); for (let i = 0; i < 9; i += 1) expect((await explain(token, '81.2.69.142', 'GB')).status).toBe(403); expect((await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id).aiActive).toBe(true); await explain(token, '81.2.69.142', 'GB'); const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id); expect(owner.aiBlocked.reason).toMatch(/outside the allowed region/); }); }); describe('location helpers', () => { it('recognise private and loopback addresses', () => { for (const ip of ['10.1.2.3', '192.168.0.5', '172.20.1.1', '127.0.0.1', '::1', 'fd00::1', '::ffff:192.168.1.2']) expect(isPrivateAddress(ip)).toBe(true); for (const ip of ['8.8.8.8', '172.32.0.1', '2001:4860:4860::8888']) expect(isPrivateAddress(ip)).toBe(false); }); it('migration 8 adds the share AI columns', () => { expect(MIGRATIONS.find((m) => m.id === 8)?.name).toBe('share-ai-safety'); const cols = t.ctx.db.all<{ name: string }>('PRAGMA table_info(shares)').map((c) => c.name); expect(cols).toEqual(expect.arrayContaining(['ai_expires_at', 'ai_blocked'])); }); });