diff --git a/scripts/seed-demo.mjs b/scripts/seed-demo.mjs new file mode 100644 index 0000000..255e26f --- /dev/null +++ b/scripts/seed-demo.mjs @@ -0,0 +1,109 @@ +// Fills a development instance with sample content. Never run against real data. +// node scripts/seed-demo.mjs [http://localhost:3726] +// Reads the admin password from ADMIN_PASSWORD (or .env). +try { + process.loadEnvFile('.env'); +} catch { + /* optional */ +} +const base = process.argv[2] ?? `http://localhost:${process.env.PORT ?? 3726}`; +const password = process.env.ADMIN_PASSWORD; +if (!password) throw new Error('Set ADMIN_PASSWORD (or put it in .env) to seed demo data.'); + +let cookie = ''; +async function call(method, path, body, form) { + const res = await fetch(base + path, { + method, + headers: { 'X-Requested-With': 'studydesk', ...(cookie ? { cookie } : {}), ...(body && !form ? { 'content-type': 'application/json' } : {}) }, + body: form ?? (body ? JSON.stringify(body) : undefined), + }); + const set = res.headers.get('set-cookie'); + if (set) cookie = set.split(';')[0]; + const text = await res.text(); + if (!res.ok) throw new Error(`${method} ${path} -> ${res.status} ${text}`); + return text ? JSON.parse(text) : null; +} + +await call('POST', '/api/auth/login', { password }); + +const pharm = (await call('POST', '/api/classes', { name: 'Pharmacology', code: 'NUR 356', description: 'Drug classes, mechanisms, adverse effects and nursing considerations.' })).class; +const peds = (await call('POST', '/api/classes', { name: 'Pediatrics', code: 'NUR 410' })).class; +const medsurg = (await call('POST', '/api/classes', { name: 'Medical-Surgical Nursing', code: 'NUR 320' })).class; +const unit1 = (await call('POST', `/api/classes/${pharm.id}/folders`, { name: 'Unit 1: Cardiovascular' })).folder; + +const q = (type, stem, options, rationale, topic, difficulty = 'medium') => ({ type, stem, options: options.map(([text, correct, why]) => ({ text, correct: Boolean(correct), why: why ?? '' })), rationale, topic, difficulty }); +const cardiac = [ + q('single', 'A client taking digoxin reports nausea, fatigue and seeing yellow halos around lights. Which action should the nurse take **first**?', [ + ['Hold the next dose and check the apical pulse and serum digoxin level', 1, 'Visual changes with GI upset are classic early toxicity; assess and hold before giving more.'], + ['Administer the scheduled dose with food', 0, 'Giving more drug worsens toxicity.'], + ['Reassure the client that these effects are expected', 0, 'These are signs of toxicity, not expected effects.'], + ['Encourage oral fluids and reassess in 4 hours', 0, 'Delays recognition of a potentially life-threatening problem.'], + ], 'Digoxin has a narrow therapeutic range (0.5 to 2 ng/mL). Early toxicity presents with **GI symptoms and visual disturbances**; dysrhythmias follow. Hold the drug, assess the apical pulse for a full minute, and obtain a level.', 'Cardiac glycosides'), + q('multi', 'A nurse is teaching a client who is starting furosemide. Which statements should be included? **Select all that apply.**', [ + ['Take the medication in the morning', 1, 'Prevents nocturia.'], + ['Eat foods high in potassium, such as bananas and potatoes', 1, 'Loop diuretics waste potassium.'], + ['Rise slowly from sitting or lying down', 1, 'Volume depletion causes orthostatic hypotension.'], + ['Expect a weight gain of 1 kg per day', 0, 'Weight should fall or stay stable; a gain suggests fluid retention.'], + ['Report ringing in the ears', 1, 'Ototoxicity is a known adverse effect.'], + ], 'Furosemide is a loop diuretic. Key teaching: morning dosing, potassium intake, orthostatic precautions, daily weights, and reporting tinnitus or hearing changes.', 'Diuretics'), + q('single', 'Which laboratory value should the nurse review before administering lisinopril?', [ + ['Serum potassium', 1, 'ACE inhibitors reduce aldosterone and can cause hyperkalemia.'], + ['Serum calcium', 0], + ['Platelet count', 0], + ['Serum amylase', 0], + ], 'ACE inhibitors can cause **hyperkalemia** and a rise in creatinine. Review potassium and renal function, and monitor blood pressure.', 'ACE inhibitors', 'easy'), + q('single', 'A client receiving a heparin infusion has an aPTT of 120 seconds (control 30 seconds). Which action is appropriate?', [ + ['Stop the infusion and notify the provider', 1, 'The aPTT is four times control; the therapeutic goal is 1.5 to 2.5 times control.'], + ['Increase the infusion rate per protocol', 0, 'Would increase bleeding risk.'], + ['Continue the infusion and recheck in 6 hours', 0], + ['Administer vitamin K', 0, 'Vitamin K reverses warfarin. Protamine sulfate reverses heparin.'], + ], 'Therapeutic aPTT on heparin is about **1.5 to 2.5 times control**. A value of 120 seconds is supratherapeutic. The antidote is protamine sulfate.', 'Anticoagulants', 'hard'), + q('multi', 'Which findings indicate a therapeutic response to metoprolol? **Select all that apply.**', [ + ['Heart rate decreases from 104 to 78/min', 1], + ['Blood pressure decreases from 162/94 to 134/82 mm Hg', 1], + ['Client reports fewer episodes of chest pain', 1], + ['Wheezing is heard on auscultation', 0, 'Bronchospasm is an adverse effect.'], + ], 'Beta-1 blockade lowers heart rate, contractility and blood pressure, reducing myocardial oxygen demand.', 'Beta blockers'), + q('single', 'A client taking warfarin has an INR of 5.8 with no bleeding. Which prescription should the nurse anticipate?', [ + ['Hold warfarin and administer oral vitamin K', 1], + ['Administer protamine sulfate', 0, 'Protamine reverses heparin.'], + ['Increase the warfarin dose', 0], + ['Administer fresh frozen plasma immediately', 0, 'Reserved for serious bleeding.'], + ], 'The usual target INR is 2 to 3. For a markedly elevated INR without bleeding, hold warfarin and give low-dose oral vitamin K (phytonadione).', 'Anticoagulants'), +]; +const cardiacQuiz = (await call('POST', '/api/quizzes', { classId: pharm.id, folderId: unit1.id, title: 'Cardiovascular medications', description: 'Glycosides, diuretics, ACE inhibitors, beta blockers and anticoagulants.', questions: cardiac })).quiz; +await call('POST', '/api/quizzes', { classId: pharm.id, title: 'Antibiotics check', questions: [ + q('single', 'Which assessment is the priority before the first dose of IV penicillin?', [['Ask about previous allergic reactions', 1], ['Measure urine output', 0], ['Check blood glucose', 0], ['Assess deep tendon reflexes', 0]], 'Penicillins are a leading cause of drug anaphylaxis.', 'Antibiotics', 'easy'), + q('single', 'A client on gentamicin reports new ringing in the ears. The nurse should suspect:', [['Ototoxicity', 1], ['Hepatotoxicity', 0], ['Superinfection', 0], ['Photosensitivity', 0]], 'Aminoglycosides are ototoxic and nephrotoxic. Monitor peak and trough levels.', 'Antibiotics'), +] }); + +const cards = [ + ['Furosemide: class', '**Loop diuretic**'], ['Furosemide: key adverse effects', '- Hypokalemia\n- Ototoxicity\n- Orthostatic hypotension\n- Dehydration'], + ['Digoxin: therapeutic range', '**0.5 to 2 ng/mL**'], ['Digoxin toxicity: early signs', 'Anorexia, nausea, vomiting, fatigue, **yellow-green halos**'], + ['Antidote for heparin', 'Protamine sulfate'], ['Antidote for warfarin', 'Vitamin K (phytonadione)'], + ['Lisinopril: suffix and class', '**-pril** = ACE inhibitor'], ['ACE inhibitor: classic adverse effects', 'Dry **cough**, hyperkalemia, angioedema, first-dose hypotension'], + ['Metoprolol: hold when', 'Heart rate below **60/min** or systolic BP below 100 mm Hg (per protocol)'], ['Normal serum potassium', '3.5 to 5.0 mEq/L'], + ['Therapeutic INR on warfarin', '2 to 3 (2.5 to 3.5 for mechanical valves)'], ['Spironolactone: class', 'Potassium-sparing diuretic (aldosterone antagonist)'], +].map(([front, back]) => ({ front, back })); +const deck = (await call('POST', '/api/decks', { classId: pharm.id, folderId: unit1.id, title: 'Cardiac drug cards', description: 'High-yield facts for exam 1.', cards })).deck; +await call('POST', '/api/decks', { classId: peds.id, title: 'Growth and development milestones', cards: [ + { front: 'Age an infant sits without support', back: 'About **6 to 8 months**' }, { front: 'Age of first words', back: 'About 12 months' }, { front: 'Erikson stage: toddler', back: 'Autonomy vs. shame and doubt' }, +] }); + +await call('POST', '/api/notes', { classId: pharm.id, title: 'Exam 1 plan', content: '## Topics\n- Cardiac glycosides\n- Diuretics\n- Anticoagulants\n\n| Drug | Antidote |\n| --- | --- |\n| Heparin | Protamine sulfate |\n| Warfarin | Vitamin K |\n' }); +await call('POST', '/api/documents/text', { classId: pharm.id, title: 'Lecture 3: Heart failure drugs', text: '# Heart failure pharmacology\n\n## ACE inhibitors\nACE inhibitors such as lisinopril block the conversion of angiotensin I to angiotensin II. They reduce afterload and preload and slow ventricular remodeling. Adverse effects include a dry cough, hyperkalemia, angioedema and first-dose hypotension. Monitor potassium and creatinine.\n\n## Beta blockers\nMetoprolol succinate, carvedilol and bisoprolol reduce mortality in heart failure with reduced ejection fraction. Start low and titrate slowly. Hold for a heart rate below 60 per minute.\n\n## Loop diuretics\nFurosemide relieves congestion by inhibiting sodium and chloride reabsorption in the loop of Henle. Monitor daily weights, intake and output, potassium, and hearing.\n\n## Digoxin\nDigoxin increases contractility and slows conduction through the AV node. The therapeutic range is 0.5 to 2 ng/mL. Hypokalemia increases the risk of toxicity. Early signs of toxicity are anorexia, nausea, vomiting and visual disturbances such as yellow halos.\n' }); + +// Some study history so dashboards have something to show. +const attempt = (await call('POST', '/api/attempts', { mode: 'practice', source: { type: 'quiz', quizId: cardiacQuiz.id }, shuffleQuestions: false, shuffleOptions: false })).attempt; +const answers = {}; +attempt.questions.forEach((question, i) => { + const right = question.options.filter((o) => o.correct).map((o) => o.id); + const wrong = question.options.filter((o) => !o.correct).map((o) => o.id); + answers[question.id] = { selected: i === 1 ? right.slice(0, 2) : i === 3 ? wrong.slice(0, 1) : right, checked: true }; +}); +await call('PATCH', `/api/attempts/${attempt.id}`, { answers, elapsedSec: 312 }); +await call('POST', `/api/attempts/${attempt.id}/complete`, { elapsedSec: 312 }); +const queue = await call('GET', `/api/review/queue?deckId=${deck.id}`); +for (const [i, item] of queue.cards.slice(0, 7).entries()) await call('POST', '/api/review/answer', { cardId: item.card.id, rating: i % 4 === 0 ? 1 : 3 }); + +console.log(`Seeded demo data at ${base}: ${[pharm, peds, medsurg].map((c) => c.name).join(', ')}`); diff --git a/server/ai/generate.ts b/server/ai/generate.ts index e99656f..57a226e 100644 --- a/server/ai/generate.ts +++ b/server/ai/generate.ts @@ -545,9 +545,13 @@ export function startGeneration(app: AppContext, input: z.input { const message = err instanceof AIError ? err.message : err instanceof Error && 'status' in err ? err.message : 'Generation failed unexpectedly.'; if (!(err instanceof AIError) && !(err instanceof Error && 'status' in err)) app.log('error', 'Generation failed', err); - const current = app.db.get<{ status: string }>('SELECT status FROM drafts WHERE id = ?', draftId); - if (current?.status === 'generating') { - app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ?`, controller.signal.aborted ? 'Generation was cancelled.' : message, now(), draftId); + try { + const current = app.db.get<{ status: string }>('SELECT status FROM drafts WHERE id = ?', draftId); + if (current?.status === 'generating') { + app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ?`, controller.signal.aborted ? 'Generation was cancelled.' : message, now(), draftId); + } + } catch { + /* the database was closed during shutdown; tidyDrafts marks the job failed on the next start */ } }) .then(() => { @@ -865,7 +869,11 @@ Return the questions in the same order, one output question per input question.$ })() .catch((err) => { const message = err instanceof AIError || (err instanceof Error && 'status' in err) ? (err as Error).message : 'Improving the questions failed unexpectedly.'; - app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ? AND status = 'generating'`, message, now(), draftId); + try { + app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ? AND status = 'generating'`, message, now(), draftId); + } catch { + /* database closed during shutdown */ + } }) .then(() => { running.delete(draftId); diff --git a/tests/ai.test.ts b/tests/ai.test.ts new file mode 100644 index 0000000..2b847a2 --- /dev/null +++ b/tests/ai.test.ts @@ -0,0 +1,377 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { queueIdle } from '../server/services/documents.ts'; +import { buildDocx, buildPdf, TINY_PNG } from '../server/documents/testFixtures.ts'; +import { createTestApp, sampleQuestions, type TestApp } from './helpers.ts'; +import { startMockAi, type MockAi } from './mockAi.ts'; + +let mock: MockAi; +let t: TestApp; + +beforeAll(async () => { + mock = await startMockAi(); +}); +afterAll(() => mock.close()); +beforeEach(async () => { + t = await createTestApp(); + mock.requests.length = 0; +}); +afterEach(() => t.close()); + +/** Connects the mock provider and switches on its chat models. */ +async function connectAi(opts: { tools?: boolean } = {}) { + const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider; + const found = await t.api('POST', `/api/ai/providers/${provider.id}/discover`); + const models = found.body.models as any[]; + for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: opts.tools ?? true } }); + return { provider, models }; +} + +async function upload(classId: string, filename: string, bytes: Uint8Array, mime?: string) { + const form = new FormData(); + form.append('classId', classId); + form.append('file', new File([Buffer.from(bytes)], filename, mime ? { type: mime } : undefined)); + const res = await t.raw('POST', '/api/documents', { body: form }); + const body = (await res.json()) as any; + await queueIdle(t.ctx); + return { status: res.status, body }; +} + +async function sse(url: string, body: unknown): Promise { + const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toMatch(/text\/event-stream/); + const raw = await res.text(); + return raw + .split(/\n\n/) + .map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join('')) + .filter(Boolean) + .map((d) => JSON.parse(d)); +} + +async function waitForDraft(id: string) { + for (let i = 0; i < 200; i += 1) { + const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft; + if (d.status !== 'generating') return d; + await new Promise((r) => setTimeout(r, 25)); + } + throw new Error('draft never finished'); +} + +const LECTURE = `# Heart failure pharmacology + +## Loop diuretics +Furosemide relieves congestion by blocking sodium and chloride reabsorption in the loop of Henle. Monitor potassium, daily weights and hearing. + +## Digoxin +Digoxin increases contractility. The therapeutic range is 0.5 to 2 ng/mL. Hypokalemia increases the risk of toxicity. Early toxicity causes nausea and yellow halos. +`; + +describe('AI providers and models', () => { + it('stores keys encrypted, never returns them, discovers models and routes tasks', async () => { + expect((await t.api('GET', '/api/ai/status')).body.ready).toBe(false); + const { provider, models } = await connectAi(); + expect(provider.hasKey).toBe(true); + expect(JSON.stringify(provider)).not.toContain('mock-key'); + expect(provider.keyPreview).toMatch(/•/); + // The embedding model is filtered out of the chat model list. + expect(models.map((m) => m.model).sort()).toEqual(['mock-large', 'mock-small']); + + // Nothing sensitive in any listing, and the database holds only ciphertext. + const listing = await t.api('GET', '/api/ai/providers'); + expect(JSON.stringify(listing.body)).not.toContain('mock-key'); + const row = t.ctx.db.get<{ api_key_enc: string }>('SELECT api_key_enc FROM ai_providers')!; + expect(row.api_key_enc).toMatch(/^v1\./); + expect(row.api_key_enc).not.toContain('mock-key'); + expect(fs.readFileSync(path.join(t.dir, 'study.db')).includes('mock-key')).toBe(false); + + const test = (await t.api('POST', `/api/ai/providers/${provider.id}/test`)).body.result; + expect(test.ok).toBe(true); + expect(mock.requests.at(-1)!.headers.authorization).toBe('Bearer mock-key'); + + // Per-task routing with parameters. + const small = models.find((m) => m.model === 'mock-small'); + const large = models.find((m) => m.model === 'mock-large'); + await t.api('PUT', '/api/ai/tasks', { default: { modelId: large.id, temperature: null, reasoningEffort: null, maxOutputTokens: null }, quick: { modelId: small.id, temperature: 0.2, reasoningEffort: null, maxOutputTokens: 800 } }); + const status = (await t.api('GET', '/api/ai/status')).body; + expect(status.ready).toBe(true); + expect(status.resolved.quick.label).toBe('mock-small'); + expect(status.resolved.generate.label).toBe('mock-large'); + + const events = await sse('/api/ai/explain', { kind: 'card', front: 'Antidote for heparin', back: 'Protamine sulfate' }); + expect(events.filter((e) => e.type === 'text').map((e) => e.delta).join('')).toContain('mock study guide'); + const call = mock.requests.at(-1)!; + expect(call.body.model).toBe('mock-small'); + expect(call.body.temperature).toBe(0.2); + + // A wrong key is reported clearly, and the key is not echoed back. + const bad = (await t.api('POST', '/api/ai/providers', { name: 'Bad', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'sk-wrong-secret-value' })).body.provider; + const badTest = (await t.api('POST', `/api/ai/providers/${bad.id}/test`)).body.result; + expect(badTest.ok).toBe(false); + expect(badTest.message).not.toContain('sk-wrong-secret-value'); + + const usage = (await t.api('GET', '/api/ai/usage')).body; + expect(usage.total.calls).toBeGreaterThanOrEqual(1); + + // Removing a provider clears task assignments that pointed at it. + await t.api('DELETE', `/api/ai/providers/${provider.id}`); + expect((await t.api('GET', '/api/ai/tasks')).body.tasks.quick.modelId).toBeNull(); + }); + + it('explains that AI is not set up instead of failing obscurely', async () => { + const cls = (await t.api('POST', '/api/classes', { name: 'C' })).body.class; + const res = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'anything' }); + expect(res.status).toBe(409); + expect(res.body.error.code).toBe('ai_not_configured'); + const events = await sse('/api/ai/chat', { message: 'hello' }); + expect(events.find((e) => e.type === 'error').message).toMatch(/No AI model/); + }); +}); + +describe('documents', () => { + it('uploads, extracts, chunks, searches, serves and deletes files', async () => { + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + const pdf = buildPdf([{ lines: ['Loop diuretics', 'Furosemide can cause hypokalemia and ototoxicity.'] }, { lines: ['Digoxin toxicity', 'Yellow halos and nausea are early signs.'] }], { title: 'Cardiac drugs' }); + const up = await upload(cls.id, 'cardiac-drugs.pdf', pdf, 'application/pdf'); + expect(up.status).toBe(201); + const id = up.body.documents[0].id; + const detail = (await t.api('GET', `/api/documents/${id}`)).body; + expect(detail.document).toMatchObject({ status: 'ready', docKind: 'pdf', pageCount: 2, needsOcr: false }); + expect(detail.document.chunkCount).toBeGreaterThan(0); + + const chunks = (await t.api('GET', `/api/documents/${id}/chunks`)).body.chunks; + expect(chunks.map((c: any) => c.text).join(' ')).toMatch(/ototoxicity/); + expect(chunks.at(-1).pageEnd).toBe(2); + + const search = (await t.api('GET', '/api/search?q=halos')).body.results; + expect(search.some((r: any) => r.kind === 'passage' && r.parentId === id)).toBe(true); + + // The stored file is served back intact, only to the signed-in owner. + const file = await t.raw('GET', `/api/documents/${id}/file`); + expect(file.headers.get('content-type')).toBe('application/pdf'); + expect(new Uint8Array(await file.arrayBuffer()).length).toBe(pdf.length); + expect((await t.raw('GET', `/api/documents/${id}/file`, { cookie: null })).status).toBe(401); + + // Word documents and pasted text go through the same pipeline. + const docx = await upload(cls.id, 'notes.docx', buildDocx([{ type: 'heading', level: 1, text: 'Anticoagulants' }, { type: 'p', text: 'Protamine sulfate reverses heparin.' }])); + expect((await t.api('GET', `/api/documents/${docx.body.documents[0].id}`)).body.document.status).toBe('ready'); + const pasted = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE })).body.document; + await queueIdle(t.ctx); + expect((await t.api('GET', `/api/documents/${pasted.id}`)).body.outline.map((o: any) => o.heading)).toContain('Heart failure pharmacology'); + + // Unsupported and disguised files are rejected with a helpful message. + const exe = await upload(cls.id, 'notes.pdf', new Uint8Array([0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00])); + expect(exe.status).toBe(415); + expect(exe.body.error.message).toMatch(/not a supported file type/); + + // Images are accepted and flagged for AI reading when no vision model exists. + const img = await upload(cls.id, 'slide.png', TINY_PNG, 'image/png'); + const imgDoc = (await t.api('GET', `/api/documents/${img.body.documents[0].id}`)).body.document; + expect(imgDoc).toMatchObject({ status: 'ready', needsOcr: true, chunkCount: 0 }); + expect(imgDoc.warnings.join(' ')).toMatch(/vision/); + + const stored = fs.readdirSync(path.join(t.dir, 'uploads')); + expect(stored.length).toBe(4); + await t.api('DELETE', `/api/documents/${id}`); + expect(fs.readdirSync(path.join(t.dir, 'uploads')).length).toBe(3); + expect((await t.api('GET', '/api/search?q=ototoxicity')).body.results.filter((r: any) => r.kind === 'passage')).toHaveLength(0); + }); +}); + +describe('generation', () => { + it('writes a quiz from a document in batches, as a draft that can be edited and saved', async () => { + await connectAi(); + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure lecture', text: LECTURE })).body.document; + await queueIdle(t.ctx); + + const started = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { documentIds: [doc.id] }, count: 12, style: 'nclex', focus: 'digoxin toxicity' }); + expect(started.status).toBe(202); + const draft = await waitForDraft(started.body.draftId); + expect(draft.status).toBe('ready'); + expect(draft.payload.questions).toHaveLength(12); + expect(draft.meta.sources[0].title).toBe('Heart failure lecture'); + // Citations were mapped from excerpt ids to readable labels. + expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/); + // 12 questions at 8 per batch means two model calls, each carrying the source excerpts. + const calls = mock.requests.filter((r) => r.path.endsWith('/chat/completions')); + expect(calls).toHaveLength(2); + expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/); + expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/); + expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } }); + + // Nothing is in the library until the draft is saved. + expect((await t.api('GET', `/api/classes/${cls.id}`)).body.items.filter((i: any) => i.kind === 'quiz')).toHaveLength(0); + + // Edit: break one question, and the save is refused with a pointer to it. + const payload = structuredClone(draft.payload); + payload.title = 'Digoxin practice'; + payload.questions[1].options.forEach((o: any) => (o.correct = false)); + await t.api('PUT', `/api/drafts/${draft.id}`, { payload }); + const refused = await t.api('POST', `/api/drafts/${draft.id}/save`, {}); + expect(refused.status).toBe(400); + expect(refused.body.error.message).toMatch(/Question 2/); + payload.questions.splice(1, 1); + await t.api('PUT', `/api/drafts/${draft.id}`, { payload }); + const saved = (await t.api('POST', `/api/drafts/${draft.id}/save`, {})).body.saved; + expect(saved).toMatchObject({ kind: 'quiz', count: 11 }); + const quiz = (await t.api('GET', `/api/quizzes/${saved.id}`)).body; + expect(quiz.quiz.title).toBe('Digoxin practice'); + expect(quiz.questions).toHaveLength(11); + expect(quiz.questions[0].rationale).toMatch(/Source: Heart failure lecture/); + // Saving twice does not duplicate. + expect((await t.api('POST', `/api/drafts/${draft.id}/save`, {})).body.saved.id).toBe(saved.id); + }); + + it('makes flashcards and study guides, converts between formats, and appends to existing items', async () => { + await connectAi(); + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE })).body.document; + await queueIdle(t.ctx); + + const cards = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { documentIds: [doc.id] }, count: 10, cardStyle: 'medication' })).body.draftId); + expect(cards.payload.cards).toHaveLength(10); + const deckId = (await t.api('POST', `/api/drafts/${cards.id}/save`, {})).body.saved.id; + + // Deck -> quiz conversion uses the cards as the source. + const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId); + expect(converted.status).toBe('ready'); + expect(JSON.stringify(mock.requests.at(-1)!.body.messages)).toMatch(/Flashcards from the deck/); + + // Append more cards to the existing deck. + const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId); + await t.api('POST', `/api/drafts/${more.id}/save`, { mode: 'append', targetId: deckId }); + expect((await t.api('GET', `/api/decks/${deckId}`)).body.cards).toHaveLength(14); + + const guide = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'note', classId: cls.id, source: { documentIds: [doc.id] }, noteStyle: 'study_guide' })).body.draftId); + expect(guide.payload.content).toMatch(/## Overview/); + // The raw [c12] marker was replaced with a readable citation. + expect(guide.payload.content).not.toMatch(/\[c\d+\]/); + expect(guide.payload.content).toMatch(/\(Lecture/); + const note = (await t.api('POST', `/api/drafts/${guide.id}/save`, {})).body.saved; + expect(note.kind).toBe('note'); + + // Discarding removes it from the list of drafts. + const junk = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'x y z', count: 2 })).body.draftId); + await t.api('DELETE', `/api/drafts/${junk.id}`); + expect((await t.api('GET', '/api/drafts')).body.drafts.map((d: any) => d.id)).not.toContain(junk.id); + }); + + it('falls back to prompt-JSON for models without tool support and survives a transient failure', async () => { + await connectAi({ tools: false }); + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + mock.failNext(500, 1); + const draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId); + expect(draft.status).toBe('ready'); + expect(draft.payload.questions).toHaveLength(3); + const call = mock.requests.at(-1)!; + expect(call.body.tools).toBeUndefined(); + expect(call.body.response_format).toEqual({ type: 'json_object' }); + }); + + it('reports a clear failure when the provider keeps erroring', async () => { + await connectAi(); + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + mock.failNext(401, 10); + let draft; + try { + draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId); + } finally { + mock.failNext(0, 0); + } + expect(draft.status).toBe('failed'); + expect(draft.error).toMatch(/API key|rejected|auth/i); + }); + + it('improves existing questions and applies the revisions in place', async () => { + await connectAi(); + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + const quizId = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Rough draft', questions: sampleQuestions })).body.quiz.id; + const before = (await t.api('GET', `/api/quizzes/${quizId}`)).body.questions; + const draft = await waitForDraft((await t.api('POST', '/api/ai/improve', { quizId })).body.draftId); + expect(draft.meta.replaceQuestionIds).toEqual(before.map((q: any) => q.id)); + await t.api('POST', `/api/drafts/${draft.id}/save`, { mode: 'replace' }); + const after = (await t.api('GET', `/api/quizzes/${quizId}`)).body.questions; + expect(after.map((q: any) => q.id)).toEqual(before.map((q: any) => q.id)); + expect(after[0].stem).toMatch(/Mock question 1/); + }); +}); + +describe('assistant', () => { + it('uses tools to search documents and answers with citations', async () => { + await connectAi(); + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE }); + await queueIdle(t.ctx); + + const events = await sse('/api/ai/chat', { message: 'What should I monitor with furosemide?', context: { classId: cls.id } }); + const types = events.map((e) => e.type); + expect(types[0]).toBe('start'); + expect(types.at(-1)).toBe('done'); + const tool = events.filter((e) => e.type === 'tool').at(-1).tool; + expect(tool).toMatchObject({ name: 'search_documents', status: 'done' }); + const sources = events.find((e) => e.type === 'ui' && e.item.type === 'sources').item.items; + const answer = events.filter((e) => e.type === 'text').map((e) => e.delta).join(''); + expect(answer).toContain(`[${sources[0].ref}]`); + + // The conversation is stored and reloads as one collapsed assistant message. + const conversationId = events[0].conversationId; + const convo = (await t.api('GET', `/api/ai/conversations/${conversationId}`)).body; + expect(convo.messages.map((m: any) => m.role)).toEqual(['user', 'assistant']); + expect(convo.messages[1].tools[0].name).toBe('search_documents'); + expect(convo.messages[1].text).toContain('potassium'); + + // The system prompt tells the model which class is in view; the API key never reaches the model. + const first = mock.requests.find((r) => r.path.endsWith('/chat/completions'))!; + expect(JSON.stringify(first.body.messages[0])).toMatch(/currently viewing the class \\"Pharm\\"/); + expect(JSON.stringify(first.body)).not.toContain('mock-key'); + }); + + it('creates drafts through tools, never saving by itself', async () => { + await connectAi(); + const cls = (await t.api('POST', '/api/classes', { name: 'Peds' })).body.class; + await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Respiratory', text: LECTURE }); + await queueIdle(t.ctx); + + const events = await sse('/api/ai/chat', { message: 'Create a 4-question quiz using the documents in this class', context: { classId: cls.id } }); + const draftItem = events.find((e) => e.type === 'ui' && e.item.type === 'draft').item; + expect(draftItem).toMatchObject({ kind: 'quiz', count: 4, status: 'ready' }); + expect(events.filter((e) => e.type === 'text').map((e) => e.delta).join('')).toMatch(/Review it and save/); + expect((await t.api('GET', `/api/classes/${cls.id}`)).body.items.filter((i: any) => i.kind === 'quiz')).toHaveLength(0); + + const saved = (await t.api('POST', `/api/drafts/${draftItem.draftId}/save`, {})).body.saved; + expect((await t.api('GET', `/api/quizzes/${saved.id}`)).body.questions).toHaveLength(4); + // Reloading the conversation shows the draft as saved. + const convo = (await t.api('GET', `/api/ai/conversations/${events[0].conversationId}`)).body; + expect(convo.messages[1].ui.find((u: any) => u.type === 'draft').status).toBe('saved'); + }); + + it('only changes the library after the user confirms', async () => { + await connectAi(); + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + const quizId = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Old quiz', questions: sampleQuestions })).body.quiz.id; + + const events = await sse('/api/ai/chat', { message: `Please delete quiz ${quizId}`, context: { classId: cls.id } }); + const action = events.find((e) => e.type === 'ui' && e.item.type === 'action').item; + expect(action.status).toBe('pending'); + expect(action.changes[0]).toBe('Delete quiz "Old quiz"'); + // Still there. + expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(200); + + // Declining leaves it alone; a second proposal that is confirmed removes it. + expect((await t.api('POST', `/api/ai/actions/${action.actionId}`, { confirm: false })).body.status).toBe('cancelled'); + expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(200); + expect((await t.api('POST', `/api/ai/actions/${action.actionId}`, { confirm: true })).status).toBe(400); + + const again = await sse('/api/ai/chat', { conversationId: events[0].conversationId, message: `I changed my mind, delete quiz ${quizId}` }); + const second = again.find((e) => e.type === 'ui' && e.item.type === 'action').item; + expect((await t.api('POST', `/api/ai/actions/${second.actionId}`, { confirm: true })).body.status).toBe('confirmed'); + expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(404); + + // An id that does not exist is reported back to the model as a tool error, not a crash. + const bogus = await sse('/api/ai/chat', { message: 'delete quiz nonexistent-id-123' }); + expect(bogus.filter((e) => e.type === 'tool').at(-1).tool.status).toBe('error'); + expect(bogus.at(-1).type).toBe('done'); + }); +}); diff --git a/tests/mockAi.ts b/tests/mockAi.ts new file mode 100644 index 0000000..8d21337 --- /dev/null +++ b/tests/mockAi.ts @@ -0,0 +1,181 @@ +/** + * A small OpenAI-compatible server with scripted behaviour, for tests and for trying the + * AI features locally without a real provider (see scripts/mock-ai.mjs). + * + * - Forced tool call -> returns that tool call with plausible arguments. + * - Tools + user turn -> picks a tool from keywords in the message. + * - Tool result turn -> answers in text, citing the first [cN] ref it was given. + * - No tools -> streams a short Markdown answer. + */ +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; + +export interface MockAi { + url: string; + requests: { path: string; headers: http.IncomingHttpHeaders; body: any }[]; + /** Force the next N chat calls to fail with this status. */ + failNext: (status: number, times?: number) => void; + close: () => Promise; +} + +const text = (content: unknown): string => + typeof content === 'string' ? content : Array.isArray(content) ? content.map((p: any) => (typeof p?.text === 'string' ? p.text : '')).join('\n') : ''; + +function questions(n: number, refs: string[], offset = 0) { + return Array.from({ length: n }, (_, k) => { + const i = k + offset; + const multi = i % 3 === 2; + return { + type: multi ? 'multi' : 'single', + stem: `Mock question ${i + 1}: which action is the priority for the client described in case ${i + 1}?`, + options: [ + { text: `Correct action ${i + 1}`, correct: true, why: 'This addresses the immediate risk.' }, + { text: `Plausible distractor ${i + 1}a`, correct: multi, why: multi ? 'Also indicated.' : 'Lower priority.' }, + { text: `Plausible distractor ${i + 1}b`, correct: false, why: 'Not indicated.' }, + { text: `Plausible distractor ${i + 1}c`, correct: false, why: 'Contraindicated.' }, + ], + rationale: `Rationale for question ${i + 1}.`, + topic: i % 2 === 0 ? 'Heart failure' : 'Pharmacology', + difficulty: 'medium', + source: refs.length ? refs[i % refs.length] : '', + }; + }); +} + +function toolArgs(name: string, userText: string, all: string): Record { + const count = Number(/(?:Write|write)\s+(\d+)/.exec(userText)?.[1] ?? /(\d+)[- ](?:question|card|flashcard)/.exec(userText)?.[1] ?? 3); + // Only refs from the user's turn count: the system prompt contains an example id. + const refs = [...new Set([...userText.matchAll(/\[(c\d+)\]/g)].map((m) => m[1]))]; + // Continue numbering after anything already written, so later batches are not duplicates. + const offset = (userText.split('Already written')[1]?.match(/^- /gm) ?? []).length; + void all; + switch (name) { + case 'submit_questions': + return { questions: questions(count, refs, offset) }; + case 'submit_flashcards': + return { cards: Array.from({ length: count }, (_, i) => ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, hint: '', tags: ['mock'] })) }; + case 'generate_quiz': + return { count: Math.min(count, 5), topic: 'heart failure', style: 'nclex', use_all_class_documents: /document/i.test(userText), from_weak_areas: /wrong|weak|miss/i.test(userText) }; + case 'generate_flashcards': + return { count: Math.min(count, 6), topic: 'medications', use_all_class_documents: /document/i.test(userText) }; + case 'search_documents': + return { query: userText.replace(/[^\w\s]/g, ' ').split(/\s+/).filter((w) => w.length > 3).slice(0, 6).join(' ') }; + case 'get_performance': + case 'list_classes': + case 'list_materials': + return {}; + case 'propose_changes': { + const m = /(delete|rename)\s+(quiz|deck|note|document)\s+([\w-]{8,})/i.exec(userText); + return { reason: 'Requested by the user.', changes: [{ op: m?.[1]?.toLowerCase() ?? 'delete', kind: m?.[2]?.toLowerCase() ?? 'quiz', id: m?.[3] ?? 'missing', title: 'Renamed by mock' }] }; + } + default: + return {}; + } +} + +function chooseTool(userText: string, names: string[]): string | null { + const pick = (name: string) => (names.includes(name) ? name : null); + if (/\b(delete|rename)\b/i.test(userText)) return pick('propose_changes'); + if (/flashcard/i.test(userText)) return pick('generate_flashcards'); + if (/\bquiz\b|practice questions/i.test(userText)) return pick('generate_quiz'); + if (/should i study|performance|how am i doing/i.test(userText)) return pick('get_performance'); + if (/\?|what|explain|summar/i.test(userText)) return pick('search_documents'); + return null; +} + +export function startMockAi(port = 0): Promise { + const requests: MockAi['requests'] = []; + let fail = { status: 0, times: 0 }; + + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (c) => chunks.push(c)); + req.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf8'); + let body: any = null; + try { + body = raw ? JSON.parse(raw) : null; + } catch { + /* ignore */ + } + const path = req.url ?? ''; + requests.push({ path, headers: req.headers, body }); + const json = (status: number, data: unknown) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(data)); + }; + + if (req.headers.authorization !== 'Bearer mock-key') return json(401, { error: { message: 'Incorrect API key provided: ' + String(req.headers.authorization), type: 'invalid_request_error' } }); + if (path.endsWith('/models')) return json(200, { object: 'list', data: [{ id: 'mock-large' }, { id: 'mock-small' }, { id: 'mock-embedding-1' }] }); + if (!path.endsWith('/chat/completions')) return json(404, { error: { message: 'not found' } }); + if (fail.times > 0) { + fail.times -= 1; + const kind = fail.status === 401 ? ['Incorrect API key provided', 'invalid_request_error'] : fail.status === 429 ? ['Rate limit reached', 'rate_limit_error'] : ['The server had an error', 'server_error']; + return json(fail.status, { error: { message: kind[0], type: kind[1] } }); + } + + const messages: any[] = body?.messages ?? []; + const tools: any[] = body?.tools ?? []; + const names = tools.map((t) => t.function?.name); + const last = messages[messages.length - 1] ?? {}; + const lastUser = [...messages].reverse().find((m) => m.role === 'user'); + const userText = text(lastUser?.content); + const all = messages.map((m) => text(m.content)).join('\n'); + + res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' }); + const send = (delta: Record, finish: string | null = null) => + res.write(`data: ${JSON.stringify({ id: 'mock', object: 'chat.completion.chunk', model: body?.model, choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`); + const end = (finish: string, outTokens: number) => { + send({}, finish); + res.write(`data: ${JSON.stringify({ id: 'mock', object: 'chat.completion.chunk', choices: [], usage: { prompt_tokens: Math.ceil(raw.length / 4), completion_tokens: outTokens } })}\n\n`); + res.write('data: [DONE]\n\n'); + res.end(); + }; + const callTool = (name: string) => { + const args = JSON.stringify(toolArgs(name, userText, all)); + send({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: `call_${Math.random().toString(36).slice(2, 10)}`, type: 'function', function: { name, arguments: '' } }] }); + // Arguments arrive in fragments, like a real provider. + for (let i = 0; i < args.length; i += 37) send({ tool_calls: [{ index: 0, function: { arguments: args.slice(i, i + 37) } }] }); + end('tool_calls', Math.ceil(args.length / 4)); + }; + const say = (content: string) => { + send({ role: 'assistant', content: '' }); + for (const word of content.match(/\S+\s*/g) ?? []) send({ content: word }); + end('stop', Math.ceil(content.length / 4)); + }; + + const forced = body?.tool_choice?.function?.name as string | undefined; + if (forced) return callTool(forced); + if (last.role === 'tool') { + const ref = /"ref":"(c\d+)"/.exec(text(last.content))?.[1]; + if (/draft_ready|revisions_ready/.test(text(last.content))) return say('The draft is ready. Review it and save it when you are happy with it.'); + if (/awaiting_user_confirmation/.test(text(last.content))) return say('I have prepared the change. Press **Confirm** to apply it.'); + return say(ref ? `Based on your documents, loop diuretics relieve congestion and need potassium monitoring [${ref}].` : 'Here is what I found in your library.'); + } + if (names.length > 0 && body?.tool_choice !== 'none') { + const tool = chooseTool(userText, names); + if (tool) return callTool(tool); + } + if (/JSON Schema/.test(text(messages[0]?.content))) { + // Prompt-JSON fallback for models flagged as not supporting tools. + return say(JSON.stringify(/cards/.test(text(messages[0]?.content)) ? toolArgs('submit_flashcards', userText, all) : toolArgs('submit_questions', userText, all))); + } + const ref = /\[(c\d+)\]/.exec(userText)?.[1]; + return say(`## Overview\n\nThis is a mock study guide.\n\n- Key point one${ref ? ` [${ref}]` : ''}\n- Key point two\n\n## Check yourself\n\n1. What is the priority?`); + }); + }); + + return new Promise((resolve) => { + server.listen(port, '127.0.0.1', () => { + const { port: actual } = server.address() as AddressInfo; + resolve({ + url: `http://127.0.0.1:${actual}/v1`, + requests, + failNext: (status, times = 1) => { + fail = { status, times }; + }, + close: () => new Promise((done) => server.close(() => done())), + }); + }); + }); +} diff --git a/web/src/app/Shell.tsx b/web/src/app/Shell.tsx index 13aee3a..4ed648d 100644 --- a/web/src/app/Shell.tsx +++ b/web/src/app/Shell.tsx @@ -304,8 +304,8 @@ function useLogout() { try { await api.post('/api/auth/logout'); } finally { - qc.clear(); - await qc.invalidateQueries({ queryKey: qk.auth }); + qc.setQueryData(qk.auth, { configured: true, authenticated: false }); + qc.removeQueries({ predicate: (q) => q.queryKey[0] !== 'auth' }); } }; } diff --git a/web/src/features/flashcards.tsx b/web/src/features/flashcards.tsx index 40b6da6..d44c698 100644 --- a/web/src/features/flashcards.tsx +++ b/web/src/features/flashcards.tsx @@ -28,13 +28,21 @@ export function FlipCard({ footer?: ReactNode; }) { const [showHint, setShowHint] = useState(false); - useEffect(() => setShowHint(false), [cardKey]); + // Only animate the front when it comes back from the answer, not when a new card arrives. + const [turned, setTurned] = useState(false); + useEffect(() => { + setShowHint(false); + setTurned(false); + }, [cardKey]); + useEffect(() => { + if (flipped) setTurned(true); + }, [flipped]); const face = 'flip-face flex min-h-[320px] flex-col rounded-[22px] border border-line bg-surface shadow-2 sm:min-h-[380px]'; // Short answers read best large and centred; long ones as normal text. const size = (text: string) => (text.length < 60 ? 'text-[28px] sm:text-[34px] leading-tight text-center' : text.length < 220 ? 'text-[21px] sm:text-[24px] leading-snug text-center' : 'text-[17px] sm:text-[18px]'); return ( -
-
+
+