All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
- Stems no longer all open the same way: the writing rules, batch prompt and planner ask for varied openings (age openings stay part of the mix), and when one opener covers more than a quarter of a quiz the excess stems have only their opening reworded, keeping every number, before the check. - Rationales are asked for as Markdown (a bold key point, then short bullets); older one-paragraph rationales are laid out the same way on display. - The practice score in the quiz header is a plain "x/y correct", with no percentage or colour. - Fix a flaky assertion: the mock's answer-key fix is a no-op when the shuffle already put the answer in place. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
457 lines
27 KiB
TypeScript
457 lines
27 KiB
TypeScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
|
import { queueIdle } from '../server/services/documents.ts';
|
|
import { buildDocx, buildPdf, TINY_PNG } from '../server/documents/testFixtures.ts';
|
|
import { createTestApp, sampleQuestions, type TestApp } from './helpers.ts';
|
|
import { startMockAi, type MockAi } from './mockAi.ts';
|
|
|
|
let mock: MockAi;
|
|
let t: TestApp;
|
|
|
|
beforeAll(async () => {
|
|
mock = await startMockAi();
|
|
});
|
|
afterAll(() => mock.close());
|
|
beforeEach(async () => {
|
|
t = await createTestApp();
|
|
mock.requests.length = 0;
|
|
});
|
|
afterEach(() => t.close());
|
|
|
|
/** Connects the mock provider and switches on its chat models. */
|
|
async function connectAi(opts: { tools?: boolean } = {}) {
|
|
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
|
|
const found = await t.api('POST', `/api/ai/providers/${provider.id}/discover`);
|
|
const models = found.body.models as any[];
|
|
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: opts.tools ?? true } });
|
|
return { provider, models };
|
|
}
|
|
|
|
async function upload(classId: string, filename: string, bytes: Uint8Array, mime?: string) {
|
|
const form = new FormData();
|
|
form.append('classId', classId);
|
|
form.append('file', new File([Buffer.from(bytes)], filename, mime ? { type: mime } : undefined));
|
|
const res = await t.raw('POST', '/api/documents', { body: form });
|
|
const body = (await res.json()) as any;
|
|
await queueIdle(t.ctx);
|
|
return { status: res.status, body };
|
|
}
|
|
|
|
async function sse(url: string, body: unknown): Promise<any[]> {
|
|
const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
|
|
expect(res.status).toBe(200);
|
|
expect(res.headers.get('content-type')).toMatch(/text\/event-stream/);
|
|
const raw = await res.text();
|
|
return raw
|
|
.split(/\n\n/)
|
|
.map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join(''))
|
|
.filter(Boolean)
|
|
.map((d) => JSON.parse(d));
|
|
}
|
|
|
|
/** A request from the Python quiz check rather than from generation itself. */
|
|
const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review');
|
|
/** The quick call that groups topic labels after a quiz is written. */
|
|
const isGrouping = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_topics');
|
|
const isNaming = (r: { body: any }) => ['name_set', 'submit_proofread', 'plan_questions', 'submit_stems'].includes(r.body?.tool_choice?.function?.name);
|
|
|
|
async function waitForDraft(id: string) {
|
|
for (let i = 0; i < 200; i += 1) {
|
|
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
|
|
if (d.status !== 'generating') return d;
|
|
await new Promise((r) => setTimeout(r, 25));
|
|
}
|
|
throw new Error('draft never finished');
|
|
}
|
|
|
|
const LECTURE = `# Heart failure pharmacology
|
|
|
|
## Loop diuretics
|
|
Furosemide relieves congestion by blocking sodium and chloride reabsorption in the loop of Henle. Monitor potassium, daily weights and hearing.
|
|
|
|
## Digoxin
|
|
Digoxin increases contractility. The therapeutic range is 0.5 to 2 ng/mL. Hypokalemia increases the risk of toxicity. Early toxicity causes nausea and yellow halos.
|
|
`;
|
|
|
|
describe('AI providers and models', () => {
|
|
it('stores keys encrypted, never returns them, discovers models and routes tasks', async () => {
|
|
expect((await t.api('GET', '/api/ai/status')).body.ready).toBe(false);
|
|
const { provider, models } = await connectAi();
|
|
expect(provider.hasKey).toBe(true);
|
|
expect(JSON.stringify(provider)).not.toContain('mock-key');
|
|
expect(provider.keyPreview).toMatch(/•/);
|
|
// The embedding model is filtered out of the chat model list.
|
|
expect(models.map((m) => m.model).sort()).toEqual(['mock-large', 'mock-small']);
|
|
|
|
// Nothing sensitive in any listing, and the database holds only ciphertext.
|
|
const listing = await t.api('GET', '/api/ai/providers');
|
|
expect(JSON.stringify(listing.body)).not.toContain('mock-key');
|
|
const row = t.ctx.db.get<{ api_key_enc: string }>('SELECT api_key_enc FROM ai_providers')!;
|
|
expect(row.api_key_enc).toMatch(/^v1\./);
|
|
expect(row.api_key_enc).not.toContain('mock-key');
|
|
expect(fs.readFileSync(path.join(t.dir, 'study.db')).includes('mock-key')).toBe(false);
|
|
|
|
const test = (await t.api('POST', `/api/ai/providers/${provider.id}/test`)).body.result;
|
|
expect(test.ok).toBe(true);
|
|
expect(mock.requests.at(-1)!.headers.authorization).toBe('Bearer mock-key');
|
|
|
|
// Per-task routing with parameters.
|
|
const small = models.find((m) => m.model === 'mock-small');
|
|
const large = models.find((m) => m.model === 'mock-large');
|
|
await t.api('PUT', '/api/ai/tasks', { default: { modelId: large.id, temperature: null, reasoningEffort: null, maxOutputTokens: null }, quick: { modelId: small.id, temperature: 0.2, reasoningEffort: null, maxOutputTokens: 800 } });
|
|
const status = (await t.api('GET', '/api/ai/status')).body;
|
|
expect(status.ready).toBe(true);
|
|
expect(status.resolved.quick.label).toBe('mock-small');
|
|
expect(status.resolved.generate.label).toBe('mock-large');
|
|
|
|
const events = await sse('/api/ai/explain', { kind: 'card', front: 'Antidote for heparin', back: 'Protamine sulfate' });
|
|
expect(events.filter((e) => e.type === 'text').map((e) => e.delta).join('')).toContain('mock study guide');
|
|
const call = mock.requests.at(-1)!;
|
|
expect(call.body.model).toBe('mock-small');
|
|
expect(call.body.temperature).toBe(0.2);
|
|
|
|
// A wrong key is reported clearly, and the key is not echoed back.
|
|
const bad = (await t.api('POST', '/api/ai/providers', { name: 'Bad', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'sk-wrong-secret-value' })).body.provider;
|
|
const badTest = (await t.api('POST', `/api/ai/providers/${bad.id}/test`)).body.result;
|
|
expect(badTest.ok).toBe(false);
|
|
expect(badTest.message).not.toContain('sk-wrong-secret-value');
|
|
|
|
const usage = (await t.api('GET', '/api/ai/usage')).body;
|
|
expect(usage.total.calls).toBeGreaterThanOrEqual(1);
|
|
|
|
// Removing a provider clears task assignments that pointed at it.
|
|
await t.api('DELETE', `/api/ai/providers/${provider.id}`);
|
|
expect((await t.api('GET', '/api/ai/tasks')).body.tasks.quick.modelId).toBeNull();
|
|
});
|
|
|
|
it('explains that AI is not set up instead of failing obscurely', async () => {
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'C' })).body.class;
|
|
const res = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'anything' });
|
|
expect(res.status).toBe(409);
|
|
expect(res.body.error.code).toBe('ai_not_configured');
|
|
const events = await sse('/api/ai/chat', { message: 'hello' });
|
|
expect(events.find((e) => e.type === 'error').message).toMatch(/No AI model/);
|
|
});
|
|
});
|
|
|
|
describe('documents', () => {
|
|
it('uploads, extracts, chunks, searches, serves and deletes files', async () => {
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
const pdf = buildPdf([{ lines: ['Loop diuretics', 'Furosemide can cause hypokalemia and ototoxicity.'] }, { lines: ['Digoxin toxicity', 'Yellow halos and nausea are early signs.'] }], { title: 'Cardiac drugs' });
|
|
const up = await upload(cls.id, 'cardiac-drugs.pdf', pdf, 'application/pdf');
|
|
expect(up.status).toBe(201);
|
|
const id = up.body.documents[0].id;
|
|
const detail = (await t.api('GET', `/api/documents/${id}`)).body;
|
|
expect(detail.document).toMatchObject({ status: 'ready', docKind: 'pdf', pageCount: 2, needsOcr: false });
|
|
expect(detail.document.chunkCount).toBeGreaterThan(0);
|
|
|
|
const chunks = (await t.api('GET', `/api/documents/${id}/chunks`)).body.chunks;
|
|
expect(chunks.map((c: any) => c.text).join(' ')).toMatch(/ototoxicity/);
|
|
expect(chunks.at(-1).pageEnd).toBe(2);
|
|
|
|
const search = (await t.api('GET', '/api/search?q=halos')).body.results;
|
|
expect(search.some((r: any) => r.kind === 'passage' && r.parentId === id)).toBe(true);
|
|
|
|
// The stored file is served back intact, only to the signed-in owner.
|
|
const file = await t.raw('GET', `/api/documents/${id}/file`);
|
|
expect(file.headers.get('content-type')).toBe('application/pdf');
|
|
expect(new Uint8Array(await file.arrayBuffer()).length).toBe(pdf.length);
|
|
expect((await t.raw('GET', `/api/documents/${id}/file`, { cookie: null })).status).toBe(401);
|
|
|
|
// Word documents and pasted text go through the same pipeline.
|
|
const docx = await upload(cls.id, 'notes.docx', buildDocx([{ type: 'heading', level: 1, text: 'Anticoagulants' }, { type: 'p', text: 'Protamine sulfate reverses heparin.' }]));
|
|
expect((await t.api('GET', `/api/documents/${docx.body.documents[0].id}`)).body.document.status).toBe('ready');
|
|
const pasted = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE })).body.document;
|
|
await queueIdle(t.ctx);
|
|
expect((await t.api('GET', `/api/documents/${pasted.id}`)).body.outline.map((o: any) => o.heading)).toContain('Heart failure pharmacology');
|
|
|
|
// Unsupported and disguised files are rejected with a helpful message.
|
|
const exe = await upload(cls.id, 'notes.pdf', new Uint8Array([0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00]));
|
|
expect(exe.status).toBe(415);
|
|
expect(exe.body.error.message).toMatch(/not a supported file type/);
|
|
|
|
// Images are accepted and flagged for AI reading when no vision model exists.
|
|
const img = await upload(cls.id, 'slide.png', TINY_PNG, 'image/png');
|
|
const imgDoc = (await t.api('GET', `/api/documents/${img.body.documents[0].id}`)).body.document;
|
|
expect(imgDoc).toMatchObject({ status: 'ready', needsOcr: true, chunkCount: 0 });
|
|
expect(imgDoc.warnings.join(' ')).toMatch(/vision/);
|
|
|
|
const stored = fs.readdirSync(path.join(t.dir, 'uploads'));
|
|
expect(stored.length).toBe(4);
|
|
await t.api('DELETE', `/api/documents/${id}`);
|
|
expect(fs.readdirSync(path.join(t.dir, 'uploads')).length).toBe(3);
|
|
expect((await t.api('GET', '/api/search?q=ototoxicity')).body.results.filter((r: any) => r.kind === 'passage')).toHaveLength(0);
|
|
});
|
|
});
|
|
|
|
describe('generation', () => {
|
|
it('writes a quiz from a document in batches, as a draft that can be edited and saved', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure lecture', text: LECTURE })).body.document;
|
|
await queueIdle(t.ctx);
|
|
|
|
const started = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { documentIds: [doc.id] }, count: 12, style: 'nclex', focus: 'digoxin toxicity' });
|
|
expect(started.status).toBe(202);
|
|
const draft = await waitForDraft(started.body.draftId);
|
|
expect(draft.status).toBe('ready');
|
|
expect(draft.payload.questions).toHaveLength(12);
|
|
expect(draft.meta.sources[0].title).toBe('Heart failure lecture');
|
|
// Citations were mapped from excerpt ids to readable labels.
|
|
expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/);
|
|
// 12 questions at 8 per batch means two model calls, each carrying the source excerpts.
|
|
const all = mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
|
|
const calls = all.filter((r) => !isCheck(r) && !isGrouping(r) && !isNaming(r));
|
|
expect(all.filter(isGrouping)).toHaveLength(1);
|
|
expect(calls).toHaveLength(2);
|
|
// Then the Python check: one run, a review, and its fix applied to question 2.
|
|
const checks = all.filter(isCheck);
|
|
expect(checks.length).toBeGreaterThanOrEqual(2);
|
|
const pythonResult = JSON.parse(checks[1].body.messages.find((m: any) => m.role === 'tool').content);
|
|
// 12 questions against a request for 12, then the select-all count and correct answers per question.
|
|
expect(pythonResult.ok).toBe(true);
|
|
expect(pythonResult.stdout).toMatch(/^12 12 \d+ \[(?:[12], ){11}[12]\]\n$/);
|
|
expect(draft.payload.questions[1].options.map((o: any) => o.correct)).toEqual([false, true, false, false]);
|
|
// The mock's fix keys option 2; when the shuffle already put the answer there it changes
|
|
// nothing and is skipped, so "1 fix applied" only shows when it did something.
|
|
expect(draft.meta.notes.join(' ')).toMatch(/Checked with Python \(1 run\): 2 of 2 checks passed(?:, 1 fix applied|\.)/);
|
|
// Every mock stem opens "Mock question…": the excess ones were reworded (stem only) before the check.
|
|
expect(draft.meta.notes.join(' ')).toMatch(/Reworded how \d+ questions open/);
|
|
expect(draft.payload.questions.filter((q: any) => q.stem.startsWith('On the unit, Mock question')).length).toBeGreaterThan(4);
|
|
expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/);
|
|
// The questions are planned first; both batches then run at the same time, each with its own planned angles.
|
|
expect(all.filter((r) => r.body?.tool_choice?.function?.name === 'plan_questions')).toHaveLength(1);
|
|
expect(JSON.stringify(calls[0].body.messages)).toMatch(/Mock angle 1\b/);
|
|
expect(JSON.stringify(calls[1].body.messages)).toMatch(/SOURCE EXCERPTS/);
|
|
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Mock angle 9\b/);
|
|
expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } });
|
|
|
|
// Nothing is in the library until the draft is saved.
|
|
expect((await t.api('GET', `/api/classes/${cls.id}`)).body.items.filter((i: any) => i.kind === 'quiz')).toHaveLength(0);
|
|
|
|
// Edit: break one question, and the save is refused with a pointer to it.
|
|
const payload = structuredClone(draft.payload);
|
|
payload.title = 'Digoxin practice';
|
|
payload.questions[1].options.forEach((o: any) => (o.correct = false));
|
|
await t.api('PUT', `/api/drafts/${draft.id}`, { payload });
|
|
const refused = await t.api('POST', `/api/drafts/${draft.id}/save`, {});
|
|
expect(refused.status).toBe(400);
|
|
expect(refused.body.error.message).toMatch(/Question 2/);
|
|
payload.questions.splice(1, 1);
|
|
await t.api('PUT', `/api/drafts/${draft.id}`, { payload });
|
|
const saved = (await t.api('POST', `/api/drafts/${draft.id}/save`, {})).body.saved;
|
|
expect(saved).toMatchObject({ kind: 'quiz', count: 11 });
|
|
const quiz = (await t.api('GET', `/api/quizzes/${saved.id}`)).body;
|
|
expect(quiz.quiz.title).toBe('Digoxin practice');
|
|
expect(quiz.questions).toHaveLength(11);
|
|
expect(quiz.questions[0].rationale).toMatch(/Source: Heart failure lecture/);
|
|
// Saving twice does not duplicate.
|
|
expect((await t.api('POST', `/api/drafts/${draft.id}/save`, {})).body.saved.id).toBe(saved.id);
|
|
});
|
|
|
|
it('makes flashcards and study guides, converts between formats, and appends to existing items', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE })).body.document;
|
|
await queueIdle(t.ctx);
|
|
|
|
const cards = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { documentIds: [doc.id] }, count: 10, cardStyle: 'medication' })).body.draftId);
|
|
expect(cards.payload.cards).toHaveLength(10);
|
|
const deckId = (await t.api('POST', `/api/drafts/${cards.id}/save`, {})).body.saved.id;
|
|
|
|
// Deck -> quiz conversion uses the cards as the source.
|
|
const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId);
|
|
expect(converted.status).toBe('ready');
|
|
expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r) && !isGrouping(r) && !isNaming(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
|
|
|
|
// Append more cards to the existing deck.
|
|
const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId);
|
|
await t.api('POST', `/api/drafts/${more.id}/save`, { mode: 'append', targetId: deckId });
|
|
expect((await t.api('GET', `/api/decks/${deckId}`)).body.cards).toHaveLength(14);
|
|
|
|
const guide = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'note', classId: cls.id, source: { documentIds: [doc.id] }, noteStyle: 'study_guide' })).body.draftId);
|
|
expect(guide.payload.content).toMatch(/## Overview/);
|
|
// The raw [c12] marker was replaced with a readable citation.
|
|
expect(guide.payload.content).not.toMatch(/\[c\d+\]/);
|
|
expect(guide.payload.content).toMatch(/\(Lecture/);
|
|
const note = (await t.api('POST', `/api/drafts/${guide.id}/save`, {})).body.saved;
|
|
expect(note.kind).toBe('note');
|
|
|
|
// Discarding removes it from the list of drafts.
|
|
const junk = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'x y z', count: 2 })).body.draftId);
|
|
await t.api('DELETE', `/api/drafts/${junk.id}`);
|
|
expect((await t.api('GET', '/api/drafts')).body.drafts.map((d: any) => d.id)).not.toContain(junk.id);
|
|
});
|
|
|
|
it('falls back to prompt-JSON for models without tool support and survives a transient failure', async () => {
|
|
await connectAi({ tools: false });
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
mock.failNext(500, 1);
|
|
const draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId);
|
|
expect(draft.status).toBe('ready');
|
|
expect(draft.payload.questions).toHaveLength(3);
|
|
expect(draft.meta.notes.join(' ')).toMatch(/check was skipped/);
|
|
const call = mock.requests.at(-1)!;
|
|
expect(call.body.tools).toBeUndefined();
|
|
expect(call.body.response_format).toEqual({ type: 'json_object' });
|
|
});
|
|
|
|
it('reports a clear failure when the provider keeps erroring', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
mock.failNext(401, 10);
|
|
let draft;
|
|
try {
|
|
draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId);
|
|
} finally {
|
|
mock.failNext(0, 0);
|
|
}
|
|
expect(draft.status).toBe('failed');
|
|
expect(draft.error).toMatch(/API key|rejected|auth/i);
|
|
});
|
|
|
|
it('improves existing questions and applies the revisions in place', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
const quizId = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Rough draft', questions: sampleQuestions })).body.quiz.id;
|
|
const before = (await t.api('GET', `/api/quizzes/${quizId}`)).body.questions;
|
|
const draft = await waitForDraft((await t.api('POST', '/api/ai/improve', { quizId })).body.draftId);
|
|
expect(draft.meta.replaceQuestionIds).toEqual(before.map((q: any) => q.id));
|
|
await t.api('POST', `/api/drafts/${draft.id}/save`, { mode: 'replace' });
|
|
const after = (await t.api('GET', `/api/quizzes/${quizId}`)).body.questions;
|
|
expect(after.map((q: any) => q.id)).toEqual(before.map((q: any) => q.id));
|
|
expect(after[0].stem).toMatch(/Mock question 1/);
|
|
});
|
|
});
|
|
|
|
describe('assistant', () => {
|
|
it('uses tools to search documents and answers with citations', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE });
|
|
await queueIdle(t.ctx);
|
|
|
|
const events = await sse('/api/ai/chat', { message: 'What should I monitor with furosemide?', context: { classId: cls.id } });
|
|
const types = events.map((e) => e.type);
|
|
expect(types[0]).toBe('start');
|
|
expect(types.at(-1)).toBe('done');
|
|
const tool = events.filter((e) => e.type === 'tool').at(-1).tool;
|
|
expect(tool).toMatchObject({ name: 'search_documents', status: 'done' });
|
|
const sources = events.find((e) => e.type === 'ui' && e.item.type === 'sources').item.items;
|
|
const answer = events.filter((e) => e.type === 'text').map((e) => e.delta).join('');
|
|
expect(answer).toContain(`[${sources[0].ref}]`);
|
|
|
|
// The conversation is stored and reloads as one collapsed assistant message.
|
|
const conversationId = events[0].conversationId;
|
|
const convo = (await t.api('GET', `/api/ai/conversations/${conversationId}`)).body;
|
|
expect(convo.messages.map((m: any) => m.role)).toEqual(['user', 'assistant']);
|
|
expect(convo.messages[1].tools[0].name).toBe('search_documents');
|
|
expect(convo.messages[1].text).toContain('potassium');
|
|
|
|
// The system prompt tells the model which class is in view; the API key never reaches the model.
|
|
const first = mock.requests.find((r) => r.path.endsWith('/chat/completions'))!;
|
|
expect(JSON.stringify(first.body.messages[0])).toMatch(/currently viewing the class \\"Pharm\\"/);
|
|
expect(JSON.stringify(first.body)).not.toContain('mock-key');
|
|
});
|
|
|
|
it('creates drafts through tools, never saving by itself', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Peds' })).body.class;
|
|
await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Respiratory', text: LECTURE });
|
|
await queueIdle(t.ctx);
|
|
|
|
const events = await sse('/api/ai/chat', { message: 'Create a 4-question quiz using the documents in this class', context: { classId: cls.id } });
|
|
const draftItem = events.find((e) => e.type === 'ui' && e.item.type === 'draft').item;
|
|
expect(draftItem).toMatchObject({ kind: 'quiz', count: 4, status: 'ready' });
|
|
expect(events.filter((e) => e.type === 'text').map((e) => e.delta).join('')).toMatch(/Review it and save/);
|
|
expect((await t.api('GET', `/api/classes/${cls.id}`)).body.items.filter((i: any) => i.kind === 'quiz')).toHaveLength(0);
|
|
|
|
const saved = (await t.api('POST', `/api/drafts/${draftItem.draftId}/save`, {})).body.saved;
|
|
expect((await t.api('GET', `/api/quizzes/${saved.id}`)).body.questions).toHaveLength(4);
|
|
// Reloading the conversation shows the draft as saved.
|
|
const convo = (await t.api('GET', `/api/ai/conversations/${events[0].conversationId}`)).body;
|
|
expect(convo.messages[1].ui.find((u: any) => u.type === 'draft').status).toBe('saved');
|
|
});
|
|
|
|
it('only changes the library after the user confirms', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
const quizId = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Old quiz', questions: sampleQuestions })).body.quiz.id;
|
|
|
|
const events = await sse('/api/ai/chat', { message: `Please delete quiz ${quizId}`, context: { classId: cls.id } });
|
|
const action = events.find((e) => e.type === 'ui' && e.item.type === 'action').item;
|
|
expect(action.status).toBe('pending');
|
|
expect(action.changes[0]).toBe('Delete quiz "Old quiz"');
|
|
// Still there.
|
|
expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(200);
|
|
|
|
// Declining leaves it alone; a second proposal that is confirmed removes it.
|
|
expect((await t.api('POST', `/api/ai/actions/${action.actionId}`, { confirm: false })).body.status).toBe('cancelled');
|
|
expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(200);
|
|
expect((await t.api('POST', `/api/ai/actions/${action.actionId}`, { confirm: true })).status).toBe(400);
|
|
|
|
const again = await sse('/api/ai/chat', { conversationId: events[0].conversationId, message: `I changed my mind, delete quiz ${quizId}` });
|
|
const second = again.find((e) => e.type === 'ui' && e.item.type === 'action').item;
|
|
expect((await t.api('POST', `/api/ai/actions/${second.actionId}`, { confirm: true })).body.status).toBe('confirmed');
|
|
expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(404);
|
|
|
|
// An id that does not exist is reported back to the model as a tool error, not a crash.
|
|
const bogus = await sse('/api/ai/chat', { message: 'delete quiz nonexistent-id-123' });
|
|
expect(bogus.filter((e) => e.type === 'tool').at(-1).tool.status).toBe('error');
|
|
expect(bogus.at(-1).type).toBe('done');
|
|
});
|
|
});
|
|
|
|
describe('reading long scanned PDFs', () => {
|
|
async function scannedPdf(pages: number) {
|
|
const { PDFDocument } = await import('pdf-lib');
|
|
const doc = await PDFDocument.create();
|
|
for (let i = 0; i < pages; i++) doc.addPage([612, 792]);
|
|
return doc.save();
|
|
}
|
|
const ocrCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions') && JSON.stringify(r.body?.messages ?? '').includes('Transcribe this study material'));
|
|
|
|
it('reads the PDF in page batches, keeps real page numbers and caches finished batches', async () => {
|
|
const { models } = await connectAi();
|
|
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { capabilities: { vision: true, pdf: true } });
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Med-Surg' })).body.class;
|
|
|
|
const up = await upload(cls.id, 'scanned-textbook.pdf', await scannedPdf(20), 'application/pdf');
|
|
expect(up.status).toBe(201);
|
|
const id = up.body.documents[0].id;
|
|
const doc = (await t.api('GET', `/api/documents/${id}`)).body.document;
|
|
expect(doc).toMatchObject({ status: 'ready', needsOcr: false, pageCount: 20 });
|
|
|
|
// 20 pages at 8 per batch: pages 1-8, 9-16, 17-20, each sent as its own PDF.
|
|
const calls = ocrCalls();
|
|
expect(calls).toHaveLength(3);
|
|
const prompts = calls.map((c) => JSON.stringify(c.body.messages));
|
|
expect(prompts[0]).toContain('pages 1-8 of a 20-page document');
|
|
expect(prompts[2]).toContain('pages 17-20 of a 20-page document');
|
|
expect(prompts[1]).toContain('data:application/pdf;base64,');
|
|
|
|
const chunks = (await t.api('GET', `/api/documents/${id}/chunks`)).body.chunks as { text: string; pageStart: number | null; pageEnd: number | null }[];
|
|
const text = chunks.map((c) => c.text).join('\n');
|
|
for (const n of [1, 8, 9, 20]) expect(text).toContain(`page ${n}:`);
|
|
expect(Math.min(...chunks.map((c) => c.pageStart ?? 99))).toBe(1);
|
|
expect(Math.max(...chunks.map((c) => c.pageEnd ?? 0))).toBe(20);
|
|
expect(fs.readdirSync(path.join(t.dir, 'uploads')).some((f) => f.endsWith('.ocr.json'))).toBe(true);
|
|
|
|
// Reprocessing without asking for a fresh read reuses the cached batches: no new AI calls.
|
|
mock.requests.length = 0;
|
|
await t.api('POST', `/api/documents/${id}/reprocess`, {});
|
|
await queueIdle(t.ctx);
|
|
expect(ocrCalls()).toHaveLength(0);
|
|
expect((await t.api('GET', `/api/documents/${id}`)).body.document.needsOcr).toBe(false);
|
|
|
|
// "Read with AI" asks for a fresh read and sends every batch again.
|
|
await t.api('POST', `/api/documents/${id}/reprocess`, { ocr: true });
|
|
await queueIdle(t.ctx);
|
|
expect(ocrCalls()).toHaveLength(3);
|
|
|
|
// Deleting the document removes the cache with the file.
|
|
await t.api('DELETE', `/api/documents/${id}`);
|
|
expect(fs.readdirSync(path.join(t.dir, 'uploads')).some((f) => f.endsWith('.ocr.json'))).toBe(false);
|
|
});
|
|
});
|