All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 24s
On a class link with exams, the visitor's Generate dialog asks which exam the set is for (opening on the latest exam with materials, with "Not in an exam" when there is loose material). Only that exam's class materials are offered, and the set is saved into it. The server requires the choice and refuses materials from another exam. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
316 lines
17 KiB
TypeScript
316 lines
17 KiB
TypeScript
/**
|
|
* Fourth revision: live stages on the assistant's tool steps, Explain without practice
|
|
* questions, interactive blocks by default, asking before searching the web, exams offered in
|
|
* clarifying questions, and OpenRouter replacing the Ollama connection type.
|
|
*/
|
|
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
|
import { pastTense } from '../server/ai/assistant.ts';
|
|
import { genUiPrompt } from '../server/ai/genui.ts';
|
|
import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts';
|
|
import { MIGRATIONS } from '../server/migrations.ts';
|
|
import { queueIdle } from '../server/services/documents.ts';
|
|
import { newId, now } from '../server/util.ts';
|
|
import { createTestApp, type TestApp } from './helpers.ts';
|
|
import { startMockAi, type MockAi } from './mockAi.ts';
|
|
|
|
let mock: MockAi;
|
|
let t: TestApp;
|
|
|
|
beforeAll(async () => {
|
|
mock = await startMockAi();
|
|
});
|
|
afterAll(async () => {
|
|
await mock.close();
|
|
});
|
|
beforeEach(async () => {
|
|
t = await createTestApp();
|
|
mock.requests.length = 0;
|
|
mock.delayChat(0);
|
|
});
|
|
afterEach(() => t.close());
|
|
|
|
async function connectAi() {
|
|
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
|
|
const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[];
|
|
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } });
|
|
}
|
|
|
|
async function sse(url: string, body: unknown): Promise<any[]> {
|
|
const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
|
|
expect(res.status).toBe(200);
|
|
return (await res.text())
|
|
.split(/\n\n/)
|
|
.map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join(''))
|
|
.filter(Boolean)
|
|
.map((d) => JSON.parse(d));
|
|
}
|
|
|
|
const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
|
|
|
|
describe('assistant tool steps', () => {
|
|
it('report each stage of a generation while it runs, then a past-tense line', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' });
|
|
await queueIdle(t.ctx);
|
|
const events = await sse('/api/ai/chat', { message: 'Write 3 quiz questions from my documents', context: { classId: cls.id } });
|
|
const steps = events.filter((e) => e.type === 'tool' && e.tool.name === 'generate_quiz').map((e) => e.tool);
|
|
const stages = steps.filter((s) => s.status === 'running' && s.stage).map((s) => s.stage);
|
|
expect(stages).toContain('Reading your material');
|
|
expect(stages.some((s: string) => /^Writing questions/.test(s))).toBe(true);
|
|
const last = steps.at(-1);
|
|
expect(last.status).toBe('done');
|
|
expect(last.stage).toBeUndefined();
|
|
expect(last.label).toMatch(/^Wrote \d+ quiz questions?$/);
|
|
});
|
|
|
|
it('turn labels into the past tense once done', () => {
|
|
expect(pastTense('Searching documents for "sepsis"')).toBe('Searched documents for "sepsis"');
|
|
expect(pastTense('Setting up a practice session')).toBe('Set up a practice session');
|
|
expect(pastTense('Running Python')).toBe('Ran Python');
|
|
expect(pastTense('Something else')).toBe('Something else');
|
|
});
|
|
});
|
|
|
|
describe('interactive blocks', () => {
|
|
it('are encouraged in chats, and Explain never quizzes', () => {
|
|
const chat = genUiPrompt('chat');
|
|
expect(chat).toContain('Reach for a block whenever');
|
|
expect(chat).toContain('Question(stem: string');
|
|
const explain = genUiPrompt('explain');
|
|
expect(explain).not.toContain('Question(stem: string');
|
|
expect(explain).not.toContain('Flashcards(');
|
|
expect(explain).not.toContain('self-check Question');
|
|
expect(explain).toContain('never include a Question, Flashcards');
|
|
});
|
|
|
|
it('are part of the Explain request', async () => {
|
|
await connectAi();
|
|
await sse('/api/ai/explain', { kind: 'text', text: 'Loop diuretics waste potassium.' });
|
|
const system = chatCalls()[0].body.messages[0].content as string;
|
|
expect(system).toContain('Give information only');
|
|
expect(system).not.toContain('Question(stem: string');
|
|
});
|
|
});
|
|
|
|
describe('the assistant prompt', () => {
|
|
it('lists every class with its exams and keeps quizzes NCLEX style', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Med-Surg' })).body.class;
|
|
const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 3' })).body.folder;
|
|
await sse('/api/ai/chat', { message: 'hello there' });
|
|
const system = chatCalls()[0].body.messages[0].content as string;
|
|
expect(system).toContain(`- Med-Surg — id ${cls.id}; exams: Exam 3 (exam_id ${exam.id})`);
|
|
expect(system).toContain('Never ask about the question format or style');
|
|
expect(system).toContain('interactive without being asked');
|
|
expect(system).toContain('class materials');
|
|
});
|
|
|
|
it('asks before searching the web when the materials do not cover a question', () => {
|
|
const mcp: LoadedMcp = { tools: [], unavailable: [], servers: [{ id: 'web-search', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web' }] };
|
|
const chat = mcpPromptNote(mcp, { askBeforeWeb: true });
|
|
expect(chat).toContain('Should I look it up on the web?');
|
|
expect(chat).toContain('do not answer from your own knowledge');
|
|
const elsewhere = mcpPromptNote(mcp);
|
|
expect(elsewhere).not.toContain('Should I look it up on the web?');
|
|
expect(elsewhere).toContain("anything the student's own material does not cover");
|
|
});
|
|
});
|
|
|
|
describe('OpenRouter replaces the Ollama type', () => {
|
|
it('is offered as a provider, and Ollama is not', async () => {
|
|
const kinds = (await t.api('GET', '/api/ai/meta')).body.adapters.map((a: any) => a.kind);
|
|
expect(kinds).toContain('openrouter');
|
|
expect(kinds).not.toContain('ollama');
|
|
const refused = await t.api('POST', '/api/ai/providers', { name: 'Old', kind: 'ollama' });
|
|
expect(refused.status).toBe(422);
|
|
});
|
|
|
|
it('moves existing Ollama connections to OpenAI-compatible at /v1', () => {
|
|
const insert = (base: string | null) => {
|
|
const id = newId();
|
|
t.ctx.db.run('INSERT INTO ai_providers (id, name, kind, base_url, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)', id, 'Ollama', 'ollama', base, now(), now());
|
|
return id;
|
|
};
|
|
const ids = [insert(null), insert('http://box:11434'), insert('http://box:11434/v1/'), insert('http://box:11434/api')];
|
|
t.ctx.db.exec(MIGRATIONS.find((m) => m.id === 6)!.sql);
|
|
const rows = ids.map((id) => t.ctx.db.get<{ kind: string; base_url: string }>('SELECT kind, base_url FROM ai_providers WHERE id = ?', id)!);
|
|
expect(rows.map((r) => r.kind)).toEqual(['openai_compatible', 'openai_compatible', 'openai_compatible', 'openai_compatible']);
|
|
expect(rows.map((r) => r.base_url)).toEqual(['http://localhost:11434/v1', 'http://box:11434/v1', 'http://box:11434/v1', 'http://box:11434/v1']);
|
|
});
|
|
});
|
|
|
|
/* ------------------------------ med math and share-link generation ------------------------------ */
|
|
|
|
async function waitDraft(id: string): Promise<any> {
|
|
for (let i = 0; i < 400; i += 1) {
|
|
const draft = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
|
|
if (draft.status !== 'generating') return draft;
|
|
await new Promise((r) => setTimeout(r, 25));
|
|
}
|
|
throw new Error('generation did not finish');
|
|
}
|
|
|
|
async function classWithMaterial() {
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' })).body.document;
|
|
await queueIdle(t.ctx);
|
|
return { cls, doc };
|
|
}
|
|
|
|
describe('med math on request', () => {
|
|
it('writes exactly that many calculations, spread through the quiz, even without calculations in the material', async () => {
|
|
await connectAi();
|
|
const { cls } = await classWithMaterial();
|
|
const started = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 8, mathQuestions: 2, verify: false });
|
|
expect(started.status).toBe(202);
|
|
const draft = await waitDraft(started.body.draftId);
|
|
expect(draft.status).toBe('ready');
|
|
const types = draft.payload.questions.map((q: any) => q.type);
|
|
expect(types).toHaveLength(8);
|
|
expect(types.filter((x: string) => x === 'numeric')).toHaveLength(2);
|
|
// Not bunched at the end.
|
|
expect(types.slice(-2)).not.toEqual(['numeric', 'numeric']);
|
|
const sent = JSON.stringify(chatCalls().map((r) => r.body.messages));
|
|
expect(sent).toContain('medication math');
|
|
});
|
|
|
|
it('spreads items evenly', async () => {
|
|
const { spreadEvenly } = await import('../server/ai/generate.ts');
|
|
const list = [...Array.from({ length: 8 }, (_, i) => `r${i}`), 'm0', 'm1'];
|
|
expect(spreadEvenly(list, (x) => x.startsWith('m'))).toEqual(['r0', 'r1', 'm0', 'r2', 'r3', 'r4', 'r5', 'm1', 'r6', 'r7']);
|
|
});
|
|
});
|
|
|
|
describe('share-link visitors can generate', () => {
|
|
beforeEach(async () => {
|
|
const { resetShareAiLimits } = await import('../server/routes/shares.ts');
|
|
resetShareAiLimits();
|
|
});
|
|
|
|
async function waitSets(token: string): Promise<any[]> {
|
|
for (let i = 0; i < 400; i += 1) {
|
|
const res = await t.api('GET', `/api/public/${token}/generated`);
|
|
if (!res.body.sets.some((s: any) => s.status === 'generating')) return res.body.sets;
|
|
await new Promise((r) => setTimeout(r, 25));
|
|
}
|
|
throw new Error('generation did not finish');
|
|
}
|
|
|
|
it('is off until the owner allows it, then adds quizzes and flashcards to the class within daily limits', async () => {
|
|
await connectAi();
|
|
const { cls, doc } = await classWithMaterial();
|
|
const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder;
|
|
await t.api('PATCH', `/api/documents/${doc.id}`, { folderId: exam.id });
|
|
const share = (await t.api('POST', '/api/shares', { kind: 'folder', targetId: exam.id })).body.share;
|
|
const token = share.token;
|
|
expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 4 })).status).toBe(403);
|
|
await t.api('PATCH', `/api/shares/${share.id}`, { ai: { generate: true, explain: true } });
|
|
expect((await t.api('GET', `/api/public/${token}`)).body.ai.generate).toBe(true);
|
|
|
|
// At most 50 questions or 75 cards, materials from the link only.
|
|
expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 51 })).status).toBe(400);
|
|
expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 76 })).status).toBe(422);
|
|
expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', documentIds: ['not-shared'] })).status).toBe(404);
|
|
|
|
mock.requests.length = 0;
|
|
mock.delayChat(150);
|
|
const first = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 8, difficulty: 'hard', documentIds: [doc.id], math: true });
|
|
expect(first.status).toBe(202);
|
|
expect(first.body.usage.quiz).toEqual({ used: 1, limit: 4 });
|
|
// In the owner's drafts while it is being written.
|
|
expect((await t.api('GET', '/api/drafts')).body.drafts.find((d: any) => d.id === first.body.set.id)?.status).toBe('generating');
|
|
mock.delayChat(0);
|
|
const [set] = await waitSets(token);
|
|
expect(set).toMatchObject({ kind: 'quiz', status: 'ready', count: 8 });
|
|
expect(set.itemId).toBeTruthy();
|
|
|
|
// Saved into the exam as if the owner made it, and gone from Drafts.
|
|
expect((await t.api('GET', '/api/drafts')).body.drafts.some((d: any) => d.id === set.id)).toBe(false);
|
|
const saved = (await t.api('GET', `/api/quizzes/${set.itemId}`)).body;
|
|
expect(saved.quiz.folderId).toBe(exam.id);
|
|
expect(saved.quiz.description).toBe('Made with AI on a share link.');
|
|
const shown = (await t.api('GET', `/api/public/${token}`)).body.items.map((i: any) => i.id);
|
|
expect(shown).toContain(set.itemId);
|
|
const quiz = (await t.api('GET', `/api/public/${token}/items/quiz/${set.itemId}`)).body;
|
|
expect(quiz.questions.filter((q: any) => q.type === 'numeric')).toHaveLength(5);
|
|
const sent = JSON.stringify(chatCalls().map((r) => r.body.messages));
|
|
expect(sent).toContain('Difficulty: hard');
|
|
expect(sent).toMatch(/Exactly 1 of them must be select-all-that-apply/);
|
|
const explained = await t.raw('POST', `/api/public/${token}/explain`, { headers: { 'content-type': 'application/json' }, body: JSON.stringify({ kind: 'question', quizId: set.itemId, questionId: quiz.questions[0].id }) });
|
|
expect(explained.status).toBe(200);
|
|
const listed = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
|
|
expect(listed.generated.sets.map((s: any) => s.itemId)).toEqual([set.itemId]);
|
|
|
|
// Four quizzes and two flashcard sets a day.
|
|
for (let i = 0; i < 3; i += 1) expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 2 })).status).toBe(202);
|
|
const over = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 2 });
|
|
expect(over.status).toBe(429);
|
|
expect(over.body.error.message).toContain('4 quizzes');
|
|
for (let i = 0; i < 2; i += 1) expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3 })).status).toBe(202);
|
|
expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3 })).status).toBe(429);
|
|
await waitSets(token);
|
|
|
|
// The owner resets the allowance.
|
|
const reset = await t.api('POST', `/api/shares/${share.id}/generate/reset`);
|
|
expect(reset.body.generated.usage.quiz.used).toBe(0);
|
|
|
|
// Deleting the link keeps everything made on it, including a set still being written.
|
|
mock.delayChat(150);
|
|
const running = (await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3 })).body.set;
|
|
expect((await t.api('DELETE', `/api/shares/${share.id}`)).status).toBe(200);
|
|
mock.delayChat(0);
|
|
const finished = await waitDraft(running.id);
|
|
expect(finished.status).toBe('saved');
|
|
expect((await t.api('GET', `/api/decks/${finished.savedId}`)).status).toBe(200);
|
|
expect((await t.api('GET', `/api/quizzes/${set.itemId}`)).status).toBe(200);
|
|
});
|
|
|
|
it('defaults to a 50-question quiz', async () => {
|
|
await connectAi();
|
|
const { cls } = await classWithMaterial();
|
|
const share = (await t.api('POST', '/api/shares', { kind: 'class', targetId: cls.id })).body.share;
|
|
await t.api('PATCH', `/api/shares/${share.id}`, { ai: { generate: true } });
|
|
const res = await t.api('POST', `/api/public/${share.token}/generate`, { kind: 'quiz' });
|
|
expect(res.status).toBe(202);
|
|
const draft = await waitDraft(res.body.set.id);
|
|
expect(draft.meta.request).toMatchObject({ count: 50, sataShare: 0.25, format: 'exam' });
|
|
});
|
|
});
|
|
|
|
describe('class links: visitors choose the exam', () => {
|
|
beforeEach(async () => {
|
|
const { resetShareAiLimits } = await import('../server/routes/shares.ts');
|
|
resetShareAiLimits();
|
|
});
|
|
|
|
it('requires an exam, uses only its materials and saves the set into it', async () => {
|
|
await connectAi();
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Med-Surg' })).body.class;
|
|
const exam1 = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder;
|
|
const exam2 = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 2' })).body.folder;
|
|
const cardiac = (await t.api('POST', '/api/documents/text', { classId: cls.id, folderId: exam1.id, title: 'Cardiac', text: '# Cardiac\n\nDigoxin toxicity causes nausea and visual halos.' })).body.document;
|
|
await t.api('POST', '/api/documents/text', { classId: cls.id, folderId: exam2.id, title: 'Renal', text: '# Renal\n\nHyperkalemia in kidney failure shows peaked T waves. Kayexalate removes potassium.' });
|
|
await queueIdle(t.ctx);
|
|
const share = (await t.api('POST', '/api/shares', { kind: 'class', targetId: cls.id })).body.share;
|
|
await t.api('PATCH', `/api/shares/${share.id}`, { ai: { generate: true } });
|
|
const token = share.token;
|
|
|
|
const missing = await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3 });
|
|
expect(missing.status).toBe(400);
|
|
expect(missing.body.error.message).toContain('Choose which exam');
|
|
// Materials from another exam are refused.
|
|
expect((await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3, examId: exam2.id, documentIds: [cardiac.id] })).status).toBe(404);
|
|
|
|
mock.requests.length = 0;
|
|
const res = await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 3, examId: exam2.id });
|
|
expect(res.status).toBe(202);
|
|
const draft = await waitDraft(res.body.set.id);
|
|
expect(draft.status).toBe('saved');
|
|
const deck = (await t.api('GET', `/api/decks/${draft.savedId}`)).body.deck;
|
|
expect(deck.folderId).toBe(exam2.id);
|
|
const sent = JSON.stringify(chatCalls().map((r) => r.body.messages));
|
|
expect(sent).toContain('Kayexalate');
|
|
expect(sent).not.toContain('Digoxin');
|
|
});
|
|
});
|