Add mock AI provider and AI integration tests; fix sign-in transition and card flip

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 13:30:27 -07:00
parent 74a98ac1f1
commit 75158c4449
8 changed files with 720 additions and 33 deletions

377
tests/ai.test.ts Normal file
View file

@ -0,0 +1,377 @@
import fs from 'node:fs';
import path from 'node:path';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { queueIdle } from '../server/services/documents.ts';
import { buildDocx, buildPdf, TINY_PNG } from '../server/documents/testFixtures.ts';
import { createTestApp, sampleQuestions, type TestApp } from './helpers.ts';
import { startMockAi, type MockAi } from './mockAi.ts';
let mock: MockAi;
let t: TestApp;
beforeAll(async () => {
mock = await startMockAi();
});
afterAll(() => mock.close());
beforeEach(async () => {
t = await createTestApp();
mock.requests.length = 0;
});
afterEach(() => t.close());
/** Connects the mock provider and switches on its chat models. */
async function connectAi(opts: { tools?: boolean } = {}) {
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
const found = await t.api('POST', `/api/ai/providers/${provider.id}/discover`);
const models = found.body.models as any[];
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: opts.tools ?? true } });
return { provider, models };
}
async function upload(classId: string, filename: string, bytes: Uint8Array, mime?: string) {
const form = new FormData();
form.append('classId', classId);
form.append('file', new File([Buffer.from(bytes)], filename, mime ? { type: mime } : undefined));
const res = await t.raw('POST', '/api/documents', { body: form });
const body = (await res.json()) as any;
await queueIdle(t.ctx);
return { status: res.status, body };
}
async function sse(url: string, body: unknown): Promise<any[]> {
const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
expect(res.status).toBe(200);
expect(res.headers.get('content-type')).toMatch(/text\/event-stream/);
const raw = await res.text();
return raw
.split(/\n\n/)
.map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join(''))
.filter(Boolean)
.map((d) => JSON.parse(d));
}
async function waitForDraft(id: string) {
for (let i = 0; i < 200; i += 1) {
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
if (d.status !== 'generating') return d;
await new Promise((r) => setTimeout(r, 25));
}
throw new Error('draft never finished');
}
const LECTURE = `# Heart failure pharmacology
## Loop diuretics
Furosemide relieves congestion by blocking sodium and chloride reabsorption in the loop of Henle. Monitor potassium, daily weights and hearing.
## Digoxin
Digoxin increases contractility. The therapeutic range is 0.5 to 2 ng/mL. Hypokalemia increases the risk of toxicity. Early toxicity causes nausea and yellow halos.
`;
describe('AI providers and models', () => {
it('stores keys encrypted, never returns them, discovers models and routes tasks', async () => {
expect((await t.api('GET', '/api/ai/status')).body.ready).toBe(false);
const { provider, models } = await connectAi();
expect(provider.hasKey).toBe(true);
expect(JSON.stringify(provider)).not.toContain('mock-key');
expect(provider.keyPreview).toMatch(/•/);
// The embedding model is filtered out of the chat model list.
expect(models.map((m) => m.model).sort()).toEqual(['mock-large', 'mock-small']);
// Nothing sensitive in any listing, and the database holds only ciphertext.
const listing = await t.api('GET', '/api/ai/providers');
expect(JSON.stringify(listing.body)).not.toContain('mock-key');
const row = t.ctx.db.get<{ api_key_enc: string }>('SELECT api_key_enc FROM ai_providers')!;
expect(row.api_key_enc).toMatch(/^v1\./);
expect(row.api_key_enc).not.toContain('mock-key');
expect(fs.readFileSync(path.join(t.dir, 'study.db')).includes('mock-key')).toBe(false);
const test = (await t.api('POST', `/api/ai/providers/${provider.id}/test`)).body.result;
expect(test.ok).toBe(true);
expect(mock.requests.at(-1)!.headers.authorization).toBe('Bearer mock-key');
// Per-task routing with parameters.
const small = models.find((m) => m.model === 'mock-small');
const large = models.find((m) => m.model === 'mock-large');
await t.api('PUT', '/api/ai/tasks', { default: { modelId: large.id, temperature: null, reasoningEffort: null, maxOutputTokens: null }, quick: { modelId: small.id, temperature: 0.2, reasoningEffort: null, maxOutputTokens: 800 } });
const status = (await t.api('GET', '/api/ai/status')).body;
expect(status.ready).toBe(true);
expect(status.resolved.quick.label).toBe('mock-small');
expect(status.resolved.generate.label).toBe('mock-large');
const events = await sse('/api/ai/explain', { kind: 'card', front: 'Antidote for heparin', back: 'Protamine sulfate' });
expect(events.filter((e) => e.type === 'text').map((e) => e.delta).join('')).toContain('mock study guide');
const call = mock.requests.at(-1)!;
expect(call.body.model).toBe('mock-small');
expect(call.body.temperature).toBe(0.2);
// A wrong key is reported clearly, and the key is not echoed back.
const bad = (await t.api('POST', '/api/ai/providers', { name: 'Bad', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'sk-wrong-secret-value' })).body.provider;
const badTest = (await t.api('POST', `/api/ai/providers/${bad.id}/test`)).body.result;
expect(badTest.ok).toBe(false);
expect(badTest.message).not.toContain('sk-wrong-secret-value');
const usage = (await t.api('GET', '/api/ai/usage')).body;
expect(usage.total.calls).toBeGreaterThanOrEqual(1);
// Removing a provider clears task assignments that pointed at it.
await t.api('DELETE', `/api/ai/providers/${provider.id}`);
expect((await t.api('GET', '/api/ai/tasks')).body.tasks.quick.modelId).toBeNull();
});
it('explains that AI is not set up instead of failing obscurely', async () => {
const cls = (await t.api('POST', '/api/classes', { name: 'C' })).body.class;
const res = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'anything' });
expect(res.status).toBe(409);
expect(res.body.error.code).toBe('ai_not_configured');
const events = await sse('/api/ai/chat', { message: 'hello' });
expect(events.find((e) => e.type === 'error').message).toMatch(/No AI model/);
});
});
describe('documents', () => {
it('uploads, extracts, chunks, searches, serves and deletes files', async () => {
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
const pdf = buildPdf([{ lines: ['Loop diuretics', 'Furosemide can cause hypokalemia and ototoxicity.'] }, { lines: ['Digoxin toxicity', 'Yellow halos and nausea are early signs.'] }], { title: 'Cardiac drugs' });
const up = await upload(cls.id, 'cardiac-drugs.pdf', pdf, 'application/pdf');
expect(up.status).toBe(201);
const id = up.body.documents[0].id;
const detail = (await t.api('GET', `/api/documents/${id}`)).body;
expect(detail.document).toMatchObject({ status: 'ready', docKind: 'pdf', pageCount: 2, needsOcr: false });
expect(detail.document.chunkCount).toBeGreaterThan(0);
const chunks = (await t.api('GET', `/api/documents/${id}/chunks`)).body.chunks;
expect(chunks.map((c: any) => c.text).join(' ')).toMatch(/ototoxicity/);
expect(chunks.at(-1).pageEnd).toBe(2);
const search = (await t.api('GET', '/api/search?q=halos')).body.results;
expect(search.some((r: any) => r.kind === 'passage' && r.parentId === id)).toBe(true);
// The stored file is served back intact, only to the signed-in owner.
const file = await t.raw('GET', `/api/documents/${id}/file`);
expect(file.headers.get('content-type')).toBe('application/pdf');
expect(new Uint8Array(await file.arrayBuffer()).length).toBe(pdf.length);
expect((await t.raw('GET', `/api/documents/${id}/file`, { cookie: null })).status).toBe(401);
// Word documents and pasted text go through the same pipeline.
const docx = await upload(cls.id, 'notes.docx', buildDocx([{ type: 'heading', level: 1, text: 'Anticoagulants' }, { type: 'p', text: 'Protamine sulfate reverses heparin.' }]));
expect((await t.api('GET', `/api/documents/${docx.body.documents[0].id}`)).body.document.status).toBe('ready');
const pasted = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE })).body.document;
await queueIdle(t.ctx);
expect((await t.api('GET', `/api/documents/${pasted.id}`)).body.outline.map((o: any) => o.heading)).toContain('Heart failure pharmacology');
// Unsupported and disguised files are rejected with a helpful message.
const exe = await upload(cls.id, 'notes.pdf', new Uint8Array([0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00]));
expect(exe.status).toBe(415);
expect(exe.body.error.message).toMatch(/not a supported file type/);
// Images are accepted and flagged for AI reading when no vision model exists.
const img = await upload(cls.id, 'slide.png', TINY_PNG, 'image/png');
const imgDoc = (await t.api('GET', `/api/documents/${img.body.documents[0].id}`)).body.document;
expect(imgDoc).toMatchObject({ status: 'ready', needsOcr: true, chunkCount: 0 });
expect(imgDoc.warnings.join(' ')).toMatch(/vision/);
const stored = fs.readdirSync(path.join(t.dir, 'uploads'));
expect(stored.length).toBe(4);
await t.api('DELETE', `/api/documents/${id}`);
expect(fs.readdirSync(path.join(t.dir, 'uploads')).length).toBe(3);
expect((await t.api('GET', '/api/search?q=ototoxicity')).body.results.filter((r: any) => r.kind === 'passage')).toHaveLength(0);
});
});
describe('generation', () => {
it('writes a quiz from a document in batches, as a draft that can be edited and saved', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure lecture', text: LECTURE })).body.document;
await queueIdle(t.ctx);
const started = await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { documentIds: [doc.id] }, count: 12, style: 'nclex', focus: 'digoxin toxicity' });
expect(started.status).toBe(202);
const draft = await waitForDraft(started.body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.questions).toHaveLength(12);
expect(draft.meta.sources[0].title).toBe('Heart failure lecture');
// Citations were mapped from excerpt ids to readable labels.
expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/);
// 12 questions at 8 per batch means two model calls, each carrying the source excerpts.
const calls = mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
expect(calls).toHaveLength(2);
expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/);
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/);
expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } });
// Nothing is in the library until the draft is saved.
expect((await t.api('GET', `/api/classes/${cls.id}`)).body.items.filter((i: any) => i.kind === 'quiz')).toHaveLength(0);
// Edit: break one question, and the save is refused with a pointer to it.
const payload = structuredClone(draft.payload);
payload.title = 'Digoxin practice';
payload.questions[1].options.forEach((o: any) => (o.correct = false));
await t.api('PUT', `/api/drafts/${draft.id}`, { payload });
const refused = await t.api('POST', `/api/drafts/${draft.id}/save`, {});
expect(refused.status).toBe(400);
expect(refused.body.error.message).toMatch(/Question 2/);
payload.questions.splice(1, 1);
await t.api('PUT', `/api/drafts/${draft.id}`, { payload });
const saved = (await t.api('POST', `/api/drafts/${draft.id}/save`, {})).body.saved;
expect(saved).toMatchObject({ kind: 'quiz', count: 11 });
const quiz = (await t.api('GET', `/api/quizzes/${saved.id}`)).body;
expect(quiz.quiz.title).toBe('Digoxin practice');
expect(quiz.questions).toHaveLength(11);
expect(quiz.questions[0].rationale).toMatch(/Source: Heart failure lecture/);
// Saving twice does not duplicate.
expect((await t.api('POST', `/api/drafts/${draft.id}/save`, {})).body.saved.id).toBe(saved.id);
});
it('makes flashcards and study guides, converts between formats, and appends to existing items', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE })).body.document;
await queueIdle(t.ctx);
const cards = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { documentIds: [doc.id] }, count: 10, cardStyle: 'medication' })).body.draftId);
expect(cards.payload.cards).toHaveLength(10);
const deckId = (await t.api('POST', `/api/drafts/${cards.id}/save`, {})).body.saved.id;
// Deck -> quiz conversion uses the cards as the source.
const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId);
expect(converted.status).toBe('ready');
expect(JSON.stringify(mock.requests.at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
// Append more cards to the existing deck.
const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId);
await t.api('POST', `/api/drafts/${more.id}/save`, { mode: 'append', targetId: deckId });
expect((await t.api('GET', `/api/decks/${deckId}`)).body.cards).toHaveLength(14);
const guide = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'note', classId: cls.id, source: { documentIds: [doc.id] }, noteStyle: 'study_guide' })).body.draftId);
expect(guide.payload.content).toMatch(/## Overview/);
// The raw [c12] marker was replaced with a readable citation.
expect(guide.payload.content).not.toMatch(/\[c\d+\]/);
expect(guide.payload.content).toMatch(/\(Lecture/);
const note = (await t.api('POST', `/api/drafts/${guide.id}/save`, {})).body.saved;
expect(note.kind).toBe('note');
// Discarding removes it from the list of drafts.
const junk = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'x y z', count: 2 })).body.draftId);
await t.api('DELETE', `/api/drafts/${junk.id}`);
expect((await t.api('GET', '/api/drafts')).body.drafts.map((d: any) => d.id)).not.toContain(junk.id);
});
it('falls back to prompt-JSON for models without tool support and survives a transient failure', async () => {
await connectAi({ tools: false });
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
mock.failNext(500, 1);
const draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.questions).toHaveLength(3);
const call = mock.requests.at(-1)!;
expect(call.body.tools).toBeUndefined();
expect(call.body.response_format).toEqual({ type: 'json_object' });
});
it('reports a clear failure when the provider keeps erroring', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
mock.failNext(401, 10);
let draft;
try {
draft = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, focus: 'electrolytes', count: 3 })).body.draftId);
} finally {
mock.failNext(0, 0);
}
expect(draft.status).toBe('failed');
expect(draft.error).toMatch(/API key|rejected|auth/i);
});
it('improves existing questions and applies the revisions in place', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
const quizId = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Rough draft', questions: sampleQuestions })).body.quiz.id;
const before = (await t.api('GET', `/api/quizzes/${quizId}`)).body.questions;
const draft = await waitForDraft((await t.api('POST', '/api/ai/improve', { quizId })).body.draftId);
expect(draft.meta.replaceQuestionIds).toEqual(before.map((q: any) => q.id));
await t.api('POST', `/api/drafts/${draft.id}/save`, { mode: 'replace' });
const after = (await t.api('GET', `/api/quizzes/${quizId}`)).body.questions;
expect(after.map((q: any) => q.id)).toEqual(before.map((q: any) => q.id));
expect(after[0].stem).toMatch(/Mock question 1/);
});
});
describe('assistant', () => {
it('uses tools to search documents and answers with citations', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Lecture', text: LECTURE });
await queueIdle(t.ctx);
const events = await sse('/api/ai/chat', { message: 'What should I monitor with furosemide?', context: { classId: cls.id } });
const types = events.map((e) => e.type);
expect(types[0]).toBe('start');
expect(types.at(-1)).toBe('done');
const tool = events.filter((e) => e.type === 'tool').at(-1).tool;
expect(tool).toMatchObject({ name: 'search_documents', status: 'done' });
const sources = events.find((e) => e.type === 'ui' && e.item.type === 'sources').item.items;
const answer = events.filter((e) => e.type === 'text').map((e) => e.delta).join('');
expect(answer).toContain(`[${sources[0].ref}]`);
// The conversation is stored and reloads as one collapsed assistant message.
const conversationId = events[0].conversationId;
const convo = (await t.api('GET', `/api/ai/conversations/${conversationId}`)).body;
expect(convo.messages.map((m: any) => m.role)).toEqual(['user', 'assistant']);
expect(convo.messages[1].tools[0].name).toBe('search_documents');
expect(convo.messages[1].text).toContain('potassium');
// The system prompt tells the model which class is in view; the API key never reaches the model.
const first = mock.requests.find((r) => r.path.endsWith('/chat/completions'))!;
expect(JSON.stringify(first.body.messages[0])).toMatch(/currently viewing the class \\"Pharm\\"/);
expect(JSON.stringify(first.body)).not.toContain('mock-key');
});
it('creates drafts through tools, never saving by itself', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Peds' })).body.class;
await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Respiratory', text: LECTURE });
await queueIdle(t.ctx);
const events = await sse('/api/ai/chat', { message: 'Create a 4-question quiz using the documents in this class', context: { classId: cls.id } });
const draftItem = events.find((e) => e.type === 'ui' && e.item.type === 'draft').item;
expect(draftItem).toMatchObject({ kind: 'quiz', count: 4, status: 'ready' });
expect(events.filter((e) => e.type === 'text').map((e) => e.delta).join('')).toMatch(/Review it and save/);
expect((await t.api('GET', `/api/classes/${cls.id}`)).body.items.filter((i: any) => i.kind === 'quiz')).toHaveLength(0);
const saved = (await t.api('POST', `/api/drafts/${draftItem.draftId}/save`, {})).body.saved;
expect((await t.api('GET', `/api/quizzes/${saved.id}`)).body.questions).toHaveLength(4);
// Reloading the conversation shows the draft as saved.
const convo = (await t.api('GET', `/api/ai/conversations/${events[0].conversationId}`)).body;
expect(convo.messages[1].ui.find((u: any) => u.type === 'draft').status).toBe('saved');
});
it('only changes the library after the user confirms', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
const quizId = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Old quiz', questions: sampleQuestions })).body.quiz.id;
const events = await sse('/api/ai/chat', { message: `Please delete quiz ${quizId}`, context: { classId: cls.id } });
const action = events.find((e) => e.type === 'ui' && e.item.type === 'action').item;
expect(action.status).toBe('pending');
expect(action.changes[0]).toBe('Delete quiz "Old quiz"');
// Still there.
expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(200);
// Declining leaves it alone; a second proposal that is confirmed removes it.
expect((await t.api('POST', `/api/ai/actions/${action.actionId}`, { confirm: false })).body.status).toBe('cancelled');
expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(200);
expect((await t.api('POST', `/api/ai/actions/${action.actionId}`, { confirm: true })).status).toBe(400);
const again = await sse('/api/ai/chat', { conversationId: events[0].conversationId, message: `I changed my mind, delete quiz ${quizId}` });
const second = again.find((e) => e.type === 'ui' && e.item.type === 'action').item;
expect((await t.api('POST', `/api/ai/actions/${second.actionId}`, { confirm: true })).body.status).toBe('confirmed');
expect((await t.api('GET', `/api/quizzes/${quizId}`)).status).toBe(404);
// An id that does not exist is reported back to the model as a tool error, not a crash.
const bogus = await sse('/api/ai/chat', { message: 'delete quiz nonexistent-id-123' });
expect(bogus.filter((e) => e.type === 'tool').at(-1).tool.status).toBe('error');
expect(bogus.at(-1).type).toBe('done');
});
});

181
tests/mockAi.ts Normal file
View file

@ -0,0 +1,181 @@
/**
* A small OpenAI-compatible server with scripted behaviour, for tests and for trying the
* AI features locally without a real provider (see scripts/mock-ai.mjs).
*
* - Forced tool call -> returns that tool call with plausible arguments.
* - Tools + user turn -> picks a tool from keywords in the message.
* - Tool result turn -> answers in text, citing the first [cN] ref it was given.
* - No tools -> streams a short Markdown answer.
*/
import http from 'node:http';
import type { AddressInfo } from 'node:net';
export interface MockAi {
url: string;
requests: { path: string; headers: http.IncomingHttpHeaders; body: any }[];
/** Force the next N chat calls to fail with this status. */
failNext: (status: number, times?: number) => void;
close: () => Promise<void>;
}
const text = (content: unknown): string =>
typeof content === 'string' ? content : Array.isArray(content) ? content.map((p: any) => (typeof p?.text === 'string' ? p.text : '')).join('\n') : '';
function questions(n: number, refs: string[], offset = 0) {
return Array.from({ length: n }, (_, k) => {
const i = k + offset;
const multi = i % 3 === 2;
return {
type: multi ? 'multi' : 'single',
stem: `Mock question ${i + 1}: which action is the priority for the client described in case ${i + 1}?`,
options: [
{ text: `Correct action ${i + 1}`, correct: true, why: 'This addresses the immediate risk.' },
{ text: `Plausible distractor ${i + 1}a`, correct: multi, why: multi ? 'Also indicated.' : 'Lower priority.' },
{ text: `Plausible distractor ${i + 1}b`, correct: false, why: 'Not indicated.' },
{ text: `Plausible distractor ${i + 1}c`, correct: false, why: 'Contraindicated.' },
],
rationale: `Rationale for question ${i + 1}.`,
topic: i % 2 === 0 ? 'Heart failure' : 'Pharmacology',
difficulty: 'medium',
source: refs.length ? refs[i % refs.length] : '',
};
});
}
function toolArgs(name: string, userText: string, all: string): Record<string, unknown> {
const count = Number(/(?:Write|write)\s+(\d+)/.exec(userText)?.[1] ?? /(\d+)[- ](?:question|card|flashcard)/.exec(userText)?.[1] ?? 3);
// Only refs from the user's turn count: the system prompt contains an example id.
const refs = [...new Set([...userText.matchAll(/\[(c\d+)\]/g)].map((m) => m[1]))];
// Continue numbering after anything already written, so later batches are not duplicates.
const offset = (userText.split('Already written')[1]?.match(/^- /gm) ?? []).length;
void all;
switch (name) {
case 'submit_questions':
return { questions: questions(count, refs, offset) };
case 'submit_flashcards':
return { cards: Array.from({ length: count }, (_, i) => ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, hint: '', tags: ['mock'] })) };
case 'generate_quiz':
return { count: Math.min(count, 5), topic: 'heart failure', style: 'nclex', use_all_class_documents: /document/i.test(userText), from_weak_areas: /wrong|weak|miss/i.test(userText) };
case 'generate_flashcards':
return { count: Math.min(count, 6), topic: 'medications', use_all_class_documents: /document/i.test(userText) };
case 'search_documents':
return { query: userText.replace(/[^\w\s]/g, ' ').split(/\s+/).filter((w) => w.length > 3).slice(0, 6).join(' ') };
case 'get_performance':
case 'list_classes':
case 'list_materials':
return {};
case 'propose_changes': {
const m = /(delete|rename)\s+(quiz|deck|note|document)\s+([\w-]{8,})/i.exec(userText);
return { reason: 'Requested by the user.', changes: [{ op: m?.[1]?.toLowerCase() ?? 'delete', kind: m?.[2]?.toLowerCase() ?? 'quiz', id: m?.[3] ?? 'missing', title: 'Renamed by mock' }] };
}
default:
return {};
}
}
function chooseTool(userText: string, names: string[]): string | null {
const pick = (name: string) => (names.includes(name) ? name : null);
if (/\b(delete|rename)\b/i.test(userText)) return pick('propose_changes');
if (/flashcard/i.test(userText)) return pick('generate_flashcards');
if (/\bquiz\b|practice questions/i.test(userText)) return pick('generate_quiz');
if (/should i study|performance|how am i doing/i.test(userText)) return pick('get_performance');
if (/\?|what|explain|summar/i.test(userText)) return pick('search_documents');
return null;
}
export function startMockAi(port = 0): Promise<MockAi> {
const requests: MockAi['requests'] = [];
let fail = { status: 0, times: 0 };
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on('data', (c) => chunks.push(c));
req.on('end', () => {
const raw = Buffer.concat(chunks).toString('utf8');
let body: any = null;
try {
body = raw ? JSON.parse(raw) : null;
} catch {
/* ignore */
}
const path = req.url ?? '';
requests.push({ path, headers: req.headers, body });
const json = (status: number, data: unknown) => {
res.writeHead(status, { 'content-type': 'application/json' });
res.end(JSON.stringify(data));
};
if (req.headers.authorization !== 'Bearer mock-key') return json(401, { error: { message: 'Incorrect API key provided: ' + String(req.headers.authorization), type: 'invalid_request_error' } });
if (path.endsWith('/models')) return json(200, { object: 'list', data: [{ id: 'mock-large' }, { id: 'mock-small' }, { id: 'mock-embedding-1' }] });
if (!path.endsWith('/chat/completions')) return json(404, { error: { message: 'not found' } });
if (fail.times > 0) {
fail.times -= 1;
const kind = fail.status === 401 ? ['Incorrect API key provided', 'invalid_request_error'] : fail.status === 429 ? ['Rate limit reached', 'rate_limit_error'] : ['The server had an error', 'server_error'];
return json(fail.status, { error: { message: kind[0], type: kind[1] } });
}
const messages: any[] = body?.messages ?? [];
const tools: any[] = body?.tools ?? [];
const names = tools.map((t) => t.function?.name);
const last = messages[messages.length - 1] ?? {};
const lastUser = [...messages].reverse().find((m) => m.role === 'user');
const userText = text(lastUser?.content);
const all = messages.map((m) => text(m.content)).join('\n');
res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' });
const send = (delta: Record<string, unknown>, finish: string | null = null) =>
res.write(`data: ${JSON.stringify({ id: 'mock', object: 'chat.completion.chunk', model: body?.model, choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`);
const end = (finish: string, outTokens: number) => {
send({}, finish);
res.write(`data: ${JSON.stringify({ id: 'mock', object: 'chat.completion.chunk', choices: [], usage: { prompt_tokens: Math.ceil(raw.length / 4), completion_tokens: outTokens } })}\n\n`);
res.write('data: [DONE]\n\n');
res.end();
};
const callTool = (name: string) => {
const args = JSON.stringify(toolArgs(name, userText, all));
send({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: `call_${Math.random().toString(36).slice(2, 10)}`, type: 'function', function: { name, arguments: '' } }] });
// Arguments arrive in fragments, like a real provider.
for (let i = 0; i < args.length; i += 37) send({ tool_calls: [{ index: 0, function: { arguments: args.slice(i, i + 37) } }] });
end('tool_calls', Math.ceil(args.length / 4));
};
const say = (content: string) => {
send({ role: 'assistant', content: '' });
for (const word of content.match(/\S+\s*/g) ?? []) send({ content: word });
end('stop', Math.ceil(content.length / 4));
};
const forced = body?.tool_choice?.function?.name as string | undefined;
if (forced) return callTool(forced);
if (last.role === 'tool') {
const ref = /"ref":"(c\d+)"/.exec(text(last.content))?.[1];
if (/draft_ready|revisions_ready/.test(text(last.content))) return say('The draft is ready. Review it and save it when you are happy with it.');
if (/awaiting_user_confirmation/.test(text(last.content))) return say('I have prepared the change. Press **Confirm** to apply it.');
return say(ref ? `Based on your documents, loop diuretics relieve congestion and need potassium monitoring [${ref}].` : 'Here is what I found in your library.');
}
if (names.length > 0 && body?.tool_choice !== 'none') {
const tool = chooseTool(userText, names);
if (tool) return callTool(tool);
}
if (/JSON Schema/.test(text(messages[0]?.content))) {
// Prompt-JSON fallback for models flagged as not supporting tools.
return say(JSON.stringify(/cards/.test(text(messages[0]?.content)) ? toolArgs('submit_flashcards', userText, all) : toolArgs('submit_questions', userText, all)));
}
const ref = /\[(c\d+)\]/.exec(userText)?.[1];
return say(`## Overview\n\nThis is a mock study guide.\n\n- Key point one${ref ? ` [${ref}]` : ''}\n- Key point two\n\n## Check yourself\n\n1. What is the priority?`);
});
});
return new Promise((resolve) => {
server.listen(port, '127.0.0.1', () => {
const { port: actual } = server.address() as AddressInfo;
resolve({
url: `http://127.0.0.1:${actual}/v1`,
requests,
failNext: (status, times = 1) => {
fail = { status, times };
},
close: () => new Promise((done) => server.close(() => done())),
});
});
});
}