Study/tests/revision4.test.ts
Elijah 25bc03ea75
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s
Fourth revision: OpenRouter, live tool stages, Explain layout, share sections
- Interactive blocks are encouraged by default; Explain gets a reduced library
  and never writes practice questions.
- Assistant tool steps breathe while running, show live stages from generation
  and the Python check, and read in the past tense when done.
- The assistant asks before searching the web when class materials do not
  cover a question; clarifying questions list every class's exams and never
  ask about format (always NCLEX).
- Explain opens beside the question from 900px wide, moving the question left
  while open and back on the next one.
- Generate dialog drops the practice set/exam switch (presets cover it) and the
  Generate buttons lose their icon.
- Python check notes are green when every check passed, applied fixes neutral.
- Share pages group by exam, then Quizzes, Flashcards, Notes, Class Materials,
  hiding empty groups; the class page uses the same order. "Documents" is now
  "Class Materials".
- OpenRouter provider replaces the Ollama type; migration 6 turns Ollama
  connections into OpenAI-compatible ones at /v1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 21:54:39 -07:00

141 lines
7.1 KiB
TypeScript

/**
* Fourth revision: live stages on the assistant's tool steps, Explain without practice
* questions, interactive blocks by default, asking before searching the web, exams offered in
* clarifying questions, and OpenRouter replacing the Ollama connection type.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { pastTense } from '../server/ai/assistant.ts';
import { genUiPrompt } from '../server/ai/genui.ts';
import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts';
import { MIGRATIONS } from '../server/migrations.ts';
import { queueIdle } from '../server/services/documents.ts';
import { newId, now } from '../server/util.ts';
import { createTestApp, type TestApp } from './helpers.ts';
import { startMockAi, type MockAi } from './mockAi.ts';
let mock: MockAi;
let t: TestApp;
beforeAll(async () => {
mock = await startMockAi();
});
afterAll(async () => {
await mock.close();
});
beforeEach(async () => {
t = await createTestApp();
mock.requests.length = 0;
mock.delayChat(0);
});
afterEach(() => t.close());
async function connectAi() {
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[];
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } });
}
async function sse(url: string, body: unknown): Promise<any[]> {
const res = await t.raw('POST', url, { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
expect(res.status).toBe(200);
return (await res.text())
.split(/\n\n/)
.map((block) => block.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim()).join(''))
.filter(Boolean)
.map((d) => JSON.parse(d));
}
const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
describe('assistant tool steps', () => {
it('report each stage of a generation while it runs, then a past-tense line', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' });
await queueIdle(t.ctx);
const events = await sse('/api/ai/chat', { message: 'Write 3 quiz questions from my documents', context: { classId: cls.id } });
const steps = events.filter((e) => e.type === 'tool' && e.tool.name === 'generate_quiz').map((e) => e.tool);
const stages = steps.filter((s) => s.status === 'running' && s.stage).map((s) => s.stage);
expect(stages).toContain('Reading your material');
expect(stages.some((s: string) => /^Writing questions/.test(s))).toBe(true);
const last = steps.at(-1);
expect(last.status).toBe('done');
expect(last.stage).toBeUndefined();
expect(last.label).toMatch(/^Wrote \d+ quiz questions?$/);
});
it('turn labels into the past tense once done', () => {
expect(pastTense('Searching documents for "sepsis"')).toBe('Searched documents for "sepsis"');
expect(pastTense('Setting up a practice session')).toBe('Set up a practice session');
expect(pastTense('Running Python')).toBe('Ran Python');
expect(pastTense('Something else')).toBe('Something else');
});
});
describe('interactive blocks', () => {
it('are encouraged in chats, and Explain never quizzes', () => {
const chat = genUiPrompt('chat');
expect(chat).toContain('Reach for a block whenever');
expect(chat).toContain('Question(stem: string');
const explain = genUiPrompt('explain');
expect(explain).not.toContain('Question(stem: string');
expect(explain).not.toContain('Flashcards(');
expect(explain).not.toContain('self-check Question');
expect(explain).toContain('never include a Question, Flashcards');
});
it('are part of the Explain request', async () => {
await connectAi();
await sse('/api/ai/explain', { kind: 'text', text: 'Loop diuretics waste potassium.' });
const system = chatCalls()[0].body.messages[0].content as string;
expect(system).toContain('Give information only');
expect(system).not.toContain('Question(stem: string');
});
});
describe('the assistant prompt', () => {
it('lists every class with its exams and keeps quizzes NCLEX style', async () => {
await connectAi();
const cls = (await t.api('POST', '/api/classes', { name: 'Med-Surg' })).body.class;
const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 3' })).body.folder;
await sse('/api/ai/chat', { message: 'hello there' });
const system = chatCalls()[0].body.messages[0].content as string;
expect(system).toContain(`- Med-Surg — id ${cls.id}; exams: Exam 3 (exam_id ${exam.id})`);
expect(system).toContain('Never ask about the question format or style');
expect(system).toContain('interactive without being asked');
expect(system).toContain('class materials');
});
it('asks before searching the web when the materials do not cover a question', () => {
const mcp: LoadedMcp = { tools: [], unavailable: [], servers: [{ id: 'web-search', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web' }] };
const chat = mcpPromptNote(mcp, { askBeforeWeb: true });
expect(chat).toContain('Should I look it up on the web?');
expect(chat).toContain('do not answer from your own knowledge');
const elsewhere = mcpPromptNote(mcp);
expect(elsewhere).not.toContain('Should I look it up on the web?');
expect(elsewhere).toContain("anything the student's own material does not cover");
});
});
describe('OpenRouter replaces the Ollama type', () => {
it('is offered as a provider, and Ollama is not', async () => {
const kinds = (await t.api('GET', '/api/ai/meta')).body.adapters.map((a: any) => a.kind);
expect(kinds).toContain('openrouter');
expect(kinds).not.toContain('ollama');
const refused = await t.api('POST', '/api/ai/providers', { name: 'Old', kind: 'ollama' });
expect(refused.status).toBe(422);
});
it('moves existing Ollama connections to OpenAI-compatible at /v1', () => {
const insert = (base: string | null) => {
const id = newId();
t.ctx.db.run('INSERT INTO ai_providers (id, name, kind, base_url, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)', id, 'Ollama', 'ollama', base, now(), now());
return id;
};
const ids = [insert(null), insert('http://box:11434'), insert('http://box:11434/v1/'), insert('http://box:11434/api')];
t.ctx.db.exec(MIGRATIONS.find((m) => m.id === 6)!.sql);
const rows = ids.map((id) => t.ctx.db.get<{ kind: string; base_url: string }>('SELECT kind, base_url FROM ai_providers WHERE id = ?', id)!);
expect(rows.map((r) => r.kind)).toEqual(['openai_compatible', 'openai_compatible', 'openai_compatible', 'openai_compatible']);
expect(rows.map((r) => r.base_url)).toEqual(['http://localhost:11434/v1', 'http://box:11434/v1', 'http://box:11434/v1', 'http://box:11434/v1']);
});
});