Study/tests/revision5.test.ts
Elijah 10be5c39da
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s
Separate settings per generation step; explain Practice and Exam on shares
- Settings -> AI: Planning, Writing, Python check and Second review each get
  their own model, reasoning effort, temperature and output limit, shown under
  Content generation. Unset choices follow Content generation, then the
  default model.
- Shared quizzes say what Practice and Exam mean under the mode toggle.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 11:08:21 -07:00

386 lines
22 KiB
TypeScript

/**
* Fifth revision: malformed model output is repaired or retried (and a whole job restarted
* once), failed share-link sets can be dismissed by visitors, generated sets get their own
* names, flashcards are atomic without hints, the assistant never answers from general
* knowledge, and share-link AI expires after a week, is limited to a region and is locked by
* abuse protection.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts';
import { applyQuizFixes, suspiciousWording } from '../server/ai/verify.ts';
import { MIGRATIONS } from '../server/migrations.ts';
import { queueIdle } from '../server/services/documents.ts';
import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts';
import { createTestApp, type TestApp } from './helpers.ts';
import { startMockAi, type MockAi } from './mockAi.ts';
let mock: MockAi;
let t: TestApp;
beforeAll(async () => {
mock = await startMockAi();
});
afterAll(async () => {
await mock.close();
});
beforeEach(async () => {
t = await createTestApp();
mock.requests.length = 0;
mock.delayChat(0);
mock.badJsonNext('', 0);
mock.scriptReviews([]);
const { resetShareAiLimits } = await import('../server/routes/shares.ts');
resetShareAiLimits();
resetGeoForTests();
});
afterEach(() => t.close());
async function connectAi() {
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[];
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } });
}
async function classWithMaterial() {
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' })).body.document;
await queueIdle(t.ctx);
return { cls, doc };
}
async function waitDraft(id: string) {
for (let i = 0; i < 400; i += 1) {
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
if (d.status !== 'generating') return d;
await new Promise((r) => setTimeout(r, 25));
}
throw new Error('generation did not finish');
}
const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
const isRepair = (r: { body: any }) => JSON.stringify(r.body.messages ?? []).includes('You repair malformed JSON');
describe('malformed model output', () => {
it('is sent back to the model for repair instead of failing', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.badJsonNext('submit_flashcards', 1);
const started = await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 });
const draft = await waitDraft(started.body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.cards).toHaveLength(5);
expect(chatCalls().filter(isRepair)).toHaveLength(1);
});
it('starts the whole job over once when repair and a retry both fail, then fails for good', async () => {
await connectAi();
const { cls } = await classWithMaterial();
// Original, repair and retry all broken: the job starts over and succeeds.
mock.badJsonNext('submit_flashcards', 3);
let draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.cards).toHaveLength(5);
// Broken on the second run too: failed, with a readable error.
mock.badJsonNext('submit_flashcards', 6);
draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId);
expect(draft.status).toBe('failed');
expect(draft.error).toMatch(/not valid JSON/);
});
});
describe('the quiz check', () => {
const isReview = (r: { body: any }) => (r.body?.tools ?? []).some((x: any) => x.function?.name === 'submit_review');
const pass = { summary: 'All good.', checks: [{ name: 'Question count', passed: true }, { name: 'Answer keys', passed: true }], fixes: [] };
it('rewrites broken questions, replaces removed ones and checks again until the quiz passes', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.scriptReviews([
{
summary: 'Two problems.',
checks: [
{ name: 'Question count vs request', passed: false, detail: 'Q3 duplicates Q2; removing it leaves 5.' },
{ name: 'Stem integrity', passed: true, detail: 'Q1 rewritten.' },
],
fixes: [
{ question: 3, action: 'remove', reason: 'duplicate of Q2' },
{
question: 1,
action: 'rewrite',
reason: 'garbled stem',
question_text: { type: 'single', stem: 'An 80 kg client in septic shock needs norepinephrine. Which finding is the priority?', options: [{ text: 'MAP 58', correct: true }, { text: 'Temp 38.1', correct: false }, { text: 'HR 104', correct: false }, { text: 'Urine 40 mL/h', correct: false }], rationale: 'Perfusion first.' },
},
],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 6 })).body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.questions).toHaveLength(6);
expect(draft.payload.questions.some((q: any) => q.stem.startsWith('An 80 kg client'))).toBe(true);
// Two full checks: the first, then the re-check after the fixes and the replacement.
expect(chatCalls().filter((r) => isReview(r) && r.body.messages.filter((m: any) => m.role !== 'system').length === 1)).toHaveLength(2);
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/Checked with Python in 2 rounds/);
expect(notes).toMatch(/1 removed question was replaced/);
expect(notes).not.toMatch(/Check failed/);
});
it('checks rewritten questions again', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.scriptReviews([
{
summary: 'One rewrite.',
checks: [{ name: 'Stem integrity', passed: true, detail: 'Q2 fixed.' }],
fixes: [
{ question: 2, action: 'rewrite', reason: 'garbled', question_text: { type: 'single', stem: 'Which lab value is monitored for heparin?', options: [{ text: 'aPTT', correct: true }, { text: 'INR', correct: false }, { text: 'BNP', correct: false }, { text: 'TSH', correct: false }], rationale: 'aPTT guides heparin.' } },
],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/in 2 rounds/);
expect(notes).toMatch(/Question 2: rewritten/);
});
it('ignores a key fix that changes nothing', () => {
const q = { type: 'single' as const, stem: 'S', options: [{ text: 'a', correct: true }, { text: 'b', correct: false }, { text: 'c', correct: false }, { text: 'd', correct: false }], rationale: '', topic: 'T', difficulty: 'medium' as const, tags: [] };
const out = applyQuizFixes([q], [{ question: 1, action: 'set_correct', correct_options: [1], reason: 'Informational only' }]);
expect(out.applied).toEqual([]);
expect(out.refused).toEqual([]);
});
it('has a second reviewer rewrite a question whose writer changed its mind', async () => {
await connectAi();
const { cls } = await classWithMaterial();
// The first check leaves a self-correction in Q1 (as a model once did); the second review catches it.
mock.scriptReviews([
{
summary: 'Fine.',
checks: [{ name: 'Answer keys', passed: true }],
fixes: [{ question: 1, action: 'rewrite', reason: 'test setup', question_text: { type: 'numeric', stem: 'A 78 kg? No — an 80 kg client receives 4 mg in 250 mL at 15 mL/h. Dose in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000/250*15/60/80' }], rationale: 'Convert, then divide.' } }],
},
pass,
]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
const proofreads = chatCalls().filter((r) => r.body.tool_choice?.function?.name === 'submit_proofread');
expect(proofreads.length).toBeGreaterThanOrEqual(2);
// The leftover was pointed out to the reviewer, and the question no longer has it.
// (The first second-review runs alongside the check that planted it, so a later one sees it.)
expect(proofreads.some((p) => /looks like a leftover/.test(JSON.stringify(p.body.messages)))).toBe(true);
expect(draft.payload.questions.some((q: any) => /\?\s*No\b/.test(q.stem))).toBe(false);
expect(draft.meta.notes.join(' | ')).toMatch(/Second review: Question \d+: rewritten/);
});
it('spots leftover wording', () => {
const q = (stem: string) => ({ stem, options: [{ text: 'a', correct: true }], rationale: '' });
expect(suspiciousWording([q('A 78 kg? No — an 80 kg client'), q('A client with heart failure asks what to eat.'), q('Actually, the dose is 5 mg.')])).toHaveLength(2);
});
it('stops after three rounds when problems remain', async () => {
await connectAi();
const { cls } = await classWithMaterial();
const failing = { summary: 'Still wrong.', checks: [{ name: 'Answer keys', passed: false, detail: 'Q1 is disputed.' }], fixes: [] };
mock.scriptReviews([failing, failing, failing, failing]);
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
expect(draft.status).toBe('ready');
const notes = draft.meta.notes.join(' | ');
expect(notes).toMatch(/in 3 rounds/);
expect(notes).toMatch(/Check failed: Answer keys/);
});
});
describe('generation steps in Settings', () => {
it('use Content generation settings until a step has its own', async () => {
await connectAi();
const { resolveModel } = await import('../server/ai/service.ts');
const models = (await t.api('GET', '/api/ai/models')).body.models.filter((m: any) => m.enabled);
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { capabilities: { reasoning: true } });
const [a, b] = models;
const none = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null };
await t.api('PUT', '/api/ai/tasks', { default: { ...none, modelId: a.id }, generate: { ...none, reasoningEffort: 'high' } });
// Nothing set on the steps: they follow Content generation.
for (const step of ['plan', 'write', 'check', 'review'] as const) expect(resolveModel(t.ctx, step).params.reasoningEffort).toBe('high');
// A step's own effort wins; the others keep following.
await t.api('PUT', '/api/ai/tasks', { plan: { ...none, reasoningEffort: 'low' } });
expect(resolveModel(t.ctx, 'plan').params.reasoningEffort).toBe('low');
expect(resolveModel(t.ctx, 'write').params.reasoningEffort).toBe('high');
// A step with its own model does not take Content generation's parameters for it.
if (b) {
await t.api('PUT', '/api/ai/tasks', { review: { ...none, modelId: b.id } });
const review = resolveModel(t.ctx, 'review');
expect(review.model.id).toBe(b.id);
expect(review.params.reasoningEffort).toBeUndefined();
}
const meta = (await t.api('GET', '/api/ai/meta')).body.tasks;
expect(meta.plan).toMatchObject({ label: 'Planning', parent: 'generate' });
});
});
describe('generated sets', () => {
it('get their own name and description, and flashcards are atomic without hints', async () => {
await connectAi();
const { cls } = await classWithMaterial();
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
expect(draft.payload.title).toBe('Mock Terms');
expect(draft.payload.description).toBe('Covers the mock terms.');
const writing = chatCalls().find((r) => r.body.tool_choice?.function?.name === 'submit_flashcards')!;
const tool = writing.body.tools.find((x: any) => x.function.name === 'submit_flashcards');
expect(Object.keys(tool.function.parameters.properties.cards.items.properties)).not.toContain('hint');
expect(writing.body.messages[0].content).toMatch(/Atomic cards/);
// The same name twice in a class gets a number.
await t.api('POST', `/api/drafts/${draft.id}/save`, {});
const second = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
expect(second.payload.title).toBe('Mock Terms 2');
});
});
describe('the assistant', () => {
it('never offers to answer from general knowledge', () => {
const mcp: LoadedMcp = { tools: [], servers: [{ id: 'web', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web', research: false }], unavailable: [] };
const note = mcpPromptNote(mcp, { askBeforeWeb: true });
expect(note).not.toMatch(/Answer from general knowledge/);
expect(note).toMatch(/never answer from your own knowledge/);
expect(note).toMatch(/do not have information/);
});
});
/* ------------------------------ share links ------------------------------ */
async function sharedExam(ai: Record<string, boolean>) {
const { cls, doc } = await classWithMaterial();
const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder;
await t.api('PATCH', `/api/documents/${doc.id}`, { folderId: exam.id });
const share = (await t.api('POST', '/api/shares', { kind: 'folder', targetId: exam.id })).body.share;
const updated = (await t.api('PATCH', `/api/shares/${share.id}`, { ai })).body.share;
return { cls, doc, exam, share: updated, token: share.token as string };
}
const from = (ip: string, country?: string) => ({ headers: { 'x-forwarded-for': ip, ...(country ? { 'cf-ipcountry': country } : {}) } });
const explain = (token: string, ip: string, country?: string) => t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'none', cardId: 'none' }, from(ip, country));
describe('failed share-link sets', () => {
it('can be dismissed by the visitor and stay in the owner drafts', async () => {
await connectAi();
const { token } = await sharedExam({ generate: true });
mock.badJsonNext('submit_flashcards', 6);
const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 5 });
expect(started.status).toBe(202);
const id = started.body.set.id;
await waitDraft(id);
const sets = (await t.api('GET', `/api/public/${token}/generated`)).body.sets;
expect(sets.find((s: any) => s.id === id)?.status).toBe('failed');
expect((await t.api('POST', `/api/public/${token}/generated/${id}/dismiss`)).status).toBe(200);
expect((await t.api('GET', `/api/public/${token}/generated`)).body.sets.some((s: any) => s.id === id)).toBe(false);
expect((await t.api('GET', '/api/drafts')).body.drafts.find((d: any) => d.id === id)?.status).toBe('failed');
});
it('show visitors plain progress steps', async () => {
await connectAi();
const { token } = await sharedExam({ generate: true });
mock.delayChat(300);
const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 4 });
t.ctx.db.run('UPDATE drafts SET progress_label = ? WHERE id = ?', 'Checking the questions with Python (run 1)', started.body.set.id);
const set = (await t.api('GET', `/api/public/${token}/generated`)).body.sets.find((s: any) => s.id === started.body.set.id);
expect(set.progressLabel).toBe('Verifying questions');
mock.delayChat(0);
await waitDraft(started.body.set.id);
});
});
describe('share-link AI', () => {
it('turns itself off after a week and can be given another week', async () => {
await connectAi();
const { share, token } = await sharedExam({ explain: true, generate: true });
expect(share.aiActive).toBe(true);
expect(share.aiExpiresAt - Date.now()).toBeGreaterThan(6.9 * 86_400_000);
expect((await t.api('GET', `/api/public/${token}`)).body.ai.explain).toBe(true);
t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id);
expect((await t.api('GET', `/api/public/${token}`)).body.ai).toEqual({ explain: false, assistant: false, generate: false });
expect((await explain(token, '10.0.0.2')).status).toBe(403);
const lapsed = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
expect(lapsed.aiActive).toBe(false);
const renewed = (await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share;
expect(renewed.aiActive).toBe(true);
expect(renewed.ai).toMatchObject({ explain: true, generate: true });
expect((await explain(token, '10.0.0.2')).status).toBe(404); // allowed through; the card does not exist
// Switching a feature on after a lapse turns on only that feature, for a fresh week.
t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id);
const fresh = (await t.api('PATCH', `/api/shares/${share.id}`, { ai: { assistant: true } })).body.share;
expect(fresh.ai).toEqual({ explain: false, assistant: true, generate: false });
expect(fresh.aiActive).toBe(true);
});
it('works only from the allowed region, failing closed when a visitor cannot be placed', async () => {
await connectAi();
const { token } = await sharedExam({ explain: true });
expect((await t.api('PUT', '/api/share-settings/ai', { region: 'us' })).status).toBe(200);
// Outside the US: the page works without AI, and the API refuses.
const page = await t.api('GET', `/api/public/${token}`, undefined, from('81.2.69.142', 'GB'));
expect(page.status).toBe(200);
expect(page.body.ai.explain).toBe(false);
const refused = await explain(token, '81.2.69.142', 'GB');
expect(refused.status).toBe(403);
expect(refused.body.error.code).toBe('ai_region');
expect((await explain(token, '8.8.8.8', 'US')).status).toBe(404);
// The local network counts as home; an unplaceable public address does not.
expect((await explain(token, '192.168.1.20')).status).toBe(404);
expect((await explain(token, '8.8.4.4')).status).toBe(403);
// Arizona: the state has to match too.
await t.api('PUT', '/api/share-settings/ai', { region: 'arizona' });
expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'CA' } })).status).toBe(403);
expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'AZ' } })).status).toBe(404);
const status = (await t.api('GET', '/api/share-settings/ai')).body;
expect(status.geo.region).toBe('arizona');
expect(status.geo.lastDay.blocked).toBeGreaterThan(0);
});
it('is locked by abuse protection when dozens of networks use one link', async () => {
await connectAi();
const { share, token } = await sharedExam({ explain: true });
for (let i = 1; i < 25; i += 1) expect((await explain(token, `9.9.9.${i}`)).status).toBe(404);
expect((await explain(token, '9.9.9.200')).status).toBe(403);
// Stays off for everyone, and the owner sees why.
expect((await explain(token, '9.9.9.1')).status).toBe(403);
const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
expect(owner.aiActive).toBe(false);
expect(owner.aiBlocked.reason).toMatch(/25 different networks/);
// Turning it back on lifts the lock.
expect((await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share.aiActive).toBe(true);
expect((await explain(token, '9.9.9.1')).status).toBe(404);
});
it('is locked by a flood of requests or repeated requests from outside the region', async () => {
await connectAi();
const { share, token } = await sharedExam({ explain: true });
await t.api('PUT', '/api/share-settings/ai', { region: 'us' });
for (let i = 0; i < 9; i += 1) expect((await explain(token, '81.2.69.142', 'GB')).status).toBe(403);
expect((await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id).aiActive).toBe(true);
await explain(token, '81.2.69.142', 'GB');
const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
expect(owner.aiBlocked.reason).toMatch(/outside the allowed region/);
});
});
describe('location helpers', () => {
it('recognise private and loopback addresses', () => {
for (const ip of ['10.1.2.3', '192.168.0.5', '172.20.1.1', '127.0.0.1', '::1', 'fd00::1', '::ffff:192.168.1.2']) expect(isPrivateAddress(ip)).toBe(true);
for (const ip of ['8.8.8.8', '172.32.0.1', '2001:4860:4860::8888']) expect(isPrivateAddress(ip)).toBe(false);
});
it('migration 8 adds the share AI columns', () => {
expect(MIGRATIONS.find((m) => m.id === 8)?.name).toBe('share-ai-safety');
const cols = t.ctx.db.all<{ name: string }>('PRAGMA table_info(shares)').map((c) => c.name);
expect(cols).toEqual(expect.arrayContaining(['ai_expires_at', 'ai_blocked']));
});
});