All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 1m0s
- Malformed model JSON is sent back for repair, then retried; a failed job starts over once before it fails. Visitors can dismiss failed sets. - Generated sets get a short distinct name and description; flashcards are atomic and have no hints; standard study shows known/learning counts. - Share links: "In progress" list, finished sets join the normal lists with a "New" tag, plain progress steps, newest exam first. - Share-link AI expires after a week, can be limited to the US or Arizona (DB-IP Lite or Cloudflare headers) and is locked by abuse protection (migration 8). - The assistant never answers from general knowledge; MCP and web search tools load without a manual test. - Plain-text copy, no question count in exam mode, filler text removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
253 lines
14 KiB
TypeScript
253 lines
14 KiB
TypeScript
/**
|
|
* Fifth revision: malformed model output is repaired or retried (and a whole job restarted
|
|
* once), failed share-link sets can be dismissed by visitors, generated sets get their own
|
|
* names, flashcards are atomic without hints, the assistant never answers from general
|
|
* knowledge, and share-link AI expires after a week, is limited to a region and is locked by
|
|
* abuse protection.
|
|
*/
|
|
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
|
import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts';
|
|
import { MIGRATIONS } from '../server/migrations.ts';
|
|
import { queueIdle } from '../server/services/documents.ts';
|
|
import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts';
|
|
import { createTestApp, type TestApp } from './helpers.ts';
|
|
import { startMockAi, type MockAi } from './mockAi.ts';
|
|
|
|
let mock: MockAi;
|
|
let t: TestApp;
|
|
|
|
beforeAll(async () => {
|
|
mock = await startMockAi();
|
|
});
|
|
afterAll(async () => {
|
|
await mock.close();
|
|
});
|
|
beforeEach(async () => {
|
|
t = await createTestApp();
|
|
mock.requests.length = 0;
|
|
mock.delayChat(0);
|
|
mock.badJsonNext('', 0);
|
|
const { resetShareAiLimits } = await import('../server/routes/shares.ts');
|
|
resetShareAiLimits();
|
|
resetGeoForTests();
|
|
});
|
|
afterEach(() => t.close());
|
|
|
|
async function connectAi() {
|
|
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
|
|
const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[];
|
|
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } });
|
|
}
|
|
|
|
async function classWithMaterial() {
|
|
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
|
|
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' })).body.document;
|
|
await queueIdle(t.ctx);
|
|
return { cls, doc };
|
|
}
|
|
|
|
async function waitDraft(id: string) {
|
|
for (let i = 0; i < 400; i += 1) {
|
|
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
|
|
if (d.status !== 'generating') return d;
|
|
await new Promise((r) => setTimeout(r, 25));
|
|
}
|
|
throw new Error('generation did not finish');
|
|
}
|
|
|
|
const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
|
|
const isRepair = (r: { body: any }) => JSON.stringify(r.body.messages ?? []).includes('You repair malformed JSON');
|
|
|
|
describe('malformed model output', () => {
|
|
it('is sent back to the model for repair instead of failing', async () => {
|
|
await connectAi();
|
|
const { cls } = await classWithMaterial();
|
|
mock.badJsonNext('submit_flashcards', 1);
|
|
const started = await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 });
|
|
const draft = await waitDraft(started.body.draftId);
|
|
expect(draft.status).toBe('ready');
|
|
expect(draft.payload.cards).toHaveLength(5);
|
|
expect(chatCalls().filter(isRepair)).toHaveLength(1);
|
|
});
|
|
|
|
it('starts the whole job over once when repair and a retry both fail, then fails for good', async () => {
|
|
await connectAi();
|
|
const { cls } = await classWithMaterial();
|
|
// Original, repair and retry all broken: the job starts over and succeeds.
|
|
mock.badJsonNext('submit_flashcards', 3);
|
|
let draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId);
|
|
expect(draft.status).toBe('ready');
|
|
expect(draft.payload.cards).toHaveLength(5);
|
|
|
|
// Broken on the second run too: failed, with a readable error.
|
|
mock.badJsonNext('submit_flashcards', 6);
|
|
draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId);
|
|
expect(draft.status).toBe('failed');
|
|
expect(draft.error).toMatch(/not valid JSON/);
|
|
});
|
|
});
|
|
|
|
describe('generated sets', () => {
|
|
it('get their own name and description, and flashcards are atomic without hints', async () => {
|
|
await connectAi();
|
|
const { cls } = await classWithMaterial();
|
|
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
|
|
expect(draft.payload.title).toBe('Mock Terms');
|
|
expect(draft.payload.description).toBe('Covers the mock terms.');
|
|
const writing = chatCalls().find((r) => r.body.tool_choice?.function?.name === 'submit_flashcards')!;
|
|
const tool = writing.body.tools.find((x: any) => x.function.name === 'submit_flashcards');
|
|
expect(Object.keys(tool.function.parameters.properties.cards.items.properties)).not.toContain('hint');
|
|
expect(writing.body.messages[0].content).toMatch(/Atomic cards/);
|
|
// The same name twice in a class gets a number.
|
|
await t.api('POST', `/api/drafts/${draft.id}/save`, {});
|
|
const second = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
|
|
expect(second.payload.title).toBe('Mock Terms 2');
|
|
});
|
|
});
|
|
|
|
describe('the assistant', () => {
|
|
it('never offers to answer from general knowledge', () => {
|
|
const mcp: LoadedMcp = { tools: [], servers: [{ id: 'web', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web', research: false }], unavailable: [] };
|
|
const note = mcpPromptNote(mcp, { askBeforeWeb: true });
|
|
expect(note).not.toMatch(/Answer from general knowledge/);
|
|
expect(note).toMatch(/never answer from your own knowledge/);
|
|
expect(note).toMatch(/do not have information/);
|
|
});
|
|
});
|
|
|
|
/* ------------------------------ share links ------------------------------ */
|
|
|
|
async function sharedExam(ai: Record<string, boolean>) {
|
|
const { cls, doc } = await classWithMaterial();
|
|
const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder;
|
|
await t.api('PATCH', `/api/documents/${doc.id}`, { folderId: exam.id });
|
|
const share = (await t.api('POST', '/api/shares', { kind: 'folder', targetId: exam.id })).body.share;
|
|
const updated = (await t.api('PATCH', `/api/shares/${share.id}`, { ai })).body.share;
|
|
return { cls, doc, exam, share: updated, token: share.token as string };
|
|
}
|
|
|
|
const from = (ip: string, country?: string) => ({ headers: { 'x-forwarded-for': ip, ...(country ? { 'cf-ipcountry': country } : {}) } });
|
|
const explain = (token: string, ip: string, country?: string) => t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'none', cardId: 'none' }, from(ip, country));
|
|
|
|
describe('failed share-link sets', () => {
|
|
it('can be dismissed by the visitor and stay in the owner drafts', async () => {
|
|
await connectAi();
|
|
const { token } = await sharedExam({ generate: true });
|
|
mock.badJsonNext('submit_flashcards', 6);
|
|
const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 5 });
|
|
expect(started.status).toBe(202);
|
|
const id = started.body.set.id;
|
|
await waitDraft(id);
|
|
const sets = (await t.api('GET', `/api/public/${token}/generated`)).body.sets;
|
|
expect(sets.find((s: any) => s.id === id)?.status).toBe('failed');
|
|
expect((await t.api('POST', `/api/public/${token}/generated/${id}/dismiss`)).status).toBe(200);
|
|
expect((await t.api('GET', `/api/public/${token}/generated`)).body.sets.some((s: any) => s.id === id)).toBe(false);
|
|
expect((await t.api('GET', '/api/drafts')).body.drafts.find((d: any) => d.id === id)?.status).toBe('failed');
|
|
});
|
|
|
|
it('show visitors plain progress steps', async () => {
|
|
await connectAi();
|
|
const { token } = await sharedExam({ generate: true });
|
|
mock.delayChat(300);
|
|
const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 4 });
|
|
t.ctx.db.run('UPDATE drafts SET progress_label = ? WHERE id = ?', 'Checking the questions with Python (run 1)', started.body.set.id);
|
|
const set = (await t.api('GET', `/api/public/${token}/generated`)).body.sets.find((s: any) => s.id === started.body.set.id);
|
|
expect(set.progressLabel).toBe('Verifying questions');
|
|
mock.delayChat(0);
|
|
await waitDraft(started.body.set.id);
|
|
});
|
|
});
|
|
|
|
describe('share-link AI', () => {
|
|
it('turns itself off after a week and can be given another week', async () => {
|
|
await connectAi();
|
|
const { share, token } = await sharedExam({ explain: true, generate: true });
|
|
expect(share.aiActive).toBe(true);
|
|
expect(share.aiExpiresAt - Date.now()).toBeGreaterThan(6.9 * 86_400_000);
|
|
expect((await t.api('GET', `/api/public/${token}`)).body.ai.explain).toBe(true);
|
|
|
|
t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id);
|
|
expect((await t.api('GET', `/api/public/${token}`)).body.ai).toEqual({ explain: false, assistant: false, generate: false });
|
|
expect((await explain(token, '10.0.0.2')).status).toBe(403);
|
|
const lapsed = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
|
|
expect(lapsed.aiActive).toBe(false);
|
|
|
|
const renewed = (await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share;
|
|
expect(renewed.aiActive).toBe(true);
|
|
expect(renewed.ai).toMatchObject({ explain: true, generate: true });
|
|
expect((await explain(token, '10.0.0.2')).status).toBe(404); // allowed through; the card does not exist
|
|
|
|
// Switching a feature on after a lapse turns on only that feature, for a fresh week.
|
|
t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id);
|
|
const fresh = (await t.api('PATCH', `/api/shares/${share.id}`, { ai: { assistant: true } })).body.share;
|
|
expect(fresh.ai).toEqual({ explain: false, assistant: true, generate: false });
|
|
expect(fresh.aiActive).toBe(true);
|
|
});
|
|
|
|
it('works only from the allowed region, failing closed when a visitor cannot be placed', async () => {
|
|
await connectAi();
|
|
const { token } = await sharedExam({ explain: true });
|
|
expect((await t.api('PUT', '/api/share-settings/ai', { region: 'us' })).status).toBe(200);
|
|
|
|
// Outside the US: the page works without AI, and the API refuses.
|
|
const page = await t.api('GET', `/api/public/${token}`, undefined, from('81.2.69.142', 'GB'));
|
|
expect(page.status).toBe(200);
|
|
expect(page.body.ai.explain).toBe(false);
|
|
const refused = await explain(token, '81.2.69.142', 'GB');
|
|
expect(refused.status).toBe(403);
|
|
expect(refused.body.error.code).toBe('ai_region');
|
|
|
|
expect((await explain(token, '8.8.8.8', 'US')).status).toBe(404);
|
|
// The local network counts as home; an unplaceable public address does not.
|
|
expect((await explain(token, '192.168.1.20')).status).toBe(404);
|
|
expect((await explain(token, '8.8.4.4')).status).toBe(403);
|
|
|
|
// Arizona: the state has to match too.
|
|
await t.api('PUT', '/api/share-settings/ai', { region: 'arizona' });
|
|
expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'CA' } })).status).toBe(403);
|
|
expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'AZ' } })).status).toBe(404);
|
|
|
|
const status = (await t.api('GET', '/api/share-settings/ai')).body;
|
|
expect(status.geo.region).toBe('arizona');
|
|
expect(status.geo.lastDay.blocked).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('is locked by abuse protection when dozens of networks use one link', async () => {
|
|
await connectAi();
|
|
const { share, token } = await sharedExam({ explain: true });
|
|
for (let i = 1; i < 25; i += 1) expect((await explain(token, `9.9.9.${i}`)).status).toBe(404);
|
|
expect((await explain(token, '9.9.9.200')).status).toBe(403);
|
|
// Stays off for everyone, and the owner sees why.
|
|
expect((await explain(token, '9.9.9.1')).status).toBe(403);
|
|
const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
|
|
expect(owner.aiActive).toBe(false);
|
|
expect(owner.aiBlocked.reason).toMatch(/25 different networks/);
|
|
// Turning it back on lifts the lock.
|
|
expect((await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share.aiActive).toBe(true);
|
|
expect((await explain(token, '9.9.9.1')).status).toBe(404);
|
|
});
|
|
|
|
it('is locked by a flood of requests or repeated requests from outside the region', async () => {
|
|
await connectAi();
|
|
const { share, token } = await sharedExam({ explain: true });
|
|
await t.api('PUT', '/api/share-settings/ai', { region: 'us' });
|
|
for (let i = 0; i < 9; i += 1) expect((await explain(token, '81.2.69.142', 'GB')).status).toBe(403);
|
|
expect((await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id).aiActive).toBe(true);
|
|
await explain(token, '81.2.69.142', 'GB');
|
|
const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
|
|
expect(owner.aiBlocked.reason).toMatch(/outside the allowed region/);
|
|
});
|
|
});
|
|
|
|
describe('location helpers', () => {
|
|
it('recognise private and loopback addresses', () => {
|
|
for (const ip of ['10.1.2.3', '192.168.0.5', '172.20.1.1', '127.0.0.1', '::1', 'fd00::1', '::ffff:192.168.1.2']) expect(isPrivateAddress(ip)).toBe(true);
|
|
for (const ip of ['8.8.8.8', '172.32.0.1', '2001:4860:4860::8888']) expect(isPrivateAddress(ip)).toBe(false);
|
|
});
|
|
|
|
it('migration 8 adds the share AI columns', () => {
|
|
expect(MIGRATIONS.find((m) => m.id === 8)?.name).toBe('share-ai-safety');
|
|
const cols = t.ctx.db.all<{ name: string }>('PRAGMA table_info(shares)').map((c) => c.name);
|
|
expect(cols).toEqual(expect.arrayContaining(['ai_expires_at', 'ai_blocked']));
|
|
});
|
|
});
|