Fifth revision: share-link AI safety, sturdier generation, cleaner UI
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 1m0s

- Malformed model JSON is sent back for repair, then retried; a failed job
  starts over once before it fails. Visitors can dismiss failed sets.
- Generated sets get a short distinct name and description; flashcards are
  atomic and have no hints; standard study shows known/learning counts.
- Share links: "In progress" list, finished sets join the normal lists with a
  "New" tag, plain progress steps, newest exam first.
- Share-link AI expires after a week, can be limited to the US or Arizona
  (DB-IP Lite or Cloudflare headers) and is locked by abuse protection
  (migration 8).
- The assistant never answers from general knowledge; MCP and web search
  tools load without a manual test.
- Plain-text copy, no question count in exam mode, filler text removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 09:17:50 -07:00
parent d8df5edf30
commit 85d5c9186d
46 changed files with 1242 additions and 283 deletions

View file

@ -54,6 +54,7 @@ async function sse(url: string, body: unknown): Promise<any[]> {
const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review');
/** The quick call that groups topic labels after a quiz is written. */
const isGrouping = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_topics');
const isNaming = (r: { body: any }) => r.body?.tool_choice?.function?.name === 'name_set';
async function waitForDraft(id: string) {
for (let i = 0; i < 200; i += 1) {
@ -201,7 +202,7 @@ describe('generation', () => {
expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/);
// 12 questions at 8 per batch means two model calls, each carrying the source excerpts.
const all = mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
const calls = all.filter((r) => !isCheck(r) && !isGrouping(r));
const calls = all.filter((r) => !isCheck(r) && !isGrouping(r) && !isNaming(r));
expect(all.filter(isGrouping)).toHaveLength(1);
expect(calls).toHaveLength(2);
// Then the Python check: one run, a review, and its fix applied to question 2.
@ -251,7 +252,7 @@ describe('generation', () => {
// Deck -> quiz conversion uses the cards as the source.
const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId);
expect(converted.status).toBe('ready');
expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r) && !isGrouping(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r) && !isGrouping(r) && !isNaming(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/);
// Append more cards to the existing deck.
const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId);

View file

@ -18,6 +18,8 @@ export interface MockAi {
failNext: (status: number, times?: number) => void;
/** Delay every chat call by this many milliseconds (0 turns it off). */
delayChat: (ms: number) => void;
/** The next N calls of this tool send arguments that are not valid JSON (beyond jsonrepair). */
badJsonNext: (tool: string, times?: number) => void;
close: () => Promise<void>;
}
@ -90,7 +92,9 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
}
return { questions: questions(count, refs, offset) };
case 'submit_flashcards':
return { cards: Array.from({ length: count }, (_, i) => ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, hint: '', tags: ['mock'] })) };
return { cards: Array.from({ length: count }, (_, i) => ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, tags: ['mock'] })) };
case 'name_set':
return /quiz of/i.test(userText) ? { title: 'Heart Failure and Pharmacology', description: 'Covers heart failure care and the drugs used for it.' } : { title: 'Mock Terms', description: 'Covers the mock terms.' };
case 'generate_quiz':
return { count: Math.min(count, 5), topic: 'heart failure', style: 'nclex', use_all_class_documents: /document/i.test(userText), from_weak_areas: /wrong|weak|miss/i.test(userText) };
case 'generate_flashcards':
@ -146,6 +150,7 @@ export function startMockAi(port = 0): Promise<MockAi> {
const requests: MockAi['requests'] = [];
let fail = { status: 0, times: 0 };
let delayMs = 0;
let badJson: { tool: string; times: number } | null = null;
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
@ -198,7 +203,11 @@ export function startMockAi(port = 0): Promise<MockAi> {
res.end();
};
const callTool = (name: string) => {
const args = JSON.stringify(toolArgs(name, userText, all));
let args = JSON.stringify(toolArgs(name, userText, all));
if (badJson && badJson.tool === name && badJson.times > 0) {
badJson.times -= 1;
args = '{"cards": [{"front": "Broken" "back": ]]]} :: not json';
}
send({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: `call_${Math.random().toString(36).slice(2, 10)}`, type: 'function', function: { name, arguments: '' } }] });
// Arguments arrive in fragments, like a real provider.
for (let i = 0; i < args.length; i += 37) send({ tool_calls: [{ index: 0, function: { arguments: args.slice(i, i + 37) } }] });
@ -265,6 +274,9 @@ export function startMockAi(port = 0): Promise<MockAi> {
delayChat: (ms) => {
delayMs = ms;
},
badJsonNext: (tool, times = 1) => {
badJson = { tool, times };
},
close: () => new Promise((done) => server.close(() => done())),
});
});

View file

@ -110,7 +110,7 @@ describe('the assistant prompt', () => {
const mcp: LoadedMcp = { tools: [], unavailable: [], servers: [{ id: 'web-search', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web' }] };
const chat = mcpPromptNote(mcp, { askBeforeWeb: true });
expect(chat).toContain('Should I look it up on the web?');
expect(chat).toContain('do not answer from your own knowledge');
expect(chat).toContain('never answer from your own knowledge');
const elsewhere = mcpPromptNote(mcp);
expect(elsewhere).not.toContain('Should I look it up on the web?');
expect(elsewhere).toContain("anything the student's own material does not cover");
@ -229,7 +229,9 @@ describe('share-link visitors can generate', () => {
expect((await t.api('GET', '/api/drafts')).body.drafts.some((d: any) => d.id === set.id)).toBe(false);
const saved = (await t.api('GET', `/api/quizzes/${set.itemId}`)).body;
expect(saved.quiz.folderId).toBe(exam.id);
expect(saved.quiz.description).toBe('Made with AI on a share link.');
// Named and described by the model from what it covers.
expect(saved.quiz.title).toBe('Heart Failure and Pharmacology');
expect(saved.quiz.description).toBe('Covers heart failure care and the drugs used for it.');
const shown = (await t.api('GET', `/api/public/${token}`)).body.items.map((i: any) => i.id);
expect(shown).toContain(set.itemId);
const quiz = (await t.api('GET', `/api/public/${token}/items/quiz/${set.itemId}`)).body;

253
tests/revision5.test.ts Normal file
View file

@ -0,0 +1,253 @@
/**
* Fifth revision: malformed model output is repaired or retried (and a whole job restarted
* once), failed share-link sets can be dismissed by visitors, generated sets get their own
* names, flashcards are atomic without hints, the assistant never answers from general
* knowledge, and share-link AI expires after a week, is limited to a region and is locked by
* abuse protection.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts';
import { MIGRATIONS } from '../server/migrations.ts';
import { queueIdle } from '../server/services/documents.ts';
import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts';
import { createTestApp, type TestApp } from './helpers.ts';
import { startMockAi, type MockAi } from './mockAi.ts';
let mock: MockAi;
let t: TestApp;
beforeAll(async () => {
mock = await startMockAi();
});
afterAll(async () => {
await mock.close();
});
beforeEach(async () => {
t = await createTestApp();
mock.requests.length = 0;
mock.delayChat(0);
mock.badJsonNext('', 0);
const { resetShareAiLimits } = await import('../server/routes/shares.ts');
resetShareAiLimits();
resetGeoForTests();
});
afterEach(() => t.close());
async function connectAi() {
const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider;
const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[];
for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } });
}
async function classWithMaterial() {
const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class;
const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' })).body.document;
await queueIdle(t.ctx);
return { cls, doc };
}
async function waitDraft(id: string) {
for (let i = 0; i < 400; i += 1) {
const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft;
if (d.status !== 'generating') return d;
await new Promise((r) => setTimeout(r, 25));
}
throw new Error('generation did not finish');
}
const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions'));
const isRepair = (r: { body: any }) => JSON.stringify(r.body.messages ?? []).includes('You repair malformed JSON');
describe('malformed model output', () => {
it('is sent back to the model for repair instead of failing', async () => {
await connectAi();
const { cls } = await classWithMaterial();
mock.badJsonNext('submit_flashcards', 1);
const started = await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 });
const draft = await waitDraft(started.body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.cards).toHaveLength(5);
expect(chatCalls().filter(isRepair)).toHaveLength(1);
});
it('starts the whole job over once when repair and a retry both fail, then fails for good', async () => {
await connectAi();
const { cls } = await classWithMaterial();
// Original, repair and retry all broken: the job starts over and succeeds.
mock.badJsonNext('submit_flashcards', 3);
let draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId);
expect(draft.status).toBe('ready');
expect(draft.payload.cards).toHaveLength(5);
// Broken on the second run too: failed, with a readable error.
mock.badJsonNext('submit_flashcards', 6);
draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId);
expect(draft.status).toBe('failed');
expect(draft.error).toMatch(/not valid JSON/);
});
});
describe('generated sets', () => {
it('get their own name and description, and flashcards are atomic without hints', async () => {
await connectAi();
const { cls } = await classWithMaterial();
const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
expect(draft.payload.title).toBe('Mock Terms');
expect(draft.payload.description).toBe('Covers the mock terms.');
const writing = chatCalls().find((r) => r.body.tool_choice?.function?.name === 'submit_flashcards')!;
const tool = writing.body.tools.find((x: any) => x.function.name === 'submit_flashcards');
expect(Object.keys(tool.function.parameters.properties.cards.items.properties)).not.toContain('hint');
expect(writing.body.messages[0].content).toMatch(/Atomic cards/);
// The same name twice in a class gets a number.
await t.api('POST', `/api/drafts/${draft.id}/save`, {});
const second = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId);
expect(second.payload.title).toBe('Mock Terms 2');
});
});
describe('the assistant', () => {
it('never offers to answer from general knowledge', () => {
const mcp: LoadedMcp = { tools: [], servers: [{ id: 'web', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web', research: false }], unavailable: [] };
const note = mcpPromptNote(mcp, { askBeforeWeb: true });
expect(note).not.toMatch(/Answer from general knowledge/);
expect(note).toMatch(/never answer from your own knowledge/);
expect(note).toMatch(/do not have information/);
});
});
/* ------------------------------ share links ------------------------------ */
async function sharedExam(ai: Record<string, boolean>) {
const { cls, doc } = await classWithMaterial();
const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder;
await t.api('PATCH', `/api/documents/${doc.id}`, { folderId: exam.id });
const share = (await t.api('POST', '/api/shares', { kind: 'folder', targetId: exam.id })).body.share;
const updated = (await t.api('PATCH', `/api/shares/${share.id}`, { ai })).body.share;
return { cls, doc, exam, share: updated, token: share.token as string };
}
const from = (ip: string, country?: string) => ({ headers: { 'x-forwarded-for': ip, ...(country ? { 'cf-ipcountry': country } : {}) } });
const explain = (token: string, ip: string, country?: string) => t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'none', cardId: 'none' }, from(ip, country));
describe('failed share-link sets', () => {
it('can be dismissed by the visitor and stay in the owner drafts', async () => {
await connectAi();
const { token } = await sharedExam({ generate: true });
mock.badJsonNext('submit_flashcards', 6);
const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 5 });
expect(started.status).toBe(202);
const id = started.body.set.id;
await waitDraft(id);
const sets = (await t.api('GET', `/api/public/${token}/generated`)).body.sets;
expect(sets.find((s: any) => s.id === id)?.status).toBe('failed');
expect((await t.api('POST', `/api/public/${token}/generated/${id}/dismiss`)).status).toBe(200);
expect((await t.api('GET', `/api/public/${token}/generated`)).body.sets.some((s: any) => s.id === id)).toBe(false);
expect((await t.api('GET', '/api/drafts')).body.drafts.find((d: any) => d.id === id)?.status).toBe('failed');
});
it('show visitors plain progress steps', async () => {
await connectAi();
const { token } = await sharedExam({ generate: true });
mock.delayChat(300);
const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 4 });
t.ctx.db.run('UPDATE drafts SET progress_label = ? WHERE id = ?', 'Checking the questions with Python (run 1)', started.body.set.id);
const set = (await t.api('GET', `/api/public/${token}/generated`)).body.sets.find((s: any) => s.id === started.body.set.id);
expect(set.progressLabel).toBe('Verifying questions');
mock.delayChat(0);
await waitDraft(started.body.set.id);
});
});
describe('share-link AI', () => {
it('turns itself off after a week and can be given another week', async () => {
await connectAi();
const { share, token } = await sharedExam({ explain: true, generate: true });
expect(share.aiActive).toBe(true);
expect(share.aiExpiresAt - Date.now()).toBeGreaterThan(6.9 * 86_400_000);
expect((await t.api('GET', `/api/public/${token}`)).body.ai.explain).toBe(true);
t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id);
expect((await t.api('GET', `/api/public/${token}`)).body.ai).toEqual({ explain: false, assistant: false, generate: false });
expect((await explain(token, '10.0.0.2')).status).toBe(403);
const lapsed = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
expect(lapsed.aiActive).toBe(false);
const renewed = (await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share;
expect(renewed.aiActive).toBe(true);
expect(renewed.ai).toMatchObject({ explain: true, generate: true });
expect((await explain(token, '10.0.0.2')).status).toBe(404); // allowed through; the card does not exist
// Switching a feature on after a lapse turns on only that feature, for a fresh week.
t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id);
const fresh = (await t.api('PATCH', `/api/shares/${share.id}`, { ai: { assistant: true } })).body.share;
expect(fresh.ai).toEqual({ explain: false, assistant: true, generate: false });
expect(fresh.aiActive).toBe(true);
});
it('works only from the allowed region, failing closed when a visitor cannot be placed', async () => {
await connectAi();
const { token } = await sharedExam({ explain: true });
expect((await t.api('PUT', '/api/share-settings/ai', { region: 'us' })).status).toBe(200);
// Outside the US: the page works without AI, and the API refuses.
const page = await t.api('GET', `/api/public/${token}`, undefined, from('81.2.69.142', 'GB'));
expect(page.status).toBe(200);
expect(page.body.ai.explain).toBe(false);
const refused = await explain(token, '81.2.69.142', 'GB');
expect(refused.status).toBe(403);
expect(refused.body.error.code).toBe('ai_region');
expect((await explain(token, '8.8.8.8', 'US')).status).toBe(404);
// The local network counts as home; an unplaceable public address does not.
expect((await explain(token, '192.168.1.20')).status).toBe(404);
expect((await explain(token, '8.8.4.4')).status).toBe(403);
// Arizona: the state has to match too.
await t.api('PUT', '/api/share-settings/ai', { region: 'arizona' });
expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'CA' } })).status).toBe(403);
expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'AZ' } })).status).toBe(404);
const status = (await t.api('GET', '/api/share-settings/ai')).body;
expect(status.geo.region).toBe('arizona');
expect(status.geo.lastDay.blocked).toBeGreaterThan(0);
});
it('is locked by abuse protection when dozens of networks use one link', async () => {
await connectAi();
const { share, token } = await sharedExam({ explain: true });
for (let i = 1; i < 25; i += 1) expect((await explain(token, `9.9.9.${i}`)).status).toBe(404);
expect((await explain(token, '9.9.9.200')).status).toBe(403);
// Stays off for everyone, and the owner sees why.
expect((await explain(token, '9.9.9.1')).status).toBe(403);
const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
expect(owner.aiActive).toBe(false);
expect(owner.aiBlocked.reason).toMatch(/25 different networks/);
// Turning it back on lifts the lock.
expect((await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share.aiActive).toBe(true);
expect((await explain(token, '9.9.9.1')).status).toBe(404);
});
it('is locked by a flood of requests or repeated requests from outside the region', async () => {
await connectAi();
const { share, token } = await sharedExam({ explain: true });
await t.api('PUT', '/api/share-settings/ai', { region: 'us' });
for (let i = 0; i < 9; i += 1) expect((await explain(token, '81.2.69.142', 'GB')).status).toBe(403);
expect((await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id).aiActive).toBe(true);
await explain(token, '81.2.69.142', 'GB');
const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id);
expect(owner.aiBlocked.reason).toMatch(/outside the allowed region/);
});
});
describe('location helpers', () => {
it('recognise private and loopback addresses', () => {
for (const ip of ['10.1.2.3', '192.168.0.5', '172.20.1.1', '127.0.0.1', '::1', 'fd00::1', '::ffff:192.168.1.2']) expect(isPrivateAddress(ip)).toBe(true);
for (const ip of ['8.8.8.8', '172.32.0.1', '2001:4860:4860::8888']) expect(isPrivateAddress(ip)).toBe(false);
});
it('migration 8 adds the share AI columns', () => {
expect(MIGRATIONS.find((m) => m.id === 8)?.name).toBe('share-ai-safety');
const cols = t.ctx.db.all<{ name: string }>('PRAGMA table_info(shares)').map((c) => c.name);
expect(cols).toEqual(expect.arrayContaining(['ai_expires_at', 'ai_blocked']));
});
});