From 85d5c9186deea9f63f17f0d5b566584f58027701 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 16 Nov 2026 09:17:50 -0700 Subject: [PATCH] Fifth revision: share-link AI safety, sturdier generation, cleaner UI - Malformed model JSON is sent back for repair, then retried; a failed job starts over once before it fails. Visitors can dismiss failed sets. - Generated sets get a short distinct name and description; flashcards are atomic and have no hints; standard study shows known/learning counts. - Share links: "In progress" list, finished sets join the normal lists with a "New" tag, plain progress steps, newest exam first. - Share-link AI expires after a week, can be limited to the US or Arizona (DB-IP Lite or Cloudflare headers) and is locked by abuse protection (migration 8). - The assistant never answers from general knowledge; MCP and web search tools load without a manual test. - Plain-text copy, no question count in exam mode, filler text removed. Co-Authored-By: Claude Opus 5.5 --- PROGRESS.md | 21 ++- README.md | 2 + docs/AI.md | 53 +++++- docs/DEPLOYMENT.md | 5 + package-lock.json | 11 ++ package.json | 1 + server/ai/assistant.ts | 6 +- server/ai/generate.ts | 114 +++++++++++- server/ai/mcp.ts | 45 ++++- server/ai/providers/common.ts | 1 + server/ai/service.ts | 41 ++++- server/ai/types.ts | 8 +- server/main.ts | 6 + server/migrations.ts | 14 ++ server/routes/ai.ts | 21 ++- server/routes/shares.ts | 173 ++++++++++++++++-- server/services/geo.ts | 231 +++++++++++++++++++++++ shared/importer.ts | 9 +- tests/ai.test.ts | 5 +- tests/mockAi.ts | 16 +- tests/revision4.test.ts | 6 +- tests/revision5.test.ts | 253 ++++++++++++++++++++++++++ web/src/app/Shell.tsx | 3 +- web/src/features/AiSettings.tsx | 11 +- web/src/features/AssistantPanel.tsx | 1 - web/src/features/GenerateDialog.tsx | 10 +- web/src/features/ImportDialog.tsx | 2 +- web/src/features/McpSettings.tsx | 1 - web/src/features/SemanticSettings.tsx | 1 - web/src/features/SharedGenerate.tsx | 109 ++++------- web/src/features/TemplateSettings.tsx | 12 +- web/src/features/dialogs.tsx | 60 +++--- web/src/features/editors.tsx | 21 +-- web/src/features/flashcards.tsx | 43 ++--- web/src/features/quiz.tsx | 2 +- web/src/lib/types.ts | 20 +- web/src/main.tsx | 13 ++ web/src/pages/ClassPage.tsx | 4 +- web/src/pages/DeckPage.tsx | 1 - web/src/pages/DraftPage.tsx | 1 - web/src/pages/InsightsPage.tsx | 2 +- web/src/pages/NotePage.tsx | 1 - web/src/pages/QuizPage.tsx | 45 ++--- web/src/pages/ReviewPage.tsx | 3 +- web/src/pages/SettingsPage.tsx | 74 +++++++- web/src/pages/SharedPage.tsx | 43 ++--- 46 files changed, 1242 insertions(+), 283 deletions(-) create mode 100644 server/services/geo.ts create mode 100644 tests/revision5.test.ts diff --git a/PROGRESS.md b/PROGRESS.md index a001c1d..6dd73f7 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -159,10 +159,29 @@ migration 7). Sets show in Drafts while written, then save into the class or exa automatically, and stay when the link is deleted. The owner's Generate dialog gained "Include med math" with a count (default 5). +## 2026-11-16: fifth revision (owner's 18-item list) + +Explanatory filler removed across the UI (share page, Generate dialogs, start screens, settings, +flashcards); share-link "In progress" list for sets being written, finished sets join the normal +lists with a "New" tag for two hours; visitors see "Verifying questions"; exam mode has no +question-count slider (an exam is always the whole quiz); newest exam first on share pages; +malformed JSON repaired by the model, retried, then the whole job restarted once; failed sets +dismissable on the link only; plain-text copy everywhere; generated sets get their own short +name and description; atomic flashcards without hints; a known / still learning counter in +standard flashcard study; the assistant never answers from general knowledge (search or say it +has no information); MCP and web search tools load automatically; share-link AI expires after a +week, can be limited to the US or Arizona (`server/services/geo.ts`, DB-IP Lite) and is locked +by abuse protection (migration 8). + +Tests: `tests/revision5.test.ts`. Live check on the rev1 copy with DeepSeek flash: a visitor's +10-card set was written, named ("Heparin, Furosemide, and Dosage Calculations"), saved and +listed once with the New tag; cards were single facts with no hints. The rev1 copy's 401 was its +stored DeepSeek connection still holding the old key; it was updated from `.env`. + ## Verification status Baseline: 162 tests passed, TypeScript passed and production web/server builds passed. -Added six regression tests. Full `npm run check` passes: **261 tests** (share-link generation, 2026-11-15; 257 after the fourth revision; 247 after the third, 208 after the second, 181 on 2026-11-12), TypeScript +Added six regression tests. Full `npm run check` passes: **274 tests** (fifth revision, 2026-11-16; 261 after share-link generation, 2026-11-15; 257 after the fourth revision; 247 after the third, 208 after the second, 181 on 2026-11-12), TypeScript and both production builds. The Docker `test` stage runs the same 181 tests on Node 24. Tests were run with loopback access for the mock provider. The earlier agent manually exercised quiz completion/results, flashcards, streak effects, diff --git a/README.md b/README.md index ad298fc..a77d666 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ is an opt-in development tool, never part of startup. - Built-in web search (Parallel; no account needed, optional sign-in for higher limits). - Share links for classes and exams can let visitors generate their own quizzes and flashcards (daily limits per link), and quizzes can include med math on request. +- AI on share links turns itself off after a week, can be limited to the US or Arizona, and is + locked automatically when a link looks leaked or abused. - Interactive charts, practice questions and other blocks inline in assistant replies, and clarifying questions before the assistant generates. - MCP servers (Streamable HTTP, with OAuth sign-in) as extra assistant tools, each with your own instructions and diff --git a/docs/AI.md b/docs/AI.md index 5f68aef..dcbc785 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -91,6 +91,17 @@ has no calculations. Topics are kept broad ("Stroke", not "Right hemisphere stro prompt lists the class's existing topics, and narrower labels that contain an existing one are merged into it after generation. +Flashcards are atomic: one fact per card, the back only the answer (a word, number or short +phrase), and no hints (the schema has no hint field any more). When generation finishes, one +small call names the set: a 2-6 word title for what it covers, different from the class's other +titles (a clash gets a number), and a one-sentence description. The generic title stays if +naming fails. + +Malformed output: tool-call arguments that `jsonrepair` cannot fix are sent back to the model to +be rewritten as valid JSON (`repairStructured` in `server/ai/service.ts`); if that fails the +request is asked once more. A job that still fails on bad output or a passing provider error +starts over once with the same request ("Starting over"); a second failure fails the draft. + ## MCP servers Settings → AI → MCP servers connects Model Context Protocol servers over Streamable HTTP @@ -140,6 +151,16 @@ arguments (`session_id`, `client_model`) are hidden from the model, and ids, usa warnings are stripped from results. It is on for chat and Explain by default (off in tests), with its own instructions and area switches like any server. +Tools load without a manual test: the server connects to every enabled MCP server (and the web +search) a few seconds after start, again every few minutes, and right after a server is saved +or signed in to. A listing that fails is retried once on a fresh connection before the server +is reported as unavailable for that turn. + +The assistant never answers factual questions from its own knowledge. When the class materials +do not cover a question it asks "Search the web" or "No, skip it", and if the student skips it +says it has no information to answer. Without the web search it says so straight away. The +shared assistant answers only from the shared material (and web results, when allowed). + ### Tool steps Every tool call shows as one line: an icon for the kind of tool, a friendly name ("Searched @@ -214,6 +235,29 @@ Each share link has two switches, both off by default: Neither has library tools; MCP servers join only when allowed on share links. Both use the owner's models and keys and show only when an AI model is configured. +AI on a link lasts a week: turning any switch on starts it, and it switches itself off +afterwards (`shares.ai_expires_at`, migration 8). The share dialog shows the end date with +**Extend a week**; once lapsed, **Turn on for a week** brings back the same switches, while +turning a single switch on starts a fresh week with only that feature. + +**Region** (Settings → Sharing → AI on public links): Anywhere, United States (default) or +Arizona. Visitors elsewhere still see and study everything, but the page shows no AI and the +API answers 403 `ai_region`. The location comes from Cloudflare's `CF-IPCountry` and +`CF-Region-Code` headers when `TRUST_PROXY` is on, otherwise from the free DB-IP Lite database +(CC BY 4.0), downloaded monthly into `DATA_DIR/geoip` from db-ip.com with the ip-location-db +mirror on jsDelivr as a fallback (country file about 8 MB; Arizona also needs the city file, +over 100 MB, held in memory). Visitor addresses are never sent anywhere. Private and loopback +addresses count as local and are allowed; a public address that can't be placed is refused. +State-level location is approximate: mobile networks can appear to be in another state. +Settings shows the database dates, the last day's allowed and blocked checks, and a warning when +every visitor looks local (the proxy is not passing real addresses). + +**Abuse protection** turns a link's AI off (`shares.ai_blocked`, with the reason) when, within +an hour, it is used from 25 different networks or 40 different browsers, gets 60 requests in +five minutes, or keeps being refused (10 requests from outside the region, which the page never +offers, or 30 over the per-device limit). The owner sees why in the share dialog and can turn it +back on. Thresholds are `ABUSE` in `server/routes/shares.ts`. + Limits are per device, not per IP address, so students on one campus network do not share an allowance: each browser gets a random visitor id in a signed, HTTP-only cookie (`sd_visitor`, scoped to `/api/public`), limited to 20 requests per 10 minutes per link. A @@ -232,9 +276,12 @@ select-all and the Python check. A link may make 4 quizzes and 2 flashcard sets midnight, failed attempts not counted), on top of the per-device request limit; **Reset limits** in the share dialog starts a fresh allowance. Each set is a draft tied to the link (`drafts.share_id`, migration 7): it shows in the owner's Drafts while it is written, then is -saved into the class (and the exam, for an exam link) as if the owner had approved it, with the -description "Made with AI on a share link". From then on it is ordinary class material: the -link lists it with everything else, and Explain works on it. If the automatic save fails, the +saved into the class (and the exam, for an exam link) as if the owner had approved it, with its +generated name and description. From then on it is ordinary class material: the link lists it +with everything else (marked "New" for two hours), and Explain works on it. While it is written +the link shows it under "In progress" with plain steps ("Verifying questions" rather than the +Python details). A failed set stays there until a visitor dismisses it +(`drafts.share_hidden`); the owner still sees it in Drafts. If the automatic save fails, the draft waits in Drafts for review. Deleting the link keeps what was made on it (running jobs finish and save too). diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index d298811..bd2ec55 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -59,6 +59,11 @@ direct access to the application to your proxy/network: forwarded headers are tr Use `COOKIE_SECURE=true` for the HTTPS public hostname. Plain HTTP cannot carry Secure cookies. Keep proxy upload limits at least as large as `MAX_UPLOAD_MB`, and allow long AI responses. Disable buffering for `/api/ai/chat` and `/api/ai/explain`; allow a long read timeout there. +Share-link AI is limited to a region by visitor address, so the proxy must pass the real one +(Settings → Sharing warns when every visitor looks local). Behind Cloudflare, the +`CF-IPCountry` header (and `CF-Region-Code`, from the "Add visitor location headers" managed +transform) is used directly; otherwise the server downloads the DB-IP Lite database into +`/data/geoip` monthly and needs outbound HTTPS to download.db-ip.com or cdn.jsdelivr.net. Do not cache `/api/`, `/media/` or shared study data. `/api/health` returns 200 when SQLite works. ## Configuration diff --git a/package-lock.json b/package-lock.json index 9c435e0..1379193 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "jsonrepair": "^3.15.0", "lucide-react": "^1.71.0", "mammoth": "^1.13.0", + "mmdb-lib": "3.0.3", "pdf-lib": "^1.17.1", "pyodide": "^314.0.7", "radix-ui": "^1.8.1", @@ -6250,6 +6251,16 @@ ], "license": "MIT" }, + "node_modules/mmdb-lib": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-3.0.3.tgz", + "integrity": "sha512-xQPoBXcNjjHiOvOraFBKtA++uNWF6aCVHL9dRKFXEov8eI3QJwtgiw3qApsonFT5SpoqsEVISUTg3HIDs2DiXw==", + "license": "MIT", + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", diff --git a/package.json b/package.json index 610e0aa..e1fe3f4 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "jsonrepair": "^3.15.0", "lucide-react": "^1.71.0", "mammoth": "^1.13.0", + "mmdb-lib": "3.0.3", "pdf-lib": "^1.17.1", "pyodide": "^314.0.7", "radix-ui": "^1.8.1", diff --git a/server/ai/assistant.ts b/server/ai/assistant.ts index e69d173..f160caa 100644 --- a/server/ai/assistant.ts +++ b/server/ai/assistant.ts @@ -571,7 +571,7 @@ const TOOLS: ToolSpec[] = [ ['type', 'stem', 'options', 'rationale'], ), }, - cards: { type: 'array', items: obj({ front: S, back: S, hint: S }, ['front', 'back']) }, + cards: { type: 'array', items: obj({ front: S, back: S }, ['front', 'back']) }, content: { ...S, description: 'Markdown body, for kind = note.' }, }, ['kind', 'title'], @@ -1110,8 +1110,8 @@ function systemPrompt(app: AppContext, context: AssistantContext, opts: { web?: - Use tools instead of guessing. To answer questions about their material, search or read the documents first and base the answer on what you find. - Cite document passages inline with the ref id in square brackets right after the claim, exactly as given, e.g. [c42]. Only cite refs that a tool returned in this conversation. ${ opts.web - ? 'If the documents do not cover something, say so and ask whether to search the web (see Web search below) instead of answering from your own knowledge.' - : 'If the documents do not cover something, say so, then answer from general knowledge and label it as such.' + ? 'Never answer factual questions from your own general knowledge. If the documents do not cover something, say so and ask whether to search the web (see Web search below); if the user says no, tell them you do not have information to answer it.' + : 'Never answer factual questions from your own general knowledge. If the documents do not cover something, say that you do not have information on it in their class materials.' } - You can browse everything: list_materials and search_library for quizzes, flashcards and notes; get_quiz, get_deck and read_note to open them; search_documents and read_document for uploaded files. - When you mention a specific quiz, deck, note or document, link it with Markdown using the url the tool gave you, for example [Cardiac drug cards](/decks/abc123). Never invent a url. diff --git a/server/ai/generate.ts b/server/ai/generate.ts index 1de5a30..a7c7a61 100644 --- a/server/ai/generate.ts +++ b/server/ai/generate.ts @@ -366,9 +366,8 @@ const CARD_SCHEMA = { items: { type: 'object', properties: { - front: { type: 'string', description: 'Prompt side: one specific question or term.' }, - back: { type: 'string', description: 'Answer side. Markdown allowed.' }, - hint: { type: 'string', description: 'Optional memory hook or mnemonic. Empty string if none.' }, + front: { type: 'string', description: 'Prompt side: one specific question about one fact.' }, + back: { type: 'string', description: 'Only the answer: a word, number or short phrase. No explanation.' }, tags: { type: 'array', items: { type: 'string' }, description: '1-2 short topic tags.' }, }, required: ['front', 'back'], @@ -402,8 +401,9 @@ ${NUMERIC_RULE}`; const CARD_RULES: Record = { - concise: 'Each card tests exactly one fact. Fronts are specific questions or terms; backs are short (a phrase or one or two lines).', - detailed: 'Each card covers one concept. Backs may use a short bullet list (up to 5 bullets) for the key points.', + concise: + `Atomic cards: each card tests exactly one fact, so it can be learned in one look. The front asks one specific question (never "describe" or "explain"). The back is only the answer: a word, a number with its unit, or a short phrase, ideally under 10 words and never more than one short sentence. No explanations, reasons, lists or paragraphs on the back. Split anything with several parts (a drug's class, action and side effects; the signs of a condition) into separate cards.`, + detailed: 'Each card covers one concept. The back is at most 3 short bullet points (a few words each), no paragraphs.', medication: 'Make medication cards. Front: the generic drug name (with a common brand name in parentheses when well known). Back: a Markdown list with **Class**, **Action**, **Uses**, **Adverse effects** (the most important ones), **Nursing considerations** and **Client teaching**. Only include what the material supports.', definition: 'Make term and definition cards. Front: the term. Back: a precise definition in one or two sentences, plus an example when useful.', @@ -787,7 +787,7 @@ export function startGeneration( const controller = new AbortController(); running.set(draftId, controller); if (opts.onProgress) progressListeners.set(draftId, opts.onProgress); - const done = runGeneration(app, draftId, req, controller.signal) + const done = runWithRestart(app, draftId, req, controller.signal) .catch((err) => { const message = err instanceof AIError ? err.message : err instanceof Error && 'status' in err ? err.message : 'Generation failed unexpectedly.'; if (!(err instanceof AIError) && !(err instanceof Error && 'status' in err)) app.log('error', 'Generation failed', err); @@ -824,6 +824,31 @@ function autoSaveDraft(app: AppContext, draftId: string, description: string): v } } +/** Failures worth starting the whole job over for: bad output and passing provider trouble. */ +const RESTARTABLE = new Set(['invalid_response', 'server', 'timeout', 'network', 'rate_limit']); + +/** + * Runs a generation job, and when it fails on bad model output (or a passing provider error) + * starts it over once with the same request. A second failure fails the draft. + */ +async function runWithRestart(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal): Promise { + try { + await runGeneration(app, draftId, req, signal); + } catch (err) { + if (signal.aborted || !(err instanceof AIError) || !RESTARTABLE.has(err.code)) throw err; + app.log('warn', 'Generation failed; starting it over once', err.message); + const kind = req.kind; + app.db.run( + `UPDATE drafts SET payload = ?, progress = 0.02, progress_label = 'Starting over', updated_at = ? WHERE id = ? AND status = 'generating'`, + JSON.stringify(emptyPayload(kind, req.title ?? '')), + now(), + draftId, + ); + progressListeners.get(draftId)?.('Starting over'); + await runGeneration(app, draftId, req, signal); + } +} + async function runGeneration(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal): Promise { const { db } = app; const resolved = resolveModel(app, 'generate', req.modelId); @@ -904,7 +929,8 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ : [ `Write ${want} flashcards${req.focus ? ` about: ${req.focus}` : ''}.`, sources.groups.length > 0 ? 'Cover the most testable facts across all of the excerpts.' : '', - req.source.quizId ? 'Turn the quiz questions into flashcards: the front asks for the key fact, the back states it with the essential reasoning.' : '', + req.source.quizId ? 'Turn the quiz questions into flashcards: the front asks for the key fact, the back states only that fact.' : '', + 'Do not write hints.', ]; if (written.length > 0) ask.push(`Already written (do not repeat or closely paraphrase these):\n${written.slice(-80).join('\n')}`); // Med math is standard material: the excerpts would only cost tokens. @@ -1011,6 +1037,16 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ notes.push(...checked.notes); } + // A short name and description of what this particular set covers. + if (!req.title) { + setProgress(db, draftId, 0.99, 'Naming the set', draft); + const named = await nameSet(app, draft, { classId: req.classId ?? null, modelId: req.modelId, signal, difficulty, math: numericCount() > 0 }); + if (named) { + draft.title = named.title; + draft.description = named.description; + } + } + db.run( `UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`, draft.title, @@ -1021,6 +1057,70 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ ); } +const NAME_SCHEMA = { + type: 'object', + properties: { + title: { type: 'string', description: 'Short, specific name for this set: 2-6 words naming what it covers.' }, + description: { type: 'string', description: 'One sentence (at most 25 words) saying what the set covers.' }, + }, + required: ['title', 'description'], +}; + +/** + * Asks the model for a short, distinctive title and a one-line description of a finished quiz + * or flashcard set, different from the class's other titles. Null when naming fails (the + * generic title stays). + */ +export async function nameSet( + app: AppContext, + draft: QuizDraft | DeckDraft, + opts: { classId: string | null; modelId?: string | null; signal?: AbortSignal; difficulty?: string; math?: boolean }, +): Promise<{ title: string; description: string } | null> { + const isQuiz = draft.kind === 'quiz'; + const taken = opts.classId + ? app.db.all<{ title: string }>(`SELECT title FROM ${isQuiz ? 'quizzes' : 'decks'} WHERE class_id = ? ORDER BY created_at DESC LIMIT 80`, opts.classId).map((r) => r.title) + : []; + const lines = isQuiz + ? (() => { + const counts = new Map(); + for (const q of draft.questions) counts.set(q.topic || 'General', (counts.get(q.topic || 'General') ?? 0) + 1); + const topics = [...counts].sort((a, b) => b[1] - a[1]).map(([t, n]) => `${t} (${n})`).join(', '); + return [`A quiz of ${draft.questions.length} questions. Topics: ${topics}.`, ...draft.questions.slice(0, 40).map((q) => `- ${q.stem.replace(/\s+/g, ' ').slice(0, 110)}`)]; + })() + : [`A flashcard set of ${draft.cards.length} cards.`, ...draft.cards.slice(0, 60).map((c) => `- ${c.front.replace(/\s+/g, ' ').slice(0, 90)}`)]; + try { + const result = await generateStructured(app, { + task: 'generate', + purpose: 'naming a set', + modelId: opts.modelId, + system: `You name study sets for a nursing student. The title is 2-6 words in title case and names the actual subject matter (for example "Heart Failure and Diuretics" or "Stroke Priorities"). Never use generic words such as Quiz, Flashcards, Practice, NCLEX, Exam, Review or Set, and no dates or numbers of items.${ + opts.math ? ' The set includes medication math, so the title may mention it.' : '' + } The description is one plain sentence on what it covers.`, + messages: [{ role: 'user', content: [{ type: 'text', text: `${lines.join('\n')}${taken.length ? `\n\nTitles already used in this class (yours must differ from all of them):\n${taken.map((t) => `- ${t}`).join('\n')}` : ''}` }] }], + toolName: 'name_set', + toolDescription: 'Submit the title and description.', + schema: NAME_SCHEMA, + maxOutputTokens: 400, + signal: opts.signal, + }); + const value = result.data as { title?: unknown; description?: unknown }; + let title = typeof value.title === 'string' ? value.title.replace(/\s+/g, ' ').replace(/^["'“]+|["'”.]+$/g, '').trim().slice(0, 80) : ''; + const description = typeof value.description === 'string' ? value.description.replace(/\s+/g, ' ').trim().slice(0, 300) : ''; + if (!title) return null; + const lower = new Set(taken.map((t) => t.toLowerCase())); + if (lower.has(title.toLowerCase())) { + let n = 2; + while (lower.has(`${title} ${n}`.toLowerCase())) n += 1; + title = `${title} ${n}`; + } + return { title, description }; + } catch (err) { + if (opts.signal?.aborted) throw err; + app.log('warn', 'Naming a generated set failed; keeping the default title', err instanceof Error ? err.message : err); + return null; + } +} + async function generateNote(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal, budgetTokens: number, window: number): Promise { const { db } = app; const resolved = resolveModel(app, 'generate', req.modelId); diff --git a/server/ai/mcp.ts b/server/ai/mcp.ts index cb5ca62..0909f8c 100644 --- a/server/ai/mcp.ts +++ b/server/ai/mcp.ts @@ -27,6 +27,8 @@ export type McpAreas = Record; export const DEFAULT_MCP_AREAS: McpAreas = { chat: true, explain: true, generate: false, check: false, share: false }; const PROTOCOL_VERSION = '2025-06-18'; const LIST_TIMEOUT_MS = 10_000; +/** A first connection (sign-in refresh, session set-up) can take longer than a warm listing. */ +const FIRST_LIST_TIMEOUT_MS = 20_000; const CALL_TIMEOUT_MS = 90_000; const TOOLS_TTL_MS = 5 * 60_000; const MAX_RESPONSE_BYTES = 4 * 1024 * 1024; @@ -742,11 +744,19 @@ export async function loadMcpTools(app: AppContext, area: McpArea, signal?: Abor const lists = await Promise.all( servers.map(async (s) => { try { - return { s, tools: await listTools(app, s, anySignal([signal, AbortSignal.timeout(LIST_TIMEOUT_MS)])) }; - } catch (err) { - app.log('warn', `MCP server "${s.name}" is unavailable`, err instanceof Error ? err.message : err); - unavailable.push(s.name); - return { s, tools: [] as McpTool[] }; + const warm = sessions.has(s.id); + return { s, tools: await listTools(app, s, anySignal([signal, AbortSignal.timeout(warm ? LIST_TIMEOUT_MS : FIRST_LIST_TIMEOUT_MS)])) }; + } catch (first) { + if (signal?.aborted) return { s, tools: [] as McpTool[] }; + // One more try on a fresh connection before giving up on the server for this turn. + try { + forget(s.id); + return { s, tools: await listTools(app, s, anySignal([signal, AbortSignal.timeout(FIRST_LIST_TIMEOUT_MS)]), true) }; + } catch (err) { + app.log('warn', `MCP server "${s.name}" is unavailable`, err instanceof Error ? err.message : (first instanceof Error ? first.message : err)); + unavailable.push(s.name); + return { s, tools: [] as McpTool[] }; + } } }), ); @@ -802,7 +812,7 @@ export function mcpPromptNote(mcp: LoadedMcp, opts: { askBeforeWeb?: boolean } = const others = mcp.servers.filter((s) => s.builtin !== 'web'); if (web) { const when = opts.askBeforeWeb - ? 'Their own documents come first. When the documents do not answer a question, do not answer from your own knowledge and do not search on your own: call ask_user with one choice question, e.g. "Your class materials do not cover this. Should I look it up on the web?" with the options "Search the web" (first, the default) and "Answer from general knowledge", then stop. Search straight away only when the user asked you to search or look something up online, or already said yes to searching in this conversation.' + ? 'Their own documents come first. When the documents do not answer a question, never answer from your own knowledge and do not search on your own: call ask_user with one choice question, e.g. "Your class materials do not cover this. Should I look it up on the web?" with the options "Search the web" (first, the default) and "No, skip it", then stop. If they choose not to search, say you do not have information to answer it. Search straight away only when the user asked you to search or look something up online, or already said yes to searching in this conversation.' : "Use it for current facts, guidelines, drug information and anything the student's own material does not cover. For questions about their course material, use their documents first."; lines.push( `Web search: you can search the web with web_search and read specific pages with web_fetch${web.research ? ', and answer a hard question in depth with web_research (slower; use it sparingly)' : ''}. ${when}${ @@ -823,6 +833,29 @@ export function mcpPromptNote(mcp: LoadedMcp, opts: { askBeforeWeb?: boolean } = } /** Connects afresh, lists the tools and records the outcome on the server. */ +/** + * Connects to every enabled server and lists its tools in the background, so the web search and + * MCP tools are ready the first time the AI needs them (no manual connection test). + */ +export function warmMcpServers(app: AppContext): void { + if (app.config.env === 'test') return; + for (const s of load(app).filter((x) => x.enabled)) { + void listTools(app, s, AbortSignal.timeout(FIRST_LIST_TIMEOUT_MS), true).catch((err) => app.log('warn', `MCP server "${s.name}" could not be reached yet`, err instanceof Error ? err.message : err)); + } +} + +/** Keeps the tool lists warm while the server runs. */ +export function startMcpWarmup(app: AppContext): () => void { + const first = setTimeout(() => warmMcpServers(app), 3_000); + const timer = setInterval(() => warmMcpServers(app), TOOLS_TTL_MS - 30_000); + first.unref(); + timer.unref(); + return () => { + clearTimeout(first); + clearInterval(timer); + }; +} + export async function testMcpServer(app: AppContext, id: string): Promise<{ ok: boolean; message: string; tools: { name: string; description: string }[]; latencyMs: number }> { const servers = load(app); const s = servers.find((x) => x.id === id); diff --git a/server/ai/providers/common.ts b/server/ai/providers/common.ts index 13fed7a..2f40988 100644 --- a/server/ai/providers/common.ts +++ b/server/ai/providers/common.ts @@ -41,6 +41,7 @@ export function parseToolArguments(raw: unknown, provider: string, toolName: str } throw new AIError('invalid_response', `${provider} produced a tool call for "${toolName}" with arguments that are not valid JSON.`, { providerMessage: text.length > 300 ? `${text.slice(0, 300)}…` : text, + rawOutput: text, }); } diff --git a/server/ai/service.ts b/server/ai/service.ts index c90a6fb..5ff9c25 100644 --- a/server/ai/service.ts +++ b/server/ai/service.ts @@ -661,21 +661,54 @@ export async function generateStructured(app: AppContext, opts: StructuredOption return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' }; }; if (!resolved.model.capabilities.tools) return promptJson(); - try { - const chatOpts: ChatOptions = { ...opts, tools: [tool], toolChoice: { name: opts.toolName } }; + const chatOpts: ChatOptions = { ...opts, tools: [tool], toolChoice: { name: opts.toolName } }; + const once = async () => { const res = opts.mcpArea ? await runChatWithMcp(app, chatOpts, opts.mcpArea) : await runChat(app, chatOpts); const call = res.message.content.find((p) => p.type === 'tool_call'); - if (call && call.type === 'tool_call') return { data: call.input, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' }; + if (call && call.type === 'tool_call') return { data: call.input as unknown, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' }; if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' }; if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.'); if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.'); throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true }); + }; + try { + return await once(); } catch (err) { if (err instanceof AIError && err.code === 'unsupported') return promptJson(); - throw err; + if (!(err instanceof AIError) || err.code !== 'invalid_response' || opts.signal?.aborted) throw err; + // Malformed output (jsonrepair could not fix it): the model repairs its own JSON when we + // have it, otherwise it gets one more try at the same request. + if (err.rawOutput) { + try { + return await repairStructured(app, opts, tool, err.rawOutput); + } catch (repairErr) { + app.log('warn', `Repairing malformed ${opts.toolName} output failed; asking again`, repairErr instanceof Error ? repairErr.message : repairErr); + } + } + return await once(); } } +/** Sends malformed tool-call arguments back to the model to be rewritten as valid JSON. */ +async function repairStructured(app: AppContext, opts: StructuredOptions, tool: ToolDef, raw: string) { + const res = await runChat(app, { + task: opts.task, + purpose: `${opts.purpose ?? 'structured output'} (JSON repair)`, + modelId: opts.modelId, + system: 'You repair malformed JSON. Call the tool once with the same content, corrected so it is valid JSON that matches the tool schema. Keep every item and all of its wording; do not add, drop or rewrite content.', + messages: [{ role: 'user', content: [{ type: 'text', text: `These arguments for "${tool.name}" are not valid JSON: + +${raw}` }] }], + tools: [tool], + toolChoice: { name: tool.name }, + maxOutputTokens: opts.maxOutputTokens, + signal: opts.signal, + }); + const call = res.message.content.find((p) => p.type === 'tool_call'); + if (!call || call.type !== 'tool_call') throw new AIError('invalid_response', 'The repair returned no tool call.'); + return { data: call.input as unknown, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' }; +} + /* ------------------------------ reporting ------------------------------ */ export function usageSummary(db: DB, days: number) { diff --git a/server/ai/types.ts b/server/ai/types.ts index 2d124e7..5750160 100644 --- a/server/ai/types.ts +++ b/server/ai/types.ts @@ -184,14 +184,20 @@ export class AIError extends Error { retryable: boolean; /** Raw (already secret-scrubbed) provider message, for the details disclosure in the UI. */ providerMessage?: string; + /** + * The model's unparseable output (tool-call arguments), kept so it can be sent back for + * repair. Server-side only: never serialised to a response. + */ + declare rawOutput?: string; - constructor(code: AIErrorCode, message: string, opts: { status?: number; retryable?: boolean; providerMessage?: string; cause?: unknown } = {}) { + constructor(code: AIErrorCode, message: string, opts: { status?: number; retryable?: boolean; providerMessage?: string; rawOutput?: string; cause?: unknown } = {}) { super(message, opts.cause !== undefined ? { cause: opts.cause } : undefined); this.name = 'AIError'; this.code = code; this.status = opts.status; this.retryable = opts.retryable ?? (code === 'rate_limit' || code === 'server' || code === 'network' || code === 'timeout'); this.providerMessage = opts.providerMessage; + if (opts.rawOutput !== undefined) Object.defineProperty(this, 'rawOutput', { value: opts.rawOutput, enumerable: false }); } } diff --git a/server/main.ts b/server/main.ts index 85bcfd8..2bfe8c2 100644 --- a/server/main.ts +++ b/server/main.ts @@ -1,5 +1,7 @@ import { serve } from '@hono/node-server'; import { tidyDrafts } from './ai/generate.ts'; +import { startGeoSchedule } from './services/geo.ts'; +import { startMcpWarmup } from './ai/mcp.ts'; import { createApp } from './app.ts'; import { initAuth } from './auth.ts'; import { loadConfig } from './config.ts'; @@ -39,6 +41,8 @@ async function main(): Promise { if (resumed > 0) log('info', `Resuming processing of ${resumed} document(s)`); scheduleIndexing(app); const stopBackups = startBackupSchedule(app); + const stopGeo = startGeoSchedule(app); + const stopMcp = startMcpWarmup(app); const server = serve({ fetch: createApp(app).fetch, port: config.port, hostname: config.host }, (info) => { log('info', `Listening on http://${info.address === '::' || info.address === '0.0.0.0' ? 'localhost' : info.address}:${info.port}`); @@ -52,6 +56,8 @@ async function main(): Promise { closing = true; log('info', `${signal} received, shutting down`); stopBackups(); + stopGeo(); + stopMcp(); server.close(() => { try { db.close(); diff --git a/server/migrations.ts b/server/migrations.ts index 42c8daf..62ab928 100644 --- a/server/migrations.ts +++ b/server/migrations.ts @@ -450,6 +450,19 @@ CREATE INDEX drafts_share ON drafts(share_id, created_at); ALTER TABLE shares ADD COLUMN generate_reset_at INTEGER; `; +/** + * Share-link AI safety: AI features switch themselves off a week after they are turned on + * (`ai_expires_at`), and abuse protection can lock them (`ai_blocked`, JSON {at, reason}). + * Links that already have AI on get their week from now. A visitor can dismiss a failed set + * from the link (`share_hidden`); the owner still sees it in Drafts. + */ +const shareAiSafety = ` +ALTER TABLE shares ADD COLUMN ai_expires_at INTEGER; +ALTER TABLE shares ADD COLUMN ai_blocked TEXT; +UPDATE shares SET ai_expires_at = CAST(strftime('%s', 'now') AS INTEGER) * 1000 + 604800000 WHERE ai LIKE '%true%'; +ALTER TABLE drafts ADD COLUMN share_hidden INTEGER NOT NULL DEFAULT 0; +`; + export const MIGRATIONS: Migration[] = [ { id: 1, name: 'init', sql: init }, { id: 2, name: 'embeddings', sql: embeddings }, @@ -458,4 +471,5 @@ export const MIGRATIONS: Migration[] = [ { id: 5, name: 'revision3', sql: revision3 }, { id: 6, name: 'revision4', sql: revision4 }, { id: 7, name: 'share-generation', sql: shareGeneration }, + { id: 8, name: 'share-ai-safety', sql: shareAiSafety }, ]; diff --git a/server/routes/ai.ts b/server/routes/ai.ts index c198e6f..2bd64e1 100644 --- a/server/routes/ai.ts +++ b/server/routes/ai.ts @@ -34,6 +34,7 @@ import { testMcpServer, WEB_SEARCH_ID, webSearchPatchSchema, + warmMcpServers, } from '../ai/mcp.ts'; import type { ToolActivity } from '../ai/activity.ts'; import { genUiPrompt } from '../ai/genui.ts'; @@ -160,8 +161,17 @@ export function aiRoutes(app: AppContext): Hono { // MCP servers the assistant can call as tools. r.get('/ai/mcp', (c) => c.json({ servers: listMcpServers(app) })); - r.post('/ai/mcp', async (c) => c.json({ server: saveMcpServer(app, null, await readBody(c, mcpServerInputSchema)) }, 201)); - r.patch('/ai/mcp/:id', async (c) => c.json({ server: saveMcpServer(app, idParam(c), await readBody(c, mcpServerPatchSchema)) })); + // Saving (or signing in) connects in the background, so the tools are ready without a manual test. + r.post('/ai/mcp', async (c) => { + const server = saveMcpServer(app, null, await readBody(c, mcpServerInputSchema)); + warmMcpServers(app); + return c.json({ server }, 201); + }); + r.patch('/ai/mcp/:id', async (c) => { + const server = saveMcpServer(app, idParam(c), await readBody(c, mcpServerPatchSchema)); + warmMcpServers(app); + return c.json({ server }); + }); r.delete('/ai/mcp/:id', (c) => { deleteMcpServer(app, idParam(c)); return c.json({ ok: true }); @@ -188,6 +198,7 @@ export function aiRoutes(app: AppContext): Hono { if (c.req.query('error')) return back('/settings/ai', 'error', c.req.query('error_description') || 'The sign-in was cancelled.'); try { const done = await finishMcpOAuth(app, state, code); + warmMcpServers(app); return back(done.returnTo, 'ok', `Signed in to ${done.name}.`); } catch (err) { app.log('warn', 'MCP sign-in failed', err instanceof Error ? err.message : err); @@ -198,7 +209,11 @@ export function aiRoutes(app: AppContext): Hono { // The built-in web search. r.get('/ai/web-search', (c) => c.json({ webSearch: getWebSearch(app) })); - r.patch('/ai/web-search', async (c) => c.json({ webSearch: saveWebSearch(app, await readBody(c, webSearchPatchSchema)) })); + r.patch('/ai/web-search', async (c) => { + const webSearch = saveWebSearch(app, await readBody(c, webSearchPatchSchema)); + warmMcpServers(app); + return c.json({ webSearch }); + }); r.post('/ai/web-search/test', async (c) => c.json({ result: await testMcpServer(app, WEB_SEARCH_ID), webSearch: getWebSearch(app) })); /** Tool sources the chat may use, for the per-conversation switches. */ diff --git a/server/routes/shares.ts b/server/routes/shares.ts index 4abf15e..16e0f43 100644 --- a/server/routes/shares.ts +++ b/server/routes/shares.ts @@ -19,6 +19,7 @@ import { listChunks, searchChunks, sourceLabel, type ChunkHit } from '../service import { hybridSearch } from '../services/semantic.ts'; import { newId, newToken, now, parseJson, startOfDay } from '../util.ts'; import type { ToolActivity } from '../ai/activity.ts'; +import { checkVisitorRegion, geoRegionSchema, geoStatus, setGeoRegion } from '../services/geo.ts'; import { sourceActivity, sse, type ExplainEvent } from './ai.ts'; const KINDS = ['quiz', 'deck', 'folder', 'class', 'note', 'document'] as const; @@ -36,6 +37,10 @@ interface ShareRow { updated_at: number; ai: string; generate_reset_at: number | null; + /** AI features switch themselves off at this time (a week after they were turned on). */ + ai_expires_at: number | null; + /** Set when abuse protection turned AI off: JSON {at, reason}. */ + ai_blocked: string | null; } /** AI features a share's visitors may use. Both are off until the owner turns them on. */ @@ -48,7 +53,24 @@ export interface ShareAi { generate: boolean; } -const shareAi = (row: Pick): ShareAi => ({ explain: false, assistant: false, generate: false, ...parseJson>(row.ai, {}) }); +const NO_AI: ShareAi = { explain: false, assistant: false, generate: false }; +/** The AI switches as the owner set them. */ +const shareAi = (row: Pick): ShareAi => ({ ...NO_AI, ...parseJson>(row.ai, {}) }); +const anyAi = (ai: ShareAi) => ai.explain || ai.assistant || ai.generate; + +/** AI on a link lasts a week from when it was turned on; the owner can give it another week. */ +export const SHARE_AI_DAYS = 7; +const SHARE_AI_MS = SHARE_AI_DAYS * 86_400_000; + +/** Whether the link's AI is running: switched on, within its week, and not locked by abuse protection. */ +function aiState(row: Pick) { + const blocked = row.ai_blocked ? parseJson<{ at: number; reason: string } | null>(row.ai_blocked, null) : null; + const expired = row.ai_expires_at !== null && row.ai_expires_at <= now(); + return { expiresAt: row.ai_expires_at, expired, blocked, active: anyAi(shareAi(row)) && !expired && !blocked }; +} + +/** The AI features that work on the link right now. */ +const liveAi = (row: Pick): ShareAi => (aiState(row).active ? shareAi(row) : NO_AI); /** Sets a share link may generate per day (the app's time zone), until the owner resets it. */ export const SHARE_GENERATE_LIMITS = { quiz: 4, deck: 2 } as const; @@ -76,10 +98,10 @@ function generateUsage(app: AppContext, share: Pick( - `SELECT id, kind, title, status, progress, progress_label, payload, saved_kind, saved_id, created_at FROM drafts WHERE share_id = ? AND status IN ('generating', 'ready', 'failed', 'saved') ORDER BY created_at DESC LIMIT 30`, + `SELECT id, kind, title, status, progress, progress_label, payload, saved_kind, saved_id, created_at FROM drafts WHERE share_id = ? AND status IN ('generating', 'ready', 'failed', 'saved')${opts.visitor ? ' AND share_hidden = 0' : ''} ORDER BY created_at DESC LIMIT 30`, shareId, ) .flatMap((d) => { @@ -96,7 +118,8 @@ function generatedSets(app: AppContext, shareId: string) { status, itemId: saved ? d.saved_id : null, progress: d.progress, - progressLabel: d.progress_label, + // Visitors see plain steps; the owner's drafts keep the detail. + progressLabel: opts.visitor && /python|checking/i.test(d.progress_label) ? 'Verifying questions' : d.progress_label, error: status === 'failed' ? 'This set could not be made. Try again later.' : status === 'review' ? 'Made, and waiting for the owner to check it.' : '', count: (d.kind === 'quiz' ? payload.questions?.length : payload.cards?.length) ?? 0, createdAt: d.created_at, @@ -125,6 +148,9 @@ function mapShare(row: ShareRow) { lastViewedAt: row.last_viewed_at, createdAt: row.created_at, ai: shareAi(row), + aiExpiresAt: row.ai_expires_at, + aiBlocked: aiState(row).blocked, + aiActive: aiState(row).active, }; } @@ -186,11 +212,39 @@ export function shareRoutes(app: AppContext): Hono { const row = db.get('SELECT * FROM shares WHERE id = ?', id); if (!row) throw notFound('Share link'); const enabled = body.enabled ?? Boolean(row.enabled); - const ai = { ...shareAi(row), ...body.ai }; - db.run('UPDATE shares SET enabled = ?, ai = ?, updated_at = ? WHERE id = ?', enabled ? 1 : 0, JSON.stringify(ai), now(), id); + const state = aiState(row); + // Switching AI on after it lapsed (or was locked) starts from scratch: only what is turned on now. + const ai = { ...(state.active ? shareAi(row) : NO_AI), ...body.ai }; + let expiresAt = row.ai_expires_at; + let blocked = row.ai_blocked; + if (!anyAi(ai)) { + expiresAt = null; + } else if (!state.active) { + expiresAt = now() + SHARE_AI_MS; + blocked = null; + } + db.run('UPDATE shares SET enabled = ?, ai = ?, ai_expires_at = ?, ai_blocked = ?, updated_at = ? WHERE id = ?', enabled ? 1 : 0, JSON.stringify(ai), expiresAt, blocked, now(), id); return c.json({ share: mapShare(db.get('SELECT * FROM shares WHERE id = ?', id)!) }); }); + /** Another week of AI on the link, with the features it had (also lifts an abuse lock). */ + r.post('/shares/:id/ai/renew', (c) => { + const id = idParam(c); + const row = db.get('SELECT * FROM shares WHERE id = ?', id); + if (!row) throw notFound('Share link'); + if (!anyAi(shareAi(row))) throw new HttpError(400, 'ai_off', 'Turn on an AI feature first.'); + db.run('UPDATE shares SET ai_expires_at = ?, ai_blocked = NULL, updated_at = ? WHERE id = ?', now() + SHARE_AI_MS, now(), id); + return c.json({ share: mapShare(db.get('SELECT * FROM shares WHERE id = ?', id)!) }); + }); + + /** Where visitors may use AI from, and how the location check is doing. */ + r.get('/share-settings/ai', (c) => c.json({ geo: geoStatus(app), abuse: ABUSE })); + r.put('/share-settings/ai', async (c) => { + const body = await readBody(c, geoRegionSchema); + setGeoRegion(app, body.region); + return c.json({ geo: geoStatus(app), abuse: ABUSE }); + }); + /** Gives the link a fresh daily allowance of generated quizzes and flashcards. */ r.post('/shares/:id/generate/reset', (c) => { const id = idParam(c); @@ -263,8 +317,51 @@ function visitorId(app: AppContext, c: Context): string { return fresh; } +/** + * Abuse protection: signs that a link has leaked or is being scripted turn its AI off until the + * owner turns it back on. Counted over the last hour per link (in memory). + */ +export const ABUSE = { + windowMs: 60 * 60_000, + /** Different networks (IP addresses) using AI. A class shares a handful; dozens is a leak. */ + addresses: 25, + /** Different browsers (visitor cookies), which clearing cookies inflates. */ + devices: 40, + /** A flood: this many AI requests within five minutes. */ + burst: 60, + burstMs: 5 * 60_000, + /** Requests from outside the allowed region (the page hides AI there, so these are scripted). */ + outside: 10, + /** Requests refused by the per-device limit. */ + limited: 30, +} as const; + +type GuardEvent = { at: number; ip: string; device: string; kind: 'ok' | 'outside' | 'limited' }; +const guardEvents = new Map(); + +/** Records one AI request on a link; returns why AI should be turned off, if it should. */ +function guard(shareId: string, event: Omit): string | null { + const at = Date.now(); + if (guardEvents.size > 5000) guardEvents.clear(); + const list = (guardEvents.get(shareId) ?? []).filter((e) => at - e.at < ABUSE.windowMs); + list.push({ ...event, at }); + guardEvents.set(shareId, list); + const addresses = new Set(list.map((e) => e.ip)).size; + const devices = new Set(list.map((e) => e.device)).size; + const burst = list.filter((e) => at - e.at < ABUSE.burstMs).length; + const outside = list.filter((e) => e.kind === 'outside').length; + const limited = list.filter((e) => e.kind === 'limited').length; + if (addresses >= ABUSE.addresses) return `AI was used from ${addresses} different networks within an hour.`; + if (devices >= ABUSE.devices) return `AI was used from ${devices} different browsers within an hour.`; + if (burst >= ABUSE.burst) return `${burst} AI requests arrived within five minutes.`; + if (outside >= ABUSE.outside) return `${outside} AI requests came from outside the allowed region within an hour.`; + if (limited >= ABUSE.limited) return `${limited} AI requests went over the rate limit within an hour.`; + return null; +} + export function resetShareAiLimits(): void { aiHits.clear(); + guardEvents.clear(); } /** What a visitor sees of a tool step: the step itself and its pages, not its raw input and output. */ @@ -355,19 +452,19 @@ function collectionItems(app: AppContext, share: ShareRow) { const byFolder = share.kind === 'folder'; const where = byFolder ? 'folder_id = ?' : 'class_id = ?'; const quizzes = db.all>( - `SELECT z.id, z.title, z.description, z.folder_id, (SELECT COUNT(*) FROM questions q WHERE q.quiz_id = z.id) AS n FROM quizzes z WHERE z.${where} ORDER BY z.sort_order, z.created_at`, + `SELECT z.id, z.title, z.description, z.folder_id, z.created_at, (SELECT COUNT(*) FROM questions q WHERE q.quiz_id = z.id) AS n FROM quizzes z WHERE z.${where} ORDER BY z.sort_order, z.created_at`, share.target_id, ); const decks = db.all>( - `SELECT d.id, d.title, d.description, d.folder_id, (SELECT COUNT(*) FROM cards c WHERE c.deck_id = d.id) AS n FROM decks d WHERE d.${where} ORDER BY d.sort_order, d.created_at`, + `SELECT d.id, d.title, d.description, d.folder_id, d.created_at, (SELECT COUNT(*) FROM cards c WHERE c.deck_id = d.id) AS n FROM decks d WHERE d.${where} ORDER BY d.sort_order, d.created_at`, share.target_id, ); const notes = db.all>(`SELECT id, title, folder_id FROM notes WHERE ${where} ORDER BY sort_order, created_at`, share.target_id); // Documents are shared once they have been processed (a failed or half-read upload stays private). const documents = db.all>(`SELECT id, title, folder_id, page_count, kind FROM documents WHERE ${where} AND status = 'ready' ORDER BY sort_order, created_at`, share.target_id); return [ - ...quizzes.filter((q) => q.n > 0).map((q) => ({ kind: 'quiz' as const, id: q.id as string, title: q.title as string, description: q.description as string, folderId: q.folder_id as string | null, count: q.n as number })), - ...decks.filter((d) => d.n > 0).map((d) => ({ kind: 'deck' as const, id: d.id as string, title: d.title as string, description: d.description as string, folderId: d.folder_id as string | null, count: d.n as number })), + ...quizzes.filter((q) => q.n > 0).map((q) => ({ kind: 'quiz' as const, id: q.id as string, title: q.title as string, description: q.description as string, folderId: q.folder_id as string | null, count: q.n as number, createdAt: q.created_at as number })), + ...decks.filter((d) => d.n > 0).map((d) => ({ kind: 'deck' as const, id: d.id as string, title: d.title as string, description: d.description as string, folderId: d.folder_id as string | null, count: d.n as number, createdAt: d.created_at as number })), ...notes.map((n) => ({ kind: 'note' as const, id: n.id as string, title: n.title as string, description: '', folderId: n.folder_id as string | null, count: 0 })), ...documents.map((d) => ({ kind: 'document' as const, id: d.id as string, title: d.title as string, description: String(d.kind ?? ''), folderId: d.folder_id as string | null, count: (d.page_count as number | null) ?? 0 })), ]; @@ -378,11 +475,13 @@ export function publicRoutes(app: AppContext): Hono { const r = new Hono(); /** The switches visitors see: on only when the owner allowed it and a model is configured. */ - const publicAi = (share: ShareRow): ShareAi => { - const flags = { ...shareAi(share) }; + const publicAi = (share: ShareRow, c: Context): ShareAi => { + const flags = { ...liveAi(share) }; // Generating needs class materials, which only class and exam links have. if (share.kind !== 'class' && share.kind !== 'folder') flags.generate = false; if (!flags.explain && !flags.assistant && !flags.generate) return flags; + // Outside the allowed region the page works, without AI. + if (!checkVisitorRegion(app, c).allowed) return NO_AI; const ready = aiStatus(app).ready; return { explain: flags.explain && ready, assistant: flags.assistant && ready, generate: flags.generate && ready }; }; @@ -399,7 +498,7 @@ export function publicRoutes(app: AppContext): Hono { db.run('UPDATE shares SET view_count = view_count + 1, last_viewed_at = ? WHERE id = ?', now(), share.id); c.header('Cache-Control', 'no-store'); c.header('X-Robots-Tag', 'noindex, nofollow'); - const ai = publicAi(share); + const ai = publicAi(share, c); switch (share.kind) { case 'quiz': return c.json({ ...publicQuiz(app, share.target_id), ai }); @@ -436,7 +535,7 @@ export function publicRoutes(app: AppContext): Hono { if (!allowed) throw gone(); c.header('Cache-Control', 'no-store'); c.header('X-Robots-Tag', 'noindex, nofollow'); - const ai = publicAi(share); + const ai = publicAi(share, c); if (kind === 'quiz') return c.json({ ...publicQuiz(app, id), ai }); if (kind === 'deck') return c.json({ ...publicDeck(app, id), ai }); if (kind === 'document') return c.json({ ...publicDocument(app, id), ai }); @@ -474,10 +573,36 @@ export function publicRoutes(app: AppContext): Hono { const inScope = (share: ShareRow, kind: 'quiz' | 'deck' | 'note' | 'document', id: string): boolean => share.kind === kind ? share.target_id === id : (share.kind === 'class' || share.kind === 'folder') && collectionItems(app, share).some((i) => i.kind === kind && i.id === id); + /** Turns a link's AI off (abuse protection); the owner sees why and can turn it back on. */ + const lockAi = (share: ShareRow, reason: string) => { + db.run('UPDATE shares SET ai_blocked = ?, updated_at = ? WHERE id = ?', JSON.stringify({ at: now(), reason }), now(), share.id); + guardEvents.delete(share.id); + app.log('warn', `AI turned off on a share link by abuse protection: ${reason}`); + }; + const requireAi = (c: Context, share: ShareRow, feature: keyof ShareAi) => { - if (!shareAi(share)[feature]) throw new HttpError(403, 'ai_disabled', 'The owner has not turned this on for this link.'); + if (!liveAi(share)[feature]) throw new HttpError(403, 'ai_disabled', 'AI is not turned on for this link.'); if (!aiStatus(app).ready) throw new HttpError(503, 'ai_unavailable', 'AI is not available right now.'); - limitVisitor(share.id, visitorId(app, c)); + const device = visitorId(app, c); + const geo = checkVisitorRegion(app, c); + const trip = (kind: GuardEvent['kind']) => { + const reason = guard(share.id, { ip: geo.ip, device, kind }); + if (reason) { + lockAi(share, reason); + throw new HttpError(403, 'ai_disabled', 'AI is not turned on for this link.'); + } + }; + if (!geo.allowed) { + trip('outside'); + throw new HttpError(403, 'ai_region', "AI features aren't available from your location."); + } + try { + limitVisitor(share.id, device); + } catch (err) { + trip('limited'); + throw err; + } + trip('ok'); }; /** The class a class or exam link belongs to. */ @@ -487,9 +612,9 @@ export function publicRoutes(app: AppContext): Hono { /** What visitors can generate on this link, and what was made on it. */ r.get('/:token/generated', (c) => { const share = load(c.req.param('token')); - if (!publicAi(share).generate) throw new HttpError(403, 'ai_disabled', 'The owner has not turned this on for this link.'); + if (!publicAi(share, c).generate) throw new HttpError(403, 'ai_disabled', 'AI is not turned on for this link.'); c.header('Cache-Control', 'no-store'); - return c.json({ sets: generatedSets(app, share.id), usage: generateUsage(app, share), limits: { quizMax: SHARE_QUIZ_MAX, deckMax: SHARE_DECK_MAX, mathQuestions: SHARE_MATH_QUESTIONS } }); + return c.json({ sets: generatedSets(app, share.id, { visitor: true }), usage: generateUsage(app, share), limits: { quizMax: SHARE_QUIZ_MAX, deckMax: SHARE_DECK_MAX, mathQuestions: SHARE_MATH_QUESTIONS } }); }); const generateSchema = z.object({ @@ -553,7 +678,15 @@ export function publicRoutes(app: AppContext): Hono { // Saved into the class (and exam) as if the owner had approved it; in Drafts while it is written. { origin: 'share', shareId: share.id, autoSave: { description: 'Made with AI on a share link.' } }, ); - return c.json({ set: generatedSets(app, share.id).find((s) => s.id === draftId), usage: generateUsage(app, share) }, 202); + return c.json({ set: generatedSets(app, share.id, { visitor: true }).find((s) => s.id === draftId), usage: generateUsage(app, share) }, 202); + }); + + /** A visitor clears a failed set from the link; the owner still finds it in Drafts. */ + r.post('/:token/generated/:id/dismiss', (c) => { + const share = load(c.req.param('token')); + const { changes } = db.run(`UPDATE drafts SET share_hidden = 1 WHERE id = ? AND share_id = ? AND status = 'failed'`, c.req.param('id'), share.id); + if (!changes) throw gone(); + return c.json({ ok: true }); }); const explainSchema = z.discriminatedUnion('kind', [ @@ -630,7 +763,7 @@ export function publicRoutes(app: AppContext): Hono { const excerpts = hits.map((h) => `[${sourceLabel(h)}]\n${h.text}`).join('\n\n---\n\n'); const system = [ `You are a study assistant on a page someone shared: "${material.title}". Help the visitor understand and remember this material.`, - 'Answer from the shared material below and from general nursing knowledge. You cannot see anything else of the owner\'s, and you cannot change anything. If a question has nothing to do with studying this material, politely say you can only help with it.', + 'Answer only from the shared material and excerpts below (and web search results, when that tool is offered). Never answer factual questions from your own general knowledge: if the material does not cover something, say you do not have information on it in these materials. You cannot see anything else of the owner\'s, and you cannot change anything. If a question has nothing to do with studying this material, politely say you can only help with it.', 'When you use a document excerpt, name its source in parentheses, e.g. (Lecture 3, p. 4). Be concise and use Markdown.', material.text ? `SHARED MATERIAL:\n${material.text}` : '', excerpts ? `RELEVANT DOCUMENT EXCERPTS:\n${excerpts}` : '', diff --git a/server/services/geo.ts b/server/services/geo.ts new file mode 100644 index 0000000..12e4d39 --- /dev/null +++ b/server/services/geo.ts @@ -0,0 +1,231 @@ +/** + * Where a share-link visitor is, for limiting AI features to a region (Settings → Sharing). + * + * Sources, in order: Cloudflare's visitor-location headers when the app trusts its proxy + * (`CF-IPCountry`, and `CF-Region-Code` with the "Add visitor location headers" transform), then + * a local copy of the free DB-IP Lite database (CC BY 4.0, https://db-ip.com), downloaded + * monthly into DATA_DIR/geoip. No visitor address is ever sent anywhere. + * + * Private and loopback addresses count as local and are allowed. A public address that can't + * be placed is refused: the check fails closed. + */ +import fs from 'node:fs'; +import net from 'node:net'; +import path from 'node:path'; +import zlib from 'node:zlib'; +import type { Context } from 'hono'; +import { Reader } from 'mmdb-lib'; +import { z } from 'zod'; +import { clientIp } from '../auth.ts'; +import type { AppContext } from '../http.ts'; +import { getSetting, setSetting } from '../settings.ts'; + +export const GEO_REGIONS = ['anywhere', 'us', 'arizona'] as const; +export type GeoRegion = (typeof GEO_REGIONS)[number]; +const SETTING = 'share.aiRegion'; +export const geoRegionSchema = z.object({ region: z.enum(GEO_REGIONS) }); + +type DbKind = 'country' | 'city'; +const MAX_AGE_MS = 32 * 86_400_000; +const RETRY_MS = 6 * 3_600_000; + +/** Official monthly files (MaxMind-style records), then the ip-location-db mirror on jsDelivr. */ +function sources(kind: DbKind, at = new Date()): { url: string; gzip: boolean }[] { + const month = (d: Date) => `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`; + const prev = new Date(Date.UTC(at.getUTCFullYear(), at.getUTCMonth() - 1, 1)); + const official = [month(at), month(prev)].map((m) => ({ url: `https://download.db-ip.com/free/dbip-${kind}-lite-${m}.mmdb.gz`, gzip: true })); + const mirror = + kind === 'country' + ? [{ url: 'https://cdn.jsdelivr.net/npm/@ip-location-db/dbip-country-mmdb/dbip-country.mmdb', gzip: false }] + : [{ url: 'https://cdn.jsdelivr.net/npm/@ip-location-db/dbip-city-mmdb/dbip-city-ipv4.mmdb', gzip: false }]; + return [...official, ...mirror]; +} + +interface LoadedDb { + reader: Reader; + builtAt: number; + downloadedAt: number; +} + +const dbs: Partial> = {}; +const lastError: Partial> = {}; +const downloading: Partial>> = {}; +const lastAttempt: Partial> = {}; + +/** Recent decisions, for the status line in Settings (in memory, last 24 hours). */ +const recent: { at: number; outcome: 'allowed' | 'blocked' | 'local' | 'unknown' }[] = []; + +export function getGeoRegion(app: AppContext): GeoRegion { + // Tests run without client addresses, so they start unrestricted. + const value = getSetting(app.db, SETTING, app.config.env === 'test' ? 'anywhere' : 'us'); + return (GEO_REGIONS as readonly string[]).includes(value) ? (value as GeoRegion) : 'us'; +} + +export function setGeoRegion(app: AppContext, region: GeoRegion): void { + setSetting(app.db, SETTING, region); + ensureGeoData(app); +} + +const dir = (app: AppContext) => path.join(app.config.dataDir, 'geoip'); +const file = (app: AppContext, kind: DbKind) => path.join(dir(app), `${kind}.mmdb`); + +function loadFromDisk(app: AppContext, kind: DbKind): boolean { + try { + const buf = fs.readFileSync(file(app, kind)); + const reader = new Reader(buf); + dbs[kind] = { reader, builtAt: Number(reader.metadata.buildEpoch) || 0, downloadedAt: fs.statSync(file(app, kind)).mtimeMs }; + return true; + } catch { + return false; + } +} + +async function download(app: AppContext, kind: DbKind): Promise { + lastAttempt[kind] = Date.now(); + const errors: string[] = []; + for (const src of sources(kind)) { + try { + const res = await fetch(src.url, { signal: AbortSignal.timeout(kind === 'city' ? 600_000 : 120_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const body = Buffer.from(await res.arrayBuffer()); + const data = src.gzip ? zlib.gunzipSync(body) : body; + new Reader(data); // validates the file before it replaces the old one + fs.mkdirSync(dir(app), { recursive: true }); + const tmp = `${file(app, kind)}.tmp`; + fs.writeFileSync(tmp, data); + fs.renameSync(tmp, file(app, kind)); + loadFromDisk(app, kind); + delete lastError[kind]; + app.log('info', `Location database (${kind}) updated from ${new URL(src.url).host}`); + return; + } catch (err) { + errors.push(`${new URL(src.url).host}: ${err instanceof Error ? err.message : String(err)}`); + } + } + lastError[kind] = `Download failed (${errors.join('; ')})`; + app.log('warn', `Location database (${kind}) could not be downloaded`, lastError[kind]); +} + +/** The databases the chosen region needs, loaded from disk and refreshed in the background. */ +export function ensureGeoData(app: AppContext): void { + const region = getGeoRegion(app); + if (region === 'anywhere') return; + const needed: DbKind[] = region === 'arizona' ? ['country', 'city'] : ['country']; + for (const kind of needed) { + if (!dbs[kind]) loadFromDisk(app, kind); + const stale = !dbs[kind] || Date.now() - dbs[kind]!.downloadedAt > MAX_AGE_MS; + // Tests never reach the internet. + if (!stale || app.config.env === 'test' || downloading[kind]) continue; + if (lastAttempt[kind] && Date.now() - lastAttempt[kind]! < RETRY_MS) continue; + downloading[kind] = download(app, kind).finally(() => delete downloading[kind]); + } +} + +/** Keeps the databases fresh while the server runs. */ +export function startGeoSchedule(app: AppContext): () => void { + ensureGeoData(app); + const timer = setInterval(() => ensureGeoData(app), 6 * 3_600_000); + timer.unref(); + return () => clearInterval(timer); +} + +export function isPrivateAddress(ip: string): boolean { + const v = ip.replace(/^::ffff:/i, ''); + if (net.isIPv4(v)) { + const [a, b] = v.split('.').map(Number); + return a === 10 || a === 127 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 169 && b === 254) || (a === 100 && b >= 64 && b <= 127) || a === 0; + } + if (net.isIPv6(v)) { + const lower = v.toLowerCase(); + return lower === '::1' || lower === '::' || lower.startsWith('fc') || lower.startsWith('fd') || lower.startsWith('fe80'); + } + return false; +} + +export interface GeoCheck { + allowed: boolean; + /** Where the answer came from. */ + source: 'off' | 'local' | 'header' | 'database' | 'unknown'; + country: string | null; + region: string | null; + ip: string; +} + +/** Country and (when known) state from a database record: official DB-IP or the mirror's format. */ +function fromRecord(rec: any): { country: string | null; region: string | null } { + if (!rec) return { country: null, region: null }; + const country = rec.country?.iso_code ?? rec.country_code ?? null; + const sub = rec.subdivisions?.[0]; + const region = sub?.iso_code ? String(sub.iso_code) : (sub?.names?.en ?? rec.state1 ?? null); + return { country: country ? String(country).toUpperCase() : null, region: region ? String(region) : null }; +} + +const isArizona = (region: string | null) => Boolean(region && /^(az|arizona|us-az)$/i.test(region.trim())); + +function record(outcome: (typeof recent)[number]['outcome']): void { + const at = Date.now(); + recent.push({ at, outcome }); + while (recent.length && (at - recent[0].at > 86_400_000 || recent.length > 5000)) recent.shift(); +} + +/** Whether this request comes from the region allowed to use AI on share links. */ +export function checkVisitorRegion(app: AppContext, c: Context): GeoCheck { + const region = getGeoRegion(app); + const ip = clientIp(c, app.config.trustProxy); + if (region === 'anywhere') return { allowed: true, source: 'off', country: null, region: null, ip }; + if (isPrivateAddress(ip)) { + record('local'); + return { allowed: true, source: 'local', country: null, region: null, ip }; + } + let country: string | null = null; + let state: string | null = null; + let source: GeoCheck['source'] = 'unknown'; + if (app.config.trustProxy) { + const cf = c.req.header('cf-ipcountry')?.trim().toUpperCase(); + if (cf && /^[A-Z][A-Z0-9]$/.test(cf) && cf !== 'XX') { + country = cf; + state = c.req.header('cf-region-code')?.trim() || null; + source = 'header'; + } + } + if (!country || (region === 'arizona' && !state)) { + ensureGeoData(app); + try { + const fromCountry = fromRecord(dbs.country?.reader.get(ip.replace(/^::ffff:/i, ''))); + const fromCity = region === 'arizona' ? fromRecord(dbs.city?.reader.get(ip.replace(/^::ffff:/i, ''))) : { country: null, region: null }; + if (!country) country = fromCity.country ?? fromCountry.country; + if (!state) state = fromCity.region; + if (country && source === 'unknown') source = 'database'; + } catch { + /* an unreadable address: unknown */ + } + } + // Anything not placed in the region is refused (Cloudflare marks Tor as T1, which is not US). + const allowed = country === 'US' && (region === 'us' || isArizona(state)); + record(country ? (allowed ? 'allowed' : 'blocked') : 'unknown'); + return { allowed, source: country ? source : 'unknown', country, region: state, ip }; +} + +/** For Settings: the chosen region, the databases and what the check has been doing. */ +export function geoStatus(app: AppContext) { + const region = getGeoRegion(app); + ensureGeoData(app); + const db = (kind: DbKind) => { + const d = dbs[kind]; + return { ready: Boolean(d), builtAt: d?.builtAt ?? null, downloading: Boolean(downloading[kind]), error: d ? null : (lastError[kind] ?? null) }; + }; + const count = (o: (typeof recent)[number]['outcome']) => recent.filter((r) => r.outcome === o).length; + return { + region, + trustProxy: app.config.trustProxy, + country: db('country'), + city: region === 'arizona' ? db('city') : null, + lastDay: { allowed: count('allowed'), blocked: count('blocked'), local: count('local'), unknown: count('unknown') }, + }; +} + +/** Test hook: forget loaded databases and recent decisions. */ +export function resetGeoForTests(): void { + for (const k of Object.keys(dbs) as DbKind[]) delete dbs[k]; + recent.length = 0; +} diff --git a/shared/importer.ts b/shared/importer.ts index 0244776..978d377 100644 --- a/shared/importer.ts +++ b/shared/importer.ts @@ -563,7 +563,7 @@ export const DECK_JSON_TEMPLATE = JSON.stringify( title: 'Example deck', description: 'Optional description', cards: [ - { front: 'Furosemide: drug class', back: 'Loop diuretic', hint: 'Acts on the loop of Henle', tags: ['pharm'] }, + { front: 'Furosemide: drug class', back: 'Loop diuretic', tags: ['pharm'] }, { front: 'Normal serum potassium', back: '3.5 to 5.0 mEq/L', tags: ['labs'] }, ], }), @@ -657,14 +657,13 @@ Use exactly this shape: "title": "Short title", "description": "One sentence about what this covers", "cards": [ - { "front": "Term or question", "back": "Answer", "hint": "Optional memory aid", "tags": ["topic"] } + { "front": "Term or question", "back": "Short answer", "tags": ["topic"] } ] } Rules: -- One fact or idea per card. Fronts are specific questions or terms; backs are short. -- For medications, put the generic name on the front and use a short Markdown list on the back: class, action, key adverse effects, nursing considerations. -- "hint" is optional. Leave it out or use "" when there is no good memory aid. +- Atomic cards: one fact per card. The front asks one specific question; the back is only the answer, a word, number or short phrase (ideally under 10 words). No explanations or paragraphs on the back. +- Split anything with several parts (a drug's class, action and side effects) into separate cards. - Markdown is allowed inside strings (bold, lists, tables). Write line breaks inside strings as \\n. - The output must be valid JSON: double quotes around every key and string, no trailing commas. - Base everything on the material below. Do not add facts that are not in it. diff --git a/tests/ai.test.ts b/tests/ai.test.ts index 30107de..6849034 100644 --- a/tests/ai.test.ts +++ b/tests/ai.test.ts @@ -54,6 +54,7 @@ async function sse(url: string, body: unknown): Promise { const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review'); /** The quick call that groups topic labels after a quiz is written. */ const isGrouping = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_topics'); +const isNaming = (r: { body: any }) => r.body?.tool_choice?.function?.name === 'name_set'; async function waitForDraft(id: string) { for (let i = 0; i < 200; i += 1) { @@ -201,7 +202,7 @@ describe('generation', () => { expect(draft.payload.questions[0].source).toMatch(/Heart failure lecture/); // 12 questions at 8 per batch means two model calls, each carrying the source excerpts. const all = mock.requests.filter((r) => r.path.endsWith('/chat/completions')); - const calls = all.filter((r) => !isCheck(r) && !isGrouping(r)); + const calls = all.filter((r) => !isCheck(r) && !isGrouping(r) && !isNaming(r)); expect(all.filter(isGrouping)).toHaveLength(1); expect(calls).toHaveLength(2); // Then the Python check: one run, a review, and its fix applied to question 2. @@ -251,7 +252,7 @@ describe('generation', () => { // Deck -> quiz conversion uses the cards as the source. const converted = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'quiz', classId: cls.id, source: { deckId } })).body.draftId); expect(converted.status).toBe('ready'); - expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r) && !isGrouping(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/); + expect(JSON.stringify(mock.requests.filter((r) => r.path.endsWith('/chat/completions') && !isCheck(r) && !isGrouping(r) && !isNaming(r)).at(-1)!.body.messages)).toMatch(/Flashcards from the deck/); // Append more cards to the existing deck. const more = await waitForDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, focus: 'beta blockers', count: 4 })).body.draftId); diff --git a/tests/mockAi.ts b/tests/mockAi.ts index 4993165..ee620e3 100644 --- a/tests/mockAi.ts +++ b/tests/mockAi.ts @@ -18,6 +18,8 @@ export interface MockAi { failNext: (status: number, times?: number) => void; /** Delay every chat call by this many milliseconds (0 turns it off). */ delayChat: (ms: number) => void; + /** The next N calls of this tool send arguments that are not valid JSON (beyond jsonrepair). */ + badJsonNext: (tool: string, times?: number) => void; close: () => Promise; } @@ -90,7 +92,9 @@ function toolArgs(name: string, userText: string, all: string): Record ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, hint: '', tags: ['mock'] })) }; + return { cards: Array.from({ length: count }, (_, i) => ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, tags: ['mock'] })) }; + case 'name_set': + return /quiz of/i.test(userText) ? { title: 'Heart Failure and Pharmacology', description: 'Covers heart failure care and the drugs used for it.' } : { title: 'Mock Terms', description: 'Covers the mock terms.' }; case 'generate_quiz': return { count: Math.min(count, 5), topic: 'heart failure', style: 'nclex', use_all_class_documents: /document/i.test(userText), from_weak_areas: /wrong|weak|miss/i.test(userText) }; case 'generate_flashcards': @@ -146,6 +150,7 @@ export function startMockAi(port = 0): Promise { const requests: MockAi['requests'] = []; let fail = { status: 0, times: 0 }; let delayMs = 0; + let badJson: { tool: string; times: number } | null = null; const server = http.createServer((req, res) => { const chunks: Buffer[] = []; @@ -198,7 +203,11 @@ export function startMockAi(port = 0): Promise { res.end(); }; const callTool = (name: string) => { - const args = JSON.stringify(toolArgs(name, userText, all)); + let args = JSON.stringify(toolArgs(name, userText, all)); + if (badJson && badJson.tool === name && badJson.times > 0) { + badJson.times -= 1; + args = '{"cards": [{"front": "Broken" "back": ]]]} :: not json'; + } send({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: `call_${Math.random().toString(36).slice(2, 10)}`, type: 'function', function: { name, arguments: '' } }] }); // Arguments arrive in fragments, like a real provider. for (let i = 0; i < args.length; i += 37) send({ tool_calls: [{ index: 0, function: { arguments: args.slice(i, i + 37) } }] }); @@ -265,6 +274,9 @@ export function startMockAi(port = 0): Promise { delayChat: (ms) => { delayMs = ms; }, + badJsonNext: (tool, times = 1) => { + badJson = { tool, times }; + }, close: () => new Promise((done) => server.close(() => done())), }); }); diff --git a/tests/revision4.test.ts b/tests/revision4.test.ts index 825ed17..72e81e1 100644 --- a/tests/revision4.test.ts +++ b/tests/revision4.test.ts @@ -110,7 +110,7 @@ describe('the assistant prompt', () => { const mcp: LoadedMcp = { tools: [], unavailable: [], servers: [{ id: 'web-search', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web' }] }; const chat = mcpPromptNote(mcp, { askBeforeWeb: true }); expect(chat).toContain('Should I look it up on the web?'); - expect(chat).toContain('do not answer from your own knowledge'); + expect(chat).toContain('never answer from your own knowledge'); const elsewhere = mcpPromptNote(mcp); expect(elsewhere).not.toContain('Should I look it up on the web?'); expect(elsewhere).toContain("anything the student's own material does not cover"); @@ -229,7 +229,9 @@ describe('share-link visitors can generate', () => { expect((await t.api('GET', '/api/drafts')).body.drafts.some((d: any) => d.id === set.id)).toBe(false); const saved = (await t.api('GET', `/api/quizzes/${set.itemId}`)).body; expect(saved.quiz.folderId).toBe(exam.id); - expect(saved.quiz.description).toBe('Made with AI on a share link.'); + // Named and described by the model from what it covers. + expect(saved.quiz.title).toBe('Heart Failure and Pharmacology'); + expect(saved.quiz.description).toBe('Covers heart failure care and the drugs used for it.'); const shown = (await t.api('GET', `/api/public/${token}`)).body.items.map((i: any) => i.id); expect(shown).toContain(set.itemId); const quiz = (await t.api('GET', `/api/public/${token}/items/quiz/${set.itemId}`)).body; diff --git a/tests/revision5.test.ts b/tests/revision5.test.ts new file mode 100644 index 0000000..876800d --- /dev/null +++ b/tests/revision5.test.ts @@ -0,0 +1,253 @@ +/** + * Fifth revision: malformed model output is repaired or retried (and a whole job restarted + * once), failed share-link sets can be dismissed by visitors, generated sets get their own + * names, flashcards are atomic without hints, the assistant never answers from general + * knowledge, and share-link AI expires after a week, is limited to a region and is locked by + * abuse protection. + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { mcpPromptNote, type LoadedMcp } from '../server/ai/mcp.ts'; +import { MIGRATIONS } from '../server/migrations.ts'; +import { queueIdle } from '../server/services/documents.ts'; +import { isPrivateAddress, resetGeoForTests } from '../server/services/geo.ts'; +import { createTestApp, type TestApp } from './helpers.ts'; +import { startMockAi, type MockAi } from './mockAi.ts'; + +let mock: MockAi; +let t: TestApp; + +beforeAll(async () => { + mock = await startMockAi(); +}); +afterAll(async () => { + await mock.close(); +}); +beforeEach(async () => { + t = await createTestApp(); + mock.requests.length = 0; + mock.delayChat(0); + mock.badJsonNext('', 0); + const { resetShareAiLimits } = await import('../server/routes/shares.ts'); + resetShareAiLimits(); + resetGeoForTests(); +}); +afterEach(() => t.close()); + +async function connectAi() { + const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider; + const models = (await t.api('POST', `/api/ai/providers/${provider.id}/discover`)).body.models as any[]; + for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { enabled: true, capabilities: { tools: true } }); +} + +async function classWithMaterial() { + const cls = (await t.api('POST', '/api/classes', { name: 'Pharm' })).body.class; + const doc = (await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Heart failure', text: '# Heart failure\n\nFurosemide is a loop diuretic. Monitor potassium and daily weights.' })).body.document; + await queueIdle(t.ctx); + return { cls, doc }; +} + +async function waitDraft(id: string) { + for (let i = 0; i < 400; i += 1) { + const d = (await t.api('GET', `/api/drafts/${id}`)).body.draft; + if (d.status !== 'generating') return d; + await new Promise((r) => setTimeout(r, 25)); + } + throw new Error('generation did not finish'); +} + +const chatCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions')); +const isRepair = (r: { body: any }) => JSON.stringify(r.body.messages ?? []).includes('You repair malformed JSON'); + +describe('malformed model output', () => { + it('is sent back to the model for repair instead of failing', async () => { + await connectAi(); + const { cls } = await classWithMaterial(); + mock.badJsonNext('submit_flashcards', 1); + const started = await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 }); + const draft = await waitDraft(started.body.draftId); + expect(draft.status).toBe('ready'); + expect(draft.payload.cards).toHaveLength(5); + expect(chatCalls().filter(isRepair)).toHaveLength(1); + }); + + it('starts the whole job over once when repair and a retry both fail, then fails for good', async () => { + await connectAi(); + const { cls } = await classWithMaterial(); + // Original, repair and retry all broken: the job starts over and succeeds. + mock.badJsonNext('submit_flashcards', 3); + let draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId); + expect(draft.status).toBe('ready'); + expect(draft.payload.cards).toHaveLength(5); + + // Broken on the second run too: failed, with a readable error. + mock.badJsonNext('submit_flashcards', 6); + draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 5 })).body.draftId); + expect(draft.status).toBe('failed'); + expect(draft.error).toMatch(/not valid JSON/); + }); +}); + +describe('generated sets', () => { + it('get their own name and description, and flashcards are atomic without hints', async () => { + await connectAi(); + const { cls } = await classWithMaterial(); + const draft = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); + expect(draft.payload.title).toBe('Mock Terms'); + expect(draft.payload.description).toBe('Covers the mock terms.'); + const writing = chatCalls().find((r) => r.body.tool_choice?.function?.name === 'submit_flashcards')!; + const tool = writing.body.tools.find((x: any) => x.function.name === 'submit_flashcards'); + expect(Object.keys(tool.function.parameters.properties.cards.items.properties)).not.toContain('hint'); + expect(writing.body.messages[0].content).toMatch(/Atomic cards/); + // The same name twice in a class gets a number. + await t.api('POST', `/api/drafts/${draft.id}/save`, {}); + const second = await waitDraft((await t.api('POST', '/api/ai/generate', { kind: 'deck', classId: cls.id, source: { allClassDocuments: true }, count: 4 })).body.draftId); + expect(second.payload.title).toBe('Mock Terms 2'); + }); +}); + +describe('the assistant', () => { + it('never offers to answer from general knowledge', () => { + const mcp: LoadedMcp = { tools: [], servers: [{ id: 'web', name: 'Web search', instructions: '', prefix: 'web_', builtin: 'web', research: false }], unavailable: [] }; + const note = mcpPromptNote(mcp, { askBeforeWeb: true }); + expect(note).not.toMatch(/Answer from general knowledge/); + expect(note).toMatch(/never answer from your own knowledge/); + expect(note).toMatch(/do not have information/); + }); +}); + +/* ------------------------------ share links ------------------------------ */ + +async function sharedExam(ai: Record) { + const { cls, doc } = await classWithMaterial(); + const exam = (await t.api('POST', `/api/classes/${cls.id}/folders`, { name: 'Exam 1' })).body.folder; + await t.api('PATCH', `/api/documents/${doc.id}`, { folderId: exam.id }); + const share = (await t.api('POST', '/api/shares', { kind: 'folder', targetId: exam.id })).body.share; + const updated = (await t.api('PATCH', `/api/shares/${share.id}`, { ai })).body.share; + return { cls, doc, exam, share: updated, token: share.token as string }; +} + +const from = (ip: string, country?: string) => ({ headers: { 'x-forwarded-for': ip, ...(country ? { 'cf-ipcountry': country } : {}) } }); +const explain = (token: string, ip: string, country?: string) => t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'none', cardId: 'none' }, from(ip, country)); + +describe('failed share-link sets', () => { + it('can be dismissed by the visitor and stay in the owner drafts', async () => { + await connectAi(); + const { token } = await sharedExam({ generate: true }); + mock.badJsonNext('submit_flashcards', 6); + const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'deck', count: 5 }); + expect(started.status).toBe(202); + const id = started.body.set.id; + await waitDraft(id); + const sets = (await t.api('GET', `/api/public/${token}/generated`)).body.sets; + expect(sets.find((s: any) => s.id === id)?.status).toBe('failed'); + expect((await t.api('POST', `/api/public/${token}/generated/${id}/dismiss`)).status).toBe(200); + expect((await t.api('GET', `/api/public/${token}/generated`)).body.sets.some((s: any) => s.id === id)).toBe(false); + expect((await t.api('GET', '/api/drafts')).body.drafts.find((d: any) => d.id === id)?.status).toBe('failed'); + }); + + it('show visitors plain progress steps', async () => { + await connectAi(); + const { token } = await sharedExam({ generate: true }); + mock.delayChat(300); + const started = await t.api('POST', `/api/public/${token}/generate`, { kind: 'quiz', count: 4 }); + t.ctx.db.run('UPDATE drafts SET progress_label = ? WHERE id = ?', 'Checking the questions with Python (run 1)', started.body.set.id); + const set = (await t.api('GET', `/api/public/${token}/generated`)).body.sets.find((s: any) => s.id === started.body.set.id); + expect(set.progressLabel).toBe('Verifying questions'); + mock.delayChat(0); + await waitDraft(started.body.set.id); + }); +}); + +describe('share-link AI', () => { + it('turns itself off after a week and can be given another week', async () => { + await connectAi(); + const { share, token } = await sharedExam({ explain: true, generate: true }); + expect(share.aiActive).toBe(true); + expect(share.aiExpiresAt - Date.now()).toBeGreaterThan(6.9 * 86_400_000); + expect((await t.api('GET', `/api/public/${token}`)).body.ai.explain).toBe(true); + + t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id); + expect((await t.api('GET', `/api/public/${token}`)).body.ai).toEqual({ explain: false, assistant: false, generate: false }); + expect((await explain(token, '10.0.0.2')).status).toBe(403); + const lapsed = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id); + expect(lapsed.aiActive).toBe(false); + + const renewed = (await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share; + expect(renewed.aiActive).toBe(true); + expect(renewed.ai).toMatchObject({ explain: true, generate: true }); + expect((await explain(token, '10.0.0.2')).status).toBe(404); // allowed through; the card does not exist + + // Switching a feature on after a lapse turns on only that feature, for a fresh week. + t.ctx.db.run('UPDATE shares SET ai_expires_at = ? WHERE id = ?', Date.now() - 1000, share.id); + const fresh = (await t.api('PATCH', `/api/shares/${share.id}`, { ai: { assistant: true } })).body.share; + expect(fresh.ai).toEqual({ explain: false, assistant: true, generate: false }); + expect(fresh.aiActive).toBe(true); + }); + + it('works only from the allowed region, failing closed when a visitor cannot be placed', async () => { + await connectAi(); + const { token } = await sharedExam({ explain: true }); + expect((await t.api('PUT', '/api/share-settings/ai', { region: 'us' })).status).toBe(200); + + // Outside the US: the page works without AI, and the API refuses. + const page = await t.api('GET', `/api/public/${token}`, undefined, from('81.2.69.142', 'GB')); + expect(page.status).toBe(200); + expect(page.body.ai.explain).toBe(false); + const refused = await explain(token, '81.2.69.142', 'GB'); + expect(refused.status).toBe(403); + expect(refused.body.error.code).toBe('ai_region'); + + expect((await explain(token, '8.8.8.8', 'US')).status).toBe(404); + // The local network counts as home; an unplaceable public address does not. + expect((await explain(token, '192.168.1.20')).status).toBe(404); + expect((await explain(token, '8.8.4.4')).status).toBe(403); + + // Arizona: the state has to match too. + await t.api('PUT', '/api/share-settings/ai', { region: 'arizona' }); + expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'CA' } })).status).toBe(403); + expect((await t.api('POST', `/api/public/${token}/explain`, { kind: 'card', deckId: 'x', cardId: 'x' }, { headers: { 'x-forwarded-for': '8.8.8.8', 'cf-ipcountry': 'US', 'cf-region-code': 'AZ' } })).status).toBe(404); + + const status = (await t.api('GET', '/api/share-settings/ai')).body; + expect(status.geo.region).toBe('arizona'); + expect(status.geo.lastDay.blocked).toBeGreaterThan(0); + }); + + it('is locked by abuse protection when dozens of networks use one link', async () => { + await connectAi(); + const { share, token } = await sharedExam({ explain: true }); + for (let i = 1; i < 25; i += 1) expect((await explain(token, `9.9.9.${i}`)).status).toBe(404); + expect((await explain(token, '9.9.9.200')).status).toBe(403); + // Stays off for everyone, and the owner sees why. + expect((await explain(token, '9.9.9.1')).status).toBe(403); + const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id); + expect(owner.aiActive).toBe(false); + expect(owner.aiBlocked.reason).toMatch(/25 different networks/); + // Turning it back on lifts the lock. + expect((await t.api('POST', `/api/shares/${share.id}/ai/renew`)).body.share.aiActive).toBe(true); + expect((await explain(token, '9.9.9.1')).status).toBe(404); + }); + + it('is locked by a flood of requests or repeated requests from outside the region', async () => { + await connectAi(); + const { share, token } = await sharedExam({ explain: true }); + await t.api('PUT', '/api/share-settings/ai', { region: 'us' }); + for (let i = 0; i < 9; i += 1) expect((await explain(token, '81.2.69.142', 'GB')).status).toBe(403); + expect((await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id).aiActive).toBe(true); + await explain(token, '81.2.69.142', 'GB'); + const owner = (await t.api('GET', '/api/shares')).body.shares.find((s: any) => s.id === share.id); + expect(owner.aiBlocked.reason).toMatch(/outside the allowed region/); + }); +}); + +describe('location helpers', () => { + it('recognise private and loopback addresses', () => { + for (const ip of ['10.1.2.3', '192.168.0.5', '172.20.1.1', '127.0.0.1', '::1', 'fd00::1', '::ffff:192.168.1.2']) expect(isPrivateAddress(ip)).toBe(true); + for (const ip of ['8.8.8.8', '172.32.0.1', '2001:4860:4860::8888']) expect(isPrivateAddress(ip)).toBe(false); + }); + + it('migration 8 adds the share AI columns', () => { + expect(MIGRATIONS.find((m) => m.id === 8)?.name).toBe('share-ai-safety'); + const cols = t.ctx.db.all<{ name: string }>('PRAGMA table_info(shares)').map((c) => c.name); + expect(cols).toEqual(expect.arrayContaining(['ai_expires_at', 'ai_blocked'])); + }); +}); diff --git a/web/src/app/Shell.tsx b/web/src/app/Shell.tsx index ab5374c..1a6b4ac 100644 --- a/web/src/app/Shell.tsx +++ b/web/src/app/Shell.tsx @@ -61,7 +61,6 @@ export function ClassDialog({ open, onOpenChange, existing }: { open: boolean; o open={open} onOpenChange={onOpenChange} title={existing ? 'Edit class' : 'New class'} - description={existing ? undefined : 'A class holds the quizzes, flashcards, notes and class materials for one course.'} footer={ <>