diff --git a/PROGRESS.md b/PROGRESS.md index 1c97dd0..6771dd3 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -183,6 +183,10 @@ Speed (timed live on a 50-question quiz): output limits now leave room for reaso four at a time without overlapping, replacements are planned and parallel too, and the two reviews run side by side. 50 questions: 7 min 21 s, 0 duplicates, all checks passed. +Settings → AI now has separate Planning, Writing, Python check and Second review rows under +Content generation (each inherits until set), and shared quizzes say what Practice and Exam +mean under the mode toggle. + Tests: `tests/revision5.test.ts`. Live check on the rev1 copy with DeepSeek flash: a visitor's 10-card set was written, named ("Heparin, Furosemide, and Dosage Calculations"), saved and listed once with the New tag; cards were single facts with no hints. The rev1 copy's 401 was its @@ -191,7 +195,7 @@ stored DeepSeek connection still holding the old key; it was updated from `.env` ## Verification status Baseline: 162 tests passed, TypeScript passed and production web/server builds passed. -Added six regression tests. Full `npm run check` passes: **280 tests** (check loop and second review, 2026-11-16; 274 after the fifth revision; 261 after share-link generation, 2026-11-15; 257 after the fourth revision; 247 after the third, 208 after the second, 181 on 2026-11-12), TypeScript +Added six regression tests. Full `npm run check` passes: **281 tests** (generation steps, 2026-11-16; 280 after the check loop and second review; 274 after the fifth revision; 261 after share-link generation, 2026-11-15; 257 after the fourth revision; 247 after the third, 208 after the second, 181 on 2026-11-12), TypeScript and both production builds. The Docker `test` stage runs the same 181 tests on Node 24. Tests were run with loopback access for the mock provider. The earlier agent manually exercised quiz completion/results, flashcards, streak effects, diff --git a/docs/AI.md b/docs/AI.md index 45a8f34..29630ad 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -105,6 +105,12 @@ has no calculations. Topics are kept broad ("Stroke", not "Right hemisphere stro prompt lists the class's existing topics, and narrower labels that contain an existing one are merged into it after generation. +Settings → AI → "Which model does what" lists four steps under Content generation: Planning, +Writing (quiz, flashcard and study-guide writing, and improving questions), Python check and +Second review (`plan`, `write`, `check`, `review` tasks). Each can have its own model, +reasoning effort, temperature and output limit; anything left unset uses Content generation's +choice, then the default model's. Topic grouping and naming use Content generation directly. + Speed: a quiz of more than one batch is planned first. One call lists a distinct angle per question (client situation plus the single decision or fact tested), then the batches of 8 run four at a time (`WRITE_WAVE`), each writing its own angles from its own excerpts, so diff --git a/server/ai/generate.ts b/server/ai/generate.ts index b34f738..fcb82a3 100644 --- a/server/ai/generate.ts +++ b/server/ai/generate.ts @@ -778,7 +778,7 @@ export function startGeneration( if (!folder || (req.classId && folder.class_id !== req.classId)) throw badRequest('That exam is not part of this class.'); } // Fail fast (before creating a draft) when no model is configured. - const resolved = resolveModel(app, 'generate', req.modelId); + const resolved = resolveModel(app, 'write', req.modelId); const draftId = createDraft(app.db, { kind: req.kind, classId: req.classId ?? null, @@ -856,7 +856,7 @@ async function runWithRestart(app: AppContext, draftId: string, req: GenerateReq async function runGeneration(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal): Promise { const { db } = app; - const resolved = resolveModel(app, 'generate', req.modelId); + const resolved = resolveModel(app, 'write', req.modelId); const window = resolved.model.contextWindow ?? 32_000; const budgetTokens = Math.max(2_500, Math.min(Math.floor(window * 0.4), 28_000)); setProgress(db, draftId, 0.03, 'Reading your material'); @@ -944,7 +944,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ const user = [math ? '' : sourceBlock(sources, index), ask.filter(Boolean).join('\n')].filter(Boolean).join('\n\n======\n\n'); const result = await generateStructured(app, { - task: 'generate', + task: 'write', purpose: isQuiz ? 'quiz generation' : 'flashcard generation', modelId: req.modelId, system, @@ -992,7 +992,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ if (!isQuiz || n <= 0) return []; try { const result = await generateStructured(app, { - task: 'generate', + task: 'plan', purpose: 'planning questions', modelId: req.modelId, system: `${baseSystem(db, req, hasSources)}\n\nYou plan NCLEX-style quizzes. List distinct angles, one per question: each names the client situation and the single decision or fact it tests, in one line (for example "Continuous IV heparin, aPTT 3x control with gum bleeding: first action"). No two angles may test the same decision, fact or teaching point, even in a different scenario. Spread them over the client-needs areas the material supports (pharmacology, safety, prioritisation, delegation, teaching, lab values, complications, psychosocial). When the material is short, vary the situations and the area tested rather than repeating its points.`, @@ -1016,7 +1016,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ toolName: 'plan_questions', toolDescription: 'Submit the planned angles.', schema: { type: 'object', properties: { angles: { type: 'array', items: { type: 'string' } } }, required: ['angles'] }, - maxOutputTokens: outputBudget(resolved, 8_000), + maxOutputTokens: outputBudget(resolveModel(app, 'plan', req.modelId), 8_000), signal, }); const list = (result.data as { angles?: unknown }).angles; @@ -1271,7 +1271,7 @@ export async function nameSet( async function generateNote(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal, budgetTokens: number, window: number): Promise { const { db } = app; - const resolved = resolveModel(app, 'generate', req.modelId); + const resolved = resolveModel(app, 'write', req.modelId); const style = req.noteStyle ?? 'study_guide'; // A guide reads the material in as few passes as possible; allow a bigger slice per pass than item generation. const perPass = Math.max(budgetTokens, Math.min(Math.floor(window * 0.55), 90_000)); @@ -1304,7 +1304,7 @@ async function generateNote(app: AppContext, draftId: string, req: GenerateReque if (passes.length <= 1) { setProgress(db, draftId, 0.2, 'Writing'); const body = [passes[0]?.length ? `SOURCE EXCERPTS:\n\n${formatExcerpts(passes[0])}` : '', sources.inline, ask].filter(Boolean).join('\n\n======\n\n'); - const res = await runChatWithMcp(app, { task: 'generate', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: outputBudget(resolved, 16_000), signal }, 'generate'); + const res = await runChatWithMcp(app, { task: 'write', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: outputBudget(resolved, 16_000), signal }, 'generate'); content = res.text; } else { // Long material: condense each part, then write the final piece from the condensed notes. @@ -1313,7 +1313,7 @@ async function generateNote(app: AppContext, draftId: string, req: GenerateReque if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.'); setProgress(db, draftId, 0.05 + (0.7 * i) / passes.length, `Reading part ${i + 1} of ${passes.length}`); const res = await runChat(app, { - task: 'generate', + task: 'write', purpose: 'study guide', modelId: req.modelId, system: `${baseSystem(db, req, true)}\n\nCondense this part of the material into detailed notes: every key concept, value, medication, cause, sign, intervention and warning, as Markdown bullet points under short headings. Keep the bracketed excerpt ids (e.g. [c42]) next to the points they support. Do not add outside information.`, @@ -1327,7 +1327,7 @@ async function generateNote(app: AppContext, draftId: string, req: GenerateReque const res = await runChatWithMcp( app, { - task: 'generate', + task: 'write', purpose: 'study guide', modelId: req.modelId, system, @@ -1387,7 +1387,7 @@ export function startImprove(app: AppContext, input: z.input req.questionIds!.includes(q.id)); if (questions.length === 0) throw badRequest('No questions to improve'); questions = questions.slice(0, 60); - const resolved = resolveModel(app, 'generate', req.modelId); + const resolved = resolveModel(app, 'write', req.modelId); const draftId = createDraft(app.db, { kind: 'quiz', classId: quiz.class_id, @@ -1420,7 +1420,7 @@ Return the questions in the same order, one output question per input question.$ ) .join('\n\n'); const result = await generateStructured(app, { - task: 'generate', + task: 'write', purpose: 'improve questions', modelId: req.modelId, system, diff --git a/server/ai/service.ts b/server/ai/service.ts index d1640c1..2b89c32 100644 --- a/server/ai/service.ts +++ b/server/ai/service.ts @@ -64,12 +64,20 @@ interface ModelRow { price_cached_in: number | null; } -export const TASKS = ['chat', 'generate', 'vision', 'quick'] as const; +export const TASKS = ['chat', 'generate', 'plan', 'write', 'check', 'review', 'vision', 'quick'] as const; export type TaskName = (typeof TASKS)[number]; -export const TASK_INFO: Record = { +/** + * Tasks, and the steps of content generation. A step with nothing set uses Content generation's + * model and parameters (then the default's), so the steps only matter once the user sets them. + */ +export const TASK_INFO: Record = { chat: { label: 'Assistant', description: 'The assistant that searches your library, answers questions and runs tools.' }, - generate: { label: 'Content generation', description: 'Writing quizzes, flashcards and study guides. Benefits most from a strong model.' }, + generate: { label: 'Content generation', description: 'Quizzes, flashcards and study guides. Each step below can have its own settings.' }, + plan: { label: 'Planning', description: 'Lists a distinct angle for every question before writing.', parent: 'generate' }, + write: { label: 'Writing', description: 'Writes the questions, flashcards and study guides.', parent: 'generate' }, + check: { label: 'Python check', description: 'Checks counts, answer keys and calculations with Python.', parent: 'generate' }, + review: { label: 'Second review', description: 'Reads every question for badly written items.', parent: 'generate' }, vision: { label: 'Vision and OCR', description: 'Reading scanned PDFs and images. Needs a model that accepts images.' }, quick: { label: 'Quick explanations', description: 'Short explanations while studying. A fast, inexpensive model works well.' }, }; @@ -379,7 +387,7 @@ export function getTaskSettings(db: DB): TaskSettings { } export function saveTaskSettings(db: DB, input: unknown): TaskSettings { - const schema = z.object({ default: taskConfigSchema, chat: taskConfigSchema, generate: taskConfigSchema, vision: taskConfigSchema, quick: taskConfigSchema }).partial(); + const schema = z.object(Object.fromEntries(['default', ...TASKS].map((t) => [t, taskConfigSchema])) as Record).partial(); const patch = schema.parse(input); const next = { ...getTaskSettings(db), ...patch } as TaskSettings; setSetting(db, 'ai.tasks', next); @@ -399,7 +407,10 @@ export interface ResolvedModel { export function resolveModel(app: AppContext, task: TaskName, overrideModelId?: string | null): ResolvedModel { const tasks = getTaskSettings(app.db); const cfg = tasks[task]; - const candidates = [overrideModelId, cfg.modelId, tasks.default.modelId].filter((x): x is string => Boolean(x)); + // A generation step falls back to Content generation, then to the default. + const parent = TASK_INFO[task].parent; + const chain = parent ? [cfg, tasks[parent]] : [cfg]; + const candidates = [overrideModelId, ...chain.map((c) => c.modelId), tasks.default.modelId].filter((x): x is string => Boolean(x)); let model: ModelDto | undefined; for (const id of candidates) { const row = app.db.get(`${MODEL_SELECT} WHERE m.id = ? AND p.enabled = 1`, id); @@ -420,12 +431,17 @@ export function resolveModel(app: AppContext, task: TaskName, overrideModelId?: ); } const provider = providerConfig(app, providerRow(app.db, model.providerId)); - const own = cfg.modelId === model.id || (!cfg.modelId && tasks.default.modelId === model.id) ? cfg : EMPTY_TASK; + // A level's parameters apply when it chose this model, or chose none and inherits this model. + const applies = chain.filter((level, i) => { + const inherited = [...chain.slice(i + 1), tasks.default].find((l) => l.modelId)?.modelId ?? null; + return level.modelId ? level.modelId === model.id : inherited === model.id; + }); const fallback = tasks.default; + const pick = (key: K): TaskConfig[K] => applies.map((l) => l[key]).find((v) => v !== null) ?? fallback[key]; const params: ResolvedModel['params'] = {}; - const temperature = own.temperature ?? fallback.temperature; - const effort = own.reasoningEffort ?? fallback.reasoningEffort; - const maxOut = own.maxOutputTokens ?? fallback.maxOutputTokens; + const temperature = pick('temperature'); + const effort = pick('reasoningEffort'); + const maxOut = pick('maxOutputTokens'); if (temperature !== null && model.capabilities.temperature) params.temperature = temperature; if (effort !== null && model.capabilities.reasoning) params.reasoningEffort = effort; if (maxOut !== null) params.maxOutputTokens = maxOut; diff --git a/server/ai/verify.ts b/server/ai/verify.ts index 6c92b72..eda33be 100644 --- a/server/ai/verify.ts +++ b/server/ai/verify.ts @@ -274,7 +274,7 @@ export async function verifyQuizDraft( req: QuizRequestFacts, opts: { modelId?: string | null; signal?: AbortSignal; onStep?: (label: string) => void } = {}, ): Promise { - const resolved = resolveModel(app, 'generate', opts.modelId); + const resolved = resolveModel(app, 'check', opts.modelId); if (!resolved.model.capabilities.tools) { return { draft, notes: ['The Python answer check was skipped: the generation model cannot use tools.'], review: null, pythonRuns: 0, failed: [], applied: [], refused: [], removed: 0, rewritten: 0, incomplete: true }; } @@ -306,7 +306,7 @@ export async function verifyQuizDraft( else messages.push({ role: 'user', content: [nudge] }); } const res = await runChat(app, { - task: 'generate', + task: 'check', purpose: 'quiz check', modelId: opts.modelId, system, @@ -474,7 +474,7 @@ export async function proofreadQuiz( opts: { modelId?: string | null; signal?: AbortSignal } = {}, ): Promise<{ draft: QuizDraft; flagged: number; fixes: QuizFix[]; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> { const none = { draft, flagged: 0, fixes: [], applied: [], refused: [], rewritten: 0, removed: 0 }; - const resolved = resolveModel(app, 'generate', opts.modelId); + const resolved = resolveModel(app, 'review', opts.modelId); if (!resolved.model.capabilities.tools) return { ...none, incomplete: true }; const data = quizForPython(draft); const listing = data.questions @@ -484,7 +484,7 @@ export async function proofreadQuiz( const prompt = `${hints.length ? `Wording that looks like a leftover or a change of mind (confirm and fix each):\n${hints.map((h) => `- ${h}`).join('\n')}\n\n` : ''}THE QUIZ (${data.question_count} questions):\n\n${listing}`; try { const res = await runChat(app, { - task: 'generate', + task: 'review', purpose: 'quiz second review', modelId: opts.modelId, system: PROOFREAD_SYSTEM, diff --git a/tests/revision5.test.ts b/tests/revision5.test.ts index 269805f..307f45d 100644 --- a/tests/revision5.test.ts +++ b/tests/revision5.test.ts @@ -193,6 +193,33 @@ describe('the quiz check', () => { }); }); +describe('generation steps in Settings', () => { + it('use Content generation settings until a step has its own', async () => { + await connectAi(); + const { resolveModel } = await import('../server/ai/service.ts'); + const models = (await t.api('GET', '/api/ai/models')).body.models.filter((m: any) => m.enabled); + for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { capabilities: { reasoning: true } }); + const [a, b] = models; + const none = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null }; + await t.api('PUT', '/api/ai/tasks', { default: { ...none, modelId: a.id }, generate: { ...none, reasoningEffort: 'high' } }); + // Nothing set on the steps: they follow Content generation. + for (const step of ['plan', 'write', 'check', 'review'] as const) expect(resolveModel(t.ctx, step).params.reasoningEffort).toBe('high'); + // A step's own effort wins; the others keep following. + await t.api('PUT', '/api/ai/tasks', { plan: { ...none, reasoningEffort: 'low' } }); + expect(resolveModel(t.ctx, 'plan').params.reasoningEffort).toBe('low'); + expect(resolveModel(t.ctx, 'write').params.reasoningEffort).toBe('high'); + // A step with its own model does not take Content generation's parameters for it. + if (b) { + await t.api('PUT', '/api/ai/tasks', { review: { ...none, modelId: b.id } }); + const review = resolveModel(t.ctx, 'review'); + expect(review.model.id).toBe(b.id); + expect(review.params.reasoningEffort).toBeUndefined(); + } + const meta = (await t.api('GET', '/api/ai/meta')).body.tasks; + expect(meta.plan).toMatchObject({ label: 'Planning', parent: 'generate' }); + }); +}); + describe('generated sets', () => { it('get their own name and description, and flashcards are atomic without hints', async () => { await connectAi(); diff --git a/web/src/features/AiSettings.tsx b/web/src/features/AiSettings.tsx index d78ff82..9fee796 100644 --- a/web/src/features/AiSettings.tsx +++ b/web/src/features/AiSettings.tsx @@ -12,7 +12,7 @@ import { Badge, Button, Callout, Dialog, EmptyState, Field, IconButton, Input, M interface Meta { adapters: AdapterMeta[]; - tasks: Record; + tasks: Record; catalogUpdated: string; } @@ -495,7 +495,27 @@ function ProviderCard({ provider, models, adapter, onEdit, reload }: { provider: /* ------------------------------ tasks ------------------------------ */ -function TaskRow({ name, label, description, config, models, resolved, onChange }: { name: TaskName | 'default'; label: string; description?: string; config: TaskConfig; models: AiModel[]; resolved?: string; onChange: (c: TaskConfig) => void }) { +function TaskRow({ + name, + label, + description, + config, + models, + resolved, + inherits, + onChange, +}: { + name: TaskName | 'default'; + label: string; + description?: string; + config: TaskConfig; + models: AiModel[]; + resolved?: string; + /** A step of another task: unset choices use that task's (e.g. "Content generation"). */ + inherits?: string; + onChange: (c: TaskConfig) => void; +}) { + const same = inherits ? `Same as ${inherits.toLowerCase()}` : null; const model = models.find((m) => m.id === config.modelId); const hasOverrides = config.temperature !== null || config.reasoningEffort !== null || config.maxOutputTokens !== null; const byProvider = useMemo(() => { @@ -504,14 +524,14 @@ function TaskRow({ name, label, description, config, models, resolved, onChange return [...map.entries()]; }, [models]); return ( -
+
{label}
{description &&
{description}
}
onChange({ ...config, reasoningEffort: (e.target.value || null) as TaskConfig['reasoningEffort'] })}> - + {REASONING_EFFORTS.map((e) => (
@@ -726,9 +746,24 @@ export default function AiSettings() {
saveTask('default', c)} /> - {(Object.keys(meta.data.tasks) as TaskName[]).map((t) => ( - saveTask(t, c)} /> - ))} + {(Object.keys(meta.data.tasks) as TaskName[]) + .filter((t) => tasks.data![t]) + .map((t) => { + const parent = meta.data!.tasks[t].parent; + return ( + saveTask(t, c)} + /> + ); + })}
)} diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 4347522..6d9418e 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -447,7 +447,8 @@ export interface AiModel { /** USD per million cached input tokens; null = the provider's usual cache discount. */ priceCachedIn: number | null; } -export type TaskName = 'chat' | 'generate' | 'vision' | 'quick'; +/** Tasks, and the steps of content generation (plan, write, check, review). */ +export type TaskName = 'chat' | 'generate' | 'plan' | 'write' | 'check' | 'review' | 'vision' | 'quick'; /** Reasoning effort levels, lowest to highest. Each provider maps them onto what it accepts. */ export const REASONING_EFFORTS = [ { value: 'none', label: 'Off' }, diff --git a/web/src/pages/SharedPage.tsx b/web/src/pages/SharedPage.tsx index 1c645e0..035c984 100644 --- a/web/src/pages/SharedPage.tsx +++ b/web/src/pages/SharedPage.tsx @@ -263,6 +263,9 @@ function SharedQuiz({ quiz, storageKey, back, explainUrl, onImmersive }: { quiz: ]} className="w-full" /> +

+ {prefs.mode === 'practice' ? 'Practice: see the answer and rationale right after each question.' : 'Exam: no answers along the way; submit at the end for your score and every rationale.'} +

setPrefs((p) => ({ ...p, shuffleQ: v }))} label="Shuffle questions" /> setPrefs((p) => ({ ...p, shuffleO: v }))} label="Shuffle answer choices" />