Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

431
server/services/attempts.ts Normal file
View file

@ -0,0 +1,431 @@
/**
* Quiz sessions. An attempt freezes its questions (and their shuffled answer order) in a
* snapshot when it starts, so editing or deleting the quiz later can never corrupt a session
* in progress or the review of a finished one.
*/
import { z } from 'zod';
import { isFullyCorrect, scoreQuestion, seededShuffle, type Option, type QuestionType, type SataScoring } from '../../shared/content.ts';
import type { DB } from '../db.ts';
import { badRequest, notFound } from '../http.ts';
import { newId, now, parseJson } from '../util.ts';
import { getClass, getQuiz, mapQuestion, type QuestionRow } from './content.ts';
export interface SnapshotQuestion {
id: string;
quizId: string | null;
quizTitle: string;
type: QuestionType;
stem: string;
options: Option[];
rationale: string;
topic: string;
difficulty: string;
}
export interface AnswerState {
selected: string[];
/** Practice mode: the answer has been checked and feedback shown. */
checked: boolean;
at: number;
}
export interface AttemptSettings {
shuffleQuestions: boolean;
shuffleOptions: boolean;
timeLimitSec: number | null;
sataScoring: SataScoring;
}
interface AttemptRow {
id: string;
quiz_id: string | null;
class_id: string | null;
title: string;
mode: 'practice' | 'exam';
kind: string;
status: 'active' | 'completed';
settings: string;
snapshot: string;
answers: string;
flags: string;
current_index: number;
elapsed_sec: number;
question_count: number;
score: number | null;
max_score: number | null;
started_at: number;
updated_at: number;
completed_at: number | null;
}
export const attemptCreateSchema = z.object({
mode: z.enum(['practice', 'exam']).default('practice'),
source: z.discriminatedUnion('type', [
z.object({ type: z.literal('quiz'), quizId: z.string() }),
/** Questions missed in one finished attempt. */
z.object({ type: z.literal('retake'), attemptId: z.string() }),
/** Questions answered wrong most often, across a class or everything. */
z.object({ type: z.literal('missed'), classId: z.string().optional(), quizId: z.string().optional() }),
/** Every question on the given topics within a class. */
z.object({ type: z.literal('topics'), classId: z.string(), topics: z.array(z.string()).min(1).max(50) }),
z.object({ type: z.literal('questions'), questionIds: z.array(z.string()).min(1).max(1000), title: z.string().max(200).optional() }),
]),
shuffleQuestions: z.boolean().default(true),
shuffleOptions: z.boolean().default(true),
count: z.number().int().min(1).max(1000).optional(),
timeLimitMin: z.number().min(1).max(600).nullish(),
});
export type AttemptCreate = z.input<typeof attemptCreateSchema>;
const PINNED_OPTION = /^(all|none|both|neither)\b.*\b(above|these|options|answers)\b|^(a|b|c|d)\s*(and|&)\s*(a|b|c|d)\b/i;
/** Shuffles options but keeps "All of the above"-style choices at the end. */
function shuffleOptions(options: Option[], seed: number): Option[] {
const pinned = options.filter((o) => PINNED_OPTION.test(o.text.trim()));
const free = options.filter((o) => !PINNED_OPTION.test(o.text.trim()));
return [...seededShuffle(free, seed), ...pinned];
}
function loadQuestionRows(db: DB, ids: string[]): (QuestionRow & { quiz_title: string })[] {
if (ids.length === 0) return [];
const out: (QuestionRow & { quiz_title: string })[] = [];
for (let i = 0; i < ids.length; i += 400) {
const batch = ids.slice(i, i + 400);
out.push(
...db.all<QuestionRow & { quiz_title: string }>(
`SELECT q.*, z.title AS quiz_title FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE q.id IN (${batch.map(() => '?').join(',')})`,
...batch,
),
);
}
const order = new Map(ids.map((id, i) => [id, i]));
return out.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
}
/** Question ids ordered from most to least troublesome, using recent responses. */
export function missedQuestionIds(db: DB, scope: { classId?: string; quizId?: string }, limit = 200): string[] {
const where = scope.quizId ? 'AND r.quiz_id = ?' : scope.classId ? 'AND r.class_id = ?' : '';
const params = scope.quizId ? [scope.quizId] : scope.classId ? [scope.classId] : [];
const rows = db.all<{ question_id: string }>(
`SELECT r.question_id, COUNT(*) AS seen, SUM(CASE WHEN r.correct = 0 THEN 1 ELSE 0 END) AS wrong,
(SELECT r2.correct FROM responses r2 WHERE r2.question_id = r.question_id ORDER BY r2.answered_at DESC LIMIT 1) AS last_correct
FROM responses r JOIN questions q ON q.id = r.question_id
WHERE 1 = 1 ${where}
GROUP BY r.question_id
HAVING wrong > 0 AND (last_correct = 0 OR wrong * 2 >= seen)
ORDER BY last_correct ASC, (wrong * 1.0 / seen) DESC, wrong DESC
LIMIT ?`,
...params,
limit,
);
return rows.map((r) => r.question_id);
}
export function createAttempt(db: DB, input: AttemptCreate, sataScoring: SataScoring): string {
const opts = attemptCreateSchema.parse(input);
const src = opts.source;
let quizId: string | null = null;
let classId: string | null = null;
let title = '';
let kind = 'full';
let rows: (QuestionRow & { quiz_title: string })[] = [];
let frozen: SnapshotQuestion[] | null = null;
if (src.type === 'quiz') {
const quiz = getQuiz(db, src.quizId);
quizId = quiz.id;
classId = quiz.class_id;
title = quiz.title;
rows = db
.all<QuestionRow>('SELECT * FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quiz.id)
.map((r) => ({ ...r, quiz_title: quiz.title }));
} else if (src.type === 'retake') {
const prev = db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', src.attemptId);
if (!prev) throw notFound('Attempt');
const snap = parseJson<{ questions: SnapshotQuestion[] }>(prev.snapshot, { questions: [] });
const answers = parseJson<Record<string, AnswerState>>(prev.answers, {});
const prevSettings = parseJson<AttemptSettings>(prev.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring });
const missed = snap.questions.filter((q) => !isFullyCorrect(scoreQuestion(q, answers[q.id]?.selected ?? [], prevSettings.sataScoring)));
const current = new Map(loadQuestionRows(db, missed.map((q) => q.id)).map((r) => [r.id, r]));
// Prefer the live question (it may have been corrected since); fall back to the frozen copy.
frozen = missed.map((q) => {
const live = current.get(q.id);
if (!live) return q;
const m = mapQuestion(live);
return { id: m.id, quizId: m.quizId, quizTitle: live.quiz_title, type: m.type, stem: m.stem, options: m.options, rationale: m.rationale, topic: m.topic, difficulty: m.difficulty };
});
quizId = prev.quiz_id;
classId = prev.class_id;
title = `${prev.title.replace(/ · missed$/, '')} · missed`;
kind = 'retake';
} else if (src.type === 'missed') {
const ids = missedQuestionIds(db, { classId: src.classId, quizId: src.quizId });
rows = loadQuestionRows(db, ids);
kind = 'missed';
if (src.quizId) {
const quiz = getQuiz(db, src.quizId);
quizId = quiz.id;
classId = quiz.class_id;
title = `${quiz.title} · missed`;
} else if (src.classId) {
classId = src.classId;
title = `${getClass(db, src.classId).name} · missed questions`;
} else {
title = 'Missed questions';
}
} else if (src.type === 'topics') {
const cls = getClass(db, src.classId);
classId = cls.id;
rows = db.all<QuestionRow & { quiz_title: string }>(
`SELECT q.*, z.title AS quiz_title FROM questions q JOIN quizzes z ON z.id = q.quiz_id
WHERE z.class_id = ? AND q.topic IN (${src.topics.map(() => '?').join(',')}) ORDER BY z.sort_order, q.sort_order`,
cls.id,
...src.topics,
);
title = src.topics.length === 1 ? `${src.topics[0]} · practice` : `${cls.name} · ${src.topics.length} topics`;
kind = 'topics';
} else {
rows = loadQuestionRows(db, src.questionIds);
title = src.title ?? 'Custom practice';
kind = 'custom';
const classes = new Set(rows.map((r) => db.get<{ class_id: string }>('SELECT class_id FROM quizzes WHERE id = ?', r.quiz_id)?.class_id));
if (classes.size === 1) classId = [...classes][0] ?? null;
}
let questions: SnapshotQuestion[] =
frozen ??
rows.map((r) => {
const m = mapQuestion(r);
return { id: m.id, quizId: m.quizId, quizTitle: r.quiz_title, type: m.type, stem: m.stem, options: m.options, rationale: m.rationale, topic: m.topic, difficulty: m.difficulty };
});
if (questions.length === 0) {
throw badRequest(src.type === 'quiz' ? 'This quiz has no questions yet' : src.type === 'missed' || src.type === 'retake' ? 'No missed questions to review' : 'No questions matched');
}
const seed = (Math.random() * 2 ** 32) >>> 0;
if (opts.shuffleQuestions) questions = seededShuffle(questions, seed);
if (opts.count && opts.count < questions.length) questions = questions.slice(0, opts.count);
if (opts.shuffleOptions) questions = questions.map((q, i) => ({ ...q, options: shuffleOptions(q.options, seed + i + 1) }));
const settings: AttemptSettings = {
shuffleQuestions: opts.shuffleQuestions,
shuffleOptions: opts.shuffleOptions,
timeLimitSec: opts.timeLimitMin ? Math.round(opts.timeLimitMin * 60) : null,
sataScoring,
};
const id = newId();
const at = now();
db.run(
`INSERT INTO attempts (id, quiz_id, class_id, title, mode, kind, status, settings, snapshot, question_count, started_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?)`,
id,
quizId,
classId,
title,
opts.mode,
kind,
JSON.stringify(settings),
JSON.stringify({ questions }),
questions.length,
at,
at,
);
return id;
}
function loadAttempt(db: DB, id: string): AttemptRow {
const row = db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', id);
if (!row) throw notFound('Session');
return row;
}
function recordResponse(db: DB, row: AttemptRow, q: SnapshotQuestion, selected: string[], sata: SataScoring, at: number): void {
const score = scoreQuestion(q, selected, sata);
db.run(
`INSERT INTO responses (attempt_id, question_id, quiz_id, class_id, topic, score, correct, answered_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(attempt_id, question_id) DO UPDATE SET score = excluded.score, correct = excluded.correct, answered_at = excluded.answered_at`,
row.id,
q.id,
q.quizId ?? row.quiz_id,
row.class_id,
q.topic,
score,
isFullyCorrect(score) ? 1 : 0,
at,
);
}
export function mapAttempt(row: AttemptRow) {
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const results =
row.status === 'completed'
? Object.fromEntries(
snapshot.questions.map((q) => {
const selected = answers[q.id]?.selected ?? [];
const score = scoreQuestion(q, selected, settings.sataScoring);
return [q.id, { score, correct: isFullyCorrect(score), answered: selected.length > 0 }];
}),
)
: null;
return {
id: row.id,
quizId: row.quiz_id,
classId: row.class_id,
title: row.title,
mode: row.mode,
kind: row.kind,
status: row.status,
settings,
questions: snapshot.questions,
answers,
flags: parseJson<string[]>(row.flags, []),
currentIndex: row.current_index,
elapsedSec: row.elapsed_sec,
score: row.score,
maxScore: row.max_score,
startedAt: row.started_at,
updatedAt: row.updated_at,
completedAt: row.completed_at,
results,
};
}
export type AttemptDto = ReturnType<typeof mapAttempt>;
export const mapAttemptSummary = (row: AttemptRow) => {
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
return {
id: row.id,
quizId: row.quiz_id,
classId: row.class_id,
title: row.title,
mode: row.mode,
kind: row.kind,
status: row.status,
questionCount: row.question_count,
answeredCount: Object.values(answers).filter((a) => a.selected.length > 0).length,
currentIndex: row.current_index,
elapsedSec: row.elapsed_sec,
score: row.score,
maxScore: row.max_score,
startedAt: row.started_at,
updatedAt: row.updated_at,
completedAt: row.completed_at,
};
};
export type AttemptSummary = ReturnType<typeof mapAttemptSummary>;
export function getAttempt(db: DB, id: string): AttemptDto {
return mapAttempt(loadAttempt(db, id));
}
export const attemptPatchSchema = z.object({
answers: z
.record(z.string(), z.object({ selected: z.array(z.string().max(40)).max(12), checked: z.boolean().optional().default(false) }))
.optional(),
flags: z.array(z.string()).max(1000).optional(),
currentIndex: z.number().int().min(0).optional(),
elapsedSec: z.number().int().min(0).max(1_000_000).optional(),
});
export function patchAttempt(db: DB, id: string, input: z.input<typeof attemptPatchSchema>): AttemptDto {
const patch = attemptPatchSchema.parse(input);
const row = loadAttempt(db, id);
if (row.status !== 'active') throw badRequest('This session is already finished');
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const at = now();
db.tx(() => {
if (patch.answers) {
const byId = new Map(snapshot.questions.map((q) => [q.id, q]));
for (const [qid, a] of Object.entries(patch.answers)) {
const q = byId.get(qid);
if (!q) continue;
const prev = answers[qid];
// A checked practice answer is final: feedback has been shown.
if (prev?.checked && row.mode === 'practice') continue;
const valid = new Set(q.options.map((o) => o.id));
const selected = [...new Set(a.selected.filter((s) => valid.has(s)))];
const checked = row.mode === 'practice' && a.checked && selected.length > 0;
answers[qid] = { selected, checked, at };
if (checked) recordResponse(db, row, q, selected, settings.sataScoring, at);
}
}
db.run(
'UPDATE attempts SET answers = ?, flags = ?, current_index = ?, elapsed_sec = ?, updated_at = ? WHERE id = ?',
JSON.stringify(answers),
patch.flags ? JSON.stringify(patch.flags) : row.flags,
patch.currentIndex !== undefined ? Math.min(patch.currentIndex, Math.max(0, snapshot.questions.length - 1)) : row.current_index,
patch.elapsedSec !== undefined ? Math.max(row.elapsed_sec, patch.elapsedSec) : row.elapsed_sec,
at,
id,
);
});
return getAttempt(db, id);
}
/**
* Finishes a session. Exams score every question (blank = wrong). Practice sessions that are
* ended early are scored on the questions actually answered.
*/
export function completeAttempt(db: DB, id: string, elapsedSec?: number): AttemptDto {
const row = loadAttempt(db, id);
if (row.status === 'completed') return mapAttempt(row);
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const at = now();
let score = 0;
let max = 0;
db.tx(() => {
for (const q of snapshot.questions) {
const selected = answers[q.id]?.selected ?? [];
const answered = selected.length > 0;
if (row.mode === 'practice' && !answered) continue;
max += 1;
score += scoreQuestion(q, selected, settings.sataScoring);
const already = row.mode === 'practice' && answers[q.id]?.checked;
if (!already) recordResponse(db, row, q, selected, settings.sataScoring, answers[q.id]?.at ?? at);
}
db.run(
`UPDATE attempts SET status = 'completed', score = ?, max_score = ?, completed_at = ?, updated_at = ?, elapsed_sec = ? WHERE id = ?`,
score,
max,
at,
at,
elapsedSec !== undefined ? Math.max(row.elapsed_sec, elapsedSec) : row.elapsed_sec,
id,
);
});
return getAttempt(db, id);
}
export function deleteAttempt(db: DB, id: string): void {
const row = loadAttempt(db, id);
db.run('DELETE FROM attempts WHERE id = ?', row.id);
}
export function listAttempts(db: DB, filter: { status?: 'active' | 'completed'; quizId?: string; classId?: string; limit?: number }): AttemptSummary[] {
const where: string[] = [];
const params: (string | number)[] = [];
if (filter.status) {
where.push('status = ?');
params.push(filter.status);
}
if (filter.quizId) {
where.push('quiz_id = ?');
params.push(filter.quizId);
}
if (filter.classId) {
where.push('class_id = ?');
params.push(filter.classId);
}
const rows = db.all<AttemptRow>(
`SELECT * FROM attempts ${where.length ? `WHERE ${where.join(' AND ')}` : ''} ORDER BY COALESCE(completed_at, updated_at) DESC LIMIT ?`,
...params,
Math.min(filter.limit ?? 50, 500),
);
return rows.map(mapAttemptSummary);
}

518
server/services/content.ts Normal file
View file

@ -0,0 +1,518 @@
/**
* Classes, folders, quizzes, questions, decks, cards and notes.
* Used by the HTTP routes, the importer and the AI tools, so every write path
* shares one set of validation and ordering rules.
*/
import { z } from 'zod';
import {
CLASS_COLORS,
cardInputSchema,
questionInputSchema,
type Card,
type Difficulty,
type DraftCard,
type DraftQuestion,
type Option,
type Question,
type QuestionType,
} from '../../shared/content.ts';
import type { DB } from '../db.ts';
import { notFound } from '../http.ts';
import { newId, now, parseJson } from '../util.ts';
/* ------------------------------ row types ------------------------------ */
export interface ClassRow {
id: string;
name: string;
code: string;
description: string;
color: string;
archived: number;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface FolderRow {
id: string;
class_id: string;
name: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface QuizRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
description: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface QuestionRow {
id: string;
quiz_id: string;
type: QuestionType;
stem: string;
options: string;
rationale: string;
topic: string;
difficulty: Difficulty;
tags: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface DeckRow extends QuizRow {}
export interface CardRow {
id: string;
deck_id: string;
front: string;
back: string;
hint: string;
tags: string;
sort_order: number;
suspended: number;
state: number;
due: number | null;
stability: number;
difficulty: number;
scheduled_days: number;
learning_steps: number;
reps: number;
lapses: number;
last_review: number | null;
introduced_at: number | null;
created_at: number;
updated_at: number;
}
export interface NoteRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
content: string;
kind: string;
source: string;
sort_order: number;
created_at: number;
updated_at: number;
}
/* ------------------------------ mappers ------------------------------ */
export const mapClass = (r: ClassRow) => ({
id: r.id,
name: r.name,
code: r.code,
description: r.description,
color: r.color,
archived: Boolean(r.archived),
sortOrder: r.sort_order,
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export type ClassDto = ReturnType<typeof mapClass>;
export const mapFolder = (r: FolderRow) => ({ id: r.id, classId: r.class_id, name: r.name, sortOrder: r.sort_order });
export const mapQuiz = (r: QuizRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
description: r.description,
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export const mapDeck = mapQuiz;
export function mapQuestion(r: QuestionRow): Question {
return {
id: r.id,
quizId: r.quiz_id,
type: r.type,
stem: r.stem,
options: parseJson<Option[]>(r.options, []).map((o) => ({ id: o.id, text: o.text, correct: Boolean(o.correct), why: o.why ?? '' })),
rationale: r.rationale,
topic: r.topic,
difficulty: r.difficulty,
tags: parseJson<string[]>(r.tags, []),
sortOrder: r.sort_order,
createdAt: r.created_at,
updatedAt: r.updated_at,
};
}
export function mapCard(r: CardRow): Card {
return {
id: r.id,
deckId: r.deck_id,
front: r.front,
back: r.back,
hint: r.hint,
tags: parseJson<string[]>(r.tags, []),
sortOrder: r.sort_order,
suspended: Boolean(r.suspended),
state: r.state,
due: r.due,
stability: r.stability,
difficulty: r.difficulty,
scheduledDays: r.scheduled_days,
reps: r.reps,
lapses: r.lapses,
lastReview: r.last_review,
createdAt: r.created_at,
updatedAt: r.updated_at,
};
}
export const mapNote = (r: NoteRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
content: r.content,
kind: r.kind,
source: parseJson<Record<string, unknown>>(r.source, {}),
createdAt: r.created_at,
updatedAt: r.updated_at,
});
/* ------------------------------ lookups ------------------------------ */
export function getClass(db: DB, id: string): ClassRow {
const row = db.get<ClassRow>('SELECT * FROM classes WHERE id = ?', id);
if (!row) throw notFound('Class');
return row;
}
export function getFolder(db: DB, id: string): FolderRow {
const row = db.get<FolderRow>('SELECT * FROM folders WHERE id = ?', id);
if (!row) throw notFound('Folder');
return row;
}
export function getQuiz(db: DB, id: string): QuizRow {
const row = db.get<QuizRow>('SELECT * FROM quizzes WHERE id = ?', id);
if (!row) throw notFound('Quiz');
return row;
}
export function getDeck(db: DB, id: string): DeckRow {
const row = db.get<DeckRow>('SELECT * FROM decks WHERE id = ?', id);
if (!row) throw notFound('Deck');
return row;
}
export function getNote(db: DB, id: string): NoteRow {
const row = db.get<NoteRow>('SELECT * FROM notes WHERE id = ?', id);
if (!row) throw notFound('Note');
return row;
}
export function listQuestions(db: DB, quizId: string): Question[] {
return db.all<QuestionRow>('SELECT * FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quizId).map(mapQuestion);
}
export function listCards(db: DB, deckId: string): Card[] {
return db.all<CardRow>('SELECT * FROM cards WHERE deck_id = ? ORDER BY sort_order, created_at', deckId).map(mapCard);
}
/** Resolves and validates a folder id against a class; returns null for "no folder". */
export function resolveFolder(db: DB, classId: string, folderId: string | null | undefined): string | null {
if (!folderId) return null;
const folder = db.get<FolderRow>('SELECT * FROM folders WHERE id = ?', folderId);
return folder && folder.class_id === classId ? folder.id : null;
}
function nextSort(db: DB, table: string, where: string, value: string): number {
const row = db.get<{ m: number | null }>(`SELECT MAX(sort_order) AS m FROM ${table} WHERE ${where} = ?`, value);
return (row?.m ?? -1) + 1;
}
const touch = (db: DB, table: 'quizzes' | 'decks' | 'classes', id: string) => db.run(`UPDATE ${table} SET updated_at = ? WHERE id = ?`, now(), id);
/* ------------------------------ classes + folders ------------------------------ */
export const classInputSchema = z.object({
name: z.string().trim().min(1, 'Name is required').max(120),
code: z.string().trim().max(40).optional().default(''),
description: z.string().max(2000).optional().default(''),
color: z.enum(CLASS_COLORS).optional(),
});
export function createClass(db: DB, input: z.input<typeof classInputSchema>): ClassRow {
const data = classInputSchema.parse(input);
const id = newId();
const at = now();
const count = db.get<{ n: number }>('SELECT COUNT(*) AS n FROM classes')?.n ?? 0;
const color = data.color ?? CLASS_COLORS[count % CLASS_COLORS.length];
db.run(
'INSERT INTO classes (id, name, code, description, color, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
data.name,
data.code,
data.description,
color,
count,
at,
at,
);
return getClass(db, id);
}
export function createFolder(db: DB, classId: string, name: string): FolderRow {
getClass(db, classId);
const id = newId();
const at = now();
db.run(
'INSERT INTO folders (id, class_id, name, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)',
id,
classId,
name.trim().slice(0, 120) || 'Untitled folder',
nextSort(db, 'folders', 'class_id', classId),
at,
at,
);
return getFolder(db, id);
}
/* ------------------------------ quizzes + questions ------------------------------ */
type ParsedQuestion = z.output<typeof questionInputSchema>;
function optionsJson(q: ParsedQuestion, keep?: Option[]): string {
const used = new Set<string>();
const options: Option[] = q.options.map((o) => {
let id = o.id && !used.has(o.id) ? o.id : undefined;
// Keep option ids stable across edits when the text is unchanged, so old attempts still line up.
if (!id && keep) id = keep.find((k) => k.text === o.text && !used.has(k.id))?.id;
if (!id) id = newId().slice(0, 8);
used.add(id);
return { id, text: o.text, correct: o.correct, why: o.why ?? '' };
});
return JSON.stringify(options);
}
export function insertQuestions(db: DB, quizId: string, inputs: unknown[], atIndex?: number): Question[] {
const parsed = inputs.map((q) => questionInputSchema.parse(q));
const at = now();
const ids: string[] = [];
db.tx(() => {
let sort = nextSort(db, 'questions', 'quiz_id', quizId);
if (atIndex !== undefined) {
const ordered = db.all<{ id: string }>('SELECT id FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quizId);
const insertAt = Math.max(0, Math.min(atIndex, ordered.length));
ordered.forEach((row, i) => db.run('UPDATE questions SET sort_order = ? WHERE id = ?', i < insertAt ? i : i + parsed.length, row.id));
sort = insertAt;
}
for (const q of parsed) {
const id = newId();
ids.push(id);
db.run(
'INSERT INTO questions (id, quiz_id, type, stem, options, rationale, topic, difficulty, tags, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
quizId,
q.type,
q.stem,
optionsJson(q),
q.rationale,
q.topic,
q.difficulty,
JSON.stringify(q.tags),
sort,
at,
at,
);
sort += 1;
}
touch(db, 'quizzes', quizId);
});
const all = listQuestions(db, quizId);
return ids.map((id) => all.find((q) => q.id === id)!);
}
export function updateQuestion(db: DB, id: string, input: unknown): Question {
const existing = db.get<QuestionRow>('SELECT * FROM questions WHERE id = ?', id);
if (!existing) throw notFound('Question');
const q = questionInputSchema.parse(input);
db.run(
'UPDATE questions SET type = ?, stem = ?, options = ?, rationale = ?, topic = ?, difficulty = ?, tags = ?, updated_at = ? WHERE id = ?',
q.type,
q.stem,
optionsJson(q, mapQuestion(existing).options),
q.rationale,
q.topic,
q.difficulty,
JSON.stringify(q.tags),
now(),
id,
);
touch(db, 'quizzes', existing.quiz_id);
return mapQuestion(db.get<QuestionRow>('SELECT * FROM questions WHERE id = ?', id)!);
}
export function createQuiz(
db: DB,
input: { classId: string; folderId?: string | null; title: string; description?: string; questions?: unknown[] },
): QuizRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.tx(() => {
db.run(
'INSERT INTO quizzes (id, class_id, folder_id, title, description, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim(),
input.description ?? '',
nextSort(db, 'quizzes', 'class_id', input.classId),
at,
at,
);
if (input.questions?.length) insertQuestions(db, id, input.questions);
});
return getQuiz(db, id);
}
export function duplicateQuiz(db: DB, id: string, target?: { classId?: string; folderId?: string | null; title?: string }): QuizRow {
const quiz = getQuiz(db, id);
const questions = listQuestions(db, id);
const classId = target?.classId ?? quiz.class_id;
return createQuiz(db, {
classId,
folderId: target?.folderId !== undefined ? target.folderId : classId === quiz.class_id ? quiz.folder_id : null,
title: target?.title ?? `${quiz.title} (copy)`,
description: quiz.description,
questions: questions.map((q) => ({
type: q.type,
stem: q.stem,
options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why })),
rationale: q.rationale,
topic: q.topic,
difficulty: q.difficulty,
tags: q.tags,
})),
});
}
/* ------------------------------ decks + cards ------------------------------ */
export function insertCards(db: DB, deckId: string, inputs: unknown[]): Card[] {
const parsed = inputs.map((c) => cardInputSchema.parse(c));
const at = now();
const ids: string[] = [];
db.tx(() => {
let sort = nextSort(db, 'cards', 'deck_id', deckId);
for (const c of parsed) {
const id = newId();
ids.push(id);
db.run(
'INSERT INTO cards (id, deck_id, front, back, hint, tags, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
deckId,
c.front,
c.back,
c.hint,
JSON.stringify(c.tags),
sort,
at,
at,
);
sort += 1;
}
touch(db, 'decks', deckId);
});
const all = listCards(db, deckId);
return ids.map((id) => all.find((c) => c.id === id)!);
}
export function updateCard(db: DB, id: string, input: unknown): Card {
const existing = db.get<CardRow>('SELECT * FROM cards WHERE id = ?', id);
if (!existing) throw notFound('Card');
const c = cardInputSchema.parse(input);
db.run('UPDATE cards SET front = ?, back = ?, hint = ?, tags = ?, updated_at = ? WHERE id = ?', c.front, c.back, c.hint, JSON.stringify(c.tags), now(), id);
touch(db, 'decks', existing.deck_id);
return mapCard(db.get<CardRow>('SELECT * FROM cards WHERE id = ?', id)!);
}
export function createDeck(
db: DB,
input: { classId: string; folderId?: string | null; title: string; description?: string; cards?: unknown[] },
): DeckRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.tx(() => {
db.run(
'INSERT INTO decks (id, class_id, folder_id, title, description, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim(),
input.description ?? '',
nextSort(db, 'decks', 'class_id', input.classId),
at,
at,
);
if (input.cards?.length) insertCards(db, id, input.cards);
});
return getDeck(db, id);
}
export function duplicateDeck(db: DB, id: string, target?: { classId?: string; folderId?: string | null; title?: string }): DeckRow {
const deck = getDeck(db, id);
const cards = listCards(db, id);
const classId = target?.classId ?? deck.class_id;
return createDeck(db, {
classId,
folderId: target?.folderId !== undefined ? target.folderId : classId === deck.class_id ? deck.folder_id : null,
title: target?.title ?? `${deck.title} (copy)`,
description: deck.description,
cards: cards.map((c) => ({ front: c.front, back: c.back, hint: c.hint, tags: c.tags })),
});
}
/* ------------------------------ notes ------------------------------ */
export function createNote(
db: DB,
input: { classId: string; folderId?: string | null; title: string; content?: string; kind?: string; source?: unknown },
): NoteRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.run(
'INSERT INTO notes (id, class_id, folder_id, title, content, kind, source, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim().slice(0, 200) || 'Untitled note',
input.content ?? '',
input.kind ?? 'note',
JSON.stringify(input.source ?? {}),
nextSort(db, 'notes', 'class_id', input.classId),
at,
at,
);
return getNote(db, id);
}
/* ------------------------------ draft conversion ------------------------------ */
/** Turns a (validated) draft question into a strict question input, folding the citation into the rationale. */
export function draftQuestionToInput(q: DraftQuestion) {
const source = q.source?.trim();
const rationale = source && !q.rationale.includes(source) ? `${q.rationale.trim()}\n\n*Source: ${source}*`.trim() : q.rationale;
return {
type: q.type,
stem: q.stem,
options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why ?? '' })),
rationale,
topic: (q.topic ?? '').slice(0, 80),
difficulty: q.difficulty ?? 'medium',
tags: (q.tags ?? []).map((t) => t.slice(0, 40)).filter(Boolean).slice(0, 12),
};
}
export function draftCardToInput(c: DraftCard) {
return { front: c.front, back: c.back, hint: c.hint ?? '', tags: (c.tags ?? []).map((t) => t.slice(0, 40)).filter(Boolean).slice(0, 12) };
}

View file

@ -0,0 +1,500 @@
/**
* Uploaded study materials: storage, the background processing queue
* (extract -> optional AI transcription -> chunk -> index) and retrieval.
*/
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { resolveModel, runChat } from '../ai/service.ts';
import { AIError, type ContentPart } from '../ai/types.ts';
import type { DB } from '../db.ts';
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
import { detectKind, extractDocument } from '../documents/extract.ts';
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
import { HttpError, notFound, type AppContext } from '../http.ts';
import { ftsQuery, newId, now, parseJson } from '../util.ts';
import { getClass, resolveFolder } from './content.ts';
export interface DocumentRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
filename: string;
mime: string;
kind: DocKind;
size: number;
sha256: string;
storage_name: string;
status: 'pending' | 'processing' | 'ready' | 'failed';
error: string;
progress: number;
page_count: number | null;
char_count: number;
token_count: number;
chunk_count: number;
needs_ocr: number;
warnings: string;
created_at: number;
updated_at: number;
}
export const mapDocument = (r: DocumentRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
filename: r.filename,
mime: r.mime,
docKind: r.kind,
size: r.size,
status: r.status,
error: r.error,
progress: r.progress,
pageCount: r.page_count,
charCount: r.char_count,
tokenCount: r.token_count,
chunkCount: r.chunk_count,
needsOcr: Boolean(r.needs_ocr),
warnings: parseJson<string[]>(r.warnings, []),
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export type DocumentDto = ReturnType<typeof mapDocument>;
const MIME_BY_KIND: Record<DocKind, string> = {
pdf: 'application/pdf',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
text: 'text/plain; charset=utf-8',
markdown: 'text/markdown; charset=utf-8',
image: 'application/octet-stream',
};
function imageMime(bytes: Uint8Array): string {
if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'image/png';
if (bytes[0] === 0xff && bytes[1] === 0xd8) return 'image/jpeg';
if (bytes[0] === 0x47 && bytes[1] === 0x49) return 'image/gif';
if (bytes[0] === 0x52 && bytes[8] === 0x57) return 'image/webp';
return 'application/octet-stream';
}
export function getDocument(db: DB, id: string): DocumentRow {
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) throw notFound('Document');
return row;
}
export function documentPath(app: AppContext, row: Pick<DocumentRow, 'storage_name'>): string {
// basename() guarantees a stored name can never escape the uploads folder.
return path.join(app.config.uploadsDir, path.basename(row.storage_name));
}
function titleFromFilename(filename: string): string {
const base = filename.replace(/\.[^.]+$/, '').replace(/[_]+/g, ' ').replace(/\s+/g, ' ').trim();
return (base || 'Untitled document').slice(0, 200);
}
/* ------------------------------ ingest ------------------------------ */
export function ingestUpload(
app: AppContext,
input: { classId: string; folderId?: string | null; filename: string; mime?: string; bytes: Uint8Array; title?: string },
): DocumentRow {
const { db } = app;
getClass(db, input.classId);
if (input.bytes.length === 0) throw new HttpError(422, 'empty_file', `"${input.filename}" is empty.`);
if (input.bytes.length > app.config.maxUploadBytes) {
throw new HttpError(413, 'too_large', `"${input.filename}" is larger than the ${Math.round(app.config.maxUploadBytes / 1048576)} MB upload limit.`);
}
const filename = path.basename(input.filename.replace(/\\/g, '/')).slice(0, 200) || 'upload';
const kind = detectKind(filename, input.mime, input.bytes);
if (!kind) {
throw new HttpError(415, 'unsupported_type', `"${filename}" is not a supported file type. Upload PDF, Word (.docx), PowerPoint (.pptx), text, Markdown, or an image.`);
}
const id = newId();
const ext = { pdf: '.pdf', docx: '.docx', pptx: '.pptx', text: '.txt', markdown: '.md', image: path.extname(filename).toLowerCase().replace(/[^.a-z0-9]/g, '') || '.img' }[kind];
const storageName = `${id}${ext}`;
fs.writeFileSync(path.join(app.config.uploadsDir, storageName), input.bytes);
const at = now();
const sort = (db.get<{ m: number | null }>('SELECT MAX(sort_order) AS m FROM documents WHERE class_id = ?', input.classId)?.m ?? -1) + 1;
db.run(
`INSERT INTO documents (id, class_id, folder_id, title, filename, mime, kind, size, sha256, storage_name, status, sort_order, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`,
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
(input.title?.trim() || titleFromFilename(filename)).slice(0, 200),
filename,
kind === 'image' ? imageMime(input.bytes) : MIME_BY_KIND[kind],
kind,
input.bytes.length,
crypto.createHash('sha256').update(input.bytes).digest('hex'),
storageName,
sort,
at,
at,
);
enqueue(app, id);
return getDocument(db, id);
}
export function deleteDocument(app: AppContext, id: string): void {
const row = getDocument(app.db, id);
app.db.tx(() => {
app.db.run(`DELETE FROM shares WHERE kind = 'document' AND target_id = ?`, id);
app.db.run('DELETE FROM documents WHERE id = ?', id);
});
fs.rmSync(documentPath(app, row), { force: true });
}
/* ------------------------------ processing queue ------------------------------ */
const queues = new WeakMap<DB, { pending: string[]; running: boolean; ocr: Set<string>; idle: Promise<void>; wake: () => void }>();
function queueFor(app: AppContext) {
let q = queues.get(app.db);
if (!q) {
q = { pending: [], running: false, ocr: new Set(), idle: Promise.resolve(), wake: () => undefined };
queues.set(app.db, q);
}
return q;
}
export function enqueue(app: AppContext, id: string, opts: { ocr?: boolean } = {}): void {
const q = queueFor(app);
if (opts.ocr) q.ocr.add(id);
if (!q.pending.includes(id)) q.pending.push(id);
if (!q.running) {
q.running = true;
q.idle = (async () => {
// Yield first so the HTTP response for the upload is sent before heavy parsing starts.
await new Promise((r) => setTimeout(r, 10));
while (q.pending.length > 0) {
const next = q.pending.shift()!;
const forceOcr = q.ocr.delete(next);
try {
await processDocument(app, next, { forceOcr });
} catch (e) {
app.log('error', `Document ${next} failed unexpectedly`, e);
}
}
q.running = false;
})();
}
}
/** Resolves when the queue is empty. Used by tests and graceful shutdown. */
export function queueIdle(app: AppContext): Promise<void> {
return queueFor(app).idle;
}
/** Re-queues anything a restart interrupted. */
export function resumeQueue(app: AppContext): number {
const rows = app.db.all<{ id: string }>(`SELECT id FROM documents WHERE status IN ('pending', 'processing') ORDER BY created_at`);
for (const r of rows) enqueue(app, r.id);
return rows.length;
}
const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
- Reproduce all readable text faithfully, in reading order. Do not summarise and do not add information.
- Keep headings, lists and tables (as Markdown tables).
- For diagrams, charts and labelled figures, add a short description in italics that includes every label and value shown.
- For a multi-page document, start each page with a line of the form "## Page N".
- If something is illegible, write [illegible].
Return only the transcription.`;
/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */
async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise<ExtractedSection[] | null> {
const data = Buffer.from(bytes).toString('base64');
const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename };
if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) {
throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.');
}
let result;
try {
const vision = resolveModel(app, 'vision');
if (!vision.model.capabilities.vision) return null;
if (row.kind === 'pdf' && !vision.model.capabilities.pdf) {
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
}
result = await runChat(app, {
task: 'vision',
purpose: 'document transcription',
messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }],
maxOutputTokens: 32000,
reasoningEffort: 'low',
});
} catch (err) {
if (err instanceof AIError && err.code === 'not_configured') return null;
throw err;
}
const text = result.text.trim();
if (!text) return [];
// Split on the "## Page N" markers the prompt asks for so citations keep page numbers.
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }];
const sections: ExtractedSection[] = [];
if (pages[0].trim()) sections.push({ text: pages[0].trim() });
for (let i = 1; i < pages.length; i += 2) {
const body = (pages[i + 1] ?? '').trim();
if (body) sections.push({ text: body, page: Number(pages[i]) });
}
return sections;
}
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
const { db } = app;
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) return;
const setProgress = (p: number) => db.run('UPDATE documents SET progress = ?, updated_at = ? WHERE id = ?', Math.round(p * 100) / 100, now(), id);
db.run(`UPDATE documents SET status = 'processing', error = '', progress = 0, updated_at = ? WHERE id = ?`, now(), id);
try {
const file = documentPath(app, row);
if (!fs.existsSync(file)) throw new ExtractionError('failed', 'The uploaded file is missing from storage. Upload it again.');
const bytes = new Uint8Array(await fs.promises.readFile(file));
let lastWrite = 0;
const result = await extractDocument({
bytes,
filename: row.filename,
mime: row.mime,
onProgress: (done, total) => {
const t = Date.now();
if (t - lastWrite > 400 && total > 0) {
lastWrite = t;
setProgress((done / total) * 0.8);
}
},
});
const warnings = [...result.warnings];
let sections = result.sections;
let needsOcr = result.needsOcr;
if (needsOcr || opts.forceOcr) {
setProgress(0.82);
if (row.kind === 'image' || row.kind === 'pdf') {
try {
const transcribed = await transcribeWithVision(app, row, bytes);
if (transcribed === null) {
warnings.push(
row.kind === 'image'
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
);
} else if (transcribed.length > 0) {
sections = transcribed;
needsOcr = false;
} else {
warnings.push('The AI model did not find any readable text in this file.');
}
} catch (err) {
warnings.push(`Reading with AI failed: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
}
const chunks = chunkSections(sections);
const chars = sections.reduce((n, s) => n + s.text.length, 0);
const tokens = chunks.reduce((n, c) => n + c.tokens, 0);
db.tx(() => {
db.run('DELETE FROM chunks WHERE document_id = ?', id);
const current = db.get<{ class_id: string }>('SELECT class_id FROM documents WHERE id = ?', id);
if (!current) return; // deleted while processing
for (const c of chunks) {
db.run(
'INSERT INTO chunks (document_id, class_id, seq, text, heading, page_start, page_end, tokens) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
current.class_id,
c.seq,
c.text,
c.heading,
c.pageStart,
c.pageEnd,
c.tokens,
);
}
db.run(
`UPDATE documents SET status = 'ready', error = '', progress = 1, page_count = ?, char_count = ?, token_count = ?, chunk_count = ?, needs_ocr = ?, warnings = ?, updated_at = ? WHERE id = ?`,
result.pageCount ?? null,
chars,
tokens,
chunks.length,
needsOcr ? 1 : 0,
JSON.stringify([...new Set(warnings)].slice(0, 8)),
now(),
id,
);
});
} catch (err) {
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
db.run(`UPDATE documents SET status = 'failed', error = ?, progress = 0, updated_at = ? WHERE id = ?`, message, now(), id);
}
}
/* ------------------------------ retrieval ------------------------------ */
export interface ChunkHit {
id: number;
documentId: string;
documentTitle: string;
seq: number;
text: string;
heading: string;
pageStart: number | null;
pageEnd: number | null;
tokens: number;
score?: number;
}
type ChunkJoin = { id: number; document_id: string; doc_title: string; seq: number; text: string; heading: string; page_start: number | null; page_end: number | null; tokens: number; score?: number };
const mapHit = (r: ChunkJoin): ChunkHit => ({
id: r.id,
documentId: r.document_id,
documentTitle: r.doc_title,
seq: r.seq,
text: r.text,
heading: r.heading,
pageStart: r.page_start,
pageEnd: r.page_end,
tokens: r.tokens,
score: r.score,
});
export interface ChunkScope {
classId?: string | null;
documentIds?: string[];
}
function scopeSql(scope: ChunkScope): { sql: string; params: string[] } {
if (scope.documentIds?.length) return { sql: `AND ch.document_id IN (${scope.documentIds.map(() => '?').join(',')})`, params: scope.documentIds };
if (scope.classId) return { sql: 'AND ch.class_id = ?', params: [scope.classId] };
return { sql: '', params: [] };
}
/** BM25 full-text search over document chunks. */
export function searchChunks(db: DB, query: string, scope: ChunkScope, limit = 8): ChunkHit[] {
const match = ftsQuery(query, 'or');
if (!match) return [];
const { sql, params } = scopeSql(scope);
try {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens, bm25(chunks_fts, 1.0, 2.0) AS score
FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id
WHERE chunks_fts MATCH ? ${sql} ORDER BY score LIMIT ?`,
match,
...params,
limit,
)
.map(mapHit);
} catch {
return [];
}
}
export function listChunks(db: DB, documentId: string, offset = 0, limit = 50): ChunkHit[] {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.document_id = ? AND ch.seq >= ? ORDER BY ch.seq LIMIT ?`,
documentId,
offset,
limit,
)
.map(mapHit);
}
export function chunkById(db: DB, id: number): ChunkHit | null {
const row = db.get<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.id = ?`,
id,
);
return row ? mapHit(row) : null;
}
export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' | 'pageEnd'>): string {
if (hit.pageStart === null) return hit.documentTitle;
return hit.pageEnd && hit.pageEnd !== hit.pageStart ? `${hit.documentTitle}, pp. ${hit.pageStart}-${hit.pageEnd}` : `${hit.documentTitle}, p. ${hit.pageStart}`;
}
/**
* Chooses source material for a generation job and splits it into `groups` batches that
* each fit `budgetTokens`.
* - With a focus, the best-matching chunks are used (kept in document order).
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
* so every part of the document is represented.
*/
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number }): ChunkHit[][] {
const { sql, params } = scopeSql(scope);
if (!sql) return [];
const all = db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE 1 = 1 ${sql} ORDER BY d.sort_order, d.created_at, ch.seq`,
...params,
)
.map(mapHit);
if (all.length === 0) return [];
const capacity = opts.budgetTokens * opts.groups;
const total = all.reduce((n, c) => n + c.tokens, 0);
let chosen = all;
if (opts.focus?.trim()) {
const ranked = searchChunks(db, opts.focus, scope, 400);
if (ranked.length >= 3) {
const keep = new Set<number>();
let used = 0;
for (const hit of ranked) {
if (used + hit.tokens > capacity) break;
keep.add(hit.id);
used += hit.tokens;
}
chosen = all.filter((c) => keep.has(c.id));
}
}
if (chosen === all && total > capacity) {
// Even sampling: walk the document and keep chunks at a steady stride.
const stride = total / capacity;
const picked: ChunkHit[] = [];
let acc = 0;
let next = 0;
for (const c of all) {
if (acc >= next) {
picked.push(c);
next += c.tokens * stride;
}
acc += c.tokens;
}
chosen = picked;
}
// Contiguous groups of roughly equal size, none above the budget.
const chosenTotal = chosen.reduce((n, c) => n + c.tokens, 0);
const groupCount = Math.max(1, Math.min(opts.groups, chosen.length));
const target = Math.min(opts.budgetTokens, Math.ceil(chosenTotal / groupCount));
const groups: ChunkHit[][] = [];
let current: ChunkHit[] = [];
let size = 0;
for (const c of chosen) {
if (current.length > 0 && size + c.tokens > target && groups.length < groupCount - 1) {
groups.push(current);
current = [];
size = 0;
}
if (size + c.tokens > opts.budgetTokens && current.length > 0) continue; // last group is full
current.push(c);
size += c.tokens;
}
if (current.length > 0) groups.push(current);
return groups;
}
export function formatExcerpts(chunks: ChunkHit[]): string {
return chunks.map((c) => `[c${c.id}] ${sourceLabel(c)}${c.heading ? ` — ${c.heading}` : ''}\n${c.text}`).join('\n\n---\n\n');
}
export { estimateTokens };

309
server/services/srs.ts Normal file
View file

@ -0,0 +1,309 @@
/**
* Spaced repetition with FSRS (the algorithm Anki adopted in 2023), via ts-fsrs.
* Scheduling state lives on each card row; every review is logged so it can be undone
* and analysed later.
*/
import { createEmptyCard, fsrs, Rating, State, type Card as FsrsCard, type Grade } from 'ts-fsrs';
import type { DB } from '../db.ts';
import { badRequest, notFound } from '../http.ts';
import type { AppSettings } from '../settings.ts';
import { DAY_MS, now as clock, parseJson, startOfDay } from '../util.ts';
import { mapCard, type CardRow } from './content.ts';
export const LEARN_AHEAD_MS = 20 * 60_000;
export type RatingValue = 1 | 2 | 3 | 4;
export const RATING_LABELS: Record<RatingValue, string> = { 1: 'Again', 2: 'Hard', 3: 'Good', 4: 'Easy' };
const schedulers = new Map<string, ReturnType<typeof fsrs>>();
function scheduler(retention: number, fuzz: boolean) {
const key = `${retention}:${fuzz}`;
let s = schedulers.get(key);
if (!s) {
s = fsrs({ request_retention: retention, enable_fuzz: fuzz, enable_short_term: true, maximum_interval: 36500 });
schedulers.set(key, s);
}
return s;
}
function toFsrs(row: CardRow, at: number): FsrsCard {
if (row.state === State.New || row.due === null) return createEmptyCard(new Date(at));
return {
due: new Date(row.due),
stability: row.stability,
difficulty: row.difficulty,
scheduled_days: row.scheduled_days,
learning_steps: row.learning_steps,
reps: row.reps,
lapses: row.lapses,
state: row.state as State,
last_review: row.last_review ? new Date(row.last_review) : undefined,
} as FsrsCard;
}
export function intervalLabel(ms: number): string {
if (ms < 60_000) return '<1m';
const minutes = ms / 60_000;
if (minutes < 60) return `${Math.round(minutes)}m`;
const hours = minutes / 60;
if (hours < 24) return `${Math.round(hours)}h`;
const days = hours / 24;
if (days < 30.5) return `${Math.round(days)}d`;
const months = days / 30.44;
if (months < 12) return `${months.toFixed(months < 3 ? 1 : 0).replace(/\.0$/, '')}mo`;
return `${(days / 365.25).toFixed(1).replace(/\.0$/, '')}y`;
}
export interface RatingPreview {
rating: RatingValue;
label: string;
interval: string;
due: number;
}
export function previewRatings(row: CardRow, settings: AppSettings, at: number): RatingPreview[] {
// Previews never use fuzz so the labels are stable while the card is on screen.
const options = scheduler(settings.desiredRetention, false).repeat(toFsrs(row, at), new Date(at));
return ([1, 2, 3, 4] as RatingValue[]).map((rating) => {
const due = options[rating as Grade].card.due.getTime();
return { rating, label: RATING_LABELS[rating], interval: intervalLabel(due - at), due };
});
}
export interface QueueScope {
deckId?: string;
classId?: string;
}
function scopeSql(scope: QueueScope): { where: string; params: string[] } {
if (scope.deckId) return { where: 'AND c.deck_id = ?', params: [scope.deckId] };
if (scope.classId) return { where: 'AND d.class_id = ?', params: [scope.classId] };
return { where: '', params: [] };
}
const BASE = `FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.suspended = 0 AND k.archived = 0`;
export interface DueCounts {
learning: number;
review: number;
new: number;
total: number;
}
/** New cards still allowed today under the global daily limit. */
export function newCardsLeftToday(db: DB, settings: AppSettings, at: number): number {
const since = startOfDay(at, settings.timezone);
const introduced = db.get<{ n: number }>('SELECT COUNT(*) AS n FROM cards WHERE introduced_at >= ?', since)?.n ?? 0;
return Math.max(0, settings.newCardsPerDay - introduced);
}
export function dueCounts(db: DB, scope: QueueScope, settings: AppSettings, at = clock()): DueCounts {
const { where, params } = scopeSql(scope);
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const row = db.get<{ learning: number; review: number; fresh: number }>(
`SELECT
COALESCE(SUM(CASE WHEN c.state IN (1,3) AND c.due <= ? THEN 1 ELSE 0 END), 0) AS learning,
COALESCE(SUM(CASE WHEN c.state = 2 AND c.due < ? THEN 1 ELSE 0 END), 0) AS review,
COALESCE(SUM(CASE WHEN c.state = 0 THEN 1 ELSE 0 END), 0) AS fresh
${BASE} ${where}`,
at + LEARN_AHEAD_MS,
endOfDay,
...params,
)!;
const fresh = Math.min(row.fresh, newCardsLeftToday(db, settings, at));
return { learning: row.learning, review: row.review, new: fresh, total: row.learning + row.review + fresh };
}
export interface QueueCard {
card: ReturnType<typeof mapCard>;
deckTitle: string;
classId: string;
className: string;
classColor: string;
previews: RatingPreview[];
}
type JoinedRow = CardRow & { deck_title: string; class_id: string; class_name: string; class_color: string };
const SELECT = `SELECT c.*, d.title AS deck_title, d.class_id AS class_id, k.name AS class_name, k.color AS class_color`;
function toQueueCard(row: JoinedRow, settings: AppSettings, at: number): QueueCard {
return {
card: mapCard(row),
deckTitle: row.deck_title,
classId: row.class_id,
className: row.class_name,
classColor: row.class_color,
previews: previewRatings(row, settings, at),
};
}
/**
* Builds the study queue: learning cards first (they are time-critical), then reviews
* due today with new cards spread evenly among them.
*/
export function buildQueue(db: DB, scope: QueueScope, settings: AppSettings, limit = 100, at = clock()): { cards: QueueCard[]; counts: DueCounts } {
const { where, params } = scopeSql(scope);
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const learning = db.all<JoinedRow>(`${SELECT} ${BASE} ${where} AND c.state IN (1,3) AND c.due <= ? ORDER BY c.due LIMIT ?`, ...params, at + LEARN_AHEAD_MS, limit);
const reviews = db.all<JoinedRow>(`${SELECT} ${BASE} ${where} AND c.state = 2 AND c.due < ? ORDER BY c.due LIMIT ?`, ...params, endOfDay, limit);
const newLeft = newCardsLeftToday(db, settings, at);
const fresh =
newLeft > 0
? db.all<JoinedRow>(
`${SELECT} ${BASE} ${where} AND c.state = 0 ORDER BY k.sort_order, d.sort_order, d.created_at, c.sort_order, c.created_at LIMIT ?`,
...params,
Math.min(newLeft, limit),
)
: [];
const mixed: JoinedRow[] = [];
const every = fresh.length > 0 ? Math.max(1, Math.floor(reviews.length / fresh.length)) : 0;
let f = 0;
reviews.forEach((r, i) => {
mixed.push(r);
if (every > 0 && (i + 1) % every === 0 && f < fresh.length) mixed.push(fresh[f++]);
});
while (f < fresh.length) mixed.push(fresh[f++]);
const cards = [...learning, ...mixed].slice(0, limit).map((r) => toQueueCard(r, settings, at));
return { cards, counts: dueCounts(db, scope, settings, at) };
}
const SCHEDULE_FIELDS = ['state', 'due', 'stability', 'difficulty', 'scheduled_days', 'learning_steps', 'reps', 'lapses', 'last_review', 'introduced_at'] as const;
export function answerCard(
db: DB,
cardId: string,
rating: RatingValue,
settings: AppSettings,
opts: { at?: number; fuzz?: boolean } = {},
): QueueCard {
const at = opts.at ?? clock();
const row = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId);
if (!row) throw notFound('Card');
if (![1, 2, 3, 4].includes(rating)) throw badRequest('Rating must be 1-4');
const result = scheduler(settings.desiredRetention, opts.fuzz ?? true).next(toFsrs(row, at), new Date(at), rating as Grade);
const next = result.card;
const prev: Record<string, unknown> = {};
for (const f of SCHEDULE_FIELDS) prev[f] = row[f];
db.tx(() => {
db.run(
`UPDATE cards SET state = ?, due = ?, stability = ?, difficulty = ?, scheduled_days = ?, learning_steps = ?, reps = ?, lapses = ?, last_review = ?,
introduced_at = COALESCE(introduced_at, ?) WHERE id = ?`,
next.state,
next.due.getTime(),
next.stability,
next.difficulty,
next.scheduled_days,
next.learning_steps,
next.reps,
next.lapses,
at,
at,
cardId,
);
db.run(
'INSERT INTO reviews (card_id, deck_id, class_id, rating, state_before, scheduled_days, mode, prev, reviewed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
cardId,
row.deck_id,
row.class_id,
rating,
row.state,
next.scheduled_days,
'srs',
JSON.stringify(prev),
at,
);
});
const updated = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId)!;
return toQueueCard(updated, settings, at);
}
/** Reverts the most recent scheduled review of a card and returns it to its previous state. */
export function undoReview(db: DB, cardId: string, settings: AppSettings, at = clock()): QueueCard {
const review = db.get<{ id: number; prev: string }>(`SELECT id, prev FROM reviews WHERE card_id = ? AND mode = 'srs' ORDER BY id DESC LIMIT 1`, cardId);
if (!review || !review.prev) throw badRequest('Nothing to undo for this card');
const prev = parseJson<Record<string, number | null>>(review.prev, {});
db.tx(() => {
db.run(
`UPDATE cards SET state = ?, due = ?, stability = ?, difficulty = ?, scheduled_days = ?, learning_steps = ?, reps = ?, lapses = ?, last_review = ?, introduced_at = ? WHERE id = ?`,
prev.state ?? 0,
prev.due ?? null,
prev.stability ?? 0,
prev.difficulty ?? 0,
prev.scheduled_days ?? 0,
prev.learning_steps ?? 0,
prev.reps ?? 0,
prev.lapses ?? 0,
prev.last_review ?? null,
prev.introduced_at ?? null,
cardId,
);
db.run('DELETE FROM reviews WHERE id = ?', review.id);
});
const row = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId);
if (!row) throw notFound('Card');
return toQueueCard(row, settings, at);
}
export function resetCards(db: DB, where: { cardId?: string; deckId?: string }): number {
const sql = `UPDATE cards SET state = 0, due = NULL, stability = 0, difficulty = 0, scheduled_days = 0, learning_steps = 0, reps = 0, lapses = 0, last_review = NULL, introduced_at = NULL WHERE `;
if (where.cardId) return db.run(`${sql} id = ?`, where.cardId).changes;
if (where.deckId) return db.run(`${sql} deck_id = ?`, where.deckId).changes;
return 0;
}
/** Logs a self-graded cram flip so it counts toward activity without touching the schedule. */
export function logCram(db: DB, cardId: string, known: boolean, at = clock()): void {
const row = db.get<{ deck_id: string; class_id: string; state: number }>(
'SELECT c.deck_id, d.class_id, c.state FROM cards c JOIN decks d ON d.id = c.deck_id WHERE c.id = ?',
cardId,
);
if (!row) return;
db.run(
`INSERT INTO reviews (card_id, deck_id, class_id, rating, state_before, scheduled_days, mode, prev, reviewed_at) VALUES (?, ?, ?, ?, ?, 0, 'cram', '', ?)`,
cardId,
row.deck_id,
row.class_id,
known ? Rating.Good : Rating.Again,
row.state,
at,
);
}
export interface DeckStats {
total: number;
new: number;
learning: number;
young: number;
mature: number;
suspended: number;
dueNow: number;
}
export function deckStats(db: DB, deckId: string, settings: AppSettings, at = clock()): DeckStats {
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const r = db.get<Record<string, number>>(
`SELECT COUNT(*) AS total,
COALESCE(SUM(suspended), 0) AS suspended,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 0 THEN 1 ELSE 0 END), 0) AS fresh,
COALESCE(SUM(CASE WHEN suspended = 0 AND state IN (1,3) THEN 1 ELSE 0 END), 0) AS learning,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 2 AND scheduled_days < 21 THEN 1 ELSE 0 END), 0) AS young,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 2 AND scheduled_days >= 21 THEN 1 ELSE 0 END), 0) AS mature,
COALESCE(SUM(CASE WHEN suspended = 0 AND ((state IN (1,3) AND due <= ?) OR (state = 2 AND due < ?)) THEN 1 ELSE 0 END), 0) AS due_now
FROM cards WHERE deck_id = ?`,
at + LEARN_AHEAD_MS,
endOfDay,
deckId,
)!;
return { total: r.total, new: r.fresh, learning: r.learning, young: r.young, mature: r.mature, suspended: r.suspended, dueNow: r.due_now };
}
/** Reviews coming due per day for the next `days` days (index 0 = today, including overdue). */
export function forecast(db: DB, scope: QueueScope, settings: AppSettings, days = 14, at = clock()): number[] {
const { where, params } = scopeSql(scope);
const start = startOfDay(at, settings.timezone);
const rows = db.all<{ due: number }>(`SELECT c.due AS due ${BASE} ${where} AND c.state != 0 AND c.due < ?`, ...params, start + days * DAY_MS);
const out = new Array<number>(days).fill(0);
for (const r of rows) out[Math.max(0, Math.min(days - 1, Math.floor((r.due - start) / DAY_MS)))] += 1;
return out;
}

185
server/services/stats.ts Normal file
View file

@ -0,0 +1,185 @@
/** Study activity, streaks, accuracy and weak-topic analytics. All derived from the review and response logs. */
import type { DB } from '../db.ts';
import type { AppSettings } from '../settings.ts';
import { addDaysToKey, DAY_MS, dayKey, now as clock, startOfDay } from '../util.ts';
export interface DayActivity {
date: string;
cards: number;
questions: number;
total: number;
}
/** Per-day totals for the trailing `days` days (oldest first, always dense). */
export function activityByDay(db: DB, settings: AppSettings, days: number, scope: { classId?: string } = {}, at = clock()): DayActivity[] {
const tz = settings.timezone;
const today = dayKey(at, tz);
const since = startOfDay(at, tz) - (days - 1) * DAY_MS - DAY_MS; // one extra day absorbs DST shifts
const map = new Map<string, DayActivity>();
const bump = (ts: number, field: 'cards' | 'questions') => {
const key = dayKey(ts, tz);
let d = map.get(key);
if (!d) {
d = { date: key, cards: 0, questions: 0, total: 0 };
map.set(key, d);
}
d[field] += 1;
d.total += 1;
};
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
for (const r of db.all<{ t: number }>(`SELECT reviewed_at AS t FROM reviews WHERE reviewed_at >= ? ${cls}`, since, ...p)) bump(r.t, 'cards');
for (const r of db.all<{ t: number }>(`SELECT answered_at AS t FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)) bump(r.t, 'questions');
const out: DayActivity[] = [];
for (let i = days - 1; i >= 0; i -= 1) {
const key = addDaysToKey(today, -i);
out.push(map.get(key) ?? { date: key, cards: 0, questions: 0, total: 0 });
}
return out;
}
export interface Streak {
current: number;
longest: number;
/** Today already meets the daily goal. */
todayDone: boolean;
todayTotal: number;
goal: number;
}
export function computeStreak(days: DayActivity[], goal: number): Streak {
const met = days.map((d) => d.total >= goal);
const todayDone = met[met.length - 1] ?? false;
let current = 0;
// Today only counts once the goal is met; an unfinished today does not break yesterday's streak.
for (let i = met.length - (todayDone ? 1 : 2); i >= 0 && met[i]; i -= 1) current += 1;
let longest = 0;
let run = 0;
for (const m of met) {
run = m ? run + 1 : 0;
longest = Math.max(longest, run);
}
return { current, longest, todayDone, todayTotal: days[days.length - 1]?.total ?? 0, goal };
}
export interface TopicStat {
topic: string;
classId: string | null;
className: string;
classColor: string;
seen: number;
accuracy: number;
/** Distinct questions available on this topic right now. */
questions: number;
}
/** Accuracy per topic over the trailing window. Topics need a few answers before they count. */
export function topicStats(db: DB, scope: { classId?: string }, windowDays = 90, at = clock()): TopicStat[] {
const since = at - windowDays * DAY_MS;
const cls = scope.classId ? 'AND r.class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const rows = db.all<{ topic: string; class_id: string | null; class_name: string | null; class_color: string | null; seen: number; acc: number; questions: number }>(
`SELECT r.topic AS topic, r.class_id AS class_id, k.name AS class_name, k.color AS class_color,
COUNT(*) AS seen, AVG(r.score) AS acc,
(SELECT COUNT(*) FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE q.topic = r.topic AND z.class_id = r.class_id) AS questions
FROM responses r LEFT JOIN classes k ON k.id = r.class_id
WHERE r.answered_at >= ? AND r.topic != '' ${cls}
GROUP BY r.topic, r.class_id
ORDER BY acc ASC, seen DESC`,
since,
...p,
);
return rows.map((r) => ({
topic: r.topic,
classId: r.class_id,
className: r.class_name ?? '',
classColor: r.class_color ?? 'slate',
seen: r.seen,
accuracy: r.acc,
questions: r.questions,
}));
}
export function weakTopics(db: DB, scope: { classId?: string }, limit = 6): TopicStat[] {
return topicStats(db, scope)
.filter((t) => t.seen >= 3 && t.accuracy < 0.75 && t.questions > 0)
.slice(0, limit);
}
export interface QuestionStat {
seen: number;
correct: number;
lastCorrect: boolean;
}
export function questionStats(db: DB, quizId: string): Record<string, QuestionStat> {
const rows = db.all<{ question_id: string; seen: number; correct: number; last_correct: number }>(
`SELECT r.question_id, COUNT(*) AS seen, SUM(r.correct) AS correct,
(SELECT r2.correct FROM responses r2 WHERE r2.question_id = r.question_id ORDER BY r2.answered_at DESC LIMIT 1) AS last_correct
FROM responses r JOIN questions q ON q.id = r.question_id WHERE q.quiz_id = ? GROUP BY r.question_id`,
quizId,
);
return Object.fromEntries(rows.map((r) => [r.question_id, { seen: r.seen, correct: r.correct, lastCorrect: Boolean(r.last_correct) }]));
}
export interface AccuracyPoint {
date: string;
answered: number;
accuracy: number | null;
}
export function accuracyTrend(db: DB, settings: AppSettings, days: number, scope: { classId?: string } = {}, at = clock()): AccuracyPoint[] {
const tz = settings.timezone;
const since = startOfDay(at, tz) - days * DAY_MS;
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const sums = new Map<string, { n: number; s: number }>();
for (const r of db.all<{ t: number; score: number }>(`SELECT answered_at AS t, score FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)) {
const key = dayKey(r.t, tz);
const cur = sums.get(key) ?? { n: 0, s: 0 };
cur.n += 1;
cur.s += r.score;
sums.set(key, cur);
}
const today = dayKey(at, tz);
const out: AccuracyPoint[] = [];
for (let i = days - 1; i >= 0; i -= 1) {
const key = addDaysToKey(today, -i);
const v = sums.get(key);
out.push({ date: key, answered: v?.n ?? 0, accuracy: v ? v.s / v.n : null });
}
return out;
}
export interface Totals {
questionsAnswered: number;
accuracy: number | null;
cardsReviewed: number;
retention: number | null;
quizzesCompleted: number;
studyDays: number;
}
export function totals(db: DB, settings: AppSettings, scope: { classId?: string } = {}, windowDays = 30, at = clock()): Totals {
const since = at - windowDays * DAY_MS;
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const resp = db.get<{ n: number; acc: number | null }>(`SELECT COUNT(*) AS n, AVG(score) AS acc FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)!;
const rev = db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM reviews WHERE reviewed_at >= ? ${cls}`, since, ...p)!;
// Retention: share of reviews of already-learned cards that were remembered (not "Again").
const ret = db.get<{ n: number; ok: number | null }>(
`SELECT COUNT(*) AS n, SUM(CASE WHEN rating > 1 THEN 1 ELSE 0 END) AS ok FROM reviews WHERE reviewed_at >= ? AND mode = 'srs' AND state_before = 2 ${cls}`,
since,
...p,
)!;
const done = db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM attempts WHERE status = 'completed' AND completed_at >= ? ${cls}`, since, ...p)!;
const days = activityByDay(db, settings, windowDays, scope, at).filter((d) => d.total > 0).length;
return {
questionsAnswered: resp.n,
accuracy: resp.n > 0 ? resp.acc : null,
cardsReviewed: rev.n,
retention: ret.n >= 5 ? (ret.ok ?? 0) / ret.n : null,
quizzesCompleted: done.n,
studyDays: days,
};
}