Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

561
shared/importer.ts Normal file
View file

@ -0,0 +1,561 @@
/**
* Lenient import parser.
*
* Accepts Study Desk V2 exports, V1 exports, loosely-shaped AI output, plain-text
* question blocks and delimited flashcard lists, and normalises all of them into
* editable drafts plus a list of issues. Nothing is saved from here: the preview UI
* shows the drafts, lets the user fix problems, and only then posts strict payloads.
*/
import { jsonrepair } from 'jsonrepair';
import {
DIFFICULTIES,
type ContentDraft,
type DeckDraft,
type Difficulty,
type DraftCard,
type DraftOption,
type DraftQuestion,
type Issue,
type NoteDraft,
type QuizDraft,
validateDraft,
} from './content.ts';
export const EXPORT_FORMAT = 'studydesk';
export const EXPORT_VERSION = 2;
export interface ImportResult {
items: ContentDraft[];
/** Problems that are not tied to one item (unparseable input, skipped entries...). */
notices: Issue[];
/** True when the JSON had to be repaired before it could be parsed. */
repaired: boolean;
/** How the input was interpreted, for display. */
detected: 'json' | 'text-quiz' | 'text-cards' | 'none';
}
type Raw = Record<string, unknown>;
const isObj = (v: unknown): v is Raw => typeof v === 'object' && v !== null && !Array.isArray(v);
const str = (v: unknown): string => (typeof v === 'string' ? v : typeof v === 'number' || typeof v === 'boolean' ? String(v) : '');
const pick = (o: Raw, ...keys: string[]): unknown => {
for (const k of keys) if (o[k] !== undefined && o[k] !== null) return o[k];
return undefined;
};
function toTags(v: unknown): string[] {
if (Array.isArray(v)) return v.map(str).map((s) => s.trim()).filter(Boolean).slice(0, 12);
if (typeof v === 'string') return v.split(/[,;]/).map((s) => s.trim()).filter(Boolean).slice(0, 12);
return [];
}
function toDifficulty(v: unknown): Difficulty {
const s = str(v).toLowerCase().trim();
if ((DIFFICULTIES as readonly string[]).includes(s)) return s as Difficulty;
if (['1', 'beginner', 'basic', 'low'].includes(s)) return 'easy';
if (['3', 'advanced', 'difficult', 'high', 'challenging'].includes(s)) return 'hard';
return 'medium';
}
function truthy(v: unknown): boolean {
if (typeof v === 'boolean') return v;
if (typeof v === 'number') return v !== 0;
const s = str(v).trim().toLowerCase();
return s === 'true' || s === 'yes' || s === 'y' || s === '1' || s === 'correct' || s === 'x';
}
/** Maps "B", "b", "2", 2 or the option text to option indexes. */
function resolveAnswerRefs(answer: unknown, options: DraftOption[]): number[] {
const refs = Array.isArray(answer) ? answer : typeof answer === 'string' ? answer.split(/[,;&]|\band\b/i) : [answer];
const out = new Set<number>();
for (const ref of refs) {
if (typeof ref === 'number' && Number.isInteger(ref)) {
// Ambiguous 0/1-based: treat as 0-based only when 0 appears, otherwise 1-based is far more common in hand-written keys.
const idx = ref >= 1 && ref <= options.length ? ref - 1 : ref;
if (idx >= 0 && idx < options.length) out.add(idx);
continue;
}
const s = str(ref).trim();
if (!s) continue;
const byText = options.findIndex((o) => o.text.trim().toLowerCase() === s.toLowerCase());
if (byText >= 0) {
out.add(byText);
continue;
}
const letter = s.replace(/[().\s]/g, '').toUpperCase();
if (/^[A-J]$/.test(letter)) {
const idx = letter.charCodeAt(0) - 65;
if (idx < options.length) out.add(idx);
} else if (/^\d+$/.test(letter)) {
const idx = Number(letter) - 1;
if (idx >= 0 && idx < options.length) out.add(idx);
}
}
return [...out];
}
function normalizeOptions(raw: unknown): DraftOption[] {
if (Array.isArray(raw)) {
return raw
.map((o): DraftOption | null => {
if (typeof o === 'string' || typeof o === 'number') return { text: String(o).trim(), correct: false, why: '' };
if (!isObj(o)) return null;
return {
text: str(pick(o, 'text', 'option', 'answer', 'label', 'value', 'content')).trim(),
correct: truthy(pick(o, 'correct', 'isCorrect', 'is_correct', 'right', 'answer_correct')),
why: str(pick(o, 'why', 'rationale', 'explanation', 'feedback')).trim(),
};
})
.filter((o): o is DraftOption => o !== null);
}
// { "A": "text", "B": "text" }
if (isObj(raw)) {
return Object.keys(raw)
.sort()
.map((k) => ({ text: str(raw[k]).trim(), correct: false, why: '' }));
}
return [];
}
function normalizeQuestion(raw: unknown, notices: Issue[], index: number): DraftQuestion | null {
if (!isObj(raw)) {
notices.push({ level: 'warning', index, message: `Question ${index + 1} is not an object and was skipped` });
return null;
}
const options = normalizeOptions(pick(raw, 'options', 'choices', 'answers', 'answerOptions', 'answer_options'));
const answerKey = pick(raw, 'correctAnswers', 'correct_answers', 'correctAnswer', 'correct_answer', 'answer', 'correct', 'key');
if (answerKey !== undefined && !options.some((o) => o.correct)) {
for (const idx of resolveAnswerRefs(answerKey, options)) options[idx].correct = true;
}
const correctCount = options.filter((o) => o.correct).length;
const rawType = str(pick(raw, 'type', 'questionType', 'question_type', 'kind')).toLowerCase().replace(/[\s-]/g, '_');
let type: DraftQuestion['type'];
if (['multi', 'sata', 'select_all', 'select_all_that_apply', 'multiple_response', 'multiple_select', 'multi_select', 'checkbox'].includes(rawType)) {
type = 'multi';
} else if (['single', 'multiple_choice', 'mcq', 'mc', 'single_choice', 'true_false', 'radio'].includes(rawType)) {
type = 'single';
} else {
type = correctCount > 1 ? 'multi' : 'single';
}
// A "multiple choice" question with several keyed answers can only be select-all.
if (type === 'single' && correctCount > 1) type = 'multi';
return {
type,
stem: str(pick(raw, 'stem', 'prompt', 'question', 'text', 'q')).trim(),
options,
rationale: str(pick(raw, 'rationale', 'explanation', 'reasoning', 'feedback')).trim(),
topic: str(pick(raw, 'topic', 'category', 'subject', 'concept')).trim().slice(0, 80),
difficulty: toDifficulty(pick(raw, 'difficulty', 'level')),
tags: toTags(pick(raw, 'tags', 'keywords')),
source: str(pick(raw, 'source', 'citation', 'reference')).trim() || undefined,
};
}
function normalizeCard(raw: unknown, notices: Issue[], index: number): DraftCard | null {
if (Array.isArray(raw) && raw.length >= 2) return { front: str(raw[0]).trim(), back: str(raw[1]).trim(), hint: '', tags: [] };
if (!isObj(raw)) {
notices.push({ level: 'warning', index, message: `Card ${index + 1} is not an object and was skipped` });
return null;
}
return {
front: str(pick(raw, 'front', 'term', 'question', 'prompt', 'q', 'word')).trim(),
back: str(pick(raw, 'back', 'definition', 'answer', 'a', 'response', 'meaning')).trim(),
hint: str(pick(raw, 'hint', 'clue')).trim(),
tags: toTags(pick(raw, 'tags', 'keywords')),
};
}
function quizFrom(raw: Raw, notices: Issue[]): QuizDraft {
const list = pick(raw, 'questions', 'items', 'quiz') as unknown;
const questions = (Array.isArray(list) ? list : [])
.map((q, i) => normalizeQuestion(q, notices, i))
.filter((q): q is DraftQuestion => q !== null);
return {
kind: 'quiz',
title: str(pick(raw, 'title', 'quizName', 'quiz_name', 'name')).trim(),
description: str(pick(raw, 'description', 'summary')).trim(),
questions,
};
}
function deckFrom(raw: Raw, notices: Issue[]): DeckDraft {
const list = pick(raw, 'cards', 'flashcards', 'items') as unknown;
const cards = (Array.isArray(list) ? list : [])
.map((c, i) => normalizeCard(c, notices, i))
.filter((c): c is DraftCard => c !== null);
return {
kind: 'deck',
title: str(pick(raw, 'title', 'deckName', 'deck_name', 'name')).trim(),
description: str(pick(raw, 'description', 'summary')).trim(),
cards,
};
}
function noteFrom(raw: Raw): NoteDraft {
return { kind: 'note', title: str(pick(raw, 'title', 'name')).trim(), content: str(pick(raw, 'content', 'markdown', 'body', 'text')) };
}
function looksLikeQuestion(v: unknown): boolean {
return isObj(v) && pick(v, 'options', 'choices', 'answers', 'answerOptions') !== undefined;
}
function looksLikeCard(v: unknown): boolean {
return isObj(v) && pick(v, 'front', 'term') !== undefined && pick(v, 'back', 'definition') !== undefined;
}
function collect(value: unknown, items: ContentDraft[], notices: Issue[], depth = 0): void {
if (depth > 4) return;
if (Array.isArray(value)) {
if (value.length === 0) return;
if (value.every(looksLikeQuestion)) {
items.push(quizFrom({ questions: value }, notices));
return;
}
if (value.every((v) => looksLikeCard(v) || (Array.isArray(v) && v.length === 2))) {
items.push(deckFrom({ cards: value }, notices));
return;
}
for (const v of value) collect(v, items, notices, depth + 1);
return;
}
if (!isObj(value)) return;
const kind = str(pick(value, 'kind', 'type')).toLowerCase();
// Bundles / backups: { classes: [{ quizzes, decks, notes }] } or { items: [...] }
if (Array.isArray(value.classes)) {
for (const cls of value.classes) collect(cls, items, notices, depth + 1);
return;
}
const containers = ['quizzes', 'decks', 'notes', 'quizSets'].filter((k) => Array.isArray(value[k]));
if (containers.length > 0 && !Array.isArray(value.questions) && !Array.isArray(value.cards)) {
for (const k of containers) {
for (const entry of value[k] as unknown[]) {
if (!isObj(entry)) continue;
if (k === 'notes') items.push(noteFrom(entry));
else if (k === 'decks') items.push(deckFrom(entry, notices));
else items.push(quizFrom(entry, notices));
}
}
return;
}
if (kind === 'bundle' && Array.isArray(value.items)) {
for (const entry of value.items) collect(entry, items, notices, depth + 1);
return;
}
if (kind === 'note' || kind === 'guide') {
items.push(noteFrom(value));
return;
}
if (kind === 'deck' || kind === 'flashcards' || Array.isArray(value.cards) || Array.isArray(value.flashcards)) {
items.push(deckFrom(value, notices));
return;
}
if (kind === 'quiz' || Array.isArray(value.questions)) {
items.push(quizFrom(value, notices));
return;
}
if (looksLikeQuestion(value)) {
items.push(quizFrom({ questions: [value] }, notices));
return;
}
if (looksLikeCard(value)) {
items.push(deckFrom({ cards: [value] }, notices));
return;
}
// Unknown wrapper: look one level down for something recognisable.
for (const v of Object.values(value)) if (typeof v === 'object' && v !== null) collect(v, items, notices, depth + 1);
}
/** Strips a Markdown code fence and any prose around the outermost JSON value. */
function extractJsonText(text: string): string | null {
let t = text.trim();
const fence = /```(?:json|JSON)?\s*\n([\s\S]*?)```/.exec(t);
if (fence) t = fence[1].trim();
const start = t.search(/[[{]/);
if (start < 0) return null;
const open = t[start];
const close = open === '{' ? '}' : ']';
const end = t.lastIndexOf(close);
return end > start ? t.slice(start, end + 1) : t.slice(start);
}
/* ------------------------------------------------------------------ */
/* Plain-text formats */
/* ------------------------------------------------------------------ */
const OPTION_LINE = /^\s*(\*)?\s*(?:\(?([A-Ja-j])[.)]|[-•])\s+(.*)$/;
const ANSWER_LINE = /^\s*(?:correct\s+)?answers?\s*[:=]\s*(.+)$/i;
const RATIONALE_LINE = /^\s*(?:rationale|explanation|why)\s*[:=]\s*(.*)$/i;
const TOPIC_LINE = /^\s*(?:topic|category)\s*[:=]\s*(.*)$/i;
/**
* Parses hand-typed question blocks separated by blank lines:
*
* 1. Which finding requires immediate action?
* A. Heart rate 88
* *B. Oxygen saturation 84%
* C. Temperature 37.2
* Answer: B (alternative to the * marker)
* Rationale: ...
* Topic: Respiratory
*/
export function parseTextQuestions(text: string): DraftQuestion[] {
const blocks = text.replace(/\r\n?/g, '\n').split(/\n\s*\n/);
const questions: DraftQuestion[] = [];
for (const block of blocks) {
const lines = block.split('\n').filter((l) => l.trim());
if (lines.length < 3) continue;
const stem: string[] = [];
const options: DraftOption[] = [];
let rationale = '';
let topic = '';
let answer: string | null = null;
let section: 'stem' | 'options' | 'rationale' = 'stem';
for (const line of lines) {
let m: RegExpExecArray | null;
if ((m = ANSWER_LINE.exec(line))) {
answer = m[1];
continue;
}
if ((m = RATIONALE_LINE.exec(line))) {
rationale = m[1].trim();
section = 'rationale';
continue;
}
if ((m = TOPIC_LINE.exec(line))) {
topic = m[1].trim();
continue;
}
if ((m = OPTION_LINE.exec(line)) && section !== 'rationale') {
let body = m[3].trim();
let correct = Boolean(m[1]);
const tail = /\s*(?:\*|\(correct\)|✓|✔)\s*$/i.exec(body);
if (tail) {
correct = true;
body = body.slice(0, tail.index).trim();
}
options.push({ text: body, correct, why: '' });
section = 'options';
continue;
}
if (section === 'stem') stem.push(line.replace(/^\s*(?:Q?\d+[.):]|Q[.:])\s*/i, '').trim());
else if (section === 'rationale') rationale += `\n${line.trim()}`;
else if (options.length) options[options.length - 1].text += ` ${line.trim()}`;
}
if (options.length < 2 || stem.length === 0) continue;
if (answer && !options.some((o) => o.correct)) for (const idx of resolveAnswerRefs(answer, options)) options[idx].correct = true;
const correctCount = options.filter((o) => o.correct).length;
questions.push({
type: correctCount > 1 ? 'multi' : 'single',
stem: stem.join('\n'),
options,
rationale: rationale.trim(),
topic,
difficulty: 'medium',
tags: [],
});
}
return questions;
}
/** Parses "term <sep> definition" lines; the separator is detected from the input. */
export function parseTextCards(text: string): DraftCard[] {
const lines = text.replace(/\r\n?/g, '\n').split('\n').map((l) => l.trim()).filter(Boolean);
if (lines.length === 0) return [];
const seps = ['\t', ' :: ', '::', ' | ', ' — ', ' – ', ' - ', ';', ','];
const sep = seps.find((s) => lines.filter((l) => l.includes(s)).length >= Math.ceil(lines.length * 0.8));
if (!sep) return [];
const cards: DraftCard[] = [];
for (const line of lines) {
const at = line.indexOf(sep);
if (at <= 0) continue;
const front = line.slice(0, at).trim().replace(/^"|"$/g, '');
const back = line.slice(at + sep.length).trim().replace(/^"|"$/g, '');
if (front && back) cards.push({ front, back, hint: '', tags: [] });
}
return cards;
}
/* ------------------------------------------------------------------ */
/* Entry point */
/* ------------------------------------------------------------------ */
export function parseImport(text: string, opts: { prefer?: 'quiz' | 'deck'; fallbackTitle?: string } = {}): ImportResult {
const notices: Issue[] = [];
const items: ContentDraft[] = [];
let repaired = false;
let detected: ImportResult['detected'] = 'none';
const trimmed = text.trim();
if (!trimmed) return { items, notices: [{ level: 'error', message: 'Nothing to import yet. Paste content or choose a file.' }], repaired, detected };
const jsonText = /^[\s`]*(?:```|[[{])/.test(trimmed) || /^[^\n]{0,200}[[{]/.test(trimmed) ? extractJsonText(trimmed) : null;
if (jsonText) {
let parsed: unknown;
let ok = false;
try {
parsed = JSON.parse(jsonText);
ok = true;
} catch {
try {
parsed = JSON.parse(jsonrepair(jsonText));
ok = true;
repaired = true;
} catch (e) {
notices.push({ level: 'error', message: `The JSON could not be read: ${e instanceof Error ? e.message : 'syntax error'}` });
}
}
if (ok) {
detected = 'json';
collect(parsed, items, notices);
if (items.length === 0) notices.push({ level: 'error', message: 'No quizzes or flashcards were found in that JSON. Download a template to see the expected shape.' });
if (repaired) notices.push({ level: 'warning', message: 'The JSON had syntax problems that were repaired automatically. Check the preview carefully.' });
}
}
if (items.length === 0 && detected !== 'json') {
const questions = opts.prefer === 'deck' ? [] : parseTextQuestions(trimmed);
const cards = questions.length > 0 ? [] : parseTextCards(trimmed);
if (questions.length > 0) {
detected = 'text-quiz';
items.push({ kind: 'quiz', title: '', description: '', questions });
} else if (cards.length > 0) {
detected = 'text-cards';
items.push({ kind: 'deck', title: '', description: '', cards });
} else if (notices.length === 0) {
notices.push({
level: 'error',
message: 'Could not recognise this content. Use JSON, question blocks with lettered choices, or one "term [tab] definition" per line.',
});
}
}
for (const item of items) {
if (!item.title && opts.fallbackTitle) item.title = opts.fallbackTitle;
}
return { items, notices, repaired, detected };
}
/** Issues for one draft, including a friendly summary count. */
export function summarizeIssues(draft: ContentDraft): { errors: number; warnings: number; issues: Issue[] } {
const issues = validateDraft(draft);
return {
errors: issues.filter((i) => i.level === 'error').length,
warnings: issues.filter((i) => i.level === 'warning').length,
issues,
};
}
/* ------------------------------------------------------------------ */
/* Export shapes + templates */
/* ------------------------------------------------------------------ */
export function exportQuiz(q: { title: string; description: string; questions: DraftQuestion[] }) {
return {
format: EXPORT_FORMAT,
version: EXPORT_VERSION,
kind: 'quiz' as const,
title: q.title,
description: q.description,
questions: q.questions.map((x) => ({
type: x.type,
stem: x.stem,
options: x.options.map((o) => (o.why ? { text: o.text, correct: o.correct, why: o.why } : { text: o.text, correct: o.correct })),
rationale: x.rationale,
topic: x.topic,
difficulty: x.difficulty,
tags: x.tags,
})),
};
}
export function exportDeck(d: { title: string; description: string; cards: DraftCard[] }) {
return {
format: EXPORT_FORMAT,
version: EXPORT_VERSION,
kind: 'deck' as const,
title: d.title,
description: d.description,
cards: d.cards.map((c) => ({ front: c.front, back: c.back, ...(c.hint ? { hint: c.hint } : {}), tags: c.tags })),
};
}
export const QUIZ_JSON_TEMPLATE = JSON.stringify(
exportQuiz({
title: 'Example quiz',
description: 'Optional description',
questions: [
{
type: 'single',
stem: 'A client with heart failure reports a 1.4 kg weight gain overnight. Which action should the nurse take first?',
options: [
{ text: 'Assess lung sounds and oxygen saturation', correct: true, why: 'Rapid weight gain signals fluid retention; assess for pulmonary congestion first.' },
{ text: 'Document the weight and continue to monitor', correct: false },
{ text: 'Encourage increased oral fluid intake', correct: false },
{ text: 'Schedule a dietitian consult for next week', correct: false },
],
rationale: 'Assessment comes first. A gain of more than 1 kg in a day suggests fluid volume overload.',
topic: 'Heart failure',
difficulty: 'medium',
tags: ['cardiac', 'prioritization'],
},
{
type: 'multi',
stem: 'Which findings are expected with hypokalemia? Select all that apply.',
options: [
{ text: 'Muscle weakness', correct: true },
{ text: 'Flattened T waves', correct: true },
{ text: 'Peaked T waves', correct: false },
{ text: 'Decreased bowel sounds', correct: true },
],
rationale: 'Low potassium slows smooth and skeletal muscle activity and flattens T waves. Peaked T waves indicate hyperkalemia.',
topic: 'Electrolytes',
difficulty: 'medium',
tags: [],
},
],
}),
null,
2,
);
export const DECK_JSON_TEMPLATE = JSON.stringify(
exportDeck({
title: 'Example deck',
description: 'Optional description',
cards: [
{ front: 'Furosemide: drug class', back: 'Loop diuretic', hint: 'Acts on the loop of Henle', tags: ['pharm'] },
{ front: 'Normal serum potassium', back: '3.5 to 5.0 mEq/L', tags: ['labs'] },
],
}),
null,
2,
);
export const TEXT_QUIZ_TEMPLATE = `1. Which electrolyte imbalance causes peaked T waves?
A. Hypokalemia
*B. Hyperkalemia
C. Hyponatremia
D. Hypercalcemia
Rationale: High potassium speeds repolarisation, producing tall, peaked T waves.
Topic: Electrolytes
2. Which are signs of digoxin toxicity? Select all that apply.
*A. Visual halos
*B. Nausea and vomiting
C. Hypertension
*D. Bradycardia
Rationale: Early toxicity is gastrointestinal and visual; bradycardia and dysrhythmias follow.
Topic: Pharmacology`;
export const TEXT_CARDS_TEMPLATE = `Furosemide\tLoop diuretic; monitor potassium
Metoprolol\tBeta-1 selective blocker; hold for HR below 60
Warfarin\tVitamin K antagonist; monitor INR`;
/** Normalises already-parsed data (for example the arguments of an AI tool call) into drafts. */
export function draftsFromValue(value: unknown): { items: ContentDraft[]; notices: Issue[] } {
const items: ContentDraft[] = [];
const notices: Issue[] = [];
collect(value, items, notices);
return { items, notices };
}