Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

View file

@ -1,20 +0,0 @@
node_modules
.next
.git
.forgejo
.github
.env*
!.env.example
dev.db*
study.db*
data
coverage
.test-databases
.npm-cache
.prisma-cache
audit-results
*.log
*.tsbuildinfo
out.css
temp.css
test.css

View file

@ -1,70 +0,0 @@
name: Verify and publish container
on:
push:
branches:
- main
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
# The changed-file lint step compares the pushed commit with its parent.
fetch-depth: 2
- name: Log into Local Registry
run: |
echo "${{ secrets.FORGEJO_PAT }}" | docker login git.elijahkuntz.com -u "${{ gitea.actor }}" --password-stdin
- name: Install locked dependencies
run: npm ci
- name: Run tests
run: npm test
- name: Validate Prisma schema and migration drift
run: |
npx prisma validate
DATABASE_URL=file:./ci-migration.test.db npx prisma migrate deploy
npx prisma migrate diff --from-migrations prisma/migrations --to-schema prisma/schema.prisma --exit-code
- name: Build application
run: npm run build
- name: Lint changed source files
run: |
if BASE=$(git rev-parse HEAD^ 2>/dev/null); then
git diff --name-only --diff-filter=ACMR -z "$BASE" HEAD -- '*.ts' '*.tsx' '*.js' '*.mjs' |
xargs -0 -r npx eslint
else
git ls-files -z -- '*.ts' '*.tsx' '*.js' '*.mjs' |
xargs -0 -r npx eslint
fi
- name: Build and smoke production image
run: |
IMAGE_PATH=$(echo "git.elijahkuntz.com/${{ gitea.actor }}/${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]')
IMAGE_SHA="$IMAGE_PATH:${{ github.sha }}"
CI_SECRET=$(openssl rand -hex 32)
SMOKE_CONTAINER="study-smoke-$(openssl rand -hex 8)"
cleanup_smoke() {
docker rm -f "$SMOKE_CONTAINER" > /dev/null 2>&1 || true
}
trap cleanup_smoke EXIT
docker build -t "$IMAGE_SHA" .
docker run -d --name "$SMOKE_CONTAINER" -e SESSION_SECRET="$CI_SECRET" -e ALLOW_INITIAL_SETUP=true "$IMAGE_SHA"
for attempt in $(seq 1 30); do
if [ "$(docker inspect --format='{{.State.Health.Status}}' "$SMOKE_CONTAINER")" = "healthy" ]; then break; fi
sleep 2
done
test "$(docker inspect --format='{{.State.Health.Status}}' "$SMOKE_CONTAINER")" = "healthy"
docker exec "$SMOKE_CONTAINER" node -e "fetch('http://127.0.0.1:3726/login').then(r => { if (!r.ok) process.exit(1) }).catch(() => process.exit(1))"
docker exec "$SMOKE_CONTAINER" node -e "fetch('http://127.0.0.1:3726/api/auth/setup-status').then(async r => { const body = await r.json(); if (!r.ok || body.setupRequired !== true) process.exit(1) }).catch(() => process.exit(1))"
cleanup_smoke
trap - EXIT
docker tag "$IMAGE_SHA" "$IMAGE_PATH:latest"
docker push "$IMAGE_SHA"
docker push "$IMAGE_PATH:latest"

9
.gitattributes vendored
View file

@ -1,2 +1,7 @@
# Prisma 7 generated doc comments contain trailing spaces; do not hand-edit generated output.
src/generated/prisma/** -whitespace
* text=auto eol=lf
*.png binary
*.jpg binary
*.pdf binary
*.docx binary
*.pptx binary
*.woff2 binary

68
.gitignore vendored
View file

@ -1,53 +1,17 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.*
.yarn/*
!.yarn/patches
!.yarn/plugins
!.yarn/releases
!.yarn/versions
# testing
/coverage
/.test-databases/
# local SQLite study data
/dev.db
/dev.db-journal
/dev.db-shm
/dev.db-wal
/data/
/study.db
/study.db-journal
/study.db-shm
/study.db-wal
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# env files (can opt-in for committing if needed)
.env*
# vercel
.vercel
# typescript
node_modules/
dist/
data/
.data/
*.db
*.db-wal
*.db-shm
*.db-journal
.env
.env.*
!.env.example
coverage/
.vite/
*.tsbuildinfo
next-env.d.ts
.DS_Store
npm-debug.log*
.test-data/

350
AGENTS.md
View file

@ -1,350 +0,0 @@
# Study Desk - AI Contributor Guide
This file is the operating guide for AI agents and human contributors working in this repository.
## Project summary
Study Desk is a self-hosted, single-user study application built with Next.js, React, TypeScript, Prisma, and SQLite.
Core capabilities:
- Flashcard decks with Markdown content, self-grading, shuffle persistence, resume state, and card management
- Practice quizzes with multiple-choice and SATA questions, partial-credit scoring, rationales, category breakdowns, retakes, and attempt history
- JSON import, repair, validation, and generation workflows
- Material groups for organizing decks and quizzes
- Password-protected private pages
- Public, read-only share links for decks, quizzes, and groups
- Responsive light and dark themes
This is intentionally a single-user container, not a multi-tenant SaaS application. There is no `User` model. Content ownership is implicit in the local installation.
When documentation conflicts with implementation, prefer:
1. Current source code
2. `prisma/schema.prisma`
3. Applied migrations
4. This file
5. Older planning documents and the stock README
## Mandatory working rules
- Inspect `git status --short` before editing.
- Preserve unrelated user changes.
- Keep changes scoped to the requested behavior.
- Do not use destructive commands such as `git reset --hard` or `git checkout --` unless explicitly requested.
- Prefer narrow, patch-based edits over rewriting entire files.
- Do not commit secrets, `.env` files, database contents, generated credentials, or private study material.
- Do not manually edit generated Prisma files under `src/generated/prisma/`.
- Do not modify an already-applied migration. Change `prisma/schema.prisma`, then create a new migration.
- Prefer TypeScript types over `any`.
- Add dependencies only when the existing stack cannot reasonably solve the problem.
- When dependencies change, update `package-lock.json` and verify the build.
- Separate UI, route handling, services, and low-level utilities by responsibility.
- Ask a clarifying question only when missing information would materially change the implementation or create a risky assumption.
## Stack and source of truth
Confirm exact versions in `package.json`. The project currently uses:
- Next.js 16 App Router
- React 19
- TypeScript in strict mode
- Tailwind CSS v4
- Prisma 7 with SQLite and `@prisma/adapter-better-sqlite3`
- `iron-session` for encrypted session cookies
- Argon2 password hashing
- Zod validation
- `react-markdown` with `remark-gfm`
- `@dnd-kit` for sortable UI
- Node 22 in Docker
Important files:
- `package.json` - scripts and dependency versions
- `prisma/schema.prisma` - database source of truth
- `src/lib/db.ts` - shared Prisma client
- `src/lib/auth.ts` - session helpers
- `src/services/` - business logic and database access
- `src/app/api/` - route handlers
- `src/app/globals.css` - design tokens and global styling
- `src/config/studyModes.ts` - shared study-mode navigation
- `SKILL.md` - project workflow for adding API routes
- `study-app-implementation-plan.md` - historical architectural context only
- `README.md` - currently not authoritative
For framework-sensitive changes involving routing, layouts, dynamic params, middleware, server/client boundaries, or framework APIs, follow nearby repository patterns and consult version-matched Next.js documentation. Do not perform unrelated framework migrations.
## Local development
For a clean checkout:
```bash
npm ci
npx prisma generate
npm run dev
```
On Windows PowerShell, use `npm.cmd` and `npx.cmd` if script execution policy blocks the default shims.
Useful scripts:
```bash
npm run dev
npm run build
npm run start
npm run lint
```
There is currently no configured test runner. Use focused manual verification plus `npm run build`. Run lint as well, but distinguish pre-existing failures from failures introduced by the current change.
## Environment and authentication
Common environment variables:
- `DATABASE_URL` - SQLite URL
- `SESSION_SECRET` - `iron-session` secret, at least 32 characters outside throwaway local development
- `SECURE_COOKIES` - marks the session cookie secure when set to `true`
- `ADMIN_PASSWORD_HASH` - present in `.env.example`, but currently not used by the login flow
Current password behavior:
1. On first login, the submitted password is Argon2-hashed.
2. The hash is stored in the `Setting` row `admin_password_hash`.
3. Later logins verify against that stored hash.
Do not change password provisioning casually. Any change must update login, setup-status behavior, deployment documentation, and security handling together.
Authentication state is stored in the `study-app-session` cookie.
Public paths include:
- `/login`
- `/api/auth/*`
- `/shared/*`
- Static assets and Next internals
Everything else requires authentication unless explicitly designed and reviewed as public.
Do not log passwords, secrets, full share tokens, or private study content.
## Database and migration safety
The local `dev.db` may contain real study data.
- Do not delete, reset, replace, or recreate it during normal work.
- Back up the database before destructive migration work.
- Use the shared Prisma client from `@/lib/db`.
- Do not create additional `PrismaClient` instances in request handlers or components.
- Use transactions when partial writes would leave inconsistent state.
- Return 404 responses for missing records instead of exposing raw Prisma errors.
After changing the Prisma schema:
```bash
npx prisma generate
npx prisma migrate dev --name describe_the_change
```
Use `npx prisma migrate deploy` for deployed environments.
## Architecture conventions
### Server and client components
- Use Server Components by default.
- Add `"use client"` only when state, effects, event handlers, browser APIs, drag and drop, or client-side fetching are required.
- Keep database and authentication modules out of client import graphs.
- Do not access `window`, `localStorage`, `navigator`, or clipboard APIs from Server Components.
- Prefer `Link` for ordinary internal navigation.
- Use `useRouter` when navigation follows an action or must be imperative.
- Follow nearby Next.js 16 patterns for dynamic params and route context types.
### API routes and services
Preferred route-handler flow:
1. Parse and validate external input.
2. Delegate business logic to focused service or utility functions.
3. Return a useful status and JSON response.
Business logic and Prisma calls normally belong in `src/services/` or a focused `src/lib/` module. Reuse existing services before creating new abstractions.
Some older routes still contain direct Prisma calls or loose input handling. Improve the requested path without turning a small task into a broad refactor.
### Validation
- Reuse existing Zod schemas where available.
- Never rely only on client-side validation.
- Validate IDs, required strings, enums, arrays, and import payloads at the API boundary.
- Keep stored JSON parseable.
- Use `jsonRepair.ts` only for user- or LLM-generated import text, not to hide malformed database state.
## UI and accessibility
Use the existing design system in `src/app/globals.css`.
Common tokens include:
- `bg-bg-base`
- `bg-bg-surface`
- `bg-bg-surface-alt`
- `text-text-heading`
- `text-text-secondary`
- `text-text-muted`
- `primary`
- `primary-hover`
- `border`
- `border-light`
- `success`
- `error`
Preserve:
- Light and dark themes
- Responsive mobile layouts
- Existing card, modal, shadow, and rounded-corner language
- `editorial-title` for major serif headings
- Manrope for interface text
- Existing Markdown rendering through `ReactMarkdown`, `remarkGfm`, and `.markdown-content`
- Existing `@dnd-kit` patterns for sortable content
Accessibility requirements:
- Use semantic buttons and links.
- Preserve visible keyboard focus states.
- Give icon-only controls an accessible name with visible text or `aria-label`.
- Keep interactive controls keyboard accessible.
- Avoid horizontal overflow on study pages, library cards, modals, and action rows.
- Do not attach drag listeners to nested action buttons.
## Domain invariants
### Flashcards
- A deck contains ordered `Flashcard` rows with `front` and `back`.
- Study progress stores order and results as JSON.
- Study mode may be sequential or shuffled.
- Saved progress may contain stale card IDs after deletion. Resume helpers must filter and clamp stale IDs safely.
- Preserve both study and manage views.
- Preserve existing sharing entry points when adding new ones.
### Quizzes
- A quiz contains ordered questions with ordered answer options.
- `MULTIPLE_CHOICE` questions must have exactly one correct option.
- `SATA` questions may have multiple correct options.
- SATA uses partial-credit scoring in `src/lib/scoring.ts`.
- Partial retakes score only questions present in the submitted answer set.
- Attempts persist score, maximum score, submitted answers, retake state, and completion time.
- In-progress answers and position persist through `/api/progress`.
### Imports and generation
Flashcard import shape:
```json
{
"type": "flashcards",
"deckName": "Deck name",
"description": "Optional description",
"cards": [{ "front": "...", "back": "..." }]
}
```
Quiz import shape:
```json
{
"type": "quiz",
"quizName": "Quiz name",
"description": "Optional description",
"questions": [
{
"type": "multiple_choice",
"prompt": "...",
"category": "topic",
"options": [{ "text": "...", "correct": true }],
"rationale": "..."
}
]
}
```
The app does not call an LLM directly. The Generate UI prepares instructions and accepts generated JSON from the user.
Preserve Markdown inside JSON string values and render it through existing Markdown components.
Keep the current separation between `ImportModal`, `CreateTab`, `ImportTab`, and `GenerateTab`.
### Sharing
- `ShareLink` may target a `DECK`, `QUIZ`, or `GROUP`.
- Reuse `src/components/ui/ShareMenu.tsx`.
- Public routes are `/shared/<classSlug>/<type>/<token>`.
- Public viewers are read-only.
- Public viewers must not write authenticated progress.
- Validate that token, type, and class slug agree before returning shared content.
- A deck or quiz may appear publicly shared through its group. Preserve inherited group-share behavior and `?itemId=...` handling.
## Verification checklist
Before editing:
- Read `git status --short`.
- Inspect the target component, route, service, and schema paths relevant to the requested behavior.
- Check whether the change affects authentication, public sharing, migrations, generated Prisma output, or real study data.
During implementation:
- Keep route, service, utility, and component responsibilities clear.
- Add validation for new external input.
- Reuse existing design tokens, components, and services.
- Consider loading, error, empty, mobile, keyboard, light-theme, and dark-theme states.
- Avoid unrelated cleanup.
After editing:
```bash
npm run lint
npm run build
```
If lint already fails, report baseline failures separately.
For UI work, manually verify:
1. The affected authenticated route
2. The corresponding public/shared route when applicable
3. Loading, error, and empty states
4. Study resume and restart behavior when relevant
5. Drag and drop when relevant
6. Light and dark themes
7. A narrow mobile viewport
8. No horizontal overflow
9. Keyboard access for changed controls
## Known repository conditions
- `README.md` is still mostly stock create-next-app documentation.
- `study-app-implementation-plan.md` contains useful historical context but may be stale.
- `task.md` records completed implementation phases.
- `SKILL.md` describes the expected workflow for new API routes.
- The current lint baseline includes existing errors such as `any` usage and React effect-rule violations.
- A successful `next build` is the primary repository-wide compile and type check.
- The current middleware convention produces a known deprecation warning.
- `docker-compose.yml` may publish port `3000`, while the production Docker image uses port `3726`. Verify the mapping before relying on production Compose.
- Do not silently fix known baseline issues as unrelated cleanup.
## Extending the application
For a genuinely new study mode:
1. Add or update the Prisma model and migration when persistence is required.
2. Add a focused service in `src/services/`.
3. Add API routes under `src/app/api/`.
4. Add page routes and client components.
5. Add the mode to `src/config/studyModes.ts` when it belongs in shared navigation.
6. Extend progress, import, scoring, or sharing only when required.
7. Verify authentication, public/private boundaries, mobile layout, and both themes.
Do not add a new mode by placing unrelated logic into `deckService.ts`, `quizService.ts`, or a large page component.

View file

@ -1 +0,0 @@
@AGENTS.md

View file

@ -1,47 +0,0 @@
# ---- base ----
FROM node:22-slim AS base
RUN apt-get update -y && apt-get install -y openssl && rm -rf /var/lib/apt/lists/*
# ---- deps ----
FROM base AS deps
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci
# ---- production dependencies ----
FROM deps AS runtime-deps
RUN npm prune --omit=dev
# ---- builder ----
FROM base AS builder
WORKDIR /app
COPY --from=deps /app/node_modules ./node_modules
COPY . .
RUN npx prisma generate
RUN npm run build
# ---- runner ----
FROM base AS runner
WORKDIR /app
ENV NODE_ENV=production
ENV PORT=3726
ENV HOSTNAME=0.0.0.0
ENV DATABASE_URL="file:/app/data/study.db"
RUN useradd --system --create-home appuser && mkdir -p /app/data && chown -R appuser:appuser /app
COPY --from=builder /app/.next/standalone ./
COPY --from=builder /app/.next/static ./.next/static
COPY --from=builder /app/public ./public
COPY --from=builder /app/prisma ./prisma
COPY --from=builder /app/prisma.config.ts ./
COPY --from=builder /app/scripts ./scripts
COPY --from=runtime-deps /app/node_modules ./node_modules
COPY docker-entrypoint.sh ./
RUN chmod +x docker-entrypoint.sh
# Running as root to avoid permission denied on Unraid host-mounted volumes
EXPOSE 3726
VOLUME ["/app/data"]
HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=6 \
CMD ["node", "-e", "fetch('http://127.0.0.1:3726/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
ENTRYPOINT ["./docker-entrypoint.sh"]

115
README.md
View file

@ -1,115 +0,0 @@
Study Desk is a self-hosted, single-user study application.
## Local development
```powershell
npm.cmd ci
npx.cmd prisma generate
npm.cmd run dev
```
The automated test harness always creates a uniquely named database under
`.test-databases/`, applies committed migrations, and removes the database after
the run. It refuses `dev.db`, `/app/data/study.db`, existing databases, and paths
that are not explicitly test-named.
## Production container
Set a unique session secret of at least 32 characters. Production startup fails
before migrations or the HTTP server when the secret is missing, short, or the
published development fallback.
```powershell
$env:SESSION_SECRET = '<at-least-32-random-characters>'
docker compose -f docker-compose.yml up --build
```
The host listens on `http://localhost:3000`; the container listens on port 3726.
Development Compose is intentionally explicit and is never auto-merged:
```powershell
docker compose -f docker-compose.dev.yml up --build
```
For first-time production setup, either provide an Argon2 encoded
`ADMIN_PASSWORD_HASH`, or temporarily set `ALLOW_INITIAL_SETUP=true` for the
one-time setup request. Remove the flag after setup. Production HTTP password
reset initiation is unavailable; run this on the server instead:
```powershell
npm.cmd run auth:reset
```
If TLS terminates at a trusted reverse proxy, forward requests only from that
proxy and set `SECURE_COOKIES=true`. Leave it false for intentional plain HTTP;
Secure cookies cannot be used over plain HTTP.
## Database backup, migration adoption, and restore
Never run migration tests against the only copy of a study database. Create a
lock-safe SQLite backup through the backup API and verify it before deployment:
```powershell
$env:DATABASE_URL = 'file:./data/study.db'
npm.cmd run db:backup -- --output ./backups/study-before-upgrade.db
npm.cmd run db:preflight
npx.cmd prisma migrate deploy
```
The material-group preflight reports one of four states:
- `APPLY`: migration history is complete and the group schema is absent; run
`prisma migrate deploy` normally.
- `ADOPT`: prior migrations are tracked and the database exactly matches the
intended schema of one or more unrecorded committed migrations. This includes
recovery from P3018 when an already schema-pushed table or column exists.
After reviewing the verified backup, explicitly run
`npm run db:preflight -- --resolve --backup <backup-path>`.
- `CURRENT`: migration and schema already agree.
- `CONFLICT`: partial or unknown state. Stop and recover manually; do not use
`db push`, reset, edit applied migrations, or mark anything applied.
The production entrypoint runs this preflight before `prisma migrate deploy`.
It refuses `ADOPT` and `CONFLICT` states without changing migration history;
adoption always remains an explicit backup-gated operator action.
Restore drill: stop Study Desk, keep the damaged database as evidence, restore
the verified backup to a new path, run SQLite `integrity_check` plus
`npm run db:preflight`, start the same prior application image against the
restored path, and compare class/deck/quiz counts plus representative content.
Only swap the production path after those checks pass.
## Getting Started
First, run the development server:
```bash
npm run dev
# or
yarn dev
# or
pnpm dev
# or
bun dev
```
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
## Learn More
To learn more about Next.js, take a look at the following resources:
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
## Deploy on Vercel
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.

View file

@ -1,24 +0,0 @@
---
name: adding-an-api-route
description: Use when adding a new Next.js API route to this project. Keeps routes thin and logic in the right layer instead of inline in the route file.
---
# Adding a new API route
1. Check `study-app-implementation-plan.md`'s Section 9 file tree first. If
this route is already named there, follow that path and method exactly
rather than inventing a different shape.
2. Create the route file under `src/app/api/<resource>/route.ts`, or
`src/app/api/<resource>/[id]/route.ts` for a single-resource endpoint.
3. The route handler does exactly three things: validate the request body
(reuse a Zod schema from `lib/validation/` if one already exists for this
shape, write one if it doesn't), call one function in the matching
`services/<resource>Service.ts`, and return a `NextResponse`. No Prisma
calls and no business logic directly in the route file.
4. If the logic doesn't fit an existing service file, add a new exported
function to the closest matching one. Only create a new
`services/<resource>Service.ts` if this is genuinely a new resource, don't
bolt unrelated logic onto an existing service to avoid creating a file.
5. Run `npm run lint` before considering the route done. A failing
`max-lines` or `filename-case` error means something is in the wrong
place, not that the rule needs an exception.

View file

@ -1,67 +0,0 @@
# Confirmed Findings
Severity: Critical / High / Medium / Low / Observation. Confidence: Confirmed / High-confidence inference / Unverified risk.
Full structured detail for every finding (files, functions, scenario, expected/actual, root cause, impact, evidence, fix direction, complexity, regression test, runtime-confirmation flag) is in `FINDINGS.json` (validated). Source IDs from specialist auditors are in parentheses.
| ID | Severity | Confidence | Title | Area |
|---|---|---|---|---|
| FIN-01 (DBAUD-01, GRP-01) | **Critical** | Confirmed | Schema/migration drift: `MaterialGroup` table + `groupId` columns never migrated — fresh deployments' databases are incompatible with the app (verified: `no such column: Deck.groupId`, `no such table: MaterialGroup`) | DB/migrations |
| FIN-02 (AUTH-01, OPS-03) | **Critical** | High-confidence inference | Hardcoded fallback `SESSION_SECRET` (`"dev-session-secret-change-in-production-must-be-32-chars"`) in `src/lib/auth.ts:15` + `src/proxy.ts:8-9`; empty env var in shipped compose → forgeable session cookie → full auth bypass | Auth/deploy |
| FIN-03 (QUIZ-01, ADV-01) | **High** | Confirmed | In-viewer "Retake Missed" never sets `retakeIds` prop → persisted as a full attempt with a wrong, lower score; pollutes SEQUENTIAL progress | Quiz |
| FIN-04 (CARD-01, IMPT-01, QUIZ-06) | **High** | Confirmed | Stale card ids in saved progress never filtered on resume (`filterAndClampOrder` dead code) → deleting a card mid-session bricks the flashcard study session | Flashcards |
| FIN-05 (OPS-01) | **High** | Confirmed | Port mismatch: container listens on 3726, compose publishes 3000:3000 → production deployment unreachable | Deploy |
| FIN-06 (OPS-06) | **High** | High-confidence inference | No `.dockerignore`: Windows-host builds inject win32 native modules (better-sqlite3, argon2) into the Linux image | Deploy |
| FIN-07 (OPS-08) | **High** | Confirmed | `docker-compose.override.yml` auto-merges on plain `docker compose up` → production command silently runs dev mode | Deploy |
| FIN-08 (AUTH-04, ADV-09) | Medium | High-confidence inference | First-login provisioning takeover; `ADMIN_PASSWORD_HASH` env is dead config | Auth |
| FIN-09 (AUTH-02) | Medium | High-confidence inference | Proxy auth bypass for any path containing `.` (latent today; would expose any future non-UUID-id route) | Auth |
| FIN-10 (AUTH-03) | Medium | High-confidence inference | Password-reset abuse: token overwrite DoS, spoofable `x-forwarded-for` rate limits, unthrottled `complete` | Auth |
| FIN-11 (QUIZ-02) | Medium | High-confidence inference | Attempt route stores unvalidated `answersJson`; duplicate option ids inflate SATA credit; malformed shapes → 500 | Quiz |
| FIN-12 (QUIZ-03, FE-04) | Medium | Confirmed | Quiz Finish double-submit creates duplicate attempts; failure path silent | Quiz/FE |
| FIN-13 (QUIZ-04) | Medium | High-confidence inference | Progress PATCH can land after Finish's DELETE → stale "Continue" row for a completed quiz | Quiz |
| FIN-14 (CARD-02, FE-05, QUIZ-09) | Medium | High-confidence inference | Fire-and-forget, unsequenced progress autosaves: out-of-order PATCHes regress resume state; multi-tab last-write-wins | Flashcards/Quiz |
| FIN-15 (CARD-03, ADV-07) | Medium | Confirmed | Completed flashcard session never persisted as complete → resume re-shows the last graded card | Flashcards |
| FIN-16 (CARD-04) | Medium | Confirmed | SRS review endpoint enforces membership only — no due-date or `newCardsPerDay` enforcement server-side | SRS |
| FIN-17 (CARD-05, ADV-02) | Medium | Confirmed | Unbounded DB scans: whole `StudyActivity` table per fetch; all SRS card states per set; Navbar polls every 60 s | Perf/DB |
| FIN-18 (FE-01, GRP-03, FE-02, GRP-05) | Medium | Confirmed | Library optimistic mutations are fire-and-forget (no `res.ok`, no rollback); cross-group reorder omits origin group from payload | FE/Groups |
| FIN-19 (GRP-02, CARD-07, AUTH-06) | Medium | Confirmed | Reorder/create endpoints trust client `groupId`/`sortOrder` — no class/type/existence validation; cross-class group membership can leak content into another class's share link | Groups |
| FIN-20 (FE-03) | Medium | Confirmed | Shared quiz session resets and options reshuffle on any parent re-render (unstable inline `quiz` prop) | FE |
| FIN-21 (IMPT-02, FE-08, QUIZ-05, ADV-10) | Medium | Confirmed | `/api/progress` PATCH completely unvalidated; unguarded client `JSON.parse` of persisted progress crashes pages | Quiz/FE |
| FIN-22 (OPS-02) | Medium | High-confidence inference | Entrypoint migrate-deploy chain fragile: crash loop on conflicting DB state; runner prisma CLI install unverified/unpinned | Deploy |
| FIN-23 (OPS-05) | Medium | Confirmed | Non-reproducible installs: `npm install` (not `ci`); unpinned `prisma@^7.8.0` re-install re-resolves the whole tree in the runner | Deploy |
| FIN-24 (OPS-07) | Medium | Confirmed | No healthcheck, no backup, no documented recovery | Deploy |
| FIN-25 (OPS-10, TEST-10) | Medium | Confirmed | CI builds/pushes without tests, lint, prisma validate, or container smoke test — deployment-breaking defects ship green | CI |
| FIN-26 (QUIZ-07) | Low | Confirmed | SATA scoring divides by `correctIds.length` with no zero-guard → NaN | Quiz |
| FIN-27 (QUIZ-08) | Low | Confirmed | Historical review/category breakdown recompute scores from current content, not stored attempt | Quiz |
| FIN-28 (IMPT-03) | Low | Confirmed | Card add/edit endpoints bypass Zod: empty strings storable; CreateTab silently drops incomplete cards | Imports |
| FIN-29 (IMPT-04) | Low | Confirmed | No payload size/string-length/array-length caps on import schemas/routes; whitespace-only names accepted | Imports |
| FIN-30 (IMPT-05) | Low | Confirmed | SATA constraint mismatch: instructions say ≥2 correct, schema enforces ≥1 | Imports |
| FIN-31 (IMPT-06) | Low | Confirmed | Unvalidated `name`/`groupId` override fields on import POST routes → 500; PATCH routes mask failures as 404 | Imports |
| FIN-32 (GRP-04) | Low | Confirmed | Group deletion leaves duplicate sortOrder values in Uncategorized; no unique constraint/tie-break | Groups |
| FIN-33 (GRP-06) | Low | Confirmed | Keyboard users cannot move items between groups (empty `handleDragOver`) | Groups/a11y |
| FIN-34 (CARD-08) | Low | High-confidence inference | Restart deletes progress fire-and-forget; slow DELETE can remove the fresh session's progress row | Flashcards |
| FIN-35 (CARD-09) | Low | Confirmed | Concurrent first review of same new card → P2002 → generic 500 instead of 409 | SRS |
| FIN-36 (CARD-10) | Low | High-confidence inference | "Previous card" during the 350 ms grade animation races the pending timeout | Flashcards |
| FIN-37 (CARD-11) | Low | Confirmed | SRS set page never refreshes on focus → stale membership after deck deletion in another tab | SRS/FE |
| FIN-38 (CARD-06) | Low | Confirmed | Day boundary hardcoded to Arizona (UTC-7): "today"/new-card limit/streak roll over at 07:00 UTC | SRS |
| FIN-39 (AUTH-05) | Low | Confirmed | Proxy `destroy()` cookie-clear lost on redirect; layout check ignores `sessionGeneration` | Auth |
| FIN-40 (AUTH-08) | Low | Confirmed | Session cookie Secure flag off in shipped compose (plain HTTP, no TLS story) | Auth/deploy |
| FIN-41 (DBAUD-02, AUTH-07) | Low | Confirmed | `/api/share` accepts arbitrary `targetType` → repeatable junk all-NULL `ShareLink` rows | Auth/DB |
| FIN-42 (DBAUD-03) | Low | High-confidence inference | `dev.db` committed in git history (8 commits), remains in blobs on the LAN remote | Repo hygiene |
| FIN-43 (OPS-09) | Low | Confirmed | `.gitignore` gaps (`data/`, `study.db*`); scratch files `out.css`/`temp.css`/`test.css` tracked | Repo hygiene |
| FIN-44 (FE-07, ADV-04) | Low | Confirmed | Dashboard `fetchClasses` no `.catch`/`res.ok` → unhandled rejection, misleading empty state (or render crash) | FE |
| FIN-45 (FE-09) | Low | Confirmed | Effect fetches without `.catch` in GenerateTab and ShareMenu | FE |
| FIN-46 (FE-10) | Low | High-confidence inference | Cross-class navigation fetch race renders wrong-class data; module cache never invalidated | FE |
| FIN-47 (ADV-03) | Low | Confirmed | Logout has no error handling — failed logout strands the user | FE |
| FIN-48 (ADV-05) | Low | Confirmed | `slugify` can yield an empty slug (unreachable class); renames never update the URL slug | Classes |
| FIN-49 (ADV-08) | Low | High-confidence inference | Shared-viewer localStorage keys item-scoped, not token-scoped → session bleed across tokens | Sharing/FE |
| FIN-50 (GRP-07) | Observation | Confirmed | Group `sortOrder` uses inverted desc convention with unvalidated PATCH — latent trap | Groups |
| FIN-51 (FE-11) | Observation | Confirmed | Collapsed-groups state read in effect → one-frame expand flash | FE |
| FIN-52 (AUTH-10) | Observation | Confirmed | Shared quiz links ship the full answer key to anonymous viewers — by design, worth a warning | Sharing |
| FIN-53 (TEST-01) | Low | Confirmed | False-confidence test: "distinct valid state for every rating" never asserts distinctness | Tests |
## Notes on consolidation
- Duplicate findings across specialist reports were merged under one root cause (see `sourceIds`).
- FE-06 ("all /api/* routes unauthenticated") was **refuted** by the adversarial review — `src/proxy.ts` IS the compiled middleware and enforces session checks; the residual truth is FIN-09 (dot-bypass). See REJECTED_FINDINGS.md.
- OPS-02's fresh-volume crash-loop framing was weakened by the adversarial review (on a fresh volume `migrate deploy` succeeds; the app then fails per FIN-01) — reframed as FIN-22.
- FIN-18 severity downgraded from the frontend auditor's High to Medium after adversarial review (no visible break for a single user; silent divergence self-heals on reload).

View file

@ -1,38 +0,0 @@
# Coverage Map
Mapping of repository areas → audit status → specialist report reference. Statuses: done / done+verified (parent-verified with runtime evidence) / partial.
| Area | Files | Auditor | Status | Report ref |
|---|---|---|---|---|
| Prisma schema/migrations/transactions/cascades | prisma/schema.prisma, prisma/migrations/*, prisma.config.ts, src/lib/db.ts, services | DB specialist (task-1) | done+verified (drift reproduced) | sa_20260806_054030_000000000_5ed877563a0d |
| Quiz scoring/attempts/history/retakes/progress | src/lib/scoring.ts, quizService, progressService, quizzes/*, api/quizzes/**, api/progress | Quiz specialist (task-2) | done+verified (server scoring path re-checked) | sa_20260806_054030_000000000_e75a8ce575f9 |
| Flashcards/SRS/progress/deletion/ordering | flashcards/*, spaced-repetition/*, cardService, deckService, spacedRepetitionService, api/cards|decks|spaced-repetition-sets|progress | Flashcard specialist (task-3) | done | sa_20260806_054030_000000000_a222804c133c |
| Auth/protected routes/API authz/sharing | src/proxy.ts, lib/auth.ts, authService, shareService, shareMetadata, api/auth/**, api/share, shared/*, all api routes | Auth specialist (task-4) | done+verified (proxy + fallback secret read directly) | sa_20260806_054030_000000000_aab0c59dbad3 |
| Imports/exports/generation/Zod/malformed input | lib/validation/*, jsonRepair, components/import/*, api/decks|quizzes|material-groups|settings | Import specialist (task-5) | done | sa_20260806_054030_000000000_5e72c2713989 |
| Material groups/library/drag-drop/deletion/orphans | api/material-groups/**, reorder routes, library pages, classService | Groups specialist (task-6) | done | sa_20260806_054030_000000000_9ae1f3dfa282 |
| Frontend state/refresh/races/localStorage/boundaries | all components + pages + shared viewers | Frontend specialist (task-7) | done | sa_20260806_054030_000000000_0e17fd155702 |
| Test quality/coverage gaps | *.test.ts, vitest.config.ts, package.json, CI workflow | Test specialist (task-8) | done | sa_20260806_054030_000000000_cd3652deb3f7 |
| Docker/startup/env/scripts/production/recovery | Dockerfile, docker-entrypoint.sh, compose files, package.json, prisma.config.ts, next.config.ts, .gitignore, CI | Ops specialist (task-9) | done | sa_20260806_054030_000000000_f49cdaacaabf |
| Adversarial review (challenge Critical/High + missed issues) | (all of the above + lightly-covered files) | Adversarial specialist (task-10) | done — 9/10 challenged findings confirmed/weakened, FE-06 refuted, 10 new issues added | sa_20260806_055608_000000000_c550c742d5dc |
## Parent-level verification performed (see VERIFICATION_LOG.md and audit-results/tmp/)
- `npx prisma validate` — PASS
- `npm test` — PASS (30 tests: 20 arcade + 10 spacedRepetition)
- `npm run lint` — 28 baseline problems (9 errors / 19 warnings), all pre-existing
- `npm run build` — PASS (route table incl. "ƒ Proxy (Middleware)")
- `prisma migrate deploy` on fresh temp DB — PASS (creates DB **missing** MaterialGroup/groupId → FIN-01 drift confirmed)
- Temp DB introspection (`audit-results/tmp/inspect-db.cjs`) — `MaterialGroup` table and `groupId` columns absent
- Prisma-shaped SQL against temp DB (`audit-results/tmp/reproduce-drift.cjs`) — `no such column: Deck.groupId`, `no such table: MaterialGroup`, `no such column: ShareLink.groupId`
- `FINDINGS.json` validity — PASS (53 findings; validator `audit-results/tmp/validate-findings.cjs`)
## Coverage gaps (areas NOT fully audited)
1. **Arcade feature internals** — intentionally out of scope (may be removed). Only checked for shared-DB/build/deploy impact (Arcade models/indexes verified in migrations).
2. **Real browser interaction** — no browser available in the audit environment: drag-drop, resume flows, share pages, keyboard a11y, theme behavior were code-verified only; findings needing runtime confirmation are flagged in UNVERIFIED_RISKS.md.
3. **Container build/run** — docker CLI unavailable: OPS-02/FIN-22, FIN-06, FIN-24 need a container run to observe exact failure modes.
4. **git history deep-dive** — limited to drift commit (7af0935), dev.db commits, and migration commits; other commits not diffed line-by-line.
5. **Committed dev.db blob contents** — could not be fully enumerated (no sqlite3 CLI; blob grep only; no admin_password hash found in checked blobs).
6. **Live dev.db** — none exists in the workspace; runtime behavior on a populated database (perf findings FIN-17) not measured.
7. **Network/remote behavior** — the Forgejo remote and CI execution were not reachable; CI findings are static analysis of `.forgejo/workflows/build.yml`.
8. **External services** — none (app calls no LLM/third-party APIs).

View file

@ -1,69 +0,0 @@
# Study Desk — Overnight Read-Only Audit: Executive Summary
Status: **COMPLETE** (2026-08-06)
## Scope and method
Comprehensive read-only audit of the Study repository (Next.js 16 / React 19 / Prisma 7 / SQLite, self-hosted single-user study app). Ten specialist subagents audited: (1) Prisma schema/migrations/transactions/cascades/drift, (2) quiz scoring/attempts/retakes/progress, (3) flashcards/SRS/progress/ordering, (4) auth/protected routes/API authorization/sharing, (5) imports/exports/generation/Zod validation, (6) material groups/library/drag-drop/deletion, (7) frontend state/races/localStorage/boundaries, (8) test quality, (9) Docker/env/production/recovery, (10) adversarial review challenging every Critical/High finding and hunting missed issues. The parent agent independently re-verified all Critical/High claims (code tracing, git history, temp-DB migration chain, DB introspection, Prisma-shaped SQL probes). The Arcade feature set was excluded per scope unless it affects the main app, shared dependencies, database integrity, build, or deployment.
## Constraints honored
- **No source code, tests, migrations, config, package files, lockfiles, docs, or databases modified.** Final `git status --short`: only untracked `.reasonix/` and `audit-results/`; `git diff --check` clean; `src/generated/prisma/` byte-identical before/after all prisma commands (sha256).
- All reports and diagnostic artifacts live under `audit-results/` (reports + `tmp/`: `audit-migration.db`, `inspect-db.cjs`, `reproduce-drift.cjs`, `validate-findings.cjs`, `generated-before.sha256`).
- Existing databases: none found in the workspace; the migration-chain test used a brand-new temp DB.
## Baseline verification
| Check | Result |
|---|---|
| `npm test` | ✅ PASS — 30 tests / 6 files (20 arcade out of scope, 10 spacedRepetition) |
| `npm run lint` | ⚠️ 28 problems (9 errors, 19 warnings) — all pre-existing baseline |
| `npm run build` | ✅ PASS — full route table incl. "ƒ Proxy (Middleware)" |
| `npx prisma validate` | ✅ PASS |
| Migration chain on fresh temp DB | ✅ PASS — 4/4 migrations apply, but the resulting DB is **missing** `MaterialGroup` + `groupId` columns (drift, see below) |
## Findings (53 total: 2 Critical, 5 High, 18 Medium, 25 Low, 3 Observation)
Full detail in `FINDINGS.json` (validated) and `CONFIRMED_FINDINGS.md`.
### Critical
1. **FIN-01 — Schema/migration drift (Confirmed, runtime-verified).** `prisma/schema.prisma` defines `MaterialGroup` and `Deck/QuizSet/ShareLink.groupId` (added by commit 7af0935, which also rewrote a committed `dev.db` via `prisma db push`) but **no migration creates them**. Every DB built by `prisma migrate deploy` — the `predev` hook and the Docker entrypoint — lacks these objects; the generated client then fails every Deck/QuizSet/ShareLink/MaterialGroup query (`no such column: Deck.groupId`, `no such table: MaterialGroup` — reproduced against a temp DB). **Any fresh install/deploy is non-functional.** The app only works on the developer's db-push-synced local database.
2. **FIN-02 — Hardcoded session-secret fallback (High-confidence inference).** `src/lib/auth.ts:15` and `src/proxy.ts:8-9` fall back to the public constant `"dev-session-secret-change-in-production-must-be-32-chars"` whenever `SESSION_SECRET` is unset — and the shipped `docker-compose.yml` passes `${SESSION_SECRET}`, which is empty by default. Anyone with the source can forge an iron-session cookie (`{isAuthenticated:true, sessionGeneration:1}`) and fully bypass authentication on such deployments.
### High
3. **FIN-03 — "Retake Missed" persists as a full attempt with a wrong lower score** (in-viewer retake never sets the `retakeIds` prop; server scores all unanswered questions as 0) and pollutes SEQUENTIAL progress.
4. **FIN-04 — Deleting a card mid-session bricks the flashcard study session** (`filterAndClampOrder` is dead code; resume never filters stale ids; no skip UI).
5. **FIN-05 — Production compose is unreachable**: container listens on 3726 (`ENV PORT=3726`), compose publishes 3000:3000.
6. **FIN-06 — No `.dockerignore`**: Windows-host builds inject win32 native modules (better-sqlite3, argon2) into the Linux image.
7. **FIN-07 — `docker-compose.override.yml` auto-merges**: plain `docker compose up` silently runs dev mode in production.
### Medium (selected)
First-login provisioning takeover + dead `ADMIN_PASSWORD_HASH` (FIN-08); proxy auth bypass for dot-containing paths (FIN-09); password-reset abuse (FIN-10); unvalidated attempt `answersJson` (FIN-11); finish double-submit duplicates attempts (FIN-12); progress PATCH/DELETE race (FIN-13); unsequenced progress autosaves (FIN-14); completed session resumes on last card (FIN-15); SRS review endpoint lacks due/new-card-limit enforcement (FIN-16); unbounded DB scans (FIN-17); library optimistic mutations without rollback (FIN-18); unvalidated `groupId`/`sortOrder` incl. cross-class group membership (FIN-19); shared-quiz reshuffle on re-render (FIN-20); unvalidated `/api/progress` + unguarded `JSON.parse` crashes (FIN-21); entrypoint migrate fragility (FIN-22); non-reproducible installs (FIN-23); no healthcheck/backup (FIN-24); CI ships without tests/lint/prisma/smoke gates (FIN-25).
### Low / Observation
25 Low (e.g. SATA div-by-zero NaN, historical score recompute, card-edit endpoints without Zod, no import size caps, sortOrder duplicates after group deletion, keyboard cross-group moves impossible, restart DELETE race, P2002 → 500, git history containing `dev.db`, `.gitignore` gaps, dashboard fetch crash, empty slug from `slugify`, shared-viewer localStorage key collisions) and 3 Observations (group sortOrder desc convention, collapse flash, shared quizzes ship the answer key by design).
## Verification highlights
- FIN-01 disproved attempts: checked whether the client avoids selecting `groupId` (it doesn't — generated `Deck.ts` includes it in every payload), whether pages swallow errors (they don't), and whether any migration creates the objects (grep = 0 hits). Confirmed via live probes instead: see `audit-results/tmp/reproduce-drift.cjs` output in VERIFICATION_LOG.md.
- Adversarial review **refuted** the frontend auditor's "no auth on API routes" claim (proxy is compiled middleware — verified in `.next` artifacts), **weakened** the entrypoint crash-loop framing (fresh-volume `migrate deploy` succeeds), and downgraded the optimistic-reorder severity. It independently confirmed all other Critical/High findings and added 10 missed issues, merged into the final list.
## Repository areas NOT fully audited
1. Arcade feature internals (out of scope; shared-DB/build/deploy impact only — verified in migrations).
2. Real browser interaction (no browser available): drag-drop, resume flows, share pages, keyboard a11y, themes — code-verified; flagged for runtime confirmation in UNVERIFIED_RISKS.md.
3. Container build/run (docker CLI unavailable): FIN-06, FIN-22, FIN-24 need a container run.
4. Deep git-history diff of every commit; full content audit of the committed `dev.db` blobs (no sqlite3 CLI; no credentials found in the checked blobs).
5. Runtime performance on a populated database (FIN-17 not measured).
6. CI/remote execution (Forgejo pipeline analyzed statically).
## Recommended priority (when fixes are authorized)
1. **FIN-01** — generate and commit the missing migration (`prisma migrate dev --name add_material_groups`); add a CI drift gate. Everything else depends on a working fresh deployment.
2. **FIN-02 / FIN-05 / FIN-06 / FIN-07** — harden the deployment path: require `SESSION_SECRET`, fix the port mapping, add `.dockerignore`, rename the dev override, and smoke-test the image in CI.
3. **FIN-03 / FIN-04** — the two most user-visible application bugs (wrong retake scores; stuck flashcard sessions).
4. Then the Medium cluster (validation, races, SRS enforcement, progress handling) and finally Low items + TEST_GAPS.md.

File diff suppressed because it is too large Load diff

View file

@ -1,15 +0,0 @@
# Rejected Findings
Findings that were investigated and disproven, or removed as duplicate/stylistic/insignificant, with the reason.
| ID | Original claim | Rejection reason |
|---|---|---|
| FE-06 | "All `/api/*` routes are unauthenticated (no middleware, no per-route checks) — curl without a session cookie returns data" | **Refuted by adversarial review + parent verification.** `src/proxy.ts` IS the compiled Next.js 16 middleware (verified in `.next/server/functions-config-manifest.json`, `middleware.js`, and the compiled chunk; build output shows "ƒ Proxy (Middleware)"). It decrypts the iron-session cookie and enforces `isAuthenticated` + `sessionGeneration` on every path except `/login`, `/api/auth`, `/shared`, `/_next*`, `/favicon*`, and paths containing `.`. The auditor searched for `middleware.ts` (the Next 15 name) and missed `proxy.ts`. The residual truth is the dot-bypass (FIN-09). The claim that "no frontend code handles 401s because there is no auth" is also wrong — there is no 401 because the proxy redirects to /login. |
| QUIZ-06 (standalone) | "Quiz resume with stale question ids crashes the viewer" | Merged into FIN-04. The quiz-side path is currently unreachable (no question editing/deletion exists in the app; quiz deletion cascades progress), so it is a latent variant of the same root cause (dead `filterAndClampOrder`), not a separate live defect. |
| OPS-02 (original framing) | "Fresh-volume deployment crash-loops because `migrate deploy` fails" | **Weakened by adversarial review.** On a fresh volume `prisma migrate deploy` *succeeds* (verified against a temp DB); the fresh-volume failure is FIN-01 (schema drift → app 500s while the container runs). The crash loop requires a secondary trigger (db-push DB, tampering, lock). Reframed as FIN-22 with the crash-loop path demoted. |
| FE-01 (original severity) | Optimistic drag-drop reorder without rollback rated High | **Severity downgraded to Medium by adversarial review.** Mechanics confirmed (fire-and-forget fetch, in-place mutation, origin group omitted from cross-group payload) but impact analysis: sortOrder gaps only, silent divergence self-heals on reload, no visible break for a single user. Merged into FIN-18. |
| AUTH-06 (standalone) | "Cross-class group membership via reorder API exposes another class's content" | Merged into FIN-19 (same root cause: `groupId`/`sortOrder` never validated server-side). The sharing-exposure angle is recorded as impact of FIN-19. |
| DBAUD-04 | "Relative SQLite path resolution may cause dev dual-DB split-brain" | **Disproven.** CLI (`prisma.config.ts`) and app adapter both resolve `file:./dev.db` cwd-relative; `predev` and the app run from the project root → same file. Docker uses absolute `file:/app/data/study.db`. Recorded as verified-safe in UNVERIFIED_RISKS.md. |
| DBAUD-05/06/07, QUIZ-08-verdicts, GRP-01-verdicts (various) | "Cascades broken / transactions partial / StudyProgress duplicates via NULL tricks / id regeneration on card edit / XSS via markdown / client-server boundary violations" | **Disproven** by the respective auditors with evidence: FKs enforced by compile default; nested creates implicit-transactional; service-level guards block duplicates; PATCH edits in place (ids preserved); no `rehype-raw`; no server-module imports in client components. Recorded as verified-safe. |
| FE-11 (as bug) | "Collapsed-groups expand flash" | Retained only as Observation (FIN-51) — cosmetic, no data impact. |
| Various | Style preferences, harmless duplication, generic best practices, speculative micro-optimizations, Arcade-only issues | Excluded per audit scope rules. |

View file

@ -1,340 +0,0 @@
# Study Desk audit evaluation and remediation plan
Date: 2026-08-06
Scope: independent evaluation of `CONFIRMED_FINDINGS.md`, `FINDINGS.json`, the supporting audit reports, and the current repository source.
Change policy for this evaluation: this plan is the only new file; no application source, tests, migrations, configuration, generated files, database, or existing audit artifact was changed.
## Executive verdict
The audit is credible and the central findings hold up against the current source. The most urgent problems are not cosmetic:
1. The committed migration chain does not create `MaterialGroup` or the three `groupId` columns, while the generated Prisma client selects those fields. A database created only with `prisma migrate deploy` is incompatible with the app.
2. Production authentication can use a public fallback session secret, making a deployment with an empty `SESSION_SECRET` forgeable.
3. The in-viewer quiz retake flow loses the retake scope and records the retake as a full, incorrectly scored attempt.
4. Flashcard resume trusts stale IDs and unguarded JSON, so deletion or malformed progress can leave the study viewer blank or crash it.
5. The production container path is presently unreliable: the port mapping is wrong without the development override, the conventional override silently changes normal Compose behavior, the build context can include host `node_modules`, and the runner installs a ranged Prisma CLI outside the lockfile.
I agree with most of the remaining findings as defects or resilience gaps, but not always with their severity or proposed remedy. In particular:
- FIN-04 needs more than wiring in the existing `filterAndClampOrder`: that helper does not preserve the logical current card when a stale ID occurs before the saved index, and it turns a completed index back into the last card.
- FIN-05 should normally use `3000:3726`, preserving the documented host port while mapping it to the actual container port.
- FIN-22 should not be fixed with an automatic migration retry loop. Schema conflicts should fail fast with actionable diagnostics and a documented recovery path.
- FIN-27 is more important than Low because changing quiz content can rewrite the meaning of persisted history.
- FIN-34 is only partially present: the dedicated deck page already awaits both DELETE requests, but the library start/restart path fires them without waiting.
- FIN-38 is not currently a defect for this Arizona-based installation. It becomes a portability requirement only if the app is meant to support a configurable local day boundary.
- FIN-42 does not justify a destructive history rewrite without evidence that the old database contained sensitive data and without coordinating every clone/remote.
- FIN-50 is an intentional convention. Newest named groups must remain first and Uncategorized must remain last; validation should make that convention harder to corrupt.
- FIN-52 is an architectural consequence of local grading in an anonymous read-only viewer. It merits a sharing warning, not removal of the answer key unless the product adopts server-side anonymous grading.
## Finding-by-finding disposition
Priority definitions: P0 = release/deployment blocker; P1 = next remediation batch; P2 = important hardening; P3 = worthwhile backlog; Conditional = verify a stated condition before changing behavior; No separate fix = intentional behavior or covered by another item.
| ID | Verdict | Worth fixing? | Priority and disposition |
|---|---|---:|---|
| FIN-01 | Agree | Yes | **P0.** Add the missing migration plus a safe adoption runbook for already-`db push`-synchronized databases. |
| FIN-02 | Agree, conditional on an unset/blank production secret | Yes | **P0.** Production must fail before serving requests; development may use an explicitly development-only value. |
| FIN-03 | Agree | Yes | **P1.** Make attempted-question scope explicit and derive partial-retake status on the server. |
| FIN-04 | Agree; proposed helper is insufficient as written | Yes | **P1.** Normalize order, index, and results while preserving the logical current card and completed state. |
| FIN-05 | Agree | Yes | **P0.** Map host 3000 to container 3726 and smoke-test it. |
| FIN-06 | Agree with the mechanics; exact runtime failure is unconfirmed | Yes | **P0.** Add `.dockerignore` and prove native modules load in the Linux image. |
| FIN-07 | Agree | Yes | **P0.** Rename the development override and document explicit dev/prod commands. |
| FIN-08 | Partly agree: first-login setup is intentional, accidental exposure and dead config are real | Yes | **P1.** Honor a pre-provisioned admin hash or require an explicit one-time setup mode. |
| FIN-09 | Agree; latent with current IDs | Yes | **P1.** Replace the broad dot bypass with exact public/static path rules. |
| FIN-10 | Agree | Yes | **P1.** Prevent token overwrite, remove spoofable per-IP trust, and make reset initiation a local/admin operation. |
| FIN-11 | Agree | Yes | **P1.** Validate the whole attempt envelope, question scope, option ownership, array uniqueness, and types. |
| FIN-12 | Agree | Yes | **P1.** Add an in-flight guard, disable Finish, and surface failure/retry state. |
| FIN-13 | Agree as a credible timing race | Yes | **P1.** Persist the attempt and clear full-attempt progress in one server transaction. |
| FIN-14 | Agree as a credible timing race | Yes | **P2.** Add session identity and monotonic revisions, not only client debouncing. |
| FIN-15 | Agree | Yes | **P1.** Persist `currentIndex === order.length` and restore the summary state. |
| FIN-16 | Agree; impact is lower in a protected single-user app | Yes | **P2.** Accept only the queue-eligible card, including the intended learn-ahead rule. |
| FIN-17 | Agree that the queries are unbounded; user impact is unmeasured | Conditional | **P3.** Benchmark realistic data first, then optimize queries that exceed the budget. |
| FIN-18 | Agree | Yes | **P2.** Stop mutating shared objects, capture the origin group before mutation, await writes, and rollback/refetch on error. |
| FIN-19 | Agree | Yes | **P1.** Enforce class/type/group membership and server-owned ordering; retain a share-page defense. |
| FIN-20 | Agree | Yes | **P1.** Give the viewer a stable session input so unrelated parent renders cannot reset it. |
| FIN-21 | Agree | Yes | **P1.** Validate progress writes and defensively normalize legacy/corrupt progress on reads. |
| FIN-22 | Partly agree with the operational gap, not the retry remedy | No separate fix | Fail fast; cover CLI pinning in FIN-23 and recovery/health in FIN-24. Do not retry a conflicting migration automatically. |
| FIN-23 | Agree | Yes | **P0.** Use `npm ci` and ship a lockfile-pinned Prisma CLI/runtime path. |
| FIN-24 | Agree | Yes | **P1.** Add a schema-aware health check and tested backup/restore instructions. |
| FIN-25 | Agree | Yes | **P0/P1.** Gate test, build, Prisma validation/drift, and container smoke. Stage lint until its known baseline is resolved. |
| FIN-26 | Agree | Yes | **P1.** Return zero for an invalid zero-correct SATA question and test finite totals. |
| FIN-27 | Agree; severity should be Medium | Yes | **P2.** Persist an immutable result/review snapshot for new attempts, with legacy fallback. |
| FIN-28 | Agree | Yes | **P2.** Reuse a trimmed card-content schema and reject rather than silently drop incomplete Create-tab rows. |
| FIN-29 | Agree | Yes | **P2.** Add reasonable string, array, and request-size limits with clear 400/413 errors. |
| FIN-30 | Agree that the contract is inconsistent | Yes | **P2.** Make SATA imports require at least two correct options, matching the shipped generation instructions; preserve legacy stored data. |
| FIN-31 | Agree | Yes | **P2.** Validate complete request envelopes and map only actual not-found errors to 404. |
| FIN-32 | Agree | Yes | **P2.** Delete/reassign/renumber in one transaction and add deterministic item tie-breaks. |
| FIN-33 | Agree | Yes | **P2.** Add an explicit keyboard-accessible “Move to group” action; do not depend on drag gestures. |
| FIN-34 | Partly agree | Yes | **P2.** Fix the library path that does not await deletion; make deletion session-aware with FIN-14. |
| FIN-35 | Agree | Yes | **P3.** Translate concurrent first-review uniqueness conflicts to 409 and refetch. |
| FIN-36 | Agree as a credible timing race | Yes | **P2.** Disable navigation while grading and cancel/ignore stale animation callbacks. |
| FIN-37 | Agree | Yes | **P3.** Refresh set/deck membership on focus and after local deck-change events. |
| FIN-38 | Factually correct, but Arizona is the current intended boundary | Not now | Keep the existing behavior. Revisit only with an explicit portability requirement and a timezone setting/migration plan. |
| FIN-39 | Agree about the lost clear-cookie response; layout impact is overstated because proxy checks generation | Yes | **P2.** Destroy the cookie on the actual redirect response and test its header. |
| FIN-40 | Deployment-policy gap, not a bug on intentional HTTP | Yes, as documentation/config | **P1.** Document a TLS proxy profile and `SECURE_COOKIES=true`; do not enable Secure cookies on plain HTTP. |
| FIN-41 | Agree | Yes | **P1.** Validate a strict target enum and target existence; reject impossible/null-target rows. |
| FIN-42 | Agree that blobs exist; sensitivity is unproven | Conditional | Do not rewrite history now. Inventory the old DB offline before any public remote; rewrite only with explicit coordination if sensitive data is found. |
| FIN-43 | Partly agree | Yes | **P3.** Ignore `data/` and production DB sidecars. Confirm ownership, then remove unreferenced CSS scratch files. Do not automatically ignore the audit plan. |
| FIN-44 | Agree | Yes | **P2.** Add `res.ok`, error, and retry handling; never present a failed load as an empty dashboard. |
| FIN-45 | Agree | Yes | **P3.** Add abort/error handling to settings/share fetches. |
| FIN-46 | Agree as a credible timing race | Yes | **P2.** Use `AbortController` or a request generation tied to `classSlug`; remove or bound the module cache. |
| FIN-47 | Agree | Yes | **P3.** Show logout failure and retry; do not falsely claim logout if the server call failed. |
| FIN-48 | Partly agree | Yes | **P2.** Guarantee a non-empty unique slug. Keep slugs stable on rename unless redirect/history semantics are deliberately designed. |
| FIN-49 | Agree; impact is same-browser only | Yes | **P3.** Include the share token in local session keys and migrate/ignore old keys safely. |
| FIN-50 | Intentional newest-first convention | No separate fix | Preserve descending named-group order and final Uncategorized placement; restrict arbitrary sort-order writes under FIN-19/31. |
| FIN-51 | Agree, cosmetic only | Not now | Defer unless it can be removed without hydration mismatch or new effect-rule violations. |
| FIN-52 | Expected local-grading design | Warning only | **P3.** Explain in ShareMenu that recipients can inspect correct answers; do not imply answer secrecy. |
| FIN-53 | Agree | Yes | **P1.** Replace the false-confidence assertion and add scoring/scheduling edge cases. |
## Remediation sequence
### Phase 0 — safeguards and test foundation
Do this before behavior or schema changes.
1. Create a database-backed Vitest harness that always points Prisma at a unique disposable SQLite path. It must refuse to run if the resolved path is `dev.db`, `/app/data/study.db`, or any existing non-test database.
2. Add helpers to apply committed migrations to the disposable database and dispose of it after the test process.
3. Record the current lint baseline separately. Do not make a failing baseline a nominally “green” CI gate, and do not mix unrelated lint cleanup into P0 fixes.
4. Add focused pure tests first for scoring, progress normalization, import schemas, and rate limiting. These give fast feedback before route/service integration tests.
5. Before testing any real deployment database, make and verify a restorable backup. No remediation command should run against the only copy.
Exit criteria:
- Tests demonstrably create and use only a disposable DB.
- A deliberate attempt to point the harness at the real DB aborts.
- Existing `npm.cmd test` remains green.
### Phase 1 — restore a safe, reproducible deployment path
Addresses FIN-01, FIN-02, FIN-05–10, FIN-23–25, FIN-39, and FIN-40.
#### 1A. Repair migration drift without breaking already-pushed databases
1. Generate a new migration; never edit the four applied migrations. It must add:
- `MaterialGroup` with its Class cascade foreign key.
- nullable `groupId` on Deck and QuizSet with `ON DELETE SET NULL`.
- nullable unique `groupId` on ShareLink with `ON DELETE CASCADE`.
- the indexes Prisma expects.
2. Review the generated SQLite table-rebuild SQL by hand for preserved rows, foreign keys, defaults, and unique indexes.
3. Publish a one-time preflight/adoption script that classifies a database as:
- migration-tracked and missing the group schema: apply the migration normally;
- already schema-pushed and exactly matching the intended DDL: after backup and exact introspection, mark the new migration applied with `prisma migrate resolve --applied`;
- partially matching or otherwise inconsistent: stop with diagnostics and require manual recovery; never guess or auto-resolve.
4. Keep the adoption operation explicit. The normal entrypoint should not silently mutate migration history based on table existence.
Verification:
- Fresh empty DB: `prisma migrate deploy` succeeds; `prisma migrate diff --from-migrations ... --to-schema ...` is empty; Deck, QuizSet, ShareLink, and MaterialGroup queries succeed.
- Populated pre-group DB: seed classes/decks/quizzes/shares, apply the new migration, and prove every row/count/relationship is preserved.
- Disposable `db push`-style DB: introspect exact equivalence, resolve the migration, run deploy again, and prove an empty schema diff.
- Partial/conflicting DB: preflight exits nonzero without changing schema or `_prisma_migrations`.
#### 1B. Make session configuration unforgeable and setup explicit
1. Centralize session option construction so `src/lib/auth.ts` and `src/proxy.ts` cannot drift.
2. Require a nonblank, at-least-32-character production `SESSION_SECRET` at runtime. Do not require or bake the real secret during `docker build`.
3. Make the container entrypoint fail before migrations/server startup when the secret is absent or equals the known fallback. Make Compose use required-variable expansion.
4. Allow a development-only fallback only when `NODE_ENV !== "production"`, clearly label it non-production, and cover both branches with tests.
5. Replace `pathname.includes(".")` with exact public/static rules. Ensure every `/api/*` path other than the intentional auth endpoints remains protected even when the URL contains a dot or encoded dot.
6. Construct the unauthenticated redirect first and bind `getIronSession` to that response before `destroy()`, so the returned redirect carries the clearing cookie.
7. Honor a valid `ADMIN_PASSWORD_HASH` when the database has no configured password, or require an explicit one-time setup flag/token. The default production state must not let the first remote request select the admin password.
8. Prefer a local console/CLI initiation for password reset. If the HTTP request endpoint remains, use a global single-user throttle, never replace an unexpired token, rate-limit verify/complete, and do not trust arbitrary `x-forwarded-for` unless a trusted proxy is explicitly configured.
Verification:
- Production startup with missing, blank, short, or known fallback secret exits nonzero before serving.
- A cookie sealed with the old fallback does not authenticate when a real secret is configured.
- Valid login, logout, reset completion, and session-generation invalidation still work.
- Dot-containing protected page/API requests redirect or reject; known static assets still load.
- A stale-generation response contains a `Set-Cookie` deletion header.
- Pre-provisioned hash rejects a different first password and reports setup complete.
#### 1C. Make the image and Compose definitions deterministic
1. Add `.dockerignore` for `node_modules`, `.next`, VCS metadata, environment files, databases/data, coverage, temporary caches, and audit scratch artifacts while retaining source, Prisma schema/migrations, lockfile, and public assets.
2. Change the dependency stage to `npm ci`.
3. Remove `npm install prisma@^7.8.0` from the runner. Ship a Prisma CLI/runtime installed from `package-lock.json` at the exact repository version. If Prisma is needed at runtime, classify it as a runtime dependency and prune/copy dependencies deterministically rather than re-resolving them.
4. Map `3000:3726` in production Compose. Keep `PORT=3726` and `EXPOSE 3726` internally unless there is a deliberate decision to standardize everything on 3000.
5. Rename `docker-compose.override.yml` to a non-auto-merged development filename such as `docker-compose.dev.yml`. Document exact production and development invocations.
6. Add a minimal internal health route that performs schema-aware DB checks (including a query that touches `Deck.groupId` and MaterialGroup), returning no sensitive details. The container health check must call the actual internal port.
7. Add a lock-safe SQLite backup command using the SQLite backup API, plus a documented restore drill. A raw copy of only the main DB while WAL writes are active is not an acceptable backup procedure.
Verification:
- `docker compose -f docker-compose.yml config` shows the production command, production environment, and `3000:3726` only.
- The explicit development Compose command shows dev mode and source mounts.
- Build on a Windows-host context and run on Linux; requiring `better-sqlite3` and `argon2` succeeds in the final image.
- Fresh-volume container becomes healthy and `/login` returns 200 through host port 3000.
- A migration-drifted disposable volume stays unhealthy with an actionable log.
- Backup a populated disposable volume, destroy the disposable container/volume, restore it, and compare row counts plus representative content.
#### 1D. Add CI gates in a sequence that can actually be green
1. Gate `npm ci`, `npm test`, Prisma validate, migration/schema drift, and `npm run build` immediately.
2. Build the production image, start it on a fresh disposable volume with a generated CI secret, wait for health, and smoke `/login` plus setup status.
3. Tag immutable images by commit SHA; optionally move the mutable `latest` tag only after all gates pass.
4. Add strict lint only after the existing 9-error baseline is resolved in an explicit cleanup or after a changed-files lint gate is implemented. Never report baseline lint as passing.
### Phase 2 — make quiz attempts authoritative and immutable
Addresses FIN-03, FIN-11–13, FIN-20, FIN-26, FIN-27, and FIN-53.
1. Replace the client-controlled `isPartialRetake` contract with an explicit ordered `questionIds` scope. The server validates that IDs are unique and belong to the quiz, then derives whether the attempt is partial.
2. Accept a structured answers object at the route boundary and persist only canonical server-serialized JSON. Validate that:
- each key is in the attempted scope;
- every selected option belongs to that question;
- selected IDs are unique;
- multiple-choice has at most one selection;
- missing answers score zero rather than silently disappearing.
3. Make the in-viewer “Retake Missed” and “Retake Full Quiz” set explicit local attempt scope. Do not infer retake state from the original prop after the viewer has transitioned internally.
4. Add an `isFinishing` state/ref, disable the Finish button during submission, and show a retryable error without discarding answers.
5. Move attempt creation and full-attempt SEQUENTIAL-progress deletion into one service transaction. Partial retakes must not overwrite or delete a full in-progress session.
6. Guard zero-correct SATA scoring with zero points and deduplicate selections defensively in the pure scorer even though the route also validates them.
7. Add an optional immutable review snapshot to QuizAttempt for new attempts: attempted question/order, prompt/category/rationale, options and correctness, selections, and per-question points. Render new history from that snapshot; keep a clearly tested legacy fallback for old rows.
8. Stabilize QuizViewer initialization. A parent render that changes topics or another surrounding control must not regenerate option order, clear answers, or reset the index. A deliberate restart/retake must use a new session key.
Focused verification:
- Full 5-question attempt scores out of 5 and clears only its matching full progress row.
- Two-question retake scores out of 2, is stored as partial, and leaves full progress untouched.
- An unanswered question in the explicit scope scores zero and remains in the review snapshot.
- Duplicate/unknown question or option IDs, wrong shapes, and client-forged partial flags return 400 without creating an attempt.
- Delayed double-click Finish produces exactly one attempt.
- Changing or deleting current quiz content after an attempt does not change the stored score, category breakdown, or review copy for snapshot-backed attempts.
- Parent rerender preserves question order, option order, current index, and answers; explicit restart changes only the intended session.
### Phase 3 — make progress and flashcard resume race-safe
Addresses FIN-04, FIN-14, FIN-15, FIN-21, FIN-34, and FIN-36.
1. Replace or correct `filterAndClampOrder` with a normalization function that accepts saved order, live IDs, saved index, results/answers, and completion state. It must:
- remove stale and duplicate IDs;
- preserve the saved current ID when it still exists;
- when it was deleted, select the next surviving saved card, otherwise the previous survivor;
- preserve `index === oldOrder.length` as completion by returning `index === newOrder.length`;
- filter result/answer keys to live IDs;
- safely fall back when any JSON field is malformed.
2. Apply normalization before any viewer state initializer calls `JSON.parse`. Treat corrupt legacy progress as recoverable, show a small restore warning, and allow a fresh start.
3. Add Zod schemas for GET/PATCH/DELETE progress inputs, enum values, nonnegative index, content-specific JSON shape, and referenced content existence.
4. Add `sessionId` and monotonic `revision` to progress. The server accepts only a newer revision for the same session; a stale session cannot overwrite or delete a newer session. Serialize client saves but retain server-side revision enforcement because request completion order is not guaranteed.
5. On the last flashcard, persist `currentIndex = order.length`. Restoring that row must show the existing completion summary, not the last card.
6. Await restart deletion before remount/navigation. Make DELETE conditional on the session being cleared so a late old request cannot delete fresh progress.
7. Disable Previous, shuffle, and repeated grading during the 350 ms transition, and clear/cancel the timer on restart/unmount.
Focused verification:
- Stale ID before, at, and after the current index all resume on the correct logical card.
- All saved cards deleted yields a clear empty/completed state rather than a blank viewer.
- Completed sessions reopen on the summary; restart opens card 1.
- Invalid JSON and invalid PATCH payloads produce a safe UI fallback or 400, never a render crash/500.
- Deliver revisions 3, 1, and 2 in that order; the DB retains revision 3.
- Delay an old-session DELETE until after a new-session PATCH; the new session remains.
- Grade then immediately try Previous/restart; no index jump or stale result write occurs.
### Phase 4 — enforce group, ordering, and sharing invariants on the server
Addresses FIN-18, FIN-19, FIN-32, FIN-33, FIN-41, FIN-49, FIN-50, and FIN-52.
1. Move reorder logic into focused services. The route must identify the owning class and content type from the database, not trust the client.
2. Validate every target group exists, belongs to the same class, and matches DECK/QUIZ type. Reject duplicate IDs, foreign-class IDs, type mismatches, and unknown IDs.
3. Prefer an ordered list/group assignment contract and compute contiguous `sortOrder` values server-side. If a complete-set contract is required, verify completeness explicitly.
4. In the library, capture origin group before any update, update state immutably, await `res.ok`, disable conflicting mutations in flight, and rollback/refetch on failure.
5. Delete a group in one transaction: capture affected items, delete/reassign through the FK behavior, then renumber Uncategorized items deterministically. Add `createdAt`/`id` tie-breakers to reads so legacy duplicates are stable.
6. Add a keyboard-accessible Move-to-group menu on every item. Preserve drag-and-drop for pointer users and keep drag listeners off action controls.
7. Validate ShareLink target type and target existence. Enforce exactly one populated target in service logic; consider CHECK constraints in a later SQLite migration after compatibility testing.
8. Retain the shared-page class/type/token checks and additionally assert each selected group item has the group’s class and expected content type.
9. Pass the share token/session namespace to shared viewers and include it in localStorage keys. Avoid copying an old item-only session into a different token namespace without explicit user confirmation.
10. Preserve named-group ordering as `sortOrder desc, createdAt desc`, prepend newly created groups locally, and render Uncategorized last. Remove arbitrary `sortOrder` from the ordinary group rename PATCH contract.
11. Add a concise ShareMenu warning that a shared quiz necessarily sends answer/rationale data to the recipient’s browser for local grading.
Focused verification:
- Cross-class and cross-type group assignment returns 400/409 and changes no rows.
- A failing reorder restores/refetches the visible order.
- Deleting a group yields unique contiguous Uncategorized item order and does not delete decks/quizzes.
- Newer named groups stay above older groups; Uncategorized remains last in quizzes, flashcards, and import selectors.
- A keyboard-only user can move an item between two groups and hear/see confirmation.
- Invalid share target types and missing IDs create zero rows.
- A tampered group share cannot render a foreign-class item.
- Two tokens for the same content maintain independent local sessions.
### Phase 5 — validation, SRS integrity, and user-visible error handling
Addresses FIN-16, FIN-28–31, FIN-35, FIN-37, FIN-43–48, and the measurable part of FIN-17.
1. Reuse shared Zod schemas for card create/edit, deck/quiz create/edit, classes, imports, and material groups. Trim before minimum checks and add documented maximums for names, descriptions, Markdown content, options, questions, and cards.
2. Reject a Create-tab submission containing any partially filled card; identify the row(s) instead of silently dropping them.
3. Enforce at least two correct options for imported SATA questions while leaving existing stored questions readable. Add a targeted message explaining how to repair invalid generated JSON.
4. Narrow Prisma error mapping: P2025 becomes 404, P2002 becomes 409 where appropriate, validation is 400/422, and unexpected failures remain 500 with non-sensitive server diagnostics.
5. In SRS review, verify the submitted card is the queue-eligible card for the current set/time/new-card allowance (including deliberate learn-ahead). Retain the existing state-version comparison. Map concurrent first-review P2002 to 409 and return/refetch current study state.
6. Refresh SRS membership on focus and after same-tab deck changes.
7. Add abortable, checked fetch helpers or a small consistent pattern for Dashboard, GenerateTab, ShareMenu, logout, and class library requests. Distinguish loading, empty, error, and retry states.
8. Tie library responses to the active `classSlug`; abort or ignore stale responses. Bound or remove module-level caches that can outlive their class data.
9. Guarantee class slug generation produces a non-empty unique slug, for example a stable `class-<short-id>` fallback. Keep existing slugs stable on rename.
10. Add `data/`, `study.db`, and sidecars to `.gitignore`. Confirm `out.css`, `temp.css`, and `test.css` are unreferenced scratch artifacts before removing them in a separate cleanup change.
11. For performance, first seed a disposable benchmark DB approximating expected upper use (for example 100k activity rows, 10k cards, and several SRS sets). Add date predicates/aggregation and narrower SRS queries only where the measured request/poll budget is exceeded. Preserve long-streak correctness when limiting the activity window.
Focused verification:
- Whitespace-only and oversized content is rejected with an actionable client error and no partial write.
- SATA 1-correct import fails; 2+-correct passes; multiple-choice still requires exactly one.
- SRS cannot review a not-due/non-selected/new-limit-exhausted card; valid due and learn-ahead reviews still work.
- Concurrent first review produces one success and one handled conflict, not a generic 500.
- Focus refresh removes a deleted deck from the SRS set UI.
- Failed dashboard/settings/share/logout requests have visible, retryable states and no unhandled promise rejection.
- Rapid A→B class navigation cannot render A data under B.
- Creating a class named only punctuation yields a navigable, unique slug; rename does not break the old URL.
- Performance tests record query counts and latency before/after and prove the 53-week display plus longer current streak remain correct.
## Cross-cutting release verification
Run after every phase, with focused tests first and the full suite last.
Automated gates:
1. `npm.cmd test`
2. `npx.cmd prisma validate`
3. migration deploy + empty migration/schema diff against a fresh disposable DB whenever schema changes
4. `npm.cmd run build`
5. `git diff --check`
6. production image build, fresh-volume health wait, and HTTP smoke when Docker/deployment files change
7. lint against changed files; full `npm.cmd run lint` only becomes blocking after the known baseline is cleared
Manual browser matrix for affected phases:
1. Authenticated routes and corresponding public/shared routes.
2. Fresh, resumed, completed, restarted, partial-retake, and stale/deleted-content sessions.
3. Loading, empty, server-error, malformed-persisted-data, and retry states.
4. Light and dark themes.
5. Narrow mobile viewport with no horizontal overflow.
6. Keyboard-only navigation, including group moves and all modified icon controls.
7. Throttled-network checks for Finish, autosave, restart, class navigation, reorder, and logout.
8. Two-tab checks for progress revisions, SRS conflicts, and focus refresh.
Release/rollback rules:
- Back up and restore-test the SQLite database before the first migration-bearing release.
- Deploy schema and code as one versioned release; do not run a newer generated Prisma client against an older database.
- Keep the prior image and verified pre-migration backup until post-deploy smoke and representative data checks pass.
- If migration preflight sees a partial or unknown schema state, stop. Do not run `db push`, edit an applied migration, reset the database, or auto-mark the migration applied.
## Explicitly deferred or rejected work
- **FIN-38:** no timezone-setting work without a product requirement to support a non-Arizona study day.
- **FIN-42:** no history rewrite without an offline sensitivity review and explicit remote/clone coordination.
- **FIN-51:** no cosmetic state-initialization change unless it avoids both hydration mismatch and the existing effect-rule class of lint failures.
- **FIN-22 retry proposal:** no automatic migration retry/backoff for schema conflicts; fail fast with health diagnostics and the adoption/recovery runbook.
- No changes to the audit’s already rejected hypotheses (global API auth absence, split-brain SQLite paths, broken cascades/transactions, Markdown XSS, or client/server import violations) unless new evidence appears.
## Recommended delivery slices
Keep the implementation reviewable rather than landing all remediation at once:
1. **Release blocker:** FIN-01/02/05/06/07/23/25 plus migration adoption, container smoke, and backup prerequisites.
2. **Quiz correctness:** FIN-03/11/12/13/20/26/27/53.
3. **Progress integrity:** FIN-04/14/15/21/34/36.
4. **Group/share integrity:** FIN-18/19/32/33/41/49/50/52.
5. **Auth recovery and deployment posture:** FIN-08/09/10/24/39/40.
6. **Validation, SRS, UI resilience, and measured performance:** the remaining accepted items.
Each slice should be independently releasable, have its own focused regression tests, and finish with the cross-cutting gates above.

View file

@ -1,20 +0,0 @@
# Subagent Reports
Full specialist reports were produced by 10 read-only subagents. Each returned a structured markdown report with a findings table and per-finding detail blocks (ID, severity, confidence, files:lines, scenario, expected/actual, root cause, impact, evidence, test coverage, fix direction, complexity, regression test, runtime-confirmation flag). The parent agent re-verified the Critical/High claims directly (see VERIFICATION_LOG.md) and consolidated/deduplicated into `FINDINGS.json` (53 findings).
## Report index (all reports were reviewed in full by the parent)
| # | Auditor | Ref | Key output |
|---|---|---|---|
| 1 | DB / migrations / transactions / cascades / drift | sa_20260806_054030_000000000_5ed877563a0d | **DBAUD-01 Critical migration drift** (MaterialGroup/groupId never migrated; commit 7af0935 db-push artifact); DBAUD-02 junk ShareLink rows; DBAUD-03 dev.db in git history; verified-safe: cascades, transactions, path resolution, single-row AuthSecurity |
| 2 | Quiz scoring / attempts / retakes / progress | sa_20260806_054030_000000000_e75a8ce575f9 | **QUIZ-01 High retake-scoring bug** (in-viewer retake never sets retakeIds); QUIZ-02..09 (unvalidated answersJson, double-submit, progress races, dead filter, div-by-zero, historical recompute, multi-tab) |
| 3 | Flashcards / SRS / progress / deletion / ordering | sa_20260806_054030_000000000_a222804c133c | CARD-01..11: dead stale-id filter (stuck sessions), unsequenced saves, completed-session resume, SRS due/limit not enforced, unbounded scans, Arizona day boundary, restart race, P2002, animation race, focus staleness |
| 4 | Auth / protected routes / API authz / sharing | sa_20260806_054030_000000000_aab0c59dbad3 | **AUTH-01 Critical hardcoded session secret**; AUTH-02 dot-bypass; AUTH-03 reset abuse; AUTH-04 first-login takeover + dead ADMIN_PASSWORD_HASH; AUTH-05..10 (cookie-clear loss, cross-class group membership, share validation, secure flag, no auth tests, answer-key exposure); verified proxy compiled as middleware |
| 5 | Imports / exports / generation / Zod / malformed input | sa_20260806_054030_000000000_5e72c2713989 | IMPT-01..06: dead filterAndClampOrder, unvalidated /api/progress + unguarded JSON.parse, card edit endpoints without Zod, no size caps, SATA doc mismatch, override-field 500s; verified: shared schemas, atomic imports, no XSS, no id regeneration |
| 6 | Material groups / ordering / drag-drop / deletion / orphans | sa_20260806_054030_000000000_9ae1f3dfa282 | **GRP-01 Critical drift** (independent confirmation); GRP-02..07: unvalidated groupId/sortOrder, fire-and-forget reorder, sortOrder duplicates after group deletion, keyboard a11y, desc-convention fragility; verified: FKs enforced, uncategorized mapping, cascade safety |
| 7 | Frontend state / refresh / races / localStorage / boundaries | sa_20260806_054030_000000000_0e17fd155702 | FE-01..11: optimistic reorder without rollback, CRUD res.ok ignored, shared quiz reshuffle (unstable prop), finish double-submit, progress race, **FE-06 (later refuted — proxy exists)**, dashboard fetch gap, unguarded JSON.parse, effect fetches, cross-class race, collapse flash; verified: shared viewers read-only, localStorage clean, no boundary violations |
| 8 | Test quality / coverage / false confidence | sa_20260806_054030_000000000_cd3652deb3f7 | TEST-01..10: false-confidence SRS test, zero tests for scoring/auth/services/routes, no DB harness, CI never runs tests; ranked missing-test gaps by impact |
| 9 | Docker / startup / env / scripts / production / recovery | sa_20260806_054030_000000000_f49cdaacaabf | OPS-01..10: port mismatch 3726 vs 3000:3000, entrypoint fragility, session secret in prod path, dead ADMIN_PASSWORD_HASH, non-reproducible installs, no .dockerignore (win32 leak), no healthcheck/backup, override auto-merge, .gitignore gaps, CI without gates; argon2/standalone tracing verified OK |
| 10 | Adversarial review | sa_20260806_055608_000000000_c550c742d5dc | Challenged all Critical/High findings: confirmed drift, secret, retake, port, dockerignore, override; weakened OPS-02 (fresh-volume migrate succeeds) and FE-01 severity; **refuted FE-06** (proxy active — verified in compiled .next artifacts); added ADV-01..10 (retake progress pollution, activity scan, logout, dashboard crash, slugify empty slug, unguarded parses, completed-resume, localStorage key collisions, first-login race, progress validation) |
All reports are preserved in full in the session; their findings are consolidated (deduplicated, source IDs preserved) in `FINDINGS.json` and `CONFIRMED_FINDINGS.md`.

View file

@ -1,21 +0,0 @@
# Test Gaps
Current suite: 6 files / 30 tests — 20 in `src/lib/arcade/*` (out of scope) + 10 in `src/lib/spacedRepetition.test.ts` (pure scheduling math only). **No test touches a service, an API route, the database, auth, scoring, imports, or progress.** CI never runs `npm test`. Gaps ranked by user-impact × regression-likelihood.
| ID | Area | Gap | Risk it would fail to catch | Suggested test |
|---|---|---|---|---|
| TEST-GAP-01 | Auth (FIN-02/08/09/10) | `authService.ts` (login, lockout, first-login provisioning, reset token, generation bump) and session config have zero tests | Session-secret fallback regression, lockout bypass, generation-bump loss (stolen sessions stay valid) | Extract + unit-test `getLockoutDuration` thresholds, `parseResetTokenRecord`; service tests with temp SQLite + fake timers: 5 wrong passwords → 423; locked-out login rejected; success resets counter; `completePasswordReset` bumps generation |
| TEST-GAP-02 | Scoring (FIN-03/11/26/27) | `src/lib/scoring.ts` SATA partial credit has zero tests | Wrong grades silently persisted into every attempt — permanent data corruption | `scoreQuestion`: SATA 2/4 correct select both → 1; 1 correct + 1 wrong → 0; over-select → 0; MC multi-select first-wrong → 0; `scoreQuiz` with missing question keys; 0-correct SATA → 0 (not NaN) |
| TEST-GAP-03 | Quiz attempt persistence (FIN-03/12/13) | `POST /api/quizzes/[id]/attempt` + partial-retake filter + `progressService.upsertProgress` asymmetry (DECK writes cardResultsJson, QUIZ writes answersJson) have zero tests | Retake-scoring regression (FIN-03 class), duplicate attempts, progress column clobbering | Route/service test: partial retake answering 2 of 5 → maxScore 2, score from those 2; upsert round-trip preserves the respective JSON columns |
| TEST-GAP-04 | SRS service (FIN-16/35) | `spacedRepetitionService.ts` (reviewCard optimistic concurrency, findNextCard precedence, removeDeck transaction, addDeck validations) has zero tests — only the pure lib is tested | Double-review stale write, due/new-card precedence break, cross-class membership, P2002 → 500 | Seed class→deck→cards→set on temp DB: stale `expectedStateVersion` → 409; due before new before learn-ahead; new card with limit 0 rejected; removeDeck atomically deletes states |
| TEST-GAP-05 | Import/repair pipeline (FIN-28/29/30) | `jsonRepair.ts`, `importSchemas.ts`, `shuffle.ts`/`filterAndClampOrder` have zero tests | AI-import regression (fences/repair), schema constraint drift (MC exactly-one), resume clamp bugs | jsonRepair fence/trailing-comma/broken-JSON cases; schema whitespace/oversize/exactly-one cases; `filterAndClampOrder` stale-id + overflow cases |
| TEST-GAP-06 | Rate limiter (FIN-10) | `rateLimiter.ts` has zero tests | Brute-force guard silently disabled or locking the owner out | Fake timers: 10 allowed / 11th blocked with retryAfterMs; window expiry; per-key isolation; custom limits |
| TEST-GAP-07 | Integration harness (FIN-01/21/25) | No route tests, no DB-backed tests, no setup file; `src/lib/db.ts` defaults to `file:./dev.db` (a naive service test would touch the real dev DB) | Migration drift (FIN-01) — nothing would catch a schema/migration mismatch; progress/share/attempt route validation gaps | `setupFiles` + `DATABASE_URL=file:<tmp>` + `prisma migrate deploy` in globalSetup; route tests with `NextRequest`: attempt 200/400/404, reviews 409, share unknown token, login 429 |
| TEST-GAP-08 | CI gate (FIN-25) | `.forgejo/workflows/build.yml` runs no tests/lint/prisma validate; no coverage config; no drift check | Entire suite can regress with a green pipeline; FIN-01-class drift ships | Add `npm ci && npm test && npm run lint && npx prisma validate` job; `prisma migrate diff --from-migrations --to-schema-datamodel` gate; container smoke (curl /login on fresh volume); coverage floor |
| TEST-GAP-09 | False confidence (FIN-53) | `spacedRepetition.test.ts:15-23` "distinct valid state for every rating" never compares ratings | `scheduleRating` collapsing all ratings to one schedule passes the suite | Assert pairwise due ordering AGAIN < HARD < GOOD < EASY and distinct due timestamps; graduated-state lapse case |
## Test-quality positives (verified)
- `spacedRepetition.test.ts` is deterministic (fixed `now`, `enable_fuzz: false`, no `Math.random`), tests the real pure module, and asserts exact FSRS intervals.
- Arcade tests (out of scope) use seeded shuffles and meaningful assertions.
- No test uses wall-clock time, locale, or network — no flakiness found.

View file

@ -1,35 +0,0 @@
# Unverified Risks
Risks that could not be fully confirmed within audit constraints (need a browser, a container run, a live deployment, or timing-dependent reproduction). All are code-verified as plausible; the missing piece is runtime confirmation.
| ID | Title | Why unverified | Suggested verification |
|---|---|---|---|
| FIN-02 | Hardcoded fallback `SESSION_SECRET` → session forgery | Code path fully traced (seal/verify semantics verified in iron-session), but no live production deployment exists here to demonstrate an actual forged-cookie login | Boot production build without `SESSION_SECRET`, forge a cookie with the fallback password (iron-webcrypto seal script), assert full access; then with a real secret assert rejection |
| FIN-06 | No `.dockerignore` → win32 modules in Linux image | Mechanics verified (PE32+ DLL present in local node_modules; Dockerfile COPY order), exact failure mode (build error vs `ERR_DLOPEN_FAILED`) not observed | Run `docker build .` on this Windows host; inspect `better_sqlite3.node`/argon2 prebuilds in the image (must be ELF) |
| FIN-13 | Progress PATCH-after-DELETE race | Timing-dependent; needs throttled network | DevTools throttling: submit final answer + Finish immediately; inspect `StudyProgress` after |
| FIN-14 | Out-of-order progress PATCHes | Timing-dependent | DevTools throttling with rapid grading; reload and compare resume point/results |
| FIN-22 | Entrypoint migrate crash loop / runner CLI install | Requires container build + run on `node:22-slim` (docker CLI unavailable in audit environment) | `docker build` + fresh-volume run (assert 200 + `_prisma_migrations`); corrupt-volume run (assert actionable failure, not silent loop) |
| FIN-24 | Healthcheck/backup gaps | Behavior observable only in a real container run | Compose up with a corrupt volume; observe restart loop and absence of health status |
| FIN-32 | Group-delete sortOrder duplicates | Deterministic from code; visual impact needs a browser | Delete a group in the app, inspect Uncategorized order + DB sortOrders |
| FIN-33 | Keyboard cross-group moves impossible | Deterministic from code (empty `handleDragOver`), needs manual keyboard test | Tab to a drag handle, attempt keyboard cross-group move |
| FIN-34 | Restart DELETE race | Timing-dependent | Throttled network: Restart, grade first card, inspect progress row |
| FIN-36 | Previous-card animation race | Timing-dependent (350 ms window) | Grade then immediately click Previous; observe index jump |
| FIN-37 | SRS page stale on focus | Deterministic from code; browser needed to observe | Two tabs: delete deck in one, focus the other |
| FIN-39 | Proxy destroy cookie lost on redirect | Deterministic from code; browser needed to observe Set-Cookie | Stale-generation cookie → follow redirect → inspect response headers |
| FIN-40 | Secure cookie flag off | Deterministic from code (flag logic); deployment-dependent | Inspect `Set-Cookie` in a production container |
| FIN-46 | Cross-class fetch race | Timing-dependent | Throttled network + fast class switching |
| FIN-48 | Empty slug / stale slug on rename | Deterministic from code (slugify verified); UI behavior needs browser | Create class named "!!!", try to navigate to it; rename a class, check old URL |
| FIN-49 | Shared-viewer localStorage key collisions | Code-verified key construction; symptom needs browser | Open two share tokens for the same content, answer in one, reload the other |
| FIN-51 | Collapsed-groups expand flash | Cosmetic; browser-only | Visit library with saved collapsed state |
## Items verified safe (hypotheses disproven — recorded for completeness)
- SQLite relative-path split-brain between Prisma CLI and app (both cwd-relative → same `dev.db`; Docker uses absolute path) — verified safe.
- Foreign-key enforcement (better-sqlite3 compiled with `SQLITE_DEFAULT_FOREIGN_KEYS=1`) — cascades/SetNull fire; verified safe.
- Transactionality of deck/quiz/attempt/reorder writes — nested creates + `$transaction`; verified safe.
- `StudyProgress`/`ShareLink` NULL-uniqueness — service-level guards prevent duplicates via app paths; only direct API misuse (FIN-41/FIN-21) can create junk rows.
- Client/server boundary violations — none found (no client import of `@/lib/db`/services/prisma).
- XSS via Markdown — react-markdown without `rehype-raw`; no `dangerouslySetInnerHTML` on user data; verified safe.
- Password-reset nonce/token crypto (192-bit token, SHA-256 digest, `timingSafeEqual`, expiry) — sound; abuse vectors are FIN-10.
- SRS optimistic concurrency (`expectedStateVersion` → 409, rollback via refetch) — correct.
- Migrations are purely additive; no NOT NULL-without-default, no DROP — safe on populated migration-tracked DBs (except FIN-01 drift).

View file

@ -1,38 +0,0 @@
# Verification Log
Chronological record of every command run, its outcome, and classification (baseline / environment / possible defect).
## Environment notes
- OS: Windows, shell: bash (git-bash)
- `node_modules` was **absent** at audit start → `npm test` initially failed with `'vitest' is not recognized`. This is an environment condition, not a repo defect. `npm ci` completed successfully during the first audit session (518 packages present).
- No existing application database (`dev.db*`) exists anywhere in the repo — no real user data to protect; the migration-chain test uses a brand-new temp DB under `audit-results/tmp`.
- `src/generated/prisma/` (tracked) hashed BEFORE any prisma command: `audit-results/tmp/generated-before.sha256`.
## Entries
| # | Time | Command | Result | Classification |
|---|---|---|---|---|
| 1 | session start | `git status --short` | clean; only `?? .reasonix/` and `?? audit-results/` untracked | baseline |
| 2 | session start | `npm test` (first run, prior session) | FAILED: `'vitest' is not recognized` (no node_modules) | environment |
| 3 | session start | `npm ci` (background, prior session) | completed; 518 packages in node_modules | environment |
| 4 | 22:36 | `npx prisma validate` | PASS — "The schema at prisma\schema.prisma is valid" (exit 0) | baseline |
| 5 | 22:36 | `npm test` | PASS — 6 files, 30 tests, all green (arcade ×20, spacedRepetition ×10) | baseline |
| 6 | 22:37 | `npm run lint` | 28 problems: 9 errors, 19 warnings (react-hooks/set-state-in-effect errors + unused-vars warnings). All pre-existing; matches AGENTS.md known baseline. | baseline |
| 7 | 22:37 | `npm run build` (background) | PASS — route table emitted, no compile errors | baseline |
| 8 | 22:38 | `DATABASE_URL=file:./audit-results/tmp/audit-migration.db npx prisma migrate deploy` | PASS — all 4 migrations applied to brand-new temp DB | baseline |
| 9 | 22:38 | `sha256sum -c audit-results/tmp/generated-before.sha256` | PASS — every generated Prisma file byte-identical (all ": OK") | baseline |
| 10 | 22:38 | `git status --short` | only untracked `.reasonix/`, `audit-results/`; no tracked modifications | baseline |
| 11 | 22:38 | `git diff --check` | clean (no whitespace errors) | baseline |
| 12 | 22:40 | `git log --all --oneline -- prisma/migrations/` + `git show 7af0935 --stat` | Confirms drift: commit 7af0935 changed `prisma/schema.prisma` (+31) and `dev.db` (139264→155648 bytes) with **no migration**; `dev.db` present in 8 commits of history | possible defect (FIN-01, FIN-42) |
| 13 | 22:41 | `node audit-results/tmp/inspect-db.cjs audit-results/tmp/audit-migration.db` | Temp DB (built solely by `migrate deploy`) has **no** `MaterialGroup` table and **no** `groupId` column on Deck/QuizSet/ShareLink | possible defect (FIN-01) |
| 14 | 22:42 | `node audit-results/tmp/reproduce-drift.cjs` | Prisma-shaped queries fail: `no such column: Deck.groupId`, `no such table: MaterialGroup`, `no such column: ShareLink.groupId` — fresh-deploy breakage proven | possible defect (FIN-01) |
| 15 | 22:45 | `node --check audit-results/tmp/inspect-db.cjs && node --check audit-results/tmp/reproduce-drift.cjs` | diagnostic scripts syntax-valid | baseline |
| 16 | 22:50 | `node audit-results/tmp/validate-findings.cjs` | `FINDINGS.json` VALID JSON — 53 findings: 2 Critical, 5 High, 18 Medium, 25 Low, 3 Observation; no duplicate ids | baseline |
| 17 | 23:00 | `git status --short` + `git diff --check` (final) | Only untracked `.reasonix/`, `audit-results/`; no tracked file modified; diff clean | baseline |
## Classification legend
- **baseline** — repository behaves as expected; recorded for reference.
- **environment** — failure caused by local machine state, not repo code.
- **possible defect** — outcome may indicate a repo defect; investigated further in findings.

Binary file not shown.

View file

@ -1,26 +0,0 @@
cb737289f5b5f6cdb8b90fc23d2c89ebd41d6602ce99e88bf6dfd751beab7103 *src/generated/prisma/browser.ts
2ea331124cc56c9c9a360b4a85f42f029992811ece7efe9db0ec1300a43f59c3 *src/generated/prisma/client.ts
f1f6280ff65c1e8aab5ce30307c8777ce1795f81bf1a513396263fc1c9a30e5c *src/generated/prisma/commonInputTypes.ts
8bcc37ae19ee9c55424b735ce5f3a0972b7e741958ef86b350c9a54dc2238a0d *src/generated/prisma/enums.ts
d52628a3dc8285d9014e2fda489badce6c88604810e2975310ab6c40ec3c708d *src/generated/prisma/internal/class.ts
91794eb9b3c395e57b96d18f2bae895149f7b70ddf0fda8b817e7aa9da17a950 *src/generated/prisma/internal/prismaNamespace.ts
6645109792ed08507d79da319aaecc2752bffa6e8bd5a3364dccc5ae93e2f23c *src/generated/prisma/internal/prismaNamespaceBrowser.ts
b8ffef1ad4428179847aafe66abee3ad5eb74a84fff46265532551b10ab2537d *src/generated/prisma/models.ts
daecbb1c94a96d19da4accfdf81fe8f11fa73c543958016bc7d6060d454f3943 *src/generated/prisma/models/AnswerOption.ts
87e18727cec17a341d32709bc0c1602c66e6c6dcc57672b9c9fbe38a6ce12801 *src/generated/prisma/models/ArcadeAttempt.ts
0c9de62bdbefafe8b5ff5300d1d764cc73235a077f8e3b3e03e6144287610c9d *src/generated/prisma/models/ArcadePack.ts
67b8ea01f0a866d14db7fdff3178a080e2879990129c5e08690463fb9940f48d *src/generated/prisma/models/AuthSecurity.ts
6885465a57019e8cba28755d9c6abaa087e913d666d02d8912abf0790c41a77b *src/generated/prisma/models/Class.ts
ea873177511c6f7354dc01ec0ad228d9f777bca2fbfb15d281d3f2ab8f817ebf *src/generated/prisma/models/Deck.ts
1082ff29bec6e4f920c93d778032ffffa324c312b05efd6c5fa7ada1be6674f7 *src/generated/prisma/models/Flashcard.ts
de94c2da405c08a0b690aeb1b0cdc03582c94ccddaf913f4c6d1b18d8c413b61 *src/generated/prisma/models/MaterialGroup.ts
941998d2c098e9fe5cd0a888bac470190108238c936c1d7cfce2742ed1e40d8a *src/generated/prisma/models/Question.ts
3d54ed93e516f4d773c200b166a84ab2c9c8b7b0d7f7ddfd61afc3ca057ee755 *src/generated/prisma/models/QuizAttempt.ts
454f19f97a82b8f37fc5c5179e87afe6ea185a79810043471d55c0a8ca26f14a *src/generated/prisma/models/QuizSet.ts
3b884aaaad311e8f22c1390cf24ebe69b74b52a97c213781d2a4cbb56288b8a7 *src/generated/prisma/models/Setting.ts
1ccf32b2f56b1d35c9b7152d2e1817b4c1b7853549a1ff187e9fb7c29fa49f2a *src/generated/prisma/models/ShareLink.ts
cba950d431c702afe67f3ac92d8397275d3aeeebbc59d357fd20dec2123e0ee2 *src/generated/prisma/models/SpacedRepetitionCardState.ts
2ef43d1c5d7e2eb0d85080fb29703ae99c1704ba8f5200703e2bb8e7a617bf04 *src/generated/prisma/models/SpacedRepetitionSet.ts
0bf9eab9f937ef2d621ccf0cea8e209eae9372a9b333dfe6b18267e563a6b2b0 *src/generated/prisma/models/SpacedRepetitionSetDeck.ts
916bda8f95647f0dc3560bc6848c4a97d4c1f37fd5ed54e6736d9d7d45e08adc *src/generated/prisma/models/StudyActivity.ts
36724157e76b49df49e7827eacbcb0ab8336917f35913777701d8a6830770500 *src/generated/prisma/models/StudyProgress.ts

View file

@ -1,52 +0,0 @@
// Read-only diagnostic: introspect the temp DB created by `prisma migrate deploy`
// to check whether the migrated schema matches schema.prisma (drift check).
// Usage: node audit-results/tmp/inspect-db.cjs <path-to-sqlite-db>
const path = require("node:path");
const Database = require("better-sqlite3");
const dbPath = process.argv[2];
if (!dbPath) {
console.error("usage: node inspect-db.cjs <db-path>");
process.exit(2);
}
const db = new Database(path.resolve(dbPath), { readonly: true });
const tables = db
.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
.all()
.map((r) => r.name);
console.log("=== TABLES ===");
console.log(tables.join(", "));
console.log("Has MaterialGroup table:", tables.includes("MaterialGroup"));
for (const table of ["Deck", "QuizSet", "ShareLink", "MaterialGroup"]) {
if (tables.includes(table)) {
const cols = db
.prepare(`PRAGMA table_info('${table}')`)
.all()
.map((r) => r.name);
console.log(`--- ${table} columns ---`);
console.log(cols.join(", "));
if (table === "Deck" || table === "QuizSet" || table === "ShareLink") {
console.log(` -> has groupId:`, cols.includes("groupId"));
}
} else {
console.log(`--- ${table}: MISSING TABLE ---`);
}
}
const indexes = db
.prepare("SELECT name, tbl_name FROM sqlite_master WHERE type='index' ORDER BY name")
.all()
.map((r) => `${r.tbl_name}.${r.name}`);
console.log("=== INDEXES (count) ===", indexes.length);
// Confirm migrations table
const applied = db.prepare("SELECT migration_name FROM _prisma_migrations ORDER BY started_at").all();
console.log("=== APPLIED MIGRATIONS ===");
applied.forEach((m) => console.log(" -", m.migration_name));
db.close();
console.log("DONE");

View file

@ -1,40 +0,0 @@
// Read-only diagnostic: execute the exact SQL shape the generated Prisma client
// emits for deck.findMany() / materialGroup.findMany() against a DB built purely
// from `prisma migrate deploy` (predev + Docker entrypoint flow).
// Proves DBAUD-01: migrated DB is incompatible with the shipped client.
const path = require("node:path");
const Database = require("better-sqlite3");
const dbPath = path.resolve("audit-results/tmp/audit-migration.db");
const db = new Database(dbPath, { readonly: true });
// 1. Shape of prisma.deck.findMany({ include: { class: true } }) — Prisma selects
// every scalar field, including groupId (schema.prisma:33-34, generated client).
const deckSql =
'SELECT "Deck"."id", "Deck"."classId", "Deck"."name", "Deck"."description", "Deck"."sortOrder", "Deck"."createdAt", "Deck"."groupId" FROM "Deck"';
try {
db.prepare(deckSql).all();
console.log("deck.findMany SQL: OK");
} catch (e) {
console.log("deck.findMany SQL: FAILED ->", e.message);
}
// 2. Shape of prisma.materialGroup.findMany()
const groupSql = 'SELECT "MaterialGroup"."id", "MaterialGroup"."classId", "MaterialGroup"."name", "MaterialGroup"."type", "MaterialGroup"."sortOrder", "MaterialGroup"."createdAt" FROM "MaterialGroup"';
try {
db.prepare(groupSql).all();
console.log("materialGroup.findMany SQL: OK");
} catch (e) {
console.log("materialGroup.findMany SQL: FAILED ->", e.message);
}
// 3. Shape of prisma.shareLink.findFirst() (share page validation)
const shareSql = 'SELECT "ShareLink"."id", "ShareLink"."targetType", "ShareLink"."deckId", "ShareLink"."quizSetId", "ShareLink"."groupId", "ShareLink"."createdAt" FROM "ShareLink" LIMIT 1';
try {
db.prepare(shareSql).all();
console.log("shareLink.findFirst SQL: OK");
} catch (e) {
console.log("shareLink.findFirst SQL: FAILED ->", e.message);
}
db.close();

View file

@ -1,27 +0,0 @@
// Validates audit-results/FINDINGS.json parses and reports counts per severity.
const fs = require("node:fs");
const path = require("node:path");
const file = path.resolve("audit-results/FINDINGS.json");
const raw = fs.readFileSync(file, "utf8");
const data = JSON.parse(raw); // throws if invalid
const findings = data.findings;
const bySeverity = {};
for (const f of findings) {
bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
}
const byConfidence = {};
for (const f of findings) {
byConfidence[f.confidence] = (byConfidence[f.confidence] || 0) + 1;
}
const ids = findings.map((f) => f.id);
const dupes = ids.filter((id, i) => ids.indexOf(id) !== i);
if (dupes.length) throw new Error("duplicate ids: " + dupes.join(","));
console.log("FINDINGS.json: VALID JSON");
console.log("total findings:", findings.length);
console.log("by severity:", JSON.stringify(bySeverity));
console.log("by confidence:", JSON.stringify(byConfidence));
console.log("ids:", ids.join(", "));

View file

@ -1,145 +0,0 @@
import { performance } from "node:perf_hooks";
import path from "node:path";
import Database from "better-sqlite3";
import { afterAll, describe, expect, it } from "vitest";
import { prisma } from "@/lib/db";
import { getActivitySummary } from "@/services/activityService";
import { listClasses } from "@/services/classService";
import { getStudyAvailabilityByClass } from "@/services/spacedRepetitionService";
const ACTIVITY_ROWS = 100_000;
const CARD_ROWS = 10_000;
const SETS = 5;
const REQUEST_BUDGET_MS = 250;
const NOW = new Date("2026-08-07T18:00:00.000Z");
afterAll(async () => {
await prisma.$disconnect();
});
function databasePath() {
const url = process.env.DATABASE_URL;
if (!url?.startsWith("file:./")) {
throw new Error("Benchmark requires the disposable Vitest database");
}
return path.resolve(process.cwd(), url.slice("file:".length));
}
function seedRealApplicationSchema() {
const database = new Database(databasePath());
try {
database.pragma("foreign_keys = ON");
const insertActivity = database.prepare(
'INSERT INTO "StudyActivity" ("id", "type", "occurredAt") VALUES (?, ?, ?)'
);
const insertDeck = database.prepare(
'INSERT INTO "Deck" ("id", "classId", "name", "sortOrder") VALUES (?, ?, ?, ?)'
);
const insertSet = database.prepare(
'INSERT INTO "SpacedRepetitionSet" ("id", "classId", "name", "newCardsPerDay", "sortOrder", "updatedAt") VALUES (?, ?, ?, ?, ?, ?)'
);
const insertMembership = database.prepare(
'INSERT INTO "SpacedRepetitionSetDeck" ("setId", "deckId", "sortOrder") VALUES (?, ?, ?)'
);
const insertCard = database.prepare(
'INSERT INTO "Flashcard" ("id", "deckId", "front", "back", "sortOrder") VALUES (?, ?, ?, ?, ?)'
);
const insertState = database.prepare(`
INSERT INTO "SpacedRepetitionCardState"
("id", "setId", "flashcardId", "due", "stability", "difficulty", "elapsedDays", "scheduledDays", "learningSteps", "reps", "lapses", "state", "firstReviewedAt", "lastReview", "createdAt", "updatedAt")
VALUES (?, ?, ?, ?, 1, 5, 1, ?, 0, 1, 0, ?, ?, ?, ?, ?)
`);
database.transaction(() => {
database.prepare(
'INSERT INTO "Class" ("id", "slug", "name", "sortOrder") VALUES (?, ?, ?, 0)'
).run("benchmark-class", "benchmark-class", "Benchmark Class");
for (let index = 0; index < ACTIVITY_ROWS; index += 1) {
const ageDays = index % 730;
const type = index % 2 === 0 ? "FLASHCARD" : "QUIZ_QUESTION";
insertActivity.run(
`activity-${index}`,
type,
NOW.getTime() - ageDays * 86_400_000
);
}
for (let setIndex = 0; setIndex < SETS; setIndex += 1) {
const deckId = `benchmark-deck-${setIndex}`;
const setId = `benchmark-set-${setIndex}`;
insertDeck.run(deckId, "benchmark-class", `Deck ${setIndex}`, setIndex);
insertSet.run(setId, "benchmark-class", `Set ${setIndex}`, 30, setIndex, NOW.getTime());
insertMembership.run(setId, deckId, 0);
for (let cardIndex = setIndex; cardIndex < CARD_ROWS; cardIndex += SETS) {
const cardId = `benchmark-card-${cardIndex}`;
const due = NOW.getTime() + ((cardIndex % 200) - 100) * 60_000;
const firstReviewed = NOW.getTime() - (cardIndex % 30) * 86_400_000;
const scheduledDays = cardIndex % 4 === 0 ? 1 : 0;
const state = cardIndex % 4;
insertCard.run(cardId, deckId, `Front ${cardIndex}`, `Back ${cardIndex}`, cardIndex);
insertState.run(
`benchmark-state-${cardIndex}`,
setId,
cardId,
due,
scheduledDays,
state,
firstReviewed,
firstReviewed,
firstReviewed,
NOW.getTime()
);
}
}
})();
} finally {
database.close();
}
}
async function measure<T>(label: string, operation: () => Promise<T>) {
const start = performance.now();
const value = await operation();
return {
label,
latencyMs: Number((performance.now() - start).toFixed(2)),
value,
};
}
describe("remediation performance condition", () => {
it("measures actual migrated services at the expected upper-use fixture", async () => {
seedRealApplicationSchema();
const activity = await measure("getActivitySummary", () => getActivitySummary(NOW));
const availability = await measure("getStudyAvailabilityByClass", () =>
getStudyAvailabilityByClass(NOW)
);
const classPolling = await measure("listClasses/navbar-poll", () => listClasses());
const measurements = [activity, availability, classPolling].map((result) => ({
label: result.label,
latencyMs: result.latencyMs,
}));
console.info(JSON.stringify({
fixture: { activityRows: ACTIVITY_ROWS, cards: CARD_ROWS, sets: SETS },
budgetMs: REQUEST_BUDGET_MS,
prismaServiceOperations: { activity: 1, availability: 1, classPolling: 2 },
measurements,
activity: {
displayedDays: activity.value.days.length,
currentStreak: activity.value.currentStreak,
},
readyCards: availability.value["benchmark-class"]?.readyCards,
}, null, 2));
expect(activity.value.days).toHaveLength(53 * 7);
expect(activity.value.currentStreak).toBeGreaterThan(53 * 7);
expect(availability.value["benchmark-class"]).toBeDefined();
expect(classPolling.value).toHaveLength(1);
for (const measurement of measurements) {
expect(measurement.latencyMs, measurement.label).toBeLessThanOrEqual(
REQUEST_BUDGET_MS
);
}
});
});

View file

@ -1,15 +0,0 @@
services:
study-app:
build:
context: .
target: deps
command: npm run dev
ports:
- "3000:3000"
environment:
- DATABASE_URL=file:/app/data/study.db
- NODE_ENV=development
volumes:
- .:/app
- /app/node_modules
- ./data:/app/data

View file

@ -1,16 +0,0 @@
services:
study-app:
build: .
container_name: study-app
restart: unless-stopped
ports:
- "3000:3726"
environment:
- DATABASE_URL=file:/app/data/study.db
- SESSION_SECRET=${SESSION_SECRET:?SESSION_SECRET must be set to at least 32 characters}
- ADMIN_PASSWORD_HASH=${ADMIN_PASSWORD_HASH:-}
- ALLOW_INITIAL_SETUP=${ALLOW_INITIAL_SETUP:-false}
- SECURE_COOKIES=${SECURE_COOKIES:-false}
- NODE_ENV=production
volumes:
- ./data:/app/data

View file

@ -1,13 +0,0 @@
#!/bin/sh
set -e
DEFAULT_SECRET='dev-session-secret-change-in-production-must-be-32-chars'
TRIMMED_SECRET=$(printf '%s' "${SESSION_SECRET:-}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
if [ -z "$TRIMMED_SECRET" ] || [ "${#TRIMMED_SECRET}" -lt 32 ] || [ "$TRIMMED_SECRET" = "$DEFAULT_SECRET" ]; then
echo >&2 "SESSION_SECRET must be a non-default value of at least 32 characters"
exit 1
fi
node scripts/migration-preflight.mjs --startup
./node_modules/.bin/prisma migrate deploy
exec node server.js

View file

@ -1,19 +0,0 @@
import { defineConfig, globalIgnores } from "eslint/config";
import nextVitals from "eslint-config-next/core-web-vitals";
import nextTs from "eslint-config-next/typescript";
const eslintConfig = defineConfig([
...nextVitals,
...nextTs,
// Override default ignores of eslint-config-next.
globalIgnores([
// Default ignores of eslint-config-next:
".next/**",
"out/**",
"build/**",
"next-env.d.ts",
"audit-results/tmp/**",
]),
]);
export default eslintConfig;

View file

@ -1,8 +0,0 @@
import type { NextConfig } from "next";
const nextConfig: NextConfig = {
output: "standalone",
serverExternalPackages: ["better-sqlite3"],
};
export default nextConfig;

11116
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -1,48 +1,61 @@
{
"name": "study-app",
"version": "0.1.0",
"name": "study-desk",
"version": "2.0.0",
"private": true,
"type": "module",
"description": "Study Desk V2 - a self-hosted, single-user study platform with quizzes, spaced-repetition flashcards, documents, and AI.",
"engines": {
"node": ">=24"
},
"scripts": {
"predev": "prisma migrate deploy",
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "eslint",
"dev": "concurrently -k -n api,web -c blue,magenta \"npm:dev:server\" \"npm:dev:web\"",
"dev:server": "tsx watch --clear-screen=false server/main.ts",
"dev:web": "vite",
"build": "npm run build:web && npm run build:server",
"build:web": "vite build",
"build:server": "node scripts/build-server.mjs",
"start": "node dist/server/main.js",
"typecheck": "tsc -p tsconfig.server.json --noEmit && tsc -p tsconfig.web.json --noEmit",
"test": "vitest run",
"benchmark:remediation": "vitest run --config vitest.benchmark.config.ts",
"db:preflight": "node scripts/migration-preflight.mjs",
"db:backup": "node scripts/backup-database.mjs",
"auth:reset": "node scripts/request-password-reset.mjs"
"test:watch": "vitest",
"check": "npm run typecheck && npm run test && npm run build"
},
"dependencies": {
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@prisma/adapter-better-sqlite3": "^7.8.0",
"@prisma/client": "^7.8.0",
"argon2": "^0.44.0",
"better-sqlite3": "^12.11.1",
"iron-session": "^8.0.4",
"jsonrepair": "^3.14.1",
"next": "16.3.0",
"prisma": "^7.8.0",
"react": "19.2.4",
"react-dom": "19.2.4",
"@fontsource-variable/instrument-sans": "^5.3.0",
"@fontsource-variable/jetbrains-mono": "^5.3.0",
"@fontsource-variable/newsreader": "^5.3.0",
"@hono/node-server": "^2.1.6",
"@tanstack/react-query": "^5.107.1",
"clsx": "^2.1.1",
"fast-xml-parser": "^5.12.0",
"fflate": "^0.8.3",
"hono": "^4.13.17",
"jsonrepair": "^3.15.0",
"lucide-react": "^1.71.0",
"mammoth": "^1.13.0",
"radix-ui": "^1.8.1",
"react": "^19.3.1",
"react-dom": "^19.3.1",
"react-markdown": "^10.1.0",
"react-router": "^8.5.1",
"remark-gfm": "^4.0.1",
"ts-fsrs": "^5.4.1",
"zod": "^4.4.3"
"sonner": "^2.0.8",
"ts-fsrs": "^6.0.0",
"unpdf": "^2.1.0",
"zod": "^4.6.7"
},
"devDependencies": {
"@tailwindcss/postcss": "^4",
"@types/better-sqlite3": "^9.6.0",
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",
"eslint": "^9",
"eslint-config-next": "16.2.9",
"tailwindcss": "^4",
"typescript": "^5",
"vitest": "^4.1.10"
"@tailwindcss/vite": "^4.3.5",
"@types/node": "^26.6.6",
"@types/react": "^19.3.0",
"@types/react-dom": "^19.3.0",
"@vitejs/plugin-react": "^6.3.0",
"concurrently": "^10.0.7",
"esbuild": "^0.28.2",
"tailwindcss": "^4.3.5",
"tsx": "^4.23.15",
"typescript": "^7.0.2",
"vite": "^8.3.8",
"vitest": "^5.1.0"
}
}

View file

@ -1,7 +0,0 @@
const config = {
plugins: {
"@tailwindcss/postcss": {},
},
};
export default config;

View file

@ -1,9 +0,0 @@
import path from "node:path";
import { defineConfig } from "prisma/config";
export default defineConfig({
schema: path.join(__dirname, "prisma", "schema.prisma"),
datasource: {
url: process.env.DATABASE_URL ?? "file:./dev.db",
},
});

View file

@ -1,130 +0,0 @@
-- CreateTable
CREATE TABLE "Class" (
"id" TEXT NOT NULL PRIMARY KEY,
"slug" TEXT NOT NULL,
"name" TEXT NOT NULL,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- CreateTable
CREATE TABLE "Deck" (
"id" TEXT NOT NULL PRIMARY KEY,
"classId" TEXT NOT NULL,
"name" TEXT NOT NULL,
"description" TEXT,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "Deck_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "Flashcard" (
"id" TEXT NOT NULL PRIMARY KEY,
"deckId" TEXT NOT NULL,
"front" TEXT NOT NULL,
"back" TEXT NOT NULL,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
CONSTRAINT "Flashcard_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "QuizSet" (
"id" TEXT NOT NULL PRIMARY KEY,
"classId" TEXT NOT NULL,
"name" TEXT NOT NULL,
"description" TEXT,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "QuizSet_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "Question" (
"id" TEXT NOT NULL PRIMARY KEY,
"quizSetId" TEXT NOT NULL,
"type" TEXT NOT NULL,
"prompt" TEXT NOT NULL,
"rationale" TEXT NOT NULL,
"category" TEXT NOT NULL,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
CONSTRAINT "Question_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "AnswerOption" (
"id" TEXT NOT NULL PRIMARY KEY,
"questionId" TEXT NOT NULL,
"text" TEXT NOT NULL,
"isCorrect" BOOLEAN NOT NULL DEFAULT false,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
CONSTRAINT "AnswerOption_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "StudyProgress" (
"id" TEXT NOT NULL PRIMARY KEY,
"contentType" TEXT NOT NULL,
"deckId" TEXT,
"quizSetId" TEXT,
"mode" TEXT NOT NULL,
"currentIndex" INTEGER NOT NULL DEFAULT 0,
"orderJson" TEXT NOT NULL,
"answersJson" TEXT,
"cardResultsJson" TEXT,
"updatedAt" DATETIME NOT NULL,
CONSTRAINT "StudyProgress_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "StudyProgress_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "QuizAttempt" (
"id" TEXT NOT NULL PRIMARY KEY,
"quizSetId" TEXT NOT NULL,
"score" REAL NOT NULL,
"maxScore" INTEGER NOT NULL,
"answersJson" TEXT NOT NULL,
"isPartialRetake" BOOLEAN NOT NULL DEFAULT false,
"completedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "QuizAttempt_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "ShareLink" (
"id" TEXT NOT NULL PRIMARY KEY,
"targetType" TEXT NOT NULL,
"deckId" TEXT,
"quizSetId" TEXT,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "ShareLink_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "ShareLink_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "AuthSecurity" (
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT DEFAULT 1,
"failedAttempts" INTEGER NOT NULL DEFAULT 0,
"lockedUntil" DATETIME,
"lastAttemptAt" DATETIME
);
-- CreateTable
CREATE TABLE "Setting" (
"key" TEXT NOT NULL PRIMARY KEY,
"value" TEXT NOT NULL
);
-- CreateIndex
CREATE UNIQUE INDEX "Class_slug_key" ON "Class"("slug");
-- CreateIndex
CREATE UNIQUE INDEX "StudyProgress_deckId_mode_key" ON "StudyProgress"("deckId", "mode");
-- CreateIndex
CREATE UNIQUE INDEX "StudyProgress_quizSetId_mode_key" ON "StudyProgress"("quizSetId", "mode");
-- CreateIndex
CREATE UNIQUE INDEX "ShareLink_deckId_key" ON "ShareLink"("deckId");
-- CreateIndex
CREATE UNIQUE INDEX "ShareLink_quizSetId_key" ON "ShareLink"("quizSetId");

View file

@ -1,9 +0,0 @@
-- CreateTable
CREATE TABLE "StudyActivity" (
"id" TEXT NOT NULL PRIMARY KEY,
"type" TEXT NOT NULL,
"occurredAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- CreateIndex
CREATE INDEX "StudyActivity_occurredAt_idx" ON "StudyActivity"("occurredAt");

View file

@ -1,64 +0,0 @@
-- CreateTable
CREATE TABLE "SpacedRepetitionSet" (
"id" TEXT NOT NULL PRIMARY KEY,
"classId" TEXT NOT NULL,
"name" TEXT NOT NULL,
"description" TEXT,
"newCardsPerDay" INTEGER NOT NULL DEFAULT 30,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" DATETIME NOT NULL,
CONSTRAINT "SpacedRepetitionSet_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "SpacedRepetitionSetDeck" (
"setId" TEXT NOT NULL,
"deckId" TEXT NOT NULL,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"addedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY ("setId", "deckId"),
CONSTRAINT "SpacedRepetitionSetDeck_setId_fkey" FOREIGN KEY ("setId") REFERENCES "SpacedRepetitionSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "SpacedRepetitionSetDeck_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "SpacedRepetitionCardState" (
"id" TEXT NOT NULL PRIMARY KEY,
"setId" TEXT NOT NULL,
"flashcardId" TEXT NOT NULL,
"due" DATETIME NOT NULL,
"stability" REAL NOT NULL,
"difficulty" REAL NOT NULL,
"elapsedDays" INTEGER NOT NULL,
"scheduledDays" INTEGER NOT NULL,
"learningSteps" INTEGER NOT NULL DEFAULT 0,
"reps" INTEGER NOT NULL,
"lapses" INTEGER NOT NULL,
"state" INTEGER NOT NULL,
"firstReviewedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"lastReview" DATETIME,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" DATETIME NOT NULL,
CONSTRAINT "SpacedRepetitionCardState_setId_fkey" FOREIGN KEY ("setId") REFERENCES "SpacedRepetitionSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "SpacedRepetitionCardState_flashcardId_fkey" FOREIGN KEY ("flashcardId") REFERENCES "Flashcard" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateIndex
CREATE INDEX "SpacedRepetitionSet_classId_sortOrder_idx" ON "SpacedRepetitionSet"("classId", "sortOrder");
-- CreateIndex
CREATE INDEX "SpacedRepetitionSetDeck_deckId_idx" ON "SpacedRepetitionSetDeck"("deckId");
-- CreateIndex
CREATE INDEX "SpacedRepetitionCardState_setId_due_idx" ON "SpacedRepetitionCardState"("setId", "due");
-- CreateIndex
CREATE INDEX "SpacedRepetitionCardState_setId_firstReviewedAt_idx" ON "SpacedRepetitionCardState"("setId", "firstReviewedAt");
-- CreateIndex
CREATE INDEX "SpacedRepetitionCardState_flashcardId_idx" ON "SpacedRepetitionCardState"("flashcardId");
-- CreateIndex
CREATE UNIQUE INDEX "SpacedRepetitionCardState_setId_flashcardId_key" ON "SpacedRepetitionCardState"("setId", "flashcardId");

View file

@ -1,40 +0,0 @@
-- CreateTable
CREATE TABLE "ArcadePack" (
"id" TEXT NOT NULL PRIMARY KEY,
"classId" TEXT NOT NULL,
"gameType" TEXT NOT NULL,
"name" TEXT NOT NULL,
"description" TEXT,
"schemaVersion" INTEGER NOT NULL,
"sourceJson" TEXT NOT NULL,
"normalizedJson" TEXT NOT NULL,
"validationReportJson" TEXT NOT NULL,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" DATETIME NOT NULL,
CONSTRAINT "ArcadePack_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateTable
CREATE TABLE "ArcadeAttempt" (
"id" TEXT NOT NULL PRIMARY KEY,
"arcadePackId" TEXT NOT NULL,
"mode" TEXT NOT NULL,
"score" INTEGER NOT NULL,
"maxScore" INTEGER NOT NULL,
"accuracy" REAL NOT NULL,
"durationSeconds" INTEGER NOT NULL,
"mistakes" INTEGER NOT NULL,
"hintsUsed" INTEGER NOT NULL,
"settingsJson" TEXT NOT NULL,
"resultsJson" TEXT NOT NULL,
"seed" TEXT NOT NULL,
"completedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "ArcadeAttempt_arcadePackId_fkey" FOREIGN KEY ("arcadePackId") REFERENCES "ArcadePack" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- CreateIndex
CREATE INDEX "ArcadePack_classId_gameType_sortOrder_idx" ON "ArcadePack"("classId", "gameType", "sortOrder");
-- CreateIndex
CREATE INDEX "ArcadeAttempt_arcadePackId_completedAt_idx" ON "ArcadeAttempt"("arcadePackId", "completedAt");

View file

@ -1,68 +0,0 @@
PRAGMA defer_foreign_keys=ON;
PRAGMA foreign_keys=OFF;
-- CreateTable
CREATE TABLE "MaterialGroup" (
"id" TEXT NOT NULL PRIMARY KEY,
"classId" TEXT NOT NULL,
"name" TEXT NOT NULL,
"type" TEXT NOT NULL,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "MaterialGroup_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
-- RedefineTables
CREATE TABLE "new_Deck" (
"id" TEXT NOT NULL PRIMARY KEY,
"classId" TEXT NOT NULL,
"name" TEXT NOT NULL,
"description" TEXT,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"groupId" TEXT,
CONSTRAINT "Deck_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "Deck_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "MaterialGroup" ("id") ON DELETE SET NULL ON UPDATE CASCADE
);
INSERT INTO "new_Deck" ("classId", "createdAt", "description", "id", "name", "sortOrder")
SELECT "classId", "createdAt", "description", "id", "name", "sortOrder" FROM "Deck";
DROP TABLE "Deck";
ALTER TABLE "new_Deck" RENAME TO "Deck";
CREATE TABLE "new_QuizSet" (
"id" TEXT NOT NULL PRIMARY KEY,
"classId" TEXT NOT NULL,
"name" TEXT NOT NULL,
"description" TEXT,
"sortOrder" INTEGER NOT NULL DEFAULT 0,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
"groupId" TEXT,
CONSTRAINT "QuizSet_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "QuizSet_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "MaterialGroup" ("id") ON DELETE SET NULL ON UPDATE CASCADE
);
INSERT INTO "new_QuizSet" ("classId", "createdAt", "description", "id", "name", "sortOrder")
SELECT "classId", "createdAt", "description", "id", "name", "sortOrder" FROM "QuizSet";
DROP TABLE "QuizSet";
ALTER TABLE "new_QuizSet" RENAME TO "QuizSet";
CREATE TABLE "new_ShareLink" (
"id" TEXT NOT NULL PRIMARY KEY,
"targetType" TEXT NOT NULL,
"deckId" TEXT,
"quizSetId" TEXT,
"groupId" TEXT,
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "ShareLink_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "ShareLink_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
CONSTRAINT "ShareLink_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "MaterialGroup" ("id") ON DELETE CASCADE ON UPDATE CASCADE
);
INSERT INTO "new_ShareLink" ("createdAt", "deckId", "id", "quizSetId", "targetType")
SELECT "createdAt", "deckId", "id", "quizSetId", "targetType" FROM "ShareLink";
DROP TABLE "ShareLink";
ALTER TABLE "new_ShareLink" RENAME TO "ShareLink";
CREATE UNIQUE INDEX "ShareLink_deckId_key" ON "ShareLink"("deckId");
CREATE UNIQUE INDEX "ShareLink_quizSetId_key" ON "ShareLink"("quizSetId");
CREATE UNIQUE INDEX "ShareLink_groupId_key" ON "ShareLink"("groupId");
PRAGMA foreign_keys=ON;
PRAGMA defer_foreign_keys=OFF;

View file

@ -1,2 +0,0 @@
-- AlterTable
ALTER TABLE "QuizAttempt" ADD COLUMN "reviewSnapshotJson" TEXT;

View file

@ -1,3 +0,0 @@
-- AlterTable
ALTER TABLE "StudyProgress" ADD COLUMN "sessionId" TEXT NOT NULL DEFAULT 'legacy';
ALTER TABLE "StudyProgress" ADD COLUMN "revision" INTEGER NOT NULL DEFAULT 0;

View file

@ -1,10 +0,0 @@
-- Remove Arcade-generated activity and saved instruction settings.
DELETE FROM "StudyActivity"
WHERE "type" IN ('ARCADE_GROUP', 'ARCADE_WORD');
DELETE FROM "Setting"
WHERE "key" IN ('llmInstructionsConnections', 'llmInstructionsCrossword');
-- Drop the dependent table before its parent while SQLite foreign keys are enabled.
DROP TABLE IF EXISTS "ArcadeAttempt";
DROP TABLE IF EXISTS "ArcadePack";

View file

@ -1,3 +0,0 @@
# Please do not edit this file manually
# It should be added in your version-control system (e.g., Git)
provider = "sqlite"

View file

@ -1,227 +0,0 @@
datasource db {
provider = "sqlite"
}
generator client {
provider = "prisma-client"
output = "../src/generated/prisma"
}
model Class {
id String @id @default(uuid())
slug String @unique
name String
sortOrder Int @default(0)
createdAt DateTime @default(now())
decks Deck[]
quizSets QuizSet[]
materialGroups MaterialGroup[]
spacedRepetitionSets SpacedRepetitionSet[]
}
model Deck {
id String @id @default(uuid())
classId String
name String
description String?
sortOrder Int @default(0)
createdAt DateTime @default(now())
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
groupId String?
group MaterialGroup? @relation(fields: [groupId], references: [id], onDelete: SetNull)
cards Flashcard[]
shareLink ShareLink?
progress StudyProgress[]
spacedRepetitionMemberships SpacedRepetitionSetDeck[]
}
model Flashcard {
id String @id @default(uuid())
deckId String
front String
back String
sortOrder Int @default(0)
deck Deck @relation(fields: [deckId], references: [id], onDelete: Cascade)
spacedRepetitionStates SpacedRepetitionCardState[]
}
model QuizSet {
id String @id @default(uuid())
classId String
name String
description String?
sortOrder Int @default(0)
createdAt DateTime @default(now())
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
groupId String?
group MaterialGroup? @relation(fields: [groupId], references: [id], onDelete: SetNull)
questions Question[]
attempts QuizAttempt[]
shareLink ShareLink?
progress StudyProgress[]
}
model Question {
id String @id @default(uuid())
quizSetId String
type String // "MULTIPLE_CHOICE" | "SATA"
prompt String
rationale String
category String
sortOrder Int @default(0)
quizSet QuizSet @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
options AnswerOption[]
}
model AnswerOption {
id String @id @default(uuid())
questionId String
text String
isCorrect Boolean @default(false)
sortOrder Int @default(0)
question Question @relation(fields: [questionId], references: [id], onDelete: Cascade)
}
model StudyProgress {
id String @id @default(uuid())
contentType String // "DECK" | "QUIZ"
deckId String?
quizSetId String?
mode String // "SEQUENTIAL" | "SHUFFLED"
currentIndex Int @default(0)
orderJson String // JSON array of card/question ids
answersJson String? // in-progress quiz answers
cardResultsJson String? // per-card grades for decks
sessionId String @default("legacy")
revision Int @default(0)
updatedAt DateTime @updatedAt
deck Deck? @relation(fields: [deckId], references: [id], onDelete: Cascade)
quizSet QuizSet? @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
@@unique([deckId, mode])
@@unique([quizSetId, mode])
}
model QuizAttempt {
id String @id @default(uuid())
quizSetId String
score Float
maxScore Int
answersJson String
reviewSnapshotJson String?
isPartialRetake Boolean @default(false)
completedAt DateTime @default(now())
quizSet QuizSet @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
}
model ShareLink {
id String @id @default(uuid())
targetType String // "DECK" | "QUIZ" | "GROUP"
deckId String? @unique
quizSetId String? @unique
groupId String? @unique
createdAt DateTime @default(now())
deck Deck? @relation(fields: [deckId], references: [id], onDelete: Cascade)
quizSet QuizSet? @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
group MaterialGroup? @relation(fields: [groupId], references: [id], onDelete: Cascade)
}
model AuthSecurity {
id Int @id @default(1)
failedAttempts Int @default(0)
lockedUntil DateTime?
lastAttemptAt DateTime?
}
model Setting {
key String @id
value String
}
model StudyActivity {
id String @id @default(uuid())
type String // "FLASHCARD" | "QUIZ_QUESTION"
occurredAt DateTime @default(now())
@@index([occurredAt])
}
model MaterialGroup {
id String @id @default(uuid())
classId String
name String
type String // "DECK" | "QUIZ"
sortOrder Int @default(0)
createdAt DateTime @default(now())
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
decks Deck[]
quizSets QuizSet[]
shareLink ShareLink?
}
model SpacedRepetitionSet {
id String @id @default(uuid())
classId String
name String
description String?
newCardsPerDay Int @default(30)
sortOrder Int @default(0)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
decks SpacedRepetitionSetDeck[]
cardStates SpacedRepetitionCardState[]
@@index([classId, sortOrder])
}
model SpacedRepetitionSetDeck {
setId String
deckId String
sortOrder Int @default(0)
addedAt DateTime @default(now())
set SpacedRepetitionSet @relation(fields: [setId], references: [id], onDelete: Cascade)
deck Deck @relation(fields: [deckId], references: [id], onDelete: Cascade)
@@id([setId, deckId])
@@index([deckId])
}
model SpacedRepetitionCardState {
id String @id @default(uuid())
setId String
flashcardId String
due DateTime
stability Float
difficulty Float
elapsedDays Int
scheduledDays Int
learningSteps Int @default(0)
reps Int
lapses Int
state Int
firstReviewedAt DateTime @default(now())
lastReview DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
set SpacedRepetitionSet @relation(fields: [setId], references: [id], onDelete: Cascade)
flashcard Flashcard @relation(fields: [flashcardId], references: [id], onDelete: Cascade)
@@unique([setId, flashcardId])
@@index([setId, due])
@@index([setId, firstReviewedAt])
@@index([flashcardId])
}

View file

@ -1 +0,0 @@
<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>

Before

Width:  |  Height:  |  Size: 391 B

View file

@ -1 +0,0 @@
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>

Before

Width:  |  Height:  |  Size: 1 KiB

View file

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

Before

Width:  |  Height:  |  Size: 1.3 KiB

View file

@ -1 +0,0 @@
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>

Before

Width:  |  Height:  |  Size: 128 B

View file

@ -1 +0,0 @@
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>

Before

Width:  |  Height:  |  Size: 385 B

View file

@ -1,42 +0,0 @@
import { existsSync } from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import Database from "better-sqlite3";
import { databasePathFromUrl } from "./databasePath.mjs";
function argument(name) {
const index = process.argv.indexOf(name);
return index >= 0 ? process.argv[index + 1] : undefined;
}
export async function backupDatabase(sourcePath, outputPath) {
const source = path.resolve(sourcePath);
const output = path.resolve(outputPath);
if (!existsSync(source)) throw new Error(`Source database does not exist: ${source}`);
if (existsSync(output)) throw new Error(`Backup destination already exists: ${output}`);
if (source === output) throw new Error("Backup destination must differ from source");
const database = new Database(source, { readonly: true, fileMustExist: true });
try {
await database.backup(output);
} finally {
database.close();
}
const backup = new Database(output, { readonly: true, fileMustExist: true });
try {
const integrity = backup.pragma("integrity_check", { simple: true });
if (integrity !== "ok") throw new Error(`Backup integrity check failed: ${integrity}`);
} finally {
backup.close();
}
return output;
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
const source = argument("--source") ?? databasePathFromUrl(process.env.DATABASE_URL);
const output = argument("--output");
if (!output) throw new Error("Usage: npm run db:backup -- --output <new-backup.db>");
const completedPath = await backupDatabase(source, output);
console.info(`Verified SQLite backup created: ${completedPath}`);
}

View file

@ -1,10 +0,0 @@
import path from "node:path";
export function databasePathFromUrl(databaseUrl, cwd = process.cwd()) {
if (!databaseUrl?.startsWith("file:")) {
throw new Error("DATABASE_URL must be a file: SQLite URL");
}
const rawPath = databaseUrl.slice("file:".length).split("?")[0];
if (!rawPath) throw new Error("DATABASE_URL does not contain a database path");
return path.resolve(cwd, rawPath);
}

View file

@ -1,358 +0,0 @@
import { existsSync, readFileSync, readdirSync } from "node:fs";
import path from "node:path";
import { spawnSync } from "node:child_process";
import { pathToFileURL } from "node:url";
import Database from "better-sqlite3";
import { databasePathFromUrl } from "./databasePath.mjs";
export const GROUP_MIGRATION = "20260807090000_add_material_groups";
function tableExists(database, table) {
return Boolean(
database
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?")
.get(table)
);
}
function hasColumn(database, table, column) {
if (!tableExists(database, table)) return false;
return database.pragma(`table_info(${table})`).some((item) => item.name === column);
}
function hasForeignKey(database, table, from, target, onDelete) {
return database
.pragma(`foreign_key_list(${table})`)
.some(
(item) =>
item.from === from && item.table === target && item.on_delete === onDelete
);
}
function hasUniqueIndex(database, table, indexName, column) {
const index = database
.pragma(`index_list(${table})`)
.find((item) => item.name === indexName && item.unique === 1);
if (!index) return false;
const columns = database.pragma(`index_info(${indexName})`);
return columns.length === 1 && columns[0].name === column;
}
function schemaSnapshot(database, includeMigrationTable = false) {
const tables = database
.prepare(
`SELECT name FROM sqlite_master
WHERE type = 'table'
AND name NOT LIKE 'sqlite_%'
${includeMigrationTable ? "" : "AND name <> '_prisma_migrations'"}
ORDER BY name`
)
.all()
.map((row) => row.name);
return tables.map((table) => ({
table,
columns: database.pragma(`table_info(${JSON.stringify(table)})`).map((column) => ({
name: column.name,
type: column.type,
notnull: column.notnull,
defaultValue: column.dflt_value,
primaryKey: column.pk,
})),
foreignKeys: database
.pragma(`foreign_key_list(${JSON.stringify(table)})`)
.map((foreignKey) => ({
id: foreignKey.id,
sequence: foreignKey.seq,
target: foreignKey.table,
from: foreignKey.from,
to: foreignKey.to,
onUpdate: foreignKey.on_update,
onDelete: foreignKey.on_delete,
match: foreignKey.match,
}))
.sort((left, right) => left.id - right.id || left.sequence - right.sequence),
indexes: database
.pragma(`index_list(${JSON.stringify(table)})`)
.filter((index) => !index.name.startsWith("sqlite_autoindex_"))
.map((index) => ({
name: index.name,
unique: index.unique,
partial: index.partial,
columns: database
.pragma(`index_info(${JSON.stringify(index.name)})`)
.map((column) => column.name),
}))
.sort((left, right) => left.name.localeCompare(right.name)),
}));
}
function expectedSchemaSnapshot(migrationNames) {
const database = new Database(":memory:");
try {
database.pragma("foreign_keys = ON");
for (const migrationName of migrationNames) {
database.exec(
readFileSync(
path.join(
process.cwd(),
"prisma",
"migrations",
migrationName,
"migration.sql"
),
"utf8"
)
);
}
return schemaSnapshot(database);
} finally {
database.close();
}
}
export function inspectMigrationState(databasePath) {
if (!existsSync(databasePath)) {
throw new Error(`Database does not exist: ${databasePath}`);
}
const database = new Database(databasePath, { readonly: true, fileMustExist: true });
try {
const migrationRecords = tableExists(database, "_prisma_migrations")
? database
.prepare(
'SELECT "migration_name", "finished_at", "rolled_back_at" FROM "_prisma_migrations"'
)
.all()
: [];
const activeRecords = migrationRecords.filter((row) => row.rolled_back_at === null);
const migrationNames = activeRecords
.filter((row) => row.finished_at !== null)
.map((row) => row.migration_name);
const failedMigrationNames = activeRecords
.filter((row) => row.finished_at === null)
.map((row) => row.migration_name);
const committedMigrations = readdirSync(
path.join(process.cwd(), "prisma", "migrations"),
{ withFileTypes: true }
)
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort();
const priorMigrations = committedMigrations.filter(
(name) => name < GROUP_MIGRATION
);
const groupMigrationIndex = committedMigrations.indexOf(GROUP_MIGRATION);
const artifacts = {
materialGroup: tableExists(database, "MaterialGroup"),
deckColumn: hasColumn(database, "Deck", "groupId"),
quizColumn: hasColumn(database, "QuizSet", "groupId"),
shareColumn: hasColumn(database, "ShareLink", "groupId"),
deckForeignKey: hasForeignKey(
database,
"Deck",
"groupId",
"MaterialGroup",
"SET NULL"
),
quizForeignKey: hasForeignKey(
database,
"QuizSet",
"groupId",
"MaterialGroup",
"SET NULL"
),
shareForeignKey: hasForeignKey(
database,
"ShareLink",
"groupId",
"MaterialGroup",
"CASCADE"
),
shareUnique: hasUniqueIndex(
database,
"ShareLink",
"ShareLink_groupId_key",
"groupId"
),
};
const priorHistoryComplete = priorMigrations.every((name) =>
migrationNames.includes(name)
);
const knownMigrationNames = new Set(committedMigrations);
const historyHasUnknown = activeRecords.some(
(row) => !knownMigrationNames.has(row.migration_name)
);
const actualSchema = schemaSnapshot(database);
const matches = (expected) =>
JSON.stringify(actualSchema) === JSON.stringify(expected);
const preGroupSchemaExact = matches(expectedSchemaSnapshot(priorMigrations));
const adoptedGroupSchemaExact = matches(
expectedSchemaSnapshot([...priorMigrations, GROUP_MIGRATION])
);
let historyPrefixLength = 0;
while (
historyPrefixLength < committedMigrations.length &&
migrationNames.includes(committedMigrations[historyPrefixLength])
) {
historyPrefixLength += 1;
}
const historyIsExactPrefix =
!historyHasUnknown &&
migrationNames.length === historyPrefixLength &&
migrationNames.every((name) =>
committedMigrations.slice(0, historyPrefixLength).includes(name)
);
const trackedSchemaExact =
historyIsExactPrefix &&
matches(expectedSchemaSnapshot(committedMigrations.slice(0, historyPrefixLength)));
let schemaPrefixLength = -1;
for (let index = 0; index <= committedMigrations.length; index += 1) {
if (matches(expectedSchemaSnapshot(committedMigrations.slice(0, index)))) {
schemaPrefixLength = index;
break;
}
}
const migrationsToAdopt =
schemaPrefixLength > historyPrefixLength
? committedMigrations.slice(historyPrefixLength, schemaPrefixLength)
: [];
const failuresAreAdoptable = failedMigrationNames.every((name) =>
migrationsToAdopt.includes(name)
);
let classification = "CONFLICT";
if (
historyIsExactPrefix &&
failedMigrationNames.length === 0 &&
schemaPrefixLength === historyPrefixLength
) {
classification =
historyPrefixLength === committedMigrations.length ? "CURRENT" : "APPLY";
} else if (
historyIsExactPrefix &&
priorHistoryComplete &&
historyPrefixLength >= groupMigrationIndex &&
schemaPrefixLength > historyPrefixLength &&
failuresAreAdoptable
) {
classification = "ADOPT";
}
return {
classification,
artifacts,
migrationNames,
failedMigrationNames,
migrationsToAdopt,
priorHistoryComplete,
schemaMigrationPrefix:
schemaPrefixLength >= 0
? committedMigrations[schemaPrefixLength - 1] ?? null
: null,
exactSchema: {
preGroup: preGroupSchemaExact,
adoptedGroup: adoptedGroupSchemaExact,
tracked: trackedSchemaExact,
},
};
} finally {
database.close();
}
}
function verifyBackup(backupPath, sourcePath) {
if (!backupPath || !existsSync(backupPath)) {
throw new Error("--resolve requires an existing verified --backup file");
}
if (path.resolve(backupPath) === path.resolve(sourcePath)) {
throw new Error("--backup must be a separate database file, not the source database");
}
const backup = new Database(backupPath, { readonly: true, fileMustExist: true });
const source = new Database(sourcePath, { readonly: true, fileMustExist: true });
try {
if (backup.pragma("integrity_check", { simple: true }) !== "ok") {
throw new Error("Backup failed SQLite integrity_check");
}
if (
JSON.stringify(schemaSnapshot(backup, true)) !==
JSON.stringify(schemaSnapshot(source, true))
) {
throw new Error("Backup schema does not match the source database");
}
const sourceCounts = Object.fromEntries(
schemaSnapshot(source, true).map(({ table }) => [
table,
source.prepare(`SELECT COUNT(*) AS count FROM ${JSON.stringify(table)}`).get().count,
])
);
const backupCounts = Object.fromEntries(
schemaSnapshot(backup, true).map(({ table }) => [
table,
backup.prepare(`SELECT COUNT(*) AS count FROM ${JSON.stringify(table)}`).get().count,
])
);
if (JSON.stringify(sourceCounts) !== JSON.stringify(backupCounts)) {
throw new Error("Backup row counts do not match the source database");
}
} finally {
backup.close();
source.close();
}
}
function resolveMigration(databaseUrl, migrationName) {
const cli = path.join(process.cwd(), "node_modules", "prisma", "build", "index.js");
const result = spawnSync(
process.execPath,
[cli, "migrate", "resolve", "--applied", migrationName],
{
cwd: process.cwd(),
env: { ...process.env, DATABASE_URL: databaseUrl },
encoding: "utf8",
}
);
if (result.status !== 0) {
throw new Error(`Prisma migration resolve failed:\n${result.stdout}\n${result.stderr}`);
}
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
const databaseUrl = process.env.DATABASE_URL;
const databasePath = databasePathFromUrl(databaseUrl);
const startup = process.argv.includes("--startup");
const state =
startup && !existsSync(databasePath)
? {
classification: "EMPTY",
migrationNames: [],
failedMigrationNames: [],
migrationsToAdopt: [],
}
: inspectMigrationState(databasePath);
console.info(JSON.stringify(state, null, 2));
if (process.argv.includes("--resolve")) {
if (state.classification !== "ADOPT") {
throw new Error(`Refusing adoption for ${state.classification} database state`);
}
const backupIndex = process.argv.indexOf("--backup");
verifyBackup(
backupIndex >= 0 ? process.argv[backupIndex + 1] : undefined,
databasePath
);
for (const migrationName of state.migrationsToAdopt) {
resolveMigration(databaseUrl, migrationName);
}
console.info(
`Marked ${state.migrationsToAdopt.join(", ")} applied after exact-schema preflight.`
);
} else if (startup && state.classification === "ADOPT") {
console.error(
"Database schema requires explicit migration adoption. Stop, create and verify a backup, then run db:preflight -- --resolve --backup <backup-path>."
);
process.exitCode = 3;
} else if (state.classification === "CONFLICT") {
process.exitCode = 2;
}
}

View file

@ -1,56 +0,0 @@
import { createHash, randomBytes, randomUUID } from "node:crypto";
import { existsSync } from "node:fs";
import { pathToFileURL } from "node:url";
import Database from "better-sqlite3";
import { databasePathFromUrl } from "./databasePath.mjs";
const PASSWORD_HASH_KEY = "admin_password_hash";
const RESET_TOKEN_KEY = "admin_password_reset";
const RESET_TOKEN_LIFETIME_MS = 15 * 60 * 1000;
function activeRecord(value) {
try {
const record = JSON.parse(value);
return new Date(record.expiresAt).getTime() > Date.now();
} catch {
return false;
}
}
export function createResetToken(databasePath) {
if (!existsSync(databasePath)) throw new Error(`Database does not exist: ${databasePath}`);
const database = new Database(databasePath, { fileMustExist: true });
try {
if (!database.prepare('SELECT 1 FROM "Setting" WHERE "key" = ?').get(PASSWORD_HASH_KEY)) {
throw new Error("Password recovery is unavailable before initial setup");
}
const existing = database
.prepare('SELECT "value" FROM "Setting" WHERE "key" = ?')
.get(RESET_TOKEN_KEY);
if (existing && activeRecord(existing.value)) {
throw new Error("An unexpired reset token already exists; it was not replaced");
}
const token = randomBytes(24).toString("base64url");
const expiresAt = new Date(Date.now() + RESET_TOKEN_LIFETIME_MS).toISOString();
const value = JSON.stringify({
digest: createHash("sha256").update(token, "utf8").digest("hex"),
nonce: randomUUID(),
expiresAt,
});
database
.prepare(
'INSERT INTO "Setting" ("key", "value") VALUES (?, ?) ON CONFLICT("key") DO UPDATE SET "value" = excluded."value"'
)
.run(RESET_TOKEN_KEY, value);
return { token, expiresAt };
} finally {
database.close();
}
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
const result = createResetToken(databasePathFromUrl(process.env.DATABASE_URL));
console.info(`Password reset token: ${result.token}`);
console.info(`Expires: ${result.expiresAt}`);
}

View file

@ -1,197 +0,0 @@
import { randomUUID } from "node:crypto";
import {
copyFileSync,
existsSync,
mkdirSync,
readFileSync,
readdirSync,
rmSync,
statSync,
} from "node:fs";
import path from "node:path";
import Database from "better-sqlite3";
import { backupDatabase } from "./backup-database.mjs";
const root = path.join(process.cwd(), ".test-databases");
const directory = path.join(root, `backup-restore-${randomUUID()}`);
if (!path.resolve(directory).startsWith(`${path.resolve(root)}${path.sep}`)) {
throw new Error("Unsafe disposable backup verification path");
}
mkdirSync(directory, { recursive: true });
const sourcePath = path.join(directory, "source.test.db");
const backupPath = path.join(directory, "backup.test.db");
const restoredPath = path.join(directory, "restored.test.db");
let source;
function rowCounts(database) {
const tables = database
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
)
.all()
.map((row) => row.name);
return Object.fromEntries(
tables.map((table) => [
table,
database.prepare(`SELECT COUNT(*) AS count FROM ${JSON.stringify(table)}`).get().count,
])
);
}
try {
source = new Database(sourcePath);
source.pragma("foreign_keys = ON");
source.pragma("journal_mode = WAL");
source.pragma("wal_autocheckpoint = 0");
const migrations = readdirSync(path.join(process.cwd(), "prisma", "migrations"), {
withFileTypes: true,
})
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort();
for (const migration of migrations) {
source.exec(
readFileSync(
path.join(process.cwd(), "prisma", "migrations", migration, "migration.sql"),
"utf8"
)
);
}
source.transaction(() => {
source
.prepare('INSERT INTO "Class" ("id", "name", "slug", "sortOrder") VALUES (?, ?, ?, ?)')
.run("backup-class", "Representative content", "representative-content", 0);
source
.prepare(
'INSERT INTO "MaterialGroup" ("id", "classId", "name", "type", "sortOrder") VALUES (?, ?, ?, ?, ?)'
)
.run("backup-group", "backup-class", "Backup group", "DECK", 0);
source
.prepare(
'INSERT INTO "Deck" ("id", "classId", "groupId", "name", "description", "sortOrder") VALUES (?, ?, ?, ?, ?, ?)'
)
.run(
"backup-deck",
"backup-class",
"backup-group",
"Backup deck",
"Restore drill content",
0
);
source
.prepare(
'INSERT INTO "Flashcard" ("id", "deckId", "front", "back", "sortOrder") VALUES (?, ?, ?, ?, ?)'
)
.run("backup-card", "backup-deck", "Representative front", "Representative back", 0);
source
.prepare(
'INSERT INTO "QuizSet" ("id", "classId", "name", "description", "sortOrder") VALUES (?, ?, ?, ?, ?)'
)
.run("backup-quiz", "backup-class", "Backup quiz", "Quiz relationship", 0);
source
.prepare(
'INSERT INTO "Question" ("id", "quizSetId", "type", "prompt", "rationale", "category", "sortOrder") VALUES (?, ?, ?, ?, ?, ?, ?)'
)
.run(
"backup-question",
"backup-quiz",
"MULTIPLE_CHOICE",
"Representative prompt",
"Representative rationale",
"backup",
0
);
source
.prepare(
'INSERT INTO "AnswerOption" ("id", "questionId", "text", "isCorrect", "sortOrder") VALUES (?, ?, ?, ?, ?)'
)
.run("backup-option", "backup-question", "Correct option", 1, 0);
source
.prepare(
'INSERT INTO "SpacedRepetitionSet" ("id", "classId", "name", "sortOrder", "updatedAt") VALUES (?, ?, ?, ?, ?)'
)
.run("backup-srs", "backup-class", "Backup SRS", 0, Date.now());
source
.prepare(
'INSERT INTO "SpacedRepetitionSetDeck" ("setId", "deckId", "sortOrder") VALUES (?, ?, ?)'
)
.run("backup-srs", "backup-deck", 0);
source
.prepare(
'INSERT INTO "ShareLink" ("id", "targetType", "groupId") VALUES (?, ?, ?)'
)
.run("backup-share", "GROUP", "backup-group");
})();
const walPath = `${sourcePath}-wal`;
if (!existsSync(walPath) || statSync(walPath).size === 0) {
throw new Error("Backup drill did not create an active WAL database");
}
const sourceWalBytes = statSync(walPath).size;
const sourceCounts = rowCounts(source);
await backupDatabase(sourcePath, backupPath);
copyFileSync(backupPath, restoredPath);
source.close();
source = undefined;
const restored = new Database(restoredPath, { readonly: true, fileMustExist: true });
try {
const integrity = restored.pragma("integrity_check", { simple: true });
const foreignKeyErrors = restored.pragma("foreign_key_check");
const restoredCounts = rowCounts(restored);
const representative = restored
.prepare(`
SELECT c.name AS className, g.name AS groupName, d.description,
f.front, s.name AS setName, sl.targetType
FROM "Class" c
JOIN "MaterialGroup" g ON g.classId = c.id
JOIN "Deck" d ON d.groupId = g.id
JOIN "Flashcard" f ON f.deckId = d.id
JOIN "SpacedRepetitionSetDeck" sd ON sd.deckId = d.id
JOIN "SpacedRepetitionSet" s ON s.id = sd.setId
JOIN "ShareLink" sl ON sl.groupId = g.id
WHERE d.id = ?
`)
.get("backup-deck");
const quizRelationship = restored
.prepare(`
SELECT q.name AS quizName, question.prompt, option.text, option.isCorrect
FROM "QuizSet" q
JOIN "Question" question ON question.quizSetId = q.id
JOIN "AnswerOption" option ON option.questionId = question.id
WHERE q.id = ?
`)
.get("backup-quiz");
if (
integrity !== "ok" ||
foreignKeyErrors.length !== 0 ||
JSON.stringify(restoredCounts) !== JSON.stringify(sourceCounts) ||
representative?.description !== "Restore drill content" ||
representative?.setName !== "Backup SRS" ||
representative?.targetType !== "GROUP" ||
quizRelationship?.isCorrect !== 1
) {
throw new Error("Restored database did not preserve integrity, counts, content, and relationships");
}
console.info(
JSON.stringify(
{
method: "better-sqlite3 online backup while the WAL source remained open",
sourceWalBytes,
integrity,
foreignKeyErrors,
counts: restoredCounts,
representative,
quizRelationship,
},
null,
2
)
);
} finally {
restored.close();
}
} finally {
source?.close();
rmSync(directory, { recursive: true, force: true });
}

View file

@ -1,83 +0,0 @@
import { randomUUID } from "node:crypto";
import { mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
import path from "node:path";
import { spawn } from "node:child_process";
import Database from "better-sqlite3";
const root = path.join(process.cwd(), ".test-databases");
const directory = path.join(root, `http-smoke-${randomUUID()}`);
if (!path.resolve(directory).startsWith(`${path.resolve(root)}${path.sep}`)) {
throw new Error("Unsafe disposable HTTP smoke path");
}
mkdirSync(directory, { recursive: true });
const databasePath = path.join(directory, "http-smoke.test.db");
const database = new Database(databasePath);
try {
database.pragma("foreign_keys = ON");
for (const migration of readdirSync(path.join(process.cwd(), "prisma", "migrations"), { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort()) {
database.exec(readFileSync(path.join(process.cwd(), "prisma", "migrations", migration, "migration.sql"), "utf8"));
}
} finally {
database.close();
}
const port = 3789;
const server = spawn(
process.execPath,
[path.join(process.cwd(), "node_modules", "next", "dist", "bin", "next"), "start", "-p", String(port)],
{
cwd: process.cwd(),
env: {
...process.env,
NODE_ENV: "production",
DATABASE_URL: `file:${databasePath.replaceAll("\\", "/")}`,
SESSION_SECRET: "http-smoke-secret-0123456789abcdef0123456789",
ALLOW_INITIAL_SETUP: "true",
},
stdio: ["ignore", "pipe", "pipe"],
windowsHide: true,
}
);
let output = "";
server.stdout.on("data", (chunk) => { output += chunk.toString(); });
server.stderr.on("data", (chunk) => { output += chunk.toString(); });
try {
let health;
for (let attempt = 0; attempt < 30; attempt += 1) {
try {
health = await fetch(`http://127.0.0.1:${port}/api/health`);
if (health.ok) break;
} catch {}
await new Promise((resolve) => setTimeout(resolve, 500));
}
if (!health?.ok) throw new Error(`Health check did not become ready.\n${output}`);
const login = await fetch(`http://127.0.0.1:${port}/login`);
const setup = await fetch(`http://127.0.0.1:${port}/api/auth/setup-status`);
const setupBody = await setup.json();
const protectedDot = await fetch(`http://127.0.0.1:${port}/api/decks/file.json`, { redirect: "manual" });
if (!login.ok || !setup.ok || setupBody.setupRequired !== true || setupBody.setupAllowed !== true || protectedDot.status !== 307) {
throw new Error("HTTP smoke responses did not match the production contract");
}
console.info(JSON.stringify({
health: health.status,
login: login.status,
setup: setupBody,
protectedDot: { status: protectedDot.status, location: protectedDot.headers.get("location") },
}, null, 2));
if (process.argv.includes("--stay")) {
console.info(`Browser smoke server ready at http://127.0.0.1:${port}/login`);
await new Promise(() => {});
}
} finally {
server.kill();
await new Promise((resolve) => {
if (server.exitCode !== null) return resolve();
server.once("exit", resolve);
setTimeout(resolve, 2_000);
});
rmSync(directory, { recursive: true, force: true });
}

1000
server/ai/assistant.ts Normal file

File diff suppressed because it is too large Load diff

875
server/ai/generate.ts Normal file
View file

@ -0,0 +1,875 @@
/**
* Content generation: quizzes, flashcards and study guides from documents, pasted text,
* existing decks/quizzes, weak areas, or a plain topic.
*
* Every job writes into a draft row. Nothing reaches the library until the user has
* reviewed the draft and saved it. Long jobs are split into batches, each with its own
* slice of source material, so output limits and context windows are never a problem.
*/
import { z } from 'zod';
import {
validateDraftCard,
validateDraftQuestion,
type ContentDraft,
type DeckDraft,
type DraftCard,
type DraftQuestion,
type NoteDraft,
type QuizDraft,
} from '../../shared/content.ts';
import { draftsFromValue } from '../../shared/importer.ts';
import type { DB } from '../db.ts';
import { badRequest, notFound, type AppContext } from '../http.ts';
import { missedQuestionIds } from '../services/attempts.ts';
import {
createDeck,
createNote,
createQuiz,
draftCardToInput,
draftQuestionToInput,
getClass,
getDeck,
getQuiz,
insertCards,
insertQuestions,
listCards,
listQuestions,
updateQuestion,
} from '../services/content.ts';
import { chunkById, formatExcerpts, searchChunks, selectContext, sourceLabel, type ChunkHit } from '../services/documents.ts';
import { weakTopics } from '../services/stats.ts';
import { newId, now, parseJson } from '../util.ts';
import { generateStructured, resolveModel, runChat } from './service.ts';
import { AIError } from './types.ts';
/* ------------------------------ drafts ------------------------------ */
interface DraftRow {
id: string;
kind: 'quiz' | 'deck' | 'note';
class_id: string | null;
folder_id: string | null;
title: string;
payload: string;
meta: string;
status: 'generating' | 'ready' | 'failed' | 'saved' | 'discarded';
progress: number;
progress_label: string;
error: string;
saved_kind: string | null;
saved_id: string | null;
conversation_id: string | null;
created_at: number;
updated_at: number;
}
export interface DraftMeta {
origin?: 'generate' | 'assistant' | 'convert' | 'improve' | 'import';
model?: string;
request?: Record<string, unknown>;
sources?: { documentId: string; title: string }[];
/** For "improve" drafts: the stored question each draft question replaces (same order). */
replaceQuestionIds?: string[];
/** Suggested target when the draft is meant to extend an existing item. */
targetKind?: 'quiz' | 'deck';
targetId?: string;
notes?: string[];
}
function emptyPayload(kind: DraftRow['kind'], title = ''): ContentDraft {
if (kind === 'quiz') return { kind: 'quiz', title, description: '', questions: [] };
if (kind === 'deck') return { kind: 'deck', title, description: '', cards: [] };
return { kind: 'note', title, content: '' };
}
export function mapDraft(row: DraftRow) {
const payload = parseJson<ContentDraft>(row.payload, emptyPayload(row.kind, row.title));
return {
id: row.id,
kind: row.kind,
classId: row.class_id,
folderId: row.folder_id,
title: row.title,
payload,
meta: parseJson<DraftMeta>(row.meta, {}),
status: row.status,
progress: row.progress,
progressLabel: row.progress_label,
error: row.error,
savedKind: row.saved_kind,
savedId: row.saved_id,
conversationId: row.conversation_id,
itemCount: payload.kind === 'quiz' ? payload.questions.length : payload.kind === 'deck' ? payload.cards.length : payload.content.length,
createdAt: row.created_at,
updatedAt: row.updated_at,
};
}
export type DraftDto = ReturnType<typeof mapDraft>;
export function getDraft(db: DB, id: string): DraftDto {
const row = db.get<DraftRow>('SELECT * FROM drafts WHERE id = ?', id);
if (!row) throw notFound('Draft');
return mapDraft(row);
}
export function createDraft(
db: DB,
input: { kind: DraftRow['kind']; classId?: string | null; folderId?: string | null; title: string; payload?: ContentDraft; meta?: DraftMeta; status?: DraftRow['status']; conversationId?: string | null },
): string {
const id = newId();
const at = now();
db.run(
`INSERT INTO drafts (id, kind, class_id, folder_id, title, payload, meta, status, conversation_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
id,
input.kind,
input.classId ?? null,
input.folderId ?? null,
input.title.slice(0, 200),
JSON.stringify(input.payload ?? emptyPayload(input.kind, input.title)),
JSON.stringify(input.meta ?? {}),
input.status ?? 'ready',
input.conversationId ?? null,
at,
at,
);
return id;
}
export function listDrafts(db: DB) {
return db
.all<DraftRow>(`SELECT * FROM drafts WHERE status IN ('generating', 'ready', 'failed') ORDER BY updated_at DESC LIMIT 50`)
.map(mapDraft)
.map(({ payload: _payload, ...rest }) => rest);
}
export function updateDraftPayload(db: DB, id: string, payload: ContentDraft, extra: { classId?: string | null; folderId?: string | null } = {}): DraftDto {
const draft = getDraft(db, id);
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
if (payload.kind !== draft.kind) throw badRequest('Draft type cannot change');
db.run(
'UPDATE drafts SET payload = ?, title = ?, class_id = ?, folder_id = ?, updated_at = ? WHERE id = ?',
JSON.stringify(payload),
(payload.title ?? '').slice(0, 200),
extra.classId === undefined ? draft.classId : extra.classId,
extra.folderId === undefined ? draft.folderId : extra.folderId,
now(),
id,
);
return getDraft(db, id);
}
export const draftSaveSchema = z.object({
classId: z.string().optional(),
folderId: z.string().nullish(),
/** new: create an item. append: add to an existing quiz/deck. replace: apply improved questions in place. */
mode: z.enum(['new', 'append', 'replace']).default('new'),
targetId: z.string().optional(),
});
/** Turns a reviewed draft into real content. Invalid items are rejected, never silently dropped. */
export function saveDraft(app: AppContext, id: string, input: z.input<typeof draftSaveSchema>): { kind: string; id: string; count: number } {
const { db } = app;
const opts = draftSaveSchema.parse(input);
const draft = getDraft(db, id);
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
if (draft.status === 'saved' && draft.savedId) return { kind: draft.savedKind ?? draft.kind, id: draft.savedId, count: 0 };
const payload = draft.payload;
const classId = opts.classId ?? draft.classId;
let result: { kind: string; id: string; count: number };
if (payload.kind === 'quiz') {
const bad = payload.questions.flatMap((q, i) => validateDraftQuestion(q, i)).filter((x) => x.level === 'error');
if (bad.length > 0) throw badRequest(`Question ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
if (payload.questions.length === 0) throw badRequest('There are no questions to save');
const inputs = payload.questions.map(draftQuestionToInput);
if (opts.mode === 'replace') {
const ids = draft.meta.replaceQuestionIds ?? [];
const quizId = opts.targetId ?? draft.meta.targetId;
if (!quizId) throw badRequest('No quiz to update');
getQuiz(db, quizId);
db.tx(() => {
inputs.forEach((q, i) => {
const target = ids[i];
if (target && db.get('SELECT id FROM questions WHERE id = ? AND quiz_id = ?', target, quizId)) updateQuestion(db, target, q);
else insertQuestions(db, quizId, [q]);
});
});
result = { kind: 'quiz', id: quizId, count: inputs.length };
} else if (opts.mode === 'append') {
const quizId = opts.targetId ?? draft.meta.targetId;
if (!quizId) throw badRequest('Choose a quiz to add these questions to');
getQuiz(db, quizId);
insertQuestions(db, quizId, inputs);
result = { kind: 'quiz', id: quizId, count: inputs.length };
} else {
if (!classId) throw badRequest('Choose a class to save this quiz in');
const quiz = createQuiz(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled quiz', description: payload.description, questions: inputs });
result = { kind: 'quiz', id: quiz.id, count: inputs.length };
}
} else if (payload.kind === 'deck') {
const bad = payload.cards.flatMap((c, i) => validateDraftCard(c, i)).filter((x) => x.level === 'error');
if (bad.length > 0) throw badRequest(`Card ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
if (payload.cards.length === 0) throw badRequest('There are no cards to save');
const inputs = payload.cards.map(draftCardToInput);
if (opts.mode === 'append') {
const deckId = opts.targetId ?? draft.meta.targetId;
if (!deckId) throw badRequest('Choose a deck to add these cards to');
getDeck(db, deckId);
insertCards(db, deckId, inputs);
result = { kind: 'deck', id: deckId, count: inputs.length };
} else {
if (!classId) throw badRequest('Choose a class to save this deck in');
const deck = createDeck(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled deck', description: payload.description, cards: inputs });
result = { kind: 'deck', id: deck.id, count: inputs.length };
}
} else {
if (!payload.content.trim()) throw badRequest('The note is empty');
if (!classId) throw badRequest('Choose a class to save this note in');
const note = createNote(db, {
classId,
folderId: opts.folderId ?? draft.folderId,
title: payload.title.trim() || 'Untitled note',
content: payload.content,
kind: 'guide',
source: { model: draft.meta.model, sources: draft.meta.sources },
});
result = { kind: 'note', id: note.id, count: 1 };
}
db.run(`UPDATE drafts SET status = 'saved', saved_kind = ?, saved_id = ?, updated_at = ? WHERE id = ?`, result.kind, result.id, now(), id);
return result;
}
export function discardDraft(db: DB, id: string): void {
cancelGeneration(id);
const { changes } = db.run(`UPDATE drafts SET status = 'discarded', updated_at = ? WHERE id = ?`, now(), id);
if (!changes) throw notFound('Draft');
}
/** Removes finished drafts after a while and marks jobs interrupted by a restart as failed. */
export function tidyDrafts(db: DB): void {
db.run(`UPDATE drafts SET status = 'failed', error = 'Generation was interrupted by a restart. Start it again.' WHERE status = 'generating'`);
db.run(`DELETE FROM drafts WHERE status IN ('saved', 'discarded') AND updated_at < ?`, now() - 14 * 86_400_000);
db.run(`DELETE FROM drafts WHERE status IN ('ready', 'failed') AND updated_at < ?`, now() - 60 * 86_400_000);
}
/* ------------------------------ request ------------------------------ */
export const generateRequestSchema = z.object({
kind: z.enum(['quiz', 'deck', 'note']),
classId: z.string().nullish(),
folderId: z.string().nullish(),
title: z.string().trim().max(200).optional(),
source: z
.object({
documentIds: z.array(z.string()).max(50).optional(),
/** Use every processed document in the class. */
allClassDocuments: z.boolean().optional(),
text: z.string().max(400_000).optional(),
quizId: z.string().optional(),
deckId: z.string().optional(),
noteId: z.string().optional(),
/** Build from the questions the user keeps missing. */
weakAreas: z.boolean().optional(),
})
.default({}),
/** Topic or focus, e.g. "pediatric respiratory disorders". */
focus: z.string().trim().max(600).optional(),
instructions: z.string().trim().max(4000).optional(),
count: z.number().int().min(1).max(200).optional(),
style: z.enum(['nclex', 'standard', 'recall']).optional(),
difficulty: z.enum(['easy', 'medium', 'hard', 'mixed']).optional(),
/** Share of select-all-that-apply questions, 0..1. */
sataShare: z.number().min(0).max(1).optional(),
cardStyle: z.enum(['concise', 'detailed', 'medication', 'definition']).optional(),
noteStyle: z.enum(['study_guide', 'summary', 'outline', 'cheat_sheet']).optional(),
modelId: z.string().nullish(),
});
export type GenerateRequest = z.infer<typeof generateRequestSchema>;
/* ------------------------------ prompts ------------------------------ */
const QUESTION_SCHEMA = {
type: 'object',
properties: {
questions: {
type: 'array',
items: {
type: 'object',
properties: {
type: { type: 'string', enum: ['single', 'multi'], description: 'single = one correct answer; multi = select all that apply' },
stem: { type: 'string', description: 'The question. Markdown allowed.' },
options: {
type: 'array',
items: {
type: 'object',
properties: {
text: { type: 'string' },
correct: { type: 'boolean' },
why: { type: 'string', description: 'One sentence on why this choice is right or wrong.' },
},
required: ['text', 'correct', 'why'],
},
},
rationale: { type: 'string', description: 'Teaching explanation of the correct answer. Markdown allowed.' },
topic: { type: 'string', description: 'Short topic label, reused across related questions (2-4 words).' },
difficulty: { type: 'string', enum: ['easy', 'medium', 'hard'] },
source: { type: 'string', description: 'Id of the excerpt this question is based on, e.g. "c42". Empty if none.' },
},
required: ['type', 'stem', 'options', 'rationale', 'topic', 'difficulty'],
},
},
},
required: ['questions'],
};
const CARD_SCHEMA = {
type: 'object',
properties: {
cards: {
type: 'array',
items: {
type: 'object',
properties: {
front: { type: 'string', description: 'Prompt side: one specific question or term.' },
back: { type: 'string', description: 'Answer side. Markdown allowed.' },
hint: { type: 'string', description: 'Optional memory hook or mnemonic. Empty string if none.' },
tags: { type: 'array', items: { type: 'string' }, description: '1-2 short topic tags.' },
},
required: ['front', 'back'],
},
},
},
required: ['cards'],
};
const NCLEX_RULES = `Write NCLEX-style (Next Generation NCLEX) items:
- Test clinical judgment at the application and analysis level: recognise cues, analyse, prioritise, choose and evaluate actions. Avoid pure recall.
- Open with a brief, realistic client scenario containing only relevant data (age, diagnosis, key findings, vital signs or lab values with units).
- Ask one clear question. Use priority wording where it fits (first, best, most important, requires immediate follow-up) and build it on a real framework: ABCs, Maslow, the nursing process, acute before chronic, unstable before stable, safety, least invasive first.
- Cover the client-needs areas where the material allows: safe and effective care, pharmacology, physiological adaptation, reduction of risk, health promotion, psychosocial integrity, delegation and assignment.
- Every distractor must be plausible to a student with partial knowledge and wrong for a specific reason. Keep choices parallel in grammar and similar in length. Never use "all of the above" or "none of the above". Do not give the answer away through wording or length.
- Multiple-choice items have exactly 4 choices with exactly one best answer. Select-all-that-apply items have 5 or 6 choices with 2 to 5 correct, and the stem ends with "Select all that apply."
- Use generic drug names. Use current, standard practice.
- The rationale teaches: state why the correct answer is correct, name the principle behind it, and give each choice its own "why".`;
const STANDARD_RULES = `Write clear exam-quality questions:
- Each question tests one idea and has one defensible answer set.
- Mix understanding and application; avoid trivia and trick wording.
- Distractors are plausible and parallel. Never use "all of the above" or "none of the above".
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
- The rationale explains the correct answer, and each choice gets its own "why".`;
const RECALL_RULES = `Write straightforward recall questions that check key facts, definitions, normal values and classifications.
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
- Keep stems short and unambiguous. Each choice gets a brief "why".`;
const CARD_RULES: Record<string, string> = {
concise: 'Each card tests exactly one fact. Fronts are specific questions or terms; backs are short (a phrase or one or two lines).',
detailed: 'Each card covers one concept. Backs may use a short bullet list (up to 5 bullets) for the key points.',
medication:
'Make medication cards. Front: the generic drug name (with a common brand name in parentheses when well known). Back: a Markdown list with **Class**, **Action**, **Uses**, **Adverse effects** (the most important ones), **Nursing considerations** and **Client teaching**. Only include what the material supports.',
definition: 'Make term and definition cards. Front: the term. Back: a precise definition in one or two sentences, plus an example when useful.',
};
const NOTE_RULES: Record<string, string> = {
study_guide:
'Write a study guide in Markdown: a one-paragraph overview, then sections with headings. Under each heading give the key concepts as tight bullet points, important values and medications in tables where that helps, common pitfalls, and finish with a "Check yourself" list of 5-10 self-test questions (answers at the very end).',
summary: 'Write a faithful summary in Markdown: a short overview paragraph, then the main points grouped under headings. Keep it compact and do not add outside information.',
outline: 'Write a hierarchical outline in Markdown using nested bullet points. Capture the structure and every key point of the material without full sentences where a phrase will do.',
cheat_sheet: 'Write a one-page cheat sheet in Markdown: dense, scannable, organised under short headings, using tables for values, drug classes and comparisons, and bold for must-know facts.',
};
function baseSystem(req: GenerateRequest, hasSources: boolean): string {
const lines = [
'You are an expert educator writing study material for a nursing student. Accuracy is critical: this material is used to prepare for licensing exams.',
hasSources
? 'Base everything on the provided source material. Do not introduce facts that contradict it. You may rely on standard, well-established knowledge only to write plausible distractors and clear explanations.'
: 'No source material was provided, so rely on current, well-established knowledge. If something is uncertain or varies by institution, prefer the most widely taught standard.',
'Write in clear, natural English. Use Markdown inside text fields where it aids clarity (bold key terms, short lists, tables). Do not wrap output in code fences.',
];
if (req.instructions) lines.push(`Additional instructions from the student (follow them unless they conflict with accuracy):\n${req.instructions}`);
return lines.join('\n\n');
}
/* ------------------------------ source gathering ------------------------------ */
interface Sources {
/** Batches of excerpts (documents). */
groups: ChunkHit[][];
/** Free text used as-is (pasted notes, an existing deck or quiz, missed questions). */
inline: string;
docs: { documentId: string; title: string }[];
refs: Map<string, string>;
totalTokens: number;
}
function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens: number, groups: number): Sources {
const { db } = app;
const src = req.source;
const out: Sources = { groups: [], inline: '', docs: [], refs: new Map(), totalTokens: 0 };
const inline: string[] = [];
let documentIds = src.documentIds ?? [];
if (src.allClassDocuments && req.classId) {
documentIds = db.all<{ id: string }>(`SELECT id FROM documents WHERE class_id = ? AND status = 'ready' AND chunk_count > 0 ORDER BY sort_order, created_at`, req.classId).map((d) => d.id);
}
if (documentIds.length > 0) {
const rows = db.all<{ id: string; title: string; status: string; chunk_count: number }>(
`SELECT id, title, status, chunk_count FROM documents WHERE id IN (${documentIds.map(() => '?').join(',')})`,
...documentIds,
);
const usable = rows.filter((r) => r.status === 'ready' && r.chunk_count > 0);
if (usable.length === 0) {
throw badRequest(rows.some((r) => r.status !== 'ready') ? 'Those documents are still being processed. Try again in a moment.' : 'Those documents have no readable text yet.');
}
out.docs = usable.map((r) => ({ documentId: r.id, title: r.title }));
out.groups = selectContext(db, { documentIds: usable.map((r) => r.id) }, { focus: req.focus, budgetTokens, groups });
}
if (src.text?.trim()) inline.push(`Notes provided by the student:\n${src.text.trim()}`);
if (src.noteId) {
const note = db.get<{ title: string; content: string }>('SELECT title, content FROM notes WHERE id = ?', src.noteId);
if (note) inline.push(`Note "${note.title}":\n${note.content}`);
}
if (src.deckId) {
const deck = getDeck(db, src.deckId);
const cards = listCards(db, deck.id);
if (cards.length === 0) throw badRequest('That deck has no cards');
inline.push(`Flashcards from the deck "${deck.title}":\n${cards.map((c, i) => `${i + 1}. ${c.front}\n -> ${c.back}`).join('\n')}`);
}
if (src.quizId) {
const quiz = getQuiz(db, src.quizId);
const questions = listQuestions(db, quiz.id);
if (questions.length === 0) throw badRequest('That quiz has no questions');
inline.push(
`Questions from the quiz "${quiz.title}":\n${questions
.map((q, i) => `${i + 1}. ${q.stem}\n Correct: ${q.options.filter((o) => o.correct).map((o) => o.text).join('; ')}\n Rationale: ${q.rationale}`)
.join('\n')}`,
);
}
if (src.weakAreas) {
const ids = missedQuestionIds(db, { classId: req.classId ?? undefined }, 40);
const topics = weakTopics(db, { classId: req.classId ?? undefined }, 10);
if (ids.length === 0 && topics.length === 0) throw badRequest('There is not enough quiz history yet to find weak areas. Take a few quizzes first.');
const missed = ids
.map((id) => db.get<{ stem: string; topic: string; rationale: string }>('SELECT stem, topic, rationale FROM questions WHERE id = ?', id))
.filter((q): q is { stem: string; topic: string; rationale: string } => Boolean(q));
inline.push(
[
'The student keeps missing these concepts. Write NEW items that test the same underlying knowledge from different angles. Do not repeat or lightly reword the original questions.',
topics.length ? `Weakest topics: ${topics.map((t) => `${t.topic} (${Math.round(t.accuracy * 100)}% correct)`).join(', ')}` : '',
missed.length ? `Questions answered incorrectly:\n${missed.map((q, i) => `${i + 1}. [${q.topic || 'General'}] ${q.stem}\n Key point: ${q.rationale.slice(0, 400)}`).join('\n')}` : '',
]
.filter(Boolean)
.join('\n\n'),
);
// Pull supporting passages from the class documents for those topics.
if (req.classId && out.groups.length === 0) {
const query = [...topics.map((t) => t.topic), ...missed.slice(0, 10).map((q) => q.topic)].filter(Boolean).join(' ');
const hits = query ? searchChunks(db, query, { classId: req.classId }, 24) : [];
if (hits.length > 0) {
const perGroup = Math.max(1, Math.ceil(hits.length / groups));
for (let i = 0; i < hits.length; i += perGroup) out.groups.push(hits.slice(i, i + perGroup));
}
}
}
out.inline = inline.join('\n\n');
for (const g of out.groups) for (const c of g) out.refs.set(`c${c.id}`, sourceLabel(c));
out.totalTokens = out.groups.reduce((n, g) => n + g.reduce((m, c) => m + c.tokens, 0), 0) + Math.ceil(out.inline.length / 4);
return out;
}
function sourceBlock(sources: Sources, groupIndex: number): string {
const parts: string[] = [];
const group = sources.groups.length > 0 ? sources.groups[groupIndex % sources.groups.length] : [];
if (group.length > 0) parts.push(`SOURCE EXCERPTS (each starts with its id in brackets):\n\n${formatExcerpts(group)}`);
if (sources.inline) parts.push(sources.inline);
return parts.join('\n\n======\n\n');
}
/* ------------------------------ job control ------------------------------ */
const running = new Map<string, AbortController>();
export function cancelGeneration(draftId: string): void {
running.get(draftId)?.abort();
running.delete(draftId);
}
function setProgress(db: DB, id: string, progress: number, label: string, payload?: ContentDraft): void {
if (payload) {
db.run('UPDATE drafts SET progress = ?, progress_label = ?, payload = ?, updated_at = ? WHERE id = ?', progress, label, JSON.stringify(payload), now(), id);
} else {
db.run('UPDATE drafts SET progress = ?, progress_label = ?, updated_at = ? WHERE id = ?', progress, label, now(), id);
}
}
function defaultTitle(app: AppContext, req: GenerateRequest, sources: Sources): string {
if (req.title) return req.title;
const subject =
req.focus?.replace(/\s+/g, ' ').slice(0, 80) ||
(sources.docs.length === 1 ? sources.docs[0].title : '') ||
(req.source.deckId ? getDeck(app.db, req.source.deckId).title : '') ||
(req.source.quizId ? getQuiz(app.db, req.source.quizId).title : '') ||
(req.source.weakAreas ? 'Weak areas' : '') ||
(req.classId ? getClass(app.db, req.classId).name : 'Untitled');
const suffix = req.kind === 'quiz' ? (req.source.weakAreas ? 'practice' : req.style === 'nclex' ? 'NCLEX practice' : 'quiz') : req.kind === 'deck' ? 'flashcards' : NOTE_TITLES[req.noteStyle ?? 'study_guide'];
const cap = subject.charAt(0).toUpperCase() + subject.slice(1);
return `${cap} · ${suffix}`.slice(0, 200);
}
const NOTE_TITLES: Record<string, string> = { study_guide: 'study guide', summary: 'summary', outline: 'outline', cheat_sheet: 'cheat sheet' };
/**
* Starts a generation job in the background and returns the draft id immediately.
* The caller polls the draft (or awaits `done`) for progress.
*/
export function startGeneration(app: AppContext, input: z.input<typeof generateRequestSchema>, opts: { conversationId?: string | null; origin?: DraftMeta['origin'] } = {}): { draftId: string; done: Promise<DraftDto> } {
const req = generateRequestSchema.parse(input);
if (req.classId) getClass(app.db, req.classId);
// Fail fast (before creating a draft) when no model is configured.
const resolved = resolveModel(app, 'generate', req.modelId);
const draftId = createDraft(app.db, {
kind: req.kind,
classId: req.classId ?? null,
folderId: req.folderId ?? null,
title: req.title ?? '',
status: 'generating',
conversationId: opts.conversationId ?? null,
meta: { origin: opts.origin ?? 'generate', model: resolved.model.label, request: { ...req, source: { ...req.source, text: req.source.text ? `${req.source.text.slice(0, 200)}…` : undefined } } },
});
const controller = new AbortController();
running.set(draftId, controller);
const done = runGeneration(app, draftId, req, controller.signal)
.catch((err) => {
const message = err instanceof AIError ? err.message : err instanceof Error && 'status' in err ? err.message : 'Generation failed unexpectedly.';
if (!(err instanceof AIError) && !(err instanceof Error && 'status' in err)) app.log('error', 'Generation failed', err);
const current = app.db.get<{ status: string }>('SELECT status FROM drafts WHERE id = ?', draftId);
if (current?.status === 'generating') {
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ?`, controller.signal.aborted ? 'Generation was cancelled.' : message, now(), draftId);
}
})
.then(() => {
running.delete(draftId);
return getDraft(app.db, draftId);
});
return { draftId, done };
}
async function runGeneration(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal): Promise<void> {
const { db } = app;
const resolved = resolveModel(app, 'generate', req.modelId);
const window = resolved.model.contextWindow ?? 32_000;
const budgetTokens = Math.max(2_500, Math.min(Math.floor(window * 0.4), 28_000));
setProgress(db, draftId, 0.03, 'Reading your material');
if (req.kind === 'note') {
await generateNote(app, draftId, req, signal, budgetTokens, window);
return;
}
const isQuiz = req.kind === 'quiz';
const perBatch = isQuiz ? 8 : 25;
const defaultCount = isQuiz ? 15 : 30;
let count = req.count ?? defaultCount;
if ((req.source.deckId || req.source.quizId) && !req.count) {
// Conversions default to one item per source item.
count = req.source.deckId ? listCards(db, req.source.deckId).length : listQuestions(db, req.source.quizId!).length;
count = Math.min(Math.max(count, 1), 100);
}
const batches = Math.ceil(count / perBatch);
const sources = gatherSources(app, req, budgetTokens, batches);
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
if (!hasSources && !req.focus && !req.instructions) throw badRequest('Choose source material or describe a topic to generate from.');
const title = defaultTitle(app, req, sources);
const draft: QuizDraft | DeckDraft = isQuiz ? { kind: 'quiz', title, description: '', questions: [] } : { kind: 'deck', title, description: '', cards: [] };
const items = (): (DraftQuestion | DraftCard)[] => (draft.kind === 'quiz' ? draft.questions : draft.cards);
const key = (x: DraftQuestion | DraftCard) => ('stem' in x ? x.stem : x.front).toLowerCase().replace(/\W+/g, ' ').trim().slice(0, 120);
const seen = new Set<string>();
const notes: string[] = [];
const style = req.style ?? 'nclex';
const rules = isQuiz ? (style === 'nclex' ? NCLEX_RULES : style === 'recall' ? RECALL_RULES : STANDARD_RULES) : CARD_RULES[req.cardStyle ?? 'concise'];
const system = `${baseSystem(req, hasSources)}\n\n${rules}`;
const sata = req.sataShare ?? (style === 'nclex' ? 0.3 : 0.2);
const difficulty = req.difficulty ?? 'mixed';
const runBatch = async (index: number, want: number): Promise<number> => {
const written = items().map((x) => `- ${('stem' in x ? x.stem : x.front).replace(/\s+/g, ' ').slice(0, 140)}`);
const sataCount = Math.round(want * sata);
const ask = isQuiz
? [
`Write ${want} ${style === 'nclex' ? 'NCLEX-style ' : ''}questions${req.focus ? ` about: ${req.focus}` : ''}.`,
sata <= 0 ? 'Use only multiple-choice (single) questions.' : sata >= 1 ? 'Use only select-all-that-apply (multi) questions.' : `About ${sataCount} of them should be select-all-that-apply (multi); the rest multiple choice (single).`,
difficulty === 'mixed' ? 'Mix difficulty: mostly medium, with a few easy and a few hard.' : `Difficulty: ${difficulty}.`,
sources.groups.length > 0 ? 'Spread the questions across the excerpts instead of clustering on one. Set "source" to the id of the excerpt each question is based on.' : '',
req.source.deckId ? 'Turn the flashcards into questions: use each card\'s answer as the correct choice and write distractors that a student could confuse with it.' : '',
]
: [
`Write ${want} flashcards${req.focus ? ` about: ${req.focus}` : ''}.`,
sources.groups.length > 0 ? 'Cover the most testable facts across all of the excerpts.' : '',
req.source.quizId ? 'Turn the quiz questions into flashcards: the front asks for the key fact, the back states it with the essential reasoning.' : '',
];
if (written.length > 0) ask.push(`Already written (do not repeat or closely paraphrase these):\n${written.slice(-80).join('\n')}`);
const user = [sourceBlock(sources, index), ask.filter(Boolean).join('\n')].filter(Boolean).join('\n\n======\n\n');
const result = await generateStructured(app, {
task: 'generate',
purpose: isQuiz ? 'quiz generation' : 'flashcard generation',
modelId: req.modelId,
system,
messages: [{ role: 'user', content: [{ type: 'text', text: user }] }],
toolName: isQuiz ? 'submit_questions' : 'submit_flashcards',
toolDescription: isQuiz ? 'Submit the finished questions.' : 'Submit the finished flashcards.',
schema: isQuiz ? QUESTION_SCHEMA : CARD_SCHEMA,
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
signal,
});
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === req.kind) as QuizDraft | DeckDraft | undefined;
const incoming: (DraftQuestion | DraftCard)[] = parsed ? (parsed.kind === 'quiz' ? parsed.questions : parsed.cards) : [];
let added = 0;
for (const item of incoming) {
const k = key(item);
if (!k || seen.has(k)) continue;
const problems = ('stem' in item ? validateDraftQuestion(item, 0) : validateDraftCard(item, 0)).filter((p) => p.level === 'error');
if (problems.length > 0) continue; // malformed items are dropped here and replaced by a top-up batch
if ('stem' in item) {
const ref = (item.source ?? '').replace(/[[\]\s]/g, '');
item.source = sources.refs.get(ref) ?? undefined;
}
seen.add(k);
(items() as (DraftQuestion | DraftCard)[]).push(item);
added += 1;
if (items().length >= count) break;
}
if (result.truncated) notes.push('One batch hit the model output limit, so it produced fewer items than requested.');
return added;
};
let failures = 0;
for (let i = 0; i < batches && items().length < count; i += 1) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
const want = Math.min(perBatch, count - items().length);
setProgress(db, draftId, 0.05 + (0.9 * items().length) / count, `Writing ${isQuiz ? 'questions' : 'cards'} ${items().length + 1}-${Math.min(count, items().length + want)} of ${count}`, draft);
try {
await runBatch(i, want);
} catch (err) {
// One bad batch should not sink a long job that already has good content.
failures += 1;
if (items().length === 0 || failures > 2 || (err instanceof AIError && !['invalid_response', 'server', 'rate_limit', 'timeout', 'network'].includes(err.code))) throw err;
notes.push(`A batch failed and was skipped: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
// Top up when validation or de-duplication left the draft short.
for (let extra = 0; extra < 2 && items().length < count && items().length > 0; extra += 1) {
if (signal.aborted) break;
const want = Math.min(perBatch, count - items().length);
setProgress(db, draftId, 0.92, `Writing the last ${want} ${isQuiz ? 'questions' : 'cards'}`, draft);
try {
if ((await runBatch(batches + extra, want)) === 0) break;
} catch {
break;
}
}
if (items().length === 0) throw new AIError('invalid_response', 'The model did not produce any usable items. Try again, or choose a different model for content generation.');
if (items().length < count) notes.push(`Generated ${items().length} of the ${count} requested.`);
db.run(
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
draft.title,
JSON.stringify(draft),
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs, notes: [...new Set(notes)] } satisfies DraftMeta),
now(),
draftId,
);
}
async function generateNote(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal, budgetTokens: number, window: number): Promise<void> {
const { db } = app;
const resolved = resolveModel(app, 'generate', req.modelId);
const style = req.noteStyle ?? 'study_guide';
// A guide reads the material in as few passes as possible; allow a bigger slice per pass than item generation.
const perPass = Math.max(budgetTokens, Math.min(Math.floor(window * 0.55), 90_000));
const sources = gatherSources(app, req, perPass, 12);
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
if (!hasSources && !req.focus) throw badRequest('Choose source material or describe a topic to write about.');
const title = defaultTitle(app, req, sources);
const system = `${baseSystem(req, hasSources)}\n\n${NOTE_RULES[style]}\n\n${
sources.groups.length > 0 ? 'Cite the excerpts you draw on by putting their id in brackets at the end of the relevant line or paragraph, e.g. [c42]. Use only ids that appear in the source.' : ''
}\nStart directly with the content (no preamble, no title line).`;
// Merge excerpt groups into as few passes as fit.
const passes: ChunkHit[][] = [];
let current: ChunkHit[] = [];
let size = 0;
for (const g of sources.groups) {
const t = g.reduce((n, c) => n + c.tokens, 0);
if (current.length > 0 && size + t > perPass) {
passes.push(current);
current = [];
size = 0;
}
current.push(...g);
size += t;
}
if (current.length > 0) passes.push(current);
const ask = `${req.focus ? `Focus on: ${req.focus}\n` : ''}Write the ${NOTE_TITLES[style]} now.`;
let content = '';
if (passes.length <= 1) {
setProgress(db, draftId, 0.2, 'Writing');
const body = [passes[0]?.length ? `SOURCE EXCERPTS:\n\n${formatExcerpts(passes[0])}` : '', sources.inline, ask].filter(Boolean).join('\n\n======\n\n');
const res = await runChat(app, { task: 'generate', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000, signal });
content = res.text;
} else {
// Long material: condense each part, then write the final piece from the condensed notes.
const partials: string[] = [];
for (let i = 0; i < passes.length; i += 1) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
setProgress(db, draftId, 0.05 + (0.7 * i) / passes.length, `Reading part ${i + 1} of ${passes.length}`);
const res = await runChat(app, {
task: 'generate',
purpose: 'study guide',
modelId: req.modelId,
system: `${baseSystem(req, true)}\n\nCondense this part of the material into detailed notes: every key concept, value, medication, cause, sign, intervention and warning, as Markdown bullet points under short headings. Keep the bracketed excerpt ids (e.g. [c42]) next to the points they support. Do not add outside information.`,
messages: [{ role: 'user', content: [{ type: 'text', text: `${req.focus ? `Focus on: ${req.focus}\n\n` : ''}SOURCE EXCERPTS (part ${i + 1} of ${passes.length}):\n\n${formatExcerpts(passes[i])}` }] }],
maxOutputTokens: 6_000,
signal,
});
partials.push(res.text.trim());
}
setProgress(db, draftId, 0.8, 'Writing');
const res = await runChat(app, {
task: 'generate',
purpose: 'study guide',
modelId: req.modelId,
system,
messages: [{ role: 'user', content: [{ type: 'text', text: `CONDENSED NOTES FROM THE FULL MATERIAL:\n\n${partials.join('\n\n---\n\n')}\n\n======\n\n${sources.inline ? `${sources.inline}\n\n======\n\n` : ''}${ask}` }] }],
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
signal,
});
content = res.text;
}
content = content.trim().replace(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i, '$1');
if (!content) throw new AIError('invalid_response', 'The model returned an empty document. Try again.');
// Turn [c42] markers into readable citations and append a source list.
const used = new Map<string, string>();
content = content.replace(/\[(c\d+(?:\s*,\s*c\d+)*)\]/g, (_m, list: string) => {
const labels = list
.split(',')
.map((r) => r.trim())
.map((r) => {
const label = sources.refs.get(r) ?? (() => {
const hit = chunkById(db, Number(r.slice(1)));
return hit ? sourceLabel(hit) : null;
})();
if (label) used.set(label, label);
return label;
})
.filter(Boolean);
return labels.length ? ` *(${[...new Set(labels)].join('; ')})*` : '';
});
const payload: NoteDraft = { kind: 'note', title, content };
db.run(
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
title,
JSON.stringify(payload),
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs } satisfies DraftMeta),
now(),
draftId,
);
}
/* ------------------------------ improve existing questions ------------------------------ */
export const improveRequestSchema = z.object({
quizId: z.string(),
questionIds: z.array(z.string()).max(60).optional(),
instructions: z.string().trim().max(2000).optional(),
modelId: z.string().nullish(),
});
/** Rewrites existing questions (clearer stems, better distractors, fuller rationales) into a draft the user can apply. */
export function startImprove(app: AppContext, input: z.input<typeof improveRequestSchema>, opts: { conversationId?: string | null } = {}): { draftId: string; done: Promise<DraftDto> } {
const req = improveRequestSchema.parse(input);
const quiz = getQuiz(app.db, req.quizId);
let questions = listQuestions(app.db, quiz.id);
if (req.questionIds?.length) questions = questions.filter((q) => req.questionIds!.includes(q.id));
if (questions.length === 0) throw badRequest('No questions to improve');
questions = questions.slice(0, 60);
const resolved = resolveModel(app, 'generate', req.modelId);
const draftId = createDraft(app.db, {
kind: 'quiz',
classId: quiz.class_id,
title: quiz.title,
status: 'generating',
conversationId: opts.conversationId ?? null,
meta: { origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: [] },
});
const controller = new AbortController();
running.set(draftId, controller);
const done = (async () => {
const out: DraftQuestion[] = [];
const ids: string[] = [];
const system = `You are an expert nursing educator and item writer reviewing existing exam questions.
Improve each question while keeping what it tests and its correct answer(s) unless the keyed answer is factually wrong (if so, fix it and say so at the start of the rationale).
- Make the stem clear, complete and free of clues; fix grammar.
- Replace weak or implausible distractors; keep choices parallel and similar in length; remove "all/none of the above".
- Write a teaching rationale, and give every choice a one-sentence "why".
- Keep the same question type unless it is clearly wrong for the content. Set a short topic label and a difficulty.
Return the questions in the same order, one output question per input question.${req.instructions ? `\n\nAdditional instructions: ${req.instructions}` : ''}`;
for (let i = 0; i < questions.length; i += 8) {
if (controller.signal.aborted) throw new AIError('aborted', 'Cancelled.');
const batch = questions.slice(i, i + 8);
setProgress(app.db, draftId, 0.05 + (0.9 * i) / questions.length, `Reviewing questions ${i + 1}-${i + batch.length} of ${questions.length}`);
const body = batch
.map(
(q, n) =>
`Question ${n + 1} (${q.type})\nStem: ${q.stem}\nChoices:\n${q.options.map((o) => `- [${o.correct ? 'x' : ' '}] ${o.text}`).join('\n')}\nRationale: ${q.rationale || '(none)'}\nTopic: ${q.topic || '(none)'}`,
)
.join('\n\n');
const result = await generateStructured(app, {
task: 'generate',
purpose: 'improve questions',
modelId: req.modelId,
system,
messages: [{ role: 'user', content: [{ type: 'text', text: body }] }],
toolName: 'submit_questions',
toolDescription: 'Submit the improved questions in the original order.',
schema: QUESTION_SCHEMA,
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
signal: controller.signal,
});
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === 'quiz') as QuizDraft | undefined;
const improved = parsed?.questions ?? [];
batch.forEach((orig, n) => {
const next = improved[n];
if (!next || validateDraftQuestion(next, 0).some((p) => p.level === 'error')) return;
next.source = undefined;
out.push(next);
ids.push(orig.id);
});
}
if (out.length === 0) throw new AIError('invalid_response', 'The model did not return usable revisions. Try again or use a different model.');
const payload: QuizDraft = { kind: 'quiz', title: quiz.title, description: quiz.description, questions: out };
app.db.run(
`UPDATE drafts SET status = 'ready', payload = ?, meta = ?, progress = 1, progress_label = '', updated_at = ? WHERE id = ?`,
JSON.stringify(payload),
JSON.stringify({ origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: ids } satisfies DraftMeta),
now(),
draftId,
);
})()
.catch((err) => {
const message = err instanceof AIError || (err instanceof Error && 'status' in err) ? (err as Error).message : 'Improving the questions failed unexpectedly.';
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ? AND status = 'generating'`, message, now(), draftId);
})
.then(() => {
running.delete(draftId);
return getDraft(app.db, draftId);
});
return { draftId, done };
}

View file

@ -0,0 +1,581 @@
/**
* Anthropic — Messages API (`POST /v1/messages`, SSE).
*
* Verified against (2026-11):
* https://platform.claude.com/docs/en/api/messages
* https://platform.claude.com/docs/en/build-with-claude/streaming
* https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking
* https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
* https://platform.claude.com/docs/en/api/migration-2026-10-21
*
* Quirks handled here:
* - API version 2026-10-21 rejects temperature/top_p/top_k on EVERY model, so
* this adapter never sends a temperature (the catalog marks Claude models
* `temperature: false`).
* - `max_tokens` is required; thinking counts against it.
* - Thinking configuration differs per model generation (see `claudeProfile`).
* - Thinking blocks (with their signatures) must be replayed unchanged, so the
* raw content blocks of each turn are kept in `providerData`.
* - Newer models bind thinking blocks to the conversation prefix (system
* prompt, tool set, earlier messages). If the app changed any of those, the
* API answers 400 "Invalid `signature` in `thinking` block"; the documented
* recovery is to strip the thinking blocks and retry once.
* - Safety classifiers can end a turn with `stop_reason: "refusal"` (HTTP 200).
* For the models that support it we opt into server-side fallback, which
* re-runs a declined request on another Claude model inside the same call.
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { lookupModelDefaults } from './catalog.ts';
import { isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext } from './http.ts';
type Json = Record<string, unknown>;
const LABEL = 'Anthropic';
const DEFAULT_BASE_URL = 'https://api.anthropic.com';
const API_VERSION = '2026-10-21';
/** Beta that enables the `fallbacks` request parameter. */
const FALLBACK_BETA = 'server-side-fallback-2026-07-01';
/** Streaming has no HTTP-timeout concern, so give the model room (thinking counts against this). */
const DEFAULT_MAX_TOKENS = 64_000;
/* ------------------------------------------------------------------ */
/* Per-model behaviour */
/* ------------------------------------------------------------------ */
export interface ClaudeProfile {
/**
* always – thinking cannot be turned off (Fable/Mythos, Opus 5.5+)
* default_on – adaptive thinking runs unless disabled (Opus 5, Sonnet 5+, Haiku 5.5)
* opt_in – adaptive thinking must be requested (Opus 4.6–4.8, Sonnet 4.6)
* budget – legacy `budget_tokens` thinking (Haiku 4.5, Opus/Sonnet 4.5 and older)
* none – no thinking support (Claude 3 and earlier)
*/
thinking: 'always' | 'default_on' | 'opt_in' | 'budget' | 'none';
/** How thinking is switched off, when the model allows it. */
off: 'disabled' | 'between_tools' | null;
/** Accepts `output_config.effort`. */
effort: boolean;
/** Accepts `thinking.display` (and defaults to hiding the reasoning text). */
display: boolean;
/** Accepts `tool_choice` of type `any` / `tool`. */
forcedToolChoice: boolean;
/** Supports the server-side refusal `fallbacks` parameter. */
fallbacks: boolean;
}
/** Thinking / tool-choice rules by model, from the model migration guides. */
export function claudeProfile(model: string): ClaudeProfile {
const m = /claude-(fable|mythos|opus|sonnet|haiku)-(\d+)(?:-(\d{1,2}))?(?!\d)/.exec(model);
if (!m) {
// "claude-3-5-sonnet-…" and older put the version first and never had adaptive thinking.
if (/claude-(?:[123]\b|instant)/.test(model)) {
return { thinking: 'none', off: null, effort: false, display: false, forcedToolChoice: true, fallbacks: false };
}
// Unknown naming scheme: assume it behaves like the newest generation.
return { thinking: 'default_on', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: false };
}
const family = m[1];
const v = Number(m[2]) * 10 + Number(m[3] ?? 0); // 4.6 -> 46, 5 -> 50, 5.5 -> 55
if (family === 'fable' || family === 'mythos') {
return {
thinking: 'always',
off: null,
effort: true,
display: true,
forcedToolChoice: v <= 50,
fallbacks: family === 'fable' && (v === 51 || v === 52),
};
}
if (family === 'opus') {
if (v >= 55) return { thinking: 'always', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: v === 50 };
if (v >= 47) return { thinking: 'opt_in', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
return { thinking: 'budget', off: 'disabled', effort: v === 45, display: false, forcedToolChoice: true, fallbacks: false };
}
if (family === 'sonnet') {
// Sonnet 5.5 rejects {type:"disabled"}; its lowest setting is {type:"between_tools"}.
if (v >= 55) return { thinking: 'default_on', off: 'between_tools', effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
}
// haiku
if (v >= 55) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
}
/** Thinking budgets for the legacy `budget_tokens` models. */
const LEGACY_BUDGET = { low: 2_048, medium: 8_192, high: 16_384 } as const;
/* ------------------------------------------------------------------ */
/* Request building */
/* ------------------------------------------------------------------ */
interface BuildOptions {
/** Official endpoint: enables features a proxy may not understand. */
official: boolean;
stripThinking: boolean;
useFallbacks: boolean;
eagerToolInput: boolean;
promptCache: boolean;
}
/** Tool-call ids must match ^[a-zA-Z0-9_-]+$; ids minted by other providers may not. */
function safeId(id: string): string {
const cleaned = id.replace(/[^a-zA-Z0-9_-]/g, '_');
return cleaned || 'tool_call';
}
function userBlocks(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
const results: Json[] = [];
const rest: Json[] = [];
for (const p of parts) {
switch (p.type) {
case 'tool_result': {
// All results of one assistant turn go into ONE user message, before any other content.
const block: Json = { type: 'tool_result', tool_use_id: safeId(p.toolCallId), content: p.output || '(no output)' };
if (p.isError) block.is_error = true;
results.push(block);
break;
}
case 'text':
if (p.text) rest.push({ type: 'text', text: p.text });
break;
case 'image':
if (caps.vision === false) rest.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
else rest.push({ type: 'image', source: { type: 'base64', media_type: p.mime, data: p.data } });
break;
case 'file':
if (p.mime === 'application/pdf' && caps.pdf !== false) {
const doc: Json = { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: p.data } };
if (p.filename) doc.title = p.filename;
rest.push(doc);
} else if (p.mime.startsWith('text/')) {
const doc: Json = { type: 'document', source: { type: 'text', media_type: 'text/plain', data: Buffer.from(p.data, 'base64').toString('utf8') } };
if (p.filename) doc.title = p.filename;
rest.push(doc);
} else {
rest.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
break;
case 'tool_call':
break; // not valid in a user message
}
}
return [...results, ...rest];
}
function assistantBlocks(msg: ChatMessage, stripThinking: boolean): Json[] {
const pd = msg.providerData;
if (pd?.kind === 'anthropic' && isRecord(pd.data) && Array.isArray(pd.data.content)) {
// Same provider: replay the turn exactly as it was received, thinking blocks
// and signatures included.
const blocks = pd.data.content.filter(isRecord);
return stripThinking ? blocks.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking') : blocks;
}
// Another provider produced this turn: rebuild it from the neutral parts.
const blocks: Json[] = [];
for (const p of msg.content) {
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
else if (p.type === 'tool_call') blocks.push({ type: 'tool_use', id: safeId(p.id), name: p.name, input: p.input });
}
return blocks;
}
function toWireMessages(messages: ChatMessage[], caps: Partial<ModelCapabilities>, stripThinking: boolean): Json[] {
const out: Json[] = [];
for (const msg of messages) {
const content = msg.role === 'user' ? userBlocks(msg.content, caps) : assistantBlocks(msg, stripThinking);
if (content.length === 0) continue; // the API rejects empty turns
out.push({ role: msg.role, content });
}
if (out[0]?.role === 'assistant') out.unshift({ role: 'user', content: [{ type: 'text', text: '(continuing the conversation)' }] });
return out;
}
function buildBody(req: ChatRequest, profile: ClaudeProfile, opts: BuildOptions): Json {
const caps = req.capabilities ?? {};
const defaults = lookupModelDefaults('anthropic', req.model);
const ceiling = defaults.maxOutput ?? Number.POSITIVE_INFINITY;
const maxTokens = Math.max(1, Math.min(req.maxOutputTokens ?? DEFAULT_MAX_TOKENS, ceiling));
const systemNotes: string[] = [];
const body: Json = { model: req.model, max_tokens: maxTokens, stream: true };
// --- tools -------------------------------------------------------
const tools = caps.tools === false ? [] : (req.tools ?? []);
let forcing = false;
if (tools.length > 0) {
body.tools = tools.map((t) => {
const tool: Json = { name: t.name, description: t.description, input_schema: { type: 'object', ...t.parameters } };
// Without this the API buffers each tool-input parameter and the stream goes
// silent for the whole time a large input (a generated quiz, say) is written.
// The price is that the input is no longer validated server-side, so it is
// parsed strictly below and truncated calls are dropped.
if (opts.eagerToolInput) tool.eager_input_streaming = true;
return tool;
});
const choice = req.toolChoice;
if (choice === 'none') {
body.tool_choice = { type: 'none' };
} else if (choice === 'required' || isRecord(choice)) {
const name = isRecord(choice) ? str(choice.name) : '';
if (profile.forcedToolChoice) {
body.tool_choice = name ? { type: 'tool', name } : { type: 'any' };
forcing = true;
} else {
// Fable 5.1+, Opus 5.5 and Sonnet 5.5 answer 400 to forced tool use; the
// documented replacement is `auto` plus an explicit instruction.
systemNotes.push(
name
? `For this turn you must call the \`${name}\` tool. Begin your response with that tool call.`
: 'For this turn you must call one of the available tools. Begin your response with the tool call.',
);
}
}
}
// --- thinking ----------------------------------------------------
if (caps.reasoning !== false && profile.thinking !== 'none') {
const effort = req.reasoningEffort;
const adaptive: Json = profile.display ? { type: 'adaptive', display: 'summarized' } : { type: 'adaptive' };
const outputConfig: Json = {};
if (effort === 'none') {
if (profile.thinking === 'always') {
// Cannot be disabled: the lowest effort is the closest thing to "off".
body.thinking = adaptive;
outputConfig.effort = 'low';
} else if (profile.thinking === 'default_on') {
body.thinking = { type: profile.off ?? 'disabled' }; // `between_tools` takes no other field
}
// opt_in / budget models do not think unless asked.
} else if (profile.thinking === 'always' || profile.thinking === 'default_on') {
// `display: "summarized"` because these models hide the reasoning text by
// default, which would look like a long pause in the UI.
body.thinking = adaptive;
if (effort) outputConfig.effort = effort;
} else if (effort) {
// Forced tool use cannot be combined with thinking on the older generations.
if (profile.thinking === 'opt_in' && !forcing) body.thinking = adaptive;
if (profile.thinking === 'budget' && !forcing) {
const budget = Math.min(LEGACY_BUDGET[effort], maxTokens - 1_024); // must be < max_tokens, min 1024
if (budget >= 1_024) body.thinking = { type: 'enabled', budget_tokens: budget };
}
if (profile.effort) outputConfig.effort = effort;
}
if (Object.keys(outputConfig).length > 0) body.output_config = outputConfig;
}
// --- system prompt -----------------------------------------------
// There is no schema-less JSON mode (structured outputs need a schema), and
// assistant prefill is rejected by current models, so JSON mode is an instruction.
if (req.jsonMode) systemNotes.push('Respond with a single valid JSON object and nothing else — no prose, no Markdown code fences.');
const system = [req.system?.trim(), ...systemNotes].filter(Boolean).join('\n\n');
if (system) body.system = system;
body.messages = toWireMessages(req.messages, caps, opts.stripThinking);
if (opts.promptCache) body.cache_control = { type: 'ephemeral' }; // automatic caching of the reusable prefix
if (opts.useFallbacks) body.fallbacks = 'default';
return body;
}
/* ------------------------------------------------------------------ */
/* Adapter */
/* ------------------------------------------------------------------ */
function rootUrl(cfg: ProviderConfig): string {
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1$/, '');
}
function isOfficial(cfg: ProviderConfig): boolean {
try {
return new URL(rootUrl(cfg)).hostname === 'api.anthropic.com';
} catch {
return false;
}
}
function errorContext(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig, betas: string[] = []): Record<string, string> {
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an Anthropic API key in Settings to use Claude models.');
const h: Record<string, string> = { ...cfg.headers, 'x-api-key': cfg.apiKey, 'anthropic-version': API_VERSION };
const userBeta = Object.entries(cfg.headers ?? {}).find(([k]) => k.toLowerCase() === 'anthropic-beta')?.[1];
const all = [...(userBeta ? userBeta.split(',').map((s) => s.trim()) : []), ...betas].filter(Boolean);
for (const k of Object.keys(h)) if (k.toLowerCase() === 'anthropic-beta') delete h[k];
if (all.length > 0) h['anthropic-beta'] = [...new Set(all)].join(',');
return h;
}
function mapStopReason(reason: string): StopReason {
switch (reason) {
case 'end_turn':
case 'stop_sequence':
return 'end';
case 'tool_use':
return 'tool_use';
case 'max_tokens':
case 'model_context_window_exceeded':
return 'max_tokens';
case 'refusal':
return 'refusal';
default:
return 'other'; // pause_turn (server tools only) and anything new
}
}
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(cfg);
const profile = claudeProfile(req.model);
const official = isOfficial(cfg);
const opts: BuildOptions = {
official,
stripThinking: false,
// The beta-gated extras are only sent to api.anthropic.com; gateways and
// proxies frequently reject fields they do not know.
useFallbacks: official && profile.fallbacks,
eagerToolInput: official,
promptCache: official,
};
// Open the stream, degrading once per recognised 400 (see file header).
let http;
for (let attempt = 0; ; attempt++) {
http = await httpRequest(`${rootUrl(cfg)}/v1/messages`, {
...ctx,
headers: headers(cfg, opts.useFallbacks ? [FALLBACK_BETA] : []),
body: buildBody(req, profile, opts),
signal: req.signal,
});
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
const detail = err.providerMessage ?? '';
if (!opts.stripThinking && /signature/i.test(detail) && /thinking/i.test(detail)) opts.stripThinking = true;
else if (opts.useFallbacks && /fallback|anthropic-beta/i.test(detail)) opts.useFallbacks = false;
else if (opts.eagerToolInput && /eager_input_streaming/i.test(detail)) opts.eagerToolInput = false;
else if (opts.promptCache && /cache_control/i.test(detail)) opts.promptCache = false;
else throw err;
}
}
// Content blocks by index, accumulated exactly as the API defines them.
const blocks: Json[] = [];
const toolJson = new Map<number, string>();
let stopReason = '';
let sawStop = false;
let reasoningEmitted = false;
const usage = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
const readUsage = (u: unknown) => {
if (!isRecord(u)) return;
if (typeof u.input_tokens === 'number') usage.input = u.input_tokens;
if (typeof u.cache_read_input_tokens === 'number') usage.cacheRead = u.cache_read_input_tokens;
if (typeof u.cache_creation_input_tokens === 'number') usage.cacheWrite = u.cache_creation_input_tokens;
if (typeof u.output_tokens === 'number') usage.output = u.output_tokens;
};
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
switch (data.type) {
case 'message_start':
if (isRecord(data.message)) readUsage(data.message.usage);
break;
case 'content_block_start': {
const index = num(data.index);
const block: Json = isRecord(data.content_block) ? { ...data.content_block } : { type: 'unknown' };
blocks[index] = block;
if (block.type === 'tool_use') toolJson.set(index, '');
if (block.type === 'text' && typeof block.text === 'string' && block.text) yield { type: 'text', delta: block.text };
if (block.type === 'thinking' && reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
break;
}
case 'content_block_delta': {
const index = num(data.index);
const block = blocks[index];
const delta = data.delta;
if (!block || !isRecord(delta)) break;
if (delta.type === 'text_delta') {
const text = str(delta.text);
block.text = str(block.text) + text;
if (text) yield { type: 'text', delta: text };
} else if (delta.type === 'thinking_delta') {
const text = str(delta.thinking);
block.thinking = str(block.thinking) + text;
if (text) {
reasoningEmitted = true;
yield { type: 'reasoning', delta: text };
}
} else if (delta.type === 'signature_delta') {
block.signature = str(block.signature) + str(delta.signature);
} else if (delta.type === 'input_json_delta') {
toolJson.set(index, (toolJson.get(index) ?? '') + str(delta.partial_json));
} else if (delta.type === 'citations_delta') {
const list = Array.isArray(block.citations) ? block.citations : [];
list.push(delta.citation);
block.citations = list;
}
break;
}
case 'message_delta':
if (isRecord(data.delta) && typeof data.delta.stop_reason === 'string') stopReason = data.delta.stop_reason;
readUsage(data.usage);
break;
case 'message_stop':
sawStop = true;
break;
case 'error':
// e.g. overloaded_error delivered after the 200 response started
throw errorFromProvider(ctx, undefined, data);
default:
break; // ping, and event types added later
}
}
if (!sawStop && !stopReason) {
throw new AIError('network', 'The connection to Anthropic closed before the response finished.', { retryable: true });
}
// Tool calls are surfaced only now, at the end of the message, because the
// stop reason decides whether they are usable:
// - `refusal` can cut a tool_use off mid-input — never run that turn's tools;
// - `max_tokens` can truncate the last call — drop it rather than "repair" it;
// - after a mid-output fallback, tool_use blocks from before the last
// `fallback` block belong to the declined attempt and must not be echoed.
let boundary = -1;
blocks.forEach((b, i) => {
if (b?.type === 'fallback') boundary = i;
});
const content: ContentPart[] = [];
const replay: Json[] = [];
const calls: Array<{ id: string; name: string; input: Record<string, unknown> }> = [];
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
if (!block) continue;
const beforeBoundary = i < boundary;
if (block.type === 'text') {
const text = str(block.text);
if (!text) continue; // empty text blocks are rejected when echoed back
content.push({ type: 'text', text });
replay.push(block);
} else if (block.type === 'tool_use') {
if (beforeBoundary || stopReason === 'refusal') continue;
const raw = toolJson.get(i) ?? '';
const name = str(block.name);
let input = parseArgsStrict(raw);
if (!input) {
if (stopReason === 'max_tokens') continue; // truncated mid-input
input = parseToolArguments(raw, LABEL, name);
}
const call = { id: str(block.id), name, input };
calls.push(call);
content.push({ type: 'tool_call', ...call });
replay.push({ ...block, input });
} else if (block.type === 'fallback') {
replay.push(block); // must stay exactly where it appeared
} else if (!beforeBoundary) {
replay.push(block); // thinking, redacted_thinking, …: echoed back unchanged
}
}
for (const call of calls) yield { type: 'tool_call', call };
let mapped = mapStopReason(stopReason);
if (mapped === 'tool_use' && calls.length === 0) mapped = 'other';
if (mapped === 'end' && calls.length > 0) mapped = 'tool_use';
const message: ChatMessage = { role: 'assistant', content };
if (replay.length > 0) message.providerData = { kind: 'anthropic', data: { content: replay } };
yield {
type: 'done',
message,
stopReason: mapped,
// Anthropic reports uncached, cache-read and cache-write input separately.
usage: makeUsage({
input: usage.input + usage.cacheRead + usage.cacheWrite,
output: usage.output,
cacheRead: usage.cacheRead,
cacheWrite: usage.cacheWrite,
}),
};
}
function supported(caps: unknown, key: string): boolean | undefined {
if (!isRecord(caps) || !isRecord(caps[key])) return undefined;
const v = (caps[key] as Json).supported;
return typeof v === 'boolean' ? v : undefined;
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const ctx = errorContext(cfg);
const found: DiscoveredModel[] = [];
let afterId = '';
// https://platform.claude.com/docs/en/api/models/list — cursor pagination via has_more / last_id.
for (let page = 0; page < 20; page++) {
const url = `${rootUrl(cfg)}/v1/models?limit=1000${afterId ? `&after_id=${encodeURIComponent(afterId)}` : ''}`;
const res = await requestJson<Json>(url, { ...ctx, headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
for (const m of data) {
const id = str(m.id);
if (!id || !isChatModelId(id)) continue;
const model: DiscoveredModel = {
id,
// The API version in use rejects sampling parameters on every model.
capabilities: { tools: true, temperature: false },
};
if (typeof m.display_name === 'string') model.label = m.display_name;
if (typeof m.max_input_tokens === 'number') model.contextWindow = m.max_input_tokens;
if (typeof m.max_tokens === 'number') model.maxOutput = m.max_tokens;
const vision = supported(m.capabilities, 'image_input');
const pdf = supported(m.capabilities, 'pdf_input');
const thinking = supported(m.capabilities, 'thinking');
if (vision !== undefined) model.capabilities = { ...model.capabilities, vision };
if (pdf !== undefined) model.capabilities = { ...model.capabilities, pdf };
if (thinking !== undefined) model.capabilities = { ...model.capabilities, reasoning: thinking };
found.push(model);
}
if (res.has_more !== true || typeof res.last_id !== 'string' || !res.last_id) break;
afterId = res.last_id;
}
return withCatalogDefaults('anthropic', found);
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
// Listing models is free and proves both connectivity and the key.
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
}
export const anthropicAdapter: ProviderAdapter = {
kind: 'anthropic',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: true,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://platform.claude.com/settings/keys',
keyHint: 'sk-ant-…',
},
stream,
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

View file

@ -0,0 +1,206 @@
/**
* Static defaults for well-known models: capabilities, limits and list prices.
*
* Discovery APIs return ids (and sometimes limits) but almost never prices or
* feature flags, so `listModels` merges its results with this table. Unknown
* models simply get no defaults — nothing here is required for a model to work.
*
* Everything below was read from the official pages listed per provider on the
* dates noted. Prices are USD per million tokens, standard (non-batch,
* non-priority) tier, short-context tier where a provider has several. A field
* is omitted when it could not be confirmed from the page — do not guess.
*/
import type { DiscoveredModel, ModelCapabilities, ProviderKind } from '../types.ts';
export const CATALOG_UPDATED = '2026-11-10';
type Entry = Omit<DiscoveredModel, 'id' | 'capabilities'> & { capabilities?: Partial<ModelCapabilities> };
/* ------------------------------------------------------------------ */
/* OpenAI */
/* Prices: https://developers.openai.com/api/docs/pricing (fetched 2026-11-11)
/* Limits: https://developers.openai.com/api/docs/models (fetched 2026-11-11;
/* only the four current flagship models list limits there — the
/* limits on older models below are the long-published values and were
/* not re-fetched)
/* Reasoning models reject a custom `temperature`, hence temperature:false.
/* ------------------------------------------------------------------ */
const OA_REASONING: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
const OA_CLASSIC: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: false, temperature: true };
const OA_FLAGSHIP_LIMITS = { contextWindow: 1_050_000, maxOutput: 128_000 };
const OPENAI: Record<string, Entry> = {
'gpt-6-astra': { label: 'GPT-6 Astra', ...OA_FLAGSHIP_LIMITS, priceIn: 10, priceOut: 50, capabilities: OA_REASONING },
'gpt-6.1-sol': { label: 'GPT-6.1 Sol', ...OA_FLAGSHIP_LIMITS, priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
'gpt-6.1-terra': { label: 'GPT-6.1 Terra', ...OA_FLAGSHIP_LIMITS, priceIn: 1, priceOut: 5, capabilities: OA_REASONING },
'gpt-6-luna': { label: 'GPT-6 Luna', ...OA_FLAGSHIP_LIMITS, priceIn: 0.1, priceOut: 0.5, capabilities: OA_REASONING },
'gpt-6-sol': { label: 'GPT-6 Sol', priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
'gpt-5.6-sol': { label: 'GPT-5.6 Sol', priceIn: 4, priceOut: 20, capabilities: OA_REASONING },
'gpt-5.6-terra': { label: 'GPT-5.6 Terra', priceIn: 2, priceOut: 12, capabilities: OA_REASONING },
'gpt-5.6-luna': { label: 'GPT-5.6 Luna', priceIn: 0.2, priceOut: 1.2, capabilities: OA_REASONING },
'gpt-5.5': { label: 'GPT-5.5', priceIn: 5, priceOut: 30, capabilities: OA_REASONING },
'gpt-5.5-pro': { label: 'GPT-5.5 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
'gpt-5.4': { label: 'GPT-5.4', priceIn: 2.5, priceOut: 15, capabilities: OA_REASONING },
'gpt-5.4-pro': { label: 'GPT-5.4 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
'gpt-5.4-mini': { label: 'GPT-5.4 mini', priceIn: 0.75, priceOut: 4.5, capabilities: OA_REASONING },
'gpt-5.4-nano': { label: 'GPT-5.4 nano', priceIn: 0.2, priceOut: 1.25, capabilities: OA_REASONING },
'gpt-5.3-codex': { label: 'GPT-5.3 Codex', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
'gpt-5.2': { label: 'GPT-5.2', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
'gpt-5.2-pro': { label: 'GPT-5.2 Pro', priceIn: 21, priceOut: 168, capabilities: OA_REASONING },
'gpt-5.1': { label: 'GPT-5.1', priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
'gpt-5': { label: 'GPT-5', contextWindow: 400_000, maxOutput: 128_000, priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
'gpt-5-mini': { label: 'GPT-5 mini', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.25, priceOut: 2, capabilities: OA_REASONING },
'gpt-5-nano': { label: 'GPT-5 nano', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.05, priceOut: 0.4, capabilities: OA_REASONING },
'gpt-5-pro': { label: 'GPT-5 Pro', priceIn: 15, priceOut: 120, capabilities: OA_REASONING },
'chat-latest': { label: 'ChatGPT (latest)', priceIn: 5, priceOut: 30, capabilities: OA_CLASSIC },
'gpt-4.1': { label: 'GPT-4.1', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 2, priceOut: 8, capabilities: OA_CLASSIC },
'gpt-4.1-mini': { label: 'GPT-4.1 mini', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 0.4, priceOut: 1.6, capabilities: OA_CLASSIC },
'gpt-4o': { label: 'GPT-4o', contextWindow: 128_000, maxOutput: 16_384, priceIn: 2.5, priceOut: 10, capabilities: OA_CLASSIC },
'gpt-4o-mini': { label: 'GPT-4o mini', contextWindow: 128_000, maxOutput: 16_384, priceIn: 0.15, priceOut: 0.6, capabilities: OA_CLASSIC },
o3: { label: 'o3', contextWindow: 200_000, maxOutput: 100_000, priceIn: 2, priceOut: 8, capabilities: OA_REASONING },
'o3-pro': { label: 'o3-pro', contextWindow: 200_000, maxOutput: 100_000, priceIn: 20, priceOut: 80, capabilities: OA_REASONING },
};
/* ------------------------------------------------------------------ */
/* Anthropic */
/* Prices: https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-11-11)
/* Ids and limits: https://platform.claude.com/docs/en/about-claude/models/overview
/* (via Anthropic's Claude API reference, cached 2026-11-03)
/* temperature:false for every Claude model: the adapter speaks API version
/* 2026-10-21, which rejects temperature/top_p/top_k on all models
/* (https://platform.claude.com/docs/en/api/migration-2026-10-21).
/* Claude Haiku 5.5 prices are for prompts up to 100K tokens ($0.50/$2.50 above).
/* ------------------------------------------------------------------ */
const CLAUDE: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
const CLAUDE_1M = { contextWindow: 1_000_000, maxOutput: 128_000 };
const ANTHROPIC: Record<string, Entry> = {
'claude-fable-5-2': { label: 'Claude Fable 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
'claude-mythos-5-2': { label: 'Claude Mythos 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
'claude-fable-5-1': { label: 'Claude Fable 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-mythos-5-1': { label: 'Claude Mythos 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-fable-5': { label: 'Claude Fable 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-mythos-5': { label: 'Claude Mythos 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-opus-5-5': { label: 'Claude Opus 5.5', ...CLAUDE_1M, priceIn: 4, priceOut: 20, capabilities: CLAUDE },
'claude-opus-5': { label: 'Claude Opus 5', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-8': { label: 'Claude Opus 4.8', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-7': { label: 'Claude Opus 4.7', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-6': { label: 'Claude Opus 4.6', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-5': { label: 'Claude Opus 4.5', priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-sonnet-5-5': { label: 'Claude Sonnet 5.5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
'claude-sonnet-5': { label: 'Claude Sonnet 5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
'claude-sonnet-4-6': { label: 'Claude Sonnet 4.6', ...CLAUDE_1M, priceIn: 3, priceOut: 15, capabilities: CLAUDE },
'claude-sonnet-4-5': { label: 'Claude Sonnet 4.5', priceIn: 3, priceOut: 15, capabilities: CLAUDE },
'claude-haiku-5-5': { label: 'Claude Haiku 5.5', ...CLAUDE_1M, priceIn: 0.1, priceOut: 0.5, capabilities: CLAUDE },
'claude-haiku-4-5': { label: 'Claude Haiku 4.5', contextWindow: 200_000, maxOutput: 64_000, priceIn: 1, priceOut: 5, capabilities: CLAUDE },
};
/* ------------------------------------------------------------------ */
/* Google Gemini */
/* Prices: https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-11-11,
/* now served from https://aistudio.google.com/docs/pricing)
/* Limits: https://ai.google.dev/gemini-api/docs/models (fetched 2026-11-11;
/* only the Gemini 4 models list limits — `GET /v1beta/models`
/* supplies inputTokenLimit/outputTokenLimit for the rest at runtime)
/* Scheduled changes announced on the pricing page (NOT applied here):
/* - gemini-4-carbon / gemini-4-argon: $4 / $20 from 2026-12-01
/* - gemini-3.8-flash / gemini-3.6-flash: $1.50 / $7.50 from 2027-01-01
/* - gemini-2.5-pro / gemini-2.5-flash are deprecated and are routed to
/* gemini-3.8-flash / gemini-3.5-flash-lite (and billed as such) from 2026-12-01
/* Pro-tier prices are for prompts up to 200K tokens.
/* ------------------------------------------------------------------ */
const GEMINI_CAPS: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: true };
const GEMINI_4_LIMITS = { contextWindow: 1_048_576, maxOutput: 65_536 };
const GEMINI: Record<string, Entry> = {
'gemini-4-carbon': { label: 'Gemini 4 Carbon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
'gemini-4-argon': { label: 'Gemini 4 Argon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
'gemini-3.8-flash': { label: 'Gemini 3.8 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
'gemini-3.6-flash': { label: 'Gemini 3.6 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
'gemini-3.5-flash-lite': { label: 'Gemini 3.5 Flash-Lite', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
'gemini-3.1-pro-preview': { label: 'Gemini 3.1 Pro (preview)', priceIn: 2, priceOut: 12, capabilities: GEMINI_CAPS },
'gemini-3.1-flash-lite': { label: 'Gemini 3.1 Flash-Lite', priceIn: 0.25, priceOut: 1.5, capabilities: GEMINI_CAPS },
'gemini-3-flash-preview': { label: 'Gemini 3 Flash (preview)', priceIn: 0.5, priceOut: 3, capabilities: GEMINI_CAPS },
'gemini-2.5-pro': { label: 'Gemini 2.5 Pro', priceIn: 1.25, priceOut: 10, capabilities: GEMINI_CAPS },
'gemini-2.5-flash': { label: 'Gemini 2.5 Flash', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
'gemini-2.5-flash-lite': { label: 'Gemini 2.5 Flash-Lite', priceIn: 0.1, priceOut: 0.4, capabilities: GEMINI_CAPS },
};
const TABLES: Record<ProviderKind, Record<string, Entry>> = {
openai: OPENAI,
anthropic: ANTHROPIC,
gemini: GEMINI,
// Aggregators and local servers host arbitrary models at their own prices.
openai_compatible: {},
ollama: {},
};
/**
* What may follow a known id for it to still be "the same model at the same
* price": a dated snapshot or a moving alias, e.g. `gpt-5-2025-08-07`,
* `claude-haiku-4-5-20251001`, `claude-opus-4-5@20251101`, `gemini-2.5-flash-latest`.
*/
const SNAPSHOT_SUFFIX = /^[-@:](\d{4}-\d{2}-\d{2}|\d{6,8}|\d{3,4}|latest|preview|preview-\d{2}-\d{2,4}|v\d+(:\d+)?)$/;
function clone(e: Entry): Partial<DiscoveredModel> {
return { ...e, ...(e.capabilities ? { capabilities: { ...e.capabilities } } : {}) };
}
function lookupIn(table: Record<string, Entry>, id: string): Partial<DiscoveredModel> {
const exact = table[id];
if (exact) return clone(exact);
// Longest known id that is a prefix of `id` and ends on a separator.
let best = '';
for (const key of Object.keys(table)) {
if (key.length > best.length && id.startsWith(key) && /[-@:]/.test(id.charAt(key.length))) best = key;
}
if (!best) return {};
const entry = table[best];
if (!entry) return {};
const out = clone(entry);
delete out.label; // the label names the base model, not this variant
if (!SNAPSHOT_SUFFIX.test(id.slice(best.length))) {
// A genuinely different variant (`o3-deep-research`, `gpt-4o-audio-...`):
// capabilities are a fair guess, the price is not.
delete out.priceIn;
delete out.priceOut;
}
return out;
}
/**
* Defaults for a model: exact id first, then the longest known id that prefixes
* it (so dated snapshots match their base model). Returns `{}` when unknown.
*/
export function lookupModelDefaults(kind: ProviderKind, modelId: string): Partial<DiscoveredModel> {
// Gemini's listing names models "models/<id>".
const id = modelId.trim().replace(/^models\//, '');
if (!id) return {};
if (kind === 'openai_compatible') {
// Routers name models "<vendor>/<model>" (`anthropic/claude-sonnet-5-5`).
// Borrow capabilities and limits from the vendor's table, never its prices.
const bare = id.slice(id.lastIndexOf('/') + 1).replace(/:[a-z]+$/, '');
for (const table of [OPENAI, ANTHROPIC, GEMINI]) {
const hit = lookupIn(table, bare);
if (Object.keys(hit).length > 0) {
delete hit.priceIn;
delete hit.priceOut;
// Temperature limits are properties of the first-party APIs, not of the router.
if (hit.capabilities) delete hit.capabilities.temperature;
return hit;
}
}
return {};
}
return lookupIn(TABLES[kind], id);
}
/** Every catalogued model id for a provider (used as a fallback list when discovery is unavailable). */
export function catalogModelIds(kind: ProviderKind): string[] {
return Object.keys(TABLES[kind]);
}

View file

@ -0,0 +1,138 @@
/**
* Helpers shared by the adapters that are about our neutral contract rather
* than about HTTP: tool-argument parsing, model-list post-processing, the
* connection-test wrapper.
*/
import { jsonrepair } from 'jsonrepair';
import { AIError, type DiscoveredModel, type ProviderKind, type TestResult, type Usage } from '../types.ts';
import { lookupModelDefaults } from './catalog.ts';
import { isRecord } from './http.ts';
/** Strict parse of tool-call arguments; `undefined` when they are not a JSON object. */
export function parseArgsStrict(raw: string): Record<string, unknown> | undefined {
const text = raw.trim();
if (text === '') return {};
try {
const value: unknown = JSON.parse(text);
return isRecord(value) ? value : undefined;
} catch {
return undefined;
}
}
/**
* Parses the JSON arguments of a completed tool call. Models occasionally emit
* slightly broken JSON (trailing commas, unescaped newlines); `jsonrepair`
* fixes those. Anything still unparseable is an `invalid_response`.
*
* Do NOT use this on a call that was cut off by the output-token limit —
* repairing a truncated call would silently invent a shorter input.
*/
export function parseToolArguments(raw: unknown, provider: string, toolName: string): Record<string, unknown> {
if (isRecord(raw)) return raw; // some servers send an object instead of a JSON string
const text = typeof raw === 'string' ? raw : raw == null ? '' : String(raw);
const strict = parseArgsStrict(text);
if (strict) return strict;
try {
const repaired: unknown = JSON.parse(jsonrepair(text));
if (isRecord(repaired)) return repaired;
} catch {
/* fall through */
}
throw new AIError('invalid_response', `${provider} produced a tool call for "${toolName}" with arguments that are not valid JSON.`, {
providerMessage: text.length > 300 ? `${text.slice(0, 300)}…` : text,
});
}
/** Stable id for providers that do not issue tool-call ids themselves. */
export function newToolCallId(prefix: string): string {
return `${prefix}_${globalThis.crypto.randomUUID().replace(/-/g, '').slice(0, 24)}`;
}
export function dataUrl(mime: string, base64: string): string {
return `data:${mime};base64,${base64}`;
}
export function num(v: unknown): number {
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
}
export function str(v: unknown): string {
return typeof v === 'string' ? v : '';
}
/**
* Usage convention used by every adapter:
* inputTokens = ALL prompt tokens, including cached ones
* cacheRead/cacheWriteTokens = the cached subsets of inputTokens
* outputTokens = ALL generated tokens, including reasoning
* reasoningTokens = the reasoning subset of outputTokens
*/
export function makeUsage(u: { input: number; output: number; cacheRead?: number; cacheWrite?: number; reasoning?: number }): Usage {
const usage: Usage = { inputTokens: u.input, outputTokens: u.output };
if (u.cacheRead) usage.cacheReadTokens = u.cacheRead;
if (u.cacheWrite) usage.cacheWriteTokens = u.cacheWrite;
if (u.reasoning) usage.reasoningTokens = u.reasoning;
return usage;
}
/**
* Ids that are obviously not text-chat models: embeddings, speech, image and
* video generation, moderation, realtime/audio-only, rerankers, legacy
* completion-only models.
*/
const NON_CHAT_RE = new RegExp(
[
'embed', // text-embedding-3-large, gemini-embedding-001, nomic-embed-text
'\\btts\\b|whisper|transcri|speech|\\baudio\\b|realtime|\\blive\\b',
'dall-e|gpt-image|image-generation|imagen|-image\\b|image-preview|\\bsora\\b|\\bveo\\b|lyria|stable-diffusion',
'moderation|-guard\\b|rerank|\\baqa\\b',
'computer-use|robotics|deep-research',
'^(babbage|davinci|curie|ada)\\b|^text-|turbo-instruct',
].join('|'),
'i',
);
export function isChatModelId(id: string): boolean {
return !NON_CHAT_RE.test(id);
}
/** Merges discovery results with catalog defaults; values the provider reported always win. */
export function withCatalogDefaults(kind: ProviderKind, models: DiscoveredModel[]): DiscoveredModel[] {
const seen = new Set<string>();
const out: DiscoveredModel[] = [];
for (const m of models) {
if (!m.id || seen.has(m.id)) continue;
seen.add(m.id);
const d = lookupModelDefaults(kind, m.id);
const merged: DiscoveredModel = { ...d, ...stripUndefined(m), id: m.id };
if (d.capabilities || m.capabilities) merged.capabilities = { ...d.capabilities, ...stripUndefined(m.capabilities ?? {}) };
out.push(merged);
}
return out;
}
function stripUndefined<T extends object>(obj: T): T {
const out = {} as Record<string, unknown>;
for (const [k, v] of Object.entries(obj)) if (v !== undefined && v !== null) out[k] = v;
return out as T;
}
/** Shared implementation of `ProviderAdapter.test()`: never throws, always friendly. */
export async function runTest(label: string, check: () => Promise<{ modelCount?: number; note?: string }>): Promise<TestResult> {
const started = Date.now();
try {
const { modelCount, note } = await check();
const latencyMs = Date.now() - started;
const message =
note ?? (modelCount !== undefined ? `Connected to ${label} — ${modelCount} model${modelCount === 1 ? '' : 's'} available.` : `Connected to ${label}.`);
return modelCount !== undefined ? { ok: true, message, latencyMs, modelCount } : { ok: true, message, latencyMs };
} catch (err) {
const latencyMs = Date.now() - started;
const message = err instanceof AIError ? err.message : `Could not connect to ${label}.`;
return { ok: false, message, latencyMs };
}
}
/** Timeout used by `test()` so a dead endpoint fails fast instead of hanging the settings page. */
export const TEST_TIMEOUT_MS = 20_000;

View file

@ -0,0 +1,342 @@
/**
* Google Gemini — `models/{model}:streamGenerateContent?alt=sse`.
* https://ai.google.dev/api/generate-content
* https://ai.google.dev/gemini-api/docs/function-calling
* https://ai.google.dev/gemini-api/docs/thinking
* https://ai.google.dev/gemini-api/docs/thought-signatures
*
* Quirks handled here:
* - The API key travels in the `x-goog-api-key` header, never in the URL.
* - Function declarations accept only a subset of JSON Schema, so schemas are
* down-converted (`toGeminiSchema`).
* - Gemini issues no tool-call ids; stable ones are synthesised.
* - Thinking models attach a `thoughtSignature` to parts (notably function calls)
* that must be sent back unchanged, so each turn's raw parts live in `providerData`.
* - Thinking is configured with `thinkingLevel` on Gemini 3 and later and with a
* numeric `thinkingBudget` on 2.5; an unsupported config is dropped and retried.
* - A response can be blocked (`promptFeedback.blockReason`, finishReason SAFETY).
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { isChatModelId, makeUsage, newToolCallId, num, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
type Json = Record<string, unknown>;
const LABEL = 'Google Gemini';
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com';
function root(cfg: ProviderConfig): string {
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1(beta)?$/, '');
}
function errorContext(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig): Record<string, string> {
if (!cfg.apiKey) throw new AIError('not_configured', 'Add a Gemini API key in Settings to use Gemini models.');
return { ...cfg.headers, 'x-goog-api-key': cfg.apiKey };
}
const SCHEMA_KEYS = new Set(['type', 'description', 'enum', 'properties', 'required', 'items', 'nullable', 'format', 'minimum', 'maximum', 'minItems', 'maxItems', 'anyOf', 'title']);
const SCHEMA_FORMATS = new Set(['date-time', 'enum', 'int32', 'int64', 'float', 'double']);
/** Reduces JSON Schema to the OpenAPI-style subset Gemini's `parameters` field accepts. */
export function toGeminiSchema(schema: unknown): Json {
if (!isRecord(schema)) return { type: 'string' };
const out: Json = {};
for (const [key, value] of Object.entries(schema)) {
if (!SCHEMA_KEYS.has(key)) continue; // additionalProperties, $schema, default, pattern, const, ...
if (key === 'properties' && isRecord(value)) {
out.properties = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, toGeminiSchema(v)]));
} else if (key === 'items') {
out.items = toGeminiSchema(value);
} else if (key === 'anyOf' && Array.isArray(value)) {
out.anyOf = value.map(toGeminiSchema);
} else if (key === 'type' && Array.isArray(value)) {
// ["string","null"] -> nullable string
const real = value.filter((t) => t !== 'null');
out.type = real[0] ?? 'string';
if (value.includes('null')) out.nullable = true;
} else if (key === 'format') {
if (typeof value === 'string' && SCHEMA_FORMATS.has(value)) out.format = value;
} else {
out[key] = value;
}
}
if (out.enum && !out.type) out.type = 'string';
if (out.type === 'object' && !out.properties) out.properties = {};
return out;
}
function userParts(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
const out: Json[] = [];
for (const p of parts) {
if (p.type === 'tool_result') {
out.push({ functionResponse: { name: p.name, response: p.isError ? { error: p.output } : { result: p.output } } });
} else if (p.type === 'text' && p.text) {
out.push({ text: p.text });
} else if (p.type === 'image') {
if (caps.vision === false) out.push({ text: '[An image was attached, but this model cannot read images.]' });
else out.push({ inlineData: { mimeType: p.mime, data: p.data } });
} else if (p.type === 'file') {
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ inlineData: { mimeType: p.mime, data: p.data } });
else if (p.mime.startsWith('text/')) out.push({ text: Buffer.from(p.data, 'base64').toString('utf8') });
else out.push({ text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
}
return out;
}
function toContents(messages: ChatMessage[], caps: Partial<ModelCapabilities>): Json[] {
const contents: Json[] = [];
for (const msg of messages) {
let parts: Json[];
if (msg.role === 'assistant') {
const pd = msg.providerData;
if (pd?.kind === 'gemini' && isRecord(pd.data) && Array.isArray(pd.data.parts)) {
parts = pd.data.parts.filter(isRecord); // thought signatures included, exactly as received
} else {
parts = [];
for (const p of msg.content) {
if (p.type === 'text' && p.text) parts.push({ text: p.text });
else if (p.type === 'tool_call') parts.push({ functionCall: { name: p.name, args: p.input ?? {} } });
}
}
} else {
parts = userParts(msg.content, caps);
}
if (parts.length === 0) continue;
const role = msg.role === 'assistant' ? 'model' : 'user';
const last = contents[contents.length - 1];
// Gemini requires strictly alternating roles; merge consecutive turns of the same role.
if (last && last.role === role) (last.parts as Json[]).push(...parts);
else contents.push({ role, parts });
}
if (contents[0]?.role === 'model') contents.unshift({ role: 'user', parts: [{ text: '(continuing the conversation)' }] });
return contents;
}
type ThinkingStyle = 'level' | 'budget' | 'off';
function thinkingStyleFor(model: string): ThinkingStyle {
const m = /gemini-(\d+)(?:\.(\d+))?/.exec(model);
if (!m) return 'level';
const major = Number(m[1]);
if (major >= 3) return 'level';
if (major === 2 && Number(m[2] ?? 0) >= 5) return 'budget';
return 'off';
}
const BUDGETS = { none: 0, low: 2_048, medium: 8_192, high: 24_576 } as const;
function buildBody(req: ChatRequest, thinking: ThinkingStyle, jsonMode: boolean): Json {
const caps = req.capabilities ?? {};
const body: Json = { contents: toContents(req.messages, caps) };
if (req.system?.trim()) body.systemInstruction = { parts: [{ text: req.system.trim() }] };
const hasTools = caps.tools !== false && (req.tools?.length ?? 0) > 0;
if (hasTools) {
body.tools = [{ functionDeclarations: req.tools!.map((t) => ({ name: t.name, description: t.description, parameters: toGeminiSchema({ type: 'object', ...t.parameters }) })) }];
const choice = req.toolChoice;
if (choice === 'none') body.toolConfig = { functionCallingConfig: { mode: 'NONE' } };
else if (choice === 'required') body.toolConfig = { functionCallingConfig: { mode: 'ANY' } };
else if (isRecord(choice)) body.toolConfig = { functionCallingConfig: { mode: 'ANY', allowedFunctionNames: [choice.name] } };
}
const gen: Json = {};
if (req.temperature !== undefined && caps.temperature !== false) gen.temperature = req.temperature;
if (req.maxOutputTokens) gen.maxOutputTokens = req.maxOutputTokens;
// JSON mode cannot be combined with function calling on most models.
if (jsonMode && req.jsonMode && !hasTools) gen.responseMimeType = 'application/json';
if (caps.reasoning !== false && thinking !== 'off') {
const effort = req.reasoningEffort;
if (thinking === 'level') {
const cfg: Json = { includeThoughts: true };
if (effort) cfg.thinkingLevel = effort === 'none' ? 'minimal' : effort;
gen.thinkingConfig = cfg;
} else if (effort) {
// 2.5 Pro cannot switch thinking off; its minimum budget is 128 tokens.
const budget = effort === 'none' && /pro/.test(req.model) ? 128 : BUDGETS[effort];
gen.thinkingConfig = { thinkingBudget: budget, includeThoughts: budget > 0 };
} else {
gen.thinkingConfig = { includeThoughts: true };
}
}
if (Object.keys(gen).length > 0) body.generationConfig = gen;
return body;
}
function mapFinish(reason: string): StopReason {
switch (reason) {
case 'STOP':
return 'end';
case 'MAX_TOKENS':
return 'max_tokens';
case 'SAFETY':
case 'RECITATION':
case 'BLOCKLIST':
case 'PROHIBITED_CONTENT':
case 'SPII':
case 'IMAGE_SAFETY':
return 'refusal';
default:
return 'other'; // MALFORMED_FUNCTION_CALL, OTHER, UNEXPECTED_TOOL_CALL, ...
}
}
function modelPath(model: string): string {
return model.startsWith('models/') || model.startsWith('tunedModels/') ? model : `models/${model}`;
}
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(cfg);
const url = `${root(cfg)}/v1beta/${modelPath(req.model)}:streamGenerateContent?alt=sse`;
let thinking = thinkingStyleFor(req.model);
let jsonMode = true;
let http: HttpResponse;
for (let attempt = 0; ; attempt += 1) {
http = await httpRequest(url, { ...ctx, headers: headers(cfg), body: buildBody(req, thinking, jsonMode), signal: req.signal });
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
const detail = (err.providerMessage ?? '').toLowerCase();
if (thinking !== 'off' && /thinking|thought/.test(detail)) thinking = thinking === 'level' ? 'budget' : 'off';
else if (jsonMode && req.jsonMode && /response_?mime|json/.test(detail)) jsonMode = false;
else throw err;
}
}
const parts: Json[] = [];
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
let text = '';
let finish = '';
let blocked = '';
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
if (isRecord(data.promptFeedback) && str(data.promptFeedback.blockReason)) blocked = str(data.promptFeedback.blockReason);
if (isRecord(data.usageMetadata)) {
const u = data.usageMetadata;
usage.input = num(u.promptTokenCount) || usage.input;
usage.reasoning = num(u.thoughtsTokenCount) || usage.reasoning;
// candidatesTokenCount excludes thinking; both are billed as output.
usage.output = num(u.candidatesTokenCount) + num(u.thoughtsTokenCount) || usage.output;
usage.cacheRead = num(u.cachedContentTokenCount) || usage.cacheRead;
}
const candidate = Array.isArray(data.candidates) ? data.candidates[0] : undefined;
if (!isRecord(candidate)) continue;
if (str(candidate.finishReason)) finish = str(candidate.finishReason);
const chunkParts = isRecord(candidate.content) && Array.isArray(candidate.content.parts) ? candidate.content.parts.filter(isRecord) : [];
for (const part of chunkParts) {
if (isRecord(part.functionCall)) {
const call = { id: str(part.functionCall.id) || newToolCallId('gem'), name: str(part.functionCall.name), input: isRecord(part.functionCall.args) ? part.functionCall.args : {} };
calls.push(call);
parts.push(part);
yield { type: 'tool_call', call };
} else if (typeof part.text === 'string') {
if (part.thought === true) {
if (part.text) yield { type: 'reasoning', delta: part.text };
// Thought summaries are display-only; only a signature needs to survive.
if (part.thoughtSignature) parts.push({ text: '', thought: true, thoughtSignature: part.thoughtSignature });
} else {
if (part.text) {
text += part.text;
yield { type: 'text', delta: part.text };
}
// Merge streamed text fragments, but keep a part that carries a signature intact.
const last = parts[parts.length - 1];
if (last && typeof last.text === 'string' && last.thought !== true && !last.thoughtSignature && !part.thoughtSignature) last.text += part.text;
else parts.push({ ...part });
}
} else {
parts.push(part); // inlineData, executableCode, ... replayed untouched
}
}
}
if (blocked && !text && calls.length === 0) {
throw new AIError('content_filter', 'Gemini blocked this request under its safety policy.', { providerMessage: `blockReason: ${blocked}` });
}
if (!finish && !text && calls.length === 0) {
throw new AIError('network', 'The connection to Gemini closed before the response finished.', { retryable: true });
}
if (finish === 'MALFORMED_FUNCTION_CALL' && calls.length === 0) {
throw new AIError('invalid_response', 'Gemini produced a malformed tool call. Try again.', { retryable: true });
}
const content: ContentPart[] = [];
if (text) content.push({ type: 'text', text });
for (const call of calls) content.push({ type: 'tool_call', ...call });
const message: ChatMessage = { role: 'assistant', content };
const replay = parts.filter((p) => !(typeof p.text === 'string' && p.text === '' && !p.thoughtSignature));
if (replay.length > 0) message.providerData = { kind: 'gemini', data: { parts: replay } };
yield {
type: 'done',
message,
stopReason: calls.length > 0 ? 'tool_use' : mapFinish(finish || 'STOP'),
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
};
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const found: DiscoveredModel[] = [];
let pageToken = '';
for (let page = 0; page < 10; page += 1) {
const url = `${root(cfg)}/v1beta/models?pageSize=1000${pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''}`;
const res = await requestJson<Json>(url, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
for (const m of Array.isArray(res.models) ? res.models.filter(isRecord) : []) {
const id = str(m.name).replace(/^models\//, '');
const methods = Array.isArray(m.supportedGenerationMethods) ? m.supportedGenerationMethods.map(String) : [];
if (!id || !methods.includes('generateContent') || !isChatModelId(id)) continue;
if (!/^(gemini|gemma|learnlm)/.test(id)) continue;
const model: DiscoveredModel = { id };
if (str(m.displayName)) model.label = str(m.displayName);
if (num(m.inputTokenLimit)) model.contextWindow = num(m.inputTokenLimit);
if (num(m.outputTokenLimit)) model.maxOutput = num(m.outputTokenLimit);
if (typeof m.thinking === 'boolean') model.capabilities = { reasoning: m.thinking };
if (id.startsWith('gemma')) model.capabilities = { ...model.capabilities, tools: false, pdf: false };
found.push(model);
}
pageToken = str(res.nextPageToken);
if (!pageToken) break;
}
return withCatalogDefaults('gemini', found);
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
}
export const geminiAdapter: ProviderAdapter = {
kind: 'gemini',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: true,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://aistudio.google.com/apikey',
keyHint: 'AIza…',
},
stream,
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

519
server/ai/providers/http.ts Normal file
View file

@ -0,0 +1,519 @@
/**
* Shared HTTP plumbing for provider adapters: a fetch wrapper with timeouts,
* an SSE parser over Web Streams, and one place that turns provider failures
* into `AIError`s. Uses only global `fetch` — no SDKs.
*
* Security rule for this whole folder: an API key must never reach an error
* message, a log line or a thrown value. Everything user-visible that contains
* provider text goes through `scrubSecrets` first.
*/
import { AIError, type AIErrorCode } from '../types.ts';
/* ------------------------------------------------------------------ */
/* Secret scrubbing */
/* ------------------------------------------------------------------ */
const KEY_PATTERNS: RegExp[] = [
/\bsk-[A-Za-z0-9_-]{8,}/g, // OpenAI / Anthropic / OpenRouter style
/\bAIza[0-9A-Za-z_-]{16,}/g, // Google API keys
/\bgsk_[A-Za-z0-9]{8,}/g, // Groq
/(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi,
/([?&](?:key|api_key|apikey|access_token)=)[^&\s"']+/gi,
];
/** Removes known secrets (and anything that looks like an API key) from provider text. */
export function scrubSecrets(text: string, secrets: Array<string | null | undefined> = []): string {
let out = text;
for (const s of secrets) {
if (s && s.length >= 4) out = out.split(s).join('[redacted]');
}
for (const re of KEY_PATTERNS) {
out = out.replace(re, (_m, prefix?: unknown) => (typeof prefix === 'string' ? `${prefix}[redacted]` : '[redacted]'));
}
return out;
}
function clip(text: string, max = 800): string {
const t = text.trim();
return t.length > max ? `${t.slice(0, max)}…` : t;
}
/* ------------------------------------------------------------------ */
/* Error mapping */
/* ------------------------------------------------------------------ */
export interface ErrorContext {
/** Human provider label used in messages ("OpenAI", "Ollama"). */
provider: string;
/** Values that must never appear in an error (API key, custom header values). */
secrets?: Array<string | null | undefined>;
}
interface ProviderErrorFields {
message: string;
type: string;
code: string;
/** Whole body as text, used for keyword matching (quota ids, reasons, ...). */
raw: string;
}
/** Pulls message/type/code out of the error envelopes used by the providers we talk to. */
export function extractProviderError(body: unknown): ProviderErrorFields {
let node: unknown = body;
if (typeof node === 'string') {
const text = node;
try {
node = JSON.parse(text);
} catch {
return { message: text.trim().startsWith('<') ? '' : text.trim(), type: '', code: '', raw: text };
}
}
const raw = safeStringify(node);
// Gemini sometimes wraps the error object in an array.
if (Array.isArray(node)) node = node[0];
if (!isRecord(node)) return { message: '', type: '', code: '', raw };
const err = node.error ?? node;
// Ollama: {"error": "model 'x' not found"}
if (typeof err === 'string') return { message: err, type: '', code: '', raw };
if (!isRecord(err)) return { message: '', type: '', code: '', raw };
const message = typeof err.message === 'string' ? err.message : typeof node.message === 'string' ? node.message : '';
// OpenAI: error.type + error.code; Anthropic: error.type; Gemini: error.status (+ numeric code).
const type = [err.type, err.status].filter((v): v is string => typeof v === 'string').join(' ');
const code = typeof err.code === 'string' ? err.code : '';
return { message, type, code, raw };
}
const CONTEXT_RE =
/context[_ ]length|context[_ ]window|maximum context|max(?:imum)? (?:input )?tokens? (?:limit|allowed)|too many (?:input )?tokens|prompt is too long|input is too long|input token count|exceeds? the (?:model'?s? )?(?:maximum|max)[^.]{0,40}tokens|reduce the length|request[_ ]too[_ ]large|exceed_context_size|n_ctx/i;
const QUOTA_RE =
/insufficient[_ ]quota|billing|credit balance|exceeded your current quota|out of credits|insufficient (?:credits|funds|balance)|payment required|spend(?:ing)? limit/i;
const AUTH_RE = /invalid[_ ]api[_ ]key|incorrect api key|api key not valid|api_key_invalid|authentication[_ ]error|invalid x-api-key|unauthorized|no auth credentials/i;
const FILTER_RE = /content[_ ]filter|content[_ ]policy|content management policy|safety system|responsible ai/i;
function classify(status: number | undefined, f: ProviderErrorFields): AIErrorCode {
const hay = `${f.type} ${f.code} ${f.message}`;
const s = status ?? 0;
// Provider error names win over the status code: they are also all we have
// for errors delivered inside an already-open (HTTP 200) stream.
if (/authentication_error|invalid_api_key|permission_error|PERMISSION_DENIED|UNAUTHENTICATED/.test(hay)) return 'auth';
if (/insufficient_quota|billing_error/.test(hay)) return 'quota';
if (/context_length_exceeded|request_too_large/.test(hay)) return 'context_length';
if (/overloaded_error|server_error|api_error|UNAVAILABLE|INTERNAL\b/.test(`${f.type} ${f.code}`)) return 'server';
if (s === 401 || s === 403) return 'auth';
if (s === 402) return 'quota';
if (s === 429 || /rate_limit|RESOURCE_EXHAUSTED/.test(hay)) {
// Gemini words every 429 as "exceeded your current quota"; its details say
// whether it is a per-minute limit (retry soon) or a daily/billing cap.
if (/PerMinute|retryDelay|RetryInfo/.test(f.raw) && !/PerDay/.test(f.raw)) return 'rate_limit';
return QUOTA_RE.test(hay) || /PerDay/.test(f.raw) ? 'quota' : 'rate_limit';
}
if (s === 404 || /not_found_error|model_not_found|NOT_FOUND/.test(hay)) return 'not_found';
if (s === 408 || s === 504 || /DEADLINE_EXCEEDED/.test(hay)) return 'timeout';
if (s === 413) return 'context_length';
if (s >= 500) return 'server';
// Remaining 4xx (and status-less stream errors).
if (AUTH_RE.test(hay)) return 'auth'; // Gemini answers a bad key with 400 INVALID_ARGUMENT
if (CONTEXT_RE.test(hay)) return 'context_length';
if (QUOTA_RE.test(hay)) return 'quota'; // Anthropic: 400 "credit balance is too low"
if (FILTER_RE.test(hay)) return 'content_filter';
if (s === 0 && !f.message) return 'server';
return 'bad_request';
}
function friendlyMessage(code: AIErrorCode, provider: string): string {
switch (code) {
case 'auth':
return `${provider} rejected the API key.`;
case 'rate_limit':
return `${provider} is rate limiting requests. Try again in a moment.`;
case 'quota':
return `${provider} reports that the account is out of quota or credit.`;
case 'context_length':
return `The conversation is too long for this ${provider} model.`;
case 'not_found':
return `${provider} could not find that model or endpoint.`;
case 'content_filter':
return `${provider} blocked the request with its content filter.`;
case 'server':
return `${provider} is having trouble right now. Try again shortly.`;
case 'timeout':
return `${provider} took too long to respond.`;
case 'bad_request':
return `${provider} rejected the request.`;
default:
return `${provider} returned an error.`;
}
}
/**
* The single place that maps "HTTP status + provider error body" to an AIError.
* Pass `status: undefined` for errors that arrive as events inside a stream.
*/
export function errorFromProvider(ctx: ErrorContext, status: number | undefined, body: unknown): AIError {
const fields = extractProviderError(body);
const code = classify(status, fields);
const detail = fields.message || (status ? `HTTP ${status}` : '');
return new AIError(code, friendlyMessage(code, ctx.provider), {
status,
providerMessage: detail ? clip(scrubSecrets(detail, ctx.secrets)) : undefined,
});
}
/** Maps anything thrown by `fetch` / a body read to an AIError. */
export function errorFromThrown(ctx: ErrorContext, err: unknown, abortReason?: 'timeout' | 'aborted' | null): AIError {
if (err instanceof AIError) return err;
const name = isRecord(err) && typeof err.name === 'string' ? err.name : '';
if (abortReason === 'timeout' || name === 'TimeoutError') {
return new AIError('timeout', friendlyMessage('timeout', ctx.provider), { cause: err });
}
if (abortReason === 'aborted' || name === 'AbortError') {
return new AIError('aborted', 'The request was cancelled.', { cause: err });
}
// undici reports DNS/connection failures as TypeError("fetch failed") with the real reason in `cause`.
const cause = isRecord(err) && isRecord(err.cause) ? err.cause : undefined;
const parts = [
cause && typeof cause.code === 'string' ? cause.code : '',
cause && typeof cause.message === 'string' ? cause.message : '',
err instanceof Error ? err.message : String(err),
].filter(Boolean);
return new AIError('network', `Could not reach ${ctx.provider}. Check the address and your connection.`, {
providerMessage: clip(scrubSecrets(parts.join(': '), ctx.secrets)),
cause: err,
});
}
/* ------------------------------------------------------------------ */
/* fetch with timeouts */
/* ------------------------------------------------------------------ */
export interface HttpOptions extends ErrorContext {
method?: 'GET' | 'POST';
headers?: Record<string, string>;
/** Serialised as JSON when present. */
body?: unknown;
signal?: AbortSignal;
/** Time allowed until response headers arrive (default 60s). */
firstByteTimeoutMs?: number;
/** Hard cap on the whole exchange (default 30 min — streams are long). */
totalTimeoutMs?: number;
}
export interface HttpResponse {
response: Response;
ctx: ErrorContext;
/** Abort the exchange from our side (used by the idle watchdog). */
abort(reason: 'timeout' | 'aborted'): void;
abortReason(): 'timeout' | 'aborted' | null;
/** Clears timers/listeners. Safe to call more than once. */
cleanup(): void;
}
export const DEFAULT_FIRST_BYTE_TIMEOUT_MS = 60_000;
export const DEFAULT_TOTAL_TIMEOUT_MS = 30 * 60_000;
/** Longest silence tolerated inside an open stream before we give up. */
export const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60_000;
/**
* Sends a request and resolves once headers are in. Does NOT check `response.ok`
* (callers decide; see `ensureOk`). Throws AIError for abort/timeout/network.
*/
export async function httpRequest(url: string, opts: HttpOptions): Promise<HttpResponse> {
const ctx: ErrorContext = { provider: opts.provider, secrets: opts.secrets };
if (opts.signal?.aborted) throw new AIError('aborted', 'The request was cancelled.');
const controller = new AbortController();
let reason: 'timeout' | 'aborted' | null = null;
const abort = (r: 'timeout' | 'aborted') => {
if (reason) return;
reason = r;
controller.abort();
};
const onUserAbort = () => abort('aborted');
opts.signal?.addEventListener('abort', onUserAbort, { once: true });
const firstByte = setTimeout(() => abort('timeout'), opts.firstByteTimeoutMs ?? DEFAULT_FIRST_BYTE_TIMEOUT_MS);
const total = setTimeout(() => abort('timeout'), opts.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS);
let cleaned = false;
const cleanup = () => {
if (cleaned) return;
cleaned = true;
clearTimeout(firstByte);
clearTimeout(total);
opts.signal?.removeEventListener('abort', onUserAbort);
};
const headers: Record<string, string> = { ...opts.headers };
let body: string | undefined;
if (opts.body !== undefined) {
body = JSON.stringify(opts.body);
if (!hasHeader(headers, 'content-type')) headers['Content-Type'] = 'application/json';
}
try {
const response = await fetch(url, {
method: opts.method ?? (body === undefined ? 'GET' : 'POST'),
headers,
body,
signal: controller.signal,
});
clearTimeout(firstByte);
return { response, ctx, abort, abortReason: () => reason, cleanup };
} catch (err) {
cleanup();
throw errorFromThrown(ctx, err, reason);
}
}
/** Reads the body of a failed response and throws the mapped AIError. Returns normally when `ok`. */
export async function ensureOk(http: HttpResponse): Promise<void> {
if (http.response.ok) return;
let text = '';
try {
text = await http.response.text();
} catch {
/* body unreadable — the status code alone still maps to something useful */
} finally {
http.cleanup();
}
throw errorFromProvider(http.ctx, http.response.status, text);
}
/** Request + status check + JSON parse, for non-streaming calls such as model discovery. */
export async function requestJson<T = unknown>(url: string, opts: HttpOptions): Promise<T> {
const http = await httpRequest(url, opts);
await ensureOk(http);
try {
const text = await http.response.text();
try {
return JSON.parse(text) as T;
} catch {
throw new AIError('invalid_response', `${opts.provider} sent a response that was not valid JSON.`, {
providerMessage: clip(scrubSecrets(text, opts.secrets), 300),
});
}
} catch (err) {
throw errorFromThrown(http.ctx, err, http.abortReason());
} finally {
http.cleanup();
}
}
/* ------------------------------------------------------------------ */
/* Server-sent events */
/* ------------------------------------------------------------------ */
export interface SSEMessage {
/** Value of the `event:` field, when the server sent one. */
event?: string;
/** All `data:` lines of the event joined with "\n". */
data: string;
}
export interface SSEOptions {
/** Abort when no bytes arrive for this long. */
idleTimeoutMs?: number;
onIdle?: () => void;
}
/**
* Parses a text/event-stream body. Handles `\n`, `\r\n` and bare `\r` line
* endings (also when split across chunks), multi-line `data:`, comment lines,
* a UTF-8 BOM, and a final event that is not followed by a blank line.
* Iteration ends at end-of-stream or at the OpenAI-style `data: [DONE]` sentinel.
*/
export async function* parseSSE(stream: ReadableStream<Uint8Array>, opts: SSEOptions = {}): AsyncGenerator<SSEMessage> {
const reader = stream.getReader();
const decoder = new TextDecoder('utf-8');
let buffer = '';
let dataLines: string[] = [];
let eventName: string | undefined;
let sawCR = false; // previous chunk ended in "\r": swallow a leading "\n"
let first = true;
const takeLine = (line: string): SSEMessage | 'done' | null => {
if (line === '') {
// Blank line dispatches the event.
if (dataLines.length === 0) {
eventName = undefined;
return null;
}
const msg: SSEMessage = { data: dataLines.join('\n') };
if (eventName !== undefined) msg.event = eventName;
dataLines = [];
eventName = undefined;
return msg.data === '[DONE]' ? 'done' : msg;
}
if (line.startsWith(':')) return null; // comment / keep-alive
const colon = line.indexOf(':');
const field = colon === -1 ? line : line.slice(0, colon);
let value = colon === -1 ? '' : line.slice(colon + 1);
if (value.startsWith(' ')) value = value.slice(1);
if (field === 'data') dataLines.push(value);
else if (field === 'event') eventName = value;
// `id` and `retry` are irrelevant for one-shot completions.
return null;
};
try {
for (;;) {
const { done, value } = await readWithIdle(reader, opts);
if (done) break;
let chunk = decoder.decode(value, { stream: true });
if (first && chunk.length > 0) {
first = false;
if (chunk.charCodeAt(0) === 0xfeff) chunk = chunk.slice(1);
}
if (sawCR && chunk.startsWith('\n')) chunk = chunk.slice(1);
if (chunk.length > 0) sawCR = chunk.endsWith('\r');
buffer += chunk;
let start = 0;
for (;;) {
const idx = indexOfLineEnd(buffer, start);
if (idx === -1) break;
const line = buffer.slice(start, idx);
start = buffer[idx] === '\r' && buffer[idx + 1] === '\n' ? idx + 2 : idx + 1;
const out = takeLine(line);
if (out === 'done') return;
if (out) yield out;
}
buffer = buffer.slice(start);
}
// Flush: tolerate a last line / last event without the terminating newline(s).
buffer += decoder.decode();
if (buffer.length > 0) {
const out = takeLine(buffer);
if (out === 'done') return;
if (out) yield out;
}
const tail = takeLine('');
if (tail && tail !== 'done') yield tail;
} finally {
try {
await reader.cancel();
} catch {
/* already closed or errored */
}
}
}
function indexOfLineEnd(s: string, from: number): number {
for (let i = from; i < s.length; i++) {
const c = s.charCodeAt(i);
if (c === 10 || c === 13) return i;
}
return -1;
}
async function readWithIdle(
reader: ReadableStreamDefaultReader<Uint8Array>,
opts: SSEOptions,
): Promise<Awaited<ReturnType<ReadableStreamDefaultReader<Uint8Array>['read']>>> {
if (!opts.idleTimeoutMs) return reader.read();
let timer: ReturnType<typeof setTimeout> | undefined;
const idle = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
opts.onIdle?.();
const err = new Error('stream idle timeout');
err.name = 'TimeoutError';
reject(err);
}, opts.idleTimeoutMs);
});
try {
return await Promise.race([reader.read(), idle]);
} finally {
clearTimeout(timer);
}
}
/**
* Iterates the SSE events of an open response, translating read failures
* (dropped connection, abort, idle timeout) into AIErrors and always releasing
* the request's timers. `data` is JSON-parsed; unparseable events are skipped.
*/
export async function* sseJson(http: HttpResponse, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS): AsyncGenerator<{ event?: string; data: unknown }> {
const body = http.response.body;
if (!body) {
http.cleanup();
throw new AIError('invalid_response', `${http.ctx.provider} returned an empty response.`);
}
const events = parseSSE(body, { idleTimeoutMs, onIdle: () => http.abort('timeout') });
try {
for (;;) {
let next: IteratorResult<SSEMessage>;
try {
next = await events.next();
} catch (err) {
throw errorFromThrown(http.ctx, err, http.abortReason());
}
if (next.done) return;
let data: unknown;
try {
data = JSON.parse(next.value.data);
} catch {
continue; // keep-alives such as OpenRouter's ": OPENROUTER PROCESSING" variants, or junk
}
yield next.value.event !== undefined ? { event: next.value.event, data } : { data };
}
} finally {
await events.return(undefined).catch(() => undefined);
http.cleanup();
}
}
/* ------------------------------------------------------------------ */
/* Small shared utilities */
/* ------------------------------------------------------------------ */
export function isRecord(v: unknown): v is Record<string, unknown> {
return typeof v === 'object' && v !== null && !Array.isArray(v);
}
function hasHeader(headers: Record<string, string>, name: string): boolean {
const n = name.toLowerCase();
return Object.keys(headers).some((k) => k.toLowerCase() === n);
}
function safeStringify(v: unknown): string {
try {
return JSON.stringify(v) ?? '';
} catch {
return '';
}
}
/** Trims whitespace and trailing slashes from a base URL. */
export function trimBaseUrl(url: string): string {
return url.trim().replace(/\/+$/, '');
}
/** Host part of a URL for user-facing messages; never includes credentials or query. */
export function hostOf(url: string): string {
try {
return new URL(url).host;
} catch {
return url;
}
}
/** True for loopback / private-network hosts, where a local model server may need minutes to load a model. */
export function isLocalUrl(url: string): boolean {
try {
const h = new URL(url).hostname.replace(/^\[|\]$/g, '');
return (
h === 'localhost' ||
h === '::1' ||
h.endsWith('.local') ||
h.endsWith('.internal') ||
/^127\./.test(h) ||
/^10\./.test(h) ||
/^192\.168\./.test(h) ||
/^172\.(1[6-9]|2\d|3[01])\./.test(h) ||
h === 'host.docker.internal'
);
} catch {
return false;
}
}

View file

@ -0,0 +1,26 @@
import { AIError, type ProviderAdapter, type ProviderKind } from '../types.ts';
import { anthropicAdapter } from './anthropic.ts';
import { geminiAdapter } from './gemini.ts';
import { ollamaAdapter } from './ollama.ts';
import { openaiAdapter } from './openai.ts';
import { openaiCompatibleAdapter } from './openaiCompatible.ts';
const ADAPTERS: Record<ProviderKind, ProviderAdapter> = {
openai: openaiAdapter,
anthropic: anthropicAdapter,
gemini: geminiAdapter,
openai_compatible: openaiCompatibleAdapter,
ollama: ollamaAdapter,
};
export function getAdapter(kind: ProviderKind): ProviderAdapter {
const adapter = ADAPTERS[kind];
if (!adapter) throw new AIError('unsupported', `Unknown provider type "${kind}".`);
return adapter;
}
export function listAdapterMeta(): Array<{ kind: ProviderKind } & ProviderAdapter['meta']> {
return (Object.keys(ADAPTERS) as ProviderKind[]).map((kind) => ({ kind, ...ADAPTERS[kind].meta }));
}
export { CATALOG_UPDATED, lookupModelDefaults } from './catalog.ts';

View file

@ -0,0 +1,102 @@
/**
* Ollama (local models). Chat goes through Ollama's OpenAI-compatible endpoint
* (`{base}/v1/chat/completions`), which handles tools and streaming; discovery
* uses the native API because it reports what each model can do:
* GET /api/tags -> installed models
* POST /api/show -> { capabilities: ["completion","tools","vision","thinking"], model_info }
* https://docs.ollama.com/api · https://docs.ollama.com/openai
*/
import { AIError, type DiscoveredModel, type ModelCapabilities, type ProviderAdapter, type ProviderConfig, type TestResult } from '../types.ts';
import { num, runTest, str, TEST_TIMEOUT_MS } from './common.ts';
import { isRecord, requestJson, trimBaseUrl, type ErrorContext } from './http.ts';
import { streamChatCompletions, type CompatOptions } from './openaiCompatible.ts';
const LABEL = 'Ollama';
const DEFAULT_BASE_URL = 'http://localhost:11434';
function root(cfg: ProviderConfig): string {
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/(v1|api)$/, '');
}
function ctx(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig): Record<string, string> {
const h: Record<string, string> = { ...cfg.headers };
if (cfg.apiKey) h.Authorization = `Bearer ${cfg.apiKey}`; // only needed behind an authenticating proxy
return h;
}
function compat(cfg: ProviderConfig): CompatOptions {
return { kind: 'ollama', label: LABEL, baseUrl: `${root(cfg)}/v1`, apiKey: cfg.apiKey, headers: cfg.headers };
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
let tags: unknown;
try {
tags = await requestJson(`${root(cfg)}/api/tags`, { ...ctx(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs ?? 15_000, totalTimeoutMs: timeoutMs ?? 30_000 });
} catch (err) {
if (err instanceof AIError && err.code === 'network') {
throw new AIError('network', `Could not reach Ollama at ${root(cfg)}. Is it running, and reachable from the Study Desk container?`, { retryable: true });
}
throw err;
}
const list = isRecord(tags) && Array.isArray(tags.models) ? tags.models.filter(isRecord) : [];
const models: DiscoveredModel[] = [];
// /api/show is one cheap local call per model; cap it so a huge library cannot stall the settings page.
await Promise.all(
list.slice(0, 60).map(async (m) => {
const id = str(m.model) || str(m.name);
if (!id) return;
const model: DiscoveredModel = { id, label: str(m.name) || id, priceIn: 0, priceOut: 0 };
try {
const show = await requestJson<Record<string, unknown>>(`${root(cfg)}/api/show`, {
...ctx(cfg),
headers: headers(cfg),
body: { model: id },
signal,
firstByteTimeoutMs: 10_000,
totalTimeoutMs: 10_000,
});
if (Array.isArray(show.capabilities)) {
const caps = show.capabilities.map(String);
if (!caps.includes('completion')) return; // embedding-only model
const c: Partial<ModelCapabilities> = { tools: caps.includes('tools'), vision: caps.includes('vision'), reasoning: caps.includes('thinking'), pdf: false, temperature: true };
model.capabilities = c;
}
if (isRecord(show.model_info)) {
const key = Object.keys(show.model_info).find((k) => k.endsWith('.context_length'));
if (key && num(show.model_info[key])) model.contextWindow = num(show.model_info[key]);
}
} catch {
/* older Ollama versions: keep the model with unknown capabilities */
}
models.push(model);
}),
);
return models.sort((a, b) => a.id.localeCompare(b.id));
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
return runTest(LABEL, async () => {
const count = (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length;
return count === 0 ? { modelCount: 0, note: 'Connected to Ollama, but no models are installed. Run `ollama pull <model>` first.' } : { modelCount: count };
});
}
export const ollamaAdapter: ProviderAdapter = {
kind: 'ollama',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: false,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://docs.ollama.com',
keyHint: 'Not needed',
},
stream: (cfg, req) => streamChatCompletions(compat(cfg), req),
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

View file

@ -0,0 +1,313 @@
/**
* OpenAI — Responses API (`POST /v1/responses`, SSE).
* https://platform.openai.com/docs/api-reference/responses
* https://platform.openai.com/docs/guides/function-calling
* https://platform.openai.com/docs/guides/reasoning
*
* Quirks handled here:
* - Requests are stateless (`store: false`). Reasoning models need their reasoning
* items passed back between tool calls, so encrypted reasoning is requested via
* `include: ["reasoning.encrypted_content"]` and every turn's raw output items are
* kept in `providerData` and replayed verbatim.
* - Reasoning models reject `temperature`; classic models reject `reasoning`.
* - The accepted `reasoning.effort` values differ by model generation. When a model
* rejects the chosen value the request is retried once with a safe one.
* - Tool-call arguments arrive as deltas keyed by output item.
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { lookupModelDefaults } from './catalog.ts';
import { dataUrl, isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
type Json = Record<string, unknown>;
const LABEL = 'OpenAI';
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
function base(cfg: ProviderConfig): string {
const url = trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL);
return /\/v\d+$/.test(url) ? url : `${url}/v1`;
}
function errorContext(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig): Record<string, string> {
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an OpenAI API key in Settings to use OpenAI models.');
return { ...cfg.headers, Authorization: `Bearer ${cfg.apiKey}` };
}
/** Whether a model is a reasoning model, when the caller did not say. */
function isReasoningModel(req: ChatRequest): boolean {
if (req.capabilities?.reasoning !== undefined) return req.capabilities.reasoning;
const known = lookupModelDefaults('openai', req.model).capabilities?.reasoning;
if (known !== undefined) return known;
return /^(o\d|gpt-[5-9]|gpt-\d{2})/.test(req.model) && !/chat/.test(req.model);
}
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
const out: Json[] = [];
for (const p of parts) {
if (p.type === 'text' && p.text) out.push({ type: 'input_text', text: p.text });
else if (p.type === 'image') {
if (caps.vision === false) out.push({ type: 'input_text', text: '[An image was attached, but this model cannot read images.]' });
else out.push({ type: 'input_image', image_url: dataUrl(p.mime, p.data) });
} else if (p.type === 'file') {
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ type: 'input_file', filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) });
else if (p.mime.startsWith('text/')) out.push({ type: 'input_text', text: Buffer.from(p.data, 'base64').toString('utf8') });
else out.push({ type: 'input_text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
}
return out;
}
function toInput(messages: ChatMessage[], caps: Partial<ModelCapabilities>, dropReasoning: boolean): Json[] {
const input: Json[] = [];
for (const msg of messages) {
if (msg.role === 'assistant') {
const pd = msg.providerData;
if (pd?.kind === 'openai' && isRecord(pd.data) && Array.isArray(pd.data.output)) {
// Same provider: replay the output items exactly (reasoning items included).
for (const item of pd.data.output.filter(isRecord)) {
if (item.type === 'reasoning' && (dropReasoning || !item.encrypted_content)) continue;
input.push(item);
}
continue;
}
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
if (text) input.push({ role: 'assistant', content: [{ type: 'output_text', text }] });
for (const p of msg.content) {
if (p.type === 'tool_call') input.push({ type: 'function_call', call_id: p.id, name: p.name, arguments: JSON.stringify(p.input ?? {}) });
}
continue;
}
for (const p of msg.content) {
if (p.type === 'tool_result') input.push({ type: 'function_call_output', call_id: p.toolCallId, output: p.output || '(no output)' });
}
const content = userContent(msg.content.filter((p) => p.type !== 'tool_result'), caps);
if (content.length > 0) input.push({ role: 'user', content });
}
return input;
}
interface Flags {
/** Value sent as reasoning.effort, or null to omit the field. */
effort: string | null;
summary: boolean;
temperature: boolean;
dropReasoning: boolean;
include: boolean;
}
function buildBody(req: ChatRequest, flags: Flags): Json {
const caps = req.capabilities ?? {};
const body: Json = { model: req.model, input: toInput(req.messages, caps, flags.dropReasoning), stream: true, store: false };
if (req.system?.trim()) body.instructions = req.system.trim();
if (req.maxOutputTokens) body.max_output_tokens = Math.max(16, req.maxOutputTokens);
if (caps.tools !== false && req.tools?.length) {
body.tools = req.tools.map((t) => ({ type: 'function', name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters }, strict: false }));
const choice = req.toolChoice;
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
else if (isRecord(choice)) body.tool_choice = { type: 'function', name: choice.name };
}
if (isReasoningModel(req)) {
const reasoning: Json = {};
if (flags.effort) reasoning.effort = flags.effort;
if (flags.summary) reasoning.summary = 'auto';
if (Object.keys(reasoning).length > 0) body.reasoning = reasoning;
if (flags.include) body.include = ['reasoning.encrypted_content'];
} else if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) {
body.temperature = req.temperature;
}
if (req.jsonMode) body.text = { format: { type: 'json_object' } };
return body;
}
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(cfg);
const flags: Flags = {
// "none" is only understood by recent models; if it is rejected we fall back to "low".
effort: req.reasoningEffort ?? null,
summary: true,
temperature: true,
dropReasoning: false,
include: true,
};
let http: HttpResponse;
for (let attempt = 0; ; attempt += 1) {
http = await httpRequest(`${base(cfg)}/responses`, { ...ctx, headers: headers(cfg), body: buildBody(req, flags), signal: req.signal });
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 5) throw err;
const detail = (err.providerMessage ?? '').toLowerCase();
if (flags.effort && /reasoning\.effort|reasoning_effort|effort/.test(detail)) flags.effort = flags.effort === 'none' ? 'minimal' : flags.effort === 'minimal' ? 'low' : null;
else if (flags.summary && /summary/.test(detail)) flags.summary = false;
else if (flags.include && /include|encrypted_content/.test(detail)) flags.include = false;
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
else if (!flags.dropReasoning && /reasoning/.test(detail) && /item|input/.test(detail)) flags.dropReasoning = true;
else throw err;
}
}
const args = new Map<string, string>();
let final: Json | null = null;
let reasoningEmitted = false;
let streamedText = '';
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
switch (data.type) {
case 'response.output_text.delta': {
const delta = str(data.delta);
if (delta) {
streamedText += delta;
yield { type: 'text', delta };
}
break;
}
case 'response.refusal.delta': {
const delta = str(data.delta);
if (delta) yield { type: 'text', delta };
break;
}
case 'response.reasoning_summary_part.added':
if (reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
break;
case 'response.reasoning_summary_text.delta':
case 'response.reasoning_text.delta': {
const delta = str(data.delta);
if (delta) {
reasoningEmitted = true;
yield { type: 'reasoning', delta };
}
break;
}
case 'response.function_call_arguments.delta':
args.set(str(data.item_id), (args.get(str(data.item_id)) ?? '') + str(data.delta));
break;
case 'response.completed':
case 'response.incomplete':
if (isRecord(data.response)) final = data.response;
break;
case 'response.failed':
throw errorFromProvider(ctx, undefined, isRecord(data.response) && isRecord(data.response.error) ? { error: data.response.error } : data);
case 'error':
throw errorFromProvider(ctx, undefined, isRecord(data.error) ? data : { error: data });
default:
break;
}
}
if (!final) throw new AIError('network', 'The connection to OpenAI closed before the response finished.', { retryable: true });
const incomplete = isRecord(final.incomplete_details) ? str(final.incomplete_details.reason) : '';
const truncated = incomplete === 'max_output_tokens';
const output = Array.isArray(final.output) ? final.output.filter(isRecord) : [];
const content: ContentPart[] = [];
const replay: Json[] = [];
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
let refused = false;
let text = '';
for (const item of output) {
if (item.type === 'message') {
for (const part of Array.isArray(item.content) ? item.content.filter(isRecord) : []) {
if (part.type === 'output_text') text += str(part.text);
else if (part.type === 'refusal') {
refused = true;
text += str(part.refusal);
}
}
replay.push(item);
} else if (item.type === 'function_call') {
const name = str(item.name);
const raw = str(item.arguments) || args.get(str(item.id)) || '';
let input = parseArgsStrict(raw);
if (!input) {
if (truncated) continue;
input = parseToolArguments(raw, LABEL, name);
}
calls.push({ id: str(item.call_id) || str(item.id), name, input });
replay.push(item);
} else if (item.type === 'reasoning') {
replay.push(item);
}
}
if (!text && streamedText) text = streamedText;
if (text) content.push({ type: 'text', text });
for (const call of calls) {
content.push({ type: 'tool_call', ...call });
yield { type: 'tool_call', call };
}
let stopReason: StopReason = 'end';
if (calls.length > 0) stopReason = 'tool_use';
else if (refused || incomplete === 'content_filter') stopReason = 'refusal';
else if (truncated) stopReason = 'max_tokens';
else if (incomplete) stopReason = 'other';
const u = isRecord(final.usage) ? final.usage : {};
const message: ChatMessage = { role: 'assistant', content };
if (replay.length > 0) message.providerData = { kind: 'openai', data: { output: replay } };
yield {
type: 'done',
message,
stopReason,
usage: makeUsage({
input: num(u.input_tokens),
output: num(u.output_tokens),
cacheRead: isRecord(u.input_tokens_details) ? num(u.input_tokens_details.cached_tokens) : 0,
reasoning: isRecord(u.output_tokens_details) ? num(u.output_tokens_details.reasoning_tokens) : 0,
}),
};
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const res = await requestJson<Json>(`${base(cfg)}/models`, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
const found: DiscoveredModel[] = [];
for (const m of data) {
const id = str(m.id);
// Fine-tunes ("ft:...") are chat models too; search/codex-only variants are not useful here.
if (!id || !isChatModelId(id) || /search|^codex-|-codex-max/.test(id)) continue;
if (!/^(gpt|o\d|chat|ft:)/.test(id)) continue;
found.push({ id });
}
found.sort((a, b) => b.id.localeCompare(a.id, undefined, { numeric: true }));
return withCatalogDefaults('openai', found);
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
}
export const openaiAdapter: ProviderAdapter = {
kind: 'openai',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: true,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://platform.openai.com/api-keys',
keyHint: 'sk-…',
},
stream,
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

View file

@ -0,0 +1,363 @@
/**
* Generic Chat Completions (`POST {base}/chat/completions`) for OpenRouter, Groq,
* Together, LM Studio, vLLM, llama.cpp and friends, and the engine behind the
* Ollama adapter.
*
* These servers agree on the basics and disagree on everything else, so this
* adapter is deliberately forgiving:
* - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs
* `max_completion_tokens` and `reasoning_effort` are each dropped and retried
* once when a server answers 400 about them;
* - usage may be missing, and may arrive on a chunk with no choices;
* - tool-call argument fragments may arrive in any chunking, with or without ids;
* - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`;
* - the last chunk may lack a `finish_reason`.
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type ProviderKind,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
type Json = Record<string, unknown>;
/** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */
export function normalizeCompatBase(url: string): string {
let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, '');
try {
const u = new URL(base);
// A bare host ("http://localhost:1234") almost always means the conventional /v1 root.
if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`;
} catch {
/* validated by the caller */
}
return base;
}
export interface CompatOptions {
kind: ProviderKind;
label: string;
baseUrl: string;
apiKey: string | null;
headers?: Record<string, string>;
}
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): string | Json[] {
const blocks: Json[] = [];
for (const p of parts) {
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
else if (p.type === 'image') {
if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } });
} else if (p.type === 'file') {
if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') });
else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
}
// Plain strings are understood by every server; arrays only by multimodal ones.
if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n');
return blocks;
}
export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system'): Json[] {
const caps = req.capabilities ?? {};
const out: Json[] = [];
if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() });
for (const msg of req.messages) {
if (msg.role === 'assistant') {
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract<ContentPart, { type: 'tool_call' }>[];
if (!text && calls.length === 0) continue;
const wire: Json = { role: 'assistant', content: text || null };
if (calls.length > 0) {
wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } }));
}
out.push(wire);
continue;
}
// Tool results must directly follow the assistant turn that requested them.
for (const p of msg.content) {
if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' });
}
const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call');
if (rest.length > 0) {
const content = userContent(rest, caps);
if (typeof content !== 'string' || content) out.push({ role: 'user', content });
}
}
return out;
}
interface BodyFlags {
tools: boolean;
usage: boolean;
jsonMode: boolean;
temperature: boolean;
reasoning: boolean;
maxKey: 'max_tokens' | 'max_completion_tokens' | null;
}
function buildBody(req: ChatRequest, flags: BodyFlags): Json {
const caps = req.capabilities ?? {};
const body: Json = { model: req.model, messages: toChatMessages(req), stream: true };
if (flags.usage) body.stream_options = { include_usage: true };
if (flags.tools && caps.tools !== false && req.tools?.length) {
body.tools = req.tools.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } }));
const choice = req.toolChoice;
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
else if (isRecord(choice)) body.tool_choice = { type: 'function', function: { name: choice.name } };
}
if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature;
if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens;
if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = req.reasoningEffort;
if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' };
return body;
}
function mapFinish(reason: string): StopReason {
switch (reason) {
case 'stop':
case 'eos':
case 'end_turn':
return 'end';
case 'tool_calls':
case 'function_call':
return 'tool_use';
case 'length':
case 'max_tokens':
return 'max_tokens';
case 'content_filter':
return 'refusal';
default:
return 'other';
}
}
function authHeaders(o: CompatOptions): Record<string, string> {
const h: Record<string, string> = { ...o.headers };
if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`;
return h;
}
function errorContext(o: CompatOptions): ErrorContext {
return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] };
}
export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(o);
const base = normalizeCompatBase(o.baseUrl);
const local = isLocalUrl(base);
const flags: BodyFlags = { tools: true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' };
let http: HttpResponse;
for (let attempt = 0; ; attempt += 1) {
http = await httpRequest(`${base}/chat/completions`, {
...ctx,
headers: authHeaders(o),
body: buildBody(req, flags),
signal: req.signal,
// A local server may spend minutes loading a model into memory before the first byte.
firstByteTimeoutMs: local ? 10 * 60_000 : undefined,
});
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err;
const detail = (err.providerMessage ?? '').toLowerCase();
if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false;
else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens';
else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null;
else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false;
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false;
else if (flags.tools && req.tools?.length && /tool|function/.test(detail)) {
// The model (or server) cannot do tool calling at all.
if (req.toolChoice === 'required' || isRecord(req.toolChoice)) {
throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, {
status: err.status,
providerMessage: err.providerMessage,
});
}
flags.tools = false;
} else throw err;
}
}
let text = '';
let finish = '';
let reasoningSeen = false;
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
const calls: { id: string; name: string; args: string }[] = [];
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
if (isRecord(data.usage)) {
const u = data.usage;
usage.input = num(u.prompt_tokens) || usage.input;
usage.output = num(u.completion_tokens) || usage.output;
if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens);
if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens);
}
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
if (!isRecord(choice)) continue;
if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason;
const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null;
if (!delta) continue;
const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking);
if (reasoning) {
reasoningSeen = true;
yield { type: 'reasoning', delta: reasoning };
}
const piece = typeof delta.content === 'string' ? delta.content : '';
if (piece) {
text += piece;
yield { type: 'text', delta: piece };
}
if (Array.isArray(delta.tool_calls)) {
for (const raw of delta.tool_calls) {
if (!isRecord(raw)) continue;
const fn = isRecord(raw.function) ? raw.function : {};
// `index` identifies the call; servers that omit it send one whole call per entry.
const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1;
let slot = index >= 0 ? calls[index] : undefined;
if (!slot) {
slot = { id: '', name: '', args: '' };
if (index >= 0) calls[index] = slot;
else calls.push(slot);
}
if (str(raw.id)) slot.id = str(raw.id);
if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name);
if (typeof fn.arguments === 'string') slot.args += fn.arguments;
else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments);
}
}
}
const stop = finish ? mapFinish(finish) : 'end';
const content: ContentPart[] = [];
if (text) content.push({ type: 'text', text });
const done: { id: string; name: string; input: Record<string, unknown> }[] = [];
for (const c of calls) {
if (!c || !c.name) continue;
let input = parseArgsStrict(c.args);
if (!input) {
if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest
input = parseToolArguments(c.args, o.label, c.name);
}
done.push({ id: c.id || newToolCallId('call'), name: c.name, input });
}
for (const call of done) {
content.push({ type: 'tool_call', ...call });
yield { type: 'tool_call', call };
}
if (content.length === 0 && !reasoningSeen && !finish) {
throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true });
}
const message: ChatMessage = { role: 'assistant', content };
yield {
type: 'done',
message,
stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop,
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
};
}
export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const base = normalizeCompatBase(o.baseUrl);
let res: unknown;
try {
res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
} catch (err) {
if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) {
throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status });
}
throw err;
}
const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : [];
const found: DiscoveredModel[] = [];
for (const m of list) {
if (!isRecord(m)) continue;
const id = str(m.id) || str(m.name);
if (!id || !isChatModelId(id)) continue;
const model: DiscoveredModel = { id };
if (str(m.name) && str(m.name) !== id) model.label = str(m.name);
// OpenRouter-style metadata, when present.
const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length);
if (ctxLen) model.contextWindow = ctxLen;
if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens);
if (isRecord(m.pricing)) {
const pin = Number(m.pricing.prompt);
const pout = Number(m.pricing.completion);
if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3;
if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3;
}
const caps: Partial<ModelCapabilities> = {};
if (Array.isArray(m.supported_parameters)) {
caps.tools = m.supported_parameters.includes('tools');
caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort');
}
if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) {
caps.vision = m.architecture.input_modalities.includes('image');
}
if (Object.keys(caps).length > 0) model.capabilities = caps;
found.push(model);
}
return withCatalogDefaults(o.kind, found);
}
function options(cfg: ProviderConfig): CompatOptions {
if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`);
return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers };
}
async function test(cfg: ProviderConfig, model?: string): Promise<TestResult> {
return runTest(cfg.name || 'the server', async () => {
const o = options(cfg);
try {
return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length };
} catch (err) {
if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err;
// No model listing: prove the connection with a one-token completion instead.
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
try {
for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) {
if (ev.type === 'done') break;
}
} finally {
clearTimeout(timer);
}
return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` };
}
});
}
export const openaiCompatibleAdapter: ProviderAdapter = {
kind: 'openai_compatible',
meta: {
label: 'OpenAI-compatible',
defaultBaseUrl: null,
requiresApiKey: false,
requiresBaseUrl: true,
supportsDiscovery: true,
docsUrl: 'https://openrouter.ai/docs/quickstart',
keyHint: 'Optional for local servers',
},
stream: (cfg, req) => streamChatCompletions(options(cfg), req),
listModels: (cfg, signal) => listCompatModels(options(cfg), signal),
test,
};

View file

@ -0,0 +1,932 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/**
* Provider adapter tests. `fetch` is always stubbed; no network is touched.
* Wire shapes below are derived from what each adapter's parser reads.
*/
import { afterEach, describe, expect, it, vi } from 'vitest';
import { AIError, type ChatMessage, type ChatRequest, type ProviderAdapter, type ProviderConfig, type ProviderKind, type StreamEvent, type ToolDef, type Usage } from '../types.ts';
import { anthropicAdapter } from './anthropic.ts';
import { lookupModelDefaults } from './catalog.ts';
import { toGeminiSchema, geminiAdapter } from './gemini.ts';
import { parseSSE } from './http.ts';
import { ollamaAdapter } from './ollama.ts';
import { openaiAdapter } from './openai.ts';
import { normalizeCompatBase, openaiCompatibleAdapter } from './openaiCompatible.ts';
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
const enc = new TextEncoder();
function bytesStream(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
return new ReadableStream({
start(c) {
for (const ch of chunks) c.enqueue(ch);
c.close();
},
});
}
const textStream = (chunks: string[]) => bytesStream(chunks.map((c) => enc.encode(c)));
/** One SSE event. */
function ev(data: unknown, event?: string): string {
return `${event ? `event: ${event}\n` : ''}data: ${JSON.stringify(data)}\n\n`;
}
const DONE = 'data: [DONE]\n\n';
function rechunk(s: string, n: number): string[] {
const out: string[] = [];
for (let i = 0; i < s.length; i += n) out.push(s.slice(i, i + n));
return out;
}
/** SSE response whose chunk boundaries deliberately fall mid-line. */
function sseResponse(chunks: string[], status = 200): Response {
return new Response(textStream(rechunk(chunks.join(''), 23)), { status, headers: { 'content-type': 'text/event-stream' } });
}
function jsonResponse(body: unknown, status = 200): Response {
return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
}
interface Call {
url: string;
method?: string;
headers: Record<string, string>;
body: any;
}
let calls: Call[] = [];
type Reply = Response | Error;
type Handler = (call: Call, n: number) => Reply | Promise<Reply>;
function stubFetch(handler: Handler): void {
calls = [];
vi.stubGlobal(
'fetch',
vi.fn(async (input: unknown, init?: RequestInit) => {
const call: Call = {
url: String(input),
method: init?.method,
headers: (init?.headers ?? {}) as Record<string, string>,
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined,
};
calls.push(call);
const r = await handler(call, calls.length - 1);
if (r instanceof Error) throw r;
return r;
}),
);
}
/** Replies with the given thunks in order (the last one repeats). */
function queue(...items: Array<() => Reply>): Handler {
return (_c, n) => items[Math.min(n, items.length - 1)]!();
}
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
async function drain<T>(it: AsyncIterable<T>): Promise<T[]> {
const out: T[] = [];
for await (const x of it) out.push(x);
return out;
}
async function run(adapter: ProviderAdapter, cfg: ProviderConfig, req: ChatRequest) {
const events = await drain(adapter.stream(cfg, req));
const done = events[events.length - 1];
if (!done || done.type !== 'done') throw new Error('stream did not end with a done event');
return { events, done };
}
async function failure(p: Promise<unknown>): Promise<AIError> {
const err = await p.then(
() => undefined,
(e: unknown) => e,
);
expect(err).toBeInstanceOf(AIError);
return err as AIError;
}
const TOOL: ToolDef = {
name: 'lookup',
description: 'Look things up',
parameters: { type: 'object', properties: { q: { type: 'string' } }, required: ['q'] },
};
const FRAGS = ['{"q":"ca', 'ts","n"', ':2}'];
const user = (text: string): ChatMessage => ({ role: 'user', content: [{ type: 'text', text }] });
const toolCallMsg = (extra: Partial<ChatMessage> = {}): ChatMessage => ({
role: 'assistant',
content: [{ type: 'tool_call', id: 'call_1', name: 'lookup', input: { q: 'cats' } }],
...extra,
});
const toolResultMsg: ChatMessage = { role: 'user', content: [{ type: 'tool_result', toolCallId: 'call_1', name: 'lookup', output: 'result text' }] };
const cfgOf = (kind: ProviderKind, apiKey: string | null, baseUrl: string | null = null): ProviderConfig => ({ id: kind, kind, name: 'Test Server', baseUrl, apiKey });
/* ------------------------------------------------------------------ */
/* Per-adapter fixtures */
/* ------------------------------------------------------------------ */
const EXPECTED_FULL_USAGE: Usage = { inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, reasoningTokens: 10 };
interface Kit {
name: string;
adapter: ProviderAdapter;
key: string;
cfg: ProviderConfig;
model: string;
otherKind: ProviderKind;
streamUrl: string;
toolId: unknown;
assertAuth(h: Record<string, string>): void;
assertRequest(body: any): void; // system prompt + tool definitions
assertHistory(body: any): void; // tool_call + tool_result mapping
text(parts: string[]): string[];
tool(frags: string[]): string[];
usage(): { chunks: string[]; expected: Usage };
errors: { auth: [number, unknown]; rate: [number, unknown]; ctx: [number, unknown] };
}
const openaiStyleErrors = (key: string): Kit['errors'] => ({
auth: [401, { error: { message: `Incorrect API key provided: ${key}.`, type: 'invalid_request_error', code: 'invalid_api_key' } }],
rate: [429, { error: { message: 'Rate limit reached for requests', type: 'rate_limit_error', code: 'rate_limit_exceeded' } }],
ctx: [400, { error: { message: 'This model maximum context length is 8192 tokens, request too long', type: 'invalid_request_error', code: 'context_length_exceeded' } }],
});
/* ---- Anthropic ---- */
const anthropicKit: Kit = {
name: 'anthropic',
adapter: anthropicAdapter,
key: 'sk-ant-TESTKEY-1234567890',
cfg: cfgOf('anthropic', 'sk-ant-TESTKEY-1234567890'),
model: 'claude-sonnet-5-5',
otherKind: 'openai',
streamUrl: 'https://api.anthropic.com/v1/messages',
toolId: 'call_1',
assertAuth(h) {
expect(h['x-api-key']).toBe('sk-ant-TESTKEY-1234567890');
expect(h['anthropic-version']).toBe('2026-10-21');
expect(h.Authorization).toBeUndefined();
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.system).toBe('SYS');
expect(b.messages.some((m: any) => m.role === 'system')).toBe(false);
expect(b.tools).toEqual([{ name: 'lookup', description: 'Look things up', input_schema: TOOL.parameters, eager_input_streaming: true }]);
},
assertHistory(b) {
expect(b.messages).toEqual([
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
{ role: 'assistant', content: [{ type: 'tool_use', id: 'call_1', name: 'lookup', input: { q: 'cats' } }] },
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call_1', content: 'result text' }] },
]);
},
text: (parts) => [
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }, 'message_start'),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, 'content_block_start'),
...parts.map((p) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: p } }, 'content_block_delta')),
ev({ type: 'content_block_stop', index: 0 }, 'content_block_stop'),
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 3 } }, 'message_delta'),
ev({ type: 'message_stop' }, 'message_stop'),
],
tool: (frags) => [
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} } }),
...frags.map((f) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: f } })),
ev({ type: 'content_block_stop', index: 0 }),
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
ev({ type: 'message_stop' }),
],
// Anthropic reports uncached/cache-read/cache-write input separately and has no reasoning-token field.
usage: () => ({
chunks: [
ev({ type: 'message_start', message: { usage: { input_tokens: 10, cache_read_input_tokens: 5, cache_creation_input_tokens: 3, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'ok' } }),
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 20 } }),
ev({ type: 'message_stop' }),
],
expected: { inputTokens: 18, outputTokens: 20, cacheReadTokens: 5, cacheWriteTokens: 3 },
}),
errors: {
auth: [401, { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key sk-ant-TESTKEY-1234567890' } }],
rate: [429, { type: 'error', error: { type: 'rate_limit_error', message: 'Number of requests has exceeded your rate limit' } }],
ctx: [400, { type: 'error', error: { type: 'invalid_request_error', message: 'prompt is too long: 250000 tokens > 200000 maximum' } }],
},
};
/* ---- OpenAI (Responses API) ---- */
const oaMessage = (text: string) => ({ type: 'message', id: 'msg_1', role: 'assistant', content: [{ type: 'output_text', text }] });
const openaiKit: Kit = {
name: 'openai',
adapter: openaiAdapter,
key: 'sk-proj-TESTKEY-1234567890',
cfg: cfgOf('openai', 'sk-proj-TESTKEY-1234567890'),
model: 'gpt-4o',
otherKind: 'anthropic',
streamUrl: 'https://api.openai.com/v1/responses',
toolId: 'call_1',
assertAuth(h) {
expect(h.Authorization).toBe('Bearer sk-proj-TESTKEY-1234567890');
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.store).toBe(false);
expect(b.instructions).toBe('SYS');
expect(b.tools).toEqual([{ type: 'function', name: 'lookup', description: 'Look things up', parameters: TOOL.parameters, strict: false }]);
},
assertHistory(b) {
expect(b.input).toEqual([
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
{ type: 'function_call', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
]);
},
text: (parts) => [
...parts.map((p) => ev({ type: 'response.output_text.delta', item_id: 'msg_1', delta: p }, 'response.output_text.delta')),
ev({ type: 'response.completed', response: { output: [oaMessage(parts.join(''))], usage: { input_tokens: 4, output_tokens: 3 } } }, 'response.completed'),
],
// `arguments` is empty in the final item here so that the streamed deltas are what gets assembled.
tool: (frags) => [
...frags.map((d) => ev({ type: 'response.function_call_arguments.delta', item_id: 'fc_1', delta: d })),
ev({
type: 'response.completed',
response: { output: [{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '' }], usage: { input_tokens: 4, output_tokens: 3 } },
}),
],
usage: () => ({
chunks: [
ev({ type: 'response.output_text.delta', delta: 'ok' }),
ev({
type: 'response.completed',
response: {
output: [oaMessage('ok')],
usage: { input_tokens: 100, output_tokens: 50, input_tokens_details: { cached_tokens: 20 }, output_tokens_details: { reasoning_tokens: 10 } },
},
}),
],
expected: EXPECTED_FULL_USAGE,
}),
errors: openaiStyleErrors('sk-proj-TESTKEY-1234567890'),
};
/* ---- Gemini ---- */
const gemChunk = (parts: unknown[], extra: Record<string, unknown> = {}, cand: Record<string, unknown> = {}) =>
ev({ candidates: [{ content: { role: 'model', parts }, ...cand }], ...extra });
const geminiKit: Kit = {
name: 'gemini',
adapter: geminiAdapter,
key: 'AIzaSyTESTKEY1234567890abcd',
cfg: cfgOf('gemini', 'AIzaSyTESTKEY1234567890abcd'),
model: 'gemini-3.8-flash',
otherKind: 'anthropic',
streamUrl: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse',
toolId: expect.stringMatching(/^gem_/),
assertAuth(h) {
expect(h['x-goog-api-key']).toBe('AIzaSyTESTKEY1234567890abcd');
expect(h.Authorization).toBeUndefined();
},
assertRequest(b) {
expect(b.systemInstruction).toEqual({ parts: [{ text: 'SYS' }] });
expect(b.tools).toEqual([{ functionDeclarations: [{ name: 'lookup', description: 'Look things up', parameters: TOOL.parameters }] }]);
},
assertHistory(b) {
expect(b.contents).toEqual([
{ role: 'user', parts: [{ text: 'hi' }] },
{ role: 'model', parts: [{ functionCall: { name: 'lookup', args: { q: 'cats' } } }] },
{ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] },
]);
},
text: (parts) => [...parts.map((p) => gemChunk([{ text: p }])), gemChunk([{ text: '' }], {}, { finishReason: 'STOP' })],
// Gemini delivers a function call whole (args is an object), never as fragments.
tool: (frags) => [gemChunk([{ functionCall: { name: 'lookup', args: JSON.parse(frags.join('')) } }], {}, { finishReason: 'STOP' })],
usage: () => ({
chunks: [
gemChunk([{ text: 'ok' }], { usageMetadata: { promptTokenCount: 100, candidatesTokenCount: 40, thoughtsTokenCount: 10, cachedContentTokenCount: 20 } }, { finishReason: 'STOP' }),
],
expected: EXPECTED_FULL_USAGE,
}),
errors: {
auth: [401, { error: { code: 401, message: 'API key not valid. AIzaSyTESTKEY1234567890abcd', status: 'UNAUTHENTICATED' } }],
rate: [429, { error: { code: 429, message: 'Resource has been exhausted', status: 'RESOURCE_EXHAUSTED' } }],
ctx: [400, { error: { code: 400, message: 'The input token count (2000000) exceeds the maximum number of tokens allowed (1048576).', status: 'INVALID_ARGUMENT' } }],
},
};
/* ---- Chat Completions family ---- */
const ccChunk = (delta: Record<string, unknown>, finish: string | null = null) => ev({ choices: [{ index: 0, delta, finish_reason: finish }] });
function chatCompletionsKit(over: Pick<Kit, 'name' | 'adapter' | 'key' | 'cfg' | 'model' | 'streamUrl'>): Kit {
return {
...over,
otherKind: 'gemini',
toolId: 'call_1',
assertAuth(h) {
expect(h.Authorization).toBe(`Bearer ${over.key}`);
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.messages[0]).toEqual({ role: 'system', content: 'SYS' });
expect(b.tools).toEqual([{ type: 'function', function: { name: 'lookup', description: 'Look things up', parameters: TOOL.parameters } }]);
},
assertHistory(b) {
expect(b.messages).toEqual([
{ role: 'system', content: 'SYS' },
{ role: 'user', content: 'hi' },
{ role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{"q":"cats"}' } }] },
{ role: 'tool', tool_call_id: 'call_1', content: 'result text' },
]);
},
text: (parts) => [ccChunk({ role: 'assistant', content: '' }), ...parts.map((p) => ccChunk({ content: p })), ccChunk({}, 'stop'), DONE],
tool: (frags) => [
ccChunk({ role: 'assistant', tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '' } }] }),
...frags.map((f) => ccChunk({ tool_calls: [{ index: 0, function: { arguments: f } }] })),
ccChunk({}, 'tool_calls'),
DONE,
],
usage: () => ({
chunks: [
ccChunk({ content: 'ok' }, 'stop'),
ev({
choices: [],
usage: { prompt_tokens: 100, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 20 }, completion_tokens_details: { reasoning_tokens: 10 } },
}),
DONE,
],
expected: EXPECTED_FULL_USAGE,
}),
errors: openaiStyleErrors(over.key),
};
}
const compatKit = chatCompletionsKit({
name: 'openai_compatible',
adapter: openaiCompatibleAdapter,
key: 'sk-or-TESTKEY-1234567890',
cfg: cfgOf('openai_compatible', 'sk-or-TESTKEY-1234567890', 'https://router.example.com/api/v1'),
model: 'meta/llama-3',
streamUrl: 'https://router.example.com/api/v1/chat/completions',
});
const ollamaKit = chatCompletionsKit({
name: 'ollama',
adapter: ollamaAdapter,
key: 'ollama-proxy-secret-1234',
cfg: cfgOf('ollama', 'ollama-proxy-secret-1234'),
model: 'llama3',
streamUrl: 'http://localhost:11434/v1/chat/completions',
});
const kits = [anthropicKit, openaiKit, geminiKit, compatKit, ollamaKit];
/* ------------------------------------------------------------------ */
/* Shared behaviour across all five adapters */
/* ------------------------------------------------------------------ */
describe.each(kits)('$name adapter', (kit) => {
const baseReq = (): ChatRequest => ({ model: kit.model, system: 'SYS', messages: [user('hi')], tools: [TOOL] });
it('sends the expected URL, auth header, system prompt and tool definitions; key stays out of the URL', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
await run(kit.adapter, kit.cfg, baseReq());
expect(calls).toHaveLength(1);
expect(calls[0]!.url).toBe(kit.streamUrl);
expect(calls[0]!.url).not.toContain(kit.key);
expect(calls[0]!.method).toBe('POST');
kit.assertAuth(calls[0]!.headers);
kit.assertRequest(calls[0]!.body);
});
it('maps tool_call and tool_result history into the provider format', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg(), toolResultMsg] });
kit.assertHistory(calls[0]!.body);
});
it('rebuilds from neutral content when providerData belongs to a different provider kind', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
// The foreign payload carries every shape key so it would leak through if the kind were not checked.
const foreign = { kind: kit.otherKind, data: { content: [{ type: 'text', text: 'MARKER' }], output: [{ type: 'message', marker: 'MARKER' }], parts: [{ text: 'MARKER' }] } };
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg({ providerData: foreign }), toolResultMsg] });
expect(JSON.stringify(calls[0]!.body)).not.toContain('MARKER');
kit.assertHistory(calls[0]!.body);
});
it('streams text events in order and concatenates them in done', async () => {
stubFetch(queue(() => sseResponse(kit.text(['Hel', 'lo', ' world']))));
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
expect(events.filter((e) => e.type === 'text').map((e) => (e as { delta: string }).delta)).toEqual(['Hel', 'lo', ' world']);
expect(done.message.role).toBe('assistant');
expect(done.message.content).toEqual([{ type: 'text', text: 'Hello world' }]);
expect(done.stopReason).toBe('end');
});
it('assembles a fragmented tool call into one tool_call event', async () => {
stubFetch(queue(() => sseResponse(kit.tool(FRAGS))));
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
const toolEvents = events.filter((e): e is Extract<StreamEvent, { type: 'tool_call' }> => e.type === 'tool_call');
expect(toolEvents).toHaveLength(1);
expect(toolEvents[0]!.call).toEqual({ id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
expect(done.stopReason).toBe('tool_use');
expect(done.message.content).toContainEqual({ type: 'tool_call', id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
expect(events[events.length - 1]).toBe(done);
});
it('maps usage onto the neutral convention', async () => {
const { chunks, expected } = kit.usage();
stubFetch(queue(() => sseResponse(chunks)));
const { done } = await run(kit.adapter, kit.cfg, baseReq());
expect(done.usage).toEqual(expected);
});
it.each([
['auth', 'auth', 401],
['rate', 'rate_limit', 429],
['ctx', 'context_length', 400],
] as const)('maps HTTP %s failure to AIError code %s', async (which, code, status) => {
const [st, body] = kit.errors[which];
stubFetch(queue(() => jsonResponse(body, st)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.code).toBe(code);
expect(err.status).toBe(status);
expect(err.message).not.toContain(kit.key);
expect(err.providerMessage ?? '').not.toContain(kit.key);
});
it('scrubs the API key from provider text', async () => {
const [st, body] = kit.errors.auth;
stubFetch(queue(() => jsonResponse(body, st)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.providerMessage).toContain('[redacted]');
expect(JSON.stringify(err)).not.toContain(kit.key);
});
it('maps a fetch TypeError to a network error without leaking the key', async () => {
stubFetch(queue(() => new TypeError(`fetch failed (${kit.key})`)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.code).toBe('network');
expect(err.retryable).toBe(true);
expect(err.message).not.toContain(kit.key);
expect(err.providerMessage ?? '').not.toContain(kit.key);
});
});
/* ------------------------------------------------------------------ */
/* Anthropic specifics */
/* ------------------------------------------------------------------ */
describe('anthropic providerData', () => {
const thinkingTurn = [
ev({ type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'Let me ' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'think' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'abc' } }),
ev({ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } }),
ev({ type: 'content_block_delta', index: 1, delta: { type: 'text_delta', text: 'Sure' } }),
ev({ type: 'content_block_start', index: 2, content_block: { type: 'tool_use', id: 'toolu_1', name: 'lookup', input: {} } }),
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '{"q":' } }),
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '"cats"}' } }),
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
ev({ type: 'message_stop' }),
];
const replayed = [
{ type: 'thinking', thinking: 'Let me think', signature: 'sig-abc' },
{ type: 'text', text: 'Sure' },
{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { q: 'cats' } },
];
const followUp = (assistant: ChatMessage): ChatRequest => ({
model: 'claude-sonnet-5-5',
messages: [user('hi'), assistant, { role: 'user', content: [{ type: 'tool_result', toolCallId: 'toolu_1', name: 'lookup', output: 'r' }] }],
});
it('requests adaptive thinking, emits reasoning, and replays thinking blocks with signatures', async () => {
stubFetch(queue(() => sseResponse(thinkingTurn), () => sseResponse(anthropicKit.text(['done']))));
const { events, done } = await run(anthropicAdapter, anthropicKit.cfg, { model: 'claude-sonnet-5-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'high' });
expect(calls[0]!.body.thinking).toEqual({ type: 'adaptive', display: 'summarized' });
expect(calls[0]!.body.output_config).toEqual({ effort: 'high' });
expect(events.filter((e) => e.type === 'reasoning').map((e) => (e as { delta: string }).delta)).toEqual(['Let me ', 'think']);
expect(done.message.providerData).toEqual({ kind: 'anthropic', data: { content: replayed } });
await run(anthropicAdapter, anthropicKit.cfg, followUp(done.message));
expect(calls[1]!.body.messages[1]).toEqual({ role: 'assistant', content: replayed });
expect(calls[1]!.body.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'toolu_1', content: 'r' });
});
it('strips thinking blocks and retries once on an invalid-signature 400', async () => {
const msg: ChatMessage = { ...toolCallMsg(), providerData: { kind: 'anthropic', data: { content: replayed } } };
stubFetch(
queue(
() => jsonResponse({ type: 'error', error: { type: 'invalid_request_error', message: 'Invalid `signature` in `thinking` block' } }, 400),
() => sseResponse(anthropicKit.text(['ok'])),
),
);
await run(anthropicAdapter, anthropicKit.cfg, followUp(msg));
expect(calls).toHaveLength(2);
expect(JSON.stringify(calls[0]!.body)).toContain('"thinking"');
expect(calls[1]!.body.messages[1].content.map((b: any) => b.type)).toEqual(['text', 'tool_use']);
});
});
describe('anthropic listModels', () => {
it('paginates, filters non-chat ids and merges API data over catalog defaults', async () => {
stubFetch(
queue(
() =>
jsonResponse({
data: [
{
id: 'claude-haiku-4-5',
display_name: 'Haiku API Name',
max_input_tokens: 200000,
max_tokens: 64000,
capabilities: { image_input: { supported: true }, pdf_input: { supported: false }, thinking: { supported: true } },
},
{ id: 'claude-embed-1' },
],
has_more: true,
last_id: 'claude-haiku-4-5',
}),
() => jsonResponse({ data: [{ id: 'claude-unknown-9' }], has_more: false }),
),
);
const models = await anthropicAdapter.listModels(anthropicKit.cfg);
expect(calls[0]!.url).toBe('https://api.anthropic.com/v1/models?limit=1000');
expect(calls[1]!.url).toContain('after_id=claude-haiku-4-5');
expect(calls[0]!.url).not.toContain(anthropicKit.key);
anthropicKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id)).toEqual(['claude-haiku-4-5', 'claude-unknown-9']);
expect(models[0]).toMatchObject({
label: 'Haiku API Name', // reported by the API, wins over the catalog
contextWindow: 200000,
maxOutput: 64000,
priceIn: 1, // from the catalog
priceOut: 5,
capabilities: { tools: true, vision: true, pdf: false, reasoning: true, temperature: false },
});
expect(models[1]).toEqual({ id: 'claude-unknown-9', capabilities: { tools: true, temperature: false } });
});
});
/* ------------------------------------------------------------------ */
/* OpenAI specifics */
/* ------------------------------------------------------------------ */
describe('openai providerData and reasoning', () => {
const output = [
{ type: 'reasoning', id: 'rs_1', encrypted_content: 'ENC', summary: [] },
oaMessage('Hi'),
{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
];
const turn = [
ev({ type: 'response.reasoning_summary_text.delta', delta: 'Thinking' }),
ev({ type: 'response.output_text.delta', delta: 'Hi' }),
ev({ type: 'response.completed', response: { output, usage: { input_tokens: 3, output_tokens: 2 } } }),
];
it('asks for encrypted reasoning and replays output items (reasoning item included) verbatim', async () => {
stubFetch(queue(() => sseResponse(turn), () => sseResponse(openaiKit.text(['ok']))));
const { events, done } = await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.2 });
const body = calls[0]!.body;
expect(body.reasoning).toEqual({ effort: 'medium', summary: 'auto' });
expect(body.include).toEqual(['reasoning.encrypted_content']);
expect(body.temperature).toBeUndefined(); // reasoning models reject it
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Thinking' }]);
expect(done.stopReason).toBe('tool_use');
expect(done.message.providerData).toEqual({ kind: 'openai', data: { output } });
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi'), done.message, toolResultMsg] });
expect(calls[1]!.body.input).toEqual([
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
...output,
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
]);
});
it('retries once without reasoning.effort when the model rejects it', async () => {
stubFetch(
queue(
() => jsonResponse({ error: { message: "Unsupported value: 'reasoning.effort' does not support 'none'", type: 'invalid_request_error' } }, 400),
() => sseResponse(openaiKit.text(['ok'])),
),
);
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], reasoningEffort: 'none' });
expect(calls[0]!.body.reasoning.effort).toBe('none');
expect(calls[1]!.body.reasoning.effort).toBe('minimal');
});
});
describe('openai listModels', () => {
it('keeps chat models only, sorts them and merges catalog defaults', async () => {
const ids = ['gpt-4o', 'gpt-4o-mini', 'o3', 'text-embedding-3-large', 'whisper-1', 'dall-e-3', 'gpt-4o-mini-tts', 'gpt-4o-realtime-preview', 'gpt-5-search-api', 'davinci-002', 'gpt-3.5-turbo-instruct', 'omni-moderation-latest'];
stubFetch(queue(() => jsonResponse({ data: ids.map((id) => ({ id })) })));
const models = await openaiAdapter.listModels(openaiKit.cfg);
expect(calls[0]!.url).toBe('https://api.openai.com/v1/models');
openaiKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id).sort()).toEqual(['gpt-4o', 'gpt-4o-mini', 'o3']);
expect(models.find((m) => m.id === 'gpt-4o')).toMatchObject({ label: 'GPT-4o', contextWindow: 128000, priceIn: 2.5, priceOut: 10, capabilities: { reasoning: false, temperature: true } });
expect(models.find((m) => m.id === 'o3')?.capabilities).toMatchObject({ reasoning: true, temperature: false });
});
});
/* ------------------------------------------------------------------ */
/* Gemini specifics */
/* ------------------------------------------------------------------ */
describe('gemini providerData and request config', () => {
const turn = [
gemChunk([{ text: 'Pondering', thought: true, thoughtSignature: 'SIGTHOUGHT' }]),
gemChunk([{ text: 'Hello' }]),
gemChunk([{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' }], { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 2 } }, { finishReason: 'STOP' }),
];
const replayed = [
{ text: '', thought: true, thoughtSignature: 'SIGTHOUGHT' },
{ text: 'Hello' },
{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' },
];
it('keeps thought signatures in providerData and replays the raw parts', async () => {
stubFetch(queue(() => sseResponse(turn), () => sseResponse(geminiKit.text(['ok']))));
const { events, done } = await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.3, maxOutputTokens: 100 });
expect(calls[0]!.body.generationConfig).toEqual({ temperature: 0.3, maxOutputTokens: 100, thinkingConfig: { includeThoughts: true, thinkingLevel: 'medium' } });
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Pondering' }]);
expect(done.stopReason).toBe('tool_use');
expect(done.message.providerData).toEqual({ kind: 'gemini', data: { parts: replayed } });
await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi'), done.message, toolResultMsg] });
expect(calls[1]!.body.contents[1]).toEqual({ role: 'model', parts: replayed });
expect(calls[1]!.body.contents[2]).toEqual({ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] });
});
it('turns a blocked prompt into a content_filter error', async () => {
stubFetch(queue(() => sseResponse([ev({ promptFeedback: { blockReason: 'SAFETY' } })])));
const err = await failure(run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')] }));
expect(err.code).toBe('content_filter');
});
});
describe('gemini listModels', () => {
it('paginates, keeps generateContent chat models and merges catalog defaults', async () => {
stubFetch(
queue(
() =>
jsonResponse({
models: [
{ name: 'models/gemini-3.8-flash', displayName: 'Gemini API Name', inputTokenLimit: 1000000, outputTokenLimit: 65536, supportedGenerationMethods: ['generateContent'], thinking: true },
{ name: 'models/gemini-embedding-001', supportedGenerationMethods: ['embedContent'] },
{ name: 'models/gemini-2.5-flash-image', supportedGenerationMethods: ['generateContent'] },
],
nextPageToken: 'tok1',
}),
() => jsonResponse({ models: [{ name: 'models/gemma-3-27b-it', supportedGenerationMethods: ['generateContent'] }, { name: 'models/text-bison-001', supportedGenerationMethods: ['generateContent'] }] }),
),
);
const models = await geminiAdapter.listModels(geminiKit.cfg);
expect(calls[0]!.url).toBe('https://generativelanguage.googleapis.com/v1beta/models?pageSize=1000');
expect(calls[1]!.url).toContain('pageToken=tok1');
for (const c of calls) {
expect(c.url).not.toContain(geminiKit.key);
geminiKit.assertAuth(c.headers);
}
expect(models.map((m) => m.id)).toEqual(['gemini-3.8-flash', 'gemma-3-27b-it']);
expect(models[0]).toMatchObject({ label: 'Gemini API Name', contextWindow: 1000000, maxOutput: 65536, priceIn: 0.75, priceOut: 3.75, capabilities: { reasoning: true, tools: true } });
expect(models[1]!.capabilities).toMatchObject({ tools: false, pdf: false });
});
});
describe('toGeminiSchema', () => {
it('strips unsupported keywords and converts nullable/enum/object shapes', () => {
const out = toGeminiSchema({
$schema: 'http://json-schema.org/draft-07/schema#',
type: 'object',
additionalProperties: false,
properties: {
name: { type: ['string', 'null'], default: 'a', pattern: '^a', format: 'email' },
when: { type: 'string', format: 'date-time' },
tags: { type: 'array', items: { type: 'string', const: 'x' }, minItems: 1, uniqueItems: true },
mode: { enum: ['a', 'b'] },
nested: { type: 'object' },
},
required: ['name'],
});
expect(out).toEqual({
type: 'object',
properties: {
name: { type: 'string', nullable: true },
when: { type: 'string', format: 'date-time' },
tags: { type: 'array', items: { type: 'string' }, minItems: 1 },
mode: { enum: ['a', 'b'], type: 'string' },
nested: { type: 'object', properties: {} },
},
required: ['name'],
});
});
it('falls back to a string schema for non-objects', () => {
expect(toGeminiSchema('nope')).toEqual({ type: 'string' });
});
});
/* ------------------------------------------------------------------ */
/* OpenAI-compatible and Ollama specifics */
/* ------------------------------------------------------------------ */
describe('openai-compatible stream_options fallback', () => {
it('retries without stream_options when the first 400 complains about it', async () => {
stubFetch(
queue(
() => jsonResponse({ error: { message: "Unknown parameter: 'stream_options'.", type: 'invalid_request_error' } }, 400),
() => sseResponse(compatKit.text(['ok'])),
),
);
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
expect(calls).toHaveLength(2);
expect(calls[0]!.body.stream_options).toEqual({ include_usage: true });
expect(calls[1]!.body.stream_options).toBeUndefined();
expect(done.message.content).toEqual([{ type: 'text', text: 'ok' }]);
});
it('does not write providerData (it only speaks neutral content)', async () => {
stubFetch(queue(() => sseResponse(compatKit.text(['ok']))));
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
expect(done.message.providerData).toBeUndefined();
});
});
describe('openai-compatible listModels', () => {
it('parses OpenRouter-style metadata, filters embeddings and merges vendor catalog defaults', async () => {
stubFetch(
queue(() =>
jsonResponse({
data: [
{
id: 'anthropic/claude-sonnet-5-5',
name: 'Claude Sonnet 5.5',
context_length: 1000000,
top_provider: { max_completion_tokens: 64000 },
pricing: { prompt: '0.000002', completion: '0.00001' },
supported_parameters: ['tools', 'reasoning'],
architecture: { input_modalities: ['text', 'image'] },
},
{ id: 'openai/text-embedding-3-small' },
{ id: 'vendor/plain' },
],
}),
),
);
const models = await openaiCompatibleAdapter.listModels(compatKit.cfg);
expect(calls[0]!.url).toBe('https://router.example.com/api/v1/models');
compatKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id)).toEqual(['anthropic/claude-sonnet-5-5', 'vendor/plain']);
expect(models[0]).toMatchObject({
label: 'Claude Sonnet 5.5',
contextWindow: 1000000,
maxOutput: 64000,
priceIn: 2,
priceOut: 10,
capabilities: { tools: true, reasoning: true, vision: true, pdf: true }, // pdf borrowed from the Anthropic catalog entry
});
expect(models[0]!.capabilities).not.toHaveProperty('temperature');
expect(models[1]).toEqual({ id: 'vendor/plain' });
});
it('reports "unsupported" when the server has no /models endpoint', async () => {
stubFetch(queue(() => jsonResponse({ error: 'not found' }, 404)));
const err = await failure(openaiCompatibleAdapter.listModels(compatKit.cfg));
expect(err.code).toBe('unsupported');
});
});
describe('ollama listModels', () => {
it('uses /api/tags + /api/show, drops embedding-only models and keeps models whose /show fails', async () => {
stubFetch((c) => {
if (c.url.endsWith('/api/tags')) {
return jsonResponse({
models: [
{ name: 'old:1b', model: 'old:1b' },
{ name: 'nomic-embed-text:latest', model: 'nomic-embed-text:latest' },
{ name: 'llama3:latest', model: 'llama3:latest' },
],
});
}
if (c.url.endsWith('/api/show')) {
if (c.body.model === 'llama3:latest') return jsonResponse({ capabilities: ['completion', 'tools'], model_info: { 'llama.context_length': 8192 } });
if (c.body.model === 'nomic-embed-text:latest') return jsonResponse({ capabilities: ['embedding'] });
return jsonResponse({ error: 'unsupported' }, 404);
}
return new Response('?', { status: 404 });
});
const models = await ollamaAdapter.listModels(ollamaKit.cfg);
expect(calls[0]!.url).toBe('http://localhost:11434/api/tags');
expect(calls.filter((c) => c.url.endsWith('/api/show')).map((c) => c.method)).toEqual(['POST', 'POST', 'POST']);
ollamaKit.assertAuth(calls[0]!.headers);
expect(models).toEqual([
{
id: 'llama3:latest',
label: 'llama3:latest',
priceIn: 0,
priceOut: 0,
contextWindow: 8192,
capabilities: { tools: true, vision: false, reasoning: false, pdf: false, temperature: true },
},
{ id: 'old:1b', label: 'old:1b', priceIn: 0, priceOut: 0 },
]);
});
it('strips /v1 from a configured base URL', async () => {
stubFetch(queue(() => jsonResponse({ models: [] })));
await ollamaAdapter.listModels({ ...ollamaKit.cfg, baseUrl: 'http://box:11434/v1/' });
expect(calls[0]!.url).toBe('http://box:11434/api/tags');
});
});
describe('normalizeCompatBase', () => {
it.each([
['https://api.x.com/v1', 'https://api.x.com/v1'],
['https://api.x.com/v1/', 'https://api.x.com/v1'],
[' https://x.com/api/v1 ', 'https://x.com/api/v1'],
['https://api.x.com/v1/chat/completions', 'https://api.x.com/v1'],
['https://api.x.com/v1/completions', 'https://api.x.com/v1'],
['https://api.x.com/v1/models', 'https://api.x.com/v1'],
['http://localhost:1234', 'http://localhost:1234/v1'],
['http://localhost:1234/', 'http://localhost:1234/v1'],
])('%s -> %s', (input, expected) => {
expect(normalizeCompatBase(input)).toBe(expected);
});
});
/* ------------------------------------------------------------------ */
/* catalog */
/* ------------------------------------------------------------------ */
describe('lookupModelDefaults', () => {
it('matches exact ids', () => {
expect(lookupModelDefaults('anthropic', 'claude-haiku-4-5')).toMatchObject({ label: 'Claude Haiku 4.5', priceIn: 1, priceOut: 5, contextWindow: 200000 });
});
it('matches dated snapshots by prefix, without the base label', () => {
const d = lookupModelDefaults('anthropic', 'claude-haiku-4-5-20251001');
expect(d).toMatchObject({ priceIn: 1, contextWindow: 200000 });
expect(d.label).toBeUndefined();
expect(lookupModelDefaults('openai', 'gpt-5-2025-08-07')).toMatchObject({ priceIn: 1.25 });
expect(lookupModelDefaults('gemini', 'models/gemini-2.5-flash-latest')).toMatchObject({ priceIn: 0.3 });
});
it('keeps capabilities but drops prices for a different variant of a known model', () => {
const d = lookupModelDefaults('openai', 'o3-deep-research');
expect(d.capabilities?.reasoning).toBe(true);
expect(d.priceIn).toBeUndefined();
expect(d.priceOut).toBeUndefined();
});
it('returns {} for unknown models and for kinds without a table', () => {
expect(lookupModelDefaults('openai', 'totally-unknown-model')).toEqual({});
expect(lookupModelDefaults('anthropic', '')).toEqual({});
expect(lookupModelDefaults('ollama', 'llama3')).toEqual({});
});
it('borrows capabilities but not prices or temperature for router ids', () => {
const d = lookupModelDefaults('openai_compatible', 'anthropic/claude-sonnet-5-5');
expect(d.priceIn).toBeUndefined();
expect(d.capabilities).toMatchObject({ tools: true, vision: true, pdf: true });
expect(d.capabilities).not.toHaveProperty('temperature');
});
it('returns copies that are safe to mutate', () => {
const a = lookupModelDefaults('openai', 'gpt-4o');
a.capabilities!.tools = false;
expect(lookupModelDefaults('openai', 'gpt-4o').capabilities!.tools).toBe(true);
});
});
/* ------------------------------------------------------------------ */
/* parseSSE */
/* ------------------------------------------------------------------ */
describe('parseSSE', () => {
const parse = (chunks: string[]) => drain(parseSSE(textStream(chunks)));
it('handles CRLF line endings', async () => {
expect(await parse(['event: ping\r\ndata: {"a":1}\r\n\r\ndata: two\r\n\r\n'])).toEqual([{ event: 'ping', data: '{"a":1}' }, { data: 'two' }]);
});
it('joins multi-line data with newlines', async () => {
expect(await parse(['data: a\ndata: b\n\n'])).toEqual([{ data: 'a\nb' }]);
});
it('ignores comments and keep-alives', async () => {
expect(await parse([': keepalive\n\ndata: x\n: mid-event comment\n\n'])).toEqual([{ data: 'x' }]);
});
it('stops at [DONE]', async () => {
expect(await parse(['data: a\n\ndata: [DONE]\n\ndata: b\n\n'])).toEqual([{ data: 'a' }]);
});
it('handles chunk boundaries inside a line', async () => {
expect(await parse(['da', 'ta: hel', 'lo\n', '\ndata: wor', 'ld\n\n'])).toEqual([{ data: 'hello' }, { data: 'world' }]);
});
it('handles CRLF split across chunks', async () => {
expect(await parse(['data: x\r', '\n\r', '\ndata: y\n\n'])).toEqual([{ data: 'x' }, { data: 'y' }]);
});
it('handles bare CR line endings', async () => {
expect(await parse(['data: x\r\rdata: y\r\r'])).toEqual([{ data: 'x' }, { data: 'y' }]);
});
it('flushes a final event with no trailing blank line', async () => {
expect(await parse(['data: a\n\ndata: tail'])).toEqual([{ data: 'a' }, { data: 'tail' }]);
});
it('strips a BOM and decodes multi-byte characters split across chunks', async () => {
const bytes = enc.encode('data: €uro\n\n');
const euro = bytes.indexOf(0xe2); // first byte of the 3-byte euro sign
const stream = bytesStream([bytes.slice(0, euro + 1), bytes.slice(euro + 1, euro + 2), bytes.slice(euro + 2)]);
expect(await drain(parseSSE(stream))).toEqual([{ data: '€uro' }]);
});
});

624
server/ai/service.ts Normal file
View file

@ -0,0 +1,624 @@
/**
* Provider + model configuration, task routing, usage accounting and the two call
* primitives everything else builds on: `runChat` (one streamed model turn) and
* `generateStructured` (schema-constrained JSON through a forced tool call).
*/
import { jsonrepair } from 'jsonrepair';
import { z } from 'zod';
import { decryptSecret, encryptSecret, maskSecret } from '../crypto.ts';
import type { DB } from '../db.ts';
import { HttpError, notFound, type AppContext } from '../http.ts';
import { getSetting, setSetting } from '../settings.ts';
import { newId, now, parseJson } from '../util.ts';
import { getAdapter, lookupModelDefaults } from './providers/index.ts';
import {
AIError,
DEFAULT_CAPABILITIES,
PROVIDER_KINDS,
type ChatMessage,
type ChatRequest,
type DiscoveredModel,
type ModelCapabilities,
type ProviderConfig,
type ProviderKind,
type ReasoningEffort,
type StopReason,
type StreamEvent,
type ToolDef,
type Usage,
} from './types.ts';
/* ------------------------------ rows + DTOs ------------------------------ */
interface ProviderRow {
id: string;
name: string;
kind: ProviderKind;
base_url: string | null;
api_key_enc: string | null;
headers_enc: string | null;
enabled: number;
last_test_at: number | null;
last_test_ok: number | null;
last_test_message: string;
created_at: number;
updated_at: number;
}
interface ModelRow {
id: string;
provider_id: string;
model: string;
label: string;
source: string;
enabled: number;
capabilities: string;
context_window: number | null;
max_output: number | null;
price_in: number | null;
price_out: number | null;
}
export const TASKS = ['chat', 'generate', 'vision', 'quick'] as const;
export type TaskName = (typeof TASKS)[number];
export const TASK_INFO: Record<TaskName, { label: string; description: string }> = {
chat: { label: 'Assistant', description: 'The assistant that searches your library, answers questions and runs tools.' },
generate: { label: 'Content generation', description: 'Writing quizzes, flashcards and study guides. Benefits most from a strong model.' },
vision: { label: 'Vision and OCR', description: 'Reading scanned PDFs and images. Needs a model that accepts images.' },
quick: { label: 'Quick explanations', description: 'Short explanations while studying. A fast, inexpensive model works well.' },
};
export interface TaskConfig {
modelId: string | null;
temperature: number | null;
reasoningEffort: ReasoningEffort | null;
maxOutputTokens: number | null;
}
export type TaskSettings = Record<TaskName | 'default', TaskConfig>;
const EMPTY_TASK: TaskConfig = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null };
export const taskConfigSchema = z.object({
modelId: z.string().nullable(),
temperature: z.number().min(0).max(2).nullable(),
reasoningEffort: z.enum(['none', 'low', 'medium', 'high']).nullable(),
maxOutputTokens: z.number().int().min(256).max(200000).nullable(),
});
function mapProvider(app: AppContext, row: ProviderRow) {
const key = decryptSecret(row.api_key_enc, app.config.secretKey);
const headers = parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {});
return {
id: row.id,
name: row.name,
kind: row.kind,
baseUrl: row.base_url,
hasKey: Boolean(row.api_key_enc),
/** Only ever a masked preview; the key itself never leaves the server. */
keyPreview: key ? maskSecret(key) : row.api_key_enc ? 'unreadable (re-enter key)' : '',
headerNames: Object.keys(headers),
enabled: Boolean(row.enabled),
lastTestAt: row.last_test_at,
lastTestOk: row.last_test_ok === null ? null : Boolean(row.last_test_ok),
lastTestMessage: row.last_test_message,
createdAt: row.created_at,
};
}
function mapModel(row: ModelRow & { provider_name?: string; provider_kind?: string; provider_enabled?: number }) {
return {
id: row.id,
providerId: row.provider_id,
providerName: row.provider_name ?? '',
providerKind: (row.provider_kind ?? '') as ProviderKind,
model: row.model,
label: row.label || row.model,
source: row.source,
enabled: Boolean(row.enabled),
available: Boolean(row.enabled) && row.provider_enabled !== 0,
capabilities: { ...DEFAULT_CAPABILITIES, ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}) },
contextWindow: row.context_window,
maxOutput: row.max_output,
priceIn: row.price_in,
priceOut: row.price_out,
};
}
export type ModelDto = ReturnType<typeof mapModel>;
const MODEL_SELECT = `SELECT m.*, p.name AS provider_name, p.kind AS provider_kind, p.enabled AS provider_enabled FROM ai_models m JOIN ai_providers p ON p.id = m.provider_id`;
/* ------------------------------ providers ------------------------------ */
export const providerInputSchema = z.object({
name: z.string().trim().min(1).max(80),
kind: z.enum(PROVIDER_KINDS as [ProviderKind, ...ProviderKind[]]),
baseUrl: z
.string()
.trim()
.max(500)
.nullish()
.transform((v) => v || null)
.refine((v) => v === null || /^https?:\/\//i.test(v), 'URL must start with http:// or https://'),
/** undefined = keep the stored key, '' or null = remove it. */
apiKey: z.string().max(2000).nullish(),
headers: z.record(z.string().max(100), z.string().max(2000)).optional(),
enabled: z.boolean().optional(),
});
function providerConfig(app: AppContext, row: ProviderRow): ProviderConfig {
return {
id: row.id,
kind: row.kind,
name: row.name,
baseUrl: row.base_url,
apiKey: decryptSecret(row.api_key_enc, app.config.secretKey),
headers: parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {}),
};
}
function providerRow(db: DB, id: string): ProviderRow {
const row = db.get<ProviderRow>('SELECT * FROM ai_providers WHERE id = ?', id);
if (!row) throw notFound('Provider');
return row;
}
export function listProviders(app: AppContext) {
return app.db.all<ProviderRow>('SELECT * FROM ai_providers ORDER BY created_at').map((r) => mapProvider(app, r));
}
export function saveProvider(app: AppContext, id: string | null, input: z.input<typeof providerInputSchema>) {
const data = providerInputSchema.parse(input);
const meta = getAdapter(data.kind).meta;
if (meta.requiresBaseUrl && !data.baseUrl) throw new HttpError(422, 'validation', 'This provider type needs a server URL');
const at = now();
const key = app.config.secretKey;
if (id) {
const row = providerRow(app.db, id);
const apiKeyEnc = data.apiKey === undefined ? row.api_key_enc : data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null;
const headersEnc = data.headers === undefined ? row.headers_enc : Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null;
app.db.run(
'UPDATE ai_providers SET name = ?, kind = ?, base_url = ?, api_key_enc = ?, headers_enc = ?, enabled = ?, updated_at = ? WHERE id = ?',
data.name,
data.kind,
data.baseUrl,
apiKeyEnc,
headersEnc,
data.enabled === undefined ? row.enabled : data.enabled ? 1 : 0,
at,
id,
);
return mapProvider(app, providerRow(app.db, id));
}
const newProviderId = newId();
app.db.run(
'INSERT INTO ai_providers (id, name, kind, base_url, api_key_enc, headers_enc, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
newProviderId,
data.name,
data.kind,
data.baseUrl,
data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null,
data.headers && Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null,
data.enabled === false ? 0 : 1,
at,
at,
);
return mapProvider(app, providerRow(app.db, newProviderId));
}
export function deleteProvider(app: AppContext, id: string): void {
providerRow(app.db, id);
const modelIds = new Set(app.db.all<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ?', id).map((m) => m.id));
app.db.run('DELETE FROM ai_providers WHERE id = ?', id);
const tasks = getTaskSettings(app.db);
let changed = false;
for (const name of Object.keys(tasks) as (keyof TaskSettings)[]) {
if (tasks[name].modelId && modelIds.has(tasks[name].modelId!)) {
tasks[name].modelId = null;
changed = true;
}
}
if (changed) setSetting(app.db, 'ai.tasks', tasks);
}
export async function testProvider(app: AppContext, id: string) {
const row = providerRow(app.db, id);
const anyModel = app.db.get<{ model: string }>('SELECT model FROM ai_models WHERE provider_id = ? ORDER BY enabled DESC LIMIT 1', id)?.model;
const result = await getAdapter(row.kind).test(providerConfig(app, row), anyModel);
app.db.run('UPDATE ai_providers SET last_test_at = ?, last_test_ok = ?, last_test_message = ? WHERE id = ?', now(), result.ok ? 1 : 0, result.message.slice(0, 500), id);
return result;
}
/** Looks up the provider's model list and merges it into ai_models without disturbing user edits. */
export async function discoverModels(app: AppContext, id: string): Promise<{ added: number; total: number }> {
const row = providerRow(app.db, id);
const found = await getAdapter(row.kind).listModels(providerConfig(app, row));
const existing = new Map(app.db.all<ModelRow>('SELECT * FROM ai_models WHERE provider_id = ?', id).map((m) => [m.model, m]));
const at = now();
let added = 0;
app.db.tx(() => {
for (const m of found) {
const current = existing.get(m.id);
if (current) {
// Refresh facts the provider reports, but never overwrite prices or flags the user may have edited.
app.db.run(
'UPDATE ai_models SET context_window = COALESCE(?, context_window), max_output = COALESCE(?, max_output), price_in = COALESCE(price_in, ?), price_out = COALESCE(price_out, ?), updated_at = ? WHERE id = ?',
m.contextWindow ?? null,
m.maxOutput ?? null,
m.priceIn ?? null,
m.priceOut ?? null,
at,
current.id,
);
continue;
}
insertModel(app.db, id, m, 'discovered', false);
added += 1;
}
});
return { added, total: found.length };
}
function insertModel(db: DB, providerId: string, m: DiscoveredModel, source: string, enabled: boolean): string {
const id = newId();
const at = now();
db.run(
`INSERT INTO ai_models (id, provider_id, model, label, source, enabled, capabilities, context_window, max_output, price_in, price_out, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
id,
providerId,
m.id,
m.label ?? '',
source,
enabled ? 1 : 0,
JSON.stringify(m.capabilities ?? {}),
m.contextWindow ?? null,
m.maxOutput ?? null,
m.priceIn ?? null,
m.priceOut ?? null,
at,
at,
);
return id;
}
/* ------------------------------ models ------------------------------ */
export function listModels(db: DB): ModelDto[] {
return db.all<ModelRow>(`${MODEL_SELECT} ORDER BY p.created_at, m.enabled DESC, m.model`).map(mapModel);
}
export function getModel(db: DB, id: string): ModelDto {
const row = db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ?`, id);
if (!row) throw notFound('Model');
return mapModel(row);
}
export const manualModelSchema = z.object({
providerId: z.string(),
model: z.string().trim().min(1).max(200),
label: z.string().trim().max(120).optional(),
});
export function addManualModel(app: AppContext, input: z.input<typeof manualModelSchema>): ModelDto {
const data = manualModelSchema.parse(input);
const provider = providerRow(app.db, data.providerId);
const existing = app.db.get<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ? AND model = ?', provider.id, data.model);
if (existing) {
app.db.run('UPDATE ai_models SET enabled = 1, updated_at = ? WHERE id = ?', now(), existing.id);
return getModel(app.db, existing.id);
}
const defaults = lookupModelDefaults(provider.kind, data.model);
const id = insertModel(app.db, provider.id, { ...defaults, id: data.model, label: data.label || defaults.label }, 'manual', true);
return getModel(app.db, id);
}
export const modelPatchSchema = z.object({
enabled: z.boolean().optional(),
label: z.string().trim().max(120).optional(),
capabilities: z
.object({ tools: z.boolean(), vision: z.boolean(), pdf: z.boolean(), reasoning: z.boolean(), temperature: z.boolean() })
.partial()
.optional(),
contextWindow: z.number().int().min(256).max(100_000_000).nullable().optional(),
maxOutput: z.number().int().min(16).max(10_000_000).nullable().optional(),
priceIn: z.number().min(0).max(100000).nullable().optional(),
priceOut: z.number().min(0).max(100000).nullable().optional(),
});
export function patchModel(db: DB, id: string, input: z.input<typeof modelPatchSchema>): ModelDto {
const patch = modelPatchSchema.parse(input);
const row = db.get<ModelRow>('SELECT * FROM ai_models WHERE id = ?', id);
if (!row) throw notFound('Model');
const caps = { ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}), ...patch.capabilities };
db.run(
'UPDATE ai_models SET enabled = ?, label = ?, capabilities = ?, context_window = ?, max_output = ?, price_in = ?, price_out = ?, updated_at = ? WHERE id = ?',
patch.enabled === undefined ? row.enabled : patch.enabled ? 1 : 0,
patch.label ?? row.label,
JSON.stringify(caps),
patch.contextWindow === undefined ? row.context_window : patch.contextWindow,
patch.maxOutput === undefined ? row.max_output : patch.maxOutput,
patch.priceIn === undefined ? row.price_in : patch.priceIn,
patch.priceOut === undefined ? row.price_out : patch.priceOut,
now(),
id,
);
return getModel(db, id);
}
export function deleteModel(db: DB, id: string): void {
const { changes } = db.run('DELETE FROM ai_models WHERE id = ?', id);
if (!changes) throw notFound('Model');
}
/* ------------------------------ task routing ------------------------------ */
export function getTaskSettings(db: DB): TaskSettings {
const stored = getSetting<Partial<TaskSettings>>(db, 'ai.tasks', {});
const out = { default: { ...EMPTY_TASK, ...stored.default } } as TaskSettings;
for (const t of TASKS) out[t] = { ...EMPTY_TASK, ...stored[t] };
return out;
}
export function saveTaskSettings(db: DB, input: unknown): TaskSettings {
const schema = z.object({ default: taskConfigSchema, chat: taskConfigSchema, generate: taskConfigSchema, vision: taskConfigSchema, quick: taskConfigSchema }).partial();
const patch = schema.parse(input);
const next = { ...getTaskSettings(db), ...patch } as TaskSettings;
setSetting(db, 'ai.tasks', next);
return next;
}
export interface ResolvedModel {
provider: ProviderConfig;
model: ModelDto;
params: { temperature?: number; reasoningEffort?: ReasoningEffort; maxOutputTokens?: number };
}
/**
* Picks the model for a task: explicit override, then the task's own model, then the
* default model, then (so a fresh install works after adding one model) any enabled model.
*/
export function resolveModel(app: AppContext, task: TaskName, overrideModelId?: string | null): ResolvedModel {
const tasks = getTaskSettings(app.db);
const cfg = tasks[task];
const candidates = [overrideModelId, cfg.modelId, tasks.default.modelId].filter((x): x is string => Boolean(x));
let model: ModelDto | undefined;
for (const id of candidates) {
const row = app.db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ? AND p.enabled = 1`, id);
if (row) {
model = mapModel(row);
break;
}
}
if (!model) {
const needVision = task === 'vision';
const rows = app.db.all<ModelRow>(`${MODEL_SELECT} WHERE m.enabled = 1 AND p.enabled = 1 ORDER BY p.created_at, m.created_at`).map(mapModel);
model = needVision ? rows.find((m) => m.capabilities.vision) : rows[0];
}
if (!model) {
throw new AIError(
'not_configured',
task === 'vision' ? 'No vision-capable AI model is set up. Add one in Settings, then choose it for "Vision and OCR".' : 'No AI model is set up yet. Add a provider and enable a model in Settings.',
);
}
const provider = providerConfig(app, providerRow(app.db, model.providerId));
const own = cfg.modelId === model.id || (!cfg.modelId && tasks.default.modelId === model.id) ? cfg : EMPTY_TASK;
const fallback = tasks.default;
const params: ResolvedModel['params'] = {};
const temperature = own.temperature ?? fallback.temperature;
const effort = own.reasoningEffort ?? fallback.reasoningEffort;
const maxOut = own.maxOutputTokens ?? fallback.maxOutputTokens;
if (temperature !== null && model.capabilities.temperature) params.temperature = temperature;
if (effort !== null && model.capabilities.reasoning) params.reasoningEffort = effort;
if (maxOut !== null) params.maxOutputTokens = maxOut;
return { provider, model, params };
}
/* ------------------------------ usage ------------------------------ */
export function estimateCost(model: Pick<ModelDto, 'priceIn' | 'priceOut'>, usage: Usage): number | null {
if (model.priceIn === null && model.priceOut === null) return null;
// Cached input is billed at a fraction of the list price by every provider; 10% is a close, conservative stand-in.
const cached = usage.cacheReadTokens ?? 0;
const fresh = Math.max(0, usage.inputTokens - cached);
return ((fresh + cached * 0.1) * (model.priceIn ?? 0) + usage.outputTokens * (model.priceOut ?? 0)) / 1_000_000;
}
function recordUsage(app: AppContext, r: ResolvedModel, task: string, usage: Usage | null, started: number, error?: string): void {
try {
app.db.run(
`INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cost, ok, error, duration_ms)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
now(),
r.provider.id,
r.provider.name,
r.model.model,
task,
usage?.inputTokens ?? 0,
usage?.outputTokens ?? 0,
usage?.cacheReadTokens ?? 0,
usage ? estimateCost(r.model, usage) : null,
error ? 0 : 1,
(error ?? '').slice(0, 300),
now() - started,
);
} catch {
/* accounting must never break a request */
}
}
/* ------------------------------ call primitives ------------------------------ */
export interface ChatOptions {
task: TaskName;
/** Label stored with usage rows ("assistant", "quiz generation"...). Defaults to the task. */
purpose?: string;
modelId?: string | null;
system?: string;
messages: ChatMessage[];
tools?: ToolDef[];
toolChoice?: ChatRequest['toolChoice'];
jsonMode?: boolean;
temperature?: number;
reasoningEffort?: ReasoningEffort;
maxOutputTokens?: number;
signal?: AbortSignal;
onEvent?: (event: StreamEvent) => void;
}
export interface ChatResult {
message: ChatMessage;
text: string;
stopReason: StopReason;
usage: Usage;
model: ModelDto;
}
const sleep = (ms: number, signal?: AbortSignal) =>
new Promise<void>((resolve, reject) => {
const t = setTimeout(resolve, ms);
signal?.addEventListener('abort', () => {
clearTimeout(t);
reject(new AIError('aborted', 'The request was cancelled.'));
});
});
/** Runs one model turn. Retries transient failures, but only while nothing has been streamed yet. */
export async function runChat(app: AppContext, opts: ChatOptions): Promise<ChatResult> {
const resolved = resolveModel(app, opts.task, opts.modelId);
const caps = resolved.model.capabilities;
const req: ChatRequest = {
model: resolved.model.model,
system: opts.system,
messages: opts.messages,
tools: opts.tools,
toolChoice: opts.toolChoice,
jsonMode: opts.jsonMode,
capabilities: caps,
signal: opts.signal,
temperature: opts.temperature ?? resolved.params.temperature,
reasoningEffort: opts.reasoningEffort ?? resolved.params.reasoningEffort,
maxOutputTokens: opts.maxOutputTokens ?? resolved.params.maxOutputTokens ?? undefined,
};
if (req.maxOutputTokens && resolved.model.maxOutput) req.maxOutputTokens = Math.min(req.maxOutputTokens, resolved.model.maxOutput);
if (opts.tools?.length && !caps.tools && (opts.toolChoice === 'required' || typeof opts.toolChoice === 'object')) {
throw new AIError('unsupported', `${resolved.model.label} is marked as not supporting tools. Choose another model for this task.`);
}
const adapter = getAdapter(resolved.provider.kind);
const purpose = opts.purpose ?? opts.task;
for (let attempt = 0; ; attempt += 1) {
const started = now();
let streamed = false;
try {
for await (const event of adapter.stream(resolved.provider, req)) {
if (event.type === 'done') {
recordUsage(app, resolved, purpose, event.usage, started);
const text = event.message.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
return { message: event.message, text, stopReason: event.stopReason, usage: event.usage, model: resolved.model };
}
streamed = true;
opts.onEvent?.(event);
}
throw new AIError('invalid_response', `${resolved.provider.name} ended the response unexpectedly.`, { retryable: true });
} catch (raw) {
const err = raw instanceof AIError ? raw : new AIError('server', 'The AI request failed unexpectedly.', { cause: raw });
if (!(raw instanceof AIError)) app.log('error', 'Unexpected AI adapter error', raw);
recordUsage(app, resolved, purpose, null, started, `${err.code}: ${err.message}`);
if (!err.retryable || streamed || attempt >= 2 || opts.signal?.aborted) throw err;
await sleep(1500 * (attempt + 1) ** 2, opts.signal);
}
}
}
/** Finds the first JSON value in model text (handles code fences and stray prose). */
export function parseLooseJson(text: string): unknown {
let t = text.trim();
const fence = /```(?:json)?\s*\n?([\s\S]*?)```/i.exec(t);
if (fence) t = fence[1].trim();
const start = t.search(/[[{]/);
if (start < 0) throw new AIError('invalid_response', 'The model did not return structured data.');
t = t.slice(start);
try {
return JSON.parse(t);
} catch {
try {
return JSON.parse(jsonrepair(t));
} catch {
throw new AIError('invalid_response', 'The model returned data that could not be read.', { retryable: true });
}
}
}
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
/** Name + description of the "function" whose arguments are the result. */
toolName: string;
toolDescription: string;
schema: Record<string, unknown>;
}
/**
* Structured output that works on every provider: the schema is offered as a single tool the
* model must call. Models without tool support (many local ones) get the schema in the prompt
* and their text is parsed leniently instead.
*/
export async function generateStructured(app: AppContext, opts: StructuredOptions): Promise<{ data: unknown; usage: Usage; model: ModelDto; truncated: boolean }> {
const resolved = resolveModel(app, opts.task, opts.modelId);
const tool: ToolDef = { name: opts.toolName, description: opts.toolDescription, parameters: opts.schema };
const promptJson = async () => {
const system = `${opts.system ?? ''}\n\nRespond with ONLY a JSON object matching this JSON Schema. No prose, no Markdown fences.\n${JSON.stringify(opts.schema)}`.trim();
const res = await runChat(app, { ...opts, system, jsonMode: true });
return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
};
if (!resolved.model.capabilities.tools) return promptJson();
try {
const res = await runChat(app, { ...opts, tools: [tool], toolChoice: { name: opts.toolName } });
const call = res.message.content.find((p) => p.type === 'tool_call');
if (call && call.type === 'tool_call') return { data: call.input, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
} catch (err) {
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
throw err;
}
}
/* ------------------------------ reporting ------------------------------ */
export function usageSummary(db: DB, days: number) {
const since = now() - days * 86_400_000;
const byModel = db.all<Record<string, any>>(
`SELECT provider_name AS provider, model, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost,
SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors, SUM(CASE WHEN cost IS NULL AND ok = 1 THEN 1 ELSE 0 END) AS unpriced
FROM ai_usage WHERE at >= ? GROUP BY provider_name, model ORDER BY cost DESC, calls DESC`,
since,
);
const byTask = db.all<Record<string, any>>(
`SELECT task, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost FROM ai_usage WHERE at >= ? AND ok = 1 GROUP BY task ORDER BY cost DESC`,
since,
);
const total = db.get<Record<string, any>>(
`SELECT COUNT(*) AS calls, COALESCE(SUM(input_tokens), 0) AS input, COALESCE(SUM(output_tokens), 0) AS output, COALESCE(SUM(cost), 0) AS cost FROM ai_usage WHERE at >= ?`,
since,
)!;
const recentErrors = db.all<Record<string, any>>(`SELECT at, provider_name AS provider, model, task, error FROM ai_usage WHERE ok = 0 AND at >= ? ORDER BY at DESC LIMIT 10`, since);
return { days, total, byModel, byTask, recentErrors };
}
/** True when at least one usable model exists; lets the UI show setup prompts instead of errors. */
export function aiStatus(app: AppContext) {
const models = listModels(app.db).filter((m) => m.available);
const tasks = getTaskSettings(app.db);
const resolved: Record<string, { modelId: string; label: string; provider: string } | null> = {};
for (const t of TASKS) {
try {
const r = resolveModel(app, t);
resolved[t] = { modelId: r.model.id, label: r.model.label, provider: r.provider.name };
} catch {
resolved[t] = null;
}
}
return { ready: models.length > 0, modelCount: models.length, tasks, resolved };
}

180
server/ai/types.ts Normal file
View file

@ -0,0 +1,180 @@
/**
* Provider-neutral AI contract.
*
* Every provider adapter (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama)
* translates between this shape and the provider's wire format. Nothing outside
* `server/ai/providers/` may depend on a provider-specific request or response.
*/
export type ProviderKind = 'openai' | 'anthropic' | 'gemini' | 'openai_compatible' | 'ollama';
export const PROVIDER_KINDS: ProviderKind[] = ['openai', 'anthropic', 'gemini', 'openai_compatible', 'ollama'];
/** Decrypted provider connection, only ever held server-side. */
export interface ProviderConfig {
id: string;
kind: ProviderKind;
name: string;
/** Null means "use the provider's official default endpoint". */
baseUrl: string | null;
apiKey: string | null;
/** Extra HTTP headers (e.g. OpenRouter attribution, org ids). */
headers?: Record<string, string>;
}
export type ContentPart =
| { type: 'text'; text: string }
/** Base64 image data (no data: prefix). */
| { type: 'image'; mime: string; data: string }
/** Base64 file, used for native PDF input where a provider supports it. */
| { type: 'file'; mime: string; data: string; filename?: string }
| { type: 'tool_call'; id: string; name: string; input: Record<string, unknown> }
| { type: 'tool_result'; toolCallId: string; name: string; output: string; isError?: boolean };
export interface ChatMessage {
/** Tool results travel as `tool_result` parts inside a `user` message. */
role: 'user' | 'assistant';
content: ContentPart[];
/**
* Opaque provider state for an assistant turn (thinking blocks + signatures,
* Gemini thought signatures, OpenAI reasoning items...). An adapter writes it
* on the `done` event and, when the next request goes to the same provider
* kind, replays it verbatim instead of rebuilding the turn from `content`.
*/
providerData?: { kind: ProviderKind; data: unknown };
}
export interface ToolDef {
name: string;
description: string;
/** JSON Schema (object). Adapters down-convert for providers with a restricted dialect. */
parameters: Record<string, unknown>;
}
export type ReasoningEffort = 'none' | 'low' | 'medium' | 'high';
export interface ModelCapabilities {
tools: boolean;
vision: boolean;
/** Accepts PDF files directly as input. */
pdf: boolean;
/** Has a configurable reasoning / thinking effort. */
reasoning: boolean;
/** Accepts a custom temperature (some reasoning models reject it). */
temperature: boolean;
}
export interface ChatRequest {
model: string;
system?: string;
messages: ChatMessage[];
tools?: ToolDef[];
toolChoice?: 'auto' | 'none' | 'required' | { name: string };
temperature?: number;
maxOutputTokens?: number;
reasoningEffort?: ReasoningEffort;
/** Ask for a JSON object as the text output when the provider has a JSON mode. */
jsonMode?: boolean;
/** Known capabilities of the model, when the caller has them. Adapters must not send unsupported params. */
capabilities?: Partial<ModelCapabilities>;
signal?: AbortSignal;
}
export interface Usage {
inputTokens: number;
outputTokens: number;
cacheReadTokens?: number;
cacheWriteTokens?: number;
reasoningTokens?: number;
}
export type StopReason = 'end' | 'tool_use' | 'max_tokens' | 'refusal' | 'other';
export type StreamEvent =
| { type: 'text'; delta: string }
| { type: 'reasoning'; delta: string }
/** Emitted once per tool call, after its arguments are complete and parsed. */
| { type: 'tool_call'; call: { id: string; name: string; input: Record<string, unknown> } }
/** Always the final event of a successful stream. */
| { type: 'done'; message: ChatMessage; stopReason: StopReason; usage: Usage };
export interface DiscoveredModel {
/** Identifier sent to the API. */
id: string;
label?: string;
contextWindow?: number;
maxOutput?: number;
capabilities?: Partial<ModelCapabilities>;
/** USD per million tokens, when known. */
priceIn?: number;
priceOut?: number;
}
export interface TestResult {
ok: boolean;
message: string;
latencyMs: number;
/** Number of models visible to the key, when discovery works. */
modelCount?: number;
}
export interface ProviderAdapter {
kind: ProviderKind;
/** Human label and defaults used by the settings UI. */
meta: {
label: string;
defaultBaseUrl: string | null;
requiresApiKey: boolean;
requiresBaseUrl: boolean;
supportsDiscovery: boolean;
docsUrl: string;
keyHint: string;
};
/** Streams one model turn. Throws AIError; never yields after `done`. */
stream(cfg: ProviderConfig, req: ChatRequest): AsyncIterable<StreamEvent>;
/** Lists chat-capable models. Throws AIError('unsupported') when a provider has no listing API. */
listModels(cfg: ProviderConfig, signal?: AbortSignal): Promise<DiscoveredModel[]>;
/** Cheap credential / connectivity check. Never throws. */
test(cfg: ProviderConfig, model?: string): Promise<TestResult>;
}
export type AIErrorCode =
| 'auth'
| 'rate_limit'
| 'quota'
| 'context_length'
| 'not_found'
| 'bad_request'
| 'content_filter'
| 'server'
| 'network'
| 'timeout'
| 'aborted'
| 'unsupported'
| 'invalid_response'
| 'not_configured';
export class AIError extends Error {
code: AIErrorCode;
status?: number;
retryable: boolean;
/** Raw (already secret-scrubbed) provider message, for the details disclosure in the UI. */
providerMessage?: string;
constructor(code: AIErrorCode, message: string, opts: { status?: number; retryable?: boolean; providerMessage?: string; cause?: unknown } = {}) {
super(message, opts.cause !== undefined ? { cause: opts.cause } : undefined);
this.name = 'AIError';
this.code = code;
this.status = opts.status;
this.retryable = opts.retryable ?? (code === 'rate_limit' || code === 'server' || code === 'network' || code === 'timeout');
this.providerMessage = opts.providerMessage;
}
}
export const DEFAULT_CAPABILITIES: ModelCapabilities = {
tools: true,
vision: false,
pdf: false,
reasoning: false,
temperature: true,
};

277
server/auth.ts Normal file
View file

@ -0,0 +1,277 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { Hono, type Context, type MiddlewareHandler } from 'hono';
import { deleteCookie, getCookie, setCookie } from 'hono/cookie';
import { getConnInfo } from '@hono/node-server/conninfo';
import { z } from 'zod';
import { hashPassword, safeEqual, verifyPassword } from './crypto.ts';
import { HttpError, readBody, type AppContext } from './http.ts';
import { deleteSetting, getSetting, setSetting } from './settings.ts';
import { newToken, now, sha256 } from './util.ts';
const COOKIE = 'sd_session';
const SESSION_MS = 30 * 86_400_000;
const PASSWORD_MIN = 8;
const passwordSchema = z.string().min(PASSWORD_MIN, `Use at least ${PASSWORD_MIN} characters`).max(256);
/* ------------------------------ rate limiting ------------------------------ */
interface Bucket {
failures: number[];
}
const WINDOW_MS = 15 * 60_000;
const MAX_PER_IP = 8;
const MAX_GLOBAL = 40;
const buckets = new Map<string, Bucket>();
function recent(key: string, at: number): number[] {
const bucket = buckets.get(key) ?? { failures: [] };
bucket.failures = bucket.failures.filter((t) => at - t < WINDOW_MS);
buckets.set(key, bucket);
return bucket.failures;
}
function assertNotLimited(ip: string): void {
const at = now();
const mine = recent(`ip:${ip}`, at);
const all = recent('global', at);
if (mine.length >= MAX_PER_IP || all.length >= MAX_GLOBAL) {
const oldest = (mine.length >= MAX_PER_IP ? mine : all)[0];
const retry = Math.max(1, Math.ceil((oldest + WINDOW_MS - at) / 1000));
throw new HttpError(429, 'rate_limited', `Too many attempts. Try again in ${Math.ceil(retry / 60)} minute${retry > 60 ? 's' : ''}.`, { retryAfter: retry });
}
}
function recordFailure(ip: string): void {
const at = now();
recent(`ip:${ip}`, at).push(at);
recent('global', at).push(at);
}
export function resetRateLimits(): void {
buckets.clear();
}
/* ------------------------------ request helpers ------------------------------ */
export function clientIp(c: Context, trustProxy: boolean): string {
if (trustProxy) {
const fwd = c.req.header('x-forwarded-for');
if (fwd) return fwd.split(',')[0].trim();
const real = c.req.header('x-real-ip');
if (real) return real.trim();
}
try {
return getConnInfo(c).remote.address ?? 'unknown';
} catch {
return 'unknown';
}
}
export function isHttps(c: Context, trustProxy: boolean): boolean {
if (trustProxy) {
const proto = c.req.header('x-forwarded-proto');
if (proto) return proto.split(',')[0].trim() === 'https';
}
return new URL(c.req.url).protocol === 'https:';
}
/* ------------------------------ sessions ------------------------------ */
function createSession(app: AppContext, c: Context): void {
const token = newToken();
const at = now();
app.db.run(
'INSERT INTO sessions (token_hash, created_at, expires_at, last_seen_at, user_agent, ip) VALUES (?, ?, ?, ?, ?, ?)',
sha256(token),
at,
at + SESSION_MS,
at,
(c.req.header('user-agent') ?? '').slice(0, 300),
clientIp(c, app.config.trustProxy),
);
app.db.run('DELETE FROM sessions WHERE expires_at < ?', at);
const secure = app.config.cookieSecure === 'auto' ? isHttps(c, app.config.trustProxy) : app.config.cookieSecure;
setCookie(c, COOKIE, token, { httpOnly: true, sameSite: 'Lax', secure, path: '/', maxAge: Math.floor(SESSION_MS / 1000) });
}
export function isAuthenticated(app: AppContext, c: Context): boolean {
const token = getCookie(c, COOKIE);
if (!token) return false;
const hash = sha256(token);
const at = now();
const row = app.db.get<{ expires_at: number; last_seen_at: number }>('SELECT expires_at, last_seen_at FROM sessions WHERE token_hash = ?', hash);
if (!row || row.expires_at < at) return false;
// Sliding expiry, written at most once an hour.
if (at - row.last_seen_at > 3_600_000) {
app.db.run('UPDATE sessions SET last_seen_at = ?, expires_at = ? WHERE token_hash = ?', at, at + SESSION_MS, hash);
}
return true;
}
export function requireAuth(app: AppContext): MiddlewareHandler {
return async (c, next) => {
if (!isAuthenticated(app, c)) throw new HttpError(401, 'unauthorized', 'Sign in to continue');
await next();
};
}
/**
* CSRF defence for cookie-authenticated requests: state-changing calls must carry a custom
* header (which a cross-site form cannot send, and which forces a CORS preflight that this
* server never approves), and must not be flagged cross-site by the browser.
*/
export function csrfGuard(): MiddlewareHandler {
return async (c, next) => {
const method = c.req.method;
if (method !== 'GET' && method !== 'HEAD' && method !== 'OPTIONS') {
if (c.req.header('x-requested-with') !== 'studydesk') throw new HttpError(403, 'csrf', 'Missing request header');
// Fetch metadata is set by the browser and cannot be forged by page script. Using it
// (rather than comparing Origin to Host) keeps this working behind proxies that rewrite Host.
if (c.req.header('sec-fetch-site') === 'cross-site') throw new HttpError(403, 'csrf', 'Cross-site request blocked');
}
await next();
};
}
/* ------------------------------ password storage ------------------------------ */
const HASH_KEY = 'auth.password_hash';
const SETUP_KEY = 'auth.setup_code';
export function isConfigured(app: AppContext): boolean {
return Boolean(getSetting<string | null>(app.db, HASH_KEY, null));
}
function setupCodeFile(app: AppContext): string {
return path.join(app.config.dataDir, 'setup-code.txt');
}
async function storePassword(app: AppContext, password: string): Promise<void> {
setSetting(app.db, HASH_KEY, await hashPassword(password));
deleteSetting(app.db, SETUP_KEY);
fs.rmSync(setupCodeFile(app), { force: true });
}
/**
* Runs at startup. Applies ADMIN_PASSWORD / RESET_ADMIN_PASSWORD, or prepares a one-time
* setup code (printed to the log and written to the data folder) so that only someone with
* access to the server can claim a fresh install.
*/
export async function initAuth(app: AppContext): Promise<void> {
const { config, db } = app;
if (config.resetAdminPassword) {
passwordSchema.parse(config.resetAdminPassword);
await storePassword(app, config.resetAdminPassword);
db.run('DELETE FROM sessions');
app.log('warn', 'Administrator password was reset from RESET_ADMIN_PASSWORD. Remove that variable and restart.');
return;
}
if (isConfigured(app)) return;
if (config.adminPassword) {
passwordSchema.parse(config.adminPassword);
await storePassword(app, config.adminPassword);
app.log('info', 'Administrator password initialised from ADMIN_PASSWORD.');
return;
}
let code = getSetting<string | null>(db, SETUP_KEY, null);
if (!code) {
code = crypto.randomBytes(5).toString('hex').toUpperCase().replace(/(.{5})/, '$1-');
setSetting(db, SETUP_KEY, code);
}
try {
fs.writeFileSync(setupCodeFile(app), `${code}\n`, { mode: 0o600 });
} catch {
/* the log line below is enough */
}
app.log('info', `First-run setup code: ${code} (also saved to ${setupCodeFile(app)})`);
}
/* ------------------------------ routes ------------------------------ */
export function authRoutes(app: AppContext): Hono {
const r = new Hono();
r.get('/status', (c) => c.json({ configured: isConfigured(app), authenticated: isAuthenticated(app, c) }));
r.post('/setup', async (c) => {
if (isConfigured(app)) throw new HttpError(409, 'already_configured', 'Study Desk is already set up');
const ip = clientIp(c, app.config.trustProxy);
assertNotLimited(ip);
const body = await readBody(c, z.object({ code: z.string().max(40), password: passwordSchema }));
const expected = getSetting<string | null>(app.db, SETUP_KEY, null);
const given = body.code.trim().toUpperCase();
if (!expected || !(safeEqual(given, expected) || safeEqual(given, expected.replace('-', '')))) {
recordFailure(ip);
throw new HttpError(403, 'bad_setup_code', 'That setup code is not correct. Check the server log or setup-code.txt in the data folder.');
}
await storePassword(app, body.password);
createSession(app, c);
return c.json({ ok: true });
});
r.post('/login', async (c) => {
const ip = clientIp(c, app.config.trustProxy);
assertNotLimited(ip);
const body = await readBody(c, z.object({ password: z.string().max(256) }));
const hash = getSetting<string | null>(app.db, HASH_KEY, null);
if (!hash) throw new HttpError(409, 'not_configured', 'Study Desk has not been set up yet');
if (!(await verifyPassword(body.password, hash))) {
recordFailure(ip);
throw new HttpError(401, 'bad_password', 'Incorrect password');
}
createSession(app, c);
return c.json({ ok: true });
});
r.post('/logout', (c) => {
const token = getCookie(c, COOKIE);
if (token) app.db.run('DELETE FROM sessions WHERE token_hash = ?', sha256(token));
deleteCookie(c, COOKIE, { path: '/' });
return c.json({ ok: true });
});
r.post('/password', requireAuth(app), async (c) => {
const ip = clientIp(c, app.config.trustProxy);
assertNotLimited(ip);
const body = await readBody(c, z.object({ current: z.string().max(256), next: passwordSchema }));
const hash = getSetting<string | null>(app.db, HASH_KEY, null);
if (!hash || !(await verifyPassword(body.current, hash))) {
recordFailure(ip);
throw new HttpError(403, 'bad_password', 'Current password is incorrect');
}
await storePassword(app, body.next);
// Sign out every other device.
const token = getCookie(c, COOKIE);
app.db.run('DELETE FROM sessions WHERE token_hash != ?', token ? sha256(token) : '');
return c.json({ ok: true });
});
r.get('/sessions', requireAuth(app), (c) => {
const current = sha256(getCookie(c, COOKIE) ?? '');
const rows = app.db.all<{ token_hash: string; created_at: number; last_seen_at: number; user_agent: string; ip: string }>(
'SELECT token_hash, created_at, last_seen_at, user_agent, ip FROM sessions WHERE expires_at > ? ORDER BY last_seen_at DESC',
now(),
);
return c.json({
sessions: rows.map((s) => ({
id: s.token_hash.slice(0, 12),
createdAt: s.created_at,
lastSeenAt: s.last_seen_at,
userAgent: s.user_agent,
ip: s.ip,
current: s.token_hash === current,
})),
});
});
r.post('/sessions/revoke-others', requireAuth(app), (c) => {
const current = sha256(getCookie(c, COOKIE) ?? '');
const { changes } = app.db.run('DELETE FROM sessions WHERE token_hash != ?', current);
return c.json({ revoked: changes });
});
return r;
}

89
server/config.ts Normal file
View file

@ -0,0 +1,89 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
export interface Config {
env: 'development' | 'production' | 'test';
host: string;
port: number;
dataDir: string;
dbPath: string;
uploadsDir: string;
mediaDir: string;
backupsDir: string;
/** Directory with the built web app, or null when the Vite dev server serves it. */
webDir: string | null;
/** 32-byte key used to encrypt provider API keys at rest. */
secretKey: Buffer;
/** 'auto' marks the cookie Secure whenever the request arrived over HTTPS (directly or via proxy). */
cookieSecure: 'auto' | boolean;
trustProxy: boolean;
adminPassword: string | null;
resetAdminPassword: string | null;
maxUploadBytes: number;
backupKeep: number;
timezone: string;
}
function bool(v: string | undefined, fallback: boolean): boolean {
if (v === undefined || v === '') return fallback;
return ['1', 'true', 'yes', 'on'].includes(v.toLowerCase());
}
function loadSecret(dataDir: string, fromEnv: string | undefined): Buffer {
if (fromEnv && fromEnv.trim().length >= 16) {
return crypto.createHash('sha256').update(`studydesk:${fromEnv.trim()}`).digest();
}
const file = path.join(dataDir, 'secret.key');
if (fs.existsSync(file)) {
const raw = Buffer.from(fs.readFileSync(file, 'utf8').trim(), 'base64');
if (raw.length === 32) return raw;
}
const key = crypto.randomBytes(32);
fs.writeFileSync(file, key.toString('base64'), { mode: 0o600 });
return key;
}
export function loadConfig(overrides: Partial<Config> = {}, envVars: NodeJS.ProcessEnv = process.env): Config {
const env = (overrides.env ?? (envVars.NODE_ENV === 'production' ? 'production' : envVars.NODE_ENV === 'test' ? 'test' : 'development')) as Config['env'];
const dataDir = path.resolve(overrides.dataDir ?? envVars.DATA_DIR ?? (env === 'production' ? '/data' : './data'));
const uploadsDir = path.join(dataDir, 'uploads');
const mediaDir = path.join(dataDir, 'media');
const backupsDir = path.join(dataDir, 'backups');
for (const dir of [dataDir, uploadsDir, mediaDir, backupsDir]) fs.mkdirSync(dir, { recursive: true });
const secure = (envVars.COOKIE_SECURE ?? 'auto').toLowerCase();
let webDir: string | null = overrides.webDir ?? envVars.WEB_DIR ?? null;
if (webDir === null) {
const candidate = path.resolve('dist/web');
if (fs.existsSync(path.join(candidate, 'index.html'))) webDir = candidate;
}
let timezone = envVars.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
try {
new Intl.DateTimeFormat('en-US', { timeZone: timezone });
} catch {
timezone = 'UTC';
}
return {
env,
host: envVars.HOST ?? '0.0.0.0',
port: Number(envVars.PORT ?? 3726),
dataDir,
dbPath: path.join(dataDir, 'study.db'),
uploadsDir,
mediaDir,
backupsDir,
webDir,
secretKey: loadSecret(dataDir, envVars.APP_SECRET),
cookieSecure: secure === 'auto' ? 'auto' : bool(secure, false),
trustProxy: bool(envVars.TRUST_PROXY, true),
adminPassword: envVars.ADMIN_PASSWORD?.trim() || null,
resetAdminPassword: envVars.RESET_ADMIN_PASSWORD?.trim() || null,
maxUploadBytes: Math.max(1, Number(envVars.MAX_UPLOAD_MB ?? 100)) * 1024 * 1024,
backupKeep: Math.max(1, Number(envVars.BACKUP_KEEP ?? 7)),
timezone,
...overrides,
};
}

62
server/crypto.ts Normal file
View file

@ -0,0 +1,62 @@
import crypto from 'node:crypto';
const SCRYPT = { N: 32768, r: 8, p: 1, keylen: 32 };
function scrypt(password: string, salt: Buffer, N: number, r: number, p: number, keylen: number): Promise<Buffer> {
return new Promise((resolve, reject) => {
crypto.scrypt(password.normalize('NFKC'), salt, keylen, { N, r, p, maxmem: 256 * N * r }, (err, key) => (err ? reject(err) : resolve(key)));
});
}
/** scrypt password hash: `scrypt$N$r$p$salt$hash` (base64url fields). */
export async function hashPassword(password: string): Promise<string> {
const salt = crypto.randomBytes(16);
const key = await scrypt(password, salt, SCRYPT.N, SCRYPT.r, SCRYPT.p, SCRYPT.keylen);
return ['scrypt', SCRYPT.N, SCRYPT.r, SCRYPT.p, salt.toString('base64url'), key.toString('base64url')].join('$');
}
export async function verifyPassword(password: string, stored: string): Promise<boolean> {
const parts = stored.split('$');
if (parts.length !== 6 || parts[0] !== 'scrypt') return false;
const [, n, r, p, saltB64, hashB64] = parts;
const expected = Buffer.from(hashB64, 'base64url');
try {
const actual = await scrypt(password, Buffer.from(saltB64, 'base64url'), Number(n), Number(r), Number(p), expected.length);
return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
} catch {
return false;
}
}
/** AES-256-GCM. Output: `v1.<iv>.<tag>.<ciphertext>` (base64url). */
export function encryptSecret(plain: string, key: Buffer): string {
const iv = crypto.randomBytes(12);
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
const data = Buffer.concat([cipher.update(plain, 'utf8'), cipher.final()]);
return ['v1', iv.toString('base64url'), cipher.getAuthTag().toString('base64url'), data.toString('base64url')].join('.');
}
/** Returns null when the value cannot be decrypted (wrong key, tampering). */
export function decryptSecret(value: string | null | undefined, key: Buffer): string | null {
if (!value) return null;
const parts = value.split('.');
if (parts.length !== 4 || parts[0] !== 'v1') return null;
try {
const decipher = crypto.createDecipheriv('aes-256-gcm', key, Buffer.from(parts[1], 'base64url'));
decipher.setAuthTag(Buffer.from(parts[2], 'base64url'));
return Buffer.concat([decipher.update(Buffer.from(parts[3], 'base64url')), decipher.final()]).toString('utf8');
} catch {
return null;
}
}
export function maskSecret(secret: string | null): string {
if (!secret) return '';
return secret.length <= 8 ? '••••' : `${secret.slice(0, 3)}••••${secret.slice(-4)}`;
}
export function safeEqual(a: string, b: string): boolean {
const x = crypto.createHash('sha256').update(a).digest();
const y = crypto.createHash('sha256').update(b).digest();
return crypto.timingSafeEqual(x, y);
}

140
server/db.ts Normal file
View file

@ -0,0 +1,140 @@
import fs from 'node:fs';
import path from 'node:path';
import { DatabaseSync, type StatementSync } from 'node:sqlite';
import { MIGRATIONS } from './migrations.ts';
type Param = string | number | bigint | boolean | null | undefined | Uint8Array;
function bind(params: Param[]): (string | number | bigint | null | Uint8Array)[] {
return params.map((p) => (p === undefined ? null : typeof p === 'boolean' ? (p ? 1 : 0) : p));
}
/**
* Thin synchronous wrapper around node:sqlite. Single process, single writer:
* exactly what a one-user app in one container needs, with no native add-on to build.
*/
export class DB {
readonly raw: DatabaseSync;
readonly path: string;
private statements = new Map<string, StatementSync>();
private txDepth = 0;
constructor(file: string) {
this.path = file;
this.raw = new DatabaseSync(file);
this.raw.exec('PRAGMA journal_mode = WAL');
this.raw.exec('PRAGMA synchronous = NORMAL');
this.raw.exec('PRAGMA foreign_keys = ON');
this.raw.exec('PRAGMA busy_timeout = 5000');
}
private stmt(sql: string): StatementSync {
let s = this.statements.get(sql);
if (!s) {
s = this.raw.prepare(sql);
if (this.statements.size > 500) this.statements.clear();
this.statements.set(sql, s);
}
return s;
}
all<T = Record<string, unknown>>(sql: string, ...params: Param[]): T[] {
return this.stmt(sql).all(...bind(params)) as T[];
}
get<T = Record<string, unknown>>(sql: string, ...params: Param[]): T | undefined {
return this.stmt(sql).get(...bind(params)) as T | undefined;
}
run(sql: string, ...params: Param[]): { changes: number; lastId: number } {
const r = this.stmt(sql).run(...bind(params));
return { changes: Number(r.changes), lastId: Number(r.lastInsertRowid) };
}
exec(sql: string): void {
this.raw.exec(sql);
}
/** Runs fn atomically. Nested calls become savepoints. */
tx<T>(fn: () => T): T {
const depth = this.txDepth;
const name = `sp_${depth}`;
this.raw.exec(depth === 0 ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
this.txDepth += 1;
try {
const out = fn();
this.txDepth -= 1;
this.raw.exec(depth === 0 ? 'COMMIT' : `RELEASE ${name}`);
return out;
} catch (e) {
this.txDepth -= 1;
try {
this.raw.exec(depth === 0 ? 'ROLLBACK' : `ROLLBACK TO ${name}; RELEASE ${name}`);
} catch {
/* the original error matters more */
}
throw e;
}
}
/** Writes a consistent, compacted copy of the database (safe while the app is running). */
snapshot(target: string): void {
if (fs.existsSync(target)) fs.rmSync(target);
this.raw.exec(`VACUUM INTO '${target.replace(/'/g, "''")}'`);
}
close(): void {
this.statements.clear();
try {
this.raw.exec('PRAGMA wal_checkpoint(TRUNCATE)');
} catch {
/* ignore */
}
this.raw.close();
}
}
export interface MigrationReport {
applied: string[];
backup: string | null;
}
/**
* Applies pending migrations in order, each in its own transaction. When an existing
* database is about to change, a snapshot is written to the backups folder first.
*/
export function migrate(db: DB, backupsDir?: string): MigrationReport {
db.exec('CREATE TABLE IF NOT EXISTS schema_migrations (id INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at INTEGER NOT NULL)');
const done = new Set(db.all<{ id: number }>('SELECT id FROM schema_migrations').map((r) => r.id));
const known = new Set(MIGRATIONS.map((m) => m.id));
const unknown = [...done].filter((id) => !known.has(id));
if (unknown.length > 0) {
throw new Error(
`This database was created by a newer version of Study Desk (unknown migration ${unknown.join(', ')}). Upgrade the app instead of downgrading it.`,
);
}
const pending = MIGRATIONS.filter((m) => !done.has(m.id)).sort((a, b) => a.id - b.id);
const report: MigrationReport = { applied: [], backup: null };
if (pending.length === 0) return report;
if (done.size > 0 && backupsDir && db.path !== ':memory:') {
fs.mkdirSync(backupsDir, { recursive: true });
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
report.backup = path.join(backupsDir, `pre-migration-${stamp}.db`);
db.snapshot(report.backup);
}
for (const m of pending) {
db.tx(() => {
db.exec(m.sql);
db.run('INSERT INTO schema_migrations (id, name, applied_at) VALUES (?, ?, ?)', m.id, m.name, Date.now());
});
report.applied.push(`${m.id}_${m.name}`);
}
return report;
}
export function openDatabase(file: string, backupsDir?: string): { db: DB; report: MigrationReport } {
const db = new DB(file);
const report = migrate(db, backupsDir);
return { db, report };
}

514
server/documents/chunk.ts Normal file
View file

@ -0,0 +1,514 @@
/**
* Text normalisation, token estimation and retrieval chunking.
* Pure functions, no dependencies. Everything here is linear-time on its input.
*/
import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts';
// ---------------------------------------------------------------------------
// Token estimation
// ---------------------------------------------------------------------------
/**
* Un-rounded token estimate. Sub-additive under concatenation
* (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and
* still guarantee the final `estimateTokens(chunk)` stays under the ceiling.
*/
function rawTokens(text: string): number {
let total = 0;
let prevWhitespace = false;
for (let i = 0; i < text.length; i++) {
const c = text.charCodeAt(i);
if (c === 32 || (c >= 9 && c <= 13)) {
// A run of whitespace costs the same as one character of prose.
if (!prevWhitespace) total += 0.25;
prevWhitespace = true;
continue;
}
prevWhitespace = false;
if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25;
else if (c >= 48 && c <= 57) total += 0.4;
else if (c < 128) total += 0.5;
else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character
else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each
else total += 0.5; // accented Latin, Greek, symbols
}
return total;
}
/** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */
export function estimateTokens(text: string): number {
if (!text) return 0;
return Math.ceil(rawTokens(text));
}
// ---------------------------------------------------------------------------
// cleanText
// ---------------------------------------------------------------------------
const LIGATURES: Record<string, string> = {
'ff': 'ff',
'fi': 'fi',
'fl': 'fl',
'ffi': 'ffi',
'ffl': 'ffl',
'ſt': 'st',
'st': 'st',
};
/** First halves that normally form a real hyphenated compound ("well-known"). */
const HYPHEN_KEEP_LEFT = new Set([
'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first',
'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part',
'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid',
'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed',
]);
/** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */
const HYPHEN_KEEP_RIGHT = new Set([
'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced',
'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk',
'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive',
'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset',
]);
function isLower(code: number): boolean {
return code >= 97 && code <= 122;
}
function isLetter(code: number): boolean {
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90);
}
/** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */
function joinHyphenatedLineBreaks(text: string): string {
if (!text.includes('-\n')) return text;
return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => {
// Left fragment: lowercase letters, optionally led by one capital, at most 30 long.
let start = offset;
while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--;
if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--;
const left = whole.slice(start, offset);
if (left.length < 2) return match;
if (start > 0) {
const before = whole.charCodeAt(start - 1);
// Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it.
if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match;
}
const rightStart = offset + match.length;
let rightEnd = rightStart;
while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++;
const right = whole.slice(rightStart, rightEnd);
if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z"
if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match;
return '';
});
}
/**
* Normalises extracted text: newlines, unicode spaces, control characters,
* ligatures, trailing spaces, runs of blank lines and words hyphenated across
* a line break. Idempotent.
*/
export function cleanText(text: string): string {
if (!text) return '';
let t = text;
if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n');
t = t.replace(/[\u2028\u2029\f\v]/g, '\n');
t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' ');
// NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM.
t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, '');
t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch);
// Trailing whitespace per line (done by splitting so a huge run of spaces stays linear).
t = t
.split('\n')
.map((line) => line.trimEnd())
.join('\n');
t = joinHyphenatedLineBreaks(t);
t = t.replace(/\n{3,}/g, '\n\n');
return t.trim();
}
// ---------------------------------------------------------------------------
// Repeated page headers / footers
// ---------------------------------------------------------------------------
function edgeLineKey(line: string): string | null {
const trimmed = line.trim();
if (!trimmed || trimmed.length > 100) return null;
return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
}
/**
* Removes running headers/footers and page numbers from per-page text, but only
* when the same first or last line (digits ignored) shows up on at least 60% of
* the pages that have text, and there are at least four such pages. Chapter
* headings that change through a book are therefore left alone.
*/
export function stripRepeatedPageLines(pages: string[]): string[] {
let current = pages.map((page) => page.split('\n'));
for (let pass = 0; pass < 2; pass++) {
const firstIdx: number[] = [];
const lastIdx: number[] = [];
const topCounts = new Map<string, number>();
const bottomCounts = new Map<string, number>();
let textPages = 0;
for (let p = 0; p < current.length; p++) {
const lines = current[p];
let first = 0;
while (first < lines.length && !lines[first].trim()) first++;
let last = lines.length - 1;
while (last >= 0 && !lines[last].trim()) last--;
firstIdx.push(first < lines.length ? first : -1);
lastIdx.push(last);
if (last < 0) continue;
textPages++;
const top = edgeLineKey(lines[first]);
if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1);
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1);
}
}
if (textPages < 4) break;
const threshold = Math.max(3, Math.ceil(textPages * 0.6));
let removed = false;
current = current.map((lines, p) => {
const first = firstIdx[p];
const last = lastIdx[p];
if (last < 0) return lines;
let from = 0;
let to = lines.length;
const top = edgeLineKey(lines[first]);
if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1;
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last;
}
if (from === 0 && to === lines.length) return lines;
removed = true;
return lines.slice(from, to);
});
if (!removed) break;
}
return current.map((lines) => lines.join('\n'));
}
// ---------------------------------------------------------------------------
// Chunking
// ---------------------------------------------------------------------------
/** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */
interface Atom {
text: string;
raw: number;
sep: string;
sepRaw: number;
/** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */
strength: number;
}
const WHITESPACE_RAW = 0.25;
/** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */
const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/;
const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/;
const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/;
const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/;
function sepRawOf(sep: string): number {
return sep ? WHITESPACE_RAW : 0;
}
/** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */
function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void {
let start = 0;
let acc = 0;
let first = true;
const push = (end: number) => {
const text = word.slice(start, end);
out.push({
text,
raw: rawTokens(text),
sep: first ? sep : '',
sepRaw: first ? sepRawOf(sep) : 0,
strength: first ? strength : -1,
});
first = false;
start = end;
acc = 0;
};
for (let i = 0; i < word.length; i++) {
const c = word.charCodeAt(i);
const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff;
const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost
if (acc + weight > limit && i > start && !isLowSurrogate) push(i);
acc += weight;
}
if (start < word.length) push(word.length);
}
function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void {
const raw = rawTokens(sentence);
if (raw <= atomMax) {
out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength });
return;
}
// Oversized sentence (or table row, or blob): fall back to groups of words.
const groupMax = Math.max(1, Math.min(atomMax, 24));
const words = sentence.split(/[ \t]+/);
// Keep leading indentation attached to the first word.
const indent = /^[ \t]+/.exec(sentence)?.[0] ?? '';
let group = '';
let groupRaw = 0;
let first = true;
const flush = () => {
if (!group) return;
out.push({
text: group,
raw: rawTokens(group),
sep: first ? sep : ' ',
sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW,
strength: first ? strength : 0,
});
first = false;
group = '';
groupRaw = 0;
};
let pendingIndent = indent;
for (const word of words) {
if (!word) continue;
const piece = pendingIndent + word;
pendingIndent = '';
const wordRaw = rawTokens(piece);
if (wordRaw > atomMax) {
flush();
hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0);
first = false;
continue;
}
if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush();
group = group ? `${group} ${piece}` : piece;
groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0);
}
flush();
}
/** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */
function atomize(text: string, atomMax: number): Atom[] {
const atoms: Atom[] = [];
let prevLine = '';
for (const para of text.split(/\n{2,}/)) {
let firstInPara = true;
for (const rawLine of para.split('\n')) {
const line = rawLine.trimEnd();
if (!line.trim()) continue;
let sep = '';
let strength = 0;
if (atoms.length > 0) {
const afterHeading = HEADING_LINE.test(prevLine);
const isHeading = HEADING_LINE.test(line);
if (firstInPara) {
sep = '\n\n';
strength = isHeading ? 4 : afterHeading ? -0.5 : 3;
} else {
sep = '\n';
if (isHeading) strength = 4;
else if (afterHeading) strength = -0.5;
else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2;
else strength = 0.5;
}
}
firstInPara = false;
prevLine = line;
if (rawTokens(line) <= Math.min(atomMax, 12)) {
// Short line: not worth looking for sentences.
pushSentence(line, atomMax, atoms, sep, strength);
continue;
}
let firstSentence = true;
for (const sentence of line.split(SENTENCE_BOUNDARY)) {
if (!sentence) continue;
pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1);
firstSentence = false;
}
}
}
return atoms;
}
function joinAtoms(atoms: Atom[], from: number, to: number): string {
let text = '';
for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text;
return text;
}
/** Trailing whole words of `text` worth at most `budget` tokens. */
function tailWords(text: string, budget: number): string {
let end = text.length;
let cut = text.length;
let acc = 0;
while (end > 0) {
let wordEnd = end;
while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--;
if (wordEnd === 0) break;
const space = text.lastIndexOf(' ', wordEnd - 1);
const word = text.slice(space + 1, wordEnd);
const cost = rawTokens(word) + WHITESPACE_RAW;
if (acc + cost > budget) break;
acc += cost;
cut = space + 1;
end = space < 0 ? 0 : space;
}
return text.slice(cut).trim();
}
/**
* Splits one oversized section into balanced pieces of at most `bodyBudget`
* tokens, cutting at the strongest boundary near the balanced size, and
* prefixing each later piece with up to `overlap` tokens of trailing context.
*/
function splitSection(text: string, bodyBudget: number, overlap: number): string[] {
const atomMax = Math.max(1, bodyBudget / 4);
const atoms = atomize(text, atomMax);
const n = atoms.length;
const pieces: string[] = [];
if (n === 0) return pieces;
// suffix[i] = tokens of atoms i..n-1 including the separators before each of them.
const suffix = new Float64Array(n + 1);
for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw;
let start = 0;
let prevStart = -1;
while (start < n) {
const remaining = suffix[start] - atoms[start].sepRaw;
let end: number;
if (remaining <= bodyBudget) {
end = n;
} else {
const goal = remaining / Math.ceil(remaining / bodyBudget);
const minFill = goal * 0.6;
let acc = 0;
let best = -1;
let bestStrength = -Infinity;
let bestDistance = Infinity;
let i = start;
for (; i < n; i++) {
const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0);
if (i > start && acc + add > bodyBudget) break;
acc += add;
if (i + 1 < n && acc >= minFill) {
const strength = atoms[i + 1].strength;
const distance = Math.abs(acc - goal);
if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) {
best = i;
bestStrength = strength;
bestDistance = distance;
}
}
}
end = best >= 0 ? best + 1 : Math.max(start + 1, i);
}
let prefix = '';
if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) {
// Trailing context from the previous piece: whole atoms if they fit, else trailing words.
let from = start;
let acc = 0;
while (from - 1 > prevStart) {
const candidate = atoms[from - 1];
const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0);
if (acc + cost > overlap) break;
acc += cost;
from--;
if (candidate.strength >= 4) break; // do not reach back past a heading
}
if (from < start) prefix = joinAtoms(atoms, from, start);
else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap);
}
const body = joinAtoms(atoms, start, end);
pieces.push(prefix ? prefix + atoms[start].sep + body : body);
prevStart = start;
start = end;
}
return pieces;
}
function positiveInt(value: number | undefined, fallback: number, min: number): number {
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback;
return Math.max(min, Math.floor(value));
}
interface Pending {
text: string;
heading: string;
page: number | null;
}
/**
* Builds retrieval chunks from the sections of ONE document.
*
* - Consecutive small sections (short slides, short pages) are packed together
* up to `targetTokens`; the chunk keeps the first section's heading and the
* full page range.
* - A section that fits comfortably under the ceiling stays whole so chunks
* line up with pages/slides.
* - Larger sections are split into balanced pieces at paragraph, then
* sentence, then word boundaries, with `overlapTokens` of repeated context
* between consecutive pieces of the same section only.
*/
export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] {
const target = positiveInt(opts.targetTokens, 450, 8);
const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8));
const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2));
/** Sections up to this size are kept in one piece. */
const wholeLimit = Math.floor(target + (max - target) / 2);
/** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */
const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1));
const chunks: DocChunk[] = [];
const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => {
const trimmed = text.trim();
if (!trimmed) return;
chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) });
};
let pending: Pending[] = [];
let pendingRaw = 0;
const flush = () => {
if (pending.length === 0) return;
let pageStart: number | null = null;
let pageEnd: number | null = null;
for (const item of pending) {
if (item.page === null) continue;
pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page);
pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page);
}
emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd);
pending = [];
pendingRaw = 0;
};
let lastHeading = '';
for (const section of sections ?? []) {
if (!section || typeof section.text !== 'string') continue;
const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : '';
if (ownHeading) lastHeading = ownHeading;
const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim();
if (!text) continue;
const heading = lastHeading;
const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null;
const raw = rawTokens(text);
if (raw > wholeLimit) {
flush();
for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page);
continue;
}
if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush();
pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0);
pending.push({ text, heading, page });
}
flush();
return chunks;
}

View file

@ -0,0 +1,420 @@
import { strToU8, zipSync } from 'fflate';
import { describe, expect, it } from 'vitest';
import { chunkSections, cleanText, estimateTokens } from './chunk.ts';
import { detectKind, extractDocument } from './extract.ts';
import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts';
import { ExtractionError, type ExtractedSection } from './types.ts';
const enc = (s: string) => new TextEncoder().encode(s);
const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n');
const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]);
/** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */
const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s;
async function codeOf(promise: Promise<unknown>): Promise<string> {
try {
await promise;
} catch (e) {
expect(e).toBeInstanceOf(ExtractionError);
return (e as ExtractionError).code;
}
throw new Error('expected rejection');
}
describe('detectKind', () => {
const pdf = buildPdf([{ lines: ['hello'] }]);
const docx = buildDocx([{ type: 'p', text: 'hi' }]);
const pptx = buildPptx([{ file: 1, title: 'T' }]);
it('detects by magic bytes', () => {
expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf');
expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image');
expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image');
expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image');
expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image');
expect(detectKind('a.docx', undefined, docx)).toBe('docx');
expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx');
});
it('trusts magic bytes over a wrong extension', () => {
expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf');
expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx');
expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image');
expect(detectKind('thing.bin', undefined, docx)).toBe('docx');
});
it('uses the extension for text and markdown', () => {
expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text');
expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text');
expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown');
});
it('rejects unknown binary, legacy .doc, plain zips and empty input', () => {
expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull();
expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull();
expect(detectKind('old.doc', undefined, OLE)).toBeNull();
expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull();
expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull();
});
});
describe('extractDocument: PDF', () => {
it('extracts multi-page text with page numbers, count and title', async () => {
const bytes = buildPdf(
[{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }],
{ title: 'Cardiology Notes' },
);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.kind).toBe('pdf');
expect(r.pageCount).toBe(2);
expect(r.title).toBe('Cardiology Notes');
expect(r.needsOcr).toBe(false);
expect(r.warnings).toEqual([]);
expect(r.sections.map((s) => s.page)).toEqual([1, 2]);
expect(r.sections[0].text).toContain('Heart failure overview');
expect(r.sections[1].text).toContain('Digoxin');
});
it('warns about a page without text but keeps page numbers', async () => {
const long = 'Readable text on this page, long enough to avoid OCR. ';
const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.pageCount).toBe(3);
expect(r.sections.map((s) => s.page)).toEqual([1, 3]);
expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true);
});
it('flags an all-empty scanned PDF as needing OCR', async () => {
const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' });
expect(r.needsOcr).toBe(true);
expect(r.sections).toEqual([]);
expect(r.pageCount).toBe(2);
expect(r.warnings.join(' ')).toMatch(/scanned/);
});
it('reports progress per page', async () => {
const seen: Array<[number, number]> = [];
await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) });
expect(seen).toEqual([
[1, 2],
[2, 2],
]);
});
});
describe('extractDocument: DOCX', () => {
it('turns headings into sections and keeps lists and tables readable', async () => {
const bytes = buildDocx(
[
{ type: 'heading', level: 1, text: 'Cardiac Drugs' },
{ type: 'p', text: 'Intro paragraph about drugs.' },
{ type: 'bullet', text: 'Beta blockers' },
{ type: 'bullet', text: 'ACE inhibitors' },
{ type: 'heading', level: 2, text: 'Dosing' },
{ type: 'number', text: 'Check heart rate' },
{ type: 'number', text: 'Give the dose' },
{
type: 'table',
rows: [
['Drug', 'Dose'],
['Digoxin', '0.25 mg'],
],
},
],
{ title: 'Drug Guide' },
);
const r = await extractDocument({ bytes, filename: 'a.docx' });
expect(r.kind).toBe('docx');
expect(r.title).toBe('Drug Guide');
expect(r.needsOcr).toBe(false);
expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']);
const first = r.sections[0].text;
expect(first).toContain('Intro paragraph');
expect(first).toContain('Beta blockers');
expect(first).toContain('ACE inhibitors');
const second = r.sections[1].text;
expect(second).toContain('Check heart rate');
expect(second).toContain('Digoxin');
expect(second).toContain('0.25 mg');
});
it('rejects a docx with no text as empty', async () => {
const bytes = buildDocx([{ type: 'p', text: '' }]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty');
});
});
describe('extractDocument: PPTX', () => {
it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => {
const bytes = buildPptx(
[
{ file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] },
{ file: 1, title: 'Middle', textBox: ['Free text box content here'] },
{ file: 2 },
{ file: 4, title: 'Closing', bullets: ['Thanks'] },
],
{ title: 'Deck Title' },
);
const r = await extractDocument({ bytes, filename: 'a.pptx' });
expect(r.kind).toBe('pptx');
expect(r.title).toBe('Deck Title');
expect(r.pageCount).toBe(4);
expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]);
expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']);
expect(r.sections[0].text).toContain('Welcome');
expect(r.sections[0].text).toContain('Sub point');
expect(r.sections[0].text).toContain('Say hello slowly');
expect(r.sections[1].text).toContain('Free text box content here');
expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored
expect(r.warnings).toEqual([]); // one empty slide is not worth a warning
});
it('reads tables', async () => {
const table = [
['A', 'B'],
['x', 'y'],
];
const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' });
expect(allText(r)).toContain('x');
expect(allText(r)).toContain('B');
});
it('warns when most slides have no text', async () => {
const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' });
expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true);
});
});
describe('extractDocument: text, markdown, image', () => {
it('splits markdown by heading and ignores headings inside code fences', async () => {
const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.';
const r = await extractDocument({ bytes: enc(md), filename: 'a.md' });
expect(r.kind).toBe('markdown');
expect(r.title).toBe('Title');
expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']);
expect(r.sections[1].text).toContain('# not a heading');
expect(r.needsOcr).toBe(false);
});
it('extracts plain text and falls back for non-UTF-8 with a warning', async () => {
const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' });
expect(r.kind).toBe('text');
expect(allText(r)).toContain('line two');
const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute
const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' });
expect(allText(r2)).toBe('café');
expect(r2.warnings.length).toBe(1);
});
it('groups long plain text into several bounded sections', async () => {
const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n');
const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' });
expect(r.sections.length).toBeGreaterThan(1);
for (const s of r.sections) expect(s.text.length).toBeLessThan(7000);
});
it('returns images with needsOcr and no sections', async () => {
const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' });
expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true });
});
});
describe('extractDocument: errors', () => {
it('rejects an empty file', async () => {
expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty');
});
it('rejects whitespace-only text as empty', async () => {
expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty');
});
it('rejects unsupported types and legacy .doc', async () => {
expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported');
await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/);
expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported');
});
it('rejects a password-protected OOXML wrapper as encrypted', async () => {
const marker = Buffer.from('EncryptedPackage', 'utf16le');
const bytes = Uint8Array.from([...OLE, ...marker]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted');
});
it('rejects corrupt PDFs', async () => {
expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt');
});
it('rejects an encrypted PDF', async () => {
const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted');
});
it('rejects a corrupt zip posing as docx/pptx', async () => {
const good = buildDocx([{ type: 'p', text: 'hello world' }]);
const broken = good.slice(0, Math.floor(good.length / 2));
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt');
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a pptx zip that has no readable slides', async () => {
const bytes = zipSync({ 'ppt/presentation.xml': strToU8('<p:presentation/>') });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a zip bomb whose headers lie about entry size', async () => {
const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024);
const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' }));
expect(['too_large', 'corrupt']).toContain(code);
}, 30000);
it('honours an already-aborted signal', async () => {
const ac = new AbortController();
ac.abort();
expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed');
});
});
describe('chunkSections', () => {
const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 };
it('keeps every chunk within maxTokens with contiguous 0-based seq', () => {
const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({
text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '),
page: i + 1,
}));
const chunks = chunkSections(sections, opts);
expect(chunks.length).toBeGreaterThan(12);
chunks.forEach((c, i) => {
expect(c.seq).toBe(i);
expect(c.tokens).toBe(estimateTokens(c.text));
expect(c.tokens).toBeLessThanOrEqual(150);
});
});
it('packs small sections together with correct page range and first heading', () => {
const sections: ExtractedSection[] = [
{ text: 'Alpha text.', page: 1, heading: 'First' },
{ text: 'Beta text.', page: 2 },
{ text: 'Gamma text.', page: 3, heading: 'Third' },
];
const chunks = chunkSections(sections, opts);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 });
expect(chunks[0].text).toContain('Gamma');
});
it('starts a new chunk when packing would exceed the target and tracks ranges', () => {
const big = (i: number) => ({ text: words(60, `s${i}x`), page: i });
const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 });
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(c.pageStart).not.toBeNull();
expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!);
}
expect(chunks[0].pageStart).toBe(1);
expect(chunks[chunks.length - 1].pageEnd).toBe(4);
});
it('uses null pages and empty heading when sections have none', () => {
const [c] = chunkSections([{ text: 'No pages here.' }]);
expect(c.pageStart).toBeNull();
expect(c.pageEnd).toBeNull();
expect(c.heading).toBe('');
});
it('splits oversized sections without cutting words', () => {
const text = words(800, 'term');
const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts);
expect(chunks.length).toBeGreaterThan(3);
const vocabulary = new Set(text.split(' '));
for (const c of chunks) {
expect(c.pageStart).toBe(5);
expect(c.pageEnd).toBe(5);
expect(c.heading).toBe('Big');
expect(c.tokens).toBeLessThanOrEqual(150);
for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true);
}
const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/)));
expect(seen.size).toBe(800);
});
it('survives a giant unbroken string within the limit', () => {
const blob = 'x'.repeat(200_000);
const t0 = Date.now();
const chunks = chunkSections([{ text: blob }], opts);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150);
expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000);
});
it('does not split surrogate pairs in unbroken emoji runs', () => {
const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(wellFormed(c.text)).toBe(true);
expect(c.tokens).toBeLessThanOrEqual(150);
}
});
it('is deterministic', () => {
const sections = [
{ text: words(900), page: 1, heading: 'H' },
{ text: 'Short one.', page: 2 },
];
expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts));
});
it('drops empty and whitespace-only sections', () => {
expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]);
const chunks = chunkSections([
{ text: ' ', page: 1 },
{ text: 'Real content.', page: 2 },
{ text: '\n', page: 3 },
]);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 });
});
it('chunks about 2 MB of text quickly', () => {
const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n';
const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length));
const t0 = Date.now();
const chunks = chunkSections([{ text }]);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(100);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700);
});
});
describe('cleanText', () => {
it('joins words hyphenated across a line break', () => {
expect(cleanText('the medica-\ntion was given')).toBe('the medication was given');
});
it('keeps real compounds across a line break', () => {
expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan');
});
it('collapses 3+ newlines to a paragraph break', () => {
expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb');
expect(cleanText('a\n\nb')).toBe('a\n\nb');
});
it('strips control characters but keeps tabs and newlines', () => {
expect(cleanText('he\u0000l\u0007lo​ ­world\ttab\nnext')).toBe('hello world\ttab\nnext');
});
it('normalises CRLF and trims', () => {
expect(cleanText(' a\r\nb \r\n')).toBe('a\nb');
expect(cleanText('')).toBe('');
});
it('is idempotent', () => {
const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny  z');
expect(cleanText(once)).toBe(once);
});
});

105
server/documents/docx.ts Normal file
View file

@ -0,0 +1,105 @@
/** DOCX extraction through mammoth: HTML -> Markdown-ish text, split into sections at H1-H3. */
import { zipSync } from 'fflate';
import mammoth from 'mammoth';
import { cleanText } from './chunk.ts';
import { htmlToBlocks } from './markup.ts';
import { readCoreTitle, type OfficeContext } from './pptx.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
import { ZipReader } from './zip.ts';
const STYLE_MAP = ["p[style-name='Title'] => h1:fresh", "p[style-name='Subtitle'] => p:fresh"];
const MAX_WARNINGS = 3;
/**
* Re-packs only the XML parts mammoth reads (no media, fonts or embeddings)
* into a fresh stored zip. Every byte went through the size-limited reader, so
* mammoth's own unzip can never be handed a bomb, and it never spends time
* base64-encoding images we would throw away.
*/
function sanitisedPackage(zip: ZipReader): { data: Uint8Array; images: number } {
const files: Record<string, Uint8Array> = {};
let images = 0;
for (const entry of zip.entries) {
const name = entry.name;
if (name.endsWith('/')) continue;
if (/^word\/media\//i.test(name)) {
images++;
continue;
}
if (!/\.(xml|rels)$/i.test(name)) continue;
if (/^(customXml|docProps\/thumbnail)/i.test(name)) continue;
const data = zip.read(name);
if (data) files[name] = data;
}
return { data: zipSync(files, { level: 0 }), images };
}
function friendlyWarnings(messages: Array<{ type: string; message: string }>): string[] {
const seen = new Set<string>();
for (const item of messages) {
const message = String(item?.message ?? '').trim();
// Unknown style names never lose text; they are noise for a reader.
if (!message || /^Unrecognised (paragraph|run|table) style/i.test(message)) continue;
seen.add(message.length > 160 ? `${message.slice(0, 157)}...` : message);
}
const unique = [...seen];
const out = unique.slice(0, MAX_WARNINGS).map((message) => `Some content may be missing: ${message}`);
if (unique.length > MAX_WARNINGS) out.push(`...and ${unique.length - MAX_WARNINGS} more conversion notes.`);
return out;
}
export async function extractDocx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
const zip = new ZipReader(bytes, ctx.limits);
if (!zip.names().some((name) => /^word\/document\d*\.xml$/i.test(name))) {
throw new ExtractionError('corrupt', "This file doesn't look like a valid Word document. Try saving it again as .docx and re-uploading.");
}
const { data, images } = sanitisedPackage(zip);
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
const converted = await mammoth.convertToHtml(
{ buffer: Buffer.from(data.buffer, data.byteOffset, data.byteLength) },
{
styleMap: STYLE_MAP,
ignoreEmptyParagraphs: true,
// Images are not read locally; never decode them.
convertImage: mammoth.images.imgElement(async () => ({ src: '' })),
},
);
const sections: ExtractedSection[] = [];
let heading: string | undefined;
let parts: string[] = [];
const flush = () => {
const text = cleanText(parts.join('\n\n'));
parts = [];
if (!text) return;
const section: ExtractedSection = { text };
if (heading) section.heading = heading;
sections.push(section);
};
for (const block of htmlToBlocks(converted.value)) {
if (block.level >= 1 && block.level <= 3) {
flush();
heading = block.text.slice(0, 200);
}
parts.push(block.level > 0 ? `${'#'.repeat(block.level)} ${block.text}` : block.text);
}
flush();
ctx.onProgress?.(1, 1);
const warnings = friendlyWarnings(converted.messages ?? []);
const totalChars = sections.reduce((sum, section) => sum + section.text.length, 0);
let needsOcr = false;
if (totalChars < 25) {
if (images === 0) {
throw new ExtractionError('empty', "This Word document doesn't contain any text.");
}
needsOcr = true;
warnings.unshift(`This document has almost no text, only ${images} image${images === 1 ? '' : 's'} - pictures of text are not read.`);
}
const result: ExtractResult = { kind: 'docx', sections, warnings, needsOcr };
const title = readCoreTitle(zip);
if (title) result.title = title;
return result;
}

362
server/documents/extract.ts Normal file
View file

@ -0,0 +1,362 @@
/**
* Document text extraction: kind detection plus one extractor per format.
* Every failure surfaces as an ExtractionError with a user-facing message.
*/
import { cleanText } from './chunk.ts';
import { extractDocx } from './docx.ts';
import { extractPdf } from './pdf.ts';
import { extractPptx } from './pptx.ts';
import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts';
import { listZip, ZipError, type ZipLimits } from './zip.ts';
/** Largest upload accepted (100 MB). */
export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024;
/** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */
const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 };
export interface ExtractInput {
bytes: Uint8Array;
filename: string;
mime?: string;
/** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */
onProgress?: (done: number, total: number) => void;
/** Aborting rejects with ExtractionError('failed') at the next page boundary. */
signal?: AbortSignal;
}
const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']);
const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']);
const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']);
const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']);
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation';
const SNIFF_BYTES = 256 * 1024;
function extensionOf(filename: string): string {
const base = (filename ?? '').split(/[\\/]/).pop() ?? '';
const dot = base.lastIndexOf('.');
return dot > 0 ? base.slice(dot + 1).toLowerCase() : '';
}
function baseMime(mime: string | undefined): string {
return (mime ?? '').split(';')[0].trim().toLowerCase();
}
function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean {
if (bytes.length < offset + signature.length) return false;
for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false;
return true;
}
function isImage(bytes: Uint8Array): boolean {
return (
startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG
startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a
(startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP
);
}
/** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */
function hasPdfHeader(bytes: Uint8Array): boolean {
const limit = Math.min(bytes.length - 5, 1024);
for (let i = 0; i <= limit; i++) {
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true;
}
return false;
}
function isZip(bytes: Uint8Array): boolean {
return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]);
}
/** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */
function isOle(bytes: Uint8Array): boolean {
return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
}
function isUtf16Bom(bytes: Uint8Array): boolean {
return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff));
}
function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean {
try {
new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail });
return true;
} catch {
return false;
}
}
/** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */
function looksLikeSingleByteText(sample: Uint8Array): boolean {
let high = 0;
for (let i = 0; i < sample.length; i++) {
const b = sample[i];
if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false;
if (b === 0x7f) return false;
if (b >= 0x80) high++;
}
return high <= sample.length * 0.3;
}
function isProbablyText(bytes: Uint8Array): boolean {
if (isUtf16Bom(bytes)) return bytes.length % 2 === 0;
const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes;
if (sample.includes(0)) return false;
return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample);
}
/**
* Works out what a file is. Magic bytes win over the name; the extension and
* MIME type are only a fallback. Returns null for anything unsupported
* (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk).
*/
export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null {
const ext = extensionOf(filename);
const type = baseMime(mime);
if (!bytes || bytes.length === 0) return null;
if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf';
if (isImage(bytes)) return 'image';
if (isZip(bytes)) {
let names: string[] | null = null;
try {
names = listZip(bytes).map((entry) => entry.name);
} catch {
names = null;
}
if (names) {
if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx';
if (names.includes('ppt/presentation.xml')) return 'pptx';
return null; // xlsx, odt, epub, jar, plain zip...
}
// Unreadable archive: trust the name so extraction can report it as damaged.
if (ext === 'docx' || type === DOCX_MIME) return 'docx';
if (ext === 'pptx' || type === PPTX_MIME) return 'pptx';
return null;
}
if (isOle(bytes)) {
// Password-protected .docx/.pptx are OLE containers; extraction explains that.
if (ext === 'docx') return 'docx';
if (ext === 'pptx') return 'pptx';
return null;
}
// No recognisable signature: fall back to the name / declared type.
if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf';
if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx';
if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx';
if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null;
if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null;
return null;
}
// ---------------------------------------------------------------------------
// Text and Markdown
// ---------------------------------------------------------------------------
function decodeText(bytes: Uint8Array, warnings: string[]): string {
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
return new TextDecoder('utf-8').decode(bytes.subarray(3));
}
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2));
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2));
try {
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
} catch {
if (bytes.includes(0)) {
throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.");
}
warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.');
return new TextDecoder('windows-1252').decode(bytes);
}
}
/** Target size of a plain-text section, in characters. */
const TEXT_SECTION_CHARS = 6000;
function splitPlainText(text: string): ExtractedSection[] {
const sections: ExtractedSection[] = [];
let current: string[] = [];
let size = 0;
const flush = (separator: string) => {
if (current.length === 0) return;
sections.push({ text: current.join(separator) });
current = [];
size = 0;
};
for (const block of text.split(/\n{2,}/)) {
if (!block.trim()) continue;
if (block.length > TEXT_SECTION_CHARS) {
// One huge block (CSV, logs, text without blank lines): fall back to line boundaries.
flush('\n\n');
for (const line of block.split('\n')) {
if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n');
current.push(line);
size += line.length + 1;
}
flush('\n');
continue;
}
if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n');
current.push(block);
size += block.length + 2;
}
flush('\n\n');
return sections;
}
function markdownHeadingText(raw: string): string {
return raw
.replace(/[ \t]+#+[ \t]*$/, '')
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
.replace(/\*\*|__|[*`]/g, '')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 200);
}
function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } {
const sections: ExtractedSection[] = [];
let title: string | undefined;
let heading: string | undefined;
let lines: string[] = [];
let fence: string | null = null;
const flush = () => {
const body = lines.join('\n').trim();
lines = [];
if (!body) return;
const section: ExtractedSection = { text: body };
if (heading) section.heading = heading;
sections.push(section);
};
for (const line of text.split('\n')) {
const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null;
if (fenceMatch) {
const marker = fenceMatch[1][0];
if (fence === null) fence = marker;
else if (fence === marker) fence = null;
} else if (fence === null && line.length < 400) {
const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line);
if (match) {
const headingText = markdownHeadingText(match[2]);
if (headingText) {
flush();
heading = headingText;
if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText;
}
}
}
lines.push(line);
}
flush();
return { sections, title };
}
function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult {
const warnings: string[] = [];
const text = cleanText(decodeText(bytes, warnings));
if (!text) throw new ExtractionError('empty', "This file doesn't contain any text.");
if (kind === 'markdown') {
const { sections, title } = splitMarkdown(text);
const result: ExtractResult = { kind, sections, warnings, needsOcr: false };
if (title) result.title = title;
return result;
}
return { kind, sections: splitPlainText(text), warnings, needsOcr: false };
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le');
function unsupportedMessage(filename: string, bytes: Uint8Array): string {
const ext = extensionOf(filename);
if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead.";
if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead.";
if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead.";
if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') {
return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`;
}
if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) {
return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.";
}
if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead.";
return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file.";
}
function fromZipError(error: ZipError, kind: DocKind): ExtractionError {
const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document';
if (error.code === 'too_large') {
return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error });
}
return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error });
}
/**
* Extracts the text of a document as ordered sections. Images return no
* sections and `needsOcr: true`; the caller runs vision transcription.
*/
export async function extractDocument(input: ExtractInput): Promise<ExtractResult> {
const bytes = input?.bytes;
const filename = typeof input?.filename === 'string' ? input.filename : '';
if (!(bytes instanceof Uint8Array) || bytes.length === 0) {
throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.');
}
if (bytes.length > MAX_DOCUMENT_BYTES) {
throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`);
}
if (input.signal?.aborted) {
throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason });
}
let kind: DocKind | null;
try {
kind = detectKind(filename, input.mime, bytes);
} catch (error) {
throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error });
}
if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes));
try {
switch (kind) {
case 'pdf':
return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal });
case 'docx':
case 'pptx': {
if (isOle(bytes)) {
const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) {
throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`);
}
throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes));
}
const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal };
return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx);
}
case 'text':
case 'markdown': {
const result = extractText(bytes, kind);
input.onProgress?.(1, 1);
return result;
}
case 'image':
return { kind: 'image', sections: [], needsOcr: true, warnings: [] };
}
} catch (error) {
if (error instanceof ExtractionError) throw error;
if (error instanceof ZipError) throw fromZipError(error, kind);
const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file';
if (kind === 'docx' || kind === 'pptx') {
throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error });
}
throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error });
}
}

272
server/documents/markup.ts Normal file
View file

@ -0,0 +1,272 @@
/** Small markup helpers shared by the DOCX and PPTX extractors. */
const NAMED_ENTITIES: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
};
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
export function decodeEntities(text: string): string {
if (!text.includes('&')) return text;
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
return String.fromCodePoint(code);
}
return NAMED_ENTITIES[body] ?? match;
});
}
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
export function renderTable(rows: string[][]): string {
const cleaned = rows
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
.filter((row) => row.some((cell) => cell.length > 0));
if (cleaned.length === 0) return '';
const width = Math.max(...cleaned.map((row) => row.length));
const line = (row: string[]) => {
const cells = row.slice();
while (cells.length < width) cells.push('');
return `| ${cells.join(' | ')} |`;
};
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
return out.join('\n');
}
// ---------------------------------------------------------------------------
// HTML (as produced by mammoth) -> Markdown blocks
// ---------------------------------------------------------------------------
export interface MarkdownBlock {
/** 1-6 for headings, 0 for everything else. */
level: number;
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
text: string;
}
interface HtmlElement {
tag: string;
children: HtmlNode[];
}
type HtmlNode = HtmlElement | string;
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
const HEADING_TAG = /^h[1-6]$/;
function parseHtml(html: string): HtmlElement {
const root: HtmlElement = { tag: '#root', children: [] };
const stack: HtmlElement[] = [root];
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
let match: RegExpExecArray | null;
while ((match = tokens.exec(html)) !== null) {
const top = stack[stack.length - 1];
if (match[4] !== undefined) {
top.children.push(decodeEntities(match[4]));
} else if (match[2] === undefined) {
top.children.push('<');
} else {
const tag = match[2].toLowerCase();
if (match[1]) {
// Close the nearest matching open element; ignore stray closers.
for (let i = stack.length - 1; i > 0; i--) {
if (stack[i].tag === tag) {
stack.length = i;
break;
}
}
} else {
const element: HtmlElement = { tag, children: [] };
top.children.push(element);
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
}
}
}
return root;
}
function collapse(text: string): string {
return text.replace(/[ \t\r\n]+/g, ' ');
}
function wrap(marker: string, inner: string): string {
const core = inner.trim();
if (!core) return inner;
const lead = inner.slice(0, inner.length - inner.trimStart().length);
const trail = inner.slice(inner.trimEnd().length);
return `${lead}${marker}${core}${marker}${trail}`;
}
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
function inlineText(nodes: HtmlNode[], plain: boolean): string {
let out = '';
for (const node of nodes) {
if (typeof node === 'string') {
out += collapse(node);
continue;
}
switch (node.tag) {
case 'br':
out += '\n';
break;
case 'img':
case 'hr':
break;
case 'strong':
case 'b':
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
break;
case 'em':
case 'i':
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
break;
case 'a': {
const text = inlineText(node.children, plain);
// mammoth's footnote back-references
if (text.trim() !== '↑') out += text;
break;
}
case 'p':
case 'li':
case 'tr':
case 'td':
case 'th':
case 'ul':
case 'ol':
case 'table': {
const inner = inlineText(node.children, plain).trim();
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
break;
}
default:
out += inlineText(node.children, plain);
}
}
return out;
}
function tidyInline(text: string): string {
return text
.split('\n')
.map((line) => line.replace(/ {2,}/g, ' ').trim())
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
.join('\n')
.trim();
}
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
const ordered = list.tag === 'ol';
let counter = 0;
for (const item of list.children) {
if (typeof item === 'string' || item.tag !== 'li') continue;
counter++;
const marker = ordered ? `${counter}. ` : '- ';
const inlineNodes: HtmlNode[] = [];
const nested: HtmlElement[] = [];
const paragraphs: string[] = [];
const flushInline = () => {
const text = tidyInline(inlineText(inlineNodes, false));
if (text) paragraphs.push(text);
inlineNodes.length = 0;
};
for (const child of item.children) {
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
flushInline();
nested.push(child);
} else if (typeof child !== 'string' && child.tag === 'p') {
flushInline();
inlineNodes.push(...child.children);
flushInline();
} else if (typeof child !== 'string' && child.tag === 'table') {
flushInline();
const table = renderHtmlTable(child);
if (table) paragraphs.push(table);
} else {
inlineNodes.push(child);
}
}
flushInline();
const pad = indent + ' '.repeat(marker.length);
const body = paragraphs.join('\n').split('\n');
if (body.length > 0 && body[0] !== '') {
lines.push(indent + marker + body[0]);
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
} else if (nested.length === 0) {
continue;
}
for (const child of nested) renderList(child, pad, lines);
}
}
function renderHtmlTable(table: HtmlElement): string {
const rows: string[][] = [];
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') continue;
if (node.tag === 'tr') {
const cells: string[] = [];
for (const cell of node.children) {
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
cells.push(tidyInline(inlineText(cell.children, false)));
}
rows.push(cells);
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
visit(node.children);
}
}
};
visit(table.children);
return renderTable(rows);
}
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
export function htmlToBlocks(html: string): MarkdownBlock[] {
const root = parseHtml(html);
const blocks: MarkdownBlock[] = [];
const loose: HtmlNode[] = [];
const flushLoose = () => {
if (loose.length === 0) return;
const text = tidyInline(inlineText(loose, false));
if (text) blocks.push({ level: 0, text });
loose.length = 0;
};
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') {
loose.push(node);
continue;
}
if (HEADING_TAG.test(node.tag)) {
flushLoose();
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
if (text) blocks.push({ level: Number(node.tag[1]), text });
} else if (node.tag === 'p') {
flushLoose();
const text = tidyInline(inlineText(node.children, false));
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'ul' || node.tag === 'ol') {
flushLoose();
const lines: string[] = [];
renderList(node, '', lines);
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
} else if (node.tag === 'table') {
flushLoose();
const text = renderHtmlTable(node);
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
flushLoose();
visit(node.children);
} else {
loose.push(node);
}
}
};
visit(root.children);
flushLoose();
return blocks;
}

249
server/documents/pdf.ts Normal file
View file

@ -0,0 +1,249 @@
/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
import { getDocumentProxy } from 'unpdf';
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
export interface PdfContext {
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
/** Below this average of characters per page the PDF is treated as a scan. */
const OCR_CHARS_PER_PAGE = 25;
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
const CLEANUP_EVERY = 40;
interface TextItemLike {
str?: unknown;
hasEOL?: unknown;
transform?: unknown;
height?: unknown;
}
interface Line {
text: string;
y: number | null;
height: number;
}
/**
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
* order (which keeps multi-column pages readable) with `hasEOL` marking line
* ends; the vertical gaps between lines are then used to restore paragraph
* breaks, which pdf.js does not report.
*/
export function textItemsToText(items: readonly TextItemLike[]): string {
const lines: Line[] = [];
let text = '';
let y: number | null = null;
let height = 0;
const endLine = () => {
lines.push({ text, y, height });
text = '';
y = null;
height = 0;
};
for (const item of items) {
if (!item || typeof item.str !== 'string') continue;
const str = item.str;
if (str.length > 0) {
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
const t = item.transform as number[];
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
if (upright && Number.isFinite(t[5])) {
y = t[5];
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
height = Number.isFinite(h) ? h : 0;
}
}
text += str;
}
if (item.hasEOL === true) endLine();
}
if (text.length > 0) endLine();
// Typical line advance on this page (median of the downward steps).
const advances: number[] = [];
for (let i = 1; i < lines.length; i++) {
const a = lines[i - 1];
const b = lines[i];
if (a.y === null || b.y === null) continue;
const dy = a.y - b.y;
const ref = Math.max(a.height, b.height, 1);
if (dy > 0 && dy < ref * 4) advances.push(dy);
}
advances.sort((p, q) => p - q);
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
let out = '';
let prev: Line | null = null;
for (const line of lines) {
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
if (!content) {
// An empty line is an explicit gap.
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
continue;
}
if (out && !out.endsWith('\n')) {
let gap = false;
if (prev && prev.y !== null && line.y !== null) {
const dy = prev.y - line.y;
const ref = Math.max(prev.height, line.height, 1);
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
else gap = dy > ref * 1.9;
}
out += gap ? '\n\n' : '\n';
}
out += content;
prev = line;
}
return out;
}
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
export function tidyTitle(raw: unknown): string | undefined {
if (typeof raw !== 'string') return undefined;
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
return title;
}
function errorName(error: unknown): string {
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
}
function toExtractionError(error: unknown): ExtractionError {
if (error instanceof ExtractionError) return error;
const name = errorName(error);
const message = error instanceof Error ? error.message : String(error);
if (name === 'PasswordException' || /password/i.test(message)) {
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
}
if (
name === 'InvalidPDFException' ||
name === 'FormatError' ||
name === 'XRefParseException' ||
name === 'XRefEntryException' ||
name === 'MissingDataException' ||
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
) {
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
}
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
}
function cancelled(signal: AbortSignal): ExtractionError {
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
}
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
const { onProgress, signal } = ctx;
if (signal?.aborted) throw cancelled(signal);
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
try {
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
pdf = await getDocumentProxy(bytes.slice(), options);
} catch (error) {
throw toExtractionError(error);
}
try {
const total = pdf.numPages;
if (!Number.isFinite(total) || total < 1) {
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
}
let title: string | undefined;
try {
const meta = await pdf.getMetadata();
const info = (meta?.info ?? {}) as Record<string, unknown>;
title = tidyTitle(info.Title);
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
} catch {
// Metadata is optional.
}
const pageTexts: string[] = new Array(total).fill('');
let failedPages = 0;
let lastYield = Date.now();
for (let n = 1; n <= total; n++) {
if (signal?.aborted) throw cancelled(signal);
try {
const page = await pdf.getPage(n);
try {
const content = await page.getTextContent();
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
} finally {
page.cleanup();
}
} catch (error) {
if (errorName(error) === 'PasswordException') throw error;
failedPages++;
}
if (n % CLEANUP_EVERY === 0) {
try {
await pdf.cleanup();
} catch {
// Best effort.
}
}
onProgress?.(n, total);
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
if (failedPages === total) {
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
}
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
const sections: ExtractedSection[] = [];
let empty = 0;
let chars = 0;
for (let i = 0; i < total; i++) {
const text = cleaned[i];
if (!text) {
empty++;
continue;
}
chars += text.length;
sections.push({ text, page: i + 1 });
}
const warnings: string[] = [];
if (empty > 0) {
warnings.push(
empty === total
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
: `${empty} of ${total} pages had no extractable text.`,
);
}
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
const result: ExtractResult = {
kind: 'pdf',
sections,
pageCount: total,
warnings,
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
};
if (title) result.title = title;
return result;
} catch (error) {
throw toExtractionError(error);
} finally {
try {
await pdf.loadingTask.destroy();
} catch {
// Nothing useful to do.
}
}
}

424
server/documents/pptx.ts Normal file
View file

@ -0,0 +1,424 @@
/** PPTX extraction: one section per slide, in presentation order, with titles, bullets, tables and notes. */
import { XMLParser } from 'fast-xml-parser';
import { cleanText } from './chunk.ts';
import { decodeEntities, renderTable } from './markup.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
import { ZipReader, type ZipLimits } from './zip.ts';
export interface OfficeContext {
limits: ZipLimits;
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
type XmlNode = Record<string, unknown>;
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '',
preserveOrder: true,
trimValues: false,
parseTagValue: false,
parseAttributeValue: false,
// Entities are decoded by decodeEntities so numeric references work and nothing is expanded twice.
processEntities: false,
ignoreDeclaration: true,
ignorePiTags: true,
});
function parseXml(xml: string): XmlNode[] {
const parsed = parser.parse(xml) as unknown;
return Array.isArray(parsed) ? (parsed as XmlNode[]) : [];
}
function tagOf(node: XmlNode): string {
for (const key in node) if (key !== ':@') return key;
return '';
}
/** Element name without its namespace prefix. */
function localName(node: XmlNode): string {
const tag = tagOf(node);
const colon = tag.indexOf(':');
return colon < 0 ? tag : tag.slice(colon + 1);
}
function childrenOf(node: XmlNode): XmlNode[] {
const value = node[tagOf(node)];
return Array.isArray(value) ? (value as XmlNode[]) : [];
}
function attr(node: XmlNode, name: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
const direct = attrs[name];
if (typeof direct === 'string') return direct;
return undefined;
}
/** Attribute lookup that ignores the namespace prefix (r:id, rel:id, ...). */
function attrLocal(node: XmlNode, local: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
for (const key in attrs) {
if (key.endsWith(`:${local}`) && typeof attrs[key] === 'string') return attrs[key] as string;
}
return undefined;
}
function child(node: XmlNode, local: string): XmlNode | undefined {
for (const c of childrenOf(node)) if (localName(c) === local) return c;
return undefined;
}
function findDeep(nodes: XmlNode[], local: string, depth = 0): XmlNode | undefined {
if (depth > 40) return undefined;
for (const node of nodes) {
if (localName(node) === local) return node;
const found = findDeep(childrenOf(node), local, depth + 1);
if (found) return found;
}
return undefined;
}
/** Concatenated text of every a:t / m:t descendant. */
function deepText(node: XmlNode, depth = 0): string {
if (depth > 40) return '';
const name = localName(node);
if (name === '#text') return '';
if (name === 't') {
let out = '';
for (const c of childrenOf(node)) if (typeof c['#text'] === 'string') out += c['#text'] as string;
return decodeEntities(out);
}
let out = '';
for (const c of childrenOf(node)) out += deepText(c, depth + 1);
return out;
}
interface Paragraph {
text: string;
level: number;
/** true: explicit bullet/number; false: bullets explicitly off; undefined: inherited. */
bullet: boolean | undefined;
}
function readParagraph(p: XmlNode): Paragraph {
let text = '';
let level = 0;
let bullet: boolean | undefined;
for (const c of childrenOf(p)) {
switch (localName(c)) {
case 'pPr': {
const lvl = Number(attr(c, 'lvl'));
if (Number.isFinite(lvl) && lvl > 0) level = Math.min(8, Math.floor(lvl));
for (const b of childrenOf(c)) {
const name = localName(b);
if (name === 'buNone') bullet = false;
else if (name === 'buChar' || name === 'buAutoNum' || name === 'buBlip') bullet = true;
}
break;
}
case 'br':
text += '\n';
break;
case 'fld': {
const type = attr(c, 'type') ?? '';
if (!/^(slidenum|datetime)/i.test(type)) text += deepText(c);
break;
}
case 'endParaRPr':
case '#text':
break;
default:
text += deepText(c);
}
}
text = text
.replace(/[\u000B\u2028]/g, '\n')
.split('\n')
.map((line) => line.replace(/[ \t\u00A0]+/g, ' ').trim())
.filter((line) => line.length > 0)
.join('\n');
return { text, level, bullet };
}
function readParagraphs(txBody: XmlNode): Paragraph[] {
const out: Paragraph[] = [];
for (const c of childrenOf(txBody)) {
if (localName(c) !== 'p') continue;
const paragraph = readParagraph(c);
if (paragraph.text) out.push(paragraph);
}
return out;
}
function renderParagraphs(paragraphs: Paragraph[], bulletsByDefault: boolean): string {
const lines: string[] = [];
for (const p of paragraphs) {
const bulleted = p.bullet ?? (bulletsByDefault || p.level > 0);
if (!bulleted) {
lines.push(p.text);
continue;
}
const indent = ' '.repeat(p.level);
const parts = p.text.split('\n');
lines.push(`${indent}- ${parts[0]}`);
for (let i = 1; i < parts.length; i++) lines.push(`${indent} ${parts[i]}`);
}
return lines.join('\n');
}
function readTable(tbl: XmlNode): string {
const rows: string[][] = [];
for (const tr of childrenOf(tbl)) {
if (localName(tr) !== 'tr') continue;
const cells: string[] = [];
for (const tc of childrenOf(tr)) {
if (localName(tc) !== 'tc') continue;
const txBody = child(tc, 'txBody');
cells.push(txBody ? readParagraphs(txBody).map((p) => p.text.replace(/\n/g, ' ')).join(' ') : '');
}
rows.push(cells);
}
return renderTable(rows);
}
interface SlideContent {
title: string;
blocks: string[];
}
const SKIPPED_PLACEHOLDERS = new Set(['sldNum', 'dt', 'ftr', 'hdr', 'sldImg']);
function placeholderType(shape: XmlNode): string | null {
const nv = child(shape, 'nvSpPr');
const nvPr = nv ? child(nv, 'nvPr') : undefined;
const ph = nvPr ? child(nvPr, 'ph') : undefined;
if (!ph) return null;
return attr(ph, 'type') ?? 'body';
}
function walkShapes(nodes: XmlNode[], into: SlideContent, depth: number): void {
if (depth > 30) return;
for (const node of nodes) {
switch (localName(node)) {
case 'sp': {
const type = placeholderType(node);
if (type && SKIPPED_PLACEHOLDERS.has(type)) break;
const txBody = child(node, 'txBody');
if (!txBody) break;
const paragraphs = readParagraphs(txBody);
if (paragraphs.length === 0) break;
if ((type === 'title' || type === 'ctrTitle') && !into.title) {
into.title = paragraphs.map((p) => p.text.replace(/\n/g, ' ')).join(' ').replace(/\s+/g, ' ').trim();
} else {
const bullets = type !== null && type !== 'subTitle' && type !== 'title' && type !== 'ctrTitle';
const text = renderParagraphs(paragraphs, bullets);
if (text) into.blocks.push(text);
}
break;
}
case 'grpSp':
walkShapes(childrenOf(node), into, depth + 1);
break;
case 'graphicFrame': {
const tbl = findDeep(childrenOf(node), 'tbl');
if (tbl) {
const table = readTable(tbl);
if (table) into.blocks.push(table);
}
break;
}
case 'AlternateContent': {
const choice = child(node, 'Choice') ?? child(node, 'Fallback');
if (choice) walkShapes(childrenOf(choice), into, depth + 1);
break;
}
default:
break;
}
}
}
function readSlide(xml: string): SlideContent {
const content: SlideContent = { title: '', blocks: [] };
const spTree = findDeep(parseXml(xml), 'spTree');
if (spTree) walkShapes(childrenOf(spTree), content, 0);
return content;
}
function readNotes(xml: string): string {
const spTree = findDeep(parseXml(xml), 'spTree');
if (!spTree) return '';
const parts: string[] = [];
for (const shape of childrenOf(spTree)) {
if (localName(shape) !== 'sp' || placeholderType(shape) !== 'body') continue;
const txBody = child(shape, 'txBody');
if (!txBody) continue;
const text = renderParagraphs(readParagraphs(txBody), false);
if (text) parts.push(text);
}
return parts.join('\n');
}
interface Relationship {
id: string;
type: string;
target: string;
external: boolean;
}
function readRelationships(xml: string | null): Relationship[] {
if (!xml) return [];
const out: Relationship[] = [];
const tags = /<(?:[A-Za-z0-9]+:)?Relationship\b([^>]*)>/g;
let match: RegExpExecArray | null;
while ((match = tags.exec(xml)) !== null) {
const attrs: Record<string, string> = {};
const pairs = /([A-Za-z:]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
let pair: RegExpExecArray | null;
while ((pair = pairs.exec(match[1])) !== null) attrs[pair[1]] = decodeEntities(pair[2] ?? pair[3] ?? '');
if (attrs.Id && attrs.Target) {
out.push({ id: attrs.Id, type: attrs.Type ?? '', target: attrs.Target, external: attrs.TargetMode === 'External' });
}
}
return out;
}
/** Resolves a relationship target against the directory of the part that owns it. */
function resolveTarget(baseDir: string, target: string): string {
const parts = target.startsWith('/') ? [] : baseDir.split('/').filter(Boolean);
for (const segment of target.split('/')) {
if (!segment || segment === '.') continue;
if (segment === '..') parts.pop();
else parts.push(segment);
}
return parts.join('/');
}
function relsPathFor(part: string): string {
const slash = part.lastIndexOf('/');
return `${part.slice(0, slash + 1)}_rels/${part.slice(slash + 1)}.rels`;
}
function slideOrder(zip: ZipReader): string[] {
const ordered: string[] = [];
const presentation = zip.readText('ppt/presentation.xml');
if (presentation) {
const targets = new Map<string, string>();
for (const rel of readRelationships(zip.readText('ppt/_rels/presentation.xml.rels'))) {
if (!rel.external) targets.set(rel.id, resolveTarget('ppt', rel.target));
}
const list = findDeep(parseXml(presentation), 'sldIdLst');
if (list) {
for (const item of childrenOf(list)) {
if (localName(item) !== 'sldId') continue;
const id = attrLocal(item, 'id');
const path = id ? targets.get(id) : undefined;
if (path && zip.has(path) && !ordered.includes(path)) ordered.push(path);
}
}
}
if (ordered.length > 0) return ordered;
// Damaged or unusual package: fall back to numeric file order.
return zip
.names()
.map((name) => ({ name, match: /^ppt\/slides\/slide(\d+)\.xml$/i.exec(name) }))
.filter((item): item is { name: string; match: RegExpExecArray } => item.match !== null)
.sort((a, b) => Number(a.match[1]) - Number(b.match[1]))
.map((item) => item.name);
}
export function readCoreTitle(zip: ZipReader): string | undefined {
let xml: string | null = null;
try {
xml = zip.readText('docProps/core.xml');
} catch {
return undefined;
}
if (!xml) return undefined;
const match = /<(?:[A-Za-z0-9]+:)?title\b[^>]*>([^<]*)</.exec(xml);
const title = match ? decodeEntities(match[1]).replace(/\s+/g, ' ').trim() : '';
return title || undefined;
}
export async function extractPptx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
const zip = new ZipReader(bytes, ctx.limits);
const slides = slideOrder(zip);
if (slides.length === 0) {
throw new ExtractionError('corrupt', "This PowerPoint file doesn't contain any slides we can read. Try saving it again and re-uploading.");
}
const sections: ExtractedSection[] = [];
const warnings: string[] = [];
let withoutText = 0;
let unreadable = 0;
let totalChars = 0;
let lastYield = Date.now();
for (let i = 0; i < slides.length; i++) {
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
const path = slides[i];
let text = '';
let title = '';
try {
const xml = zip.readText(path);
const slide = xml ? readSlide(xml) : { title: '', blocks: [] };
title = slide.title;
const parts: string[] = [];
if (slide.title) parts.push(`## ${slide.title}`);
parts.push(...slide.blocks);
const notesRel = readRelationships(zip.readText(relsPathFor(path))).find(
(rel) => !rel.external && /\/notesSlide$/.test(rel.type),
);
if (notesRel) {
const notesXml = zip.readText(resolveTarget(path.slice(0, path.lastIndexOf('/')), notesRel.target));
const notes = notesXml ? readNotes(notesXml) : '';
if (notes) parts.push(`Notes:\n${notes}`);
}
text = cleanText(parts.join('\n\n'));
} catch (error) {
if (error instanceof ExtractionError) throw error;
if ((error as { code?: string })?.code === 'too_large') throw error;
unreadable++;
}
if (text) {
totalChars += text.length;
const section: ExtractedSection = { text, page: i + 1 };
if (title) section.heading = title;
sections.push(section);
} else {
withoutText++;
}
ctx.onProgress?.(i + 1, slides.length);
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
const total = slides.length;
if (unreadable === total) {
throw new ExtractionError('corrupt', 'This PowerPoint file looks damaged and could not be read. Try saving it again and re-uploading.');
}
if (unreadable > 0) warnings.push(`${unreadable} of ${total} slides could not be read.`);
if (withoutText === total) {
warnings.push(`None of the ${total} slides had any text - the slides are probably images.`);
} else if (withoutText * 2 > total) {
warnings.push(`${withoutText} of ${total} slides had no text - most of this deck is probably images, which are not read.`);
}
const result: ExtractResult = {
kind: 'pptx',
sections,
pageCount: total,
warnings,
needsOcr: totalChars < total * 25,
};
const title = readCoreTitle(zip);
if (title) result.title = title;
return result;
}

View file

@ -0,0 +1,270 @@
/** Programmatic fixtures for the document tests. Nothing here is used by production code. */
import { strToU8, zipSync } from 'fflate';
const latin1 = (text: string): Uint8Array => Uint8Array.from(text, (ch) => ch.charCodeAt(0) & 0xff);
function pdfString(text: string): string {
return `(${text.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')})`;
}
export interface PdfPage {
/** Lines of text, drawn top-down at 12pt with 14pt leading. `''` leaves a paragraph gap. */
lines?: string[];
/** Raw content stream, used instead of `lines`. */
raw?: string;
}
export interface PdfOptions {
title?: string;
/** Adds a standard security handler whose user password is not empty. */
encrypted?: boolean;
}
/** Builds a small but valid PDF (Helvetica, one content stream per page). */
export function buildPdf(pages: PdfPage[], options: PdfOptions = {}): Uint8Array {
const objects: string[] = [];
const add = (body: string): number => {
objects.push(body);
return objects.length;
};
const catalogId = add('');
const pagesId = add('');
const fontId = add('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>');
const pageIds: number[] = [];
for (const page of pages) {
let stream = page.raw ?? '';
if (page.raw === undefined && page.lines && page.lines.length > 0) {
const ops: string[] = ['BT', '/F1 12 Tf', '14 TL', '72 720 Td'];
for (const line of page.lines) {
if (line === '') ops.push('0 -14 Td');
else ops.push(`${pdfString(line)} Tj`, 'T*');
}
ops.push('ET');
stream = ops.join('\n');
}
const contentId = add(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
pageIds.push(
add(
`<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
),
);
}
objects[catalogId - 1] = `<< /Type /Catalog /Pages ${pagesId} 0 R >>`;
objects[pagesId - 1] = `<< /Type /Pages /Count ${pageIds.length} /Kids [${pageIds.map((id) => `${id} 0 R`).join(' ')}] >>`;
let infoId = 0;
if (options.title !== undefined && !options.encrypted) infoId = add(`<< /Title ${pdfString(options.title)} /Producer (study-desk tests) >>`);
let encryptId = 0;
if (options.encrypted) {
const o = 'a1'.repeat(32);
const u = 'b2'.repeat(32);
encryptId = add(`<< /Filter /Standard /V 1 /R 2 /Length 40 /O <${o}> /U <${u}> /P -44 >>`);
}
let out = '%PDF-1.4\n%\xE2\xE3\xCF\xD3\n';
const offsets: number[] = [];
objects.forEach((body, index) => {
offsets.push(out.length);
out += `${index + 1} 0 obj\n${body}\nendobj\n`;
});
const xrefOffset = out.length;
out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`;
out += `trailer\n<< /Size ${objects.length + 1} /Root ${catalogId} 0 R`;
if (infoId) out += ` /Info ${infoId} 0 R`;
if (encryptId) out += ` /Encrypt ${encryptId} 0 R /ID [<0123456789abcdef0123456789abcdef> <0123456789abcdef0123456789abcdef>]`;
out += ` >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
return latin1(out);
}
const xmlEscape = (text: string): string => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
function coreProps(title: string | undefined): string {
return `${XML_HEADER}<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">${
title === undefined ? '' : `<dc:title>${xmlEscape(title)}</dc:title>`
}<dc:creator>Tests</dc:creator></cp:coreProperties>`;
}
// ---------------------------------------------------------------------------
// DOCX
// ---------------------------------------------------------------------------
export type DocxBlock =
| { type: 'heading'; level: number; text: string }
| { type: 'p'; text: string; bold?: string }
| { type: 'bullet'; text: string; level?: number }
| { type: 'number'; text: string; level?: number }
| { type: 'table'; rows: string[][] }
| { type: 'image' };
const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';
function docxRun(text: string, bold = false): string {
return `<w:r>${bold ? '<w:rPr><w:b/></w:rPr>' : ''}<w:t xml:space="preserve">${xmlEscape(text)}</w:t></w:r>`;
}
function docxBlock(block: DocxBlock): string {
switch (block.type) {
case 'heading':
return `<w:p><w:pPr><w:pStyle w:val="Heading${block.level}"/></w:pPr>${docxRun(block.text)}</w:p>`;
case 'p':
return `<w:p>${block.bold ? docxRun(block.bold, true) : ''}${docxRun(block.text)}</w:p>`;
case 'bullet':
case 'number':
return `<w:p><w:pPr><w:numPr><w:ilvl w:val="${block.level ?? 0}"/><w:numId w:val="${block.type === 'bullet' ? 1 : 2}"/></w:numPr></w:pPr>${docxRun(block.text)}</w:p>`;
case 'table':
return `<w:tbl>${block.rows
.map((row) => `<w:tr>${row.map((cell) => `<w:tc><w:p>${docxRun(cell)}</w:p></w:tc>`).join('')}</w:tr>`)
.join('')}</w:tbl>`;
case 'image':
return '<w:p/>';
}
}
/** Builds a minimal DOCX with headings (style ids Heading1..6), lists, tables. */
export function buildDocx(blocks: DocxBlock[], options: { title?: string; extraFiles?: Record<string, Uint8Array> } = {}): Uint8Array {
const levels = (format: string) =>
[0, 1, 2].map((lvl) => `<w:lvl w:ilvl="${lvl}"><w:start w:val="1"/><w:numFmt w:val="${format}"/><w:lvlText w:val="%${lvl + 1}."/></w:lvl>`).join('');
const files: Record<string, Uint8Array> = {
'[Content_Types].xml': strToU8(
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Default Extension="png" ContentType="image/png"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/></Types>`,
),
'_rels/.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/></Relationships>`,
),
'word/_rels/document.xml.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/></Relationships>`,
),
'word/numbering.xml': strToU8(
`${XML_HEADER}<w:numbering ${W_NS}><w:abstractNum w:abstractNumId="0">${levels('bullet')}</w:abstractNum><w:abstractNum w:abstractNumId="1">${levels('decimal')}</w:abstractNum><w:num w:numId="1"><w:abstractNumId w:val="0"/></w:num><w:num w:numId="2"><w:abstractNumId w:val="1"/></w:num></w:numbering>`,
),
'word/document.xml': strToU8(`${XML_HEADER}<w:document ${W_NS}><w:body>${blocks.map(docxBlock).join('')}<w:sectPr/></w:body></w:document>`),
'docProps/core.xml': strToU8(coreProps(options.title)),
...options.extraFiles,
};
return zipSync(files);
}
// ---------------------------------------------------------------------------
// PPTX
// ---------------------------------------------------------------------------
export interface PptxSlide {
/** File number: the slide is stored as ppt/slides/slide<file>.xml regardless of its position. */
file: number;
title?: string;
/** Body placeholder paragraphs. */
bullets?: Array<string | { text: string; level: number }>;
/** A free text box (not a placeholder). */
textBox?: string[];
table?: string[][];
notes?: string[];
}
const P_NS =
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"';
function aParagraph(text: string, level = 0): string {
return `<a:p>${level ? `<a:pPr lvl="${level}"/>` : ''}<a:r><a:rPr lang="en-US"/><a:t>${xmlEscape(text)}</a:t></a:r></a:p>`;
}
function shape(id: number, placeholder: string | null, paragraphs: string): string {
const ph = placeholder === null ? '' : placeholder === 'body' ? '<p:ph idx="1"/>' : `<p:ph type="${placeholder}"/>`;
return `<p:sp><p:nvSpPr><p:cNvPr id="${id}" name="Shape ${id}"/><p:cNvSpPr/><p:nvPr>${ph}</p:nvPr></p:nvSpPr><p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>${paragraphs}</p:txBody></p:sp>`;
}
function slideXml(slide: PptxSlide): string {
const shapes: string[] = [];
let id = 2;
if (slide.title !== undefined) shapes.push(shape(id++, 'title', aParagraph(slide.title)));
if (slide.bullets) {
shapes.push(
shape(id++, 'body', slide.bullets.map((b) => (typeof b === 'string' ? aParagraph(b) : aParagraph(b.text, b.level))).join('')),
);
}
if (slide.textBox) shapes.push(shape(id++, null, slide.textBox.map((line) => aParagraph(line)).join('')));
if (slide.table) {
const rows = slide.table
.map((row) => `<a:tr h="370840">${row.map((cell) => `<a:tc><a:txBody><a:bodyPr/><a:lstStyle/>${aParagraph(cell)}</a:txBody><a:tcPr/></a:tc>`).join('')}</a:tr>`)
.join('');
shapes.push(
`<p:graphicFrame><p:nvGraphicFramePr><p:cNvPr id="${id++}" name="Table"/><p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr><p:xfrm/><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/table"><a:tbl><a:tblPr/><a:tblGrid/>${rows}</a:tbl></a:graphicData></a:graphic></p:graphicFrame>`,
);
}
// Every slide also carries a slide-number placeholder, which must be ignored.
shapes.push(shape(id++, 'sldNum', '<a:p><a:fld id="{1}" type="slidenum"><a:t>9</a:t></a:fld></a:p>'));
return `${XML_HEADER}<p:sld ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shapes.join('')}</p:spTree></p:cSld></p:sld>`;
}
function notesXml(lines: string[]): string {
return `${XML_HEADER}<p:notes ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shape(
2,
'sldImg',
'',
)}${shape(3, 'body', lines.map((line) => aParagraph(line)).join(''))}${shape(4, 'sldNum', aParagraph('9'))}</p:spTree></p:cSld></p:notes>`;
}
/** Builds a minimal PPTX. Slides appear in the order given, whatever their file numbers. */
export function buildPptx(slides: PptxSlide[], options: { title?: string } = {}): Uint8Array {
const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships';
const files: Record<string, Uint8Array> = {
'[Content_Types].xml': strToU8(
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>`,
),
'_rels/.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/officeDocument" Target="ppt/presentation.xml"/></Relationships>`,
),
'docProps/core.xml': strToU8(coreProps(options.title)),
'ppt/presentation.xml': strToU8(
`${XML_HEADER}<p:presentation ${P_NS}><p:sldMasterIdLst/><p:sldIdLst>${slides
.map((slide, index) => `<p:sldId id="${256 + index}" r:id="rId${slide.file + 10}"/>`)
.join('')}</p:sldIdLst></p:presentation>`,
),
'ppt/_rels/presentation.xml.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">${slides
.map((slide) => `<Relationship Id="rId${slide.file + 10}" Type="${REL}/slide" Target="slides/slide${slide.file}.xml"/>`)
.join('')}</Relationships>`,
),
};
for (const slide of slides) {
files[`ppt/slides/slide${slide.file}.xml`] = strToU8(slideXml(slide));
if (slide.notes) {
files[`ppt/slides/_rels/slide${slide.file}.xml.rels`] = strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/notesSlide" Target="../notesSlides/notesSlide${slide.file}.xml"/></Relationships>`,
);
files[`ppt/notesSlides/notesSlide${slide.file}.xml`] = strToU8(notesXml(slide.notes));
}
}
return zipSync(files);
}
// ---------------------------------------------------------------------------
// Misc
// ---------------------------------------------------------------------------
/** PNG signature plus an IHDR chunk header: enough for signature sniffing. */
export const TINY_PNG = Uint8Array.from([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06,
0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89,
]);
export const TINY_JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01]);
export const TINY_GIF = Uint8Array.from([...'GIF89a'].map((ch) => ch.charCodeAt(0)).concat([1, 0, 1, 0, 0, 0, 0]));
export const TINY_WEBP = Uint8Array.from([...'RIFF'].map((c) => c.charCodeAt(0)).concat([4, 0, 0, 0], [...'WEBP'].map((c) => c.charCodeAt(0))));
/** A stored zip whose central directory claims a tiny size for a highly compressible entry. */
export function buildLyingZip(name: string, realBytes: number): Uint8Array {
const zip = zipSync({ [name]: new Uint8Array(realBytes), 'word/document.xml': strToU8('<w:document/>') }, { level: 9 });
// Patch the uncompressed size of `name` in its central directory record to 16 bytes.
const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
for (let i = 0; i + 46 < zip.length; i++) {
if (view.getUint32(i, true) !== 0x02014b50) continue;
const nameLen = view.getUint16(i + 28, true);
const entryName = new TextDecoder().decode(zip.subarray(i + 46, i + 46 + nameLen));
if (entryName === name) view.setUint32(i + 24, 16, true);
}
return zip;
}

53
server/documents/types.ts Normal file
View file

@ -0,0 +1,53 @@
/** Contract for the document extraction + chunking pipeline. */
export type DocKind = 'pdf' | 'docx' | 'pptx' | 'text' | 'markdown' | 'image';
export interface ExtractedSection {
/** Plain text (Markdown allowed) for this page / slide / section. */
text: string;
/** 1-based page (PDF) or slide (PPTX) number, when the format has one. */
page?: number;
/** Nearest heading or slide title, when known. */
heading?: string;
}
export interface ExtractResult {
kind: DocKind;
sections: ExtractedSection[];
/** Page or slide count for paged formats. */
pageCount?: number;
/** Non-fatal problems worth showing the user (e.g. "3 pages had no extractable text"). */
warnings: string[];
/** True when the file has little or no machine-readable text (scans, images) and needs AI vision/OCR. */
needsOcr: boolean;
/** Document title from metadata, when present. */
title?: string;
}
export interface DocChunk {
seq: number;
text: string;
heading: string;
pageStart: number | null;
pageEnd: number | null;
tokens: number;
}
export interface ChunkOptions {
/** Preferred chunk size in estimated tokens. */
targetTokens?: number;
/** Hard ceiling; a chunk is split before exceeding it. */
maxTokens?: number;
/** Tokens of trailing context repeated at the start of the next chunk. */
overlapTokens?: number;
}
/** Thrown for unrecoverable extraction problems; `message` is user-facing. */
export class ExtractionError extends Error {
code: 'unsupported' | 'encrypted' | 'corrupt' | 'empty' | 'too_large' | 'failed';
constructor(code: ExtractionError['code'], message: string, opts?: { cause?: unknown }) {
super(message, opts);
this.name = 'ExtractionError';
this.code = code;
}
}

213
server/documents/zip.ts Normal file
View file

@ -0,0 +1,213 @@
/**
* Minimal, defensive ZIP reader for OOXML packages (DOCX / PPTX).
*
* Reads the central directory itself and inflates entries through fflate's
* streaming inflater in small input slices, so both memory and CPU are bounded
* by the limits below even when the archive lies about its sizes (zip bombs).
*/
import { Inflate } from 'fflate';
export interface ZipEntry {
name: string;
method: number;
compressedSize: number;
size: number;
offset: number;
encrypted: boolean;
}
export interface ZipLimits {
/** Largest uncompressed size accepted for a single entry. */
maxEntryBytes: number;
/** Largest total of uncompressed bytes read through one reader. */
maxTotalBytes: number;
}
export class ZipError extends Error {
code: 'corrupt' | 'too_large';
constructor(code: ZipError['code'], message: string) {
super(message);
this.name = 'ZipError';
this.code = code;
}
}
const EOCD_SIG = 0x06054b50;
const ZIP64_LOCATOR_SIG = 0x07064b50;
const ZIP64_EOCD_SIG = 0x06064b50;
const CENTRAL_SIG = 0x02014b50;
const LOCAL_SIG = 0x04034b50;
const MAX_ENTRIES = 20_000;
const INFLATE_STEP = 64 * 1024;
const nameDecoder = new TextDecoder('utf-8');
function u16(b: Uint8Array, o: number): number {
return b[o] | (b[o + 1] << 8);
}
function u32(b: Uint8Array, o: number): number {
return (b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)) >>> 0;
}
function u64(b: Uint8Array, o: number): number {
return u32(b, o) + u32(b, o + 4) * 0x1_0000_0000;
}
/** Parses the central directory. Throws ZipError('corrupt') when it cannot be read. */
export function listZip(bytes: Uint8Array): ZipEntry[] {
const len = bytes.length;
if (len < 22) throw new ZipError('corrupt', 'Not a zip archive');
let eocd = -1;
const stop = Math.max(0, len - 22 - 0xffff);
for (let i = len - 22; i >= stop; i--) {
if (bytes[i] === 0x50 && u32(bytes, i) === EOCD_SIG) {
eocd = i;
break;
}
}
if (eocd < 0) throw new ZipError('corrupt', 'Zip directory not found');
let count = u16(bytes, eocd + 10);
let dirOffset = u32(bytes, eocd + 16);
if (count === 0xffff || dirOffset === 0xffffffff) {
const locator = eocd - 20;
if (locator < 0 || u32(bytes, locator) !== ZIP64_LOCATOR_SIG) throw new ZipError('corrupt', 'Bad zip64 locator');
const z = u64(bytes, locator + 8);
if (z + 56 > len || u32(bytes, z) !== ZIP64_EOCD_SIG) throw new ZipError('corrupt', 'Bad zip64 directory');
count = u64(bytes, z + 32);
dirOffset = u64(bytes, z + 48);
}
if (count > MAX_ENTRIES) throw new ZipError('too_large', 'Too many files in archive');
const entries: ZipEntry[] = [];
let p = dirOffset;
for (let i = 0; i < count; i++) {
if (p + 46 > len || u32(bytes, p) !== CENTRAL_SIG) throw new ZipError('corrupt', 'Bad zip directory entry');
const flags = u16(bytes, p + 8);
const method = u16(bytes, p + 10);
let compressedSize = u32(bytes, p + 20);
let size = u32(bytes, p + 24);
const nameLen = u16(bytes, p + 28);
const extraLen = u16(bytes, p + 30);
const commentLen = u16(bytes, p + 32);
let offset = u32(bytes, p + 42);
const nameStart = p + 46;
const extraStart = nameStart + nameLen;
const next = extraStart + extraLen + commentLen;
if (next > len) throw new ZipError('corrupt', 'Truncated zip directory');
if (size === 0xffffffff || compressedSize === 0xffffffff || offset === 0xffffffff) {
// zip64 extended information: only the saturated fields are present, in this order.
let e = extraStart;
const extraEnd = extraStart + extraLen;
while (e + 4 <= extraEnd) {
const id = u16(bytes, e);
const dataLen = u16(bytes, e + 2);
if (id === 1) {
let q = e + 4;
if (size === 0xffffffff && q + 8 <= extraEnd) (size = u64(bytes, q)), (q += 8);
if (compressedSize === 0xffffffff && q + 8 <= extraEnd) (compressedSize = u64(bytes, q)), (q += 8);
if (offset === 0xffffffff && q + 8 <= extraEnd) offset = u64(bytes, q);
break;
}
e += 4 + dataLen;
}
}
const name = nameDecoder.decode(bytes.subarray(nameStart, extraStart)).replace(/\\/g, '/');
entries.push({ name, method, compressedSize, size, offset, encrypted: (flags & 1) === 1 });
p = next;
}
return entries;
}
/** Random-access reader over a zip held in memory, with per-entry and total size limits. */
export class ZipReader {
private readonly bytes: Uint8Array;
private readonly limits: ZipLimits;
private readonly byName = new Map<string, ZipEntry>();
private totalRead = 0;
readonly entries: ZipEntry[];
constructor(bytes: Uint8Array, limits: ZipLimits) {
this.bytes = bytes;
this.limits = limits;
this.entries = listZip(bytes);
for (const entry of this.entries) if (!this.byName.has(entry.name)) this.byName.set(entry.name, entry);
}
has(name: string): boolean {
return this.byName.has(name);
}
names(): string[] {
return this.entries.map((entry) => entry.name);
}
/** Uncompressed bytes of an entry, or null when it does not exist. */
read(name: string): Uint8Array | null {
const entry = this.byName.get(name);
if (!entry) return null;
if (entry.encrypted) throw new ZipError('corrupt', 'Encrypted zip entry');
if (entry.size > this.limits.maxEntryBytes) throw new ZipError('too_large', `Entry too large: ${name}`);
if (this.totalRead + entry.size > this.limits.maxTotalBytes) throw new ZipError('too_large', 'Archive too large');
const b = this.bytes;
const o = entry.offset;
if (o + 30 > b.length || u32(b, o) !== LOCAL_SIG) throw new ZipError('corrupt', 'Bad zip file header');
const dataStart = o + 30 + u16(b, o + 26) + u16(b, o + 28);
const dataEnd = dataStart + entry.compressedSize;
if (dataEnd > b.length) throw new ZipError('corrupt', 'Truncated zip entry');
const compressed = b.subarray(dataStart, dataEnd);
let out: Uint8Array;
if (entry.method === 0) {
if (compressed.length !== entry.size) throw new ZipError('corrupt', 'Zip entry size mismatch');
out = compressed;
} else if (entry.method === 8) {
out = this.inflate(compressed, entry.size);
} else {
throw new ZipError('corrupt', `Unsupported zip compression method ${entry.method}`);
}
this.totalRead += out.length;
return out;
}
readText(name: string): string | null {
const data = this.read(name);
if (!data) return null;
let text = new TextDecoder('utf-8').decode(data);
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
return text;
}
private inflate(compressed: Uint8Array, expected: number): Uint8Array {
const out = new Uint8Array(expected);
let written = 0;
let overflow = false;
const inflater = new Inflate((chunk) => {
if (overflow) return;
if (written + chunk.length > expected) {
overflow = true;
return;
}
out.set(chunk, written);
written += chunk.length;
});
try {
if (compressed.length === 0) {
inflater.push(compressed, true);
} else {
for (let p = 0; p < compressed.length && !overflow; p += INFLATE_STEP) {
const end = Math.min(p + INFLATE_STEP, compressed.length);
inflater.push(compressed.subarray(p, end), end === compressed.length);
}
}
} catch {
throw new ZipError('corrupt', 'Zip entry could not be decompressed');
}
// More data than the directory declared is the signature of a zip bomb.
if (overflow) throw new ZipError('too_large', 'Zip entry larger than declared');
if (written !== expected) throw new ZipError('corrupt', 'Zip entry size mismatch');
return out;
}
}

63
server/http.ts Normal file
View file

@ -0,0 +1,63 @@
import type { Context } from 'hono';
import { z } from 'zod';
import type { Config } from './config.ts';
import type { DB } from './db.ts';
/** Everything a route needs. Built once in main.ts (or per test). */
export interface AppContext {
db: DB;
config: Config;
log: (level: 'info' | 'warn' | 'error', message: string, extra?: unknown) => void;
}
export class HttpError extends Error {
status: number;
code: string;
details?: unknown;
constructor(status: number, code: string, message: string, details?: unknown) {
super(message);
this.name = 'HttpError';
this.status = status;
this.code = code;
this.details = details;
}
}
export const notFound = (what = 'Item') => new HttpError(404, 'not_found', `${what} not found`);
export const badRequest = (message: string, details?: unknown) => new HttpError(400, 'bad_request', message, details);
export const conflict = (message: string) => new HttpError(409, 'conflict', message);
export interface FieldIssue {
path: string;
message: string;
}
export function zodIssues(error: z.ZodError): FieldIssue[] {
return error.issues.map((i) => ({ path: i.path.join('.'), message: i.message }));
}
export function parse<S extends z.ZodType>(schema: S, data: unknown): z.output<S> {
const result = schema.safeParse(data);
if (!result.success) {
const issues = zodIssues(result.error);
const first = issues[0];
throw new HttpError(422, 'validation', first ? `${first.path ? `${first.path}: ` : ''}${first.message}` : 'Invalid input', issues);
}
return result.data;
}
export async function readBody<S extends z.ZodType>(c: Context, schema: S): Promise<z.output<S>> {
let json: unknown;
try {
json = await c.req.json();
} catch {
throw badRequest('Request body must be JSON');
}
return parse(schema, json);
}
export const idParam = (c: Context, name = 'id'): string => {
const v = c.req.param(name);
if (!v || v.length > 64) throw badRequest('Invalid id');
return v;
};

372
server/migrations.ts Normal file
View file

@ -0,0 +1,372 @@
/**
* Schema migrations. Append only: never edit a migration that has shipped.
* Timestamps are integer milliseconds since the Unix epoch. JSON lives in TEXT columns.
*/
export interface Migration {
id: number;
name: string;
sql: string;
}
const init = `
CREATE TABLE settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE sessions (
token_hash TEXT PRIMARY KEY,
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL,
last_seen_at INTEGER NOT NULL,
user_agent TEXT NOT NULL DEFAULT '',
ip TEXT NOT NULL DEFAULT ''
);
CREATE TABLE classes (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
code TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT '',
color TEXT NOT NULL DEFAULT 'cobalt',
archived INTEGER NOT NULL DEFAULT 0,
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE folders (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
name TEXT NOT NULL,
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX folders_class ON folders(class_id, sort_order);
CREATE TABLE quizzes (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX quizzes_class ON quizzes(class_id, sort_order);
CREATE TABLE questions (
id TEXT PRIMARY KEY,
quiz_id TEXT NOT NULL REFERENCES quizzes(id) ON DELETE CASCADE,
type TEXT NOT NULL CHECK (type IN ('single','multi')),
stem TEXT NOT NULL,
options TEXT NOT NULL,
rationale TEXT NOT NULL DEFAULT '',
topic TEXT NOT NULL DEFAULT '',
difficulty TEXT NOT NULL DEFAULT 'medium',
tags TEXT NOT NULL DEFAULT '[]',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX questions_quiz ON questions(quiz_id, sort_order);
CREATE TABLE attempts (
id TEXT PRIMARY KEY,
quiz_id TEXT REFERENCES quizzes(id) ON DELETE SET NULL,
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
title TEXT NOT NULL,
mode TEXT NOT NULL CHECK (mode IN ('practice','exam')),
kind TEXT NOT NULL DEFAULT 'full',
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','completed')),
settings TEXT NOT NULL DEFAULT '{}',
snapshot TEXT NOT NULL,
answers TEXT NOT NULL DEFAULT '{}',
flags TEXT NOT NULL DEFAULT '[]',
current_index INTEGER NOT NULL DEFAULT 0,
elapsed_sec INTEGER NOT NULL DEFAULT 0,
question_count INTEGER NOT NULL DEFAULT 0,
score REAL,
max_score REAL,
started_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
completed_at INTEGER
);
CREATE INDEX attempts_quiz ON attempts(quiz_id, status, completed_at);
CREATE INDEX attempts_status ON attempts(status, updated_at);
CREATE TABLE responses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
attempt_id TEXT NOT NULL REFERENCES attempts(id) ON DELETE CASCADE,
question_id TEXT NOT NULL,
quiz_id TEXT,
class_id TEXT,
topic TEXT NOT NULL DEFAULT '',
score REAL NOT NULL,
correct INTEGER NOT NULL,
answered_at INTEGER NOT NULL,
UNIQUE (attempt_id, question_id)
);
CREATE INDEX responses_question ON responses(question_id, answered_at);
CREATE INDEX responses_time ON responses(answered_at);
CREATE INDEX responses_class ON responses(class_id, answered_at);
CREATE TABLE decks (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX decks_class ON decks(class_id, sort_order);
CREATE TABLE cards (
id TEXT PRIMARY KEY,
deck_id TEXT NOT NULL REFERENCES decks(id) ON DELETE CASCADE,
front TEXT NOT NULL,
back TEXT NOT NULL,
hint TEXT NOT NULL DEFAULT '',
tags TEXT NOT NULL DEFAULT '[]',
sort_order INTEGER NOT NULL DEFAULT 0,
suspended INTEGER NOT NULL DEFAULT 0,
state INTEGER NOT NULL DEFAULT 0,
due INTEGER,
stability REAL NOT NULL DEFAULT 0,
difficulty REAL NOT NULL DEFAULT 0,
elapsed_days INTEGER NOT NULL DEFAULT 0,
scheduled_days INTEGER NOT NULL DEFAULT 0,
learning_steps INTEGER NOT NULL DEFAULT 0,
reps INTEGER NOT NULL DEFAULT 0,
lapses INTEGER NOT NULL DEFAULT 0,
last_review INTEGER,
introduced_at INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX cards_deck ON cards(deck_id, sort_order);
CREATE INDEX cards_due ON cards(state, due);
CREATE TABLE reviews (
id INTEGER PRIMARY KEY AUTOINCREMENT,
card_id TEXT NOT NULL,
deck_id TEXT,
class_id TEXT,
rating INTEGER NOT NULL,
state_before INTEGER NOT NULL DEFAULT 0,
scheduled_days INTEGER NOT NULL DEFAULT 0,
mode TEXT NOT NULL DEFAULT 'srs',
prev TEXT NOT NULL DEFAULT '',
reviewed_at INTEGER NOT NULL
);
CREATE INDEX reviews_time ON reviews(reviewed_at);
CREATE INDEX reviews_card ON reviews(card_id, reviewed_at);
CREATE TABLE deck_sessions (
deck_id TEXT PRIMARY KEY REFERENCES decks(id) ON DELETE CASCADE,
state TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE notes (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
content TEXT NOT NULL DEFAULT '',
kind TEXT NOT NULL DEFAULT 'note',
source TEXT NOT NULL DEFAULT '{}',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX notes_class ON notes(class_id, sort_order);
CREATE TABLE documents (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
filename TEXT NOT NULL,
mime TEXT NOT NULL DEFAULT '',
kind TEXT NOT NULL,
size INTEGER NOT NULL DEFAULT 0,
sha256 TEXT NOT NULL DEFAULT '',
storage_name TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
error TEXT NOT NULL DEFAULT '',
progress REAL NOT NULL DEFAULT 0,
page_count INTEGER,
char_count INTEGER NOT NULL DEFAULT 0,
token_count INTEGER NOT NULL DEFAULT 0,
chunk_count INTEGER NOT NULL DEFAULT 0,
needs_ocr INTEGER NOT NULL DEFAULT 0,
warnings TEXT NOT NULL DEFAULT '[]',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX documents_class ON documents(class_id, sort_order);
CREATE TABLE chunks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
class_id TEXT NOT NULL,
seq INTEGER NOT NULL,
text TEXT NOT NULL,
heading TEXT NOT NULL DEFAULT '',
page_start INTEGER,
page_end INTEGER,
tokens INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX chunks_doc ON chunks(document_id, seq);
CREATE VIRTUAL TABLE chunks_fts USING fts5(text, heading, content='chunks', content_rowid='id', tokenize='porter unicode61');
CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN
INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
END;
CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text, heading) VALUES ('delete', old.id, old.text, old.heading);
END;
CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text, heading) VALUES ('delete', old.id, old.text, old.heading);
INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
END;
CREATE TABLE shares (
id TEXT PRIMARY KEY,
token TEXT NOT NULL UNIQUE,
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','folder','class','note','document')),
target_id TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
view_count INTEGER NOT NULL DEFAULT 0,
last_viewed_at INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (kind, target_id)
);
CREATE TABLE templates (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','generation')),
name TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
payload TEXT NOT NULL,
builtin INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE media (
id TEXT PRIMARY KEY,
filename TEXT NOT NULL,
mime TEXT NOT NULL,
size INTEGER NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE ai_providers (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
kind TEXT NOT NULL,
base_url TEXT,
api_key_enc TEXT,
headers_enc TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
last_test_at INTEGER,
last_test_ok INTEGER,
last_test_message TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE ai_models (
id TEXT PRIMARY KEY,
provider_id TEXT NOT NULL REFERENCES ai_providers(id) ON DELETE CASCADE,
model TEXT NOT NULL,
label TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT 'discovered',
enabled INTEGER NOT NULL DEFAULT 0,
capabilities TEXT NOT NULL DEFAULT '{}',
context_window INTEGER,
max_output INTEGER,
price_in REAL,
price_out REAL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (provider_id, model)
);
CREATE TABLE ai_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
at INTEGER NOT NULL,
provider_id TEXT,
provider_name TEXT NOT NULL DEFAULT '',
model TEXT NOT NULL DEFAULT '',
task TEXT NOT NULL DEFAULT '',
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cost REAL,
ok INTEGER NOT NULL DEFAULT 1,
error TEXT NOT NULL DEFAULT '',
duration_ms INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX ai_usage_at ON ai_usage(at);
CREATE TABLE conversations (
id TEXT PRIMARY KEY,
title TEXT NOT NULL DEFAULT '',
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
role TEXT NOT NULL,
content TEXT NOT NULL,
meta TEXT NOT NULL DEFAULT '{}',
created_at INTEGER NOT NULL
);
CREATE INDEX messages_conv ON messages(conversation_id, id);
CREATE TABLE drafts (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','note')),
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
folder_id TEXT,
title TEXT NOT NULL DEFAULT '',
payload TEXT NOT NULL DEFAULT '{}',
meta TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL DEFAULT 'ready',
progress REAL NOT NULL DEFAULT 0,
progress_label TEXT NOT NULL DEFAULT '',
error TEXT NOT NULL DEFAULT '',
saved_kind TEXT,
saved_id TEXT,
conversation_id TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX drafts_status ON drafts(status, updated_at);
CREATE TABLE ai_actions (
id TEXT PRIMARY KEY,
conversation_id TEXT,
kind TEXT NOT NULL,
summary TEXT NOT NULL DEFAULT '',
payload TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL DEFAULT 'pending',
result TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
resolved_at INTEGER
);
`;
export const MIGRATIONS: Migration[] = [{ id: 1, name: 'init', sql: init }];

564
server/routes/library.ts Normal file
View file

@ -0,0 +1,564 @@
import fs from 'node:fs';
import path from 'node:path';
import { Hono } from 'hono';
import { z } from 'zod';
import { CLASS_COLORS, deckCreateSchema, descriptionSchema, questionToDraft, quizCreateSchema, titleSchema } from '../../shared/content.ts';
import { exportDeck, exportQuiz } from '../../shared/importer.ts';
import { badRequest, idParam, notFound, readBody, type AppContext } from '../http.ts';
import { getAppSettings } from '../settings.ts';
import { listAttempts } from '../services/attempts.ts';
import {
classInputSchema,
createClass,
createDeck,
createFolder,
createNote,
createQuiz,
duplicateDeck,
duplicateQuiz,
getClass,
getDeck,
getFolder,
getNote,
getQuiz,
insertCards,
insertQuestions,
listCards,
listQuestions,
mapClass,
mapDeck,
mapFolder,
mapNote,
mapQuiz,
resolveFolder,
updateCard,
updateQuestion,
type ClassRow,
type DeckRow,
type FolderRow,
type NoteRow,
type QuizRow,
} from '../services/content.ts';
import { deckStats, dueCounts, resetCards } from '../services/srs.ts';
import { questionStats } from '../services/stats.ts';
import { now, parseJson, slugify } from '../util.ts';
function download(c: import('hono').Context, filename: string, data: unknown) {
c.header('Content-Disposition', `attachment; filename="${filename}"`);
return c.json(data);
}
export function shareFor(app: AppContext, kind: string, targetId: string) {
const row = app.db.get<{ id: string; token: string; enabled: number; view_count: number; last_viewed_at: number | null; created_at: number }>(
'SELECT * FROM shares WHERE kind = ? AND target_id = ?',
kind,
targetId,
);
return row
? { id: row.id, token: row.token, enabled: Boolean(row.enabled), viewCount: row.view_count, lastViewedAt: row.last_viewed_at, createdAt: row.created_at }
: null;
}
export function libraryRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
const settings = () => getAppSettings(db, app.config.timezone);
/* ------------------------------ classes ------------------------------ */
r.get('/classes', (c) => {
const rows = db.all<ClassRow>('SELECT * FROM classes ORDER BY archived, sort_order, created_at');
const count = (table: string) =>
new Map(db.all<{ class_id: string; n: number }>(`SELECT class_id, COUNT(*) AS n FROM ${table} GROUP BY class_id`).map((x) => [x.class_id, x.n]));
const quizzes = count('quizzes');
const decks = count('decks');
const documents = count('documents');
const notes = count('notes');
const s = settings();
return c.json({
classes: rows.map((row) => ({
...mapClass(row),
counts: { quizzes: quizzes.get(row.id) ?? 0, decks: decks.get(row.id) ?? 0, documents: documents.get(row.id) ?? 0, notes: notes.get(row.id) ?? 0 },
due: row.archived ? 0 : dueCounts(db, { classId: row.id }, s).total,
})),
});
});
r.post('/classes', async (c) => {
const body = await readBody(c, classInputSchema);
return c.json({ class: mapClass(createClass(db, body)) }, 201);
});
r.post('/classes/reorder', async (c) => {
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(500) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE classes SET sort_order = ? WHERE id = ?', i, id)));
return c.json({ ok: true });
});
r.get('/classes/:id', (c) => {
const cls = getClass(db, idParam(c));
const s = settings();
const shares = new Map(
db.all<{ kind: string; target_id: string; enabled: number }>('SELECT kind, target_id, enabled FROM shares').map((x) => [`${x.kind}:${x.target_id}`, Boolean(x.enabled)]),
);
const shared = (kind: string, id: string) => shares.get(`${kind}:${id}`) ?? false;
const quizzes = db.all<QuizRow & { qn: number; attempts: number; last_score: number | null; last_max: number | null; last_at: number | null; active_id: string | null }>(
`SELECT z.*,
(SELECT COUNT(*) FROM questions q WHERE q.quiz_id = z.id) AS qn,
(SELECT COUNT(*) FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed') AS attempts,
(SELECT a.score FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed' AND a.kind = 'full' ORDER BY a.completed_at DESC LIMIT 1) AS last_score,
(SELECT a.max_score FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed' AND a.kind = 'full' ORDER BY a.completed_at DESC LIMIT 1) AS last_max,
(SELECT a.completed_at FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed' ORDER BY a.completed_at DESC LIMIT 1) AS last_at,
(SELECT a.id FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'active' ORDER BY a.updated_at DESC LIMIT 1) AS active_id
FROM quizzes z WHERE z.class_id = ?`,
cls.id,
);
const decks = db.all<DeckRow & { cn: number }>(`SELECT d.*, (SELECT COUNT(*) FROM cards c WHERE c.deck_id = d.id) AS cn FROM decks d WHERE d.class_id = ?`, cls.id);
const documents = db.all<Record<string, any>>(
'SELECT id, folder_id, title, filename, kind, size, status, error, progress, page_count, token_count, needs_ocr, created_at, updated_at FROM documents WHERE class_id = ?',
cls.id,
);
const notes = db.all<NoteRow>('SELECT * FROM notes WHERE class_id = ?', cls.id);
const items = [
...quizzes.map((q) => ({
kind: 'quiz' as const,
id: q.id,
title: q.title,
description: q.description,
folderId: q.folder_id,
createdAt: q.created_at,
updatedAt: q.updated_at,
shared: shared('quiz', q.id),
questionCount: q.qn,
attempts: q.attempts,
lastPercent: q.last_max ? Math.round(((q.last_score ?? 0) / q.last_max) * 100) : null,
lastStudiedAt: q.last_at,
activeAttemptId: q.active_id,
})),
...decks.map((d) => {
const st = deckStats(db, d.id, s);
return {
kind: 'deck' as const,
id: d.id,
title: d.title,
description: d.description,
folderId: d.folder_id,
createdAt: d.created_at,
updatedAt: d.updated_at,
shared: shared('deck', d.id),
cardCount: d.cn,
dueCount: st.dueNow,
newCount: st.new,
learnedCount: st.young + st.mature,
};
}),
...documents.map((d) => ({
kind: 'document' as const,
id: d.id as string,
title: d.title as string,
description: '',
folderId: d.folder_id as string | null,
createdAt: d.created_at as number,
updatedAt: d.updated_at as number,
shared: shared('document', d.id),
docKind: d.kind as string,
filename: d.filename as string,
size: d.size as number,
status: d.status as string,
error: d.error as string,
progress: d.progress as number,
pageCount: d.page_count as number | null,
tokenCount: d.token_count as number,
needsOcr: Boolean(d.needs_ocr),
})),
...notes.map((n) => ({
kind: 'note' as const,
id: n.id,
title: n.title,
description: n.content.replace(/[#*_>`|-]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 160),
folderId: n.folder_id,
createdAt: n.created_at,
updatedAt: n.updated_at,
shared: shared('note', n.id),
noteKind: n.kind,
})),
];
const folders = db.all<FolderRow>('SELECT * FROM folders WHERE class_id = ? ORDER BY sort_order, created_at', cls.id);
return c.json({
class: mapClass(cls),
folders: folders.map((f) => ({ ...mapFolder(f), shared: shared('folder', f.id) })),
items,
due: dueCounts(db, { classId: cls.id }, s),
share: shareFor(app, 'class', cls.id),
});
});
r.patch('/classes/:id', async (c) => {
const cls = getClass(db, idParam(c));
const body = await readBody(
c,
z.object({
name: z.string().trim().min(1).max(120).optional(),
code: z.string().trim().max(40).optional(),
description: z.string().max(2000).optional(),
color: z.enum(CLASS_COLORS).optional(),
archived: z.boolean().optional(),
}),
);
db.run(
'UPDATE classes SET name = ?, code = ?, description = ?, color = ?, archived = ?, updated_at = ? WHERE id = ?',
body.name ?? cls.name,
body.code ?? cls.code,
body.description ?? cls.description,
body.color ?? cls.color,
body.archived === undefined ? cls.archived : body.archived ? 1 : 0,
now(),
cls.id,
);
return c.json({ class: mapClass(getClass(db, cls.id)) });
});
r.delete('/classes/:id', (c) => {
const cls = getClass(db, idParam(c));
const files = db.all<{ storage_name: string }>('SELECT storage_name FROM documents WHERE class_id = ?', cls.id);
db.tx(() => {
db.run(
`DELETE FROM shares WHERE (kind = 'class' AND target_id = ?)
OR (kind = 'folder' AND target_id IN (SELECT id FROM folders WHERE class_id = ?))
OR (kind = 'quiz' AND target_id IN (SELECT id FROM quizzes WHERE class_id = ?))
OR (kind = 'deck' AND target_id IN (SELECT id FROM decks WHERE class_id = ?))
OR (kind = 'note' AND target_id IN (SELECT id FROM notes WHERE class_id = ?))
OR (kind = 'document' AND target_id IN (SELECT id FROM documents WHERE class_id = ?))`,
cls.id,
cls.id,
cls.id,
cls.id,
cls.id,
cls.id,
);
db.run('DELETE FROM classes WHERE id = ?', cls.id);
});
for (const f of files) fs.rmSync(path.join(app.config.uploadsDir, path.basename(f.storage_name)), { force: true });
return c.json({ ok: true });
});
/* ------------------------------ folders ------------------------------ */
r.post('/classes/:id/folders', async (c) => {
const { name } = await readBody(c, z.object({ name: z.string().trim().min(1).max(120) }));
return c.json({ folder: mapFolder(createFolder(db, idParam(c), name)) }, 201);
});
r.patch('/folders/:id', async (c) => {
const folder = getFolder(db, idParam(c));
const { name } = await readBody(c, z.object({ name: z.string().trim().min(1).max(120) }));
db.run('UPDATE folders SET name = ?, updated_at = ? WHERE id = ?', name, now(), folder.id);
return c.json({ folder: mapFolder(getFolder(db, folder.id)) });
});
r.delete('/folders/:id', (c) => {
const folder = getFolder(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'folder' AND target_id = ?`, folder.id);
db.run('DELETE FROM folders WHERE id = ?', folder.id);
});
return c.json({ ok: true });
});
r.post('/classes/:id/folders/reorder', async (c) => {
const classId = idParam(c);
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(500) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE folders SET sort_order = ? WHERE id = ? AND class_id = ?', i, id, classId)));
return c.json({ ok: true });
});
/* ------------------------------ move (any kind) ------------------------------ */
const TABLES = { quiz: 'quizzes', deck: 'decks', note: 'notes', document: 'documents' } as const;
r.post('/items/move', async (c) => {
const body = await readBody(
c,
z.object({
items: z.array(z.object({ kind: z.enum(['quiz', 'deck', 'note', 'document']), id: z.string() })).min(1).max(500),
classId: z.string(),
folderId: z.string().nullish(),
}),
);
getClass(db, body.classId);
const folderId = resolveFolder(db, body.classId, body.folderId);
db.tx(() => {
for (const item of body.items) {
const table = TABLES[item.kind];
db.run(`UPDATE ${table} SET class_id = ?, folder_id = ?, updated_at = ? WHERE id = ?`, body.classId, folderId, now(), item.id);
if (item.kind === 'document') db.run('UPDATE chunks SET class_id = ? WHERE document_id = ?', body.classId, item.id);
}
});
return c.json({ ok: true });
});
/* ------------------------------ quizzes ------------------------------ */
r.post('/quizzes', async (c) => {
const body = await readBody(c, quizCreateSchema);
const quiz = createQuiz(db, body);
return c.json({ quiz: mapQuiz(quiz) }, 201);
});
r.get('/quizzes/:id', (c) => {
const quiz = getQuiz(db, idParam(c));
const cls = getClass(db, quiz.class_id);
const history = listAttempts(db, { quizId: quiz.id, status: 'completed', limit: 100 });
const active = listAttempts(db, { quizId: quiz.id, status: 'active', limit: 1 })[0] ?? null;
return c.json({
quiz: mapQuiz(quiz),
class: mapClass(cls),
questions: listQuestions(db, quiz.id),
questionStats: questionStats(db, quiz.id),
history,
activeAttempt: active,
share: shareFor(app, 'quiz', quiz.id),
});
});
const itemPatch = z.object({ title: titleSchema.optional(), description: descriptionSchema.optional(), folderId: z.string().nullish() });
r.patch('/quizzes/:id', async (c) => {
const quiz = getQuiz(db, idParam(c));
const body = await readBody(c, itemPatch);
db.run(
'UPDATE quizzes SET title = ?, description = ?, folder_id = ?, updated_at = ? WHERE id = ?',
body.title ?? quiz.title,
body.description ?? quiz.description,
body.folderId === undefined ? quiz.folder_id : resolveFolder(db, quiz.class_id, body.folderId),
now(),
quiz.id,
);
return c.json({ quiz: mapQuiz(getQuiz(db, quiz.id)) });
});
r.delete('/quizzes/:id', (c) => {
const quiz = getQuiz(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'quiz' AND target_id = ?`, quiz.id);
db.run(`DELETE FROM attempts WHERE quiz_id = ? AND status = 'active'`, quiz.id);
db.run('DELETE FROM quizzes WHERE id = ?', quiz.id);
});
return c.json({ ok: true });
});
const duplicateSchema = z.object({ classId: z.string().optional(), folderId: z.string().nullish(), title: titleSchema.optional() }).optional();
r.post('/quizzes/:id/duplicate', async (c) => {
const body = await c.req.json().catch(() => ({}));
const target = duplicateSchema.parse(body) ?? {};
return c.json({ quiz: mapQuiz(duplicateQuiz(db, idParam(c), target)) }, 201);
});
r.get('/quizzes/:id/export', (c) => {
const quiz = getQuiz(db, idParam(c));
const data = exportQuiz({ title: quiz.title, description: quiz.description, questions: listQuestions(db, quiz.id).map(questionToDraft) });
return download(c, `${slugify(quiz.title)}.quiz.json`, data);
});
r.post('/quizzes/:id/questions', async (c) => {
const quiz = getQuiz(db, idParam(c));
const body = await readBody(c, z.object({ questions: z.array(z.unknown()).min(1).max(1000), index: z.number().int().min(0).optional() }));
return c.json({ questions: insertQuestions(db, quiz.id, body.questions, body.index) }, 201);
});
r.post('/quizzes/:id/questions/reorder', async (c) => {
const quiz = getQuiz(db, idParam(c));
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(2000) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE questions SET sort_order = ? WHERE id = ? AND quiz_id = ?', i, id, quiz.id)));
return c.json({ ok: true });
});
r.patch('/questions/:id', async (c) => {
const body = await c.req.json().catch(() => {
throw badRequest('Request body must be JSON');
});
return c.json({ question: updateQuestion(db, idParam(c), body) });
});
r.delete('/questions/:id', (c) => {
const { changes } = db.run('DELETE FROM questions WHERE id = ?', idParam(c));
if (!changes) throw notFound('Question');
return c.json({ ok: true });
});
r.post('/questions/delete', async (c) => {
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).min(1).max(2000) }));
db.tx(() => ids.forEach((id) => db.run('DELETE FROM questions WHERE id = ?', id)));
return c.json({ ok: true });
});
/* ------------------------------ decks ------------------------------ */
r.post('/decks', async (c) => {
const body = await readBody(c, deckCreateSchema);
return c.json({ deck: mapDeck(createDeck(db, body)) }, 201);
});
r.get('/decks/:id', (c) => {
const deck = getDeck(db, idParam(c));
const session = db.get<{ state: string; updated_at: number }>('SELECT state, updated_at FROM deck_sessions WHERE deck_id = ?', deck.id);
return c.json({
deck: mapDeck(deck),
class: mapClass(getClass(db, deck.class_id)),
cards: listCards(db, deck.id),
stats: deckStats(db, deck.id, settings()),
due: dueCounts(db, { deckId: deck.id }, settings()),
session: session ? { state: parseJson<unknown>(session.state, null), updatedAt: session.updated_at } : null,
share: shareFor(app, 'deck', deck.id),
});
});
r.patch('/decks/:id', async (c) => {
const deck = getDeck(db, idParam(c));
const body = await readBody(c, itemPatch);
db.run(
'UPDATE decks SET title = ?, description = ?, folder_id = ?, updated_at = ? WHERE id = ?',
body.title ?? deck.title,
body.description ?? deck.description,
body.folderId === undefined ? deck.folder_id : resolveFolder(db, deck.class_id, body.folderId),
now(),
deck.id,
);
return c.json({ deck: mapDeck(getDeck(db, deck.id)) });
});
r.delete('/decks/:id', (c) => {
const deck = getDeck(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'deck' AND target_id = ?`, deck.id);
db.run('DELETE FROM decks WHERE id = ?', deck.id);
});
return c.json({ ok: true });
});
r.post('/decks/:id/duplicate', async (c) => {
const body = await c.req.json().catch(() => ({}));
const target = duplicateSchema.parse(body) ?? {};
return c.json({ deck: mapDeck(duplicateDeck(db, idParam(c), target)) }, 201);
});
r.get('/decks/:id/export', (c) => {
const deck = getDeck(db, idParam(c));
const data = exportDeck({ title: deck.title, description: deck.description, cards: listCards(db, deck.id).map((x) => ({ front: x.front, back: x.back, hint: x.hint, tags: x.tags })) });
return download(c, `${slugify(deck.title)}.deck.json`, data);
});
r.post('/decks/:id/cards', async (c) => {
const deck = getDeck(db, idParam(c));
const body = await readBody(c, z.object({ cards: z.array(z.unknown()).min(1).max(5000) }));
return c.json({ cards: insertCards(db, deck.id, body.cards) }, 201);
});
r.post('/decks/:id/cards/reorder', async (c) => {
const deck = getDeck(db, idParam(c));
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(10000) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE cards SET sort_order = ? WHERE id = ? AND deck_id = ?', i, id, deck.id)));
return c.json({ ok: true });
});
r.post('/decks/:id/reset', (c) => {
const deck = getDeck(db, idParam(c));
return c.json({ reset: resetCards(db, { deckId: deck.id }) });
});
r.put('/decks/:id/session', async (c) => {
const deck = getDeck(db, idParam(c));
const body = await readBody(c, z.object({ state: z.unknown() }));
const json = JSON.stringify(body.state ?? null);
if (json.length > 2_000_000) throw badRequest('Session state is too large');
db.run(
'INSERT INTO deck_sessions (deck_id, state, updated_at) VALUES (?, ?, ?) ON CONFLICT(deck_id) DO UPDATE SET state = excluded.state, updated_at = excluded.updated_at',
deck.id,
json,
now(),
);
return c.json({ ok: true });
});
r.delete('/decks/:id/session', (c) => {
db.run('DELETE FROM deck_sessions WHERE deck_id = ?', idParam(c));
return c.json({ ok: true });
});
r.patch('/cards/:id', async (c) => {
const id = idParam(c);
const body = (await c.req.json().catch(() => {
throw badRequest('Request body must be JSON');
})) as Record<string, unknown>;
if (typeof body.suspended === 'boolean') {
const { changes } = db.run('UPDATE cards SET suspended = ?, updated_at = ? WHERE id = ?', body.suspended ? 1 : 0, now(), id);
if (!changes) throw notFound('Card');
}
if (body.reset === true) resetCards(db, { cardId: id });
if (body.front !== undefined || body.back !== undefined) return c.json({ card: updateCard(db, id, body) });
const row = db.get<{ deck_id: string }>('SELECT deck_id FROM cards WHERE id = ?', id);
if (!row) throw notFound('Card');
return c.json({ card: listCards(db, row.deck_id).find((x) => x.id === id) });
});
r.delete('/cards/:id', (c) => {
const { changes } = db.run('DELETE FROM cards WHERE id = ?', idParam(c));
if (!changes) throw notFound('Card');
return c.json({ ok: true });
});
r.post('/cards/delete', async (c) => {
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).min(1).max(10000) }));
db.tx(() => ids.forEach((id) => db.run('DELETE FROM cards WHERE id = ?', id)));
return c.json({ ok: true });
});
/* ------------------------------ notes ------------------------------ */
r.post('/notes', async (c) => {
const body = await readBody(
c,
z.object({
classId: z.string(),
folderId: z.string().nullish(),
title: titleSchema,
content: z.string().max(400000).optional().default(''),
kind: z.enum(['note', 'guide', 'summary']).optional().default('note'),
}),
);
return c.json({ note: mapNote(createNote(db, body)) }, 201);
});
r.get('/notes/:id', (c) => {
const note = getNote(db, idParam(c));
return c.json({ note: mapNote(note), class: mapClass(getClass(db, note.class_id)), share: shareFor(app, 'note', note.id) });
});
r.patch('/notes/:id', async (c) => {
const note = getNote(db, idParam(c));
const body = await readBody(c, z.object({ title: titleSchema.optional(), content: z.string().max(400000).optional(), folderId: z.string().nullish() }));
db.run(
'UPDATE notes SET title = ?, content = ?, folder_id = ?, updated_at = ? WHERE id = ?',
body.title ?? note.title,
body.content ?? note.content,
body.folderId === undefined ? note.folder_id : resolveFolder(db, note.class_id, body.folderId),
now(),
note.id,
);
return c.json({ note: mapNote(getNote(db, note.id)) });
});
r.delete('/notes/:id', (c) => {
const note = getNote(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'note' AND target_id = ?`, note.id);
db.run('DELETE FROM notes WHERE id = ?', note.id);
});
return c.json({ ok: true });
});
return r;
}

260
server/routes/shares.ts Normal file
View file

@ -0,0 +1,260 @@
import fs from 'node:fs';
import path from 'node:path';
import { Hono } from 'hono';
import { z } from 'zod';
import { HttpError, idParam, notFound, readBody, type AppContext } from '../http.ts';
import { listCards, listQuestions } from '../services/content.ts';
import { newId, newToken, now } from '../util.ts';
const KINDS = ['quiz', 'deck', 'folder', 'class', 'note', 'document'] as const;
type ShareKind = (typeof KINDS)[number];
interface ShareRow {
id: string;
token: string;
kind: ShareKind;
target_id: string;
enabled: number;
view_count: number;
last_viewed_at: number | null;
created_at: number;
updated_at: number;
}
const TABLE: Record<ShareKind, { table: string; title: string }> = {
quiz: { table: 'quizzes', title: 'title' },
deck: { table: 'decks', title: 'title' },
note: { table: 'notes', title: 'title' },
document: { table: 'documents', title: 'title' },
folder: { table: 'folders', title: 'name' },
class: { table: 'classes', title: 'name' },
};
function mapShare(row: ShareRow) {
return {
id: row.id,
token: row.token,
kind: row.kind,
targetId: row.target_id,
enabled: Boolean(row.enabled),
viewCount: row.view_count,
lastViewedAt: row.last_viewed_at,
createdAt: row.created_at,
};
}
/* ------------------------------ admin ------------------------------ */
export function shareRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
r.get('/shares', (c) => {
const rows = db.all<ShareRow>('SELECT * FROM shares ORDER BY created_at DESC');
const out = [];
for (const row of rows) {
const meta = TABLE[row.kind];
const target = db.get<{ title: string; class_id?: string }>(
`SELECT ${meta.title} AS title${row.kind === 'class' ? '' : ', class_id'} FROM ${meta.table} WHERE id = ?`,
row.target_id,
);
if (!target) {
db.run('DELETE FROM shares WHERE id = ?', row.id);
continue;
}
const classId = row.kind === 'class' ? row.target_id : target.class_id;
const cls = classId ? db.get<{ name: string; color: string }>('SELECT name, color FROM classes WHERE id = ?', classId) : undefined;
out.push({ ...mapShare(row), title: target.title, classId: classId ?? null, className: cls?.name ?? '', classColor: cls?.color ?? 'slate' });
}
return c.json({ shares: out });
});
r.post('/shares', async (c) => {
const body = await readBody(c, z.object({ kind: z.enum(KINDS), targetId: z.string() }));
const meta = TABLE[body.kind];
if (!db.get(`SELECT id FROM ${meta.table} WHERE id = ?`, body.targetId)) throw notFound('Item');
const existing = db.get<ShareRow>('SELECT * FROM shares WHERE kind = ? AND target_id = ?', body.kind, body.targetId);
const at = now();
if (existing) {
db.run('UPDATE shares SET enabled = 1, updated_at = ? WHERE id = ?', at, existing.id);
return c.json({ share: mapShare({ ...existing, enabled: 1 }) });
}
const id = newId();
db.run('INSERT INTO shares (id, token, kind, target_id, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, 1, ?, ?)', id, newToken(), body.kind, body.targetId, at, at);
return c.json({ share: mapShare(db.get<ShareRow>('SELECT * FROM shares WHERE id = ?', id)!) }, 201);
});
r.patch('/shares/:id', async (c) => {
const id = idParam(c);
const body = await readBody(c, z.object({ enabled: z.boolean() }));
const { changes } = db.run('UPDATE shares SET enabled = ?, updated_at = ? WHERE id = ?', body.enabled ? 1 : 0, now(), id);
if (!changes) throw notFound('Share link');
return c.json({ share: mapShare(db.get<ShareRow>('SELECT * FROM shares WHERE id = ?', id)!) });
});
/** Issues a new token: the old link stops working immediately. */
r.post('/shares/:id/rotate', (c) => {
const id = idParam(c);
const { changes } = db.run('UPDATE shares SET token = ?, view_count = 0, last_viewed_at = NULL, updated_at = ? WHERE id = ?', newToken(), now(), id);
if (!changes) throw notFound('Share link');
return c.json({ share: mapShare(db.get<ShareRow>('SELECT * FROM shares WHERE id = ?', id)!) });
});
r.delete('/shares/:id', (c) => {
const { changes } = db.run('DELETE FROM shares WHERE id = ?', idParam(c));
if (!changes) throw notFound('Share link');
return c.json({ ok: true });
});
return r;
}
/* ------------------------------ public (no auth) ------------------------------ */
const gone = () => new HttpError(404, 'share_not_found', 'This link is no longer available');
function publicQuiz(app: AppContext, id: string) {
const quiz = app.db.get<{ id: string; title: string; description: string; class_id: string }>('SELECT id, title, description, class_id FROM quizzes WHERE id = ?', id);
if (!quiz) throw gone();
return {
kind: 'quiz' as const,
id: quiz.id,
title: quiz.title,
description: quiz.description,
questions: listQuestions(app.db, quiz.id).map((q) => ({
id: q.id,
type: q.type,
stem: q.stem,
options: q.options,
rationale: q.rationale,
topic: q.topic,
difficulty: q.difficulty,
})),
};
}
function publicDeck(app: AppContext, id: string) {
const deck = app.db.get<{ id: string; title: string; description: string }>('SELECT id, title, description FROM decks WHERE id = ?', id);
if (!deck) throw gone();
return {
kind: 'deck' as const,
id: deck.id,
title: deck.title,
description: deck.description,
cards: listCards(app.db, deck.id).map((c) => ({ id: c.id, front: c.front, back: c.back, hint: c.hint })),
};
}
function publicNote(app: AppContext, id: string) {
const note = app.db.get<{ id: string; title: string; content: string; updated_at: number }>('SELECT id, title, content, updated_at FROM notes WHERE id = ?', id);
if (!note) throw gone();
return { kind: 'note' as const, id: note.id, title: note.title, content: note.content, updatedAt: note.updated_at };
}
function publicDocument(app: AppContext, id: string) {
const doc = app.db.get<Record<string, any>>('SELECT id, title, filename, kind, size, page_count, mime FROM documents WHERE id = ?', id);
if (!doc) throw gone();
return { kind: 'document' as const, id: doc.id as string, title: doc.title as string, filename: doc.filename as string, docKind: doc.kind as string, size: doc.size as number, pageCount: doc.page_count as number | null };
}
/** Quizzes, decks and notes inside a class or folder. Uploaded documents are never included in a collection share. */
function collectionItems(app: AppContext, share: ShareRow) {
const { db } = app;
const byFolder = share.kind === 'folder';
const where = byFolder ? 'folder_id = ?' : 'class_id = ?';
const quizzes = db.all<Record<string, any>>(
`SELECT z.id, z.title, z.description, z.folder_id, (SELECT COUNT(*) FROM questions q WHERE q.quiz_id = z.id) AS n FROM quizzes z WHERE z.${where} ORDER BY z.sort_order, z.created_at`,
share.target_id,
);
const decks = db.all<Record<string, any>>(
`SELECT d.id, d.title, d.description, d.folder_id, (SELECT COUNT(*) FROM cards c WHERE c.deck_id = d.id) AS n FROM decks d WHERE d.${where} ORDER BY d.sort_order, d.created_at`,
share.target_id,
);
const notes = db.all<Record<string, any>>(`SELECT id, title, folder_id FROM notes WHERE ${where} ORDER BY sort_order, created_at`, share.target_id);
return [
...quizzes.filter((q) => q.n > 0).map((q) => ({ kind: 'quiz' as const, id: q.id as string, title: q.title as string, description: q.description as string, folderId: q.folder_id as string | null, count: q.n as number })),
...decks.filter((d) => d.n > 0).map((d) => ({ kind: 'deck' as const, id: d.id as string, title: d.title as string, description: d.description as string, folderId: d.folder_id as string | null, count: d.n as number })),
...notes.map((n) => ({ kind: 'note' as const, id: n.id as string, title: n.title as string, description: '', folderId: n.folder_id as string | null, count: 0 })),
];
}
export function publicRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
const load = (token: string): ShareRow => {
if (!token || token.length < 16 || token.length > 64) throw gone();
const share = db.get<ShareRow>('SELECT * FROM shares WHERE token = ?', token);
if (!share || !share.enabled) throw gone();
return share;
};
r.get('/:token', (c) => {
const share = load(c.req.param('token'));
db.run('UPDATE shares SET view_count = view_count + 1, last_viewed_at = ? WHERE id = ?', now(), share.id);
c.header('Cache-Control', 'no-store');
c.header('X-Robots-Tag', 'noindex, nofollow');
switch (share.kind) {
case 'quiz':
return c.json(publicQuiz(app, share.target_id));
case 'deck':
return c.json(publicDeck(app, share.target_id));
case 'note':
return c.json(publicNote(app, share.target_id));
case 'document':
return c.json(publicDocument(app, share.target_id));
case 'folder': {
const folder = db.get<{ id: string; name: string; class_id: string }>('SELECT id, name, class_id FROM folders WHERE id = ?', share.target_id);
if (!folder) throw gone();
const cls = db.get<{ name: string; color: string }>('SELECT name, color FROM classes WHERE id = ?', folder.class_id);
return c.json({ kind: 'collection' as const, title: folder.name, subtitle: cls?.name ?? '', color: cls?.color ?? 'cobalt', folders: [], items: collectionItems(app, share) });
}
case 'class': {
const cls = db.get<{ id: string; name: string; code: string; description: string; color: string }>('SELECT * FROM classes WHERE id = ?', share.target_id);
if (!cls) throw gone();
const folders = db.all<{ id: string; name: string }>('SELECT id, name FROM folders WHERE class_id = ? ORDER BY sort_order, created_at', cls.id);
return c.json({ kind: 'collection' as const, title: cls.name, subtitle: cls.code, description: cls.description, color: cls.color, folders, items: collectionItems(app, share) });
}
default:
throw gone();
}
});
/** One item inside a shared class or folder. Membership is re-checked on every request. */
r.get('/:token/items/:kind/:id', (c) => {
const share = load(c.req.param('token'));
if (share.kind !== 'class' && share.kind !== 'folder') throw gone();
const kind = c.req.param('kind');
const id = c.req.param('id');
const allowed = collectionItems(app, share).some((i) => i.kind === kind && i.id === id);
if (!allowed) throw gone();
c.header('Cache-Control', 'no-store');
c.header('X-Robots-Tag', 'noindex, nofollow');
if (kind === 'quiz') return c.json(publicQuiz(app, id));
if (kind === 'deck') return c.json(publicDeck(app, id));
return c.json(publicNote(app, id));
});
/** Download for an explicitly shared document. */
r.get('/:token/file', (c) => {
const share = load(c.req.param('token'));
if (share.kind !== 'document') throw gone();
const doc = db.get<{ filename: string; mime: string; storage_name: string }>('SELECT filename, mime, storage_name FROM documents WHERE id = ?', share.target_id);
if (!doc) throw gone();
const file = path.join(app.config.uploadsDir, path.basename(doc.storage_name));
if (!fs.existsSync(file)) throw gone();
const inline = doc.mime === 'application/pdf' || doc.mime.startsWith('image/');
return new Response(fs.readFileSync(file), {
headers: {
'Content-Type': doc.mime || 'application/octet-stream',
'Content-Disposition': `${inline ? 'inline' : 'attachment'}; filename*=UTF-8''${encodeURIComponent(doc.filename)}`,
'X-Content-Type-Options': 'nosniff',
'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; sandbox",
'Cache-Control': 'private, no-store',
'X-Robots-Tag': 'noindex, nofollow',
},
});
});
return r;
}

201
server/routes/study.ts Normal file
View file

@ -0,0 +1,201 @@
import { Hono } from 'hono';
import { z } from 'zod';
import { badRequest, idParam, readBody, type AppContext } from '../http.ts';
import { getAppSettings } from '../settings.ts';
import {
attemptCreateSchema,
attemptPatchSchema,
completeAttempt,
createAttempt,
deleteAttempt,
getAttempt,
listAttempts,
missedQuestionIds,
patchAttempt,
} from '../services/attempts.ts';
import { answerCard, buildQueue, dueCounts, forecast, logCram, undoReview, type RatingValue } from '../services/srs.ts';
import { accuracyTrend, activityByDay, computeStreak, topicStats, totals, weakTopics } from '../services/stats.ts';
import { ftsQuery } from '../util.ts';
export function studyRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
const settings = () => getAppSettings(db, app.config.timezone);
/* ------------------------------ quiz sessions ------------------------------ */
r.post('/attempts', async (c) => {
const body = await readBody(c, attemptCreateSchema);
const id = createAttempt(db, body, settings().sataScoring);
return c.json({ attempt: getAttempt(db, id) }, 201);
});
r.get('/attempts', (c) => {
const status = c.req.query('status');
return c.json({
attempts: listAttempts(db, {
status: status === 'active' || status === 'completed' ? status : undefined,
quizId: c.req.query('quizId') || undefined,
classId: c.req.query('classId') || undefined,
limit: Number(c.req.query('limit') ?? 50) || 50,
}),
});
});
r.get('/attempts/:id', (c) => c.json({ attempt: getAttempt(db, idParam(c)) }));
r.patch('/attempts/:id', async (c) => {
const body = await readBody(c, attemptPatchSchema);
return c.json({ attempt: patchAttempt(db, idParam(c), body) });
});
r.post('/attempts/:id/complete', async (c) => {
const body = (await c.req.json().catch(() => ({}))) as { elapsedSec?: unknown };
const elapsed = typeof body.elapsedSec === 'number' && Number.isFinite(body.elapsedSec) ? Math.max(0, Math.round(body.elapsedSec)) : undefined;
return c.json({ attempt: completeAttempt(db, idParam(c), elapsed) });
});
r.delete('/attempts/:id', (c) => {
deleteAttempt(db, idParam(c));
return c.json({ ok: true });
});
/* ------------------------------ spaced repetition ------------------------------ */
r.get('/review/queue', (c) => {
const scope = { deckId: c.req.query('deckId') || undefined, classId: c.req.query('classId') || undefined };
const limit = Math.min(Number(c.req.query('limit') ?? 100) || 100, 300);
return c.json(buildQueue(db, scope, settings(), limit));
});
r.post('/review/answer', async (c) => {
const body = await readBody(c, z.object({ cardId: z.string(), rating: z.number().int().min(1).max(4) }));
return c.json({ result: answerCard(db, body.cardId, body.rating as RatingValue, settings()) });
});
r.post('/review/undo', async (c) => {
const body = await readBody(c, z.object({ cardId: z.string() }));
return c.json({ result: undoReview(db, body.cardId, settings()) });
});
r.post('/review/cram', async (c) => {
const body = await readBody(c, z.object({ cardId: z.string(), known: z.boolean() }));
logCram(db, body.cardId, body.known);
return c.json({ ok: true });
});
/* ------------------------------ dashboards ------------------------------ */
r.get('/overview', (c) => {
const s = settings();
const days = activityByDay(db, s, 26 * 7);
return c.json({
settings: s,
due: dueCounts(db, {}, s),
streak: computeStreak(days, s.dailyGoal),
activity: days,
activeAttempts: listAttempts(db, { status: 'active', limit: 6 }),
recentAttempts: listAttempts(db, { status: 'completed', limit: 6 }),
weakTopics: weakTopics(db, {}, 6),
missedCount: missedQuestionIds(db, {}, 500).length,
totals: totals(db, s, {}, 7),
forecast: forecast(db, {}, s, 7),
});
});
r.get('/insights', (c) => {
const s = settings();
const classId = c.req.query('classId') || undefined;
const range = Math.min(Math.max(Number(c.req.query('days') ?? 30) || 30, 7), 365);
const scope = { classId };
const days = activityByDay(db, s, 53 * 7, scope);
return c.json({
range,
totals: totals(db, s, scope, range),
streak: computeStreak(activityByDay(db, s, 53 * 7), s.dailyGoal),
activity: days,
accuracy: accuracyTrend(db, s, range, scope),
topics: topicStats(db, scope, range),
forecast: forecast(db, scope, s, 14),
due: dueCounts(db, scope, s),
missedCount: missedQuestionIds(db, scope, 500).length,
recentAttempts: listAttempts(db, { status: 'completed', classId, limit: 15 }),
});
});
/* ------------------------------ search ------------------------------ */
r.get('/search', (c) => {
const q = (c.req.query('q') ?? '').trim().slice(0, 120);
if (q.length < 2) return c.json({ query: q, results: [] });
const like = `%${q.replace(/[\\%_]/g, (m) => `\\${m}`)}%`;
const classId = c.req.query('classId') || null;
const scoped = (col: string) => (classId ? `AND ${col} = ?` : '');
const p = (...base: string[]) => (classId ? [...base, classId] : base);
type Hit = { kind: string; id: string; title: string; snippet: string; classId: string; className: string; classColor: string; parentId?: string; page?: number | null };
const results: Hit[] = [];
for (const row of db.all<Record<string, string>>(`SELECT id, name, color FROM classes WHERE name LIKE ? ESCAPE '\\' OR code LIKE ? ESCAPE '\\' LIMIT 5`, like, like)) {
results.push({ kind: 'class', id: row.id, title: row.name, snippet: '', classId: row.id, className: row.name, classColor: row.color });
}
const simple = (kind: string, table: string) => {
for (const row of db.all<Record<string, string>>(
`SELECT t.id, t.title, k.id AS class_id, k.name AS class_name, k.color AS class_color FROM ${table} t JOIN classes k ON k.id = t.class_id
WHERE t.title LIKE ? ESCAPE '\\' ${scoped('t.class_id')} ORDER BY t.updated_at DESC LIMIT 8`,
...p(like),
)) {
results.push({ kind, id: row.id, title: row.title, snippet: '', classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
};
simple('quiz', 'quizzes');
simple('deck', 'decks');
simple('note', 'notes');
simple('document', 'documents');
for (const row of db.all<Record<string, string>>(
`SELECT q.id, q.stem, z.id AS quiz_id, z.title AS quiz_title, k.id AS class_id, k.name AS class_name, k.color AS class_color
FROM questions q JOIN quizzes z ON z.id = q.quiz_id JOIN classes k ON k.id = z.class_id
WHERE (q.stem LIKE ? ESCAPE '\\' OR q.topic LIKE ? ESCAPE '\\') ${scoped('z.class_id')} LIMIT 10`,
...p(like, like),
)) {
results.push({ kind: 'question', id: row.id, parentId: row.quiz_id, title: row.stem.slice(0, 140), snippet: row.quiz_title, classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
for (const row of db.all<Record<string, string>>(
`SELECT c.id, c.front, c.back, d.id AS deck_id, d.title AS deck_title, k.id AS class_id, k.name AS class_name, k.color AS class_color
FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id
WHERE (c.front LIKE ? ESCAPE '\\' OR c.back LIKE ? ESCAPE '\\') ${scoped('d.class_id')} LIMIT 10`,
...p(like, like),
)) {
results.push({ kind: 'card', id: row.id, parentId: row.deck_id, title: row.front.slice(0, 140), snippet: row.deck_title, classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
const match = ftsQuery(q, 'and');
if (match) {
try {
for (const row of db.all<Record<string, any>>(
`SELECT ch.id, ch.page_start, d.id AS doc_id, d.title AS doc_title, k.id AS class_id, k.name AS class_name, k.color AS class_color,
snippet(chunks_fts, 0, '', '', ' … ', 18) AS snip
FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id JOIN classes k ON k.id = d.class_id
WHERE chunks_fts MATCH ? ${scoped('d.class_id')} ORDER BY bm25(chunks_fts) LIMIT 10`,
...p(match),
)) {
results.push({ kind: 'passage', id: String(row.id), parentId: row.doc_id, page: row.page_start, title: row.doc_title, snippet: row.snip, classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
} catch {
/* a malformed FTS query should never break search */
}
}
return c.json({ query: q, results });
});
r.get('/topics', (c) => {
const classId = c.req.query('classId');
if (!classId) throw badRequest('classId is required');
const rows = db.all<{ topic: string; n: number }>(
`SELECT q.topic AS topic, COUNT(*) AS n FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE z.class_id = ? AND q.topic != '' GROUP BY q.topic ORDER BY n DESC`,
classId,
);
return c.json({ topics: rows });
});
return r;
}

431
server/services/attempts.ts Normal file
View file

@ -0,0 +1,431 @@
/**
* Quiz sessions. An attempt freezes its questions (and their shuffled answer order) in a
* snapshot when it starts, so editing or deleting the quiz later can never corrupt a session
* in progress or the review of a finished one.
*/
import { z } from 'zod';
import { isFullyCorrect, scoreQuestion, seededShuffle, type Option, type QuestionType, type SataScoring } from '../../shared/content.ts';
import type { DB } from '../db.ts';
import { badRequest, notFound } from '../http.ts';
import { newId, now, parseJson } from '../util.ts';
import { getClass, getQuiz, mapQuestion, type QuestionRow } from './content.ts';
export interface SnapshotQuestion {
id: string;
quizId: string | null;
quizTitle: string;
type: QuestionType;
stem: string;
options: Option[];
rationale: string;
topic: string;
difficulty: string;
}
export interface AnswerState {
selected: string[];
/** Practice mode: the answer has been checked and feedback shown. */
checked: boolean;
at: number;
}
export interface AttemptSettings {
shuffleQuestions: boolean;
shuffleOptions: boolean;
timeLimitSec: number | null;
sataScoring: SataScoring;
}
interface AttemptRow {
id: string;
quiz_id: string | null;
class_id: string | null;
title: string;
mode: 'practice' | 'exam';
kind: string;
status: 'active' | 'completed';
settings: string;
snapshot: string;
answers: string;
flags: string;
current_index: number;
elapsed_sec: number;
question_count: number;
score: number | null;
max_score: number | null;
started_at: number;
updated_at: number;
completed_at: number | null;
}
export const attemptCreateSchema = z.object({
mode: z.enum(['practice', 'exam']).default('practice'),
source: z.discriminatedUnion('type', [
z.object({ type: z.literal('quiz'), quizId: z.string() }),
/** Questions missed in one finished attempt. */
z.object({ type: z.literal('retake'), attemptId: z.string() }),
/** Questions answered wrong most often, across a class or everything. */
z.object({ type: z.literal('missed'), classId: z.string().optional(), quizId: z.string().optional() }),
/** Every question on the given topics within a class. */
z.object({ type: z.literal('topics'), classId: z.string(), topics: z.array(z.string()).min(1).max(50) }),
z.object({ type: z.literal('questions'), questionIds: z.array(z.string()).min(1).max(1000), title: z.string().max(200).optional() }),
]),
shuffleQuestions: z.boolean().default(true),
shuffleOptions: z.boolean().default(true),
count: z.number().int().min(1).max(1000).optional(),
timeLimitMin: z.number().min(1).max(600).nullish(),
});
export type AttemptCreate = z.input<typeof attemptCreateSchema>;
const PINNED_OPTION = /^(all|none|both|neither)\b.*\b(above|these|options|answers)\b|^(a|b|c|d)\s*(and|&)\s*(a|b|c|d)\b/i;
/** Shuffles options but keeps "All of the above"-style choices at the end. */
function shuffleOptions(options: Option[], seed: number): Option[] {
const pinned = options.filter((o) => PINNED_OPTION.test(o.text.trim()));
const free = options.filter((o) => !PINNED_OPTION.test(o.text.trim()));
return [...seededShuffle(free, seed), ...pinned];
}
function loadQuestionRows(db: DB, ids: string[]): (QuestionRow & { quiz_title: string })[] {
if (ids.length === 0) return [];
const out: (QuestionRow & { quiz_title: string })[] = [];
for (let i = 0; i < ids.length; i += 400) {
const batch = ids.slice(i, i + 400);
out.push(
...db.all<QuestionRow & { quiz_title: string }>(
`SELECT q.*, z.title AS quiz_title FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE q.id IN (${batch.map(() => '?').join(',')})`,
...batch,
),
);
}
const order = new Map(ids.map((id, i) => [id, i]));
return out.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
}
/** Question ids ordered from most to least troublesome, using recent responses. */
export function missedQuestionIds(db: DB, scope: { classId?: string; quizId?: string }, limit = 200): string[] {
const where = scope.quizId ? 'AND r.quiz_id = ?' : scope.classId ? 'AND r.class_id = ?' : '';
const params = scope.quizId ? [scope.quizId] : scope.classId ? [scope.classId] : [];
const rows = db.all<{ question_id: string }>(
`SELECT r.question_id, COUNT(*) AS seen, SUM(CASE WHEN r.correct = 0 THEN 1 ELSE 0 END) AS wrong,
(SELECT r2.correct FROM responses r2 WHERE r2.question_id = r.question_id ORDER BY r2.answered_at DESC LIMIT 1) AS last_correct
FROM responses r JOIN questions q ON q.id = r.question_id
WHERE 1 = 1 ${where}
GROUP BY r.question_id
HAVING wrong > 0 AND (last_correct = 0 OR wrong * 2 >= seen)
ORDER BY last_correct ASC, (wrong * 1.0 / seen) DESC, wrong DESC
LIMIT ?`,
...params,
limit,
);
return rows.map((r) => r.question_id);
}
export function createAttempt(db: DB, input: AttemptCreate, sataScoring: SataScoring): string {
const opts = attemptCreateSchema.parse(input);
const src = opts.source;
let quizId: string | null = null;
let classId: string | null = null;
let title = '';
let kind = 'full';
let rows: (QuestionRow & { quiz_title: string })[] = [];
let frozen: SnapshotQuestion[] | null = null;
if (src.type === 'quiz') {
const quiz = getQuiz(db, src.quizId);
quizId = quiz.id;
classId = quiz.class_id;
title = quiz.title;
rows = db
.all<QuestionRow>('SELECT * FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quiz.id)
.map((r) => ({ ...r, quiz_title: quiz.title }));
} else if (src.type === 'retake') {
const prev = db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', src.attemptId);
if (!prev) throw notFound('Attempt');
const snap = parseJson<{ questions: SnapshotQuestion[] }>(prev.snapshot, { questions: [] });
const answers = parseJson<Record<string, AnswerState>>(prev.answers, {});
const prevSettings = parseJson<AttemptSettings>(prev.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring });
const missed = snap.questions.filter((q) => !isFullyCorrect(scoreQuestion(q, answers[q.id]?.selected ?? [], prevSettings.sataScoring)));
const current = new Map(loadQuestionRows(db, missed.map((q) => q.id)).map((r) => [r.id, r]));
// Prefer the live question (it may have been corrected since); fall back to the frozen copy.
frozen = missed.map((q) => {
const live = current.get(q.id);
if (!live) return q;
const m = mapQuestion(live);
return { id: m.id, quizId: m.quizId, quizTitle: live.quiz_title, type: m.type, stem: m.stem, options: m.options, rationale: m.rationale, topic: m.topic, difficulty: m.difficulty };
});
quizId = prev.quiz_id;
classId = prev.class_id;
title = `${prev.title.replace(/ · missed$/, '')} · missed`;
kind = 'retake';
} else if (src.type === 'missed') {
const ids = missedQuestionIds(db, { classId: src.classId, quizId: src.quizId });
rows = loadQuestionRows(db, ids);
kind = 'missed';
if (src.quizId) {
const quiz = getQuiz(db, src.quizId);
quizId = quiz.id;
classId = quiz.class_id;
title = `${quiz.title} · missed`;
} else if (src.classId) {
classId = src.classId;
title = `${getClass(db, src.classId).name} · missed questions`;
} else {
title = 'Missed questions';
}
} else if (src.type === 'topics') {
const cls = getClass(db, src.classId);
classId = cls.id;
rows = db.all<QuestionRow & { quiz_title: string }>(
`SELECT q.*, z.title AS quiz_title FROM questions q JOIN quizzes z ON z.id = q.quiz_id
WHERE z.class_id = ? AND q.topic IN (${src.topics.map(() => '?').join(',')}) ORDER BY z.sort_order, q.sort_order`,
cls.id,
...src.topics,
);
title = src.topics.length === 1 ? `${src.topics[0]} · practice` : `${cls.name} · ${src.topics.length} topics`;
kind = 'topics';
} else {
rows = loadQuestionRows(db, src.questionIds);
title = src.title ?? 'Custom practice';
kind = 'custom';
const classes = new Set(rows.map((r) => db.get<{ class_id: string }>('SELECT class_id FROM quizzes WHERE id = ?', r.quiz_id)?.class_id));
if (classes.size === 1) classId = [...classes][0] ?? null;
}
let questions: SnapshotQuestion[] =
frozen ??
rows.map((r) => {
const m = mapQuestion(r);
return { id: m.id, quizId: m.quizId, quizTitle: r.quiz_title, type: m.type, stem: m.stem, options: m.options, rationale: m.rationale, topic: m.topic, difficulty: m.difficulty };
});
if (questions.length === 0) {
throw badRequest(src.type === 'quiz' ? 'This quiz has no questions yet' : src.type === 'missed' || src.type === 'retake' ? 'No missed questions to review' : 'No questions matched');
}
const seed = (Math.random() * 2 ** 32) >>> 0;
if (opts.shuffleQuestions) questions = seededShuffle(questions, seed);
if (opts.count && opts.count < questions.length) questions = questions.slice(0, opts.count);
if (opts.shuffleOptions) questions = questions.map((q, i) => ({ ...q, options: shuffleOptions(q.options, seed + i + 1) }));
const settings: AttemptSettings = {
shuffleQuestions: opts.shuffleQuestions,
shuffleOptions: opts.shuffleOptions,
timeLimitSec: opts.timeLimitMin ? Math.round(opts.timeLimitMin * 60) : null,
sataScoring,
};
const id = newId();
const at = now();
db.run(
`INSERT INTO attempts (id, quiz_id, class_id, title, mode, kind, status, settings, snapshot, question_count, started_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?)`,
id,
quizId,
classId,
title,
opts.mode,
kind,
JSON.stringify(settings),
JSON.stringify({ questions }),
questions.length,
at,
at,
);
return id;
}
function loadAttempt(db: DB, id: string): AttemptRow {
const row = db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', id);
if (!row) throw notFound('Session');
return row;
}
function recordResponse(db: DB, row: AttemptRow, q: SnapshotQuestion, selected: string[], sata: SataScoring, at: number): void {
const score = scoreQuestion(q, selected, sata);
db.run(
`INSERT INTO responses (attempt_id, question_id, quiz_id, class_id, topic, score, correct, answered_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(attempt_id, question_id) DO UPDATE SET score = excluded.score, correct = excluded.correct, answered_at = excluded.answered_at`,
row.id,
q.id,
q.quizId ?? row.quiz_id,
row.class_id,
q.topic,
score,
isFullyCorrect(score) ? 1 : 0,
at,
);
}
export function mapAttempt(row: AttemptRow) {
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const results =
row.status === 'completed'
? Object.fromEntries(
snapshot.questions.map((q) => {
const selected = answers[q.id]?.selected ?? [];
const score = scoreQuestion(q, selected, settings.sataScoring);
return [q.id, { score, correct: isFullyCorrect(score), answered: selected.length > 0 }];
}),
)
: null;
return {
id: row.id,
quizId: row.quiz_id,
classId: row.class_id,
title: row.title,
mode: row.mode,
kind: row.kind,
status: row.status,
settings,
questions: snapshot.questions,
answers,
flags: parseJson<string[]>(row.flags, []),
currentIndex: row.current_index,
elapsedSec: row.elapsed_sec,
score: row.score,
maxScore: row.max_score,
startedAt: row.started_at,
updatedAt: row.updated_at,
completedAt: row.completed_at,
results,
};
}
export type AttemptDto = ReturnType<typeof mapAttempt>;
export const mapAttemptSummary = (row: AttemptRow) => {
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
return {
id: row.id,
quizId: row.quiz_id,
classId: row.class_id,
title: row.title,
mode: row.mode,
kind: row.kind,
status: row.status,
questionCount: row.question_count,
answeredCount: Object.values(answers).filter((a) => a.selected.length > 0).length,
currentIndex: row.current_index,
elapsedSec: row.elapsed_sec,
score: row.score,
maxScore: row.max_score,
startedAt: row.started_at,
updatedAt: row.updated_at,
completedAt: row.completed_at,
};
};
export type AttemptSummary = ReturnType<typeof mapAttemptSummary>;
export function getAttempt(db: DB, id: string): AttemptDto {
return mapAttempt(loadAttempt(db, id));
}
export const attemptPatchSchema = z.object({
answers: z
.record(z.string(), z.object({ selected: z.array(z.string().max(40)).max(12), checked: z.boolean().optional().default(false) }))
.optional(),
flags: z.array(z.string()).max(1000).optional(),
currentIndex: z.number().int().min(0).optional(),
elapsedSec: z.number().int().min(0).max(1_000_000).optional(),
});
export function patchAttempt(db: DB, id: string, input: z.input<typeof attemptPatchSchema>): AttemptDto {
const patch = attemptPatchSchema.parse(input);
const row = loadAttempt(db, id);
if (row.status !== 'active') throw badRequest('This session is already finished');
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const at = now();
db.tx(() => {
if (patch.answers) {
const byId = new Map(snapshot.questions.map((q) => [q.id, q]));
for (const [qid, a] of Object.entries(patch.answers)) {
const q = byId.get(qid);
if (!q) continue;
const prev = answers[qid];
// A checked practice answer is final: feedback has been shown.
if (prev?.checked && row.mode === 'practice') continue;
const valid = new Set(q.options.map((o) => o.id));
const selected = [...new Set(a.selected.filter((s) => valid.has(s)))];
const checked = row.mode === 'practice' && a.checked && selected.length > 0;
answers[qid] = { selected, checked, at };
if (checked) recordResponse(db, row, q, selected, settings.sataScoring, at);
}
}
db.run(
'UPDATE attempts SET answers = ?, flags = ?, current_index = ?, elapsed_sec = ?, updated_at = ? WHERE id = ?',
JSON.stringify(answers),
patch.flags ? JSON.stringify(patch.flags) : row.flags,
patch.currentIndex !== undefined ? Math.min(patch.currentIndex, Math.max(0, snapshot.questions.length - 1)) : row.current_index,
patch.elapsedSec !== undefined ? Math.max(row.elapsed_sec, patch.elapsedSec) : row.elapsed_sec,
at,
id,
);
});
return getAttempt(db, id);
}
/**
* Finishes a session. Exams score every question (blank = wrong). Practice sessions that are
* ended early are scored on the questions actually answered.
*/
export function completeAttempt(db: DB, id: string, elapsedSec?: number): AttemptDto {
const row = loadAttempt(db, id);
if (row.status === 'completed') return mapAttempt(row);
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const at = now();
let score = 0;
let max = 0;
db.tx(() => {
for (const q of snapshot.questions) {
const selected = answers[q.id]?.selected ?? [];
const answered = selected.length > 0;
if (row.mode === 'practice' && !answered) continue;
max += 1;
score += scoreQuestion(q, selected, settings.sataScoring);
const already = row.mode === 'practice' && answers[q.id]?.checked;
if (!already) recordResponse(db, row, q, selected, settings.sataScoring, answers[q.id]?.at ?? at);
}
db.run(
`UPDATE attempts SET status = 'completed', score = ?, max_score = ?, completed_at = ?, updated_at = ?, elapsed_sec = ? WHERE id = ?`,
score,
max,
at,
at,
elapsedSec !== undefined ? Math.max(row.elapsed_sec, elapsedSec) : row.elapsed_sec,
id,
);
});
return getAttempt(db, id);
}
export function deleteAttempt(db: DB, id: string): void {
const row = loadAttempt(db, id);
db.run('DELETE FROM attempts WHERE id = ?', row.id);
}
export function listAttempts(db: DB, filter: { status?: 'active' | 'completed'; quizId?: string; classId?: string; limit?: number }): AttemptSummary[] {
const where: string[] = [];
const params: (string | number)[] = [];
if (filter.status) {
where.push('status = ?');
params.push(filter.status);
}
if (filter.quizId) {
where.push('quiz_id = ?');
params.push(filter.quizId);
}
if (filter.classId) {
where.push('class_id = ?');
params.push(filter.classId);
}
const rows = db.all<AttemptRow>(
`SELECT * FROM attempts ${where.length ? `WHERE ${where.join(' AND ')}` : ''} ORDER BY COALESCE(completed_at, updated_at) DESC LIMIT ?`,
...params,
Math.min(filter.limit ?? 50, 500),
);
return rows.map(mapAttemptSummary);
}

518
server/services/content.ts Normal file
View file

@ -0,0 +1,518 @@
/**
* Classes, folders, quizzes, questions, decks, cards and notes.
* Used by the HTTP routes, the importer and the AI tools, so every write path
* shares one set of validation and ordering rules.
*/
import { z } from 'zod';
import {
CLASS_COLORS,
cardInputSchema,
questionInputSchema,
type Card,
type Difficulty,
type DraftCard,
type DraftQuestion,
type Option,
type Question,
type QuestionType,
} from '../../shared/content.ts';
import type { DB } from '../db.ts';
import { notFound } from '../http.ts';
import { newId, now, parseJson } from '../util.ts';
/* ------------------------------ row types ------------------------------ */
export interface ClassRow {
id: string;
name: string;
code: string;
description: string;
color: string;
archived: number;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface FolderRow {
id: string;
class_id: string;
name: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface QuizRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
description: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface QuestionRow {
id: string;
quiz_id: string;
type: QuestionType;
stem: string;
options: string;
rationale: string;
topic: string;
difficulty: Difficulty;
tags: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface DeckRow extends QuizRow {}
export interface CardRow {
id: string;
deck_id: string;
front: string;
back: string;
hint: string;
tags: string;
sort_order: number;
suspended: number;
state: number;
due: number | null;
stability: number;
difficulty: number;
scheduled_days: number;
learning_steps: number;
reps: number;
lapses: number;
last_review: number | null;
introduced_at: number | null;
created_at: number;
updated_at: number;
}
export interface NoteRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
content: string;
kind: string;
source: string;
sort_order: number;
created_at: number;
updated_at: number;
}
/* ------------------------------ mappers ------------------------------ */
export const mapClass = (r: ClassRow) => ({
id: r.id,
name: r.name,
code: r.code,
description: r.description,
color: r.color,
archived: Boolean(r.archived),
sortOrder: r.sort_order,
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export type ClassDto = ReturnType<typeof mapClass>;
export const mapFolder = (r: FolderRow) => ({ id: r.id, classId: r.class_id, name: r.name, sortOrder: r.sort_order });
export const mapQuiz = (r: QuizRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
description: r.description,
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export const mapDeck = mapQuiz;
export function mapQuestion(r: QuestionRow): Question {
return {
id: r.id,
quizId: r.quiz_id,
type: r.type,
stem: r.stem,
options: parseJson<Option[]>(r.options, []).map((o) => ({ id: o.id, text: o.text, correct: Boolean(o.correct), why: o.why ?? '' })),
rationale: r.rationale,
topic: r.topic,
difficulty: r.difficulty,
tags: parseJson<string[]>(r.tags, []),
sortOrder: r.sort_order,
createdAt: r.created_at,
updatedAt: r.updated_at,
};
}
export function mapCard(r: CardRow): Card {
return {
id: r.id,
deckId: r.deck_id,
front: r.front,
back: r.back,
hint: r.hint,
tags: parseJson<string[]>(r.tags, []),
sortOrder: r.sort_order,
suspended: Boolean(r.suspended),
state: r.state,
due: r.due,
stability: r.stability,
difficulty: r.difficulty,
scheduledDays: r.scheduled_days,
reps: r.reps,
lapses: r.lapses,
lastReview: r.last_review,
createdAt: r.created_at,
updatedAt: r.updated_at,
};
}
export const mapNote = (r: NoteRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
content: r.content,
kind: r.kind,
source: parseJson<Record<string, unknown>>(r.source, {}),
createdAt: r.created_at,
updatedAt: r.updated_at,
});
/* ------------------------------ lookups ------------------------------ */
export function getClass(db: DB, id: string): ClassRow {
const row = db.get<ClassRow>('SELECT * FROM classes WHERE id = ?', id);
if (!row) throw notFound('Class');
return row;
}
export function getFolder(db: DB, id: string): FolderRow {
const row = db.get<FolderRow>('SELECT * FROM folders WHERE id = ?', id);
if (!row) throw notFound('Folder');
return row;
}
export function getQuiz(db: DB, id: string): QuizRow {
const row = db.get<QuizRow>('SELECT * FROM quizzes WHERE id = ?', id);
if (!row) throw notFound('Quiz');
return row;
}
export function getDeck(db: DB, id: string): DeckRow {
const row = db.get<DeckRow>('SELECT * FROM decks WHERE id = ?', id);
if (!row) throw notFound('Deck');
return row;
}
export function getNote(db: DB, id: string): NoteRow {
const row = db.get<NoteRow>('SELECT * FROM notes WHERE id = ?', id);
if (!row) throw notFound('Note');
return row;
}
export function listQuestions(db: DB, quizId: string): Question[] {
return db.all<QuestionRow>('SELECT * FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quizId).map(mapQuestion);
}
export function listCards(db: DB, deckId: string): Card[] {
return db.all<CardRow>('SELECT * FROM cards WHERE deck_id = ? ORDER BY sort_order, created_at', deckId).map(mapCard);
}
/** Resolves and validates a folder id against a class; returns null for "no folder". */
export function resolveFolder(db: DB, classId: string, folderId: string | null | undefined): string | null {
if (!folderId) return null;
const folder = db.get<FolderRow>('SELECT * FROM folders WHERE id = ?', folderId);
return folder && folder.class_id === classId ? folder.id : null;
}
function nextSort(db: DB, table: string, where: string, value: string): number {
const row = db.get<{ m: number | null }>(`SELECT MAX(sort_order) AS m FROM ${table} WHERE ${where} = ?`, value);
return (row?.m ?? -1) + 1;
}
const touch = (db: DB, table: 'quizzes' | 'decks' | 'classes', id: string) => db.run(`UPDATE ${table} SET updated_at = ? WHERE id = ?`, now(), id);
/* ------------------------------ classes + folders ------------------------------ */
export const classInputSchema = z.object({
name: z.string().trim().min(1, 'Name is required').max(120),
code: z.string().trim().max(40).optional().default(''),
description: z.string().max(2000).optional().default(''),
color: z.enum(CLASS_COLORS).optional(),
});
export function createClass(db: DB, input: z.input<typeof classInputSchema>): ClassRow {
const data = classInputSchema.parse(input);
const id = newId();
const at = now();
const count = db.get<{ n: number }>('SELECT COUNT(*) AS n FROM classes')?.n ?? 0;
const color = data.color ?? CLASS_COLORS[count % CLASS_COLORS.length];
db.run(
'INSERT INTO classes (id, name, code, description, color, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
data.name,
data.code,
data.description,
color,
count,
at,
at,
);
return getClass(db, id);
}
export function createFolder(db: DB, classId: string, name: string): FolderRow {
getClass(db, classId);
const id = newId();
const at = now();
db.run(
'INSERT INTO folders (id, class_id, name, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)',
id,
classId,
name.trim().slice(0, 120) || 'Untitled folder',
nextSort(db, 'folders', 'class_id', classId),
at,
at,
);
return getFolder(db, id);
}
/* ------------------------------ quizzes + questions ------------------------------ */
type ParsedQuestion = z.output<typeof questionInputSchema>;
function optionsJson(q: ParsedQuestion, keep?: Option[]): string {
const used = new Set<string>();
const options: Option[] = q.options.map((o) => {
let id = o.id && !used.has(o.id) ? o.id : undefined;
// Keep option ids stable across edits when the text is unchanged, so old attempts still line up.
if (!id && keep) id = keep.find((k) => k.text === o.text && !used.has(k.id))?.id;
if (!id) id = newId().slice(0, 8);
used.add(id);
return { id, text: o.text, correct: o.correct, why: o.why ?? '' };
});
return JSON.stringify(options);
}
export function insertQuestions(db: DB, quizId: string, inputs: unknown[], atIndex?: number): Question[] {
const parsed = inputs.map((q) => questionInputSchema.parse(q));
const at = now();
const ids: string[] = [];
db.tx(() => {
let sort = nextSort(db, 'questions', 'quiz_id', quizId);
if (atIndex !== undefined) {
const ordered = db.all<{ id: string }>('SELECT id FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quizId);
const insertAt = Math.max(0, Math.min(atIndex, ordered.length));
ordered.forEach((row, i) => db.run('UPDATE questions SET sort_order = ? WHERE id = ?', i < insertAt ? i : i + parsed.length, row.id));
sort = insertAt;
}
for (const q of parsed) {
const id = newId();
ids.push(id);
db.run(
'INSERT INTO questions (id, quiz_id, type, stem, options, rationale, topic, difficulty, tags, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
quizId,
q.type,
q.stem,
optionsJson(q),
q.rationale,
q.topic,
q.difficulty,
JSON.stringify(q.tags),
sort,
at,
at,
);
sort += 1;
}
touch(db, 'quizzes', quizId);
});
const all = listQuestions(db, quizId);
return ids.map((id) => all.find((q) => q.id === id)!);
}
export function updateQuestion(db: DB, id: string, input: unknown): Question {
const existing = db.get<QuestionRow>('SELECT * FROM questions WHERE id = ?', id);
if (!existing) throw notFound('Question');
const q = questionInputSchema.parse(input);
db.run(
'UPDATE questions SET type = ?, stem = ?, options = ?, rationale = ?, topic = ?, difficulty = ?, tags = ?, updated_at = ? WHERE id = ?',
q.type,
q.stem,
optionsJson(q, mapQuestion(existing).options),
q.rationale,
q.topic,
q.difficulty,
JSON.stringify(q.tags),
now(),
id,
);
touch(db, 'quizzes', existing.quiz_id);
return mapQuestion(db.get<QuestionRow>('SELECT * FROM questions WHERE id = ?', id)!);
}
export function createQuiz(
db: DB,
input: { classId: string; folderId?: string | null; title: string; description?: string; questions?: unknown[] },
): QuizRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.tx(() => {
db.run(
'INSERT INTO quizzes (id, class_id, folder_id, title, description, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim(),
input.description ?? '',
nextSort(db, 'quizzes', 'class_id', input.classId),
at,
at,
);
if (input.questions?.length) insertQuestions(db, id, input.questions);
});
return getQuiz(db, id);
}
export function duplicateQuiz(db: DB, id: string, target?: { classId?: string; folderId?: string | null; title?: string }): QuizRow {
const quiz = getQuiz(db, id);
const questions = listQuestions(db, id);
const classId = target?.classId ?? quiz.class_id;
return createQuiz(db, {
classId,
folderId: target?.folderId !== undefined ? target.folderId : classId === quiz.class_id ? quiz.folder_id : null,
title: target?.title ?? `${quiz.title} (copy)`,
description: quiz.description,
questions: questions.map((q) => ({
type: q.type,
stem: q.stem,
options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why })),
rationale: q.rationale,
topic: q.topic,
difficulty: q.difficulty,
tags: q.tags,
})),
});
}
/* ------------------------------ decks + cards ------------------------------ */
export function insertCards(db: DB, deckId: string, inputs: unknown[]): Card[] {
const parsed = inputs.map((c) => cardInputSchema.parse(c));
const at = now();
const ids: string[] = [];
db.tx(() => {
let sort = nextSort(db, 'cards', 'deck_id', deckId);
for (const c of parsed) {
const id = newId();
ids.push(id);
db.run(
'INSERT INTO cards (id, deck_id, front, back, hint, tags, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
deckId,
c.front,
c.back,
c.hint,
JSON.stringify(c.tags),
sort,
at,
at,
);
sort += 1;
}
touch(db, 'decks', deckId);
});
const all = listCards(db, deckId);
return ids.map((id) => all.find((c) => c.id === id)!);
}
export function updateCard(db: DB, id: string, input: unknown): Card {
const existing = db.get<CardRow>('SELECT * FROM cards WHERE id = ?', id);
if (!existing) throw notFound('Card');
const c = cardInputSchema.parse(input);
db.run('UPDATE cards SET front = ?, back = ?, hint = ?, tags = ?, updated_at = ? WHERE id = ?', c.front, c.back, c.hint, JSON.stringify(c.tags), now(), id);
touch(db, 'decks', existing.deck_id);
return mapCard(db.get<CardRow>('SELECT * FROM cards WHERE id = ?', id)!);
}
export function createDeck(
db: DB,
input: { classId: string; folderId?: string | null; title: string; description?: string; cards?: unknown[] },
): DeckRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.tx(() => {
db.run(
'INSERT INTO decks (id, class_id, folder_id, title, description, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim(),
input.description ?? '',
nextSort(db, 'decks', 'class_id', input.classId),
at,
at,
);
if (input.cards?.length) insertCards(db, id, input.cards);
});
return getDeck(db, id);
}
export function duplicateDeck(db: DB, id: string, target?: { classId?: string; folderId?: string | null; title?: string }): DeckRow {
const deck = getDeck(db, id);
const cards = listCards(db, id);
const classId = target?.classId ?? deck.class_id;
return createDeck(db, {
classId,
folderId: target?.folderId !== undefined ? target.folderId : classId === deck.class_id ? deck.folder_id : null,
title: target?.title ?? `${deck.title} (copy)`,
description: deck.description,
cards: cards.map((c) => ({ front: c.front, back: c.back, hint: c.hint, tags: c.tags })),
});
}
/* ------------------------------ notes ------------------------------ */
export function createNote(
db: DB,
input: { classId: string; folderId?: string | null; title: string; content?: string; kind?: string; source?: unknown },
): NoteRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.run(
'INSERT INTO notes (id, class_id, folder_id, title, content, kind, source, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim().slice(0, 200) || 'Untitled note',
input.content ?? '',
input.kind ?? 'note',
JSON.stringify(input.source ?? {}),
nextSort(db, 'notes', 'class_id', input.classId),
at,
at,
);
return getNote(db, id);
}
/* ------------------------------ draft conversion ------------------------------ */
/** Turns a (validated) draft question into a strict question input, folding the citation into the rationale. */
export function draftQuestionToInput(q: DraftQuestion) {
const source = q.source?.trim();
const rationale = source && !q.rationale.includes(source) ? `${q.rationale.trim()}\n\n*Source: ${source}*`.trim() : q.rationale;
return {
type: q.type,
stem: q.stem,
options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why ?? '' })),
rationale,
topic: (q.topic ?? '').slice(0, 80),
difficulty: q.difficulty ?? 'medium',
tags: (q.tags ?? []).map((t) => t.slice(0, 40)).filter(Boolean).slice(0, 12),
};
}
export function draftCardToInput(c: DraftCard) {
return { front: c.front, back: c.back, hint: c.hint ?? '', tags: (c.tags ?? []).map((t) => t.slice(0, 40)).filter(Boolean).slice(0, 12) };
}

View file

@ -0,0 +1,500 @@
/**
* Uploaded study materials: storage, the background processing queue
* (extract -> optional AI transcription -> chunk -> index) and retrieval.
*/
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { resolveModel, runChat } from '../ai/service.ts';
import { AIError, type ContentPart } from '../ai/types.ts';
import type { DB } from '../db.ts';
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
import { detectKind, extractDocument } from '../documents/extract.ts';
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
import { HttpError, notFound, type AppContext } from '../http.ts';
import { ftsQuery, newId, now, parseJson } from '../util.ts';
import { getClass, resolveFolder } from './content.ts';
export interface DocumentRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
filename: string;
mime: string;
kind: DocKind;
size: number;
sha256: string;
storage_name: string;
status: 'pending' | 'processing' | 'ready' | 'failed';
error: string;
progress: number;
page_count: number | null;
char_count: number;
token_count: number;
chunk_count: number;
needs_ocr: number;
warnings: string;
created_at: number;
updated_at: number;
}
export const mapDocument = (r: DocumentRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
filename: r.filename,
mime: r.mime,
docKind: r.kind,
size: r.size,
status: r.status,
error: r.error,
progress: r.progress,
pageCount: r.page_count,
charCount: r.char_count,
tokenCount: r.token_count,
chunkCount: r.chunk_count,
needsOcr: Boolean(r.needs_ocr),
warnings: parseJson<string[]>(r.warnings, []),
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export type DocumentDto = ReturnType<typeof mapDocument>;
const MIME_BY_KIND: Record<DocKind, string> = {
pdf: 'application/pdf',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
text: 'text/plain; charset=utf-8',
markdown: 'text/markdown; charset=utf-8',
image: 'application/octet-stream',
};
function imageMime(bytes: Uint8Array): string {
if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'image/png';
if (bytes[0] === 0xff && bytes[1] === 0xd8) return 'image/jpeg';
if (bytes[0] === 0x47 && bytes[1] === 0x49) return 'image/gif';
if (bytes[0] === 0x52 && bytes[8] === 0x57) return 'image/webp';
return 'application/octet-stream';
}
export function getDocument(db: DB, id: string): DocumentRow {
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) throw notFound('Document');
return row;
}
export function documentPath(app: AppContext, row: Pick<DocumentRow, 'storage_name'>): string {
// basename() guarantees a stored name can never escape the uploads folder.
return path.join(app.config.uploadsDir, path.basename(row.storage_name));
}
function titleFromFilename(filename: string): string {
const base = filename.replace(/\.[^.]+$/, '').replace(/[_]+/g, ' ').replace(/\s+/g, ' ').trim();
return (base || 'Untitled document').slice(0, 200);
}
/* ------------------------------ ingest ------------------------------ */
export function ingestUpload(
app: AppContext,
input: { classId: string; folderId?: string | null; filename: string; mime?: string; bytes: Uint8Array; title?: string },
): DocumentRow {
const { db } = app;
getClass(db, input.classId);
if (input.bytes.length === 0) throw new HttpError(422, 'empty_file', `"${input.filename}" is empty.`);
if (input.bytes.length > app.config.maxUploadBytes) {
throw new HttpError(413, 'too_large', `"${input.filename}" is larger than the ${Math.round(app.config.maxUploadBytes / 1048576)} MB upload limit.`);
}
const filename = path.basename(input.filename.replace(/\\/g, '/')).slice(0, 200) || 'upload';
const kind = detectKind(filename, input.mime, input.bytes);
if (!kind) {
throw new HttpError(415, 'unsupported_type', `"${filename}" is not a supported file type. Upload PDF, Word (.docx), PowerPoint (.pptx), text, Markdown, or an image.`);
}
const id = newId();
const ext = { pdf: '.pdf', docx: '.docx', pptx: '.pptx', text: '.txt', markdown: '.md', image: path.extname(filename).toLowerCase().replace(/[^.a-z0-9]/g, '') || '.img' }[kind];
const storageName = `${id}${ext}`;
fs.writeFileSync(path.join(app.config.uploadsDir, storageName), input.bytes);
const at = now();
const sort = (db.get<{ m: number | null }>('SELECT MAX(sort_order) AS m FROM documents WHERE class_id = ?', input.classId)?.m ?? -1) + 1;
db.run(
`INSERT INTO documents (id, class_id, folder_id, title, filename, mime, kind, size, sha256, storage_name, status, sort_order, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`,
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
(input.title?.trim() || titleFromFilename(filename)).slice(0, 200),
filename,
kind === 'image' ? imageMime(input.bytes) : MIME_BY_KIND[kind],
kind,
input.bytes.length,
crypto.createHash('sha256').update(input.bytes).digest('hex'),
storageName,
sort,
at,
at,
);
enqueue(app, id);
return getDocument(db, id);
}
export function deleteDocument(app: AppContext, id: string): void {
const row = getDocument(app.db, id);
app.db.tx(() => {
app.db.run(`DELETE FROM shares WHERE kind = 'document' AND target_id = ?`, id);
app.db.run('DELETE FROM documents WHERE id = ?', id);
});
fs.rmSync(documentPath(app, row), { force: true });
}
/* ------------------------------ processing queue ------------------------------ */
const queues = new WeakMap<DB, { pending: string[]; running: boolean; ocr: Set<string>; idle: Promise<void>; wake: () => void }>();
function queueFor(app: AppContext) {
let q = queues.get(app.db);
if (!q) {
q = { pending: [], running: false, ocr: new Set(), idle: Promise.resolve(), wake: () => undefined };
queues.set(app.db, q);
}
return q;
}
export function enqueue(app: AppContext, id: string, opts: { ocr?: boolean } = {}): void {
const q = queueFor(app);
if (opts.ocr) q.ocr.add(id);
if (!q.pending.includes(id)) q.pending.push(id);
if (!q.running) {
q.running = true;
q.idle = (async () => {
// Yield first so the HTTP response for the upload is sent before heavy parsing starts.
await new Promise((r) => setTimeout(r, 10));
while (q.pending.length > 0) {
const next = q.pending.shift()!;
const forceOcr = q.ocr.delete(next);
try {
await processDocument(app, next, { forceOcr });
} catch (e) {
app.log('error', `Document ${next} failed unexpectedly`, e);
}
}
q.running = false;
})();
}
}
/** Resolves when the queue is empty. Used by tests and graceful shutdown. */
export function queueIdle(app: AppContext): Promise<void> {
return queueFor(app).idle;
}
/** Re-queues anything a restart interrupted. */
export function resumeQueue(app: AppContext): number {
const rows = app.db.all<{ id: string }>(`SELECT id FROM documents WHERE status IN ('pending', 'processing') ORDER BY created_at`);
for (const r of rows) enqueue(app, r.id);
return rows.length;
}
const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
- Reproduce all readable text faithfully, in reading order. Do not summarise and do not add information.
- Keep headings, lists and tables (as Markdown tables).
- For diagrams, charts and labelled figures, add a short description in italics that includes every label and value shown.
- For a multi-page document, start each page with a line of the form "## Page N".
- If something is illegible, write [illegible].
Return only the transcription.`;
/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */
async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise<ExtractedSection[] | null> {
const data = Buffer.from(bytes).toString('base64');
const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename };
if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) {
throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.');
}
let result;
try {
const vision = resolveModel(app, 'vision');
if (!vision.model.capabilities.vision) return null;
if (row.kind === 'pdf' && !vision.model.capabilities.pdf) {
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
}
result = await runChat(app, {
task: 'vision',
purpose: 'document transcription',
messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }],
maxOutputTokens: 32000,
reasoningEffort: 'low',
});
} catch (err) {
if (err instanceof AIError && err.code === 'not_configured') return null;
throw err;
}
const text = result.text.trim();
if (!text) return [];
// Split on the "## Page N" markers the prompt asks for so citations keep page numbers.
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }];
const sections: ExtractedSection[] = [];
if (pages[0].trim()) sections.push({ text: pages[0].trim() });
for (let i = 1; i < pages.length; i += 2) {
const body = (pages[i + 1] ?? '').trim();
if (body) sections.push({ text: body, page: Number(pages[i]) });
}
return sections;
}
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
const { db } = app;
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) return;
const setProgress = (p: number) => db.run('UPDATE documents SET progress = ?, updated_at = ? WHERE id = ?', Math.round(p * 100) / 100, now(), id);
db.run(`UPDATE documents SET status = 'processing', error = '', progress = 0, updated_at = ? WHERE id = ?`, now(), id);
try {
const file = documentPath(app, row);
if (!fs.existsSync(file)) throw new ExtractionError('failed', 'The uploaded file is missing from storage. Upload it again.');
const bytes = new Uint8Array(await fs.promises.readFile(file));
let lastWrite = 0;
const result = await extractDocument({
bytes,
filename: row.filename,
mime: row.mime,
onProgress: (done, total) => {
const t = Date.now();
if (t - lastWrite > 400 && total > 0) {
lastWrite = t;
setProgress((done / total) * 0.8);
}
},
});
const warnings = [...result.warnings];
let sections = result.sections;
let needsOcr = result.needsOcr;
if (needsOcr || opts.forceOcr) {
setProgress(0.82);
if (row.kind === 'image' || row.kind === 'pdf') {
try {
const transcribed = await transcribeWithVision(app, row, bytes);
if (transcribed === null) {
warnings.push(
row.kind === 'image'
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
);
} else if (transcribed.length > 0) {
sections = transcribed;
needsOcr = false;
} else {
warnings.push('The AI model did not find any readable text in this file.');
}
} catch (err) {
warnings.push(`Reading with AI failed: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
}
const chunks = chunkSections(sections);
const chars = sections.reduce((n, s) => n + s.text.length, 0);
const tokens = chunks.reduce((n, c) => n + c.tokens, 0);
db.tx(() => {
db.run('DELETE FROM chunks WHERE document_id = ?', id);
const current = db.get<{ class_id: string }>('SELECT class_id FROM documents WHERE id = ?', id);
if (!current) return; // deleted while processing
for (const c of chunks) {
db.run(
'INSERT INTO chunks (document_id, class_id, seq, text, heading, page_start, page_end, tokens) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
current.class_id,
c.seq,
c.text,
c.heading,
c.pageStart,
c.pageEnd,
c.tokens,
);
}
db.run(
`UPDATE documents SET status = 'ready', error = '', progress = 1, page_count = ?, char_count = ?, token_count = ?, chunk_count = ?, needs_ocr = ?, warnings = ?, updated_at = ? WHERE id = ?`,
result.pageCount ?? null,
chars,
tokens,
chunks.length,
needsOcr ? 1 : 0,
JSON.stringify([...new Set(warnings)].slice(0, 8)),
now(),
id,
);
});
} catch (err) {
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
db.run(`UPDATE documents SET status = 'failed', error = ?, progress = 0, updated_at = ? WHERE id = ?`, message, now(), id);
}
}
/* ------------------------------ retrieval ------------------------------ */
export interface ChunkHit {
id: number;
documentId: string;
documentTitle: string;
seq: number;
text: string;
heading: string;
pageStart: number | null;
pageEnd: number | null;
tokens: number;
score?: number;
}
type ChunkJoin = { id: number; document_id: string; doc_title: string; seq: number; text: string; heading: string; page_start: number | null; page_end: number | null; tokens: number; score?: number };
const mapHit = (r: ChunkJoin): ChunkHit => ({
id: r.id,
documentId: r.document_id,
documentTitle: r.doc_title,
seq: r.seq,
text: r.text,
heading: r.heading,
pageStart: r.page_start,
pageEnd: r.page_end,
tokens: r.tokens,
score: r.score,
});
export interface ChunkScope {
classId?: string | null;
documentIds?: string[];
}
function scopeSql(scope: ChunkScope): { sql: string; params: string[] } {
if (scope.documentIds?.length) return { sql: `AND ch.document_id IN (${scope.documentIds.map(() => '?').join(',')})`, params: scope.documentIds };
if (scope.classId) return { sql: 'AND ch.class_id = ?', params: [scope.classId] };
return { sql: '', params: [] };
}
/** BM25 full-text search over document chunks. */
export function searchChunks(db: DB, query: string, scope: ChunkScope, limit = 8): ChunkHit[] {
const match = ftsQuery(query, 'or');
if (!match) return [];
const { sql, params } = scopeSql(scope);
try {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens, bm25(chunks_fts, 1.0, 2.0) AS score
FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id
WHERE chunks_fts MATCH ? ${sql} ORDER BY score LIMIT ?`,
match,
...params,
limit,
)
.map(mapHit);
} catch {
return [];
}
}
export function listChunks(db: DB, documentId: string, offset = 0, limit = 50): ChunkHit[] {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.document_id = ? AND ch.seq >= ? ORDER BY ch.seq LIMIT ?`,
documentId,
offset,
limit,
)
.map(mapHit);
}
export function chunkById(db: DB, id: number): ChunkHit | null {
const row = db.get<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.id = ?`,
id,
);
return row ? mapHit(row) : null;
}
export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' | 'pageEnd'>): string {
if (hit.pageStart === null) return hit.documentTitle;
return hit.pageEnd && hit.pageEnd !== hit.pageStart ? `${hit.documentTitle}, pp. ${hit.pageStart}-${hit.pageEnd}` : `${hit.documentTitle}, p. ${hit.pageStart}`;
}
/**
* Chooses source material for a generation job and splits it into `groups` batches that
* each fit `budgetTokens`.
* - With a focus, the best-matching chunks are used (kept in document order).
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
* so every part of the document is represented.
*/
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number }): ChunkHit[][] {
const { sql, params } = scopeSql(scope);
if (!sql) return [];
const all = db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE 1 = 1 ${sql} ORDER BY d.sort_order, d.created_at, ch.seq`,
...params,
)
.map(mapHit);
if (all.length === 0) return [];
const capacity = opts.budgetTokens * opts.groups;
const total = all.reduce((n, c) => n + c.tokens, 0);
let chosen = all;
if (opts.focus?.trim()) {
const ranked = searchChunks(db, opts.focus, scope, 400);
if (ranked.length >= 3) {
const keep = new Set<number>();
let used = 0;
for (const hit of ranked) {
if (used + hit.tokens > capacity) break;
keep.add(hit.id);
used += hit.tokens;
}
chosen = all.filter((c) => keep.has(c.id));
}
}
if (chosen === all && total > capacity) {
// Even sampling: walk the document and keep chunks at a steady stride.
const stride = total / capacity;
const picked: ChunkHit[] = [];
let acc = 0;
let next = 0;
for (const c of all) {
if (acc >= next) {
picked.push(c);
next += c.tokens * stride;
}
acc += c.tokens;
}
chosen = picked;
}
// Contiguous groups of roughly equal size, none above the budget.
const chosenTotal = chosen.reduce((n, c) => n + c.tokens, 0);
const groupCount = Math.max(1, Math.min(opts.groups, chosen.length));
const target = Math.min(opts.budgetTokens, Math.ceil(chosenTotal / groupCount));
const groups: ChunkHit[][] = [];
let current: ChunkHit[] = [];
let size = 0;
for (const c of chosen) {
if (current.length > 0 && size + c.tokens > target && groups.length < groupCount - 1) {
groups.push(current);
current = [];
size = 0;
}
if (size + c.tokens > opts.budgetTokens && current.length > 0) continue; // last group is full
current.push(c);
size += c.tokens;
}
if (current.length > 0) groups.push(current);
return groups;
}
export function formatExcerpts(chunks: ChunkHit[]): string {
return chunks.map((c) => `[c${c.id}] ${sourceLabel(c)}${c.heading ? ` — ${c.heading}` : ''}\n${c.text}`).join('\n\n---\n\n');
}
export { estimateTokens };

309
server/services/srs.ts Normal file
View file

@ -0,0 +1,309 @@
/**
* Spaced repetition with FSRS (the algorithm Anki adopted in 2023), via ts-fsrs.
* Scheduling state lives on each card row; every review is logged so it can be undone
* and analysed later.
*/
import { createEmptyCard, fsrs, Rating, State, type Card as FsrsCard, type Grade } from 'ts-fsrs';
import type { DB } from '../db.ts';
import { badRequest, notFound } from '../http.ts';
import type { AppSettings } from '../settings.ts';
import { DAY_MS, now as clock, parseJson, startOfDay } from '../util.ts';
import { mapCard, type CardRow } from './content.ts';
export const LEARN_AHEAD_MS = 20 * 60_000;
export type RatingValue = 1 | 2 | 3 | 4;
export const RATING_LABELS: Record<RatingValue, string> = { 1: 'Again', 2: 'Hard', 3: 'Good', 4: 'Easy' };
const schedulers = new Map<string, ReturnType<typeof fsrs>>();
function scheduler(retention: number, fuzz: boolean) {
const key = `${retention}:${fuzz}`;
let s = schedulers.get(key);
if (!s) {
s = fsrs({ request_retention: retention, enable_fuzz: fuzz, enable_short_term: true, maximum_interval: 36500 });
schedulers.set(key, s);
}
return s;
}
function toFsrs(row: CardRow, at: number): FsrsCard {
if (row.state === State.New || row.due === null) return createEmptyCard(new Date(at));
return {
due: new Date(row.due),
stability: row.stability,
difficulty: row.difficulty,
scheduled_days: row.scheduled_days,
learning_steps: row.learning_steps,
reps: row.reps,
lapses: row.lapses,
state: row.state as State,
last_review: row.last_review ? new Date(row.last_review) : undefined,
} as FsrsCard;
}
export function intervalLabel(ms: number): string {
if (ms < 60_000) return '<1m';
const minutes = ms / 60_000;
if (minutes < 60) return `${Math.round(minutes)}m`;
const hours = minutes / 60;
if (hours < 24) return `${Math.round(hours)}h`;
const days = hours / 24;
if (days < 30.5) return `${Math.round(days)}d`;
const months = days / 30.44;
if (months < 12) return `${months.toFixed(months < 3 ? 1 : 0).replace(/\.0$/, '')}mo`;
return `${(days / 365.25).toFixed(1).replace(/\.0$/, '')}y`;
}
export interface RatingPreview {
rating: RatingValue;
label: string;
interval: string;
due: number;
}
export function previewRatings(row: CardRow, settings: AppSettings, at: number): RatingPreview[] {
// Previews never use fuzz so the labels are stable while the card is on screen.
const options = scheduler(settings.desiredRetention, false).repeat(toFsrs(row, at), new Date(at));
return ([1, 2, 3, 4] as RatingValue[]).map((rating) => {
const due = options[rating as Grade].card.due.getTime();
return { rating, label: RATING_LABELS[rating], interval: intervalLabel(due - at), due };
});
}
export interface QueueScope {
deckId?: string;
classId?: string;
}
function scopeSql(scope: QueueScope): { where: string; params: string[] } {
if (scope.deckId) return { where: 'AND c.deck_id = ?', params: [scope.deckId] };
if (scope.classId) return { where: 'AND d.class_id = ?', params: [scope.classId] };
return { where: '', params: [] };
}
const BASE = `FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.suspended = 0 AND k.archived = 0`;
export interface DueCounts {
learning: number;
review: number;
new: number;
total: number;
}
/** New cards still allowed today under the global daily limit. */
export function newCardsLeftToday(db: DB, settings: AppSettings, at: number): number {
const since = startOfDay(at, settings.timezone);
const introduced = db.get<{ n: number }>('SELECT COUNT(*) AS n FROM cards WHERE introduced_at >= ?', since)?.n ?? 0;
return Math.max(0, settings.newCardsPerDay - introduced);
}
export function dueCounts(db: DB, scope: QueueScope, settings: AppSettings, at = clock()): DueCounts {
const { where, params } = scopeSql(scope);
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const row = db.get<{ learning: number; review: number; fresh: number }>(
`SELECT
COALESCE(SUM(CASE WHEN c.state IN (1,3) AND c.due <= ? THEN 1 ELSE 0 END), 0) AS learning,
COALESCE(SUM(CASE WHEN c.state = 2 AND c.due < ? THEN 1 ELSE 0 END), 0) AS review,
COALESCE(SUM(CASE WHEN c.state = 0 THEN 1 ELSE 0 END), 0) AS fresh
${BASE} ${where}`,
at + LEARN_AHEAD_MS,
endOfDay,
...params,
)!;
const fresh = Math.min(row.fresh, newCardsLeftToday(db, settings, at));
return { learning: row.learning, review: row.review, new: fresh, total: row.learning + row.review + fresh };
}
export interface QueueCard {
card: ReturnType<typeof mapCard>;
deckTitle: string;
classId: string;
className: string;
classColor: string;
previews: RatingPreview[];
}
type JoinedRow = CardRow & { deck_title: string; class_id: string; class_name: string; class_color: string };
const SELECT = `SELECT c.*, d.title AS deck_title, d.class_id AS class_id, k.name AS class_name, k.color AS class_color`;
function toQueueCard(row: JoinedRow, settings: AppSettings, at: number): QueueCard {
return {
card: mapCard(row),
deckTitle: row.deck_title,
classId: row.class_id,
className: row.class_name,
classColor: row.class_color,
previews: previewRatings(row, settings, at),
};
}
/**
* Builds the study queue: learning cards first (they are time-critical), then reviews
* due today with new cards spread evenly among them.
*/
export function buildQueue(db: DB, scope: QueueScope, settings: AppSettings, limit = 100, at = clock()): { cards: QueueCard[]; counts: DueCounts } {
const { where, params } = scopeSql(scope);
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const learning = db.all<JoinedRow>(`${SELECT} ${BASE} ${where} AND c.state IN (1,3) AND c.due <= ? ORDER BY c.due LIMIT ?`, ...params, at + LEARN_AHEAD_MS, limit);
const reviews = db.all<JoinedRow>(`${SELECT} ${BASE} ${where} AND c.state = 2 AND c.due < ? ORDER BY c.due LIMIT ?`, ...params, endOfDay, limit);
const newLeft = newCardsLeftToday(db, settings, at);
const fresh =
newLeft > 0
? db.all<JoinedRow>(
`${SELECT} ${BASE} ${where} AND c.state = 0 ORDER BY k.sort_order, d.sort_order, d.created_at, c.sort_order, c.created_at LIMIT ?`,
...params,
Math.min(newLeft, limit),
)
: [];
const mixed: JoinedRow[] = [];
const every = fresh.length > 0 ? Math.max(1, Math.floor(reviews.length / fresh.length)) : 0;
let f = 0;
reviews.forEach((r, i) => {
mixed.push(r);
if (every > 0 && (i + 1) % every === 0 && f < fresh.length) mixed.push(fresh[f++]);
});
while (f < fresh.length) mixed.push(fresh[f++]);
const cards = [...learning, ...mixed].slice(0, limit).map((r) => toQueueCard(r, settings, at));
return { cards, counts: dueCounts(db, scope, settings, at) };
}
const SCHEDULE_FIELDS = ['state', 'due', 'stability', 'difficulty', 'scheduled_days', 'learning_steps', 'reps', 'lapses', 'last_review', 'introduced_at'] as const;
export function answerCard(
db: DB,
cardId: string,
rating: RatingValue,
settings: AppSettings,
opts: { at?: number; fuzz?: boolean } = {},
): QueueCard {
const at = opts.at ?? clock();
const row = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId);
if (!row) throw notFound('Card');
if (![1, 2, 3, 4].includes(rating)) throw badRequest('Rating must be 1-4');
const result = scheduler(settings.desiredRetention, opts.fuzz ?? true).next(toFsrs(row, at), new Date(at), rating as Grade);
const next = result.card;
const prev: Record<string, unknown> = {};
for (const f of SCHEDULE_FIELDS) prev[f] = row[f];
db.tx(() => {
db.run(
`UPDATE cards SET state = ?, due = ?, stability = ?, difficulty = ?, scheduled_days = ?, learning_steps = ?, reps = ?, lapses = ?, last_review = ?,
introduced_at = COALESCE(introduced_at, ?) WHERE id = ?`,
next.state,
next.due.getTime(),
next.stability,
next.difficulty,
next.scheduled_days,
next.learning_steps,
next.reps,
next.lapses,
at,
at,
cardId,
);
db.run(
'INSERT INTO reviews (card_id, deck_id, class_id, rating, state_before, scheduled_days, mode, prev, reviewed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
cardId,
row.deck_id,
row.class_id,
rating,
row.state,
next.scheduled_days,
'srs',
JSON.stringify(prev),
at,
);
});
const updated = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId)!;
return toQueueCard(updated, settings, at);
}
/** Reverts the most recent scheduled review of a card and returns it to its previous state. */
export function undoReview(db: DB, cardId: string, settings: AppSettings, at = clock()): QueueCard {
const review = db.get<{ id: number; prev: string }>(`SELECT id, prev FROM reviews WHERE card_id = ? AND mode = 'srs' ORDER BY id DESC LIMIT 1`, cardId);
if (!review || !review.prev) throw badRequest('Nothing to undo for this card');
const prev = parseJson<Record<string, number | null>>(review.prev, {});
db.tx(() => {
db.run(
`UPDATE cards SET state = ?, due = ?, stability = ?, difficulty = ?, scheduled_days = ?, learning_steps = ?, reps = ?, lapses = ?, last_review = ?, introduced_at = ? WHERE id = ?`,
prev.state ?? 0,
prev.due ?? null,
prev.stability ?? 0,
prev.difficulty ?? 0,
prev.scheduled_days ?? 0,
prev.learning_steps ?? 0,
prev.reps ?? 0,
prev.lapses ?? 0,
prev.last_review ?? null,
prev.introduced_at ?? null,
cardId,
);
db.run('DELETE FROM reviews WHERE id = ?', review.id);
});
const row = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId);
if (!row) throw notFound('Card');
return toQueueCard(row, settings, at);
}
export function resetCards(db: DB, where: { cardId?: string; deckId?: string }): number {
const sql = `UPDATE cards SET state = 0, due = NULL, stability = 0, difficulty = 0, scheduled_days = 0, learning_steps = 0, reps = 0, lapses = 0, last_review = NULL, introduced_at = NULL WHERE `;
if (where.cardId) return db.run(`${sql} id = ?`, where.cardId).changes;
if (where.deckId) return db.run(`${sql} deck_id = ?`, where.deckId).changes;
return 0;
}
/** Logs a self-graded cram flip so it counts toward activity without touching the schedule. */
export function logCram(db: DB, cardId: string, known: boolean, at = clock()): void {
const row = db.get<{ deck_id: string; class_id: string; state: number }>(
'SELECT c.deck_id, d.class_id, c.state FROM cards c JOIN decks d ON d.id = c.deck_id WHERE c.id = ?',
cardId,
);
if (!row) return;
db.run(
`INSERT INTO reviews (card_id, deck_id, class_id, rating, state_before, scheduled_days, mode, prev, reviewed_at) VALUES (?, ?, ?, ?, ?, 0, 'cram', '', ?)`,
cardId,
row.deck_id,
row.class_id,
known ? Rating.Good : Rating.Again,
row.state,
at,
);
}
export interface DeckStats {
total: number;
new: number;
learning: number;
young: number;
mature: number;
suspended: number;
dueNow: number;
}
export function deckStats(db: DB, deckId: string, settings: AppSettings, at = clock()): DeckStats {
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const r = db.get<Record<string, number>>(
`SELECT COUNT(*) AS total,
COALESCE(SUM(suspended), 0) AS suspended,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 0 THEN 1 ELSE 0 END), 0) AS fresh,
COALESCE(SUM(CASE WHEN suspended = 0 AND state IN (1,3) THEN 1 ELSE 0 END), 0) AS learning,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 2 AND scheduled_days < 21 THEN 1 ELSE 0 END), 0) AS young,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 2 AND scheduled_days >= 21 THEN 1 ELSE 0 END), 0) AS mature,
COALESCE(SUM(CASE WHEN suspended = 0 AND ((state IN (1,3) AND due <= ?) OR (state = 2 AND due < ?)) THEN 1 ELSE 0 END), 0) AS due_now
FROM cards WHERE deck_id = ?`,
at + LEARN_AHEAD_MS,
endOfDay,
deckId,
)!;
return { total: r.total, new: r.fresh, learning: r.learning, young: r.young, mature: r.mature, suspended: r.suspended, dueNow: r.due_now };
}
/** Reviews coming due per day for the next `days` days (index 0 = today, including overdue). */
export function forecast(db: DB, scope: QueueScope, settings: AppSettings, days = 14, at = clock()): number[] {
const { where, params } = scopeSql(scope);
const start = startOfDay(at, settings.timezone);
const rows = db.all<{ due: number }>(`SELECT c.due AS due ${BASE} ${where} AND c.state != 0 AND c.due < ?`, ...params, start + days * DAY_MS);
const out = new Array<number>(days).fill(0);
for (const r of rows) out[Math.max(0, Math.min(days - 1, Math.floor((r.due - start) / DAY_MS)))] += 1;
return out;
}

185
server/services/stats.ts Normal file
View file

@ -0,0 +1,185 @@
/** Study activity, streaks, accuracy and weak-topic analytics. All derived from the review and response logs. */
import type { DB } from '../db.ts';
import type { AppSettings } from '../settings.ts';
import { addDaysToKey, DAY_MS, dayKey, now as clock, startOfDay } from '../util.ts';
export interface DayActivity {
date: string;
cards: number;
questions: number;
total: number;
}
/** Per-day totals for the trailing `days` days (oldest first, always dense). */
export function activityByDay(db: DB, settings: AppSettings, days: number, scope: { classId?: string } = {}, at = clock()): DayActivity[] {
const tz = settings.timezone;
const today = dayKey(at, tz);
const since = startOfDay(at, tz) - (days - 1) * DAY_MS - DAY_MS; // one extra day absorbs DST shifts
const map = new Map<string, DayActivity>();
const bump = (ts: number, field: 'cards' | 'questions') => {
const key = dayKey(ts, tz);
let d = map.get(key);
if (!d) {
d = { date: key, cards: 0, questions: 0, total: 0 };
map.set(key, d);
}
d[field] += 1;
d.total += 1;
};
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
for (const r of db.all<{ t: number }>(`SELECT reviewed_at AS t FROM reviews WHERE reviewed_at >= ? ${cls}`, since, ...p)) bump(r.t, 'cards');
for (const r of db.all<{ t: number }>(`SELECT answered_at AS t FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)) bump(r.t, 'questions');
const out: DayActivity[] = [];
for (let i = days - 1; i >= 0; i -= 1) {
const key = addDaysToKey(today, -i);
out.push(map.get(key) ?? { date: key, cards: 0, questions: 0, total: 0 });
}
return out;
}
export interface Streak {
current: number;
longest: number;
/** Today already meets the daily goal. */
todayDone: boolean;
todayTotal: number;
goal: number;
}
export function computeStreak(days: DayActivity[], goal: number): Streak {
const met = days.map((d) => d.total >= goal);
const todayDone = met[met.length - 1] ?? false;
let current = 0;
// Today only counts once the goal is met; an unfinished today does not break yesterday's streak.
for (let i = met.length - (todayDone ? 1 : 2); i >= 0 && met[i]; i -= 1) current += 1;
let longest = 0;
let run = 0;
for (const m of met) {
run = m ? run + 1 : 0;
longest = Math.max(longest, run);
}
return { current, longest, todayDone, todayTotal: days[days.length - 1]?.total ?? 0, goal };
}
export interface TopicStat {
topic: string;
classId: string | null;
className: string;
classColor: string;
seen: number;
accuracy: number;
/** Distinct questions available on this topic right now. */
questions: number;
}
/** Accuracy per topic over the trailing window. Topics need a few answers before they count. */
export function topicStats(db: DB, scope: { classId?: string }, windowDays = 90, at = clock()): TopicStat[] {
const since = at - windowDays * DAY_MS;
const cls = scope.classId ? 'AND r.class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const rows = db.all<{ topic: string; class_id: string | null; class_name: string | null; class_color: string | null; seen: number; acc: number; questions: number }>(
`SELECT r.topic AS topic, r.class_id AS class_id, k.name AS class_name, k.color AS class_color,
COUNT(*) AS seen, AVG(r.score) AS acc,
(SELECT COUNT(*) FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE q.topic = r.topic AND z.class_id = r.class_id) AS questions
FROM responses r LEFT JOIN classes k ON k.id = r.class_id
WHERE r.answered_at >= ? AND r.topic != '' ${cls}
GROUP BY r.topic, r.class_id
ORDER BY acc ASC, seen DESC`,
since,
...p,
);
return rows.map((r) => ({
topic: r.topic,
classId: r.class_id,
className: r.class_name ?? '',
classColor: r.class_color ?? 'slate',
seen: r.seen,
accuracy: r.acc,
questions: r.questions,
}));
}
export function weakTopics(db: DB, scope: { classId?: string }, limit = 6): TopicStat[] {
return topicStats(db, scope)
.filter((t) => t.seen >= 3 && t.accuracy < 0.75 && t.questions > 0)
.slice(0, limit);
}
export interface QuestionStat {
seen: number;
correct: number;
lastCorrect: boolean;
}
export function questionStats(db: DB, quizId: string): Record<string, QuestionStat> {
const rows = db.all<{ question_id: string; seen: number; correct: number; last_correct: number }>(
`SELECT r.question_id, COUNT(*) AS seen, SUM(r.correct) AS correct,
(SELECT r2.correct FROM responses r2 WHERE r2.question_id = r.question_id ORDER BY r2.answered_at DESC LIMIT 1) AS last_correct
FROM responses r JOIN questions q ON q.id = r.question_id WHERE q.quiz_id = ? GROUP BY r.question_id`,
quizId,
);
return Object.fromEntries(rows.map((r) => [r.question_id, { seen: r.seen, correct: r.correct, lastCorrect: Boolean(r.last_correct) }]));
}
export interface AccuracyPoint {
date: string;
answered: number;
accuracy: number | null;
}
export function accuracyTrend(db: DB, settings: AppSettings, days: number, scope: { classId?: string } = {}, at = clock()): AccuracyPoint[] {
const tz = settings.timezone;
const since = startOfDay(at, tz) - days * DAY_MS;
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const sums = new Map<string, { n: number; s: number }>();
for (const r of db.all<{ t: number; score: number }>(`SELECT answered_at AS t, score FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)) {
const key = dayKey(r.t, tz);
const cur = sums.get(key) ?? { n: 0, s: 0 };
cur.n += 1;
cur.s += r.score;
sums.set(key, cur);
}
const today = dayKey(at, tz);
const out: AccuracyPoint[] = [];
for (let i = days - 1; i >= 0; i -= 1) {
const key = addDaysToKey(today, -i);
const v = sums.get(key);
out.push({ date: key, answered: v?.n ?? 0, accuracy: v ? v.s / v.n : null });
}
return out;
}
export interface Totals {
questionsAnswered: number;
accuracy: number | null;
cardsReviewed: number;
retention: number | null;
quizzesCompleted: number;
studyDays: number;
}
export function totals(db: DB, settings: AppSettings, scope: { classId?: string } = {}, windowDays = 30, at = clock()): Totals {
const since = at - windowDays * DAY_MS;
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const resp = db.get<{ n: number; acc: number | null }>(`SELECT COUNT(*) AS n, AVG(score) AS acc FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)!;
const rev = db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM reviews WHERE reviewed_at >= ? ${cls}`, since, ...p)!;
// Retention: share of reviews of already-learned cards that were remembered (not "Again").
const ret = db.get<{ n: number; ok: number | null }>(
`SELECT COUNT(*) AS n, SUM(CASE WHEN rating > 1 THEN 1 ELSE 0 END) AS ok FROM reviews WHERE reviewed_at >= ? AND mode = 'srs' AND state_before = 2 ${cls}`,
since,
...p,
)!;
const done = db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM attempts WHERE status = 'completed' AND completed_at >= ? ${cls}`, since, ...p)!;
const days = activityByDay(db, settings, windowDays, scope, at).filter((d) => d.total > 0).length;
return {
questionsAnswered: resp.n,
accuracy: resp.n > 0 ? resp.acc : null,
cardsReviewed: rev.n,
retention: ret.n >= 5 ? (ret.ok ?? 0) / ret.n : null,
quizzesCompleted: done.n,
studyDays: days,
};
}

42
server/settings.ts Normal file
View file

@ -0,0 +1,42 @@
import { z } from 'zod';
import type { DB } from './db.ts';
import { parseJson } from './util.ts';
export function getSetting<T>(db: DB, key: string, fallback: T): T {
const row = db.get<{ value: string }>('SELECT value FROM settings WHERE key = ?', key);
return row ? parseJson<T>(row.value, fallback) : fallback;
}
export function setSetting(db: DB, key: string, value: unknown): void {
db.run('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', key, JSON.stringify(value));
}
export function deleteSetting(db: DB, key: string): void {
db.run('DELETE FROM settings WHERE key = ?', key);
}
export const appSettingsSchema = z.object({
timezone: z.string().min(1).max(64),
/** New flashcards introduced per day across all decks. */
newCardsPerDay: z.number().int().min(0).max(500),
/** Target recall probability used by FSRS (0.7 - 0.97). */
desiredRetention: z.number().min(0.7).max(0.97),
sataScoring: z.enum(['partial', 'strict']),
/** Cards + questions per day that count as "studied" for the streak. */
dailyGoal: z.number().int().min(1).max(1000),
});
export type AppSettings = z.infer<typeof appSettingsSchema>;
export function getAppSettings(db: DB, defaultTimezone: string): AppSettings {
const stored = getSetting<Partial<AppSettings>>(db, 'app', {});
const merged = {
timezone: defaultTimezone,
newCardsPerDay: 20,
desiredRetention: 0.9,
sataScoring: 'partial' as const,
dailyGoal: 20,
...stored,
};
const parsed = appSettingsSchema.safeParse(merged);
return parsed.success ? parsed.data : { timezone: defaultTimezone, newCardsPerDay: 20, desiredRetention: 0.9, sataScoring: 'partial', dailyGoal: 20 };
}

97
server/util.ts Normal file
View file

@ -0,0 +1,97 @@
import crypto from 'node:crypto';
/** 16-character URL-safe id (96 bits). */
export function newId(): string {
return crypto.randomBytes(12).toString('base64url');
}
/** 32-character URL-safe token (192 bits) for share links and sessions. */
export function newToken(): string {
return crypto.randomBytes(24).toString('base64url');
}
export function sha256(input: string | Uint8Array): string {
return crypto.createHash('sha256').update(input).digest('hex');
}
export const now = (): number => Date.now();
export function parseJson<T>(text: string | null | undefined, fallback: T): T {
if (!text) return fallback;
try {
return JSON.parse(text) as T;
} catch {
return fallback;
}
}
export const DAY_MS = 86_400_000;
/** YYYY-MM-DD for a timestamp in the given IANA timezone. */
export function dayKey(ts: number, timeZone: string): string {
const parts = new Intl.DateTimeFormat('en-CA', { timeZone, year: 'numeric', month: '2-digit', day: '2-digit' }).formatToParts(new Date(ts));
const get = (t: string) => parts.find((p) => p.type === t)?.value ?? '';
return `${get('year')}-${get('month')}-${get('day')}`;
}
/** Offset in ms between UTC and the timezone at the given instant (positive = ahead of UTC). */
export function tzOffsetMs(ts: number, timeZone: string): number {
const parts = new Intl.DateTimeFormat('en-US', {
timeZone,
hourCycle: 'h23',
year: 'numeric',
month: 'numeric',
day: 'numeric',
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
}).formatToParts(new Date(ts));
const get = (t: string) => Number(parts.find((p) => p.type === t)?.value ?? 0);
const asUtc = Date.UTC(get('year'), get('month') - 1, get('day'), get('hour'), get('minute'), get('second'));
return asUtc - Math.floor(ts / 1000) * 1000;
}
/** Timestamp of local midnight that starts the day containing `ts` in the timezone. */
export function startOfDay(ts: number, timeZone: string): number {
const key = dayKey(ts, timeZone);
const utcMidnight = Date.parse(`${key}T00:00:00.000Z`);
// Two passes handle days where the offset at midnight differs from the offset at `ts` (DST).
let guess = utcMidnight - tzOffsetMs(ts, timeZone);
guess = utcMidnight - tzOffsetMs(guess, timeZone);
return guess;
}
export function addDaysToKey(key: string, days: number): string {
const d = new Date(`${key}T00:00:00.000Z`);
d.setUTCDate(d.getUTCDate() + days);
return d.toISOString().slice(0, 10);
}
export function clamp(n: number, min: number, max: number): number {
return Math.min(max, Math.max(min, n));
}
export function slugify(input: string): string {
return (
input
.toLowerCase()
.normalize('NFKD')
.replace(/[^\w\s-]/g, '')
.trim()
.replace(/[\s_]+/g, '-')
.slice(0, 60) || 'export'
);
}
/** Builds an FTS5 MATCH expression from free text: quoted terms OR-ed together, prefix match on the last term. */
export function ftsQuery(input: string, mode: 'or' | 'and' = 'or'): string | null {
const terms = (input.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []).filter((t) => !STOP_WORDS.has(t)).slice(0, 24);
if (terms.length === 0) return null;
return [...new Set(terms)].map((t) => `"${t}"`).join(mode === 'or' ? ' OR ' : ' AND ');
}
const STOP_WORDS = new Set(
'a an and are as at be but by for from has have how if in into is it its of on or that the their them then there these this to was what when where which who why will with about your you can do does i me my we our should would could not no nor so than too very'.split(
' ',
),
);

352
shared/content.ts Normal file
View file

@ -0,0 +1,352 @@
/**
* Content model shared by the server, the admin app and the public study pages.
* Zod schemas here are the single source of truth for what a valid quiz, question,
* deck or card looks like.
*/
import { z } from 'zod';
export const LIMITS = {
title: 200,
description: 4000,
markdown: 20000,
option: 4000,
topic: 80,
tag: 40,
tags: 12,
options: 10,
questionsPerQuiz: 1000,
cardsPerDeck: 5000,
noteContent: 400000,
} as const;
export const QUESTION_TYPES = ['single', 'multi'] as const;
export type QuestionType = (typeof QUESTION_TYPES)[number];
export const DIFFICULTIES = ['easy', 'medium', 'hard'] as const;
export type Difficulty = (typeof DIFFICULTIES)[number];
export const CLASS_COLORS = ['cobalt', 'teal', 'moss', 'amber', 'coral', 'rose', 'plum', 'slate'] as const;
export type ClassColor = (typeof CLASS_COLORS)[number];
export const QUESTION_TYPE_LABEL: Record<QuestionType, string> = {
single: 'Multiple choice',
multi: 'Select all that apply',
};
/* ------------------------------------------------------------------ */
/* Stored shapes */
/* ------------------------------------------------------------------ */
export interface Option {
id: string;
text: string;
correct: boolean;
/** Optional per-option rationale ("why this is right / wrong"). */
why: string;
}
export interface Question {
id: string;
quizId: string;
type: QuestionType;
stem: string;
options: Option[];
rationale: string;
topic: string;
difficulty: Difficulty;
tags: string[];
sortOrder: number;
createdAt: number;
updatedAt: number;
}
export interface Card {
id: string;
deckId: string;
front: string;
back: string;
hint: string;
tags: string[];
sortOrder: number;
suspended: boolean;
/** FSRS state: 0 new, 1 learning, 2 review, 3 relearning. */
state: number;
due: number | null;
stability: number;
difficulty: number;
scheduledDays: number;
reps: number;
lapses: number;
lastReview: number | null;
createdAt: number;
updatedAt: number;
}
/* ------------------------------------------------------------------ */
/* Input schemas */
/* ------------------------------------------------------------------ */
const tagSchema = z.string().trim().min(1).max(LIMITS.tag);
export const optionInputSchema = z.object({
id: z.string().min(1).max(40).optional(),
text: z.string().trim().min(1, 'Answer choice is empty').max(LIMITS.option),
correct: z.boolean(),
why: z.string().max(LIMITS.markdown).optional().default(''),
});
export const questionInputSchema = z
.object({
id: z.string().min(1).max(40).optional(),
type: z.enum(QUESTION_TYPES),
stem: z.string().trim().min(1, 'Question text is required').max(LIMITS.markdown),
options: z.array(optionInputSchema).min(2, 'Add at least two answer choices').max(LIMITS.options),
rationale: z.string().max(LIMITS.markdown).optional().default(''),
topic: z.string().trim().max(LIMITS.topic).optional().default(''),
difficulty: z.enum(DIFFICULTIES).optional().default('medium'),
tags: z.array(tagSchema).max(LIMITS.tags).optional().default([]),
})
.superRefine((q, ctx) => {
const correct = q.options.filter((o) => o.correct).length;
if (q.type === 'single' && correct !== 1) {
ctx.addIssue({ code: 'custom', path: ['options'], message: 'Multiple choice needs exactly one correct answer' });
}
if (q.type === 'multi' && correct < 1) {
ctx.addIssue({ code: 'custom', path: ['options'], message: 'Select-all needs at least one correct answer' });
}
});
export type QuestionInput = z.input<typeof questionInputSchema>;
export const cardInputSchema = z.object({
id: z.string().min(1).max(40).optional(),
front: z.string().trim().min(1, 'Front is required').max(LIMITS.markdown),
back: z.string().trim().min(1, 'Back is required').max(LIMITS.markdown),
hint: z.string().max(LIMITS.markdown).optional().default(''),
tags: z.array(tagSchema).max(LIMITS.tags).optional().default([]),
});
export type CardInput = z.input<typeof cardInputSchema>;
export const titleSchema = z.string().trim().min(1, 'Title is required').max(LIMITS.title);
export const descriptionSchema = z.string().max(LIMITS.description).optional().default('');
export const quizCreateSchema = z.object({
classId: z.string().min(1),
folderId: z.string().min(1).nullish(),
title: titleSchema,
description: descriptionSchema,
questions: z.array(questionInputSchema).max(LIMITS.questionsPerQuiz).optional().default([]),
});
export const deckCreateSchema = z.object({
classId: z.string().min(1),
folderId: z.string().min(1).nullish(),
title: titleSchema,
description: descriptionSchema,
cards: z.array(cardInputSchema).max(LIMITS.cardsPerDeck).optional().default([]),
});
/* ------------------------------------------------------------------ */
/* Drafts: lenient shapes used by import previews and AI generation */
/* ------------------------------------------------------------------ */
export interface DraftOption {
text: string;
correct: boolean;
why?: string;
}
export interface DraftQuestion {
type: QuestionType;
stem: string;
options: DraftOption[];
rationale: string;
topic: string;
difficulty: Difficulty;
tags: string[];
/** Citation for generated questions ("Cardiac lecture, p. 12"). Stored at the end of the rationale on save. */
source?: string;
}
export interface DraftCard {
front: string;
back: string;
hint?: string;
tags: string[];
}
export interface QuizDraft {
kind: 'quiz';
title: string;
description: string;
questions: DraftQuestion[];
}
export interface DeckDraft {
kind: 'deck';
title: string;
description: string;
cards: DraftCard[];
}
export interface NoteDraft {
kind: 'note';
title: string;
content: string;
}
export type ContentDraft = QuizDraft | DeckDraft | NoteDraft;
export interface Issue {
level: 'error' | 'warning';
/** Index of the question / card the issue belongs to; undefined for item-level issues. */
index?: number;
field?: string;
message: string;
}
export function validateDraftQuestion(q: DraftQuestion, index: number): Issue[] {
const issues: Issue[] = [];
const err = (field: string, message: string) => issues.push({ level: 'error', index, field, message });
const warn = (field: string, message: string) => issues.push({ level: 'warning', index, field, message });
if (!q.stem?.trim()) err('stem', 'Question text is missing');
const options = q.options ?? [];
if (options.length < 2) err('options', 'Needs at least two answer choices');
if (options.length > LIMITS.options) err('options', `No more than ${LIMITS.options} answer choices`);
if (options.some((o) => !o.text?.trim())) err('options', 'An answer choice is empty');
const correct = options.filter((o) => o.correct).length;
if (q.type === 'single' && correct !== 1) {
err('options', correct === 0 ? 'No correct answer is marked' : 'Multiple choice can only have one correct answer');
}
if (q.type === 'multi' && correct < 1) err('options', 'No correct answer is marked');
const seen = new Set<string>();
for (const o of options) {
const key = o.text?.trim().toLowerCase();
if (key && seen.has(key)) {
warn('options', 'Two answer choices are identical');
break;
}
if (key) seen.add(key);
}
if (!q.rationale?.trim() && !options.some((o) => o.why?.trim())) warn('rationale', 'No rationale');
return issues;
}
export function validateDraftCard(c: DraftCard, index: number): Issue[] {
const issues: Issue[] = [];
if (!c.front?.trim()) issues.push({ level: 'error', index, field: 'front', message: 'Front is empty' });
if (!c.back?.trim()) issues.push({ level: 'error', index, field: 'back', message: 'Back is empty' });
return issues;
}
export function validateDraft(draft: ContentDraft): Issue[] {
const issues: Issue[] = [];
if (!draft.title?.trim()) issues.push({ level: 'error', field: 'title', message: 'Title is required' });
if (draft.kind === 'quiz') {
if (draft.questions.length === 0) issues.push({ level: 'error', field: 'questions', message: 'Add at least one question' });
draft.questions.forEach((q, i) => issues.push(...validateDraftQuestion(q, i)));
} else if (draft.kind === 'deck') {
if (draft.cards.length === 0) issues.push({ level: 'error', field: 'cards', message: 'Add at least one card' });
draft.cards.forEach((c, i) => issues.push(...validateDraftCard(c, i)));
} else if (!draft.content?.trim()) {
issues.push({ level: 'error', field: 'content', message: 'The note is empty' });
}
return issues;
}
export function blankQuestion(type: QuestionType = 'single'): DraftQuestion {
return {
type,
stem: '',
options: [
{ text: '', correct: false, why: '' },
{ text: '', correct: false, why: '' },
{ text: '', correct: false, why: '' },
{ text: '', correct: false, why: '' },
],
rationale: '',
topic: '',
difficulty: 'medium',
tags: [],
};
}
export function blankCard(): DraftCard {
return { front: '', back: '', hint: '', tags: [] };
}
/** Converts a stored question into the editable draft shape. */
export function questionToDraft(q: Pick<Question, 'type' | 'stem' | 'options' | 'rationale' | 'topic' | 'difficulty' | 'tags'>): DraftQuestion {
return {
type: q.type,
stem: q.stem,
options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why ?? '' })),
rationale: q.rationale,
topic: q.topic,
difficulty: q.difficulty,
tags: [...q.tags],
};
}
/* ------------------------------------------------------------------ */
/* Scoring */
/* ------------------------------------------------------------------ */
export type SataScoring = 'partial' | 'strict';
export interface ScorableQuestion {
type: QuestionType;
options: { id: string; correct: boolean }[];
}
/**
* Scores one question on a 0..1 scale.
* - single: 1 if the one selected option is correct.
* - multi (partial): NCLEX-style +/- scoring. Each correct pick earns a point, each wrong pick
* costs one, floored at zero, divided by the number of correct options.
* - multi (strict): all-or-nothing.
*/
export function scoreQuestion(question: ScorableQuestion, selected: string[], sata: SataScoring = 'partial'): number {
const picked = new Set(selected);
const correctIds = question.options.filter((o) => o.correct).map((o) => o.id);
if (correctIds.length === 0 || picked.size === 0) return 0;
if (question.type === 'single') {
return picked.size === 1 && correctIds.includes([...picked][0]) ? 1 : 0;
}
let right = 0;
let wrong = 0;
for (const id of picked) {
if (correctIds.includes(id)) right += 1;
else if (question.options.some((o) => o.id === id)) wrong += 1;
}
if (sata === 'strict') return right === correctIds.length && wrong === 0 ? 1 : 0;
return Math.max(0, right - wrong) / correctIds.length;
}
export function isFullyCorrect(score: number): boolean {
return score >= 0.999;
}
export function percent(score: number, max: number): number {
if (!max) return 0;
return Math.round((score / max) * 100);
}
/* ------------------------------------------------------------------ */
/* Misc helpers */
/* ------------------------------------------------------------------ */
/** Deterministic shuffle (mulberry32) so a saved session can be reproduced from its seed. */
export function seededShuffle<T>(items: readonly T[], seed: number): T[] {
const out = [...items];
let s = seed >>> 0;
const rand = () => {
s = (s + 0x6d2b79f5) >>> 0;
let t = s;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
for (let i = out.length - 1; i > 0; i -= 1) {
const j = Math.floor(rand() * (i + 1));
[out[i], out[j]] = [out[j], out[i]];
}
return out;
}
export function optionLetter(index: number): string {
return String.fromCharCode(65 + index);
}

561
shared/importer.ts Normal file
View file

@ -0,0 +1,561 @@
/**
* Lenient import parser.
*
* Accepts Study Desk V2 exports, V1 exports, loosely-shaped AI output, plain-text
* question blocks and delimited flashcard lists, and normalises all of them into
* editable drafts plus a list of issues. Nothing is saved from here: the preview UI
* shows the drafts, lets the user fix problems, and only then posts strict payloads.
*/
import { jsonrepair } from 'jsonrepair';
import {
DIFFICULTIES,
type ContentDraft,
type DeckDraft,
type Difficulty,
type DraftCard,
type DraftOption,
type DraftQuestion,
type Issue,
type NoteDraft,
type QuizDraft,
validateDraft,
} from './content.ts';
export const EXPORT_FORMAT = 'studydesk';
export const EXPORT_VERSION = 2;
export interface ImportResult {
items: ContentDraft[];
/** Problems that are not tied to one item (unparseable input, skipped entries...). */
notices: Issue[];
/** True when the JSON had to be repaired before it could be parsed. */
repaired: boolean;
/** How the input was interpreted, for display. */
detected: 'json' | 'text-quiz' | 'text-cards' | 'none';
}
type Raw = Record<string, unknown>;
const isObj = (v: unknown): v is Raw => typeof v === 'object' && v !== null && !Array.isArray(v);
const str = (v: unknown): string => (typeof v === 'string' ? v : typeof v === 'number' || typeof v === 'boolean' ? String(v) : '');
const pick = (o: Raw, ...keys: string[]): unknown => {
for (const k of keys) if (o[k] !== undefined && o[k] !== null) return o[k];
return undefined;
};
function toTags(v: unknown): string[] {
if (Array.isArray(v)) return v.map(str).map((s) => s.trim()).filter(Boolean).slice(0, 12);
if (typeof v === 'string') return v.split(/[,;]/).map((s) => s.trim()).filter(Boolean).slice(0, 12);
return [];
}
function toDifficulty(v: unknown): Difficulty {
const s = str(v).toLowerCase().trim();
if ((DIFFICULTIES as readonly string[]).includes(s)) return s as Difficulty;
if (['1', 'beginner', 'basic', 'low'].includes(s)) return 'easy';
if (['3', 'advanced', 'difficult', 'high', 'challenging'].includes(s)) return 'hard';
return 'medium';
}
function truthy(v: unknown): boolean {
if (typeof v === 'boolean') return v;
if (typeof v === 'number') return v !== 0;
const s = str(v).trim().toLowerCase();
return s === 'true' || s === 'yes' || s === 'y' || s === '1' || s === 'correct' || s === 'x';
}
/** Maps "B", "b", "2", 2 or the option text to option indexes. */
function resolveAnswerRefs(answer: unknown, options: DraftOption[]): number[] {
const refs = Array.isArray(answer) ? answer : typeof answer === 'string' ? answer.split(/[,;&]|\band\b/i) : [answer];
const out = new Set<number>();
for (const ref of refs) {
if (typeof ref === 'number' && Number.isInteger(ref)) {
// Ambiguous 0/1-based: treat as 0-based only when 0 appears, otherwise 1-based is far more common in hand-written keys.
const idx = ref >= 1 && ref <= options.length ? ref - 1 : ref;
if (idx >= 0 && idx < options.length) out.add(idx);
continue;
}
const s = str(ref).trim();
if (!s) continue;
const byText = options.findIndex((o) => o.text.trim().toLowerCase() === s.toLowerCase());
if (byText >= 0) {
out.add(byText);
continue;
}
const letter = s.replace(/[().\s]/g, '').toUpperCase();
if (/^[A-J]$/.test(letter)) {
const idx = letter.charCodeAt(0) - 65;
if (idx < options.length) out.add(idx);
} else if (/^\d+$/.test(letter)) {
const idx = Number(letter) - 1;
if (idx >= 0 && idx < options.length) out.add(idx);
}
}
return [...out];
}
function normalizeOptions(raw: unknown): DraftOption[] {
if (Array.isArray(raw)) {
return raw
.map((o): DraftOption | null => {
if (typeof o === 'string' || typeof o === 'number') return { text: String(o).trim(), correct: false, why: '' };
if (!isObj(o)) return null;
return {
text: str(pick(o, 'text', 'option', 'answer', 'label', 'value', 'content')).trim(),
correct: truthy(pick(o, 'correct', 'isCorrect', 'is_correct', 'right', 'answer_correct')),
why: str(pick(o, 'why', 'rationale', 'explanation', 'feedback')).trim(),
};
})
.filter((o): o is DraftOption => o !== null);
}
// { "A": "text", "B": "text" }
if (isObj(raw)) {
return Object.keys(raw)
.sort()
.map((k) => ({ text: str(raw[k]).trim(), correct: false, why: '' }));
}
return [];
}
function normalizeQuestion(raw: unknown, notices: Issue[], index: number): DraftQuestion | null {
if (!isObj(raw)) {
notices.push({ level: 'warning', index, message: `Question ${index + 1} is not an object and was skipped` });
return null;
}
const options = normalizeOptions(pick(raw, 'options', 'choices', 'answers', 'answerOptions', 'answer_options'));
const answerKey = pick(raw, 'correctAnswers', 'correct_answers', 'correctAnswer', 'correct_answer', 'answer', 'correct', 'key');
if (answerKey !== undefined && !options.some((o) => o.correct)) {
for (const idx of resolveAnswerRefs(answerKey, options)) options[idx].correct = true;
}
const correctCount = options.filter((o) => o.correct).length;
const rawType = str(pick(raw, 'type', 'questionType', 'question_type', 'kind')).toLowerCase().replace(/[\s-]/g, '_');
let type: DraftQuestion['type'];
if (['multi', 'sata', 'select_all', 'select_all_that_apply', 'multiple_response', 'multiple_select', 'multi_select', 'checkbox'].includes(rawType)) {
type = 'multi';
} else if (['single', 'multiple_choice', 'mcq', 'mc', 'single_choice', 'true_false', 'radio'].includes(rawType)) {
type = 'single';
} else {
type = correctCount > 1 ? 'multi' : 'single';
}
// A "multiple choice" question with several keyed answers can only be select-all.
if (type === 'single' && correctCount > 1) type = 'multi';
return {
type,
stem: str(pick(raw, 'stem', 'prompt', 'question', 'text', 'q')).trim(),
options,
rationale: str(pick(raw, 'rationale', 'explanation', 'reasoning', 'feedback')).trim(),
topic: str(pick(raw, 'topic', 'category', 'subject', 'concept')).trim().slice(0, 80),
difficulty: toDifficulty(pick(raw, 'difficulty', 'level')),
tags: toTags(pick(raw, 'tags', 'keywords')),
source: str(pick(raw, 'source', 'citation', 'reference')).trim() || undefined,
};
}
function normalizeCard(raw: unknown, notices: Issue[], index: number): DraftCard | null {
if (Array.isArray(raw) && raw.length >= 2) return { front: str(raw[0]).trim(), back: str(raw[1]).trim(), hint: '', tags: [] };
if (!isObj(raw)) {
notices.push({ level: 'warning', index, message: `Card ${index + 1} is not an object and was skipped` });
return null;
}
return {
front: str(pick(raw, 'front', 'term', 'question', 'prompt', 'q', 'word')).trim(),
back: str(pick(raw, 'back', 'definition', 'answer', 'a', 'response', 'meaning')).trim(),
hint: str(pick(raw, 'hint', 'clue')).trim(),
tags: toTags(pick(raw, 'tags', 'keywords')),
};
}
function quizFrom(raw: Raw, notices: Issue[]): QuizDraft {
const list = pick(raw, 'questions', 'items', 'quiz') as unknown;
const questions = (Array.isArray(list) ? list : [])
.map((q, i) => normalizeQuestion(q, notices, i))
.filter((q): q is DraftQuestion => q !== null);
return {
kind: 'quiz',
title: str(pick(raw, 'title', 'quizName', 'quiz_name', 'name')).trim(),
description: str(pick(raw, 'description', 'summary')).trim(),
questions,
};
}
function deckFrom(raw: Raw, notices: Issue[]): DeckDraft {
const list = pick(raw, 'cards', 'flashcards', 'items') as unknown;
const cards = (Array.isArray(list) ? list : [])
.map((c, i) => normalizeCard(c, notices, i))
.filter((c): c is DraftCard => c !== null);
return {
kind: 'deck',
title: str(pick(raw, 'title', 'deckName', 'deck_name', 'name')).trim(),
description: str(pick(raw, 'description', 'summary')).trim(),
cards,
};
}
function noteFrom(raw: Raw): NoteDraft {
return { kind: 'note', title: str(pick(raw, 'title', 'name')).trim(), content: str(pick(raw, 'content', 'markdown', 'body', 'text')) };
}
function looksLikeQuestion(v: unknown): boolean {
return isObj(v) && pick(v, 'options', 'choices', 'answers', 'answerOptions') !== undefined;
}
function looksLikeCard(v: unknown): boolean {
return isObj(v) && pick(v, 'front', 'term') !== undefined && pick(v, 'back', 'definition') !== undefined;
}
function collect(value: unknown, items: ContentDraft[], notices: Issue[], depth = 0): void {
if (depth > 4) return;
if (Array.isArray(value)) {
if (value.length === 0) return;
if (value.every(looksLikeQuestion)) {
items.push(quizFrom({ questions: value }, notices));
return;
}
if (value.every((v) => looksLikeCard(v) || (Array.isArray(v) && v.length === 2))) {
items.push(deckFrom({ cards: value }, notices));
return;
}
for (const v of value) collect(v, items, notices, depth + 1);
return;
}
if (!isObj(value)) return;
const kind = str(pick(value, 'kind', 'type')).toLowerCase();
// Bundles / backups: { classes: [{ quizzes, decks, notes }] } or { items: [...] }
if (Array.isArray(value.classes)) {
for (const cls of value.classes) collect(cls, items, notices, depth + 1);
return;
}
const containers = ['quizzes', 'decks', 'notes', 'quizSets'].filter((k) => Array.isArray(value[k]));
if (containers.length > 0 && !Array.isArray(value.questions) && !Array.isArray(value.cards)) {
for (const k of containers) {
for (const entry of value[k] as unknown[]) {
if (!isObj(entry)) continue;
if (k === 'notes') items.push(noteFrom(entry));
else if (k === 'decks') items.push(deckFrom(entry, notices));
else items.push(quizFrom(entry, notices));
}
}
return;
}
if (kind === 'bundle' && Array.isArray(value.items)) {
for (const entry of value.items) collect(entry, items, notices, depth + 1);
return;
}
if (kind === 'note' || kind === 'guide') {
items.push(noteFrom(value));
return;
}
if (kind === 'deck' || kind === 'flashcards' || Array.isArray(value.cards) || Array.isArray(value.flashcards)) {
items.push(deckFrom(value, notices));
return;
}
if (kind === 'quiz' || Array.isArray(value.questions)) {
items.push(quizFrom(value, notices));
return;
}
if (looksLikeQuestion(value)) {
items.push(quizFrom({ questions: [value] }, notices));
return;
}
if (looksLikeCard(value)) {
items.push(deckFrom({ cards: [value] }, notices));
return;
}
// Unknown wrapper: look one level down for something recognisable.
for (const v of Object.values(value)) if (typeof v === 'object' && v !== null) collect(v, items, notices, depth + 1);
}
/** Strips a Markdown code fence and any prose around the outermost JSON value. */
function extractJsonText(text: string): string | null {
let t = text.trim();
const fence = /```(?:json|JSON)?\s*\n([\s\S]*?)```/.exec(t);
if (fence) t = fence[1].trim();
const start = t.search(/[[{]/);
if (start < 0) return null;
const open = t[start];
const close = open === '{' ? '}' : ']';
const end = t.lastIndexOf(close);
return end > start ? t.slice(start, end + 1) : t.slice(start);
}
/* ------------------------------------------------------------------ */
/* Plain-text formats */
/* ------------------------------------------------------------------ */
const OPTION_LINE = /^\s*(\*)?\s*(?:\(?([A-Ja-j])[.)]|[-•])\s+(.*)$/;
const ANSWER_LINE = /^\s*(?:correct\s+)?answers?\s*[:=]\s*(.+)$/i;
const RATIONALE_LINE = /^\s*(?:rationale|explanation|why)\s*[:=]\s*(.*)$/i;
const TOPIC_LINE = /^\s*(?:topic|category)\s*[:=]\s*(.*)$/i;
/**
* Parses hand-typed question blocks separated by blank lines:
*
* 1. Which finding requires immediate action?
* A. Heart rate 88
* *B. Oxygen saturation 84%
* C. Temperature 37.2
* Answer: B (alternative to the * marker)
* Rationale: ...
* Topic: Respiratory
*/
export function parseTextQuestions(text: string): DraftQuestion[] {
const blocks = text.replace(/\r\n?/g, '\n').split(/\n\s*\n/);
const questions: DraftQuestion[] = [];
for (const block of blocks) {
const lines = block.split('\n').filter((l) => l.trim());
if (lines.length < 3) continue;
const stem: string[] = [];
const options: DraftOption[] = [];
let rationale = '';
let topic = '';
let answer: string | null = null;
let section: 'stem' | 'options' | 'rationale' = 'stem';
for (const line of lines) {
let m: RegExpExecArray | null;
if ((m = ANSWER_LINE.exec(line))) {
answer = m[1];
continue;
}
if ((m = RATIONALE_LINE.exec(line))) {
rationale = m[1].trim();
section = 'rationale';
continue;
}
if ((m = TOPIC_LINE.exec(line))) {
topic = m[1].trim();
continue;
}
if ((m = OPTION_LINE.exec(line)) && section !== 'rationale') {
let body = m[3].trim();
let correct = Boolean(m[1]);
const tail = /\s*(?:\*|\(correct\)|✓|✔)\s*$/i.exec(body);
if (tail) {
correct = true;
body = body.slice(0, tail.index).trim();
}
options.push({ text: body, correct, why: '' });
section = 'options';
continue;
}
if (section === 'stem') stem.push(line.replace(/^\s*(?:Q?\d+[.):]|Q[.:])\s*/i, '').trim());
else if (section === 'rationale') rationale += `\n${line.trim()}`;
else if (options.length) options[options.length - 1].text += ` ${line.trim()}`;
}
if (options.length < 2 || stem.length === 0) continue;
if (answer && !options.some((o) => o.correct)) for (const idx of resolveAnswerRefs(answer, options)) options[idx].correct = true;
const correctCount = options.filter((o) => o.correct).length;
questions.push({
type: correctCount > 1 ? 'multi' : 'single',
stem: stem.join('\n'),
options,
rationale: rationale.trim(),
topic,
difficulty: 'medium',
tags: [],
});
}
return questions;
}
/** Parses "term <sep> definition" lines; the separator is detected from the input. */
export function parseTextCards(text: string): DraftCard[] {
const lines = text.replace(/\r\n?/g, '\n').split('\n').map((l) => l.trim()).filter(Boolean);
if (lines.length === 0) return [];
const seps = ['\t', ' :: ', '::', ' | ', ' — ', ' – ', ' - ', ';', ','];
const sep = seps.find((s) => lines.filter((l) => l.includes(s)).length >= Math.ceil(lines.length * 0.8));
if (!sep) return [];
const cards: DraftCard[] = [];
for (const line of lines) {
const at = line.indexOf(sep);
if (at <= 0) continue;
const front = line.slice(0, at).trim().replace(/^"|"$/g, '');
const back = line.slice(at + sep.length).trim().replace(/^"|"$/g, '');
if (front && back) cards.push({ front, back, hint: '', tags: [] });
}
return cards;
}
/* ------------------------------------------------------------------ */
/* Entry point */
/* ------------------------------------------------------------------ */
export function parseImport(text: string, opts: { prefer?: 'quiz' | 'deck'; fallbackTitle?: string } = {}): ImportResult {
const notices: Issue[] = [];
const items: ContentDraft[] = [];
let repaired = false;
let detected: ImportResult['detected'] = 'none';
const trimmed = text.trim();
if (!trimmed) return { items, notices: [{ level: 'error', message: 'Nothing to import yet. Paste content or choose a file.' }], repaired, detected };
const jsonText = /^[\s`]*(?:```|[[{])/.test(trimmed) || /^[^\n]{0,200}[[{]/.test(trimmed) ? extractJsonText(trimmed) : null;
if (jsonText) {
let parsed: unknown;
let ok = false;
try {
parsed = JSON.parse(jsonText);
ok = true;
} catch {
try {
parsed = JSON.parse(jsonrepair(jsonText));
ok = true;
repaired = true;
} catch (e) {
notices.push({ level: 'error', message: `The JSON could not be read: ${e instanceof Error ? e.message : 'syntax error'}` });
}
}
if (ok) {
detected = 'json';
collect(parsed, items, notices);
if (items.length === 0) notices.push({ level: 'error', message: 'No quizzes or flashcards were found in that JSON. Download a template to see the expected shape.' });
if (repaired) notices.push({ level: 'warning', message: 'The JSON had syntax problems that were repaired automatically. Check the preview carefully.' });
}
}
if (items.length === 0 && detected !== 'json') {
const questions = opts.prefer === 'deck' ? [] : parseTextQuestions(trimmed);
const cards = questions.length > 0 ? [] : parseTextCards(trimmed);
if (questions.length > 0) {
detected = 'text-quiz';
items.push({ kind: 'quiz', title: '', description: '', questions });
} else if (cards.length > 0) {
detected = 'text-cards';
items.push({ kind: 'deck', title: '', description: '', cards });
} else if (notices.length === 0) {
notices.push({
level: 'error',
message: 'Could not recognise this content. Use JSON, question blocks with lettered choices, or one "term [tab] definition" per line.',
});
}
}
for (const item of items) {
if (!item.title && opts.fallbackTitle) item.title = opts.fallbackTitle;
}
return { items, notices, repaired, detected };
}
/** Issues for one draft, including a friendly summary count. */
export function summarizeIssues(draft: ContentDraft): { errors: number; warnings: number; issues: Issue[] } {
const issues = validateDraft(draft);
return {
errors: issues.filter((i) => i.level === 'error').length,
warnings: issues.filter((i) => i.level === 'warning').length,
issues,
};
}
/* ------------------------------------------------------------------ */
/* Export shapes + templates */
/* ------------------------------------------------------------------ */
export function exportQuiz(q: { title: string; description: string; questions: DraftQuestion[] }) {
return {
format: EXPORT_FORMAT,
version: EXPORT_VERSION,
kind: 'quiz' as const,
title: q.title,
description: q.description,
questions: q.questions.map((x) => ({
type: x.type,
stem: x.stem,
options: x.options.map((o) => (o.why ? { text: o.text, correct: o.correct, why: o.why } : { text: o.text, correct: o.correct })),
rationale: x.rationale,
topic: x.topic,
difficulty: x.difficulty,
tags: x.tags,
})),
};
}
export function exportDeck(d: { title: string; description: string; cards: DraftCard[] }) {
return {
format: EXPORT_FORMAT,
version: EXPORT_VERSION,
kind: 'deck' as const,
title: d.title,
description: d.description,
cards: d.cards.map((c) => ({ front: c.front, back: c.back, ...(c.hint ? { hint: c.hint } : {}), tags: c.tags })),
};
}
export const QUIZ_JSON_TEMPLATE = JSON.stringify(
exportQuiz({
title: 'Example quiz',
description: 'Optional description',
questions: [
{
type: 'single',
stem: 'A client with heart failure reports a 1.4 kg weight gain overnight. Which action should the nurse take first?',
options: [
{ text: 'Assess lung sounds and oxygen saturation', correct: true, why: 'Rapid weight gain signals fluid retention; assess for pulmonary congestion first.' },
{ text: 'Document the weight and continue to monitor', correct: false },
{ text: 'Encourage increased oral fluid intake', correct: false },
{ text: 'Schedule a dietitian consult for next week', correct: false },
],
rationale: 'Assessment comes first. A gain of more than 1 kg in a day suggests fluid volume overload.',
topic: 'Heart failure',
difficulty: 'medium',
tags: ['cardiac', 'prioritization'],
},
{
type: 'multi',
stem: 'Which findings are expected with hypokalemia? Select all that apply.',
options: [
{ text: 'Muscle weakness', correct: true },
{ text: 'Flattened T waves', correct: true },
{ text: 'Peaked T waves', correct: false },
{ text: 'Decreased bowel sounds', correct: true },
],
rationale: 'Low potassium slows smooth and skeletal muscle activity and flattens T waves. Peaked T waves indicate hyperkalemia.',
topic: 'Electrolytes',
difficulty: 'medium',
tags: [],
},
],
}),
null,
2,
);
export const DECK_JSON_TEMPLATE = JSON.stringify(
exportDeck({
title: 'Example deck',
description: 'Optional description',
cards: [
{ front: 'Furosemide: drug class', back: 'Loop diuretic', hint: 'Acts on the loop of Henle', tags: ['pharm'] },
{ front: 'Normal serum potassium', back: '3.5 to 5.0 mEq/L', tags: ['labs'] },
],
}),
null,
2,
);
export const TEXT_QUIZ_TEMPLATE = `1. Which electrolyte imbalance causes peaked T waves?
A. Hypokalemia
*B. Hyperkalemia
C. Hyponatremia
D. Hypercalcemia
Rationale: High potassium speeds repolarisation, producing tall, peaked T waves.
Topic: Electrolytes
2. Which are signs of digoxin toxicity? Select all that apply.
*A. Visual halos
*B. Nausea and vomiting
C. Hypertension
*D. Bradycardia
Rationale: Early toxicity is gastrointestinal and visual; bradycardia and dysrhythmias follow.
Topic: Pharmacology`;
export const TEXT_CARDS_TEMPLATE = `Furosemide\tLoop diuretic; monitor potassium
Metoprolol\tBeta-1 selective blocker; hold for HR below 60
Warfarin\tVitamin K antagonist; monitor INR`;
/** Normalises already-parsed data (for example the arguments of an AI tool call) into drafts. */
export function draftsFromValue(value: unknown): { items: ContentDraft[]; notices: Issue[] } {
const items: ContentDraft[] = [];
const notices: Issue[] = [];
collect(value, items, notices);
return { items, notices };
}

View file

@ -1,185 +0,0 @@
"use client";
import { useState, useEffect, useCallback } from "react";
import { useParams, useRouter } from "next/navigation";
import { FlashcardViewer } from "@/components/flashcards/FlashcardViewer";
import { CardManager } from "@/components/flashcards/CardManager";
import { ShareMenu } from "@/components/ui/ShareMenu";
interface Card {
id: string;
front: string;
back: string;
sortOrder: number;
}
interface DeckData {
id: string;
name: string;
description: string | null;
cards: Card[];
class: { slug: string; name: string };
progress: Array<{
mode: "SEQUENTIAL" | "SHUFFLED";
currentIndex: number;
orderJson: string;
cardResultsJson: string | null;
sessionId: string;
revision: number;
updatedAt: string;
}>;
}
export default function DeckStudyPage() {
const params = useParams();
const router = useRouter();
const classSlug = params.classSlug as string;
const deckId = params.deckId as string;
const [deck, setDeck] = useState<DeckData | null>(null);
const [loading, setLoading] = useState(true);
const [view, setView] = useState<"study" | "manage">("study");
const [restartKey, setRestartKey] = useState(0);
async function handleRestart() {
if (!deck) return;
const responses = await Promise.all(
deck.progress.map((progress) =>
fetch("/api/progress", {
method: "DELETE",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
contentType: "DECK",
contentId: deck.id,
mode: progress.mode,
sessionId: progress.sessionId,
}),
})
)
).catch(() => null);
if (!responses || responses.some((response) => !response.ok)) {
window.alert("The session could not be restarted. Your saved progress was kept.");
return;
}
// Clear locally and force remount
setDeck({ ...deck, progress: [] });
setRestartKey(k => k + 1);
}
const fetchDeck = useCallback(async () => {
try {
const res = await fetch(`/api/decks/${deckId}`);
if (!res.ok) throw new Error("Not found");
const data = await res.json();
setDeck(data);
} catch {
router.push(`/${classSlug}/flashcards`);
} finally {
setLoading(false);
}
}, [deckId, classSlug, router]);
useEffect(() => {
const timer = window.setTimeout(() => void fetchDeck(), 0);
return () => window.clearTimeout(timer);
}, [fetchDeck]);
if (loading || !deck) {
return (
<div className="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8 py-8">
<div className="animate-subtle-pulse">
<div className="h-8 bg-bg-surface-alt rounded w-1/3 mb-4" />
<div className="h-64 bg-bg-surface rounded-xl border border-border-light" />
</div>
</div>
);
}
return (
<div className="mx-auto w-full max-w-5xl py-1 md:py-3">
{/* Header */}
<div className="mb-7 flex flex-col items-start justify-between gap-4 rounded-2xl border border-border-light bg-bg-surface/70 p-4 shadow-[var(--shadow-card)] sm:p-5 md:flex-row md:items-center">
<div>
<button
onClick={() => router.push(`/${classSlug}/flashcards`)}
className="inline-flex items-center gap-1 text-sm text-text-muted hover:text-text-heading mb-2 transition-colors cursor-pointer"
>
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M15 19l-7-7 7-7" />
</svg>
Back to decks
</button>
<p className="text-xs font-bold uppercase tracking-[0.16em] text-primary">Flashcard session</p>
<h1 className="editorial-title mt-1 text-3xl text-text-heading">{deck.name}</h1>
{deck.description && (
<p className="text-sm text-text-secondary mt-1">{deck.description}</p>
)}
</div>
{/* View toggle & Share */}
<div className="flex items-center justify-between w-full md:w-auto gap-2 md:gap-4 self-end md:self-auto">
{view === "study" && (
<button
onClick={handleRestart}
className="p-1.5 rounded-lg text-text-muted hover:text-text-heading hover:bg-bg-surface-alt transition-all duration-200 cursor-pointer"
title="Restart Session"
>
<svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
</svg>
</button>
)}
<div className="flex items-center gap-2 md:gap-4 ml-auto">
<ShareMenu targetType="DECK" contentId={deck.id} classSlug={classSlug} />
<div className="flex rounded-xl border border-border-light bg-bg-surface-alt p-1">
<button
onClick={() => setView("study")}
className={`px-4 py-2 rounded-md text-sm font-medium transition-all duration-200 cursor-pointer ${
view === "study"
? "bg-bg-surface text-text-heading shadow-sm"
: "text-text-muted hover:text-text-heading"
}`}
>
Study
</button>
<button
onClick={() => setView("manage")}
className={`px-4 py-2 rounded-md text-sm font-medium transition-all duration-200 cursor-pointer ${
view === "manage"
? "bg-bg-surface text-text-heading shadow-sm"
: "text-text-muted hover:text-text-heading"
}`}
>
Manage
</button>
</div>
</div>
</div>
</div>
{/* Content */}
{view === "study" ? (
<FlashcardViewer
key={restartKey}
cards={deck.cards}
deckId={deck.id}
initialProgress={
deck.progress?.length
? deck.progress.sort((a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime())[0]
: null
}
/>
) : (
<CardManager
cards={deck.cards}
deckId={deck.id}
onCardsChanged={fetchDeck}
/>
)}
</div>
);
}

View file

@ -1,674 +0,0 @@
"use client";
import { useState, useEffect, useCallback, useRef } from "react";
import Link from "next/link";
import { useParams } from "next/navigation";
import { ImportModal } from "@/components/import/ImportModal";
import { ShareMenu } from "@/components/ui/ShareMenu";
import {
DndContext,
pointerWithin,
KeyboardSensor,
PointerSensor,
useSensor,
useSensors,
DragEndEvent,
DragStartEvent,
DragOverlay,
useDroppable,
useDndContext,
} from "@dnd-kit/core";
import {
arrayMove,
SortableContext,
sortableKeyboardCoordinates,
verticalListSortingStrategy,
useSortable,
} from "@dnd-kit/sortable";
import { CSS } from "@dnd-kit/utilities";
import type { ReorderItem } from "@/types/study";
interface DeckItem {
id: string;
name: string;
description: string | null;
groupId: string | null;
sortOrder: number;
_count: { cards: number };
progress: Array<{
mode: string;
currentIndex: number;
orderJson: string;
cardResultsJson: string | null;
sessionId: string;
revision: number;
}>;
}
interface MaterialGroup {
id: string;
name: string;
sortOrder: number;
}
function getProgressLabel(deck: DeckItem) {
if (!deck.progress?.length) return null;
const prog = deck.progress[0];
let order: string[];
let results: Record<string, string>;
try {
const parsedOrder: unknown = JSON.parse(prog.orderJson);
const parsedResults: unknown = prog.cardResultsJson
? JSON.parse(prog.cardResultsJson)
: {};
if (!Array.isArray(parsedOrder)) throw new Error("Invalid progress order");
order = parsedOrder.filter((id): id is string => typeof id === "string");
results =
typeof parsedResults === "object" && parsedResults !== null
? (parsedResults as Record<string, string>)
: {};
} catch {
return "Saved session needs repair";
}
const correctCount = Object.values(results).filter((r) => r === "correct").length;
const total = order.length;
const current = Math.min(prog.currentIndex + 1, total);
const modeLabel = prog.mode === "SHUFFLED" ? "shuffled" : "sequential";
return `${current}/${total}, ${modeLabel} · ${correctCount} ✓`;
}
interface SortableDeckCardProps {
deck: DeckItem;
onEdit: (deck: DeckItem) => void;
onDelete: (deck: DeckItem) => void;
groups: MaterialGroup[];
onMove: (deckId: string, groupId: string | null) => void;
moveDisabled?: boolean;
classSlug: string;
}
function SortableDeckCard({
deck,
onEdit,
onDelete,
groups,
onMove,
moveDisabled = false,
classSlug,
}: SortableDeckCardProps) {
const { attributes, listeners, setNodeRef, transform, transition, isDragging } = useSortable({ id: deck.id });
const style = {
transform: CSS.Transform.toString(transform),
transition,
opacity: isDragging ? 0.4 : 1,
};
const progressLabel = getProgressLabel(deck);
return (
<div ref={setNodeRef} style={style} className="group flex h-full flex-col rounded-2xl border border-border-light bg-bg-surface shadow-[var(--shadow-card)] transition-all duration-300 hover:-translate-y-0.5 hover:border-primary/25 hover:shadow-[var(--shadow-card-hover)]">
<div className="flex h-full flex-1 flex-col p-5">
<div className="flex items-start gap-2">
<div {...attributes} {...listeners} className="mt-1 cursor-grab active:cursor-grabbing text-text-muted hover:text-text-heading">
<svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 8h16M4 16h16" />
</svg>
</div>
<div className="flex-1">
<h3 className="mb-1 text-lg font-extrabold text-text-heading">{deck.name}</h3>
{deck.description && <p className="text-sm text-text-secondary mb-3 line-clamp-2">{deck.description}</p>}
</div>
</div>
<div className="flex items-center gap-3 mt-auto mb-4 pl-7">
<span className="inline-flex items-center gap-1 px-2.5 py-1 rounded-full bg-badge-bg text-badge-text text-xs font-medium">
{deck._count.cards} {deck._count.cards === 1 ? "card" : "cards"}
</span>
{progressLabel && (
<span className="text-xs text-text-muted">{progressLabel}</span>
)}
</div>
<div className="flex flex-col gap-2 border-t border-border-light pt-4 pl-7 sm:flex-row sm:items-center">
<div className="flex w-full min-w-0 gap-2 sm:flex-1">
{deck.progress?.length ? (
<>
<Link href={`/${classSlug}/flashcards/${deck.id}`} className="flex-1 text-center py-2 px-4 rounded-lg bg-primary text-white text-sm font-medium hover:bg-primary-hover transition-all duration-200">
Continue
</Link>
<button
onClick={async () => {
if (!confirm("Start over from the beginning? This will clear your current progress for this deck.")) return;
const responses = await Promise.all(
deck.progress.map((progress) =>
fetch("/api/progress", {
method: "DELETE",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
contentType: "DECK",
contentId: deck.id,
mode: progress.mode,
sessionId: progress.sessionId,
}),
})
)
).catch(() => null);
if (!responses || responses.some((response) => !response.ok)) {
window.alert("The deck could not be restarted. Your saved progress was kept.");
return;
}
window.location.href = `/${classSlug}/flashcards/${deck.id}`;
}}
className="flex-1 text-center py-2 px-4 rounded-lg bg-bg-surface-alt border border-border text-text-heading text-sm font-medium hover:bg-border transition-all duration-200 cursor-pointer shadow-sm"
title="Restart"
>
Restart
</button>
</>
) : (
<Link href={`/${classSlug}/flashcards/${deck.id}`} className="flex-1 text-center py-2 px-4 rounded-lg bg-primary text-white text-sm font-medium hover:bg-primary-hover transition-all duration-200">
Study
</Link>
)}
</div>
<div className="flex items-center justify-end gap-2">
<label className="sr-only" htmlFor={`move-deck-${deck.id}`}>Move {deck.name} to group</label>
<select
id={`move-deck-${deck.id}`}
value={deck.groupId ?? ""}
onChange={(event) => onMove(deck.id, event.target.value || null)}
disabled={moveDisabled}
className="min-h-9 max-w-36 rounded-lg border border-border bg-bg-surface-alt px-2 text-xs text-text-heading disabled:opacity-50"
aria-label={`Move ${deck.name} to group`}
>
<option value="">Uncategorized</option>
{groups.map((group) => (
<option key={group.id} value={group.id}>{group.name}</option>
))}
</select>
<button onClick={() => onEdit(deck)} className="p-2 rounded-lg text-text-muted hover:text-text-heading hover:bg-bg-surface-alt transition-all duration-200 cursor-pointer" title="Edit">
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z" />
</svg>
</button>
<button onClick={() => onDelete(deck)} className="p-2 rounded-lg text-text-muted hover:text-error hover:bg-error-bg transition-all duration-200 cursor-pointer" title="Delete">
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" />
</svg>
</button>
<ShareMenu targetType="DECK" contentId={deck.id} classSlug={classSlug} compact />
</div>
</div>
</div>
</div>
);
}
function DroppableContainer({ id, children }: { id: string; children: React.ReactNode }) {
const { setNodeRef } = useDroppable({ id });
const { over } = useDndContext();
const isOverContainer = over?.id === id;
const isOverChild = over?.data?.current?.sortable?.containerId === id;
const isOver = isOverContainer || isOverChild;
return (
<div ref={setNodeRef} className={`grid grid-cols-1 md:grid-cols-2 gap-4 min-h-[60px] p-2 -mx-2 rounded-lg border-2 transition-colors duration-200 ${isOver ? "border-primary/40 bg-primary/5 shadow-inner" : "border-transparent"}`}>
{children}
</div>
);
}
export default function FlashcardsPage() {
const params = useParams();
const classSlug = params.classSlug as string;
const [decks, setDecks] = useState<DeckItem[]>([]);
const [groups, setGroups] = useState<MaterialGroup[]>([]);
const [loading, setLoading] = useState(true);
const [loadError, setLoadError] = useState<string | null>(null);
const [reloadKey, setReloadKey] = useState(0);
const requestGeneration = useRef(0);
const [showImport, setShowImport] = useState(false);
const [classId, setClassId] = useState<string>("");
const [editingId, setEditingId] = useState<string | null>(null);
const [editName, setEditName] = useState("");
const [editDescription, setEditDescription] = useState("");
const [isCreatingGroup, setIsCreatingGroup] = useState(false);
const [newGroupName, setNewGroupName] = useState("");
const [editingGroupId, setEditingGroupId] = useState<string | null>(null);
const [editGroupName, setEditGroupName] = useState("");
const [actionMessage, setActionMessage] = useState<string | null>(null);
const [collapsedGroups, setCollapsedGroups] = useState<Record<string, boolean>>({});
useEffect(() => {
const timer = window.setTimeout(() => {
const saved = localStorage.getItem('flashcards_collapsed_groups');
if (saved) {
try { setCollapsedGroups(JSON.parse(saved)); } catch {}
}
}, 0);
return () => window.clearTimeout(timer);
}, []);
function toggleGroup(id: string) {
setCollapsedGroups(prev => {
const next = { ...prev, [id]: !prev[id] };
localStorage.setItem('flashcards_collapsed_groups', JSON.stringify(next));
return next;
});
}
const fetchAll = useCallback(async (cId: string, generation = requestGeneration.current, signal?: AbortSignal) => {
try {
const [deckRes, groupRes] = await Promise.all([
fetch(`/api/decks/list?classId=${cId}`, { signal }),
fetch(`/api/material-groups?classId=${cId}&type=DECK`, { signal }),
]);
if (!deckRes.ok || !groupRes.ok) throw new Error("Unable to load flashcard library");
const ds = await deckRes.json();
const gs = await groupRes.json();
if (generation !== requestGeneration.current) return;
setDecks(ds);
setGroups(gs);
setLoadError(null);
} catch (error) {
if (signal?.aborted || generation !== requestGeneration.current) return;
setLoadError(error instanceof Error ? error.message : "Unable to load flashcard library");
} finally {
if (generation === requestGeneration.current) setLoading(false);
}
}, []);
useEffect(() => {
const controller = new AbortController();
const generation = ++requestGeneration.current;
async function init() {
try {
setLoading(true);
setLoadError(null);
const classRes = await fetch("/api/classes", { signal: controller.signal });
if (!classRes.ok) throw new Error("Unable to load class");
const classes = await classRes.json();
const cls = classes.find((c: { slug: string }) => c.slug === classSlug);
if (!cls) throw new Error("Class not found");
if (generation !== requestGeneration.current) return;
setClassId(cls.id);
await fetchAll(cls.id, generation, controller.signal);
} catch (error) {
if (controller.signal.aborted || generation !== requestGeneration.current) return;
setLoadError(error instanceof Error ? error.message : "Unable to load flashcard library");
setLoading(false);
}
}
const timer = window.setTimeout(() => void init(), 0);
return () => {
window.clearTimeout(timer);
controller.abort();
};
}, [classSlug, fetchAll, reloadKey]);
// Group Management
async function handleCreateGroup() {
if (!newGroupName.trim()) return;
const res = await fetch("/api/material-groups", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ classId, name: newGroupName.trim(), type: "DECK" }),
});
if (res.ok) {
const g = await res.json();
setGroups((prev) => [g, ...prev]);
setIsCreatingGroup(false);
setNewGroupName("");
setActionMessage(`Created group ${g.name}.`);
} else {
setActionMessage("The group could not be created. Please retry.");
}
}
async function handleRenameGroup(id: string) {
if (!editGroupName.trim()) return;
const response = await fetch(`/api/material-groups/${id}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name: editGroupName.trim() }),
});
if (!response.ok) {
setActionMessage("The group could not be renamed. No changes were applied.");
return;
}
setGroups((prev) => prev.map((g) => (g.id === id ? { ...g, name: editGroupName.trim() } : g)));
setEditingGroupId(null);
setActionMessage("Group renamed.");
}
async function handleDeleteGroup(id: string, name: string) {
if (!confirm(`Delete group "${name}"? Decks inside will be moved to Uncategorized.`)) return;
const response = await fetch(`/api/material-groups/${id}`, { method: "DELETE" });
if (!response.ok) {
setActionMessage("The group could not be deleted. No changes were applied.");
return;
}
await fetchAll(classId);
setActionMessage(`Deleted group ${name}; its decks are now Uncategorized.`);
}
// Deck Management
async function handleDeleteDeck(deck: DeckItem) {
if (!confirm(`Delete "${deck.name}" and all its cards?`)) return;
const response = await fetch(`/api/decks/${deck.id}`, { method: "DELETE" });
if (!response.ok) {
setActionMessage("The deck could not be deleted. No changes were applied.");
return;
}
setDecks((prev) => prev.filter((d) => d.id !== deck.id));
window.dispatchEvent(new Event("study-decks-changed"));
setActionMessage(`Deleted deck ${deck.name}.`);
}
async function handleRenameDeck(id: string) {
if (!editName.trim()) return;
const response = await fetch(`/api/decks/${id}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name: editName.trim(), description: editDescription.trim() || null }),
});
if (!response.ok) {
setActionMessage("The deck could not be updated. No changes were applied.");
return;
}
setDecks((prev) =>
prev.map((d) => (d.id === id ? { ...d, name: editName.trim(), description: editDescription.trim() || null } : d))
);
setEditingId(null);
setActionMessage("Deck updated.");
}
// Drag and drop setup
const sensors = useSensors(
useSensor(PointerSensor, { activationConstraint: { distance: 8 } }),
useSensor(KeyboardSensor, { coordinateGetter: sortableKeyboardCoordinates })
);
const [activeId, setActiveId] = useState<string | null>(null);
const [isReordering, setIsReordering] = useState(false);
function handleDragStart(event: DragStartEvent) {
if (isReordering) return;
setActiveId(event.active.id as string);
}
async function handleDragEnd(event: DragEndEvent) {
setActiveId(null);
const { active, over } = event;
if (!over) return;
const activeId = active.id as string;
const overId = over.id as string;
const activeDeck = decks.find((d) => d.id === activeId);
if (!activeDeck) return;
let targetGroupId: string | null = null;
const overContainerId = over.data.current?.sortable?.containerId;
if (overContainerId) {
targetGroupId = overContainerId === "uncategorized" ? null : overContainerId;
} else if (groups.find(g => g.id === overId) || overId === "uncategorized") {
targetGroupId = overId === "uncategorized" ? null : overId;
} else {
const overDeck = decks.find((d) => d.id === overId);
if (overDeck) {
targetGroupId = overDeck.groupId;
}
}
const oldIndex = decks.findIndex((deck) => deck.id === activeId);
const overIndex = decks.findIndex((deck) => deck.id === overId);
let nextItems = decks.map((deck) =>
deck.id === activeId ? { ...deck, groupId: targetGroupId } : { ...deck }
);
if (overIndex >= 0 && overIndex !== oldIndex) {
nextItems = arrayMove(nextItems, oldIndex, overIndex);
} else {
const [moved] = nextItems.splice(oldIndex, 1);
nextItems.push(moved);
}
await persistDeckReorder(
decks,
nextItems,
new Set([activeDeck.groupId, targetGroupId])
);
}
async function persistDeckReorder(
previousItems: DeckItem[],
proposedItems: DeckItem[],
affectedGroups: Set<string | null>
) {
const counters = new Map<string | null, number>();
const nextItems = proposedItems.map((item) => {
if (!affectedGroups.has(item.groupId)) return item;
const sortOrder = counters.get(item.groupId) ?? 0;
counters.set(item.groupId, sortOrder + 1);
return { ...item, sortOrder };
});
const updates: ReorderItem[] = nextItems
.filter((item) => affectedGroups.has(item.groupId))
.map((item) => ({ id: item.id, groupId: item.groupId }));
setDecks(nextItems);
setIsReordering(true);
try {
const response = await fetch("/api/decks/reorder", {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ items: updates }),
});
if (!response.ok) throw new Error("Reorder rejected");
return true;
} catch {
setDecks(previousItems);
window.alert("The deck move could not be saved. The library was refreshed.");
await fetchAll(classId);
return false;
} finally {
setIsReordering(false);
}
}
async function handleMoveDeck(deckId: string, targetGroupId: string | null) {
if (isReordering) return;
const activeDeck = decks.find((deck) => deck.id === deckId);
if (!activeDeck || activeDeck.groupId === targetGroupId) return;
const nextItems = decks
.filter((deck) => deck.id !== deckId)
.map((deck) => ({ ...deck }));
nextItems.push({ ...activeDeck, groupId: targetGroupId });
const saved = await persistDeckReorder(
decks,
nextItems,
new Set([activeDeck.groupId, targetGroupId])
);
if (saved) {
const destination = groups.find((group) => group.id === targetGroupId)?.name ?? "Uncategorized";
setActionMessage(`Moved ${activeDeck.name} to ${destination}.`);
}
}
const groupedDecks = groups.map(g => ({
...g,
decks: decks.filter(d => d.groupId === g.id).sort((a,b) => a.sortOrder - b.sortOrder)
}));
const uncategorizedDecks = decks.filter(d => d.groupId === null).sort((a,b) => a.sortOrder - b.sortOrder);
const activeDeck = decks.find(d => d.id === activeId);
return (
<div className="pb-20">
{actionMessage && (
<p className="mb-4 rounded-xl bg-badge-bg px-4 py-3 text-sm text-text-secondary" role="status" aria-live="polite">
{actionMessage}
</p>
)}
{/* Header */}
<div className="mb-6 flex flex-col gap-4 sm:flex-row sm:items-center sm:justify-between">
<div><p className="text-xs font-bold uppercase tracking-[0.16em] text-primary">Collection</p><h2 className="editorial-title mt-1 text-3xl text-text-heading">Flashcard decks</h2></div>
<div className="grid grid-cols-2 gap-3 sm:flex">
<button onClick={() => setIsCreatingGroup(true)} className="inline-flex min-h-12 items-center justify-center gap-2 rounded-xl border border-border-light bg-bg-surface px-4 font-bold text-text-heading shadow-sm transition-colors hover:bg-bg-surface-alt">
Add Group
</button>
<button onClick={() => setShowImport(true)} className="inline-flex min-h-12 items-center justify-center gap-2 rounded-xl bg-primary px-4 font-bold text-white shadow-sm transition-colors hover:bg-primary-hover">
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4" />
</svg>
Import Deck
</button>
</div>
</div>
{isCreatingGroup && (
<div className="animate-slide-up mb-6 flex flex-col gap-3 rounded-2xl border border-primary/20 bg-bg-surface p-5 shadow-sm sm:flex-row sm:items-center">
<input
autoFocus
type="text"
value={newGroupName}
onChange={(e) => setNewGroupName(e.target.value)}
placeholder="e.g. Exam 1 Flashcards"
className="flex-1 px-3 py-2 rounded-lg border border-border bg-bg-surface-alt/50 text-text-heading focus:outline-none focus:ring-2 focus:ring-primary/30"
/>
<button onClick={handleCreateGroup} className="px-4 py-2 bg-primary text-white font-medium rounded-md hover:bg-primary-hover cursor-pointer">
Save
</button>
<button onClick={() => { setIsCreatingGroup(false); setNewGroupName(""); }} className="px-4 py-2 bg-bg-surface-alt text-text-heading font-medium rounded-md hover:bg-border cursor-pointer">
Cancel
</button>
</div>
)}
{editingId && (
<div className="fixed inset-0 z-50 flex items-center justify-center p-4 bg-black/40 backdrop-blur-sm">
<div className="w-full max-w-md rounded-3xl border border-border bg-bg-surface p-6 shadow-[var(--shadow-modal)]">
<h3 className="editorial-title mb-4 text-2xl text-text-heading">Edit deck</h3>
<input
type="text"
value={editName}
onChange={(e) => setEditName(e.target.value)}
placeholder="Deck Name"
autoFocus
className="w-full px-3 py-2 mb-3 rounded-lg border border-border bg-bg-surface-alt/50 text-text-heading focus:outline-none focus:ring-2 focus:ring-primary/30"
/>
<textarea
value={editDescription}
onChange={(e) => setEditDescription(e.target.value)}
placeholder="Description (optional)"
rows={3}
className="w-full px-3 py-2 mb-4 rounded-lg border border-border bg-bg-surface-alt/50 text-text-heading focus:outline-none focus:ring-2 focus:ring-primary/30 resize-none"
/>
<div className="flex gap-2 justify-end">
<button onClick={() => setEditingId(null)} className="px-4 py-2 bg-bg-surface-alt text-text-heading font-medium rounded-md hover:bg-border cursor-pointer">Cancel</button>
<button onClick={() => handleRenameDeck(editingId)} className="px-4 py-2 bg-primary text-white font-medium rounded-md hover:bg-primary-hover cursor-pointer">Save Changes</button>
</div>
</div>
</div>
)}
{!loading && loadError && (
<div role="alert" className="mb-6 rounded-2xl border border-error/30 bg-error-bg p-5 text-error">
<p className="font-bold">Flashcard library could not be loaded.</p>
<p className="mt-1 text-sm">{loadError}</p>
<button onClick={() => setReloadKey((value) => value + 1)} className="mt-3 min-h-10 rounded-lg border border-error/40 px-4 text-sm font-bold">Retry</button>
</div>
)}
{!loading && !loadError && decks.length === 0 && groups.length === 0 && (
<div className="text-center py-16">
<h3 className="text-lg font-semibold text-text-heading mb-1">No decks yet</h3>
<p className="text-text-secondary mb-4">Import your first deck or create a group</p>
</div>
)}
{!loading && !loadError && (
<DndContext sensors={sensors} collisionDetection={pointerWithin} onDragStart={handleDragStart} onDragEnd={handleDragEnd}>
<div className="space-y-8">
{groupedDecks.map(group => (
<div key={group.id} className="rounded-2xl border border-border-light bg-bg-surface/70 p-4 shadow-[var(--shadow-card)] sm:p-5">
<div className={`flex items-center justify-between ${collapsedGroups[group.id] ? "" : "mb-4"}`}>
{editingGroupId === group.id ? (
<div className="flex items-center gap-2">
<input autoFocus value={editGroupName} onChange={e => setEditGroupName(e.target.value)} className="px-2 py-1 rounded border border-border bg-bg-surface text-text-heading focus:ring-1 focus:ring-primary" />
<button onClick={() => handleRenameGroup(group.id)} className="text-xs px-2 py-1 bg-primary text-white rounded">Save</button>
<button onClick={() => setEditingGroupId(null)} className="text-xs px-2 py-1 bg-bg-surface text-text-heading rounded border border-border-light">Cancel</button>
</div>
) : (
<div className="flex items-center gap-2">
<button onClick={() => toggleGroup(group.id)} className="p-1 rounded text-text-muted hover:text-text-heading hover:bg-bg-surface transition-colors cursor-pointer" title={collapsedGroups[group.id] ? "Expand" : "Collapse"}>
<svg className={`w-5 h-5 transition-transform duration-200 ${collapsedGroups[group.id] ? '-rotate-90' : ''}`} fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" /></svg>
</button>
<h3 className="text-lg font-extrabold text-text-heading">{group.name}</h3>
</div>
)}
<div className="flex items-center gap-2">
<ShareMenu targetType="GROUP" contentId={group.id} classSlug={params.classSlug as string} />
<button onClick={() => { setEditingGroupId(group.id); setEditGroupName(group.name); }} className="text-text-muted hover:text-text-heading"><svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z" /></svg></button>
<button onClick={() => handleDeleteGroup(group.id, group.name)} className="text-text-muted hover:text-error"><svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" /></svg></button>
</div>
</div>
<div className={collapsedGroups[group.id] ? "hidden" : "block"}>
<SortableContext id={group.id} items={group.decks.map(d => d.id)} strategy={verticalListSortingStrategy}>
<DroppableContainer id={group.id}>
{group.decks.length === 0 ? (
<div className="col-span-1 md:col-span-2 text-center py-4 text-text-muted text-sm border-2 border-dashed border-border-light rounded-lg">Drop decks here</div>
) : (
group.decks.map(deck => (
<SortableDeckCard key={deck.id} deck={deck} onEdit={(d: DeckItem) => { setEditingId(d.id); setEditName(d.name); setEditDescription(d.description || ""); }} onDelete={handleDeleteDeck} groups={groups} onMove={handleMoveDeck} moveDisabled={isReordering} classSlug={classSlug} />
))
)}
</DroppableContainer>
</SortableContext>
</div>
</div>
))}
{/* Uncategorized */}
<div className="rounded-2xl border border-border-light bg-bg-surface/70 p-4 shadow-[var(--shadow-card)] sm:p-5">
<div className={`flex items-center gap-2 ${collapsedGroups["uncategorized"] ? "" : "mb-4"}`}>
<button onClick={() => toggleGroup("uncategorized")} className="p-1 rounded text-text-muted hover:text-text-heading hover:bg-bg-surface transition-colors cursor-pointer" title={collapsedGroups["uncategorized"] ? "Expand" : "Collapse"}>
<svg className={`w-5 h-5 transition-transform duration-200 ${collapsedGroups["uncategorized"] ? '-rotate-90' : ''}`} fill="none" stroke="currentColor" viewBox="0 0 24 24"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" /></svg>
</button>
<h3 className="text-lg font-semibold text-text-heading">Uncategorized</h3>
</div>
<div className={collapsedGroups["uncategorized"] ? "hidden" : "block"}>
<SortableContext id="uncategorized" items={uncategorizedDecks.map(d => d.id)} strategy={verticalListSortingStrategy}>
<DroppableContainer id="uncategorized">
{uncategorizedDecks.length === 0 ? (
<div className="col-span-1 md:col-span-2 text-center py-4 text-text-muted text-sm border-2 border-dashed border-border-light rounded-lg">No uncategorized decks</div>
) : (
uncategorizedDecks.map(deck => (
<SortableDeckCard key={deck.id} deck={deck} onEdit={(d: DeckItem) => { setEditingId(d.id); setEditName(d.name); setEditDescription(d.description || ""); }} onDelete={handleDeleteDeck} groups={groups} onMove={handleMoveDeck} moveDisabled={isReordering} classSlug={classSlug} />
))
)}
</DroppableContainer>
</SortableContext>
</div>
</div>
</div>
<DragOverlay>
{activeDeck ? (
<div className="opacity-80 scale-105 shadow-xl rotate-2">
<SortableDeckCard deck={activeDeck} onEdit={()=>{}} onDelete={()=>{}} groups={groups} onMove={()=>{}} moveDisabled classSlug={classSlug} />
</div>
) : null}
</DragOverlay>
</DndContext>
)}
{showImport && (
<ImportModal classId={classId} importType="flashcards" onClose={() => setShowImport(false)} onImported={() => { setShowImport(false); setLoading(true); window.location.reload(); }} />
)}
</div>
);
}

View file

@ -1,22 +0,0 @@
import { getClassBySlug } from "@/services/classService";
import { notFound } from "next/navigation";
import { ClassHeader } from "@/components/ui/ClassHeader";
export default async function ClassLayout(props: LayoutProps<"/[classSlug]">) {
const { classSlug } = await props.params;
const classData = await getClassBySlug(classSlug);
if (!classData) {
notFound();
}
return (
<div className="app-page">
{/* Dynamic class header */}
<ClassHeader className={classData.name} />
{/* Page content */}
{props.children}
</div>
);
}

Some files were not shown because too many files have changed in this diff Show more