Compare commits
No commits in common. "4bf121728bd141b8f4244220f923176f643138f1" and "7bd0a9aad46f5f5f074de763c2e3689ef27d2e4d" have entirely different histories.
4bf121728b
...
7bd0a9aad4
338 changed files with 55341 additions and 33595 deletions
|
|
@ -1,17 +1,20 @@
|
||||||
node_modules
|
node_modules
|
||||||
dist
|
.next
|
||||||
.git
|
.git
|
||||||
.env
|
.forgejo
|
||||||
.env.*
|
.github
|
||||||
.data
|
.env*
|
||||||
|
!.env.example
|
||||||
|
dev.db*
|
||||||
|
study.db*
|
||||||
data
|
data
|
||||||
.test-data
|
|
||||||
.claude
|
|
||||||
.codex
|
|
||||||
.agents
|
|
||||||
coverage
|
coverage
|
||||||
*.db
|
.test-databases
|
||||||
*.db-wal
|
.npm-cache
|
||||||
*.db-shm
|
.prisma-cache
|
||||||
*.key
|
audit-results
|
||||||
npm-debug.log*
|
*.log
|
||||||
|
*.tsbuildinfo
|
||||||
|
out.css
|
||||||
|
temp.css
|
||||||
|
test.css
|
||||||
|
|
|
||||||
36
.env.example
36
.env.example
|
|
@ -1,36 +0,0 @@
|
||||||
# Study Desk configuration. Every value is optional.
|
|
||||||
# In Docker, set these as container environment variables instead of using this file.
|
|
||||||
|
|
||||||
# Where the database, uploads and backups live. Default: /data in the container, ./data otherwise.
|
|
||||||
# DATA_DIR=/data
|
|
||||||
|
|
||||||
# Port and bind address.
|
|
||||||
# PORT=3726
|
|
||||||
# HOST=0.0.0.0
|
|
||||||
|
|
||||||
# First-run administrator password. If unset, the first visit shows a setup screen that asks
|
|
||||||
# for the one-time code printed in the log (also written to DATA_DIR/setup-code.txt).
|
|
||||||
# ADMIN_PASSWORD=
|
|
||||||
|
|
||||||
# Forgot the password? Set this, restart once, sign in, then remove it again.
|
|
||||||
# RESET_ADMIN_PASSWORD=
|
|
||||||
|
|
||||||
# Secret used to encrypt AI provider API keys at rest. If unset, a random key is generated and
|
|
||||||
# stored in DATA_DIR/secret.key. Set it explicitly if you want the key kept outside the data folder.
|
|
||||||
# APP_SECRET=
|
|
||||||
|
|
||||||
# Cookie "Secure" flag: auto (default: on whenever the request arrived over HTTPS), true or false.
|
|
||||||
# COOKIE_SECURE=auto
|
|
||||||
|
|
||||||
# Trust X-Forwarded-* headers from a reverse proxy (default true). Set false if the container
|
|
||||||
# is exposed directly to the internet without a proxy.
|
|
||||||
# TRUST_PROXY=true
|
|
||||||
|
|
||||||
# Largest upload accepted, in megabytes.
|
|
||||||
# MAX_UPLOAD_MB=100
|
|
||||||
|
|
||||||
# Number of automatic daily database snapshots to keep.
|
|
||||||
# BACKUP_KEEP=7
|
|
||||||
|
|
||||||
# Timezone used for "today", streaks and due dates until you change it in Settings.
|
|
||||||
# TZ=America/Phoenix
|
|
||||||
70
.forgejo/workflows/build.yml
Normal file
70
.forgejo/workflows/build.yml
Normal file
|
|
@ -0,0 +1,70 @@
|
||||||
|
name: Verify and publish container
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-and-push:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout Code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
# The changed-file lint step compares the pushed commit with its parent.
|
||||||
|
fetch-depth: 2
|
||||||
|
|
||||||
|
- name: Log into Local Registry
|
||||||
|
run: |
|
||||||
|
echo "${{ secrets.FORGEJO_PAT }}" | docker login git.elijahkuntz.com -u "${{ gitea.actor }}" --password-stdin
|
||||||
|
|
||||||
|
- name: Install locked dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Run tests
|
||||||
|
run: npm test
|
||||||
|
|
||||||
|
- name: Validate Prisma schema and migration drift
|
||||||
|
run: |
|
||||||
|
npx prisma validate
|
||||||
|
DATABASE_URL=file:./ci-migration.test.db npx prisma migrate deploy
|
||||||
|
npx prisma migrate diff --from-migrations prisma/migrations --to-schema prisma/schema.prisma --exit-code
|
||||||
|
|
||||||
|
- name: Build application
|
||||||
|
run: npm run build
|
||||||
|
|
||||||
|
- name: Lint changed source files
|
||||||
|
run: |
|
||||||
|
if BASE=$(git rev-parse HEAD^ 2>/dev/null); then
|
||||||
|
git diff --name-only --diff-filter=ACMR -z "$BASE" HEAD -- '*.ts' '*.tsx' '*.js' '*.mjs' |
|
||||||
|
xargs -0 -r npx eslint
|
||||||
|
else
|
||||||
|
git ls-files -z -- '*.ts' '*.tsx' '*.js' '*.mjs' |
|
||||||
|
xargs -0 -r npx eslint
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Build and smoke production image
|
||||||
|
run: |
|
||||||
|
IMAGE_PATH=$(echo "git.elijahkuntz.com/${{ gitea.actor }}/${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]')
|
||||||
|
IMAGE_SHA="$IMAGE_PATH:${{ github.sha }}"
|
||||||
|
CI_SECRET=$(openssl rand -hex 32)
|
||||||
|
SMOKE_CONTAINER="study-smoke-$(openssl rand -hex 8)"
|
||||||
|
cleanup_smoke() {
|
||||||
|
docker rm -f "$SMOKE_CONTAINER" > /dev/null 2>&1 || true
|
||||||
|
}
|
||||||
|
trap cleanup_smoke EXIT
|
||||||
|
docker build -t "$IMAGE_SHA" .
|
||||||
|
docker run -d --name "$SMOKE_CONTAINER" -e SESSION_SECRET="$CI_SECRET" -e ALLOW_INITIAL_SETUP=true "$IMAGE_SHA"
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
if [ "$(docker inspect --format='{{.State.Health.Status}}' "$SMOKE_CONTAINER")" = "healthy" ]; then break; fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
test "$(docker inspect --format='{{.State.Health.Status}}' "$SMOKE_CONTAINER")" = "healthy"
|
||||||
|
docker exec "$SMOKE_CONTAINER" node -e "fetch('http://127.0.0.1:3726/login').then(r => { if (!r.ok) process.exit(1) }).catch(() => process.exit(1))"
|
||||||
|
docker exec "$SMOKE_CONTAINER" node -e "fetch('http://127.0.0.1:3726/api/auth/setup-status').then(async r => { const body = await r.json(); if (!r.ok || body.setupRequired !== true) process.exit(1) }).catch(() => process.exit(1))"
|
||||||
|
cleanup_smoke
|
||||||
|
trap - EXIT
|
||||||
|
docker tag "$IMAGE_SHA" "$IMAGE_PATH:latest"
|
||||||
|
docker push "$IMAGE_SHA"
|
||||||
|
docker push "$IMAGE_PATH:latest"
|
||||||
9
.gitattributes
vendored
9
.gitattributes
vendored
|
|
@ -1,7 +1,2 @@
|
||||||
* text=auto eol=lf
|
# Prisma 7 generated doc comments contain trailing spaces; do not hand-edit generated output.
|
||||||
*.png binary
|
src/generated/prisma/** -whitespace
|
||||||
*.jpg binary
|
|
||||||
*.pdf binary
|
|
||||||
*.docx binary
|
|
||||||
*.pptx binary
|
|
||||||
*.woff2 binary
|
|
||||||
|
|
|
||||||
68
.gitignore
vendored
68
.gitignore
vendored
|
|
@ -1,19 +1,53 @@
|
||||||
node_modules/
|
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||||
dist/
|
|
||||||
data/
|
# dependencies
|
||||||
.data/
|
/node_modules
|
||||||
*.db
|
/.pnp
|
||||||
*.db-wal
|
.pnp.*
|
||||||
*.db-shm
|
.yarn/*
|
||||||
*.db-journal
|
!.yarn/patches
|
||||||
.env
|
!.yarn/plugins
|
||||||
.env.*
|
!.yarn/releases
|
||||||
!.env.example
|
!.yarn/versions
|
||||||
coverage/
|
|
||||||
.vite/
|
# testing
|
||||||
*.tsbuildinfo
|
/coverage
|
||||||
|
/.test-databases/
|
||||||
|
|
||||||
|
# local SQLite study data
|
||||||
|
/dev.db
|
||||||
|
/dev.db-journal
|
||||||
|
/dev.db-shm
|
||||||
|
/dev.db-wal
|
||||||
|
/data/
|
||||||
|
/study.db
|
||||||
|
/study.db-journal
|
||||||
|
/study.db-shm
|
||||||
|
/study.db-wal
|
||||||
|
|
||||||
|
# next.js
|
||||||
|
/.next/
|
||||||
|
/out/
|
||||||
|
|
||||||
|
# production
|
||||||
|
/build
|
||||||
|
|
||||||
|
# misc
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
*.pem
|
||||||
|
|
||||||
|
# debug
|
||||||
npm-debug.log*
|
npm-debug.log*
|
||||||
.test-data/
|
yarn-debug.log*
|
||||||
.claude/
|
yarn-error.log*
|
||||||
data-v2/
|
.pnpm-debug.log*
|
||||||
|
|
||||||
|
# env files (can opt-in for committing if needed)
|
||||||
|
.env*
|
||||||
|
|
||||||
|
# vercel
|
||||||
|
.vercel
|
||||||
|
|
||||||
|
# typescript
|
||||||
|
*.tsbuildinfo
|
||||||
|
next-env.d.ts
|
||||||
|
|
|
||||||
350
AGENTS.md
Normal file
350
AGENTS.md
Normal file
|
|
@ -0,0 +1,350 @@
|
||||||
|
# Study Desk - AI Contributor Guide
|
||||||
|
|
||||||
|
This file is the operating guide for AI agents and human contributors working in this repository.
|
||||||
|
|
||||||
|
## Project summary
|
||||||
|
|
||||||
|
Study Desk is a self-hosted, single-user study application built with Next.js, React, TypeScript, Prisma, and SQLite.
|
||||||
|
|
||||||
|
Core capabilities:
|
||||||
|
|
||||||
|
- Flashcard decks with Markdown content, self-grading, shuffle persistence, resume state, and card management
|
||||||
|
- Practice quizzes with multiple-choice and SATA questions, partial-credit scoring, rationales, category breakdowns, retakes, and attempt history
|
||||||
|
- JSON import, repair, validation, and generation workflows
|
||||||
|
- Material groups for organizing decks and quizzes
|
||||||
|
- Password-protected private pages
|
||||||
|
- Public, read-only share links for decks, quizzes, and groups
|
||||||
|
- Responsive light and dark themes
|
||||||
|
|
||||||
|
This is intentionally a single-user container, not a multi-tenant SaaS application. There is no `User` model. Content ownership is implicit in the local installation.
|
||||||
|
|
||||||
|
When documentation conflicts with implementation, prefer:
|
||||||
|
|
||||||
|
1. Current source code
|
||||||
|
2. `prisma/schema.prisma`
|
||||||
|
3. Applied migrations
|
||||||
|
4. This file
|
||||||
|
5. Older planning documents and the stock README
|
||||||
|
|
||||||
|
## Mandatory working rules
|
||||||
|
|
||||||
|
- Inspect `git status --short` before editing.
|
||||||
|
- Preserve unrelated user changes.
|
||||||
|
- Keep changes scoped to the requested behavior.
|
||||||
|
- Do not use destructive commands such as `git reset --hard` or `git checkout --` unless explicitly requested.
|
||||||
|
- Prefer narrow, patch-based edits over rewriting entire files.
|
||||||
|
- Do not commit secrets, `.env` files, database contents, generated credentials, or private study material.
|
||||||
|
- Do not manually edit generated Prisma files under `src/generated/prisma/`.
|
||||||
|
- Do not modify an already-applied migration. Change `prisma/schema.prisma`, then create a new migration.
|
||||||
|
- Prefer TypeScript types over `any`.
|
||||||
|
- Add dependencies only when the existing stack cannot reasonably solve the problem.
|
||||||
|
- When dependencies change, update `package-lock.json` and verify the build.
|
||||||
|
- Separate UI, route handling, services, and low-level utilities by responsibility.
|
||||||
|
- Ask a clarifying question only when missing information would materially change the implementation or create a risky assumption.
|
||||||
|
|
||||||
|
## Stack and source of truth
|
||||||
|
|
||||||
|
Confirm exact versions in `package.json`. The project currently uses:
|
||||||
|
|
||||||
|
- Next.js 16 App Router
|
||||||
|
- React 19
|
||||||
|
- TypeScript in strict mode
|
||||||
|
- Tailwind CSS v4
|
||||||
|
- Prisma 7 with SQLite and `@prisma/adapter-better-sqlite3`
|
||||||
|
- `iron-session` for encrypted session cookies
|
||||||
|
- Argon2 password hashing
|
||||||
|
- Zod validation
|
||||||
|
- `react-markdown` with `remark-gfm`
|
||||||
|
- `@dnd-kit` for sortable UI
|
||||||
|
- Node 22 in Docker
|
||||||
|
|
||||||
|
Important files:
|
||||||
|
|
||||||
|
- `package.json` - scripts and dependency versions
|
||||||
|
- `prisma/schema.prisma` - database source of truth
|
||||||
|
- `src/lib/db.ts` - shared Prisma client
|
||||||
|
- `src/lib/auth.ts` - session helpers
|
||||||
|
- `src/services/` - business logic and database access
|
||||||
|
- `src/app/api/` - route handlers
|
||||||
|
- `src/app/globals.css` - design tokens and global styling
|
||||||
|
- `src/config/studyModes.ts` - shared study-mode navigation
|
||||||
|
- `SKILL.md` - project workflow for adding API routes
|
||||||
|
- `study-app-implementation-plan.md` - historical architectural context only
|
||||||
|
- `README.md` - currently not authoritative
|
||||||
|
|
||||||
|
For framework-sensitive changes involving routing, layouts, dynamic params, middleware, server/client boundaries, or framework APIs, follow nearby repository patterns and consult version-matched Next.js documentation. Do not perform unrelated framework migrations.
|
||||||
|
|
||||||
|
## Local development
|
||||||
|
|
||||||
|
For a clean checkout:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm ci
|
||||||
|
npx prisma generate
|
||||||
|
npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
On Windows PowerShell, use `npm.cmd` and `npx.cmd` if script execution policy blocks the default shims.
|
||||||
|
|
||||||
|
Useful scripts:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run dev
|
||||||
|
npm run build
|
||||||
|
npm run start
|
||||||
|
npm run lint
|
||||||
|
```
|
||||||
|
|
||||||
|
There is currently no configured test runner. Use focused manual verification plus `npm run build`. Run lint as well, but distinguish pre-existing failures from failures introduced by the current change.
|
||||||
|
|
||||||
|
## Environment and authentication
|
||||||
|
|
||||||
|
Common environment variables:
|
||||||
|
|
||||||
|
- `DATABASE_URL` - SQLite URL
|
||||||
|
- `SESSION_SECRET` - `iron-session` secret, at least 32 characters outside throwaway local development
|
||||||
|
- `SECURE_COOKIES` - marks the session cookie secure when set to `true`
|
||||||
|
- `ADMIN_PASSWORD_HASH` - present in `.env.example`, but currently not used by the login flow
|
||||||
|
|
||||||
|
Current password behavior:
|
||||||
|
|
||||||
|
1. On first login, the submitted password is Argon2-hashed.
|
||||||
|
2. The hash is stored in the `Setting` row `admin_password_hash`.
|
||||||
|
3. Later logins verify against that stored hash.
|
||||||
|
|
||||||
|
Do not change password provisioning casually. Any change must update login, setup-status behavior, deployment documentation, and security handling together.
|
||||||
|
|
||||||
|
Authentication state is stored in the `study-app-session` cookie.
|
||||||
|
|
||||||
|
Public paths include:
|
||||||
|
|
||||||
|
- `/login`
|
||||||
|
- `/api/auth/*`
|
||||||
|
- `/shared/*`
|
||||||
|
- Static assets and Next internals
|
||||||
|
|
||||||
|
Everything else requires authentication unless explicitly designed and reviewed as public.
|
||||||
|
|
||||||
|
Do not log passwords, secrets, full share tokens, or private study content.
|
||||||
|
|
||||||
|
## Database and migration safety
|
||||||
|
|
||||||
|
The local `dev.db` may contain real study data.
|
||||||
|
|
||||||
|
- Do not delete, reset, replace, or recreate it during normal work.
|
||||||
|
- Back up the database before destructive migration work.
|
||||||
|
- Use the shared Prisma client from `@/lib/db`.
|
||||||
|
- Do not create additional `PrismaClient` instances in request handlers or components.
|
||||||
|
- Use transactions when partial writes would leave inconsistent state.
|
||||||
|
- Return 404 responses for missing records instead of exposing raw Prisma errors.
|
||||||
|
|
||||||
|
After changing the Prisma schema:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npx prisma generate
|
||||||
|
npx prisma migrate dev --name describe_the_change
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `npx prisma migrate deploy` for deployed environments.
|
||||||
|
|
||||||
|
## Architecture conventions
|
||||||
|
|
||||||
|
### Server and client components
|
||||||
|
|
||||||
|
- Use Server Components by default.
|
||||||
|
- Add `"use client"` only when state, effects, event handlers, browser APIs, drag and drop, or client-side fetching are required.
|
||||||
|
- Keep database and authentication modules out of client import graphs.
|
||||||
|
- Do not access `window`, `localStorage`, `navigator`, or clipboard APIs from Server Components.
|
||||||
|
- Prefer `Link` for ordinary internal navigation.
|
||||||
|
- Use `useRouter` when navigation follows an action or must be imperative.
|
||||||
|
- Follow nearby Next.js 16 patterns for dynamic params and route context types.
|
||||||
|
|
||||||
|
### API routes and services
|
||||||
|
|
||||||
|
Preferred route-handler flow:
|
||||||
|
|
||||||
|
1. Parse and validate external input.
|
||||||
|
2. Delegate business logic to focused service or utility functions.
|
||||||
|
3. Return a useful status and JSON response.
|
||||||
|
|
||||||
|
Business logic and Prisma calls normally belong in `src/services/` or a focused `src/lib/` module. Reuse existing services before creating new abstractions.
|
||||||
|
|
||||||
|
Some older routes still contain direct Prisma calls or loose input handling. Improve the requested path without turning a small task into a broad refactor.
|
||||||
|
|
||||||
|
### Validation
|
||||||
|
|
||||||
|
- Reuse existing Zod schemas where available.
|
||||||
|
- Never rely only on client-side validation.
|
||||||
|
- Validate IDs, required strings, enums, arrays, and import payloads at the API boundary.
|
||||||
|
- Keep stored JSON parseable.
|
||||||
|
- Use `jsonRepair.ts` only for user- or LLM-generated import text, not to hide malformed database state.
|
||||||
|
|
||||||
|
## UI and accessibility
|
||||||
|
|
||||||
|
Use the existing design system in `src/app/globals.css`.
|
||||||
|
|
||||||
|
Common tokens include:
|
||||||
|
|
||||||
|
- `bg-bg-base`
|
||||||
|
- `bg-bg-surface`
|
||||||
|
- `bg-bg-surface-alt`
|
||||||
|
- `text-text-heading`
|
||||||
|
- `text-text-secondary`
|
||||||
|
- `text-text-muted`
|
||||||
|
- `primary`
|
||||||
|
- `primary-hover`
|
||||||
|
- `border`
|
||||||
|
- `border-light`
|
||||||
|
- `success`
|
||||||
|
- `error`
|
||||||
|
|
||||||
|
Preserve:
|
||||||
|
|
||||||
|
- Light and dark themes
|
||||||
|
- Responsive mobile layouts
|
||||||
|
- Existing card, modal, shadow, and rounded-corner language
|
||||||
|
- `editorial-title` for major serif headings
|
||||||
|
- Manrope for interface text
|
||||||
|
- Existing Markdown rendering through `ReactMarkdown`, `remarkGfm`, and `.markdown-content`
|
||||||
|
- Existing `@dnd-kit` patterns for sortable content
|
||||||
|
|
||||||
|
Accessibility requirements:
|
||||||
|
|
||||||
|
- Use semantic buttons and links.
|
||||||
|
- Preserve visible keyboard focus states.
|
||||||
|
- Give icon-only controls an accessible name with visible text or `aria-label`.
|
||||||
|
- Keep interactive controls keyboard accessible.
|
||||||
|
- Avoid horizontal overflow on study pages, library cards, modals, and action rows.
|
||||||
|
- Do not attach drag listeners to nested action buttons.
|
||||||
|
|
||||||
|
## Domain invariants
|
||||||
|
|
||||||
|
### Flashcards
|
||||||
|
|
||||||
|
- A deck contains ordered `Flashcard` rows with `front` and `back`.
|
||||||
|
- Study progress stores order and results as JSON.
|
||||||
|
- Study mode may be sequential or shuffled.
|
||||||
|
- Saved progress may contain stale card IDs after deletion. Resume helpers must filter and clamp stale IDs safely.
|
||||||
|
- Preserve both study and manage views.
|
||||||
|
- Preserve existing sharing entry points when adding new ones.
|
||||||
|
|
||||||
|
### Quizzes
|
||||||
|
|
||||||
|
- A quiz contains ordered questions with ordered answer options.
|
||||||
|
- `MULTIPLE_CHOICE` questions must have exactly one correct option.
|
||||||
|
- `SATA` questions may have multiple correct options.
|
||||||
|
- SATA uses partial-credit scoring in `src/lib/scoring.ts`.
|
||||||
|
- Partial retakes score only questions present in the submitted answer set.
|
||||||
|
- Attempts persist score, maximum score, submitted answers, retake state, and completion time.
|
||||||
|
- In-progress answers and position persist through `/api/progress`.
|
||||||
|
|
||||||
|
### Imports and generation
|
||||||
|
|
||||||
|
Flashcard import shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "flashcards",
|
||||||
|
"deckName": "Deck name",
|
||||||
|
"description": "Optional description",
|
||||||
|
"cards": [{ "front": "...", "back": "..." }]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Quiz import shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "quiz",
|
||||||
|
"quizName": "Quiz name",
|
||||||
|
"description": "Optional description",
|
||||||
|
"questions": [
|
||||||
|
{
|
||||||
|
"type": "multiple_choice",
|
||||||
|
"prompt": "...",
|
||||||
|
"category": "topic",
|
||||||
|
"options": [{ "text": "...", "correct": true }],
|
||||||
|
"rationale": "..."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The app does not call an LLM directly. The Generate UI prepares instructions and accepts generated JSON from the user.
|
||||||
|
|
||||||
|
Preserve Markdown inside JSON string values and render it through existing Markdown components.
|
||||||
|
|
||||||
|
Keep the current separation between `ImportModal`, `CreateTab`, `ImportTab`, and `GenerateTab`.
|
||||||
|
|
||||||
|
### Sharing
|
||||||
|
|
||||||
|
- `ShareLink` may target a `DECK`, `QUIZ`, or `GROUP`.
|
||||||
|
- Reuse `src/components/ui/ShareMenu.tsx`.
|
||||||
|
- Public routes are `/shared/<classSlug>/<type>/<token>`.
|
||||||
|
- Public viewers are read-only.
|
||||||
|
- Public viewers must not write authenticated progress.
|
||||||
|
- Validate that token, type, and class slug agree before returning shared content.
|
||||||
|
- A deck or quiz may appear publicly shared through its group. Preserve inherited group-share behavior and `?itemId=...` handling.
|
||||||
|
|
||||||
|
## Verification checklist
|
||||||
|
|
||||||
|
Before editing:
|
||||||
|
|
||||||
|
- Read `git status --short`.
|
||||||
|
- Inspect the target component, route, service, and schema paths relevant to the requested behavior.
|
||||||
|
- Check whether the change affects authentication, public sharing, migrations, generated Prisma output, or real study data.
|
||||||
|
|
||||||
|
During implementation:
|
||||||
|
|
||||||
|
- Keep route, service, utility, and component responsibilities clear.
|
||||||
|
- Add validation for new external input.
|
||||||
|
- Reuse existing design tokens, components, and services.
|
||||||
|
- Consider loading, error, empty, mobile, keyboard, light-theme, and dark-theme states.
|
||||||
|
- Avoid unrelated cleanup.
|
||||||
|
|
||||||
|
After editing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run lint
|
||||||
|
npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
If lint already fails, report baseline failures separately.
|
||||||
|
|
||||||
|
For UI work, manually verify:
|
||||||
|
|
||||||
|
1. The affected authenticated route
|
||||||
|
2. The corresponding public/shared route when applicable
|
||||||
|
3. Loading, error, and empty states
|
||||||
|
4. Study resume and restart behavior when relevant
|
||||||
|
5. Drag and drop when relevant
|
||||||
|
6. Light and dark themes
|
||||||
|
7. A narrow mobile viewport
|
||||||
|
8. No horizontal overflow
|
||||||
|
9. Keyboard access for changed controls
|
||||||
|
|
||||||
|
## Known repository conditions
|
||||||
|
|
||||||
|
- `README.md` is still mostly stock create-next-app documentation.
|
||||||
|
- `study-app-implementation-plan.md` contains useful historical context but may be stale.
|
||||||
|
- `task.md` records completed implementation phases.
|
||||||
|
- `SKILL.md` describes the expected workflow for new API routes.
|
||||||
|
- The current lint baseline includes existing errors such as `any` usage and React effect-rule violations.
|
||||||
|
- A successful `next build` is the primary repository-wide compile and type check.
|
||||||
|
- The current middleware convention produces a known deprecation warning.
|
||||||
|
- `docker-compose.yml` may publish port `3000`, while the production Docker image uses port `3726`. Verify the mapping before relying on production Compose.
|
||||||
|
- Do not silently fix known baseline issues as unrelated cleanup.
|
||||||
|
|
||||||
|
## Extending the application
|
||||||
|
|
||||||
|
For a genuinely new study mode:
|
||||||
|
|
||||||
|
1. Add or update the Prisma model and migration when persistence is required.
|
||||||
|
2. Add a focused service in `src/services/`.
|
||||||
|
3. Add API routes under `src/app/api/`.
|
||||||
|
4. Add page routes and client components.
|
||||||
|
5. Add the mode to `src/config/studyModes.ts` when it belongs in shared navigation.
|
||||||
|
6. Extend progress, import, scoring, or sharing only when required.
|
||||||
|
7. Verify authentication, public/private boundaries, mobile layout, and both themes.
|
||||||
|
|
||||||
|
Do not add a new mode by placing unrelated logic into `deckService.ts`, `quizService.ts`, or a large page component.
|
||||||
1
CLAUDE.md
Normal file
1
CLAUDE.md
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
@AGENTS.md
|
||||||
55
Dockerfile
55
Dockerfile
|
|
@ -1,20 +1,47 @@
|
||||||
# syntax=docker/dockerfile:1
|
# ---- base ----
|
||||||
FROM node:24-bookworm-slim AS build
|
FROM node:22-slim AS base
|
||||||
|
RUN apt-get update -y && apt-get install -y openssl && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# ---- deps ----
|
||||||
|
FROM base AS deps
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY package.json package-lock.json ./
|
COPY package.json package-lock.json ./
|
||||||
RUN npm ci
|
RUN npm ci
|
||||||
COPY . .
|
|
||||||
RUN npm run build && npm prune --omit=dev
|
|
||||||
|
|
||||||
FROM node:24-bookworm-slim AS runtime
|
# ---- production dependencies ----
|
||||||
ENV NODE_ENV=production HOST=0.0.0.0 PORT=3726 DATA_DIR=/data WEB_DIR=/app/dist/web
|
FROM deps AS runtime-deps
|
||||||
|
RUN npm prune --omit=dev
|
||||||
|
|
||||||
|
# ---- builder ----
|
||||||
|
FROM base AS builder
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=build --chown=node:node /app/package.json ./
|
COPY --from=deps /app/node_modules ./node_modules
|
||||||
COPY --from=build --chown=node:node /app/node_modules ./node_modules
|
COPY . .
|
||||||
COPY --from=build --chown=node:node /app/dist ./dist
|
RUN npx prisma generate
|
||||||
RUN mkdir -p /data && chown node:node /data
|
RUN npm run build
|
||||||
USER node
|
|
||||||
VOLUME ["/data"]
|
# ---- runner ----
|
||||||
|
FROM base AS runner
|
||||||
|
WORKDIR /app
|
||||||
|
ENV NODE_ENV=production
|
||||||
|
ENV PORT=3726
|
||||||
|
ENV HOSTNAME=0.0.0.0
|
||||||
|
ENV DATABASE_URL="file:/app/data/study.db"
|
||||||
|
|
||||||
|
RUN useradd --system --create-home appuser && mkdir -p /app/data && chown -R appuser:appuser /app
|
||||||
|
|
||||||
|
COPY --from=builder /app/.next/standalone ./
|
||||||
|
COPY --from=builder /app/.next/static ./.next/static
|
||||||
|
COPY --from=builder /app/public ./public
|
||||||
|
COPY --from=builder /app/prisma ./prisma
|
||||||
|
COPY --from=builder /app/prisma.config.ts ./
|
||||||
|
COPY --from=builder /app/scripts ./scripts
|
||||||
|
COPY --from=runtime-deps /app/node_modules ./node_modules
|
||||||
|
COPY docker-entrypoint.sh ./
|
||||||
|
RUN chmod +x docker-entrypoint.sh
|
||||||
|
# Running as root to avoid permission denied on Unraid host-mounted volumes
|
||||||
EXPOSE 3726
|
EXPOSE 3726
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD node -e "fetch('http://127.0.0.1:'+process.env.PORT+'/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"
|
VOLUME ["/app/data"]
|
||||||
CMD ["node", "dist/server/main.js"]
|
HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=6 \
|
||||||
|
CMD ["node", "-e", "fetch('http://127.0.0.1:3726/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
|
||||||
|
ENTRYPOINT ["./docker-entrypoint.sh"]
|
||||||
|
|
|
||||||
95
PROGRESS.md
95
PROGRESS.md
|
|
@ -1,95 +0,0 @@
|
||||||
# Study Desk V2 — progress and resume notes
|
|
||||||
|
|
||||||
Updated 2026-11-11. Branch: **v2**. Original V1: **main**, preserved at `7bd0a9a`.
|
|
||||||
Work remains local; no push, repository creation or production deployment has been performed.
|
|
||||||
|
|
||||||
## Product and architecture
|
|
||||||
|
|
||||||
This is a rebuilt app using React + Vite + TypeScript, Hono/Node HTTP server, built-in
|
|
||||||
`node:sqlite`, SQLite FTS5, local document storage and a persistent document queue. One process,
|
|
||||||
one data volume, one administrator. Anonymous visitors get public read-only shares and
|
|
||||||
browser-local study progress. Administrator progress stays server-side.
|
|
||||||
|
|
||||||
The previous agent implemented the core product and committed it through `c549c34`:
|
|
||||||
classes/folders, CRUD/duplicate/move/search, quiz sessions/results/history, FSRS flashcards,
|
|
||||||
imports/exports/templates, documents/extraction/retrieval/OCR, provider adapters, editable
|
|
||||||
AI drafts, tool-based assistant actions, sharing, analytics and the responsive themed UI.
|
|
||||||
User feedback fixes are already included: larger clickable flashcards, arrow keys, streak
|
|
||||||
effects, persistent review sidebar, live quiz score, shared quiz timers, SATA credit details,
|
|
||||||
type-grouped materials, copyable import instructions and assistant library links.
|
|
||||||
|
|
||||||
## Current continuation
|
|
||||||
|
|
||||||
- Confirmed original 162 tests pass; sandbox networking initially blocked the mock provider.
|
|
||||||
- Added encryption-key corruption/missing-key refusal, static-path boundary validation,
|
|
||||||
private embedded-media authorization and share-revocation checks.
|
|
||||||
- Full backups include the generated encryption key and fail on unreadable files.
|
|
||||||
- Added backup restore, media authorization, key durability, static-path traversal, V1 import and failed-migration
|
|
||||||
rollback tests. The new V1 test checks source bytes are unchanged and state is preserved.
|
|
||||||
- Added Dockerfile, Compose, .dockerignore, README and deployment/migration/AI documentation.
|
|
||||||
- Updated obsolete ChatGPT subscription wording after checking official documentation:
|
|
||||||
account integration exists for eligible clients, but hosted app approval for this private
|
|
||||||
Unraid web deployment has not been established. API integration remains the working path.
|
|
||||||
- Terra completed isolated UI/workflow and visual QA; see results below.
|
|
||||||
- Guarded quiz exit/submission when the latest save failed, and prevented repeated automatic
|
|
||||||
timer submissions after a failed request.
|
|
||||||
- Preserved the V1 card elapsed-days scheduling field during import.
|
|
||||||
|
|
||||||
## Verification status
|
|
||||||
|
|
||||||
Baseline: 162 tests passed, TypeScript passed and production web/server builds passed.
|
|
||||||
Added six regression tests. Full `npm run check` passes: **168 tests**, TypeScript and both
|
|
||||||
production builds. Tests were run with loopback access for the mock provider.
|
|
||||||
|
|
||||||
The earlier agent manually exercised quiz completion/results, flashcards, streak effects,
|
|
||||||
mock AI generation through the assistant, editable drafts, and a 375px home page without
|
|
||||||
horizontal overflow. Treat these as historical evidence; record Terra's fresh results below.
|
|
||||||
|
|
||||||
Not verified: actual Docker image build/runtime, Unraid filesystem permissions and proxy,
|
|
||||||
paid provider inference, real ChatGPT OAuth/eligibility. Docker is not installed here. Do not
|
|
||||||
claim these have been tested. The installable app caches its shell, not private study data;
|
|
||||||
full offline administrator study is not implemented.
|
|
||||||
|
|
||||||
## Remaining work
|
|
||||||
|
|
||||||
1. On Unraid later: execute the deployment/restore checklist in docs/DEPLOYMENT.md.
|
|
||||||
2. With owner-supplied provider connections: verify live discovery, streamed chat, generation,
|
|
||||||
OCR and task routing. No real-provider calls should be inferred from mock test results.
|
|
||||||
3. Future optional work: approved ChatGPT account OAuth, more robust offline studying and
|
|
||||||
semantic embeddings (current retrieval is lexical FTS5).
|
|
||||||
|
|
||||||
## Resume safely
|
|
||||||
|
|
||||||
1. `git status --short`, read this file and recent commits. Do not overwrite user changes.
|
|
||||||
2. The ignored `.env` currently selects `.data/`, containing dev/demo content that the user
|
|
||||||
has tried. Never print its secrets or reset/seed it casually. Use a new `.test-data/...`
|
|
||||||
directory and a separate port for destructive QA.
|
|
||||||
3. `npm.cmd run check` (tests require localhost networking for their mock AI server).
|
|
||||||
4. `npm.cmd run dev` for Vite + API, or `npm.cmd run build` then `npm.cmd start` for the built
|
|
||||||
app. See README and .env.example. All automated test databases are disposable copies.
|
|
||||||
5. Keep regular local commits on `v2`. Do not push until the owner explicitly changes the
|
|
||||||
original local-only instruction. Publishing later can use `git push -u origin v2` after
|
|
||||||
reviewing the diff and confirming credentials; no force push is needed.
|
|
||||||
|
|
||||||
## Fresh UI QA findings
|
|
||||||
|
|
||||||
Terra tested through Edge in an isolated volume on port 3731. Observed successful login,
|
|
||||||
class creation, quiz/question editing, practice start, save/exit, resume, answer check and
|
|
||||||
100% results; deck/card creation, browse flip, FSRS review, pasted question import; desktop
|
|
||||||
and emulated 400×933 mobile layouts in both themes. No obvious clipping or overlap observed
|
|
||||||
on those screens. The test server was stopped and the temporary password removed.
|
|
||||||
|
|
||||||
Visual finding: secondary labels appeared faint, particularly in dark mode. Adjusted the
|
|
||||||
shared `ink-3` colors. Computed WCAG contrast against paper/surface/sunken backgrounds:
|
|
||||||
light now **5.15 / 5.38 / 4.76** (previously 3.48 / 3.63 / 3.21); dark now
|
|
||||||
**7.42 / 6.86 / 7.71**. These ratios cover these token pairs, not a full accessibility audit.
|
|
||||||
|
|
||||||
Screenshots remain ignored under `.test-data/ui-terra/`: desktop-dark-quiz.png,
|
|
||||||
desktop-light-quiz.png, mobile-dark-quiz.png, mobile-light-quiz.png and mobile-light-session.png.
|
|
||||||
Post-fix Terra recheck: desktop light/dark screenshots show improved secondary-text legibility
|
|
||||||
(`postfix-dark-desktop.png`, `postfix-light-desktop.png`). An Edge InPrivate visitor opened
|
|
||||||
the isolated shared quiz without login, completed practice, saw its rationale and reached
|
|
||||||
100% results. Shared-deck UI, long-label layout stress tests and post-fix mobile contrast were
|
|
||||||
not revisited in this follow-up. Public deck data isolation/revocation is covered by API tests.
|
|
||||||
No known functional failures were found in the UI paths exercised. This is sampled QA, not
|
|
||||||
an exhaustive audit of every layout and browser. Terra's isolated server is stopped.
|
|
||||||
164
README.md
164
README.md
|
|
@ -1,81 +1,115 @@
|
||||||
# Study Desk V2
|
Study Desk is a self-hosted, single-user study application.
|
||||||
|
|
||||||
A personal study platform for nursing school: classes and folders, quizzes, FSRS flashcards,
|
## Local development
|
||||||
notes, private documents, and AI-assisted content creation. One administrator; friends study
|
|
||||||
through revocable public links without accounts.
|
|
||||||
|
|
||||||
## Run locally
|
|
||||||
|
|
||||||
Requires Node.js 24 or newer and npm. The development VM currently uses Node 26.
|
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
npm.cmd ci
|
npm.cmd ci
|
||||||
|
npx.cmd prisma generate
|
||||||
npm.cmd run dev
|
npm.cmd run dev
|
||||||
```
|
```
|
||||||
|
|
||||||
Open the Vite address printed in the terminal (normally http://localhost:5173).
|
The automated test harness always creates a uniquely named database under
|
||||||
The API listens on port 3726. For the production build locally:
|
`.test-databases/`, applies committed migrations, and removes the database after
|
||||||
|
the run. It refuses `dev.db`, `/app/data/study.db`, existing databases, and paths
|
||||||
|
that are not explicitly test-named.
|
||||||
|
|
||||||
|
## Production container
|
||||||
|
|
||||||
|
Set a unique session secret of at least 32 characters. Production startup fails
|
||||||
|
before migrations or the HTTP server when the secret is missing, short, or the
|
||||||
|
published development fallback.
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
npm.cmd run build
|
$env:SESSION_SECRET = '<at-least-32-random-characters>'
|
||||||
npm.cmd start
|
docker compose -f docker-compose.yml up --build
|
||||||
```
|
```
|
||||||
|
|
||||||
Open http://localhost:3726. On a fresh install, enter the setup code from the server log or
|
The host listens on `http://localhost:3000`; the container listens on port 3726.
|
||||||
`data/setup-code.txt`, then choose your administrator password. Alternatively set
|
Development Compose is intentionally explicit and is never auto-merged:
|
||||||
`ADMIN_PASSWORD` for first startup. An optional `.env` file is loaded by the server; see
|
|
||||||
[.env.example](.env.example). Existing environment variables take precedence.
|
|
||||||
|
|
||||||
**This checkout already has an ignored `.env` pointing at `.data/` from development.**
|
|
||||||
Do not delete it or run reset/seed commands against content you want to keep. The demo seeder
|
|
||||||
is an opt-in development tool, never part of startup.
|
|
||||||
|
|
||||||
## What is implemented
|
|
||||||
|
|
||||||
- Classes, folders, search, edit/duplicate/move/delete actions; Markdown and embedded images.
|
|
||||||
- Multiple-choice and select-all quizzes; practice feedback, exam timers, shuffle, partial
|
|
||||||
credit, saved sessions, missed-question review, results and history.
|
|
||||||
- Flashcard editing, browse/self-assessment sessions, FSRS scheduling, persistent reviews,
|
|
||||||
undo, keyboard shortcuts and streak effects.
|
|
||||||
- Public quiz/deck/class/folder/note links and explicitly shared documents. Collection shares
|
|
||||||
include current and future quizzes, decks and notes; uploaded documents stay private.
|
|
||||||
- JSON import previews, validation, templates, content exports and a V1 SQLite import path.
|
|
||||||
- PDF/DOCX/PPTX/TXT/Markdown/image uploads; text extraction, OCR via a configured vision model,
|
|
||||||
chunking and SQLite FTS retrieval with source references.
|
|
||||||
- OpenAI, Anthropic, Gemini, OpenAI-compatible and Ollama provider connections, manual/discovered
|
|
||||||
models, task routing, encrypted keys and usage estimates.
|
|
||||||
- AI generation with editable drafts; an assistant that searches materials, reads notes,
|
|
||||||
creates drafts, finds weak topics and proposes confirmed content changes.
|
|
||||||
- Responsive light/dark interface and an installable app shell. Offline studying is limited:
|
|
||||||
private data is not cached; most operations need a server connection.
|
|
||||||
|
|
||||||
## Deploy, back up, migrate
|
|
||||||
|
|
||||||
See [deployment and restore](docs/DEPLOYMENT.md), [V1 migration](docs/MIGRATION.md), and
|
|
||||||
[AI providers and account integration](docs/AI.md).
|
|
||||||
|
|
||||||
The image runs a single Node process with SQLite and local file storage. No database or queue
|
|
||||||
container is required. The container runs as UID/GID 1000 and persists everything under `/data`.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
docker compose up -d --build
|
|
||||||
```
|
|
||||||
|
|
||||||
Default V2 host port: **3727**, deliberately separate from the old application. Use a separate
|
|
||||||
data folder. Docker is not installed in this VM; image build/runtime checks remain to be run
|
|
||||||
on Unraid. No production deployment has been performed.
|
|
||||||
|
|
||||||
## Verification and resuming work
|
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
npm.cmd run check
|
docker compose -f docker-compose.dev.yml up --build
|
||||||
```
|
```
|
||||||
|
|
||||||
Tests create isolated temporary databases. AI integration tests use a local mock server and
|
For first-time production setup, either provide an Argon2 encoded
|
||||||
need loopback networking. They do not spend provider credits. Real provider credentials,
|
`ADMIN_PASSWORD_HASH`, or temporarily set `ALLOW_INITIAL_SETUP=true` for the
|
||||||
account-specific model access, Linux container behavior and Unraid proxy configuration must
|
one-time setup request. Remove the flag after setup. Production HTTP password
|
||||||
be checked in their target environments.
|
reset initiation is unavailable; run this on the server instead:
|
||||||
|
|
||||||
[PROGRESS.md](PROGRESS.md) records completed work, remaining verification, decisions and the
|
```powershell
|
||||||
resume procedure. V1 remains on `main` and in Git history; all V2 work is on local `v2`.
|
npm.cmd run auth:reset
|
||||||
Nothing is automatically pushed or published.
|
```
|
||||||
|
|
||||||
|
If TLS terminates at a trusted reverse proxy, forward requests only from that
|
||||||
|
proxy and set `SECURE_COOKIES=true`. Leave it false for intentional plain HTTP;
|
||||||
|
Secure cookies cannot be used over plain HTTP.
|
||||||
|
|
||||||
|
## Database backup, migration adoption, and restore
|
||||||
|
|
||||||
|
Never run migration tests against the only copy of a study database. Create a
|
||||||
|
lock-safe SQLite backup through the backup API and verify it before deployment:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$env:DATABASE_URL = 'file:./data/study.db'
|
||||||
|
npm.cmd run db:backup -- --output ./backups/study-before-upgrade.db
|
||||||
|
npm.cmd run db:preflight
|
||||||
|
npx.cmd prisma migrate deploy
|
||||||
|
```
|
||||||
|
|
||||||
|
The material-group preflight reports one of four states:
|
||||||
|
|
||||||
|
- `APPLY`: migration history is complete and the group schema is absent; run
|
||||||
|
`prisma migrate deploy` normally.
|
||||||
|
- `ADOPT`: prior migrations are tracked and the database exactly matches the
|
||||||
|
intended schema of one or more unrecorded committed migrations. This includes
|
||||||
|
recovery from P3018 when an already schema-pushed table or column exists.
|
||||||
|
After reviewing the verified backup, explicitly run
|
||||||
|
`npm run db:preflight -- --resolve --backup <backup-path>`.
|
||||||
|
- `CURRENT`: migration and schema already agree.
|
||||||
|
- `CONFLICT`: partial or unknown state. Stop and recover manually; do not use
|
||||||
|
`db push`, reset, edit applied migrations, or mark anything applied.
|
||||||
|
|
||||||
|
The production entrypoint runs this preflight before `prisma migrate deploy`.
|
||||||
|
It refuses `ADOPT` and `CONFLICT` states without changing migration history;
|
||||||
|
adoption always remains an explicit backup-gated operator action.
|
||||||
|
|
||||||
|
Restore drill: stop Study Desk, keep the damaged database as evidence, restore
|
||||||
|
the verified backup to a new path, run SQLite `integrity_check` plus
|
||||||
|
`npm run db:preflight`, start the same prior application image against the
|
||||||
|
restored path, and compare class/deck/quiz counts plus representative content.
|
||||||
|
Only swap the production path after those checks pass.
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
First, run the development server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run dev
|
||||||
|
# or
|
||||||
|
yarn dev
|
||||||
|
# or
|
||||||
|
pnpm dev
|
||||||
|
# or
|
||||||
|
bun dev
|
||||||
|
```
|
||||||
|
|
||||||
|
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
|
||||||
|
|
||||||
|
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
|
||||||
|
|
||||||
|
This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
|
||||||
|
|
||||||
|
## Learn More
|
||||||
|
|
||||||
|
To learn more about Next.js, take a look at the following resources:
|
||||||
|
|
||||||
|
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
|
||||||
|
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
|
||||||
|
|
||||||
|
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
|
||||||
|
|
||||||
|
## Deploy on Vercel
|
||||||
|
|
||||||
|
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
|
||||||
|
|
||||||
|
Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.
|
||||||
|
|
|
||||||
24
SKILL.md
Normal file
24
SKILL.md
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
---
|
||||||
|
name: adding-an-api-route
|
||||||
|
description: Use when adding a new Next.js API route to this project. Keeps routes thin and logic in the right layer instead of inline in the route file.
|
||||||
|
---
|
||||||
|
|
||||||
|
# Adding a new API route
|
||||||
|
|
||||||
|
1. Check `study-app-implementation-plan.md`'s Section 9 file tree first. If
|
||||||
|
this route is already named there, follow that path and method exactly
|
||||||
|
rather than inventing a different shape.
|
||||||
|
2. Create the route file under `src/app/api/<resource>/route.ts`, or
|
||||||
|
`src/app/api/<resource>/[id]/route.ts` for a single-resource endpoint.
|
||||||
|
3. The route handler does exactly three things: validate the request body
|
||||||
|
(reuse a Zod schema from `lib/validation/` if one already exists for this
|
||||||
|
shape, write one if it doesn't), call one function in the matching
|
||||||
|
`services/<resource>Service.ts`, and return a `NextResponse`. No Prisma
|
||||||
|
calls and no business logic directly in the route file.
|
||||||
|
4. If the logic doesn't fit an existing service file, add a new exported
|
||||||
|
function to the closest matching one. Only create a new
|
||||||
|
`services/<resource>Service.ts` if this is genuinely a new resource, don't
|
||||||
|
bolt unrelated logic onto an existing service to avoid creating a file.
|
||||||
|
5. Run `npm run lint` before considering the route done. A failing
|
||||||
|
`max-lines` or `filename-case` error means something is in the wrong
|
||||||
|
place, not that the rule needs an exception.
|
||||||
67
audit-results/CONFIRMED_FINDINGS.md
Normal file
67
audit-results/CONFIRMED_FINDINGS.md
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
# Confirmed Findings
|
||||||
|
|
||||||
|
Severity: Critical / High / Medium / Low / Observation. Confidence: Confirmed / High-confidence inference / Unverified risk.
|
||||||
|
Full structured detail for every finding (files, functions, scenario, expected/actual, root cause, impact, evidence, fix direction, complexity, regression test, runtime-confirmation flag) is in `FINDINGS.json` (validated). Source IDs from specialist auditors are in parentheses.
|
||||||
|
|
||||||
|
| ID | Severity | Confidence | Title | Area |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| FIN-01 (DBAUD-01, GRP-01) | **Critical** | Confirmed | Schema/migration drift: `MaterialGroup` table + `groupId` columns never migrated — fresh deployments' databases are incompatible with the app (verified: `no such column: Deck.groupId`, `no such table: MaterialGroup`) | DB/migrations |
|
||||||
|
| FIN-02 (AUTH-01, OPS-03) | **Critical** | High-confidence inference | Hardcoded fallback `SESSION_SECRET` (`"dev-session-secret-change-in-production-must-be-32-chars"`) in `src/lib/auth.ts:15` + `src/proxy.ts:8-9`; empty env var in shipped compose → forgeable session cookie → full auth bypass | Auth/deploy |
|
||||||
|
| FIN-03 (QUIZ-01, ADV-01) | **High** | Confirmed | In-viewer "Retake Missed" never sets `retakeIds` prop → persisted as a full attempt with a wrong, lower score; pollutes SEQUENTIAL progress | Quiz |
|
||||||
|
| FIN-04 (CARD-01, IMPT-01, QUIZ-06) | **High** | Confirmed | Stale card ids in saved progress never filtered on resume (`filterAndClampOrder` dead code) → deleting a card mid-session bricks the flashcard study session | Flashcards |
|
||||||
|
| FIN-05 (OPS-01) | **High** | Confirmed | Port mismatch: container listens on 3726, compose publishes 3000:3000 → production deployment unreachable | Deploy |
|
||||||
|
| FIN-06 (OPS-06) | **High** | High-confidence inference | No `.dockerignore`: Windows-host builds inject win32 native modules (better-sqlite3, argon2) into the Linux image | Deploy |
|
||||||
|
| FIN-07 (OPS-08) | **High** | Confirmed | `docker-compose.override.yml` auto-merges on plain `docker compose up` → production command silently runs dev mode | Deploy |
|
||||||
|
| FIN-08 (AUTH-04, ADV-09) | Medium | High-confidence inference | First-login provisioning takeover; `ADMIN_PASSWORD_HASH` env is dead config | Auth |
|
||||||
|
| FIN-09 (AUTH-02) | Medium | High-confidence inference | Proxy auth bypass for any path containing `.` (latent today; would expose any future non-UUID-id route) | Auth |
|
||||||
|
| FIN-10 (AUTH-03) | Medium | High-confidence inference | Password-reset abuse: token overwrite DoS, spoofable `x-forwarded-for` rate limits, unthrottled `complete` | Auth |
|
||||||
|
| FIN-11 (QUIZ-02) | Medium | High-confidence inference | Attempt route stores unvalidated `answersJson`; duplicate option ids inflate SATA credit; malformed shapes → 500 | Quiz |
|
||||||
|
| FIN-12 (QUIZ-03, FE-04) | Medium | Confirmed | Quiz Finish double-submit creates duplicate attempts; failure path silent | Quiz/FE |
|
||||||
|
| FIN-13 (QUIZ-04) | Medium | High-confidence inference | Progress PATCH can land after Finish's DELETE → stale "Continue" row for a completed quiz | Quiz |
|
||||||
|
| FIN-14 (CARD-02, FE-05, QUIZ-09) | Medium | High-confidence inference | Fire-and-forget, unsequenced progress autosaves: out-of-order PATCHes regress resume state; multi-tab last-write-wins | Flashcards/Quiz |
|
||||||
|
| FIN-15 (CARD-03, ADV-07) | Medium | Confirmed | Completed flashcard session never persisted as complete → resume re-shows the last graded card | Flashcards |
|
||||||
|
| FIN-16 (CARD-04) | Medium | Confirmed | SRS review endpoint enforces membership only — no due-date or `newCardsPerDay` enforcement server-side | SRS |
|
||||||
|
| FIN-17 (CARD-05, ADV-02) | Medium | Confirmed | Unbounded DB scans: whole `StudyActivity` table per fetch; all SRS card states per set; Navbar polls every 60 s | Perf/DB |
|
||||||
|
| FIN-18 (FE-01, GRP-03, FE-02, GRP-05) | Medium | Confirmed | Library optimistic mutations are fire-and-forget (no `res.ok`, no rollback); cross-group reorder omits origin group from payload | FE/Groups |
|
||||||
|
| FIN-19 (GRP-02, CARD-07, AUTH-06) | Medium | Confirmed | Reorder/create endpoints trust client `groupId`/`sortOrder` — no class/type/existence validation; cross-class group membership can leak content into another class's share link | Groups |
|
||||||
|
| FIN-20 (FE-03) | Medium | Confirmed | Shared quiz session resets and options reshuffle on any parent re-render (unstable inline `quiz` prop) | FE |
|
||||||
|
| FIN-21 (IMPT-02, FE-08, QUIZ-05, ADV-10) | Medium | Confirmed | `/api/progress` PATCH completely unvalidated; unguarded client `JSON.parse` of persisted progress crashes pages | Quiz/FE |
|
||||||
|
| FIN-22 (OPS-02) | Medium | High-confidence inference | Entrypoint migrate-deploy chain fragile: crash loop on conflicting DB state; runner prisma CLI install unverified/unpinned | Deploy |
|
||||||
|
| FIN-23 (OPS-05) | Medium | Confirmed | Non-reproducible installs: `npm install` (not `ci`); unpinned `prisma@^7.8.0` re-install re-resolves the whole tree in the runner | Deploy |
|
||||||
|
| FIN-24 (OPS-07) | Medium | Confirmed | No healthcheck, no backup, no documented recovery | Deploy |
|
||||||
|
| FIN-25 (OPS-10, TEST-10) | Medium | Confirmed | CI builds/pushes without tests, lint, prisma validate, or container smoke test — deployment-breaking defects ship green | CI |
|
||||||
|
| FIN-26 (QUIZ-07) | Low | Confirmed | SATA scoring divides by `correctIds.length` with no zero-guard → NaN | Quiz |
|
||||||
|
| FIN-27 (QUIZ-08) | Low | Confirmed | Historical review/category breakdown recompute scores from current content, not stored attempt | Quiz |
|
||||||
|
| FIN-28 (IMPT-03) | Low | Confirmed | Card add/edit endpoints bypass Zod: empty strings storable; CreateTab silently drops incomplete cards | Imports |
|
||||||
|
| FIN-29 (IMPT-04) | Low | Confirmed | No payload size/string-length/array-length caps on import schemas/routes; whitespace-only names accepted | Imports |
|
||||||
|
| FIN-30 (IMPT-05) | Low | Confirmed | SATA constraint mismatch: instructions say ≥2 correct, schema enforces ≥1 | Imports |
|
||||||
|
| FIN-31 (IMPT-06) | Low | Confirmed | Unvalidated `name`/`groupId` override fields on import POST routes → 500; PATCH routes mask failures as 404 | Imports |
|
||||||
|
| FIN-32 (GRP-04) | Low | Confirmed | Group deletion leaves duplicate sortOrder values in Uncategorized; no unique constraint/tie-break | Groups |
|
||||||
|
| FIN-33 (GRP-06) | Low | Confirmed | Keyboard users cannot move items between groups (empty `handleDragOver`) | Groups/a11y |
|
||||||
|
| FIN-34 (CARD-08) | Low | High-confidence inference | Restart deletes progress fire-and-forget; slow DELETE can remove the fresh session's progress row | Flashcards |
|
||||||
|
| FIN-35 (CARD-09) | Low | Confirmed | Concurrent first review of same new card → P2002 → generic 500 instead of 409 | SRS |
|
||||||
|
| FIN-36 (CARD-10) | Low | High-confidence inference | "Previous card" during the 350 ms grade animation races the pending timeout | Flashcards |
|
||||||
|
| FIN-37 (CARD-11) | Low | Confirmed | SRS set page never refreshes on focus → stale membership after deck deletion in another tab | SRS/FE |
|
||||||
|
| FIN-38 (CARD-06) | Low | Confirmed | Day boundary hardcoded to Arizona (UTC-7): "today"/new-card limit/streak roll over at 07:00 UTC | SRS |
|
||||||
|
| FIN-39 (AUTH-05) | Low | Confirmed | Proxy `destroy()` cookie-clear lost on redirect; layout check ignores `sessionGeneration` | Auth |
|
||||||
|
| FIN-40 (AUTH-08) | Low | Confirmed | Session cookie Secure flag off in shipped compose (plain HTTP, no TLS story) | Auth/deploy |
|
||||||
|
| FIN-41 (DBAUD-02, AUTH-07) | Low | Confirmed | `/api/share` accepts arbitrary `targetType` → repeatable junk all-NULL `ShareLink` rows | Auth/DB |
|
||||||
|
| FIN-42 (DBAUD-03) | Low | High-confidence inference | `dev.db` committed in git history (8 commits), remains in blobs on the LAN remote | Repo hygiene |
|
||||||
|
| FIN-43 (OPS-09) | Low | Confirmed | `.gitignore` gaps (`data/`, `study.db*`); scratch files `out.css`/`temp.css`/`test.css` tracked | Repo hygiene |
|
||||||
|
| FIN-44 (FE-07, ADV-04) | Low | Confirmed | Dashboard `fetchClasses` no `.catch`/`res.ok` → unhandled rejection, misleading empty state (or render crash) | FE |
|
||||||
|
| FIN-45 (FE-09) | Low | Confirmed | Effect fetches without `.catch` in GenerateTab and ShareMenu | FE |
|
||||||
|
| FIN-46 (FE-10) | Low | High-confidence inference | Cross-class navigation fetch race renders wrong-class data; module cache never invalidated | FE |
|
||||||
|
| FIN-47 (ADV-03) | Low | Confirmed | Logout has no error handling — failed logout strands the user | FE |
|
||||||
|
| FIN-48 (ADV-05) | Low | Confirmed | `slugify` can yield an empty slug (unreachable class); renames never update the URL slug | Classes |
|
||||||
|
| FIN-49 (ADV-08) | Low | High-confidence inference | Shared-viewer localStorage keys item-scoped, not token-scoped → session bleed across tokens | Sharing/FE |
|
||||||
|
| FIN-50 (GRP-07) | Observation | Confirmed | Group `sortOrder` uses inverted desc convention with unvalidated PATCH — latent trap | Groups |
|
||||||
|
| FIN-51 (FE-11) | Observation | Confirmed | Collapsed-groups state read in effect → one-frame expand flash | FE |
|
||||||
|
| FIN-52 (AUTH-10) | Observation | Confirmed | Shared quiz links ship the full answer key to anonymous viewers — by design, worth a warning | Sharing |
|
||||||
|
| FIN-53 (TEST-01) | Low | Confirmed | False-confidence test: "distinct valid state for every rating" never asserts distinctness | Tests |
|
||||||
|
|
||||||
|
## Notes on consolidation
|
||||||
|
|
||||||
|
- Duplicate findings across specialist reports were merged under one root cause (see `sourceIds`).
|
||||||
|
- FE-06 ("all /api/* routes unauthenticated") was **refuted** by the adversarial review — `src/proxy.ts` IS the compiled middleware and enforces session checks; the residual truth is FIN-09 (dot-bypass). See REJECTED_FINDINGS.md.
|
||||||
|
- OPS-02's fresh-volume crash-loop framing was weakened by the adversarial review (on a fresh volume `migrate deploy` succeeds; the app then fails per FIN-01) — reframed as FIN-22.
|
||||||
|
- FIN-18 severity downgraded from the frontend auditor's High to Medium after adversarial review (no visible break for a single user; silent divergence self-heals on reload).
|
||||||
38
audit-results/COVERAGE_MAP.md
Normal file
38
audit-results/COVERAGE_MAP.md
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
# Coverage Map
|
||||||
|
|
||||||
|
Mapping of repository areas → audit status → specialist report reference. Statuses: done / done+verified (parent-verified with runtime evidence) / partial.
|
||||||
|
|
||||||
|
| Area | Files | Auditor | Status | Report ref |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| Prisma schema/migrations/transactions/cascades | prisma/schema.prisma, prisma/migrations/*, prisma.config.ts, src/lib/db.ts, services | DB specialist (task-1) | done+verified (drift reproduced) | sa_20260806_054030_000000000_5ed877563a0d |
|
||||||
|
| Quiz scoring/attempts/history/retakes/progress | src/lib/scoring.ts, quizService, progressService, quizzes/*, api/quizzes/**, api/progress | Quiz specialist (task-2) | done+verified (server scoring path re-checked) | sa_20260806_054030_000000000_e75a8ce575f9 |
|
||||||
|
| Flashcards/SRS/progress/deletion/ordering | flashcards/*, spaced-repetition/*, cardService, deckService, spacedRepetitionService, api/cards|decks|spaced-repetition-sets|progress | Flashcard specialist (task-3) | done | sa_20260806_054030_000000000_a222804c133c |
|
||||||
|
| Auth/protected routes/API authz/sharing | src/proxy.ts, lib/auth.ts, authService, shareService, shareMetadata, api/auth/**, api/share, shared/*, all api routes | Auth specialist (task-4) | done+verified (proxy + fallback secret read directly) | sa_20260806_054030_000000000_aab0c59dbad3 |
|
||||||
|
| Imports/exports/generation/Zod/malformed input | lib/validation/*, jsonRepair, components/import/*, api/decks|quizzes|material-groups|settings | Import specialist (task-5) | done | sa_20260806_054030_000000000_5e72c2713989 |
|
||||||
|
| Material groups/library/drag-drop/deletion/orphans | api/material-groups/**, reorder routes, library pages, classService | Groups specialist (task-6) | done | sa_20260806_054030_000000000_9ae1f3dfa282 |
|
||||||
|
| Frontend state/refresh/races/localStorage/boundaries | all components + pages + shared viewers | Frontend specialist (task-7) | done | sa_20260806_054030_000000000_0e17fd155702 |
|
||||||
|
| Test quality/coverage gaps | *.test.ts, vitest.config.ts, package.json, CI workflow | Test specialist (task-8) | done | sa_20260806_054030_000000000_cd3652deb3f7 |
|
||||||
|
| Docker/startup/env/scripts/production/recovery | Dockerfile, docker-entrypoint.sh, compose files, package.json, prisma.config.ts, next.config.ts, .gitignore, CI | Ops specialist (task-9) | done | sa_20260806_054030_000000000_f49cdaacaabf |
|
||||||
|
| Adversarial review (challenge Critical/High + missed issues) | (all of the above + lightly-covered files) | Adversarial specialist (task-10) | done — 9/10 challenged findings confirmed/weakened, FE-06 refuted, 10 new issues added | sa_20260806_055608_000000000_c550c742d5dc |
|
||||||
|
|
||||||
|
## Parent-level verification performed (see VERIFICATION_LOG.md and audit-results/tmp/)
|
||||||
|
|
||||||
|
- `npx prisma validate` — PASS
|
||||||
|
- `npm test` — PASS (30 tests: 20 arcade + 10 spacedRepetition)
|
||||||
|
- `npm run lint` — 28 baseline problems (9 errors / 19 warnings), all pre-existing
|
||||||
|
- `npm run build` — PASS (route table incl. "ƒ Proxy (Middleware)")
|
||||||
|
- `prisma migrate deploy` on fresh temp DB — PASS (creates DB **missing** MaterialGroup/groupId → FIN-01 drift confirmed)
|
||||||
|
- Temp DB introspection (`audit-results/tmp/inspect-db.cjs`) — `MaterialGroup` table and `groupId` columns absent
|
||||||
|
- Prisma-shaped SQL against temp DB (`audit-results/tmp/reproduce-drift.cjs`) — `no such column: Deck.groupId`, `no such table: MaterialGroup`, `no such column: ShareLink.groupId`
|
||||||
|
- `FINDINGS.json` validity — PASS (53 findings; validator `audit-results/tmp/validate-findings.cjs`)
|
||||||
|
|
||||||
|
## Coverage gaps (areas NOT fully audited)
|
||||||
|
|
||||||
|
1. **Arcade feature internals** — intentionally out of scope (may be removed). Only checked for shared-DB/build/deploy impact (Arcade models/indexes verified in migrations).
|
||||||
|
2. **Real browser interaction** — no browser available in the audit environment: drag-drop, resume flows, share pages, keyboard a11y, theme behavior were code-verified only; findings needing runtime confirmation are flagged in UNVERIFIED_RISKS.md.
|
||||||
|
3. **Container build/run** — docker CLI unavailable: OPS-02/FIN-22, FIN-06, FIN-24 need a container run to observe exact failure modes.
|
||||||
|
4. **git history deep-dive** — limited to drift commit (7af0935), dev.db commits, and migration commits; other commits not diffed line-by-line.
|
||||||
|
5. **Committed dev.db blob contents** — could not be fully enumerated (no sqlite3 CLI; blob grep only; no admin_password hash found in checked blobs).
|
||||||
|
6. **Live dev.db** — none exists in the workspace; runtime behavior on a populated database (perf findings FIN-17) not measured.
|
||||||
|
7. **Network/remote behavior** — the Forgejo remote and CI execution were not reachable; CI findings are static analysis of `.forgejo/workflows/build.yml`.
|
||||||
|
8. **External services** — none (app calls no LLM/third-party APIs).
|
||||||
69
audit-results/EXECUTIVE_SUMMARY.md
Normal file
69
audit-results/EXECUTIVE_SUMMARY.md
Normal file
|
|
@ -0,0 +1,69 @@
|
||||||
|
# Study Desk — Overnight Read-Only Audit: Executive Summary
|
||||||
|
|
||||||
|
Status: **COMPLETE** (2026-08-06)
|
||||||
|
|
||||||
|
## Scope and method
|
||||||
|
|
||||||
|
Comprehensive read-only audit of the Study repository (Next.js 16 / React 19 / Prisma 7 / SQLite, self-hosted single-user study app). Ten specialist subagents audited: (1) Prisma schema/migrations/transactions/cascades/drift, (2) quiz scoring/attempts/retakes/progress, (3) flashcards/SRS/progress/ordering, (4) auth/protected routes/API authorization/sharing, (5) imports/exports/generation/Zod validation, (6) material groups/library/drag-drop/deletion, (7) frontend state/races/localStorage/boundaries, (8) test quality, (9) Docker/env/production/recovery, (10) adversarial review challenging every Critical/High finding and hunting missed issues. The parent agent independently re-verified all Critical/High claims (code tracing, git history, temp-DB migration chain, DB introspection, Prisma-shaped SQL probes). The Arcade feature set was excluded per scope unless it affects the main app, shared dependencies, database integrity, build, or deployment.
|
||||||
|
|
||||||
|
## Constraints honored
|
||||||
|
|
||||||
|
- **No source code, tests, migrations, config, package files, lockfiles, docs, or databases modified.** Final `git status --short`: only untracked `.reasonix/` and `audit-results/`; `git diff --check` clean; `src/generated/prisma/` byte-identical before/after all prisma commands (sha256).
|
||||||
|
- All reports and diagnostic artifacts live under `audit-results/` (reports + `tmp/`: `audit-migration.db`, `inspect-db.cjs`, `reproduce-drift.cjs`, `validate-findings.cjs`, `generated-before.sha256`).
|
||||||
|
- Existing databases: none found in the workspace; the migration-chain test used a brand-new temp DB.
|
||||||
|
|
||||||
|
## Baseline verification
|
||||||
|
|
||||||
|
| Check | Result |
|
||||||
|
|---|---|
|
||||||
|
| `npm test` | ✅ PASS — 30 tests / 6 files (20 arcade out of scope, 10 spacedRepetition) |
|
||||||
|
| `npm run lint` | ⚠️ 28 problems (9 errors, 19 warnings) — all pre-existing baseline |
|
||||||
|
| `npm run build` | ✅ PASS — full route table incl. "ƒ Proxy (Middleware)" |
|
||||||
|
| `npx prisma validate` | ✅ PASS |
|
||||||
|
| Migration chain on fresh temp DB | ✅ PASS — 4/4 migrations apply, but the resulting DB is **missing** `MaterialGroup` + `groupId` columns (drift, see below) |
|
||||||
|
|
||||||
|
## Findings (53 total: 2 Critical, 5 High, 18 Medium, 25 Low, 3 Observation)
|
||||||
|
|
||||||
|
Full detail in `FINDINGS.json` (validated) and `CONFIRMED_FINDINGS.md`.
|
||||||
|
|
||||||
|
### Critical
|
||||||
|
|
||||||
|
1. **FIN-01 — Schema/migration drift (Confirmed, runtime-verified).** `prisma/schema.prisma` defines `MaterialGroup` and `Deck/QuizSet/ShareLink.groupId` (added by commit 7af0935, which also rewrote a committed `dev.db` via `prisma db push`) but **no migration creates them**. Every DB built by `prisma migrate deploy` — the `predev` hook and the Docker entrypoint — lacks these objects; the generated client then fails every Deck/QuizSet/ShareLink/MaterialGroup query (`no such column: Deck.groupId`, `no such table: MaterialGroup` — reproduced against a temp DB). **Any fresh install/deploy is non-functional.** The app only works on the developer's db-push-synced local database.
|
||||||
|
2. **FIN-02 — Hardcoded session-secret fallback (High-confidence inference).** `src/lib/auth.ts:15` and `src/proxy.ts:8-9` fall back to the public constant `"dev-session-secret-change-in-production-must-be-32-chars"` whenever `SESSION_SECRET` is unset — and the shipped `docker-compose.yml` passes `${SESSION_SECRET}`, which is empty by default. Anyone with the source can forge an iron-session cookie (`{isAuthenticated:true, sessionGeneration:1}`) and fully bypass authentication on such deployments.
|
||||||
|
|
||||||
|
### High
|
||||||
|
|
||||||
|
3. **FIN-03 — "Retake Missed" persists as a full attempt with a wrong lower score** (in-viewer retake never sets the `retakeIds` prop; server scores all unanswered questions as 0) and pollutes SEQUENTIAL progress.
|
||||||
|
4. **FIN-04 — Deleting a card mid-session bricks the flashcard study session** (`filterAndClampOrder` is dead code; resume never filters stale ids; no skip UI).
|
||||||
|
5. **FIN-05 — Production compose is unreachable**: container listens on 3726 (`ENV PORT=3726`), compose publishes 3000:3000.
|
||||||
|
6. **FIN-06 — No `.dockerignore`**: Windows-host builds inject win32 native modules (better-sqlite3, argon2) into the Linux image.
|
||||||
|
7. **FIN-07 — `docker-compose.override.yml` auto-merges**: plain `docker compose up` silently runs dev mode in production.
|
||||||
|
|
||||||
|
### Medium (selected)
|
||||||
|
|
||||||
|
First-login provisioning takeover + dead `ADMIN_PASSWORD_HASH` (FIN-08); proxy auth bypass for dot-containing paths (FIN-09); password-reset abuse (FIN-10); unvalidated attempt `answersJson` (FIN-11); finish double-submit duplicates attempts (FIN-12); progress PATCH/DELETE race (FIN-13); unsequenced progress autosaves (FIN-14); completed session resumes on last card (FIN-15); SRS review endpoint lacks due/new-card-limit enforcement (FIN-16); unbounded DB scans (FIN-17); library optimistic mutations without rollback (FIN-18); unvalidated `groupId`/`sortOrder` incl. cross-class group membership (FIN-19); shared-quiz reshuffle on re-render (FIN-20); unvalidated `/api/progress` + unguarded `JSON.parse` crashes (FIN-21); entrypoint migrate fragility (FIN-22); non-reproducible installs (FIN-23); no healthcheck/backup (FIN-24); CI ships without tests/lint/prisma/smoke gates (FIN-25).
|
||||||
|
|
||||||
|
### Low / Observation
|
||||||
|
|
||||||
|
25 Low (e.g. SATA div-by-zero NaN, historical score recompute, card-edit endpoints without Zod, no import size caps, sortOrder duplicates after group deletion, keyboard cross-group moves impossible, restart DELETE race, P2002 → 500, git history containing `dev.db`, `.gitignore` gaps, dashboard fetch crash, empty slug from `slugify`, shared-viewer localStorage key collisions) and 3 Observations (group sortOrder desc convention, collapse flash, shared quizzes ship the answer key by design).
|
||||||
|
|
||||||
|
## Verification highlights
|
||||||
|
|
||||||
|
- FIN-01 disproved attempts: checked whether the client avoids selecting `groupId` (it doesn't — generated `Deck.ts` includes it in every payload), whether pages swallow errors (they don't), and whether any migration creates the objects (grep = 0 hits). Confirmed via live probes instead: see `audit-results/tmp/reproduce-drift.cjs` output in VERIFICATION_LOG.md.
|
||||||
|
- Adversarial review **refuted** the frontend auditor's "no auth on API routes" claim (proxy is compiled middleware — verified in `.next` artifacts), **weakened** the entrypoint crash-loop framing (fresh-volume `migrate deploy` succeeds), and downgraded the optimistic-reorder severity. It independently confirmed all other Critical/High findings and added 10 missed issues, merged into the final list.
|
||||||
|
|
||||||
|
## Repository areas NOT fully audited
|
||||||
|
|
||||||
|
1. Arcade feature internals (out of scope; shared-DB/build/deploy impact only — verified in migrations).
|
||||||
|
2. Real browser interaction (no browser available): drag-drop, resume flows, share pages, keyboard a11y, themes — code-verified; flagged for runtime confirmation in UNVERIFIED_RISKS.md.
|
||||||
|
3. Container build/run (docker CLI unavailable): FIN-06, FIN-22, FIN-24 need a container run.
|
||||||
|
4. Deep git-history diff of every commit; full content audit of the committed `dev.db` blobs (no sqlite3 CLI; no credentials found in the checked blobs).
|
||||||
|
5. Runtime performance on a populated database (FIN-17 not measured).
|
||||||
|
6. CI/remote execution (Forgejo pipeline analyzed statically).
|
||||||
|
|
||||||
|
## Recommended priority (when fixes are authorized)
|
||||||
|
|
||||||
|
1. **FIN-01** — generate and commit the missing migration (`prisma migrate dev --name add_material_groups`); add a CI drift gate. Everything else depends on a working fresh deployment.
|
||||||
|
2. **FIN-02 / FIN-05 / FIN-06 / FIN-07** — harden the deployment path: require `SESSION_SECRET`, fix the port mapping, add `.dockerignore`, rename the dev override, and smoke-test the image in CI.
|
||||||
|
3. **FIN-03 / FIN-04** — the two most user-visible application bugs (wrong retake scores; stuck flashcard sessions).
|
||||||
|
4. Then the Medium cluster (validation, races, SRS enforcement, progress handling) and finally Low items + TEST_GAPS.md.
|
||||||
1024
audit-results/FINDINGS.json
Normal file
1024
audit-results/FINDINGS.json
Normal file
File diff suppressed because it is too large
Load diff
15
audit-results/REJECTED_FINDINGS.md
Normal file
15
audit-results/REJECTED_FINDINGS.md
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
# Rejected Findings
|
||||||
|
|
||||||
|
Findings that were investigated and disproven, or removed as duplicate/stylistic/insignificant, with the reason.
|
||||||
|
|
||||||
|
| ID | Original claim | Rejection reason |
|
||||||
|
|---|---|---|
|
||||||
|
| FE-06 | "All `/api/*` routes are unauthenticated (no middleware, no per-route checks) — curl without a session cookie returns data" | **Refuted by adversarial review + parent verification.** `src/proxy.ts` IS the compiled Next.js 16 middleware (verified in `.next/server/functions-config-manifest.json`, `middleware.js`, and the compiled chunk; build output shows "ƒ Proxy (Middleware)"). It decrypts the iron-session cookie and enforces `isAuthenticated` + `sessionGeneration` on every path except `/login`, `/api/auth`, `/shared`, `/_next*`, `/favicon*`, and paths containing `.`. The auditor searched for `middleware.ts` (the Next 15 name) and missed `proxy.ts`. The residual truth is the dot-bypass (FIN-09). The claim that "no frontend code handles 401s because there is no auth" is also wrong — there is no 401 because the proxy redirects to /login. |
|
||||||
|
| QUIZ-06 (standalone) | "Quiz resume with stale question ids crashes the viewer" | Merged into FIN-04. The quiz-side path is currently unreachable (no question editing/deletion exists in the app; quiz deletion cascades progress), so it is a latent variant of the same root cause (dead `filterAndClampOrder`), not a separate live defect. |
|
||||||
|
| OPS-02 (original framing) | "Fresh-volume deployment crash-loops because `migrate deploy` fails" | **Weakened by adversarial review.** On a fresh volume `prisma migrate deploy` *succeeds* (verified against a temp DB); the fresh-volume failure is FIN-01 (schema drift → app 500s while the container runs). The crash loop requires a secondary trigger (db-push DB, tampering, lock). Reframed as FIN-22 with the crash-loop path demoted. |
|
||||||
|
| FE-01 (original severity) | Optimistic drag-drop reorder without rollback rated High | **Severity downgraded to Medium by adversarial review.** Mechanics confirmed (fire-and-forget fetch, in-place mutation, origin group omitted from cross-group payload) but impact analysis: sortOrder gaps only, silent divergence self-heals on reload, no visible break for a single user. Merged into FIN-18. |
|
||||||
|
| AUTH-06 (standalone) | "Cross-class group membership via reorder API exposes another class's content" | Merged into FIN-19 (same root cause: `groupId`/`sortOrder` never validated server-side). The sharing-exposure angle is recorded as impact of FIN-19. |
|
||||||
|
| DBAUD-04 | "Relative SQLite path resolution may cause dev dual-DB split-brain" | **Disproven.** CLI (`prisma.config.ts`) and app adapter both resolve `file:./dev.db` cwd-relative; `predev` and the app run from the project root → same file. Docker uses absolute `file:/app/data/study.db`. Recorded as verified-safe in UNVERIFIED_RISKS.md. |
|
||||||
|
| DBAUD-05/06/07, QUIZ-08-verdicts, GRP-01-verdicts (various) | "Cascades broken / transactions partial / StudyProgress duplicates via NULL tricks / id regeneration on card edit / XSS via markdown / client-server boundary violations" | **Disproven** by the respective auditors with evidence: FKs enforced by compile default; nested creates implicit-transactional; service-level guards block duplicates; PATCH edits in place (ids preserved); no `rehype-raw`; no server-module imports in client components. Recorded as verified-safe. |
|
||||||
|
| FE-11 (as bug) | "Collapsed-groups expand flash" | Retained only as Observation (FIN-51) — cosmetic, no data impact. |
|
||||||
|
| Various | Style preferences, harmless duplication, generic best practices, speculative micro-optimizations, Arcade-only issues | Excluded per audit scope rules. |
|
||||||
340
audit-results/REMEDIATION_PLAN.md
Normal file
340
audit-results/REMEDIATION_PLAN.md
Normal file
|
|
@ -0,0 +1,340 @@
|
||||||
|
# Study Desk audit evaluation and remediation plan
|
||||||
|
|
||||||
|
Date: 2026-08-06
|
||||||
|
Scope: independent evaluation of `CONFIRMED_FINDINGS.md`, `FINDINGS.json`, the supporting audit reports, and the current repository source.
|
||||||
|
Change policy for this evaluation: this plan is the only new file; no application source, tests, migrations, configuration, generated files, database, or existing audit artifact was changed.
|
||||||
|
|
||||||
|
## Executive verdict
|
||||||
|
|
||||||
|
The audit is credible and the central findings hold up against the current source. The most urgent problems are not cosmetic:
|
||||||
|
|
||||||
|
1. The committed migration chain does not create `MaterialGroup` or the three `groupId` columns, while the generated Prisma client selects those fields. A database created only with `prisma migrate deploy` is incompatible with the app.
|
||||||
|
2. Production authentication can use a public fallback session secret, making a deployment with an empty `SESSION_SECRET` forgeable.
|
||||||
|
3. The in-viewer quiz retake flow loses the retake scope and records the retake as a full, incorrectly scored attempt.
|
||||||
|
4. Flashcard resume trusts stale IDs and unguarded JSON, so deletion or malformed progress can leave the study viewer blank or crash it.
|
||||||
|
5. The production container path is presently unreliable: the port mapping is wrong without the development override, the conventional override silently changes normal Compose behavior, the build context can include host `node_modules`, and the runner installs a ranged Prisma CLI outside the lockfile.
|
||||||
|
|
||||||
|
I agree with most of the remaining findings as defects or resilience gaps, but not always with their severity or proposed remedy. In particular:
|
||||||
|
|
||||||
|
- FIN-04 needs more than wiring in the existing `filterAndClampOrder`: that helper does not preserve the logical current card when a stale ID occurs before the saved index, and it turns a completed index back into the last card.
|
||||||
|
- FIN-05 should normally use `3000:3726`, preserving the documented host port while mapping it to the actual container port.
|
||||||
|
- FIN-22 should not be fixed with an automatic migration retry loop. Schema conflicts should fail fast with actionable diagnostics and a documented recovery path.
|
||||||
|
- FIN-27 is more important than Low because changing quiz content can rewrite the meaning of persisted history.
|
||||||
|
- FIN-34 is only partially present: the dedicated deck page already awaits both DELETE requests, but the library start/restart path fires them without waiting.
|
||||||
|
- FIN-38 is not currently a defect for this Arizona-based installation. It becomes a portability requirement only if the app is meant to support a configurable local day boundary.
|
||||||
|
- FIN-42 does not justify a destructive history rewrite without evidence that the old database contained sensitive data and without coordinating every clone/remote.
|
||||||
|
- FIN-50 is an intentional convention. Newest named groups must remain first and Uncategorized must remain last; validation should make that convention harder to corrupt.
|
||||||
|
- FIN-52 is an architectural consequence of local grading in an anonymous read-only viewer. It merits a sharing warning, not removal of the answer key unless the product adopts server-side anonymous grading.
|
||||||
|
|
||||||
|
## Finding-by-finding disposition
|
||||||
|
|
||||||
|
Priority definitions: P0 = release/deployment blocker; P1 = next remediation batch; P2 = important hardening; P3 = worthwhile backlog; Conditional = verify a stated condition before changing behavior; No separate fix = intentional behavior or covered by another item.
|
||||||
|
|
||||||
|
| ID | Verdict | Worth fixing? | Priority and disposition |
|
||||||
|
|---|---|---:|---|
|
||||||
|
| FIN-01 | Agree | Yes | **P0.** Add the missing migration plus a safe adoption runbook for already-`db push`-synchronized databases. |
|
||||||
|
| FIN-02 | Agree, conditional on an unset/blank production secret | Yes | **P0.** Production must fail before serving requests; development may use an explicitly development-only value. |
|
||||||
|
| FIN-03 | Agree | Yes | **P1.** Make attempted-question scope explicit and derive partial-retake status on the server. |
|
||||||
|
| FIN-04 | Agree; proposed helper is insufficient as written | Yes | **P1.** Normalize order, index, and results while preserving the logical current card and completed state. |
|
||||||
|
| FIN-05 | Agree | Yes | **P0.** Map host 3000 to container 3726 and smoke-test it. |
|
||||||
|
| FIN-06 | Agree with the mechanics; exact runtime failure is unconfirmed | Yes | **P0.** Add `.dockerignore` and prove native modules load in the Linux image. |
|
||||||
|
| FIN-07 | Agree | Yes | **P0.** Rename the development override and document explicit dev/prod commands. |
|
||||||
|
| FIN-08 | Partly agree: first-login setup is intentional, accidental exposure and dead config are real | Yes | **P1.** Honor a pre-provisioned admin hash or require an explicit one-time setup mode. |
|
||||||
|
| FIN-09 | Agree; latent with current IDs | Yes | **P1.** Replace the broad dot bypass with exact public/static path rules. |
|
||||||
|
| FIN-10 | Agree | Yes | **P1.** Prevent token overwrite, remove spoofable per-IP trust, and make reset initiation a local/admin operation. |
|
||||||
|
| FIN-11 | Agree | Yes | **P1.** Validate the whole attempt envelope, question scope, option ownership, array uniqueness, and types. |
|
||||||
|
| FIN-12 | Agree | Yes | **P1.** Add an in-flight guard, disable Finish, and surface failure/retry state. |
|
||||||
|
| FIN-13 | Agree as a credible timing race | Yes | **P1.** Persist the attempt and clear full-attempt progress in one server transaction. |
|
||||||
|
| FIN-14 | Agree as a credible timing race | Yes | **P2.** Add session identity and monotonic revisions, not only client debouncing. |
|
||||||
|
| FIN-15 | Agree | Yes | **P1.** Persist `currentIndex === order.length` and restore the summary state. |
|
||||||
|
| FIN-16 | Agree; impact is lower in a protected single-user app | Yes | **P2.** Accept only the queue-eligible card, including the intended learn-ahead rule. |
|
||||||
|
| FIN-17 | Agree that the queries are unbounded; user impact is unmeasured | Conditional | **P3.** Benchmark realistic data first, then optimize queries that exceed the budget. |
|
||||||
|
| FIN-18 | Agree | Yes | **P2.** Stop mutating shared objects, capture the origin group before mutation, await writes, and rollback/refetch on error. |
|
||||||
|
| FIN-19 | Agree | Yes | **P1.** Enforce class/type/group membership and server-owned ordering; retain a share-page defense. |
|
||||||
|
| FIN-20 | Agree | Yes | **P1.** Give the viewer a stable session input so unrelated parent renders cannot reset it. |
|
||||||
|
| FIN-21 | Agree | Yes | **P1.** Validate progress writes and defensively normalize legacy/corrupt progress on reads. |
|
||||||
|
| FIN-22 | Partly agree with the operational gap, not the retry remedy | No separate fix | Fail fast; cover CLI pinning in FIN-23 and recovery/health in FIN-24. Do not retry a conflicting migration automatically. |
|
||||||
|
| FIN-23 | Agree | Yes | **P0.** Use `npm ci` and ship a lockfile-pinned Prisma CLI/runtime path. |
|
||||||
|
| FIN-24 | Agree | Yes | **P1.** Add a schema-aware health check and tested backup/restore instructions. |
|
||||||
|
| FIN-25 | Agree | Yes | **P0/P1.** Gate test, build, Prisma validation/drift, and container smoke. Stage lint until its known baseline is resolved. |
|
||||||
|
| FIN-26 | Agree | Yes | **P1.** Return zero for an invalid zero-correct SATA question and test finite totals. |
|
||||||
|
| FIN-27 | Agree; severity should be Medium | Yes | **P2.** Persist an immutable result/review snapshot for new attempts, with legacy fallback. |
|
||||||
|
| FIN-28 | Agree | Yes | **P2.** Reuse a trimmed card-content schema and reject rather than silently drop incomplete Create-tab rows. |
|
||||||
|
| FIN-29 | Agree | Yes | **P2.** Add reasonable string, array, and request-size limits with clear 400/413 errors. |
|
||||||
|
| FIN-30 | Agree that the contract is inconsistent | Yes | **P2.** Make SATA imports require at least two correct options, matching the shipped generation instructions; preserve legacy stored data. |
|
||||||
|
| FIN-31 | Agree | Yes | **P2.** Validate complete request envelopes and map only actual not-found errors to 404. |
|
||||||
|
| FIN-32 | Agree | Yes | **P2.** Delete/reassign/renumber in one transaction and add deterministic item tie-breaks. |
|
||||||
|
| FIN-33 | Agree | Yes | **P2.** Add an explicit keyboard-accessible “Move to group” action; do not depend on drag gestures. |
|
||||||
|
| FIN-34 | Partly agree | Yes | **P2.** Fix the library path that does not await deletion; make deletion session-aware with FIN-14. |
|
||||||
|
| FIN-35 | Agree | Yes | **P3.** Translate concurrent first-review uniqueness conflicts to 409 and refetch. |
|
||||||
|
| FIN-36 | Agree as a credible timing race | Yes | **P2.** Disable navigation while grading and cancel/ignore stale animation callbacks. |
|
||||||
|
| FIN-37 | Agree | Yes | **P3.** Refresh set/deck membership on focus and after local deck-change events. |
|
||||||
|
| FIN-38 | Factually correct, but Arizona is the current intended boundary | Not now | Keep the existing behavior. Revisit only with an explicit portability requirement and a timezone setting/migration plan. |
|
||||||
|
| FIN-39 | Agree about the lost clear-cookie response; layout impact is overstated because proxy checks generation | Yes | **P2.** Destroy the cookie on the actual redirect response and test its header. |
|
||||||
|
| FIN-40 | Deployment-policy gap, not a bug on intentional HTTP | Yes, as documentation/config | **P1.** Document a TLS proxy profile and `SECURE_COOKIES=true`; do not enable Secure cookies on plain HTTP. |
|
||||||
|
| FIN-41 | Agree | Yes | **P1.** Validate a strict target enum and target existence; reject impossible/null-target rows. |
|
||||||
|
| FIN-42 | Agree that blobs exist; sensitivity is unproven | Conditional | Do not rewrite history now. Inventory the old DB offline before any public remote; rewrite only with explicit coordination if sensitive data is found. |
|
||||||
|
| FIN-43 | Partly agree | Yes | **P3.** Ignore `data/` and production DB sidecars. Confirm ownership, then remove unreferenced CSS scratch files. Do not automatically ignore the audit plan. |
|
||||||
|
| FIN-44 | Agree | Yes | **P2.** Add `res.ok`, error, and retry handling; never present a failed load as an empty dashboard. |
|
||||||
|
| FIN-45 | Agree | Yes | **P3.** Add abort/error handling to settings/share fetches. |
|
||||||
|
| FIN-46 | Agree as a credible timing race | Yes | **P2.** Use `AbortController` or a request generation tied to `classSlug`; remove or bound the module cache. |
|
||||||
|
| FIN-47 | Agree | Yes | **P3.** Show logout failure and retry; do not falsely claim logout if the server call failed. |
|
||||||
|
| FIN-48 | Partly agree | Yes | **P2.** Guarantee a non-empty unique slug. Keep slugs stable on rename unless redirect/history semantics are deliberately designed. |
|
||||||
|
| FIN-49 | Agree; impact is same-browser only | Yes | **P3.** Include the share token in local session keys and migrate/ignore old keys safely. |
|
||||||
|
| FIN-50 | Intentional newest-first convention | No separate fix | Preserve descending named-group order and final Uncategorized placement; restrict arbitrary sort-order writes under FIN-19/31. |
|
||||||
|
| FIN-51 | Agree, cosmetic only | Not now | Defer unless it can be removed without hydration mismatch or new effect-rule violations. |
|
||||||
|
| FIN-52 | Expected local-grading design | Warning only | **P3.** Explain in ShareMenu that recipients can inspect correct answers; do not imply answer secrecy. |
|
||||||
|
| FIN-53 | Agree | Yes | **P1.** Replace the false-confidence assertion and add scoring/scheduling edge cases. |
|
||||||
|
|
||||||
|
## Remediation sequence
|
||||||
|
|
||||||
|
### Phase 0 — safeguards and test foundation
|
||||||
|
|
||||||
|
Do this before behavior or schema changes.
|
||||||
|
|
||||||
|
1. Create a database-backed Vitest harness that always points Prisma at a unique disposable SQLite path. It must refuse to run if the resolved path is `dev.db`, `/app/data/study.db`, or any existing non-test database.
|
||||||
|
2. Add helpers to apply committed migrations to the disposable database and dispose of it after the test process.
|
||||||
|
3. Record the current lint baseline separately. Do not make a failing baseline a nominally “green” CI gate, and do not mix unrelated lint cleanup into P0 fixes.
|
||||||
|
4. Add focused pure tests first for scoring, progress normalization, import schemas, and rate limiting. These give fast feedback before route/service integration tests.
|
||||||
|
5. Before testing any real deployment database, make and verify a restorable backup. No remediation command should run against the only copy.
|
||||||
|
|
||||||
|
Exit criteria:
|
||||||
|
|
||||||
|
- Tests demonstrably create and use only a disposable DB.
|
||||||
|
- A deliberate attempt to point the harness at the real DB aborts.
|
||||||
|
- Existing `npm.cmd test` remains green.
|
||||||
|
|
||||||
|
### Phase 1 — restore a safe, reproducible deployment path
|
||||||
|
|
||||||
|
Addresses FIN-01, FIN-02, FIN-05–10, FIN-23–25, FIN-39, and FIN-40.
|
||||||
|
|
||||||
|
#### 1A. Repair migration drift without breaking already-pushed databases
|
||||||
|
|
||||||
|
1. Generate a new migration; never edit the four applied migrations. It must add:
|
||||||
|
- `MaterialGroup` with its Class cascade foreign key.
|
||||||
|
- nullable `groupId` on Deck and QuizSet with `ON DELETE SET NULL`.
|
||||||
|
- nullable unique `groupId` on ShareLink with `ON DELETE CASCADE`.
|
||||||
|
- the indexes Prisma expects.
|
||||||
|
2. Review the generated SQLite table-rebuild SQL by hand for preserved rows, foreign keys, defaults, and unique indexes.
|
||||||
|
3. Publish a one-time preflight/adoption script that classifies a database as:
|
||||||
|
- migration-tracked and missing the group schema: apply the migration normally;
|
||||||
|
- already schema-pushed and exactly matching the intended DDL: after backup and exact introspection, mark the new migration applied with `prisma migrate resolve --applied`;
|
||||||
|
- partially matching or otherwise inconsistent: stop with diagnostics and require manual recovery; never guess or auto-resolve.
|
||||||
|
4. Keep the adoption operation explicit. The normal entrypoint should not silently mutate migration history based on table existence.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- Fresh empty DB: `prisma migrate deploy` succeeds; `prisma migrate diff --from-migrations ... --to-schema ...` is empty; Deck, QuizSet, ShareLink, and MaterialGroup queries succeed.
|
||||||
|
- Populated pre-group DB: seed classes/decks/quizzes/shares, apply the new migration, and prove every row/count/relationship is preserved.
|
||||||
|
- Disposable `db push`-style DB: introspect exact equivalence, resolve the migration, run deploy again, and prove an empty schema diff.
|
||||||
|
- Partial/conflicting DB: preflight exits nonzero without changing schema or `_prisma_migrations`.
|
||||||
|
|
||||||
|
#### 1B. Make session configuration unforgeable and setup explicit
|
||||||
|
|
||||||
|
1. Centralize session option construction so `src/lib/auth.ts` and `src/proxy.ts` cannot drift.
|
||||||
|
2. Require a nonblank, at-least-32-character production `SESSION_SECRET` at runtime. Do not require or bake the real secret during `docker build`.
|
||||||
|
3. Make the container entrypoint fail before migrations/server startup when the secret is absent or equals the known fallback. Make Compose use required-variable expansion.
|
||||||
|
4. Allow a development-only fallback only when `NODE_ENV !== "production"`, clearly label it non-production, and cover both branches with tests.
|
||||||
|
5. Replace `pathname.includes(".")` with exact public/static rules. Ensure every `/api/*` path other than the intentional auth endpoints remains protected even when the URL contains a dot or encoded dot.
|
||||||
|
6. Construct the unauthenticated redirect first and bind `getIronSession` to that response before `destroy()`, so the returned redirect carries the clearing cookie.
|
||||||
|
7. Honor a valid `ADMIN_PASSWORD_HASH` when the database has no configured password, or require an explicit one-time setup flag/token. The default production state must not let the first remote request select the admin password.
|
||||||
|
8. Prefer a local console/CLI initiation for password reset. If the HTTP request endpoint remains, use a global single-user throttle, never replace an unexpired token, rate-limit verify/complete, and do not trust arbitrary `x-forwarded-for` unless a trusted proxy is explicitly configured.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- Production startup with missing, blank, short, or known fallback secret exits nonzero before serving.
|
||||||
|
- A cookie sealed with the old fallback does not authenticate when a real secret is configured.
|
||||||
|
- Valid login, logout, reset completion, and session-generation invalidation still work.
|
||||||
|
- Dot-containing protected page/API requests redirect or reject; known static assets still load.
|
||||||
|
- A stale-generation response contains a `Set-Cookie` deletion header.
|
||||||
|
- Pre-provisioned hash rejects a different first password and reports setup complete.
|
||||||
|
|
||||||
|
#### 1C. Make the image and Compose definitions deterministic
|
||||||
|
|
||||||
|
1. Add `.dockerignore` for `node_modules`, `.next`, VCS metadata, environment files, databases/data, coverage, temporary caches, and audit scratch artifacts while retaining source, Prisma schema/migrations, lockfile, and public assets.
|
||||||
|
2. Change the dependency stage to `npm ci`.
|
||||||
|
3. Remove `npm install prisma@^7.8.0` from the runner. Ship a Prisma CLI/runtime installed from `package-lock.json` at the exact repository version. If Prisma is needed at runtime, classify it as a runtime dependency and prune/copy dependencies deterministically rather than re-resolving them.
|
||||||
|
4. Map `3000:3726` in production Compose. Keep `PORT=3726` and `EXPOSE 3726` internally unless there is a deliberate decision to standardize everything on 3000.
|
||||||
|
5. Rename `docker-compose.override.yml` to a non-auto-merged development filename such as `docker-compose.dev.yml`. Document exact production and development invocations.
|
||||||
|
6. Add a minimal internal health route that performs schema-aware DB checks (including a query that touches `Deck.groupId` and MaterialGroup), returning no sensitive details. The container health check must call the actual internal port.
|
||||||
|
7. Add a lock-safe SQLite backup command using the SQLite backup API, plus a documented restore drill. A raw copy of only the main DB while WAL writes are active is not an acceptable backup procedure.
|
||||||
|
|
||||||
|
Verification:
|
||||||
|
|
||||||
|
- `docker compose -f docker-compose.yml config` shows the production command, production environment, and `3000:3726` only.
|
||||||
|
- The explicit development Compose command shows dev mode and source mounts.
|
||||||
|
- Build on a Windows-host context and run on Linux; requiring `better-sqlite3` and `argon2` succeeds in the final image.
|
||||||
|
- Fresh-volume container becomes healthy and `/login` returns 200 through host port 3000.
|
||||||
|
- A migration-drifted disposable volume stays unhealthy with an actionable log.
|
||||||
|
- Backup a populated disposable volume, destroy the disposable container/volume, restore it, and compare row counts plus representative content.
|
||||||
|
|
||||||
|
#### 1D. Add CI gates in a sequence that can actually be green
|
||||||
|
|
||||||
|
1. Gate `npm ci`, `npm test`, Prisma validate, migration/schema drift, and `npm run build` immediately.
|
||||||
|
2. Build the production image, start it on a fresh disposable volume with a generated CI secret, wait for health, and smoke `/login` plus setup status.
|
||||||
|
3. Tag immutable images by commit SHA; optionally move the mutable `latest` tag only after all gates pass.
|
||||||
|
4. Add strict lint only after the existing 9-error baseline is resolved in an explicit cleanup or after a changed-files lint gate is implemented. Never report baseline lint as passing.
|
||||||
|
|
||||||
|
### Phase 2 — make quiz attempts authoritative and immutable
|
||||||
|
|
||||||
|
Addresses FIN-03, FIN-11–13, FIN-20, FIN-26, FIN-27, and FIN-53.
|
||||||
|
|
||||||
|
1. Replace the client-controlled `isPartialRetake` contract with an explicit ordered `questionIds` scope. The server validates that IDs are unique and belong to the quiz, then derives whether the attempt is partial.
|
||||||
|
2. Accept a structured answers object at the route boundary and persist only canonical server-serialized JSON. Validate that:
|
||||||
|
- each key is in the attempted scope;
|
||||||
|
- every selected option belongs to that question;
|
||||||
|
- selected IDs are unique;
|
||||||
|
- multiple-choice has at most one selection;
|
||||||
|
- missing answers score zero rather than silently disappearing.
|
||||||
|
3. Make the in-viewer “Retake Missed” and “Retake Full Quiz” set explicit local attempt scope. Do not infer retake state from the original prop after the viewer has transitioned internally.
|
||||||
|
4. Add an `isFinishing` state/ref, disable the Finish button during submission, and show a retryable error without discarding answers.
|
||||||
|
5. Move attempt creation and full-attempt SEQUENTIAL-progress deletion into one service transaction. Partial retakes must not overwrite or delete a full in-progress session.
|
||||||
|
6. Guard zero-correct SATA scoring with zero points and deduplicate selections defensively in the pure scorer even though the route also validates them.
|
||||||
|
7. Add an optional immutable review snapshot to QuizAttempt for new attempts: attempted question/order, prompt/category/rationale, options and correctness, selections, and per-question points. Render new history from that snapshot; keep a clearly tested legacy fallback for old rows.
|
||||||
|
8. Stabilize QuizViewer initialization. A parent render that changes topics or another surrounding control must not regenerate option order, clear answers, or reset the index. A deliberate restart/retake must use a new session key.
|
||||||
|
|
||||||
|
Focused verification:
|
||||||
|
|
||||||
|
- Full 5-question attempt scores out of 5 and clears only its matching full progress row.
|
||||||
|
- Two-question retake scores out of 2, is stored as partial, and leaves full progress untouched.
|
||||||
|
- An unanswered question in the explicit scope scores zero and remains in the review snapshot.
|
||||||
|
- Duplicate/unknown question or option IDs, wrong shapes, and client-forged partial flags return 400 without creating an attempt.
|
||||||
|
- Delayed double-click Finish produces exactly one attempt.
|
||||||
|
- Changing or deleting current quiz content after an attempt does not change the stored score, category breakdown, or review copy for snapshot-backed attempts.
|
||||||
|
- Parent rerender preserves question order, option order, current index, and answers; explicit restart changes only the intended session.
|
||||||
|
|
||||||
|
### Phase 3 — make progress and flashcard resume race-safe
|
||||||
|
|
||||||
|
Addresses FIN-04, FIN-14, FIN-15, FIN-21, FIN-34, and FIN-36.
|
||||||
|
|
||||||
|
1. Replace or correct `filterAndClampOrder` with a normalization function that accepts saved order, live IDs, saved index, results/answers, and completion state. It must:
|
||||||
|
- remove stale and duplicate IDs;
|
||||||
|
- preserve the saved current ID when it still exists;
|
||||||
|
- when it was deleted, select the next surviving saved card, otherwise the previous survivor;
|
||||||
|
- preserve `index === oldOrder.length` as completion by returning `index === newOrder.length`;
|
||||||
|
- filter result/answer keys to live IDs;
|
||||||
|
- safely fall back when any JSON field is malformed.
|
||||||
|
2. Apply normalization before any viewer state initializer calls `JSON.parse`. Treat corrupt legacy progress as recoverable, show a small restore warning, and allow a fresh start.
|
||||||
|
3. Add Zod schemas for GET/PATCH/DELETE progress inputs, enum values, nonnegative index, content-specific JSON shape, and referenced content existence.
|
||||||
|
4. Add `sessionId` and monotonic `revision` to progress. The server accepts only a newer revision for the same session; a stale session cannot overwrite or delete a newer session. Serialize client saves but retain server-side revision enforcement because request completion order is not guaranteed.
|
||||||
|
5. On the last flashcard, persist `currentIndex = order.length`. Restoring that row must show the existing completion summary, not the last card.
|
||||||
|
6. Await restart deletion before remount/navigation. Make DELETE conditional on the session being cleared so a late old request cannot delete fresh progress.
|
||||||
|
7. Disable Previous, shuffle, and repeated grading during the 350 ms transition, and clear/cancel the timer on restart/unmount.
|
||||||
|
|
||||||
|
Focused verification:
|
||||||
|
|
||||||
|
- Stale ID before, at, and after the current index all resume on the correct logical card.
|
||||||
|
- All saved cards deleted yields a clear empty/completed state rather than a blank viewer.
|
||||||
|
- Completed sessions reopen on the summary; restart opens card 1.
|
||||||
|
- Invalid JSON and invalid PATCH payloads produce a safe UI fallback or 400, never a render crash/500.
|
||||||
|
- Deliver revisions 3, 1, and 2 in that order; the DB retains revision 3.
|
||||||
|
- Delay an old-session DELETE until after a new-session PATCH; the new session remains.
|
||||||
|
- Grade then immediately try Previous/restart; no index jump or stale result write occurs.
|
||||||
|
|
||||||
|
### Phase 4 — enforce group, ordering, and sharing invariants on the server
|
||||||
|
|
||||||
|
Addresses FIN-18, FIN-19, FIN-32, FIN-33, FIN-41, FIN-49, FIN-50, and FIN-52.
|
||||||
|
|
||||||
|
1. Move reorder logic into focused services. The route must identify the owning class and content type from the database, not trust the client.
|
||||||
|
2. Validate every target group exists, belongs to the same class, and matches DECK/QUIZ type. Reject duplicate IDs, foreign-class IDs, type mismatches, and unknown IDs.
|
||||||
|
3. Prefer an ordered list/group assignment contract and compute contiguous `sortOrder` values server-side. If a complete-set contract is required, verify completeness explicitly.
|
||||||
|
4. In the library, capture origin group before any update, update state immutably, await `res.ok`, disable conflicting mutations in flight, and rollback/refetch on failure.
|
||||||
|
5. Delete a group in one transaction: capture affected items, delete/reassign through the FK behavior, then renumber Uncategorized items deterministically. Add `createdAt`/`id` tie-breakers to reads so legacy duplicates are stable.
|
||||||
|
6. Add a keyboard-accessible Move-to-group menu on every item. Preserve drag-and-drop for pointer users and keep drag listeners off action controls.
|
||||||
|
7. Validate ShareLink target type and target existence. Enforce exactly one populated target in service logic; consider CHECK constraints in a later SQLite migration after compatibility testing.
|
||||||
|
8. Retain the shared-page class/type/token checks and additionally assert each selected group item has the group’s class and expected content type.
|
||||||
|
9. Pass the share token/session namespace to shared viewers and include it in localStorage keys. Avoid copying an old item-only session into a different token namespace without explicit user confirmation.
|
||||||
|
10. Preserve named-group ordering as `sortOrder desc, createdAt desc`, prepend newly created groups locally, and render Uncategorized last. Remove arbitrary `sortOrder` from the ordinary group rename PATCH contract.
|
||||||
|
11. Add a concise ShareMenu warning that a shared quiz necessarily sends answer/rationale data to the recipient’s browser for local grading.
|
||||||
|
|
||||||
|
Focused verification:
|
||||||
|
|
||||||
|
- Cross-class and cross-type group assignment returns 400/409 and changes no rows.
|
||||||
|
- A failing reorder restores/refetches the visible order.
|
||||||
|
- Deleting a group yields unique contiguous Uncategorized item order and does not delete decks/quizzes.
|
||||||
|
- Newer named groups stay above older groups; Uncategorized remains last in quizzes, flashcards, and import selectors.
|
||||||
|
- A keyboard-only user can move an item between two groups and hear/see confirmation.
|
||||||
|
- Invalid share target types and missing IDs create zero rows.
|
||||||
|
- A tampered group share cannot render a foreign-class item.
|
||||||
|
- Two tokens for the same content maintain independent local sessions.
|
||||||
|
|
||||||
|
### Phase 5 — validation, SRS integrity, and user-visible error handling
|
||||||
|
|
||||||
|
Addresses FIN-16, FIN-28–31, FIN-35, FIN-37, FIN-43–48, and the measurable part of FIN-17.
|
||||||
|
|
||||||
|
1. Reuse shared Zod schemas for card create/edit, deck/quiz create/edit, classes, imports, and material groups. Trim before minimum checks and add documented maximums for names, descriptions, Markdown content, options, questions, and cards.
|
||||||
|
2. Reject a Create-tab submission containing any partially filled card; identify the row(s) instead of silently dropping them.
|
||||||
|
3. Enforce at least two correct options for imported SATA questions while leaving existing stored questions readable. Add a targeted message explaining how to repair invalid generated JSON.
|
||||||
|
4. Narrow Prisma error mapping: P2025 becomes 404, P2002 becomes 409 where appropriate, validation is 400/422, and unexpected failures remain 500 with non-sensitive server diagnostics.
|
||||||
|
5. In SRS review, verify the submitted card is the queue-eligible card for the current set/time/new-card allowance (including deliberate learn-ahead). Retain the existing state-version comparison. Map concurrent first-review P2002 to 409 and return/refetch current study state.
|
||||||
|
6. Refresh SRS membership on focus and after same-tab deck changes.
|
||||||
|
7. Add abortable, checked fetch helpers or a small consistent pattern for Dashboard, GenerateTab, ShareMenu, logout, and class library requests. Distinguish loading, empty, error, and retry states.
|
||||||
|
8. Tie library responses to the active `classSlug`; abort or ignore stale responses. Bound or remove module-level caches that can outlive their class data.
|
||||||
|
9. Guarantee class slug generation produces a non-empty unique slug, for example a stable `class-<short-id>` fallback. Keep existing slugs stable on rename.
|
||||||
|
10. Add `data/`, `study.db`, and sidecars to `.gitignore`. Confirm `out.css`, `temp.css`, and `test.css` are unreferenced scratch artifacts before removing them in a separate cleanup change.
|
||||||
|
11. For performance, first seed a disposable benchmark DB approximating expected upper use (for example 100k activity rows, 10k cards, and several SRS sets). Add date predicates/aggregation and narrower SRS queries only where the measured request/poll budget is exceeded. Preserve long-streak correctness when limiting the activity window.
|
||||||
|
|
||||||
|
Focused verification:
|
||||||
|
|
||||||
|
- Whitespace-only and oversized content is rejected with an actionable client error and no partial write.
|
||||||
|
- SATA 1-correct import fails; 2+-correct passes; multiple-choice still requires exactly one.
|
||||||
|
- SRS cannot review a not-due/non-selected/new-limit-exhausted card; valid due and learn-ahead reviews still work.
|
||||||
|
- Concurrent first review produces one success and one handled conflict, not a generic 500.
|
||||||
|
- Focus refresh removes a deleted deck from the SRS set UI.
|
||||||
|
- Failed dashboard/settings/share/logout requests have visible, retryable states and no unhandled promise rejection.
|
||||||
|
- Rapid A→B class navigation cannot render A data under B.
|
||||||
|
- Creating a class named only punctuation yields a navigable, unique slug; rename does not break the old URL.
|
||||||
|
- Performance tests record query counts and latency before/after and prove the 53-week display plus longer current streak remain correct.
|
||||||
|
|
||||||
|
## Cross-cutting release verification
|
||||||
|
|
||||||
|
Run after every phase, with focused tests first and the full suite last.
|
||||||
|
|
||||||
|
Automated gates:
|
||||||
|
|
||||||
|
1. `npm.cmd test`
|
||||||
|
2. `npx.cmd prisma validate`
|
||||||
|
3. migration deploy + empty migration/schema diff against a fresh disposable DB whenever schema changes
|
||||||
|
4. `npm.cmd run build`
|
||||||
|
5. `git diff --check`
|
||||||
|
6. production image build, fresh-volume health wait, and HTTP smoke when Docker/deployment files change
|
||||||
|
7. lint against changed files; full `npm.cmd run lint` only becomes blocking after the known baseline is cleared
|
||||||
|
|
||||||
|
Manual browser matrix for affected phases:
|
||||||
|
|
||||||
|
1. Authenticated routes and corresponding public/shared routes.
|
||||||
|
2. Fresh, resumed, completed, restarted, partial-retake, and stale/deleted-content sessions.
|
||||||
|
3. Loading, empty, server-error, malformed-persisted-data, and retry states.
|
||||||
|
4. Light and dark themes.
|
||||||
|
5. Narrow mobile viewport with no horizontal overflow.
|
||||||
|
6. Keyboard-only navigation, including group moves and all modified icon controls.
|
||||||
|
7. Throttled-network checks for Finish, autosave, restart, class navigation, reorder, and logout.
|
||||||
|
8. Two-tab checks for progress revisions, SRS conflicts, and focus refresh.
|
||||||
|
|
||||||
|
Release/rollback rules:
|
||||||
|
|
||||||
|
- Back up and restore-test the SQLite database before the first migration-bearing release.
|
||||||
|
- Deploy schema and code as one versioned release; do not run a newer generated Prisma client against an older database.
|
||||||
|
- Keep the prior image and verified pre-migration backup until post-deploy smoke and representative data checks pass.
|
||||||
|
- If migration preflight sees a partial or unknown schema state, stop. Do not run `db push`, edit an applied migration, reset the database, or auto-mark the migration applied.
|
||||||
|
|
||||||
|
## Explicitly deferred or rejected work
|
||||||
|
|
||||||
|
- **FIN-38:** no timezone-setting work without a product requirement to support a non-Arizona study day.
|
||||||
|
- **FIN-42:** no history rewrite without an offline sensitivity review and explicit remote/clone coordination.
|
||||||
|
- **FIN-51:** no cosmetic state-initialization change unless it avoids both hydration mismatch and the existing effect-rule class of lint failures.
|
||||||
|
- **FIN-22 retry proposal:** no automatic migration retry/backoff for schema conflicts; fail fast with health diagnostics and the adoption/recovery runbook.
|
||||||
|
- No changes to the audit’s already rejected hypotheses (global API auth absence, split-brain SQLite paths, broken cascades/transactions, Markdown XSS, or client/server import violations) unless new evidence appears.
|
||||||
|
|
||||||
|
## Recommended delivery slices
|
||||||
|
|
||||||
|
Keep the implementation reviewable rather than landing all remediation at once:
|
||||||
|
|
||||||
|
1. **Release blocker:** FIN-01/02/05/06/07/23/25 plus migration adoption, container smoke, and backup prerequisites.
|
||||||
|
2. **Quiz correctness:** FIN-03/11/12/13/20/26/27/53.
|
||||||
|
3. **Progress integrity:** FIN-04/14/15/21/34/36.
|
||||||
|
4. **Group/share integrity:** FIN-18/19/32/33/41/49/50/52.
|
||||||
|
5. **Auth recovery and deployment posture:** FIN-08/09/10/24/39/40.
|
||||||
|
6. **Validation, SRS, UI resilience, and measured performance:** the remaining accepted items.
|
||||||
|
|
||||||
|
Each slice should be independently releasable, have its own focused regression tests, and finish with the cross-cutting gates above.
|
||||||
20
audit-results/SUBAGENT_REPORTS.md
Normal file
20
audit-results/SUBAGENT_REPORTS.md
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
# Subagent Reports
|
||||||
|
|
||||||
|
Full specialist reports were produced by 10 read-only subagents. Each returned a structured markdown report with a findings table and per-finding detail blocks (ID, severity, confidence, files:lines, scenario, expected/actual, root cause, impact, evidence, test coverage, fix direction, complexity, regression test, runtime-confirmation flag). The parent agent re-verified the Critical/High claims directly (see VERIFICATION_LOG.md) and consolidated/deduplicated into `FINDINGS.json` (53 findings).
|
||||||
|
|
||||||
|
## Report index (all reports were reviewed in full by the parent)
|
||||||
|
|
||||||
|
| # | Auditor | Ref | Key output |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 1 | DB / migrations / transactions / cascades / drift | sa_20260806_054030_000000000_5ed877563a0d | **DBAUD-01 Critical migration drift** (MaterialGroup/groupId never migrated; commit 7af0935 db-push artifact); DBAUD-02 junk ShareLink rows; DBAUD-03 dev.db in git history; verified-safe: cascades, transactions, path resolution, single-row AuthSecurity |
|
||||||
|
| 2 | Quiz scoring / attempts / retakes / progress | sa_20260806_054030_000000000_e75a8ce575f9 | **QUIZ-01 High retake-scoring bug** (in-viewer retake never sets retakeIds); QUIZ-02..09 (unvalidated answersJson, double-submit, progress races, dead filter, div-by-zero, historical recompute, multi-tab) |
|
||||||
|
| 3 | Flashcards / SRS / progress / deletion / ordering | sa_20260806_054030_000000000_a222804c133c | CARD-01..11: dead stale-id filter (stuck sessions), unsequenced saves, completed-session resume, SRS due/limit not enforced, unbounded scans, Arizona day boundary, restart race, P2002, animation race, focus staleness |
|
||||||
|
| 4 | Auth / protected routes / API authz / sharing | sa_20260806_054030_000000000_aab0c59dbad3 | **AUTH-01 Critical hardcoded session secret**; AUTH-02 dot-bypass; AUTH-03 reset abuse; AUTH-04 first-login takeover + dead ADMIN_PASSWORD_HASH; AUTH-05..10 (cookie-clear loss, cross-class group membership, share validation, secure flag, no auth tests, answer-key exposure); verified proxy compiled as middleware |
|
||||||
|
| 5 | Imports / exports / generation / Zod / malformed input | sa_20260806_054030_000000000_5e72c2713989 | IMPT-01..06: dead filterAndClampOrder, unvalidated /api/progress + unguarded JSON.parse, card edit endpoints without Zod, no size caps, SATA doc mismatch, override-field 500s; verified: shared schemas, atomic imports, no XSS, no id regeneration |
|
||||||
|
| 6 | Material groups / ordering / drag-drop / deletion / orphans | sa_20260806_054030_000000000_9ae1f3dfa282 | **GRP-01 Critical drift** (independent confirmation); GRP-02..07: unvalidated groupId/sortOrder, fire-and-forget reorder, sortOrder duplicates after group deletion, keyboard a11y, desc-convention fragility; verified: FKs enforced, uncategorized mapping, cascade safety |
|
||||||
|
| 7 | Frontend state / refresh / races / localStorage / boundaries | sa_20260806_054030_000000000_0e17fd155702 | FE-01..11: optimistic reorder without rollback, CRUD res.ok ignored, shared quiz reshuffle (unstable prop), finish double-submit, progress race, **FE-06 (later refuted — proxy exists)**, dashboard fetch gap, unguarded JSON.parse, effect fetches, cross-class race, collapse flash; verified: shared viewers read-only, localStorage clean, no boundary violations |
|
||||||
|
| 8 | Test quality / coverage / false confidence | sa_20260806_054030_000000000_cd3652deb3f7 | TEST-01..10: false-confidence SRS test, zero tests for scoring/auth/services/routes, no DB harness, CI never runs tests; ranked missing-test gaps by impact |
|
||||||
|
| 9 | Docker / startup / env / scripts / production / recovery | sa_20260806_054030_000000000_f49cdaacaabf | OPS-01..10: port mismatch 3726 vs 3000:3000, entrypoint fragility, session secret in prod path, dead ADMIN_PASSWORD_HASH, non-reproducible installs, no .dockerignore (win32 leak), no healthcheck/backup, override auto-merge, .gitignore gaps, CI without gates; argon2/standalone tracing verified OK |
|
||||||
|
| 10 | Adversarial review | sa_20260806_055608_000000000_c550c742d5dc | Challenged all Critical/High findings: confirmed drift, secret, retake, port, dockerignore, override; weakened OPS-02 (fresh-volume migrate succeeds) and FE-01 severity; **refuted FE-06** (proxy active — verified in compiled .next artifacts); added ADV-01..10 (retake progress pollution, activity scan, logout, dashboard crash, slugify empty slug, unguarded parses, completed-resume, localStorage key collisions, first-login race, progress validation) |
|
||||||
|
|
||||||
|
All reports are preserved in full in the session; their findings are consolidated (deduplicated, source IDs preserved) in `FINDINGS.json` and `CONFIRMED_FINDINGS.md`.
|
||||||
21
audit-results/TEST_GAPS.md
Normal file
21
audit-results/TEST_GAPS.md
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
# Test Gaps
|
||||||
|
|
||||||
|
Current suite: 6 files / 30 tests — 20 in `src/lib/arcade/*` (out of scope) + 10 in `src/lib/spacedRepetition.test.ts` (pure scheduling math only). **No test touches a service, an API route, the database, auth, scoring, imports, or progress.** CI never runs `npm test`. Gaps ranked by user-impact × regression-likelihood.
|
||||||
|
|
||||||
|
| ID | Area | Gap | Risk it would fail to catch | Suggested test |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| TEST-GAP-01 | Auth (FIN-02/08/09/10) | `authService.ts` (login, lockout, first-login provisioning, reset token, generation bump) and session config have zero tests | Session-secret fallback regression, lockout bypass, generation-bump loss (stolen sessions stay valid) | Extract + unit-test `getLockoutDuration` thresholds, `parseResetTokenRecord`; service tests with temp SQLite + fake timers: 5 wrong passwords → 423; locked-out login rejected; success resets counter; `completePasswordReset` bumps generation |
|
||||||
|
| TEST-GAP-02 | Scoring (FIN-03/11/26/27) | `src/lib/scoring.ts` SATA partial credit has zero tests | Wrong grades silently persisted into every attempt — permanent data corruption | `scoreQuestion`: SATA 2/4 correct select both → 1; 1 correct + 1 wrong → 0; over-select → 0; MC multi-select first-wrong → 0; `scoreQuiz` with missing question keys; 0-correct SATA → 0 (not NaN) |
|
||||||
|
| TEST-GAP-03 | Quiz attempt persistence (FIN-03/12/13) | `POST /api/quizzes/[id]/attempt` + partial-retake filter + `progressService.upsertProgress` asymmetry (DECK writes cardResultsJson, QUIZ writes answersJson) have zero tests | Retake-scoring regression (FIN-03 class), duplicate attempts, progress column clobbering | Route/service test: partial retake answering 2 of 5 → maxScore 2, score from those 2; upsert round-trip preserves the respective JSON columns |
|
||||||
|
| TEST-GAP-04 | SRS service (FIN-16/35) | `spacedRepetitionService.ts` (reviewCard optimistic concurrency, findNextCard precedence, removeDeck transaction, addDeck validations) has zero tests — only the pure lib is tested | Double-review stale write, due/new-card precedence break, cross-class membership, P2002 → 500 | Seed class→deck→cards→set on temp DB: stale `expectedStateVersion` → 409; due before new before learn-ahead; new card with limit 0 rejected; removeDeck atomically deletes states |
|
||||||
|
| TEST-GAP-05 | Import/repair pipeline (FIN-28/29/30) | `jsonRepair.ts`, `importSchemas.ts`, `shuffle.ts`/`filterAndClampOrder` have zero tests | AI-import regression (fences/repair), schema constraint drift (MC exactly-one), resume clamp bugs | jsonRepair fence/trailing-comma/broken-JSON cases; schema whitespace/oversize/exactly-one cases; `filterAndClampOrder` stale-id + overflow cases |
|
||||||
|
| TEST-GAP-06 | Rate limiter (FIN-10) | `rateLimiter.ts` has zero tests | Brute-force guard silently disabled or locking the owner out | Fake timers: 10 allowed / 11th blocked with retryAfterMs; window expiry; per-key isolation; custom limits |
|
||||||
|
| TEST-GAP-07 | Integration harness (FIN-01/21/25) | No route tests, no DB-backed tests, no setup file; `src/lib/db.ts` defaults to `file:./dev.db` (a naive service test would touch the real dev DB) | Migration drift (FIN-01) — nothing would catch a schema/migration mismatch; progress/share/attempt route validation gaps | `setupFiles` + `DATABASE_URL=file:<tmp>` + `prisma migrate deploy` in globalSetup; route tests with `NextRequest`: attempt 200/400/404, reviews 409, share unknown token, login 429 |
|
||||||
|
| TEST-GAP-08 | CI gate (FIN-25) | `.forgejo/workflows/build.yml` runs no tests/lint/prisma validate; no coverage config; no drift check | Entire suite can regress with a green pipeline; FIN-01-class drift ships | Add `npm ci && npm test && npm run lint && npx prisma validate` job; `prisma migrate diff --from-migrations --to-schema-datamodel` gate; container smoke (curl /login on fresh volume); coverage floor |
|
||||||
|
| TEST-GAP-09 | False confidence (FIN-53) | `spacedRepetition.test.ts:15-23` "distinct valid state for every rating" never compares ratings | `scheduleRating` collapsing all ratings to one schedule passes the suite | Assert pairwise due ordering AGAIN < HARD < GOOD < EASY and distinct due timestamps; graduated-state lapse case |
|
||||||
|
|
||||||
|
## Test-quality positives (verified)
|
||||||
|
|
||||||
|
- `spacedRepetition.test.ts` is deterministic (fixed `now`, `enable_fuzz: false`, no `Math.random`), tests the real pure module, and asserts exact FSRS intervals.
|
||||||
|
- Arcade tests (out of scope) use seeded shuffles and meaningful assertions.
|
||||||
|
- No test uses wall-clock time, locale, or network — no flakiness found.
|
||||||
35
audit-results/UNVERIFIED_RISKS.md
Normal file
35
audit-results/UNVERIFIED_RISKS.md
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
# Unverified Risks
|
||||||
|
|
||||||
|
Risks that could not be fully confirmed within audit constraints (need a browser, a container run, a live deployment, or timing-dependent reproduction). All are code-verified as plausible; the missing piece is runtime confirmation.
|
||||||
|
|
||||||
|
| ID | Title | Why unverified | Suggested verification |
|
||||||
|
|---|---|---|---|
|
||||||
|
| FIN-02 | Hardcoded fallback `SESSION_SECRET` → session forgery | Code path fully traced (seal/verify semantics verified in iron-session), but no live production deployment exists here to demonstrate an actual forged-cookie login | Boot production build without `SESSION_SECRET`, forge a cookie with the fallback password (iron-webcrypto seal script), assert full access; then with a real secret assert rejection |
|
||||||
|
| FIN-06 | No `.dockerignore` → win32 modules in Linux image | Mechanics verified (PE32+ DLL present in local node_modules; Dockerfile COPY order), exact failure mode (build error vs `ERR_DLOPEN_FAILED`) not observed | Run `docker build .` on this Windows host; inspect `better_sqlite3.node`/argon2 prebuilds in the image (must be ELF) |
|
||||||
|
| FIN-13 | Progress PATCH-after-DELETE race | Timing-dependent; needs throttled network | DevTools throttling: submit final answer + Finish immediately; inspect `StudyProgress` after |
|
||||||
|
| FIN-14 | Out-of-order progress PATCHes | Timing-dependent | DevTools throttling with rapid grading; reload and compare resume point/results |
|
||||||
|
| FIN-22 | Entrypoint migrate crash loop / runner CLI install | Requires container build + run on `node:22-slim` (docker CLI unavailable in audit environment) | `docker build` + fresh-volume run (assert 200 + `_prisma_migrations`); corrupt-volume run (assert actionable failure, not silent loop) |
|
||||||
|
| FIN-24 | Healthcheck/backup gaps | Behavior observable only in a real container run | Compose up with a corrupt volume; observe restart loop and absence of health status |
|
||||||
|
| FIN-32 | Group-delete sortOrder duplicates | Deterministic from code; visual impact needs a browser | Delete a group in the app, inspect Uncategorized order + DB sortOrders |
|
||||||
|
| FIN-33 | Keyboard cross-group moves impossible | Deterministic from code (empty `handleDragOver`), needs manual keyboard test | Tab to a drag handle, attempt keyboard cross-group move |
|
||||||
|
| FIN-34 | Restart DELETE race | Timing-dependent | Throttled network: Restart, grade first card, inspect progress row |
|
||||||
|
| FIN-36 | Previous-card animation race | Timing-dependent (350 ms window) | Grade then immediately click Previous; observe index jump |
|
||||||
|
| FIN-37 | SRS page stale on focus | Deterministic from code; browser needed to observe | Two tabs: delete deck in one, focus the other |
|
||||||
|
| FIN-39 | Proxy destroy cookie lost on redirect | Deterministic from code; browser needed to observe Set-Cookie | Stale-generation cookie → follow redirect → inspect response headers |
|
||||||
|
| FIN-40 | Secure cookie flag off | Deterministic from code (flag logic); deployment-dependent | Inspect `Set-Cookie` in a production container |
|
||||||
|
| FIN-46 | Cross-class fetch race | Timing-dependent | Throttled network + fast class switching |
|
||||||
|
| FIN-48 | Empty slug / stale slug on rename | Deterministic from code (slugify verified); UI behavior needs browser | Create class named "!!!", try to navigate to it; rename a class, check old URL |
|
||||||
|
| FIN-49 | Shared-viewer localStorage key collisions | Code-verified key construction; symptom needs browser | Open two share tokens for the same content, answer in one, reload the other |
|
||||||
|
| FIN-51 | Collapsed-groups expand flash | Cosmetic; browser-only | Visit library with saved collapsed state |
|
||||||
|
|
||||||
|
## Items verified safe (hypotheses disproven — recorded for completeness)
|
||||||
|
|
||||||
|
- SQLite relative-path split-brain between Prisma CLI and app (both cwd-relative → same `dev.db`; Docker uses absolute path) — verified safe.
|
||||||
|
- Foreign-key enforcement (better-sqlite3 compiled with `SQLITE_DEFAULT_FOREIGN_KEYS=1`) — cascades/SetNull fire; verified safe.
|
||||||
|
- Transactionality of deck/quiz/attempt/reorder writes — nested creates + `$transaction`; verified safe.
|
||||||
|
- `StudyProgress`/`ShareLink` NULL-uniqueness — service-level guards prevent duplicates via app paths; only direct API misuse (FIN-41/FIN-21) can create junk rows.
|
||||||
|
- Client/server boundary violations — none found (no client import of `@/lib/db`/services/prisma).
|
||||||
|
- XSS via Markdown — react-markdown without `rehype-raw`; no `dangerouslySetInnerHTML` on user data; verified safe.
|
||||||
|
- Password-reset nonce/token crypto (192-bit token, SHA-256 digest, `timingSafeEqual`, expiry) — sound; abuse vectors are FIN-10.
|
||||||
|
- SRS optimistic concurrency (`expectedStateVersion` → 409, rollback via refetch) — correct.
|
||||||
|
- Migrations are purely additive; no NOT NULL-without-default, no DROP — safe on populated migration-tracked DBs (except FIN-01 drift).
|
||||||
38
audit-results/VERIFICATION_LOG.md
Normal file
38
audit-results/VERIFICATION_LOG.md
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
# Verification Log
|
||||||
|
|
||||||
|
Chronological record of every command run, its outcome, and classification (baseline / environment / possible defect).
|
||||||
|
|
||||||
|
## Environment notes
|
||||||
|
|
||||||
|
- OS: Windows, shell: bash (git-bash)
|
||||||
|
- `node_modules` was **absent** at audit start → `npm test` initially failed with `'vitest' is not recognized`. This is an environment condition, not a repo defect. `npm ci` completed successfully during the first audit session (518 packages present).
|
||||||
|
- No existing application database (`dev.db*`) exists anywhere in the repo — no real user data to protect; the migration-chain test uses a brand-new temp DB under `audit-results/tmp`.
|
||||||
|
- `src/generated/prisma/` (tracked) hashed BEFORE any prisma command: `audit-results/tmp/generated-before.sha256`.
|
||||||
|
|
||||||
|
## Entries
|
||||||
|
|
||||||
|
| # | Time | Command | Result | Classification |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 1 | session start | `git status --short` | clean; only `?? .reasonix/` and `?? audit-results/` untracked | baseline |
|
||||||
|
| 2 | session start | `npm test` (first run, prior session) | FAILED: `'vitest' is not recognized` (no node_modules) | environment |
|
||||||
|
| 3 | session start | `npm ci` (background, prior session) | completed; 518 packages in node_modules | environment |
|
||||||
|
| 4 | 22:36 | `npx prisma validate` | PASS — "The schema at prisma\schema.prisma is valid" (exit 0) | baseline |
|
||||||
|
| 5 | 22:36 | `npm test` | PASS — 6 files, 30 tests, all green (arcade ×20, spacedRepetition ×10) | baseline |
|
||||||
|
| 6 | 22:37 | `npm run lint` | 28 problems: 9 errors, 19 warnings (react-hooks/set-state-in-effect errors + unused-vars warnings). All pre-existing; matches AGENTS.md known baseline. | baseline |
|
||||||
|
| 7 | 22:37 | `npm run build` (background) | PASS — route table emitted, no compile errors | baseline |
|
||||||
|
| 8 | 22:38 | `DATABASE_URL=file:./audit-results/tmp/audit-migration.db npx prisma migrate deploy` | PASS — all 4 migrations applied to brand-new temp DB | baseline |
|
||||||
|
| 9 | 22:38 | `sha256sum -c audit-results/tmp/generated-before.sha256` | PASS — every generated Prisma file byte-identical (all ": OK") | baseline |
|
||||||
|
| 10 | 22:38 | `git status --short` | only untracked `.reasonix/`, `audit-results/`; no tracked modifications | baseline |
|
||||||
|
| 11 | 22:38 | `git diff --check` | clean (no whitespace errors) | baseline |
|
||||||
|
| 12 | 22:40 | `git log --all --oneline -- prisma/migrations/` + `git show 7af0935 --stat` | Confirms drift: commit 7af0935 changed `prisma/schema.prisma` (+31) and `dev.db` (139264→155648 bytes) with **no migration**; `dev.db` present in 8 commits of history | possible defect (FIN-01, FIN-42) |
|
||||||
|
| 13 | 22:41 | `node audit-results/tmp/inspect-db.cjs audit-results/tmp/audit-migration.db` | Temp DB (built solely by `migrate deploy`) has **no** `MaterialGroup` table and **no** `groupId` column on Deck/QuizSet/ShareLink | possible defect (FIN-01) |
|
||||||
|
| 14 | 22:42 | `node audit-results/tmp/reproduce-drift.cjs` | Prisma-shaped queries fail: `no such column: Deck.groupId`, `no such table: MaterialGroup`, `no such column: ShareLink.groupId` — fresh-deploy breakage proven | possible defect (FIN-01) |
|
||||||
|
| 15 | 22:45 | `node --check audit-results/tmp/inspect-db.cjs && node --check audit-results/tmp/reproduce-drift.cjs` | diagnostic scripts syntax-valid | baseline |
|
||||||
|
| 16 | 22:50 | `node audit-results/tmp/validate-findings.cjs` | `FINDINGS.json` VALID JSON — 53 findings: 2 Critical, 5 High, 18 Medium, 25 Low, 3 Observation; no duplicate ids | baseline |
|
||||||
|
| 17 | 23:00 | `git status --short` + `git diff --check` (final) | Only untracked `.reasonix/`, `audit-results/`; no tracked file modified; diff clean | baseline |
|
||||||
|
|
||||||
|
## Classification legend
|
||||||
|
|
||||||
|
- **baseline** — repository behaves as expected; recorded for reference.
|
||||||
|
- **environment** — failure caused by local machine state, not repo code.
|
||||||
|
- **possible defect** — outcome may indicate a repo defect; investigated further in findings.
|
||||||
BIN
audit-results/tmp/audit-migration.db
Normal file
BIN
audit-results/tmp/audit-migration.db
Normal file
Binary file not shown.
26
audit-results/tmp/generated-before.sha256
Normal file
26
audit-results/tmp/generated-before.sha256
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
cb737289f5b5f6cdb8b90fc23d2c89ebd41d6602ce99e88bf6dfd751beab7103 *src/generated/prisma/browser.ts
|
||||||
|
2ea331124cc56c9c9a360b4a85f42f029992811ece7efe9db0ec1300a43f59c3 *src/generated/prisma/client.ts
|
||||||
|
f1f6280ff65c1e8aab5ce30307c8777ce1795f81bf1a513396263fc1c9a30e5c *src/generated/prisma/commonInputTypes.ts
|
||||||
|
8bcc37ae19ee9c55424b735ce5f3a0972b7e741958ef86b350c9a54dc2238a0d *src/generated/prisma/enums.ts
|
||||||
|
d52628a3dc8285d9014e2fda489badce6c88604810e2975310ab6c40ec3c708d *src/generated/prisma/internal/class.ts
|
||||||
|
91794eb9b3c395e57b96d18f2bae895149f7b70ddf0fda8b817e7aa9da17a950 *src/generated/prisma/internal/prismaNamespace.ts
|
||||||
|
6645109792ed08507d79da319aaecc2752bffa6e8bd5a3364dccc5ae93e2f23c *src/generated/prisma/internal/prismaNamespaceBrowser.ts
|
||||||
|
b8ffef1ad4428179847aafe66abee3ad5eb74a84fff46265532551b10ab2537d *src/generated/prisma/models.ts
|
||||||
|
daecbb1c94a96d19da4accfdf81fe8f11fa73c543958016bc7d6060d454f3943 *src/generated/prisma/models/AnswerOption.ts
|
||||||
|
87e18727cec17a341d32709bc0c1602c66e6c6dcc57672b9c9fbe38a6ce12801 *src/generated/prisma/models/ArcadeAttempt.ts
|
||||||
|
0c9de62bdbefafe8b5ff5300d1d764cc73235a077f8e3b3e03e6144287610c9d *src/generated/prisma/models/ArcadePack.ts
|
||||||
|
67b8ea01f0a866d14db7fdff3178a080e2879990129c5e08690463fb9940f48d *src/generated/prisma/models/AuthSecurity.ts
|
||||||
|
6885465a57019e8cba28755d9c6abaa087e913d666d02d8912abf0790c41a77b *src/generated/prisma/models/Class.ts
|
||||||
|
ea873177511c6f7354dc01ec0ad228d9f777bca2fbfb15d281d3f2ab8f817ebf *src/generated/prisma/models/Deck.ts
|
||||||
|
1082ff29bec6e4f920c93d778032ffffa324c312b05efd6c5fa7ada1be6674f7 *src/generated/prisma/models/Flashcard.ts
|
||||||
|
de94c2da405c08a0b690aeb1b0cdc03582c94ccddaf913f4c6d1b18d8c413b61 *src/generated/prisma/models/MaterialGroup.ts
|
||||||
|
941998d2c098e9fe5cd0a888bac470190108238c936c1d7cfce2742ed1e40d8a *src/generated/prisma/models/Question.ts
|
||||||
|
3d54ed93e516f4d773c200b166a84ab2c9c8b7b0d7f7ddfd61afc3ca057ee755 *src/generated/prisma/models/QuizAttempt.ts
|
||||||
|
454f19f97a82b8f37fc5c5179e87afe6ea185a79810043471d55c0a8ca26f14a *src/generated/prisma/models/QuizSet.ts
|
||||||
|
3b884aaaad311e8f22c1390cf24ebe69b74b52a97c213781d2a4cbb56288b8a7 *src/generated/prisma/models/Setting.ts
|
||||||
|
1ccf32b2f56b1d35c9b7152d2e1817b4c1b7853549a1ff187e9fb7c29fa49f2a *src/generated/prisma/models/ShareLink.ts
|
||||||
|
cba950d431c702afe67f3ac92d8397275d3aeeebbc59d357fd20dec2123e0ee2 *src/generated/prisma/models/SpacedRepetitionCardState.ts
|
||||||
|
2ef43d1c5d7e2eb0d85080fb29703ae99c1704ba8f5200703e2bb8e7a617bf04 *src/generated/prisma/models/SpacedRepetitionSet.ts
|
||||||
|
0bf9eab9f937ef2d621ccf0cea8e209eae9372a9b333dfe6b18267e563a6b2b0 *src/generated/prisma/models/SpacedRepetitionSetDeck.ts
|
||||||
|
916bda8f95647f0dc3560bc6848c4a97d4c1f37fd5ed54e6736d9d7d45e08adc *src/generated/prisma/models/StudyActivity.ts
|
||||||
|
36724157e76b49df49e7827eacbcb0ab8336917f35913777701d8a6830770500 *src/generated/prisma/models/StudyProgress.ts
|
||||||
52
audit-results/tmp/inspect-db.cjs
Normal file
52
audit-results/tmp/inspect-db.cjs
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
// Read-only diagnostic: introspect the temp DB created by `prisma migrate deploy`
|
||||||
|
// to check whether the migrated schema matches schema.prisma (drift check).
|
||||||
|
// Usage: node audit-results/tmp/inspect-db.cjs <path-to-sqlite-db>
|
||||||
|
const path = require("node:path");
|
||||||
|
const Database = require("better-sqlite3");
|
||||||
|
|
||||||
|
const dbPath = process.argv[2];
|
||||||
|
if (!dbPath) {
|
||||||
|
console.error("usage: node inspect-db.cjs <db-path>");
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
const db = new Database(path.resolve(dbPath), { readonly: true });
|
||||||
|
|
||||||
|
const tables = db
|
||||||
|
.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
|
||||||
|
.all()
|
||||||
|
.map((r) => r.name);
|
||||||
|
|
||||||
|
console.log("=== TABLES ===");
|
||||||
|
console.log(tables.join(", "));
|
||||||
|
console.log("Has MaterialGroup table:", tables.includes("MaterialGroup"));
|
||||||
|
|
||||||
|
for (const table of ["Deck", "QuizSet", "ShareLink", "MaterialGroup"]) {
|
||||||
|
if (tables.includes(table)) {
|
||||||
|
const cols = db
|
||||||
|
.prepare(`PRAGMA table_info('${table}')`)
|
||||||
|
.all()
|
||||||
|
.map((r) => r.name);
|
||||||
|
console.log(`--- ${table} columns ---`);
|
||||||
|
console.log(cols.join(", "));
|
||||||
|
if (table === "Deck" || table === "QuizSet" || table === "ShareLink") {
|
||||||
|
console.log(` -> has groupId:`, cols.includes("groupId"));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log(`--- ${table}: MISSING TABLE ---`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const indexes = db
|
||||||
|
.prepare("SELECT name, tbl_name FROM sqlite_master WHERE type='index' ORDER BY name")
|
||||||
|
.all()
|
||||||
|
.map((r) => `${r.tbl_name}.${r.name}`);
|
||||||
|
console.log("=== INDEXES (count) ===", indexes.length);
|
||||||
|
|
||||||
|
// Confirm migrations table
|
||||||
|
const applied = db.prepare("SELECT migration_name FROM _prisma_migrations ORDER BY started_at").all();
|
||||||
|
console.log("=== APPLIED MIGRATIONS ===");
|
||||||
|
applied.forEach((m) => console.log(" -", m.migration_name));
|
||||||
|
|
||||||
|
db.close();
|
||||||
|
console.log("DONE");
|
||||||
40
audit-results/tmp/reproduce-drift.cjs
Normal file
40
audit-results/tmp/reproduce-drift.cjs
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
// Read-only diagnostic: execute the exact SQL shape the generated Prisma client
|
||||||
|
// emits for deck.findMany() / materialGroup.findMany() against a DB built purely
|
||||||
|
// from `prisma migrate deploy` (predev + Docker entrypoint flow).
|
||||||
|
// Proves DBAUD-01: migrated DB is incompatible with the shipped client.
|
||||||
|
const path = require("node:path");
|
||||||
|
const Database = require("better-sqlite3");
|
||||||
|
|
||||||
|
const dbPath = path.resolve("audit-results/tmp/audit-migration.db");
|
||||||
|
const db = new Database(dbPath, { readonly: true });
|
||||||
|
|
||||||
|
// 1. Shape of prisma.deck.findMany({ include: { class: true } }) — Prisma selects
|
||||||
|
// every scalar field, including groupId (schema.prisma:33-34, generated client).
|
||||||
|
const deckSql =
|
||||||
|
'SELECT "Deck"."id", "Deck"."classId", "Deck"."name", "Deck"."description", "Deck"."sortOrder", "Deck"."createdAt", "Deck"."groupId" FROM "Deck"';
|
||||||
|
try {
|
||||||
|
db.prepare(deckSql).all();
|
||||||
|
console.log("deck.findMany SQL: OK");
|
||||||
|
} catch (e) {
|
||||||
|
console.log("deck.findMany SQL: FAILED ->", e.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Shape of prisma.materialGroup.findMany()
|
||||||
|
const groupSql = 'SELECT "MaterialGroup"."id", "MaterialGroup"."classId", "MaterialGroup"."name", "MaterialGroup"."type", "MaterialGroup"."sortOrder", "MaterialGroup"."createdAt" FROM "MaterialGroup"';
|
||||||
|
try {
|
||||||
|
db.prepare(groupSql).all();
|
||||||
|
console.log("materialGroup.findMany SQL: OK");
|
||||||
|
} catch (e) {
|
||||||
|
console.log("materialGroup.findMany SQL: FAILED ->", e.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Shape of prisma.shareLink.findFirst() (share page validation)
|
||||||
|
const shareSql = 'SELECT "ShareLink"."id", "ShareLink"."targetType", "ShareLink"."deckId", "ShareLink"."quizSetId", "ShareLink"."groupId", "ShareLink"."createdAt" FROM "ShareLink" LIMIT 1';
|
||||||
|
try {
|
||||||
|
db.prepare(shareSql).all();
|
||||||
|
console.log("shareLink.findFirst SQL: OK");
|
||||||
|
} catch (e) {
|
||||||
|
console.log("shareLink.findFirst SQL: FAILED ->", e.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
db.close();
|
||||||
27
audit-results/tmp/validate-findings.cjs
Normal file
27
audit-results/tmp/validate-findings.cjs
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
// Validates audit-results/FINDINGS.json parses and reports counts per severity.
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = require("node:path");
|
||||||
|
|
||||||
|
const file = path.resolve("audit-results/FINDINGS.json");
|
||||||
|
const raw = fs.readFileSync(file, "utf8");
|
||||||
|
const data = JSON.parse(raw); // throws if invalid
|
||||||
|
|
||||||
|
const findings = data.findings;
|
||||||
|
const bySeverity = {};
|
||||||
|
for (const f of findings) {
|
||||||
|
bySeverity[f.severity] = (bySeverity[f.severity] || 0) + 1;
|
||||||
|
}
|
||||||
|
const byConfidence = {};
|
||||||
|
for (const f of findings) {
|
||||||
|
byConfidence[f.confidence] = (byConfidence[f.confidence] || 0) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ids = findings.map((f) => f.id);
|
||||||
|
const dupes = ids.filter((id, i) => ids.indexOf(id) !== i);
|
||||||
|
if (dupes.length) throw new Error("duplicate ids: " + dupes.join(","));
|
||||||
|
|
||||||
|
console.log("FINDINGS.json: VALID JSON");
|
||||||
|
console.log("total findings:", findings.length);
|
||||||
|
console.log("by severity:", JSON.stringify(bySeverity));
|
||||||
|
console.log("by confidence:", JSON.stringify(byConfidence));
|
||||||
|
console.log("ids:", ids.join(", "));
|
||||||
145
benchmarks/remediationBenchmark.test.ts
Normal file
145
benchmarks/remediationBenchmark.test.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
||||||
|
import { performance } from "node:perf_hooks";
|
||||||
|
import path from "node:path";
|
||||||
|
import Database from "better-sqlite3";
|
||||||
|
import { afterAll, describe, expect, it } from "vitest";
|
||||||
|
import { prisma } from "@/lib/db";
|
||||||
|
import { getActivitySummary } from "@/services/activityService";
|
||||||
|
import { listClasses } from "@/services/classService";
|
||||||
|
import { getStudyAvailabilityByClass } from "@/services/spacedRepetitionService";
|
||||||
|
|
||||||
|
const ACTIVITY_ROWS = 100_000;
|
||||||
|
const CARD_ROWS = 10_000;
|
||||||
|
const SETS = 5;
|
||||||
|
const REQUEST_BUDGET_MS = 250;
|
||||||
|
const NOW = new Date("2026-08-07T18:00:00.000Z");
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await prisma.$disconnect();
|
||||||
|
});
|
||||||
|
|
||||||
|
function databasePath() {
|
||||||
|
const url = process.env.DATABASE_URL;
|
||||||
|
if (!url?.startsWith("file:./")) {
|
||||||
|
throw new Error("Benchmark requires the disposable Vitest database");
|
||||||
|
}
|
||||||
|
return path.resolve(process.cwd(), url.slice("file:".length));
|
||||||
|
}
|
||||||
|
|
||||||
|
function seedRealApplicationSchema() {
|
||||||
|
const database = new Database(databasePath());
|
||||||
|
try {
|
||||||
|
database.pragma("foreign_keys = ON");
|
||||||
|
const insertActivity = database.prepare(
|
||||||
|
'INSERT INTO "StudyActivity" ("id", "type", "occurredAt") VALUES (?, ?, ?)'
|
||||||
|
);
|
||||||
|
const insertDeck = database.prepare(
|
||||||
|
'INSERT INTO "Deck" ("id", "classId", "name", "sortOrder") VALUES (?, ?, ?, ?)'
|
||||||
|
);
|
||||||
|
const insertSet = database.prepare(
|
||||||
|
'INSERT INTO "SpacedRepetitionSet" ("id", "classId", "name", "newCardsPerDay", "sortOrder", "updatedAt") VALUES (?, ?, ?, ?, ?, ?)'
|
||||||
|
);
|
||||||
|
const insertMembership = database.prepare(
|
||||||
|
'INSERT INTO "SpacedRepetitionSetDeck" ("setId", "deckId", "sortOrder") VALUES (?, ?, ?)'
|
||||||
|
);
|
||||||
|
const insertCard = database.prepare(
|
||||||
|
'INSERT INTO "Flashcard" ("id", "deckId", "front", "back", "sortOrder") VALUES (?, ?, ?, ?, ?)'
|
||||||
|
);
|
||||||
|
const insertState = database.prepare(`
|
||||||
|
INSERT INTO "SpacedRepetitionCardState"
|
||||||
|
("id", "setId", "flashcardId", "due", "stability", "difficulty", "elapsedDays", "scheduledDays", "learningSteps", "reps", "lapses", "state", "firstReviewedAt", "lastReview", "createdAt", "updatedAt")
|
||||||
|
VALUES (?, ?, ?, ?, 1, 5, 1, ?, 0, 1, 0, ?, ?, ?, ?, ?)
|
||||||
|
`);
|
||||||
|
database.transaction(() => {
|
||||||
|
database.prepare(
|
||||||
|
'INSERT INTO "Class" ("id", "slug", "name", "sortOrder") VALUES (?, ?, ?, 0)'
|
||||||
|
).run("benchmark-class", "benchmark-class", "Benchmark Class");
|
||||||
|
|
||||||
|
for (let index = 0; index < ACTIVITY_ROWS; index += 1) {
|
||||||
|
const ageDays = index % 730;
|
||||||
|
const type = index % 2 === 0 ? "FLASHCARD" : "QUIZ_QUESTION";
|
||||||
|
insertActivity.run(
|
||||||
|
`activity-${index}`,
|
||||||
|
type,
|
||||||
|
NOW.getTime() - ageDays * 86_400_000
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let setIndex = 0; setIndex < SETS; setIndex += 1) {
|
||||||
|
const deckId = `benchmark-deck-${setIndex}`;
|
||||||
|
const setId = `benchmark-set-${setIndex}`;
|
||||||
|
insertDeck.run(deckId, "benchmark-class", `Deck ${setIndex}`, setIndex);
|
||||||
|
insertSet.run(setId, "benchmark-class", `Set ${setIndex}`, 30, setIndex, NOW.getTime());
|
||||||
|
insertMembership.run(setId, deckId, 0);
|
||||||
|
for (let cardIndex = setIndex; cardIndex < CARD_ROWS; cardIndex += SETS) {
|
||||||
|
const cardId = `benchmark-card-${cardIndex}`;
|
||||||
|
const due = NOW.getTime() + ((cardIndex % 200) - 100) * 60_000;
|
||||||
|
const firstReviewed = NOW.getTime() - (cardIndex % 30) * 86_400_000;
|
||||||
|
const scheduledDays = cardIndex % 4 === 0 ? 1 : 0;
|
||||||
|
const state = cardIndex % 4;
|
||||||
|
insertCard.run(cardId, deckId, `Front ${cardIndex}`, `Back ${cardIndex}`, cardIndex);
|
||||||
|
insertState.run(
|
||||||
|
`benchmark-state-${cardIndex}`,
|
||||||
|
setId,
|
||||||
|
cardId,
|
||||||
|
due,
|
||||||
|
scheduledDays,
|
||||||
|
state,
|
||||||
|
firstReviewed,
|
||||||
|
firstReviewed,
|
||||||
|
firstReviewed,
|
||||||
|
NOW.getTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
} finally {
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function measure<T>(label: string, operation: () => Promise<T>) {
|
||||||
|
const start = performance.now();
|
||||||
|
const value = await operation();
|
||||||
|
return {
|
||||||
|
label,
|
||||||
|
latencyMs: Number((performance.now() - start).toFixed(2)),
|
||||||
|
value,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("remediation performance condition", () => {
|
||||||
|
it("measures actual migrated services at the expected upper-use fixture", async () => {
|
||||||
|
seedRealApplicationSchema();
|
||||||
|
const activity = await measure("getActivitySummary", () => getActivitySummary(NOW));
|
||||||
|
const availability = await measure("getStudyAvailabilityByClass", () =>
|
||||||
|
getStudyAvailabilityByClass(NOW)
|
||||||
|
);
|
||||||
|
const classPolling = await measure("listClasses/navbar-poll", () => listClasses());
|
||||||
|
const measurements = [activity, availability, classPolling].map((result) => ({
|
||||||
|
label: result.label,
|
||||||
|
latencyMs: result.latencyMs,
|
||||||
|
}));
|
||||||
|
|
||||||
|
console.info(JSON.stringify({
|
||||||
|
fixture: { activityRows: ACTIVITY_ROWS, cards: CARD_ROWS, sets: SETS },
|
||||||
|
budgetMs: REQUEST_BUDGET_MS,
|
||||||
|
prismaServiceOperations: { activity: 1, availability: 1, classPolling: 2 },
|
||||||
|
measurements,
|
||||||
|
activity: {
|
||||||
|
displayedDays: activity.value.days.length,
|
||||||
|
currentStreak: activity.value.currentStreak,
|
||||||
|
},
|
||||||
|
readyCards: availability.value["benchmark-class"]?.readyCards,
|
||||||
|
}, null, 2));
|
||||||
|
|
||||||
|
expect(activity.value.days).toHaveLength(53 * 7);
|
||||||
|
expect(activity.value.currentStreak).toBeGreaterThan(53 * 7);
|
||||||
|
expect(availability.value["benchmark-class"]).toBeDefined();
|
||||||
|
expect(classPolling.value).toHaveLength(1);
|
||||||
|
for (const measurement of measurements) {
|
||||||
|
expect(measurement.latencyMs, measurement.label).toBeLessThanOrEqual(
|
||||||
|
REQUEST_BUDGET_MS
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
20
compose.yaml
20
compose.yaml
|
|
@ -1,20 +0,0 @@
|
||||||
services:
|
|
||||||
study-v2:
|
|
||||||
build: .
|
|
||||||
image: study-desk:v2-local
|
|
||||||
container_name: study-v2
|
|
||||||
restart: unless-stopped
|
|
||||||
init: true
|
|
||||||
ports:
|
|
||||||
- "${STUDY_PORT:-3727}:3726"
|
|
||||||
volumes:
|
|
||||||
- "${STUDY_DATA_DIR:-./data-v2}:/data"
|
|
||||||
environment:
|
|
||||||
TZ: ${TZ:-America/Phoenix}
|
|
||||||
TRUST_PROXY: ${TRUST_PROXY:-false}
|
|
||||||
COOKIE_SECURE: ${COOKIE_SECURE:-auto}
|
|
||||||
MAX_UPLOAD_MB: ${MAX_UPLOAD_MB:-100}
|
|
||||||
BACKUP_KEEP: ${BACKUP_KEEP:-7}
|
|
||||||
stop_grace_period: 15s
|
|
||||||
security_opt:
|
|
||||||
- no-new-privileges:true
|
|
||||||
15
docker-compose.dev.yml
Normal file
15
docker-compose.dev.yml
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
services:
|
||||||
|
study-app:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
target: deps
|
||||||
|
command: npm run dev
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=file:/app/data/study.db
|
||||||
|
- NODE_ENV=development
|
||||||
|
volumes:
|
||||||
|
- .:/app
|
||||||
|
- /app/node_modules
|
||||||
|
- ./data:/app/data
|
||||||
16
docker-compose.yml
Normal file
16
docker-compose.yml
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
services:
|
||||||
|
study-app:
|
||||||
|
build: .
|
||||||
|
container_name: study-app
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- "3000:3726"
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=file:/app/data/study.db
|
||||||
|
- SESSION_SECRET=${SESSION_SECRET:?SESSION_SECRET must be set to at least 32 characters}
|
||||||
|
- ADMIN_PASSWORD_HASH=${ADMIN_PASSWORD_HASH:-}
|
||||||
|
- ALLOW_INITIAL_SETUP=${ALLOW_INITIAL_SETUP:-false}
|
||||||
|
- SECURE_COOKIES=${SECURE_COOKIES:-false}
|
||||||
|
- NODE_ENV=production
|
||||||
|
volumes:
|
||||||
|
- ./data:/app/data
|
||||||
13
docker-entrypoint.sh
Normal file
13
docker-entrypoint.sh
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
#!/bin/sh
|
||||||
|
set -e
|
||||||
|
|
||||||
|
DEFAULT_SECRET='dev-session-secret-change-in-production-must-be-32-chars'
|
||||||
|
TRIMMED_SECRET=$(printf '%s' "${SESSION_SECRET:-}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||||
|
if [ -z "$TRIMMED_SECRET" ] || [ "${#TRIMMED_SECRET}" -lt 32 ] || [ "$TRIMMED_SECRET" = "$DEFAULT_SECRET" ]; then
|
||||||
|
echo >&2 "SESSION_SECRET must be a non-default value of at least 32 characters"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
node scripts/migration-preflight.mjs --startup
|
||||||
|
./node_modules/.bin/prisma migrate deploy
|
||||||
|
exec node server.js
|
||||||
44
docs/AI.md
44
docs/AI.md
|
|
@ -1,44 +0,0 @@
|
||||||
# AI connections and verification
|
|
||||||
|
|
||||||
Settings → AI manages provider connections and model routing. Add a connection, test it,
|
|
||||||
discover models where supported (or add a custom model), enable models and select defaults
|
|
||||||
for generation, chat, explanations and vision. Output limits, capabilities, pricing and
|
|
||||||
reasoning settings can be edited per model. Catalog prices are estimates, not billing records.
|
|
||||||
|
|
||||||
Adapters use OpenAI Responses, Anthropic Messages, Gemini generateContent, OpenAI-compatible
|
|
||||||
Chat Completions and Ollama's native chat API. Configure the endpoint as shown in Settings.
|
|
||||||
Local models are reached **from the server/container**: `localhost` inside Docker is the
|
|
||||||
container itself, not your workstation or Unraid host. Use a reachable LAN address instead.
|
|
||||||
|
|
||||||
Keys are encrypted with AES-256-GCM in SQLite using `/data/secret.key` or `APP_SECRET`.
|
|
||||||
Only the server decrypts them. Keep the key with backups; changing it breaks decryption.
|
|
||||||
Documents are sent to the selected provider when you request AI processing; extraction and
|
|
||||||
full-text indexing otherwise run locally. OCR requires a compatible vision/PDF model.
|
|
||||||
Retrieval uses chunked SQLite FTS search; it is lexical retrieval, not embedding similarity.
|
|
||||||
Review generated nursing content and source references before using it for study.
|
|
||||||
|
|
||||||
## ChatGPT account investigation (2026-11-11)
|
|
||||||
|
|
||||||
The official [Sign in with ChatGPT quickstart](https://developers.openai.com/siwc/quickstart)
|
|
||||||
now describes plan access for eligible integrations. The
|
|
||||||
[official implementation guide](https://developers.openai.com/cookbook/articles/sign-in-with-chatgpt)
|
|
||||||
distinguishes local personal/open-source tools from remotely hosted apps, which require
|
|
||||||
access approval. This Unraid/reverse-proxy deployment is a hosted web application; eligibility
|
|
||||||
for its private client has not been established. Therefore V2 currently uses conventional API
|
|
||||||
connections, and does not advertise ChatGPT plan access or reuse another client's credentials.
|
|
||||||
|
|
||||||
To add this later, obtain the appropriate approval/registration and implement the documented
|
|
||||||
OAuth/PKCE flow, verified identity, encrypted refresh tokens, approved callbacks and plan-scoped
|
|
||||||
model discovery. Keep this separate from the single administrator's local login. Never copy
|
|
||||||
ChatGPT browser session tokens or a Codex client's registration/credentials into Study Desk.
|
|
||||||
|
|
||||||
## What has been tested
|
|
||||||
|
|
||||||
Mock protocol tests cover request translation, streams, tools, usage and error handling.
|
|
||||||
Integration tests use a local OpenAI-compatible mock to generate/edit/save drafts, search
|
|
||||||
documents with citations and exercise confirmed assistant actions. They cost no API credits.
|
|
||||||
No successful real-provider inference or real ChatGPT OAuth authorization is claimed.
|
|
||||||
|
|
||||||
For manual local AI testing: `npx tsx scripts/mock-ai.ts` starts a scripted mock at
|
|
||||||
`http://localhost:4010/v1` with key `mock-key`. It returns clearly labeled synthetic material.
|
|
||||||
Do not use mock output as nursing guidance. Stop the mock and remove its connection after QA.
|
|
||||||
|
|
@ -1,96 +0,0 @@
|
||||||
# Single-container deployment and recovery
|
|
||||||
|
|
||||||
## Unraid
|
|
||||||
|
|
||||||
Build from this checkout with `docker build -t study-desk:v2-local .`.
|
|
||||||
Create a **new** container named `study-v2`, image `study-desk:v2-local`:
|
|
||||||
|
|
||||||
| Setting | Value |
|
|
||||||
| --- | --- |
|
|
||||||
| Network | bridge |
|
|
||||||
| Container port | 3726 TCP |
|
|
||||||
| Host port | 3727 (or another unused port) |
|
|
||||||
| Container path | /data |
|
|
||||||
| Host path | /mnt/user/appdata/study-v2 |
|
|
||||||
| TZ | America/Phoenix |
|
|
||||||
| TRUST_PROXY | true only behind a trusted proxy |
|
|
||||||
| COOKIE_SECURE | true with HTTPS; auto for local HTTP testing |
|
|
||||||
|
|
||||||
The image uses UID/GID **1000:1000**. Create a dedicated host directory writable by that user
|
|
||||||
before starting. For example, on Unraid: `mkdir -p /mnt/user/appdata/study-v2` then
|
|
||||||
`chown 1000:1000 /mnt/user/appdata/study-v2`. Only change permissions on this new folder.
|
|
||||||
Alternatively select an appropriate non-root UID/GID using Docker's `--user` option and grant
|
|
||||||
that identity access to the dedicated directory. Never mount V1's database or data folder.
|
|
||||||
|
|
||||||
The supplied `compose.yaml` defaults to `./data-v2` and host port 3727. Override
|
|
||||||
`STUDY_DATA_DIR` for Unraid. Do not reuse a V1 container name, mount or proxy destination.
|
|
||||||
|
|
||||||
Startup applies committed schema migrations transactionally, taking a database snapshot first
|
|
||||||
when an existing V2 schema needs upgrading. A database from a newer app version is refused.
|
|
||||||
The first visit asks for the setup code from the container log or `/data/setup-code.txt`.
|
|
||||||
After setup, the password is stored as a salted scrypt hash. No registration exists.
|
|
||||||
|
|
||||||
## Reverse proxy
|
|
||||||
|
|
||||||
Point your proxy to the container's port 3726 (or the host's mapped port). Forward `Host`,
|
|
||||||
`X-Forwarded-Proto`, and a trustworthy `X-Forwarded-For`. With `TRUST_PROXY=true`, restrict
|
|
||||||
direct access to the application to your proxy/network: forwarded headers are trusted.
|
|
||||||
Use `COOKIE_SECURE=true` for the HTTPS public hostname. Plain HTTP cannot carry Secure cookies.
|
|
||||||
Keep proxy upload limits at least as large as `MAX_UPLOAD_MB`, and allow long AI responses.
|
|
||||||
Disable buffering for `/api/ai/chat` and `/api/ai/explain`; allow a long read timeout there.
|
|
||||||
Do not cache `/api/`, `/media/` or shared study data. `/api/health` returns 200 when SQLite works.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
`.env.example` documents all server variables. Environment values override a local `.env`.
|
|
||||||
The image excludes local configuration, source databases and test content from its build context.
|
|
||||||
Do not bake passwords or keys into an image.
|
|
||||||
|
|
||||||
- `ADMIN_PASSWORD`: optional first-install password, ignored after setup.
|
|
||||||
- `RESET_ADMIN_PASSWORD`: changes the password and revokes sessions at startup. Remove it
|
|
||||||
after one successful reset or it will repeat on every restart.
|
|
||||||
- `APP_SECRET`: optional external encryption secret (16+ characters). Keep it stable.
|
|
||||||
Otherwise `/data/secret.key` is generated once. A damaged key is refused, never replaced.
|
|
||||||
- `MAX_UPLOAD_MB`: default 100; OCR/native vision input has additional provider limits.
|
|
||||||
- `BACKUP_KEEP`: default 7 daily automatic database snapshots.
|
|
||||||
|
|
||||||
## Backups
|
|
||||||
|
|
||||||
Settings → Data has portable content JSON and a full ZIP download. JSON includes quizzes,
|
|
||||||
decks, classes, folders and notes, **not** progress, uploads, credentials or all settings.
|
|
||||||
The full ZIP contains a consistent SQLite snapshot, uploads, media, a restore manifest and
|
|
||||||
the generated `secret.key` when present. It contains private content and credential material;
|
|
||||||
store it securely. An externally configured `APP_SECRET` must be saved separately.
|
|
||||||
|
|
||||||
Daily snapshots under `/data/backups` cover **the database only**. They are not a substitute
|
|
||||||
for a full off-machine backup of the data volume and encryption key. A backup stream that
|
|
||||||
cannot read an included file fails rather than silently substituting an empty file.
|
|
||||||
Avoid changing or deleting source files while making a full ZIP; filesystem content is not
|
|
||||||
transactional with SQLite.
|
|
||||||
|
|
||||||
## Restore drill
|
|
||||||
|
|
||||||
1. Download a full ZIP and verify it opens. Stop the V2 container before restoring.
|
|
||||||
2. Keep the old data folder intact. Extract the ZIP into a **new**, empty folder.
|
|
||||||
3. Check that `study.db`, `uploads/`, `media/` and the matching key are present as applicable.
|
|
||||||
If using `APP_SECRET`, set the original value. Do not copy stale `study.db-wal` or
|
|
||||||
`study.db-shm` files from another database into the restored folder.
|
|
||||||
4. Give the container user access. Start the same image version with the new folder mounted
|
|
||||||
at `/data`, initially on a separate port without public routing.
|
|
||||||
5. Verify login, class/quiz/deck counts, representative uploads, review progress, AI connection
|
|
||||||
decryption and shared links. Use SQLite `PRAGMA integrity_check` and `foreign_key_check`
|
|
||||||
if manually inspecting a copy. Do not run migration/reset experiments on your only backup.
|
|
||||||
6. Switch proxy routing only after these checks succeed. Retain the previous folder and image.
|
|
||||||
|
|
||||||
For an upgrade, make and verify a full backup first, then rebuild/recreate the container with
|
|
||||||
the same V2 volume. Never edit an already applied migration. Roll back by restoring the
|
|
||||||
pre-upgrade data into a separate directory with the prior image; do not force an older image
|
|
||||||
to open a newer schema.
|
|
||||||
|
|
||||||
## Verification still required on Unraid
|
|
||||||
|
|
||||||
Docker is unavailable in the development VM. Before production use: build the image, confirm
|
|
||||||
health status and non-root volume permissions, complete setup over HTTPS, upload a PDF and
|
|
||||||
PPTX, restart and check persistence, test public links from a private browser, confirm proxy
|
|
||||||
streaming and upload limits, and perform the restore drill above. The Dockerfile and Compose
|
|
||||||
configuration are implemented, but container deployment has **not** been executed here.
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
# Importing V1 safely
|
|
||||||
|
|
||||||
V1 code remains on `main` at `7bd0a9a` and in history. V2 has a separate schema and must use
|
|
||||||
a separate data folder. It never upgrades a production V1 database in place.
|
|
||||||
|
|
||||||
1. Use V1's backup facility to make a consistent SQLite copy. If copying files yourself,
|
|
||||||
stop V1 first; copying only a live WAL-mode `.db` can omit recent writes.
|
|
||||||
2. Keep the original and a verified backup untouched. Start V2 with an empty V2 volume.
|
|
||||||
3. In V2 Settings → Data → V1 import, select the **backup copy** and preview it.
|
|
||||||
4. Review the reported counts/skipped items, then explicitly import. V2 opens the submitted
|
|
||||||
SQLite file read-only and creates new IDs in one transaction. A failed import rolls back.
|
|
||||||
5. Compare class/folder/deck/quiz counts and representative questions, rationales, attempts and
|
|
||||||
scheduling. V1's passwords, provider settings and share URLs are not reused. Recreate links
|
|
||||||
deliberately. Running the import again creates another set of content.
|
|
||||||
|
|
||||||
The importer supports V1 classes, material groups, decks/cards, quiz sets/questions/options,
|
|
||||||
completed attempts, spaced-repetition card state and activity where present. It reports
|
|
||||||
unsupported/skipped records. Older optional game features are not migrated as games.
|
|
||||||
V1 JSON quiz/flashcard exports can also be pasted/uploaded in a class's Import dialog.
|
|
||||||
|
|
||||||
Portable V2 JSON exports are useful for migration between apps; full ZIP backups are needed
|
|
||||||
to preserve progress, documents, settings and credentials. See DEPLOYMENT.md for recovery.
|
|
||||||
19
eslint.config.mjs
Normal file
19
eslint.config.mjs
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
import { defineConfig, globalIgnores } from "eslint/config";
|
||||||
|
import nextVitals from "eslint-config-next/core-web-vitals";
|
||||||
|
import nextTs from "eslint-config-next/typescript";
|
||||||
|
|
||||||
|
const eslintConfig = defineConfig([
|
||||||
|
...nextVitals,
|
||||||
|
...nextTs,
|
||||||
|
// Override default ignores of eslint-config-next.
|
||||||
|
globalIgnores([
|
||||||
|
// Default ignores of eslint-config-next:
|
||||||
|
".next/**",
|
||||||
|
"out/**",
|
||||||
|
"build/**",
|
||||||
|
"next-env.d.ts",
|
||||||
|
"audit-results/tmp/**",
|
||||||
|
]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
export default eslintConfig;
|
||||||
8
next.config.ts
Normal file
8
next.config.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
import type { NextConfig } from "next";
|
||||||
|
|
||||||
|
const nextConfig: NextConfig = {
|
||||||
|
output: "standalone",
|
||||||
|
serverExternalPackages: ["better-sqlite3"],
|
||||||
|
};
|
||||||
|
|
||||||
|
export default nextConfig;
|
||||||
11006
package-lock.json
generated
11006
package-lock.json
generated
File diff suppressed because it is too large
Load diff
85
package.json
85
package.json
|
|
@ -1,61 +1,48 @@
|
||||||
{
|
{
|
||||||
"name": "study-desk",
|
"name": "study-app",
|
||||||
"version": "2.0.0",
|
"version": "0.1.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
|
||||||
"description": "Study Desk V2 - a self-hosted, single-user study platform with quizzes, spaced-repetition flashcards, documents, and AI.",
|
|
||||||
"engines": {
|
|
||||||
"node": ">=24"
|
|
||||||
},
|
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "concurrently -k -n api,web -c blue,magenta \"npm:dev:server\" \"npm:dev:web\"",
|
"predev": "prisma migrate deploy",
|
||||||
"dev:server": "tsx watch --clear-screen=false server/main.ts",
|
"dev": "next dev",
|
||||||
"dev:web": "vite",
|
"build": "next build",
|
||||||
"build": "npm run build:web && npm run build:server",
|
"start": "next start",
|
||||||
"build:web": "vite build",
|
"lint": "eslint",
|
||||||
"build:server": "node scripts/build-server.mjs",
|
|
||||||
"start": "node dist/server/main.js",
|
|
||||||
"typecheck": "tsc -p tsconfig.server.json --noEmit && tsc -p tsconfig.web.json --noEmit",
|
|
||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
"test:watch": "vitest",
|
"benchmark:remediation": "vitest run --config vitest.benchmark.config.ts",
|
||||||
"check": "npm run typecheck && npm run test && npm run build"
|
"db:preflight": "node scripts/migration-preflight.mjs",
|
||||||
|
"db:backup": "node scripts/backup-database.mjs",
|
||||||
|
"auth:reset": "node scripts/request-password-reset.mjs"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@fontsource-variable/instrument-sans": "^5.3.0",
|
"@dnd-kit/core": "^6.3.1",
|
||||||
"@fontsource-variable/jetbrains-mono": "^5.3.0",
|
"@dnd-kit/sortable": "^10.0.0",
|
||||||
"@fontsource-variable/newsreader": "^5.3.0",
|
"@dnd-kit/utilities": "^3.2.2",
|
||||||
"@hono/node-server": "^2.1.6",
|
"@prisma/adapter-better-sqlite3": "^7.8.0",
|
||||||
"@tanstack/react-query": "^5.107.1",
|
"@prisma/client": "^7.8.0",
|
||||||
"clsx": "^2.1.1",
|
"argon2": "^0.44.0",
|
||||||
"fast-xml-parser": "^5.12.0",
|
"better-sqlite3": "^12.11.1",
|
||||||
"fflate": "^0.8.3",
|
"iron-session": "^8.0.4",
|
||||||
"hono": "^4.13.17",
|
"jsonrepair": "^3.14.1",
|
||||||
"jsonrepair": "^3.15.0",
|
"next": "16.3.0",
|
||||||
"lucide-react": "^1.71.0",
|
"prisma": "^7.8.0",
|
||||||
"mammoth": "^1.13.0",
|
"react": "19.2.4",
|
||||||
"radix-ui": "^1.8.1",
|
"react-dom": "19.2.4",
|
||||||
"react": "^19.3.1",
|
|
||||||
"react-dom": "^19.3.1",
|
|
||||||
"react-markdown": "^10.1.0",
|
"react-markdown": "^10.1.0",
|
||||||
"react-router": "^8.5.1",
|
|
||||||
"remark-gfm": "^4.0.1",
|
"remark-gfm": "^4.0.1",
|
||||||
"sonner": "^2.0.8",
|
"ts-fsrs": "^5.4.1",
|
||||||
"ts-fsrs": "^6.0.0",
|
"zod": "^4.4.3"
|
||||||
"unpdf": "^2.1.0",
|
|
||||||
"zod": "^4.6.7"
|
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@tailwindcss/vite": "^4.3.5",
|
"@tailwindcss/postcss": "^4",
|
||||||
"@types/node": "^26.6.6",
|
"@types/better-sqlite3": "^9.6.0",
|
||||||
"@types/react": "^19.3.0",
|
"@types/node": "^20",
|
||||||
"@types/react-dom": "^19.3.0",
|
"@types/react": "^19",
|
||||||
"@vitejs/plugin-react": "^6.3.0",
|
"@types/react-dom": "^19",
|
||||||
"concurrently": "^10.0.7",
|
"eslint": "^9",
|
||||||
"esbuild": "^0.28.2",
|
"eslint-config-next": "16.2.9",
|
||||||
"tailwindcss": "^4.3.5",
|
"tailwindcss": "^4",
|
||||||
"tsx": "^4.23.15",
|
"typescript": "^5",
|
||||||
"typescript": "^7.0.2",
|
"vitest": "^4.1.10"
|
||||||
"vite": "^8.3.8",
|
|
||||||
"vitest": "^5.1.0"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
7
postcss.config.mjs
Normal file
7
postcss.config.mjs
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
const config = {
|
||||||
|
plugins: {
|
||||||
|
"@tailwindcss/postcss": {},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export default config;
|
||||||
9
prisma.config.ts
Normal file
9
prisma.config.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
import path from "node:path";
|
||||||
|
import { defineConfig } from "prisma/config";
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
schema: path.join(__dirname, "prisma", "schema.prisma"),
|
||||||
|
datasource: {
|
||||||
|
url: process.env.DATABASE_URL ?? "file:./dev.db",
|
||||||
|
},
|
||||||
|
});
|
||||||
130
prisma/migrations/20260628011409_init/migration.sql
Normal file
130
prisma/migrations/20260628011409_init/migration.sql
Normal file
|
|
@ -0,0 +1,130 @@
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "Class" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"slug" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "Deck" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"classId" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"description" TEXT,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "Deck_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "Flashcard" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"deckId" TEXT NOT NULL,
|
||||||
|
"front" TEXT NOT NULL,
|
||||||
|
"back" TEXT NOT NULL,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
CONSTRAINT "Flashcard_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "QuizSet" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"classId" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"description" TEXT,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "QuizSet_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "Question" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"quizSetId" TEXT NOT NULL,
|
||||||
|
"type" TEXT NOT NULL,
|
||||||
|
"prompt" TEXT NOT NULL,
|
||||||
|
"rationale" TEXT NOT NULL,
|
||||||
|
"category" TEXT NOT NULL,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
CONSTRAINT "Question_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "AnswerOption" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"questionId" TEXT NOT NULL,
|
||||||
|
"text" TEXT NOT NULL,
|
||||||
|
"isCorrect" BOOLEAN NOT NULL DEFAULT false,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
CONSTRAINT "AnswerOption_questionId_fkey" FOREIGN KEY ("questionId") REFERENCES "Question" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "StudyProgress" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"contentType" TEXT NOT NULL,
|
||||||
|
"deckId" TEXT,
|
||||||
|
"quizSetId" TEXT,
|
||||||
|
"mode" TEXT NOT NULL,
|
||||||
|
"currentIndex" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"orderJson" TEXT NOT NULL,
|
||||||
|
"answersJson" TEXT,
|
||||||
|
"cardResultsJson" TEXT,
|
||||||
|
"updatedAt" DATETIME NOT NULL,
|
||||||
|
CONSTRAINT "StudyProgress_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "StudyProgress_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "QuizAttempt" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"quizSetId" TEXT NOT NULL,
|
||||||
|
"score" REAL NOT NULL,
|
||||||
|
"maxScore" INTEGER NOT NULL,
|
||||||
|
"answersJson" TEXT NOT NULL,
|
||||||
|
"isPartialRetake" BOOLEAN NOT NULL DEFAULT false,
|
||||||
|
"completedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "QuizAttempt_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "ShareLink" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"targetType" TEXT NOT NULL,
|
||||||
|
"deckId" TEXT,
|
||||||
|
"quizSetId" TEXT,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "ShareLink_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "ShareLink_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "AuthSecurity" (
|
||||||
|
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT DEFAULT 1,
|
||||||
|
"failedAttempts" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"lockedUntil" DATETIME,
|
||||||
|
"lastAttemptAt" DATETIME
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "Setting" (
|
||||||
|
"key" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"value" TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "Class_slug_key" ON "Class"("slug");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "StudyProgress_deckId_mode_key" ON "StudyProgress"("deckId", "mode");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "StudyProgress_quizSetId_mode_key" ON "StudyProgress"("quizSetId", "mode");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "ShareLink_deckId_key" ON "ShareLink"("deckId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "ShareLink_quizSetId_key" ON "ShareLink"("quizSetId");
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "StudyActivity" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"type" TEXT NOT NULL,
|
||||||
|
"occurredAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "StudyActivity_occurredAt_idx" ON "StudyActivity"("occurredAt");
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "SpacedRepetitionSet" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"classId" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"description" TEXT,
|
||||||
|
"newCardsPerDay" INTEGER NOT NULL DEFAULT 30,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" DATETIME NOT NULL,
|
||||||
|
CONSTRAINT "SpacedRepetitionSet_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "SpacedRepetitionSetDeck" (
|
||||||
|
"setId" TEXT NOT NULL,
|
||||||
|
"deckId" TEXT NOT NULL,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"addedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
|
||||||
|
PRIMARY KEY ("setId", "deckId"),
|
||||||
|
CONSTRAINT "SpacedRepetitionSetDeck_setId_fkey" FOREIGN KEY ("setId") REFERENCES "SpacedRepetitionSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "SpacedRepetitionSetDeck_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "SpacedRepetitionCardState" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"setId" TEXT NOT NULL,
|
||||||
|
"flashcardId" TEXT NOT NULL,
|
||||||
|
"due" DATETIME NOT NULL,
|
||||||
|
"stability" REAL NOT NULL,
|
||||||
|
"difficulty" REAL NOT NULL,
|
||||||
|
"elapsedDays" INTEGER NOT NULL,
|
||||||
|
"scheduledDays" INTEGER NOT NULL,
|
||||||
|
"learningSteps" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"reps" INTEGER NOT NULL,
|
||||||
|
"lapses" INTEGER NOT NULL,
|
||||||
|
"state" INTEGER NOT NULL,
|
||||||
|
"firstReviewedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"lastReview" DATETIME,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" DATETIME NOT NULL,
|
||||||
|
CONSTRAINT "SpacedRepetitionCardState_setId_fkey" FOREIGN KEY ("setId") REFERENCES "SpacedRepetitionSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "SpacedRepetitionCardState_flashcardId_fkey" FOREIGN KEY ("flashcardId") REFERENCES "Flashcard" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "SpacedRepetitionSet_classId_sortOrder_idx" ON "SpacedRepetitionSet"("classId", "sortOrder");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "SpacedRepetitionSetDeck_deckId_idx" ON "SpacedRepetitionSetDeck"("deckId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "SpacedRepetitionCardState_setId_due_idx" ON "SpacedRepetitionCardState"("setId", "due");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "SpacedRepetitionCardState_setId_firstReviewedAt_idx" ON "SpacedRepetitionCardState"("setId", "firstReviewedAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "SpacedRepetitionCardState_flashcardId_idx" ON "SpacedRepetitionCardState"("flashcardId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "SpacedRepetitionCardState_setId_flashcardId_key" ON "SpacedRepetitionCardState"("setId", "flashcardId");
|
||||||
40
prisma/migrations/20260714010000_add_arcade/migration.sql
Normal file
40
prisma/migrations/20260714010000_add_arcade/migration.sql
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "ArcadePack" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"classId" TEXT NOT NULL,
|
||||||
|
"gameType" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"description" TEXT,
|
||||||
|
"schemaVersion" INTEGER NOT NULL,
|
||||||
|
"sourceJson" TEXT NOT NULL,
|
||||||
|
"normalizedJson" TEXT NOT NULL,
|
||||||
|
"validationReportJson" TEXT NOT NULL,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" DATETIME NOT NULL,
|
||||||
|
CONSTRAINT "ArcadePack_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "ArcadeAttempt" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"arcadePackId" TEXT NOT NULL,
|
||||||
|
"mode" TEXT NOT NULL,
|
||||||
|
"score" INTEGER NOT NULL,
|
||||||
|
"maxScore" INTEGER NOT NULL,
|
||||||
|
"accuracy" REAL NOT NULL,
|
||||||
|
"durationSeconds" INTEGER NOT NULL,
|
||||||
|
"mistakes" INTEGER NOT NULL,
|
||||||
|
"hintsUsed" INTEGER NOT NULL,
|
||||||
|
"settingsJson" TEXT NOT NULL,
|
||||||
|
"resultsJson" TEXT NOT NULL,
|
||||||
|
"seed" TEXT NOT NULL,
|
||||||
|
"completedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "ArcadeAttempt_arcadePackId_fkey" FOREIGN KEY ("arcadePackId") REFERENCES "ArcadePack" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "ArcadePack_classId_gameType_sortOrder_idx" ON "ArcadePack"("classId", "gameType", "sortOrder");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "ArcadeAttempt_arcadePackId_completedAt_idx" ON "ArcadeAttempt"("arcadePackId", "completedAt");
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
PRAGMA defer_foreign_keys=ON;
|
||||||
|
PRAGMA foreign_keys=OFF;
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "MaterialGroup" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"classId" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"type" TEXT NOT NULL,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "MaterialGroup_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- RedefineTables
|
||||||
|
CREATE TABLE "new_Deck" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"classId" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"description" TEXT,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"groupId" TEXT,
|
||||||
|
CONSTRAINT "Deck_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "Deck_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "MaterialGroup" ("id") ON DELETE SET NULL ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
INSERT INTO "new_Deck" ("classId", "createdAt", "description", "id", "name", "sortOrder")
|
||||||
|
SELECT "classId", "createdAt", "description", "id", "name", "sortOrder" FROM "Deck";
|
||||||
|
DROP TABLE "Deck";
|
||||||
|
ALTER TABLE "new_Deck" RENAME TO "Deck";
|
||||||
|
|
||||||
|
CREATE TABLE "new_QuizSet" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"classId" TEXT NOT NULL,
|
||||||
|
"name" TEXT NOT NULL,
|
||||||
|
"description" TEXT,
|
||||||
|
"sortOrder" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"groupId" TEXT,
|
||||||
|
CONSTRAINT "QuizSet_classId_fkey" FOREIGN KEY ("classId") REFERENCES "Class" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "QuizSet_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "MaterialGroup" ("id") ON DELETE SET NULL ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
INSERT INTO "new_QuizSet" ("classId", "createdAt", "description", "id", "name", "sortOrder")
|
||||||
|
SELECT "classId", "createdAt", "description", "id", "name", "sortOrder" FROM "QuizSet";
|
||||||
|
DROP TABLE "QuizSet";
|
||||||
|
ALTER TABLE "new_QuizSet" RENAME TO "QuizSet";
|
||||||
|
|
||||||
|
CREATE TABLE "new_ShareLink" (
|
||||||
|
"id" TEXT NOT NULL PRIMARY KEY,
|
||||||
|
"targetType" TEXT NOT NULL,
|
||||||
|
"deckId" TEXT,
|
||||||
|
"quizSetId" TEXT,
|
||||||
|
"groupId" TEXT,
|
||||||
|
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CONSTRAINT "ShareLink_deckId_fkey" FOREIGN KEY ("deckId") REFERENCES "Deck" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "ShareLink_quizSetId_fkey" FOREIGN KEY ("quizSetId") REFERENCES "QuizSet" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
|
||||||
|
CONSTRAINT "ShareLink_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "MaterialGroup" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
INSERT INTO "new_ShareLink" ("createdAt", "deckId", "id", "quizSetId", "targetType")
|
||||||
|
SELECT "createdAt", "deckId", "id", "quizSetId", "targetType" FROM "ShareLink";
|
||||||
|
DROP TABLE "ShareLink";
|
||||||
|
ALTER TABLE "new_ShareLink" RENAME TO "ShareLink";
|
||||||
|
CREATE UNIQUE INDEX "ShareLink_deckId_key" ON "ShareLink"("deckId");
|
||||||
|
CREATE UNIQUE INDEX "ShareLink_quizSetId_key" ON "ShareLink"("quizSetId");
|
||||||
|
CREATE UNIQUE INDEX "ShareLink_groupId_key" ON "ShareLink"("groupId");
|
||||||
|
|
||||||
|
PRAGMA foreign_keys=ON;
|
||||||
|
PRAGMA defer_foreign_keys=OFF;
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "QuizAttempt" ADD COLUMN "reviewSnapshotJson" TEXT;
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "StudyProgress" ADD COLUMN "sessionId" TEXT NOT NULL DEFAULT 'legacy';
|
||||||
|
ALTER TABLE "StudyProgress" ADD COLUMN "revision" INTEGER NOT NULL DEFAULT 0;
|
||||||
10
prisma/migrations/20260808100000_remove_arcade/migration.sql
Normal file
10
prisma/migrations/20260808100000_remove_arcade/migration.sql
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
-- Remove Arcade-generated activity and saved instruction settings.
|
||||||
|
DELETE FROM "StudyActivity"
|
||||||
|
WHERE "type" IN ('ARCADE_GROUP', 'ARCADE_WORD');
|
||||||
|
|
||||||
|
DELETE FROM "Setting"
|
||||||
|
WHERE "key" IN ('llmInstructionsConnections', 'llmInstructionsCrossword');
|
||||||
|
|
||||||
|
-- Drop the dependent table before its parent while SQLite foreign keys are enabled.
|
||||||
|
DROP TABLE IF EXISTS "ArcadeAttempt";
|
||||||
|
DROP TABLE IF EXISTS "ArcadePack";
|
||||||
3
prisma/migrations/migration_lock.toml
Normal file
3
prisma/migrations/migration_lock.toml
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
# Please do not edit this file manually
|
||||||
|
# It should be added in your version-control system (e.g., Git)
|
||||||
|
provider = "sqlite"
|
||||||
227
prisma/schema.prisma
Normal file
227
prisma/schema.prisma
Normal file
|
|
@ -0,0 +1,227 @@
|
||||||
|
datasource db {
|
||||||
|
provider = "sqlite"
|
||||||
|
}
|
||||||
|
|
||||||
|
generator client {
|
||||||
|
provider = "prisma-client"
|
||||||
|
output = "../src/generated/prisma"
|
||||||
|
}
|
||||||
|
|
||||||
|
model Class {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
slug String @unique
|
||||||
|
name String
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
decks Deck[]
|
||||||
|
quizSets QuizSet[]
|
||||||
|
materialGroups MaterialGroup[]
|
||||||
|
spacedRepetitionSets SpacedRepetitionSet[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model Deck {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
classId String
|
||||||
|
name String
|
||||||
|
description String?
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
|
||||||
|
groupId String?
|
||||||
|
group MaterialGroup? @relation(fields: [groupId], references: [id], onDelete: SetNull)
|
||||||
|
cards Flashcard[]
|
||||||
|
shareLink ShareLink?
|
||||||
|
progress StudyProgress[]
|
||||||
|
spacedRepetitionMemberships SpacedRepetitionSetDeck[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model Flashcard {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
deckId String
|
||||||
|
front String
|
||||||
|
back String
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
|
||||||
|
deck Deck @relation(fields: [deckId], references: [id], onDelete: Cascade)
|
||||||
|
spacedRepetitionStates SpacedRepetitionCardState[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model QuizSet {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
classId String
|
||||||
|
name String
|
||||||
|
description String?
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
|
||||||
|
groupId String?
|
||||||
|
group MaterialGroup? @relation(fields: [groupId], references: [id], onDelete: SetNull)
|
||||||
|
questions Question[]
|
||||||
|
attempts QuizAttempt[]
|
||||||
|
shareLink ShareLink?
|
||||||
|
progress StudyProgress[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model Question {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
quizSetId String
|
||||||
|
type String // "MULTIPLE_CHOICE" | "SATA"
|
||||||
|
prompt String
|
||||||
|
rationale String
|
||||||
|
category String
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
|
||||||
|
quizSet QuizSet @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
|
||||||
|
options AnswerOption[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model AnswerOption {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
questionId String
|
||||||
|
text String
|
||||||
|
isCorrect Boolean @default(false)
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
|
||||||
|
question Question @relation(fields: [questionId], references: [id], onDelete: Cascade)
|
||||||
|
}
|
||||||
|
|
||||||
|
model StudyProgress {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
contentType String // "DECK" | "QUIZ"
|
||||||
|
deckId String?
|
||||||
|
quizSetId String?
|
||||||
|
mode String // "SEQUENTIAL" | "SHUFFLED"
|
||||||
|
currentIndex Int @default(0)
|
||||||
|
orderJson String // JSON array of card/question ids
|
||||||
|
answersJson String? // in-progress quiz answers
|
||||||
|
cardResultsJson String? // per-card grades for decks
|
||||||
|
sessionId String @default("legacy")
|
||||||
|
revision Int @default(0)
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
deck Deck? @relation(fields: [deckId], references: [id], onDelete: Cascade)
|
||||||
|
quizSet QuizSet? @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@unique([deckId, mode])
|
||||||
|
@@unique([quizSetId, mode])
|
||||||
|
}
|
||||||
|
|
||||||
|
model QuizAttempt {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
quizSetId String
|
||||||
|
score Float
|
||||||
|
maxScore Int
|
||||||
|
answersJson String
|
||||||
|
reviewSnapshotJson String?
|
||||||
|
isPartialRetake Boolean @default(false)
|
||||||
|
completedAt DateTime @default(now())
|
||||||
|
|
||||||
|
quizSet QuizSet @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
|
||||||
|
}
|
||||||
|
|
||||||
|
model ShareLink {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
targetType String // "DECK" | "QUIZ" | "GROUP"
|
||||||
|
deckId String? @unique
|
||||||
|
quizSetId String? @unique
|
||||||
|
groupId String? @unique
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
deck Deck? @relation(fields: [deckId], references: [id], onDelete: Cascade)
|
||||||
|
quizSet QuizSet? @relation(fields: [quizSetId], references: [id], onDelete: Cascade)
|
||||||
|
group MaterialGroup? @relation(fields: [groupId], references: [id], onDelete: Cascade)
|
||||||
|
}
|
||||||
|
|
||||||
|
model AuthSecurity {
|
||||||
|
id Int @id @default(1)
|
||||||
|
failedAttempts Int @default(0)
|
||||||
|
lockedUntil DateTime?
|
||||||
|
lastAttemptAt DateTime?
|
||||||
|
}
|
||||||
|
|
||||||
|
model Setting {
|
||||||
|
key String @id
|
||||||
|
value String
|
||||||
|
}
|
||||||
|
|
||||||
|
model StudyActivity {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
type String // "FLASHCARD" | "QUIZ_QUESTION"
|
||||||
|
occurredAt DateTime @default(now())
|
||||||
|
|
||||||
|
@@index([occurredAt])
|
||||||
|
}
|
||||||
|
|
||||||
|
model MaterialGroup {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
classId String
|
||||||
|
name String
|
||||||
|
type String // "DECK" | "QUIZ"
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
|
||||||
|
decks Deck[]
|
||||||
|
quizSets QuizSet[]
|
||||||
|
shareLink ShareLink?
|
||||||
|
}
|
||||||
|
|
||||||
|
model SpacedRepetitionSet {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
classId String
|
||||||
|
name String
|
||||||
|
description String?
|
||||||
|
newCardsPerDay Int @default(30)
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
class Class @relation(fields: [classId], references: [id], onDelete: Cascade)
|
||||||
|
decks SpacedRepetitionSetDeck[]
|
||||||
|
cardStates SpacedRepetitionCardState[]
|
||||||
|
|
||||||
|
@@index([classId, sortOrder])
|
||||||
|
}
|
||||||
|
|
||||||
|
model SpacedRepetitionSetDeck {
|
||||||
|
setId String
|
||||||
|
deckId String
|
||||||
|
sortOrder Int @default(0)
|
||||||
|
addedAt DateTime @default(now())
|
||||||
|
|
||||||
|
set SpacedRepetitionSet @relation(fields: [setId], references: [id], onDelete: Cascade)
|
||||||
|
deck Deck @relation(fields: [deckId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@id([setId, deckId])
|
||||||
|
@@index([deckId])
|
||||||
|
}
|
||||||
|
|
||||||
|
model SpacedRepetitionCardState {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
setId String
|
||||||
|
flashcardId String
|
||||||
|
due DateTime
|
||||||
|
stability Float
|
||||||
|
difficulty Float
|
||||||
|
elapsedDays Int
|
||||||
|
scheduledDays Int
|
||||||
|
learningSteps Int @default(0)
|
||||||
|
reps Int
|
||||||
|
lapses Int
|
||||||
|
state Int
|
||||||
|
firstReviewedAt DateTime @default(now())
|
||||||
|
lastReview DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
set SpacedRepetitionSet @relation(fields: [setId], references: [id], onDelete: Cascade)
|
||||||
|
flashcard Flashcard @relation(fields: [flashcardId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@unique([setId, flashcardId])
|
||||||
|
@@index([setId, due])
|
||||||
|
@@index([setId, firstReviewedAt])
|
||||||
|
@@index([flashcardId])
|
||||||
|
}
|
||||||
1
public/file.svg
Normal file
1
public/file.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>
|
||||||
|
After Width: | Height: | Size: 391 B |
1
public/globe.svg
Normal file
1
public/globe.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>
|
||||||
|
After Width: | Height: | Size: 1 KiB |
1
public/next.svg
Normal file
1
public/next.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
|
||||||
|
After Width: | Height: | Size: 1.3 KiB |
1
public/vercel.svg
Normal file
1
public/vercel.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>
|
||||||
|
After Width: | Height: | Size: 128 B |
1
public/window.svg
Normal file
1
public/window.svg
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>
|
||||||
|
After Width: | Height: | Size: 385 B |
42
scripts/backup-database.mjs
Normal file
42
scripts/backup-database.mjs
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { pathToFileURL } from "node:url";
|
||||||
|
import Database from "better-sqlite3";
|
||||||
|
import { databasePathFromUrl } from "./databasePath.mjs";
|
||||||
|
|
||||||
|
function argument(name) {
|
||||||
|
const index = process.argv.indexOf(name);
|
||||||
|
return index >= 0 ? process.argv[index + 1] : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function backupDatabase(sourcePath, outputPath) {
|
||||||
|
const source = path.resolve(sourcePath);
|
||||||
|
const output = path.resolve(outputPath);
|
||||||
|
if (!existsSync(source)) throw new Error(`Source database does not exist: ${source}`);
|
||||||
|
if (existsSync(output)) throw new Error(`Backup destination already exists: ${output}`);
|
||||||
|
if (source === output) throw new Error("Backup destination must differ from source");
|
||||||
|
|
||||||
|
const database = new Database(source, { readonly: true, fileMustExist: true });
|
||||||
|
try {
|
||||||
|
await database.backup(output);
|
||||||
|
} finally {
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
const backup = new Database(output, { readonly: true, fileMustExist: true });
|
||||||
|
try {
|
||||||
|
const integrity = backup.pragma("integrity_check", { simple: true });
|
||||||
|
if (integrity !== "ok") throw new Error(`Backup integrity check failed: ${integrity}`);
|
||||||
|
} finally {
|
||||||
|
backup.close();
|
||||||
|
}
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
const source = argument("--source") ?? databasePathFromUrl(process.env.DATABASE_URL);
|
||||||
|
const output = argument("--output");
|
||||||
|
if (!output) throw new Error("Usage: npm run db:backup -- --output <new-backup.db>");
|
||||||
|
const completedPath = await backupDatabase(source, output);
|
||||||
|
console.info(`Verified SQLite backup created: ${completedPath}`);
|
||||||
|
}
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
// Bundles the server into one file. Dependencies stay external and are installed in the image.
|
|
||||||
import { build } from 'esbuild';
|
|
||||||
|
|
||||||
await build({
|
|
||||||
entryPoints: ['server/main.ts'],
|
|
||||||
outfile: 'dist/server/main.js',
|
|
||||||
bundle: true,
|
|
||||||
platform: 'node',
|
|
||||||
format: 'esm',
|
|
||||||
target: 'node24',
|
|
||||||
packages: 'external',
|
|
||||||
sourcemap: true,
|
|
||||||
logLevel: 'info',
|
|
||||||
});
|
|
||||||
10
scripts/databasePath.mjs
Normal file
10
scripts/databasePath.mjs
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
|
export function databasePathFromUrl(databaseUrl, cwd = process.cwd()) {
|
||||||
|
if (!databaseUrl?.startsWith("file:")) {
|
||||||
|
throw new Error("DATABASE_URL must be a file: SQLite URL");
|
||||||
|
}
|
||||||
|
const rawPath = databaseUrl.slice("file:".length).split("?")[0];
|
||||||
|
if (!rawPath) throw new Error("DATABASE_URL does not contain a database path");
|
||||||
|
return path.resolve(cwd, rawPath);
|
||||||
|
}
|
||||||
|
|
@ -1,60 +0,0 @@
|
||||||
// Renders the PNG app icons (no image library needed): a dark tile with two "cards".
|
|
||||||
import fs from 'node:fs';
|
|
||||||
import zlib from 'node:zlib';
|
|
||||||
|
|
||||||
function png(size, draw) {
|
|
||||||
const px = Buffer.alloc(size * size * 4);
|
|
||||||
draw(px, size);
|
|
||||||
const raw = Buffer.alloc((size * 4 + 1) * size);
|
|
||||||
for (let y = 0; y < size; y++) px.copy(raw, y * (size * 4 + 1) + 1, y * size * 4, (y + 1) * size * 4);
|
|
||||||
const chunk = (type, data) => {
|
|
||||||
const len = Buffer.alloc(4);
|
|
||||||
len.writeUInt32BE(data.length);
|
|
||||||
const body = Buffer.concat([Buffer.from(type), data]);
|
|
||||||
const crc = Buffer.alloc(4);
|
|
||||||
crc.writeUInt32BE(zlib.crc32(body) >>> 0);
|
|
||||||
return Buffer.concat([len, body, crc]);
|
|
||||||
};
|
|
||||||
const ihdr = Buffer.alloc(13);
|
|
||||||
ihdr.writeUInt32BE(size, 0);
|
|
||||||
ihdr.writeUInt32BE(size, 4);
|
|
||||||
ihdr[8] = 8;
|
|
||||||
ihdr[9] = 6;
|
|
||||||
return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk('IHDR', ihdr), chunk('IDAT', zlib.deflateSync(raw, { level: 9 })), chunk('IEND', Buffer.alloc(0))]);
|
|
||||||
}
|
|
||||||
|
|
||||||
const hex = (h) => [parseInt(h.slice(1, 3), 16), parseInt(h.slice(3, 5), 16), parseInt(h.slice(5, 7), 16)];
|
|
||||||
|
|
||||||
function icon(px, size) {
|
|
||||||
const s = size / 64;
|
|
||||||
const set = (x, y, [r, g, b], a = 1) => {
|
|
||||||
if (x < 0 || y < 0 || x >= size || y >= size) return;
|
|
||||||
const i = (y * size + x) * 4;
|
|
||||||
px[i] = px[i] * (1 - a) + r * a;
|
|
||||||
px[i + 1] = px[i + 1] * (1 - a) + g * a;
|
|
||||||
px[i + 2] = px[i + 2] * (1 - a) + b * a;
|
|
||||||
px[i + 3] = 255;
|
|
||||||
};
|
|
||||||
const rrect = (x0, y0, w, h, r, color, rot = 0, cx = 0, cy = 0, alpha = 1) => {
|
|
||||||
const cos = Math.cos(-rot), sin = Math.sin(-rot);
|
|
||||||
for (let y = 0; y < size; y++)
|
|
||||||
for (let x = 0; x < size; x++) {
|
|
||||||
// rotate the sample point back into the rectangle's frame
|
|
||||||
const dx = x + 0.5 - cx * s, dy = y + 0.5 - cy * s;
|
|
||||||
const lx = (dx * cos - dy * sin) / s + cx, ly = (dx * sin + dy * cos) / s + cy;
|
|
||||||
const qx = Math.max(x0 + r - lx, 0, lx - (x0 + w - r)), qy = Math.max(y0 + r - ly, 0, ly - (y0 + h - r));
|
|
||||||
const d = Math.hypot(qx, qy) - r;
|
|
||||||
const a = Math.max(0, Math.min(1, 0.5 - d * s));
|
|
||||||
if (a > 0 && lx >= x0 - 1 && lx <= x0 + w + 1 && ly >= y0 - 1 && ly <= y0 + h + 1) set(x, y, color, a * alpha);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
rrect(0, 0, 64, 64, 0, hex('#1B1A17')); // full-bleed so maskable crops look right
|
|
||||||
rrect(14, 17, 30, 36, 4, hex('#FBFAF7'), (-7 * Math.PI) / 180, 29, 35);
|
|
||||||
rrect(21, 13, 30, 36, 4, hex('#FFE36E'));
|
|
||||||
rrect(27, 23, 18, 3.4, 1.7, hex('#1B1A17'));
|
|
||||||
rrect(27, 30.5, 13, 3.4, 1.7, hex('#1B1A17'), 0, 0, 0, 0.55);
|
|
||||||
rrect(27, 38, 15.5, 3.4, 1.7, hex('#1B1A17'), 0, 0, 0, 0.55);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const size of [180, 192, 512]) fs.writeFileSync(`web/public/icon-${size}.png`, png(size, icon));
|
|
||||||
console.log('icons written');
|
|
||||||
358
scripts/migration-preflight.mjs
Normal file
358
scripts/migration-preflight.mjs
Normal file
|
|
@ -0,0 +1,358 @@
|
||||||
|
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { pathToFileURL } from "node:url";
|
||||||
|
import Database from "better-sqlite3";
|
||||||
|
import { databasePathFromUrl } from "./databasePath.mjs";
|
||||||
|
|
||||||
|
export const GROUP_MIGRATION = "20260807090000_add_material_groups";
|
||||||
|
|
||||||
|
function tableExists(database, table) {
|
||||||
|
return Boolean(
|
||||||
|
database
|
||||||
|
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?")
|
||||||
|
.get(table)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasColumn(database, table, column) {
|
||||||
|
if (!tableExists(database, table)) return false;
|
||||||
|
return database.pragma(`table_info(${table})`).some((item) => item.name === column);
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasForeignKey(database, table, from, target, onDelete) {
|
||||||
|
return database
|
||||||
|
.pragma(`foreign_key_list(${table})`)
|
||||||
|
.some(
|
||||||
|
(item) =>
|
||||||
|
item.from === from && item.table === target && item.on_delete === onDelete
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasUniqueIndex(database, table, indexName, column) {
|
||||||
|
const index = database
|
||||||
|
.pragma(`index_list(${table})`)
|
||||||
|
.find((item) => item.name === indexName && item.unique === 1);
|
||||||
|
if (!index) return false;
|
||||||
|
const columns = database.pragma(`index_info(${indexName})`);
|
||||||
|
return columns.length === 1 && columns[0].name === column;
|
||||||
|
}
|
||||||
|
|
||||||
|
function schemaSnapshot(database, includeMigrationTable = false) {
|
||||||
|
const tables = database
|
||||||
|
.prepare(
|
||||||
|
`SELECT name FROM sqlite_master
|
||||||
|
WHERE type = 'table'
|
||||||
|
AND name NOT LIKE 'sqlite_%'
|
||||||
|
${includeMigrationTable ? "" : "AND name <> '_prisma_migrations'"}
|
||||||
|
ORDER BY name`
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
.map((row) => row.name);
|
||||||
|
|
||||||
|
return tables.map((table) => ({
|
||||||
|
table,
|
||||||
|
columns: database.pragma(`table_info(${JSON.stringify(table)})`).map((column) => ({
|
||||||
|
name: column.name,
|
||||||
|
type: column.type,
|
||||||
|
notnull: column.notnull,
|
||||||
|
defaultValue: column.dflt_value,
|
||||||
|
primaryKey: column.pk,
|
||||||
|
})),
|
||||||
|
foreignKeys: database
|
||||||
|
.pragma(`foreign_key_list(${JSON.stringify(table)})`)
|
||||||
|
.map((foreignKey) => ({
|
||||||
|
id: foreignKey.id,
|
||||||
|
sequence: foreignKey.seq,
|
||||||
|
target: foreignKey.table,
|
||||||
|
from: foreignKey.from,
|
||||||
|
to: foreignKey.to,
|
||||||
|
onUpdate: foreignKey.on_update,
|
||||||
|
onDelete: foreignKey.on_delete,
|
||||||
|
match: foreignKey.match,
|
||||||
|
}))
|
||||||
|
.sort((left, right) => left.id - right.id || left.sequence - right.sequence),
|
||||||
|
indexes: database
|
||||||
|
.pragma(`index_list(${JSON.stringify(table)})`)
|
||||||
|
.filter((index) => !index.name.startsWith("sqlite_autoindex_"))
|
||||||
|
.map((index) => ({
|
||||||
|
name: index.name,
|
||||||
|
unique: index.unique,
|
||||||
|
partial: index.partial,
|
||||||
|
columns: database
|
||||||
|
.pragma(`index_info(${JSON.stringify(index.name)})`)
|
||||||
|
.map((column) => column.name),
|
||||||
|
}))
|
||||||
|
.sort((left, right) => left.name.localeCompare(right.name)),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function expectedSchemaSnapshot(migrationNames) {
|
||||||
|
const database = new Database(":memory:");
|
||||||
|
try {
|
||||||
|
database.pragma("foreign_keys = ON");
|
||||||
|
for (const migrationName of migrationNames) {
|
||||||
|
database.exec(
|
||||||
|
readFileSync(
|
||||||
|
path.join(
|
||||||
|
process.cwd(),
|
||||||
|
"prisma",
|
||||||
|
"migrations",
|
||||||
|
migrationName,
|
||||||
|
"migration.sql"
|
||||||
|
),
|
||||||
|
"utf8"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return schemaSnapshot(database);
|
||||||
|
} finally {
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function inspectMigrationState(databasePath) {
|
||||||
|
if (!existsSync(databasePath)) {
|
||||||
|
throw new Error(`Database does not exist: ${databasePath}`);
|
||||||
|
}
|
||||||
|
const database = new Database(databasePath, { readonly: true, fileMustExist: true });
|
||||||
|
try {
|
||||||
|
const migrationRecords = tableExists(database, "_prisma_migrations")
|
||||||
|
? database
|
||||||
|
.prepare(
|
||||||
|
'SELECT "migration_name", "finished_at", "rolled_back_at" FROM "_prisma_migrations"'
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
: [];
|
||||||
|
const activeRecords = migrationRecords.filter((row) => row.rolled_back_at === null);
|
||||||
|
const migrationNames = activeRecords
|
||||||
|
.filter((row) => row.finished_at !== null)
|
||||||
|
.map((row) => row.migration_name);
|
||||||
|
const failedMigrationNames = activeRecords
|
||||||
|
.filter((row) => row.finished_at === null)
|
||||||
|
.map((row) => row.migration_name);
|
||||||
|
const committedMigrations = readdirSync(
|
||||||
|
path.join(process.cwd(), "prisma", "migrations"),
|
||||||
|
{ withFileTypes: true }
|
||||||
|
)
|
||||||
|
.filter((entry) => entry.isDirectory())
|
||||||
|
.map((entry) => entry.name)
|
||||||
|
.sort();
|
||||||
|
const priorMigrations = committedMigrations.filter(
|
||||||
|
(name) => name < GROUP_MIGRATION
|
||||||
|
);
|
||||||
|
const groupMigrationIndex = committedMigrations.indexOf(GROUP_MIGRATION);
|
||||||
|
|
||||||
|
const artifacts = {
|
||||||
|
materialGroup: tableExists(database, "MaterialGroup"),
|
||||||
|
deckColumn: hasColumn(database, "Deck", "groupId"),
|
||||||
|
quizColumn: hasColumn(database, "QuizSet", "groupId"),
|
||||||
|
shareColumn: hasColumn(database, "ShareLink", "groupId"),
|
||||||
|
deckForeignKey: hasForeignKey(
|
||||||
|
database,
|
||||||
|
"Deck",
|
||||||
|
"groupId",
|
||||||
|
"MaterialGroup",
|
||||||
|
"SET NULL"
|
||||||
|
),
|
||||||
|
quizForeignKey: hasForeignKey(
|
||||||
|
database,
|
||||||
|
"QuizSet",
|
||||||
|
"groupId",
|
||||||
|
"MaterialGroup",
|
||||||
|
"SET NULL"
|
||||||
|
),
|
||||||
|
shareForeignKey: hasForeignKey(
|
||||||
|
database,
|
||||||
|
"ShareLink",
|
||||||
|
"groupId",
|
||||||
|
"MaterialGroup",
|
||||||
|
"CASCADE"
|
||||||
|
),
|
||||||
|
shareUnique: hasUniqueIndex(
|
||||||
|
database,
|
||||||
|
"ShareLink",
|
||||||
|
"ShareLink_groupId_key",
|
||||||
|
"groupId"
|
||||||
|
),
|
||||||
|
};
|
||||||
|
const priorHistoryComplete = priorMigrations.every((name) =>
|
||||||
|
migrationNames.includes(name)
|
||||||
|
);
|
||||||
|
const knownMigrationNames = new Set(committedMigrations);
|
||||||
|
const historyHasUnknown = activeRecords.some(
|
||||||
|
(row) => !knownMigrationNames.has(row.migration_name)
|
||||||
|
);
|
||||||
|
const actualSchema = schemaSnapshot(database);
|
||||||
|
const matches = (expected) =>
|
||||||
|
JSON.stringify(actualSchema) === JSON.stringify(expected);
|
||||||
|
const preGroupSchemaExact = matches(expectedSchemaSnapshot(priorMigrations));
|
||||||
|
const adoptedGroupSchemaExact = matches(
|
||||||
|
expectedSchemaSnapshot([...priorMigrations, GROUP_MIGRATION])
|
||||||
|
);
|
||||||
|
let historyPrefixLength = 0;
|
||||||
|
while (
|
||||||
|
historyPrefixLength < committedMigrations.length &&
|
||||||
|
migrationNames.includes(committedMigrations[historyPrefixLength])
|
||||||
|
) {
|
||||||
|
historyPrefixLength += 1;
|
||||||
|
}
|
||||||
|
const historyIsExactPrefix =
|
||||||
|
!historyHasUnknown &&
|
||||||
|
migrationNames.length === historyPrefixLength &&
|
||||||
|
migrationNames.every((name) =>
|
||||||
|
committedMigrations.slice(0, historyPrefixLength).includes(name)
|
||||||
|
);
|
||||||
|
const trackedSchemaExact =
|
||||||
|
historyIsExactPrefix &&
|
||||||
|
matches(expectedSchemaSnapshot(committedMigrations.slice(0, historyPrefixLength)));
|
||||||
|
let schemaPrefixLength = -1;
|
||||||
|
for (let index = 0; index <= committedMigrations.length; index += 1) {
|
||||||
|
if (matches(expectedSchemaSnapshot(committedMigrations.slice(0, index)))) {
|
||||||
|
schemaPrefixLength = index;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const migrationsToAdopt =
|
||||||
|
schemaPrefixLength > historyPrefixLength
|
||||||
|
? committedMigrations.slice(historyPrefixLength, schemaPrefixLength)
|
||||||
|
: [];
|
||||||
|
const failuresAreAdoptable = failedMigrationNames.every((name) =>
|
||||||
|
migrationsToAdopt.includes(name)
|
||||||
|
);
|
||||||
|
|
||||||
|
let classification = "CONFLICT";
|
||||||
|
if (
|
||||||
|
historyIsExactPrefix &&
|
||||||
|
failedMigrationNames.length === 0 &&
|
||||||
|
schemaPrefixLength === historyPrefixLength
|
||||||
|
) {
|
||||||
|
classification =
|
||||||
|
historyPrefixLength === committedMigrations.length ? "CURRENT" : "APPLY";
|
||||||
|
} else if (
|
||||||
|
historyIsExactPrefix &&
|
||||||
|
priorHistoryComplete &&
|
||||||
|
historyPrefixLength >= groupMigrationIndex &&
|
||||||
|
schemaPrefixLength > historyPrefixLength &&
|
||||||
|
failuresAreAdoptable
|
||||||
|
) {
|
||||||
|
classification = "ADOPT";
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
classification,
|
||||||
|
artifacts,
|
||||||
|
migrationNames,
|
||||||
|
failedMigrationNames,
|
||||||
|
migrationsToAdopt,
|
||||||
|
priorHistoryComplete,
|
||||||
|
schemaMigrationPrefix:
|
||||||
|
schemaPrefixLength >= 0
|
||||||
|
? committedMigrations[schemaPrefixLength - 1] ?? null
|
||||||
|
: null,
|
||||||
|
exactSchema: {
|
||||||
|
preGroup: preGroupSchemaExact,
|
||||||
|
adoptedGroup: adoptedGroupSchemaExact,
|
||||||
|
tracked: trackedSchemaExact,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyBackup(backupPath, sourcePath) {
|
||||||
|
if (!backupPath || !existsSync(backupPath)) {
|
||||||
|
throw new Error("--resolve requires an existing verified --backup file");
|
||||||
|
}
|
||||||
|
if (path.resolve(backupPath) === path.resolve(sourcePath)) {
|
||||||
|
throw new Error("--backup must be a separate database file, not the source database");
|
||||||
|
}
|
||||||
|
const backup = new Database(backupPath, { readonly: true, fileMustExist: true });
|
||||||
|
const source = new Database(sourcePath, { readonly: true, fileMustExist: true });
|
||||||
|
try {
|
||||||
|
if (backup.pragma("integrity_check", { simple: true }) !== "ok") {
|
||||||
|
throw new Error("Backup failed SQLite integrity_check");
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
JSON.stringify(schemaSnapshot(backup, true)) !==
|
||||||
|
JSON.stringify(schemaSnapshot(source, true))
|
||||||
|
) {
|
||||||
|
throw new Error("Backup schema does not match the source database");
|
||||||
|
}
|
||||||
|
const sourceCounts = Object.fromEntries(
|
||||||
|
schemaSnapshot(source, true).map(({ table }) => [
|
||||||
|
table,
|
||||||
|
source.prepare(`SELECT COUNT(*) AS count FROM ${JSON.stringify(table)}`).get().count,
|
||||||
|
])
|
||||||
|
);
|
||||||
|
const backupCounts = Object.fromEntries(
|
||||||
|
schemaSnapshot(backup, true).map(({ table }) => [
|
||||||
|
table,
|
||||||
|
backup.prepare(`SELECT COUNT(*) AS count FROM ${JSON.stringify(table)}`).get().count,
|
||||||
|
])
|
||||||
|
);
|
||||||
|
if (JSON.stringify(sourceCounts) !== JSON.stringify(backupCounts)) {
|
||||||
|
throw new Error("Backup row counts do not match the source database");
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
backup.close();
|
||||||
|
source.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveMigration(databaseUrl, migrationName) {
|
||||||
|
const cli = path.join(process.cwd(), "node_modules", "prisma", "build", "index.js");
|
||||||
|
const result = spawnSync(
|
||||||
|
process.execPath,
|
||||||
|
[cli, "migrate", "resolve", "--applied", migrationName],
|
||||||
|
{
|
||||||
|
cwd: process.cwd(),
|
||||||
|
env: { ...process.env, DATABASE_URL: databaseUrl },
|
||||||
|
encoding: "utf8",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (result.status !== 0) {
|
||||||
|
throw new Error(`Prisma migration resolve failed:\n${result.stdout}\n${result.stderr}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
const databaseUrl = process.env.DATABASE_URL;
|
||||||
|
const databasePath = databasePathFromUrl(databaseUrl);
|
||||||
|
const startup = process.argv.includes("--startup");
|
||||||
|
const state =
|
||||||
|
startup && !existsSync(databasePath)
|
||||||
|
? {
|
||||||
|
classification: "EMPTY",
|
||||||
|
migrationNames: [],
|
||||||
|
failedMigrationNames: [],
|
||||||
|
migrationsToAdopt: [],
|
||||||
|
}
|
||||||
|
: inspectMigrationState(databasePath);
|
||||||
|
console.info(JSON.stringify(state, null, 2));
|
||||||
|
|
||||||
|
if (process.argv.includes("--resolve")) {
|
||||||
|
if (state.classification !== "ADOPT") {
|
||||||
|
throw new Error(`Refusing adoption for ${state.classification} database state`);
|
||||||
|
}
|
||||||
|
const backupIndex = process.argv.indexOf("--backup");
|
||||||
|
verifyBackup(
|
||||||
|
backupIndex >= 0 ? process.argv[backupIndex + 1] : undefined,
|
||||||
|
databasePath
|
||||||
|
);
|
||||||
|
for (const migrationName of state.migrationsToAdopt) {
|
||||||
|
resolveMigration(databaseUrl, migrationName);
|
||||||
|
}
|
||||||
|
console.info(
|
||||||
|
`Marked ${state.migrationsToAdopt.join(", ")} applied after exact-schema preflight.`
|
||||||
|
);
|
||||||
|
} else if (startup && state.classification === "ADOPT") {
|
||||||
|
console.error(
|
||||||
|
"Database schema requires explicit migration adoption. Stop, create and verify a backup, then run db:preflight -- --resolve --backup <backup-path>."
|
||||||
|
);
|
||||||
|
process.exitCode = 3;
|
||||||
|
} else if (state.classification === "CONFLICT") {
|
||||||
|
process.exitCode = 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
// Runs the scripted OpenAI-compatible mock used by the tests, so the AI features can be tried
|
|
||||||
// locally without a real provider:
|
|
||||||
// npx tsx scripts/mock-ai.ts
|
|
||||||
// Then add an "OpenAI-compatible" provider with URL http://localhost:4010/v1 and key "mock-key".
|
|
||||||
import { startMockAi } from '../tests/mockAi.ts';
|
|
||||||
|
|
||||||
const mock = await startMockAi(Number(process.env.MOCK_AI_PORT ?? 4010));
|
|
||||||
console.log(`Mock AI provider listening at ${mock.url} (API key: mock-key)`);
|
|
||||||
56
scripts/request-password-reset.mjs
Normal file
56
scripts/request-password-reset.mjs
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { pathToFileURL } from "node:url";
|
||||||
|
import Database from "better-sqlite3";
|
||||||
|
import { databasePathFromUrl } from "./databasePath.mjs";
|
||||||
|
|
||||||
|
const PASSWORD_HASH_KEY = "admin_password_hash";
|
||||||
|
const RESET_TOKEN_KEY = "admin_password_reset";
|
||||||
|
const RESET_TOKEN_LIFETIME_MS = 15 * 60 * 1000;
|
||||||
|
|
||||||
|
function activeRecord(value) {
|
||||||
|
try {
|
||||||
|
const record = JSON.parse(value);
|
||||||
|
return new Date(record.expiresAt).getTime() > Date.now();
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createResetToken(databasePath) {
|
||||||
|
if (!existsSync(databasePath)) throw new Error(`Database does not exist: ${databasePath}`);
|
||||||
|
const database = new Database(databasePath, { fileMustExist: true });
|
||||||
|
try {
|
||||||
|
if (!database.prepare('SELECT 1 FROM "Setting" WHERE "key" = ?').get(PASSWORD_HASH_KEY)) {
|
||||||
|
throw new Error("Password recovery is unavailable before initial setup");
|
||||||
|
}
|
||||||
|
const existing = database
|
||||||
|
.prepare('SELECT "value" FROM "Setting" WHERE "key" = ?')
|
||||||
|
.get(RESET_TOKEN_KEY);
|
||||||
|
if (existing && activeRecord(existing.value)) {
|
||||||
|
throw new Error("An unexpired reset token already exists; it was not replaced");
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = randomBytes(24).toString("base64url");
|
||||||
|
const expiresAt = new Date(Date.now() + RESET_TOKEN_LIFETIME_MS).toISOString();
|
||||||
|
const value = JSON.stringify({
|
||||||
|
digest: createHash("sha256").update(token, "utf8").digest("hex"),
|
||||||
|
nonce: randomUUID(),
|
||||||
|
expiresAt,
|
||||||
|
});
|
||||||
|
database
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "Setting" ("key", "value") VALUES (?, ?) ON CONFLICT("key") DO UPDATE SET "value" = excluded."value"'
|
||||||
|
)
|
||||||
|
.run(RESET_TOKEN_KEY, value);
|
||||||
|
return { token, expiresAt };
|
||||||
|
} finally {
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||||
|
const result = createResetToken(databasePathFromUrl(process.env.DATABASE_URL));
|
||||||
|
console.info(`Password reset token: ${result.token}`);
|
||||||
|
console.info(`Expires: ${result.expiresAt}`);
|
||||||
|
}
|
||||||
|
|
@ -1,109 +0,0 @@
|
||||||
// Fills a development instance with sample content. Never run against real data.
|
|
||||||
// node scripts/seed-demo.mjs [http://localhost:3726]
|
|
||||||
// Reads the admin password from ADMIN_PASSWORD (or .env).
|
|
||||||
try {
|
|
||||||
process.loadEnvFile('.env');
|
|
||||||
} catch {
|
|
||||||
/* optional */
|
|
||||||
}
|
|
||||||
const base = process.argv[2] ?? `http://localhost:${process.env.PORT ?? 3726}`;
|
|
||||||
const password = process.env.ADMIN_PASSWORD;
|
|
||||||
if (!password) throw new Error('Set ADMIN_PASSWORD (or put it in .env) to seed demo data.');
|
|
||||||
|
|
||||||
let cookie = '';
|
|
||||||
async function call(method, path, body, form) {
|
|
||||||
const res = await fetch(base + path, {
|
|
||||||
method,
|
|
||||||
headers: { 'X-Requested-With': 'studydesk', ...(cookie ? { cookie } : {}), ...(body && !form ? { 'content-type': 'application/json' } : {}) },
|
|
||||||
body: form ?? (body ? JSON.stringify(body) : undefined),
|
|
||||||
});
|
|
||||||
const set = res.headers.get('set-cookie');
|
|
||||||
if (set) cookie = set.split(';')[0];
|
|
||||||
const text = await res.text();
|
|
||||||
if (!res.ok) throw new Error(`${method} ${path} -> ${res.status} ${text}`);
|
|
||||||
return text ? JSON.parse(text) : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
await call('POST', '/api/auth/login', { password });
|
|
||||||
|
|
||||||
const pharm = (await call('POST', '/api/classes', { name: 'Pharmacology', code: 'NUR 356', description: 'Drug classes, mechanisms, adverse effects and nursing considerations.' })).class;
|
|
||||||
const peds = (await call('POST', '/api/classes', { name: 'Pediatrics', code: 'NUR 410' })).class;
|
|
||||||
const medsurg = (await call('POST', '/api/classes', { name: 'Medical-Surgical Nursing', code: 'NUR 320' })).class;
|
|
||||||
const unit1 = (await call('POST', `/api/classes/${pharm.id}/folders`, { name: 'Unit 1: Cardiovascular' })).folder;
|
|
||||||
|
|
||||||
const q = (type, stem, options, rationale, topic, difficulty = 'medium') => ({ type, stem, options: options.map(([text, correct, why]) => ({ text, correct: Boolean(correct), why: why ?? '' })), rationale, topic, difficulty });
|
|
||||||
const cardiac = [
|
|
||||||
q('single', 'A client taking digoxin reports nausea, fatigue and seeing yellow halos around lights. Which action should the nurse take **first**?', [
|
|
||||||
['Hold the next dose and check the apical pulse and serum digoxin level', 1, 'Visual changes with GI upset are classic early toxicity; assess and hold before giving more.'],
|
|
||||||
['Administer the scheduled dose with food', 0, 'Giving more drug worsens toxicity.'],
|
|
||||||
['Reassure the client that these effects are expected', 0, 'These are signs of toxicity, not expected effects.'],
|
|
||||||
['Encourage oral fluids and reassess in 4 hours', 0, 'Delays recognition of a potentially life-threatening problem.'],
|
|
||||||
], 'Digoxin has a narrow therapeutic range (0.5 to 2 ng/mL). Early toxicity presents with **GI symptoms and visual disturbances**; dysrhythmias follow. Hold the drug, assess the apical pulse for a full minute, and obtain a level.', 'Cardiac glycosides'),
|
|
||||||
q('multi', 'A nurse is teaching a client who is starting furosemide. Which statements should be included? **Select all that apply.**', [
|
|
||||||
['Take the medication in the morning', 1, 'Prevents nocturia.'],
|
|
||||||
['Eat foods high in potassium, such as bananas and potatoes', 1, 'Loop diuretics waste potassium.'],
|
|
||||||
['Rise slowly from sitting or lying down', 1, 'Volume depletion causes orthostatic hypotension.'],
|
|
||||||
['Expect a weight gain of 1 kg per day', 0, 'Weight should fall or stay stable; a gain suggests fluid retention.'],
|
|
||||||
['Report ringing in the ears', 1, 'Ototoxicity is a known adverse effect.'],
|
|
||||||
], 'Furosemide is a loop diuretic. Key teaching: morning dosing, potassium intake, orthostatic precautions, daily weights, and reporting tinnitus or hearing changes.', 'Diuretics'),
|
|
||||||
q('single', 'Which laboratory value should the nurse review before administering lisinopril?', [
|
|
||||||
['Serum potassium', 1, 'ACE inhibitors reduce aldosterone and can cause hyperkalemia.'],
|
|
||||||
['Serum calcium', 0],
|
|
||||||
['Platelet count', 0],
|
|
||||||
['Serum amylase', 0],
|
|
||||||
], 'ACE inhibitors can cause **hyperkalemia** and a rise in creatinine. Review potassium and renal function, and monitor blood pressure.', 'ACE inhibitors', 'easy'),
|
|
||||||
q('single', 'A client receiving a heparin infusion has an aPTT of 120 seconds (control 30 seconds). Which action is appropriate?', [
|
|
||||||
['Stop the infusion and notify the provider', 1, 'The aPTT is four times control; the therapeutic goal is 1.5 to 2.5 times control.'],
|
|
||||||
['Increase the infusion rate per protocol', 0, 'Would increase bleeding risk.'],
|
|
||||||
['Continue the infusion and recheck in 6 hours', 0],
|
|
||||||
['Administer vitamin K', 0, 'Vitamin K reverses warfarin. Protamine sulfate reverses heparin.'],
|
|
||||||
], 'Therapeutic aPTT on heparin is about **1.5 to 2.5 times control**. A value of 120 seconds is supratherapeutic. The antidote is protamine sulfate.', 'Anticoagulants', 'hard'),
|
|
||||||
q('multi', 'Which findings indicate a therapeutic response to metoprolol? **Select all that apply.**', [
|
|
||||||
['Heart rate decreases from 104 to 78/min', 1],
|
|
||||||
['Blood pressure decreases from 162/94 to 134/82 mm Hg', 1],
|
|
||||||
['Client reports fewer episodes of chest pain', 1],
|
|
||||||
['Wheezing is heard on auscultation', 0, 'Bronchospasm is an adverse effect.'],
|
|
||||||
], 'Beta-1 blockade lowers heart rate, contractility and blood pressure, reducing myocardial oxygen demand.', 'Beta blockers'),
|
|
||||||
q('single', 'A client taking warfarin has an INR of 5.8 with no bleeding. Which prescription should the nurse anticipate?', [
|
|
||||||
['Hold warfarin and administer oral vitamin K', 1],
|
|
||||||
['Administer protamine sulfate', 0, 'Protamine reverses heparin.'],
|
|
||||||
['Increase the warfarin dose', 0],
|
|
||||||
['Administer fresh frozen plasma immediately', 0, 'Reserved for serious bleeding.'],
|
|
||||||
], 'The usual target INR is 2 to 3. For a markedly elevated INR without bleeding, hold warfarin and give low-dose oral vitamin K (phytonadione).', 'Anticoagulants'),
|
|
||||||
];
|
|
||||||
const cardiacQuiz = (await call('POST', '/api/quizzes', { classId: pharm.id, folderId: unit1.id, title: 'Cardiovascular medications', description: 'Glycosides, diuretics, ACE inhibitors, beta blockers and anticoagulants.', questions: cardiac })).quiz;
|
|
||||||
await call('POST', '/api/quizzes', { classId: pharm.id, title: 'Antibiotics check', questions: [
|
|
||||||
q('single', 'Which assessment is the priority before the first dose of IV penicillin?', [['Ask about previous allergic reactions', 1], ['Measure urine output', 0], ['Check blood glucose', 0], ['Assess deep tendon reflexes', 0]], 'Penicillins are a leading cause of drug anaphylaxis.', 'Antibiotics', 'easy'),
|
|
||||||
q('single', 'A client on gentamicin reports new ringing in the ears. The nurse should suspect:', [['Ototoxicity', 1], ['Hepatotoxicity', 0], ['Superinfection', 0], ['Photosensitivity', 0]], 'Aminoglycosides are ototoxic and nephrotoxic. Monitor peak and trough levels.', 'Antibiotics'),
|
|
||||||
] });
|
|
||||||
|
|
||||||
const cards = [
|
|
||||||
['Furosemide: class', '**Loop diuretic**'], ['Furosemide: key adverse effects', '- Hypokalemia\n- Ototoxicity\n- Orthostatic hypotension\n- Dehydration'],
|
|
||||||
['Digoxin: therapeutic range', '**0.5 to 2 ng/mL**'], ['Digoxin toxicity: early signs', 'Anorexia, nausea, vomiting, fatigue, **yellow-green halos**'],
|
|
||||||
['Antidote for heparin', 'Protamine sulfate'], ['Antidote for warfarin', 'Vitamin K (phytonadione)'],
|
|
||||||
['Lisinopril: suffix and class', '**-pril** = ACE inhibitor'], ['ACE inhibitor: classic adverse effects', 'Dry **cough**, hyperkalemia, angioedema, first-dose hypotension'],
|
|
||||||
['Metoprolol: hold when', 'Heart rate below **60/min** or systolic BP below 100 mm Hg (per protocol)'], ['Normal serum potassium', '3.5 to 5.0 mEq/L'],
|
|
||||||
['Therapeutic INR on warfarin', '2 to 3 (2.5 to 3.5 for mechanical valves)'], ['Spironolactone: class', 'Potassium-sparing diuretic (aldosterone antagonist)'],
|
|
||||||
].map(([front, back]) => ({ front, back }));
|
|
||||||
const deck = (await call('POST', '/api/decks', { classId: pharm.id, folderId: unit1.id, title: 'Cardiac drug cards', description: 'High-yield facts for exam 1.', cards })).deck;
|
|
||||||
await call('POST', '/api/decks', { classId: peds.id, title: 'Growth and development milestones', cards: [
|
|
||||||
{ front: 'Age an infant sits without support', back: 'About **6 to 8 months**' }, { front: 'Age of first words', back: 'About 12 months' }, { front: 'Erikson stage: toddler', back: 'Autonomy vs. shame and doubt' },
|
|
||||||
] });
|
|
||||||
|
|
||||||
await call('POST', '/api/notes', { classId: pharm.id, title: 'Exam 1 plan', content: '## Topics\n- Cardiac glycosides\n- Diuretics\n- Anticoagulants\n\n| Drug | Antidote |\n| --- | --- |\n| Heparin | Protamine sulfate |\n| Warfarin | Vitamin K |\n' });
|
|
||||||
await call('POST', '/api/documents/text', { classId: pharm.id, title: 'Lecture 3: Heart failure drugs', text: '# Heart failure pharmacology\n\n## ACE inhibitors\nACE inhibitors such as lisinopril block the conversion of angiotensin I to angiotensin II. They reduce afterload and preload and slow ventricular remodeling. Adverse effects include a dry cough, hyperkalemia, angioedema and first-dose hypotension. Monitor potassium and creatinine.\n\n## Beta blockers\nMetoprolol succinate, carvedilol and bisoprolol reduce mortality in heart failure with reduced ejection fraction. Start low and titrate slowly. Hold for a heart rate below 60 per minute.\n\n## Loop diuretics\nFurosemide relieves congestion by inhibiting sodium and chloride reabsorption in the loop of Henle. Monitor daily weights, intake and output, potassium, and hearing.\n\n## Digoxin\nDigoxin increases contractility and slows conduction through the AV node. The therapeutic range is 0.5 to 2 ng/mL. Hypokalemia increases the risk of toxicity. Early signs of toxicity are anorexia, nausea, vomiting and visual disturbances such as yellow halos.\n' });
|
|
||||||
|
|
||||||
// Some study history so dashboards have something to show.
|
|
||||||
const attempt = (await call('POST', '/api/attempts', { mode: 'practice', source: { type: 'quiz', quizId: cardiacQuiz.id }, shuffleQuestions: false, shuffleOptions: false })).attempt;
|
|
||||||
const answers = {};
|
|
||||||
attempt.questions.forEach((question, i) => {
|
|
||||||
const right = question.options.filter((o) => o.correct).map((o) => o.id);
|
|
||||||
const wrong = question.options.filter((o) => !o.correct).map((o) => o.id);
|
|
||||||
answers[question.id] = { selected: i === 1 ? right.slice(0, 2) : i === 3 ? wrong.slice(0, 1) : right, checked: true };
|
|
||||||
});
|
|
||||||
await call('PATCH', `/api/attempts/${attempt.id}`, { answers, elapsedSec: 312 });
|
|
||||||
await call('POST', `/api/attempts/${attempt.id}/complete`, { elapsedSec: 312 });
|
|
||||||
const queue = await call('GET', `/api/review/queue?deckId=${deck.id}`);
|
|
||||||
for (const [i, item] of queue.cards.slice(0, 7).entries()) await call('POST', '/api/review/answer', { cardId: item.card.id, rating: i % 4 === 0 ? 1 : 3 });
|
|
||||||
|
|
||||||
console.log(`Seeded demo data at ${base}: ${[pharm, peds, medsurg].map((c) => c.name).join(', ')}`);
|
|
||||||
197
scripts/verify-backup-restore.mjs
Normal file
197
scripts/verify-backup-restore.mjs
Normal file
|
|
@ -0,0 +1,197 @@
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import {
|
||||||
|
copyFileSync,
|
||||||
|
existsSync,
|
||||||
|
mkdirSync,
|
||||||
|
readFileSync,
|
||||||
|
readdirSync,
|
||||||
|
rmSync,
|
||||||
|
statSync,
|
||||||
|
} from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import Database from "better-sqlite3";
|
||||||
|
import { backupDatabase } from "./backup-database.mjs";
|
||||||
|
|
||||||
|
const root = path.join(process.cwd(), ".test-databases");
|
||||||
|
const directory = path.join(root, `backup-restore-${randomUUID()}`);
|
||||||
|
if (!path.resolve(directory).startsWith(`${path.resolve(root)}${path.sep}`)) {
|
||||||
|
throw new Error("Unsafe disposable backup verification path");
|
||||||
|
}
|
||||||
|
mkdirSync(directory, { recursive: true });
|
||||||
|
const sourcePath = path.join(directory, "source.test.db");
|
||||||
|
const backupPath = path.join(directory, "backup.test.db");
|
||||||
|
const restoredPath = path.join(directory, "restored.test.db");
|
||||||
|
let source;
|
||||||
|
|
||||||
|
function rowCounts(database) {
|
||||||
|
const tables = database
|
||||||
|
.prepare(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
.map((row) => row.name);
|
||||||
|
return Object.fromEntries(
|
||||||
|
tables.map((table) => [
|
||||||
|
table,
|
||||||
|
database.prepare(`SELECT COUNT(*) AS count FROM ${JSON.stringify(table)}`).get().count,
|
||||||
|
])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
source = new Database(sourcePath);
|
||||||
|
source.pragma("foreign_keys = ON");
|
||||||
|
source.pragma("journal_mode = WAL");
|
||||||
|
source.pragma("wal_autocheckpoint = 0");
|
||||||
|
const migrations = readdirSync(path.join(process.cwd(), "prisma", "migrations"), {
|
||||||
|
withFileTypes: true,
|
||||||
|
})
|
||||||
|
.filter((entry) => entry.isDirectory())
|
||||||
|
.map((entry) => entry.name)
|
||||||
|
.sort();
|
||||||
|
for (const migration of migrations) {
|
||||||
|
source.exec(
|
||||||
|
readFileSync(
|
||||||
|
path.join(process.cwd(), "prisma", "migrations", migration, "migration.sql"),
|
||||||
|
"utf8"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
source.transaction(() => {
|
||||||
|
source
|
||||||
|
.prepare('INSERT INTO "Class" ("id", "name", "slug", "sortOrder") VALUES (?, ?, ?, ?)')
|
||||||
|
.run("backup-class", "Representative content", "representative-content", 0);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "MaterialGroup" ("id", "classId", "name", "type", "sortOrder") VALUES (?, ?, ?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run("backup-group", "backup-class", "Backup group", "DECK", 0);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "Deck" ("id", "classId", "groupId", "name", "description", "sortOrder") VALUES (?, ?, ?, ?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run(
|
||||||
|
"backup-deck",
|
||||||
|
"backup-class",
|
||||||
|
"backup-group",
|
||||||
|
"Backup deck",
|
||||||
|
"Restore drill content",
|
||||||
|
0
|
||||||
|
);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "Flashcard" ("id", "deckId", "front", "back", "sortOrder") VALUES (?, ?, ?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run("backup-card", "backup-deck", "Representative front", "Representative back", 0);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "QuizSet" ("id", "classId", "name", "description", "sortOrder") VALUES (?, ?, ?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run("backup-quiz", "backup-class", "Backup quiz", "Quiz relationship", 0);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "Question" ("id", "quizSetId", "type", "prompt", "rationale", "category", "sortOrder") VALUES (?, ?, ?, ?, ?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run(
|
||||||
|
"backup-question",
|
||||||
|
"backup-quiz",
|
||||||
|
"MULTIPLE_CHOICE",
|
||||||
|
"Representative prompt",
|
||||||
|
"Representative rationale",
|
||||||
|
"backup",
|
||||||
|
0
|
||||||
|
);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "AnswerOption" ("id", "questionId", "text", "isCorrect", "sortOrder") VALUES (?, ?, ?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run("backup-option", "backup-question", "Correct option", 1, 0);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "SpacedRepetitionSet" ("id", "classId", "name", "sortOrder", "updatedAt") VALUES (?, ?, ?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run("backup-srs", "backup-class", "Backup SRS", 0, Date.now());
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "SpacedRepetitionSetDeck" ("setId", "deckId", "sortOrder") VALUES (?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run("backup-srs", "backup-deck", 0);
|
||||||
|
source
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO "ShareLink" ("id", "targetType", "groupId") VALUES (?, ?, ?)'
|
||||||
|
)
|
||||||
|
.run("backup-share", "GROUP", "backup-group");
|
||||||
|
})();
|
||||||
|
|
||||||
|
const walPath = `${sourcePath}-wal`;
|
||||||
|
if (!existsSync(walPath) || statSync(walPath).size === 0) {
|
||||||
|
throw new Error("Backup drill did not create an active WAL database");
|
||||||
|
}
|
||||||
|
const sourceWalBytes = statSync(walPath).size;
|
||||||
|
const sourceCounts = rowCounts(source);
|
||||||
|
await backupDatabase(sourcePath, backupPath);
|
||||||
|
copyFileSync(backupPath, restoredPath);
|
||||||
|
source.close();
|
||||||
|
source = undefined;
|
||||||
|
|
||||||
|
const restored = new Database(restoredPath, { readonly: true, fileMustExist: true });
|
||||||
|
try {
|
||||||
|
const integrity = restored.pragma("integrity_check", { simple: true });
|
||||||
|
const foreignKeyErrors = restored.pragma("foreign_key_check");
|
||||||
|
const restoredCounts = rowCounts(restored);
|
||||||
|
const representative = restored
|
||||||
|
.prepare(`
|
||||||
|
SELECT c.name AS className, g.name AS groupName, d.description,
|
||||||
|
f.front, s.name AS setName, sl.targetType
|
||||||
|
FROM "Class" c
|
||||||
|
JOIN "MaterialGroup" g ON g.classId = c.id
|
||||||
|
JOIN "Deck" d ON d.groupId = g.id
|
||||||
|
JOIN "Flashcard" f ON f.deckId = d.id
|
||||||
|
JOIN "SpacedRepetitionSetDeck" sd ON sd.deckId = d.id
|
||||||
|
JOIN "SpacedRepetitionSet" s ON s.id = sd.setId
|
||||||
|
JOIN "ShareLink" sl ON sl.groupId = g.id
|
||||||
|
WHERE d.id = ?
|
||||||
|
`)
|
||||||
|
.get("backup-deck");
|
||||||
|
const quizRelationship = restored
|
||||||
|
.prepare(`
|
||||||
|
SELECT q.name AS quizName, question.prompt, option.text, option.isCorrect
|
||||||
|
FROM "QuizSet" q
|
||||||
|
JOIN "Question" question ON question.quizSetId = q.id
|
||||||
|
JOIN "AnswerOption" option ON option.questionId = question.id
|
||||||
|
WHERE q.id = ?
|
||||||
|
`)
|
||||||
|
.get("backup-quiz");
|
||||||
|
if (
|
||||||
|
integrity !== "ok" ||
|
||||||
|
foreignKeyErrors.length !== 0 ||
|
||||||
|
JSON.stringify(restoredCounts) !== JSON.stringify(sourceCounts) ||
|
||||||
|
representative?.description !== "Restore drill content" ||
|
||||||
|
representative?.setName !== "Backup SRS" ||
|
||||||
|
representative?.targetType !== "GROUP" ||
|
||||||
|
quizRelationship?.isCorrect !== 1
|
||||||
|
) {
|
||||||
|
throw new Error("Restored database did not preserve integrity, counts, content, and relationships");
|
||||||
|
}
|
||||||
|
console.info(
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
method: "better-sqlite3 online backup while the WAL source remained open",
|
||||||
|
sourceWalBytes,
|
||||||
|
integrity,
|
||||||
|
foreignKeyErrors,
|
||||||
|
counts: restoredCounts,
|
||||||
|
representative,
|
||||||
|
quizRelationship,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
restored.close();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
source?.close();
|
||||||
|
rmSync(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
83
scripts/verify-http-smoke.mjs
Normal file
83
scripts/verify-http-smoke.mjs
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
import { spawn } from "node:child_process";
|
||||||
|
import Database from "better-sqlite3";
|
||||||
|
|
||||||
|
const root = path.join(process.cwd(), ".test-databases");
|
||||||
|
const directory = path.join(root, `http-smoke-${randomUUID()}`);
|
||||||
|
if (!path.resolve(directory).startsWith(`${path.resolve(root)}${path.sep}`)) {
|
||||||
|
throw new Error("Unsafe disposable HTTP smoke path");
|
||||||
|
}
|
||||||
|
mkdirSync(directory, { recursive: true });
|
||||||
|
const databasePath = path.join(directory, "http-smoke.test.db");
|
||||||
|
const database = new Database(databasePath);
|
||||||
|
try {
|
||||||
|
database.pragma("foreign_keys = ON");
|
||||||
|
for (const migration of readdirSync(path.join(process.cwd(), "prisma", "migrations"), { withFileTypes: true })
|
||||||
|
.filter((entry) => entry.isDirectory())
|
||||||
|
.map((entry) => entry.name)
|
||||||
|
.sort()) {
|
||||||
|
database.exec(readFileSync(path.join(process.cwd(), "prisma", "migrations", migration, "migration.sql"), "utf8"));
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
database.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
const port = 3789;
|
||||||
|
const server = spawn(
|
||||||
|
process.execPath,
|
||||||
|
[path.join(process.cwd(), "node_modules", "next", "dist", "bin", "next"), "start", "-p", String(port)],
|
||||||
|
{
|
||||||
|
cwd: process.cwd(),
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
NODE_ENV: "production",
|
||||||
|
DATABASE_URL: `file:${databasePath.replaceAll("\\", "/")}`,
|
||||||
|
SESSION_SECRET: "http-smoke-secret-0123456789abcdef0123456789",
|
||||||
|
ALLOW_INITIAL_SETUP: "true",
|
||||||
|
},
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
windowsHide: true,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let output = "";
|
||||||
|
server.stdout.on("data", (chunk) => { output += chunk.toString(); });
|
||||||
|
server.stderr.on("data", (chunk) => { output += chunk.toString(); });
|
||||||
|
|
||||||
|
try {
|
||||||
|
let health;
|
||||||
|
for (let attempt = 0; attempt < 30; attempt += 1) {
|
||||||
|
try {
|
||||||
|
health = await fetch(`http://127.0.0.1:${port}/api/health`);
|
||||||
|
if (health.ok) break;
|
||||||
|
} catch {}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||||
|
}
|
||||||
|
if (!health?.ok) throw new Error(`Health check did not become ready.\n${output}`);
|
||||||
|
const login = await fetch(`http://127.0.0.1:${port}/login`);
|
||||||
|
const setup = await fetch(`http://127.0.0.1:${port}/api/auth/setup-status`);
|
||||||
|
const setupBody = await setup.json();
|
||||||
|
const protectedDot = await fetch(`http://127.0.0.1:${port}/api/decks/file.json`, { redirect: "manual" });
|
||||||
|
if (!login.ok || !setup.ok || setupBody.setupRequired !== true || setupBody.setupAllowed !== true || protectedDot.status !== 307) {
|
||||||
|
throw new Error("HTTP smoke responses did not match the production contract");
|
||||||
|
}
|
||||||
|
console.info(JSON.stringify({
|
||||||
|
health: health.status,
|
||||||
|
login: login.status,
|
||||||
|
setup: setupBody,
|
||||||
|
protectedDot: { status: protectedDot.status, location: protectedDot.headers.get("location") },
|
||||||
|
}, null, 2));
|
||||||
|
if (process.argv.includes("--stay")) {
|
||||||
|
console.info(`Browser smoke server ready at http://127.0.0.1:${port}/login`);
|
||||||
|
await new Promise(() => {});
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
server.kill();
|
||||||
|
await new Promise((resolve) => {
|
||||||
|
if (server.exitCode !== null) return resolve();
|
||||||
|
server.once("exit", resolve);
|
||||||
|
setTimeout(resolve, 2_000);
|
||||||
|
});
|
||||||
|
rmSync(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,883 +0,0 @@
|
||||||
/**
|
|
||||||
* Content generation: quizzes, flashcards and study guides from documents, pasted text,
|
|
||||||
* existing decks/quizzes, weak areas, or a plain topic.
|
|
||||||
*
|
|
||||||
* Every job writes into a draft row. Nothing reaches the library until the user has
|
|
||||||
* reviewed the draft and saved it. Long jobs are split into batches, each with its own
|
|
||||||
* slice of source material, so output limits and context windows are never a problem.
|
|
||||||
*/
|
|
||||||
import { z } from 'zod';
|
|
||||||
import {
|
|
||||||
validateDraftCard,
|
|
||||||
validateDraftQuestion,
|
|
||||||
type ContentDraft,
|
|
||||||
type DeckDraft,
|
|
||||||
type DraftCard,
|
|
||||||
type DraftQuestion,
|
|
||||||
type NoteDraft,
|
|
||||||
type QuizDraft,
|
|
||||||
} from '../../shared/content.ts';
|
|
||||||
import { draftsFromValue } from '../../shared/importer.ts';
|
|
||||||
import type { DB } from '../db.ts';
|
|
||||||
import { badRequest, notFound, type AppContext } from '../http.ts';
|
|
||||||
import { missedQuestionIds } from '../services/attempts.ts';
|
|
||||||
import {
|
|
||||||
createDeck,
|
|
||||||
createNote,
|
|
||||||
createQuiz,
|
|
||||||
draftCardToInput,
|
|
||||||
draftQuestionToInput,
|
|
||||||
getClass,
|
|
||||||
getDeck,
|
|
||||||
getQuiz,
|
|
||||||
insertCards,
|
|
||||||
insertQuestions,
|
|
||||||
listCards,
|
|
||||||
listQuestions,
|
|
||||||
updateQuestion,
|
|
||||||
} from '../services/content.ts';
|
|
||||||
import { chunkById, formatExcerpts, searchChunks, selectContext, sourceLabel, type ChunkHit } from '../services/documents.ts';
|
|
||||||
import { weakTopics } from '../services/stats.ts';
|
|
||||||
import { newId, now, parseJson } from '../util.ts';
|
|
||||||
import { generateStructured, resolveModel, runChat } from './service.ts';
|
|
||||||
import { AIError } from './types.ts';
|
|
||||||
|
|
||||||
/* ------------------------------ drafts ------------------------------ */
|
|
||||||
|
|
||||||
interface DraftRow {
|
|
||||||
id: string;
|
|
||||||
kind: 'quiz' | 'deck' | 'note';
|
|
||||||
class_id: string | null;
|
|
||||||
folder_id: string | null;
|
|
||||||
title: string;
|
|
||||||
payload: string;
|
|
||||||
meta: string;
|
|
||||||
status: 'generating' | 'ready' | 'failed' | 'saved' | 'discarded';
|
|
||||||
progress: number;
|
|
||||||
progress_label: string;
|
|
||||||
error: string;
|
|
||||||
saved_kind: string | null;
|
|
||||||
saved_id: string | null;
|
|
||||||
conversation_id: string | null;
|
|
||||||
created_at: number;
|
|
||||||
updated_at: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface DraftMeta {
|
|
||||||
origin?: 'generate' | 'assistant' | 'convert' | 'improve' | 'import';
|
|
||||||
model?: string;
|
|
||||||
request?: Record<string, unknown>;
|
|
||||||
sources?: { documentId: string; title: string }[];
|
|
||||||
/** For "improve" drafts: the stored question each draft question replaces (same order). */
|
|
||||||
replaceQuestionIds?: string[];
|
|
||||||
/** Suggested target when the draft is meant to extend an existing item. */
|
|
||||||
targetKind?: 'quiz' | 'deck';
|
|
||||||
targetId?: string;
|
|
||||||
notes?: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
function emptyPayload(kind: DraftRow['kind'], title = ''): ContentDraft {
|
|
||||||
if (kind === 'quiz') return { kind: 'quiz', title, description: '', questions: [] };
|
|
||||||
if (kind === 'deck') return { kind: 'deck', title, description: '', cards: [] };
|
|
||||||
return { kind: 'note', title, content: '' };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function mapDraft(row: DraftRow) {
|
|
||||||
const payload = parseJson<ContentDraft>(row.payload, emptyPayload(row.kind, row.title));
|
|
||||||
return {
|
|
||||||
id: row.id,
|
|
||||||
kind: row.kind,
|
|
||||||
classId: row.class_id,
|
|
||||||
folderId: row.folder_id,
|
|
||||||
title: row.title,
|
|
||||||
payload,
|
|
||||||
meta: parseJson<DraftMeta>(row.meta, {}),
|
|
||||||
status: row.status,
|
|
||||||
progress: row.progress,
|
|
||||||
progressLabel: row.progress_label,
|
|
||||||
error: row.error,
|
|
||||||
savedKind: row.saved_kind,
|
|
||||||
savedId: row.saved_id,
|
|
||||||
conversationId: row.conversation_id,
|
|
||||||
itemCount: payload.kind === 'quiz' ? payload.questions.length : payload.kind === 'deck' ? payload.cards.length : payload.content.length,
|
|
||||||
createdAt: row.created_at,
|
|
||||||
updatedAt: row.updated_at,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
export type DraftDto = ReturnType<typeof mapDraft>;
|
|
||||||
|
|
||||||
export function getDraft(db: DB, id: string): DraftDto {
|
|
||||||
const row = db.get<DraftRow>('SELECT * FROM drafts WHERE id = ?', id);
|
|
||||||
if (!row) throw notFound('Draft');
|
|
||||||
return mapDraft(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function createDraft(
|
|
||||||
db: DB,
|
|
||||||
input: { kind: DraftRow['kind']; classId?: string | null; folderId?: string | null; title: string; payload?: ContentDraft; meta?: DraftMeta; status?: DraftRow['status']; conversationId?: string | null },
|
|
||||||
): string {
|
|
||||||
const id = newId();
|
|
||||||
const at = now();
|
|
||||||
db.run(
|
|
||||||
`INSERT INTO drafts (id, kind, class_id, folder_id, title, payload, meta, status, conversation_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
||||||
id,
|
|
||||||
input.kind,
|
|
||||||
input.classId ?? null,
|
|
||||||
input.folderId ?? null,
|
|
||||||
input.title.slice(0, 200),
|
|
||||||
JSON.stringify(input.payload ?? emptyPayload(input.kind, input.title)),
|
|
||||||
JSON.stringify(input.meta ?? {}),
|
|
||||||
input.status ?? 'ready',
|
|
||||||
input.conversationId ?? null,
|
|
||||||
at,
|
|
||||||
at,
|
|
||||||
);
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function listDrafts(db: DB) {
|
|
||||||
return db
|
|
||||||
.all<DraftRow>(`SELECT * FROM drafts WHERE status IN ('generating', 'ready', 'failed') ORDER BY updated_at DESC LIMIT 50`)
|
|
||||||
.map(mapDraft)
|
|
||||||
.map(({ payload: _payload, ...rest }) => rest);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function updateDraftPayload(db: DB, id: string, payload: ContentDraft, extra: { classId?: string | null; folderId?: string | null } = {}): DraftDto {
|
|
||||||
const draft = getDraft(db, id);
|
|
||||||
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
|
|
||||||
if (payload.kind !== draft.kind) throw badRequest('Draft type cannot change');
|
|
||||||
db.run(
|
|
||||||
'UPDATE drafts SET payload = ?, title = ?, class_id = ?, folder_id = ?, updated_at = ? WHERE id = ?',
|
|
||||||
JSON.stringify(payload),
|
|
||||||
(payload.title ?? '').slice(0, 200),
|
|
||||||
extra.classId === undefined ? draft.classId : extra.classId,
|
|
||||||
extra.folderId === undefined ? draft.folderId : extra.folderId,
|
|
||||||
now(),
|
|
||||||
id,
|
|
||||||
);
|
|
||||||
return getDraft(db, id);
|
|
||||||
}
|
|
||||||
|
|
||||||
export const draftSaveSchema = z.object({
|
|
||||||
classId: z.string().optional(),
|
|
||||||
folderId: z.string().nullish(),
|
|
||||||
/** new: create an item. append: add to an existing quiz/deck. replace: apply improved questions in place. */
|
|
||||||
mode: z.enum(['new', 'append', 'replace']).default('new'),
|
|
||||||
targetId: z.string().optional(),
|
|
||||||
});
|
|
||||||
|
|
||||||
/** Turns a reviewed draft into real content. Invalid items are rejected, never silently dropped. */
|
|
||||||
export function saveDraft(app: AppContext, id: string, input: z.input<typeof draftSaveSchema>): { kind: string; id: string; count: number } {
|
|
||||||
const { db } = app;
|
|
||||||
const opts = draftSaveSchema.parse(input);
|
|
||||||
const draft = getDraft(db, id);
|
|
||||||
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
|
|
||||||
if (draft.status === 'saved' && draft.savedId) return { kind: draft.savedKind ?? draft.kind, id: draft.savedId, count: 0 };
|
|
||||||
const payload = draft.payload;
|
|
||||||
const classId = opts.classId ?? draft.classId;
|
|
||||||
let result: { kind: string; id: string; count: number };
|
|
||||||
|
|
||||||
if (payload.kind === 'quiz') {
|
|
||||||
const bad = payload.questions.flatMap((q, i) => validateDraftQuestion(q, i)).filter((x) => x.level === 'error');
|
|
||||||
if (bad.length > 0) throw badRequest(`Question ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
|
|
||||||
if (payload.questions.length === 0) throw badRequest('There are no questions to save');
|
|
||||||
const inputs = payload.questions.map(draftQuestionToInput);
|
|
||||||
if (opts.mode === 'replace') {
|
|
||||||
const ids = draft.meta.replaceQuestionIds ?? [];
|
|
||||||
const quizId = opts.targetId ?? draft.meta.targetId;
|
|
||||||
if (!quizId) throw badRequest('No quiz to update');
|
|
||||||
getQuiz(db, quizId);
|
|
||||||
db.tx(() => {
|
|
||||||
inputs.forEach((q, i) => {
|
|
||||||
const target = ids[i];
|
|
||||||
if (target && db.get('SELECT id FROM questions WHERE id = ? AND quiz_id = ?', target, quizId)) updateQuestion(db, target, q);
|
|
||||||
else insertQuestions(db, quizId, [q]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
result = { kind: 'quiz', id: quizId, count: inputs.length };
|
|
||||||
} else if (opts.mode === 'append') {
|
|
||||||
const quizId = opts.targetId ?? draft.meta.targetId;
|
|
||||||
if (!quizId) throw badRequest('Choose a quiz to add these questions to');
|
|
||||||
getQuiz(db, quizId);
|
|
||||||
insertQuestions(db, quizId, inputs);
|
|
||||||
result = { kind: 'quiz', id: quizId, count: inputs.length };
|
|
||||||
} else {
|
|
||||||
if (!classId) throw badRequest('Choose a class to save this quiz in');
|
|
||||||
const quiz = createQuiz(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled quiz', description: payload.description, questions: inputs });
|
|
||||||
result = { kind: 'quiz', id: quiz.id, count: inputs.length };
|
|
||||||
}
|
|
||||||
} else if (payload.kind === 'deck') {
|
|
||||||
const bad = payload.cards.flatMap((c, i) => validateDraftCard(c, i)).filter((x) => x.level === 'error');
|
|
||||||
if (bad.length > 0) throw badRequest(`Card ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
|
|
||||||
if (payload.cards.length === 0) throw badRequest('There are no cards to save');
|
|
||||||
const inputs = payload.cards.map(draftCardToInput);
|
|
||||||
if (opts.mode === 'append') {
|
|
||||||
const deckId = opts.targetId ?? draft.meta.targetId;
|
|
||||||
if (!deckId) throw badRequest('Choose a deck to add these cards to');
|
|
||||||
getDeck(db, deckId);
|
|
||||||
insertCards(db, deckId, inputs);
|
|
||||||
result = { kind: 'deck', id: deckId, count: inputs.length };
|
|
||||||
} else {
|
|
||||||
if (!classId) throw badRequest('Choose a class to save this deck in');
|
|
||||||
const deck = createDeck(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled deck', description: payload.description, cards: inputs });
|
|
||||||
result = { kind: 'deck', id: deck.id, count: inputs.length };
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (!payload.content.trim()) throw badRequest('The note is empty');
|
|
||||||
if (!classId) throw badRequest('Choose a class to save this note in');
|
|
||||||
const note = createNote(db, {
|
|
||||||
classId,
|
|
||||||
folderId: opts.folderId ?? draft.folderId,
|
|
||||||
title: payload.title.trim() || 'Untitled note',
|
|
||||||
content: payload.content,
|
|
||||||
kind: 'guide',
|
|
||||||
source: { model: draft.meta.model, sources: draft.meta.sources },
|
|
||||||
});
|
|
||||||
result = { kind: 'note', id: note.id, count: 1 };
|
|
||||||
}
|
|
||||||
db.run(`UPDATE drafts SET status = 'saved', saved_kind = ?, saved_id = ?, updated_at = ? WHERE id = ?`, result.kind, result.id, now(), id);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function discardDraft(db: DB, id: string): void {
|
|
||||||
cancelGeneration(id);
|
|
||||||
const { changes } = db.run(`UPDATE drafts SET status = 'discarded', updated_at = ? WHERE id = ?`, now(), id);
|
|
||||||
if (!changes) throw notFound('Draft');
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Removes finished drafts after a while and marks jobs interrupted by a restart as failed. */
|
|
||||||
export function tidyDrafts(db: DB): void {
|
|
||||||
db.run(`UPDATE drafts SET status = 'failed', error = 'Generation was interrupted by a restart. Start it again.' WHERE status = 'generating'`);
|
|
||||||
db.run(`DELETE FROM drafts WHERE status IN ('saved', 'discarded') AND updated_at < ?`, now() - 14 * 86_400_000);
|
|
||||||
db.run(`DELETE FROM drafts WHERE status IN ('ready', 'failed') AND updated_at < ?`, now() - 60 * 86_400_000);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ request ------------------------------ */
|
|
||||||
|
|
||||||
export const generateRequestSchema = z.object({
|
|
||||||
kind: z.enum(['quiz', 'deck', 'note']),
|
|
||||||
classId: z.string().nullish(),
|
|
||||||
folderId: z.string().nullish(),
|
|
||||||
title: z.string().trim().max(200).optional(),
|
|
||||||
source: z
|
|
||||||
.object({
|
|
||||||
documentIds: z.array(z.string()).max(50).optional(),
|
|
||||||
/** Use every processed document in the class. */
|
|
||||||
allClassDocuments: z.boolean().optional(),
|
|
||||||
text: z.string().max(400_000).optional(),
|
|
||||||
quizId: z.string().optional(),
|
|
||||||
deckId: z.string().optional(),
|
|
||||||
noteId: z.string().optional(),
|
|
||||||
/** Build from the questions the user keeps missing. */
|
|
||||||
weakAreas: z.boolean().optional(),
|
|
||||||
})
|
|
||||||
.default({}),
|
|
||||||
/** Topic or focus, e.g. "pediatric respiratory disorders". */
|
|
||||||
focus: z.string().trim().max(600).optional(),
|
|
||||||
instructions: z.string().trim().max(4000).optional(),
|
|
||||||
count: z.number().int().min(1).max(200).optional(),
|
|
||||||
style: z.enum(['nclex', 'standard', 'recall']).optional(),
|
|
||||||
difficulty: z.enum(['easy', 'medium', 'hard', 'mixed']).optional(),
|
|
||||||
/** Share of select-all-that-apply questions, 0..1. */
|
|
||||||
sataShare: z.number().min(0).max(1).optional(),
|
|
||||||
cardStyle: z.enum(['concise', 'detailed', 'medication', 'definition']).optional(),
|
|
||||||
noteStyle: z.enum(['study_guide', 'summary', 'outline', 'cheat_sheet']).optional(),
|
|
||||||
modelId: z.string().nullish(),
|
|
||||||
});
|
|
||||||
export type GenerateRequest = z.infer<typeof generateRequestSchema>;
|
|
||||||
|
|
||||||
/* ------------------------------ prompts ------------------------------ */
|
|
||||||
|
|
||||||
const QUESTION_SCHEMA = {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
questions: {
|
|
||||||
type: 'array',
|
|
||||||
items: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
type: { type: 'string', enum: ['single', 'multi'], description: 'single = one correct answer; multi = select all that apply' },
|
|
||||||
stem: { type: 'string', description: 'The question. Markdown allowed.' },
|
|
||||||
options: {
|
|
||||||
type: 'array',
|
|
||||||
items: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
text: { type: 'string' },
|
|
||||||
correct: { type: 'boolean' },
|
|
||||||
why: { type: 'string', description: 'One sentence on why this choice is right or wrong.' },
|
|
||||||
},
|
|
||||||
required: ['text', 'correct', 'why'],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
rationale: { type: 'string', description: 'Teaching explanation of the correct answer. Markdown allowed.' },
|
|
||||||
topic: { type: 'string', description: 'Short topic label, reused across related questions (2-4 words).' },
|
|
||||||
difficulty: { type: 'string', enum: ['easy', 'medium', 'hard'] },
|
|
||||||
source: { type: 'string', description: 'Id of the excerpt this question is based on, e.g. "c42". Empty if none.' },
|
|
||||||
},
|
|
||||||
required: ['type', 'stem', 'options', 'rationale', 'topic', 'difficulty'],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
required: ['questions'],
|
|
||||||
};
|
|
||||||
|
|
||||||
const CARD_SCHEMA = {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
cards: {
|
|
||||||
type: 'array',
|
|
||||||
items: {
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
front: { type: 'string', description: 'Prompt side: one specific question or term.' },
|
|
||||||
back: { type: 'string', description: 'Answer side. Markdown allowed.' },
|
|
||||||
hint: { type: 'string', description: 'Optional memory hook or mnemonic. Empty string if none.' },
|
|
||||||
tags: { type: 'array', items: { type: 'string' }, description: '1-2 short topic tags.' },
|
|
||||||
},
|
|
||||||
required: ['front', 'back'],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
required: ['cards'],
|
|
||||||
};
|
|
||||||
|
|
||||||
const NCLEX_RULES = `Write NCLEX-style (Next Generation NCLEX) items:
|
|
||||||
- Test clinical judgment at the application and analysis level: recognise cues, analyse, prioritise, choose and evaluate actions. Avoid pure recall.
|
|
||||||
- Open with a brief, realistic client scenario containing only relevant data (age, diagnosis, key findings, vital signs or lab values with units).
|
|
||||||
- Ask one clear question. Use priority wording where it fits (first, best, most important, requires immediate follow-up) and build it on a real framework: ABCs, Maslow, the nursing process, acute before chronic, unstable before stable, safety, least invasive first.
|
|
||||||
- Cover the client-needs areas where the material allows: safe and effective care, pharmacology, physiological adaptation, reduction of risk, health promotion, psychosocial integrity, delegation and assignment.
|
|
||||||
- Every distractor must be plausible to a student with partial knowledge and wrong for a specific reason. Keep choices parallel in grammar and similar in length. Never use "all of the above" or "none of the above". Do not give the answer away through wording or length.
|
|
||||||
- Multiple-choice items have exactly 4 choices with exactly one best answer. Select-all-that-apply items have 5 or 6 choices with 2 to 5 correct, and the stem ends with "Select all that apply."
|
|
||||||
- Use generic drug names. Use current, standard practice.
|
|
||||||
- The rationale teaches: state why the correct answer is correct, name the principle behind it, and give each choice its own "why".`;
|
|
||||||
|
|
||||||
const STANDARD_RULES = `Write clear exam-quality questions:
|
|
||||||
- Each question tests one idea and has one defensible answer set.
|
|
||||||
- Mix understanding and application; avoid trivia and trick wording.
|
|
||||||
- Distractors are plausible and parallel. Never use "all of the above" or "none of the above".
|
|
||||||
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
|
|
||||||
- The rationale explains the correct answer, and each choice gets its own "why".`;
|
|
||||||
|
|
||||||
const RECALL_RULES = `Write straightforward recall questions that check key facts, definitions, normal values and classifications.
|
|
||||||
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
|
|
||||||
- Keep stems short and unambiguous. Each choice gets a brief "why".`;
|
|
||||||
|
|
||||||
const CARD_RULES: Record<string, string> = {
|
|
||||||
concise: 'Each card tests exactly one fact. Fronts are specific questions or terms; backs are short (a phrase or one or two lines).',
|
|
||||||
detailed: 'Each card covers one concept. Backs may use a short bullet list (up to 5 bullets) for the key points.',
|
|
||||||
medication:
|
|
||||||
'Make medication cards. Front: the generic drug name (with a common brand name in parentheses when well known). Back: a Markdown list with **Class**, **Action**, **Uses**, **Adverse effects** (the most important ones), **Nursing considerations** and **Client teaching**. Only include what the material supports.',
|
|
||||||
definition: 'Make term and definition cards. Front: the term. Back: a precise definition in one or two sentences, plus an example when useful.',
|
|
||||||
};
|
|
||||||
|
|
||||||
const NOTE_RULES: Record<string, string> = {
|
|
||||||
study_guide:
|
|
||||||
'Write a study guide in Markdown: a one-paragraph overview, then sections with headings. Under each heading give the key concepts as tight bullet points, important values and medications in tables where that helps, common pitfalls, and finish with a "Check yourself" list of 5-10 self-test questions (answers at the very end).',
|
|
||||||
summary: 'Write a faithful summary in Markdown: a short overview paragraph, then the main points grouped under headings. Keep it compact and do not add outside information.',
|
|
||||||
outline: 'Write a hierarchical outline in Markdown using nested bullet points. Capture the structure and every key point of the material without full sentences where a phrase will do.',
|
|
||||||
cheat_sheet: 'Write a one-page cheat sheet in Markdown: dense, scannable, organised under short headings, using tables for values, drug classes and comparisons, and bold for must-know facts.',
|
|
||||||
};
|
|
||||||
|
|
||||||
function baseSystem(req: GenerateRequest, hasSources: boolean): string {
|
|
||||||
const lines = [
|
|
||||||
'You are an expert educator writing study material for a nursing student. Accuracy is critical: this material is used to prepare for licensing exams.',
|
|
||||||
hasSources
|
|
||||||
? 'Base everything on the provided source material. Do not introduce facts that contradict it. You may rely on standard, well-established knowledge only to write plausible distractors and clear explanations.'
|
|
||||||
: 'No source material was provided, so rely on current, well-established knowledge. If something is uncertain or varies by institution, prefer the most widely taught standard.',
|
|
||||||
'Write in clear, natural English. Use Markdown inside text fields where it aids clarity (bold key terms, short lists, tables). Do not wrap output in code fences.',
|
|
||||||
];
|
|
||||||
if (req.instructions) lines.push(`Additional instructions from the student (follow them unless they conflict with accuracy):\n${req.instructions}`);
|
|
||||||
return lines.join('\n\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ source gathering ------------------------------ */
|
|
||||||
|
|
||||||
interface Sources {
|
|
||||||
/** Batches of excerpts (documents). */
|
|
||||||
groups: ChunkHit[][];
|
|
||||||
/** Free text used as-is (pasted notes, an existing deck or quiz, missed questions). */
|
|
||||||
inline: string;
|
|
||||||
docs: { documentId: string; title: string }[];
|
|
||||||
refs: Map<string, string>;
|
|
||||||
totalTokens: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens: number, groups: number): Sources {
|
|
||||||
const { db } = app;
|
|
||||||
const src = req.source;
|
|
||||||
const out: Sources = { groups: [], inline: '', docs: [], refs: new Map(), totalTokens: 0 };
|
|
||||||
const inline: string[] = [];
|
|
||||||
|
|
||||||
let documentIds = src.documentIds ?? [];
|
|
||||||
if (src.allClassDocuments && req.classId) {
|
|
||||||
documentIds = db.all<{ id: string }>(`SELECT id FROM documents WHERE class_id = ? AND status = 'ready' AND chunk_count > 0 ORDER BY sort_order, created_at`, req.classId).map((d) => d.id);
|
|
||||||
}
|
|
||||||
if (documentIds.length > 0) {
|
|
||||||
const rows = db.all<{ id: string; title: string; status: string; chunk_count: number }>(
|
|
||||||
`SELECT id, title, status, chunk_count FROM documents WHERE id IN (${documentIds.map(() => '?').join(',')})`,
|
|
||||||
...documentIds,
|
|
||||||
);
|
|
||||||
const usable = rows.filter((r) => r.status === 'ready' && r.chunk_count > 0);
|
|
||||||
if (usable.length === 0) {
|
|
||||||
throw badRequest(rows.some((r) => r.status !== 'ready') ? 'Those documents are still being processed. Try again in a moment.' : 'Those documents have no readable text yet.');
|
|
||||||
}
|
|
||||||
out.docs = usable.map((r) => ({ documentId: r.id, title: r.title }));
|
|
||||||
out.groups = selectContext(db, { documentIds: usable.map((r) => r.id) }, { focus: req.focus, budgetTokens, groups });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (src.text?.trim()) inline.push(`Notes provided by the student:\n${src.text.trim()}`);
|
|
||||||
if (src.noteId) {
|
|
||||||
const note = db.get<{ title: string; content: string }>('SELECT title, content FROM notes WHERE id = ?', src.noteId);
|
|
||||||
if (note) inline.push(`Note "${note.title}":\n${note.content}`);
|
|
||||||
}
|
|
||||||
if (src.deckId) {
|
|
||||||
const deck = getDeck(db, src.deckId);
|
|
||||||
const cards = listCards(db, deck.id);
|
|
||||||
if (cards.length === 0) throw badRequest('That deck has no cards');
|
|
||||||
inline.push(`Flashcards from the deck "${deck.title}":\n${cards.map((c, i) => `${i + 1}. ${c.front}\n -> ${c.back}`).join('\n')}`);
|
|
||||||
}
|
|
||||||
if (src.quizId) {
|
|
||||||
const quiz = getQuiz(db, src.quizId);
|
|
||||||
const questions = listQuestions(db, quiz.id);
|
|
||||||
if (questions.length === 0) throw badRequest('That quiz has no questions');
|
|
||||||
inline.push(
|
|
||||||
`Questions from the quiz "${quiz.title}":\n${questions
|
|
||||||
.map((q, i) => `${i + 1}. ${q.stem}\n Correct: ${q.options.filter((o) => o.correct).map((o) => o.text).join('; ')}\n Rationale: ${q.rationale}`)
|
|
||||||
.join('\n')}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (src.weakAreas) {
|
|
||||||
const ids = missedQuestionIds(db, { classId: req.classId ?? undefined }, 40);
|
|
||||||
const topics = weakTopics(db, { classId: req.classId ?? undefined }, 10);
|
|
||||||
if (ids.length === 0 && topics.length === 0) throw badRequest('There is not enough quiz history yet to find weak areas. Take a few quizzes first.');
|
|
||||||
const missed = ids
|
|
||||||
.map((id) => db.get<{ stem: string; topic: string; rationale: string }>('SELECT stem, topic, rationale FROM questions WHERE id = ?', id))
|
|
||||||
.filter((q): q is { stem: string; topic: string; rationale: string } => Boolean(q));
|
|
||||||
inline.push(
|
|
||||||
[
|
|
||||||
'The student keeps missing these concepts. Write NEW items that test the same underlying knowledge from different angles. Do not repeat or lightly reword the original questions.',
|
|
||||||
topics.length ? `Weakest topics: ${topics.map((t) => `${t.topic} (${Math.round(t.accuracy * 100)}% correct)`).join(', ')}` : '',
|
|
||||||
missed.length ? `Questions answered incorrectly:\n${missed.map((q, i) => `${i + 1}. [${q.topic || 'General'}] ${q.stem}\n Key point: ${q.rationale.slice(0, 400)}`).join('\n')}` : '',
|
|
||||||
]
|
|
||||||
.filter(Boolean)
|
|
||||||
.join('\n\n'),
|
|
||||||
);
|
|
||||||
// Pull supporting passages from the class documents for those topics.
|
|
||||||
if (req.classId && out.groups.length === 0) {
|
|
||||||
const query = [...topics.map((t) => t.topic), ...missed.slice(0, 10).map((q) => q.topic)].filter(Boolean).join(' ');
|
|
||||||
const hits = query ? searchChunks(db, query, { classId: req.classId }, 24) : [];
|
|
||||||
if (hits.length > 0) {
|
|
||||||
const perGroup = Math.max(1, Math.ceil(hits.length / groups));
|
|
||||||
for (let i = 0; i < hits.length; i += perGroup) out.groups.push(hits.slice(i, i + perGroup));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
out.inline = inline.join('\n\n');
|
|
||||||
for (const g of out.groups) for (const c of g) out.refs.set(`c${c.id}`, sourceLabel(c));
|
|
||||||
out.totalTokens = out.groups.reduce((n, g) => n + g.reduce((m, c) => m + c.tokens, 0), 0) + Math.ceil(out.inline.length / 4);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function sourceBlock(sources: Sources, groupIndex: number): string {
|
|
||||||
const parts: string[] = [];
|
|
||||||
const group = sources.groups.length > 0 ? sources.groups[groupIndex % sources.groups.length] : [];
|
|
||||||
if (group.length > 0) parts.push(`SOURCE EXCERPTS (each starts with its id in brackets):\n\n${formatExcerpts(group)}`);
|
|
||||||
if (sources.inline) parts.push(sources.inline);
|
|
||||||
return parts.join('\n\n======\n\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ job control ------------------------------ */
|
|
||||||
|
|
||||||
const running = new Map<string, AbortController>();
|
|
||||||
|
|
||||||
export function cancelGeneration(draftId: string): void {
|
|
||||||
running.get(draftId)?.abort();
|
|
||||||
running.delete(draftId);
|
|
||||||
}
|
|
||||||
|
|
||||||
function setProgress(db: DB, id: string, progress: number, label: string, payload?: ContentDraft): void {
|
|
||||||
if (payload) {
|
|
||||||
db.run('UPDATE drafts SET progress = ?, progress_label = ?, payload = ?, updated_at = ? WHERE id = ?', progress, label, JSON.stringify(payload), now(), id);
|
|
||||||
} else {
|
|
||||||
db.run('UPDATE drafts SET progress = ?, progress_label = ?, updated_at = ? WHERE id = ?', progress, label, now(), id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function defaultTitle(app: AppContext, req: GenerateRequest, sources: Sources): string {
|
|
||||||
if (req.title) return req.title;
|
|
||||||
const subject =
|
|
||||||
req.focus?.replace(/\s+/g, ' ').slice(0, 80) ||
|
|
||||||
(sources.docs.length === 1 ? sources.docs[0].title : '') ||
|
|
||||||
(req.source.deckId ? getDeck(app.db, req.source.deckId).title : '') ||
|
|
||||||
(req.source.quizId ? getQuiz(app.db, req.source.quizId).title : '') ||
|
|
||||||
(req.source.weakAreas ? 'Weak areas' : '') ||
|
|
||||||
(req.classId ? getClass(app.db, req.classId).name : 'Untitled');
|
|
||||||
const suffix = req.kind === 'quiz' ? (req.source.weakAreas ? 'practice' : req.style === 'nclex' ? 'NCLEX practice' : 'quiz') : req.kind === 'deck' ? 'flashcards' : NOTE_TITLES[req.noteStyle ?? 'study_guide'];
|
|
||||||
const cap = subject.charAt(0).toUpperCase() + subject.slice(1);
|
|
||||||
return `${cap} · ${suffix}`.slice(0, 200);
|
|
||||||
}
|
|
||||||
|
|
||||||
const NOTE_TITLES: Record<string, string> = { study_guide: 'study guide', summary: 'summary', outline: 'outline', cheat_sheet: 'cheat sheet' };
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Starts a generation job in the background and returns the draft id immediately.
|
|
||||||
* The caller polls the draft (or awaits `done`) for progress.
|
|
||||||
*/
|
|
||||||
export function startGeneration(app: AppContext, input: z.input<typeof generateRequestSchema>, opts: { conversationId?: string | null; origin?: DraftMeta['origin'] } = {}): { draftId: string; done: Promise<DraftDto> } {
|
|
||||||
const req = generateRequestSchema.parse(input);
|
|
||||||
if (req.classId) getClass(app.db, req.classId);
|
|
||||||
// Fail fast (before creating a draft) when no model is configured.
|
|
||||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
|
||||||
const draftId = createDraft(app.db, {
|
|
||||||
kind: req.kind,
|
|
||||||
classId: req.classId ?? null,
|
|
||||||
folderId: req.folderId ?? null,
|
|
||||||
title: req.title ?? '',
|
|
||||||
status: 'generating',
|
|
||||||
conversationId: opts.conversationId ?? null,
|
|
||||||
meta: { origin: opts.origin ?? 'generate', model: resolved.model.label, request: { ...req, source: { ...req.source, text: req.source.text ? `${req.source.text.slice(0, 200)}…` : undefined } } },
|
|
||||||
});
|
|
||||||
const controller = new AbortController();
|
|
||||||
running.set(draftId, controller);
|
|
||||||
const done = runGeneration(app, draftId, req, controller.signal)
|
|
||||||
.catch((err) => {
|
|
||||||
const message = err instanceof AIError ? err.message : err instanceof Error && 'status' in err ? err.message : 'Generation failed unexpectedly.';
|
|
||||||
if (!(err instanceof AIError) && !(err instanceof Error && 'status' in err)) app.log('error', 'Generation failed', err);
|
|
||||||
try {
|
|
||||||
const current = app.db.get<{ status: string }>('SELECT status FROM drafts WHERE id = ?', draftId);
|
|
||||||
if (current?.status === 'generating') {
|
|
||||||
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ?`, controller.signal.aborted ? 'Generation was cancelled.' : message, now(), draftId);
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
/* the database was closed during shutdown; tidyDrafts marks the job failed on the next start */
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.then(() => {
|
|
||||||
running.delete(draftId);
|
|
||||||
return getDraft(app.db, draftId);
|
|
||||||
});
|
|
||||||
return { draftId, done };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function runGeneration(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal): Promise<void> {
|
|
||||||
const { db } = app;
|
|
||||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
|
||||||
const window = resolved.model.contextWindow ?? 32_000;
|
|
||||||
const budgetTokens = Math.max(2_500, Math.min(Math.floor(window * 0.4), 28_000));
|
|
||||||
setProgress(db, draftId, 0.03, 'Reading your material');
|
|
||||||
|
|
||||||
if (req.kind === 'note') {
|
|
||||||
await generateNote(app, draftId, req, signal, budgetTokens, window);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const isQuiz = req.kind === 'quiz';
|
|
||||||
const perBatch = isQuiz ? 8 : 25;
|
|
||||||
const defaultCount = isQuiz ? 15 : 30;
|
|
||||||
let count = req.count ?? defaultCount;
|
|
||||||
if ((req.source.deckId || req.source.quizId) && !req.count) {
|
|
||||||
// Conversions default to one item per source item.
|
|
||||||
count = req.source.deckId ? listCards(db, req.source.deckId).length : listQuestions(db, req.source.quizId!).length;
|
|
||||||
count = Math.min(Math.max(count, 1), 100);
|
|
||||||
}
|
|
||||||
const batches = Math.ceil(count / perBatch);
|
|
||||||
const sources = gatherSources(app, req, budgetTokens, batches);
|
|
||||||
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
|
|
||||||
if (!hasSources && !req.focus && !req.instructions) throw badRequest('Choose source material or describe a topic to generate from.');
|
|
||||||
|
|
||||||
const title = defaultTitle(app, req, sources);
|
|
||||||
const draft: QuizDraft | DeckDraft = isQuiz ? { kind: 'quiz', title, description: '', questions: [] } : { kind: 'deck', title, description: '', cards: [] };
|
|
||||||
const items = (): (DraftQuestion | DraftCard)[] => (draft.kind === 'quiz' ? draft.questions : draft.cards);
|
|
||||||
const key = (x: DraftQuestion | DraftCard) => ('stem' in x ? x.stem : x.front).toLowerCase().replace(/\W+/g, ' ').trim().slice(0, 120);
|
|
||||||
const seen = new Set<string>();
|
|
||||||
const notes: string[] = [];
|
|
||||||
|
|
||||||
const style = req.style ?? 'nclex';
|
|
||||||
const rules = isQuiz ? (style === 'nclex' ? NCLEX_RULES : style === 'recall' ? RECALL_RULES : STANDARD_RULES) : CARD_RULES[req.cardStyle ?? 'concise'];
|
|
||||||
const system = `${baseSystem(req, hasSources)}\n\n${rules}`;
|
|
||||||
const sata = req.sataShare ?? (style === 'nclex' ? 0.3 : 0.2);
|
|
||||||
const difficulty = req.difficulty ?? 'mixed';
|
|
||||||
|
|
||||||
const runBatch = async (index: number, want: number): Promise<number> => {
|
|
||||||
const written = items().map((x) => `- ${('stem' in x ? x.stem : x.front).replace(/\s+/g, ' ').slice(0, 140)}`);
|
|
||||||
const sataCount = Math.round(want * sata);
|
|
||||||
const ask = isQuiz
|
|
||||||
? [
|
|
||||||
`Write ${want} ${style === 'nclex' ? 'NCLEX-style ' : ''}questions${req.focus ? ` about: ${req.focus}` : ''}.`,
|
|
||||||
sata <= 0 ? 'Use only multiple-choice (single) questions.' : sata >= 1 ? 'Use only select-all-that-apply (multi) questions.' : `About ${sataCount} of them should be select-all-that-apply (multi); the rest multiple choice (single).`,
|
|
||||||
difficulty === 'mixed' ? 'Mix difficulty: mostly medium, with a few easy and a few hard.' : `Difficulty: ${difficulty}.`,
|
|
||||||
sources.groups.length > 0 ? 'Spread the questions across the excerpts instead of clustering on one. Set "source" to the id of the excerpt each question is based on.' : '',
|
|
||||||
req.source.deckId ? 'Turn the flashcards into questions: use each card\'s answer as the correct choice and write distractors that a student could confuse with it.' : '',
|
|
||||||
]
|
|
||||||
: [
|
|
||||||
`Write ${want} flashcards${req.focus ? ` about: ${req.focus}` : ''}.`,
|
|
||||||
sources.groups.length > 0 ? 'Cover the most testable facts across all of the excerpts.' : '',
|
|
||||||
req.source.quizId ? 'Turn the quiz questions into flashcards: the front asks for the key fact, the back states it with the essential reasoning.' : '',
|
|
||||||
];
|
|
||||||
if (written.length > 0) ask.push(`Already written (do not repeat or closely paraphrase these):\n${written.slice(-80).join('\n')}`);
|
|
||||||
const user = [sourceBlock(sources, index), ask.filter(Boolean).join('\n')].filter(Boolean).join('\n\n======\n\n');
|
|
||||||
|
|
||||||
const result = await generateStructured(app, {
|
|
||||||
task: 'generate',
|
|
||||||
purpose: isQuiz ? 'quiz generation' : 'flashcard generation',
|
|
||||||
modelId: req.modelId,
|
|
||||||
system,
|
|
||||||
messages: [{ role: 'user', content: [{ type: 'text', text: user }] }],
|
|
||||||
toolName: isQuiz ? 'submit_questions' : 'submit_flashcards',
|
|
||||||
toolDescription: isQuiz ? 'Submit the finished questions.' : 'Submit the finished flashcards.',
|
|
||||||
schema: isQuiz ? QUESTION_SCHEMA : CARD_SCHEMA,
|
|
||||||
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === req.kind) as QuizDraft | DeckDraft | undefined;
|
|
||||||
const incoming: (DraftQuestion | DraftCard)[] = parsed ? (parsed.kind === 'quiz' ? parsed.questions : parsed.cards) : [];
|
|
||||||
let added = 0;
|
|
||||||
for (const item of incoming) {
|
|
||||||
const k = key(item);
|
|
||||||
if (!k || seen.has(k)) continue;
|
|
||||||
const problems = ('stem' in item ? validateDraftQuestion(item, 0) : validateDraftCard(item, 0)).filter((p) => p.level === 'error');
|
|
||||||
if (problems.length > 0) continue; // malformed items are dropped here and replaced by a top-up batch
|
|
||||||
if ('stem' in item) {
|
|
||||||
const ref = (item.source ?? '').replace(/[[\]\s]/g, '');
|
|
||||||
item.source = sources.refs.get(ref) ?? undefined;
|
|
||||||
}
|
|
||||||
seen.add(k);
|
|
||||||
(items() as (DraftQuestion | DraftCard)[]).push(item);
|
|
||||||
added += 1;
|
|
||||||
if (items().length >= count) break;
|
|
||||||
}
|
|
||||||
if (result.truncated) notes.push('One batch hit the model output limit, so it produced fewer items than requested.');
|
|
||||||
return added;
|
|
||||||
};
|
|
||||||
|
|
||||||
let failures = 0;
|
|
||||||
for (let i = 0; i < batches && items().length < count; i += 1) {
|
|
||||||
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
|
|
||||||
const want = Math.min(perBatch, count - items().length);
|
|
||||||
setProgress(db, draftId, 0.05 + (0.9 * items().length) / count, `Writing ${isQuiz ? 'questions' : 'cards'} ${items().length + 1}-${Math.min(count, items().length + want)} of ${count}`, draft);
|
|
||||||
try {
|
|
||||||
await runBatch(i, want);
|
|
||||||
} catch (err) {
|
|
||||||
// One bad batch should not sink a long job that already has good content.
|
|
||||||
failures += 1;
|
|
||||||
if (items().length === 0 || failures > 2 || (err instanceof AIError && !['invalid_response', 'server', 'rate_limit', 'timeout', 'network'].includes(err.code))) throw err;
|
|
||||||
notes.push(`A batch failed and was skipped: ${err instanceof Error ? err.message : 'unknown error'}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Top up when validation or de-duplication left the draft short.
|
|
||||||
for (let extra = 0; extra < 2 && items().length < count && items().length > 0; extra += 1) {
|
|
||||||
if (signal.aborted) break;
|
|
||||||
const want = Math.min(perBatch, count - items().length);
|
|
||||||
setProgress(db, draftId, 0.92, `Writing the last ${want} ${isQuiz ? 'questions' : 'cards'}`, draft);
|
|
||||||
try {
|
|
||||||
if ((await runBatch(batches + extra, want)) === 0) break;
|
|
||||||
} catch {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (items().length === 0) throw new AIError('invalid_response', 'The model did not produce any usable items. Try again, or choose a different model for content generation.');
|
|
||||||
if (items().length < count) notes.push(`Generated ${items().length} of the ${count} requested.`);
|
|
||||||
|
|
||||||
db.run(
|
|
||||||
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
|
|
||||||
draft.title,
|
|
||||||
JSON.stringify(draft),
|
|
||||||
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs, notes: [...new Set(notes)] } satisfies DraftMeta),
|
|
||||||
now(),
|
|
||||||
draftId,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function generateNote(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal, budgetTokens: number, window: number): Promise<void> {
|
|
||||||
const { db } = app;
|
|
||||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
|
||||||
const style = req.noteStyle ?? 'study_guide';
|
|
||||||
// A guide reads the material in as few passes as possible; allow a bigger slice per pass than item generation.
|
|
||||||
const perPass = Math.max(budgetTokens, Math.min(Math.floor(window * 0.55), 90_000));
|
|
||||||
const sources = gatherSources(app, req, perPass, 12);
|
|
||||||
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
|
|
||||||
if (!hasSources && !req.focus) throw badRequest('Choose source material or describe a topic to write about.');
|
|
||||||
const title = defaultTitle(app, req, sources);
|
|
||||||
const system = `${baseSystem(req, hasSources)}\n\n${NOTE_RULES[style]}\n\n${
|
|
||||||
sources.groups.length > 0 ? 'Cite the excerpts you draw on by putting their id in brackets at the end of the relevant line or paragraph, e.g. [c42]. Use only ids that appear in the source.' : ''
|
|
||||||
}\nStart directly with the content (no preamble, no title line).`;
|
|
||||||
|
|
||||||
// Merge excerpt groups into as few passes as fit.
|
|
||||||
const passes: ChunkHit[][] = [];
|
|
||||||
let current: ChunkHit[] = [];
|
|
||||||
let size = 0;
|
|
||||||
for (const g of sources.groups) {
|
|
||||||
const t = g.reduce((n, c) => n + c.tokens, 0);
|
|
||||||
if (current.length > 0 && size + t > perPass) {
|
|
||||||
passes.push(current);
|
|
||||||
current = [];
|
|
||||||
size = 0;
|
|
||||||
}
|
|
||||||
current.push(...g);
|
|
||||||
size += t;
|
|
||||||
}
|
|
||||||
if (current.length > 0) passes.push(current);
|
|
||||||
|
|
||||||
const ask = `${req.focus ? `Focus on: ${req.focus}\n` : ''}Write the ${NOTE_TITLES[style]} now.`;
|
|
||||||
let content = '';
|
|
||||||
if (passes.length <= 1) {
|
|
||||||
setProgress(db, draftId, 0.2, 'Writing');
|
|
||||||
const body = [passes[0]?.length ? `SOURCE EXCERPTS:\n\n${formatExcerpts(passes[0])}` : '', sources.inline, ask].filter(Boolean).join('\n\n======\n\n');
|
|
||||||
const res = await runChat(app, { task: 'generate', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000, signal });
|
|
||||||
content = res.text;
|
|
||||||
} else {
|
|
||||||
// Long material: condense each part, then write the final piece from the condensed notes.
|
|
||||||
const partials: string[] = [];
|
|
||||||
for (let i = 0; i < passes.length; i += 1) {
|
|
||||||
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
|
|
||||||
setProgress(db, draftId, 0.05 + (0.7 * i) / passes.length, `Reading part ${i + 1} of ${passes.length}`);
|
|
||||||
const res = await runChat(app, {
|
|
||||||
task: 'generate',
|
|
||||||
purpose: 'study guide',
|
|
||||||
modelId: req.modelId,
|
|
||||||
system: `${baseSystem(req, true)}\n\nCondense this part of the material into detailed notes: every key concept, value, medication, cause, sign, intervention and warning, as Markdown bullet points under short headings. Keep the bracketed excerpt ids (e.g. [c42]) next to the points they support. Do not add outside information.`,
|
|
||||||
messages: [{ role: 'user', content: [{ type: 'text', text: `${req.focus ? `Focus on: ${req.focus}\n\n` : ''}SOURCE EXCERPTS (part ${i + 1} of ${passes.length}):\n\n${formatExcerpts(passes[i])}` }] }],
|
|
||||||
maxOutputTokens: 6_000,
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
partials.push(res.text.trim());
|
|
||||||
}
|
|
||||||
setProgress(db, draftId, 0.8, 'Writing');
|
|
||||||
const res = await runChat(app, {
|
|
||||||
task: 'generate',
|
|
||||||
purpose: 'study guide',
|
|
||||||
modelId: req.modelId,
|
|
||||||
system,
|
|
||||||
messages: [{ role: 'user', content: [{ type: 'text', text: `CONDENSED NOTES FROM THE FULL MATERIAL:\n\n${partials.join('\n\n---\n\n')}\n\n======\n\n${sources.inline ? `${sources.inline}\n\n======\n\n` : ''}${ask}` }] }],
|
|
||||||
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
|
|
||||||
signal,
|
|
||||||
});
|
|
||||||
content = res.text;
|
|
||||||
}
|
|
||||||
content = content.trim().replace(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i, '$1');
|
|
||||||
if (!content) throw new AIError('invalid_response', 'The model returned an empty document. Try again.');
|
|
||||||
|
|
||||||
// Turn [c42] markers into readable citations and append a source list.
|
|
||||||
const used = new Map<string, string>();
|
|
||||||
content = content.replace(/\[(c\d+(?:\s*,\s*c\d+)*)\]/g, (_m, list: string) => {
|
|
||||||
const labels = list
|
|
||||||
.split(',')
|
|
||||||
.map((r) => r.trim())
|
|
||||||
.map((r) => {
|
|
||||||
const label = sources.refs.get(r) ?? (() => {
|
|
||||||
const hit = chunkById(db, Number(r.slice(1)));
|
|
||||||
return hit ? sourceLabel(hit) : null;
|
|
||||||
})();
|
|
||||||
if (label) used.set(label, label);
|
|
||||||
return label;
|
|
||||||
})
|
|
||||||
.filter(Boolean);
|
|
||||||
return labels.length ? ` *(${[...new Set(labels)].join('; ')})*` : '';
|
|
||||||
});
|
|
||||||
const payload: NoteDraft = { kind: 'note', title, content };
|
|
||||||
db.run(
|
|
||||||
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
|
|
||||||
title,
|
|
||||||
JSON.stringify(payload),
|
|
||||||
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs } satisfies DraftMeta),
|
|
||||||
now(),
|
|
||||||
draftId,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ improve existing questions ------------------------------ */
|
|
||||||
|
|
||||||
export const improveRequestSchema = z.object({
|
|
||||||
quizId: z.string(),
|
|
||||||
questionIds: z.array(z.string()).max(60).optional(),
|
|
||||||
instructions: z.string().trim().max(2000).optional(),
|
|
||||||
modelId: z.string().nullish(),
|
|
||||||
});
|
|
||||||
|
|
||||||
/** Rewrites existing questions (clearer stems, better distractors, fuller rationales) into a draft the user can apply. */
|
|
||||||
export function startImprove(app: AppContext, input: z.input<typeof improveRequestSchema>, opts: { conversationId?: string | null } = {}): { draftId: string; done: Promise<DraftDto> } {
|
|
||||||
const req = improveRequestSchema.parse(input);
|
|
||||||
const quiz = getQuiz(app.db, req.quizId);
|
|
||||||
let questions = listQuestions(app.db, quiz.id);
|
|
||||||
if (req.questionIds?.length) questions = questions.filter((q) => req.questionIds!.includes(q.id));
|
|
||||||
if (questions.length === 0) throw badRequest('No questions to improve');
|
|
||||||
questions = questions.slice(0, 60);
|
|
||||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
|
||||||
const draftId = createDraft(app.db, {
|
|
||||||
kind: 'quiz',
|
|
||||||
classId: quiz.class_id,
|
|
||||||
title: quiz.title,
|
|
||||||
status: 'generating',
|
|
||||||
conversationId: opts.conversationId ?? null,
|
|
||||||
meta: { origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: [] },
|
|
||||||
});
|
|
||||||
const controller = new AbortController();
|
|
||||||
running.set(draftId, controller);
|
|
||||||
const done = (async () => {
|
|
||||||
const out: DraftQuestion[] = [];
|
|
||||||
const ids: string[] = [];
|
|
||||||
const system = `You are an expert nursing educator and item writer reviewing existing exam questions.
|
|
||||||
Improve each question while keeping what it tests and its correct answer(s) unless the keyed answer is factually wrong (if so, fix it and say so at the start of the rationale).
|
|
||||||
- Make the stem clear, complete and free of clues; fix grammar.
|
|
||||||
- Replace weak or implausible distractors; keep choices parallel and similar in length; remove "all/none of the above".
|
|
||||||
- Write a teaching rationale, and give every choice a one-sentence "why".
|
|
||||||
- Keep the same question type unless it is clearly wrong for the content. Set a short topic label and a difficulty.
|
|
||||||
Return the questions in the same order, one output question per input question.${req.instructions ? `\n\nAdditional instructions: ${req.instructions}` : ''}`;
|
|
||||||
for (let i = 0; i < questions.length; i += 8) {
|
|
||||||
if (controller.signal.aborted) throw new AIError('aborted', 'Cancelled.');
|
|
||||||
const batch = questions.slice(i, i + 8);
|
|
||||||
setProgress(app.db, draftId, 0.05 + (0.9 * i) / questions.length, `Reviewing questions ${i + 1}-${i + batch.length} of ${questions.length}`);
|
|
||||||
const body = batch
|
|
||||||
.map(
|
|
||||||
(q, n) =>
|
|
||||||
`Question ${n + 1} (${q.type})\nStem: ${q.stem}\nChoices:\n${q.options.map((o) => `- [${o.correct ? 'x' : ' '}] ${o.text}`).join('\n')}\nRationale: ${q.rationale || '(none)'}\nTopic: ${q.topic || '(none)'}`,
|
|
||||||
)
|
|
||||||
.join('\n\n');
|
|
||||||
const result = await generateStructured(app, {
|
|
||||||
task: 'generate',
|
|
||||||
purpose: 'improve questions',
|
|
||||||
modelId: req.modelId,
|
|
||||||
system,
|
|
||||||
messages: [{ role: 'user', content: [{ type: 'text', text: body }] }],
|
|
||||||
toolName: 'submit_questions',
|
|
||||||
toolDescription: 'Submit the improved questions in the original order.',
|
|
||||||
schema: QUESTION_SCHEMA,
|
|
||||||
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
|
|
||||||
signal: controller.signal,
|
|
||||||
});
|
|
||||||
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === 'quiz') as QuizDraft | undefined;
|
|
||||||
const improved = parsed?.questions ?? [];
|
|
||||||
batch.forEach((orig, n) => {
|
|
||||||
const next = improved[n];
|
|
||||||
if (!next || validateDraftQuestion(next, 0).some((p) => p.level === 'error')) return;
|
|
||||||
next.source = undefined;
|
|
||||||
out.push(next);
|
|
||||||
ids.push(orig.id);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (out.length === 0) throw new AIError('invalid_response', 'The model did not return usable revisions. Try again or use a different model.');
|
|
||||||
const payload: QuizDraft = { kind: 'quiz', title: quiz.title, description: quiz.description, questions: out };
|
|
||||||
app.db.run(
|
|
||||||
`UPDATE drafts SET status = 'ready', payload = ?, meta = ?, progress = 1, progress_label = '', updated_at = ? WHERE id = ?`,
|
|
||||||
JSON.stringify(payload),
|
|
||||||
JSON.stringify({ origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: ids } satisfies DraftMeta),
|
|
||||||
now(),
|
|
||||||
draftId,
|
|
||||||
);
|
|
||||||
})()
|
|
||||||
.catch((err) => {
|
|
||||||
const message = err instanceof AIError || (err instanceof Error && 'status' in err) ? (err as Error).message : 'Improving the questions failed unexpectedly.';
|
|
||||||
try {
|
|
||||||
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ? AND status = 'generating'`, message, now(), draftId);
|
|
||||||
} catch {
|
|
||||||
/* database closed during shutdown */
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.then(() => {
|
|
||||||
running.delete(draftId);
|
|
||||||
return getDraft(app.db, draftId);
|
|
||||||
});
|
|
||||||
return { draftId, done };
|
|
||||||
}
|
|
||||||
|
|
@ -1,581 +0,0 @@
|
||||||
/**
|
|
||||||
* Anthropic — Messages API (`POST /v1/messages`, SSE).
|
|
||||||
*
|
|
||||||
* Verified against (2026-11):
|
|
||||||
* https://platform.claude.com/docs/en/api/messages
|
|
||||||
* https://platform.claude.com/docs/en/build-with-claude/streaming
|
|
||||||
* https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking
|
|
||||||
* https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
|
|
||||||
* https://platform.claude.com/docs/en/api/migration-2026-10-21
|
|
||||||
*
|
|
||||||
* Quirks handled here:
|
|
||||||
* - API version 2026-10-21 rejects temperature/top_p/top_k on EVERY model, so
|
|
||||||
* this adapter never sends a temperature (the catalog marks Claude models
|
|
||||||
* `temperature: false`).
|
|
||||||
* - `max_tokens` is required; thinking counts against it.
|
|
||||||
* - Thinking configuration differs per model generation (see `claudeProfile`).
|
|
||||||
* - Thinking blocks (with their signatures) must be replayed unchanged, so the
|
|
||||||
* raw content blocks of each turn are kept in `providerData`.
|
|
||||||
* - Newer models bind thinking blocks to the conversation prefix (system
|
|
||||||
* prompt, tool set, earlier messages). If the app changed any of those, the
|
|
||||||
* API answers 400 "Invalid `signature` in `thinking` block"; the documented
|
|
||||||
* recovery is to strip the thinking blocks and retry once.
|
|
||||||
* - Safety classifiers can end a turn with `stop_reason: "refusal"` (HTTP 200).
|
|
||||||
* For the models that support it we opt into server-side fallback, which
|
|
||||||
* re-runs a declined request on another Claude model inside the same call.
|
|
||||||
*/
|
|
||||||
import {
|
|
||||||
AIError,
|
|
||||||
type ChatMessage,
|
|
||||||
type ChatRequest,
|
|
||||||
type ContentPart,
|
|
||||||
type DiscoveredModel,
|
|
||||||
type ModelCapabilities,
|
|
||||||
type ProviderAdapter,
|
|
||||||
type ProviderConfig,
|
|
||||||
type StopReason,
|
|
||||||
type StreamEvent,
|
|
||||||
type TestResult,
|
|
||||||
} from '../types.ts';
|
|
||||||
import { lookupModelDefaults } from './catalog.ts';
|
|
||||||
import { isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
|
||||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext } from './http.ts';
|
|
||||||
|
|
||||||
type Json = Record<string, unknown>;
|
|
||||||
|
|
||||||
const LABEL = 'Anthropic';
|
|
||||||
const DEFAULT_BASE_URL = 'https://api.anthropic.com';
|
|
||||||
const API_VERSION = '2026-10-21';
|
|
||||||
/** Beta that enables the `fallbacks` request parameter. */
|
|
||||||
const FALLBACK_BETA = 'server-side-fallback-2026-07-01';
|
|
||||||
/** Streaming has no HTTP-timeout concern, so give the model room (thinking counts against this). */
|
|
||||||
const DEFAULT_MAX_TOKENS = 64_000;
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Per-model behaviour */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
export interface ClaudeProfile {
|
|
||||||
/**
|
|
||||||
* always – thinking cannot be turned off (Fable/Mythos, Opus 5.5+)
|
|
||||||
* default_on – adaptive thinking runs unless disabled (Opus 5, Sonnet 5+, Haiku 5.5)
|
|
||||||
* opt_in – adaptive thinking must be requested (Opus 4.6–4.8, Sonnet 4.6)
|
|
||||||
* budget – legacy `budget_tokens` thinking (Haiku 4.5, Opus/Sonnet 4.5 and older)
|
|
||||||
* none – no thinking support (Claude 3 and earlier)
|
|
||||||
*/
|
|
||||||
thinking: 'always' | 'default_on' | 'opt_in' | 'budget' | 'none';
|
|
||||||
/** How thinking is switched off, when the model allows it. */
|
|
||||||
off: 'disabled' | 'between_tools' | null;
|
|
||||||
/** Accepts `output_config.effort`. */
|
|
||||||
effort: boolean;
|
|
||||||
/** Accepts `thinking.display` (and defaults to hiding the reasoning text). */
|
|
||||||
display: boolean;
|
|
||||||
/** Accepts `tool_choice` of type `any` / `tool`. */
|
|
||||||
forcedToolChoice: boolean;
|
|
||||||
/** Supports the server-side refusal `fallbacks` parameter. */
|
|
||||||
fallbacks: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Thinking / tool-choice rules by model, from the model migration guides. */
|
|
||||||
export function claudeProfile(model: string): ClaudeProfile {
|
|
||||||
const m = /claude-(fable|mythos|opus|sonnet|haiku)-(\d+)(?:-(\d{1,2}))?(?!\d)/.exec(model);
|
|
||||||
if (!m) {
|
|
||||||
// "claude-3-5-sonnet-…" and older put the version first and never had adaptive thinking.
|
|
||||||
if (/claude-(?:[123]\b|instant)/.test(model)) {
|
|
||||||
return { thinking: 'none', off: null, effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
|
||||||
}
|
|
||||||
// Unknown naming scheme: assume it behaves like the newest generation.
|
|
||||||
return { thinking: 'default_on', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: false };
|
|
||||||
}
|
|
||||||
const family = m[1];
|
|
||||||
const v = Number(m[2]) * 10 + Number(m[3] ?? 0); // 4.6 -> 46, 5 -> 50, 5.5 -> 55
|
|
||||||
|
|
||||||
if (family === 'fable' || family === 'mythos') {
|
|
||||||
return {
|
|
||||||
thinking: 'always',
|
|
||||||
off: null,
|
|
||||||
effort: true,
|
|
||||||
display: true,
|
|
||||||
forcedToolChoice: v <= 50,
|
|
||||||
fallbacks: family === 'fable' && (v === 51 || v === 52),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (family === 'opus') {
|
|
||||||
if (v >= 55) return { thinking: 'always', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
|
|
||||||
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: v === 50 };
|
|
||||||
if (v >= 47) return { thinking: 'opt_in', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
|
||||||
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
|
|
||||||
return { thinking: 'budget', off: 'disabled', effort: v === 45, display: false, forcedToolChoice: true, fallbacks: false };
|
|
||||||
}
|
|
||||||
if (family === 'sonnet') {
|
|
||||||
// Sonnet 5.5 rejects {type:"disabled"}; its lowest setting is {type:"between_tools"}.
|
|
||||||
if (v >= 55) return { thinking: 'default_on', off: 'between_tools', effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
|
|
||||||
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
|
||||||
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
|
|
||||||
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
|
||||||
}
|
|
||||||
// haiku
|
|
||||||
if (v >= 55) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
|
||||||
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Thinking budgets for the legacy `budget_tokens` models. */
|
|
||||||
const LEGACY_BUDGET = { low: 2_048, medium: 8_192, high: 16_384 } as const;
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Request building */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
interface BuildOptions {
|
|
||||||
/** Official endpoint: enables features a proxy may not understand. */
|
|
||||||
official: boolean;
|
|
||||||
stripThinking: boolean;
|
|
||||||
useFallbacks: boolean;
|
|
||||||
eagerToolInput: boolean;
|
|
||||||
promptCache: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Tool-call ids must match ^[a-zA-Z0-9_-]+$; ids minted by other providers may not. */
|
|
||||||
function safeId(id: string): string {
|
|
||||||
const cleaned = id.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
||||||
return cleaned || 'tool_call';
|
|
||||||
}
|
|
||||||
|
|
||||||
function userBlocks(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
|
||||||
const results: Json[] = [];
|
|
||||||
const rest: Json[] = [];
|
|
||||||
for (const p of parts) {
|
|
||||||
switch (p.type) {
|
|
||||||
case 'tool_result': {
|
|
||||||
// All results of one assistant turn go into ONE user message, before any other content.
|
|
||||||
const block: Json = { type: 'tool_result', tool_use_id: safeId(p.toolCallId), content: p.output || '(no output)' };
|
|
||||||
if (p.isError) block.is_error = true;
|
|
||||||
results.push(block);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'text':
|
|
||||||
if (p.text) rest.push({ type: 'text', text: p.text });
|
|
||||||
break;
|
|
||||||
case 'image':
|
|
||||||
if (caps.vision === false) rest.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
|
|
||||||
else rest.push({ type: 'image', source: { type: 'base64', media_type: p.mime, data: p.data } });
|
|
||||||
break;
|
|
||||||
case 'file':
|
|
||||||
if (p.mime === 'application/pdf' && caps.pdf !== false) {
|
|
||||||
const doc: Json = { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: p.data } };
|
|
||||||
if (p.filename) doc.title = p.filename;
|
|
||||||
rest.push(doc);
|
|
||||||
} else if (p.mime.startsWith('text/')) {
|
|
||||||
const doc: Json = { type: 'document', source: { type: 'text', media_type: 'text/plain', data: Buffer.from(p.data, 'base64').toString('utf8') } };
|
|
||||||
if (p.filename) doc.title = p.filename;
|
|
||||||
rest.push(doc);
|
|
||||||
} else {
|
|
||||||
rest.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case 'tool_call':
|
|
||||||
break; // not valid in a user message
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return [...results, ...rest];
|
|
||||||
}
|
|
||||||
|
|
||||||
function assistantBlocks(msg: ChatMessage, stripThinking: boolean): Json[] {
|
|
||||||
const pd = msg.providerData;
|
|
||||||
if (pd?.kind === 'anthropic' && isRecord(pd.data) && Array.isArray(pd.data.content)) {
|
|
||||||
// Same provider: replay the turn exactly as it was received, thinking blocks
|
|
||||||
// and signatures included.
|
|
||||||
const blocks = pd.data.content.filter(isRecord);
|
|
||||||
return stripThinking ? blocks.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking') : blocks;
|
|
||||||
}
|
|
||||||
// Another provider produced this turn: rebuild it from the neutral parts.
|
|
||||||
const blocks: Json[] = [];
|
|
||||||
for (const p of msg.content) {
|
|
||||||
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
|
|
||||||
else if (p.type === 'tool_call') blocks.push({ type: 'tool_use', id: safeId(p.id), name: p.name, input: p.input });
|
|
||||||
}
|
|
||||||
return blocks;
|
|
||||||
}
|
|
||||||
|
|
||||||
function toWireMessages(messages: ChatMessage[], caps: Partial<ModelCapabilities>, stripThinking: boolean): Json[] {
|
|
||||||
const out: Json[] = [];
|
|
||||||
for (const msg of messages) {
|
|
||||||
const content = msg.role === 'user' ? userBlocks(msg.content, caps) : assistantBlocks(msg, stripThinking);
|
|
||||||
if (content.length === 0) continue; // the API rejects empty turns
|
|
||||||
out.push({ role: msg.role, content });
|
|
||||||
}
|
|
||||||
if (out[0]?.role === 'assistant') out.unshift({ role: 'user', content: [{ type: 'text', text: '(continuing the conversation)' }] });
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildBody(req: ChatRequest, profile: ClaudeProfile, opts: BuildOptions): Json {
|
|
||||||
const caps = req.capabilities ?? {};
|
|
||||||
const defaults = lookupModelDefaults('anthropic', req.model);
|
|
||||||
const ceiling = defaults.maxOutput ?? Number.POSITIVE_INFINITY;
|
|
||||||
const maxTokens = Math.max(1, Math.min(req.maxOutputTokens ?? DEFAULT_MAX_TOKENS, ceiling));
|
|
||||||
|
|
||||||
const systemNotes: string[] = [];
|
|
||||||
const body: Json = { model: req.model, max_tokens: maxTokens, stream: true };
|
|
||||||
|
|
||||||
// --- tools -------------------------------------------------------
|
|
||||||
const tools = caps.tools === false ? [] : (req.tools ?? []);
|
|
||||||
let forcing = false;
|
|
||||||
if (tools.length > 0) {
|
|
||||||
body.tools = tools.map((t) => {
|
|
||||||
const tool: Json = { name: t.name, description: t.description, input_schema: { type: 'object', ...t.parameters } };
|
|
||||||
// Without this the API buffers each tool-input parameter and the stream goes
|
|
||||||
// silent for the whole time a large input (a generated quiz, say) is written.
|
|
||||||
// The price is that the input is no longer validated server-side, so it is
|
|
||||||
// parsed strictly below and truncated calls are dropped.
|
|
||||||
if (opts.eagerToolInput) tool.eager_input_streaming = true;
|
|
||||||
return tool;
|
|
||||||
});
|
|
||||||
const choice = req.toolChoice;
|
|
||||||
if (choice === 'none') {
|
|
||||||
body.tool_choice = { type: 'none' };
|
|
||||||
} else if (choice === 'required' || isRecord(choice)) {
|
|
||||||
const name = isRecord(choice) ? str(choice.name) : '';
|
|
||||||
if (profile.forcedToolChoice) {
|
|
||||||
body.tool_choice = name ? { type: 'tool', name } : { type: 'any' };
|
|
||||||
forcing = true;
|
|
||||||
} else {
|
|
||||||
// Fable 5.1+, Opus 5.5 and Sonnet 5.5 answer 400 to forced tool use; the
|
|
||||||
// documented replacement is `auto` plus an explicit instruction.
|
|
||||||
systemNotes.push(
|
|
||||||
name
|
|
||||||
? `For this turn you must call the \`${name}\` tool. Begin your response with that tool call.`
|
|
||||||
: 'For this turn you must call one of the available tools. Begin your response with the tool call.',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- thinking ----------------------------------------------------
|
|
||||||
if (caps.reasoning !== false && profile.thinking !== 'none') {
|
|
||||||
const effort = req.reasoningEffort;
|
|
||||||
const adaptive: Json = profile.display ? { type: 'adaptive', display: 'summarized' } : { type: 'adaptive' };
|
|
||||||
const outputConfig: Json = {};
|
|
||||||
if (effort === 'none') {
|
|
||||||
if (profile.thinking === 'always') {
|
|
||||||
// Cannot be disabled: the lowest effort is the closest thing to "off".
|
|
||||||
body.thinking = adaptive;
|
|
||||||
outputConfig.effort = 'low';
|
|
||||||
} else if (profile.thinking === 'default_on') {
|
|
||||||
body.thinking = { type: profile.off ?? 'disabled' }; // `between_tools` takes no other field
|
|
||||||
}
|
|
||||||
// opt_in / budget models do not think unless asked.
|
|
||||||
} else if (profile.thinking === 'always' || profile.thinking === 'default_on') {
|
|
||||||
// `display: "summarized"` because these models hide the reasoning text by
|
|
||||||
// default, which would look like a long pause in the UI.
|
|
||||||
body.thinking = adaptive;
|
|
||||||
if (effort) outputConfig.effort = effort;
|
|
||||||
} else if (effort) {
|
|
||||||
// Forced tool use cannot be combined with thinking on the older generations.
|
|
||||||
if (profile.thinking === 'opt_in' && !forcing) body.thinking = adaptive;
|
|
||||||
if (profile.thinking === 'budget' && !forcing) {
|
|
||||||
const budget = Math.min(LEGACY_BUDGET[effort], maxTokens - 1_024); // must be < max_tokens, min 1024
|
|
||||||
if (budget >= 1_024) body.thinking = { type: 'enabled', budget_tokens: budget };
|
|
||||||
}
|
|
||||||
if (profile.effort) outputConfig.effort = effort;
|
|
||||||
}
|
|
||||||
if (Object.keys(outputConfig).length > 0) body.output_config = outputConfig;
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- system prompt -----------------------------------------------
|
|
||||||
// There is no schema-less JSON mode (structured outputs need a schema), and
|
|
||||||
// assistant prefill is rejected by current models, so JSON mode is an instruction.
|
|
||||||
if (req.jsonMode) systemNotes.push('Respond with a single valid JSON object and nothing else — no prose, no Markdown code fences.');
|
|
||||||
const system = [req.system?.trim(), ...systemNotes].filter(Boolean).join('\n\n');
|
|
||||||
if (system) body.system = system;
|
|
||||||
|
|
||||||
body.messages = toWireMessages(req.messages, caps, opts.stripThinking);
|
|
||||||
|
|
||||||
if (opts.promptCache) body.cache_control = { type: 'ephemeral' }; // automatic caching of the reusable prefix
|
|
||||||
if (opts.useFallbacks) body.fallbacks = 'default';
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Adapter */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
function rootUrl(cfg: ProviderConfig): string {
|
|
||||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1$/, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
function isOfficial(cfg: ProviderConfig): boolean {
|
|
||||||
try {
|
|
||||||
return new URL(rootUrl(cfg)).hostname === 'api.anthropic.com';
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
|
||||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
|
||||||
}
|
|
||||||
|
|
||||||
function headers(cfg: ProviderConfig, betas: string[] = []): Record<string, string> {
|
|
||||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an Anthropic API key in Settings to use Claude models.');
|
|
||||||
const h: Record<string, string> = { ...cfg.headers, 'x-api-key': cfg.apiKey, 'anthropic-version': API_VERSION };
|
|
||||||
const userBeta = Object.entries(cfg.headers ?? {}).find(([k]) => k.toLowerCase() === 'anthropic-beta')?.[1];
|
|
||||||
const all = [...(userBeta ? userBeta.split(',').map((s) => s.trim()) : []), ...betas].filter(Boolean);
|
|
||||||
for (const k of Object.keys(h)) if (k.toLowerCase() === 'anthropic-beta') delete h[k];
|
|
||||||
if (all.length > 0) h['anthropic-beta'] = [...new Set(all)].join(',');
|
|
||||||
return h;
|
|
||||||
}
|
|
||||||
|
|
||||||
function mapStopReason(reason: string): StopReason {
|
|
||||||
switch (reason) {
|
|
||||||
case 'end_turn':
|
|
||||||
case 'stop_sequence':
|
|
||||||
return 'end';
|
|
||||||
case 'tool_use':
|
|
||||||
return 'tool_use';
|
|
||||||
case 'max_tokens':
|
|
||||||
case 'model_context_window_exceeded':
|
|
||||||
return 'max_tokens';
|
|
||||||
case 'refusal':
|
|
||||||
return 'refusal';
|
|
||||||
default:
|
|
||||||
return 'other'; // pause_turn (server tools only) and anything new
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
|
||||||
const ctx = errorContext(cfg);
|
|
||||||
const profile = claudeProfile(req.model);
|
|
||||||
const official = isOfficial(cfg);
|
|
||||||
const opts: BuildOptions = {
|
|
||||||
official,
|
|
||||||
stripThinking: false,
|
|
||||||
// The beta-gated extras are only sent to api.anthropic.com; gateways and
|
|
||||||
// proxies frequently reject fields they do not know.
|
|
||||||
useFallbacks: official && profile.fallbacks,
|
|
||||||
eagerToolInput: official,
|
|
||||||
promptCache: official,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Open the stream, degrading once per recognised 400 (see file header).
|
|
||||||
let http;
|
|
||||||
for (let attempt = 0; ; attempt++) {
|
|
||||||
http = await httpRequest(`${rootUrl(cfg)}/v1/messages`, {
|
|
||||||
...ctx,
|
|
||||||
headers: headers(cfg, opts.useFallbacks ? [FALLBACK_BETA] : []),
|
|
||||||
body: buildBody(req, profile, opts),
|
|
||||||
signal: req.signal,
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
await ensureOk(http);
|
|
||||||
break;
|
|
||||||
} catch (err) {
|
|
||||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
|
|
||||||
const detail = err.providerMessage ?? '';
|
|
||||||
if (!opts.stripThinking && /signature/i.test(detail) && /thinking/i.test(detail)) opts.stripThinking = true;
|
|
||||||
else if (opts.useFallbacks && /fallback|anthropic-beta/i.test(detail)) opts.useFallbacks = false;
|
|
||||||
else if (opts.eagerToolInput && /eager_input_streaming/i.test(detail)) opts.eagerToolInput = false;
|
|
||||||
else if (opts.promptCache && /cache_control/i.test(detail)) opts.promptCache = false;
|
|
||||||
else throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Content blocks by index, accumulated exactly as the API defines them.
|
|
||||||
const blocks: Json[] = [];
|
|
||||||
const toolJson = new Map<number, string>();
|
|
||||||
let stopReason = '';
|
|
||||||
let sawStop = false;
|
|
||||||
let reasoningEmitted = false;
|
|
||||||
const usage = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
|
|
||||||
const readUsage = (u: unknown) => {
|
|
||||||
if (!isRecord(u)) return;
|
|
||||||
if (typeof u.input_tokens === 'number') usage.input = u.input_tokens;
|
|
||||||
if (typeof u.cache_read_input_tokens === 'number') usage.cacheRead = u.cache_read_input_tokens;
|
|
||||||
if (typeof u.cache_creation_input_tokens === 'number') usage.cacheWrite = u.cache_creation_input_tokens;
|
|
||||||
if (typeof u.output_tokens === 'number') usage.output = u.output_tokens;
|
|
||||||
};
|
|
||||||
|
|
||||||
for await (const { data } of sseJson(http)) {
|
|
||||||
if (!isRecord(data)) continue;
|
|
||||||
switch (data.type) {
|
|
||||||
case 'message_start':
|
|
||||||
if (isRecord(data.message)) readUsage(data.message.usage);
|
|
||||||
break;
|
|
||||||
case 'content_block_start': {
|
|
||||||
const index = num(data.index);
|
|
||||||
const block: Json = isRecord(data.content_block) ? { ...data.content_block } : { type: 'unknown' };
|
|
||||||
blocks[index] = block;
|
|
||||||
if (block.type === 'tool_use') toolJson.set(index, '');
|
|
||||||
if (block.type === 'text' && typeof block.text === 'string' && block.text) yield { type: 'text', delta: block.text };
|
|
||||||
if (block.type === 'thinking' && reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'content_block_delta': {
|
|
||||||
const index = num(data.index);
|
|
||||||
const block = blocks[index];
|
|
||||||
const delta = data.delta;
|
|
||||||
if (!block || !isRecord(delta)) break;
|
|
||||||
if (delta.type === 'text_delta') {
|
|
||||||
const text = str(delta.text);
|
|
||||||
block.text = str(block.text) + text;
|
|
||||||
if (text) yield { type: 'text', delta: text };
|
|
||||||
} else if (delta.type === 'thinking_delta') {
|
|
||||||
const text = str(delta.thinking);
|
|
||||||
block.thinking = str(block.thinking) + text;
|
|
||||||
if (text) {
|
|
||||||
reasoningEmitted = true;
|
|
||||||
yield { type: 'reasoning', delta: text };
|
|
||||||
}
|
|
||||||
} else if (delta.type === 'signature_delta') {
|
|
||||||
block.signature = str(block.signature) + str(delta.signature);
|
|
||||||
} else if (delta.type === 'input_json_delta') {
|
|
||||||
toolJson.set(index, (toolJson.get(index) ?? '') + str(delta.partial_json));
|
|
||||||
} else if (delta.type === 'citations_delta') {
|
|
||||||
const list = Array.isArray(block.citations) ? block.citations : [];
|
|
||||||
list.push(delta.citation);
|
|
||||||
block.citations = list;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'message_delta':
|
|
||||||
if (isRecord(data.delta) && typeof data.delta.stop_reason === 'string') stopReason = data.delta.stop_reason;
|
|
||||||
readUsage(data.usage);
|
|
||||||
break;
|
|
||||||
case 'message_stop':
|
|
||||||
sawStop = true;
|
|
||||||
break;
|
|
||||||
case 'error':
|
|
||||||
// e.g. overloaded_error delivered after the 200 response started
|
|
||||||
throw errorFromProvider(ctx, undefined, data);
|
|
||||||
default:
|
|
||||||
break; // ping, and event types added later
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!sawStop && !stopReason) {
|
|
||||||
throw new AIError('network', 'The connection to Anthropic closed before the response finished.', { retryable: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tool calls are surfaced only now, at the end of the message, because the
|
|
||||||
// stop reason decides whether they are usable:
|
|
||||||
// - `refusal` can cut a tool_use off mid-input — never run that turn's tools;
|
|
||||||
// - `max_tokens` can truncate the last call — drop it rather than "repair" it;
|
|
||||||
// - after a mid-output fallback, tool_use blocks from before the last
|
|
||||||
// `fallback` block belong to the declined attempt and must not be echoed.
|
|
||||||
let boundary = -1;
|
|
||||||
blocks.forEach((b, i) => {
|
|
||||||
if (b?.type === 'fallback') boundary = i;
|
|
||||||
});
|
|
||||||
|
|
||||||
const content: ContentPart[] = [];
|
|
||||||
const replay: Json[] = [];
|
|
||||||
const calls: Array<{ id: string; name: string; input: Record<string, unknown> }> = [];
|
|
||||||
for (let i = 0; i < blocks.length; i++) {
|
|
||||||
const block = blocks[i];
|
|
||||||
if (!block) continue;
|
|
||||||
const beforeBoundary = i < boundary;
|
|
||||||
if (block.type === 'text') {
|
|
||||||
const text = str(block.text);
|
|
||||||
if (!text) continue; // empty text blocks are rejected when echoed back
|
|
||||||
content.push({ type: 'text', text });
|
|
||||||
replay.push(block);
|
|
||||||
} else if (block.type === 'tool_use') {
|
|
||||||
if (beforeBoundary || stopReason === 'refusal') continue;
|
|
||||||
const raw = toolJson.get(i) ?? '';
|
|
||||||
const name = str(block.name);
|
|
||||||
let input = parseArgsStrict(raw);
|
|
||||||
if (!input) {
|
|
||||||
if (stopReason === 'max_tokens') continue; // truncated mid-input
|
|
||||||
input = parseToolArguments(raw, LABEL, name);
|
|
||||||
}
|
|
||||||
const call = { id: str(block.id), name, input };
|
|
||||||
calls.push(call);
|
|
||||||
content.push({ type: 'tool_call', ...call });
|
|
||||||
replay.push({ ...block, input });
|
|
||||||
} else if (block.type === 'fallback') {
|
|
||||||
replay.push(block); // must stay exactly where it appeared
|
|
||||||
} else if (!beforeBoundary) {
|
|
||||||
replay.push(block); // thinking, redacted_thinking, …: echoed back unchanged
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const call of calls) yield { type: 'tool_call', call };
|
|
||||||
|
|
||||||
let mapped = mapStopReason(stopReason);
|
|
||||||
if (mapped === 'tool_use' && calls.length === 0) mapped = 'other';
|
|
||||||
if (mapped === 'end' && calls.length > 0) mapped = 'tool_use';
|
|
||||||
|
|
||||||
const message: ChatMessage = { role: 'assistant', content };
|
|
||||||
if (replay.length > 0) message.providerData = { kind: 'anthropic', data: { content: replay } };
|
|
||||||
yield {
|
|
||||||
type: 'done',
|
|
||||||
message,
|
|
||||||
stopReason: mapped,
|
|
||||||
// Anthropic reports uncached, cache-read and cache-write input separately.
|
|
||||||
usage: makeUsage({
|
|
||||||
input: usage.input + usage.cacheRead + usage.cacheWrite,
|
|
||||||
output: usage.output,
|
|
||||||
cacheRead: usage.cacheRead,
|
|
||||||
cacheWrite: usage.cacheWrite,
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function supported(caps: unknown, key: string): boolean | undefined {
|
|
||||||
if (!isRecord(caps) || !isRecord(caps[key])) return undefined;
|
|
||||||
const v = (caps[key] as Json).supported;
|
|
||||||
return typeof v === 'boolean' ? v : undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
|
||||||
const ctx = errorContext(cfg);
|
|
||||||
const found: DiscoveredModel[] = [];
|
|
||||||
let afterId = '';
|
|
||||||
// https://platform.claude.com/docs/en/api/models/list — cursor pagination via has_more / last_id.
|
|
||||||
for (let page = 0; page < 20; page++) {
|
|
||||||
const url = `${rootUrl(cfg)}/v1/models?limit=1000${afterId ? `&after_id=${encodeURIComponent(afterId)}` : ''}`;
|
|
||||||
const res = await requestJson<Json>(url, { ...ctx, headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
|
||||||
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
|
|
||||||
for (const m of data) {
|
|
||||||
const id = str(m.id);
|
|
||||||
if (!id || !isChatModelId(id)) continue;
|
|
||||||
const model: DiscoveredModel = {
|
|
||||||
id,
|
|
||||||
// The API version in use rejects sampling parameters on every model.
|
|
||||||
capabilities: { tools: true, temperature: false },
|
|
||||||
};
|
|
||||||
if (typeof m.display_name === 'string') model.label = m.display_name;
|
|
||||||
if (typeof m.max_input_tokens === 'number') model.contextWindow = m.max_input_tokens;
|
|
||||||
if (typeof m.max_tokens === 'number') model.maxOutput = m.max_tokens;
|
|
||||||
const vision = supported(m.capabilities, 'image_input');
|
|
||||||
const pdf = supported(m.capabilities, 'pdf_input');
|
|
||||||
const thinking = supported(m.capabilities, 'thinking');
|
|
||||||
if (vision !== undefined) model.capabilities = { ...model.capabilities, vision };
|
|
||||||
if (pdf !== undefined) model.capabilities = { ...model.capabilities, pdf };
|
|
||||||
if (thinking !== undefined) model.capabilities = { ...model.capabilities, reasoning: thinking };
|
|
||||||
found.push(model);
|
|
||||||
}
|
|
||||||
if (res.has_more !== true || typeof res.last_id !== 'string' || !res.last_id) break;
|
|
||||||
afterId = res.last_id;
|
|
||||||
}
|
|
||||||
return withCatalogDefaults('anthropic', found);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
|
||||||
// Listing models is free and proves both connectivity and the key.
|
|
||||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
|
||||||
}
|
|
||||||
|
|
||||||
export const anthropicAdapter: ProviderAdapter = {
|
|
||||||
kind: 'anthropic',
|
|
||||||
meta: {
|
|
||||||
label: LABEL,
|
|
||||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
|
||||||
requiresApiKey: true,
|
|
||||||
requiresBaseUrl: false,
|
|
||||||
supportsDiscovery: true,
|
|
||||||
docsUrl: 'https://platform.claude.com/settings/keys',
|
|
||||||
keyHint: 'sk-ant-…',
|
|
||||||
},
|
|
||||||
stream,
|
|
||||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
|
||||||
test,
|
|
||||||
};
|
|
||||||
|
|
@ -1,206 +0,0 @@
|
||||||
/**
|
|
||||||
* Static defaults for well-known models: capabilities, limits and list prices.
|
|
||||||
*
|
|
||||||
* Discovery APIs return ids (and sometimes limits) but almost never prices or
|
|
||||||
* feature flags, so `listModels` merges its results with this table. Unknown
|
|
||||||
* models simply get no defaults — nothing here is required for a model to work.
|
|
||||||
*
|
|
||||||
* Everything below was read from the official pages listed per provider on the
|
|
||||||
* dates noted. Prices are USD per million tokens, standard (non-batch,
|
|
||||||
* non-priority) tier, short-context tier where a provider has several. A field
|
|
||||||
* is omitted when it could not be confirmed from the page — do not guess.
|
|
||||||
*/
|
|
||||||
import type { DiscoveredModel, ModelCapabilities, ProviderKind } from '../types.ts';
|
|
||||||
|
|
||||||
export const CATALOG_UPDATED = '2026-11-10';
|
|
||||||
|
|
||||||
type Entry = Omit<DiscoveredModel, 'id' | 'capabilities'> & { capabilities?: Partial<ModelCapabilities> };
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* OpenAI */
|
|
||||||
/* Prices: https://developers.openai.com/api/docs/pricing (fetched 2026-11-11)
|
|
||||||
/* Limits: https://developers.openai.com/api/docs/models (fetched 2026-11-11;
|
|
||||||
/* only the four current flagship models list limits there — the
|
|
||||||
/* limits on older models below are the long-published values and were
|
|
||||||
/* not re-fetched)
|
|
||||||
/* Reasoning models reject a custom `temperature`, hence temperature:false.
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
const OA_REASONING: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
|
|
||||||
const OA_CLASSIC: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: false, temperature: true };
|
|
||||||
const OA_FLAGSHIP_LIMITS = { contextWindow: 1_050_000, maxOutput: 128_000 };
|
|
||||||
|
|
||||||
const OPENAI: Record<string, Entry> = {
|
|
||||||
'gpt-6-astra': { label: 'GPT-6 Astra', ...OA_FLAGSHIP_LIMITS, priceIn: 10, priceOut: 50, capabilities: OA_REASONING },
|
|
||||||
'gpt-6.1-sol': { label: 'GPT-6.1 Sol', ...OA_FLAGSHIP_LIMITS, priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
|
|
||||||
'gpt-6.1-terra': { label: 'GPT-6.1 Terra', ...OA_FLAGSHIP_LIMITS, priceIn: 1, priceOut: 5, capabilities: OA_REASONING },
|
|
||||||
'gpt-6-luna': { label: 'GPT-6 Luna', ...OA_FLAGSHIP_LIMITS, priceIn: 0.1, priceOut: 0.5, capabilities: OA_REASONING },
|
|
||||||
'gpt-6-sol': { label: 'GPT-6 Sol', priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.6-sol': { label: 'GPT-5.6 Sol', priceIn: 4, priceOut: 20, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.6-terra': { label: 'GPT-5.6 Terra', priceIn: 2, priceOut: 12, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.6-luna': { label: 'GPT-5.6 Luna', priceIn: 0.2, priceOut: 1.2, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.5': { label: 'GPT-5.5', priceIn: 5, priceOut: 30, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.5-pro': { label: 'GPT-5.5 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.4': { label: 'GPT-5.4', priceIn: 2.5, priceOut: 15, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.4-pro': { label: 'GPT-5.4 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.4-mini': { label: 'GPT-5.4 mini', priceIn: 0.75, priceOut: 4.5, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.4-nano': { label: 'GPT-5.4 nano', priceIn: 0.2, priceOut: 1.25, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.3-codex': { label: 'GPT-5.3 Codex', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.2': { label: 'GPT-5.2', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.2-pro': { label: 'GPT-5.2 Pro', priceIn: 21, priceOut: 168, capabilities: OA_REASONING },
|
|
||||||
'gpt-5.1': { label: 'GPT-5.1', priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
|
|
||||||
'gpt-5': { label: 'GPT-5', contextWindow: 400_000, maxOutput: 128_000, priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
|
|
||||||
'gpt-5-mini': { label: 'GPT-5 mini', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.25, priceOut: 2, capabilities: OA_REASONING },
|
|
||||||
'gpt-5-nano': { label: 'GPT-5 nano', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.05, priceOut: 0.4, capabilities: OA_REASONING },
|
|
||||||
'gpt-5-pro': { label: 'GPT-5 Pro', priceIn: 15, priceOut: 120, capabilities: OA_REASONING },
|
|
||||||
'chat-latest': { label: 'ChatGPT (latest)', priceIn: 5, priceOut: 30, capabilities: OA_CLASSIC },
|
|
||||||
'gpt-4.1': { label: 'GPT-4.1', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 2, priceOut: 8, capabilities: OA_CLASSIC },
|
|
||||||
'gpt-4.1-mini': { label: 'GPT-4.1 mini', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 0.4, priceOut: 1.6, capabilities: OA_CLASSIC },
|
|
||||||
'gpt-4o': { label: 'GPT-4o', contextWindow: 128_000, maxOutput: 16_384, priceIn: 2.5, priceOut: 10, capabilities: OA_CLASSIC },
|
|
||||||
'gpt-4o-mini': { label: 'GPT-4o mini', contextWindow: 128_000, maxOutput: 16_384, priceIn: 0.15, priceOut: 0.6, capabilities: OA_CLASSIC },
|
|
||||||
o3: { label: 'o3', contextWindow: 200_000, maxOutput: 100_000, priceIn: 2, priceOut: 8, capabilities: OA_REASONING },
|
|
||||||
'o3-pro': { label: 'o3-pro', contextWindow: 200_000, maxOutput: 100_000, priceIn: 20, priceOut: 80, capabilities: OA_REASONING },
|
|
||||||
};
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Anthropic */
|
|
||||||
/* Prices: https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-11-11)
|
|
||||||
/* Ids and limits: https://platform.claude.com/docs/en/about-claude/models/overview
|
|
||||||
/* (via Anthropic's Claude API reference, cached 2026-11-03)
|
|
||||||
/* temperature:false for every Claude model: the adapter speaks API version
|
|
||||||
/* 2026-10-21, which rejects temperature/top_p/top_k on all models
|
|
||||||
/* (https://platform.claude.com/docs/en/api/migration-2026-10-21).
|
|
||||||
/* Claude Haiku 5.5 prices are for prompts up to 100K tokens ($0.50/$2.50 above).
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
const CLAUDE: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
|
|
||||||
const CLAUDE_1M = { contextWindow: 1_000_000, maxOutput: 128_000 };
|
|
||||||
|
|
||||||
const ANTHROPIC: Record<string, Entry> = {
|
|
||||||
'claude-fable-5-2': { label: 'Claude Fable 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
|
|
||||||
'claude-mythos-5-2': { label: 'Claude Mythos 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
|
|
||||||
'claude-fable-5-1': { label: 'Claude Fable 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
|
||||||
'claude-mythos-5-1': { label: 'Claude Mythos 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
|
||||||
'claude-fable-5': { label: 'Claude Fable 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
|
||||||
'claude-mythos-5': { label: 'Claude Mythos 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
|
||||||
'claude-opus-5-5': { label: 'Claude Opus 5.5', ...CLAUDE_1M, priceIn: 4, priceOut: 20, capabilities: CLAUDE },
|
|
||||||
'claude-opus-5': { label: 'Claude Opus 5', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
|
||||||
'claude-opus-4-8': { label: 'Claude Opus 4.8', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
|
||||||
'claude-opus-4-7': { label: 'Claude Opus 4.7', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
|
||||||
'claude-opus-4-6': { label: 'Claude Opus 4.6', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
|
||||||
'claude-opus-4-5': { label: 'Claude Opus 4.5', priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
|
||||||
'claude-sonnet-5-5': { label: 'Claude Sonnet 5.5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
|
|
||||||
'claude-sonnet-5': { label: 'Claude Sonnet 5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
|
|
||||||
'claude-sonnet-4-6': { label: 'Claude Sonnet 4.6', ...CLAUDE_1M, priceIn: 3, priceOut: 15, capabilities: CLAUDE },
|
|
||||||
'claude-sonnet-4-5': { label: 'Claude Sonnet 4.5', priceIn: 3, priceOut: 15, capabilities: CLAUDE },
|
|
||||||
'claude-haiku-5-5': { label: 'Claude Haiku 5.5', ...CLAUDE_1M, priceIn: 0.1, priceOut: 0.5, capabilities: CLAUDE },
|
|
||||||
'claude-haiku-4-5': { label: 'Claude Haiku 4.5', contextWindow: 200_000, maxOutput: 64_000, priceIn: 1, priceOut: 5, capabilities: CLAUDE },
|
|
||||||
};
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Google Gemini */
|
|
||||||
/* Prices: https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-11-11,
|
|
||||||
/* now served from https://aistudio.google.com/docs/pricing)
|
|
||||||
/* Limits: https://ai.google.dev/gemini-api/docs/models (fetched 2026-11-11;
|
|
||||||
/* only the Gemini 4 models list limits — `GET /v1beta/models`
|
|
||||||
/* supplies inputTokenLimit/outputTokenLimit for the rest at runtime)
|
|
||||||
/* Scheduled changes announced on the pricing page (NOT applied here):
|
|
||||||
/* - gemini-4-carbon / gemini-4-argon: $4 / $20 from 2026-12-01
|
|
||||||
/* - gemini-3.8-flash / gemini-3.6-flash: $1.50 / $7.50 from 2027-01-01
|
|
||||||
/* - gemini-2.5-pro / gemini-2.5-flash are deprecated and are routed to
|
|
||||||
/* gemini-3.8-flash / gemini-3.5-flash-lite (and billed as such) from 2026-12-01
|
|
||||||
/* Pro-tier prices are for prompts up to 200K tokens.
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
const GEMINI_CAPS: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: true };
|
|
||||||
const GEMINI_4_LIMITS = { contextWindow: 1_048_576, maxOutput: 65_536 };
|
|
||||||
|
|
||||||
const GEMINI: Record<string, Entry> = {
|
|
||||||
'gemini-4-carbon': { label: 'Gemini 4 Carbon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-4-argon': { label: 'Gemini 4 Argon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-3.8-flash': { label: 'Gemini 3.8 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-3.6-flash': { label: 'Gemini 3.6 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-3.5-flash-lite': { label: 'Gemini 3.5 Flash-Lite', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-3.1-pro-preview': { label: 'Gemini 3.1 Pro (preview)', priceIn: 2, priceOut: 12, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-3.1-flash-lite': { label: 'Gemini 3.1 Flash-Lite', priceIn: 0.25, priceOut: 1.5, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-3-flash-preview': { label: 'Gemini 3 Flash (preview)', priceIn: 0.5, priceOut: 3, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-2.5-pro': { label: 'Gemini 2.5 Pro', priceIn: 1.25, priceOut: 10, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-2.5-flash': { label: 'Gemini 2.5 Flash', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
|
|
||||||
'gemini-2.5-flash-lite': { label: 'Gemini 2.5 Flash-Lite', priceIn: 0.1, priceOut: 0.4, capabilities: GEMINI_CAPS },
|
|
||||||
};
|
|
||||||
|
|
||||||
const TABLES: Record<ProviderKind, Record<string, Entry>> = {
|
|
||||||
openai: OPENAI,
|
|
||||||
anthropic: ANTHROPIC,
|
|
||||||
gemini: GEMINI,
|
|
||||||
// Aggregators and local servers host arbitrary models at their own prices.
|
|
||||||
openai_compatible: {},
|
|
||||||
ollama: {},
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* What may follow a known id for it to still be "the same model at the same
|
|
||||||
* price": a dated snapshot or a moving alias, e.g. `gpt-5-2025-08-07`,
|
|
||||||
* `claude-haiku-4-5-20251001`, `claude-opus-4-5@20251101`, `gemini-2.5-flash-latest`.
|
|
||||||
*/
|
|
||||||
const SNAPSHOT_SUFFIX = /^[-@:](\d{4}-\d{2}-\d{2}|\d{6,8}|\d{3,4}|latest|preview|preview-\d{2}-\d{2,4}|v\d+(:\d+)?)$/;
|
|
||||||
|
|
||||||
function clone(e: Entry): Partial<DiscoveredModel> {
|
|
||||||
return { ...e, ...(e.capabilities ? { capabilities: { ...e.capabilities } } : {}) };
|
|
||||||
}
|
|
||||||
|
|
||||||
function lookupIn(table: Record<string, Entry>, id: string): Partial<DiscoveredModel> {
|
|
||||||
const exact = table[id];
|
|
||||||
if (exact) return clone(exact);
|
|
||||||
|
|
||||||
// Longest known id that is a prefix of `id` and ends on a separator.
|
|
||||||
let best = '';
|
|
||||||
for (const key of Object.keys(table)) {
|
|
||||||
if (key.length > best.length && id.startsWith(key) && /[-@:]/.test(id.charAt(key.length))) best = key;
|
|
||||||
}
|
|
||||||
if (!best) return {};
|
|
||||||
const entry = table[best];
|
|
||||||
if (!entry) return {};
|
|
||||||
const out = clone(entry);
|
|
||||||
delete out.label; // the label names the base model, not this variant
|
|
||||||
if (!SNAPSHOT_SUFFIX.test(id.slice(best.length))) {
|
|
||||||
// A genuinely different variant (`o3-deep-research`, `gpt-4o-audio-...`):
|
|
||||||
// capabilities are a fair guess, the price is not.
|
|
||||||
delete out.priceIn;
|
|
||||||
delete out.priceOut;
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Defaults for a model: exact id first, then the longest known id that prefixes
|
|
||||||
* it (so dated snapshots match their base model). Returns `{}` when unknown.
|
|
||||||
*/
|
|
||||||
export function lookupModelDefaults(kind: ProviderKind, modelId: string): Partial<DiscoveredModel> {
|
|
||||||
// Gemini's listing names models "models/<id>".
|
|
||||||
const id = modelId.trim().replace(/^models\//, '');
|
|
||||||
if (!id) return {};
|
|
||||||
|
|
||||||
if (kind === 'openai_compatible') {
|
|
||||||
// Routers name models "<vendor>/<model>" (`anthropic/claude-sonnet-5-5`).
|
|
||||||
// Borrow capabilities and limits from the vendor's table, never its prices.
|
|
||||||
const bare = id.slice(id.lastIndexOf('/') + 1).replace(/:[a-z]+$/, '');
|
|
||||||
for (const table of [OPENAI, ANTHROPIC, GEMINI]) {
|
|
||||||
const hit = lookupIn(table, bare);
|
|
||||||
if (Object.keys(hit).length > 0) {
|
|
||||||
delete hit.priceIn;
|
|
||||||
delete hit.priceOut;
|
|
||||||
// Temperature limits are properties of the first-party APIs, not of the router.
|
|
||||||
if (hit.capabilities) delete hit.capabilities.temperature;
|
|
||||||
return hit;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
return lookupIn(TABLES[kind], id);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Every catalogued model id for a provider (used as a fallback list when discovery is unavailable). */
|
|
||||||
export function catalogModelIds(kind: ProviderKind): string[] {
|
|
||||||
return Object.keys(TABLES[kind]);
|
|
||||||
}
|
|
||||||
|
|
@ -1,138 +0,0 @@
|
||||||
/**
|
|
||||||
* Helpers shared by the adapters that are about our neutral contract rather
|
|
||||||
* than about HTTP: tool-argument parsing, model-list post-processing, the
|
|
||||||
* connection-test wrapper.
|
|
||||||
*/
|
|
||||||
import { jsonrepair } from 'jsonrepair';
|
|
||||||
import { AIError, type DiscoveredModel, type ProviderKind, type TestResult, type Usage } from '../types.ts';
|
|
||||||
import { lookupModelDefaults } from './catalog.ts';
|
|
||||||
import { isRecord } from './http.ts';
|
|
||||||
|
|
||||||
/** Strict parse of tool-call arguments; `undefined` when they are not a JSON object. */
|
|
||||||
export function parseArgsStrict(raw: string): Record<string, unknown> | undefined {
|
|
||||||
const text = raw.trim();
|
|
||||||
if (text === '') return {};
|
|
||||||
try {
|
|
||||||
const value: unknown = JSON.parse(text);
|
|
||||||
return isRecord(value) ? value : undefined;
|
|
||||||
} catch {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parses the JSON arguments of a completed tool call. Models occasionally emit
|
|
||||||
* slightly broken JSON (trailing commas, unescaped newlines); `jsonrepair`
|
|
||||||
* fixes those. Anything still unparseable is an `invalid_response`.
|
|
||||||
*
|
|
||||||
* Do NOT use this on a call that was cut off by the output-token limit —
|
|
||||||
* repairing a truncated call would silently invent a shorter input.
|
|
||||||
*/
|
|
||||||
export function parseToolArguments(raw: unknown, provider: string, toolName: string): Record<string, unknown> {
|
|
||||||
if (isRecord(raw)) return raw; // some servers send an object instead of a JSON string
|
|
||||||
const text = typeof raw === 'string' ? raw : raw == null ? '' : String(raw);
|
|
||||||
const strict = parseArgsStrict(text);
|
|
||||||
if (strict) return strict;
|
|
||||||
try {
|
|
||||||
const repaired: unknown = JSON.parse(jsonrepair(text));
|
|
||||||
if (isRecord(repaired)) return repaired;
|
|
||||||
} catch {
|
|
||||||
/* fall through */
|
|
||||||
}
|
|
||||||
throw new AIError('invalid_response', `${provider} produced a tool call for "${toolName}" with arguments that are not valid JSON.`, {
|
|
||||||
providerMessage: text.length > 300 ? `${text.slice(0, 300)}…` : text,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Stable id for providers that do not issue tool-call ids themselves. */
|
|
||||||
export function newToolCallId(prefix: string): string {
|
|
||||||
return `${prefix}_${globalThis.crypto.randomUUID().replace(/-/g, '').slice(0, 24)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function dataUrl(mime: string, base64: string): string {
|
|
||||||
return `data:${mime};base64,${base64}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function num(v: unknown): number {
|
|
||||||
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function str(v: unknown): string {
|
|
||||||
return typeof v === 'string' ? v : '';
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Usage convention used by every adapter:
|
|
||||||
* inputTokens = ALL prompt tokens, including cached ones
|
|
||||||
* cacheRead/cacheWriteTokens = the cached subsets of inputTokens
|
|
||||||
* outputTokens = ALL generated tokens, including reasoning
|
|
||||||
* reasoningTokens = the reasoning subset of outputTokens
|
|
||||||
*/
|
|
||||||
export function makeUsage(u: { input: number; output: number; cacheRead?: number; cacheWrite?: number; reasoning?: number }): Usage {
|
|
||||||
const usage: Usage = { inputTokens: u.input, outputTokens: u.output };
|
|
||||||
if (u.cacheRead) usage.cacheReadTokens = u.cacheRead;
|
|
||||||
if (u.cacheWrite) usage.cacheWriteTokens = u.cacheWrite;
|
|
||||||
if (u.reasoning) usage.reasoningTokens = u.reasoning;
|
|
||||||
return usage;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Ids that are obviously not text-chat models: embeddings, speech, image and
|
|
||||||
* video generation, moderation, realtime/audio-only, rerankers, legacy
|
|
||||||
* completion-only models.
|
|
||||||
*/
|
|
||||||
const NON_CHAT_RE = new RegExp(
|
|
||||||
[
|
|
||||||
'embed', // text-embedding-3-large, gemini-embedding-001, nomic-embed-text
|
|
||||||
'\\btts\\b|whisper|transcri|speech|\\baudio\\b|realtime|\\blive\\b',
|
|
||||||
'dall-e|gpt-image|image-generation|imagen|-image\\b|image-preview|\\bsora\\b|\\bveo\\b|lyria|stable-diffusion',
|
|
||||||
'moderation|-guard\\b|rerank|\\baqa\\b',
|
|
||||||
'computer-use|robotics|deep-research',
|
|
||||||
'^(babbage|davinci|curie|ada)\\b|^text-|turbo-instruct',
|
|
||||||
].join('|'),
|
|
||||||
'i',
|
|
||||||
);
|
|
||||||
|
|
||||||
export function isChatModelId(id: string): boolean {
|
|
||||||
return !NON_CHAT_RE.test(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Merges discovery results with catalog defaults; values the provider reported always win. */
|
|
||||||
export function withCatalogDefaults(kind: ProviderKind, models: DiscoveredModel[]): DiscoveredModel[] {
|
|
||||||
const seen = new Set<string>();
|
|
||||||
const out: DiscoveredModel[] = [];
|
|
||||||
for (const m of models) {
|
|
||||||
if (!m.id || seen.has(m.id)) continue;
|
|
||||||
seen.add(m.id);
|
|
||||||
const d = lookupModelDefaults(kind, m.id);
|
|
||||||
const merged: DiscoveredModel = { ...d, ...stripUndefined(m), id: m.id };
|
|
||||||
if (d.capabilities || m.capabilities) merged.capabilities = { ...d.capabilities, ...stripUndefined(m.capabilities ?? {}) };
|
|
||||||
out.push(merged);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function stripUndefined<T extends object>(obj: T): T {
|
|
||||||
const out = {} as Record<string, unknown>;
|
|
||||||
for (const [k, v] of Object.entries(obj)) if (v !== undefined && v !== null) out[k] = v;
|
|
||||||
return out as T;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Shared implementation of `ProviderAdapter.test()`: never throws, always friendly. */
|
|
||||||
export async function runTest(label: string, check: () => Promise<{ modelCount?: number; note?: string }>): Promise<TestResult> {
|
|
||||||
const started = Date.now();
|
|
||||||
try {
|
|
||||||
const { modelCount, note } = await check();
|
|
||||||
const latencyMs = Date.now() - started;
|
|
||||||
const message =
|
|
||||||
note ?? (modelCount !== undefined ? `Connected to ${label}: ${modelCount} model${modelCount === 1 ? '' : 's'} available.` : `Connected to ${label}.`);
|
|
||||||
return modelCount !== undefined ? { ok: true, message, latencyMs, modelCount } : { ok: true, message, latencyMs };
|
|
||||||
} catch (err) {
|
|
||||||
const latencyMs = Date.now() - started;
|
|
||||||
const message = err instanceof AIError ? err.message : `Could not connect to ${label}.`;
|
|
||||||
return { ok: false, message, latencyMs };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Timeout used by `test()` so a dead endpoint fails fast instead of hanging the settings page. */
|
|
||||||
export const TEST_TIMEOUT_MS = 20_000;
|
|
||||||
|
|
@ -1,342 +0,0 @@
|
||||||
/**
|
|
||||||
* Google Gemini — `models/{model}:streamGenerateContent?alt=sse`.
|
|
||||||
* https://ai.google.dev/api/generate-content
|
|
||||||
* https://ai.google.dev/gemini-api/docs/function-calling
|
|
||||||
* https://ai.google.dev/gemini-api/docs/thinking
|
|
||||||
* https://ai.google.dev/gemini-api/docs/thought-signatures
|
|
||||||
*
|
|
||||||
* Quirks handled here:
|
|
||||||
* - The API key travels in the `x-goog-api-key` header, never in the URL.
|
|
||||||
* - Function declarations accept only a subset of JSON Schema, so schemas are
|
|
||||||
* down-converted (`toGeminiSchema`).
|
|
||||||
* - Gemini issues no tool-call ids; stable ones are synthesised.
|
|
||||||
* - Thinking models attach a `thoughtSignature` to parts (notably function calls)
|
|
||||||
* that must be sent back unchanged, so each turn's raw parts live in `providerData`.
|
|
||||||
* - Thinking is configured with `thinkingLevel` on Gemini 3 and later and with a
|
|
||||||
* numeric `thinkingBudget` on 2.5; an unsupported config is dropped and retried.
|
|
||||||
* - A response can be blocked (`promptFeedback.blockReason`, finishReason SAFETY).
|
|
||||||
*/
|
|
||||||
import {
|
|
||||||
AIError,
|
|
||||||
type ChatMessage,
|
|
||||||
type ChatRequest,
|
|
||||||
type ContentPart,
|
|
||||||
type DiscoveredModel,
|
|
||||||
type ModelCapabilities,
|
|
||||||
type ProviderAdapter,
|
|
||||||
type ProviderConfig,
|
|
||||||
type StopReason,
|
|
||||||
type StreamEvent,
|
|
||||||
type TestResult,
|
|
||||||
} from '../types.ts';
|
|
||||||
import { isChatModelId, makeUsage, newToolCallId, num, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
|
||||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
|
||||||
|
|
||||||
type Json = Record<string, unknown>;
|
|
||||||
|
|
||||||
const LABEL = 'Google Gemini';
|
|
||||||
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com';
|
|
||||||
|
|
||||||
function root(cfg: ProviderConfig): string {
|
|
||||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1(beta)?$/, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
|
||||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
|
||||||
}
|
|
||||||
|
|
||||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
|
||||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add a Gemini API key in Settings to use Gemini models.');
|
|
||||||
return { ...cfg.headers, 'x-goog-api-key': cfg.apiKey };
|
|
||||||
}
|
|
||||||
|
|
||||||
const SCHEMA_KEYS = new Set(['type', 'description', 'enum', 'properties', 'required', 'items', 'nullable', 'format', 'minimum', 'maximum', 'minItems', 'maxItems', 'anyOf', 'title']);
|
|
||||||
const SCHEMA_FORMATS = new Set(['date-time', 'enum', 'int32', 'int64', 'float', 'double']);
|
|
||||||
|
|
||||||
/** Reduces JSON Schema to the OpenAPI-style subset Gemini's `parameters` field accepts. */
|
|
||||||
export function toGeminiSchema(schema: unknown): Json {
|
|
||||||
if (!isRecord(schema)) return { type: 'string' };
|
|
||||||
const out: Json = {};
|
|
||||||
for (const [key, value] of Object.entries(schema)) {
|
|
||||||
if (!SCHEMA_KEYS.has(key)) continue; // additionalProperties, $schema, default, pattern, const, ...
|
|
||||||
if (key === 'properties' && isRecord(value)) {
|
|
||||||
out.properties = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, toGeminiSchema(v)]));
|
|
||||||
} else if (key === 'items') {
|
|
||||||
out.items = toGeminiSchema(value);
|
|
||||||
} else if (key === 'anyOf' && Array.isArray(value)) {
|
|
||||||
out.anyOf = value.map(toGeminiSchema);
|
|
||||||
} else if (key === 'type' && Array.isArray(value)) {
|
|
||||||
// ["string","null"] -> nullable string
|
|
||||||
const real = value.filter((t) => t !== 'null');
|
|
||||||
out.type = real[0] ?? 'string';
|
|
||||||
if (value.includes('null')) out.nullable = true;
|
|
||||||
} else if (key === 'format') {
|
|
||||||
if (typeof value === 'string' && SCHEMA_FORMATS.has(value)) out.format = value;
|
|
||||||
} else {
|
|
||||||
out[key] = value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (out.enum && !out.type) out.type = 'string';
|
|
||||||
if (out.type === 'object' && !out.properties) out.properties = {};
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function userParts(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
|
||||||
const out: Json[] = [];
|
|
||||||
for (const p of parts) {
|
|
||||||
if (p.type === 'tool_result') {
|
|
||||||
out.push({ functionResponse: { name: p.name, response: p.isError ? { error: p.output } : { result: p.output } } });
|
|
||||||
} else if (p.type === 'text' && p.text) {
|
|
||||||
out.push({ text: p.text });
|
|
||||||
} else if (p.type === 'image') {
|
|
||||||
if (caps.vision === false) out.push({ text: '[An image was attached, but this model cannot read images.]' });
|
|
||||||
else out.push({ inlineData: { mimeType: p.mime, data: p.data } });
|
|
||||||
} else if (p.type === 'file') {
|
|
||||||
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ inlineData: { mimeType: p.mime, data: p.data } });
|
|
||||||
else if (p.mime.startsWith('text/')) out.push({ text: Buffer.from(p.data, 'base64').toString('utf8') });
|
|
||||||
else out.push({ text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function toContents(messages: ChatMessage[], caps: Partial<ModelCapabilities>): Json[] {
|
|
||||||
const contents: Json[] = [];
|
|
||||||
for (const msg of messages) {
|
|
||||||
let parts: Json[];
|
|
||||||
if (msg.role === 'assistant') {
|
|
||||||
const pd = msg.providerData;
|
|
||||||
if (pd?.kind === 'gemini' && isRecord(pd.data) && Array.isArray(pd.data.parts)) {
|
|
||||||
parts = pd.data.parts.filter(isRecord); // thought signatures included, exactly as received
|
|
||||||
} else {
|
|
||||||
parts = [];
|
|
||||||
for (const p of msg.content) {
|
|
||||||
if (p.type === 'text' && p.text) parts.push({ text: p.text });
|
|
||||||
else if (p.type === 'tool_call') parts.push({ functionCall: { name: p.name, args: p.input ?? {} } });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
parts = userParts(msg.content, caps);
|
|
||||||
}
|
|
||||||
if (parts.length === 0) continue;
|
|
||||||
const role = msg.role === 'assistant' ? 'model' : 'user';
|
|
||||||
const last = contents[contents.length - 1];
|
|
||||||
// Gemini requires strictly alternating roles; merge consecutive turns of the same role.
|
|
||||||
if (last && last.role === role) (last.parts as Json[]).push(...parts);
|
|
||||||
else contents.push({ role, parts });
|
|
||||||
}
|
|
||||||
if (contents[0]?.role === 'model') contents.unshift({ role: 'user', parts: [{ text: '(continuing the conversation)' }] });
|
|
||||||
return contents;
|
|
||||||
}
|
|
||||||
|
|
||||||
type ThinkingStyle = 'level' | 'budget' | 'off';
|
|
||||||
|
|
||||||
function thinkingStyleFor(model: string): ThinkingStyle {
|
|
||||||
const m = /gemini-(\d+)(?:\.(\d+))?/.exec(model);
|
|
||||||
if (!m) return 'level';
|
|
||||||
const major = Number(m[1]);
|
|
||||||
if (major >= 3) return 'level';
|
|
||||||
if (major === 2 && Number(m[2] ?? 0) >= 5) return 'budget';
|
|
||||||
return 'off';
|
|
||||||
}
|
|
||||||
|
|
||||||
const BUDGETS = { none: 0, low: 2_048, medium: 8_192, high: 24_576 } as const;
|
|
||||||
|
|
||||||
function buildBody(req: ChatRequest, thinking: ThinkingStyle, jsonMode: boolean): Json {
|
|
||||||
const caps = req.capabilities ?? {};
|
|
||||||
const body: Json = { contents: toContents(req.messages, caps) };
|
|
||||||
if (req.system?.trim()) body.systemInstruction = { parts: [{ text: req.system.trim() }] };
|
|
||||||
|
|
||||||
const hasTools = caps.tools !== false && (req.tools?.length ?? 0) > 0;
|
|
||||||
if (hasTools) {
|
|
||||||
body.tools = [{ functionDeclarations: req.tools!.map((t) => ({ name: t.name, description: t.description, parameters: toGeminiSchema({ type: 'object', ...t.parameters }) })) }];
|
|
||||||
const choice = req.toolChoice;
|
|
||||||
if (choice === 'none') body.toolConfig = { functionCallingConfig: { mode: 'NONE' } };
|
|
||||||
else if (choice === 'required') body.toolConfig = { functionCallingConfig: { mode: 'ANY' } };
|
|
||||||
else if (isRecord(choice)) body.toolConfig = { functionCallingConfig: { mode: 'ANY', allowedFunctionNames: [choice.name] } };
|
|
||||||
}
|
|
||||||
|
|
||||||
const gen: Json = {};
|
|
||||||
if (req.temperature !== undefined && caps.temperature !== false) gen.temperature = req.temperature;
|
|
||||||
if (req.maxOutputTokens) gen.maxOutputTokens = req.maxOutputTokens;
|
|
||||||
// JSON mode cannot be combined with function calling on most models.
|
|
||||||
if (jsonMode && req.jsonMode && !hasTools) gen.responseMimeType = 'application/json';
|
|
||||||
if (caps.reasoning !== false && thinking !== 'off') {
|
|
||||||
const effort = req.reasoningEffort;
|
|
||||||
if (thinking === 'level') {
|
|
||||||
const cfg: Json = { includeThoughts: true };
|
|
||||||
if (effort) cfg.thinkingLevel = effort === 'none' ? 'minimal' : effort;
|
|
||||||
gen.thinkingConfig = cfg;
|
|
||||||
} else if (effort) {
|
|
||||||
// 2.5 Pro cannot switch thinking off; its minimum budget is 128 tokens.
|
|
||||||
const budget = effort === 'none' && /pro/.test(req.model) ? 128 : BUDGETS[effort];
|
|
||||||
gen.thinkingConfig = { thinkingBudget: budget, includeThoughts: budget > 0 };
|
|
||||||
} else {
|
|
||||||
gen.thinkingConfig = { includeThoughts: true };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Object.keys(gen).length > 0) body.generationConfig = gen;
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
function mapFinish(reason: string): StopReason {
|
|
||||||
switch (reason) {
|
|
||||||
case 'STOP':
|
|
||||||
return 'end';
|
|
||||||
case 'MAX_TOKENS':
|
|
||||||
return 'max_tokens';
|
|
||||||
case 'SAFETY':
|
|
||||||
case 'RECITATION':
|
|
||||||
case 'BLOCKLIST':
|
|
||||||
case 'PROHIBITED_CONTENT':
|
|
||||||
case 'SPII':
|
|
||||||
case 'IMAGE_SAFETY':
|
|
||||||
return 'refusal';
|
|
||||||
default:
|
|
||||||
return 'other'; // MALFORMED_FUNCTION_CALL, OTHER, UNEXPECTED_TOOL_CALL, ...
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function modelPath(model: string): string {
|
|
||||||
return model.startsWith('models/') || model.startsWith('tunedModels/') ? model : `models/${model}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
|
||||||
const ctx = errorContext(cfg);
|
|
||||||
const url = `${root(cfg)}/v1beta/${modelPath(req.model)}:streamGenerateContent?alt=sse`;
|
|
||||||
let thinking = thinkingStyleFor(req.model);
|
|
||||||
let jsonMode = true;
|
|
||||||
|
|
||||||
let http: HttpResponse;
|
|
||||||
for (let attempt = 0; ; attempt += 1) {
|
|
||||||
http = await httpRequest(url, { ...ctx, headers: headers(cfg), body: buildBody(req, thinking, jsonMode), signal: req.signal });
|
|
||||||
try {
|
|
||||||
await ensureOk(http);
|
|
||||||
break;
|
|
||||||
} catch (err) {
|
|
||||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
|
|
||||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
|
||||||
if (thinking !== 'off' && /thinking|thought/.test(detail)) thinking = thinking === 'level' ? 'budget' : 'off';
|
|
||||||
else if (jsonMode && req.jsonMode && /response_?mime|json/.test(detail)) jsonMode = false;
|
|
||||||
else throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const parts: Json[] = [];
|
|
||||||
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
|
||||||
let text = '';
|
|
||||||
let finish = '';
|
|
||||||
let blocked = '';
|
|
||||||
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
|
|
||||||
|
|
||||||
for await (const { data } of sseJson(http)) {
|
|
||||||
if (!isRecord(data)) continue;
|
|
||||||
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
|
|
||||||
if (isRecord(data.promptFeedback) && str(data.promptFeedback.blockReason)) blocked = str(data.promptFeedback.blockReason);
|
|
||||||
if (isRecord(data.usageMetadata)) {
|
|
||||||
const u = data.usageMetadata;
|
|
||||||
usage.input = num(u.promptTokenCount) || usage.input;
|
|
||||||
usage.reasoning = num(u.thoughtsTokenCount) || usage.reasoning;
|
|
||||||
// candidatesTokenCount excludes thinking; both are billed as output.
|
|
||||||
usage.output = num(u.candidatesTokenCount) + num(u.thoughtsTokenCount) || usage.output;
|
|
||||||
usage.cacheRead = num(u.cachedContentTokenCount) || usage.cacheRead;
|
|
||||||
}
|
|
||||||
const candidate = Array.isArray(data.candidates) ? data.candidates[0] : undefined;
|
|
||||||
if (!isRecord(candidate)) continue;
|
|
||||||
if (str(candidate.finishReason)) finish = str(candidate.finishReason);
|
|
||||||
const chunkParts = isRecord(candidate.content) && Array.isArray(candidate.content.parts) ? candidate.content.parts.filter(isRecord) : [];
|
|
||||||
for (const part of chunkParts) {
|
|
||||||
if (isRecord(part.functionCall)) {
|
|
||||||
const call = { id: str(part.functionCall.id) || newToolCallId('gem'), name: str(part.functionCall.name), input: isRecord(part.functionCall.args) ? part.functionCall.args : {} };
|
|
||||||
calls.push(call);
|
|
||||||
parts.push(part);
|
|
||||||
yield { type: 'tool_call', call };
|
|
||||||
} else if (typeof part.text === 'string') {
|
|
||||||
if (part.thought === true) {
|
|
||||||
if (part.text) yield { type: 'reasoning', delta: part.text };
|
|
||||||
// Thought summaries are display-only; only a signature needs to survive.
|
|
||||||
if (part.thoughtSignature) parts.push({ text: '', thought: true, thoughtSignature: part.thoughtSignature });
|
|
||||||
} else {
|
|
||||||
if (part.text) {
|
|
||||||
text += part.text;
|
|
||||||
yield { type: 'text', delta: part.text };
|
|
||||||
}
|
|
||||||
// Merge streamed text fragments, but keep a part that carries a signature intact.
|
|
||||||
const last = parts[parts.length - 1];
|
|
||||||
if (last && typeof last.text === 'string' && last.thought !== true && !last.thoughtSignature && !part.thoughtSignature) last.text += part.text;
|
|
||||||
else parts.push({ ...part });
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
parts.push(part); // inlineData, executableCode, ... replayed untouched
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (blocked && !text && calls.length === 0) {
|
|
||||||
throw new AIError('content_filter', 'Gemini blocked this request under its safety policy.', { providerMessage: `blockReason: ${blocked}` });
|
|
||||||
}
|
|
||||||
if (!finish && !text && calls.length === 0) {
|
|
||||||
throw new AIError('network', 'The connection to Gemini closed before the response finished.', { retryable: true });
|
|
||||||
}
|
|
||||||
if (finish === 'MALFORMED_FUNCTION_CALL' && calls.length === 0) {
|
|
||||||
throw new AIError('invalid_response', 'Gemini produced a malformed tool call. Try again.', { retryable: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
const content: ContentPart[] = [];
|
|
||||||
if (text) content.push({ type: 'text', text });
|
|
||||||
for (const call of calls) content.push({ type: 'tool_call', ...call });
|
|
||||||
const message: ChatMessage = { role: 'assistant', content };
|
|
||||||
const replay = parts.filter((p) => !(typeof p.text === 'string' && p.text === '' && !p.thoughtSignature));
|
|
||||||
if (replay.length > 0) message.providerData = { kind: 'gemini', data: { parts: replay } };
|
|
||||||
yield {
|
|
||||||
type: 'done',
|
|
||||||
message,
|
|
||||||
stopReason: calls.length > 0 ? 'tool_use' : mapFinish(finish || 'STOP'),
|
|
||||||
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
|
||||||
const found: DiscoveredModel[] = [];
|
|
||||||
let pageToken = '';
|
|
||||||
for (let page = 0; page < 10; page += 1) {
|
|
||||||
const url = `${root(cfg)}/v1beta/models?pageSize=1000${pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''}`;
|
|
||||||
const res = await requestJson<Json>(url, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
|
||||||
for (const m of Array.isArray(res.models) ? res.models.filter(isRecord) : []) {
|
|
||||||
const id = str(m.name).replace(/^models\//, '');
|
|
||||||
const methods = Array.isArray(m.supportedGenerationMethods) ? m.supportedGenerationMethods.map(String) : [];
|
|
||||||
if (!id || !methods.includes('generateContent') || !isChatModelId(id)) continue;
|
|
||||||
if (!/^(gemini|gemma|learnlm)/.test(id)) continue;
|
|
||||||
const model: DiscoveredModel = { id };
|
|
||||||
if (str(m.displayName)) model.label = str(m.displayName);
|
|
||||||
if (num(m.inputTokenLimit)) model.contextWindow = num(m.inputTokenLimit);
|
|
||||||
if (num(m.outputTokenLimit)) model.maxOutput = num(m.outputTokenLimit);
|
|
||||||
if (typeof m.thinking === 'boolean') model.capabilities = { reasoning: m.thinking };
|
|
||||||
if (id.startsWith('gemma')) model.capabilities = { ...model.capabilities, tools: false, pdf: false };
|
|
||||||
found.push(model);
|
|
||||||
}
|
|
||||||
pageToken = str(res.nextPageToken);
|
|
||||||
if (!pageToken) break;
|
|
||||||
}
|
|
||||||
return withCatalogDefaults('gemini', found);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
|
||||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
|
||||||
}
|
|
||||||
|
|
||||||
export const geminiAdapter: ProviderAdapter = {
|
|
||||||
kind: 'gemini',
|
|
||||||
meta: {
|
|
||||||
label: LABEL,
|
|
||||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
|
||||||
requiresApiKey: true,
|
|
||||||
requiresBaseUrl: false,
|
|
||||||
supportsDiscovery: true,
|
|
||||||
docsUrl: 'https://aistudio.google.com/apikey',
|
|
||||||
keyHint: 'AIza…',
|
|
||||||
},
|
|
||||||
stream,
|
|
||||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
|
||||||
test,
|
|
||||||
};
|
|
||||||
|
|
@ -1,519 +0,0 @@
|
||||||
/**
|
|
||||||
* Shared HTTP plumbing for provider adapters: a fetch wrapper with timeouts,
|
|
||||||
* an SSE parser over Web Streams, and one place that turns provider failures
|
|
||||||
* into `AIError`s. Uses only global `fetch` — no SDKs.
|
|
||||||
*
|
|
||||||
* Security rule for this whole folder: an API key must never reach an error
|
|
||||||
* message, a log line or a thrown value. Everything user-visible that contains
|
|
||||||
* provider text goes through `scrubSecrets` first.
|
|
||||||
*/
|
|
||||||
import { AIError, type AIErrorCode } from '../types.ts';
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Secret scrubbing */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
const KEY_PATTERNS: RegExp[] = [
|
|
||||||
/\bsk-[A-Za-z0-9_-]{8,}/g, // OpenAI / Anthropic / OpenRouter style
|
|
||||||
/\bAIza[0-9A-Za-z_-]{16,}/g, // Google API keys
|
|
||||||
/\bgsk_[A-Za-z0-9]{8,}/g, // Groq
|
|
||||||
/(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi,
|
|
||||||
/([?&](?:key|api_key|apikey|access_token)=)[^&\s"']+/gi,
|
|
||||||
];
|
|
||||||
|
|
||||||
/** Removes known secrets (and anything that looks like an API key) from provider text. */
|
|
||||||
export function scrubSecrets(text: string, secrets: Array<string | null | undefined> = []): string {
|
|
||||||
let out = text;
|
|
||||||
for (const s of secrets) {
|
|
||||||
if (s && s.length >= 4) out = out.split(s).join('[redacted]');
|
|
||||||
}
|
|
||||||
for (const re of KEY_PATTERNS) {
|
|
||||||
out = out.replace(re, (_m, prefix?: unknown) => (typeof prefix === 'string' ? `${prefix}[redacted]` : '[redacted]'));
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function clip(text: string, max = 800): string {
|
|
||||||
const t = text.trim();
|
|
||||||
return t.length > max ? `${t.slice(0, max)}…` : t;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Error mapping */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
export interface ErrorContext {
|
|
||||||
/** Human provider label used in messages ("OpenAI", "Ollama"). */
|
|
||||||
provider: string;
|
|
||||||
/** Values that must never appear in an error (API key, custom header values). */
|
|
||||||
secrets?: Array<string | null | undefined>;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ProviderErrorFields {
|
|
||||||
message: string;
|
|
||||||
type: string;
|
|
||||||
code: string;
|
|
||||||
/** Whole body as text, used for keyword matching (quota ids, reasons, ...). */
|
|
||||||
raw: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Pulls message/type/code out of the error envelopes used by the providers we talk to. */
|
|
||||||
export function extractProviderError(body: unknown): ProviderErrorFields {
|
|
||||||
let node: unknown = body;
|
|
||||||
if (typeof node === 'string') {
|
|
||||||
const text = node;
|
|
||||||
try {
|
|
||||||
node = JSON.parse(text);
|
|
||||||
} catch {
|
|
||||||
return { message: text.trim().startsWith('<') ? '' : text.trim(), type: '', code: '', raw: text };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const raw = safeStringify(node);
|
|
||||||
// Gemini sometimes wraps the error object in an array.
|
|
||||||
if (Array.isArray(node)) node = node[0];
|
|
||||||
if (!isRecord(node)) return { message: '', type: '', code: '', raw };
|
|
||||||
const err = node.error ?? node;
|
|
||||||
// Ollama: {"error": "model 'x' not found"}
|
|
||||||
if (typeof err === 'string') return { message: err, type: '', code: '', raw };
|
|
||||||
if (!isRecord(err)) return { message: '', type: '', code: '', raw };
|
|
||||||
const message = typeof err.message === 'string' ? err.message : typeof node.message === 'string' ? node.message : '';
|
|
||||||
// OpenAI: error.type + error.code; Anthropic: error.type; Gemini: error.status (+ numeric code).
|
|
||||||
const type = [err.type, err.status].filter((v): v is string => typeof v === 'string').join(' ');
|
|
||||||
const code = typeof err.code === 'string' ? err.code : '';
|
|
||||||
return { message, type, code, raw };
|
|
||||||
}
|
|
||||||
|
|
||||||
const CONTEXT_RE =
|
|
||||||
/context[_ ]length|context[_ ]window|maximum context|max(?:imum)? (?:input )?tokens? (?:limit|allowed)|too many (?:input )?tokens|prompt is too long|input is too long|input token count|exceeds? the (?:model'?s? )?(?:maximum|max)[^.]{0,40}tokens|reduce the length|request[_ ]too[_ ]large|exceed_context_size|n_ctx/i;
|
|
||||||
const QUOTA_RE =
|
|
||||||
/insufficient[_ ]quota|billing|credit balance|exceeded your current quota|out of credits|insufficient (?:credits|funds|balance)|payment required|spend(?:ing)? limit/i;
|
|
||||||
const AUTH_RE = /invalid[_ ]api[_ ]key|incorrect api key|api key not valid|api_key_invalid|authentication[_ ]error|invalid x-api-key|unauthorized|no auth credentials/i;
|
|
||||||
const FILTER_RE = /content[_ ]filter|content[_ ]policy|content management policy|safety system|responsible ai/i;
|
|
||||||
|
|
||||||
function classify(status: number | undefined, f: ProviderErrorFields): AIErrorCode {
|
|
||||||
const hay = `${f.type} ${f.code} ${f.message}`;
|
|
||||||
const s = status ?? 0;
|
|
||||||
|
|
||||||
// Provider error names win over the status code: they are also all we have
|
|
||||||
// for errors delivered inside an already-open (HTTP 200) stream.
|
|
||||||
if (/authentication_error|invalid_api_key|permission_error|PERMISSION_DENIED|UNAUTHENTICATED/.test(hay)) return 'auth';
|
|
||||||
if (/insufficient_quota|billing_error/.test(hay)) return 'quota';
|
|
||||||
if (/context_length_exceeded|request_too_large/.test(hay)) return 'context_length';
|
|
||||||
if (/overloaded_error|server_error|api_error|UNAVAILABLE|INTERNAL\b/.test(`${f.type} ${f.code}`)) return 'server';
|
|
||||||
|
|
||||||
if (s === 401 || s === 403) return 'auth';
|
|
||||||
if (s === 402) return 'quota';
|
|
||||||
if (s === 429 || /rate_limit|RESOURCE_EXHAUSTED/.test(hay)) {
|
|
||||||
// Gemini words every 429 as "exceeded your current quota"; its details say
|
|
||||||
// whether it is a per-minute limit (retry soon) or a daily/billing cap.
|
|
||||||
if (/PerMinute|retryDelay|RetryInfo/.test(f.raw) && !/PerDay/.test(f.raw)) return 'rate_limit';
|
|
||||||
return QUOTA_RE.test(hay) || /PerDay/.test(f.raw) ? 'quota' : 'rate_limit';
|
|
||||||
}
|
|
||||||
if (s === 404 || /not_found_error|model_not_found|NOT_FOUND/.test(hay)) return 'not_found';
|
|
||||||
if (s === 408 || s === 504 || /DEADLINE_EXCEEDED/.test(hay)) return 'timeout';
|
|
||||||
if (s === 413) return 'context_length';
|
|
||||||
if (s >= 500) return 'server';
|
|
||||||
// Remaining 4xx (and status-less stream errors).
|
|
||||||
if (AUTH_RE.test(hay)) return 'auth'; // Gemini answers a bad key with 400 INVALID_ARGUMENT
|
|
||||||
if (CONTEXT_RE.test(hay)) return 'context_length';
|
|
||||||
if (QUOTA_RE.test(hay)) return 'quota'; // Anthropic: 400 "credit balance is too low"
|
|
||||||
if (FILTER_RE.test(hay)) return 'content_filter';
|
|
||||||
if (s === 0 && !f.message) return 'server';
|
|
||||||
return 'bad_request';
|
|
||||||
}
|
|
||||||
|
|
||||||
function friendlyMessage(code: AIErrorCode, provider: string): string {
|
|
||||||
switch (code) {
|
|
||||||
case 'auth':
|
|
||||||
return `${provider} rejected the API key.`;
|
|
||||||
case 'rate_limit':
|
|
||||||
return `${provider} is rate limiting requests. Try again in a moment.`;
|
|
||||||
case 'quota':
|
|
||||||
return `${provider} reports that the account is out of quota or credit.`;
|
|
||||||
case 'context_length':
|
|
||||||
return `The conversation is too long for this ${provider} model.`;
|
|
||||||
case 'not_found':
|
|
||||||
return `${provider} could not find that model or endpoint.`;
|
|
||||||
case 'content_filter':
|
|
||||||
return `${provider} blocked the request with its content filter.`;
|
|
||||||
case 'server':
|
|
||||||
return `${provider} is having trouble right now. Try again shortly.`;
|
|
||||||
case 'timeout':
|
|
||||||
return `${provider} took too long to respond.`;
|
|
||||||
case 'bad_request':
|
|
||||||
return `${provider} rejected the request.`;
|
|
||||||
default:
|
|
||||||
return `${provider} returned an error.`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The single place that maps "HTTP status + provider error body" to an AIError.
|
|
||||||
* Pass `status: undefined` for errors that arrive as events inside a stream.
|
|
||||||
*/
|
|
||||||
export function errorFromProvider(ctx: ErrorContext, status: number | undefined, body: unknown): AIError {
|
|
||||||
const fields = extractProviderError(body);
|
|
||||||
const code = classify(status, fields);
|
|
||||||
const detail = fields.message || (status ? `HTTP ${status}` : '');
|
|
||||||
return new AIError(code, friendlyMessage(code, ctx.provider), {
|
|
||||||
status,
|
|
||||||
providerMessage: detail ? clip(scrubSecrets(detail, ctx.secrets)) : undefined,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Maps anything thrown by `fetch` / a body read to an AIError. */
|
|
||||||
export function errorFromThrown(ctx: ErrorContext, err: unknown, abortReason?: 'timeout' | 'aborted' | null): AIError {
|
|
||||||
if (err instanceof AIError) return err;
|
|
||||||
const name = isRecord(err) && typeof err.name === 'string' ? err.name : '';
|
|
||||||
if (abortReason === 'timeout' || name === 'TimeoutError') {
|
|
||||||
return new AIError('timeout', friendlyMessage('timeout', ctx.provider), { cause: err });
|
|
||||||
}
|
|
||||||
if (abortReason === 'aborted' || name === 'AbortError') {
|
|
||||||
return new AIError('aborted', 'The request was cancelled.', { cause: err });
|
|
||||||
}
|
|
||||||
// undici reports DNS/connection failures as TypeError("fetch failed") with the real reason in `cause`.
|
|
||||||
const cause = isRecord(err) && isRecord(err.cause) ? err.cause : undefined;
|
|
||||||
const parts = [
|
|
||||||
cause && typeof cause.code === 'string' ? cause.code : '',
|
|
||||||
cause && typeof cause.message === 'string' ? cause.message : '',
|
|
||||||
err instanceof Error ? err.message : String(err),
|
|
||||||
].filter(Boolean);
|
|
||||||
return new AIError('network', `Could not reach ${ctx.provider}. Check the address and your connection.`, {
|
|
||||||
providerMessage: clip(scrubSecrets(parts.join(': '), ctx.secrets)),
|
|
||||||
cause: err,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* fetch with timeouts */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
export interface HttpOptions extends ErrorContext {
|
|
||||||
method?: 'GET' | 'POST';
|
|
||||||
headers?: Record<string, string>;
|
|
||||||
/** Serialised as JSON when present. */
|
|
||||||
body?: unknown;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
/** Time allowed until response headers arrive (default 60s). */
|
|
||||||
firstByteTimeoutMs?: number;
|
|
||||||
/** Hard cap on the whole exchange (default 30 min — streams are long). */
|
|
||||||
totalTimeoutMs?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface HttpResponse {
|
|
||||||
response: Response;
|
|
||||||
ctx: ErrorContext;
|
|
||||||
/** Abort the exchange from our side (used by the idle watchdog). */
|
|
||||||
abort(reason: 'timeout' | 'aborted'): void;
|
|
||||||
abortReason(): 'timeout' | 'aborted' | null;
|
|
||||||
/** Clears timers/listeners. Safe to call more than once. */
|
|
||||||
cleanup(): void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const DEFAULT_FIRST_BYTE_TIMEOUT_MS = 60_000;
|
|
||||||
export const DEFAULT_TOTAL_TIMEOUT_MS = 30 * 60_000;
|
|
||||||
/** Longest silence tolerated inside an open stream before we give up. */
|
|
||||||
export const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60_000;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sends a request and resolves once headers are in. Does NOT check `response.ok`
|
|
||||||
* (callers decide; see `ensureOk`). Throws AIError for abort/timeout/network.
|
|
||||||
*/
|
|
||||||
export async function httpRequest(url: string, opts: HttpOptions): Promise<HttpResponse> {
|
|
||||||
const ctx: ErrorContext = { provider: opts.provider, secrets: opts.secrets };
|
|
||||||
if (opts.signal?.aborted) throw new AIError('aborted', 'The request was cancelled.');
|
|
||||||
|
|
||||||
const controller = new AbortController();
|
|
||||||
let reason: 'timeout' | 'aborted' | null = null;
|
|
||||||
const abort = (r: 'timeout' | 'aborted') => {
|
|
||||||
if (reason) return;
|
|
||||||
reason = r;
|
|
||||||
controller.abort();
|
|
||||||
};
|
|
||||||
const onUserAbort = () => abort('aborted');
|
|
||||||
opts.signal?.addEventListener('abort', onUserAbort, { once: true });
|
|
||||||
|
|
||||||
const firstByte = setTimeout(() => abort('timeout'), opts.firstByteTimeoutMs ?? DEFAULT_FIRST_BYTE_TIMEOUT_MS);
|
|
||||||
const total = setTimeout(() => abort('timeout'), opts.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS);
|
|
||||||
let cleaned = false;
|
|
||||||
const cleanup = () => {
|
|
||||||
if (cleaned) return;
|
|
||||||
cleaned = true;
|
|
||||||
clearTimeout(firstByte);
|
|
||||||
clearTimeout(total);
|
|
||||||
opts.signal?.removeEventListener('abort', onUserAbort);
|
|
||||||
};
|
|
||||||
|
|
||||||
const headers: Record<string, string> = { ...opts.headers };
|
|
||||||
let body: string | undefined;
|
|
||||||
if (opts.body !== undefined) {
|
|
||||||
body = JSON.stringify(opts.body);
|
|
||||||
if (!hasHeader(headers, 'content-type')) headers['Content-Type'] = 'application/json';
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const response = await fetch(url, {
|
|
||||||
method: opts.method ?? (body === undefined ? 'GET' : 'POST'),
|
|
||||||
headers,
|
|
||||||
body,
|
|
||||||
signal: controller.signal,
|
|
||||||
});
|
|
||||||
clearTimeout(firstByte);
|
|
||||||
return { response, ctx, abort, abortReason: () => reason, cleanup };
|
|
||||||
} catch (err) {
|
|
||||||
cleanup();
|
|
||||||
throw errorFromThrown(ctx, err, reason);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Reads the body of a failed response and throws the mapped AIError. Returns normally when `ok`. */
|
|
||||||
export async function ensureOk(http: HttpResponse): Promise<void> {
|
|
||||||
if (http.response.ok) return;
|
|
||||||
let text = '';
|
|
||||||
try {
|
|
||||||
text = await http.response.text();
|
|
||||||
} catch {
|
|
||||||
/* body unreadable — the status code alone still maps to something useful */
|
|
||||||
} finally {
|
|
||||||
http.cleanup();
|
|
||||||
}
|
|
||||||
throw errorFromProvider(http.ctx, http.response.status, text);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Request + status check + JSON parse, for non-streaming calls such as model discovery. */
|
|
||||||
export async function requestJson<T = unknown>(url: string, opts: HttpOptions): Promise<T> {
|
|
||||||
const http = await httpRequest(url, opts);
|
|
||||||
await ensureOk(http);
|
|
||||||
try {
|
|
||||||
const text = await http.response.text();
|
|
||||||
try {
|
|
||||||
return JSON.parse(text) as T;
|
|
||||||
} catch {
|
|
||||||
throw new AIError('invalid_response', `${opts.provider} sent a response that was not valid JSON.`, {
|
|
||||||
providerMessage: clip(scrubSecrets(text, opts.secrets), 300),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
throw errorFromThrown(http.ctx, err, http.abortReason());
|
|
||||||
} finally {
|
|
||||||
http.cleanup();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Server-sent events */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
export interface SSEMessage {
|
|
||||||
/** Value of the `event:` field, when the server sent one. */
|
|
||||||
event?: string;
|
|
||||||
/** All `data:` lines of the event joined with "\n". */
|
|
||||||
data: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SSEOptions {
|
|
||||||
/** Abort when no bytes arrive for this long. */
|
|
||||||
idleTimeoutMs?: number;
|
|
||||||
onIdle?: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Parses a text/event-stream body. Handles `\n`, `\r\n` and bare `\r` line
|
|
||||||
* endings (also when split across chunks), multi-line `data:`, comment lines,
|
|
||||||
* a UTF-8 BOM, and a final event that is not followed by a blank line.
|
|
||||||
* Iteration ends at end-of-stream or at the OpenAI-style `data: [DONE]` sentinel.
|
|
||||||
*/
|
|
||||||
export async function* parseSSE(stream: ReadableStream<Uint8Array>, opts: SSEOptions = {}): AsyncGenerator<SSEMessage> {
|
|
||||||
const reader = stream.getReader();
|
|
||||||
const decoder = new TextDecoder('utf-8');
|
|
||||||
let buffer = '';
|
|
||||||
let dataLines: string[] = [];
|
|
||||||
let eventName: string | undefined;
|
|
||||||
let sawCR = false; // previous chunk ended in "\r": swallow a leading "\n"
|
|
||||||
let first = true;
|
|
||||||
|
|
||||||
const takeLine = (line: string): SSEMessage | 'done' | null => {
|
|
||||||
if (line === '') {
|
|
||||||
// Blank line dispatches the event.
|
|
||||||
if (dataLines.length === 0) {
|
|
||||||
eventName = undefined;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const msg: SSEMessage = { data: dataLines.join('\n') };
|
|
||||||
if (eventName !== undefined) msg.event = eventName;
|
|
||||||
dataLines = [];
|
|
||||||
eventName = undefined;
|
|
||||||
return msg.data === '[DONE]' ? 'done' : msg;
|
|
||||||
}
|
|
||||||
if (line.startsWith(':')) return null; // comment / keep-alive
|
|
||||||
const colon = line.indexOf(':');
|
|
||||||
const field = colon === -1 ? line : line.slice(0, colon);
|
|
||||||
let value = colon === -1 ? '' : line.slice(colon + 1);
|
|
||||||
if (value.startsWith(' ')) value = value.slice(1);
|
|
||||||
if (field === 'data') dataLines.push(value);
|
|
||||||
else if (field === 'event') eventName = value;
|
|
||||||
// `id` and `retry` are irrelevant for one-shot completions.
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
for (;;) {
|
|
||||||
const { done, value } = await readWithIdle(reader, opts);
|
|
||||||
if (done) break;
|
|
||||||
let chunk = decoder.decode(value, { stream: true });
|
|
||||||
if (first && chunk.length > 0) {
|
|
||||||
first = false;
|
|
||||||
if (chunk.charCodeAt(0) === 0xfeff) chunk = chunk.slice(1);
|
|
||||||
}
|
|
||||||
if (sawCR && chunk.startsWith('\n')) chunk = chunk.slice(1);
|
|
||||||
if (chunk.length > 0) sawCR = chunk.endsWith('\r');
|
|
||||||
buffer += chunk;
|
|
||||||
|
|
||||||
let start = 0;
|
|
||||||
for (;;) {
|
|
||||||
const idx = indexOfLineEnd(buffer, start);
|
|
||||||
if (idx === -1) break;
|
|
||||||
const line = buffer.slice(start, idx);
|
|
||||||
start = buffer[idx] === '\r' && buffer[idx + 1] === '\n' ? idx + 2 : idx + 1;
|
|
||||||
const out = takeLine(line);
|
|
||||||
if (out === 'done') return;
|
|
||||||
if (out) yield out;
|
|
||||||
}
|
|
||||||
buffer = buffer.slice(start);
|
|
||||||
}
|
|
||||||
// Flush: tolerate a last line / last event without the terminating newline(s).
|
|
||||||
buffer += decoder.decode();
|
|
||||||
if (buffer.length > 0) {
|
|
||||||
const out = takeLine(buffer);
|
|
||||||
if (out === 'done') return;
|
|
||||||
if (out) yield out;
|
|
||||||
}
|
|
||||||
const tail = takeLine('');
|
|
||||||
if (tail && tail !== 'done') yield tail;
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
await reader.cancel();
|
|
||||||
} catch {
|
|
||||||
/* already closed or errored */
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function indexOfLineEnd(s: string, from: number): number {
|
|
||||||
for (let i = from; i < s.length; i++) {
|
|
||||||
const c = s.charCodeAt(i);
|
|
||||||
if (c === 10 || c === 13) return i;
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readWithIdle(
|
|
||||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
||||||
opts: SSEOptions,
|
|
||||||
): Promise<Awaited<ReturnType<ReadableStreamDefaultReader<Uint8Array>['read']>>> {
|
|
||||||
if (!opts.idleTimeoutMs) return reader.read();
|
|
||||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
||||||
const idle = new Promise<never>((_resolve, reject) => {
|
|
||||||
timer = setTimeout(() => {
|
|
||||||
opts.onIdle?.();
|
|
||||||
const err = new Error('stream idle timeout');
|
|
||||||
err.name = 'TimeoutError';
|
|
||||||
reject(err);
|
|
||||||
}, opts.idleTimeoutMs);
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
return await Promise.race([reader.read(), idle]);
|
|
||||||
} finally {
|
|
||||||
clearTimeout(timer);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Iterates the SSE events of an open response, translating read failures
|
|
||||||
* (dropped connection, abort, idle timeout) into AIErrors and always releasing
|
|
||||||
* the request's timers. `data` is JSON-parsed; unparseable events are skipped.
|
|
||||||
*/
|
|
||||||
export async function* sseJson(http: HttpResponse, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS): AsyncGenerator<{ event?: string; data: unknown }> {
|
|
||||||
const body = http.response.body;
|
|
||||||
if (!body) {
|
|
||||||
http.cleanup();
|
|
||||||
throw new AIError('invalid_response', `${http.ctx.provider} returned an empty response.`);
|
|
||||||
}
|
|
||||||
const events = parseSSE(body, { idleTimeoutMs, onIdle: () => http.abort('timeout') });
|
|
||||||
try {
|
|
||||||
for (;;) {
|
|
||||||
let next: IteratorResult<SSEMessage>;
|
|
||||||
try {
|
|
||||||
next = await events.next();
|
|
||||||
} catch (err) {
|
|
||||||
throw errorFromThrown(http.ctx, err, http.abortReason());
|
|
||||||
}
|
|
||||||
if (next.done) return;
|
|
||||||
let data: unknown;
|
|
||||||
try {
|
|
||||||
data = JSON.parse(next.value.data);
|
|
||||||
} catch {
|
|
||||||
continue; // keep-alives such as OpenRouter's ": OPENROUTER PROCESSING" variants, or junk
|
|
||||||
}
|
|
||||||
yield next.value.event !== undefined ? { event: next.value.event, data } : { data };
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
await events.return(undefined).catch(() => undefined);
|
|
||||||
http.cleanup();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Small shared utilities */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
export function isRecord(v: unknown): v is Record<string, unknown> {
|
|
||||||
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
||||||
}
|
|
||||||
|
|
||||||
function hasHeader(headers: Record<string, string>, name: string): boolean {
|
|
||||||
const n = name.toLowerCase();
|
|
||||||
return Object.keys(headers).some((k) => k.toLowerCase() === n);
|
|
||||||
}
|
|
||||||
|
|
||||||
function safeStringify(v: unknown): string {
|
|
||||||
try {
|
|
||||||
return JSON.stringify(v) ?? '';
|
|
||||||
} catch {
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Trims whitespace and trailing slashes from a base URL. */
|
|
||||||
export function trimBaseUrl(url: string): string {
|
|
||||||
return url.trim().replace(/\/+$/, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Host part of a URL for user-facing messages; never includes credentials or query. */
|
|
||||||
export function hostOf(url: string): string {
|
|
||||||
try {
|
|
||||||
return new URL(url).host;
|
|
||||||
} catch {
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** True for loopback / private-network hosts, where a local model server may need minutes to load a model. */
|
|
||||||
export function isLocalUrl(url: string): boolean {
|
|
||||||
try {
|
|
||||||
const h = new URL(url).hostname.replace(/^\[|\]$/g, '');
|
|
||||||
return (
|
|
||||||
h === 'localhost' ||
|
|
||||||
h === '::1' ||
|
|
||||||
h.endsWith('.local') ||
|
|
||||||
h.endsWith('.internal') ||
|
|
||||||
/^127\./.test(h) ||
|
|
||||||
/^10\./.test(h) ||
|
|
||||||
/^192\.168\./.test(h) ||
|
|
||||||
/^172\.(1[6-9]|2\d|3[01])\./.test(h) ||
|
|
||||||
h === 'host.docker.internal'
|
|
||||||
);
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
import { AIError, type ProviderAdapter, type ProviderKind } from '../types.ts';
|
|
||||||
import { anthropicAdapter } from './anthropic.ts';
|
|
||||||
import { geminiAdapter } from './gemini.ts';
|
|
||||||
import { ollamaAdapter } from './ollama.ts';
|
|
||||||
import { openaiAdapter } from './openai.ts';
|
|
||||||
import { openaiCompatibleAdapter } from './openaiCompatible.ts';
|
|
||||||
|
|
||||||
const ADAPTERS: Record<ProviderKind, ProviderAdapter> = {
|
|
||||||
openai: openaiAdapter,
|
|
||||||
anthropic: anthropicAdapter,
|
|
||||||
gemini: geminiAdapter,
|
|
||||||
openai_compatible: openaiCompatibleAdapter,
|
|
||||||
ollama: ollamaAdapter,
|
|
||||||
};
|
|
||||||
|
|
||||||
export function getAdapter(kind: ProviderKind): ProviderAdapter {
|
|
||||||
const adapter = ADAPTERS[kind];
|
|
||||||
if (!adapter) throw new AIError('unsupported', `Unknown provider type "${kind}".`);
|
|
||||||
return adapter;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function listAdapterMeta(): Array<{ kind: ProviderKind } & ProviderAdapter['meta']> {
|
|
||||||
return (Object.keys(ADAPTERS) as ProviderKind[]).map((kind) => ({ kind, ...ADAPTERS[kind].meta }));
|
|
||||||
}
|
|
||||||
|
|
||||||
export { CATALOG_UPDATED, lookupModelDefaults } from './catalog.ts';
|
|
||||||
|
|
@ -1,102 +0,0 @@
|
||||||
/**
|
|
||||||
* Ollama (local models). Chat goes through Ollama's OpenAI-compatible endpoint
|
|
||||||
* (`{base}/v1/chat/completions`), which handles tools and streaming; discovery
|
|
||||||
* uses the native API because it reports what each model can do:
|
|
||||||
* GET /api/tags -> installed models
|
|
||||||
* POST /api/show -> { capabilities: ["completion","tools","vision","thinking"], model_info }
|
|
||||||
* https://docs.ollama.com/api · https://docs.ollama.com/openai
|
|
||||||
*/
|
|
||||||
import { AIError, type DiscoveredModel, type ModelCapabilities, type ProviderAdapter, type ProviderConfig, type TestResult } from '../types.ts';
|
|
||||||
import { num, runTest, str, TEST_TIMEOUT_MS } from './common.ts';
|
|
||||||
import { isRecord, requestJson, trimBaseUrl, type ErrorContext } from './http.ts';
|
|
||||||
import { streamChatCompletions, type CompatOptions } from './openaiCompatible.ts';
|
|
||||||
|
|
||||||
const LABEL = 'Ollama';
|
|
||||||
const DEFAULT_BASE_URL = 'http://localhost:11434';
|
|
||||||
|
|
||||||
function root(cfg: ProviderConfig): string {
|
|
||||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/(v1|api)$/, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
function ctx(cfg: ProviderConfig): ErrorContext {
|
|
||||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
|
||||||
}
|
|
||||||
|
|
||||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
|
||||||
const h: Record<string, string> = { ...cfg.headers };
|
|
||||||
if (cfg.apiKey) h.Authorization = `Bearer ${cfg.apiKey}`; // only needed behind an authenticating proxy
|
|
||||||
return h;
|
|
||||||
}
|
|
||||||
|
|
||||||
function compat(cfg: ProviderConfig): CompatOptions {
|
|
||||||
return { kind: 'ollama', label: LABEL, baseUrl: `${root(cfg)}/v1`, apiKey: cfg.apiKey, headers: cfg.headers };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
|
||||||
let tags: unknown;
|
|
||||||
try {
|
|
||||||
tags = await requestJson(`${root(cfg)}/api/tags`, { ...ctx(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs ?? 15_000, totalTimeoutMs: timeoutMs ?? 30_000 });
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof AIError && err.code === 'network') {
|
|
||||||
throw new AIError('network', `Could not reach Ollama at ${root(cfg)}. Is it running, and reachable from the Study Desk container?`, { retryable: true });
|
|
||||||
}
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
const list = isRecord(tags) && Array.isArray(tags.models) ? tags.models.filter(isRecord) : [];
|
|
||||||
const models: DiscoveredModel[] = [];
|
|
||||||
// /api/show is one cheap local call per model; cap it so a huge library cannot stall the settings page.
|
|
||||||
await Promise.all(
|
|
||||||
list.slice(0, 60).map(async (m) => {
|
|
||||||
const id = str(m.model) || str(m.name);
|
|
||||||
if (!id) return;
|
|
||||||
const model: DiscoveredModel = { id, label: str(m.name) || id, priceIn: 0, priceOut: 0 };
|
|
||||||
try {
|
|
||||||
const show = await requestJson<Record<string, unknown>>(`${root(cfg)}/api/show`, {
|
|
||||||
...ctx(cfg),
|
|
||||||
headers: headers(cfg),
|
|
||||||
body: { model: id },
|
|
||||||
signal,
|
|
||||||
firstByteTimeoutMs: 10_000,
|
|
||||||
totalTimeoutMs: 10_000,
|
|
||||||
});
|
|
||||||
if (Array.isArray(show.capabilities)) {
|
|
||||||
const caps = show.capabilities.map(String);
|
|
||||||
if (!caps.includes('completion')) return; // embedding-only model
|
|
||||||
const c: Partial<ModelCapabilities> = { tools: caps.includes('tools'), vision: caps.includes('vision'), reasoning: caps.includes('thinking'), pdf: false, temperature: true };
|
|
||||||
model.capabilities = c;
|
|
||||||
}
|
|
||||||
if (isRecord(show.model_info)) {
|
|
||||||
const key = Object.keys(show.model_info).find((k) => k.endsWith('.context_length'));
|
|
||||||
if (key && num(show.model_info[key])) model.contextWindow = num(show.model_info[key]);
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
/* older Ollama versions: keep the model with unknown capabilities */
|
|
||||||
}
|
|
||||||
models.push(model);
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
return models.sort((a, b) => a.id.localeCompare(b.id));
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
|
||||||
return runTest(LABEL, async () => {
|
|
||||||
const count = (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length;
|
|
||||||
return count === 0 ? { modelCount: 0, note: 'Connected to Ollama, but no models are installed. Run `ollama pull <model>` first.' } : { modelCount: count };
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export const ollamaAdapter: ProviderAdapter = {
|
|
||||||
kind: 'ollama',
|
|
||||||
meta: {
|
|
||||||
label: LABEL,
|
|
||||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
|
||||||
requiresApiKey: false,
|
|
||||||
requiresBaseUrl: false,
|
|
||||||
supportsDiscovery: true,
|
|
||||||
docsUrl: 'https://docs.ollama.com',
|
|
||||||
keyHint: 'Not needed',
|
|
||||||
},
|
|
||||||
stream: (cfg, req) => streamChatCompletions(compat(cfg), req),
|
|
||||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
|
||||||
test,
|
|
||||||
};
|
|
||||||
|
|
@ -1,313 +0,0 @@
|
||||||
/**
|
|
||||||
* OpenAI — Responses API (`POST /v1/responses`, SSE).
|
|
||||||
* https://platform.openai.com/docs/api-reference/responses
|
|
||||||
* https://platform.openai.com/docs/guides/function-calling
|
|
||||||
* https://platform.openai.com/docs/guides/reasoning
|
|
||||||
*
|
|
||||||
* Quirks handled here:
|
|
||||||
* - Requests are stateless (`store: false`). Reasoning models need their reasoning
|
|
||||||
* items passed back between tool calls, so encrypted reasoning is requested via
|
|
||||||
* `include: ["reasoning.encrypted_content"]` and every turn's raw output items are
|
|
||||||
* kept in `providerData` and replayed verbatim.
|
|
||||||
* - Reasoning models reject `temperature`; classic models reject `reasoning`.
|
|
||||||
* - The accepted `reasoning.effort` values differ by model generation. When a model
|
|
||||||
* rejects the chosen value the request is retried once with a safe one.
|
|
||||||
* - Tool-call arguments arrive as deltas keyed by output item.
|
|
||||||
*/
|
|
||||||
import {
|
|
||||||
AIError,
|
|
||||||
type ChatMessage,
|
|
||||||
type ChatRequest,
|
|
||||||
type ContentPart,
|
|
||||||
type DiscoveredModel,
|
|
||||||
type ModelCapabilities,
|
|
||||||
type ProviderAdapter,
|
|
||||||
type ProviderConfig,
|
|
||||||
type StopReason,
|
|
||||||
type StreamEvent,
|
|
||||||
type TestResult,
|
|
||||||
} from '../types.ts';
|
|
||||||
import { lookupModelDefaults } from './catalog.ts';
|
|
||||||
import { dataUrl, isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
|
||||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
|
||||||
|
|
||||||
type Json = Record<string, unknown>;
|
|
||||||
|
|
||||||
const LABEL = 'OpenAI';
|
|
||||||
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
|
|
||||||
|
|
||||||
function base(cfg: ProviderConfig): string {
|
|
||||||
const url = trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL);
|
|
||||||
return /\/v\d+$/.test(url) ? url : `${url}/v1`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
|
||||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
|
||||||
}
|
|
||||||
|
|
||||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
|
||||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an OpenAI API key in Settings to use OpenAI models.');
|
|
||||||
return { ...cfg.headers, Authorization: `Bearer ${cfg.apiKey}` };
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Whether a model is a reasoning model, when the caller did not say. */
|
|
||||||
function isReasoningModel(req: ChatRequest): boolean {
|
|
||||||
if (req.capabilities?.reasoning !== undefined) return req.capabilities.reasoning;
|
|
||||||
const known = lookupModelDefaults('openai', req.model).capabilities?.reasoning;
|
|
||||||
if (known !== undefined) return known;
|
|
||||||
return /^(o\d|gpt-[5-9]|gpt-\d{2})/.test(req.model) && !/chat/.test(req.model);
|
|
||||||
}
|
|
||||||
|
|
||||||
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
|
||||||
const out: Json[] = [];
|
|
||||||
for (const p of parts) {
|
|
||||||
if (p.type === 'text' && p.text) out.push({ type: 'input_text', text: p.text });
|
|
||||||
else if (p.type === 'image') {
|
|
||||||
if (caps.vision === false) out.push({ type: 'input_text', text: '[An image was attached, but this model cannot read images.]' });
|
|
||||||
else out.push({ type: 'input_image', image_url: dataUrl(p.mime, p.data) });
|
|
||||||
} else if (p.type === 'file') {
|
|
||||||
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ type: 'input_file', filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) });
|
|
||||||
else if (p.mime.startsWith('text/')) out.push({ type: 'input_text', text: Buffer.from(p.data, 'base64').toString('utf8') });
|
|
||||||
else out.push({ type: 'input_text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function toInput(messages: ChatMessage[], caps: Partial<ModelCapabilities>, dropReasoning: boolean): Json[] {
|
|
||||||
const input: Json[] = [];
|
|
||||||
for (const msg of messages) {
|
|
||||||
if (msg.role === 'assistant') {
|
|
||||||
const pd = msg.providerData;
|
|
||||||
if (pd?.kind === 'openai' && isRecord(pd.data) && Array.isArray(pd.data.output)) {
|
|
||||||
// Same provider: replay the output items exactly (reasoning items included).
|
|
||||||
for (const item of pd.data.output.filter(isRecord)) {
|
|
||||||
if (item.type === 'reasoning' && (dropReasoning || !item.encrypted_content)) continue;
|
|
||||||
input.push(item);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
|
||||||
if (text) input.push({ role: 'assistant', content: [{ type: 'output_text', text }] });
|
|
||||||
for (const p of msg.content) {
|
|
||||||
if (p.type === 'tool_call') input.push({ type: 'function_call', call_id: p.id, name: p.name, arguments: JSON.stringify(p.input ?? {}) });
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
for (const p of msg.content) {
|
|
||||||
if (p.type === 'tool_result') input.push({ type: 'function_call_output', call_id: p.toolCallId, output: p.output || '(no output)' });
|
|
||||||
}
|
|
||||||
const content = userContent(msg.content.filter((p) => p.type !== 'tool_result'), caps);
|
|
||||||
if (content.length > 0) input.push({ role: 'user', content });
|
|
||||||
}
|
|
||||||
return input;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Flags {
|
|
||||||
/** Value sent as reasoning.effort, or null to omit the field. */
|
|
||||||
effort: string | null;
|
|
||||||
summary: boolean;
|
|
||||||
temperature: boolean;
|
|
||||||
dropReasoning: boolean;
|
|
||||||
include: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildBody(req: ChatRequest, flags: Flags): Json {
|
|
||||||
const caps = req.capabilities ?? {};
|
|
||||||
const body: Json = { model: req.model, input: toInput(req.messages, caps, flags.dropReasoning), stream: true, store: false };
|
|
||||||
if (req.system?.trim()) body.instructions = req.system.trim();
|
|
||||||
if (req.maxOutputTokens) body.max_output_tokens = Math.max(16, req.maxOutputTokens);
|
|
||||||
if (caps.tools !== false && req.tools?.length) {
|
|
||||||
body.tools = req.tools.map((t) => ({ type: 'function', name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters }, strict: false }));
|
|
||||||
const choice = req.toolChoice;
|
|
||||||
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
|
|
||||||
else if (isRecord(choice)) body.tool_choice = { type: 'function', name: choice.name };
|
|
||||||
}
|
|
||||||
if (isReasoningModel(req)) {
|
|
||||||
const reasoning: Json = {};
|
|
||||||
if (flags.effort) reasoning.effort = flags.effort;
|
|
||||||
if (flags.summary) reasoning.summary = 'auto';
|
|
||||||
if (Object.keys(reasoning).length > 0) body.reasoning = reasoning;
|
|
||||||
if (flags.include) body.include = ['reasoning.encrypted_content'];
|
|
||||||
} else if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) {
|
|
||||||
body.temperature = req.temperature;
|
|
||||||
}
|
|
||||||
if (req.jsonMode) body.text = { format: { type: 'json_object' } };
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
|
||||||
const ctx = errorContext(cfg);
|
|
||||||
const flags: Flags = {
|
|
||||||
// "none" is only understood by recent models; if it is rejected we fall back to "low".
|
|
||||||
effort: req.reasoningEffort ?? null,
|
|
||||||
summary: true,
|
|
||||||
temperature: true,
|
|
||||||
dropReasoning: false,
|
|
||||||
include: true,
|
|
||||||
};
|
|
||||||
|
|
||||||
let http: HttpResponse;
|
|
||||||
for (let attempt = 0; ; attempt += 1) {
|
|
||||||
http = await httpRequest(`${base(cfg)}/responses`, { ...ctx, headers: headers(cfg), body: buildBody(req, flags), signal: req.signal });
|
|
||||||
try {
|
|
||||||
await ensureOk(http);
|
|
||||||
break;
|
|
||||||
} catch (err) {
|
|
||||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 5) throw err;
|
|
||||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
|
||||||
if (flags.effort && /reasoning\.effort|reasoning_effort|effort/.test(detail)) flags.effort = flags.effort === 'none' ? 'minimal' : flags.effort === 'minimal' ? 'low' : null;
|
|
||||||
else if (flags.summary && /summary/.test(detail)) flags.summary = false;
|
|
||||||
else if (flags.include && /include|encrypted_content/.test(detail)) flags.include = false;
|
|
||||||
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
|
|
||||||
else if (!flags.dropReasoning && /reasoning/.test(detail) && /item|input/.test(detail)) flags.dropReasoning = true;
|
|
||||||
else throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const args = new Map<string, string>();
|
|
||||||
let final: Json | null = null;
|
|
||||||
let reasoningEmitted = false;
|
|
||||||
let streamedText = '';
|
|
||||||
|
|
||||||
for await (const { data } of sseJson(http)) {
|
|
||||||
if (!isRecord(data)) continue;
|
|
||||||
switch (data.type) {
|
|
||||||
case 'response.output_text.delta': {
|
|
||||||
const delta = str(data.delta);
|
|
||||||
if (delta) {
|
|
||||||
streamedText += delta;
|
|
||||||
yield { type: 'text', delta };
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'response.refusal.delta': {
|
|
||||||
const delta = str(data.delta);
|
|
||||||
if (delta) yield { type: 'text', delta };
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'response.reasoning_summary_part.added':
|
|
||||||
if (reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
|
|
||||||
break;
|
|
||||||
case 'response.reasoning_summary_text.delta':
|
|
||||||
case 'response.reasoning_text.delta': {
|
|
||||||
const delta = str(data.delta);
|
|
||||||
if (delta) {
|
|
||||||
reasoningEmitted = true;
|
|
||||||
yield { type: 'reasoning', delta };
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'response.function_call_arguments.delta':
|
|
||||||
args.set(str(data.item_id), (args.get(str(data.item_id)) ?? '') + str(data.delta));
|
|
||||||
break;
|
|
||||||
case 'response.completed':
|
|
||||||
case 'response.incomplete':
|
|
||||||
if (isRecord(data.response)) final = data.response;
|
|
||||||
break;
|
|
||||||
case 'response.failed':
|
|
||||||
throw errorFromProvider(ctx, undefined, isRecord(data.response) && isRecord(data.response.error) ? { error: data.response.error } : data);
|
|
||||||
case 'error':
|
|
||||||
throw errorFromProvider(ctx, undefined, isRecord(data.error) ? data : { error: data });
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!final) throw new AIError('network', 'The connection to OpenAI closed before the response finished.', { retryable: true });
|
|
||||||
|
|
||||||
const incomplete = isRecord(final.incomplete_details) ? str(final.incomplete_details.reason) : '';
|
|
||||||
const truncated = incomplete === 'max_output_tokens';
|
|
||||||
const output = Array.isArray(final.output) ? final.output.filter(isRecord) : [];
|
|
||||||
const content: ContentPart[] = [];
|
|
||||||
const replay: Json[] = [];
|
|
||||||
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
|
||||||
let refused = false;
|
|
||||||
let text = '';
|
|
||||||
|
|
||||||
for (const item of output) {
|
|
||||||
if (item.type === 'message') {
|
|
||||||
for (const part of Array.isArray(item.content) ? item.content.filter(isRecord) : []) {
|
|
||||||
if (part.type === 'output_text') text += str(part.text);
|
|
||||||
else if (part.type === 'refusal') {
|
|
||||||
refused = true;
|
|
||||||
text += str(part.refusal);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
replay.push(item);
|
|
||||||
} else if (item.type === 'function_call') {
|
|
||||||
const name = str(item.name);
|
|
||||||
const raw = str(item.arguments) || args.get(str(item.id)) || '';
|
|
||||||
let input = parseArgsStrict(raw);
|
|
||||||
if (!input) {
|
|
||||||
if (truncated) continue;
|
|
||||||
input = parseToolArguments(raw, LABEL, name);
|
|
||||||
}
|
|
||||||
calls.push({ id: str(item.call_id) || str(item.id), name, input });
|
|
||||||
replay.push(item);
|
|
||||||
} else if (item.type === 'reasoning') {
|
|
||||||
replay.push(item);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!text && streamedText) text = streamedText;
|
|
||||||
if (text) content.push({ type: 'text', text });
|
|
||||||
for (const call of calls) {
|
|
||||||
content.push({ type: 'tool_call', ...call });
|
|
||||||
yield { type: 'tool_call', call };
|
|
||||||
}
|
|
||||||
|
|
||||||
let stopReason: StopReason = 'end';
|
|
||||||
if (calls.length > 0) stopReason = 'tool_use';
|
|
||||||
else if (refused || incomplete === 'content_filter') stopReason = 'refusal';
|
|
||||||
else if (truncated) stopReason = 'max_tokens';
|
|
||||||
else if (incomplete) stopReason = 'other';
|
|
||||||
|
|
||||||
const u = isRecord(final.usage) ? final.usage : {};
|
|
||||||
const message: ChatMessage = { role: 'assistant', content };
|
|
||||||
if (replay.length > 0) message.providerData = { kind: 'openai', data: { output: replay } };
|
|
||||||
yield {
|
|
||||||
type: 'done',
|
|
||||||
message,
|
|
||||||
stopReason,
|
|
||||||
usage: makeUsage({
|
|
||||||
input: num(u.input_tokens),
|
|
||||||
output: num(u.output_tokens),
|
|
||||||
cacheRead: isRecord(u.input_tokens_details) ? num(u.input_tokens_details.cached_tokens) : 0,
|
|
||||||
reasoning: isRecord(u.output_tokens_details) ? num(u.output_tokens_details.reasoning_tokens) : 0,
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
|
||||||
const res = await requestJson<Json>(`${base(cfg)}/models`, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
|
||||||
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
|
|
||||||
const found: DiscoveredModel[] = [];
|
|
||||||
for (const m of data) {
|
|
||||||
const id = str(m.id);
|
|
||||||
// Fine-tunes ("ft:...") are chat models too; search/codex-only variants are not useful here.
|
|
||||||
if (!id || !isChatModelId(id) || /search|^codex-|-codex-max/.test(id)) continue;
|
|
||||||
if (!/^(gpt|o\d|chat|ft:)/.test(id)) continue;
|
|
||||||
found.push({ id });
|
|
||||||
}
|
|
||||||
found.sort((a, b) => b.id.localeCompare(a.id, undefined, { numeric: true }));
|
|
||||||
return withCatalogDefaults('openai', found);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
|
||||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
|
||||||
}
|
|
||||||
|
|
||||||
export const openaiAdapter: ProviderAdapter = {
|
|
||||||
kind: 'openai',
|
|
||||||
meta: {
|
|
||||||
label: LABEL,
|
|
||||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
|
||||||
requiresApiKey: true,
|
|
||||||
requiresBaseUrl: false,
|
|
||||||
supportsDiscovery: true,
|
|
||||||
docsUrl: 'https://platform.openai.com/api-keys',
|
|
||||||
keyHint: 'sk-…',
|
|
||||||
},
|
|
||||||
stream,
|
|
||||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
|
||||||
test,
|
|
||||||
};
|
|
||||||
|
|
@ -1,363 +0,0 @@
|
||||||
/**
|
|
||||||
* Generic Chat Completions (`POST {base}/chat/completions`) for OpenRouter, Groq,
|
|
||||||
* Together, LM Studio, vLLM, llama.cpp and friends, and the engine behind the
|
|
||||||
* Ollama adapter.
|
|
||||||
*
|
|
||||||
* These servers agree on the basics and disagree on everything else, so this
|
|
||||||
* adapter is deliberately forgiving:
|
|
||||||
* - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs
|
|
||||||
* `max_completion_tokens` and `reasoning_effort` are each dropped and retried
|
|
||||||
* once when a server answers 400 about them;
|
|
||||||
* - usage may be missing, and may arrive on a chunk with no choices;
|
|
||||||
* - tool-call argument fragments may arrive in any chunking, with or without ids;
|
|
||||||
* - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`;
|
|
||||||
* - the last chunk may lack a `finish_reason`.
|
|
||||||
*/
|
|
||||||
import {
|
|
||||||
AIError,
|
|
||||||
type ChatMessage,
|
|
||||||
type ChatRequest,
|
|
||||||
type ContentPart,
|
|
||||||
type DiscoveredModel,
|
|
||||||
type ModelCapabilities,
|
|
||||||
type ProviderAdapter,
|
|
||||||
type ProviderConfig,
|
|
||||||
type ProviderKind,
|
|
||||||
type StopReason,
|
|
||||||
type StreamEvent,
|
|
||||||
type TestResult,
|
|
||||||
} from '../types.ts';
|
|
||||||
import { dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
|
||||||
import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
|
||||||
|
|
||||||
type Json = Record<string, unknown>;
|
|
||||||
|
|
||||||
/** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */
|
|
||||||
export function normalizeCompatBase(url: string): string {
|
|
||||||
let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, '');
|
|
||||||
try {
|
|
||||||
const u = new URL(base);
|
|
||||||
// A bare host ("http://localhost:1234") almost always means the conventional /v1 root.
|
|
||||||
if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`;
|
|
||||||
} catch {
|
|
||||||
/* validated by the caller */
|
|
||||||
}
|
|
||||||
return base;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CompatOptions {
|
|
||||||
kind: ProviderKind;
|
|
||||||
label: string;
|
|
||||||
baseUrl: string;
|
|
||||||
apiKey: string | null;
|
|
||||||
headers?: Record<string, string>;
|
|
||||||
}
|
|
||||||
|
|
||||||
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): string | Json[] {
|
|
||||||
const blocks: Json[] = [];
|
|
||||||
for (const p of parts) {
|
|
||||||
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
|
|
||||||
else if (p.type === 'image') {
|
|
||||||
if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
|
|
||||||
else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } });
|
|
||||||
} else if (p.type === 'file') {
|
|
||||||
if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') });
|
|
||||||
else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Plain strings are understood by every server; arrays only by multimodal ones.
|
|
||||||
if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n');
|
|
||||||
return blocks;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system'): Json[] {
|
|
||||||
const caps = req.capabilities ?? {};
|
|
||||||
const out: Json[] = [];
|
|
||||||
if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() });
|
|
||||||
for (const msg of req.messages) {
|
|
||||||
if (msg.role === 'assistant') {
|
|
||||||
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
|
||||||
const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract<ContentPart, { type: 'tool_call' }>[];
|
|
||||||
if (!text && calls.length === 0) continue;
|
|
||||||
const wire: Json = { role: 'assistant', content: text || null };
|
|
||||||
if (calls.length > 0) {
|
|
||||||
wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } }));
|
|
||||||
}
|
|
||||||
out.push(wire);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Tool results must directly follow the assistant turn that requested them.
|
|
||||||
for (const p of msg.content) {
|
|
||||||
if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' });
|
|
||||||
}
|
|
||||||
const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call');
|
|
||||||
if (rest.length > 0) {
|
|
||||||
const content = userContent(rest, caps);
|
|
||||||
if (typeof content !== 'string' || content) out.push({ role: 'user', content });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BodyFlags {
|
|
||||||
tools: boolean;
|
|
||||||
usage: boolean;
|
|
||||||
jsonMode: boolean;
|
|
||||||
temperature: boolean;
|
|
||||||
reasoning: boolean;
|
|
||||||
maxKey: 'max_tokens' | 'max_completion_tokens' | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildBody(req: ChatRequest, flags: BodyFlags): Json {
|
|
||||||
const caps = req.capabilities ?? {};
|
|
||||||
const body: Json = { model: req.model, messages: toChatMessages(req), stream: true };
|
|
||||||
if (flags.usage) body.stream_options = { include_usage: true };
|
|
||||||
if (flags.tools && caps.tools !== false && req.tools?.length) {
|
|
||||||
body.tools = req.tools.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } }));
|
|
||||||
const choice = req.toolChoice;
|
|
||||||
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
|
|
||||||
else if (isRecord(choice)) body.tool_choice = { type: 'function', function: { name: choice.name } };
|
|
||||||
}
|
|
||||||
if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature;
|
|
||||||
if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens;
|
|
||||||
if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = req.reasoningEffort;
|
|
||||||
if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' };
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
function mapFinish(reason: string): StopReason {
|
|
||||||
switch (reason) {
|
|
||||||
case 'stop':
|
|
||||||
case 'eos':
|
|
||||||
case 'end_turn':
|
|
||||||
return 'end';
|
|
||||||
case 'tool_calls':
|
|
||||||
case 'function_call':
|
|
||||||
return 'tool_use';
|
|
||||||
case 'length':
|
|
||||||
case 'max_tokens':
|
|
||||||
return 'max_tokens';
|
|
||||||
case 'content_filter':
|
|
||||||
return 'refusal';
|
|
||||||
default:
|
|
||||||
return 'other';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function authHeaders(o: CompatOptions): Record<string, string> {
|
|
||||||
const h: Record<string, string> = { ...o.headers };
|
|
||||||
if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`;
|
|
||||||
return h;
|
|
||||||
}
|
|
||||||
|
|
||||||
function errorContext(o: CompatOptions): ErrorContext {
|
|
||||||
return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] };
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
|
||||||
const ctx = errorContext(o);
|
|
||||||
const base = normalizeCompatBase(o.baseUrl);
|
|
||||||
const local = isLocalUrl(base);
|
|
||||||
const flags: BodyFlags = { tools: true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' };
|
|
||||||
|
|
||||||
let http: HttpResponse;
|
|
||||||
for (let attempt = 0; ; attempt += 1) {
|
|
||||||
http = await httpRequest(`${base}/chat/completions`, {
|
|
||||||
...ctx,
|
|
||||||
headers: authHeaders(o),
|
|
||||||
body: buildBody(req, flags),
|
|
||||||
signal: req.signal,
|
|
||||||
// A local server may spend minutes loading a model into memory before the first byte.
|
|
||||||
firstByteTimeoutMs: local ? 10 * 60_000 : undefined,
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
await ensureOk(http);
|
|
||||||
break;
|
|
||||||
} catch (err) {
|
|
||||||
if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err;
|
|
||||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
|
||||||
if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false;
|
|
||||||
else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens';
|
|
||||||
else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null;
|
|
||||||
else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false;
|
|
||||||
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
|
|
||||||
else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false;
|
|
||||||
else if (flags.tools && req.tools?.length && /tool|function/.test(detail)) {
|
|
||||||
// The model (or server) cannot do tool calling at all.
|
|
||||||
if (req.toolChoice === 'required' || isRecord(req.toolChoice)) {
|
|
||||||
throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, {
|
|
||||||
status: err.status,
|
|
||||||
providerMessage: err.providerMessage,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
flags.tools = false;
|
|
||||||
} else throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let text = '';
|
|
||||||
let finish = '';
|
|
||||||
let reasoningSeen = false;
|
|
||||||
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
|
|
||||||
const calls: { id: string; name: string; args: string }[] = [];
|
|
||||||
|
|
||||||
for await (const { data } of sseJson(http)) {
|
|
||||||
if (!isRecord(data)) continue;
|
|
||||||
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
|
|
||||||
if (isRecord(data.usage)) {
|
|
||||||
const u = data.usage;
|
|
||||||
usage.input = num(u.prompt_tokens) || usage.input;
|
|
||||||
usage.output = num(u.completion_tokens) || usage.output;
|
|
||||||
if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens);
|
|
||||||
if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens);
|
|
||||||
}
|
|
||||||
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
|
|
||||||
if (!isRecord(choice)) continue;
|
|
||||||
if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason;
|
|
||||||
const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null;
|
|
||||||
if (!delta) continue;
|
|
||||||
|
|
||||||
const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking);
|
|
||||||
if (reasoning) {
|
|
||||||
reasoningSeen = true;
|
|
||||||
yield { type: 'reasoning', delta: reasoning };
|
|
||||||
}
|
|
||||||
const piece = typeof delta.content === 'string' ? delta.content : '';
|
|
||||||
if (piece) {
|
|
||||||
text += piece;
|
|
||||||
yield { type: 'text', delta: piece };
|
|
||||||
}
|
|
||||||
if (Array.isArray(delta.tool_calls)) {
|
|
||||||
for (const raw of delta.tool_calls) {
|
|
||||||
if (!isRecord(raw)) continue;
|
|
||||||
const fn = isRecord(raw.function) ? raw.function : {};
|
|
||||||
// `index` identifies the call; servers that omit it send one whole call per entry.
|
|
||||||
const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1;
|
|
||||||
let slot = index >= 0 ? calls[index] : undefined;
|
|
||||||
if (!slot) {
|
|
||||||
slot = { id: '', name: '', args: '' };
|
|
||||||
if (index >= 0) calls[index] = slot;
|
|
||||||
else calls.push(slot);
|
|
||||||
}
|
|
||||||
if (str(raw.id)) slot.id = str(raw.id);
|
|
||||||
if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name);
|
|
||||||
if (typeof fn.arguments === 'string') slot.args += fn.arguments;
|
|
||||||
else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const stop = finish ? mapFinish(finish) : 'end';
|
|
||||||
const content: ContentPart[] = [];
|
|
||||||
if (text) content.push({ type: 'text', text });
|
|
||||||
const done: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
|
||||||
for (const c of calls) {
|
|
||||||
if (!c || !c.name) continue;
|
|
||||||
let input = parseArgsStrict(c.args);
|
|
||||||
if (!input) {
|
|
||||||
if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest
|
|
||||||
input = parseToolArguments(c.args, o.label, c.name);
|
|
||||||
}
|
|
||||||
done.push({ id: c.id || newToolCallId('call'), name: c.name, input });
|
|
||||||
}
|
|
||||||
for (const call of done) {
|
|
||||||
content.push({ type: 'tool_call', ...call });
|
|
||||||
yield { type: 'tool_call', call };
|
|
||||||
}
|
|
||||||
if (content.length === 0 && !reasoningSeen && !finish) {
|
|
||||||
throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true });
|
|
||||||
}
|
|
||||||
const message: ChatMessage = { role: 'assistant', content };
|
|
||||||
yield {
|
|
||||||
type: 'done',
|
|
||||||
message,
|
|
||||||
stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop,
|
|
||||||
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
|
||||||
const base = normalizeCompatBase(o.baseUrl);
|
|
||||||
let res: unknown;
|
|
||||||
try {
|
|
||||||
res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) {
|
|
||||||
throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status });
|
|
||||||
}
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : [];
|
|
||||||
const found: DiscoveredModel[] = [];
|
|
||||||
for (const m of list) {
|
|
||||||
if (!isRecord(m)) continue;
|
|
||||||
const id = str(m.id) || str(m.name);
|
|
||||||
if (!id || !isChatModelId(id)) continue;
|
|
||||||
const model: DiscoveredModel = { id };
|
|
||||||
if (str(m.name) && str(m.name) !== id) model.label = str(m.name);
|
|
||||||
// OpenRouter-style metadata, when present.
|
|
||||||
const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length);
|
|
||||||
if (ctxLen) model.contextWindow = ctxLen;
|
|
||||||
if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens);
|
|
||||||
if (isRecord(m.pricing)) {
|
|
||||||
const pin = Number(m.pricing.prompt);
|
|
||||||
const pout = Number(m.pricing.completion);
|
|
||||||
if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3;
|
|
||||||
if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3;
|
|
||||||
}
|
|
||||||
const caps: Partial<ModelCapabilities> = {};
|
|
||||||
if (Array.isArray(m.supported_parameters)) {
|
|
||||||
caps.tools = m.supported_parameters.includes('tools');
|
|
||||||
caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort');
|
|
||||||
}
|
|
||||||
if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) {
|
|
||||||
caps.vision = m.architecture.input_modalities.includes('image');
|
|
||||||
}
|
|
||||||
if (Object.keys(caps).length > 0) model.capabilities = caps;
|
|
||||||
found.push(model);
|
|
||||||
}
|
|
||||||
return withCatalogDefaults(o.kind, found);
|
|
||||||
}
|
|
||||||
|
|
||||||
function options(cfg: ProviderConfig): CompatOptions {
|
|
||||||
if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`);
|
|
||||||
return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function test(cfg: ProviderConfig, model?: string): Promise<TestResult> {
|
|
||||||
return runTest(cfg.name || 'the server', async () => {
|
|
||||||
const o = options(cfg);
|
|
||||||
try {
|
|
||||||
return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length };
|
|
||||||
} catch (err) {
|
|
||||||
if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err;
|
|
||||||
// No model listing: prove the connection with a one-token completion instead.
|
|
||||||
const controller = new AbortController();
|
|
||||||
const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
|
|
||||||
try {
|
|
||||||
for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) {
|
|
||||||
if (ev.type === 'done') break;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
clearTimeout(timer);
|
|
||||||
}
|
|
||||||
return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` };
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export const openaiCompatibleAdapter: ProviderAdapter = {
|
|
||||||
kind: 'openai_compatible',
|
|
||||||
meta: {
|
|
||||||
label: 'OpenAI-compatible',
|
|
||||||
defaultBaseUrl: null,
|
|
||||||
requiresApiKey: false,
|
|
||||||
requiresBaseUrl: true,
|
|
||||||
supportsDiscovery: true,
|
|
||||||
docsUrl: 'https://openrouter.ai/docs/quickstart',
|
|
||||||
keyHint: 'Optional for local servers',
|
|
||||||
},
|
|
||||||
stream: (cfg, req) => streamChatCompletions(options(cfg), req),
|
|
||||||
listModels: (cfg, signal) => listCompatModels(options(cfg), signal),
|
|
||||||
test,
|
|
||||||
};
|
|
||||||
|
|
@ -1,932 +0,0 @@
|
||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
||||||
/**
|
|
||||||
* Provider adapter tests. `fetch` is always stubbed; no network is touched.
|
|
||||||
* Wire shapes below are derived from what each adapter's parser reads.
|
|
||||||
*/
|
|
||||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
||||||
import { AIError, type ChatMessage, type ChatRequest, type ProviderAdapter, type ProviderConfig, type ProviderKind, type StreamEvent, type ToolDef, type Usage } from '../types.ts';
|
|
||||||
import { anthropicAdapter } from './anthropic.ts';
|
|
||||||
import { lookupModelDefaults } from './catalog.ts';
|
|
||||||
import { toGeminiSchema, geminiAdapter } from './gemini.ts';
|
|
||||||
import { parseSSE } from './http.ts';
|
|
||||||
import { ollamaAdapter } from './ollama.ts';
|
|
||||||
import { openaiAdapter } from './openai.ts';
|
|
||||||
import { normalizeCompatBase, openaiCompatibleAdapter } from './openaiCompatible.ts';
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Helpers */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
const enc = new TextEncoder();
|
|
||||||
|
|
||||||
function bytesStream(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
|
|
||||||
return new ReadableStream({
|
|
||||||
start(c) {
|
|
||||||
for (const ch of chunks) c.enqueue(ch);
|
|
||||||
c.close();
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
const textStream = (chunks: string[]) => bytesStream(chunks.map((c) => enc.encode(c)));
|
|
||||||
|
|
||||||
/** One SSE event. */
|
|
||||||
function ev(data: unknown, event?: string): string {
|
|
||||||
return `${event ? `event: ${event}\n` : ''}data: ${JSON.stringify(data)}\n\n`;
|
|
||||||
}
|
|
||||||
const DONE = 'data: [DONE]\n\n';
|
|
||||||
|
|
||||||
function rechunk(s: string, n: number): string[] {
|
|
||||||
const out: string[] = [];
|
|
||||||
for (let i = 0; i < s.length; i += n) out.push(s.slice(i, i + n));
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** SSE response whose chunk boundaries deliberately fall mid-line. */
|
|
||||||
function sseResponse(chunks: string[], status = 200): Response {
|
|
||||||
return new Response(textStream(rechunk(chunks.join(''), 23)), { status, headers: { 'content-type': 'text/event-stream' } });
|
|
||||||
}
|
|
||||||
function jsonResponse(body: unknown, status = 200): Response {
|
|
||||||
return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Call {
|
|
||||||
url: string;
|
|
||||||
method?: string;
|
|
||||||
headers: Record<string, string>;
|
|
||||||
body: any;
|
|
||||||
}
|
|
||||||
let calls: Call[] = [];
|
|
||||||
type Reply = Response | Error;
|
|
||||||
type Handler = (call: Call, n: number) => Reply | Promise<Reply>;
|
|
||||||
|
|
||||||
function stubFetch(handler: Handler): void {
|
|
||||||
calls = [];
|
|
||||||
vi.stubGlobal(
|
|
||||||
'fetch',
|
|
||||||
vi.fn(async (input: unknown, init?: RequestInit) => {
|
|
||||||
const call: Call = {
|
|
||||||
url: String(input),
|
|
||||||
method: init?.method,
|
|
||||||
headers: (init?.headers ?? {}) as Record<string, string>,
|
|
||||||
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined,
|
|
||||||
};
|
|
||||||
calls.push(call);
|
|
||||||
const r = await handler(call, calls.length - 1);
|
|
||||||
if (r instanceof Error) throw r;
|
|
||||||
return r;
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
/** Replies with the given thunks in order (the last one repeats). */
|
|
||||||
function queue(...items: Array<() => Reply>): Handler {
|
|
||||||
return (_c, n) => items[Math.min(n, items.length - 1)]!();
|
|
||||||
}
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
vi.unstubAllGlobals();
|
|
||||||
vi.restoreAllMocks();
|
|
||||||
});
|
|
||||||
|
|
||||||
async function drain<T>(it: AsyncIterable<T>): Promise<T[]> {
|
|
||||||
const out: T[] = [];
|
|
||||||
for await (const x of it) out.push(x);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function run(adapter: ProviderAdapter, cfg: ProviderConfig, req: ChatRequest) {
|
|
||||||
const events = await drain(adapter.stream(cfg, req));
|
|
||||||
const done = events[events.length - 1];
|
|
||||||
if (!done || done.type !== 'done') throw new Error('stream did not end with a done event');
|
|
||||||
return { events, done };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function failure(p: Promise<unknown>): Promise<AIError> {
|
|
||||||
const err = await p.then(
|
|
||||||
() => undefined,
|
|
||||||
(e: unknown) => e,
|
|
||||||
);
|
|
||||||
expect(err).toBeInstanceOf(AIError);
|
|
||||||
return err as AIError;
|
|
||||||
}
|
|
||||||
|
|
||||||
const TOOL: ToolDef = {
|
|
||||||
name: 'lookup',
|
|
||||||
description: 'Look things up',
|
|
||||||
parameters: { type: 'object', properties: { q: { type: 'string' } }, required: ['q'] },
|
|
||||||
};
|
|
||||||
const FRAGS = ['{"q":"ca', 'ts","n"', ':2}'];
|
|
||||||
const user = (text: string): ChatMessage => ({ role: 'user', content: [{ type: 'text', text }] });
|
|
||||||
const toolCallMsg = (extra: Partial<ChatMessage> = {}): ChatMessage => ({
|
|
||||||
role: 'assistant',
|
|
||||||
content: [{ type: 'tool_call', id: 'call_1', name: 'lookup', input: { q: 'cats' } }],
|
|
||||||
...extra,
|
|
||||||
});
|
|
||||||
const toolResultMsg: ChatMessage = { role: 'user', content: [{ type: 'tool_result', toolCallId: 'call_1', name: 'lookup', output: 'result text' }] };
|
|
||||||
|
|
||||||
const cfgOf = (kind: ProviderKind, apiKey: string | null, baseUrl: string | null = null): ProviderConfig => ({ id: kind, kind, name: 'Test Server', baseUrl, apiKey });
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Per-adapter fixtures */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
const EXPECTED_FULL_USAGE: Usage = { inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, reasoningTokens: 10 };
|
|
||||||
|
|
||||||
interface Kit {
|
|
||||||
name: string;
|
|
||||||
adapter: ProviderAdapter;
|
|
||||||
key: string;
|
|
||||||
cfg: ProviderConfig;
|
|
||||||
model: string;
|
|
||||||
otherKind: ProviderKind;
|
|
||||||
streamUrl: string;
|
|
||||||
toolId: unknown;
|
|
||||||
assertAuth(h: Record<string, string>): void;
|
|
||||||
assertRequest(body: any): void; // system prompt + tool definitions
|
|
||||||
assertHistory(body: any): void; // tool_call + tool_result mapping
|
|
||||||
text(parts: string[]): string[];
|
|
||||||
tool(frags: string[]): string[];
|
|
||||||
usage(): { chunks: string[]; expected: Usage };
|
|
||||||
errors: { auth: [number, unknown]; rate: [number, unknown]; ctx: [number, unknown] };
|
|
||||||
}
|
|
||||||
|
|
||||||
const openaiStyleErrors = (key: string): Kit['errors'] => ({
|
|
||||||
auth: [401, { error: { message: `Incorrect API key provided: ${key}.`, type: 'invalid_request_error', code: 'invalid_api_key' } }],
|
|
||||||
rate: [429, { error: { message: 'Rate limit reached for requests', type: 'rate_limit_error', code: 'rate_limit_exceeded' } }],
|
|
||||||
ctx: [400, { error: { message: 'This model maximum context length is 8192 tokens, request too long', type: 'invalid_request_error', code: 'context_length_exceeded' } }],
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ---- Anthropic ---- */
|
|
||||||
const anthropicKit: Kit = {
|
|
||||||
name: 'anthropic',
|
|
||||||
adapter: anthropicAdapter,
|
|
||||||
key: 'sk-ant-TESTKEY-1234567890',
|
|
||||||
cfg: cfgOf('anthropic', 'sk-ant-TESTKEY-1234567890'),
|
|
||||||
model: 'claude-sonnet-5-5',
|
|
||||||
otherKind: 'openai',
|
|
||||||
streamUrl: 'https://api.anthropic.com/v1/messages',
|
|
||||||
toolId: 'call_1',
|
|
||||||
assertAuth(h) {
|
|
||||||
expect(h['x-api-key']).toBe('sk-ant-TESTKEY-1234567890');
|
|
||||||
expect(h['anthropic-version']).toBe('2026-10-21');
|
|
||||||
expect(h.Authorization).toBeUndefined();
|
|
||||||
},
|
|
||||||
assertRequest(b) {
|
|
||||||
expect(b.stream).toBe(true);
|
|
||||||
expect(b.system).toBe('SYS');
|
|
||||||
expect(b.messages.some((m: any) => m.role === 'system')).toBe(false);
|
|
||||||
expect(b.tools).toEqual([{ name: 'lookup', description: 'Look things up', input_schema: TOOL.parameters, eager_input_streaming: true }]);
|
|
||||||
},
|
|
||||||
assertHistory(b) {
|
|
||||||
expect(b.messages).toEqual([
|
|
||||||
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
|
|
||||||
{ role: 'assistant', content: [{ type: 'tool_use', id: 'call_1', name: 'lookup', input: { q: 'cats' } }] },
|
|
||||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call_1', content: 'result text' }] },
|
|
||||||
]);
|
|
||||||
},
|
|
||||||
text: (parts) => [
|
|
||||||
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }, 'message_start'),
|
|
||||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, 'content_block_start'),
|
|
||||||
...parts.map((p) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: p } }, 'content_block_delta')),
|
|
||||||
ev({ type: 'content_block_stop', index: 0 }, 'content_block_stop'),
|
|
||||||
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 3 } }, 'message_delta'),
|
|
||||||
ev({ type: 'message_stop' }, 'message_stop'),
|
|
||||||
],
|
|
||||||
tool: (frags) => [
|
|
||||||
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }),
|
|
||||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} } }),
|
|
||||||
...frags.map((f) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: f } })),
|
|
||||||
ev({ type: 'content_block_stop', index: 0 }),
|
|
||||||
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
|
|
||||||
ev({ type: 'message_stop' }),
|
|
||||||
],
|
|
||||||
// Anthropic reports uncached/cache-read/cache-write input separately and has no reasoning-token field.
|
|
||||||
usage: () => ({
|
|
||||||
chunks: [
|
|
||||||
ev({ type: 'message_start', message: { usage: { input_tokens: 10, cache_read_input_tokens: 5, cache_creation_input_tokens: 3, output_tokens: 1 } } }),
|
|
||||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'ok' } }),
|
|
||||||
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 20 } }),
|
|
||||||
ev({ type: 'message_stop' }),
|
|
||||||
],
|
|
||||||
expected: { inputTokens: 18, outputTokens: 20, cacheReadTokens: 5, cacheWriteTokens: 3 },
|
|
||||||
}),
|
|
||||||
errors: {
|
|
||||||
auth: [401, { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key sk-ant-TESTKEY-1234567890' } }],
|
|
||||||
rate: [429, { type: 'error', error: { type: 'rate_limit_error', message: 'Number of requests has exceeded your rate limit' } }],
|
|
||||||
ctx: [400, { type: 'error', error: { type: 'invalid_request_error', message: 'prompt is too long: 250000 tokens > 200000 maximum' } }],
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
/* ---- OpenAI (Responses API) ---- */
|
|
||||||
const oaMessage = (text: string) => ({ type: 'message', id: 'msg_1', role: 'assistant', content: [{ type: 'output_text', text }] });
|
|
||||||
const openaiKit: Kit = {
|
|
||||||
name: 'openai',
|
|
||||||
adapter: openaiAdapter,
|
|
||||||
key: 'sk-proj-TESTKEY-1234567890',
|
|
||||||
cfg: cfgOf('openai', 'sk-proj-TESTKEY-1234567890'),
|
|
||||||
model: 'gpt-4o',
|
|
||||||
otherKind: 'anthropic',
|
|
||||||
streamUrl: 'https://api.openai.com/v1/responses',
|
|
||||||
toolId: 'call_1',
|
|
||||||
assertAuth(h) {
|
|
||||||
expect(h.Authorization).toBe('Bearer sk-proj-TESTKEY-1234567890');
|
|
||||||
},
|
|
||||||
assertRequest(b) {
|
|
||||||
expect(b.stream).toBe(true);
|
|
||||||
expect(b.store).toBe(false);
|
|
||||||
expect(b.instructions).toBe('SYS');
|
|
||||||
expect(b.tools).toEqual([{ type: 'function', name: 'lookup', description: 'Look things up', parameters: TOOL.parameters, strict: false }]);
|
|
||||||
},
|
|
||||||
assertHistory(b) {
|
|
||||||
expect(b.input).toEqual([
|
|
||||||
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
|
|
||||||
{ type: 'function_call', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
|
|
||||||
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
|
|
||||||
]);
|
|
||||||
},
|
|
||||||
text: (parts) => [
|
|
||||||
...parts.map((p) => ev({ type: 'response.output_text.delta', item_id: 'msg_1', delta: p }, 'response.output_text.delta')),
|
|
||||||
ev({ type: 'response.completed', response: { output: [oaMessage(parts.join(''))], usage: { input_tokens: 4, output_tokens: 3 } } }, 'response.completed'),
|
|
||||||
],
|
|
||||||
// `arguments` is empty in the final item here so that the streamed deltas are what gets assembled.
|
|
||||||
tool: (frags) => [
|
|
||||||
...frags.map((d) => ev({ type: 'response.function_call_arguments.delta', item_id: 'fc_1', delta: d })),
|
|
||||||
ev({
|
|
||||||
type: 'response.completed',
|
|
||||||
response: { output: [{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '' }], usage: { input_tokens: 4, output_tokens: 3 } },
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
usage: () => ({
|
|
||||||
chunks: [
|
|
||||||
ev({ type: 'response.output_text.delta', delta: 'ok' }),
|
|
||||||
ev({
|
|
||||||
type: 'response.completed',
|
|
||||||
response: {
|
|
||||||
output: [oaMessage('ok')],
|
|
||||||
usage: { input_tokens: 100, output_tokens: 50, input_tokens_details: { cached_tokens: 20 }, output_tokens_details: { reasoning_tokens: 10 } },
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
expected: EXPECTED_FULL_USAGE,
|
|
||||||
}),
|
|
||||||
errors: openaiStyleErrors('sk-proj-TESTKEY-1234567890'),
|
|
||||||
};
|
|
||||||
|
|
||||||
/* ---- Gemini ---- */
|
|
||||||
const gemChunk = (parts: unknown[], extra: Record<string, unknown> = {}, cand: Record<string, unknown> = {}) =>
|
|
||||||
ev({ candidates: [{ content: { role: 'model', parts }, ...cand }], ...extra });
|
|
||||||
const geminiKit: Kit = {
|
|
||||||
name: 'gemini',
|
|
||||||
adapter: geminiAdapter,
|
|
||||||
key: 'AIzaSyTESTKEY1234567890abcd',
|
|
||||||
cfg: cfgOf('gemini', 'AIzaSyTESTKEY1234567890abcd'),
|
|
||||||
model: 'gemini-3.8-flash',
|
|
||||||
otherKind: 'anthropic',
|
|
||||||
streamUrl: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse',
|
|
||||||
toolId: expect.stringMatching(/^gem_/),
|
|
||||||
assertAuth(h) {
|
|
||||||
expect(h['x-goog-api-key']).toBe('AIzaSyTESTKEY1234567890abcd');
|
|
||||||
expect(h.Authorization).toBeUndefined();
|
|
||||||
},
|
|
||||||
assertRequest(b) {
|
|
||||||
expect(b.systemInstruction).toEqual({ parts: [{ text: 'SYS' }] });
|
|
||||||
expect(b.tools).toEqual([{ functionDeclarations: [{ name: 'lookup', description: 'Look things up', parameters: TOOL.parameters }] }]);
|
|
||||||
},
|
|
||||||
assertHistory(b) {
|
|
||||||
expect(b.contents).toEqual([
|
|
||||||
{ role: 'user', parts: [{ text: 'hi' }] },
|
|
||||||
{ role: 'model', parts: [{ functionCall: { name: 'lookup', args: { q: 'cats' } } }] },
|
|
||||||
{ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] },
|
|
||||||
]);
|
|
||||||
},
|
|
||||||
text: (parts) => [...parts.map((p) => gemChunk([{ text: p }])), gemChunk([{ text: '' }], {}, { finishReason: 'STOP' })],
|
|
||||||
// Gemini delivers a function call whole (args is an object), never as fragments.
|
|
||||||
tool: (frags) => [gemChunk([{ functionCall: { name: 'lookup', args: JSON.parse(frags.join('')) } }], {}, { finishReason: 'STOP' })],
|
|
||||||
usage: () => ({
|
|
||||||
chunks: [
|
|
||||||
gemChunk([{ text: 'ok' }], { usageMetadata: { promptTokenCount: 100, candidatesTokenCount: 40, thoughtsTokenCount: 10, cachedContentTokenCount: 20 } }, { finishReason: 'STOP' }),
|
|
||||||
],
|
|
||||||
expected: EXPECTED_FULL_USAGE,
|
|
||||||
}),
|
|
||||||
errors: {
|
|
||||||
auth: [401, { error: { code: 401, message: 'API key not valid. AIzaSyTESTKEY1234567890abcd', status: 'UNAUTHENTICATED' } }],
|
|
||||||
rate: [429, { error: { code: 429, message: 'Resource has been exhausted', status: 'RESOURCE_EXHAUSTED' } }],
|
|
||||||
ctx: [400, { error: { code: 400, message: 'The input token count (2000000) exceeds the maximum number of tokens allowed (1048576).', status: 'INVALID_ARGUMENT' } }],
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
/* ---- Chat Completions family ---- */
|
|
||||||
const ccChunk = (delta: Record<string, unknown>, finish: string | null = null) => ev({ choices: [{ index: 0, delta, finish_reason: finish }] });
|
|
||||||
function chatCompletionsKit(over: Pick<Kit, 'name' | 'adapter' | 'key' | 'cfg' | 'model' | 'streamUrl'>): Kit {
|
|
||||||
return {
|
|
||||||
...over,
|
|
||||||
otherKind: 'gemini',
|
|
||||||
toolId: 'call_1',
|
|
||||||
assertAuth(h) {
|
|
||||||
expect(h.Authorization).toBe(`Bearer ${over.key}`);
|
|
||||||
},
|
|
||||||
assertRequest(b) {
|
|
||||||
expect(b.stream).toBe(true);
|
|
||||||
expect(b.messages[0]).toEqual({ role: 'system', content: 'SYS' });
|
|
||||||
expect(b.tools).toEqual([{ type: 'function', function: { name: 'lookup', description: 'Look things up', parameters: TOOL.parameters } }]);
|
|
||||||
},
|
|
||||||
assertHistory(b) {
|
|
||||||
expect(b.messages).toEqual([
|
|
||||||
{ role: 'system', content: 'SYS' },
|
|
||||||
{ role: 'user', content: 'hi' },
|
|
||||||
{ role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{"q":"cats"}' } }] },
|
|
||||||
{ role: 'tool', tool_call_id: 'call_1', content: 'result text' },
|
|
||||||
]);
|
|
||||||
},
|
|
||||||
text: (parts) => [ccChunk({ role: 'assistant', content: '' }), ...parts.map((p) => ccChunk({ content: p })), ccChunk({}, 'stop'), DONE],
|
|
||||||
tool: (frags) => [
|
|
||||||
ccChunk({ role: 'assistant', tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '' } }] }),
|
|
||||||
...frags.map((f) => ccChunk({ tool_calls: [{ index: 0, function: { arguments: f } }] })),
|
|
||||||
ccChunk({}, 'tool_calls'),
|
|
||||||
DONE,
|
|
||||||
],
|
|
||||||
usage: () => ({
|
|
||||||
chunks: [
|
|
||||||
ccChunk({ content: 'ok' }, 'stop'),
|
|
||||||
ev({
|
|
||||||
choices: [],
|
|
||||||
usage: { prompt_tokens: 100, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 20 }, completion_tokens_details: { reasoning_tokens: 10 } },
|
|
||||||
}),
|
|
||||||
DONE,
|
|
||||||
],
|
|
||||||
expected: EXPECTED_FULL_USAGE,
|
|
||||||
}),
|
|
||||||
errors: openaiStyleErrors(over.key),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
const compatKit = chatCompletionsKit({
|
|
||||||
name: 'openai_compatible',
|
|
||||||
adapter: openaiCompatibleAdapter,
|
|
||||||
key: 'sk-or-TESTKEY-1234567890',
|
|
||||||
cfg: cfgOf('openai_compatible', 'sk-or-TESTKEY-1234567890', 'https://router.example.com/api/v1'),
|
|
||||||
model: 'meta/llama-3',
|
|
||||||
streamUrl: 'https://router.example.com/api/v1/chat/completions',
|
|
||||||
});
|
|
||||||
const ollamaKit = chatCompletionsKit({
|
|
||||||
name: 'ollama',
|
|
||||||
adapter: ollamaAdapter,
|
|
||||||
key: 'ollama-proxy-secret-1234',
|
|
||||||
cfg: cfgOf('ollama', 'ollama-proxy-secret-1234'),
|
|
||||||
model: 'llama3',
|
|
||||||
streamUrl: 'http://localhost:11434/v1/chat/completions',
|
|
||||||
});
|
|
||||||
|
|
||||||
const kits = [anthropicKit, openaiKit, geminiKit, compatKit, ollamaKit];
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Shared behaviour across all five adapters */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
describe.each(kits)('$name adapter', (kit) => {
|
|
||||||
const baseReq = (): ChatRequest => ({ model: kit.model, system: 'SYS', messages: [user('hi')], tools: [TOOL] });
|
|
||||||
|
|
||||||
it('sends the expected URL, auth header, system prompt and tool definitions; key stays out of the URL', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
|
||||||
await run(kit.adapter, kit.cfg, baseReq());
|
|
||||||
expect(calls).toHaveLength(1);
|
|
||||||
expect(calls[0]!.url).toBe(kit.streamUrl);
|
|
||||||
expect(calls[0]!.url).not.toContain(kit.key);
|
|
||||||
expect(calls[0]!.method).toBe('POST');
|
|
||||||
kit.assertAuth(calls[0]!.headers);
|
|
||||||
kit.assertRequest(calls[0]!.body);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('maps tool_call and tool_result history into the provider format', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
|
||||||
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg(), toolResultMsg] });
|
|
||||||
kit.assertHistory(calls[0]!.body);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rebuilds from neutral content when providerData belongs to a different provider kind', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
|
||||||
// The foreign payload carries every shape key so it would leak through if the kind were not checked.
|
|
||||||
const foreign = { kind: kit.otherKind, data: { content: [{ type: 'text', text: 'MARKER' }], output: [{ type: 'message', marker: 'MARKER' }], parts: [{ text: 'MARKER' }] } };
|
|
||||||
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg({ providerData: foreign }), toolResultMsg] });
|
|
||||||
expect(JSON.stringify(calls[0]!.body)).not.toContain('MARKER');
|
|
||||||
kit.assertHistory(calls[0]!.body);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('streams text events in order and concatenates them in done', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(kit.text(['Hel', 'lo', ' world']))));
|
|
||||||
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
|
|
||||||
expect(events.filter((e) => e.type === 'text').map((e) => (e as { delta: string }).delta)).toEqual(['Hel', 'lo', ' world']);
|
|
||||||
expect(done.message.role).toBe('assistant');
|
|
||||||
expect(done.message.content).toEqual([{ type: 'text', text: 'Hello world' }]);
|
|
||||||
expect(done.stopReason).toBe('end');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('assembles a fragmented tool call into one tool_call event', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(kit.tool(FRAGS))));
|
|
||||||
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
|
|
||||||
const toolEvents = events.filter((e): e is Extract<StreamEvent, { type: 'tool_call' }> => e.type === 'tool_call');
|
|
||||||
expect(toolEvents).toHaveLength(1);
|
|
||||||
expect(toolEvents[0]!.call).toEqual({ id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
|
|
||||||
expect(done.stopReason).toBe('tool_use');
|
|
||||||
expect(done.message.content).toContainEqual({ type: 'tool_call', id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
|
|
||||||
expect(events[events.length - 1]).toBe(done);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('maps usage onto the neutral convention', async () => {
|
|
||||||
const { chunks, expected } = kit.usage();
|
|
||||||
stubFetch(queue(() => sseResponse(chunks)));
|
|
||||||
const { done } = await run(kit.adapter, kit.cfg, baseReq());
|
|
||||||
expect(done.usage).toEqual(expected);
|
|
||||||
});
|
|
||||||
|
|
||||||
it.each([
|
|
||||||
['auth', 'auth', 401],
|
|
||||||
['rate', 'rate_limit', 429],
|
|
||||||
['ctx', 'context_length', 400],
|
|
||||||
] as const)('maps HTTP %s failure to AIError code %s', async (which, code, status) => {
|
|
||||||
const [st, body] = kit.errors[which];
|
|
||||||
stubFetch(queue(() => jsonResponse(body, st)));
|
|
||||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
|
||||||
expect(err.code).toBe(code);
|
|
||||||
expect(err.status).toBe(status);
|
|
||||||
expect(err.message).not.toContain(kit.key);
|
|
||||||
expect(err.providerMessage ?? '').not.toContain(kit.key);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('scrubs the API key from provider text', async () => {
|
|
||||||
const [st, body] = kit.errors.auth;
|
|
||||||
stubFetch(queue(() => jsonResponse(body, st)));
|
|
||||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
|
||||||
expect(err.providerMessage).toContain('[redacted]');
|
|
||||||
expect(JSON.stringify(err)).not.toContain(kit.key);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('maps a fetch TypeError to a network error without leaking the key', async () => {
|
|
||||||
stubFetch(queue(() => new TypeError(`fetch failed (${kit.key})`)));
|
|
||||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
|
||||||
expect(err.code).toBe('network');
|
|
||||||
expect(err.retryable).toBe(true);
|
|
||||||
expect(err.message).not.toContain(kit.key);
|
|
||||||
expect(err.providerMessage ?? '').not.toContain(kit.key);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Anthropic specifics */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
describe('anthropic providerData', () => {
|
|
||||||
const thinkingTurn = [
|
|
||||||
ev({ type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }),
|
|
||||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }),
|
|
||||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'Let me ' } }),
|
|
||||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'think' } }),
|
|
||||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-' } }),
|
|
||||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'abc' } }),
|
|
||||||
ev({ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } }),
|
|
||||||
ev({ type: 'content_block_delta', index: 1, delta: { type: 'text_delta', text: 'Sure' } }),
|
|
||||||
ev({ type: 'content_block_start', index: 2, content_block: { type: 'tool_use', id: 'toolu_1', name: 'lookup', input: {} } }),
|
|
||||||
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '{"q":' } }),
|
|
||||||
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '"cats"}' } }),
|
|
||||||
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
|
|
||||||
ev({ type: 'message_stop' }),
|
|
||||||
];
|
|
||||||
const replayed = [
|
|
||||||
{ type: 'thinking', thinking: 'Let me think', signature: 'sig-abc' },
|
|
||||||
{ type: 'text', text: 'Sure' },
|
|
||||||
{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { q: 'cats' } },
|
|
||||||
];
|
|
||||||
const followUp = (assistant: ChatMessage): ChatRequest => ({
|
|
||||||
model: 'claude-sonnet-5-5',
|
|
||||||
messages: [user('hi'), assistant, { role: 'user', content: [{ type: 'tool_result', toolCallId: 'toolu_1', name: 'lookup', output: 'r' }] }],
|
|
||||||
});
|
|
||||||
|
|
||||||
it('requests adaptive thinking, emits reasoning, and replays thinking blocks with signatures', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(thinkingTurn), () => sseResponse(anthropicKit.text(['done']))));
|
|
||||||
const { events, done } = await run(anthropicAdapter, anthropicKit.cfg, { model: 'claude-sonnet-5-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'high' });
|
|
||||||
expect(calls[0]!.body.thinking).toEqual({ type: 'adaptive', display: 'summarized' });
|
|
||||||
expect(calls[0]!.body.output_config).toEqual({ effort: 'high' });
|
|
||||||
expect(events.filter((e) => e.type === 'reasoning').map((e) => (e as { delta: string }).delta)).toEqual(['Let me ', 'think']);
|
|
||||||
expect(done.message.providerData).toEqual({ kind: 'anthropic', data: { content: replayed } });
|
|
||||||
|
|
||||||
await run(anthropicAdapter, anthropicKit.cfg, followUp(done.message));
|
|
||||||
expect(calls[1]!.body.messages[1]).toEqual({ role: 'assistant', content: replayed });
|
|
||||||
expect(calls[1]!.body.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'toolu_1', content: 'r' });
|
|
||||||
});
|
|
||||||
|
|
||||||
it('strips thinking blocks and retries once on an invalid-signature 400', async () => {
|
|
||||||
const msg: ChatMessage = { ...toolCallMsg(), providerData: { kind: 'anthropic', data: { content: replayed } } };
|
|
||||||
stubFetch(
|
|
||||||
queue(
|
|
||||||
() => jsonResponse({ type: 'error', error: { type: 'invalid_request_error', message: 'Invalid `signature` in `thinking` block' } }, 400),
|
|
||||||
() => sseResponse(anthropicKit.text(['ok'])),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
await run(anthropicAdapter, anthropicKit.cfg, followUp(msg));
|
|
||||||
expect(calls).toHaveLength(2);
|
|
||||||
expect(JSON.stringify(calls[0]!.body)).toContain('"thinking"');
|
|
||||||
expect(calls[1]!.body.messages[1].content.map((b: any) => b.type)).toEqual(['text', 'tool_use']);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('anthropic listModels', () => {
|
|
||||||
it('paginates, filters non-chat ids and merges API data over catalog defaults', async () => {
|
|
||||||
stubFetch(
|
|
||||||
queue(
|
|
||||||
() =>
|
|
||||||
jsonResponse({
|
|
||||||
data: [
|
|
||||||
{
|
|
||||||
id: 'claude-haiku-4-5',
|
|
||||||
display_name: 'Haiku API Name',
|
|
||||||
max_input_tokens: 200000,
|
|
||||||
max_tokens: 64000,
|
|
||||||
capabilities: { image_input: { supported: true }, pdf_input: { supported: false }, thinking: { supported: true } },
|
|
||||||
},
|
|
||||||
{ id: 'claude-embed-1' },
|
|
||||||
],
|
|
||||||
has_more: true,
|
|
||||||
last_id: 'claude-haiku-4-5',
|
|
||||||
}),
|
|
||||||
() => jsonResponse({ data: [{ id: 'claude-unknown-9' }], has_more: false }),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
const models = await anthropicAdapter.listModels(anthropicKit.cfg);
|
|
||||||
expect(calls[0]!.url).toBe('https://api.anthropic.com/v1/models?limit=1000');
|
|
||||||
expect(calls[1]!.url).toContain('after_id=claude-haiku-4-5');
|
|
||||||
expect(calls[0]!.url).not.toContain(anthropicKit.key);
|
|
||||||
anthropicKit.assertAuth(calls[0]!.headers);
|
|
||||||
expect(models.map((m) => m.id)).toEqual(['claude-haiku-4-5', 'claude-unknown-9']);
|
|
||||||
expect(models[0]).toMatchObject({
|
|
||||||
label: 'Haiku API Name', // reported by the API, wins over the catalog
|
|
||||||
contextWindow: 200000,
|
|
||||||
maxOutput: 64000,
|
|
||||||
priceIn: 1, // from the catalog
|
|
||||||
priceOut: 5,
|
|
||||||
capabilities: { tools: true, vision: true, pdf: false, reasoning: true, temperature: false },
|
|
||||||
});
|
|
||||||
expect(models[1]).toEqual({ id: 'claude-unknown-9', capabilities: { tools: true, temperature: false } });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* OpenAI specifics */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
describe('openai providerData and reasoning', () => {
|
|
||||||
const output = [
|
|
||||||
{ type: 'reasoning', id: 'rs_1', encrypted_content: 'ENC', summary: [] },
|
|
||||||
oaMessage('Hi'),
|
|
||||||
{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
|
|
||||||
];
|
|
||||||
const turn = [
|
|
||||||
ev({ type: 'response.reasoning_summary_text.delta', delta: 'Thinking' }),
|
|
||||||
ev({ type: 'response.output_text.delta', delta: 'Hi' }),
|
|
||||||
ev({ type: 'response.completed', response: { output, usage: { input_tokens: 3, output_tokens: 2 } } }),
|
|
||||||
];
|
|
||||||
|
|
||||||
it('asks for encrypted reasoning and replays output items (reasoning item included) verbatim', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(turn), () => sseResponse(openaiKit.text(['ok']))));
|
|
||||||
const { events, done } = await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.2 });
|
|
||||||
const body = calls[0]!.body;
|
|
||||||
expect(body.reasoning).toEqual({ effort: 'medium', summary: 'auto' });
|
|
||||||
expect(body.include).toEqual(['reasoning.encrypted_content']);
|
|
||||||
expect(body.temperature).toBeUndefined(); // reasoning models reject it
|
|
||||||
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Thinking' }]);
|
|
||||||
expect(done.stopReason).toBe('tool_use');
|
|
||||||
expect(done.message.providerData).toEqual({ kind: 'openai', data: { output } });
|
|
||||||
|
|
||||||
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi'), done.message, toolResultMsg] });
|
|
||||||
expect(calls[1]!.body.input).toEqual([
|
|
||||||
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
|
|
||||||
...output,
|
|
||||||
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('retries once without reasoning.effort when the model rejects it', async () => {
|
|
||||||
stubFetch(
|
|
||||||
queue(
|
|
||||||
() => jsonResponse({ error: { message: "Unsupported value: 'reasoning.effort' does not support 'none'", type: 'invalid_request_error' } }, 400),
|
|
||||||
() => sseResponse(openaiKit.text(['ok'])),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], reasoningEffort: 'none' });
|
|
||||||
expect(calls[0]!.body.reasoning.effort).toBe('none');
|
|
||||||
expect(calls[1]!.body.reasoning.effort).toBe('minimal');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('openai listModels', () => {
|
|
||||||
it('keeps chat models only, sorts them and merges catalog defaults', async () => {
|
|
||||||
const ids = ['gpt-4o', 'gpt-4o-mini', 'o3', 'text-embedding-3-large', 'whisper-1', 'dall-e-3', 'gpt-4o-mini-tts', 'gpt-4o-realtime-preview', 'gpt-5-search-api', 'davinci-002', 'gpt-3.5-turbo-instruct', 'omni-moderation-latest'];
|
|
||||||
stubFetch(queue(() => jsonResponse({ data: ids.map((id) => ({ id })) })));
|
|
||||||
const models = await openaiAdapter.listModels(openaiKit.cfg);
|
|
||||||
expect(calls[0]!.url).toBe('https://api.openai.com/v1/models');
|
|
||||||
openaiKit.assertAuth(calls[0]!.headers);
|
|
||||||
expect(models.map((m) => m.id).sort()).toEqual(['gpt-4o', 'gpt-4o-mini', 'o3']);
|
|
||||||
expect(models.find((m) => m.id === 'gpt-4o')).toMatchObject({ label: 'GPT-4o', contextWindow: 128000, priceIn: 2.5, priceOut: 10, capabilities: { reasoning: false, temperature: true } });
|
|
||||||
expect(models.find((m) => m.id === 'o3')?.capabilities).toMatchObject({ reasoning: true, temperature: false });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* Gemini specifics */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
describe('gemini providerData and request config', () => {
|
|
||||||
const turn = [
|
|
||||||
gemChunk([{ text: 'Pondering', thought: true, thoughtSignature: 'SIGTHOUGHT' }]),
|
|
||||||
gemChunk([{ text: 'Hello' }]),
|
|
||||||
gemChunk([{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' }], { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 2 } }, { finishReason: 'STOP' }),
|
|
||||||
];
|
|
||||||
const replayed = [
|
|
||||||
{ text: '', thought: true, thoughtSignature: 'SIGTHOUGHT' },
|
|
||||||
{ text: 'Hello' },
|
|
||||||
{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' },
|
|
||||||
];
|
|
||||||
|
|
||||||
it('keeps thought signatures in providerData and replays the raw parts', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(turn), () => sseResponse(geminiKit.text(['ok']))));
|
|
||||||
const { events, done } = await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.3, maxOutputTokens: 100 });
|
|
||||||
expect(calls[0]!.body.generationConfig).toEqual({ temperature: 0.3, maxOutputTokens: 100, thinkingConfig: { includeThoughts: true, thinkingLevel: 'medium' } });
|
|
||||||
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Pondering' }]);
|
|
||||||
expect(done.stopReason).toBe('tool_use');
|
|
||||||
expect(done.message.providerData).toEqual({ kind: 'gemini', data: { parts: replayed } });
|
|
||||||
|
|
||||||
await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi'), done.message, toolResultMsg] });
|
|
||||||
expect(calls[1]!.body.contents[1]).toEqual({ role: 'model', parts: replayed });
|
|
||||||
expect(calls[1]!.body.contents[2]).toEqual({ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] });
|
|
||||||
});
|
|
||||||
|
|
||||||
it('turns a blocked prompt into a content_filter error', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse([ev({ promptFeedback: { blockReason: 'SAFETY' } })])));
|
|
||||||
const err = await failure(run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')] }));
|
|
||||||
expect(err.code).toBe('content_filter');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('gemini listModels', () => {
|
|
||||||
it('paginates, keeps generateContent chat models and merges catalog defaults', async () => {
|
|
||||||
stubFetch(
|
|
||||||
queue(
|
|
||||||
() =>
|
|
||||||
jsonResponse({
|
|
||||||
models: [
|
|
||||||
{ name: 'models/gemini-3.8-flash', displayName: 'Gemini API Name', inputTokenLimit: 1000000, outputTokenLimit: 65536, supportedGenerationMethods: ['generateContent'], thinking: true },
|
|
||||||
{ name: 'models/gemini-embedding-001', supportedGenerationMethods: ['embedContent'] },
|
|
||||||
{ name: 'models/gemini-2.5-flash-image', supportedGenerationMethods: ['generateContent'] },
|
|
||||||
],
|
|
||||||
nextPageToken: 'tok1',
|
|
||||||
}),
|
|
||||||
() => jsonResponse({ models: [{ name: 'models/gemma-3-27b-it', supportedGenerationMethods: ['generateContent'] }, { name: 'models/text-bison-001', supportedGenerationMethods: ['generateContent'] }] }),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
const models = await geminiAdapter.listModels(geminiKit.cfg);
|
|
||||||
expect(calls[0]!.url).toBe('https://generativelanguage.googleapis.com/v1beta/models?pageSize=1000');
|
|
||||||
expect(calls[1]!.url).toContain('pageToken=tok1');
|
|
||||||
for (const c of calls) {
|
|
||||||
expect(c.url).not.toContain(geminiKit.key);
|
|
||||||
geminiKit.assertAuth(c.headers);
|
|
||||||
}
|
|
||||||
expect(models.map((m) => m.id)).toEqual(['gemini-3.8-flash', 'gemma-3-27b-it']);
|
|
||||||
expect(models[0]).toMatchObject({ label: 'Gemini API Name', contextWindow: 1000000, maxOutput: 65536, priceIn: 0.75, priceOut: 3.75, capabilities: { reasoning: true, tools: true } });
|
|
||||||
expect(models[1]!.capabilities).toMatchObject({ tools: false, pdf: false });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('toGeminiSchema', () => {
|
|
||||||
it('strips unsupported keywords and converts nullable/enum/object shapes', () => {
|
|
||||||
const out = toGeminiSchema({
|
|
||||||
$schema: 'http://json-schema.org/draft-07/schema#',
|
|
||||||
type: 'object',
|
|
||||||
additionalProperties: false,
|
|
||||||
properties: {
|
|
||||||
name: { type: ['string', 'null'], default: 'a', pattern: '^a', format: 'email' },
|
|
||||||
when: { type: 'string', format: 'date-time' },
|
|
||||||
tags: { type: 'array', items: { type: 'string', const: 'x' }, minItems: 1, uniqueItems: true },
|
|
||||||
mode: { enum: ['a', 'b'] },
|
|
||||||
nested: { type: 'object' },
|
|
||||||
},
|
|
||||||
required: ['name'],
|
|
||||||
});
|
|
||||||
expect(out).toEqual({
|
|
||||||
type: 'object',
|
|
||||||
properties: {
|
|
||||||
name: { type: 'string', nullable: true },
|
|
||||||
when: { type: 'string', format: 'date-time' },
|
|
||||||
tags: { type: 'array', items: { type: 'string' }, minItems: 1 },
|
|
||||||
mode: { enum: ['a', 'b'], type: 'string' },
|
|
||||||
nested: { type: 'object', properties: {} },
|
|
||||||
},
|
|
||||||
required: ['name'],
|
|
||||||
});
|
|
||||||
});
|
|
||||||
it('falls back to a string schema for non-objects', () => {
|
|
||||||
expect(toGeminiSchema('nope')).toEqual({ type: 'string' });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* OpenAI-compatible and Ollama specifics */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
describe('openai-compatible stream_options fallback', () => {
|
|
||||||
it('retries without stream_options when the first 400 complains about it', async () => {
|
|
||||||
stubFetch(
|
|
||||||
queue(
|
|
||||||
() => jsonResponse({ error: { message: "Unknown parameter: 'stream_options'.", type: 'invalid_request_error' } }, 400),
|
|
||||||
() => sseResponse(compatKit.text(['ok'])),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
|
|
||||||
expect(calls).toHaveLength(2);
|
|
||||||
expect(calls[0]!.body.stream_options).toEqual({ include_usage: true });
|
|
||||||
expect(calls[1]!.body.stream_options).toBeUndefined();
|
|
||||||
expect(done.message.content).toEqual([{ type: 'text', text: 'ok' }]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('does not write providerData (it only speaks neutral content)', async () => {
|
|
||||||
stubFetch(queue(() => sseResponse(compatKit.text(['ok']))));
|
|
||||||
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
|
|
||||||
expect(done.message.providerData).toBeUndefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('openai-compatible listModels', () => {
|
|
||||||
it('parses OpenRouter-style metadata, filters embeddings and merges vendor catalog defaults', async () => {
|
|
||||||
stubFetch(
|
|
||||||
queue(() =>
|
|
||||||
jsonResponse({
|
|
||||||
data: [
|
|
||||||
{
|
|
||||||
id: 'anthropic/claude-sonnet-5-5',
|
|
||||||
name: 'Claude Sonnet 5.5',
|
|
||||||
context_length: 1000000,
|
|
||||||
top_provider: { max_completion_tokens: 64000 },
|
|
||||||
pricing: { prompt: '0.000002', completion: '0.00001' },
|
|
||||||
supported_parameters: ['tools', 'reasoning'],
|
|
||||||
architecture: { input_modalities: ['text', 'image'] },
|
|
||||||
},
|
|
||||||
{ id: 'openai/text-embedding-3-small' },
|
|
||||||
{ id: 'vendor/plain' },
|
|
||||||
],
|
|
||||||
}),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
const models = await openaiCompatibleAdapter.listModels(compatKit.cfg);
|
|
||||||
expect(calls[0]!.url).toBe('https://router.example.com/api/v1/models');
|
|
||||||
compatKit.assertAuth(calls[0]!.headers);
|
|
||||||
expect(models.map((m) => m.id)).toEqual(['anthropic/claude-sonnet-5-5', 'vendor/plain']);
|
|
||||||
expect(models[0]).toMatchObject({
|
|
||||||
label: 'Claude Sonnet 5.5',
|
|
||||||
contextWindow: 1000000,
|
|
||||||
maxOutput: 64000,
|
|
||||||
priceIn: 2,
|
|
||||||
priceOut: 10,
|
|
||||||
capabilities: { tools: true, reasoning: true, vision: true, pdf: true }, // pdf borrowed from the Anthropic catalog entry
|
|
||||||
});
|
|
||||||
expect(models[0]!.capabilities).not.toHaveProperty('temperature');
|
|
||||||
expect(models[1]).toEqual({ id: 'vendor/plain' });
|
|
||||||
});
|
|
||||||
|
|
||||||
it('reports "unsupported" when the server has no /models endpoint', async () => {
|
|
||||||
stubFetch(queue(() => jsonResponse({ error: 'not found' }, 404)));
|
|
||||||
const err = await failure(openaiCompatibleAdapter.listModels(compatKit.cfg));
|
|
||||||
expect(err.code).toBe('unsupported');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('ollama listModels', () => {
|
|
||||||
it('uses /api/tags + /api/show, drops embedding-only models and keeps models whose /show fails', async () => {
|
|
||||||
stubFetch((c) => {
|
|
||||||
if (c.url.endsWith('/api/tags')) {
|
|
||||||
return jsonResponse({
|
|
||||||
models: [
|
|
||||||
{ name: 'old:1b', model: 'old:1b' },
|
|
||||||
{ name: 'nomic-embed-text:latest', model: 'nomic-embed-text:latest' },
|
|
||||||
{ name: 'llama3:latest', model: 'llama3:latest' },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (c.url.endsWith('/api/show')) {
|
|
||||||
if (c.body.model === 'llama3:latest') return jsonResponse({ capabilities: ['completion', 'tools'], model_info: { 'llama.context_length': 8192 } });
|
|
||||||
if (c.body.model === 'nomic-embed-text:latest') return jsonResponse({ capabilities: ['embedding'] });
|
|
||||||
return jsonResponse({ error: 'unsupported' }, 404);
|
|
||||||
}
|
|
||||||
return new Response('?', { status: 404 });
|
|
||||||
});
|
|
||||||
const models = await ollamaAdapter.listModels(ollamaKit.cfg);
|
|
||||||
expect(calls[0]!.url).toBe('http://localhost:11434/api/tags');
|
|
||||||
expect(calls.filter((c) => c.url.endsWith('/api/show')).map((c) => c.method)).toEqual(['POST', 'POST', 'POST']);
|
|
||||||
ollamaKit.assertAuth(calls[0]!.headers);
|
|
||||||
expect(models).toEqual([
|
|
||||||
{
|
|
||||||
id: 'llama3:latest',
|
|
||||||
label: 'llama3:latest',
|
|
||||||
priceIn: 0,
|
|
||||||
priceOut: 0,
|
|
||||||
contextWindow: 8192,
|
|
||||||
capabilities: { tools: true, vision: false, reasoning: false, pdf: false, temperature: true },
|
|
||||||
},
|
|
||||||
{ id: 'old:1b', label: 'old:1b', priceIn: 0, priceOut: 0 },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('strips /v1 from a configured base URL', async () => {
|
|
||||||
stubFetch(queue(() => jsonResponse({ models: [] })));
|
|
||||||
await ollamaAdapter.listModels({ ...ollamaKit.cfg, baseUrl: 'http://box:11434/v1/' });
|
|
||||||
expect(calls[0]!.url).toBe('http://box:11434/api/tags');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('normalizeCompatBase', () => {
|
|
||||||
it.each([
|
|
||||||
['https://api.x.com/v1', 'https://api.x.com/v1'],
|
|
||||||
['https://api.x.com/v1/', 'https://api.x.com/v1'],
|
|
||||||
[' https://x.com/api/v1 ', 'https://x.com/api/v1'],
|
|
||||||
['https://api.x.com/v1/chat/completions', 'https://api.x.com/v1'],
|
|
||||||
['https://api.x.com/v1/completions', 'https://api.x.com/v1'],
|
|
||||||
['https://api.x.com/v1/models', 'https://api.x.com/v1'],
|
|
||||||
['http://localhost:1234', 'http://localhost:1234/v1'],
|
|
||||||
['http://localhost:1234/', 'http://localhost:1234/v1'],
|
|
||||||
])('%s -> %s', (input, expected) => {
|
|
||||||
expect(normalizeCompatBase(input)).toBe(expected);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* catalog */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
describe('lookupModelDefaults', () => {
|
|
||||||
it('matches exact ids', () => {
|
|
||||||
expect(lookupModelDefaults('anthropic', 'claude-haiku-4-5')).toMatchObject({ label: 'Claude Haiku 4.5', priceIn: 1, priceOut: 5, contextWindow: 200000 });
|
|
||||||
});
|
|
||||||
it('matches dated snapshots by prefix, without the base label', () => {
|
|
||||||
const d = lookupModelDefaults('anthropic', 'claude-haiku-4-5-20251001');
|
|
||||||
expect(d).toMatchObject({ priceIn: 1, contextWindow: 200000 });
|
|
||||||
expect(d.label).toBeUndefined();
|
|
||||||
expect(lookupModelDefaults('openai', 'gpt-5-2025-08-07')).toMatchObject({ priceIn: 1.25 });
|
|
||||||
expect(lookupModelDefaults('gemini', 'models/gemini-2.5-flash-latest')).toMatchObject({ priceIn: 0.3 });
|
|
||||||
});
|
|
||||||
it('keeps capabilities but drops prices for a different variant of a known model', () => {
|
|
||||||
const d = lookupModelDefaults('openai', 'o3-deep-research');
|
|
||||||
expect(d.capabilities?.reasoning).toBe(true);
|
|
||||||
expect(d.priceIn).toBeUndefined();
|
|
||||||
expect(d.priceOut).toBeUndefined();
|
|
||||||
});
|
|
||||||
it('returns {} for unknown models and for kinds without a table', () => {
|
|
||||||
expect(lookupModelDefaults('openai', 'totally-unknown-model')).toEqual({});
|
|
||||||
expect(lookupModelDefaults('anthropic', '')).toEqual({});
|
|
||||||
expect(lookupModelDefaults('ollama', 'llama3')).toEqual({});
|
|
||||||
});
|
|
||||||
it('borrows capabilities but not prices or temperature for router ids', () => {
|
|
||||||
const d = lookupModelDefaults('openai_compatible', 'anthropic/claude-sonnet-5-5');
|
|
||||||
expect(d.priceIn).toBeUndefined();
|
|
||||||
expect(d.capabilities).toMatchObject({ tools: true, vision: true, pdf: true });
|
|
||||||
expect(d.capabilities).not.toHaveProperty('temperature');
|
|
||||||
});
|
|
||||||
it('returns copies that are safe to mutate', () => {
|
|
||||||
const a = lookupModelDefaults('openai', 'gpt-4o');
|
|
||||||
a.capabilities!.tools = false;
|
|
||||||
expect(lookupModelDefaults('openai', 'gpt-4o').capabilities!.tools).toBe(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
/* parseSSE */
|
|
||||||
/* ------------------------------------------------------------------ */
|
|
||||||
|
|
||||||
describe('parseSSE', () => {
|
|
||||||
const parse = (chunks: string[]) => drain(parseSSE(textStream(chunks)));
|
|
||||||
|
|
||||||
it('handles CRLF line endings', async () => {
|
|
||||||
expect(await parse(['event: ping\r\ndata: {"a":1}\r\n\r\ndata: two\r\n\r\n'])).toEqual([{ event: 'ping', data: '{"a":1}' }, { data: 'two' }]);
|
|
||||||
});
|
|
||||||
it('joins multi-line data with newlines', async () => {
|
|
||||||
expect(await parse(['data: a\ndata: b\n\n'])).toEqual([{ data: 'a\nb' }]);
|
|
||||||
});
|
|
||||||
it('ignores comments and keep-alives', async () => {
|
|
||||||
expect(await parse([': keepalive\n\ndata: x\n: mid-event comment\n\n'])).toEqual([{ data: 'x' }]);
|
|
||||||
});
|
|
||||||
it('stops at [DONE]', async () => {
|
|
||||||
expect(await parse(['data: a\n\ndata: [DONE]\n\ndata: b\n\n'])).toEqual([{ data: 'a' }]);
|
|
||||||
});
|
|
||||||
it('handles chunk boundaries inside a line', async () => {
|
|
||||||
expect(await parse(['da', 'ta: hel', 'lo\n', '\ndata: wor', 'ld\n\n'])).toEqual([{ data: 'hello' }, { data: 'world' }]);
|
|
||||||
});
|
|
||||||
it('handles CRLF split across chunks', async () => {
|
|
||||||
expect(await parse(['data: x\r', '\n\r', '\ndata: y\n\n'])).toEqual([{ data: 'x' }, { data: 'y' }]);
|
|
||||||
});
|
|
||||||
it('handles bare CR line endings', async () => {
|
|
||||||
expect(await parse(['data: x\r\rdata: y\r\r'])).toEqual([{ data: 'x' }, { data: 'y' }]);
|
|
||||||
});
|
|
||||||
it('flushes a final event with no trailing blank line', async () => {
|
|
||||||
expect(await parse(['data: a\n\ndata: tail'])).toEqual([{ data: 'a' }, { data: 'tail' }]);
|
|
||||||
});
|
|
||||||
it('strips a BOM and decodes multi-byte characters split across chunks', async () => {
|
|
||||||
const bytes = enc.encode('data: €uro\n\n');
|
|
||||||
const euro = bytes.indexOf(0xe2); // first byte of the 3-byte euro sign
|
|
||||||
const stream = bytesStream([bytes.slice(0, euro + 1), bytes.slice(euro + 1, euro + 2), bytes.slice(euro + 2)]);
|
|
||||||
expect(await drain(parseSSE(stream))).toEqual([{ data: '€uro' }]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,624 +0,0 @@
|
||||||
/**
|
|
||||||
* Provider + model configuration, task routing, usage accounting and the two call
|
|
||||||
* primitives everything else builds on: `runChat` (one streamed model turn) and
|
|
||||||
* `generateStructured` (schema-constrained JSON through a forced tool call).
|
|
||||||
*/
|
|
||||||
import { jsonrepair } from 'jsonrepair';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import { decryptSecret, encryptSecret, maskSecret } from '../crypto.ts';
|
|
||||||
import type { DB } from '../db.ts';
|
|
||||||
import { HttpError, notFound, type AppContext } from '../http.ts';
|
|
||||||
import { getSetting, setSetting } from '../settings.ts';
|
|
||||||
import { newId, now, parseJson } from '../util.ts';
|
|
||||||
import { getAdapter, lookupModelDefaults } from './providers/index.ts';
|
|
||||||
import {
|
|
||||||
AIError,
|
|
||||||
DEFAULT_CAPABILITIES,
|
|
||||||
PROVIDER_KINDS,
|
|
||||||
type ChatMessage,
|
|
||||||
type ChatRequest,
|
|
||||||
type DiscoveredModel,
|
|
||||||
type ModelCapabilities,
|
|
||||||
type ProviderConfig,
|
|
||||||
type ProviderKind,
|
|
||||||
type ReasoningEffort,
|
|
||||||
type StopReason,
|
|
||||||
type StreamEvent,
|
|
||||||
type ToolDef,
|
|
||||||
type Usage,
|
|
||||||
} from './types.ts';
|
|
||||||
|
|
||||||
/* ------------------------------ rows + DTOs ------------------------------ */
|
|
||||||
|
|
||||||
interface ProviderRow {
|
|
||||||
id: string;
|
|
||||||
name: string;
|
|
||||||
kind: ProviderKind;
|
|
||||||
base_url: string | null;
|
|
||||||
api_key_enc: string | null;
|
|
||||||
headers_enc: string | null;
|
|
||||||
enabled: number;
|
|
||||||
last_test_at: number | null;
|
|
||||||
last_test_ok: number | null;
|
|
||||||
last_test_message: string;
|
|
||||||
created_at: number;
|
|
||||||
updated_at: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ModelRow {
|
|
||||||
id: string;
|
|
||||||
provider_id: string;
|
|
||||||
model: string;
|
|
||||||
label: string;
|
|
||||||
source: string;
|
|
||||||
enabled: number;
|
|
||||||
capabilities: string;
|
|
||||||
context_window: number | null;
|
|
||||||
max_output: number | null;
|
|
||||||
price_in: number | null;
|
|
||||||
price_out: number | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const TASKS = ['chat', 'generate', 'vision', 'quick'] as const;
|
|
||||||
export type TaskName = (typeof TASKS)[number];
|
|
||||||
|
|
||||||
export const TASK_INFO: Record<TaskName, { label: string; description: string }> = {
|
|
||||||
chat: { label: 'Assistant', description: 'The assistant that searches your library, answers questions and runs tools.' },
|
|
||||||
generate: { label: 'Content generation', description: 'Writing quizzes, flashcards and study guides. Benefits most from a strong model.' },
|
|
||||||
vision: { label: 'Vision and OCR', description: 'Reading scanned PDFs and images. Needs a model that accepts images.' },
|
|
||||||
quick: { label: 'Quick explanations', description: 'Short explanations while studying. A fast, inexpensive model works well.' },
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface TaskConfig {
|
|
||||||
modelId: string | null;
|
|
||||||
temperature: number | null;
|
|
||||||
reasoningEffort: ReasoningEffort | null;
|
|
||||||
maxOutputTokens: number | null;
|
|
||||||
}
|
|
||||||
export type TaskSettings = Record<TaskName | 'default', TaskConfig>;
|
|
||||||
|
|
||||||
const EMPTY_TASK: TaskConfig = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null };
|
|
||||||
|
|
||||||
export const taskConfigSchema = z.object({
|
|
||||||
modelId: z.string().nullable(),
|
|
||||||
temperature: z.number().min(0).max(2).nullable(),
|
|
||||||
reasoningEffort: z.enum(['none', 'low', 'medium', 'high']).nullable(),
|
|
||||||
maxOutputTokens: z.number().int().min(256).max(200000).nullable(),
|
|
||||||
});
|
|
||||||
|
|
||||||
function mapProvider(app: AppContext, row: ProviderRow) {
|
|
||||||
const key = decryptSecret(row.api_key_enc, app.config.secretKey);
|
|
||||||
const headers = parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {});
|
|
||||||
return {
|
|
||||||
id: row.id,
|
|
||||||
name: row.name,
|
|
||||||
kind: row.kind,
|
|
||||||
baseUrl: row.base_url,
|
|
||||||
hasKey: Boolean(row.api_key_enc),
|
|
||||||
/** Only ever a masked preview; the key itself never leaves the server. */
|
|
||||||
keyPreview: key ? maskSecret(key) : row.api_key_enc ? 'unreadable (re-enter key)' : '',
|
|
||||||
headerNames: Object.keys(headers),
|
|
||||||
enabled: Boolean(row.enabled),
|
|
||||||
lastTestAt: row.last_test_at,
|
|
||||||
lastTestOk: row.last_test_ok === null ? null : Boolean(row.last_test_ok),
|
|
||||||
lastTestMessage: row.last_test_message,
|
|
||||||
createdAt: row.created_at,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function mapModel(row: ModelRow & { provider_name?: string; provider_kind?: string; provider_enabled?: number }) {
|
|
||||||
return {
|
|
||||||
id: row.id,
|
|
||||||
providerId: row.provider_id,
|
|
||||||
providerName: row.provider_name ?? '',
|
|
||||||
providerKind: (row.provider_kind ?? '') as ProviderKind,
|
|
||||||
model: row.model,
|
|
||||||
label: row.label || row.model,
|
|
||||||
source: row.source,
|
|
||||||
enabled: Boolean(row.enabled),
|
|
||||||
available: Boolean(row.enabled) && row.provider_enabled !== 0,
|
|
||||||
capabilities: { ...DEFAULT_CAPABILITIES, ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}) },
|
|
||||||
contextWindow: row.context_window,
|
|
||||||
maxOutput: row.max_output,
|
|
||||||
priceIn: row.price_in,
|
|
||||||
priceOut: row.price_out,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
export type ModelDto = ReturnType<typeof mapModel>;
|
|
||||||
|
|
||||||
const MODEL_SELECT = `SELECT m.*, p.name AS provider_name, p.kind AS provider_kind, p.enabled AS provider_enabled FROM ai_models m JOIN ai_providers p ON p.id = m.provider_id`;
|
|
||||||
|
|
||||||
/* ------------------------------ providers ------------------------------ */
|
|
||||||
|
|
||||||
export const providerInputSchema = z.object({
|
|
||||||
name: z.string().trim().min(1).max(80),
|
|
||||||
kind: z.enum(PROVIDER_KINDS as [ProviderKind, ...ProviderKind[]]),
|
|
||||||
baseUrl: z
|
|
||||||
.string()
|
|
||||||
.trim()
|
|
||||||
.max(500)
|
|
||||||
.nullish()
|
|
||||||
.transform((v) => v || null)
|
|
||||||
.refine((v) => v === null || /^https?:\/\//i.test(v), 'URL must start with http:// or https://'),
|
|
||||||
/** undefined = keep the stored key, '' or null = remove it. */
|
|
||||||
apiKey: z.string().max(2000).nullish(),
|
|
||||||
headers: z.record(z.string().max(100), z.string().max(2000)).optional(),
|
|
||||||
enabled: z.boolean().optional(),
|
|
||||||
});
|
|
||||||
|
|
||||||
function providerConfig(app: AppContext, row: ProviderRow): ProviderConfig {
|
|
||||||
return {
|
|
||||||
id: row.id,
|
|
||||||
kind: row.kind,
|
|
||||||
name: row.name,
|
|
||||||
baseUrl: row.base_url,
|
|
||||||
apiKey: decryptSecret(row.api_key_enc, app.config.secretKey),
|
|
||||||
headers: parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function providerRow(db: DB, id: string): ProviderRow {
|
|
||||||
const row = db.get<ProviderRow>('SELECT * FROM ai_providers WHERE id = ?', id);
|
|
||||||
if (!row) throw notFound('Provider');
|
|
||||||
return row;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function listProviders(app: AppContext) {
|
|
||||||
return app.db.all<ProviderRow>('SELECT * FROM ai_providers ORDER BY created_at').map((r) => mapProvider(app, r));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function saveProvider(app: AppContext, id: string | null, input: z.input<typeof providerInputSchema>) {
|
|
||||||
const data = providerInputSchema.parse(input);
|
|
||||||
const meta = getAdapter(data.kind).meta;
|
|
||||||
if (meta.requiresBaseUrl && !data.baseUrl) throw new HttpError(422, 'validation', 'This provider type needs a server URL');
|
|
||||||
const at = now();
|
|
||||||
const key = app.config.secretKey;
|
|
||||||
if (id) {
|
|
||||||
const row = providerRow(app.db, id);
|
|
||||||
const apiKeyEnc = data.apiKey === undefined ? row.api_key_enc : data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null;
|
|
||||||
const headersEnc = data.headers === undefined ? row.headers_enc : Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null;
|
|
||||||
app.db.run(
|
|
||||||
'UPDATE ai_providers SET name = ?, kind = ?, base_url = ?, api_key_enc = ?, headers_enc = ?, enabled = ?, updated_at = ? WHERE id = ?',
|
|
||||||
data.name,
|
|
||||||
data.kind,
|
|
||||||
data.baseUrl,
|
|
||||||
apiKeyEnc,
|
|
||||||
headersEnc,
|
|
||||||
data.enabled === undefined ? row.enabled : data.enabled ? 1 : 0,
|
|
||||||
at,
|
|
||||||
id,
|
|
||||||
);
|
|
||||||
return mapProvider(app, providerRow(app.db, id));
|
|
||||||
}
|
|
||||||
const newProviderId = newId();
|
|
||||||
app.db.run(
|
|
||||||
'INSERT INTO ai_providers (id, name, kind, base_url, api_key_enc, headers_enc, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
||||||
newProviderId,
|
|
||||||
data.name,
|
|
||||||
data.kind,
|
|
||||||
data.baseUrl,
|
|
||||||
data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null,
|
|
||||||
data.headers && Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null,
|
|
||||||
data.enabled === false ? 0 : 1,
|
|
||||||
at,
|
|
||||||
at,
|
|
||||||
);
|
|
||||||
return mapProvider(app, providerRow(app.db, newProviderId));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function deleteProvider(app: AppContext, id: string): void {
|
|
||||||
providerRow(app.db, id);
|
|
||||||
const modelIds = new Set(app.db.all<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ?', id).map((m) => m.id));
|
|
||||||
app.db.run('DELETE FROM ai_providers WHERE id = ?', id);
|
|
||||||
const tasks = getTaskSettings(app.db);
|
|
||||||
let changed = false;
|
|
||||||
for (const name of Object.keys(tasks) as (keyof TaskSettings)[]) {
|
|
||||||
if (tasks[name].modelId && modelIds.has(tasks[name].modelId!)) {
|
|
||||||
tasks[name].modelId = null;
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (changed) setSetting(app.db, 'ai.tasks', tasks);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function testProvider(app: AppContext, id: string) {
|
|
||||||
const row = providerRow(app.db, id);
|
|
||||||
const anyModel = app.db.get<{ model: string }>('SELECT model FROM ai_models WHERE provider_id = ? ORDER BY enabled DESC LIMIT 1', id)?.model;
|
|
||||||
const result = await getAdapter(row.kind).test(providerConfig(app, row), anyModel);
|
|
||||||
app.db.run('UPDATE ai_providers SET last_test_at = ?, last_test_ok = ?, last_test_message = ? WHERE id = ?', now(), result.ok ? 1 : 0, result.message.slice(0, 500), id);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Looks up the provider's model list and merges it into ai_models without disturbing user edits. */
|
|
||||||
export async function discoverModels(app: AppContext, id: string): Promise<{ added: number; total: number }> {
|
|
||||||
const row = providerRow(app.db, id);
|
|
||||||
const found = await getAdapter(row.kind).listModels(providerConfig(app, row));
|
|
||||||
const existing = new Map(app.db.all<ModelRow>('SELECT * FROM ai_models WHERE provider_id = ?', id).map((m) => [m.model, m]));
|
|
||||||
const at = now();
|
|
||||||
let added = 0;
|
|
||||||
app.db.tx(() => {
|
|
||||||
for (const m of found) {
|
|
||||||
const current = existing.get(m.id);
|
|
||||||
if (current) {
|
|
||||||
// Refresh facts the provider reports, but never overwrite prices or flags the user may have edited.
|
|
||||||
app.db.run(
|
|
||||||
'UPDATE ai_models SET context_window = COALESCE(?, context_window), max_output = COALESCE(?, max_output), price_in = COALESCE(price_in, ?), price_out = COALESCE(price_out, ?), updated_at = ? WHERE id = ?',
|
|
||||||
m.contextWindow ?? null,
|
|
||||||
m.maxOutput ?? null,
|
|
||||||
m.priceIn ?? null,
|
|
||||||
m.priceOut ?? null,
|
|
||||||
at,
|
|
||||||
current.id,
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
insertModel(app.db, id, m, 'discovered', false);
|
|
||||||
added += 1;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return { added, total: found.length };
|
|
||||||
}
|
|
||||||
|
|
||||||
function insertModel(db: DB, providerId: string, m: DiscoveredModel, source: string, enabled: boolean): string {
|
|
||||||
const id = newId();
|
|
||||||
const at = now();
|
|
||||||
db.run(
|
|
||||||
`INSERT INTO ai_models (id, provider_id, model, label, source, enabled, capabilities, context_window, max_output, price_in, price_out, created_at, updated_at)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
||||||
id,
|
|
||||||
providerId,
|
|
||||||
m.id,
|
|
||||||
m.label ?? '',
|
|
||||||
source,
|
|
||||||
enabled ? 1 : 0,
|
|
||||||
JSON.stringify(m.capabilities ?? {}),
|
|
||||||
m.contextWindow ?? null,
|
|
||||||
m.maxOutput ?? null,
|
|
||||||
m.priceIn ?? null,
|
|
||||||
m.priceOut ?? null,
|
|
||||||
at,
|
|
||||||
at,
|
|
||||||
);
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ models ------------------------------ */
|
|
||||||
|
|
||||||
export function listModels(db: DB): ModelDto[] {
|
|
||||||
return db.all<ModelRow>(`${MODEL_SELECT} ORDER BY p.created_at, m.enabled DESC, m.model`).map(mapModel);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getModel(db: DB, id: string): ModelDto {
|
|
||||||
const row = db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ?`, id);
|
|
||||||
if (!row) throw notFound('Model');
|
|
||||||
return mapModel(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
export const manualModelSchema = z.object({
|
|
||||||
providerId: z.string(),
|
|
||||||
model: z.string().trim().min(1).max(200),
|
|
||||||
label: z.string().trim().max(120).optional(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export function addManualModel(app: AppContext, input: z.input<typeof manualModelSchema>): ModelDto {
|
|
||||||
const data = manualModelSchema.parse(input);
|
|
||||||
const provider = providerRow(app.db, data.providerId);
|
|
||||||
const existing = app.db.get<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ? AND model = ?', provider.id, data.model);
|
|
||||||
if (existing) {
|
|
||||||
app.db.run('UPDATE ai_models SET enabled = 1, updated_at = ? WHERE id = ?', now(), existing.id);
|
|
||||||
return getModel(app.db, existing.id);
|
|
||||||
}
|
|
||||||
const defaults = lookupModelDefaults(provider.kind, data.model);
|
|
||||||
const id = insertModel(app.db, provider.id, { ...defaults, id: data.model, label: data.label || defaults.label }, 'manual', true);
|
|
||||||
return getModel(app.db, id);
|
|
||||||
}
|
|
||||||
|
|
||||||
export const modelPatchSchema = z.object({
|
|
||||||
enabled: z.boolean().optional(),
|
|
||||||
label: z.string().trim().max(120).optional(),
|
|
||||||
capabilities: z
|
|
||||||
.object({ tools: z.boolean(), vision: z.boolean(), pdf: z.boolean(), reasoning: z.boolean(), temperature: z.boolean() })
|
|
||||||
.partial()
|
|
||||||
.optional(),
|
|
||||||
contextWindow: z.number().int().min(256).max(100_000_000).nullable().optional(),
|
|
||||||
maxOutput: z.number().int().min(16).max(10_000_000).nullable().optional(),
|
|
||||||
priceIn: z.number().min(0).max(100000).nullable().optional(),
|
|
||||||
priceOut: z.number().min(0).max(100000).nullable().optional(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export function patchModel(db: DB, id: string, input: z.input<typeof modelPatchSchema>): ModelDto {
|
|
||||||
const patch = modelPatchSchema.parse(input);
|
|
||||||
const row = db.get<ModelRow>('SELECT * FROM ai_models WHERE id = ?', id);
|
|
||||||
if (!row) throw notFound('Model');
|
|
||||||
const caps = { ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}), ...patch.capabilities };
|
|
||||||
db.run(
|
|
||||||
'UPDATE ai_models SET enabled = ?, label = ?, capabilities = ?, context_window = ?, max_output = ?, price_in = ?, price_out = ?, updated_at = ? WHERE id = ?',
|
|
||||||
patch.enabled === undefined ? row.enabled : patch.enabled ? 1 : 0,
|
|
||||||
patch.label ?? row.label,
|
|
||||||
JSON.stringify(caps),
|
|
||||||
patch.contextWindow === undefined ? row.context_window : patch.contextWindow,
|
|
||||||
patch.maxOutput === undefined ? row.max_output : patch.maxOutput,
|
|
||||||
patch.priceIn === undefined ? row.price_in : patch.priceIn,
|
|
||||||
patch.priceOut === undefined ? row.price_out : patch.priceOut,
|
|
||||||
now(),
|
|
||||||
id,
|
|
||||||
);
|
|
||||||
return getModel(db, id);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function deleteModel(db: DB, id: string): void {
|
|
||||||
const { changes } = db.run('DELETE FROM ai_models WHERE id = ?', id);
|
|
||||||
if (!changes) throw notFound('Model');
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ task routing ------------------------------ */
|
|
||||||
|
|
||||||
export function getTaskSettings(db: DB): TaskSettings {
|
|
||||||
const stored = getSetting<Partial<TaskSettings>>(db, 'ai.tasks', {});
|
|
||||||
const out = { default: { ...EMPTY_TASK, ...stored.default } } as TaskSettings;
|
|
||||||
for (const t of TASKS) out[t] = { ...EMPTY_TASK, ...stored[t] };
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function saveTaskSettings(db: DB, input: unknown): TaskSettings {
|
|
||||||
const schema = z.object({ default: taskConfigSchema, chat: taskConfigSchema, generate: taskConfigSchema, vision: taskConfigSchema, quick: taskConfigSchema }).partial();
|
|
||||||
const patch = schema.parse(input);
|
|
||||||
const next = { ...getTaskSettings(db), ...patch } as TaskSettings;
|
|
||||||
setSetting(db, 'ai.tasks', next);
|
|
||||||
return next;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ResolvedModel {
|
|
||||||
provider: ProviderConfig;
|
|
||||||
model: ModelDto;
|
|
||||||
params: { temperature?: number; reasoningEffort?: ReasoningEffort; maxOutputTokens?: number };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Picks the model for a task: explicit override, then the task's own model, then the
|
|
||||||
* default model, then (so a fresh install works after adding one model) any enabled model.
|
|
||||||
*/
|
|
||||||
export function resolveModel(app: AppContext, task: TaskName, overrideModelId?: string | null): ResolvedModel {
|
|
||||||
const tasks = getTaskSettings(app.db);
|
|
||||||
const cfg = tasks[task];
|
|
||||||
const candidates = [overrideModelId, cfg.modelId, tasks.default.modelId].filter((x): x is string => Boolean(x));
|
|
||||||
let model: ModelDto | undefined;
|
|
||||||
for (const id of candidates) {
|
|
||||||
const row = app.db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ? AND p.enabled = 1`, id);
|
|
||||||
if (row) {
|
|
||||||
model = mapModel(row);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!model) {
|
|
||||||
const needVision = task === 'vision';
|
|
||||||
const rows = app.db.all<ModelRow>(`${MODEL_SELECT} WHERE m.enabled = 1 AND p.enabled = 1 ORDER BY p.created_at, m.created_at`).map(mapModel);
|
|
||||||
model = needVision ? rows.find((m) => m.capabilities.vision) : rows[0];
|
|
||||||
}
|
|
||||||
if (!model) {
|
|
||||||
throw new AIError(
|
|
||||||
'not_configured',
|
|
||||||
task === 'vision' ? 'No vision-capable AI model is set up. Add one in Settings, then choose it for "Vision and OCR".' : 'No AI model is set up yet. Add a provider and enable a model in Settings.',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const provider = providerConfig(app, providerRow(app.db, model.providerId));
|
|
||||||
const own = cfg.modelId === model.id || (!cfg.modelId && tasks.default.modelId === model.id) ? cfg : EMPTY_TASK;
|
|
||||||
const fallback = tasks.default;
|
|
||||||
const params: ResolvedModel['params'] = {};
|
|
||||||
const temperature = own.temperature ?? fallback.temperature;
|
|
||||||
const effort = own.reasoningEffort ?? fallback.reasoningEffort;
|
|
||||||
const maxOut = own.maxOutputTokens ?? fallback.maxOutputTokens;
|
|
||||||
if (temperature !== null && model.capabilities.temperature) params.temperature = temperature;
|
|
||||||
if (effort !== null && model.capabilities.reasoning) params.reasoningEffort = effort;
|
|
||||||
if (maxOut !== null) params.maxOutputTokens = maxOut;
|
|
||||||
return { provider, model, params };
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ usage ------------------------------ */
|
|
||||||
|
|
||||||
export function estimateCost(model: Pick<ModelDto, 'priceIn' | 'priceOut'>, usage: Usage): number | null {
|
|
||||||
if (model.priceIn === null && model.priceOut === null) return null;
|
|
||||||
// Cached input is billed at a fraction of the list price by every provider; 10% is a close, conservative stand-in.
|
|
||||||
const cached = usage.cacheReadTokens ?? 0;
|
|
||||||
const fresh = Math.max(0, usage.inputTokens - cached);
|
|
||||||
return ((fresh + cached * 0.1) * (model.priceIn ?? 0) + usage.outputTokens * (model.priceOut ?? 0)) / 1_000_000;
|
|
||||||
}
|
|
||||||
|
|
||||||
function recordUsage(app: AppContext, r: ResolvedModel, task: string, usage: Usage | null, started: number, error?: string): void {
|
|
||||||
try {
|
|
||||||
app.db.run(
|
|
||||||
`INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cost, ok, error, duration_ms)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
||||||
now(),
|
|
||||||
r.provider.id,
|
|
||||||
r.provider.name,
|
|
||||||
r.model.model,
|
|
||||||
task,
|
|
||||||
usage?.inputTokens ?? 0,
|
|
||||||
usage?.outputTokens ?? 0,
|
|
||||||
usage?.cacheReadTokens ?? 0,
|
|
||||||
usage ? estimateCost(r.model, usage) : null,
|
|
||||||
error ? 0 : 1,
|
|
||||||
(error ?? '').slice(0, 300),
|
|
||||||
now() - started,
|
|
||||||
);
|
|
||||||
} catch {
|
|
||||||
/* accounting must never break a request */
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ call primitives ------------------------------ */
|
|
||||||
|
|
||||||
export interface ChatOptions {
|
|
||||||
task: TaskName;
|
|
||||||
/** Label stored with usage rows ("assistant", "quiz generation"...). Defaults to the task. */
|
|
||||||
purpose?: string;
|
|
||||||
modelId?: string | null;
|
|
||||||
system?: string;
|
|
||||||
messages: ChatMessage[];
|
|
||||||
tools?: ToolDef[];
|
|
||||||
toolChoice?: ChatRequest['toolChoice'];
|
|
||||||
jsonMode?: boolean;
|
|
||||||
temperature?: number;
|
|
||||||
reasoningEffort?: ReasoningEffort;
|
|
||||||
maxOutputTokens?: number;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
onEvent?: (event: StreamEvent) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ChatResult {
|
|
||||||
message: ChatMessage;
|
|
||||||
text: string;
|
|
||||||
stopReason: StopReason;
|
|
||||||
usage: Usage;
|
|
||||||
model: ModelDto;
|
|
||||||
}
|
|
||||||
|
|
||||||
const sleep = (ms: number, signal?: AbortSignal) =>
|
|
||||||
new Promise<void>((resolve, reject) => {
|
|
||||||
const t = setTimeout(resolve, ms);
|
|
||||||
signal?.addEventListener('abort', () => {
|
|
||||||
clearTimeout(t);
|
|
||||||
reject(new AIError('aborted', 'The request was cancelled.'));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
/** Runs one model turn. Retries transient failures, but only while nothing has been streamed yet. */
|
|
||||||
export async function runChat(app: AppContext, opts: ChatOptions): Promise<ChatResult> {
|
|
||||||
const resolved = resolveModel(app, opts.task, opts.modelId);
|
|
||||||
const caps = resolved.model.capabilities;
|
|
||||||
const req: ChatRequest = {
|
|
||||||
model: resolved.model.model,
|
|
||||||
system: opts.system,
|
|
||||||
messages: opts.messages,
|
|
||||||
tools: opts.tools,
|
|
||||||
toolChoice: opts.toolChoice,
|
|
||||||
jsonMode: opts.jsonMode,
|
|
||||||
capabilities: caps,
|
|
||||||
signal: opts.signal,
|
|
||||||
temperature: opts.temperature ?? resolved.params.temperature,
|
|
||||||
reasoningEffort: opts.reasoningEffort ?? resolved.params.reasoningEffort,
|
|
||||||
maxOutputTokens: opts.maxOutputTokens ?? resolved.params.maxOutputTokens ?? undefined,
|
|
||||||
};
|
|
||||||
if (req.maxOutputTokens && resolved.model.maxOutput) req.maxOutputTokens = Math.min(req.maxOutputTokens, resolved.model.maxOutput);
|
|
||||||
if (opts.tools?.length && !caps.tools && (opts.toolChoice === 'required' || typeof opts.toolChoice === 'object')) {
|
|
||||||
throw new AIError('unsupported', `${resolved.model.label} is marked as not supporting tools. Choose another model for this task.`);
|
|
||||||
}
|
|
||||||
const adapter = getAdapter(resolved.provider.kind);
|
|
||||||
const purpose = opts.purpose ?? opts.task;
|
|
||||||
|
|
||||||
for (let attempt = 0; ; attempt += 1) {
|
|
||||||
const started = now();
|
|
||||||
let streamed = false;
|
|
||||||
try {
|
|
||||||
for await (const event of adapter.stream(resolved.provider, req)) {
|
|
||||||
if (event.type === 'done') {
|
|
||||||
recordUsage(app, resolved, purpose, event.usage, started);
|
|
||||||
const text = event.message.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
|
||||||
return { message: event.message, text, stopReason: event.stopReason, usage: event.usage, model: resolved.model };
|
|
||||||
}
|
|
||||||
streamed = true;
|
|
||||||
opts.onEvent?.(event);
|
|
||||||
}
|
|
||||||
throw new AIError('invalid_response', `${resolved.provider.name} ended the response unexpectedly.`, { retryable: true });
|
|
||||||
} catch (raw) {
|
|
||||||
const err = raw instanceof AIError ? raw : new AIError('server', 'The AI request failed unexpectedly.', { cause: raw });
|
|
||||||
if (!(raw instanceof AIError)) app.log('error', 'Unexpected AI adapter error', raw);
|
|
||||||
recordUsage(app, resolved, purpose, null, started, `${err.code}: ${err.message}`);
|
|
||||||
if (!err.retryable || streamed || attempt >= 2 || opts.signal?.aborted) throw err;
|
|
||||||
await sleep(1500 * (attempt + 1) ** 2, opts.signal);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Finds the first JSON value in model text (handles code fences and stray prose). */
|
|
||||||
export function parseLooseJson(text: string): unknown {
|
|
||||||
let t = text.trim();
|
|
||||||
const fence = /```(?:json)?\s*\n?([\s\S]*?)```/i.exec(t);
|
|
||||||
if (fence) t = fence[1].trim();
|
|
||||||
const start = t.search(/[[{]/);
|
|
||||||
if (start < 0) throw new AIError('invalid_response', 'The model did not return structured data.');
|
|
||||||
t = t.slice(start);
|
|
||||||
try {
|
|
||||||
return JSON.parse(t);
|
|
||||||
} catch {
|
|
||||||
try {
|
|
||||||
return JSON.parse(jsonrepair(t));
|
|
||||||
} catch {
|
|
||||||
throw new AIError('invalid_response', 'The model returned data that could not be read.', { retryable: true });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
|
|
||||||
/** Name + description of the "function" whose arguments are the result. */
|
|
||||||
toolName: string;
|
|
||||||
toolDescription: string;
|
|
||||||
schema: Record<string, unknown>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Structured output that works on every provider: the schema is offered as a single tool the
|
|
||||||
* model must call. Models without tool support (many local ones) get the schema in the prompt
|
|
||||||
* and their text is parsed leniently instead.
|
|
||||||
*/
|
|
||||||
export async function generateStructured(app: AppContext, opts: StructuredOptions): Promise<{ data: unknown; usage: Usage; model: ModelDto; truncated: boolean }> {
|
|
||||||
const resolved = resolveModel(app, opts.task, opts.modelId);
|
|
||||||
const tool: ToolDef = { name: opts.toolName, description: opts.toolDescription, parameters: opts.schema };
|
|
||||||
const promptJson = async () => {
|
|
||||||
const system = `${opts.system ?? ''}\n\nRespond with ONLY a JSON object matching this JSON Schema. No prose, no Markdown fences.\n${JSON.stringify(opts.schema)}`.trim();
|
|
||||||
const res = await runChat(app, { ...opts, system, jsonMode: true });
|
|
||||||
return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
|
||||||
};
|
|
||||||
if (!resolved.model.capabilities.tools) return promptJson();
|
|
||||||
try {
|
|
||||||
const res = await runChat(app, { ...opts, tools: [tool], toolChoice: { name: opts.toolName } });
|
|
||||||
const call = res.message.content.find((p) => p.type === 'tool_call');
|
|
||||||
if (call && call.type === 'tool_call') return { data: call.input, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
|
||||||
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
|
||||||
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
|
|
||||||
if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
|
|
||||||
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ reporting ------------------------------ */
|
|
||||||
|
|
||||||
export function usageSummary(db: DB, days: number) {
|
|
||||||
const since = now() - days * 86_400_000;
|
|
||||||
const byModel = db.all<Record<string, any>>(
|
|
||||||
`SELECT provider_name AS provider, model, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost,
|
|
||||||
SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors, SUM(CASE WHEN cost IS NULL AND ok = 1 THEN 1 ELSE 0 END) AS unpriced
|
|
||||||
FROM ai_usage WHERE at >= ? GROUP BY provider_name, model ORDER BY cost DESC, calls DESC`,
|
|
||||||
since,
|
|
||||||
);
|
|
||||||
const byTask = db.all<Record<string, any>>(
|
|
||||||
`SELECT task, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost FROM ai_usage WHERE at >= ? AND ok = 1 GROUP BY task ORDER BY cost DESC`,
|
|
||||||
since,
|
|
||||||
);
|
|
||||||
const total = db.get<Record<string, any>>(
|
|
||||||
`SELECT COUNT(*) AS calls, COALESCE(SUM(input_tokens), 0) AS input, COALESCE(SUM(output_tokens), 0) AS output, COALESCE(SUM(cost), 0) AS cost FROM ai_usage WHERE at >= ?`,
|
|
||||||
since,
|
|
||||||
)!;
|
|
||||||
const recentErrors = db.all<Record<string, any>>(`SELECT at, provider_name AS provider, model, task, error FROM ai_usage WHERE ok = 0 AND at >= ? ORDER BY at DESC LIMIT 10`, since);
|
|
||||||
return { days, total, byModel, byTask, recentErrors };
|
|
||||||
}
|
|
||||||
|
|
||||||
/** True when at least one usable model exists; lets the UI show setup prompts instead of errors. */
|
|
||||||
export function aiStatus(app: AppContext) {
|
|
||||||
const models = listModels(app.db).filter((m) => m.available);
|
|
||||||
const tasks = getTaskSettings(app.db);
|
|
||||||
const resolved: Record<string, { modelId: string; label: string; provider: string } | null> = {};
|
|
||||||
for (const t of TASKS) {
|
|
||||||
try {
|
|
||||||
const r = resolveModel(app, t);
|
|
||||||
resolved[t] = { modelId: r.model.id, label: r.model.label, provider: r.provider.name };
|
|
||||||
} catch {
|
|
||||||
resolved[t] = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return { ready: models.length > 0, modelCount: models.length, tasks, resolved };
|
|
||||||
}
|
|
||||||
|
|
@ -1,180 +0,0 @@
|
||||||
/**
|
|
||||||
* Provider-neutral AI contract.
|
|
||||||
*
|
|
||||||
* Every provider adapter (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama)
|
|
||||||
* translates between this shape and the provider's wire format. Nothing outside
|
|
||||||
* `server/ai/providers/` may depend on a provider-specific request or response.
|
|
||||||
*/
|
|
||||||
|
|
||||||
export type ProviderKind = 'openai' | 'anthropic' | 'gemini' | 'openai_compatible' | 'ollama';
|
|
||||||
|
|
||||||
export const PROVIDER_KINDS: ProviderKind[] = ['openai', 'anthropic', 'gemini', 'openai_compatible', 'ollama'];
|
|
||||||
|
|
||||||
/** Decrypted provider connection, only ever held server-side. */
|
|
||||||
export interface ProviderConfig {
|
|
||||||
id: string;
|
|
||||||
kind: ProviderKind;
|
|
||||||
name: string;
|
|
||||||
/** Null means "use the provider's official default endpoint". */
|
|
||||||
baseUrl: string | null;
|
|
||||||
apiKey: string | null;
|
|
||||||
/** Extra HTTP headers (e.g. OpenRouter attribution, org ids). */
|
|
||||||
headers?: Record<string, string>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type ContentPart =
|
|
||||||
| { type: 'text'; text: string }
|
|
||||||
/** Base64 image data (no data: prefix). */
|
|
||||||
| { type: 'image'; mime: string; data: string }
|
|
||||||
/** Base64 file, used for native PDF input where a provider supports it. */
|
|
||||||
| { type: 'file'; mime: string; data: string; filename?: string }
|
|
||||||
| { type: 'tool_call'; id: string; name: string; input: Record<string, unknown> }
|
|
||||||
| { type: 'tool_result'; toolCallId: string; name: string; output: string; isError?: boolean };
|
|
||||||
|
|
||||||
export interface ChatMessage {
|
|
||||||
/** Tool results travel as `tool_result` parts inside a `user` message. */
|
|
||||||
role: 'user' | 'assistant';
|
|
||||||
content: ContentPart[];
|
|
||||||
/**
|
|
||||||
* Opaque provider state for an assistant turn (thinking blocks + signatures,
|
|
||||||
* Gemini thought signatures, OpenAI reasoning items...). An adapter writes it
|
|
||||||
* on the `done` event and, when the next request goes to the same provider
|
|
||||||
* kind, replays it verbatim instead of rebuilding the turn from `content`.
|
|
||||||
*/
|
|
||||||
providerData?: { kind: ProviderKind; data: unknown };
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ToolDef {
|
|
||||||
name: string;
|
|
||||||
description: string;
|
|
||||||
/** JSON Schema (object). Adapters down-convert for providers with a restricted dialect. */
|
|
||||||
parameters: Record<string, unknown>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type ReasoningEffort = 'none' | 'low' | 'medium' | 'high';
|
|
||||||
|
|
||||||
export interface ModelCapabilities {
|
|
||||||
tools: boolean;
|
|
||||||
vision: boolean;
|
|
||||||
/** Accepts PDF files directly as input. */
|
|
||||||
pdf: boolean;
|
|
||||||
/** Has a configurable reasoning / thinking effort. */
|
|
||||||
reasoning: boolean;
|
|
||||||
/** Accepts a custom temperature (some reasoning models reject it). */
|
|
||||||
temperature: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ChatRequest {
|
|
||||||
model: string;
|
|
||||||
system?: string;
|
|
||||||
messages: ChatMessage[];
|
|
||||||
tools?: ToolDef[];
|
|
||||||
toolChoice?: 'auto' | 'none' | 'required' | { name: string };
|
|
||||||
temperature?: number;
|
|
||||||
maxOutputTokens?: number;
|
|
||||||
reasoningEffort?: ReasoningEffort;
|
|
||||||
/** Ask for a JSON object as the text output when the provider has a JSON mode. */
|
|
||||||
jsonMode?: boolean;
|
|
||||||
/** Known capabilities of the model, when the caller has them. Adapters must not send unsupported params. */
|
|
||||||
capabilities?: Partial<ModelCapabilities>;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface Usage {
|
|
||||||
inputTokens: number;
|
|
||||||
outputTokens: number;
|
|
||||||
cacheReadTokens?: number;
|
|
||||||
cacheWriteTokens?: number;
|
|
||||||
reasoningTokens?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type StopReason = 'end' | 'tool_use' | 'max_tokens' | 'refusal' | 'other';
|
|
||||||
|
|
||||||
export type StreamEvent =
|
|
||||||
| { type: 'text'; delta: string }
|
|
||||||
| { type: 'reasoning'; delta: string }
|
|
||||||
/** Emitted once per tool call, after its arguments are complete and parsed. */
|
|
||||||
| { type: 'tool_call'; call: { id: string; name: string; input: Record<string, unknown> } }
|
|
||||||
/** Always the final event of a successful stream. */
|
|
||||||
| { type: 'done'; message: ChatMessage; stopReason: StopReason; usage: Usage };
|
|
||||||
|
|
||||||
export interface DiscoveredModel {
|
|
||||||
/** Identifier sent to the API. */
|
|
||||||
id: string;
|
|
||||||
label?: string;
|
|
||||||
contextWindow?: number;
|
|
||||||
maxOutput?: number;
|
|
||||||
capabilities?: Partial<ModelCapabilities>;
|
|
||||||
/** USD per million tokens, when known. */
|
|
||||||
priceIn?: number;
|
|
||||||
priceOut?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface TestResult {
|
|
||||||
ok: boolean;
|
|
||||||
message: string;
|
|
||||||
latencyMs: number;
|
|
||||||
/** Number of models visible to the key, when discovery works. */
|
|
||||||
modelCount?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ProviderAdapter {
|
|
||||||
kind: ProviderKind;
|
|
||||||
/** Human label and defaults used by the settings UI. */
|
|
||||||
meta: {
|
|
||||||
label: string;
|
|
||||||
defaultBaseUrl: string | null;
|
|
||||||
requiresApiKey: boolean;
|
|
||||||
requiresBaseUrl: boolean;
|
|
||||||
supportsDiscovery: boolean;
|
|
||||||
docsUrl: string;
|
|
||||||
keyHint: string;
|
|
||||||
};
|
|
||||||
/** Streams one model turn. Throws AIError; never yields after `done`. */
|
|
||||||
stream(cfg: ProviderConfig, req: ChatRequest): AsyncIterable<StreamEvent>;
|
|
||||||
/** Lists chat-capable models. Throws AIError('unsupported') when a provider has no listing API. */
|
|
||||||
listModels(cfg: ProviderConfig, signal?: AbortSignal): Promise<DiscoveredModel[]>;
|
|
||||||
/** Cheap credential / connectivity check. Never throws. */
|
|
||||||
test(cfg: ProviderConfig, model?: string): Promise<TestResult>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type AIErrorCode =
|
|
||||||
| 'auth'
|
|
||||||
| 'rate_limit'
|
|
||||||
| 'quota'
|
|
||||||
| 'context_length'
|
|
||||||
| 'not_found'
|
|
||||||
| 'bad_request'
|
|
||||||
| 'content_filter'
|
|
||||||
| 'server'
|
|
||||||
| 'network'
|
|
||||||
| 'timeout'
|
|
||||||
| 'aborted'
|
|
||||||
| 'unsupported'
|
|
||||||
| 'invalid_response'
|
|
||||||
| 'not_configured';
|
|
||||||
|
|
||||||
export class AIError extends Error {
|
|
||||||
code: AIErrorCode;
|
|
||||||
status?: number;
|
|
||||||
retryable: boolean;
|
|
||||||
/** Raw (already secret-scrubbed) provider message, for the details disclosure in the UI. */
|
|
||||||
providerMessage?: string;
|
|
||||||
|
|
||||||
constructor(code: AIErrorCode, message: string, opts: { status?: number; retryable?: boolean; providerMessage?: string; cause?: unknown } = {}) {
|
|
||||||
super(message, opts.cause !== undefined ? { cause: opts.cause } : undefined);
|
|
||||||
this.name = 'AIError';
|
|
||||||
this.code = code;
|
|
||||||
this.status = opts.status;
|
|
||||||
this.retryable = opts.retryable ?? (code === 'rate_limit' || code === 'server' || code === 'network' || code === 'timeout');
|
|
||||||
this.providerMessage = opts.providerMessage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const DEFAULT_CAPABILITIES: ModelCapabilities = {
|
|
||||||
tools: true,
|
|
||||||
vision: false,
|
|
||||||
pdf: false,
|
|
||||||
reasoning: false,
|
|
||||||
temperature: true,
|
|
||||||
};
|
|
||||||
165
server/app.ts
165
server/app.ts
|
|
@ -1,165 +0,0 @@
|
||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import { Hono } from 'hono';
|
|
||||||
import { compress } from 'hono/compress';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import { AIError } from './ai/types.ts';
|
|
||||||
import { authRoutes, csrfGuard, requireAuth } from './auth.ts';
|
|
||||||
import { HttpError, zodIssues, type AppContext } from './http.ts';
|
|
||||||
import { aiRoutes } from './routes/ai.ts';
|
|
||||||
import { APP_VERSION, dataRoutes } from './routes/data.ts';
|
|
||||||
import { documentRoutes, mediaRoute } from './routes/documents.ts';
|
|
||||||
import { libraryRoutes } from './routes/library.ts';
|
|
||||||
import { publicRoutes, shareRoutes } from './routes/shares.ts';
|
|
||||||
import { studyRoutes } from './routes/study.ts';
|
|
||||||
|
|
||||||
const MIME: Record<string, string> = {
|
|
||||||
'.html': 'text/html; charset=utf-8',
|
|
||||||
'.js': 'text/javascript; charset=utf-8',
|
|
||||||
'.mjs': 'text/javascript; charset=utf-8',
|
|
||||||
'.css': 'text/css; charset=utf-8',
|
|
||||||
'.json': 'application/json; charset=utf-8',
|
|
||||||
'.webmanifest': 'application/manifest+json; charset=utf-8',
|
|
||||||
'.svg': 'image/svg+xml',
|
|
||||||
'.png': 'image/png',
|
|
||||||
'.jpg': 'image/jpeg',
|
|
||||||
'.ico': 'image/x-icon',
|
|
||||||
'.woff2': 'font/woff2',
|
|
||||||
'.woff': 'font/woff',
|
|
||||||
'.txt': 'text/plain; charset=utf-8',
|
|
||||||
'.map': 'application/json; charset=utf-8',
|
|
||||||
};
|
|
||||||
|
|
||||||
const CSP = [
|
|
||||||
"default-src 'self'",
|
|
||||||
"script-src 'self'",
|
|
||||||
"style-src 'self' 'unsafe-inline'",
|
|
||||||
"img-src 'self' data: blob:",
|
|
||||||
"font-src 'self' data:",
|
|
||||||
"connect-src 'self'",
|
|
||||||
"frame-src 'self' blob:",
|
|
||||||
"worker-src 'self' blob:",
|
|
||||||
"object-src 'none'",
|
|
||||||
"base-uri 'self'",
|
|
||||||
"form-action 'self'",
|
|
||||||
"frame-ancestors 'self'",
|
|
||||||
].join('; ');
|
|
||||||
|
|
||||||
const AI_STATUS: Partial<Record<AIError['code'], number>> = { not_configured: 409, bad_request: 400, unsupported: 400, context_length: 413, rate_limit: 429, quota: 402, aborted: 499 };
|
|
||||||
|
|
||||||
export function createApp(app: AppContext): Hono {
|
|
||||||
const hono = new Hono();
|
|
||||||
|
|
||||||
hono.onError((err, c) => {
|
|
||||||
if (err instanceof HttpError) {
|
|
||||||
if (err.status === 429 && typeof (err.details as { retryAfter?: number } | undefined)?.retryAfter === 'number') {
|
|
||||||
c.header('Retry-After', String((err.details as { retryAfter: number }).retryAfter));
|
|
||||||
}
|
|
||||||
return c.json({ error: { code: err.code, message: err.message, details: err.details } }, err.status as never);
|
|
||||||
}
|
|
||||||
if (err instanceof z.ZodError) {
|
|
||||||
const issues = zodIssues(err);
|
|
||||||
const first = issues[0];
|
|
||||||
return c.json({ error: { code: 'validation', message: first ? `${first.path ? `${first.path}: ` : ''}${first.message}` : 'Invalid input', details: issues } }, 422);
|
|
||||||
}
|
|
||||||
if (err instanceof AIError) {
|
|
||||||
return c.json({ error: { code: `ai_${err.code}`, message: err.message, details: err.providerMessage } }, (AI_STATUS[err.code] ?? 502) as never);
|
|
||||||
}
|
|
||||||
// Hono's own HTTP exceptions (body limit etc.)
|
|
||||||
const maybe = err as { status?: number; message?: string; getResponse?: () => Response };
|
|
||||||
if (typeof maybe.status === 'number' && maybe.status >= 400 && maybe.status < 500) {
|
|
||||||
return c.json({ error: { code: 'bad_request', message: maybe.message || 'Bad request' } }, maybe.status as never);
|
|
||||||
}
|
|
||||||
app.log('error', `Unhandled error on ${c.req.method} ${new URL(c.req.url).pathname}`, err);
|
|
||||||
return c.json({ error: { code: 'internal', message: 'Something went wrong on the server.' } }, 500);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Security headers on everything.
|
|
||||||
hono.use('*', async (c, next) => {
|
|
||||||
await next();
|
|
||||||
const h = c.res.headers;
|
|
||||||
if (!h.has('X-Content-Type-Options')) h.set('X-Content-Type-Options', 'nosniff');
|
|
||||||
if (!h.has('Referrer-Policy')) h.set('Referrer-Policy', 'same-origin');
|
|
||||||
h.set('X-Frame-Options', 'SAMEORIGIN');
|
|
||||||
h.set('Permissions-Policy', 'camera=(), microphone=(), geolocation=(), payment=()');
|
|
||||||
if ((h.get('Content-Type') ?? '').startsWith('text/html') && !h.has('Content-Security-Policy')) h.set('Content-Security-Policy', CSP);
|
|
||||||
});
|
|
||||||
|
|
||||||
// gzip for API JSON and static text; event streams must not be buffered.
|
|
||||||
const gzip = compress();
|
|
||||||
hono.use('*', async (c, next) => {
|
|
||||||
const p = c.req.path;
|
|
||||||
if (p === '/api/ai/chat' || p === '/api/ai/explain' || p.startsWith('/api/backup') || p.endsWith('/file') || p.startsWith('/media/')) return next();
|
|
||||||
return gzip(c, next);
|
|
||||||
});
|
|
||||||
|
|
||||||
hono.get('/api/health', (c) => {
|
|
||||||
try {
|
|
||||||
app.db.get('SELECT 1 AS ok');
|
|
||||||
return c.json({ ok: true, version: APP_VERSION });
|
|
||||||
} catch {
|
|
||||||
return c.json({ ok: false }, 503);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
const api = new Hono();
|
|
||||||
api.use('*', async (c, next) => {
|
|
||||||
c.header('Cache-Control', 'no-store');
|
|
||||||
await next();
|
|
||||||
});
|
|
||||||
api.use('*', csrfGuard());
|
|
||||||
api.route('/auth', authRoutes(app));
|
|
||||||
api.route('/public', publicRoutes(app));
|
|
||||||
|
|
||||||
const secured = new Hono();
|
|
||||||
secured.use('*', requireAuth(app));
|
|
||||||
secured.route('/', libraryRoutes(app));
|
|
||||||
secured.route('/', studyRoutes(app));
|
|
||||||
secured.route('/', shareRoutes(app));
|
|
||||||
secured.route('/', documentRoutes(app));
|
|
||||||
secured.route('/', aiRoutes(app));
|
|
||||||
secured.route('/', dataRoutes(app));
|
|
||||||
api.route('/', secured);
|
|
||||||
api.all('*', (c) => c.json({ error: { code: 'not_found', message: 'Unknown API route' } }, 404));
|
|
||||||
hono.route('/api', api);
|
|
||||||
|
|
||||||
hono.route('/media', mediaRoute(app));
|
|
||||||
|
|
||||||
/* ------------------------------ web app ------------------------------ */
|
|
||||||
|
|
||||||
const webDir = app.config.webDir;
|
|
||||||
const sendFile = (file: string, cache: string): Response => {
|
|
||||||
const ext = path.extname(file).toLowerCase();
|
|
||||||
return new Response(fs.readFileSync(file), { headers: { 'Content-Type': MIME[ext] ?? 'application/octet-stream', 'Cache-Control': cache } });
|
|
||||||
};
|
|
||||||
|
|
||||||
hono.get('*', (c) => {
|
|
||||||
if (!webDir) {
|
|
||||||
return c.text('The web app has not been built. Run "npm run build", or use "npm run dev" and open the Vite dev server.', 404);
|
|
||||||
}
|
|
||||||
let urlPath: string;
|
|
||||||
try {
|
|
||||||
urlPath = decodeURIComponent(new URL(c.req.url).pathname);
|
|
||||||
} catch {
|
|
||||||
return c.text('Invalid URL', 400);
|
|
||||||
}
|
|
||||||
const candidate = path.normalize(path.join(webDir, urlPath));
|
|
||||||
const relative = path.relative(webDir, candidate);
|
|
||||||
const inside = relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
|
|
||||||
if (!inside) return c.text('Not found', 404);
|
|
||||||
if (urlPath !== '/' && fs.existsSync(candidate) && fs.statSync(candidate).isFile()) {
|
|
||||||
// Hashed build assets never change; everything else is revalidated.
|
|
||||||
const immutable = urlPath.startsWith('/assets/');
|
|
||||||
return sendFile(candidate, immutable ? 'public, max-age=31536000, immutable' : 'no-cache');
|
|
||||||
}
|
|
||||||
if (path.extname(urlPath) && !urlPath.startsWith('/s/')) return c.text('Not found', 404);
|
|
||||||
const res = sendFile(path.join(webDir, 'index.html'), 'no-cache');
|
|
||||||
if (urlPath.startsWith('/s/')) {
|
|
||||||
res.headers.set('X-Robots-Tag', 'noindex, nofollow');
|
|
||||||
res.headers.set('Referrer-Policy', 'no-referrer');
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
});
|
|
||||||
|
|
||||||
return hono;
|
|
||||||
}
|
|
||||||
277
server/auth.ts
277
server/auth.ts
|
|
@ -1,277 +0,0 @@
|
||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import crypto from 'node:crypto';
|
|
||||||
import { Hono, type Context, type MiddlewareHandler } from 'hono';
|
|
||||||
import { deleteCookie, getCookie, setCookie } from 'hono/cookie';
|
|
||||||
import { getConnInfo } from '@hono/node-server/conninfo';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import { hashPassword, safeEqual, verifyPassword } from './crypto.ts';
|
|
||||||
import { HttpError, readBody, type AppContext } from './http.ts';
|
|
||||||
import { deleteSetting, getSetting, setSetting } from './settings.ts';
|
|
||||||
import { newToken, now, sha256 } from './util.ts';
|
|
||||||
|
|
||||||
const COOKIE = 'sd_session';
|
|
||||||
const SESSION_MS = 30 * 86_400_000;
|
|
||||||
const PASSWORD_MIN = 8;
|
|
||||||
|
|
||||||
const passwordSchema = z.string().min(PASSWORD_MIN, `Use at least ${PASSWORD_MIN} characters`).max(256);
|
|
||||||
|
|
||||||
/* ------------------------------ rate limiting ------------------------------ */
|
|
||||||
|
|
||||||
interface Bucket {
|
|
||||||
failures: number[];
|
|
||||||
}
|
|
||||||
const WINDOW_MS = 15 * 60_000;
|
|
||||||
const MAX_PER_IP = 8;
|
|
||||||
const MAX_GLOBAL = 40;
|
|
||||||
const buckets = new Map<string, Bucket>();
|
|
||||||
|
|
||||||
function recent(key: string, at: number): number[] {
|
|
||||||
const bucket = buckets.get(key) ?? { failures: [] };
|
|
||||||
bucket.failures = bucket.failures.filter((t) => at - t < WINDOW_MS);
|
|
||||||
buckets.set(key, bucket);
|
|
||||||
return bucket.failures;
|
|
||||||
}
|
|
||||||
|
|
||||||
function assertNotLimited(ip: string): void {
|
|
||||||
const at = now();
|
|
||||||
const mine = recent(`ip:${ip}`, at);
|
|
||||||
const all = recent('global', at);
|
|
||||||
if (mine.length >= MAX_PER_IP || all.length >= MAX_GLOBAL) {
|
|
||||||
const oldest = (mine.length >= MAX_PER_IP ? mine : all)[0];
|
|
||||||
const retry = Math.max(1, Math.ceil((oldest + WINDOW_MS - at) / 1000));
|
|
||||||
throw new HttpError(429, 'rate_limited', `Too many attempts. Try again in ${Math.ceil(retry / 60)} minute${retry > 60 ? 's' : ''}.`, { retryAfter: retry });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function recordFailure(ip: string): void {
|
|
||||||
const at = now();
|
|
||||||
recent(`ip:${ip}`, at).push(at);
|
|
||||||
recent('global', at).push(at);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function resetRateLimits(): void {
|
|
||||||
buckets.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ request helpers ------------------------------ */
|
|
||||||
|
|
||||||
export function clientIp(c: Context, trustProxy: boolean): string {
|
|
||||||
if (trustProxy) {
|
|
||||||
const fwd = c.req.header('x-forwarded-for');
|
|
||||||
if (fwd) return fwd.split(',')[0].trim();
|
|
||||||
const real = c.req.header('x-real-ip');
|
|
||||||
if (real) return real.trim();
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return getConnInfo(c).remote.address ?? 'unknown';
|
|
||||||
} catch {
|
|
||||||
return 'unknown';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isHttps(c: Context, trustProxy: boolean): boolean {
|
|
||||||
if (trustProxy) {
|
|
||||||
const proto = c.req.header('x-forwarded-proto');
|
|
||||||
if (proto) return proto.split(',')[0].trim() === 'https';
|
|
||||||
}
|
|
||||||
return new URL(c.req.url).protocol === 'https:';
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ sessions ------------------------------ */
|
|
||||||
|
|
||||||
function createSession(app: AppContext, c: Context): void {
|
|
||||||
const token = newToken();
|
|
||||||
const at = now();
|
|
||||||
app.db.run(
|
|
||||||
'INSERT INTO sessions (token_hash, created_at, expires_at, last_seen_at, user_agent, ip) VALUES (?, ?, ?, ?, ?, ?)',
|
|
||||||
sha256(token),
|
|
||||||
at,
|
|
||||||
at + SESSION_MS,
|
|
||||||
at,
|
|
||||||
(c.req.header('user-agent') ?? '').slice(0, 300),
|
|
||||||
clientIp(c, app.config.trustProxy),
|
|
||||||
);
|
|
||||||
app.db.run('DELETE FROM sessions WHERE expires_at < ?', at);
|
|
||||||
const secure = app.config.cookieSecure === 'auto' ? isHttps(c, app.config.trustProxy) : app.config.cookieSecure;
|
|
||||||
setCookie(c, COOKIE, token, { httpOnly: true, sameSite: 'Lax', secure, path: '/', maxAge: Math.floor(SESSION_MS / 1000) });
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isAuthenticated(app: AppContext, c: Context): boolean {
|
|
||||||
const token = getCookie(c, COOKIE);
|
|
||||||
if (!token) return false;
|
|
||||||
const hash = sha256(token);
|
|
||||||
const at = now();
|
|
||||||
const row = app.db.get<{ expires_at: number; last_seen_at: number }>('SELECT expires_at, last_seen_at FROM sessions WHERE token_hash = ?', hash);
|
|
||||||
if (!row || row.expires_at < at) return false;
|
|
||||||
// Sliding expiry, written at most once an hour.
|
|
||||||
if (at - row.last_seen_at > 3_600_000) {
|
|
||||||
app.db.run('UPDATE sessions SET last_seen_at = ?, expires_at = ? WHERE token_hash = ?', at, at + SESSION_MS, hash);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function requireAuth(app: AppContext): MiddlewareHandler {
|
|
||||||
return async (c, next) => {
|
|
||||||
if (!isAuthenticated(app, c)) throw new HttpError(401, 'unauthorized', 'Sign in to continue');
|
|
||||||
await next();
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* CSRF defence for cookie-authenticated requests: state-changing calls must carry a custom
|
|
||||||
* header (which a cross-site form cannot send, and which forces a CORS preflight that this
|
|
||||||
* server never approves), and must not be flagged cross-site by the browser.
|
|
||||||
*/
|
|
||||||
export function csrfGuard(): MiddlewareHandler {
|
|
||||||
return async (c, next) => {
|
|
||||||
const method = c.req.method;
|
|
||||||
if (method !== 'GET' && method !== 'HEAD' && method !== 'OPTIONS') {
|
|
||||||
if (c.req.header('x-requested-with') !== 'studydesk') throw new HttpError(403, 'csrf', 'Missing request header');
|
|
||||||
// Fetch metadata is set by the browser and cannot be forged by page script. Using it
|
|
||||||
// (rather than comparing Origin to Host) keeps this working behind proxies that rewrite Host.
|
|
||||||
if (c.req.header('sec-fetch-site') === 'cross-site') throw new HttpError(403, 'csrf', 'Cross-site request blocked');
|
|
||||||
}
|
|
||||||
await next();
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ password storage ------------------------------ */
|
|
||||||
|
|
||||||
const HASH_KEY = 'auth.password_hash';
|
|
||||||
const SETUP_KEY = 'auth.setup_code';
|
|
||||||
|
|
||||||
export function isConfigured(app: AppContext): boolean {
|
|
||||||
return Boolean(getSetting<string | null>(app.db, HASH_KEY, null));
|
|
||||||
}
|
|
||||||
|
|
||||||
function setupCodeFile(app: AppContext): string {
|
|
||||||
return path.join(app.config.dataDir, 'setup-code.txt');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function storePassword(app: AppContext, password: string): Promise<void> {
|
|
||||||
setSetting(app.db, HASH_KEY, await hashPassword(password));
|
|
||||||
deleteSetting(app.db, SETUP_KEY);
|
|
||||||
fs.rmSync(setupCodeFile(app), { force: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Runs at startup. Applies ADMIN_PASSWORD / RESET_ADMIN_PASSWORD, or prepares a one-time
|
|
||||||
* setup code (printed to the log and written to the data folder) so that only someone with
|
|
||||||
* access to the server can claim a fresh install.
|
|
||||||
*/
|
|
||||||
export async function initAuth(app: AppContext): Promise<void> {
|
|
||||||
const { config, db } = app;
|
|
||||||
if (config.resetAdminPassword) {
|
|
||||||
passwordSchema.parse(config.resetAdminPassword);
|
|
||||||
await storePassword(app, config.resetAdminPassword);
|
|
||||||
db.run('DELETE FROM sessions');
|
|
||||||
app.log('warn', 'Administrator password was reset from RESET_ADMIN_PASSWORD. Remove that variable and restart.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (isConfigured(app)) return;
|
|
||||||
if (config.adminPassword) {
|
|
||||||
passwordSchema.parse(config.adminPassword);
|
|
||||||
await storePassword(app, config.adminPassword);
|
|
||||||
app.log('info', 'Administrator password initialised from ADMIN_PASSWORD.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let code = getSetting<string | null>(db, SETUP_KEY, null);
|
|
||||||
if (!code) {
|
|
||||||
code = crypto.randomBytes(5).toString('hex').toUpperCase().replace(/(.{5})/, '$1-');
|
|
||||||
setSetting(db, SETUP_KEY, code);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
fs.writeFileSync(setupCodeFile(app), `${code}\n`, { mode: 0o600 });
|
|
||||||
} catch {
|
|
||||||
/* the log line below is enough */
|
|
||||||
}
|
|
||||||
app.log('info', `First-run setup code: ${code} (also saved to ${setupCodeFile(app)})`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ------------------------------ routes ------------------------------ */
|
|
||||||
|
|
||||||
export function authRoutes(app: AppContext): Hono {
|
|
||||||
const r = new Hono();
|
|
||||||
|
|
||||||
r.get('/status', (c) => c.json({ configured: isConfigured(app), authenticated: isAuthenticated(app, c) }));
|
|
||||||
|
|
||||||
r.post('/setup', async (c) => {
|
|
||||||
if (isConfigured(app)) throw new HttpError(409, 'already_configured', 'Study Desk is already set up');
|
|
||||||
const ip = clientIp(c, app.config.trustProxy);
|
|
||||||
assertNotLimited(ip);
|
|
||||||
const body = await readBody(c, z.object({ code: z.string().max(40), password: passwordSchema }));
|
|
||||||
const expected = getSetting<string | null>(app.db, SETUP_KEY, null);
|
|
||||||
const given = body.code.trim().toUpperCase();
|
|
||||||
if (!expected || !(safeEqual(given, expected) || safeEqual(given, expected.replace('-', '')))) {
|
|
||||||
recordFailure(ip);
|
|
||||||
throw new HttpError(403, 'bad_setup_code', 'That setup code is not correct. Check the server log or setup-code.txt in the data folder.');
|
|
||||||
}
|
|
||||||
await storePassword(app, body.password);
|
|
||||||
createSession(app, c);
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/login', async (c) => {
|
|
||||||
const ip = clientIp(c, app.config.trustProxy);
|
|
||||||
assertNotLimited(ip);
|
|
||||||
const body = await readBody(c, z.object({ password: z.string().max(256) }));
|
|
||||||
const hash = getSetting<string | null>(app.db, HASH_KEY, null);
|
|
||||||
if (!hash) throw new HttpError(409, 'not_configured', 'Study Desk has not been set up yet');
|
|
||||||
if (!(await verifyPassword(body.password, hash))) {
|
|
||||||
recordFailure(ip);
|
|
||||||
throw new HttpError(401, 'bad_password', 'Incorrect password');
|
|
||||||
}
|
|
||||||
createSession(app, c);
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/logout', (c) => {
|
|
||||||
const token = getCookie(c, COOKIE);
|
|
||||||
if (token) app.db.run('DELETE FROM sessions WHERE token_hash = ?', sha256(token));
|
|
||||||
deleteCookie(c, COOKIE, { path: '/' });
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/password', requireAuth(app), async (c) => {
|
|
||||||
const ip = clientIp(c, app.config.trustProxy);
|
|
||||||
assertNotLimited(ip);
|
|
||||||
const body = await readBody(c, z.object({ current: z.string().max(256), next: passwordSchema }));
|
|
||||||
const hash = getSetting<string | null>(app.db, HASH_KEY, null);
|
|
||||||
if (!hash || !(await verifyPassword(body.current, hash))) {
|
|
||||||
recordFailure(ip);
|
|
||||||
throw new HttpError(403, 'bad_password', 'Current password is incorrect');
|
|
||||||
}
|
|
||||||
await storePassword(app, body.next);
|
|
||||||
// Sign out every other device.
|
|
||||||
const token = getCookie(c, COOKIE);
|
|
||||||
app.db.run('DELETE FROM sessions WHERE token_hash != ?', token ? sha256(token) : '');
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/sessions', requireAuth(app), (c) => {
|
|
||||||
const current = sha256(getCookie(c, COOKIE) ?? '');
|
|
||||||
const rows = app.db.all<{ token_hash: string; created_at: number; last_seen_at: number; user_agent: string; ip: string }>(
|
|
||||||
'SELECT token_hash, created_at, last_seen_at, user_agent, ip FROM sessions WHERE expires_at > ? ORDER BY last_seen_at DESC',
|
|
||||||
now(),
|
|
||||||
);
|
|
||||||
return c.json({
|
|
||||||
sessions: rows.map((s) => ({
|
|
||||||
id: s.token_hash.slice(0, 12),
|
|
||||||
createdAt: s.created_at,
|
|
||||||
lastSeenAt: s.last_seen_at,
|
|
||||||
userAgent: s.user_agent,
|
|
||||||
ip: s.ip,
|
|
||||||
current: s.token_hash === current,
|
|
||||||
})),
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/sessions/revoke-others', requireAuth(app), (c) => {
|
|
||||||
const current = sha256(getCookie(c, COOKIE) ?? '');
|
|
||||||
const { changes } = app.db.run('DELETE FROM sessions WHERE token_hash != ?', current);
|
|
||||||
return c.json({ revoked: changes });
|
|
||||||
});
|
|
||||||
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
@ -1,94 +0,0 @@
|
||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import crypto from 'node:crypto';
|
|
||||||
|
|
||||||
export interface Config {
|
|
||||||
env: 'development' | 'production' | 'test';
|
|
||||||
host: string;
|
|
||||||
port: number;
|
|
||||||
dataDir: string;
|
|
||||||
dbPath: string;
|
|
||||||
uploadsDir: string;
|
|
||||||
mediaDir: string;
|
|
||||||
backupsDir: string;
|
|
||||||
/** Directory with the built web app, or null when the Vite dev server serves it. */
|
|
||||||
webDir: string | null;
|
|
||||||
/** 32-byte key used to encrypt provider API keys at rest. */
|
|
||||||
secretKey: Buffer;
|
|
||||||
/** 'auto' marks the cookie Secure whenever the request arrived over HTTPS (directly or via proxy). */
|
|
||||||
cookieSecure: 'auto' | boolean;
|
|
||||||
trustProxy: boolean;
|
|
||||||
adminPassword: string | null;
|
|
||||||
resetAdminPassword: string | null;
|
|
||||||
maxUploadBytes: number;
|
|
||||||
backupKeep: number;
|
|
||||||
timezone: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
function bool(v: string | undefined, fallback: boolean): boolean {
|
|
||||||
if (v === undefined || v === '') return fallback;
|
|
||||||
return ['1', 'true', 'yes', 'on'].includes(v.toLowerCase());
|
|
||||||
}
|
|
||||||
|
|
||||||
function loadSecret(dataDir: string, fromEnv: string | undefined): Buffer {
|
|
||||||
if (fromEnv && fromEnv.trim().length < 16) throw new Error('APP_SECRET must contain at least 16 characters.');
|
|
||||||
if (fromEnv) {
|
|
||||||
return crypto.createHash('sha256').update(`studydesk:${fromEnv.trim()}`).digest();
|
|
||||||
}
|
|
||||||
const file = path.join(dataDir, 'secret.key');
|
|
||||||
if (fs.existsSync(file)) {
|
|
||||||
const raw = Buffer.from(fs.readFileSync(file, 'utf8').trim(), 'base64');
|
|
||||||
if (raw.length === 32) return raw;
|
|
||||||
throw new Error('secret.key is damaged. Restore the original key from your backup; refusing to replace it.');
|
|
||||||
}
|
|
||||||
if (fs.existsSync(path.join(dataDir, 'study.db'))) {
|
|
||||||
throw new Error('Existing database has no secret.key. Restore its original key or set the original APP_SECRET; refusing to generate a replacement.');
|
|
||||||
}
|
|
||||||
const key = crypto.randomBytes(32);
|
|
||||||
fs.writeFileSync(file, key.toString('base64'), { mode: 0o600, flag: 'wx' });
|
|
||||||
return key;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function loadConfig(overrides: Partial<Config> = {}, envVars: NodeJS.ProcessEnv = process.env): Config {
|
|
||||||
const env = (overrides.env ?? (envVars.NODE_ENV === 'production' ? 'production' : envVars.NODE_ENV === 'test' ? 'test' : 'development')) as Config['env'];
|
|
||||||
const dataDir = path.resolve(overrides.dataDir ?? envVars.DATA_DIR ?? (env === 'production' ? '/data' : './data'));
|
|
||||||
const uploadsDir = path.join(dataDir, 'uploads');
|
|
||||||
const mediaDir = path.join(dataDir, 'media');
|
|
||||||
const backupsDir = path.join(dataDir, 'backups');
|
|
||||||
for (const dir of [dataDir, uploadsDir, mediaDir, backupsDir]) fs.mkdirSync(dir, { recursive: true });
|
|
||||||
|
|
||||||
const secure = (envVars.COOKIE_SECURE ?? 'auto').toLowerCase();
|
|
||||||
let webDir: string | null = overrides.webDir ?? envVars.WEB_DIR ?? null;
|
|
||||||
if (webDir === null) {
|
|
||||||
const candidate = path.resolve('dist/web');
|
|
||||||
if (fs.existsSync(path.join(candidate, 'index.html'))) webDir = candidate;
|
|
||||||
}
|
|
||||||
|
|
||||||
let timezone = envVars.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
|
|
||||||
try {
|
|
||||||
new Intl.DateTimeFormat('en-US', { timeZone: timezone });
|
|
||||||
} catch {
|
|
||||||
timezone = 'UTC';
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
env,
|
|
||||||
host: envVars.HOST ?? '0.0.0.0',
|
|
||||||
port: Number(envVars.PORT ?? 3726),
|
|
||||||
dataDir,
|
|
||||||
dbPath: path.join(dataDir, 'study.db'),
|
|
||||||
uploadsDir,
|
|
||||||
mediaDir,
|
|
||||||
backupsDir,
|
|
||||||
webDir,
|
|
||||||
secretKey: loadSecret(dataDir, envVars.APP_SECRET),
|
|
||||||
cookieSecure: secure === 'auto' ? 'auto' : bool(secure, false),
|
|
||||||
trustProxy: bool(envVars.TRUST_PROXY, true),
|
|
||||||
adminPassword: envVars.ADMIN_PASSWORD?.trim() || null,
|
|
||||||
resetAdminPassword: envVars.RESET_ADMIN_PASSWORD?.trim() || null,
|
|
||||||
maxUploadBytes: Math.max(1, Number(envVars.MAX_UPLOAD_MB ?? 100)) * 1024 * 1024,
|
|
||||||
backupKeep: Math.max(1, Number(envVars.BACKUP_KEEP ?? 7)),
|
|
||||||
timezone,
|
|
||||||
...overrides,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,62 +0,0 @@
|
||||||
import crypto from 'node:crypto';
|
|
||||||
|
|
||||||
const SCRYPT = { N: 32768, r: 8, p: 1, keylen: 32 };
|
|
||||||
|
|
||||||
function scrypt(password: string, salt: Buffer, N: number, r: number, p: number, keylen: number): Promise<Buffer> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
crypto.scrypt(password.normalize('NFKC'), salt, keylen, { N, r, p, maxmem: 256 * N * r }, (err, key) => (err ? reject(err) : resolve(key)));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** scrypt password hash: `scrypt$N$r$p$salt$hash` (base64url fields). */
|
|
||||||
export async function hashPassword(password: string): Promise<string> {
|
|
||||||
const salt = crypto.randomBytes(16);
|
|
||||||
const key = await scrypt(password, salt, SCRYPT.N, SCRYPT.r, SCRYPT.p, SCRYPT.keylen);
|
|
||||||
return ['scrypt', SCRYPT.N, SCRYPT.r, SCRYPT.p, salt.toString('base64url'), key.toString('base64url')].join('$');
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function verifyPassword(password: string, stored: string): Promise<boolean> {
|
|
||||||
const parts = stored.split('$');
|
|
||||||
if (parts.length !== 6 || parts[0] !== 'scrypt') return false;
|
|
||||||
const [, n, r, p, saltB64, hashB64] = parts;
|
|
||||||
const expected = Buffer.from(hashB64, 'base64url');
|
|
||||||
try {
|
|
||||||
const actual = await scrypt(password, Buffer.from(saltB64, 'base64url'), Number(n), Number(r), Number(p), expected.length);
|
|
||||||
return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** AES-256-GCM. Output: `v1.<iv>.<tag>.<ciphertext>` (base64url). */
|
|
||||||
export function encryptSecret(plain: string, key: Buffer): string {
|
|
||||||
const iv = crypto.randomBytes(12);
|
|
||||||
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
|
|
||||||
const data = Buffer.concat([cipher.update(plain, 'utf8'), cipher.final()]);
|
|
||||||
return ['v1', iv.toString('base64url'), cipher.getAuthTag().toString('base64url'), data.toString('base64url')].join('.');
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Returns null when the value cannot be decrypted (wrong key, tampering). */
|
|
||||||
export function decryptSecret(value: string | null | undefined, key: Buffer): string | null {
|
|
||||||
if (!value) return null;
|
|
||||||
const parts = value.split('.');
|
|
||||||
if (parts.length !== 4 || parts[0] !== 'v1') return null;
|
|
||||||
try {
|
|
||||||
const decipher = crypto.createDecipheriv('aes-256-gcm', key, Buffer.from(parts[1], 'base64url'));
|
|
||||||
decipher.setAuthTag(Buffer.from(parts[2], 'base64url'));
|
|
||||||
return Buffer.concat([decipher.update(Buffer.from(parts[3], 'base64url')), decipher.final()]).toString('utf8');
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function maskSecret(secret: string | null): string {
|
|
||||||
if (!secret) return '';
|
|
||||||
return secret.length <= 8 ? '••••' : `${secret.slice(0, 3)}••••${secret.slice(-4)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function safeEqual(a: string, b: string): boolean {
|
|
||||||
const x = crypto.createHash('sha256').update(a).digest();
|
|
||||||
const y = crypto.createHash('sha256').update(b).digest();
|
|
||||||
return crypto.timingSafeEqual(x, y);
|
|
||||||
}
|
|
||||||
140
server/db.ts
140
server/db.ts
|
|
@ -1,140 +0,0 @@
|
||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import { DatabaseSync, type StatementSync } from 'node:sqlite';
|
|
||||||
import { MIGRATIONS } from './migrations.ts';
|
|
||||||
|
|
||||||
type Param = string | number | bigint | boolean | null | undefined | Uint8Array;
|
|
||||||
|
|
||||||
function bind(params: Param[]): (string | number | bigint | null | Uint8Array)[] {
|
|
||||||
return params.map((p) => (p === undefined ? null : typeof p === 'boolean' ? (p ? 1 : 0) : p));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thin synchronous wrapper around node:sqlite. Single process, single writer:
|
|
||||||
* exactly what a one-user app in one container needs, with no native add-on to build.
|
|
||||||
*/
|
|
||||||
export class DB {
|
|
||||||
readonly raw: DatabaseSync;
|
|
||||||
readonly path: string;
|
|
||||||
private statements = new Map<string, StatementSync>();
|
|
||||||
private txDepth = 0;
|
|
||||||
|
|
||||||
constructor(file: string) {
|
|
||||||
this.path = file;
|
|
||||||
this.raw = new DatabaseSync(file);
|
|
||||||
this.raw.exec('PRAGMA journal_mode = WAL');
|
|
||||||
this.raw.exec('PRAGMA synchronous = NORMAL');
|
|
||||||
this.raw.exec('PRAGMA foreign_keys = ON');
|
|
||||||
this.raw.exec('PRAGMA busy_timeout = 5000');
|
|
||||||
}
|
|
||||||
|
|
||||||
private stmt(sql: string): StatementSync {
|
|
||||||
let s = this.statements.get(sql);
|
|
||||||
if (!s) {
|
|
||||||
s = this.raw.prepare(sql);
|
|
||||||
if (this.statements.size > 500) this.statements.clear();
|
|
||||||
this.statements.set(sql, s);
|
|
||||||
}
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
|
|
||||||
all<T = Record<string, unknown>>(sql: string, ...params: Param[]): T[] {
|
|
||||||
return this.stmt(sql).all(...bind(params)) as T[];
|
|
||||||
}
|
|
||||||
|
|
||||||
get<T = Record<string, unknown>>(sql: string, ...params: Param[]): T | undefined {
|
|
||||||
return this.stmt(sql).get(...bind(params)) as T | undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
run(sql: string, ...params: Param[]): { changes: number; lastId: number } {
|
|
||||||
const r = this.stmt(sql).run(...bind(params));
|
|
||||||
return { changes: Number(r.changes), lastId: Number(r.lastInsertRowid) };
|
|
||||||
}
|
|
||||||
|
|
||||||
exec(sql: string): void {
|
|
||||||
this.raw.exec(sql);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Runs fn atomically. Nested calls become savepoints. */
|
|
||||||
tx<T>(fn: () => T): T {
|
|
||||||
const depth = this.txDepth;
|
|
||||||
const name = `sp_${depth}`;
|
|
||||||
this.raw.exec(depth === 0 ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
|
|
||||||
this.txDepth += 1;
|
|
||||||
try {
|
|
||||||
const out = fn();
|
|
||||||
this.txDepth -= 1;
|
|
||||||
this.raw.exec(depth === 0 ? 'COMMIT' : `RELEASE ${name}`);
|
|
||||||
return out;
|
|
||||||
} catch (e) {
|
|
||||||
this.txDepth -= 1;
|
|
||||||
try {
|
|
||||||
this.raw.exec(depth === 0 ? 'ROLLBACK' : `ROLLBACK TO ${name}; RELEASE ${name}`);
|
|
||||||
} catch {
|
|
||||||
/* the original error matters more */
|
|
||||||
}
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Writes a consistent, compacted copy of the database (safe while the app is running). */
|
|
||||||
snapshot(target: string): void {
|
|
||||||
if (fs.existsSync(target)) fs.rmSync(target);
|
|
||||||
this.raw.exec(`VACUUM INTO '${target.replace(/'/g, "''")}'`);
|
|
||||||
}
|
|
||||||
|
|
||||||
close(): void {
|
|
||||||
this.statements.clear();
|
|
||||||
try {
|
|
||||||
this.raw.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
||||||
} catch {
|
|
||||||
/* ignore */
|
|
||||||
}
|
|
||||||
this.raw.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface MigrationReport {
|
|
||||||
applied: string[];
|
|
||||||
backup: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Applies pending migrations in order, each in its own transaction. When an existing
|
|
||||||
* database is about to change, a snapshot is written to the backups folder first.
|
|
||||||
*/
|
|
||||||
export function migrate(db: DB, backupsDir?: string): MigrationReport {
|
|
||||||
db.exec('CREATE TABLE IF NOT EXISTS schema_migrations (id INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at INTEGER NOT NULL)');
|
|
||||||
const done = new Set(db.all<{ id: number }>('SELECT id FROM schema_migrations').map((r) => r.id));
|
|
||||||
const known = new Set(MIGRATIONS.map((m) => m.id));
|
|
||||||
const unknown = [...done].filter((id) => !known.has(id));
|
|
||||||
if (unknown.length > 0) {
|
|
||||||
throw new Error(
|
|
||||||
`This database was created by a newer version of Study Desk (unknown migration ${unknown.join(', ')}). Upgrade the app instead of downgrading it.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const pending = MIGRATIONS.filter((m) => !done.has(m.id)).sort((a, b) => a.id - b.id);
|
|
||||||
const report: MigrationReport = { applied: [], backup: null };
|
|
||||||
if (pending.length === 0) return report;
|
|
||||||
|
|
||||||
if (done.size > 0 && backupsDir && db.path !== ':memory:') {
|
|
||||||
fs.mkdirSync(backupsDir, { recursive: true });
|
|
||||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
||||||
report.backup = path.join(backupsDir, `pre-migration-${stamp}.db`);
|
|
||||||
db.snapshot(report.backup);
|
|
||||||
}
|
|
||||||
for (const m of pending) {
|
|
||||||
db.tx(() => {
|
|
||||||
db.exec(m.sql);
|
|
||||||
db.run('INSERT INTO schema_migrations (id, name, applied_at) VALUES (?, ?, ?)', m.id, m.name, Date.now());
|
|
||||||
});
|
|
||||||
report.applied.push(`${m.id}_${m.name}`);
|
|
||||||
}
|
|
||||||
return report;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function openDatabase(file: string, backupsDir?: string): { db: DB; report: MigrationReport } {
|
|
||||||
const db = new DB(file);
|
|
||||||
const report = migrate(db, backupsDir);
|
|
||||||
return { db, report };
|
|
||||||
}
|
|
||||||
|
|
@ -1,514 +0,0 @@
|
||||||
/**
|
|
||||||
* Text normalisation, token estimation and retrieval chunking.
|
|
||||||
* Pure functions, no dependencies. Everything here is linear-time on its input.
|
|
||||||
*/
|
|
||||||
import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts';
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Token estimation
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Un-rounded token estimate. Sub-additive under concatenation
|
|
||||||
* (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and
|
|
||||||
* still guarantee the final `estimateTokens(chunk)` stays under the ceiling.
|
|
||||||
*/
|
|
||||||
function rawTokens(text: string): number {
|
|
||||||
let total = 0;
|
|
||||||
let prevWhitespace = false;
|
|
||||||
for (let i = 0; i < text.length; i++) {
|
|
||||||
const c = text.charCodeAt(i);
|
|
||||||
if (c === 32 || (c >= 9 && c <= 13)) {
|
|
||||||
// A run of whitespace costs the same as one character of prose.
|
|
||||||
if (!prevWhitespace) total += 0.25;
|
|
||||||
prevWhitespace = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
prevWhitespace = false;
|
|
||||||
if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25;
|
|
||||||
else if (c >= 48 && c <= 57) total += 0.4;
|
|
||||||
else if (c < 128) total += 0.5;
|
|
||||||
else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character
|
|
||||||
else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each
|
|
||||||
else total += 0.5; // accented Latin, Greek, symbols
|
|
||||||
}
|
|
||||||
return total;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */
|
|
||||||
export function estimateTokens(text: string): number {
|
|
||||||
if (!text) return 0;
|
|
||||||
return Math.ceil(rawTokens(text));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// cleanText
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
const LIGATURES: Record<string, string> = {
|
|
||||||
'ff': 'ff',
|
|
||||||
'fi': 'fi',
|
|
||||||
'fl': 'fl',
|
|
||||||
'ffi': 'ffi',
|
|
||||||
'ffl': 'ffl',
|
|
||||||
'ſt': 'st',
|
|
||||||
'st': 'st',
|
|
||||||
};
|
|
||||||
|
|
||||||
/** First halves that normally form a real hyphenated compound ("well-known"). */
|
|
||||||
const HYPHEN_KEEP_LEFT = new Set([
|
|
||||||
'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first',
|
|
||||||
'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part',
|
|
||||||
'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid',
|
|
||||||
'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed',
|
|
||||||
]);
|
|
||||||
|
|
||||||
/** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */
|
|
||||||
const HYPHEN_KEEP_RIGHT = new Set([
|
|
||||||
'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced',
|
|
||||||
'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk',
|
|
||||||
'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive',
|
|
||||||
'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset',
|
|
||||||
]);
|
|
||||||
|
|
||||||
function isLower(code: number): boolean {
|
|
||||||
return code >= 97 && code <= 122;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isLetter(code: number): boolean {
|
|
||||||
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */
|
|
||||||
function joinHyphenatedLineBreaks(text: string): string {
|
|
||||||
if (!text.includes('-\n')) return text;
|
|
||||||
return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => {
|
|
||||||
// Left fragment: lowercase letters, optionally led by one capital, at most 30 long.
|
|
||||||
let start = offset;
|
|
||||||
while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--;
|
|
||||||
if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--;
|
|
||||||
const left = whole.slice(start, offset);
|
|
||||||
if (left.length < 2) return match;
|
|
||||||
if (start > 0) {
|
|
||||||
const before = whole.charCodeAt(start - 1);
|
|
||||||
// Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it.
|
|
||||||
if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match;
|
|
||||||
}
|
|
||||||
const rightStart = offset + match.length;
|
|
||||||
let rightEnd = rightStart;
|
|
||||||
while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++;
|
|
||||||
const right = whole.slice(rightStart, rightEnd);
|
|
||||||
if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z"
|
|
||||||
if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match;
|
|
||||||
return '';
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Normalises extracted text: newlines, unicode spaces, control characters,
|
|
||||||
* ligatures, trailing spaces, runs of blank lines and words hyphenated across
|
|
||||||
* a line break. Idempotent.
|
|
||||||
*/
|
|
||||||
export function cleanText(text: string): string {
|
|
||||||
if (!text) return '';
|
|
||||||
let t = text;
|
|
||||||
if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n');
|
|
||||||
t = t.replace(/[\u2028\u2029\f\v]/g, '\n');
|
|
||||||
t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' ');
|
|
||||||
// NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM.
|
|
||||||
t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, '');
|
|
||||||
t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch);
|
|
||||||
// Trailing whitespace per line (done by splitting so a huge run of spaces stays linear).
|
|
||||||
t = t
|
|
||||||
.split('\n')
|
|
||||||
.map((line) => line.trimEnd())
|
|
||||||
.join('\n');
|
|
||||||
t = joinHyphenatedLineBreaks(t);
|
|
||||||
t = t.replace(/\n{3,}/g, '\n\n');
|
|
||||||
return t.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Repeated page headers / footers
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
function edgeLineKey(line: string): string | null {
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (!trimmed || trimmed.length > 100) return null;
|
|
||||||
return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Removes running headers/footers and page numbers from per-page text, but only
|
|
||||||
* when the same first or last line (digits ignored) shows up on at least 60% of
|
|
||||||
* the pages that have text, and there are at least four such pages. Chapter
|
|
||||||
* headings that change through a book are therefore left alone.
|
|
||||||
*/
|
|
||||||
export function stripRepeatedPageLines(pages: string[]): string[] {
|
|
||||||
let current = pages.map((page) => page.split('\n'));
|
|
||||||
for (let pass = 0; pass < 2; pass++) {
|
|
||||||
const firstIdx: number[] = [];
|
|
||||||
const lastIdx: number[] = [];
|
|
||||||
const topCounts = new Map<string, number>();
|
|
||||||
const bottomCounts = new Map<string, number>();
|
|
||||||
let textPages = 0;
|
|
||||||
for (let p = 0; p < current.length; p++) {
|
|
||||||
const lines = current[p];
|
|
||||||
let first = 0;
|
|
||||||
while (first < lines.length && !lines[first].trim()) first++;
|
|
||||||
let last = lines.length - 1;
|
|
||||||
while (last >= 0 && !lines[last].trim()) last--;
|
|
||||||
firstIdx.push(first < lines.length ? first : -1);
|
|
||||||
lastIdx.push(last);
|
|
||||||
if (last < 0) continue;
|
|
||||||
textPages++;
|
|
||||||
const top = edgeLineKey(lines[first]);
|
|
||||||
if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1);
|
|
||||||
if (last !== first) {
|
|
||||||
const bottom = edgeLineKey(lines[last]);
|
|
||||||
if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (textPages < 4) break;
|
|
||||||
const threshold = Math.max(3, Math.ceil(textPages * 0.6));
|
|
||||||
let removed = false;
|
|
||||||
current = current.map((lines, p) => {
|
|
||||||
const first = firstIdx[p];
|
|
||||||
const last = lastIdx[p];
|
|
||||||
if (last < 0) return lines;
|
|
||||||
let from = 0;
|
|
||||||
let to = lines.length;
|
|
||||||
const top = edgeLineKey(lines[first]);
|
|
||||||
if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1;
|
|
||||||
if (last !== first) {
|
|
||||||
const bottom = edgeLineKey(lines[last]);
|
|
||||||
if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last;
|
|
||||||
}
|
|
||||||
if (from === 0 && to === lines.length) return lines;
|
|
||||||
removed = true;
|
|
||||||
return lines.slice(from, to);
|
|
||||||
});
|
|
||||||
if (!removed) break;
|
|
||||||
}
|
|
||||||
return current.map((lines) => lines.join('\n'));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Chunking
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */
|
|
||||||
interface Atom {
|
|
||||||
text: string;
|
|
||||||
raw: number;
|
|
||||||
sep: string;
|
|
||||||
sepRaw: number;
|
|
||||||
/** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */
|
|
||||||
strength: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const WHITESPACE_RAW = 0.25;
|
|
||||||
/** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */
|
|
||||||
const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/;
|
|
||||||
const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/;
|
|
||||||
const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/;
|
|
||||||
const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/;
|
|
||||||
|
|
||||||
function sepRawOf(sep: string): number {
|
|
||||||
return sep ? WHITESPACE_RAW : 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */
|
|
||||||
function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void {
|
|
||||||
let start = 0;
|
|
||||||
let acc = 0;
|
|
||||||
let first = true;
|
|
||||||
const push = (end: number) => {
|
|
||||||
const text = word.slice(start, end);
|
|
||||||
out.push({
|
|
||||||
text,
|
|
||||||
raw: rawTokens(text),
|
|
||||||
sep: first ? sep : '',
|
|
||||||
sepRaw: first ? sepRawOf(sep) : 0,
|
|
||||||
strength: first ? strength : -1,
|
|
||||||
});
|
|
||||||
first = false;
|
|
||||||
start = end;
|
|
||||||
acc = 0;
|
|
||||||
};
|
|
||||||
for (let i = 0; i < word.length; i++) {
|
|
||||||
const c = word.charCodeAt(i);
|
|
||||||
const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff;
|
|
||||||
const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost
|
|
||||||
if (acc + weight > limit && i > start && !isLowSurrogate) push(i);
|
|
||||||
acc += weight;
|
|
||||||
}
|
|
||||||
if (start < word.length) push(word.length);
|
|
||||||
}
|
|
||||||
|
|
||||||
function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void {
|
|
||||||
const raw = rawTokens(sentence);
|
|
||||||
if (raw <= atomMax) {
|
|
||||||
out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Oversized sentence (or table row, or blob): fall back to groups of words.
|
|
||||||
const groupMax = Math.max(1, Math.min(atomMax, 24));
|
|
||||||
const words = sentence.split(/[ \t]+/);
|
|
||||||
// Keep leading indentation attached to the first word.
|
|
||||||
const indent = /^[ \t]+/.exec(sentence)?.[0] ?? '';
|
|
||||||
let group = '';
|
|
||||||
let groupRaw = 0;
|
|
||||||
let first = true;
|
|
||||||
const flush = () => {
|
|
||||||
if (!group) return;
|
|
||||||
out.push({
|
|
||||||
text: group,
|
|
||||||
raw: rawTokens(group),
|
|
||||||
sep: first ? sep : ' ',
|
|
||||||
sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW,
|
|
||||||
strength: first ? strength : 0,
|
|
||||||
});
|
|
||||||
first = false;
|
|
||||||
group = '';
|
|
||||||
groupRaw = 0;
|
|
||||||
};
|
|
||||||
let pendingIndent = indent;
|
|
||||||
for (const word of words) {
|
|
||||||
if (!word) continue;
|
|
||||||
const piece = pendingIndent + word;
|
|
||||||
pendingIndent = '';
|
|
||||||
const wordRaw = rawTokens(piece);
|
|
||||||
if (wordRaw > atomMax) {
|
|
||||||
flush();
|
|
||||||
hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0);
|
|
||||||
first = false;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush();
|
|
||||||
group = group ? `${group} ${piece}` : piece;
|
|
||||||
groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0);
|
|
||||||
}
|
|
||||||
flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */
|
|
||||||
function atomize(text: string, atomMax: number): Atom[] {
|
|
||||||
const atoms: Atom[] = [];
|
|
||||||
let prevLine = '';
|
|
||||||
for (const para of text.split(/\n{2,}/)) {
|
|
||||||
let firstInPara = true;
|
|
||||||
for (const rawLine of para.split('\n')) {
|
|
||||||
const line = rawLine.trimEnd();
|
|
||||||
if (!line.trim()) continue;
|
|
||||||
let sep = '';
|
|
||||||
let strength = 0;
|
|
||||||
if (atoms.length > 0) {
|
|
||||||
const afterHeading = HEADING_LINE.test(prevLine);
|
|
||||||
const isHeading = HEADING_LINE.test(line);
|
|
||||||
if (firstInPara) {
|
|
||||||
sep = '\n\n';
|
|
||||||
strength = isHeading ? 4 : afterHeading ? -0.5 : 3;
|
|
||||||
} else {
|
|
||||||
sep = '\n';
|
|
||||||
if (isHeading) strength = 4;
|
|
||||||
else if (afterHeading) strength = -0.5;
|
|
||||||
else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2;
|
|
||||||
else strength = 0.5;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
firstInPara = false;
|
|
||||||
prevLine = line;
|
|
||||||
|
|
||||||
if (rawTokens(line) <= Math.min(atomMax, 12)) {
|
|
||||||
// Short line: not worth looking for sentences.
|
|
||||||
pushSentence(line, atomMax, atoms, sep, strength);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let firstSentence = true;
|
|
||||||
for (const sentence of line.split(SENTENCE_BOUNDARY)) {
|
|
||||||
if (!sentence) continue;
|
|
||||||
pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1);
|
|
||||||
firstSentence = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return atoms;
|
|
||||||
}
|
|
||||||
|
|
||||||
function joinAtoms(atoms: Atom[], from: number, to: number): string {
|
|
||||||
let text = '';
|
|
||||||
for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text;
|
|
||||||
return text;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Trailing whole words of `text` worth at most `budget` tokens. */
|
|
||||||
function tailWords(text: string, budget: number): string {
|
|
||||||
let end = text.length;
|
|
||||||
let cut = text.length;
|
|
||||||
let acc = 0;
|
|
||||||
while (end > 0) {
|
|
||||||
let wordEnd = end;
|
|
||||||
while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--;
|
|
||||||
if (wordEnd === 0) break;
|
|
||||||
const space = text.lastIndexOf(' ', wordEnd - 1);
|
|
||||||
const word = text.slice(space + 1, wordEnd);
|
|
||||||
const cost = rawTokens(word) + WHITESPACE_RAW;
|
|
||||||
if (acc + cost > budget) break;
|
|
||||||
acc += cost;
|
|
||||||
cut = space + 1;
|
|
||||||
end = space < 0 ? 0 : space;
|
|
||||||
}
|
|
||||||
return text.slice(cut).trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Splits one oversized section into balanced pieces of at most `bodyBudget`
|
|
||||||
* tokens, cutting at the strongest boundary near the balanced size, and
|
|
||||||
* prefixing each later piece with up to `overlap` tokens of trailing context.
|
|
||||||
*/
|
|
||||||
function splitSection(text: string, bodyBudget: number, overlap: number): string[] {
|
|
||||||
const atomMax = Math.max(1, bodyBudget / 4);
|
|
||||||
const atoms = atomize(text, atomMax);
|
|
||||||
const n = atoms.length;
|
|
||||||
const pieces: string[] = [];
|
|
||||||
if (n === 0) return pieces;
|
|
||||||
|
|
||||||
// suffix[i] = tokens of atoms i..n-1 including the separators before each of them.
|
|
||||||
const suffix = new Float64Array(n + 1);
|
|
||||||
for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw;
|
|
||||||
|
|
||||||
let start = 0;
|
|
||||||
let prevStart = -1;
|
|
||||||
while (start < n) {
|
|
||||||
const remaining = suffix[start] - atoms[start].sepRaw;
|
|
||||||
let end: number;
|
|
||||||
if (remaining <= bodyBudget) {
|
|
||||||
end = n;
|
|
||||||
} else {
|
|
||||||
const goal = remaining / Math.ceil(remaining / bodyBudget);
|
|
||||||
const minFill = goal * 0.6;
|
|
||||||
let acc = 0;
|
|
||||||
let best = -1;
|
|
||||||
let bestStrength = -Infinity;
|
|
||||||
let bestDistance = Infinity;
|
|
||||||
let i = start;
|
|
||||||
for (; i < n; i++) {
|
|
||||||
const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0);
|
|
||||||
if (i > start && acc + add > bodyBudget) break;
|
|
||||||
acc += add;
|
|
||||||
if (i + 1 < n && acc >= minFill) {
|
|
||||||
const strength = atoms[i + 1].strength;
|
|
||||||
const distance = Math.abs(acc - goal);
|
|
||||||
if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) {
|
|
||||||
best = i;
|
|
||||||
bestStrength = strength;
|
|
||||||
bestDistance = distance;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end = best >= 0 ? best + 1 : Math.max(start + 1, i);
|
|
||||||
}
|
|
||||||
|
|
||||||
let prefix = '';
|
|
||||||
if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) {
|
|
||||||
// Trailing context from the previous piece: whole atoms if they fit, else trailing words.
|
|
||||||
let from = start;
|
|
||||||
let acc = 0;
|
|
||||||
while (from - 1 > prevStart) {
|
|
||||||
const candidate = atoms[from - 1];
|
|
||||||
const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0);
|
|
||||||
if (acc + cost > overlap) break;
|
|
||||||
acc += cost;
|
|
||||||
from--;
|
|
||||||
if (candidate.strength >= 4) break; // do not reach back past a heading
|
|
||||||
}
|
|
||||||
if (from < start) prefix = joinAtoms(atoms, from, start);
|
|
||||||
else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap);
|
|
||||||
}
|
|
||||||
const body = joinAtoms(atoms, start, end);
|
|
||||||
pieces.push(prefix ? prefix + atoms[start].sep + body : body);
|
|
||||||
prevStart = start;
|
|
||||||
start = end;
|
|
||||||
}
|
|
||||||
return pieces;
|
|
||||||
}
|
|
||||||
|
|
||||||
function positiveInt(value: number | undefined, fallback: number, min: number): number {
|
|
||||||
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback;
|
|
||||||
return Math.max(min, Math.floor(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Pending {
|
|
||||||
text: string;
|
|
||||||
heading: string;
|
|
||||||
page: number | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Builds retrieval chunks from the sections of ONE document.
|
|
||||||
*
|
|
||||||
* - Consecutive small sections (short slides, short pages) are packed together
|
|
||||||
* up to `targetTokens`; the chunk keeps the first section's heading and the
|
|
||||||
* full page range.
|
|
||||||
* - A section that fits comfortably under the ceiling stays whole so chunks
|
|
||||||
* line up with pages/slides.
|
|
||||||
* - Larger sections are split into balanced pieces at paragraph, then
|
|
||||||
* sentence, then word boundaries, with `overlapTokens` of repeated context
|
|
||||||
* between consecutive pieces of the same section only.
|
|
||||||
*/
|
|
||||||
export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] {
|
|
||||||
const target = positiveInt(opts.targetTokens, 450, 8);
|
|
||||||
const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8));
|
|
||||||
const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2));
|
|
||||||
/** Sections up to this size are kept in one piece. */
|
|
||||||
const wholeLimit = Math.floor(target + (max - target) / 2);
|
|
||||||
/** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */
|
|
||||||
const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1));
|
|
||||||
|
|
||||||
const chunks: DocChunk[] = [];
|
|
||||||
const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => {
|
|
||||||
const trimmed = text.trim();
|
|
||||||
if (!trimmed) return;
|
|
||||||
chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) });
|
|
||||||
};
|
|
||||||
|
|
||||||
let pending: Pending[] = [];
|
|
||||||
let pendingRaw = 0;
|
|
||||||
const flush = () => {
|
|
||||||
if (pending.length === 0) return;
|
|
||||||
let pageStart: number | null = null;
|
|
||||||
let pageEnd: number | null = null;
|
|
||||||
for (const item of pending) {
|
|
||||||
if (item.page === null) continue;
|
|
||||||
pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page);
|
|
||||||
pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page);
|
|
||||||
}
|
|
||||||
emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd);
|
|
||||||
pending = [];
|
|
||||||
pendingRaw = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
let lastHeading = '';
|
|
||||||
for (const section of sections ?? []) {
|
|
||||||
if (!section || typeof section.text !== 'string') continue;
|
|
||||||
const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : '';
|
|
||||||
if (ownHeading) lastHeading = ownHeading;
|
|
||||||
const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim();
|
|
||||||
if (!text) continue;
|
|
||||||
const heading = lastHeading;
|
|
||||||
const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null;
|
|
||||||
const raw = rawTokens(text);
|
|
||||||
|
|
||||||
if (raw > wholeLimit) {
|
|
||||||
flush();
|
|
||||||
for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush();
|
|
||||||
pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0);
|
|
||||||
pending.push({ text, heading, page });
|
|
||||||
}
|
|
||||||
flush();
|
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
@ -1,420 +0,0 @@
|
||||||
import { strToU8, zipSync } from 'fflate';
|
|
||||||
import { describe, expect, it } from 'vitest';
|
|
||||||
import { chunkSections, cleanText, estimateTokens } from './chunk.ts';
|
|
||||||
import { detectKind, extractDocument } from './extract.ts';
|
|
||||||
import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts';
|
|
||||||
import { ExtractionError, type ExtractedSection } from './types.ts';
|
|
||||||
|
|
||||||
const enc = (s: string) => new TextEncoder().encode(s);
|
|
||||||
const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n');
|
|
||||||
const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]);
|
|
||||||
/** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */
|
|
||||||
const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s;
|
|
||||||
|
|
||||||
async function codeOf(promise: Promise<unknown>): Promise<string> {
|
|
||||||
try {
|
|
||||||
await promise;
|
|
||||||
} catch (e) {
|
|
||||||
expect(e).toBeInstanceOf(ExtractionError);
|
|
||||||
return (e as ExtractionError).code;
|
|
||||||
}
|
|
||||||
throw new Error('expected rejection');
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('detectKind', () => {
|
|
||||||
const pdf = buildPdf([{ lines: ['hello'] }]);
|
|
||||||
const docx = buildDocx([{ type: 'p', text: 'hi' }]);
|
|
||||||
const pptx = buildPptx([{ file: 1, title: 'T' }]);
|
|
||||||
|
|
||||||
it('detects by magic bytes', () => {
|
|
||||||
expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf');
|
|
||||||
expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image');
|
|
||||||
expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image');
|
|
||||||
expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image');
|
|
||||||
expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image');
|
|
||||||
expect(detectKind('a.docx', undefined, docx)).toBe('docx');
|
|
||||||
expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('trusts magic bytes over a wrong extension', () => {
|
|
||||||
expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf');
|
|
||||||
expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx');
|
|
||||||
expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image');
|
|
||||||
expect(detectKind('thing.bin', undefined, docx)).toBe('docx');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('uses the extension for text and markdown', () => {
|
|
||||||
expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text');
|
|
||||||
expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text');
|
|
||||||
expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown');
|
|
||||||
expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown');
|
|
||||||
expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects unknown binary, legacy .doc, plain zips and empty input', () => {
|
|
||||||
expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull();
|
|
||||||
expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull();
|
|
||||||
expect(detectKind('old.doc', undefined, OLE)).toBeNull();
|
|
||||||
expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull();
|
|
||||||
expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('extractDocument: PDF', () => {
|
|
||||||
it('extracts multi-page text with page numbers, count and title', async () => {
|
|
||||||
const bytes = buildPdf(
|
|
||||||
[{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }],
|
|
||||||
{ title: 'Cardiology Notes' },
|
|
||||||
);
|
|
||||||
const r = await extractDocument({ bytes, filename: 'a.pdf' });
|
|
||||||
expect(r.kind).toBe('pdf');
|
|
||||||
expect(r.pageCount).toBe(2);
|
|
||||||
expect(r.title).toBe('Cardiology Notes');
|
|
||||||
expect(r.needsOcr).toBe(false);
|
|
||||||
expect(r.warnings).toEqual([]);
|
|
||||||
expect(r.sections.map((s) => s.page)).toEqual([1, 2]);
|
|
||||||
expect(r.sections[0].text).toContain('Heart failure overview');
|
|
||||||
expect(r.sections[1].text).toContain('Digoxin');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('warns about a page without text but keeps page numbers', async () => {
|
|
||||||
const long = 'Readable text on this page, long enough to avoid OCR. ';
|
|
||||||
const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]);
|
|
||||||
const r = await extractDocument({ bytes, filename: 'a.pdf' });
|
|
||||||
expect(r.pageCount).toBe(3);
|
|
||||||
expect(r.sections.map((s) => s.page)).toEqual([1, 3]);
|
|
||||||
expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('flags an all-empty scanned PDF as needing OCR', async () => {
|
|
||||||
const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' });
|
|
||||||
expect(r.needsOcr).toBe(true);
|
|
||||||
expect(r.sections).toEqual([]);
|
|
||||||
expect(r.pageCount).toBe(2);
|
|
||||||
expect(r.warnings.join(' ')).toMatch(/scanned/);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('reports progress per page', async () => {
|
|
||||||
const seen: Array<[number, number]> = [];
|
|
||||||
await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) });
|
|
||||||
expect(seen).toEqual([
|
|
||||||
[1, 2],
|
|
||||||
[2, 2],
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('extractDocument: DOCX', () => {
|
|
||||||
it('turns headings into sections and keeps lists and tables readable', async () => {
|
|
||||||
const bytes = buildDocx(
|
|
||||||
[
|
|
||||||
{ type: 'heading', level: 1, text: 'Cardiac Drugs' },
|
|
||||||
{ type: 'p', text: 'Intro paragraph about drugs.' },
|
|
||||||
{ type: 'bullet', text: 'Beta blockers' },
|
|
||||||
{ type: 'bullet', text: 'ACE inhibitors' },
|
|
||||||
{ type: 'heading', level: 2, text: 'Dosing' },
|
|
||||||
{ type: 'number', text: 'Check heart rate' },
|
|
||||||
{ type: 'number', text: 'Give the dose' },
|
|
||||||
{
|
|
||||||
type: 'table',
|
|
||||||
rows: [
|
|
||||||
['Drug', 'Dose'],
|
|
||||||
['Digoxin', '0.25 mg'],
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
{ title: 'Drug Guide' },
|
|
||||||
);
|
|
||||||
const r = await extractDocument({ bytes, filename: 'a.docx' });
|
|
||||||
expect(r.kind).toBe('docx');
|
|
||||||
expect(r.title).toBe('Drug Guide');
|
|
||||||
expect(r.needsOcr).toBe(false);
|
|
||||||
expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']);
|
|
||||||
const first = r.sections[0].text;
|
|
||||||
expect(first).toContain('Intro paragraph');
|
|
||||||
expect(first).toContain('Beta blockers');
|
|
||||||
expect(first).toContain('ACE inhibitors');
|
|
||||||
const second = r.sections[1].text;
|
|
||||||
expect(second).toContain('Check heart rate');
|
|
||||||
expect(second).toContain('Digoxin');
|
|
||||||
expect(second).toContain('0.25 mg');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects a docx with no text as empty', async () => {
|
|
||||||
const bytes = buildDocx([{ type: 'p', text: '' }]);
|
|
||||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('extractDocument: PPTX', () => {
|
|
||||||
it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => {
|
|
||||||
const bytes = buildPptx(
|
|
||||||
[
|
|
||||||
{ file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] },
|
|
||||||
{ file: 1, title: 'Middle', textBox: ['Free text box content here'] },
|
|
||||||
{ file: 2 },
|
|
||||||
{ file: 4, title: 'Closing', bullets: ['Thanks'] },
|
|
||||||
],
|
|
||||||
{ title: 'Deck Title' },
|
|
||||||
);
|
|
||||||
const r = await extractDocument({ bytes, filename: 'a.pptx' });
|
|
||||||
expect(r.kind).toBe('pptx');
|
|
||||||
expect(r.title).toBe('Deck Title');
|
|
||||||
expect(r.pageCount).toBe(4);
|
|
||||||
expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]);
|
|
||||||
expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']);
|
|
||||||
expect(r.sections[0].text).toContain('Welcome');
|
|
||||||
expect(r.sections[0].text).toContain('Sub point');
|
|
||||||
expect(r.sections[0].text).toContain('Say hello slowly');
|
|
||||||
expect(r.sections[1].text).toContain('Free text box content here');
|
|
||||||
expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored
|
|
||||||
expect(r.warnings).toEqual([]); // one empty slide is not worth a warning
|
|
||||||
});
|
|
||||||
|
|
||||||
it('reads tables', async () => {
|
|
||||||
const table = [
|
|
||||||
['A', 'B'],
|
|
||||||
['x', 'y'],
|
|
||||||
];
|
|
||||||
const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' });
|
|
||||||
expect(allText(r)).toContain('x');
|
|
||||||
expect(allText(r)).toContain('B');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('warns when most slides have no text', async () => {
|
|
||||||
const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' });
|
|
||||||
expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('extractDocument: text, markdown, image', () => {
|
|
||||||
it('splits markdown by heading and ignores headings inside code fences', async () => {
|
|
||||||
const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.';
|
|
||||||
const r = await extractDocument({ bytes: enc(md), filename: 'a.md' });
|
|
||||||
expect(r.kind).toBe('markdown');
|
|
||||||
expect(r.title).toBe('Title');
|
|
||||||
expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']);
|
|
||||||
expect(r.sections[1].text).toContain('# not a heading');
|
|
||||||
expect(r.needsOcr).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('extracts plain text and falls back for non-UTF-8 with a warning', async () => {
|
|
||||||
const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' });
|
|
||||||
expect(r.kind).toBe('text');
|
|
||||||
expect(allText(r)).toContain('line two');
|
|
||||||
const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute
|
|
||||||
const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' });
|
|
||||||
expect(allText(r2)).toBe('café');
|
|
||||||
expect(r2.warnings.length).toBe(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('groups long plain text into several bounded sections', async () => {
|
|
||||||
const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n');
|
|
||||||
const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' });
|
|
||||||
expect(r.sections.length).toBeGreaterThan(1);
|
|
||||||
for (const s of r.sections) expect(s.text.length).toBeLessThan(7000);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns images with needsOcr and no sections', async () => {
|
|
||||||
const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' });
|
|
||||||
expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('extractDocument: errors', () => {
|
|
||||||
it('rejects an empty file', async () => {
|
|
||||||
expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects whitespace-only text as empty', async () => {
|
|
||||||
expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects unsupported types and legacy .doc', async () => {
|
|
||||||
expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported');
|
|
||||||
await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/);
|
|
||||||
expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects a password-protected OOXML wrapper as encrypted', async () => {
|
|
||||||
const marker = Buffer.from('EncryptedPackage', 'utf16le');
|
|
||||||
const bytes = Uint8Array.from([...OLE, ...marker]);
|
|
||||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects corrupt PDFs', async () => {
|
|
||||||
expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects an encrypted PDF', async () => {
|
|
||||||
const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true });
|
|
||||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects a corrupt zip posing as docx/pptx', async () => {
|
|
||||||
const good = buildDocx([{ type: 'p', text: 'hello world' }]);
|
|
||||||
const broken = good.slice(0, Math.floor(good.length / 2));
|
|
||||||
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt');
|
|
||||||
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects a pptx zip that has no readable slides', async () => {
|
|
||||||
const bytes = zipSync({ 'ppt/presentation.xml': strToU8('<p:presentation/>') });
|
|
||||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rejects a zip bomb whose headers lie about entry size', async () => {
|
|
||||||
const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024);
|
|
||||||
const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' }));
|
|
||||||
expect(['too_large', 'corrupt']).toContain(code);
|
|
||||||
}, 30000);
|
|
||||||
|
|
||||||
it('honours an already-aborted signal', async () => {
|
|
||||||
const ac = new AbortController();
|
|
||||||
ac.abort();
|
|
||||||
expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('chunkSections', () => {
|
|
||||||
const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
|
|
||||||
const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 };
|
|
||||||
|
|
||||||
it('keeps every chunk within maxTokens with contiguous 0-based seq', () => {
|
|
||||||
const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({
|
|
||||||
text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '),
|
|
||||||
page: i + 1,
|
|
||||||
}));
|
|
||||||
const chunks = chunkSections(sections, opts);
|
|
||||||
expect(chunks.length).toBeGreaterThan(12);
|
|
||||||
chunks.forEach((c, i) => {
|
|
||||||
expect(c.seq).toBe(i);
|
|
||||||
expect(c.tokens).toBe(estimateTokens(c.text));
|
|
||||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('packs small sections together with correct page range and first heading', () => {
|
|
||||||
const sections: ExtractedSection[] = [
|
|
||||||
{ text: 'Alpha text.', page: 1, heading: 'First' },
|
|
||||||
{ text: 'Beta text.', page: 2 },
|
|
||||||
{ text: 'Gamma text.', page: 3, heading: 'Third' },
|
|
||||||
];
|
|
||||||
const chunks = chunkSections(sections, opts);
|
|
||||||
expect(chunks).toHaveLength(1);
|
|
||||||
expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 });
|
|
||||||
expect(chunks[0].text).toContain('Gamma');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('starts a new chunk when packing would exceed the target and tracks ranges', () => {
|
|
||||||
const big = (i: number) => ({ text: words(60, `s${i}x`), page: i });
|
|
||||||
const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 });
|
|
||||||
expect(chunks.length).toBeGreaterThan(1);
|
|
||||||
for (const c of chunks) {
|
|
||||||
expect(c.pageStart).not.toBeNull();
|
|
||||||
expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!);
|
|
||||||
}
|
|
||||||
expect(chunks[0].pageStart).toBe(1);
|
|
||||||
expect(chunks[chunks.length - 1].pageEnd).toBe(4);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('uses null pages and empty heading when sections have none', () => {
|
|
||||||
const [c] = chunkSections([{ text: 'No pages here.' }]);
|
|
||||||
expect(c.pageStart).toBeNull();
|
|
||||||
expect(c.pageEnd).toBeNull();
|
|
||||||
expect(c.heading).toBe('');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('splits oversized sections without cutting words', () => {
|
|
||||||
const text = words(800, 'term');
|
|
||||||
const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts);
|
|
||||||
expect(chunks.length).toBeGreaterThan(3);
|
|
||||||
const vocabulary = new Set(text.split(' '));
|
|
||||||
for (const c of chunks) {
|
|
||||||
expect(c.pageStart).toBe(5);
|
|
||||||
expect(c.pageEnd).toBe(5);
|
|
||||||
expect(c.heading).toBe('Big');
|
|
||||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
|
||||||
for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true);
|
|
||||||
}
|
|
||||||
const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/)));
|
|
||||||
expect(seen.size).toBe(800);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('survives a giant unbroken string within the limit', () => {
|
|
||||||
const blob = 'x'.repeat(200_000);
|
|
||||||
const t0 = Date.now();
|
|
||||||
const chunks = chunkSections([{ text: blob }], opts);
|
|
||||||
expect(Date.now() - t0).toBeLessThan(3000);
|
|
||||||
expect(chunks.length).toBeGreaterThan(1);
|
|
||||||
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150);
|
|
||||||
expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('does not split surrogate pairs in unbroken emoji runs', () => {
|
|
||||||
const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts);
|
|
||||||
expect(chunks.length).toBeGreaterThan(1);
|
|
||||||
for (const c of chunks) {
|
|
||||||
expect(wellFormed(c.text)).toBe(true);
|
|
||||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it('is deterministic', () => {
|
|
||||||
const sections = [
|
|
||||||
{ text: words(900), page: 1, heading: 'H' },
|
|
||||||
{ text: 'Short one.', page: 2 },
|
|
||||||
];
|
|
||||||
expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts));
|
|
||||||
});
|
|
||||||
|
|
||||||
it('drops empty and whitespace-only sections', () => {
|
|
||||||
expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]);
|
|
||||||
const chunks = chunkSections([
|
|
||||||
{ text: ' ', page: 1 },
|
|
||||||
{ text: 'Real content.', page: 2 },
|
|
||||||
{ text: '\n', page: 3 },
|
|
||||||
]);
|
|
||||||
expect(chunks).toHaveLength(1);
|
|
||||||
expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 });
|
|
||||||
});
|
|
||||||
|
|
||||||
it('chunks about 2 MB of text quickly', () => {
|
|
||||||
const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n';
|
|
||||||
const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length));
|
|
||||||
const t0 = Date.now();
|
|
||||||
const chunks = chunkSections([{ text }]);
|
|
||||||
expect(Date.now() - t0).toBeLessThan(3000);
|
|
||||||
expect(chunks.length).toBeGreaterThan(100);
|
|
||||||
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('cleanText', () => {
|
|
||||||
it('joins words hyphenated across a line break', () => {
|
|
||||||
expect(cleanText('the medica-\ntion was given')).toBe('the medication was given');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('keeps real compounds across a line break', () => {
|
|
||||||
expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('collapses 3+ newlines to a paragraph break', () => {
|
|
||||||
expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb');
|
|
||||||
expect(cleanText('a\n\nb')).toBe('a\n\nb');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('strips control characters but keeps tabs and newlines', () => {
|
|
||||||
expect(cleanText('he\u0000l\u0007lo world\ttab\nnext')).toBe('hello world\ttab\nnext');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('normalises CRLF and trims', () => {
|
|
||||||
expect(cleanText(' a\r\nb \r\n')).toBe('a\nb');
|
|
||||||
expect(cleanText('')).toBe('');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('is idempotent', () => {
|
|
||||||
const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny z');
|
|
||||||
expect(cleanText(once)).toBe(once);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,105 +0,0 @@
|
||||||
/** DOCX extraction through mammoth: HTML -> Markdown-ish text, split into sections at H1-H3. */
|
|
||||||
import { zipSync } from 'fflate';
|
|
||||||
import mammoth from 'mammoth';
|
|
||||||
import { cleanText } from './chunk.ts';
|
|
||||||
import { htmlToBlocks } from './markup.ts';
|
|
||||||
import { readCoreTitle, type OfficeContext } from './pptx.ts';
|
|
||||||
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
||||||
import { ZipReader } from './zip.ts';
|
|
||||||
|
|
||||||
const STYLE_MAP = ["p[style-name='Title'] => h1:fresh", "p[style-name='Subtitle'] => p:fresh"];
|
|
||||||
const MAX_WARNINGS = 3;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Re-packs only the XML parts mammoth reads (no media, fonts or embeddings)
|
|
||||||
* into a fresh stored zip. Every byte went through the size-limited reader, so
|
|
||||||
* mammoth's own unzip can never be handed a bomb, and it never spends time
|
|
||||||
* base64-encoding images we would throw away.
|
|
||||||
*/
|
|
||||||
function sanitisedPackage(zip: ZipReader): { data: Uint8Array; images: number } {
|
|
||||||
const files: Record<string, Uint8Array> = {};
|
|
||||||
let images = 0;
|
|
||||||
for (const entry of zip.entries) {
|
|
||||||
const name = entry.name;
|
|
||||||
if (name.endsWith('/')) continue;
|
|
||||||
if (/^word\/media\//i.test(name)) {
|
|
||||||
images++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!/\.(xml|rels)$/i.test(name)) continue;
|
|
||||||
if (/^(customXml|docProps\/thumbnail)/i.test(name)) continue;
|
|
||||||
const data = zip.read(name);
|
|
||||||
if (data) files[name] = data;
|
|
||||||
}
|
|
||||||
return { data: zipSync(files, { level: 0 }), images };
|
|
||||||
}
|
|
||||||
|
|
||||||
function friendlyWarnings(messages: Array<{ type: string; message: string }>): string[] {
|
|
||||||
const seen = new Set<string>();
|
|
||||||
for (const item of messages) {
|
|
||||||
const message = String(item?.message ?? '').trim();
|
|
||||||
// Unknown style names never lose text; they are noise for a reader.
|
|
||||||
if (!message || /^Unrecognised (paragraph|run|table) style/i.test(message)) continue;
|
|
||||||
seen.add(message.length > 160 ? `${message.slice(0, 157)}...` : message);
|
|
||||||
}
|
|
||||||
const unique = [...seen];
|
|
||||||
const out = unique.slice(0, MAX_WARNINGS).map((message) => `Some content may be missing: ${message}`);
|
|
||||||
if (unique.length > MAX_WARNINGS) out.push(`...and ${unique.length - MAX_WARNINGS} more conversion notes.`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function extractDocx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
|
|
||||||
const zip = new ZipReader(bytes, ctx.limits);
|
|
||||||
if (!zip.names().some((name) => /^word\/document\d*\.xml$/i.test(name))) {
|
|
||||||
throw new ExtractionError('corrupt', "This file doesn't look like a valid Word document. Try saving it again as .docx and re-uploading.");
|
|
||||||
}
|
|
||||||
const { data, images } = sanitisedPackage(zip);
|
|
||||||
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
|
|
||||||
|
|
||||||
const converted = await mammoth.convertToHtml(
|
|
||||||
{ buffer: Buffer.from(data.buffer, data.byteOffset, data.byteLength) },
|
|
||||||
{
|
|
||||||
styleMap: STYLE_MAP,
|
|
||||||
ignoreEmptyParagraphs: true,
|
|
||||||
// Images are not read locally; never decode them.
|
|
||||||
convertImage: mammoth.images.imgElement(async () => ({ src: '' })),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
const sections: ExtractedSection[] = [];
|
|
||||||
let heading: string | undefined;
|
|
||||||
let parts: string[] = [];
|
|
||||||
const flush = () => {
|
|
||||||
const text = cleanText(parts.join('\n\n'));
|
|
||||||
parts = [];
|
|
||||||
if (!text) return;
|
|
||||||
const section: ExtractedSection = { text };
|
|
||||||
if (heading) section.heading = heading;
|
|
||||||
sections.push(section);
|
|
||||||
};
|
|
||||||
for (const block of htmlToBlocks(converted.value)) {
|
|
||||||
if (block.level >= 1 && block.level <= 3) {
|
|
||||||
flush();
|
|
||||||
heading = block.text.slice(0, 200);
|
|
||||||
}
|
|
||||||
parts.push(block.level > 0 ? `${'#'.repeat(block.level)} ${block.text}` : block.text);
|
|
||||||
}
|
|
||||||
flush();
|
|
||||||
ctx.onProgress?.(1, 1);
|
|
||||||
|
|
||||||
const warnings = friendlyWarnings(converted.messages ?? []);
|
|
||||||
const totalChars = sections.reduce((sum, section) => sum + section.text.length, 0);
|
|
||||||
let needsOcr = false;
|
|
||||||
if (totalChars < 25) {
|
|
||||||
if (images === 0) {
|
|
||||||
throw new ExtractionError('empty', "This Word document doesn't contain any text.");
|
|
||||||
}
|
|
||||||
needsOcr = true;
|
|
||||||
warnings.unshift(`This document has almost no text, only ${images} image${images === 1 ? '' : 's'} - pictures of text are not read.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const result: ExtractResult = { kind: 'docx', sections, warnings, needsOcr };
|
|
||||||
const title = readCoreTitle(zip);
|
|
||||||
if (title) result.title = title;
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
@ -1,362 +0,0 @@
|
||||||
/**
|
|
||||||
* Document text extraction: kind detection plus one extractor per format.
|
|
||||||
* Every failure surfaces as an ExtractionError with a user-facing message.
|
|
||||||
*/
|
|
||||||
import { cleanText } from './chunk.ts';
|
|
||||||
import { extractDocx } from './docx.ts';
|
|
||||||
import { extractPdf } from './pdf.ts';
|
|
||||||
import { extractPptx } from './pptx.ts';
|
|
||||||
import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
||||||
import { listZip, ZipError, type ZipLimits } from './zip.ts';
|
|
||||||
|
|
||||||
/** Largest upload accepted (100 MB). */
|
|
||||||
export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024;
|
|
||||||
|
|
||||||
/** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */
|
|
||||||
const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 };
|
|
||||||
|
|
||||||
export interface ExtractInput {
|
|
||||||
bytes: Uint8Array;
|
|
||||||
filename: string;
|
|
||||||
mime?: string;
|
|
||||||
/** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */
|
|
||||||
onProgress?: (done: number, total: number) => void;
|
|
||||||
/** Aborting rejects with ExtractionError('failed') at the next page boundary. */
|
|
||||||
signal?: AbortSignal;
|
|
||||||
}
|
|
||||||
|
|
||||||
const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']);
|
|
||||||
const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']);
|
|
||||||
const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']);
|
|
||||||
const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']);
|
|
||||||
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
|
||||||
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation';
|
|
||||||
const SNIFF_BYTES = 256 * 1024;
|
|
||||||
|
|
||||||
function extensionOf(filename: string): string {
|
|
||||||
const base = (filename ?? '').split(/[\\/]/).pop() ?? '';
|
|
||||||
const dot = base.lastIndexOf('.');
|
|
||||||
return dot > 0 ? base.slice(dot + 1).toLowerCase() : '';
|
|
||||||
}
|
|
||||||
|
|
||||||
function baseMime(mime: string | undefined): string {
|
|
||||||
return (mime ?? '').split(';')[0].trim().toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean {
|
|
||||||
if (bytes.length < offset + signature.length) return false;
|
|
||||||
for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isImage(bytes: Uint8Array): boolean {
|
|
||||||
return (
|
|
||||||
startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG
|
|
||||||
startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG
|
|
||||||
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a
|
|
||||||
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a
|
|
||||||
(startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */
|
|
||||||
function hasPdfHeader(bytes: Uint8Array): boolean {
|
|
||||||
const limit = Math.min(bytes.length - 5, 1024);
|
|
||||||
for (let i = 0; i <= limit; i++) {
|
|
||||||
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isZip(bytes: Uint8Array): boolean {
|
|
||||||
return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */
|
|
||||||
function isOle(bytes: Uint8Array): boolean {
|
|
||||||
return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isUtf16Bom(bytes: Uint8Array): boolean {
|
|
||||||
return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff));
|
|
||||||
}
|
|
||||||
|
|
||||||
function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean {
|
|
||||||
try {
|
|
||||||
new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail });
|
|
||||||
return true;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */
|
|
||||||
function looksLikeSingleByteText(sample: Uint8Array): boolean {
|
|
||||||
let high = 0;
|
|
||||||
for (let i = 0; i < sample.length; i++) {
|
|
||||||
const b = sample[i];
|
|
||||||
if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false;
|
|
||||||
if (b === 0x7f) return false;
|
|
||||||
if (b >= 0x80) high++;
|
|
||||||
}
|
|
||||||
return high <= sample.length * 0.3;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isProbablyText(bytes: Uint8Array): boolean {
|
|
||||||
if (isUtf16Bom(bytes)) return bytes.length % 2 === 0;
|
|
||||||
const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes;
|
|
||||||
if (sample.includes(0)) return false;
|
|
||||||
return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Works out what a file is. Magic bytes win over the name; the extension and
|
|
||||||
* MIME type are only a fallback. Returns null for anything unsupported
|
|
||||||
* (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk).
|
|
||||||
*/
|
|
||||||
export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null {
|
|
||||||
const ext = extensionOf(filename);
|
|
||||||
const type = baseMime(mime);
|
|
||||||
if (!bytes || bytes.length === 0) return null;
|
|
||||||
|
|
||||||
if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf';
|
|
||||||
if (isImage(bytes)) return 'image';
|
|
||||||
|
|
||||||
if (isZip(bytes)) {
|
|
||||||
let names: string[] | null = null;
|
|
||||||
try {
|
|
||||||
names = listZip(bytes).map((entry) => entry.name);
|
|
||||||
} catch {
|
|
||||||
names = null;
|
|
||||||
}
|
|
||||||
if (names) {
|
|
||||||
if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx';
|
|
||||||
if (names.includes('ppt/presentation.xml')) return 'pptx';
|
|
||||||
return null; // xlsx, odt, epub, jar, plain zip...
|
|
||||||
}
|
|
||||||
// Unreadable archive: trust the name so extraction can report it as damaged.
|
|
||||||
if (ext === 'docx' || type === DOCX_MIME) return 'docx';
|
|
||||||
if (ext === 'pptx' || type === PPTX_MIME) return 'pptx';
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isOle(bytes)) {
|
|
||||||
// Password-protected .docx/.pptx are OLE containers; extraction explains that.
|
|
||||||
if (ext === 'docx') return 'docx';
|
|
||||||
if (ext === 'pptx') return 'pptx';
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// No recognisable signature: fall back to the name / declared type.
|
|
||||||
if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf';
|
|
||||||
if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx';
|
|
||||||
if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx';
|
|
||||||
if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null;
|
|
||||||
if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Text and Markdown
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
function decodeText(bytes: Uint8Array, warnings: string[]): string {
|
|
||||||
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
|
|
||||||
return new TextDecoder('utf-8').decode(bytes.subarray(3));
|
|
||||||
}
|
|
||||||
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2));
|
|
||||||
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2));
|
|
||||||
try {
|
|
||||||
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
||||||
} catch {
|
|
||||||
if (bytes.includes(0)) {
|
|
||||||
throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.");
|
|
||||||
}
|
|
||||||
warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.');
|
|
||||||
return new TextDecoder('windows-1252').decode(bytes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Target size of a plain-text section, in characters. */
|
|
||||||
const TEXT_SECTION_CHARS = 6000;
|
|
||||||
|
|
||||||
function splitPlainText(text: string): ExtractedSection[] {
|
|
||||||
const sections: ExtractedSection[] = [];
|
|
||||||
let current: string[] = [];
|
|
||||||
let size = 0;
|
|
||||||
const flush = (separator: string) => {
|
|
||||||
if (current.length === 0) return;
|
|
||||||
sections.push({ text: current.join(separator) });
|
|
||||||
current = [];
|
|
||||||
size = 0;
|
|
||||||
};
|
|
||||||
for (const block of text.split(/\n{2,}/)) {
|
|
||||||
if (!block.trim()) continue;
|
|
||||||
if (block.length > TEXT_SECTION_CHARS) {
|
|
||||||
// One huge block (CSV, logs, text without blank lines): fall back to line boundaries.
|
|
||||||
flush('\n\n');
|
|
||||||
for (const line of block.split('\n')) {
|
|
||||||
if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n');
|
|
||||||
current.push(line);
|
|
||||||
size += line.length + 1;
|
|
||||||
}
|
|
||||||
flush('\n');
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n');
|
|
||||||
current.push(block);
|
|
||||||
size += block.length + 2;
|
|
||||||
}
|
|
||||||
flush('\n\n');
|
|
||||||
return sections;
|
|
||||||
}
|
|
||||||
|
|
||||||
function markdownHeadingText(raw: string): string {
|
|
||||||
return raw
|
|
||||||
.replace(/[ \t]+#+[ \t]*$/, '')
|
|
||||||
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
|
|
||||||
.replace(/\*\*|__|[*`]/g, '')
|
|
||||||
.replace(/\s+/g, ' ')
|
|
||||||
.trim()
|
|
||||||
.slice(0, 200);
|
|
||||||
}
|
|
||||||
|
|
||||||
function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } {
|
|
||||||
const sections: ExtractedSection[] = [];
|
|
||||||
let title: string | undefined;
|
|
||||||
let heading: string | undefined;
|
|
||||||
let lines: string[] = [];
|
|
||||||
let fence: string | null = null;
|
|
||||||
const flush = () => {
|
|
||||||
const body = lines.join('\n').trim();
|
|
||||||
lines = [];
|
|
||||||
if (!body) return;
|
|
||||||
const section: ExtractedSection = { text: body };
|
|
||||||
if (heading) section.heading = heading;
|
|
||||||
sections.push(section);
|
|
||||||
};
|
|
||||||
for (const line of text.split('\n')) {
|
|
||||||
const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null;
|
|
||||||
if (fenceMatch) {
|
|
||||||
const marker = fenceMatch[1][0];
|
|
||||||
if (fence === null) fence = marker;
|
|
||||||
else if (fence === marker) fence = null;
|
|
||||||
} else if (fence === null && line.length < 400) {
|
|
||||||
const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line);
|
|
||||||
if (match) {
|
|
||||||
const headingText = markdownHeadingText(match[2]);
|
|
||||||
if (headingText) {
|
|
||||||
flush();
|
|
||||||
heading = headingText;
|
|
||||||
if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
lines.push(line);
|
|
||||||
}
|
|
||||||
flush();
|
|
||||||
return { sections, title };
|
|
||||||
}
|
|
||||||
|
|
||||||
function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult {
|
|
||||||
const warnings: string[] = [];
|
|
||||||
const text = cleanText(decodeText(bytes, warnings));
|
|
||||||
if (!text) throw new ExtractionError('empty', "This file doesn't contain any text.");
|
|
||||||
if (kind === 'markdown') {
|
|
||||||
const { sections, title } = splitMarkdown(text);
|
|
||||||
const result: ExtractResult = { kind, sections, warnings, needsOcr: false };
|
|
||||||
if (title) result.title = title;
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
return { kind, sections: splitPlainText(text), warnings, needsOcr: false };
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Entry point
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le');
|
|
||||||
|
|
||||||
function unsupportedMessage(filename: string, bytes: Uint8Array): string {
|
|
||||||
const ext = extensionOf(filename);
|
|
||||||
if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead.";
|
|
||||||
if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead.";
|
|
||||||
if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead.";
|
|
||||||
if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') {
|
|
||||||
return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`;
|
|
||||||
}
|
|
||||||
if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) {
|
|
||||||
return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.";
|
|
||||||
}
|
|
||||||
if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead.";
|
|
||||||
return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file.";
|
|
||||||
}
|
|
||||||
|
|
||||||
function fromZipError(error: ZipError, kind: DocKind): ExtractionError {
|
|
||||||
const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document';
|
|
||||||
if (error.code === 'too_large') {
|
|
||||||
return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error });
|
|
||||||
}
|
|
||||||
return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extracts the text of a document as ordered sections. Images return no
|
|
||||||
* sections and `needsOcr: true`; the caller runs vision transcription.
|
|
||||||
*/
|
|
||||||
export async function extractDocument(input: ExtractInput): Promise<ExtractResult> {
|
|
||||||
const bytes = input?.bytes;
|
|
||||||
const filename = typeof input?.filename === 'string' ? input.filename : '';
|
|
||||||
if (!(bytes instanceof Uint8Array) || bytes.length === 0) {
|
|
||||||
throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.');
|
|
||||||
}
|
|
||||||
if (bytes.length > MAX_DOCUMENT_BYTES) {
|
|
||||||
throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`);
|
|
||||||
}
|
|
||||||
if (input.signal?.aborted) {
|
|
||||||
throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason });
|
|
||||||
}
|
|
||||||
|
|
||||||
let kind: DocKind | null;
|
|
||||||
try {
|
|
||||||
kind = detectKind(filename, input.mime, bytes);
|
|
||||||
} catch (error) {
|
|
||||||
throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error });
|
|
||||||
}
|
|
||||||
if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes));
|
|
||||||
|
|
||||||
try {
|
|
||||||
switch (kind) {
|
|
||||||
case 'pdf':
|
|
||||||
return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal });
|
|
||||||
case 'docx':
|
|
||||||
case 'pptx': {
|
|
||||||
if (isOle(bytes)) {
|
|
||||||
const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
||||||
if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) {
|
|
||||||
throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`);
|
|
||||||
}
|
|
||||||
throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes));
|
|
||||||
}
|
|
||||||
const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal };
|
|
||||||
return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx);
|
|
||||||
}
|
|
||||||
case 'text':
|
|
||||||
case 'markdown': {
|
|
||||||
const result = extractText(bytes, kind);
|
|
||||||
input.onProgress?.(1, 1);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
case 'image':
|
|
||||||
return { kind: 'image', sections: [], needsOcr: true, warnings: [] };
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof ExtractionError) throw error;
|
|
||||||
if (error instanceof ZipError) throw fromZipError(error, kind);
|
|
||||||
const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file';
|
|
||||||
if (kind === 'docx' || kind === 'pptx') {
|
|
||||||
throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error });
|
|
||||||
}
|
|
||||||
throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,272 +0,0 @@
|
||||||
/** Small markup helpers shared by the DOCX and PPTX extractors. */
|
|
||||||
|
|
||||||
const NAMED_ENTITIES: Record<string, string> = {
|
|
||||||
amp: '&',
|
|
||||||
lt: '<',
|
|
||||||
gt: '>',
|
|
||||||
quot: '"',
|
|
||||||
apos: "'",
|
|
||||||
nbsp: ' ',
|
|
||||||
};
|
|
||||||
|
|
||||||
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
|
|
||||||
export function decodeEntities(text: string): string {
|
|
||||||
if (!text.includes('&')) return text;
|
|
||||||
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
|
|
||||||
if (body[0] === '#') {
|
|
||||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
|
||||||
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
|
|
||||||
return String.fromCodePoint(code);
|
|
||||||
}
|
|
||||||
return NAMED_ENTITIES[body] ?? match;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
|
|
||||||
export function renderTable(rows: string[][]): string {
|
|
||||||
const cleaned = rows
|
|
||||||
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
|
|
||||||
.filter((row) => row.some((cell) => cell.length > 0));
|
|
||||||
if (cleaned.length === 0) return '';
|
|
||||||
const width = Math.max(...cleaned.map((row) => row.length));
|
|
||||||
const line = (row: string[]) => {
|
|
||||||
const cells = row.slice();
|
|
||||||
while (cells.length < width) cells.push('');
|
|
||||||
return `| ${cells.join(' | ')} |`;
|
|
||||||
};
|
|
||||||
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
|
|
||||||
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
|
|
||||||
return out.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// HTML (as produced by mammoth) -> Markdown blocks
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
export interface MarkdownBlock {
|
|
||||||
/** 1-6 for headings, 0 for everything else. */
|
|
||||||
level: number;
|
|
||||||
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
|
|
||||||
text: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface HtmlElement {
|
|
||||||
tag: string;
|
|
||||||
children: HtmlNode[];
|
|
||||||
}
|
|
||||||
type HtmlNode = HtmlElement | string;
|
|
||||||
|
|
||||||
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
|
|
||||||
const HEADING_TAG = /^h[1-6]$/;
|
|
||||||
|
|
||||||
function parseHtml(html: string): HtmlElement {
|
|
||||||
const root: HtmlElement = { tag: '#root', children: [] };
|
|
||||||
const stack: HtmlElement[] = [root];
|
|
||||||
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
|
|
||||||
let match: RegExpExecArray | null;
|
|
||||||
while ((match = tokens.exec(html)) !== null) {
|
|
||||||
const top = stack[stack.length - 1];
|
|
||||||
if (match[4] !== undefined) {
|
|
||||||
top.children.push(decodeEntities(match[4]));
|
|
||||||
} else if (match[2] === undefined) {
|
|
||||||
top.children.push('<');
|
|
||||||
} else {
|
|
||||||
const tag = match[2].toLowerCase();
|
|
||||||
if (match[1]) {
|
|
||||||
// Close the nearest matching open element; ignore stray closers.
|
|
||||||
for (let i = stack.length - 1; i > 0; i--) {
|
|
||||||
if (stack[i].tag === tag) {
|
|
||||||
stack.length = i;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const element: HtmlElement = { tag, children: [] };
|
|
||||||
top.children.push(element);
|
|
||||||
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return root;
|
|
||||||
}
|
|
||||||
|
|
||||||
function collapse(text: string): string {
|
|
||||||
return text.replace(/[ \t\r\n]+/g, ' ');
|
|
||||||
}
|
|
||||||
|
|
||||||
function wrap(marker: string, inner: string): string {
|
|
||||||
const core = inner.trim();
|
|
||||||
if (!core) return inner;
|
|
||||||
const lead = inner.slice(0, inner.length - inner.trimStart().length);
|
|
||||||
const trail = inner.slice(inner.trimEnd().length);
|
|
||||||
return `${lead}${marker}${core}${marker}${trail}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
|
|
||||||
function inlineText(nodes: HtmlNode[], plain: boolean): string {
|
|
||||||
let out = '';
|
|
||||||
for (const node of nodes) {
|
|
||||||
if (typeof node === 'string') {
|
|
||||||
out += collapse(node);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
switch (node.tag) {
|
|
||||||
case 'br':
|
|
||||||
out += '\n';
|
|
||||||
break;
|
|
||||||
case 'img':
|
|
||||||
case 'hr':
|
|
||||||
break;
|
|
||||||
case 'strong':
|
|
||||||
case 'b':
|
|
||||||
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
|
|
||||||
break;
|
|
||||||
case 'em':
|
|
||||||
case 'i':
|
|
||||||
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
|
|
||||||
break;
|
|
||||||
case 'a': {
|
|
||||||
const text = inlineText(node.children, plain);
|
|
||||||
// mammoth's footnote back-references
|
|
||||||
if (text.trim() !== '↑') out += text;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'p':
|
|
||||||
case 'li':
|
|
||||||
case 'tr':
|
|
||||||
case 'td':
|
|
||||||
case 'th':
|
|
||||||
case 'ul':
|
|
||||||
case 'ol':
|
|
||||||
case 'table': {
|
|
||||||
const inner = inlineText(node.children, plain).trim();
|
|
||||||
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
out += inlineText(node.children, plain);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function tidyInline(text: string): string {
|
|
||||||
return text
|
|
||||||
.split('\n')
|
|
||||||
.map((line) => line.replace(/ {2,}/g, ' ').trim())
|
|
||||||
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
|
|
||||||
.join('\n')
|
|
||||||
.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
|
|
||||||
const ordered = list.tag === 'ol';
|
|
||||||
let counter = 0;
|
|
||||||
for (const item of list.children) {
|
|
||||||
if (typeof item === 'string' || item.tag !== 'li') continue;
|
|
||||||
counter++;
|
|
||||||
const marker = ordered ? `${counter}. ` : '- ';
|
|
||||||
const inlineNodes: HtmlNode[] = [];
|
|
||||||
const nested: HtmlElement[] = [];
|
|
||||||
const paragraphs: string[] = [];
|
|
||||||
const flushInline = () => {
|
|
||||||
const text = tidyInline(inlineText(inlineNodes, false));
|
|
||||||
if (text) paragraphs.push(text);
|
|
||||||
inlineNodes.length = 0;
|
|
||||||
};
|
|
||||||
for (const child of item.children) {
|
|
||||||
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
|
|
||||||
flushInline();
|
|
||||||
nested.push(child);
|
|
||||||
} else if (typeof child !== 'string' && child.tag === 'p') {
|
|
||||||
flushInline();
|
|
||||||
inlineNodes.push(...child.children);
|
|
||||||
flushInline();
|
|
||||||
} else if (typeof child !== 'string' && child.tag === 'table') {
|
|
||||||
flushInline();
|
|
||||||
const table = renderHtmlTable(child);
|
|
||||||
if (table) paragraphs.push(table);
|
|
||||||
} else {
|
|
||||||
inlineNodes.push(child);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
flushInline();
|
|
||||||
const pad = indent + ' '.repeat(marker.length);
|
|
||||||
const body = paragraphs.join('\n').split('\n');
|
|
||||||
if (body.length > 0 && body[0] !== '') {
|
|
||||||
lines.push(indent + marker + body[0]);
|
|
||||||
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
|
|
||||||
} else if (nested.length === 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
for (const child of nested) renderList(child, pad, lines);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderHtmlTable(table: HtmlElement): string {
|
|
||||||
const rows: string[][] = [];
|
|
||||||
const visit = (nodes: HtmlNode[]) => {
|
|
||||||
for (const node of nodes) {
|
|
||||||
if (typeof node === 'string') continue;
|
|
||||||
if (node.tag === 'tr') {
|
|
||||||
const cells: string[] = [];
|
|
||||||
for (const cell of node.children) {
|
|
||||||
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
|
|
||||||
cells.push(tidyInline(inlineText(cell.children, false)));
|
|
||||||
}
|
|
||||||
rows.push(cells);
|
|
||||||
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
|
|
||||||
visit(node.children);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
visit(table.children);
|
|
||||||
return renderTable(rows);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
|
|
||||||
export function htmlToBlocks(html: string): MarkdownBlock[] {
|
|
||||||
const root = parseHtml(html);
|
|
||||||
const blocks: MarkdownBlock[] = [];
|
|
||||||
const loose: HtmlNode[] = [];
|
|
||||||
const flushLoose = () => {
|
|
||||||
if (loose.length === 0) return;
|
|
||||||
const text = tidyInline(inlineText(loose, false));
|
|
||||||
if (text) blocks.push({ level: 0, text });
|
|
||||||
loose.length = 0;
|
|
||||||
};
|
|
||||||
const visit = (nodes: HtmlNode[]) => {
|
|
||||||
for (const node of nodes) {
|
|
||||||
if (typeof node === 'string') {
|
|
||||||
loose.push(node);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (HEADING_TAG.test(node.tag)) {
|
|
||||||
flushLoose();
|
|
||||||
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
|
|
||||||
if (text) blocks.push({ level: Number(node.tag[1]), text });
|
|
||||||
} else if (node.tag === 'p') {
|
|
||||||
flushLoose();
|
|
||||||
const text = tidyInline(inlineText(node.children, false));
|
|
||||||
if (text) blocks.push({ level: 0, text });
|
|
||||||
} else if (node.tag === 'ul' || node.tag === 'ol') {
|
|
||||||
flushLoose();
|
|
||||||
const lines: string[] = [];
|
|
||||||
renderList(node, '', lines);
|
|
||||||
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
|
|
||||||
} else if (node.tag === 'table') {
|
|
||||||
flushLoose();
|
|
||||||
const text = renderHtmlTable(node);
|
|
||||||
if (text) blocks.push({ level: 0, text });
|
|
||||||
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
|
|
||||||
flushLoose();
|
|
||||||
visit(node.children);
|
|
||||||
} else {
|
|
||||||
loose.push(node);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
visit(root.children);
|
|
||||||
flushLoose();
|
|
||||||
return blocks;
|
|
||||||
}
|
|
||||||
|
|
@ -1,249 +0,0 @@
|
||||||
/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
|
|
||||||
import { getDocumentProxy } from 'unpdf';
|
|
||||||
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
|
|
||||||
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
||||||
|
|
||||||
export interface PdfContext {
|
|
||||||
onProgress?: (done: number, total: number) => void;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Below this average of characters per page the PDF is treated as a scan. */
|
|
||||||
const OCR_CHARS_PER_PAGE = 25;
|
|
||||||
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
|
|
||||||
const CLEANUP_EVERY = 40;
|
|
||||||
|
|
||||||
interface TextItemLike {
|
|
||||||
str?: unknown;
|
|
||||||
hasEOL?: unknown;
|
|
||||||
transform?: unknown;
|
|
||||||
height?: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Line {
|
|
||||||
text: string;
|
|
||||||
y: number | null;
|
|
||||||
height: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
|
|
||||||
* order (which keeps multi-column pages readable) with `hasEOL` marking line
|
|
||||||
* ends; the vertical gaps between lines are then used to restore paragraph
|
|
||||||
* breaks, which pdf.js does not report.
|
|
||||||
*/
|
|
||||||
export function textItemsToText(items: readonly TextItemLike[]): string {
|
|
||||||
const lines: Line[] = [];
|
|
||||||
let text = '';
|
|
||||||
let y: number | null = null;
|
|
||||||
let height = 0;
|
|
||||||
const endLine = () => {
|
|
||||||
lines.push({ text, y, height });
|
|
||||||
text = '';
|
|
||||||
y = null;
|
|
||||||
height = 0;
|
|
||||||
};
|
|
||||||
for (const item of items) {
|
|
||||||
if (!item || typeof item.str !== 'string') continue;
|
|
||||||
const str = item.str;
|
|
||||||
if (str.length > 0) {
|
|
||||||
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
|
|
||||||
const t = item.transform as number[];
|
|
||||||
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
|
|
||||||
if (upright && Number.isFinite(t[5])) {
|
|
||||||
y = t[5];
|
|
||||||
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
|
|
||||||
height = Number.isFinite(h) ? h : 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
text += str;
|
|
||||||
}
|
|
||||||
if (item.hasEOL === true) endLine();
|
|
||||||
}
|
|
||||||
if (text.length > 0) endLine();
|
|
||||||
|
|
||||||
// Typical line advance on this page (median of the downward steps).
|
|
||||||
const advances: number[] = [];
|
|
||||||
for (let i = 1; i < lines.length; i++) {
|
|
||||||
const a = lines[i - 1];
|
|
||||||
const b = lines[i];
|
|
||||||
if (a.y === null || b.y === null) continue;
|
|
||||||
const dy = a.y - b.y;
|
|
||||||
const ref = Math.max(a.height, b.height, 1);
|
|
||||||
if (dy > 0 && dy < ref * 4) advances.push(dy);
|
|
||||||
}
|
|
||||||
advances.sort((p, q) => p - q);
|
|
||||||
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
|
|
||||||
|
|
||||||
let out = '';
|
|
||||||
let prev: Line | null = null;
|
|
||||||
for (const line of lines) {
|
|
||||||
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
|
|
||||||
if (!content) {
|
|
||||||
// An empty line is an explicit gap.
|
|
||||||
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (out && !out.endsWith('\n')) {
|
|
||||||
let gap = false;
|
|
||||||
if (prev && prev.y !== null && line.y !== null) {
|
|
||||||
const dy = prev.y - line.y;
|
|
||||||
const ref = Math.max(prev.height, line.height, 1);
|
|
||||||
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
|
|
||||||
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
|
|
||||||
else gap = dy > ref * 1.9;
|
|
||||||
}
|
|
||||||
out += gap ? '\n\n' : '\n';
|
|
||||||
}
|
|
||||||
out += content;
|
|
||||||
prev = line;
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
|
|
||||||
|
|
||||||
export function tidyTitle(raw: unknown): string | undefined {
|
|
||||||
if (typeof raw !== 'string') return undefined;
|
|
||||||
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
|
|
||||||
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
|
|
||||||
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
|
|
||||||
return title;
|
|
||||||
}
|
|
||||||
|
|
||||||
function errorName(error: unknown): string {
|
|
||||||
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
|
|
||||||
}
|
|
||||||
|
|
||||||
function toExtractionError(error: unknown): ExtractionError {
|
|
||||||
if (error instanceof ExtractionError) return error;
|
|
||||||
const name = errorName(error);
|
|
||||||
const message = error instanceof Error ? error.message : String(error);
|
|
||||||
if (name === 'PasswordException' || /password/i.test(message)) {
|
|
||||||
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
name === 'InvalidPDFException' ||
|
|
||||||
name === 'FormatError' ||
|
|
||||||
name === 'XRefParseException' ||
|
|
||||||
name === 'XRefEntryException' ||
|
|
||||||
name === 'MissingDataException' ||
|
|
||||||
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
|
|
||||||
) {
|
|
||||||
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
|
|
||||||
}
|
|
||||||
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
|
|
||||||
}
|
|
||||||
|
|
||||||
function cancelled(signal: AbortSignal): ExtractionError {
|
|
||||||
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
|
|
||||||
const { onProgress, signal } = ctx;
|
|
||||||
if (signal?.aborted) throw cancelled(signal);
|
|
||||||
|
|
||||||
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
|
|
||||||
try {
|
|
||||||
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
|
|
||||||
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
|
|
||||||
pdf = await getDocumentProxy(bytes.slice(), options);
|
|
||||||
} catch (error) {
|
|
||||||
throw toExtractionError(error);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const total = pdf.numPages;
|
|
||||||
if (!Number.isFinite(total) || total < 1) {
|
|
||||||
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
|
|
||||||
}
|
|
||||||
|
|
||||||
let title: string | undefined;
|
|
||||||
try {
|
|
||||||
const meta = await pdf.getMetadata();
|
|
||||||
const info = (meta?.info ?? {}) as Record<string, unknown>;
|
|
||||||
title = tidyTitle(info.Title);
|
|
||||||
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
|
|
||||||
} catch {
|
|
||||||
// Metadata is optional.
|
|
||||||
}
|
|
||||||
|
|
||||||
const pageTexts: string[] = new Array(total).fill('');
|
|
||||||
let failedPages = 0;
|
|
||||||
let lastYield = Date.now();
|
|
||||||
for (let n = 1; n <= total; n++) {
|
|
||||||
if (signal?.aborted) throw cancelled(signal);
|
|
||||||
try {
|
|
||||||
const page = await pdf.getPage(n);
|
|
||||||
try {
|
|
||||||
const content = await page.getTextContent();
|
|
||||||
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
|
|
||||||
} finally {
|
|
||||||
page.cleanup();
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
if (errorName(error) === 'PasswordException') throw error;
|
|
||||||
failedPages++;
|
|
||||||
}
|
|
||||||
if (n % CLEANUP_EVERY === 0) {
|
|
||||||
try {
|
|
||||||
await pdf.cleanup();
|
|
||||||
} catch {
|
|
||||||
// Best effort.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
onProgress?.(n, total);
|
|
||||||
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
|
|
||||||
if (Date.now() - lastYield > 20) {
|
|
||||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
||||||
lastYield = Date.now();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (failedPages === total) {
|
|
||||||
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
|
|
||||||
}
|
|
||||||
|
|
||||||
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
|
|
||||||
const sections: ExtractedSection[] = [];
|
|
||||||
let empty = 0;
|
|
||||||
let chars = 0;
|
|
||||||
for (let i = 0; i < total; i++) {
|
|
||||||
const text = cleaned[i];
|
|
||||||
if (!text) {
|
|
||||||
empty++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
chars += text.length;
|
|
||||||
sections.push({ text, page: i + 1 });
|
|
||||||
}
|
|
||||||
|
|
||||||
const warnings: string[] = [];
|
|
||||||
if (empty > 0) {
|
|
||||||
warnings.push(
|
|
||||||
empty === total
|
|
||||||
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
|
|
||||||
: `${empty} of ${total} pages had no extractable text.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
|
|
||||||
|
|
||||||
const result: ExtractResult = {
|
|
||||||
kind: 'pdf',
|
|
||||||
sections,
|
|
||||||
pageCount: total,
|
|
||||||
warnings,
|
|
||||||
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
|
|
||||||
};
|
|
||||||
if (title) result.title = title;
|
|
||||||
return result;
|
|
||||||
} catch (error) {
|
|
||||||
throw toExtractionError(error);
|
|
||||||
} finally {
|
|
||||||
try {
|
|
||||||
await pdf.loadingTask.destroy();
|
|
||||||
} catch {
|
|
||||||
// Nothing useful to do.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,424 +0,0 @@
|
||||||
/** PPTX extraction: one section per slide, in presentation order, with titles, bullets, tables and notes. */
|
|
||||||
import { XMLParser } from 'fast-xml-parser';
|
|
||||||
import { cleanText } from './chunk.ts';
|
|
||||||
import { decodeEntities, renderTable } from './markup.ts';
|
|
||||||
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
||||||
import { ZipReader, type ZipLimits } from './zip.ts';
|
|
||||||
|
|
||||||
export interface OfficeContext {
|
|
||||||
limits: ZipLimits;
|
|
||||||
onProgress?: (done: number, total: number) => void;
|
|
||||||
signal?: AbortSignal;
|
|
||||||
}
|
|
||||||
|
|
||||||
type XmlNode = Record<string, unknown>;
|
|
||||||
|
|
||||||
const parser = new XMLParser({
|
|
||||||
ignoreAttributes: false,
|
|
||||||
attributeNamePrefix: '',
|
|
||||||
preserveOrder: true,
|
|
||||||
trimValues: false,
|
|
||||||
parseTagValue: false,
|
|
||||||
parseAttributeValue: false,
|
|
||||||
// Entities are decoded by decodeEntities so numeric references work and nothing is expanded twice.
|
|
||||||
processEntities: false,
|
|
||||||
ignoreDeclaration: true,
|
|
||||||
ignorePiTags: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
function parseXml(xml: string): XmlNode[] {
|
|
||||||
const parsed = parser.parse(xml) as unknown;
|
|
||||||
return Array.isArray(parsed) ? (parsed as XmlNode[]) : [];
|
|
||||||
}
|
|
||||||
|
|
||||||
function tagOf(node: XmlNode): string {
|
|
||||||
for (const key in node) if (key !== ':@') return key;
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Element name without its namespace prefix. */
|
|
||||||
function localName(node: XmlNode): string {
|
|
||||||
const tag = tagOf(node);
|
|
||||||
const colon = tag.indexOf(':');
|
|
||||||
return colon < 0 ? tag : tag.slice(colon + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
function childrenOf(node: XmlNode): XmlNode[] {
|
|
||||||
const value = node[tagOf(node)];
|
|
||||||
return Array.isArray(value) ? (value as XmlNode[]) : [];
|
|
||||||
}
|
|
||||||
|
|
||||||
function attr(node: XmlNode, name: string): string | undefined {
|
|
||||||
const attrs = node[':@'] as Record<string, unknown> | undefined;
|
|
||||||
if (!attrs) return undefined;
|
|
||||||
const direct = attrs[name];
|
|
||||||
if (typeof direct === 'string') return direct;
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Attribute lookup that ignores the namespace prefix (r:id, rel:id, ...). */
|
|
||||||
function attrLocal(node: XmlNode, local: string): string | undefined {
|
|
||||||
const attrs = node[':@'] as Record<string, unknown> | undefined;
|
|
||||||
if (!attrs) return undefined;
|
|
||||||
for (const key in attrs) {
|
|
||||||
if (key.endsWith(`:${local}`) && typeof attrs[key] === 'string') return attrs[key] as string;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function child(node: XmlNode, local: string): XmlNode | undefined {
|
|
||||||
for (const c of childrenOf(node)) if (localName(c) === local) return c;
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function findDeep(nodes: XmlNode[], local: string, depth = 0): XmlNode | undefined {
|
|
||||||
if (depth > 40) return undefined;
|
|
||||||
for (const node of nodes) {
|
|
||||||
if (localName(node) === local) return node;
|
|
||||||
const found = findDeep(childrenOf(node), local, depth + 1);
|
|
||||||
if (found) return found;
|
|
||||||
}
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Concatenated text of every a:t / m:t descendant. */
|
|
||||||
function deepText(node: XmlNode, depth = 0): string {
|
|
||||||
if (depth > 40) return '';
|
|
||||||
const name = localName(node);
|
|
||||||
if (name === '#text') return '';
|
|
||||||
if (name === 't') {
|
|
||||||
let out = '';
|
|
||||||
for (const c of childrenOf(node)) if (typeof c['#text'] === 'string') out += c['#text'] as string;
|
|
||||||
return decodeEntities(out);
|
|
||||||
}
|
|
||||||
let out = '';
|
|
||||||
for (const c of childrenOf(node)) out += deepText(c, depth + 1);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Paragraph {
|
|
||||||
text: string;
|
|
||||||
level: number;
|
|
||||||
/** true: explicit bullet/number; false: bullets explicitly off; undefined: inherited. */
|
|
||||||
bullet: boolean | undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readParagraph(p: XmlNode): Paragraph {
|
|
||||||
let text = '';
|
|
||||||
let level = 0;
|
|
||||||
let bullet: boolean | undefined;
|
|
||||||
for (const c of childrenOf(p)) {
|
|
||||||
switch (localName(c)) {
|
|
||||||
case 'pPr': {
|
|
||||||
const lvl = Number(attr(c, 'lvl'));
|
|
||||||
if (Number.isFinite(lvl) && lvl > 0) level = Math.min(8, Math.floor(lvl));
|
|
||||||
for (const b of childrenOf(c)) {
|
|
||||||
const name = localName(b);
|
|
||||||
if (name === 'buNone') bullet = false;
|
|
||||||
else if (name === 'buChar' || name === 'buAutoNum' || name === 'buBlip') bullet = true;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'br':
|
|
||||||
text += '\n';
|
|
||||||
break;
|
|
||||||
case 'fld': {
|
|
||||||
const type = attr(c, 'type') ?? '';
|
|
||||||
if (!/^(slidenum|datetime)/i.test(type)) text += deepText(c);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'endParaRPr':
|
|
||||||
case '#text':
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
text += deepText(c);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
text = text
|
|
||||||
.replace(/[\u000B\u2028]/g, '\n')
|
|
||||||
.split('\n')
|
|
||||||
.map((line) => line.replace(/[ \t\u00A0]+/g, ' ').trim())
|
|
||||||
.filter((line) => line.length > 0)
|
|
||||||
.join('\n');
|
|
||||||
return { text, level, bullet };
|
|
||||||
}
|
|
||||||
|
|
||||||
function readParagraphs(txBody: XmlNode): Paragraph[] {
|
|
||||||
const out: Paragraph[] = [];
|
|
||||||
for (const c of childrenOf(txBody)) {
|
|
||||||
if (localName(c) !== 'p') continue;
|
|
||||||
const paragraph = readParagraph(c);
|
|
||||||
if (paragraph.text) out.push(paragraph);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderParagraphs(paragraphs: Paragraph[], bulletsByDefault: boolean): string {
|
|
||||||
const lines: string[] = [];
|
|
||||||
for (const p of paragraphs) {
|
|
||||||
const bulleted = p.bullet ?? (bulletsByDefault || p.level > 0);
|
|
||||||
if (!bulleted) {
|
|
||||||
lines.push(p.text);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const indent = ' '.repeat(p.level);
|
|
||||||
const parts = p.text.split('\n');
|
|
||||||
lines.push(`${indent}- ${parts[0]}`);
|
|
||||||
for (let i = 1; i < parts.length; i++) lines.push(`${indent} ${parts[i]}`);
|
|
||||||
}
|
|
||||||
return lines.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
function readTable(tbl: XmlNode): string {
|
|
||||||
const rows: string[][] = [];
|
|
||||||
for (const tr of childrenOf(tbl)) {
|
|
||||||
if (localName(tr) !== 'tr') continue;
|
|
||||||
const cells: string[] = [];
|
|
||||||
for (const tc of childrenOf(tr)) {
|
|
||||||
if (localName(tc) !== 'tc') continue;
|
|
||||||
const txBody = child(tc, 'txBody');
|
|
||||||
cells.push(txBody ? readParagraphs(txBody).map((p) => p.text.replace(/\n/g, ' ')).join(' ') : '');
|
|
||||||
}
|
|
||||||
rows.push(cells);
|
|
||||||
}
|
|
||||||
return renderTable(rows);
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SlideContent {
|
|
||||||
title: string;
|
|
||||||
blocks: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
const SKIPPED_PLACEHOLDERS = new Set(['sldNum', 'dt', 'ftr', 'hdr', 'sldImg']);
|
|
||||||
|
|
||||||
function placeholderType(shape: XmlNode): string | null {
|
|
||||||
const nv = child(shape, 'nvSpPr');
|
|
||||||
const nvPr = nv ? child(nv, 'nvPr') : undefined;
|
|
||||||
const ph = nvPr ? child(nvPr, 'ph') : undefined;
|
|
||||||
if (!ph) return null;
|
|
||||||
return attr(ph, 'type') ?? 'body';
|
|
||||||
}
|
|
||||||
|
|
||||||
function walkShapes(nodes: XmlNode[], into: SlideContent, depth: number): void {
|
|
||||||
if (depth > 30) return;
|
|
||||||
for (const node of nodes) {
|
|
||||||
switch (localName(node)) {
|
|
||||||
case 'sp': {
|
|
||||||
const type = placeholderType(node);
|
|
||||||
if (type && SKIPPED_PLACEHOLDERS.has(type)) break;
|
|
||||||
const txBody = child(node, 'txBody');
|
|
||||||
if (!txBody) break;
|
|
||||||
const paragraphs = readParagraphs(txBody);
|
|
||||||
if (paragraphs.length === 0) break;
|
|
||||||
if ((type === 'title' || type === 'ctrTitle') && !into.title) {
|
|
||||||
into.title = paragraphs.map((p) => p.text.replace(/\n/g, ' ')).join(' ').replace(/\s+/g, ' ').trim();
|
|
||||||
} else {
|
|
||||||
const bullets = type !== null && type !== 'subTitle' && type !== 'title' && type !== 'ctrTitle';
|
|
||||||
const text = renderParagraphs(paragraphs, bullets);
|
|
||||||
if (text) into.blocks.push(text);
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'grpSp':
|
|
||||||
walkShapes(childrenOf(node), into, depth + 1);
|
|
||||||
break;
|
|
||||||
case 'graphicFrame': {
|
|
||||||
const tbl = findDeep(childrenOf(node), 'tbl');
|
|
||||||
if (tbl) {
|
|
||||||
const table = readTable(tbl);
|
|
||||||
if (table) into.blocks.push(table);
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'AlternateContent': {
|
|
||||||
const choice = child(node, 'Choice') ?? child(node, 'Fallback');
|
|
||||||
if (choice) walkShapes(childrenOf(choice), into, depth + 1);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function readSlide(xml: string): SlideContent {
|
|
||||||
const content: SlideContent = { title: '', blocks: [] };
|
|
||||||
const spTree = findDeep(parseXml(xml), 'spTree');
|
|
||||||
if (spTree) walkShapes(childrenOf(spTree), content, 0);
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readNotes(xml: string): string {
|
|
||||||
const spTree = findDeep(parseXml(xml), 'spTree');
|
|
||||||
if (!spTree) return '';
|
|
||||||
const parts: string[] = [];
|
|
||||||
for (const shape of childrenOf(spTree)) {
|
|
||||||
if (localName(shape) !== 'sp' || placeholderType(shape) !== 'body') continue;
|
|
||||||
const txBody = child(shape, 'txBody');
|
|
||||||
if (!txBody) continue;
|
|
||||||
const text = renderParagraphs(readParagraphs(txBody), false);
|
|
||||||
if (text) parts.push(text);
|
|
||||||
}
|
|
||||||
return parts.join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Relationship {
|
|
||||||
id: string;
|
|
||||||
type: string;
|
|
||||||
target: string;
|
|
||||||
external: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readRelationships(xml: string | null): Relationship[] {
|
|
||||||
if (!xml) return [];
|
|
||||||
const out: Relationship[] = [];
|
|
||||||
const tags = /<(?:[A-Za-z0-9]+:)?Relationship\b([^>]*)>/g;
|
|
||||||
let match: RegExpExecArray | null;
|
|
||||||
while ((match = tags.exec(xml)) !== null) {
|
|
||||||
const attrs: Record<string, string> = {};
|
|
||||||
const pairs = /([A-Za-z:]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
|
|
||||||
let pair: RegExpExecArray | null;
|
|
||||||
while ((pair = pairs.exec(match[1])) !== null) attrs[pair[1]] = decodeEntities(pair[2] ?? pair[3] ?? '');
|
|
||||||
if (attrs.Id && attrs.Target) {
|
|
||||||
out.push({ id: attrs.Id, type: attrs.Type ?? '', target: attrs.Target, external: attrs.TargetMode === 'External' });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Resolves a relationship target against the directory of the part that owns it. */
|
|
||||||
function resolveTarget(baseDir: string, target: string): string {
|
|
||||||
const parts = target.startsWith('/') ? [] : baseDir.split('/').filter(Boolean);
|
|
||||||
for (const segment of target.split('/')) {
|
|
||||||
if (!segment || segment === '.') continue;
|
|
||||||
if (segment === '..') parts.pop();
|
|
||||||
else parts.push(segment);
|
|
||||||
}
|
|
||||||
return parts.join('/');
|
|
||||||
}
|
|
||||||
|
|
||||||
function relsPathFor(part: string): string {
|
|
||||||
const slash = part.lastIndexOf('/');
|
|
||||||
return `${part.slice(0, slash + 1)}_rels/${part.slice(slash + 1)}.rels`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function slideOrder(zip: ZipReader): string[] {
|
|
||||||
const ordered: string[] = [];
|
|
||||||
const presentation = zip.readText('ppt/presentation.xml');
|
|
||||||
if (presentation) {
|
|
||||||
const targets = new Map<string, string>();
|
|
||||||
for (const rel of readRelationships(zip.readText('ppt/_rels/presentation.xml.rels'))) {
|
|
||||||
if (!rel.external) targets.set(rel.id, resolveTarget('ppt', rel.target));
|
|
||||||
}
|
|
||||||
const list = findDeep(parseXml(presentation), 'sldIdLst');
|
|
||||||
if (list) {
|
|
||||||
for (const item of childrenOf(list)) {
|
|
||||||
if (localName(item) !== 'sldId') continue;
|
|
||||||
const id = attrLocal(item, 'id');
|
|
||||||
const path = id ? targets.get(id) : undefined;
|
|
||||||
if (path && zip.has(path) && !ordered.includes(path)) ordered.push(path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (ordered.length > 0) return ordered;
|
|
||||||
// Damaged or unusual package: fall back to numeric file order.
|
|
||||||
return zip
|
|
||||||
.names()
|
|
||||||
.map((name) => ({ name, match: /^ppt\/slides\/slide(\d+)\.xml$/i.exec(name) }))
|
|
||||||
.filter((item): item is { name: string; match: RegExpExecArray } => item.match !== null)
|
|
||||||
.sort((a, b) => Number(a.match[1]) - Number(b.match[1]))
|
|
||||||
.map((item) => item.name);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function readCoreTitle(zip: ZipReader): string | undefined {
|
|
||||||
let xml: string | null = null;
|
|
||||||
try {
|
|
||||||
xml = zip.readText('docProps/core.xml');
|
|
||||||
} catch {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
if (!xml) return undefined;
|
|
||||||
const match = /<(?:[A-Za-z0-9]+:)?title\b[^>]*>([^<]*)</.exec(xml);
|
|
||||||
const title = match ? decodeEntities(match[1]).replace(/\s+/g, ' ').trim() : '';
|
|
||||||
return title || undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function extractPptx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
|
|
||||||
const zip = new ZipReader(bytes, ctx.limits);
|
|
||||||
const slides = slideOrder(zip);
|
|
||||||
if (slides.length === 0) {
|
|
||||||
throw new ExtractionError('corrupt', "This PowerPoint file doesn't contain any slides we can read. Try saving it again and re-uploading.");
|
|
||||||
}
|
|
||||||
|
|
||||||
const sections: ExtractedSection[] = [];
|
|
||||||
const warnings: string[] = [];
|
|
||||||
let withoutText = 0;
|
|
||||||
let unreadable = 0;
|
|
||||||
let totalChars = 0;
|
|
||||||
let lastYield = Date.now();
|
|
||||||
|
|
||||||
for (let i = 0; i < slides.length; i++) {
|
|
||||||
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
|
|
||||||
const path = slides[i];
|
|
||||||
let text = '';
|
|
||||||
let title = '';
|
|
||||||
try {
|
|
||||||
const xml = zip.readText(path);
|
|
||||||
const slide = xml ? readSlide(xml) : { title: '', blocks: [] };
|
|
||||||
title = slide.title;
|
|
||||||
const parts: string[] = [];
|
|
||||||
if (slide.title) parts.push(`## ${slide.title}`);
|
|
||||||
parts.push(...slide.blocks);
|
|
||||||
|
|
||||||
const notesRel = readRelationships(zip.readText(relsPathFor(path))).find(
|
|
||||||
(rel) => !rel.external && /\/notesSlide$/.test(rel.type),
|
|
||||||
);
|
|
||||||
if (notesRel) {
|
|
||||||
const notesXml = zip.readText(resolveTarget(path.slice(0, path.lastIndexOf('/')), notesRel.target));
|
|
||||||
const notes = notesXml ? readNotes(notesXml) : '';
|
|
||||||
if (notes) parts.push(`Notes:\n${notes}`);
|
|
||||||
}
|
|
||||||
text = cleanText(parts.join('\n\n'));
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof ExtractionError) throw error;
|
|
||||||
if ((error as { code?: string })?.code === 'too_large') throw error;
|
|
||||||
unreadable++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (text) {
|
|
||||||
totalChars += text.length;
|
|
||||||
const section: ExtractedSection = { text, page: i + 1 };
|
|
||||||
if (title) section.heading = title;
|
|
||||||
sections.push(section);
|
|
||||||
} else {
|
|
||||||
withoutText++;
|
|
||||||
}
|
|
||||||
ctx.onProgress?.(i + 1, slides.length);
|
|
||||||
if (Date.now() - lastYield > 20) {
|
|
||||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
||||||
lastYield = Date.now();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const total = slides.length;
|
|
||||||
if (unreadable === total) {
|
|
||||||
throw new ExtractionError('corrupt', 'This PowerPoint file looks damaged and could not be read. Try saving it again and re-uploading.');
|
|
||||||
}
|
|
||||||
if (unreadable > 0) warnings.push(`${unreadable} of ${total} slides could not be read.`);
|
|
||||||
if (withoutText === total) {
|
|
||||||
warnings.push(`None of the ${total} slides had any text - the slides are probably images.`);
|
|
||||||
} else if (withoutText * 2 > total) {
|
|
||||||
warnings.push(`${withoutText} of ${total} slides had no text - most of this deck is probably images, which are not read.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const result: ExtractResult = {
|
|
||||||
kind: 'pptx',
|
|
||||||
sections,
|
|
||||||
pageCount: total,
|
|
||||||
warnings,
|
|
||||||
needsOcr: totalChars < total * 25,
|
|
||||||
};
|
|
||||||
const title = readCoreTitle(zip);
|
|
||||||
if (title) result.title = title;
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
@ -1,270 +0,0 @@
|
||||||
/** Programmatic fixtures for the document tests. Nothing here is used by production code. */
|
|
||||||
import { strToU8, zipSync } from 'fflate';
|
|
||||||
|
|
||||||
const latin1 = (text: string): Uint8Array => Uint8Array.from(text, (ch) => ch.charCodeAt(0) & 0xff);
|
|
||||||
|
|
||||||
function pdfString(text: string): string {
|
|
||||||
return `(${text.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')})`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PdfPage {
|
|
||||||
/** Lines of text, drawn top-down at 12pt with 14pt leading. `''` leaves a paragraph gap. */
|
|
||||||
lines?: string[];
|
|
||||||
/** Raw content stream, used instead of `lines`. */
|
|
||||||
raw?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PdfOptions {
|
|
||||||
title?: string;
|
|
||||||
/** Adds a standard security handler whose user password is not empty. */
|
|
||||||
encrypted?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Builds a small but valid PDF (Helvetica, one content stream per page). */
|
|
||||||
export function buildPdf(pages: PdfPage[], options: PdfOptions = {}): Uint8Array {
|
|
||||||
const objects: string[] = [];
|
|
||||||
const add = (body: string): number => {
|
|
||||||
objects.push(body);
|
|
||||||
return objects.length;
|
|
||||||
};
|
|
||||||
const catalogId = add('');
|
|
||||||
const pagesId = add('');
|
|
||||||
const fontId = add('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>');
|
|
||||||
|
|
||||||
const pageIds: number[] = [];
|
|
||||||
for (const page of pages) {
|
|
||||||
let stream = page.raw ?? '';
|
|
||||||
if (page.raw === undefined && page.lines && page.lines.length > 0) {
|
|
||||||
const ops: string[] = ['BT', '/F1 12 Tf', '14 TL', '72 720 Td'];
|
|
||||||
for (const line of page.lines) {
|
|
||||||
if (line === '') ops.push('0 -14 Td');
|
|
||||||
else ops.push(`${pdfString(line)} Tj`, 'T*');
|
|
||||||
}
|
|
||||||
ops.push('ET');
|
|
||||||
stream = ops.join('\n');
|
|
||||||
}
|
|
||||||
const contentId = add(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
|
|
||||||
pageIds.push(
|
|
||||||
add(
|
|
||||||
`<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
objects[catalogId - 1] = `<< /Type /Catalog /Pages ${pagesId} 0 R >>`;
|
|
||||||
objects[pagesId - 1] = `<< /Type /Pages /Count ${pageIds.length} /Kids [${pageIds.map((id) => `${id} 0 R`).join(' ')}] >>`;
|
|
||||||
|
|
||||||
let infoId = 0;
|
|
||||||
if (options.title !== undefined && !options.encrypted) infoId = add(`<< /Title ${pdfString(options.title)} /Producer (study-desk tests) >>`);
|
|
||||||
let encryptId = 0;
|
|
||||||
if (options.encrypted) {
|
|
||||||
const o = 'a1'.repeat(32);
|
|
||||||
const u = 'b2'.repeat(32);
|
|
||||||
encryptId = add(`<< /Filter /Standard /V 1 /R 2 /Length 40 /O <${o}> /U <${u}> /P -44 >>`);
|
|
||||||
}
|
|
||||||
|
|
||||||
let out = '%PDF-1.4\n%\xE2\xE3\xCF\xD3\n';
|
|
||||||
const offsets: number[] = [];
|
|
||||||
objects.forEach((body, index) => {
|
|
||||||
offsets.push(out.length);
|
|
||||||
out += `${index + 1} 0 obj\n${body}\nendobj\n`;
|
|
||||||
});
|
|
||||||
const xrefOffset = out.length;
|
|
||||||
out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
|
|
||||||
for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`;
|
|
||||||
out += `trailer\n<< /Size ${objects.length + 1} /Root ${catalogId} 0 R`;
|
|
||||||
if (infoId) out += ` /Info ${infoId} 0 R`;
|
|
||||||
if (encryptId) out += ` /Encrypt ${encryptId} 0 R /ID [<0123456789abcdef0123456789abcdef> <0123456789abcdef0123456789abcdef>]`;
|
|
||||||
out += ` >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
|
|
||||||
return latin1(out);
|
|
||||||
}
|
|
||||||
|
|
||||||
const xmlEscape = (text: string): string => text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
||||||
|
|
||||||
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
|
|
||||||
|
|
||||||
function coreProps(title: string | undefined): string {
|
|
||||||
return `${XML_HEADER}<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">${
|
|
||||||
title === undefined ? '' : `<dc:title>${xmlEscape(title)}</dc:title>`
|
|
||||||
}<dc:creator>Tests</dc:creator></cp:coreProperties>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// DOCX
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
export type DocxBlock =
|
|
||||||
| { type: 'heading'; level: number; text: string }
|
|
||||||
| { type: 'p'; text: string; bold?: string }
|
|
||||||
| { type: 'bullet'; text: string; level?: number }
|
|
||||||
| { type: 'number'; text: string; level?: number }
|
|
||||||
| { type: 'table'; rows: string[][] }
|
|
||||||
| { type: 'image' };
|
|
||||||
|
|
||||||
const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';
|
|
||||||
|
|
||||||
function docxRun(text: string, bold = false): string {
|
|
||||||
return `<w:r>${bold ? '<w:rPr><w:b/></w:rPr>' : ''}<w:t xml:space="preserve">${xmlEscape(text)}</w:t></w:r>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function docxBlock(block: DocxBlock): string {
|
|
||||||
switch (block.type) {
|
|
||||||
case 'heading':
|
|
||||||
return `<w:p><w:pPr><w:pStyle w:val="Heading${block.level}"/></w:pPr>${docxRun(block.text)}</w:p>`;
|
|
||||||
case 'p':
|
|
||||||
return `<w:p>${block.bold ? docxRun(block.bold, true) : ''}${docxRun(block.text)}</w:p>`;
|
|
||||||
case 'bullet':
|
|
||||||
case 'number':
|
|
||||||
return `<w:p><w:pPr><w:numPr><w:ilvl w:val="${block.level ?? 0}"/><w:numId w:val="${block.type === 'bullet' ? 1 : 2}"/></w:numPr></w:pPr>${docxRun(block.text)}</w:p>`;
|
|
||||||
case 'table':
|
|
||||||
return `<w:tbl>${block.rows
|
|
||||||
.map((row) => `<w:tr>${row.map((cell) => `<w:tc><w:p>${docxRun(cell)}</w:p></w:tc>`).join('')}</w:tr>`)
|
|
||||||
.join('')}</w:tbl>`;
|
|
||||||
case 'image':
|
|
||||||
return '<w:p/>';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Builds a minimal DOCX with headings (style ids Heading1..6), lists, tables. */
|
|
||||||
export function buildDocx(blocks: DocxBlock[], options: { title?: string; extraFiles?: Record<string, Uint8Array> } = {}): Uint8Array {
|
|
||||||
const levels = (format: string) =>
|
|
||||||
[0, 1, 2].map((lvl) => `<w:lvl w:ilvl="${lvl}"><w:start w:val="1"/><w:numFmt w:val="${format}"/><w:lvlText w:val="%${lvl + 1}."/></w:lvl>`).join('');
|
|
||||||
const files: Record<string, Uint8Array> = {
|
|
||||||
'[Content_Types].xml': strToU8(
|
|
||||||
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Default Extension="png" ContentType="image/png"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/></Types>`,
|
|
||||||
),
|
|
||||||
'_rels/.rels': strToU8(
|
|
||||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/></Relationships>`,
|
|
||||||
),
|
|
||||||
'word/_rels/document.xml.rels': strToU8(
|
|
||||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/></Relationships>`,
|
|
||||||
),
|
|
||||||
'word/numbering.xml': strToU8(
|
|
||||||
`${XML_HEADER}<w:numbering ${W_NS}><w:abstractNum w:abstractNumId="0">${levels('bullet')}</w:abstractNum><w:abstractNum w:abstractNumId="1">${levels('decimal')}</w:abstractNum><w:num w:numId="1"><w:abstractNumId w:val="0"/></w:num><w:num w:numId="2"><w:abstractNumId w:val="1"/></w:num></w:numbering>`,
|
|
||||||
),
|
|
||||||
'word/document.xml': strToU8(`${XML_HEADER}<w:document ${W_NS}><w:body>${blocks.map(docxBlock).join('')}<w:sectPr/></w:body></w:document>`),
|
|
||||||
'docProps/core.xml': strToU8(coreProps(options.title)),
|
|
||||||
...options.extraFiles,
|
|
||||||
};
|
|
||||||
return zipSync(files);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// PPTX
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
export interface PptxSlide {
|
|
||||||
/** File number: the slide is stored as ppt/slides/slide<file>.xml regardless of its position. */
|
|
||||||
file: number;
|
|
||||||
title?: string;
|
|
||||||
/** Body placeholder paragraphs. */
|
|
||||||
bullets?: Array<string | { text: string; level: number }>;
|
|
||||||
/** A free text box (not a placeholder). */
|
|
||||||
textBox?: string[];
|
|
||||||
table?: string[][];
|
|
||||||
notes?: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
const P_NS =
|
|
||||||
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"';
|
|
||||||
|
|
||||||
function aParagraph(text: string, level = 0): string {
|
|
||||||
return `<a:p>${level ? `<a:pPr lvl="${level}"/>` : ''}<a:r><a:rPr lang="en-US"/><a:t>${xmlEscape(text)}</a:t></a:r></a:p>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function shape(id: number, placeholder: string | null, paragraphs: string): string {
|
|
||||||
const ph = placeholder === null ? '' : placeholder === 'body' ? '<p:ph idx="1"/>' : `<p:ph type="${placeholder}"/>`;
|
|
||||||
return `<p:sp><p:nvSpPr><p:cNvPr id="${id}" name="Shape ${id}"/><p:cNvSpPr/><p:nvPr>${ph}</p:nvPr></p:nvSpPr><p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>${paragraphs}</p:txBody></p:sp>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function slideXml(slide: PptxSlide): string {
|
|
||||||
const shapes: string[] = [];
|
|
||||||
let id = 2;
|
|
||||||
if (slide.title !== undefined) shapes.push(shape(id++, 'title', aParagraph(slide.title)));
|
|
||||||
if (slide.bullets) {
|
|
||||||
shapes.push(
|
|
||||||
shape(id++, 'body', slide.bullets.map((b) => (typeof b === 'string' ? aParagraph(b) : aParagraph(b.text, b.level))).join('')),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (slide.textBox) shapes.push(shape(id++, null, slide.textBox.map((line) => aParagraph(line)).join('')));
|
|
||||||
if (slide.table) {
|
|
||||||
const rows = slide.table
|
|
||||||
.map((row) => `<a:tr h="370840">${row.map((cell) => `<a:tc><a:txBody><a:bodyPr/><a:lstStyle/>${aParagraph(cell)}</a:txBody><a:tcPr/></a:tc>`).join('')}</a:tr>`)
|
|
||||||
.join('');
|
|
||||||
shapes.push(
|
|
||||||
`<p:graphicFrame><p:nvGraphicFramePr><p:cNvPr id="${id++}" name="Table"/><p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr><p:xfrm/><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/table"><a:tbl><a:tblPr/><a:tblGrid/>${rows}</a:tbl></a:graphicData></a:graphic></p:graphicFrame>`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
// Every slide also carries a slide-number placeholder, which must be ignored.
|
|
||||||
shapes.push(shape(id++, 'sldNum', '<a:p><a:fld id="{1}" type="slidenum"><a:t>9</a:t></a:fld></a:p>'));
|
|
||||||
return `${XML_HEADER}<p:sld ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shapes.join('')}</p:spTree></p:cSld></p:sld>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function notesXml(lines: string[]): string {
|
|
||||||
return `${XML_HEADER}<p:notes ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shape(
|
|
||||||
2,
|
|
||||||
'sldImg',
|
|
||||||
'',
|
|
||||||
)}${shape(3, 'body', lines.map((line) => aParagraph(line)).join(''))}${shape(4, 'sldNum', aParagraph('9'))}</p:spTree></p:cSld></p:notes>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Builds a minimal PPTX. Slides appear in the order given, whatever their file numbers. */
|
|
||||||
export function buildPptx(slides: PptxSlide[], options: { title?: string } = {}): Uint8Array {
|
|
||||||
const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships';
|
|
||||||
const files: Record<string, Uint8Array> = {
|
|
||||||
'[Content_Types].xml': strToU8(
|
|
||||||
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>`,
|
|
||||||
),
|
|
||||||
'_rels/.rels': strToU8(
|
|
||||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/officeDocument" Target="ppt/presentation.xml"/></Relationships>`,
|
|
||||||
),
|
|
||||||
'docProps/core.xml': strToU8(coreProps(options.title)),
|
|
||||||
'ppt/presentation.xml': strToU8(
|
|
||||||
`${XML_HEADER}<p:presentation ${P_NS}><p:sldMasterIdLst/><p:sldIdLst>${slides
|
|
||||||
.map((slide, index) => `<p:sldId id="${256 + index}" r:id="rId${slide.file + 10}"/>`)
|
|
||||||
.join('')}</p:sldIdLst></p:presentation>`,
|
|
||||||
),
|
|
||||||
'ppt/_rels/presentation.xml.rels': strToU8(
|
|
||||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">${slides
|
|
||||||
.map((slide) => `<Relationship Id="rId${slide.file + 10}" Type="${REL}/slide" Target="slides/slide${slide.file}.xml"/>`)
|
|
||||||
.join('')}</Relationships>`,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
for (const slide of slides) {
|
|
||||||
files[`ppt/slides/slide${slide.file}.xml`] = strToU8(slideXml(slide));
|
|
||||||
if (slide.notes) {
|
|
||||||
files[`ppt/slides/_rels/slide${slide.file}.xml.rels`] = strToU8(
|
|
||||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/notesSlide" Target="../notesSlides/notesSlide${slide.file}.xml"/></Relationships>`,
|
|
||||||
);
|
|
||||||
files[`ppt/notesSlides/notesSlide${slide.file}.xml`] = strToU8(notesXml(slide.notes));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return zipSync(files);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Misc
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/** PNG signature plus an IHDR chunk header: enough for signature sniffing. */
|
|
||||||
export const TINY_PNG = Uint8Array.from([
|
|
||||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06,
|
|
||||||
0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89,
|
|
||||||
]);
|
|
||||||
|
|
||||||
export const TINY_JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01]);
|
|
||||||
export const TINY_GIF = Uint8Array.from([...'GIF89a'].map((ch) => ch.charCodeAt(0)).concat([1, 0, 1, 0, 0, 0, 0]));
|
|
||||||
export const TINY_WEBP = Uint8Array.from([...'RIFF'].map((c) => c.charCodeAt(0)).concat([4, 0, 0, 0], [...'WEBP'].map((c) => c.charCodeAt(0))));
|
|
||||||
|
|
||||||
/** A stored zip whose central directory claims a tiny size for a highly compressible entry. */
|
|
||||||
export function buildLyingZip(name: string, realBytes: number): Uint8Array {
|
|
||||||
const zip = zipSync({ [name]: new Uint8Array(realBytes), 'word/document.xml': strToU8('<w:document/>') }, { level: 9 });
|
|
||||||
// Patch the uncompressed size of `name` in its central directory record to 16 bytes.
|
|
||||||
const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
|
|
||||||
for (let i = 0; i + 46 < zip.length; i++) {
|
|
||||||
if (view.getUint32(i, true) !== 0x02014b50) continue;
|
|
||||||
const nameLen = view.getUint16(i + 28, true);
|
|
||||||
const entryName = new TextDecoder().decode(zip.subarray(i + 46, i + 46 + nameLen));
|
|
||||||
if (entryName === name) view.setUint32(i + 24, 16, true);
|
|
||||||
}
|
|
||||||
return zip;
|
|
||||||
}
|
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
/** Contract for the document extraction + chunking pipeline. */
|
|
||||||
|
|
||||||
export type DocKind = 'pdf' | 'docx' | 'pptx' | 'text' | 'markdown' | 'image';
|
|
||||||
|
|
||||||
export interface ExtractedSection {
|
|
||||||
/** Plain text (Markdown allowed) for this page / slide / section. */
|
|
||||||
text: string;
|
|
||||||
/** 1-based page (PDF) or slide (PPTX) number, when the format has one. */
|
|
||||||
page?: number;
|
|
||||||
/** Nearest heading or slide title, when known. */
|
|
||||||
heading?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ExtractResult {
|
|
||||||
kind: DocKind;
|
|
||||||
sections: ExtractedSection[];
|
|
||||||
/** Page or slide count for paged formats. */
|
|
||||||
pageCount?: number;
|
|
||||||
/** Non-fatal problems worth showing the user (e.g. "3 pages had no extractable text"). */
|
|
||||||
warnings: string[];
|
|
||||||
/** True when the file has little or no machine-readable text (scans, images) and needs AI vision/OCR. */
|
|
||||||
needsOcr: boolean;
|
|
||||||
/** Document title from metadata, when present. */
|
|
||||||
title?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface DocChunk {
|
|
||||||
seq: number;
|
|
||||||
text: string;
|
|
||||||
heading: string;
|
|
||||||
pageStart: number | null;
|
|
||||||
pageEnd: number | null;
|
|
||||||
tokens: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ChunkOptions {
|
|
||||||
/** Preferred chunk size in estimated tokens. */
|
|
||||||
targetTokens?: number;
|
|
||||||
/** Hard ceiling; a chunk is split before exceeding it. */
|
|
||||||
maxTokens?: number;
|
|
||||||
/** Tokens of trailing context repeated at the start of the next chunk. */
|
|
||||||
overlapTokens?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Thrown for unrecoverable extraction problems; `message` is user-facing. */
|
|
||||||
export class ExtractionError extends Error {
|
|
||||||
code: 'unsupported' | 'encrypted' | 'corrupt' | 'empty' | 'too_large' | 'failed';
|
|
||||||
constructor(code: ExtractionError['code'], message: string, opts?: { cause?: unknown }) {
|
|
||||||
super(message, opts);
|
|
||||||
this.name = 'ExtractionError';
|
|
||||||
this.code = code;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,213 +0,0 @@
|
||||||
/**
|
|
||||||
* Minimal, defensive ZIP reader for OOXML packages (DOCX / PPTX).
|
|
||||||
*
|
|
||||||
* Reads the central directory itself and inflates entries through fflate's
|
|
||||||
* streaming inflater in small input slices, so both memory and CPU are bounded
|
|
||||||
* by the limits below even when the archive lies about its sizes (zip bombs).
|
|
||||||
*/
|
|
||||||
import { Inflate } from 'fflate';
|
|
||||||
|
|
||||||
export interface ZipEntry {
|
|
||||||
name: string;
|
|
||||||
method: number;
|
|
||||||
compressedSize: number;
|
|
||||||
size: number;
|
|
||||||
offset: number;
|
|
||||||
encrypted: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ZipLimits {
|
|
||||||
/** Largest uncompressed size accepted for a single entry. */
|
|
||||||
maxEntryBytes: number;
|
|
||||||
/** Largest total of uncompressed bytes read through one reader. */
|
|
||||||
maxTotalBytes: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ZipError extends Error {
|
|
||||||
code: 'corrupt' | 'too_large';
|
|
||||||
constructor(code: ZipError['code'], message: string) {
|
|
||||||
super(message);
|
|
||||||
this.name = 'ZipError';
|
|
||||||
this.code = code;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const EOCD_SIG = 0x06054b50;
|
|
||||||
const ZIP64_LOCATOR_SIG = 0x07064b50;
|
|
||||||
const ZIP64_EOCD_SIG = 0x06064b50;
|
|
||||||
const CENTRAL_SIG = 0x02014b50;
|
|
||||||
const LOCAL_SIG = 0x04034b50;
|
|
||||||
const MAX_ENTRIES = 20_000;
|
|
||||||
const INFLATE_STEP = 64 * 1024;
|
|
||||||
|
|
||||||
const nameDecoder = new TextDecoder('utf-8');
|
|
||||||
|
|
||||||
function u16(b: Uint8Array, o: number): number {
|
|
||||||
return b[o] | (b[o + 1] << 8);
|
|
||||||
}
|
|
||||||
|
|
||||||
function u32(b: Uint8Array, o: number): number {
|
|
||||||
return (b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)) >>> 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
function u64(b: Uint8Array, o: number): number {
|
|
||||||
return u32(b, o) + u32(b, o + 4) * 0x1_0000_0000;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Parses the central directory. Throws ZipError('corrupt') when it cannot be read. */
|
|
||||||
export function listZip(bytes: Uint8Array): ZipEntry[] {
|
|
||||||
const len = bytes.length;
|
|
||||||
if (len < 22) throw new ZipError('corrupt', 'Not a zip archive');
|
|
||||||
let eocd = -1;
|
|
||||||
const stop = Math.max(0, len - 22 - 0xffff);
|
|
||||||
for (let i = len - 22; i >= stop; i--) {
|
|
||||||
if (bytes[i] === 0x50 && u32(bytes, i) === EOCD_SIG) {
|
|
||||||
eocd = i;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (eocd < 0) throw new ZipError('corrupt', 'Zip directory not found');
|
|
||||||
|
|
||||||
let count = u16(bytes, eocd + 10);
|
|
||||||
let dirOffset = u32(bytes, eocd + 16);
|
|
||||||
if (count === 0xffff || dirOffset === 0xffffffff) {
|
|
||||||
const locator = eocd - 20;
|
|
||||||
if (locator < 0 || u32(bytes, locator) !== ZIP64_LOCATOR_SIG) throw new ZipError('corrupt', 'Bad zip64 locator');
|
|
||||||
const z = u64(bytes, locator + 8);
|
|
||||||
if (z + 56 > len || u32(bytes, z) !== ZIP64_EOCD_SIG) throw new ZipError('corrupt', 'Bad zip64 directory');
|
|
||||||
count = u64(bytes, z + 32);
|
|
||||||
dirOffset = u64(bytes, z + 48);
|
|
||||||
}
|
|
||||||
if (count > MAX_ENTRIES) throw new ZipError('too_large', 'Too many files in archive');
|
|
||||||
|
|
||||||
const entries: ZipEntry[] = [];
|
|
||||||
let p = dirOffset;
|
|
||||||
for (let i = 0; i < count; i++) {
|
|
||||||
if (p + 46 > len || u32(bytes, p) !== CENTRAL_SIG) throw new ZipError('corrupt', 'Bad zip directory entry');
|
|
||||||
const flags = u16(bytes, p + 8);
|
|
||||||
const method = u16(bytes, p + 10);
|
|
||||||
let compressedSize = u32(bytes, p + 20);
|
|
||||||
let size = u32(bytes, p + 24);
|
|
||||||
const nameLen = u16(bytes, p + 28);
|
|
||||||
const extraLen = u16(bytes, p + 30);
|
|
||||||
const commentLen = u16(bytes, p + 32);
|
|
||||||
let offset = u32(bytes, p + 42);
|
|
||||||
const nameStart = p + 46;
|
|
||||||
const extraStart = nameStart + nameLen;
|
|
||||||
const next = extraStart + extraLen + commentLen;
|
|
||||||
if (next > len) throw new ZipError('corrupt', 'Truncated zip directory');
|
|
||||||
if (size === 0xffffffff || compressedSize === 0xffffffff || offset === 0xffffffff) {
|
|
||||||
// zip64 extended information: only the saturated fields are present, in this order.
|
|
||||||
let e = extraStart;
|
|
||||||
const extraEnd = extraStart + extraLen;
|
|
||||||
while (e + 4 <= extraEnd) {
|
|
||||||
const id = u16(bytes, e);
|
|
||||||
const dataLen = u16(bytes, e + 2);
|
|
||||||
if (id === 1) {
|
|
||||||
let q = e + 4;
|
|
||||||
if (size === 0xffffffff && q + 8 <= extraEnd) (size = u64(bytes, q)), (q += 8);
|
|
||||||
if (compressedSize === 0xffffffff && q + 8 <= extraEnd) (compressedSize = u64(bytes, q)), (q += 8);
|
|
||||||
if (offset === 0xffffffff && q + 8 <= extraEnd) offset = u64(bytes, q);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
e += 4 + dataLen;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const name = nameDecoder.decode(bytes.subarray(nameStart, extraStart)).replace(/\\/g, '/');
|
|
||||||
entries.push({ name, method, compressedSize, size, offset, encrypted: (flags & 1) === 1 });
|
|
||||||
p = next;
|
|
||||||
}
|
|
||||||
return entries;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Random-access reader over a zip held in memory, with per-entry and total size limits. */
|
|
||||||
export class ZipReader {
|
|
||||||
private readonly bytes: Uint8Array;
|
|
||||||
private readonly limits: ZipLimits;
|
|
||||||
private readonly byName = new Map<string, ZipEntry>();
|
|
||||||
private totalRead = 0;
|
|
||||||
readonly entries: ZipEntry[];
|
|
||||||
|
|
||||||
constructor(bytes: Uint8Array, limits: ZipLimits) {
|
|
||||||
this.bytes = bytes;
|
|
||||||
this.limits = limits;
|
|
||||||
this.entries = listZip(bytes);
|
|
||||||
for (const entry of this.entries) if (!this.byName.has(entry.name)) this.byName.set(entry.name, entry);
|
|
||||||
}
|
|
||||||
|
|
||||||
has(name: string): boolean {
|
|
||||||
return this.byName.has(name);
|
|
||||||
}
|
|
||||||
|
|
||||||
names(): string[] {
|
|
||||||
return this.entries.map((entry) => entry.name);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Uncompressed bytes of an entry, or null when it does not exist. */
|
|
||||||
read(name: string): Uint8Array | null {
|
|
||||||
const entry = this.byName.get(name);
|
|
||||||
if (!entry) return null;
|
|
||||||
if (entry.encrypted) throw new ZipError('corrupt', 'Encrypted zip entry');
|
|
||||||
if (entry.size > this.limits.maxEntryBytes) throw new ZipError('too_large', `Entry too large: ${name}`);
|
|
||||||
if (this.totalRead + entry.size > this.limits.maxTotalBytes) throw new ZipError('too_large', 'Archive too large');
|
|
||||||
|
|
||||||
const b = this.bytes;
|
|
||||||
const o = entry.offset;
|
|
||||||
if (o + 30 > b.length || u32(b, o) !== LOCAL_SIG) throw new ZipError('corrupt', 'Bad zip file header');
|
|
||||||
const dataStart = o + 30 + u16(b, o + 26) + u16(b, o + 28);
|
|
||||||
const dataEnd = dataStart + entry.compressedSize;
|
|
||||||
if (dataEnd > b.length) throw new ZipError('corrupt', 'Truncated zip entry');
|
|
||||||
const compressed = b.subarray(dataStart, dataEnd);
|
|
||||||
|
|
||||||
let out: Uint8Array;
|
|
||||||
if (entry.method === 0) {
|
|
||||||
if (compressed.length !== entry.size) throw new ZipError('corrupt', 'Zip entry size mismatch');
|
|
||||||
out = compressed;
|
|
||||||
} else if (entry.method === 8) {
|
|
||||||
out = this.inflate(compressed, entry.size);
|
|
||||||
} else {
|
|
||||||
throw new ZipError('corrupt', `Unsupported zip compression method ${entry.method}`);
|
|
||||||
}
|
|
||||||
this.totalRead += out.length;
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
readText(name: string): string | null {
|
|
||||||
const data = this.read(name);
|
|
||||||
if (!data) return null;
|
|
||||||
let text = new TextDecoder('utf-8').decode(data);
|
|
||||||
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
|
|
||||||
return text;
|
|
||||||
}
|
|
||||||
|
|
||||||
private inflate(compressed: Uint8Array, expected: number): Uint8Array {
|
|
||||||
const out = new Uint8Array(expected);
|
|
||||||
let written = 0;
|
|
||||||
let overflow = false;
|
|
||||||
const inflater = new Inflate((chunk) => {
|
|
||||||
if (overflow) return;
|
|
||||||
if (written + chunk.length > expected) {
|
|
||||||
overflow = true;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
out.set(chunk, written);
|
|
||||||
written += chunk.length;
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
if (compressed.length === 0) {
|
|
||||||
inflater.push(compressed, true);
|
|
||||||
} else {
|
|
||||||
for (let p = 0; p < compressed.length && !overflow; p += INFLATE_STEP) {
|
|
||||||
const end = Math.min(p + INFLATE_STEP, compressed.length);
|
|
||||||
inflater.push(compressed.subarray(p, end), end === compressed.length);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
throw new ZipError('corrupt', 'Zip entry could not be decompressed');
|
|
||||||
}
|
|
||||||
// More data than the directory declared is the signature of a zip bomb.
|
|
||||||
if (overflow) throw new ZipError('too_large', 'Zip entry larger than declared');
|
|
||||||
if (written !== expected) throw new ZipError('corrupt', 'Zip entry size mismatch');
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,63 +0,0 @@
|
||||||
import type { Context } from 'hono';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import type { Config } from './config.ts';
|
|
||||||
import type { DB } from './db.ts';
|
|
||||||
|
|
||||||
/** Everything a route needs. Built once in main.ts (or per test). */
|
|
||||||
export interface AppContext {
|
|
||||||
db: DB;
|
|
||||||
config: Config;
|
|
||||||
log: (level: 'info' | 'warn' | 'error', message: string, extra?: unknown) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class HttpError extends Error {
|
|
||||||
status: number;
|
|
||||||
code: string;
|
|
||||||
details?: unknown;
|
|
||||||
constructor(status: number, code: string, message: string, details?: unknown) {
|
|
||||||
super(message);
|
|
||||||
this.name = 'HttpError';
|
|
||||||
this.status = status;
|
|
||||||
this.code = code;
|
|
||||||
this.details = details;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const notFound = (what = 'Item') => new HttpError(404, 'not_found', `${what} not found`);
|
|
||||||
export const badRequest = (message: string, details?: unknown) => new HttpError(400, 'bad_request', message, details);
|
|
||||||
export const conflict = (message: string) => new HttpError(409, 'conflict', message);
|
|
||||||
|
|
||||||
export interface FieldIssue {
|
|
||||||
path: string;
|
|
||||||
message: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function zodIssues(error: z.ZodError): FieldIssue[] {
|
|
||||||
return error.issues.map((i) => ({ path: i.path.join('.'), message: i.message }));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function parse<S extends z.ZodType>(schema: S, data: unknown): z.output<S> {
|
|
||||||
const result = schema.safeParse(data);
|
|
||||||
if (!result.success) {
|
|
||||||
const issues = zodIssues(result.error);
|
|
||||||
const first = issues[0];
|
|
||||||
throw new HttpError(422, 'validation', first ? `${first.path ? `${first.path}: ` : ''}${first.message}` : 'Invalid input', issues);
|
|
||||||
}
|
|
||||||
return result.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function readBody<S extends z.ZodType>(c: Context, schema: S): Promise<z.output<S>> {
|
|
||||||
let json: unknown;
|
|
||||||
try {
|
|
||||||
json = await c.req.json();
|
|
||||||
} catch {
|
|
||||||
throw badRequest('Request body must be JSON');
|
|
||||||
}
|
|
||||||
return parse(schema, json);
|
|
||||||
}
|
|
||||||
|
|
||||||
export const idParam = (c: Context, name = 'id'): string => {
|
|
||||||
const v = c.req.param(name);
|
|
||||||
if (!v || v.length > 64) throw badRequest('Invalid id');
|
|
||||||
return v;
|
|
||||||
};
|
|
||||||
|
|
@ -1,77 +0,0 @@
|
||||||
import { serve } from '@hono/node-server';
|
|
||||||
import { tidyDrafts } from './ai/generate.ts';
|
|
||||||
import { createApp } from './app.ts';
|
|
||||||
import { initAuth } from './auth.ts';
|
|
||||||
import { loadConfig } from './config.ts';
|
|
||||||
import { openDatabase } from './db.ts';
|
|
||||||
import type { AppContext } from './http.ts';
|
|
||||||
import { APP_VERSION } from './routes/data.ts';
|
|
||||||
import { startBackupSchedule } from './services/backup.ts';
|
|
||||||
import { resumeQueue } from './services/documents.ts';
|
|
||||||
import { seedTemplates } from './services/templates.ts';
|
|
||||||
|
|
||||||
function log(level: 'info' | 'warn' | 'error', message: string, extra?: unknown): void {
|
|
||||||
const line = `${new Date().toISOString()} ${level.toUpperCase().padEnd(5)} ${message}`;
|
|
||||||
const detail = extra instanceof Error ? (extra.stack ?? extra.message) : extra !== undefined ? JSON.stringify(extra) : '';
|
|
||||||
(level === 'error' ? console.error : console.log)(detail ? `${line}\n${detail}` : line);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
|
||||||
// Optional .env file for running outside Docker. Real environment variables always win.
|
|
||||||
try {
|
|
||||||
process.loadEnvFile('.env');
|
|
||||||
} catch {
|
|
||||||
/* no .env file: that is the normal case in a container */
|
|
||||||
}
|
|
||||||
const config = loadConfig();
|
|
||||||
log('info', `Study Desk ${APP_VERSION} starting (data: ${config.dataDir})`);
|
|
||||||
|
|
||||||
const { db, report } = openDatabase(config.dbPath, config.backupsDir);
|
|
||||||
if (report.applied.length > 0) log('info', `Database migrated: ${report.applied.join(', ')}${report.backup ? ` (backup: ${report.backup})` : ''}`);
|
|
||||||
|
|
||||||
const app: AppContext = { db, config, log };
|
|
||||||
await initAuth(app);
|
|
||||||
seedTemplates(db);
|
|
||||||
tidyDrafts(db);
|
|
||||||
const resumed = resumeQueue(app);
|
|
||||||
if (resumed > 0) log('info', `Resuming processing of ${resumed} document(s)`);
|
|
||||||
const stopBackups = startBackupSchedule(app);
|
|
||||||
|
|
||||||
const server = serve({ fetch: createApp(app).fetch, port: config.port, hostname: config.host }, (info) => {
|
|
||||||
log('info', `Listening on http://${info.address === '::' || info.address === '0.0.0.0' ? 'localhost' : info.address}:${info.port}`);
|
|
||||||
if (!config.webDir) log('warn', 'No built web app found (dist/web). API only: use the Vite dev server for the UI.');
|
|
||||||
});
|
|
||||||
|
|
||||||
let closing = false;
|
|
||||||
const shutdown = (signal: string) => {
|
|
||||||
if (closing) return;
|
|
||||||
closing = true;
|
|
||||||
log('info', `${signal} received, shutting down`);
|
|
||||||
stopBackups();
|
|
||||||
server.close(() => {
|
|
||||||
try {
|
|
||||||
db.close();
|
|
||||||
} catch {
|
|
||||||
/* already closed */
|
|
||||||
}
|
|
||||||
process.exit(0);
|
|
||||||
});
|
|
||||||
// Do not let an open event stream hold the process forever.
|
|
||||||
setTimeout(() => {
|
|
||||||
try {
|
|
||||||
db.close();
|
|
||||||
} catch {
|
|
||||||
/* ignore */
|
|
||||||
}
|
|
||||||
process.exit(0);
|
|
||||||
}, 5000).unref();
|
|
||||||
};
|
|
||||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
|
||||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
|
||||||
process.on('unhandledRejection', (reason) => log('error', 'Unhandled promise rejection', reason));
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
log('error', 'Failed to start', err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
@ -1,372 +0,0 @@
|
||||||
/**
|
|
||||||
* Schema migrations. Append only: never edit a migration that has shipped.
|
|
||||||
* Timestamps are integer milliseconds since the Unix epoch. JSON lives in TEXT columns.
|
|
||||||
*/
|
|
||||||
export interface Migration {
|
|
||||||
id: number;
|
|
||||||
name: string;
|
|
||||||
sql: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
const init = `
|
|
||||||
CREATE TABLE settings (
|
|
||||||
key TEXT PRIMARY KEY,
|
|
||||||
value TEXT NOT NULL
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE sessions (
|
|
||||||
token_hash TEXT PRIMARY KEY,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
expires_at INTEGER NOT NULL,
|
|
||||||
last_seen_at INTEGER NOT NULL,
|
|
||||||
user_agent TEXT NOT NULL DEFAULT '',
|
|
||||||
ip TEXT NOT NULL DEFAULT ''
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE classes (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
code TEXT NOT NULL DEFAULT '',
|
|
||||||
description TEXT NOT NULL DEFAULT '',
|
|
||||||
color TEXT NOT NULL DEFAULT 'cobalt',
|
|
||||||
archived INTEGER NOT NULL DEFAULT 0,
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE folders (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX folders_class ON folders(class_id, sort_order);
|
|
||||||
|
|
||||||
CREATE TABLE quizzes (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
|
|
||||||
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
description TEXT NOT NULL DEFAULT '',
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX quizzes_class ON quizzes(class_id, sort_order);
|
|
||||||
|
|
||||||
CREATE TABLE questions (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
quiz_id TEXT NOT NULL REFERENCES quizzes(id) ON DELETE CASCADE,
|
|
||||||
type TEXT NOT NULL CHECK (type IN ('single','multi')),
|
|
||||||
stem TEXT NOT NULL,
|
|
||||||
options TEXT NOT NULL,
|
|
||||||
rationale TEXT NOT NULL DEFAULT '',
|
|
||||||
topic TEXT NOT NULL DEFAULT '',
|
|
||||||
difficulty TEXT NOT NULL DEFAULT 'medium',
|
|
||||||
tags TEXT NOT NULL DEFAULT '[]',
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX questions_quiz ON questions(quiz_id, sort_order);
|
|
||||||
|
|
||||||
CREATE TABLE attempts (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
quiz_id TEXT REFERENCES quizzes(id) ON DELETE SET NULL,
|
|
||||||
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
mode TEXT NOT NULL CHECK (mode IN ('practice','exam')),
|
|
||||||
kind TEXT NOT NULL DEFAULT 'full',
|
|
||||||
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','completed')),
|
|
||||||
settings TEXT NOT NULL DEFAULT '{}',
|
|
||||||
snapshot TEXT NOT NULL,
|
|
||||||
answers TEXT NOT NULL DEFAULT '{}',
|
|
||||||
flags TEXT NOT NULL DEFAULT '[]',
|
|
||||||
current_index INTEGER NOT NULL DEFAULT 0,
|
|
||||||
elapsed_sec INTEGER NOT NULL DEFAULT 0,
|
|
||||||
question_count INTEGER NOT NULL DEFAULT 0,
|
|
||||||
score REAL,
|
|
||||||
max_score REAL,
|
|
||||||
started_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL,
|
|
||||||
completed_at INTEGER
|
|
||||||
);
|
|
||||||
CREATE INDEX attempts_quiz ON attempts(quiz_id, status, completed_at);
|
|
||||||
CREATE INDEX attempts_status ON attempts(status, updated_at);
|
|
||||||
|
|
||||||
CREATE TABLE responses (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
attempt_id TEXT NOT NULL REFERENCES attempts(id) ON DELETE CASCADE,
|
|
||||||
question_id TEXT NOT NULL,
|
|
||||||
quiz_id TEXT,
|
|
||||||
class_id TEXT,
|
|
||||||
topic TEXT NOT NULL DEFAULT '',
|
|
||||||
score REAL NOT NULL,
|
|
||||||
correct INTEGER NOT NULL,
|
|
||||||
answered_at INTEGER NOT NULL,
|
|
||||||
UNIQUE (attempt_id, question_id)
|
|
||||||
);
|
|
||||||
CREATE INDEX responses_question ON responses(question_id, answered_at);
|
|
||||||
CREATE INDEX responses_time ON responses(answered_at);
|
|
||||||
CREATE INDEX responses_class ON responses(class_id, answered_at);
|
|
||||||
|
|
||||||
CREATE TABLE decks (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
|
|
||||||
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
description TEXT NOT NULL DEFAULT '',
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX decks_class ON decks(class_id, sort_order);
|
|
||||||
|
|
||||||
CREATE TABLE cards (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
deck_id TEXT NOT NULL REFERENCES decks(id) ON DELETE CASCADE,
|
|
||||||
front TEXT NOT NULL,
|
|
||||||
back TEXT NOT NULL,
|
|
||||||
hint TEXT NOT NULL DEFAULT '',
|
|
||||||
tags TEXT NOT NULL DEFAULT '[]',
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
suspended INTEGER NOT NULL DEFAULT 0,
|
|
||||||
state INTEGER NOT NULL DEFAULT 0,
|
|
||||||
due INTEGER,
|
|
||||||
stability REAL NOT NULL DEFAULT 0,
|
|
||||||
difficulty REAL NOT NULL DEFAULT 0,
|
|
||||||
elapsed_days INTEGER NOT NULL DEFAULT 0,
|
|
||||||
scheduled_days INTEGER NOT NULL DEFAULT 0,
|
|
||||||
learning_steps INTEGER NOT NULL DEFAULT 0,
|
|
||||||
reps INTEGER NOT NULL DEFAULT 0,
|
|
||||||
lapses INTEGER NOT NULL DEFAULT 0,
|
|
||||||
last_review INTEGER,
|
|
||||||
introduced_at INTEGER,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX cards_deck ON cards(deck_id, sort_order);
|
|
||||||
CREATE INDEX cards_due ON cards(state, due);
|
|
||||||
|
|
||||||
CREATE TABLE reviews (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
card_id TEXT NOT NULL,
|
|
||||||
deck_id TEXT,
|
|
||||||
class_id TEXT,
|
|
||||||
rating INTEGER NOT NULL,
|
|
||||||
state_before INTEGER NOT NULL DEFAULT 0,
|
|
||||||
scheduled_days INTEGER NOT NULL DEFAULT 0,
|
|
||||||
mode TEXT NOT NULL DEFAULT 'srs',
|
|
||||||
prev TEXT NOT NULL DEFAULT '',
|
|
||||||
reviewed_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX reviews_time ON reviews(reviewed_at);
|
|
||||||
CREATE INDEX reviews_card ON reviews(card_id, reviewed_at);
|
|
||||||
|
|
||||||
CREATE TABLE deck_sessions (
|
|
||||||
deck_id TEXT PRIMARY KEY REFERENCES decks(id) ON DELETE CASCADE,
|
|
||||||
state TEXT NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE notes (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
|
|
||||||
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
content TEXT NOT NULL DEFAULT '',
|
|
||||||
kind TEXT NOT NULL DEFAULT 'note',
|
|
||||||
source TEXT NOT NULL DEFAULT '{}',
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX notes_class ON notes(class_id, sort_order);
|
|
||||||
|
|
||||||
CREATE TABLE documents (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
|
|
||||||
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
filename TEXT NOT NULL,
|
|
||||||
mime TEXT NOT NULL DEFAULT '',
|
|
||||||
kind TEXT NOT NULL,
|
|
||||||
size INTEGER NOT NULL DEFAULT 0,
|
|
||||||
sha256 TEXT NOT NULL DEFAULT '',
|
|
||||||
storage_name TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL DEFAULT 'pending',
|
|
||||||
error TEXT NOT NULL DEFAULT '',
|
|
||||||
progress REAL NOT NULL DEFAULT 0,
|
|
||||||
page_count INTEGER,
|
|
||||||
char_count INTEGER NOT NULL DEFAULT 0,
|
|
||||||
token_count INTEGER NOT NULL DEFAULT 0,
|
|
||||||
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
||||||
needs_ocr INTEGER NOT NULL DEFAULT 0,
|
|
||||||
warnings TEXT NOT NULL DEFAULT '[]',
|
|
||||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX documents_class ON documents(class_id, sort_order);
|
|
||||||
|
|
||||||
CREATE TABLE chunks (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
||||||
class_id TEXT NOT NULL,
|
|
||||||
seq INTEGER NOT NULL,
|
|
||||||
text TEXT NOT NULL,
|
|
||||||
heading TEXT NOT NULL DEFAULT '',
|
|
||||||
page_start INTEGER,
|
|
||||||
page_end INTEGER,
|
|
||||||
tokens INTEGER NOT NULL DEFAULT 0
|
|
||||||
);
|
|
||||||
CREATE INDEX chunks_doc ON chunks(document_id, seq);
|
|
||||||
|
|
||||||
CREATE VIRTUAL TABLE chunks_fts USING fts5(text, heading, content='chunks', content_rowid='id', tokenize='porter unicode61');
|
|
||||||
CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN
|
|
||||||
INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
|
|
||||||
END;
|
|
||||||
CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN
|
|
||||||
INSERT INTO chunks_fts(chunks_fts, rowid, text, heading) VALUES ('delete', old.id, old.text, old.heading);
|
|
||||||
END;
|
|
||||||
CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN
|
|
||||||
INSERT INTO chunks_fts(chunks_fts, rowid, text, heading) VALUES ('delete', old.id, old.text, old.heading);
|
|
||||||
INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
|
|
||||||
END;
|
|
||||||
|
|
||||||
CREATE TABLE shares (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
token TEXT NOT NULL UNIQUE,
|
|
||||||
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','folder','class','note','document')),
|
|
||||||
target_id TEXT NOT NULL,
|
|
||||||
enabled INTEGER NOT NULL DEFAULT 1,
|
|
||||||
view_count INTEGER NOT NULL DEFAULT 0,
|
|
||||||
last_viewed_at INTEGER,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL,
|
|
||||||
UNIQUE (kind, target_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE templates (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','generation')),
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
description TEXT NOT NULL DEFAULT '',
|
|
||||||
payload TEXT NOT NULL,
|
|
||||||
builtin INTEGER NOT NULL DEFAULT 0,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE media (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
filename TEXT NOT NULL,
|
|
||||||
mime TEXT NOT NULL,
|
|
||||||
size INTEGER NOT NULL,
|
|
||||||
created_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE ai_providers (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
kind TEXT NOT NULL,
|
|
||||||
base_url TEXT,
|
|
||||||
api_key_enc TEXT,
|
|
||||||
headers_enc TEXT,
|
|
||||||
enabled INTEGER NOT NULL DEFAULT 1,
|
|
||||||
last_test_at INTEGER,
|
|
||||||
last_test_ok INTEGER,
|
|
||||||
last_test_message TEXT NOT NULL DEFAULT '',
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE ai_models (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
provider_id TEXT NOT NULL REFERENCES ai_providers(id) ON DELETE CASCADE,
|
|
||||||
model TEXT NOT NULL,
|
|
||||||
label TEXT NOT NULL DEFAULT '',
|
|
||||||
source TEXT NOT NULL DEFAULT 'discovered',
|
|
||||||
enabled INTEGER NOT NULL DEFAULT 0,
|
|
||||||
capabilities TEXT NOT NULL DEFAULT '{}',
|
|
||||||
context_window INTEGER,
|
|
||||||
max_output INTEGER,
|
|
||||||
price_in REAL,
|
|
||||||
price_out REAL,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL,
|
|
||||||
UNIQUE (provider_id, model)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE ai_usage (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
at INTEGER NOT NULL,
|
|
||||||
provider_id TEXT,
|
|
||||||
provider_name TEXT NOT NULL DEFAULT '',
|
|
||||||
model TEXT NOT NULL DEFAULT '',
|
|
||||||
task TEXT NOT NULL DEFAULT '',
|
|
||||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
|
||||||
output_tokens INTEGER NOT NULL DEFAULT 0,
|
|
||||||
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
|
||||||
cost REAL,
|
|
||||||
ok INTEGER NOT NULL DEFAULT 1,
|
|
||||||
error TEXT NOT NULL DEFAULT '',
|
|
||||||
duration_ms INTEGER NOT NULL DEFAULT 0
|
|
||||||
);
|
|
||||||
CREATE INDEX ai_usage_at ON ai_usage(at);
|
|
||||||
|
|
||||||
CREATE TABLE conversations (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
title TEXT NOT NULL DEFAULT '',
|
|
||||||
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE messages (
|
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
|
||||||
role TEXT NOT NULL,
|
|
||||||
content TEXT NOT NULL,
|
|
||||||
meta TEXT NOT NULL DEFAULT '{}',
|
|
||||||
created_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX messages_conv ON messages(conversation_id, id);
|
|
||||||
|
|
||||||
CREATE TABLE drafts (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','note')),
|
|
||||||
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
|
|
||||||
folder_id TEXT,
|
|
||||||
title TEXT NOT NULL DEFAULT '',
|
|
||||||
payload TEXT NOT NULL DEFAULT '{}',
|
|
||||||
meta TEXT NOT NULL DEFAULT '{}',
|
|
||||||
status TEXT NOT NULL DEFAULT 'ready',
|
|
||||||
progress REAL NOT NULL DEFAULT 0,
|
|
||||||
progress_label TEXT NOT NULL DEFAULT '',
|
|
||||||
error TEXT NOT NULL DEFAULT '',
|
|
||||||
saved_kind TEXT,
|
|
||||||
saved_id TEXT,
|
|
||||||
conversation_id TEXT,
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
updated_at INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX drafts_status ON drafts(status, updated_at);
|
|
||||||
|
|
||||||
CREATE TABLE ai_actions (
|
|
||||||
id TEXT PRIMARY KEY,
|
|
||||||
conversation_id TEXT,
|
|
||||||
kind TEXT NOT NULL,
|
|
||||||
summary TEXT NOT NULL DEFAULT '',
|
|
||||||
payload TEXT NOT NULL DEFAULT '{}',
|
|
||||||
status TEXT NOT NULL DEFAULT 'pending',
|
|
||||||
result TEXT NOT NULL DEFAULT '',
|
|
||||||
created_at INTEGER NOT NULL,
|
|
||||||
resolved_at INTEGER
|
|
||||||
);
|
|
||||||
`;
|
|
||||||
|
|
||||||
export const MIGRATIONS: Migration[] = [{ id: 1, name: 'init', sql: init }];
|
|
||||||
|
|
@ -1,271 +0,0 @@
|
||||||
import { Hono, type Context } from 'hono';
|
|
||||||
import { streamSSE } from 'hono/streaming';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import { assistantRequestSchema, displayMessages, listConversations, resolveAction, runAssistantTurn, type AssistantEvent } from '../ai/assistant.ts';
|
|
||||||
import {
|
|
||||||
cancelGeneration,
|
|
||||||
createDraft,
|
|
||||||
discardDraft,
|
|
||||||
draftSaveSchema,
|
|
||||||
generateRequestSchema,
|
|
||||||
getDraft,
|
|
||||||
improveRequestSchema,
|
|
||||||
listDrafts,
|
|
||||||
saveDraft,
|
|
||||||
startGeneration,
|
|
||||||
startImprove,
|
|
||||||
updateDraftPayload,
|
|
||||||
} from '../ai/generate.ts';
|
|
||||||
import { CATALOG_UPDATED, listAdapterMeta } from '../ai/providers/index.ts';
|
|
||||||
import {
|
|
||||||
addManualModel,
|
|
||||||
aiStatus,
|
|
||||||
deleteModel,
|
|
||||||
deleteProvider,
|
|
||||||
discoverModels,
|
|
||||||
getTaskSettings,
|
|
||||||
listModels,
|
|
||||||
listProviders,
|
|
||||||
manualModelSchema,
|
|
||||||
modelPatchSchema,
|
|
||||||
patchModel,
|
|
||||||
providerInputSchema,
|
|
||||||
runChat,
|
|
||||||
saveProvider,
|
|
||||||
saveTaskSettings,
|
|
||||||
TASK_INFO,
|
|
||||||
testProvider,
|
|
||||||
usageSummary,
|
|
||||||
} from '../ai/service.ts';
|
|
||||||
import { AIError } from '../ai/types.ts';
|
|
||||||
import { badRequest, idParam, notFound, readBody, type AppContext } from '../http.ts';
|
|
||||||
|
|
||||||
type Emit<T> = (event: T) => void;
|
|
||||||
|
|
||||||
/** Server-sent events with ordered writes, keep-alive pings and abort propagation. */
|
|
||||||
function sse<T extends { type: string }>(c: Context, run: (emit: Emit<T>, signal: AbortSignal) => Promise<void>) {
|
|
||||||
c.header('X-Accel-Buffering', 'no');
|
|
||||||
c.header('Cache-Control', 'no-cache, no-transform');
|
|
||||||
return streamSSE(c, async (stream) => {
|
|
||||||
const controller = new AbortController();
|
|
||||||
stream.onAbort(() => controller.abort());
|
|
||||||
let chain: Promise<unknown> = Promise.resolve();
|
|
||||||
const emit = (event: T | { type: 'ping' }) => {
|
|
||||||
chain = chain.then(() => stream.writeSSE({ data: JSON.stringify(event) })).catch(() => undefined);
|
|
||||||
};
|
|
||||||
const ping = setInterval(() => emit({ type: 'ping' }), 15_000);
|
|
||||||
try {
|
|
||||||
await run(emit, controller.signal);
|
|
||||||
} finally {
|
|
||||||
clearInterval(ping);
|
|
||||||
await chain;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const contentDraftSchema = z.discriminatedUnion('kind', [
|
|
||||||
z.object({
|
|
||||||
kind: z.literal('quiz'),
|
|
||||||
title: z.string().max(200),
|
|
||||||
description: z.string().max(4000).optional().default(''),
|
|
||||||
questions: z
|
|
||||||
.array(
|
|
||||||
z.object({
|
|
||||||
type: z.enum(['single', 'multi']),
|
|
||||||
stem: z.string().max(20000),
|
|
||||||
options: z.array(z.object({ text: z.string().max(4000), correct: z.boolean(), why: z.string().max(20000).optional() })).max(12),
|
|
||||||
rationale: z.string().max(20000).optional().default(''),
|
|
||||||
topic: z.string().max(80).optional().default(''),
|
|
||||||
difficulty: z.enum(['easy', 'medium', 'hard']).optional().default('medium'),
|
|
||||||
tags: z.array(z.string().max(40)).max(12).optional().default([]),
|
|
||||||
source: z.string().max(300).optional(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.max(1000),
|
|
||||||
}),
|
|
||||||
z.object({
|
|
||||||
kind: z.literal('deck'),
|
|
||||||
title: z.string().max(200),
|
|
||||||
description: z.string().max(4000).optional().default(''),
|
|
||||||
cards: z.array(z.object({ front: z.string().max(20000), back: z.string().max(20000), hint: z.string().max(20000).optional(), tags: z.array(z.string().max(40)).max(12).optional().default([]) })).max(5000),
|
|
||||||
}),
|
|
||||||
z.object({ kind: z.literal('note'), title: z.string().max(200), content: z.string().max(400000) }),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const EXPLAIN_SYSTEM = `You are a patient nursing instructor helping a student while they study. Explain clearly and briefly (aim for under 200 words unless asked for more).
|
|
||||||
Use Markdown. Lead with the key idea, explain the reasoning, and connect it to patient care. If answer choices are given, say why the correct one is right and, in a line each, why the others are wrong. Do not restate the whole question.`;
|
|
||||||
|
|
||||||
export function aiRoutes(app: AppContext): Hono {
|
|
||||||
const { db } = app;
|
|
||||||
const r = new Hono();
|
|
||||||
|
|
||||||
/* ------------------------------ configuration ------------------------------ */
|
|
||||||
|
|
||||||
r.get('/ai/meta', (c) => c.json({ adapters: listAdapterMeta(), tasks: TASK_INFO, catalogUpdated: CATALOG_UPDATED }));
|
|
||||||
r.get('/ai/status', (c) => c.json(aiStatus(app)));
|
|
||||||
|
|
||||||
r.get('/ai/providers', (c) => c.json({ providers: listProviders(app) }));
|
|
||||||
r.post('/ai/providers', async (c) => c.json({ provider: saveProvider(app, null, await readBody(c, providerInputSchema)) }, 201));
|
|
||||||
r.patch('/ai/providers/:id', async (c) => c.json({ provider: saveProvider(app, idParam(c), await readBody(c, providerInputSchema)) }));
|
|
||||||
r.delete('/ai/providers/:id', (c) => {
|
|
||||||
deleteProvider(app, idParam(c));
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
r.post('/ai/providers/:id/test', async (c) => c.json({ result: await testProvider(app, idParam(c)) }));
|
|
||||||
r.post('/ai/providers/:id/discover', async (c) => {
|
|
||||||
const result = await discoverModels(app, idParam(c));
|
|
||||||
return c.json({ ...result, models: listModels(db) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/ai/models', (c) => c.json({ models: listModels(db) }));
|
|
||||||
r.post('/ai/models', async (c) => c.json({ model: addManualModel(app, await readBody(c, manualModelSchema)) }, 201));
|
|
||||||
r.patch('/ai/models/:id', async (c) => c.json({ model: patchModel(db, idParam(c), await readBody(c, modelPatchSchema)) }));
|
|
||||||
r.delete('/ai/models/:id', (c) => {
|
|
||||||
deleteModel(db, idParam(c));
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/ai/tasks', (c) => c.json({ tasks: getTaskSettings(db) }));
|
|
||||||
r.put('/ai/tasks', async (c) => c.json({ tasks: saveTaskSettings(db, await c.req.json().catch(() => ({}))) }));
|
|
||||||
|
|
||||||
r.get('/ai/usage', (c) => c.json(usageSummary(db, Math.min(Math.max(Number(c.req.query('days') ?? 30) || 30, 1), 365))));
|
|
||||||
|
|
||||||
/* ------------------------------ generation + drafts ------------------------------ */
|
|
||||||
|
|
||||||
r.post('/ai/generate', async (c) => {
|
|
||||||
const body = await readBody(c, generateRequestSchema);
|
|
||||||
const { draftId } = startGeneration(app, body);
|
|
||||||
return c.json({ draftId }, 202);
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/ai/improve', async (c) => {
|
|
||||||
const body = await readBody(c, improveRequestSchema);
|
|
||||||
const { draftId } = startImprove(app, body);
|
|
||||||
return c.json({ draftId }, 202);
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/drafts', (c) => c.json({ drafts: listDrafts(db) }));
|
|
||||||
|
|
||||||
/** A blank or pre-filled draft, used by the import preview so imports and AI output share one review screen. */
|
|
||||||
r.post('/drafts', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ payload: contentDraftSchema, classId: z.string().nullish(), folderId: z.string().nullish() }));
|
|
||||||
const id = createDraft(db, { kind: body.payload.kind, classId: body.classId ?? null, folderId: body.folderId ?? null, title: body.payload.title, payload: body.payload as never, meta: { origin: 'import' } });
|
|
||||||
return c.json({ draft: getDraft(db, id) }, 201);
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/drafts/:id', (c) => c.json({ draft: getDraft(db, idParam(c)) }));
|
|
||||||
|
|
||||||
r.put('/drafts/:id', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ payload: contentDraftSchema, classId: z.string().nullish(), folderId: z.string().nullish() }));
|
|
||||||
return c.json({ draft: updateDraftPayload(db, idParam(c), body.payload as never, { classId: body.classId, folderId: body.folderId }) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/drafts/:id/save', async (c) => {
|
|
||||||
const body = await readBody(c, draftSaveSchema);
|
|
||||||
return c.json({ saved: saveDraft(app, idParam(c), body) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/drafts/:id/cancel', (c) => {
|
|
||||||
cancelGeneration(idParam(c));
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.delete('/drafts/:id', (c) => {
|
|
||||||
discardDraft(db, idParam(c));
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------ assistant ------------------------------ */
|
|
||||||
|
|
||||||
r.post('/ai/chat', async (c) => {
|
|
||||||
const body = await readBody(c, assistantRequestSchema);
|
|
||||||
return sse<AssistantEvent>(c, async (emit, signal) => {
|
|
||||||
try {
|
|
||||||
await runAssistantTurn(app, body, emit, signal);
|
|
||||||
} catch (err) {
|
|
||||||
// Failures before the turn starts (no model configured, bad input) still reach the client as an event.
|
|
||||||
const ai = err instanceof AIError ? err : null;
|
|
||||||
if (!ai) app.log('error', 'Assistant request failed', err);
|
|
||||||
emit({ type: 'error', code: ai?.code ?? 'server', message: ai?.message ?? 'The assistant could not start.', detail: ai?.providerMessage });
|
|
||||||
emit({ type: 'done', conversationId: body.conversationId ?? '', title: '' });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/ai/conversations', (c) => c.json({ conversations: listConversations(db) }));
|
|
||||||
|
|
||||||
r.get('/ai/conversations/:id', (c) => {
|
|
||||||
const id = idParam(c);
|
|
||||||
const row = db.get<{ id: string; title: string; class_id: string | null }>('SELECT id, title, class_id FROM conversations WHERE id = ?', id);
|
|
||||||
if (!row) throw notFound('Conversation');
|
|
||||||
return c.json({ conversation: { id: row.id, title: row.title, classId: row.class_id }, messages: displayMessages(db, id) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.delete('/ai/conversations/:id', (c) => {
|
|
||||||
const { changes } = db.run('DELETE FROM conversations WHERE id = ?', idParam(c));
|
|
||||||
if (!changes) throw notFound('Conversation');
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/ai/actions/:id', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ confirm: z.boolean() }));
|
|
||||||
return c.json(resolveAction(app, idParam(c), body.confirm));
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------ inline explanations ------------------------------ */
|
|
||||||
|
|
||||||
const explainSchema = z.discriminatedUnion('kind', [
|
|
||||||
z.object({
|
|
||||||
kind: z.literal('question'),
|
|
||||||
stem: z.string().max(20000),
|
|
||||||
options: z.array(z.object({ text: z.string().max(4000), correct: z.boolean() })).max(12),
|
|
||||||
selected: z.array(z.string().max(4000)).max(12).optional(),
|
|
||||||
rationale: z.string().max(20000).optional(),
|
|
||||||
ask: z.string().max(2000).optional(),
|
|
||||||
}),
|
|
||||||
z.object({ kind: z.literal('card'), front: z.string().max(20000), back: z.string().max(20000), ask: z.string().max(2000).optional() }),
|
|
||||||
z.object({ kind: z.literal('text'), text: z.string().max(40000), ask: z.string().max(2000).optional() }),
|
|
||||||
]);
|
|
||||||
|
|
||||||
r.post('/ai/explain', async (c) => {
|
|
||||||
const body = await readBody(c, explainSchema);
|
|
||||||
let prompt: string;
|
|
||||||
if (body.kind === 'question') {
|
|
||||||
prompt = [
|
|
||||||
`Question:\n${body.stem}`,
|
|
||||||
`Choices:\n${body.options.map((o) => `- ${o.text}${o.correct ? ' (correct)' : ''}`).join('\n')}`,
|
|
||||||
body.selected?.length ? `The student chose: ${body.selected.join('; ')}` : '',
|
|
||||||
body.rationale ? `Existing rationale:\n${body.rationale}` : '',
|
|
||||||
body.ask ? `The student asks: ${body.ask}` : 'Explain this question so it finally makes sense. If the student chose wrongly, address the misunderstanding behind their choice.',
|
|
||||||
]
|
|
||||||
.filter(Boolean)
|
|
||||||
.join('\n\n');
|
|
||||||
} else if (body.kind === 'card') {
|
|
||||||
prompt = `Flashcard front:\n${body.front}\n\nFlashcard back:\n${body.back}\n\n${body.ask ? `The student asks: ${body.ask}` : 'Explain this card in more depth and give one way to remember it.'}`;
|
|
||||||
} else {
|
|
||||||
prompt = `${body.text}\n\n${body.ask ? `The student asks: ${body.ask}` : 'Explain this passage in simpler terms and point out what is most important to remember.'}`;
|
|
||||||
}
|
|
||||||
if (!prompt.trim()) throw badRequest('Nothing to explain');
|
|
||||||
return sse<{ type: 'text'; delta: string } | { type: 'error'; message: string; code: string } | { type: 'done' }>(c, async (emit, signal) => {
|
|
||||||
try {
|
|
||||||
await runChat(app, {
|
|
||||||
task: 'quick',
|
|
||||||
purpose: 'explanation',
|
|
||||||
system: EXPLAIN_SYSTEM,
|
|
||||||
messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
|
|
||||||
maxOutputTokens: 1500,
|
|
||||||
signal,
|
|
||||||
onEvent: (e) => {
|
|
||||||
if (e.type === 'text') emit({ type: 'text', delta: e.delta });
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
const ai = err instanceof AIError ? err : null;
|
|
||||||
if (ai?.code !== 'aborted') emit({ type: 'error', code: ai?.code ?? 'server', message: ai?.message ?? 'The explanation could not be generated.' });
|
|
||||||
}
|
|
||||||
emit({ type: 'done' });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
@ -1,267 +0,0 @@
|
||||||
import fs from 'node:fs';
|
|
||||||
import os from 'node:os';
|
|
||||||
import path from 'node:path';
|
|
||||||
import { Hono } from 'hono';
|
|
||||||
import { bodyLimit } from 'hono/body-limit';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import { questionToDraft, validateDraft, type ContentDraft } from '../../shared/content.ts';
|
|
||||||
import { draftsFromValue, LLM_INSTRUCTIONS } from '../../shared/importer.ts';
|
|
||||||
import { badRequest, HttpError, idParam, readBody, type AppContext } from '../http.ts';
|
|
||||||
import { appSettingsSchema, deleteSetting, getAppSettings, getSetting, setSetting } from '../settings.ts';
|
|
||||||
import { createSnapshot, exportBundle, fullBackupStream, listBackups } from '../services/backup.ts';
|
|
||||||
import { createClass, createDeck, createFolder, createNote, createQuiz, draftCardToInput, draftQuestionToInput, getClass, getDeck, getQuiz, listCards, listQuestions } from '../services/content.ts';
|
|
||||||
import { createTemplate, deleteTemplate, listTemplates, templateInputSchema, updateTemplate } from '../services/templates.ts';
|
|
||||||
import { importV1 } from '../services/v1import.ts';
|
|
||||||
import { slugify } from '../util.ts';
|
|
||||||
|
|
||||||
export const APP_VERSION = '2.0.0';
|
|
||||||
|
|
||||||
function dirSize(dir: string): number {
|
|
||||||
try {
|
|
||||||
return fs.readdirSync(dir).reduce((n, f) => {
|
|
||||||
const st = fs.statSync(path.join(dir, f));
|
|
||||||
return n + (st.isFile() ? st.size : 0);
|
|
||||||
}, 0);
|
|
||||||
} catch {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Creates library items from reviewed drafts. Used by the importer and by bundle restore. */
|
|
||||||
function createFromDraft(app: AppContext, draft: ContentDraft, classId: string, folderId: string | null): { kind: string; id: string; title: string } {
|
|
||||||
const { db } = app;
|
|
||||||
if (draft.kind === 'quiz') {
|
|
||||||
const quiz = createQuiz(db, { classId, folderId, title: draft.title.trim() || 'Imported quiz', description: draft.description ?? '', questions: draft.questions.map(draftQuestionToInput) });
|
|
||||||
return { kind: 'quiz', id: quiz.id, title: quiz.title };
|
|
||||||
}
|
|
||||||
if (draft.kind === 'deck') {
|
|
||||||
const deck = createDeck(db, { classId, folderId, title: draft.title.trim() || 'Imported deck', description: draft.description ?? '', cards: draft.cards.map(draftCardToInput) });
|
|
||||||
return { kind: 'deck', id: deck.id, title: deck.title };
|
|
||||||
}
|
|
||||||
const note = createNote(db, { classId, folderId, title: draft.title.trim() || 'Imported note', content: draft.content });
|
|
||||||
return { kind: 'note', id: note.id, title: note.title };
|
|
||||||
}
|
|
||||||
|
|
||||||
export function dataRoutes(app: AppContext): Hono {
|
|
||||||
const { db } = app;
|
|
||||||
const r = new Hono();
|
|
||||||
|
|
||||||
/* ------------------------------ settings ------------------------------ */
|
|
||||||
|
|
||||||
r.get('/settings', (c) => {
|
|
||||||
const count = (table: string) => db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM ${table}`)?.n ?? 0;
|
|
||||||
let dbSize = 0;
|
|
||||||
try {
|
|
||||||
dbSize = fs.statSync(app.config.dbPath).size;
|
|
||||||
} catch {
|
|
||||||
/* in-memory */
|
|
||||||
}
|
|
||||||
return c.json({
|
|
||||||
settings: getAppSettings(db, app.config.timezone),
|
|
||||||
system: {
|
|
||||||
version: APP_VERSION,
|
|
||||||
node: process.version,
|
|
||||||
dataDir: app.config.dataDir,
|
|
||||||
databaseBytes: dbSize,
|
|
||||||
uploadBytes: dirSize(app.config.uploadsDir) + dirSize(app.config.mediaDir),
|
|
||||||
maxUploadMb: Math.round(app.config.maxUploadBytes / 1048576),
|
|
||||||
counts: { classes: count('classes'), quizzes: count('quizzes'), questions: count('questions'), decks: count('decks'), cards: count('cards'), documents: count('documents'), notes: count('notes') },
|
|
||||||
timezones: typeof Intl.supportedValuesOf === 'function' ? Intl.supportedValuesOf('timeZone') : [],
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
r.put('/settings', async (c) => {
|
|
||||||
const patch = await readBody(c, appSettingsSchema.partial());
|
|
||||||
if (patch.timezone) {
|
|
||||||
try {
|
|
||||||
new Intl.DateTimeFormat('en-US', { timeZone: patch.timezone });
|
|
||||||
} catch {
|
|
||||||
throw badRequest('Unknown timezone');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const next = { ...getAppSettings(db, app.config.timezone), ...patch };
|
|
||||||
setSetting(db, 'app', next);
|
|
||||||
return c.json({ settings: next });
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------ instructions for outside chatbots ------------------------------ */
|
|
||||||
|
|
||||||
r.get('/llm-instructions', (c) =>
|
|
||||||
c.json({
|
|
||||||
quiz: getSetting<string | null>(db, 'llm.quiz', null) ?? LLM_INSTRUCTIONS.quiz,
|
|
||||||
deck: getSetting<string | null>(db, 'llm.deck', null) ?? LLM_INSTRUCTIONS.deck,
|
|
||||||
customized: { quiz: getSetting<string | null>(db, 'llm.quiz', null) !== null, deck: getSetting<string | null>(db, 'llm.deck', null) !== null },
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
/** Saves an edited copy; `text: null` goes back to the built-in wording. */
|
|
||||||
r.put('/llm-instructions', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ kind: z.enum(['quiz', 'deck']), text: z.string().trim().min(20).max(20000).nullable() }));
|
|
||||||
if (body.text === null) deleteSetting(db, `llm.${body.kind}`);
|
|
||||||
else setSetting(db, `llm.${body.kind}`, body.text);
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------ import ------------------------------ */
|
|
||||||
|
|
||||||
r.post('/import', bodyLimit({ maxSize: 30 * 1024 * 1024 }), async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ classId: z.string(), folderId: z.string().nullish(), items: z.array(z.unknown()).min(1).max(200) }));
|
|
||||||
getClass(db, body.classId);
|
|
||||||
const drafts: ContentDraft[] = [];
|
|
||||||
body.items.forEach((raw, i) => {
|
|
||||||
const parsed = draftsFromValue(raw).items[0];
|
|
||||||
if (!parsed) throw new HttpError(422, 'validation', `Item ${i + 1} is not a quiz, deck or note`);
|
|
||||||
const errors = validateDraft(parsed).filter((x) => x.level === 'error');
|
|
||||||
if (errors.length > 0) {
|
|
||||||
const first = errors[0];
|
|
||||||
const where = first.index !== undefined ? `${parsed.kind === 'quiz' ? 'question' : 'card'} ${first.index + 1}` : first.field ?? 'item';
|
|
||||||
throw new HttpError(422, 'validation', `"${parsed.title || `Item ${i + 1}`}", ${where}: ${first.message}`, { item: i, issues: errors });
|
|
||||||
}
|
|
||||||
drafts.push(parsed);
|
|
||||||
});
|
|
||||||
const created = db.tx(() => drafts.map((d) => createFromDraft(app, d, body.classId, body.folderId ?? null)));
|
|
||||||
return c.json({ created }, 201);
|
|
||||||
});
|
|
||||||
|
|
||||||
/** Restores a JSON bundle produced by "Export everything": recreates classes, folders and their contents. */
|
|
||||||
r.post('/import/bundle', bodyLimit({ maxSize: 100 * 1024 * 1024 }), async (c) => {
|
|
||||||
const body = (await c.req.json().catch(() => {
|
|
||||||
throw badRequest('The file is not valid JSON');
|
|
||||||
})) as Record<string, unknown>;
|
|
||||||
const classes = Array.isArray(body.classes) ? (body.classes as Record<string, unknown>[]) : null;
|
|
||||||
if (!classes) throw badRequest('This file is not a Study Desk bundle. Use "Import" inside a class for single quizzes and decks.');
|
|
||||||
const existing = new Set(db.all<{ name: string }>('SELECT name FROM classes').map((x) => x.name.toLowerCase()));
|
|
||||||
const summary = { classes: 0, quizzes: 0, decks: 0, notes: 0, skipped: [] as string[] };
|
|
||||||
db.tx(() => {
|
|
||||||
for (const raw of classes) {
|
|
||||||
let name = String(raw.name ?? 'Imported class').trim().slice(0, 110) || 'Imported class';
|
|
||||||
if (existing.has(name.toLowerCase())) name = `${name} (imported)`;
|
|
||||||
existing.add(name.toLowerCase());
|
|
||||||
const color = typeof raw.color === 'string' ? raw.color : undefined;
|
|
||||||
const cls = createClass(db, { name, code: String(raw.code ?? '').slice(0, 40), description: String(raw.description ?? '').slice(0, 2000), color: color as never });
|
|
||||||
summary.classes += 1;
|
|
||||||
const folders = new Map<string, string>();
|
|
||||||
for (const f of Array.isArray(raw.folders) ? raw.folders : []) {
|
|
||||||
const fname = String(typeof f === 'string' ? f : (f as Record<string, unknown>)?.name ?? '').trim();
|
|
||||||
if (fname) folders.set(fname, createFolder(db, cls.id, fname).id);
|
|
||||||
}
|
|
||||||
for (const key of ['quizzes', 'decks', 'notes'] as const) {
|
|
||||||
for (const item of Array.isArray(raw[key]) ? (raw[key] as Record<string, unknown>[]) : []) {
|
|
||||||
const kind = key === 'quizzes' ? 'quiz' : key === 'decks' ? 'deck' : 'note';
|
|
||||||
const draft = draftsFromValue({ ...item, kind }).items[0];
|
|
||||||
const title = String(item?.title ?? 'Untitled');
|
|
||||||
if (!draft) {
|
|
||||||
summary.skipped.push(`${title} (unreadable)`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Drop invalid entries rather than failing a whole restore; report what was left out.
|
|
||||||
if (draft.kind === 'quiz') {
|
|
||||||
const before = draft.questions.length;
|
|
||||||
draft.questions = draft.questions.filter((q) => !validateDraft({ ...draft, questions: [q] }).some((x) => x.level === 'error' && x.index !== undefined));
|
|
||||||
if (draft.questions.length < before) summary.skipped.push(`${before - draft.questions.length} invalid question(s) in "${title}"`);
|
|
||||||
} else if (draft.kind === 'deck') {
|
|
||||||
const before = draft.cards.length;
|
|
||||||
draft.cards = draft.cards.filter((card) => card.front?.trim() && card.back?.trim());
|
|
||||||
if (draft.cards.length < before) summary.skipped.push(`${before - draft.cards.length} empty card(s) in "${title}"`);
|
|
||||||
}
|
|
||||||
const folderId = typeof item.folder === 'string' ? (folders.get(item.folder) ?? null) : null;
|
|
||||||
createFromDraft(app, draft, cls.id, folderId);
|
|
||||||
summary[key] += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return c.json({ summary }, 201);
|
|
||||||
});
|
|
||||||
|
|
||||||
/** Migration from a V1 database file. `?preview=1` only reports what would be imported. */
|
|
||||||
r.post('/import/v1', bodyLimit({ maxSize: 500 * 1024 * 1024 }), async (c) => {
|
|
||||||
const form = await c.req.parseBody().catch(() => {
|
|
||||||
throw badRequest('Upload could not be read');
|
|
||||||
});
|
|
||||||
const file = form.file;
|
|
||||||
if (!file || typeof file === 'string') throw badRequest('Choose your V1 study.db file');
|
|
||||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
|
||||||
if (new TextDecoder().decode(bytes.subarray(0, 15)) !== 'SQLite format 3') throw badRequest('That file is not a SQLite database.');
|
|
||||||
const tmp = path.join(os.tmpdir(), `studydesk-v1-${Date.now()}-${Math.random().toString(36).slice(2)}.db`);
|
|
||||||
fs.writeFileSync(tmp, bytes);
|
|
||||||
try {
|
|
||||||
const preview = c.req.query('preview') === '1';
|
|
||||||
if (!preview) createSnapshot(app, 'pre-v1-import');
|
|
||||||
return c.json({ preview, summary: importV1(app, tmp, { dryRun: preview }) });
|
|
||||||
} finally {
|
|
||||||
fs.rmSync(tmp, { force: true });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------ export + backup ------------------------------ */
|
|
||||||
|
|
||||||
r.get('/export/all', (c) => {
|
|
||||||
c.header('Content-Disposition', `attachment; filename="studydesk-export-${new Date().toISOString().slice(0, 10)}.json"`);
|
|
||||||
return c.json(exportBundle(app));
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/export/class/:id', (c) => {
|
|
||||||
const cls = getClass(db, idParam(c));
|
|
||||||
c.header('Content-Disposition', `attachment; filename="${slugify(cls.name)}.class.json"`);
|
|
||||||
return c.json(exportBundle(app, cls.id));
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/backup/download', () => {
|
|
||||||
const { stream, filename } = fullBackupStream(app);
|
|
||||||
return new Response(stream, { headers: { 'Content-Type': 'application/zip', 'Content-Disposition': `attachment; filename="${filename}"`, 'Cache-Control': 'no-store' } });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/backups', (c) => c.json({ backups: listBackups(app), directory: app.config.backupsDir }));
|
|
||||||
r.post('/backups', (c) => c.json({ backup: createSnapshot(app) }, 201));
|
|
||||||
|
|
||||||
r.get('/backups/:name', (c) => {
|
|
||||||
const name = path.basename(c.req.param('name'));
|
|
||||||
const file = path.join(app.config.backupsDir, name);
|
|
||||||
if (!name.endsWith('.db') || !fs.existsSync(file)) throw new HttpError(404, 'not_found', 'Backup not found');
|
|
||||||
return new Response(fs.readFileSync(file), { headers: { 'Content-Type': 'application/octet-stream', 'Content-Disposition': `attachment; filename="${name}"`, 'Cache-Control': 'no-store' } });
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------ templates ------------------------------ */
|
|
||||||
|
|
||||||
r.get('/templates', (c) => c.json({ templates: listTemplates(db, c.req.query('kind') || undefined) }));
|
|
||||||
|
|
||||||
r.post('/templates', async (c) => c.json({ template: createTemplate(db, await readBody(c, templateInputSchema)) }, 201));
|
|
||||||
|
|
||||||
/** Saves an existing quiz or deck as a reusable template. */
|
|
||||||
r.post('/templates/from', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ kind: z.enum(['quiz', 'deck']), id: z.string(), name: z.string().trim().min(1).max(120).optional() }));
|
|
||||||
if (body.kind === 'quiz') {
|
|
||||||
const quiz = getQuiz(db, body.id);
|
|
||||||
return c.json({ template: createTemplate(db, { kind: 'quiz', name: body.name ?? quiz.title, description: quiz.description.slice(0, 1000), payload: { title: quiz.title, description: quiz.description, questions: listQuestions(db, quiz.id).map(questionToDraft) } }) }, 201);
|
|
||||||
}
|
|
||||||
const deck = getDeck(db, body.id);
|
|
||||||
return c.json(
|
|
||||||
{ template: createTemplate(db, { kind: 'deck', name: body.name ?? deck.title, description: deck.description.slice(0, 1000), payload: { title: deck.title, description: deck.description, cards: listCards(db, deck.id).map((x) => ({ front: x.front, back: x.back, hint: x.hint, tags: x.tags })) } }) },
|
|
||||||
201,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
/** Creates a new quiz or deck from a content template. */
|
|
||||||
r.post('/templates/:id/use', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ classId: z.string(), folderId: z.string().nullish(), title: z.string().trim().max(200).optional() }));
|
|
||||||
const tpl = listTemplates(db).find((t) => t.id === idParam(c));
|
|
||||||
if (!tpl || tpl.kind === 'generation') throw new HttpError(404, 'not_found', 'Template not found');
|
|
||||||
const draft = draftsFromValue({ ...tpl.payload, kind: tpl.kind, title: body.title || tpl.payload.title || tpl.name }).items[0];
|
|
||||||
if (!draft) throw badRequest('This template is empty');
|
|
||||||
return c.json({ created: createFromDraft(app, draft, body.classId, body.folderId ?? null) }, 201);
|
|
||||||
});
|
|
||||||
|
|
||||||
r.patch('/templates/:id', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ name: z.string().max(120).optional(), description: z.string().max(1000).optional(), payload: z.record(z.string(), z.unknown()).optional() }));
|
|
||||||
return c.json({ template: updateTemplate(db, idParam(c), body) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.delete('/templates/:id', (c) => {
|
|
||||||
deleteTemplate(db, idParam(c));
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
@ -1,198 +0,0 @@
|
||||||
import fs from 'node:fs';
|
|
||||||
import path from 'node:path';
|
|
||||||
import crypto from 'node:crypto';
|
|
||||||
import { Hono } from 'hono';
|
|
||||||
import { bodyLimit } from 'hono/body-limit';
|
|
||||||
import { z } from 'zod';
|
|
||||||
import { titleSchema } from '../../shared/content.ts';
|
|
||||||
import { isAuthenticated } from '../auth.ts';
|
|
||||||
import { badRequest, HttpError, idParam, readBody, type AppContext } from '../http.ts';
|
|
||||||
import { getClass, mapClass, resolveFolder } from '../services/content.ts';
|
|
||||||
import { deleteDocument, documentPath, enqueue, getDocument, ingestUpload, listChunks, mapDocument, sourceLabel } from '../services/documents.ts';
|
|
||||||
import { now } from '../util.ts';
|
|
||||||
import { shareFor } from './library.ts';
|
|
||||||
|
|
||||||
const IMAGE_TYPES: Record<string, { ext: string; test: (b: Uint8Array) => boolean }> = {
|
|
||||||
'image/png': { ext: 'png', test: (b) => b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47 },
|
|
||||||
'image/jpeg': { ext: 'jpg', test: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
|
|
||||||
'image/gif': { ext: 'gif', test: (b) => b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46 },
|
|
||||||
'image/webp': { ext: 'webp', test: (b) => b[0] === 0x52 && b[1] === 0x49 && b[8] === 0x57 && b[9] === 0x45 },
|
|
||||||
};
|
|
||||||
|
|
||||||
export function documentRoutes(app: AppContext): Hono {
|
|
||||||
const { db } = app;
|
|
||||||
const r = new Hono();
|
|
||||||
|
|
||||||
const uploadLimit = bodyLimit({
|
|
||||||
maxSize: app.config.maxUploadBytes + 1024 * 1024,
|
|
||||||
onError: () => {
|
|
||||||
throw new HttpError(413, 'too_large', `That file is larger than the ${Math.round(app.config.maxUploadBytes / 1048576)} MB upload limit.`);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/documents', uploadLimit, async (c) => {
|
|
||||||
const form = await c.req.parseBody({ all: true }).catch(() => {
|
|
||||||
throw badRequest('Upload could not be read');
|
|
||||||
});
|
|
||||||
const classId = typeof form.classId === 'string' ? form.classId : '';
|
|
||||||
const folderId = typeof form.folderId === 'string' && form.folderId ? form.folderId : null;
|
|
||||||
const raw = form.file ?? form.files;
|
|
||||||
const files = (Array.isArray(raw) ? raw : [raw]).filter((f): f is File => typeof f === 'object' && f !== null && 'arrayBuffer' in f);
|
|
||||||
if (!classId) throw badRequest('classId is required');
|
|
||||||
if (files.length === 0) throw badRequest('Choose a file to upload');
|
|
||||||
const created = [];
|
|
||||||
for (const file of files) {
|
|
||||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
|
||||||
created.push(mapDocument(ingestUpload(app, { classId, folderId, filename: file.name || 'upload', mime: file.type || undefined, bytes })));
|
|
||||||
}
|
|
||||||
return c.json({ documents: created }, 201);
|
|
||||||
});
|
|
||||||
|
|
||||||
/** Pasted notes become a Markdown document, so they are searchable and usable by AI like any upload. */
|
|
||||||
r.post('/documents/text', async (c) => {
|
|
||||||
const body = await readBody(c, z.object({ classId: z.string(), folderId: z.string().nullish(), title: titleSchema, text: z.string().trim().min(1).max(2_000_000) }));
|
|
||||||
const doc = ingestUpload(app, {
|
|
||||||
classId: body.classId,
|
|
||||||
folderId: body.folderId,
|
|
||||||
filename: `${body.title.replace(/[^\w\- ]+/g, '').trim().slice(0, 80) || 'notes'}.md`,
|
|
||||||
mime: 'text/markdown',
|
|
||||||
bytes: new TextEncoder().encode(body.text),
|
|
||||||
title: body.title,
|
|
||||||
});
|
|
||||||
return c.json({ document: mapDocument(doc) }, 201);
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/documents/:id', (c) => {
|
|
||||||
const doc = getDocument(db, idParam(c));
|
|
||||||
const outline = db
|
|
||||||
.all<{ seq: number; heading: string; page_start: number | null }>(`SELECT MIN(seq) AS seq, heading, MIN(page_start) AS page_start FROM chunks WHERE document_id = ? AND heading != '' GROUP BY heading ORDER BY seq LIMIT 200`, doc.id)
|
|
||||||
.map((o) => ({ seq: o.seq, heading: o.heading, page: o.page_start }));
|
|
||||||
return c.json({ document: mapDocument(doc), class: mapClass(getClass(db, doc.class_id)), outline, share: shareFor(app, 'document', doc.id) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/documents/:id/chunks', (c) => {
|
|
||||||
const doc = getDocument(db, idParam(c));
|
|
||||||
let offset = Math.max(0, Number(c.req.query('offset') ?? 0) || 0);
|
|
||||||
const limit = Math.min(100, Math.max(1, Number(c.req.query('limit') ?? 40) || 40));
|
|
||||||
// Deep link to one passage (from search or a citation): start just before it.
|
|
||||||
const around = Number(c.req.query('around'));
|
|
||||||
if (Number.isFinite(around) && around > 0) {
|
|
||||||
const hit = db.get<{ seq: number }>('SELECT seq FROM chunks WHERE id = ? AND document_id = ?', around, doc.id);
|
|
||||||
if (hit) offset = Math.max(0, hit.seq - 1);
|
|
||||||
}
|
|
||||||
const chunks = listChunks(db, doc.id, offset, limit);
|
|
||||||
return c.json({
|
|
||||||
chunks: chunks.map((ch) => ({ id: ch.id, seq: ch.seq, text: ch.text, heading: ch.heading, pageStart: ch.pageStart, pageEnd: ch.pageEnd, label: sourceLabel(ch) })),
|
|
||||||
total: doc.chunk_count,
|
|
||||||
offset,
|
|
||||||
nextOffset: chunks.length === limit && offset + limit < doc.chunk_count ? offset + limit : null,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/documents/:id/status', (c) => {
|
|
||||||
const doc = getDocument(db, idParam(c));
|
|
||||||
return c.json({ document: mapDocument(doc) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.get('/documents/:id/file', (c) => {
|
|
||||||
const doc = getDocument(db, idParam(c));
|
|
||||||
const file = documentPath(app, doc);
|
|
||||||
if (!fs.existsSync(file)) throw new HttpError(404, 'not_found', 'The original file is missing from storage');
|
|
||||||
const inline = doc.kind === 'pdf' || doc.kind === 'image';
|
|
||||||
const download = c.req.query('download') === '1';
|
|
||||||
return new Response(fs.readFileSync(file), {
|
|
||||||
headers: {
|
|
||||||
'Content-Type': inline ? doc.mime : 'application/octet-stream',
|
|
||||||
'Content-Disposition': `${inline && !download ? 'inline' : 'attachment'}; filename*=UTF-8''${encodeURIComponent(doc.filename)}`,
|
|
||||||
'X-Content-Type-Options': 'nosniff',
|
|
||||||
// Uploaded files are untrusted. PDFs and images are served with their verified type (sniffing off);
|
|
||||||
// everything else is forced to download.
|
|
||||||
...(inline ? {} : { 'Content-Security-Policy': "default-src 'none'; sandbox" }),
|
|
||||||
'Cache-Control': 'private, max-age=3600',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
r.patch('/documents/:id', async (c) => {
|
|
||||||
const doc = getDocument(db, idParam(c));
|
|
||||||
const body = await readBody(c, z.object({ title: titleSchema.optional(), folderId: z.string().nullish() }));
|
|
||||||
db.run(
|
|
||||||
'UPDATE documents SET title = ?, folder_id = ?, updated_at = ? WHERE id = ?',
|
|
||||||
body.title ?? doc.title,
|
|
||||||
body.folderId === undefined ? doc.folder_id : resolveFolder(db, doc.class_id, body.folderId),
|
|
||||||
now(),
|
|
||||||
doc.id,
|
|
||||||
);
|
|
||||||
return c.json({ document: mapDocument(getDocument(db, doc.id)) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.post('/documents/:id/reprocess', async (c) => {
|
|
||||||
const doc = getDocument(db, idParam(c));
|
|
||||||
const body = (await c.req.json().catch(() => ({}))) as { ocr?: unknown };
|
|
||||||
db.run(`UPDATE documents SET status = 'pending', error = '', progress = 0, updated_at = ? WHERE id = ?`, now(), doc.id);
|
|
||||||
enqueue(app, doc.id, { ocr: body.ocr === true });
|
|
||||||
return c.json({ document: mapDocument(getDocument(db, doc.id)) });
|
|
||||||
});
|
|
||||||
|
|
||||||
r.delete('/documents/:id', (c) => {
|
|
||||||
deleteDocument(app, idParam(c));
|
|
||||||
return c.json({ ok: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
/* ------------------------------ images embedded in questions, cards and notes ------------------------------ */
|
|
||||||
|
|
||||||
r.post('/media', bodyLimit({ maxSize: 12 * 1024 * 1024 }), async (c) => {
|
|
||||||
const form = await c.req.parseBody().catch(() => {
|
|
||||||
throw badRequest('Upload could not be read');
|
|
||||||
});
|
|
||||||
const file = form.file;
|
|
||||||
if (!file || typeof file === 'string') throw badRequest('Choose an image');
|
|
||||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
|
||||||
if (bytes.length > 10 * 1024 * 1024) throw new HttpError(413, 'too_large', 'Images can be up to 10 MB.');
|
|
||||||
// Trust the bytes, not the declared type. SVG is deliberately not accepted (it can carry script).
|
|
||||||
const match = Object.entries(IMAGE_TYPES).find(([, t]) => t.test(bytes));
|
|
||||||
if (!match) throw new HttpError(415, 'unsupported_type', 'Use a PNG, JPEG, GIF or WebP image.');
|
|
||||||
const [mime, { ext }] = match;
|
|
||||||
const id = `${crypto.randomBytes(16).toString('base64url')}.${ext}`;
|
|
||||||
fs.writeFileSync(path.join(app.config.mediaDir, id), bytes);
|
|
||||||
db.run('INSERT INTO media (id, filename, mime, size, created_at) VALUES (?, ?, ?, ?, ?)', id, (file.name || 'image').slice(0, 200), mime, bytes.length, now());
|
|
||||||
return c.json({ url: `/media/${id}`, markdown: `![${(file.name || 'image').replace(/[[\]]/g, '')}](/media/${id})` }, 201);
|
|
||||||
});
|
|
||||||
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Images are private unless an enabled share contains a reference to them.
|
|
||||||
* Re-check on each request so revoking a share also revokes its embedded images.
|
|
||||||
*/
|
|
||||||
export function mediaRoute(app: AppContext): Hono {
|
|
||||||
const r = new Hono();
|
|
||||||
r.get('/:name', (c) => {
|
|
||||||
const name = c.req.param('name');
|
|
||||||
if (!/^[A-Za-z0-9_-]{16,40}\.(png|jpg|gif|webp)$/.test(name)) throw new HttpError(404, 'not_found', 'Not found');
|
|
||||||
if (!isAuthenticated(app, c)) {
|
|
||||||
const reference = `/media/${name}`;
|
|
||||||
const shared = app.db.get(`
|
|
||||||
SELECT 1 FROM shares s WHERE s.enabled = 1 AND (
|
|
||||||
EXISTS (SELECT 1 FROM questions q JOIN quizzes z ON z.id = q.quiz_id
|
|
||||||
WHERE ((s.kind = 'quiz' AND s.target_id = z.id) OR (s.kind = 'class' AND s.target_id = z.class_id) OR (s.kind = 'folder' AND s.target_id = z.folder_id))
|
|
||||||
AND instr(q.stem || q.options || q.rationale, ?) > 0)
|
|
||||||
OR EXISTS (SELECT 1 FROM cards c JOIN decks d ON d.id = c.deck_id
|
|
||||||
WHERE ((s.kind = 'deck' AND s.target_id = d.id) OR (s.kind = 'class' AND s.target_id = d.class_id) OR (s.kind = 'folder' AND s.target_id = d.folder_id))
|
|
||||||
AND instr(c.front || c.back || c.hint, ?) > 0)
|
|
||||||
OR EXISTS (SELECT 1 FROM notes n
|
|
||||||
WHERE ((s.kind = 'note' AND s.target_id = n.id) OR (s.kind = 'class' AND s.target_id = n.class_id) OR (s.kind = 'folder' AND s.target_id = n.folder_id))
|
|
||||||
AND instr(n.content, ?) > 0)
|
|
||||||
) LIMIT 1`, reference, reference, reference);
|
|
||||||
if (!shared) throw new HttpError(404, 'not_found', 'Not found');
|
|
||||||
}
|
|
||||||
const row = app.db.get<{ mime: string }>('SELECT mime FROM media WHERE id = ?', name);
|
|
||||||
const file = path.join(app.config.mediaDir, name);
|
|
||||||
if (!row || !fs.existsSync(file)) throw new HttpError(404, 'not_found', 'Not found');
|
|
||||||
return new Response(fs.readFileSync(file), {
|
|
||||||
headers: { 'Content-Type': row.mime, 'Cache-Control': 'private, no-store', 'X-Content-Type-Options': 'nosniff', 'Content-Security-Policy': "default-src 'none'; sandbox" },
|
|
||||||
});
|
|
||||||
});
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue