Kaleido-Desktop/src/player/enhance.ts
Elijah 9fd2b0603e Add a Quality upscaling tier and make enhancement verifiable
Quality runs ArtCNN C4F32 (its denoising model when Denoising is on),
KrigBilateral, debanding and CAS. Its network runs at any enlargement,
so 1080p in a maximized 1440p window is doubled and scaled down; Balanced
keeps FSRCNNX's 1.3x threshold.

The enhancement pipeline moves to player/enhance.ts. Mode changes are
serialized (latest wins). Shader errors in mpv's log, or network passes
missing from vo-passes, drop to the tier below with a notice, and the
chosen and running tiers are shown separately. HDR is judged from the
decoded video. Debanding, sharpening and denoising have their own
toggles, B compares with enhancement off, and the panel shows the
renderer's actual stages, GPU time and dropped frames.

Compiled shaders persist in the app cache (gpu-shader-cache-dir):
ArtCNN's first Direct3D compile takes about a minute, which the player
now announces, and later launches start immediately.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 19:37:32 -07:00

344 lines
16 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* Video enhancement: upscaling tiers built from mpv's scalers, debanding and
* the shaders bundled in src-tauri/shaders. Each shader's WHEN condition
* decides whether it runs for the current video and window size, so one list
* covers every case: the neural upscalers double luma when the video is
* enlarged (FSRCNNX above 1.3×, ArtCNN at any enlargement, so 1080p in a
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds subsampled colour,
* CAS sharpens at native size (CAS-prescale also before upscaling).
*
* The tier the user chose (preferences) and the one actually running (player
* state) are kept apart: a tier whose shaders fail drops to the one below with
* a notice instead of silently showing plain scaling, and isn't retried.
*/
import { useSettings, UPSCALING, type Upscaling } from '../app/settings';
import { isNative, resourcePath } from '../platform/native';
import { toast } from '../ui/overlays';
import { mpv } from './engine';
import { usePlayer } from './store';
export const UPSCALING_ORDER: Upscaling[] = ['off', 'performance', 'balanced', 'quality'];
/** mpv's own scaler settings, read once so Off can put them back. */
const SCALER_PROPS = ['scale', 'cscale', 'dscale', 'sigmoid-upscaling', 'deband', 'deband-iterations', 'deband-grain'] as const;
let scalerDefaults: Partial<Record<(typeof SCALER_PROPS)[number], unknown>> | null = null;
let shaderDir: Promise<string> | null = null;
/** The enlargement above which each tier's network runs (its shaders' WHEN condition). */
const NETWORK_MIN_SCALE: Partial<Record<Upscaling, number>> = { balanced: 1.3, quality: 1.0 };
const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!;
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'ArtCNN_C4F32.glsl' };
interface Plan {
props: Record<string, unknown>;
shaders: string[];
}
/**
* What a tier runs. Sharpening (CAS) and denoising are tuned for SDR levels,
* so HDR video skips them; the upscalers and debanding stay. Denoising uses
* ArtCNN's own denoising model while upscaling: a separate denoiser (nlmeans)
* took over a minute to compile for Direct3D before the first frame.
*/
function plan(mode: Upscaling, hdr: boolean): Plan {
const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState();
const sharpen = enhanceSharpen && !hdr;
let props: Record<string, unknown>;
let shaders: string[] = [];
switch (mode) {
case 'performance':
props = { scale: 'spline36', cscale: 'spline36', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 24 };
if (sharpen) shaders = ['CAS-prescale.glsl'];
break;
case 'balanced':
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 16 };
shaders = [NEURAL.balanced!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
break;
case 'quality':
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 };
shaders = [enhanceDenoise && !hdr ? 'ArtCNN_C4F32_DN.glsl' : NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
break;
default:
return { props: { ...scalerDefaults }, shaders: [] };
}
if (!enhanceDeband) props.deband = false;
return { props, shaders };
}
/** HDR as decoded (a converted stream may already be SDR), else Plex's metadata. */
function isHdr() {
const st = usePlayer.getState();
if (st.videoGamma) return st.videoGamma === 'pq' || st.videoGamma === 'hlg';
return st.delivery?.mode !== 'transcode' && !!st.item?.media[st.mediaIndex]?.dynamicRange;
}
// —— Shader failures ————————————————————————————————————————————————
const shaderErrors: { at: number; text: string }[] = [];
/** Fed every mpv log line; keeps the ones that mean a user shader failed. */
export function noteMpvLog(level: string, prefix: string, text: string) {
if (level !== 'error' && level !== 'fatal') return;
// A missing file is reported by mpv's file/stream layers; a broken one by the renderer.
if (/\.glsl/i.test(text) || (/^vo/.test(prefix) && /shader|glsl|hook|compil|spirv|parse/i.test(text))) {
shaderErrors.push({ at: Date.now(), text: text.trim() });
if (shaderErrors.length > 20) shaderErrors.shift();
}
}
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
/** Descriptions of the render passes mpv ran for the last frame (or redraw). */
type Pass = { desc: string; avg: number; samples?: number[] };
export async function renderPasses(): Promise<{ desc: string; avg: number }[]> {
const passes = await mpv.get<{ fresh?: Pass[]; redraw?: Pass[] }>('vo-passes').catch(() => null);
const list = (passes?.fresh?.length ? passes.fresh : passes?.redraw) ?? [];
// The median of recent frames: the average includes one-off shader compiles.
return list.map((p) => {
const s = [...(p.samples ?? [])].sort((a, b) => a - b);
return { desc: p.desc, avg: s.length ? s[Math.floor(s.length / 2)] : p.avg };
});
}
/**
* How much the video is enlarged on screen. The full player fills the window,
* which is known before mpv has drawn anything; the miniplayer asks mpv.
*/
export async function scaleFactor(): Promise<number> {
const st = usePlayer.getState();
if (!st.video?.width) return 1;
if (!st.mini) {
const dpr = window.devicePixelRatio || 1;
return Math.min((window.innerWidth * dpr) / st.video.width, (window.innerHeight * dpr) / st.video.height);
}
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
if (!osd) return 1;
return Math.min((osd.w - osd.ml - osd.mr) / st.video.width, (osd.h - osd.mt - osd.mb) / st.video.height);
}
/**
* After a tier is applied: wait for frames, then look for shader errors in
* mpv's log, and, when the network should be running at this size, for its
* passes in the renderer. Either failing drops to the tier below.
*/
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string) {
for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250);
await sleep(1500);
if (id !== seq || !usePlayer.getState().open) return;
const neural = NEURAL[mode];
const expectNetwork = !!neural && (await runsNetwork(mode));
const marker = mode === 'quality' ? /ArtCNN/ : /feature map|FSRCNN/i;
let error = shaderErrors.find((e) => e.at >= appliedAt);
let missing = false;
if (!error && expectNetwork) {
// A first compile can hold the renderer for over a minute; wait for the
// network's passes to appear (or an error) rather than judging too early.
// Already compiled, they appear within a few frames.
missing = true;
const patience = usePlayer.getState().enhance.preparing ? 240 : 8;
for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) {
error = shaderErrors.find((e) => e.at >= appliedAt);
if (error || (await renderPasses()).some((p) => marker.test(p.desc))) {
missing = false;
break;
}
await sleep(1000);
}
}
if (id !== seq) return;
if (!error && !missing) {
if (expectNetwork) markWarm(signature);
return;
}
failed.add(mode);
const fallback = usable(mode);
console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running');
toast(`${UPSCALING[mode].label} upscaling couldn’t start. Using ${UPSCALING[fallback].label}.`, 'error');
requestEnhancement();
}
// —— Applying ——————————————————————————————————————————————————————
let seq = 0;
let chain: Promise<unknown> = Promise.resolve();
/**
* Shader sets that have compiled on this computer. mpv keeps compiled shaders
* on disk (gpu-shader-cache-dir), but the first ArtCNN compile for Direct3D
* takes over a minute, so the player says so the first time.
*/
const WARM_KEY = 'kaleido.warmShaders';
const SLOW = /ArtCNN/;
function warmSet(): Set<string> {
try {
return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]'));
} catch {
return new Set();
}
}
function markWarm(signature: string) {
const set = warmSet();
set.add(signature);
try {
localStorage.setItem(WARM_KEY, JSON.stringify([...set]));
} catch {
/* per-device convenience only */
}
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
}
/** Tiers whose shaders failed this session. */
const failed = new Set<Upscaling>();
let missingWarned = false;
/** The chosen tier, or the nearest one below it that hasn't failed. */
function usable(mode: Upscaling): Upscaling {
let i = UPSCALING_ORDER.indexOf(mode);
while (i > 0 && failed.has(UPSCALING_ORDER[i])) i--;
return UPSCALING_ORDER[i];
}
let mpvVersion = '';
/** The last settings sent to mpv (and when), so an identical re-apply leaves mpv alone. */
let lastApplied = '';
let lastAppliedAt = 0;
async function apply(id: number, requested: Upscaling) {
if (!isNative || id !== seq) return;
if (!scalerDefaults) {
scalerDefaults = {};
for (const p of SCALER_PROPS) scalerDefaults[p] = await mpv.get(p).catch(() => undefined);
mpvVersion = (await mpv.get<string>('mpv-version').catch(() => '')) ?? '';
}
const bypass = usePlayer.getState().enhance.bypass;
let mode = usable(requested);
let { props, shaders } = plan(bypass ? 'off' : mode, isHdr());
const dir = await (shaderDir ??= resourcePath('shaders').catch(() => ''));
if (shaders.length && !dir) {
// No shaders at all: keep the scaler and debanding settings only.
if (!missingWarned) toast('The upscaling shaders are missing from this installation.', 'error');
missingWarned = true;
shaders = [];
if (mode !== 'performance') mode = 'performance';
({ props } = plan(mode, isHdr()));
}
// Decided before touching mpv: while a shader compiles, mpv holds option
// changes until it finishes, so a notice set afterwards would come too late.
// Keyed by mpv's version too: a new libmpv recompiles everything.
// WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile.
const signature = `${mpvVersion}|${shaders.join('|')}`;
const preparing = !bypass && shaders.some((n) => SLOW.test(n)) && !warmSet().has(signature) && (await runsNetwork(mode));
usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } });
if (preparing && usePlayer.getState().firstFrame)
toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for about a minute.`);
const settings = JSON.stringify({ props, shaders });
if (settings !== lastApplied) {
lastApplied = settings;
lastAppliedAt = Date.now();
for (const [k, v] of Object.entries(props)) if (v !== undefined) await mpv.set(k, v).catch(() => {});
if (shaders.length) {
// mpv takes a path list: ';'-separated on Windows, ':' elsewhere.
const windows = dir.includes('\\');
const list = shaders.map((n) => `${dir}${windows ? '\\' : '/'}${n}`).join(windows ? ';' : ':');
await mpv.set('glsl-shaders', list).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) }));
} else await mpv.command('change-list', 'glsl-shaders', 'clr', '').catch(() => {});
}
// Errors count from when these shaders actually reached mpv, even if this
// re-apply changed nothing (mpv may have already reported a failure).
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature);
}
/**
* Applies the chosen tier (and stage toggles) to the playing video. Requests
* are serialized and only the latest runs, so quick U presses can't interleave.
*/
export function requestEnhancement() {
const id = ++seq;
const run = chain.then(() => apply(id, useSettings.getState().upscaling));
chain = run.catch(() => {});
return run;
}
/** Whether a tier failed this session (shown in the player's choices). */
export const hasFailed = (mode: Upscaling) => failed.has(mode);
/** Before/after comparison: enhancement off without changing the preference. */
export function toggleBypass() {
const st = usePlayer.getState();
const bypass = !st.enhance.bypass;
usePlayer.setState({ enhance: { ...st.enhance, bypass } });
toast(bypass ? 'Enhancement off for comparison · press B to turn it back on' : `Enhancement on · ${UPSCALING[useSettings.getState().upscaling].label}`);
return requestEnhancement();
}
// —— Diagnostics ———————————————————————————————————————————————————
export interface Diagnostics {
stages: string[];
gpuMs: number;
budgetMs: number;
dropped: number;
input: string;
output: string;
}
const STAGES: [RegExp, string][] = [
[/ArtCNN C4F32 DN/, 'ArtCNN 2× with denoising'],
[/ArtCNN C4F32 \(/, 'ArtCNN 2×'],
[/feature map|FSRCNN/i, 'FSRCNNX 2×'],
[/KrigBilateral/, 'KrigBilateral colour'],
[/^debanding/, 'Debanding'],
[/FidelityFX/, 'CAS sharpening'],
];
/** What the renderer actually did for the last frame, read from mpv. */
export async function readDiagnostics(): Promise<Diagnostics> {
const passes = await renderPasses();
const stages = STAGES.filter(([re]) => passes.some((p) => re.test(p.desc))).map(([, name]) => name);
const scaler = passes.map((p) => /(up|down)scaling \(([\w-]+)\)/.exec(p.desc)).find(Boolean);
if (scaler) stages.push(`${scaler[2]} ${scaler[1]}scaling`);
const fps = (await mpv.get<number>('estimated-vf-fps').catch(() => 0)) || (await mpv.get<number>('container-fps').catch(() => 0)) || 24;
const dropped = ((await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0) + ((await mpv.get<number>('decoder-frame-drop-count').catch(() => 0)) || 0);
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
const v = usePlayer.getState().video;
return {
stages,
// Pass timings are in nanoseconds.
gpuMs: passes.reduce((a, p) => a + (p.avg || 0), 0) / 1e6,
budgetMs: 1000 / fps,
dropped,
input: v?.width ? `${v.width}×${v.height}` : '',
output: osd ? `${osd.w - osd.ml - osd.mr}×${osd.h - osd.mt - osd.mb}` : '',
};
}
/**
* While a neural tier plays, watch for dropped frames and suggest a lighter
* tier once per title if the GPU can't keep up.
*/
let watchTimer: ReturnType<typeof setInterval> | null = null;
export function watchPerformance(active: boolean) {
if (watchTimer) clearInterval(watchTimer);
watchTimer = null;
if (!active || !isNative) return;
let last = -1;
let warned = false;
watchTimer = setInterval(async () => {
const st = usePlayer.getState();
const mode = st.enhance.active;
if (!st.open || st.paused || st.enhance.bypass || (mode !== 'quality' && mode !== 'balanced')) return void (last = -1);
const dropped = (await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0;
if (last >= 0 && dropped - last >= 10 && !warned) {
warned = true;
toast(`${UPSCALING[mode].label} is dropping frames on this video. Try a lighter upscaling tier.`);
}
last = dropped;
}, 5000);
}
// Stage toggles and the tier apply straight away while something plays.
useSettings.subscribe((prefs, prev) => {
if (!isNative || !usePlayer.getState().open) return;
if (prefs.upscaling !== prev.upscaling || prefs.enhanceDeband !== prev.enhanceDeband || prefs.enhanceSharpen !== prev.enhanceSharpen || prefs.enhanceDenoise !== prev.enhanceDenoise) requestEnhancement();
});