Kaleido-Desktop/src/player/enhance.ts
Elijah d9e22dde2e Handle stopped streams, live bandwidth, and rework the upscaling tiers
When the server owner stops a stream, Plex cuts the connection and repeats
the termination in every timeline reply to the client until it reports a
stop. The player now reads those replies (also when playback stalls or
fails), stops mpv, shows the owner's message and acknowledges it; a
termination on a session's first reply is left over from an earlier stream
and is cleared instead of cancelling the new one. Each title also starts
unpaused, as mpv keeps pause across files.

The dashboard's bandwidth graph refreshes every second. Plex's real-time
samples are bytes per second with idle seconds omitted, so they are laid on
a gap-free per-second axis (they were divided by six before).

Upscaling: Balanced is now the strongest shader chain, light nlmeans,
FSRCNNX 16, KrigBilateral and debanding. Measured against 1080p references
shrunk to 480p it beat FSRCNNX 8, two network passes, RAVU and
SSimSuperRes. The denoiser is AN3223's light profile with luma strength at
60%, which keeps texture the default profile smoothed away. Quality is now
NVIDIA RTX Video Super Resolution through mpv's d3d11vpp filter, sized to
the window and re-applied when it changes; it is offered only on Windows
with an NVIDIA RTX GPU (app_info now lists the adapters), runs Balanced for
HDR or at native size, and drops a tier if the driver refuses.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 12:09:34 -07:00

430 lines
20 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* Video enhancement: upscaling tiers built from mpv's scalers, debanding, the
* shaders bundled in src-tauri/shaders and, on Windows with an NVIDIA RTX
* GPU, the driver's RTX Video Super Resolution.
*
* Shader tiers: each shader's WHEN condition decides whether it runs for the
* current video and window size, so one list covers every case. FSRCNNX 16
* doubles luma at any enlargement (1080p in a maximized 1440p window is
* doubled and scaled down), KrigBilateral rebuilds subsampled colour, CAS
* sharpens at native size (CAS-prescale also before upscaling) and nlmeans
* denoises lightly at any size. Against a 1080p reference shrunk to 480p,
* FSRCNNX 16 beat the smaller model, two network passes, RAVU and
* SSimSuperRes.
*
* Quality hands enlargement to the GPU's video processor (mpv's d3d11vpp
* filter with NVIDIA's scaling mode), sized to the window, which also
* removes compression artifacts. It needs D3D11 decoding and SDR video;
* otherwise (or at native size) it runs Balanced's chain.
*
* The tier the user chose (preferences) and the one actually running (player
* state) are kept apart: a tier that fails drops to the one below with a
* notice instead of silently showing plain scaling, and isn't retried.
*/
import { useSettings, UPSCALING, type Upscaling } from '../app/settings';
import { appInfo, isNative, resourcePath } from '../platform/native';
import { toast } from '../ui/overlays';
import { mpv } from './engine';
import { usePlayer } from './store';
export const UPSCALING_ORDER: Upscaling[] = ['off', 'performance', 'balanced', 'quality'];
/** mpv's own scaler settings, read once so Off can put them back. */
const SCALER_PROPS = ['scale', 'cscale', 'dscale', 'sigmoid-upscaling', 'deband', 'deband-iterations', 'deband-grain'] as const;
let scalerDefaults: Partial<Record<(typeof SCALER_PROPS)[number], unknown>> | null = null;
let shaderDir: Promise<string> | null = null;
const NETWORK = 'FSRCNNX_x2_16-0-4-1.glsl';
/** RTX Video needs Windows (D3D11) and an NVIDIA RTX GPU. */
let rtxVideo = false;
appInfo()
.then((info) => (rtxVideo = info.platform === 'windows' && info.gpus.some((n) => /NVIDIA/i.test(n) && /\bRTX\b/i.test(n))))
.catch(() => {});
/** Whether this computer can run a tier at all (Quality is greyed out otherwise). */
export const tierAvailable = (mode: Upscaling) => mode !== 'quality' || rtxVideo;
/** mpv label for the RTX Video filter, so other video filters are left alone. */
const VSR_LABEL = '@kaleido-vsr';
/** NVIDIA's video processor enlarges up to 4×; mpv scales any remainder. */
const VSR_MAX_SCALE = 4;
interface Plan {
props: Record<string, unknown>;
shaders: string[];
/** d3d11vpp settings for RTX Video, or '' for none. */
vsr: string;
}
/**
* What a tier runs. Sharpening (CAS) and denoising (nlmeans) are tuned for
* SDR levels, so HDR video skips them; the upscalers and debanding stay.
* Balanced denoises first, so the network enlarges the picture rather than
* the compression noise. FSRCNNX is trained on general imagery, which suits
* live action.
*/
function plan(mode: Upscaling, hdr: boolean, scale: number): Plan {
const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState();
const sharpen = enhanceSharpen && !hdr;
let props: Record<string, unknown>;
let shaders: string[] = [];
let vsr = '';
// RTX Video is SDR-only and only enlarges; otherwise Quality runs Balanced.
if (mode === 'quality' && (hdr || scale <= 1.05)) mode = 'balanced';
switch (mode) {
case 'performance':
props = { scale: 'spline36', cscale: 'spline36', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 24 };
if (sharpen) shaders = ['CAS-prescale.glsl'];
break;
case 'balanced':
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 };
shaders = [...(enhanceDenoise && !hdr ? ['nlmeans_light.glsl'] : []), NETWORK, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
break;
case 'quality':
// The video processor deblocks as it enlarges; mpv only scales any remainder.
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 8 };
vsr = `d3d11vpp=scaling-mode=nvidia:scale=${Math.min(scale, VSR_MAX_SCALE).toFixed(2)}`;
break;
default:
return { props: { ...scalerDefaults }, shaders: [], vsr };
}
if (!enhanceDeband) props.deband = false;
return { props, shaders, vsr };
}
/** HDR as decoded (a converted stream may already be SDR), else Plex's metadata. */
function isHdr() {
const st = usePlayer.getState();
if (st.videoGamma) return st.videoGamma === 'pq' || st.videoGamma === 'hlg';
return st.delivery?.mode !== 'transcode' && !!st.item?.media[st.mediaIndex]?.dynamicRange;
}
// —— Shader failures ————————————————————————————————————————————————
const shaderErrors: { at: number; text: string }[] = [];
/** Fed every mpv log line; keeps the ones that mean a user shader or RTX Video failed. */
export function noteMpvLog(level: string, prefix: string, text: string) {
if (/RTX Super Resolution/i.test(text) && /fail|not supported/i.test(text)) {
shaderErrors.push({ at: Date.now(), text: text.trim() });
return;
}
if (level !== 'error' && level !== 'fatal') return;
// A missing file is reported by mpv's file/stream layers; a broken one by the renderer.
if (/\.glsl/i.test(text) || (/^vo/.test(prefix) && /shader|glsl|hook|compil|spirv|parse/i.test(text))) {
shaderErrors.push({ at: Date.now(), text: text.trim() });
if (shaderErrors.length > 20) shaderErrors.shift();
}
}
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
/** Descriptions of the render passes mpv ran for the last frame (or redraw). */
type Pass = { desc: string; avg: number; samples?: number[] };
export async function renderPasses(): Promise<{ desc: string; avg: number }[]> {
const passes = await mpv.get<{ fresh?: Pass[]; redraw?: Pass[] }>('vo-passes').catch(() => null);
const list = (passes?.fresh?.length ? passes.fresh : passes?.redraw) ?? [];
// The median of recent frames: the average includes one-off shader compiles.
return list.map((p) => {
const s = [...(p.samples ?? [])].sort((a, b) => a - b);
return { desc: p.desc, avg: s.length ? s[Math.floor(s.length / 2)] : p.avg };
});
}
/**
* How much the video is enlarged on screen. The full player fills the window,
* which is known before mpv has drawn anything; the miniplayer asks mpv.
*/
export async function scaleFactor(): Promise<number> {
const st = usePlayer.getState();
if (!st.video?.width) return 1;
if (!st.mini) {
const dpr = window.devicePixelRatio || 1;
return Math.min((window.innerWidth * dpr) / st.video.width, (window.innerHeight * dpr) / st.video.height);
}
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
if (!osd) return 1;
return Math.min((osd.w - osd.ml - osd.mr) / st.video.width, (osd.h - osd.mt - osd.mb) / st.video.height);
}
/** Whether RTX Video is enlarging the picture (mpv's output exceeds the decoded size). */
async function vsrRunning() {
const st = usePlayer.getState();
const out = await mpv.get<{ w: number }>('video-out-params').catch(() => null);
return !!st.video?.width && !!out?.w && out.w > st.video.width * 1.05;
}
/**
* After a tier is applied: wait for frames, then look for shader or RTX
* Video errors in mpv's log, for the passes that should be running at this
* size (the denoiser always, the network when enlarging) and for RTX Video's
* enlarged output. Any failing drops to the tier below.
*/
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string, { shaders, vsr }: Plan) {
for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250);
await sleep(1500);
if (id !== seq || !usePlayer.getState().open) return;
const expected: RegExp[] = [];
if (shaders.includes('nlmeans_light.glsl')) expected.push(/Non-local means/);
if (shaders.includes(NETWORK) && (await scaleFactor()) > 1) expected.push(/feature map/);
let error = shaderErrors.find((e) => e.at >= appliedAt);
let missing = false;
if (!error && vsr) {
missing = true;
for (let i = 0; i < 8 && id === seq && usePlayer.getState().open; i++) {
error = shaderErrors.find((e) => e.at >= appliedAt);
if (error || (await vsrRunning())) {
missing = !!error;
break;
}
await sleep(1000);
}
}
if (!error && !missing && expected.length) {
// A first compile can hold the renderer for over a minute; wait for the
// expected passes to appear (or an error) rather than judging too early.
// Already compiled, they appear within a few frames.
missing = true;
const patience = usePlayer.getState().enhance.preparing ? 240 : 8;
for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) {
error = shaderErrors.find((e) => e.at >= appliedAt);
const passes = await renderPasses();
if (error || expected.every((re) => passes.some((p) => re.test(p.desc)))) {
missing = false;
break;
}
await sleep(1000);
}
}
if (id !== seq) return;
if (!error && !missing) {
// Nothing compiled at this size (e.g. the network isn't needed) leaves the set unproven.
if (expected.length) markWarm(signature);
else clearPreparing();
return;
}
clearPreparing();
failed.add(mode);
const fallback = usable(mode);
console.warn('[upscaling]', mode, 'failed:', error?.text ?? (vsr ? 'RTX Video output not enlarged' : 'network passes not running'));
toast(`${UPSCALING[mode].label} upscaling couldn’t start. Using ${UPSCALING[fallback].label}.`, 'error');
requestEnhancement();
}
// —— Applying ——————————————————————————————————————————————————————
let seq = 0;
let chain: Promise<unknown> = Promise.resolve();
/** How long preparation must take before the player mentions it. */
export const PREPARING_NOTICE_MS = 3000;
/**
* Shader sets that have compiled on this computer. mpv keeps compiled shaders
* on disk (gpu-shader-cache-dir); a first compile for Direct3D can take a
* while on some GPUs, so until a set is known the player can say so (after a
* short delay, so a quick compile shows nothing).
*/
const WARM_KEY = 'kaleido.warmShaders';
function warmSet(): Set<string> {
try {
return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]'));
} catch {
return new Set();
}
}
function clearPreparing() {
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
}
function markWarm(signature: string) {
const set = warmSet();
set.add(signature);
try {
localStorage.setItem(WARM_KEY, JSON.stringify([...set]));
} catch {
/* per-device convenience only */
}
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
}
/** Tiers that failed this session. */
const failed = new Set<Upscaling>();
let missingWarned = false;
/** The chosen tier, or the nearest one below it that this computer can run and hasn't failed. */
function usable(mode: Upscaling): Upscaling {
let i = UPSCALING_ORDER.indexOf(mode);
while (i > 0 && (failed.has(UPSCALING_ORDER[i]) || !tierAvailable(UPSCALING_ORDER[i]))) i--;
return UPSCALING_ORDER[i];
}
let mpvVersion = '';
/** The last settings sent to mpv (and when), so an identical re-apply leaves mpv alone. */
let lastApplied = '';
let lastAppliedAt = 0;
async function apply(id: number, requested: Upscaling) {
if (!isNative || id !== seq) return;
if (!scalerDefaults) {
scalerDefaults = {};
for (const p of SCALER_PROPS) scalerDefaults[p] = await mpv.get(p).catch(() => undefined);
mpvVersion = (await mpv.get<string>('mpv-version').catch(() => '')) ?? '';
}
const bypass = usePlayer.getState().enhance.bypass;
let mode = usable(requested);
const scale = await scaleFactor();
let { props, shaders, vsr } = plan(bypass ? 'off' : mode, isHdr(), scale);
const dir = await (shaderDir ??= resourcePath('shaders').catch(() => ''));
if (shaders.length && !dir) {
// No shaders at all: keep the scaler and debanding settings only.
if (!missingWarned) toast('The upscaling shaders are missing from this installation.', 'error');
missingWarned = true;
shaders = [];
if (mode !== 'performance') mode = 'performance';
({ props, vsr } = plan(mode, isHdr(), scale));
}
// Decided before touching mpv: while a shader compiles, mpv holds option
// changes until it finishes, so a notice set afterwards would come too late.
// Keyed by mpv's version too: a new libmpv recompiles everything.
// WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile.
const signature = `${mpvVersion}|${shaders.join('|')}`;
const preparing = !bypass && shaders.length > 0 && !warmSet().has(signature);
usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } });
if (preparing && usePlayer.getState().firstFrame)
setTimeout(() => {
if (id === seq && usePlayer.getState().enhance.preparing) toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for a moment.`);
}, PREPARING_NOTICE_MS);
const settings = JSON.stringify({ props, shaders, vsr });
if (settings !== lastApplied) {
lastApplied = settings;
lastAppliedAt = Date.now();
for (const [k, v] of Object.entries(props)) if (v !== undefined) await mpv.set(k, v).catch(() => {});
if (shaders.length) {
// mpv takes a path list: ';'-separated on Windows, ':' elsewhere.
const windows = dir.includes('\\');
const list = shaders.map((n) => `${dir}${windows ? '\\' : '/'}${n}`).join(windows ? ';' : ':');
await mpv.set('glsl-shaders', list).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) }));
} else await mpv.command('change-list', 'glsl-shaders', 'clr', '').catch(() => {});
// Replaced by label; removing one that isn't there is harmless.
await mpv.command('vf', 'remove', VSR_LABEL).catch(() => {});
if (vsr) await mpv.command('vf', 'add', `${VSR_LABEL}:${vsr}`).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) }));
}
// Errors count from when these shaders actually reached mpv, even if this
// re-apply changed nothing (mpv may have already reported a failure).
if (!bypass && (shaders.length || vsr)) verify(id, mode, lastAppliedAt, signature, { props, shaders, vsr });
}
/**
* Applies the chosen tier (and stage toggles) to the playing video. Requests
* are serialized and only the latest runs, so quick U presses can't interleave.
*/
export function requestEnhancement() {
const id = ++seq;
const run = chain.then(() => apply(id, useSettings.getState().upscaling));
chain = run.catch(() => {});
return run;
}
/** Whether a tier failed this session (shown in the player's choices). */
export const hasFailed = (mode: Upscaling) => failed.has(mode);
/** Before/after comparison: enhancement off without changing the preference. */
export function toggleBypass() {
const st = usePlayer.getState();
const bypass = !st.enhance.bypass;
usePlayer.setState({ enhance: { ...st.enhance, bypass } });
toast(bypass ? 'Enhancement off for comparison · press B to turn it back on' : `Enhancement on · ${UPSCALING[useSettings.getState().upscaling].label}`);
return requestEnhancement();
}
// —— Diagnostics ———————————————————————————————————————————————————
export interface Diagnostics {
stages: string[];
gpuMs: number;
budgetMs: number;
dropped: number;
input: string;
output: string;
}
const STAGES: [RegExp, string][] = [
[/Non-local means/, 'nlmeans denoising'],
[/feature map|FSRCNN/i, 'FSRCNNX 2×'],
[/KrigBilateral/, 'KrigBilateral colour'],
[/^debanding/, 'Debanding'],
[/FidelityFX/, 'CAS sharpening'],
];
/** What the renderer actually did for the last frame, read from mpv. */
export async function readDiagnostics(): Promise<Diagnostics> {
const passes = await renderPasses();
const stages = STAGES.filter(([re]) => passes.some((p) => re.test(p.desc))).map(([, name]) => name);
// RTX Video runs in the GPU's video processor, before (and outside) mpv's passes.
if (await vsrRunning()) stages.unshift('RTX Video Super Resolution');
const scaler = passes.map((p) => /(up|down)scaling \(([\w-]+)\)/.exec(p.desc)).find(Boolean);
if (scaler) stages.push(`${scaler[2]} ${scaler[1]}scaling`);
const fps = (await mpv.get<number>('estimated-vf-fps').catch(() => 0)) || (await mpv.get<number>('container-fps').catch(() => 0)) || 24;
const dropped = ((await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0) + ((await mpv.get<number>('decoder-frame-drop-count').catch(() => 0)) || 0);
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
const v = usePlayer.getState().video;
return {
stages,
// Pass timings are in nanoseconds.
gpuMs: passes.reduce((a, p) => a + (p.avg || 0), 0) / 1e6,
budgetMs: 1000 / fps,
dropped,
input: v?.width ? `${v.width}×${v.height}` : '',
output: osd ? `${osd.w - osd.ml - osd.mr}×${osd.h - osd.mt - osd.mb}` : '',
};
}
/**
* While a neural tier plays, watch for dropped frames and suggest a lighter
* tier once per title if the GPU can't keep up.
*/
let watchTimer: ReturnType<typeof setInterval> | null = null;
export function watchPerformance(active: boolean) {
if (watchTimer) clearInterval(watchTimer);
watchTimer = null;
if (!active || !isNative) return;
let last = -1;
let warned = false;
watchTimer = setInterval(async () => {
const st = usePlayer.getState();
const mode = st.enhance.active;
if (!st.open || st.paused || st.enhance.bypass || (mode !== 'quality' && mode !== 'balanced')) return void (last = -1);
const dropped = (await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0;
if (last >= 0 && dropped - last >= 10 && !warned) {
warned = true;
toast(`${UPSCALING[mode].label} is dropping frames on this video. Try a lighter upscaling tier.`);
}
last = dropped;
}, 5000);
}
// Stage toggles and the tier apply straight away while something plays.
useSettings.subscribe((prefs, prev) => {
if (!isNative || !usePlayer.getState().open) return;
if (prefs.upscaling !== prev.upscaling || prefs.enhanceDeband !== prev.enhanceDeband || prefs.enhanceSharpen !== prev.enhanceSharpen || prefs.enhanceDenoise !== prev.enhanceDenoise) requestEnhancement();
});
// RTX Video enlarges by a fixed factor, so Quality follows the video's size
// (known only once decoding starts), the miniplayer and the window.
let resizeTimer: ReturnType<typeof setTimeout> | null = null;
function followSize(delay: number) {
const st = usePlayer.getState();
if (!isNative || !st.open || usable(useSettings.getState().upscaling) !== 'quality') return;
if (resizeTimer) clearTimeout(resizeTimer);
resizeTimer = setTimeout(() => requestEnhancement(), delay);
}
let following = false;
/** Called by the player when it opens (not at import: the store imports this module). */
export function followVideoSize() {
if (following) return;
following = true;
usePlayer.subscribe((st, prev) => {
if (st.video?.width !== prev.video?.width || st.video?.height !== prev.video?.height || st.mini !== prev.mini) followSize(st.mini !== prev.mini ? 600 : 0);
});
window.addEventListener('resize', () => followSize(400));
}