/** * Video enhancement: upscaling tiers built from mpv's scalers, debanding, the * shaders bundled in src-tauri/shaders and, on Windows with an NVIDIA RTX * GPU, the driver's RTX Video Super Resolution. * * Shader tiers: each shader's WHEN condition decides whether it runs for the * current video and window size, so one list covers every case. FSRCNNX 16 * doubles luma at any enlargement (1080p in a maximized 1440p window is * doubled and scaled down), KrigBilateral rebuilds subsampled colour, CAS * sharpens at native size (CAS-prescale also before upscaling) and nlmeans * denoises lightly at any size. Against a 1080p reference shrunk to 480p, * FSRCNNX 16 beat the smaller model, two network passes, RAVU and * SSimSuperRes. * * Quality hands enlargement to the GPU's video processor (mpv's d3d11vpp * filter with NVIDIA's scaling mode), sized to the window, which also * removes compression artifacts. It needs D3D11 decoding and SDR video; * otherwise (or at native size) it runs Balanced's chain. * * The tier the user chose (preferences) and the one actually running (player * state) are kept apart: a tier that fails drops to the one below with a * notice instead of silently showing plain scaling, and isn't retried. */ import { useSettings, UPSCALING, type Upscaling } from '../app/settings'; import { appInfo, isNative, resourcePath } from '../platform/native'; import { toast } from '../ui/overlays'; import { mpv } from './engine'; import { usePlayer } from './store'; export const UPSCALING_ORDER: Upscaling[] = ['off', 'performance', 'balanced', 'quality']; /** mpv's own scaler settings, read once so Off can put them back. */ const SCALER_PROPS = ['scale', 'cscale', 'dscale', 'sigmoid-upscaling', 'deband', 'deband-iterations', 'deband-grain'] as const; let scalerDefaults: Partial> | null = null; let shaderDir: Promise | null = null; const NETWORK = 'FSRCNNX_x2_16-0-4-1.glsl'; /** RTX Video needs Windows (D3D11) and an NVIDIA RTX GPU. */ let rtxVideo = false; appInfo() .then((info) => (rtxVideo = info.platform === 'windows' && info.gpus.some((n) => /NVIDIA/i.test(n) && /\bRTX\b/i.test(n)))) .catch(() => {}); /** Whether this computer can run a tier at all (Quality is greyed out otherwise). */ export const tierAvailable = (mode: Upscaling) => mode !== 'quality' || rtxVideo; /** mpv label for the RTX Video filter, so other video filters are left alone. */ const VSR_LABEL = '@kaleido-vsr'; /** NVIDIA's video processor enlarges up to 4×; mpv scales any remainder. */ const VSR_MAX_SCALE = 4; interface Plan { props: Record; shaders: string[]; /** d3d11vpp settings for RTX Video, or '' for none. */ vsr: string; } /** * What a tier runs. Sharpening (CAS) and denoising (nlmeans) are tuned for * SDR levels, so HDR video skips them; the upscalers and debanding stay. * Balanced denoises first, so the network enlarges the picture rather than * the compression noise. FSRCNNX is trained on general imagery, which suits * live action. */ function plan(mode: Upscaling, hdr: boolean, scale: number): Plan { const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState(); const sharpen = enhanceSharpen && !hdr; let props: Record; let shaders: string[] = []; let vsr = ''; // RTX Video is SDR-only and only enlarges; otherwise Quality runs Balanced. if (mode === 'quality' && (hdr || scale <= 1.05)) mode = 'balanced'; switch (mode) { case 'performance': props = { scale: 'spline36', cscale: 'spline36', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 24 }; if (sharpen) shaders = ['CAS-prescale.glsl']; break; case 'balanced': props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 }; shaders = [...(enhanceDenoise && !hdr ? ['nlmeans_light.glsl'] : []), NETWORK, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])]; break; case 'quality': // The video processor deblocks as it enlarges; mpv only scales any remainder. props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 8 }; vsr = `d3d11vpp=scaling-mode=nvidia:scale=${Math.min(scale, VSR_MAX_SCALE).toFixed(2)}`; break; default: return { props: { ...scalerDefaults }, shaders: [], vsr }; } if (!enhanceDeband) props.deband = false; return { props, shaders, vsr }; } /** HDR as decoded (a converted stream may already be SDR), else Plex's metadata. */ function isHdr() { const st = usePlayer.getState(); if (st.videoGamma) return st.videoGamma === 'pq' || st.videoGamma === 'hlg'; return st.delivery?.mode !== 'transcode' && !!st.item?.media[st.mediaIndex]?.dynamicRange; } // —— Shader failures ———————————————————————————————————————————————— const shaderErrors: { at: number; text: string }[] = []; /** Fed every mpv log line; keeps the ones that mean a user shader or RTX Video failed. */ export function noteMpvLog(level: string, prefix: string, text: string) { if (/RTX Super Resolution/i.test(text) && /fail|not supported/i.test(text)) { shaderErrors.push({ at: Date.now(), text: text.trim() }); return; } if (level !== 'error' && level !== 'fatal') return; // A missing file is reported by mpv's file/stream layers; a broken one by the renderer. if (/\.glsl/i.test(text) || (/^vo/.test(prefix) && /shader|glsl|hook|compil|spirv|parse/i.test(text))) { shaderErrors.push({ at: Date.now(), text: text.trim() }); if (shaderErrors.length > 20) shaderErrors.shift(); } } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); /** Descriptions of the render passes mpv ran for the last frame (or redraw). */ type Pass = { desc: string; avg: number; samples?: number[] }; export async function renderPasses(): Promise<{ desc: string; avg: number }[]> { const passes = await mpv.get<{ fresh?: Pass[]; redraw?: Pass[] }>('vo-passes').catch(() => null); const list = (passes?.fresh?.length ? passes.fresh : passes?.redraw) ?? []; // The median of recent frames: the average includes one-off shader compiles. return list.map((p) => { const s = [...(p.samples ?? [])].sort((a, b) => a - b); return { desc: p.desc, avg: s.length ? s[Math.floor(s.length / 2)] : p.avg }; }); } /** * How much the video is enlarged on screen. The full player fills the window, * which is known before mpv has drawn anything; the miniplayer asks mpv. */ export async function scaleFactor(): Promise { const st = usePlayer.getState(); if (!st.video?.width) return 1; if (!st.mini) { const dpr = window.devicePixelRatio || 1; return Math.min((window.innerWidth * dpr) / st.video.width, (window.innerHeight * dpr) / st.video.height); } const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null); if (!osd) return 1; return Math.min((osd.w - osd.ml - osd.mr) / st.video.width, (osd.h - osd.mt - osd.mb) / st.video.height); } /** Whether RTX Video is enlarging the picture (mpv's output exceeds the decoded size). */ async function vsrRunning() { const st = usePlayer.getState(); const out = await mpv.get<{ w: number }>('video-out-params').catch(() => null); return !!st.video?.width && !!out?.w && out.w > st.video.width * 1.05; } /** * After a tier is applied: wait for frames, then look for shader or RTX * Video errors in mpv's log, for the passes that should be running at this * size (the denoiser always, the network when enlarging) and for RTX Video's * enlarged output. Any failing drops to the tier below. */ async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string, { shaders, vsr }: Plan) { for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250); await sleep(1500); if (id !== seq || !usePlayer.getState().open) return; const expected: RegExp[] = []; if (shaders.includes('nlmeans_light.glsl')) expected.push(/Non-local means/); if (shaders.includes(NETWORK) && (await scaleFactor()) > 1) expected.push(/feature map/); let error = shaderErrors.find((e) => e.at >= appliedAt); let missing = false; if (!error && vsr) { missing = true; for (let i = 0; i < 8 && id === seq && usePlayer.getState().open; i++) { error = shaderErrors.find((e) => e.at >= appliedAt); if (error || (await vsrRunning())) { missing = !!error; break; } await sleep(1000); } } if (!error && !missing && expected.length) { // A first compile can hold the renderer for over a minute; wait for the // expected passes to appear (or an error) rather than judging too early. // Already compiled, they appear within a few frames. missing = true; const patience = usePlayer.getState().enhance.preparing ? 240 : 8; for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) { error = shaderErrors.find((e) => e.at >= appliedAt); const passes = await renderPasses(); if (error || expected.every((re) => passes.some((p) => re.test(p.desc)))) { missing = false; break; } await sleep(1000); } } if (id !== seq) return; if (!error && !missing) { // Nothing compiled at this size (e.g. the network isn't needed) leaves the set unproven. if (expected.length) markWarm(signature); else clearPreparing(); return; } clearPreparing(); failed.add(mode); const fallback = usable(mode); console.warn('[upscaling]', mode, 'failed:', error?.text ?? (vsr ? 'RTX Video output not enlarged' : 'network passes not running')); toast(`${UPSCALING[mode].label} upscaling couldn’t start. Using ${UPSCALING[fallback].label}.`, 'error'); requestEnhancement(); } // —— Applying —————————————————————————————————————————————————————— let seq = 0; let chain: Promise = Promise.resolve(); /** How long preparation must take before the player mentions it. */ export const PREPARING_NOTICE_MS = 3000; /** * Shader sets that have compiled on this computer. mpv keeps compiled shaders * on disk (gpu-shader-cache-dir); a first compile for Direct3D can take a * while on some GPUs, so until a set is known the player can say so (after a * short delay, so a quick compile shows nothing). */ const WARM_KEY = 'kaleido.warmShaders'; function warmSet(): Set { try { return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]')); } catch { return new Set(); } } function clearPreparing() { if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } }); } function markWarm(signature: string) { const set = warmSet(); set.add(signature); try { localStorage.setItem(WARM_KEY, JSON.stringify([...set])); } catch { /* per-device convenience only */ } if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } }); } /** Tiers that failed this session. */ const failed = new Set(); let missingWarned = false; /** The chosen tier, or the nearest one below it that this computer can run and hasn't failed. */ function usable(mode: Upscaling): Upscaling { let i = UPSCALING_ORDER.indexOf(mode); while (i > 0 && (failed.has(UPSCALING_ORDER[i]) || !tierAvailable(UPSCALING_ORDER[i]))) i--; return UPSCALING_ORDER[i]; } let mpvVersion = ''; /** The last settings sent to mpv (and when), so an identical re-apply leaves mpv alone. */ let lastApplied = ''; let lastAppliedAt = 0; async function apply(id: number, requested: Upscaling) { if (!isNative || id !== seq) return; if (!scalerDefaults) { scalerDefaults = {}; for (const p of SCALER_PROPS) scalerDefaults[p] = await mpv.get(p).catch(() => undefined); mpvVersion = (await mpv.get('mpv-version').catch(() => '')) ?? ''; } const bypass = usePlayer.getState().enhance.bypass; let mode = usable(requested); const scale = await scaleFactor(); let { props, shaders, vsr } = plan(bypass ? 'off' : mode, isHdr(), scale); const dir = await (shaderDir ??= resourcePath('shaders').catch(() => '')); if (shaders.length && !dir) { // No shaders at all: keep the scaler and debanding settings only. if (!missingWarned) toast('The upscaling shaders are missing from this installation.', 'error'); missingWarned = true; shaders = []; if (mode !== 'performance') mode = 'performance'; ({ props, vsr } = plan(mode, isHdr(), scale)); } // Decided before touching mpv: while a shader compiles, mpv holds option // changes until it finishes, so a notice set afterwards would come too late. // Keyed by mpv's version too: a new libmpv recompiles everything. // WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile. const signature = `${mpvVersion}|${shaders.join('|')}`; const preparing = !bypass && shaders.length > 0 && !warmSet().has(signature); usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } }); if (preparing && usePlayer.getState().firstFrame) setTimeout(() => { if (id === seq && usePlayer.getState().enhance.preparing) toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for a moment.`); }, PREPARING_NOTICE_MS); const settings = JSON.stringify({ props, shaders, vsr }); if (settings !== lastApplied) { lastApplied = settings; lastAppliedAt = Date.now(); for (const [k, v] of Object.entries(props)) if (v !== undefined) await mpv.set(k, v).catch(() => {}); if (shaders.length) { // mpv takes a path list: ';'-separated on Windows, ':' elsewhere. const windows = dir.includes('\\'); const list = shaders.map((n) => `${dir}${windows ? '\\' : '/'}${n}`).join(windows ? ';' : ':'); await mpv.set('glsl-shaders', list).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) })); } else await mpv.command('change-list', 'glsl-shaders', 'clr', '').catch(() => {}); // Replaced by label; removing one that isn't there is harmless. await mpv.command('vf', 'remove', VSR_LABEL).catch(() => {}); if (vsr) await mpv.command('vf', 'add', `${VSR_LABEL}:${vsr}`).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) })); } // Errors count from when these shaders actually reached mpv, even if this // re-apply changed nothing (mpv may have already reported a failure). if (!bypass && (shaders.length || vsr)) verify(id, mode, lastAppliedAt, signature, { props, shaders, vsr }); } /** * Applies the chosen tier (and stage toggles) to the playing video. Requests * are serialized and only the latest runs, so quick U presses can't interleave. */ export function requestEnhancement() { const id = ++seq; const run = chain.then(() => apply(id, useSettings.getState().upscaling)); chain = run.catch(() => {}); return run; } /** Whether a tier failed this session (shown in the player's choices). */ export const hasFailed = (mode: Upscaling) => failed.has(mode); /** Before/after comparison: enhancement off without changing the preference. */ export function toggleBypass() { const st = usePlayer.getState(); const bypass = !st.enhance.bypass; usePlayer.setState({ enhance: { ...st.enhance, bypass } }); toast(bypass ? 'Enhancement off for comparison · press B to turn it back on' : `Enhancement on · ${UPSCALING[useSettings.getState().upscaling].label}`); return requestEnhancement(); } // —— Diagnostics ——————————————————————————————————————————————————— export interface Diagnostics { stages: string[]; gpuMs: number; budgetMs: number; dropped: number; input: string; output: string; } const STAGES: [RegExp, string][] = [ [/Non-local means/, 'nlmeans denoising'], [/feature map|FSRCNN/i, 'FSRCNNX 2×'], [/KrigBilateral/, 'KrigBilateral colour'], [/^debanding/, 'Debanding'], [/FidelityFX/, 'CAS sharpening'], ]; /** What the renderer actually did for the last frame, read from mpv. */ export async function readDiagnostics(): Promise { const passes = await renderPasses(); const stages = STAGES.filter(([re]) => passes.some((p) => re.test(p.desc))).map(([, name]) => name); // RTX Video runs in the GPU's video processor, before (and outside) mpv's passes. if (await vsrRunning()) stages.unshift('RTX Video Super Resolution'); const scaler = passes.map((p) => /(up|down)scaling \(([\w-]+)\)/.exec(p.desc)).find(Boolean); if (scaler) stages.push(`${scaler[2]} ${scaler[1]}scaling`); const fps = (await mpv.get('estimated-vf-fps').catch(() => 0)) || (await mpv.get('container-fps').catch(() => 0)) || 24; const dropped = ((await mpv.get('frame-drop-count').catch(() => 0)) || 0) + ((await mpv.get('decoder-frame-drop-count').catch(() => 0)) || 0); const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null); const v = usePlayer.getState().video; return { stages, // Pass timings are in nanoseconds. gpuMs: passes.reduce((a, p) => a + (p.avg || 0), 0) / 1e6, budgetMs: 1000 / fps, dropped, input: v?.width ? `${v.width}×${v.height}` : '', output: osd ? `${osd.w - osd.ml - osd.mr}×${osd.h - osd.mt - osd.mb}` : '', }; } /** * While a neural tier plays, watch for dropped frames and suggest a lighter * tier once per title if the GPU can't keep up. */ let watchTimer: ReturnType | null = null; export function watchPerformance(active: boolean) { if (watchTimer) clearInterval(watchTimer); watchTimer = null; if (!active || !isNative) return; let last = -1; let warned = false; watchTimer = setInterval(async () => { const st = usePlayer.getState(); const mode = st.enhance.active; if (!st.open || st.paused || st.enhance.bypass || (mode !== 'quality' && mode !== 'balanced')) return void (last = -1); const dropped = (await mpv.get('frame-drop-count').catch(() => 0)) || 0; if (last >= 0 && dropped - last >= 10 && !warned) { warned = true; toast(`${UPSCALING[mode].label} is dropping frames on this video. Try a lighter upscaling tier.`); } last = dropped; }, 5000); } // Stage toggles and the tier apply straight away while something plays. useSettings.subscribe((prefs, prev) => { if (!isNative || !usePlayer.getState().open) return; if (prefs.upscaling !== prev.upscaling || prefs.enhanceDeband !== prev.enhanceDeband || prefs.enhanceSharpen !== prev.enhanceSharpen || prefs.enhanceDenoise !== prev.enhanceDenoise) requestEnhancement(); }); // RTX Video enlarges by a fixed factor, so Quality follows the video's size // (known only once decoding starts), the miniplayer and the window. let resizeTimer: ReturnType | null = null; function followSize(delay: number) { const st = usePlayer.getState(); if (!isNative || !st.open || usable(useSettings.getState().upscaling) !== 'quality') return; if (resizeTimer) clearTimeout(resizeTimer); resizeTimer = setTimeout(() => requestEnhancement(), delay); } let following = false; /** Called by the player when it opens (not at import: the store imports this module). */ export function followVideoSize() { if (following) return; following = true; usePlayer.subscribe((st, prev) => { if (st.video?.width !== prev.video?.width || st.video?.height !== prev.video?.height || st.mini !== prev.mini) followSize(st.mini !== prev.mini ? 600 : 0); }); window.addEventListener('resize', () => followSize(400)); }