/** * Video enhancement: upscaling tiers built from mpv's scalers, debanding and * the shaders bundled in src-tauri/shaders. Each shader's WHEN condition * decides whether it runs for the current video and window size, so one list * covers every case: the neural upscalers double luma when the video is * enlarged (FSRCNNX above 1.3×, ArtCNN at any enlargement, so 1080p in a * maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds subsampled colour, * CAS sharpens at native size (CAS-prescale also before upscaling). * * The tier the user chose (preferences) and the one actually running (player * state) are kept apart: a tier whose shaders fail drops to the one below with * a notice instead of silently showing plain scaling, and isn't retried. */ import { useSettings, UPSCALING, type Upscaling } from '../app/settings'; import { isNative, resourcePath } from '../platform/native'; import { toast } from '../ui/overlays'; import { mpv } from './engine'; import { usePlayer } from './store'; export const UPSCALING_ORDER: Upscaling[] = ['off', 'performance', 'balanced', 'quality']; /** mpv's own scaler settings, read once so Off can put them back. */ const SCALER_PROPS = ['scale', 'cscale', 'dscale', 'sigmoid-upscaling', 'deband', 'deband-iterations', 'deband-grain'] as const; let scalerDefaults: Partial> | null = null; let shaderDir: Promise | null = null; /** The enlargement above which each tier's network runs (its shaders' WHEN condition). */ const NETWORK_MIN_SCALE: Partial> = { balanced: 1.3, quality: 1.0 }; const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!; const NEURAL: Partial> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'ArtCNN_C4F32.glsl' }; interface Plan { props: Record; shaders: string[]; } /** * What a tier runs. Sharpening (CAS) and denoising are tuned for SDR levels, * so HDR video skips them; the upscalers and debanding stay. Denoising uses * ArtCNN's own denoising model while upscaling: a separate denoiser (nlmeans) * took over a minute to compile for Direct3D before the first frame. */ function plan(mode: Upscaling, hdr: boolean): Plan { const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState(); const sharpen = enhanceSharpen && !hdr; let props: Record; let shaders: string[] = []; switch (mode) { case 'performance': props = { scale: 'spline36', cscale: 'spline36', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 24 }; if (sharpen) shaders = ['CAS-prescale.glsl']; break; case 'balanced': props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 16 }; shaders = [NEURAL.balanced!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])]; break; case 'quality': props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 }; shaders = [enhanceDenoise && !hdr ? 'ArtCNN_C4F32_DN.glsl' : NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])]; break; default: return { props: { ...scalerDefaults }, shaders: [] }; } if (!enhanceDeband) props.deband = false; return { props, shaders }; } /** HDR as decoded (a converted stream may already be SDR), else Plex's metadata. */ function isHdr() { const st = usePlayer.getState(); if (st.videoGamma) return st.videoGamma === 'pq' || st.videoGamma === 'hlg'; return st.delivery?.mode !== 'transcode' && !!st.item?.media[st.mediaIndex]?.dynamicRange; } // —— Shader failures ———————————————————————————————————————————————— const shaderErrors: { at: number; text: string }[] = []; /** Fed every mpv log line; keeps the ones that mean a user shader failed. */ export function noteMpvLog(level: string, prefix: string, text: string) { if (level !== 'error' && level !== 'fatal') return; // A missing file is reported by mpv's file/stream layers; a broken one by the renderer. if (/\.glsl/i.test(text) || (/^vo/.test(prefix) && /shader|glsl|hook|compil|spirv|parse/i.test(text))) { shaderErrors.push({ at: Date.now(), text: text.trim() }); if (shaderErrors.length > 20) shaderErrors.shift(); } } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); /** Descriptions of the render passes mpv ran for the last frame (or redraw). */ type Pass = { desc: string; avg: number; samples?: number[] }; export async function renderPasses(): Promise<{ desc: string; avg: number }[]> { const passes = await mpv.get<{ fresh?: Pass[]; redraw?: Pass[] }>('vo-passes').catch(() => null); const list = (passes?.fresh?.length ? passes.fresh : passes?.redraw) ?? []; // The median of recent frames: the average includes one-off shader compiles. return list.map((p) => { const s = [...(p.samples ?? [])].sort((a, b) => a - b); return { desc: p.desc, avg: s.length ? s[Math.floor(s.length / 2)] : p.avg }; }); } /** * How much the video is enlarged on screen. The full player fills the window, * which is known before mpv has drawn anything; the miniplayer asks mpv. */ export async function scaleFactor(): Promise { const st = usePlayer.getState(); if (!st.video?.width) return 1; if (!st.mini) { const dpr = window.devicePixelRatio || 1; return Math.min((window.innerWidth * dpr) / st.video.width, (window.innerHeight * dpr) / st.video.height); } const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null); if (!osd) return 1; return Math.min((osd.w - osd.ml - osd.mr) / st.video.width, (osd.h - osd.mt - osd.mb) / st.video.height); } /** * After a tier is applied: wait for frames, then look for shader errors in * mpv's log, and, when the network should be running at this size, for its * passes in the renderer. Either failing drops to the tier below. */ async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string) { for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250); await sleep(1500); if (id !== seq || !usePlayer.getState().open) return; const neural = NEURAL[mode]; const expectNetwork = !!neural && (await runsNetwork(mode)); const marker = mode === 'quality' ? /ArtCNN/ : /feature map|FSRCNN/i; let error = shaderErrors.find((e) => e.at >= appliedAt); let missing = false; if (!error && expectNetwork) { // A first compile can hold the renderer for over a minute; wait for the // network's passes to appear (or an error) rather than judging too early. // Already compiled, they appear within a few frames. missing = true; const patience = usePlayer.getState().enhance.preparing ? 240 : 8; for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) { error = shaderErrors.find((e) => e.at >= appliedAt); if (error || (await renderPasses()).some((p) => marker.test(p.desc))) { missing = false; break; } await sleep(1000); } } if (id !== seq) return; if (!error && !missing) { if (expectNetwork) markWarm(signature); return; } failed.add(mode); const fallback = usable(mode); console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running'); toast(`${UPSCALING[mode].label} upscaling couldn’t start. Using ${UPSCALING[fallback].label}.`, 'error'); requestEnhancement(); } // —— Applying —————————————————————————————————————————————————————— let seq = 0; let chain: Promise = Promise.resolve(); /** * Shader sets that have compiled on this computer. mpv keeps compiled shaders * on disk (gpu-shader-cache-dir), but the first ArtCNN compile for Direct3D * takes over a minute, so the player says so the first time. */ const WARM_KEY = 'kaleido.warmShaders'; const SLOW = /ArtCNN/; function warmSet(): Set { try { return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]')); } catch { return new Set(); } } function markWarm(signature: string) { const set = warmSet(); set.add(signature); try { localStorage.setItem(WARM_KEY, JSON.stringify([...set])); } catch { /* per-device convenience only */ } if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } }); } /** Tiers whose shaders failed this session. */ const failed = new Set(); let missingWarned = false; /** The chosen tier, or the nearest one below it that hasn't failed. */ function usable(mode: Upscaling): Upscaling { let i = UPSCALING_ORDER.indexOf(mode); while (i > 0 && failed.has(UPSCALING_ORDER[i])) i--; return UPSCALING_ORDER[i]; } let mpvVersion = ''; /** The last settings sent to mpv (and when), so an identical re-apply leaves mpv alone. */ let lastApplied = ''; let lastAppliedAt = 0; async function apply(id: number, requested: Upscaling) { if (!isNative || id !== seq) return; if (!scalerDefaults) { scalerDefaults = {}; for (const p of SCALER_PROPS) scalerDefaults[p] = await mpv.get(p).catch(() => undefined); mpvVersion = (await mpv.get('mpv-version').catch(() => '')) ?? ''; } const bypass = usePlayer.getState().enhance.bypass; let mode = usable(requested); let { props, shaders } = plan(bypass ? 'off' : mode, isHdr()); const dir = await (shaderDir ??= resourcePath('shaders').catch(() => '')); if (shaders.length && !dir) { // No shaders at all: keep the scaler and debanding settings only. if (!missingWarned) toast('The upscaling shaders are missing from this installation.', 'error'); missingWarned = true; shaders = []; if (mode !== 'performance') mode = 'performance'; ({ props } = plan(mode, isHdr())); } // Decided before touching mpv: while a shader compiles, mpv holds option // changes until it finishes, so a notice set afterwards would come too late. // Keyed by mpv's version too: a new libmpv recompiles everything. // WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile. const signature = `${mpvVersion}|${shaders.join('|')}`; const preparing = !bypass && shaders.some((n) => SLOW.test(n)) && !warmSet().has(signature) && (await runsNetwork(mode)); usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } }); if (preparing && usePlayer.getState().firstFrame) toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for about a minute.`); const settings = JSON.stringify({ props, shaders }); if (settings !== lastApplied) { lastApplied = settings; lastAppliedAt = Date.now(); for (const [k, v] of Object.entries(props)) if (v !== undefined) await mpv.set(k, v).catch(() => {}); if (shaders.length) { // mpv takes a path list: ';'-separated on Windows, ':' elsewhere. const windows = dir.includes('\\'); const list = shaders.map((n) => `${dir}${windows ? '\\' : '/'}${n}`).join(windows ? ';' : ':'); await mpv.set('glsl-shaders', list).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) })); } else await mpv.command('change-list', 'glsl-shaders', 'clr', '').catch(() => {}); } // Errors count from when these shaders actually reached mpv, even if this // re-apply changed nothing (mpv may have already reported a failure). if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature); } /** * Applies the chosen tier (and stage toggles) to the playing video. Requests * are serialized and only the latest runs, so quick U presses can't interleave. */ export function requestEnhancement() { const id = ++seq; const run = chain.then(() => apply(id, useSettings.getState().upscaling)); chain = run.catch(() => {}); return run; } /** Whether a tier failed this session (shown in the player's choices). */ export const hasFailed = (mode: Upscaling) => failed.has(mode); /** Before/after comparison: enhancement off without changing the preference. */ export function toggleBypass() { const st = usePlayer.getState(); const bypass = !st.enhance.bypass; usePlayer.setState({ enhance: { ...st.enhance, bypass } }); toast(bypass ? 'Enhancement off for comparison · press B to turn it back on' : `Enhancement on · ${UPSCALING[useSettings.getState().upscaling].label}`); return requestEnhancement(); } // —— Diagnostics ——————————————————————————————————————————————————— export interface Diagnostics { stages: string[]; gpuMs: number; budgetMs: number; dropped: number; input: string; output: string; } const STAGES: [RegExp, string][] = [ [/ArtCNN C4F32 DN/, 'ArtCNN 2× with denoising'], [/ArtCNN C4F32 \(/, 'ArtCNN 2×'], [/feature map|FSRCNN/i, 'FSRCNNX 2×'], [/KrigBilateral/, 'KrigBilateral colour'], [/^debanding/, 'Debanding'], [/FidelityFX/, 'CAS sharpening'], ]; /** What the renderer actually did for the last frame, read from mpv. */ export async function readDiagnostics(): Promise { const passes = await renderPasses(); const stages = STAGES.filter(([re]) => passes.some((p) => re.test(p.desc))).map(([, name]) => name); const scaler = passes.map((p) => /(up|down)scaling \(([\w-]+)\)/.exec(p.desc)).find(Boolean); if (scaler) stages.push(`${scaler[2]} ${scaler[1]}scaling`); const fps = (await mpv.get('estimated-vf-fps').catch(() => 0)) || (await mpv.get('container-fps').catch(() => 0)) || 24; const dropped = ((await mpv.get('frame-drop-count').catch(() => 0)) || 0) + ((await mpv.get('decoder-frame-drop-count').catch(() => 0)) || 0); const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null); const v = usePlayer.getState().video; return { stages, // Pass timings are in nanoseconds. gpuMs: passes.reduce((a, p) => a + (p.avg || 0), 0) / 1e6, budgetMs: 1000 / fps, dropped, input: v?.width ? `${v.width}×${v.height}` : '', output: osd ? `${osd.w - osd.ml - osd.mr}×${osd.h - osd.mt - osd.mb}` : '', }; } /** * While a neural tier plays, watch for dropped frames and suggest a lighter * tier once per title if the GPU can't keep up. */ let watchTimer: ReturnType | null = null; export function watchPerformance(active: boolean) { if (watchTimer) clearInterval(watchTimer); watchTimer = null; if (!active || !isNative) return; let last = -1; let warned = false; watchTimer = setInterval(async () => { const st = usePlayer.getState(); const mode = st.enhance.active; if (!st.open || st.paused || st.enhance.bypass || (mode !== 'quality' && mode !== 'balanced')) return void (last = -1); const dropped = (await mpv.get('frame-drop-count').catch(() => 0)) || 0; if (last >= 0 && dropped - last >= 10 && !warned) { warned = true; toast(`${UPSCALING[mode].label} is dropping frames on this video. Try a lighter upscaling tier.`); } last = dropped; }, 5000); } // Stage toggles and the tier apply straight away while something plays. useSettings.subscribe((prefs, prev) => { if (!isNative || !usePlayer.getState().open) return; if (prefs.upscaling !== prev.upscaling || prefs.enhanceDeband !== prev.enhanceDeband || prefs.enhanceSharpen !== prev.enhanceSharpen || prefs.enhanceDenoise !== prev.enhanceDenoise) requestEnhancement(); });