Quality runs ArtCNN C4F32 (its denoising model when Denoising is on), KrigBilateral, debanding and CAS. Its network runs at any enlargement, so 1080p in a maximized 1440p window is doubled and scaled down; Balanced keeps FSRCNNX's 1.3x threshold. The enhancement pipeline moves to player/enhance.ts. Mode changes are serialized (latest wins). Shader errors in mpv's log, or network passes missing from vo-passes, drop to the tier below with a notice, and the chosen and running tiers are shown separately. HDR is judged from the decoded video. Debanding, sharpening and denoising have their own toggles, B compares with enhancement off, and the panel shows the renderer's actual stages, GPU time and dropped frames. Compiled shaders persist in the app cache (gpu-shader-cache-dir): ArtCNN's first Direct3D compile takes about a minute, which the player now announces, and later launches start immediately. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
344 lines
16 KiB
TypeScript
344 lines
16 KiB
TypeScript
/**
|
||
* Video enhancement: upscaling tiers built from mpv's scalers, debanding and
|
||
* the shaders bundled in src-tauri/shaders. Each shader's WHEN condition
|
||
* decides whether it runs for the current video and window size, so one list
|
||
* covers every case: the neural upscalers double luma when the video is
|
||
* enlarged (FSRCNNX above 1.3×, ArtCNN at any enlargement, so 1080p in a
|
||
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds subsampled colour,
|
||
* CAS sharpens at native size (CAS-prescale also before upscaling).
|
||
*
|
||
* The tier the user chose (preferences) and the one actually running (player
|
||
* state) are kept apart: a tier whose shaders fail drops to the one below with
|
||
* a notice instead of silently showing plain scaling, and isn't retried.
|
||
*/
|
||
import { useSettings, UPSCALING, type Upscaling } from '../app/settings';
|
||
import { isNative, resourcePath } from '../platform/native';
|
||
import { toast } from '../ui/overlays';
|
||
import { mpv } from './engine';
|
||
import { usePlayer } from './store';
|
||
|
||
export const UPSCALING_ORDER: Upscaling[] = ['off', 'performance', 'balanced', 'quality'];
|
||
|
||
/** mpv's own scaler settings, read once so Off can put them back. */
|
||
const SCALER_PROPS = ['scale', 'cscale', 'dscale', 'sigmoid-upscaling', 'deband', 'deband-iterations', 'deband-grain'] as const;
|
||
let scalerDefaults: Partial<Record<(typeof SCALER_PROPS)[number], unknown>> | null = null;
|
||
let shaderDir: Promise<string> | null = null;
|
||
|
||
/** The enlargement above which each tier's network runs (its shaders' WHEN condition). */
|
||
const NETWORK_MIN_SCALE: Partial<Record<Upscaling, number>> = { balanced: 1.3, quality: 1.0 };
|
||
const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!;
|
||
|
||
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'ArtCNN_C4F32.glsl' };
|
||
|
||
interface Plan {
|
||
props: Record<string, unknown>;
|
||
shaders: string[];
|
||
}
|
||
|
||
/**
|
||
* What a tier runs. Sharpening (CAS) and denoising are tuned for SDR levels,
|
||
* so HDR video skips them; the upscalers and debanding stay. Denoising uses
|
||
* ArtCNN's own denoising model while upscaling: a separate denoiser (nlmeans)
|
||
* took over a minute to compile for Direct3D before the first frame.
|
||
*/
|
||
function plan(mode: Upscaling, hdr: boolean): Plan {
|
||
const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState();
|
||
const sharpen = enhanceSharpen && !hdr;
|
||
let props: Record<string, unknown>;
|
||
let shaders: string[] = [];
|
||
switch (mode) {
|
||
case 'performance':
|
||
props = { scale: 'spline36', cscale: 'spline36', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 24 };
|
||
if (sharpen) shaders = ['CAS-prescale.glsl'];
|
||
break;
|
||
case 'balanced':
|
||
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 16 };
|
||
shaders = [NEURAL.balanced!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
|
||
break;
|
||
case 'quality':
|
||
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 };
|
||
shaders = [enhanceDenoise && !hdr ? 'ArtCNN_C4F32_DN.glsl' : NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
|
||
break;
|
||
default:
|
||
return { props: { ...scalerDefaults }, shaders: [] };
|
||
}
|
||
if (!enhanceDeband) props.deband = false;
|
||
return { props, shaders };
|
||
}
|
||
|
||
/** HDR as decoded (a converted stream may already be SDR), else Plex's metadata. */
|
||
function isHdr() {
|
||
const st = usePlayer.getState();
|
||
if (st.videoGamma) return st.videoGamma === 'pq' || st.videoGamma === 'hlg';
|
||
return st.delivery?.mode !== 'transcode' && !!st.item?.media[st.mediaIndex]?.dynamicRange;
|
||
}
|
||
|
||
// —— Shader failures ————————————————————————————————————————————————
|
||
|
||
const shaderErrors: { at: number; text: string }[] = [];
|
||
|
||
/** Fed every mpv log line; keeps the ones that mean a user shader failed. */
|
||
export function noteMpvLog(level: string, prefix: string, text: string) {
|
||
if (level !== 'error' && level !== 'fatal') return;
|
||
// A missing file is reported by mpv's file/stream layers; a broken one by the renderer.
|
||
if (/\.glsl/i.test(text) || (/^vo/.test(prefix) && /shader|glsl|hook|compil|spirv|parse/i.test(text))) {
|
||
shaderErrors.push({ at: Date.now(), text: text.trim() });
|
||
if (shaderErrors.length > 20) shaderErrors.shift();
|
||
}
|
||
}
|
||
|
||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||
|
||
/** Descriptions of the render passes mpv ran for the last frame (or redraw). */
|
||
type Pass = { desc: string; avg: number; samples?: number[] };
|
||
export async function renderPasses(): Promise<{ desc: string; avg: number }[]> {
|
||
const passes = await mpv.get<{ fresh?: Pass[]; redraw?: Pass[] }>('vo-passes').catch(() => null);
|
||
const list = (passes?.fresh?.length ? passes.fresh : passes?.redraw) ?? [];
|
||
// The median of recent frames: the average includes one-off shader compiles.
|
||
return list.map((p) => {
|
||
const s = [...(p.samples ?? [])].sort((a, b) => a - b);
|
||
return { desc: p.desc, avg: s.length ? s[Math.floor(s.length / 2)] : p.avg };
|
||
});
|
||
}
|
||
|
||
/**
|
||
* How much the video is enlarged on screen. The full player fills the window,
|
||
* which is known before mpv has drawn anything; the miniplayer asks mpv.
|
||
*/
|
||
export async function scaleFactor(): Promise<number> {
|
||
const st = usePlayer.getState();
|
||
if (!st.video?.width) return 1;
|
||
if (!st.mini) {
|
||
const dpr = window.devicePixelRatio || 1;
|
||
return Math.min((window.innerWidth * dpr) / st.video.width, (window.innerHeight * dpr) / st.video.height);
|
||
}
|
||
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
|
||
if (!osd) return 1;
|
||
return Math.min((osd.w - osd.ml - osd.mr) / st.video.width, (osd.h - osd.mt - osd.mb) / st.video.height);
|
||
}
|
||
|
||
/**
|
||
* After a tier is applied: wait for frames, then look for shader errors in
|
||
* mpv's log, and, when the network should be running at this size, for its
|
||
* passes in the renderer. Either failing drops to the tier below.
|
||
*/
|
||
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string) {
|
||
for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250);
|
||
await sleep(1500);
|
||
if (id !== seq || !usePlayer.getState().open) return;
|
||
const neural = NEURAL[mode];
|
||
const expectNetwork = !!neural && (await runsNetwork(mode));
|
||
const marker = mode === 'quality' ? /ArtCNN/ : /feature map|FSRCNN/i;
|
||
let error = shaderErrors.find((e) => e.at >= appliedAt);
|
||
let missing = false;
|
||
if (!error && expectNetwork) {
|
||
// A first compile can hold the renderer for over a minute; wait for the
|
||
// network's passes to appear (or an error) rather than judging too early.
|
||
// Already compiled, they appear within a few frames.
|
||
missing = true;
|
||
const patience = usePlayer.getState().enhance.preparing ? 240 : 8;
|
||
for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) {
|
||
error = shaderErrors.find((e) => e.at >= appliedAt);
|
||
if (error || (await renderPasses()).some((p) => marker.test(p.desc))) {
|
||
missing = false;
|
||
break;
|
||
}
|
||
await sleep(1000);
|
||
}
|
||
}
|
||
if (id !== seq) return;
|
||
if (!error && !missing) {
|
||
if (expectNetwork) markWarm(signature);
|
||
return;
|
||
}
|
||
failed.add(mode);
|
||
const fallback = usable(mode);
|
||
console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running');
|
||
toast(`${UPSCALING[mode].label} upscaling couldn’t start. Using ${UPSCALING[fallback].label}.`, 'error');
|
||
requestEnhancement();
|
||
}
|
||
|
||
// —— Applying ——————————————————————————————————————————————————————
|
||
|
||
let seq = 0;
|
||
let chain: Promise<unknown> = Promise.resolve();
|
||
|
||
/**
|
||
* Shader sets that have compiled on this computer. mpv keeps compiled shaders
|
||
* on disk (gpu-shader-cache-dir), but the first ArtCNN compile for Direct3D
|
||
* takes over a minute, so the player says so the first time.
|
||
*/
|
||
const WARM_KEY = 'kaleido.warmShaders';
|
||
const SLOW = /ArtCNN/;
|
||
function warmSet(): Set<string> {
|
||
try {
|
||
return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]'));
|
||
} catch {
|
||
return new Set();
|
||
}
|
||
}
|
||
function markWarm(signature: string) {
|
||
const set = warmSet();
|
||
set.add(signature);
|
||
try {
|
||
localStorage.setItem(WARM_KEY, JSON.stringify([...set]));
|
||
} catch {
|
||
/* per-device convenience only */
|
||
}
|
||
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
|
||
}
|
||
/** Tiers whose shaders failed this session. */
|
||
const failed = new Set<Upscaling>();
|
||
let missingWarned = false;
|
||
|
||
/** The chosen tier, or the nearest one below it that hasn't failed. */
|
||
function usable(mode: Upscaling): Upscaling {
|
||
let i = UPSCALING_ORDER.indexOf(mode);
|
||
while (i > 0 && failed.has(UPSCALING_ORDER[i])) i--;
|
||
return UPSCALING_ORDER[i];
|
||
}
|
||
|
||
let mpvVersion = '';
|
||
/** The last settings sent to mpv (and when), so an identical re-apply leaves mpv alone. */
|
||
let lastApplied = '';
|
||
let lastAppliedAt = 0;
|
||
|
||
async function apply(id: number, requested: Upscaling) {
|
||
if (!isNative || id !== seq) return;
|
||
if (!scalerDefaults) {
|
||
scalerDefaults = {};
|
||
for (const p of SCALER_PROPS) scalerDefaults[p] = await mpv.get(p).catch(() => undefined);
|
||
mpvVersion = (await mpv.get<string>('mpv-version').catch(() => '')) ?? '';
|
||
}
|
||
const bypass = usePlayer.getState().enhance.bypass;
|
||
let mode = usable(requested);
|
||
let { props, shaders } = plan(bypass ? 'off' : mode, isHdr());
|
||
const dir = await (shaderDir ??= resourcePath('shaders').catch(() => ''));
|
||
if (shaders.length && !dir) {
|
||
// No shaders at all: keep the scaler and debanding settings only.
|
||
if (!missingWarned) toast('The upscaling shaders are missing from this installation.', 'error');
|
||
missingWarned = true;
|
||
shaders = [];
|
||
if (mode !== 'performance') mode = 'performance';
|
||
({ props } = plan(mode, isHdr()));
|
||
}
|
||
// Decided before touching mpv: while a shader compiles, mpv holds option
|
||
// changes until it finishes, so a notice set afterwards would come too late.
|
||
// Keyed by mpv's version too: a new libmpv recompiles everything.
|
||
// WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile.
|
||
const signature = `${mpvVersion}|${shaders.join('|')}`;
|
||
const preparing = !bypass && shaders.some((n) => SLOW.test(n)) && !warmSet().has(signature) && (await runsNetwork(mode));
|
||
usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } });
|
||
if (preparing && usePlayer.getState().firstFrame)
|
||
toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for about a minute.`);
|
||
|
||
const settings = JSON.stringify({ props, shaders });
|
||
if (settings !== lastApplied) {
|
||
lastApplied = settings;
|
||
lastAppliedAt = Date.now();
|
||
for (const [k, v] of Object.entries(props)) if (v !== undefined) await mpv.set(k, v).catch(() => {});
|
||
if (shaders.length) {
|
||
// mpv takes a path list: ';'-separated on Windows, ':' elsewhere.
|
||
const windows = dir.includes('\\');
|
||
const list = shaders.map((n) => `${dir}${windows ? '\\' : '/'}${n}`).join(windows ? ';' : ':');
|
||
await mpv.set('glsl-shaders', list).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) }));
|
||
} else await mpv.command('change-list', 'glsl-shaders', 'clr', '').catch(() => {});
|
||
}
|
||
// Errors count from when these shaders actually reached mpv, even if this
|
||
// re-apply changed nothing (mpv may have already reported a failure).
|
||
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature);
|
||
}
|
||
|
||
/**
|
||
* Applies the chosen tier (and stage toggles) to the playing video. Requests
|
||
* are serialized and only the latest runs, so quick U presses can't interleave.
|
||
*/
|
||
export function requestEnhancement() {
|
||
const id = ++seq;
|
||
const run = chain.then(() => apply(id, useSettings.getState().upscaling));
|
||
chain = run.catch(() => {});
|
||
return run;
|
||
}
|
||
|
||
/** Whether a tier failed this session (shown in the player's choices). */
|
||
export const hasFailed = (mode: Upscaling) => failed.has(mode);
|
||
|
||
/** Before/after comparison: enhancement off without changing the preference. */
|
||
export function toggleBypass() {
|
||
const st = usePlayer.getState();
|
||
const bypass = !st.enhance.bypass;
|
||
usePlayer.setState({ enhance: { ...st.enhance, bypass } });
|
||
toast(bypass ? 'Enhancement off for comparison · press B to turn it back on' : `Enhancement on · ${UPSCALING[useSettings.getState().upscaling].label}`);
|
||
return requestEnhancement();
|
||
}
|
||
|
||
// —— Diagnostics ———————————————————————————————————————————————————
|
||
|
||
export interface Diagnostics {
|
||
stages: string[];
|
||
gpuMs: number;
|
||
budgetMs: number;
|
||
dropped: number;
|
||
input: string;
|
||
output: string;
|
||
}
|
||
|
||
const STAGES: [RegExp, string][] = [
|
||
[/ArtCNN C4F32 DN/, 'ArtCNN 2× with denoising'],
|
||
[/ArtCNN C4F32 \(/, 'ArtCNN 2×'],
|
||
[/feature map|FSRCNN/i, 'FSRCNNX 2×'],
|
||
[/KrigBilateral/, 'KrigBilateral colour'],
|
||
[/^debanding/, 'Debanding'],
|
||
[/FidelityFX/, 'CAS sharpening'],
|
||
];
|
||
|
||
/** What the renderer actually did for the last frame, read from mpv. */
|
||
export async function readDiagnostics(): Promise<Diagnostics> {
|
||
const passes = await renderPasses();
|
||
const stages = STAGES.filter(([re]) => passes.some((p) => re.test(p.desc))).map(([, name]) => name);
|
||
const scaler = passes.map((p) => /(up|down)scaling \(([\w-]+)\)/.exec(p.desc)).find(Boolean);
|
||
if (scaler) stages.push(`${scaler[2]} ${scaler[1]}scaling`);
|
||
const fps = (await mpv.get<number>('estimated-vf-fps').catch(() => 0)) || (await mpv.get<number>('container-fps').catch(() => 0)) || 24;
|
||
const dropped = ((await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0) + ((await mpv.get<number>('decoder-frame-drop-count').catch(() => 0)) || 0);
|
||
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
|
||
const v = usePlayer.getState().video;
|
||
return {
|
||
stages,
|
||
// Pass timings are in nanoseconds.
|
||
gpuMs: passes.reduce((a, p) => a + (p.avg || 0), 0) / 1e6,
|
||
budgetMs: 1000 / fps,
|
||
dropped,
|
||
input: v?.width ? `${v.width}×${v.height}` : '',
|
||
output: osd ? `${osd.w - osd.ml - osd.mr}×${osd.h - osd.mt - osd.mb}` : '',
|
||
};
|
||
}
|
||
|
||
/**
|
||
* While a neural tier plays, watch for dropped frames and suggest a lighter
|
||
* tier once per title if the GPU can't keep up.
|
||
*/
|
||
let watchTimer: ReturnType<typeof setInterval> | null = null;
|
||
export function watchPerformance(active: boolean) {
|
||
if (watchTimer) clearInterval(watchTimer);
|
||
watchTimer = null;
|
||
if (!active || !isNative) return;
|
||
let last = -1;
|
||
let warned = false;
|
||
watchTimer = setInterval(async () => {
|
||
const st = usePlayer.getState();
|
||
const mode = st.enhance.active;
|
||
if (!st.open || st.paused || st.enhance.bypass || (mode !== 'quality' && mode !== 'balanced')) return void (last = -1);
|
||
const dropped = (await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0;
|
||
if (last >= 0 && dropped - last >= 10 && !warned) {
|
||
warned = true;
|
||
toast(`${UPSCALING[mode].label} is dropping frames on this video. Try a lighter upscaling tier.`);
|
||
}
|
||
last = dropped;
|
||
}, 5000);
|
||
}
|
||
|
||
// Stage toggles and the tier apply straight away while something plays.
|
||
useSettings.subscribe((prefs, prev) => {
|
||
if (!isNative || !usePlayer.getState().open) return;
|
||
if (prefs.upscaling !== prev.upscaling || prefs.enhanceDeband !== prev.enhanceDeband || prefs.enhanceSharpen !== prev.enhanceSharpen || prefs.enhanceDenoise !== prev.enhanceDenoise) requestEnhancement();
|
||
});
|