Use nlmeans and FSRCNNX 16 for the Quality upscaling tier

ArtCNN is tuned for animation and looked unnatural on live action, and
its first Direct3D compile took about a minute per model. Quality now
denoises with AN3223's nlmeans (general-purpose profile, any size) and
upscales with FSRCNNX 16-0-4-1, trained on general imagery, at any
enlargement. Both compile in seconds; 1080p in a maximized 1440p window
measured 3.2 ms per frame.

The first-compile notice now covers any shader set not yet compiled on
the machine but only shows after three seconds, and a set with nothing
to verify at the current size clears it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-10-30 19:51:47 -07:00
parent 506c69b36e
commit 5b6c063ff1
9 changed files with 3941 additions and 7897 deletions

View file

@ -10,7 +10,7 @@ import { win, isNative } from '../platform/native';
import { WindowControls } from '../app/TopBar';
import { navigate } from '../app/router';
import { mpv } from './engine';
import { UPSCALING_ORDER, hasFailed, readDiagnostics, requestEnhancement, toggleBypass, type Diagnostics } from './enhance';
import { PREPARING_NOTICE_MS, UPSCALING_ORDER, hasFailed, readDiagnostics, requestEnhancement, toggleBypass, type Diagnostics } from './enhance';
import { toast } from '../ui/overlays';
import type { Item } from '../plex/types';
@ -242,6 +242,17 @@ const SUB_BOXES: { value: SubtitleBox; label: string }[] = [
type SettingsView = 'main' | 'audio' | 'subtitles' | 'quality' | 'upscaling' | 'speed' | 'version' | 'chapters' | 'subSize' | 'subBox';
/** True once `flag` has stayed true for `ms` (a quick blip never shows). */
function useLasting(flag: boolean, ms: number) {
const [on, setOn] = useState(false);
useEffect(() => {
if (!flag) return setOn(false);
const t = setTimeout(() => setOn(true), ms);
return () => clearTimeout(t);
}, [flag, ms]);
return on;
}
/** Polls what mpv's renderer is doing while a panel shows it. */
function useDiagnostics(enabled: boolean) {
const [d, setD] = useState<Diagnostics | null>(null);
@ -322,7 +333,7 @@ function SettingsPanel({ onClose }: { onClose: () => void }) {
<div className="panel-subhead">Stages</div>
<Option selected={prefs.enhanceDeband} label="Debanding" detail="Smooths banding in gradients" onClick={() => prefs.set('enhanceDeband', !prefs.enhanceDeband)} />
<Option selected={prefs.enhanceSharpen} label="Sharpening" detail="AMD CAS · SDR only" onClick={() => prefs.set('enhanceSharpen', !prefs.enhanceSharpen)} />
<Option selected={prefs.enhanceDenoise} label="Denoising" detail="ArtCNN’s denoising model while upscaling · Quality tier, SDR only" onClick={() => prefs.set('enhanceDenoise', !prefs.enhanceDenoise)} />
<Option selected={prefs.enhanceDenoise} label="Denoising" detail="Non-local means, at any size · Quality tier, SDR only" onClick={() => prefs.set('enhanceDenoise', !prefs.enhanceDenoise)} />
<button className={`panel-compare${st.enhance.bypass ? ' on' : ''}`} onClick={() => toggleBypass()}>
{st.enhance.bypass ? 'Turn enhancement back on' : 'Compare with enhancement off'}
<kbd>B</kbd>
@ -661,6 +672,7 @@ export function PlayerOverlay() {
const { visible, poke } = useAutoHide(st.paused || st.ended, !!panel || scrubbing || st.countdown !== null);
const clickTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const seekStep = useSettings((s) => s.seekStep);
const preparing = useLasting(st.enhance.preparing, PREPARING_NOTICE_MS);
const showFlash = (icon: string) => setFlash({ icon, key: Date.now() });
@ -806,13 +818,13 @@ export function PlayerOverlay() {
{!st.fullscreen && <WindowControls />}
</div>
{(loading || st.buffering || st.seeking || st.enhance.preparing) && !st.error && (
{(loading || st.buffering || st.seeking || preparing) && !st.error && (
<div className="player-center">
<Spinner size={44} />
{st.enhance.preparing && (
{preparing && (
<p className="player-preparing">
Preparing {UPSCALING[st.enhance.active].label} upscaling
<small>First time on this computer only · about a minute</small>
<small>First time on this computer only</small>
</p>
)}
</div>

View file

@ -2,10 +2,11 @@
* Video enhancement: upscaling tiers built from mpv's scalers, debanding and
* the shaders bundled in src-tauri/shaders. Each shader's WHEN condition
* decides whether it runs for the current video and window size, so one list
* covers every case: the neural upscalers double luma when the video is
* enlarged (FSRCNNX above 1.3×, ArtCNN at any enlargement, so 1080p in a
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds subsampled colour,
* CAS sharpens at native size (CAS-prescale also before upscaling).
* covers every case: FSRCNNX doubles luma when the video is enlarged (the
* small model above 1.3×, the large one at any enlargement, so 1080p in a
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds
* subsampled colour, CAS sharpens at native size (CAS-prescale also before
* upscaling) and nlmeans denoises at any size.
*
* The tier the user chose (preferences) and the one actually running (player
* state) are kept apart: a tier whose shaders fail drops to the one below with
@ -28,7 +29,7 @@ let shaderDir: Promise<string> | null = null;
const NETWORK_MIN_SCALE: Partial<Record<Upscaling, number>> = { balanced: 1.3, quality: 1.0 };
const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!;
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'ArtCNN_C4F32.glsl' };
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'FSRCNNX_x2_16-0-4-1.glsl' };
interface Plan {
props: Record<string, unknown>;
@ -36,10 +37,11 @@ interface Plan {
}
/**
* What a tier runs. Sharpening (CAS) and denoising are tuned for SDR levels,
* so HDR video skips them; the upscalers and debanding stay. Denoising uses
* ArtCNN's own denoising model while upscaling: a separate denoiser (nlmeans)
* took over a minute to compile for Direct3D before the first frame.
* What a tier runs. Sharpening (CAS) and denoising (nlmeans) are tuned for
* SDR levels, so HDR video skips them; the upscalers and debanding stay.
* Quality denoises first, so the network enlarges the picture rather than
* the compression noise. Both FSRCNNX models are trained on general imagery,
* which suits live action.
*/
function plan(mode: Upscaling, hdr: boolean): Plan {
const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState();
@ -57,7 +59,7 @@ function plan(mode: Upscaling, hdr: boolean): Plan {
break;
case 'quality':
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 };
shaders = [enhanceDenoise && !hdr ? 'ArtCNN_C4F32_DN.glsl' : NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
shaders = [...(enhanceDenoise && !hdr ? ['nlmeans.glsl'] : []), NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
break;
default:
return { props: { ...scalerDefaults }, shaders: [] };
@ -119,27 +121,29 @@ export async function scaleFactor(): Promise<number> {
/**
* After a tier is applied: wait for frames, then look for shader errors in
* mpv's log, and, when the network should be running at this size, for its
* passes in the renderer. Either failing drops to the tier below.
* mpv's log, and for the passes that should be running at this size (the
* denoiser always, the network when enlarging). Either failing drops to the
* tier below.
*/
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string) {
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string, shaders: string[]) {
for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250);
await sleep(1500);
if (id !== seq || !usePlayer.getState().open) return;
const neural = NEURAL[mode];
const expectNetwork = !!neural && (await runsNetwork(mode));
const marker = mode === 'quality' ? /ArtCNN/ : /feature map|FSRCNN/i;
const expected: RegExp[] = [];
if (shaders.includes('nlmeans.glsl')) expected.push(/Non-local means/);
if (NEURAL[mode] && (await runsNetwork(mode))) expected.push(/feature map/);
let error = shaderErrors.find((e) => e.at >= appliedAt);
let missing = false;
if (!error && expectNetwork) {
if (!error && expected.length) {
// A first compile can hold the renderer for over a minute; wait for the
// network's passes to appear (or an error) rather than judging too early.
// expected passes to appear (or an error) rather than judging too early.
// Already compiled, they appear within a few frames.
missing = true;
const patience = usePlayer.getState().enhance.preparing ? 240 : 8;
for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) {
error = shaderErrors.find((e) => e.at >= appliedAt);
if (error || (await renderPasses()).some((p) => marker.test(p.desc))) {
const passes = await renderPasses();
if (error || expected.every((re) => passes.some((p) => re.test(p.desc)))) {
missing = false;
break;
}
@ -148,9 +152,12 @@ async function verify(id: number, mode: Upscaling, appliedAt: number, signature:
}
if (id !== seq) return;
if (!error && !missing) {
if (expectNetwork) markWarm(signature);
// Nothing compiled at this size (e.g. the network isn't needed) leaves the set unproven.
if (expected.length) markWarm(signature);
else clearPreparing();
return;
}
clearPreparing();
failed.add(mode);
const fallback = usable(mode);
console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running');
@ -162,14 +169,16 @@ async function verify(id: number, mode: Upscaling, appliedAt: number, signature:
let seq = 0;
let chain: Promise<unknown> = Promise.resolve();
/** How long preparation must take before the player mentions it. */
export const PREPARING_NOTICE_MS = 3000;
/**
* Shader sets that have compiled on this computer. mpv keeps compiled shaders
* on disk (gpu-shader-cache-dir), but the first ArtCNN compile for Direct3D
* takes over a minute, so the player says so the first time.
* on disk (gpu-shader-cache-dir); a first compile for Direct3D can take a
* while on some GPUs, so until a set is known the player can say so (after a
* short delay, so a quick compile shows nothing).
*/
const WARM_KEY = 'kaleido.warmShaders';
const SLOW = /ArtCNN/;
function warmSet(): Set<string> {
try {
return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]'));
@ -177,6 +186,9 @@ function warmSet(): Set<string> {
return new Set();
}
}
function clearPreparing() {
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
}
function markWarm(signature: string) {
const set = warmSet();
set.add(signature);
@ -227,10 +239,12 @@ async function apply(id: number, requested: Upscaling) {
// Keyed by mpv's version too: a new libmpv recompiles everything.
// WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile.
const signature = `${mpvVersion}|${shaders.join('|')}`;
const preparing = !bypass && shaders.some((n) => SLOW.test(n)) && !warmSet().has(signature) && (await runsNetwork(mode));
const preparing = !bypass && shaders.length > 0 && !warmSet().has(signature);
usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } });
if (preparing && usePlayer.getState().firstFrame)
toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for about a minute.`);
setTimeout(() => {
if (id === seq && usePlayer.getState().enhance.preparing) toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for a moment.`);
}, PREPARING_NOTICE_MS);
const settings = JSON.stringify({ props, shaders });
if (settings !== lastApplied) {
@ -246,7 +260,7 @@ async function apply(id: number, requested: Upscaling) {
}
// Errors count from when these shaders actually reached mpv, even if this
// re-apply changed nothing (mpv may have already reported a failure).
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature);
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature, shaders);
}
/**
@ -284,8 +298,7 @@ export interface Diagnostics {
}
const STAGES: [RegExp, string][] = [
[/ArtCNN C4F32 DN/, 'ArtCNN 2× with denoising'],
[/ArtCNN C4F32 \(/, 'ArtCNN 2×'],
[/Non-local means/, 'nlmeans denoising'],
[/feature map|FSRCNN/i, 'FSRCNNX 2×'],
[/KrigBilateral/, 'KrigBilateral colour'],
[/^debanding/, 'Debanding'],