add web audio, replaygain, visualizer (#1289)

* add web audio, replaygain, visualizer

* remove volume multiplication in gain

---------

Co-authored-by: Jeff <42182408+jeffvli@users.noreply.github.com>

src/main/features/core/lyrics/netease.ts

@@ -20,24 +20,6 @@ export interface Result {
     songs: Song[];
 }
 
-export interface Song {
-    album: Album;
-    alias: string[];
-    artists: Artist[];
-    copyrightId: number;
-    duration: number;
-    fee: number;
-    ftype: number;
-    id: number;
-    mark: number;
-    mvid: number;
-    name: string;
-    rtype: number;
-    rUrl: null;
-    status: number;
-    transNames?: string[];
-}
-
 interface Album {
     artist: Artist;
     copyrightId: number;
@@ -69,6 +51,24 @@ interface NetEaseResponse {
     result: Result;
 }
 
+interface Song {
+    album: Album;
+    alias: string[];
+    artists: Artist[];
+    copyrightId: number;
+    duration: number;
+    fee: number;
+    ftype: number;
+    id: number;
+    mark: number;
+    mvid: number;
+    name: string;
+    rtype: number;
+    rUrl: null;
+    status: number;
+    transNames?: string[];
+}
+
 export async function getLyricsBySongId(songId: string): Promise<null | string> {
     let result: AxiosResponse<any, any>;
     try {

src/renderer/features/player/audio-player/engine/web-player-engine.tsx

@@ -27,6 +27,8 @@ interface WebPlayerEngineProps {
     onEndedPlayer2: () => void;
     onProgressPlayer1: (e: PlayerOnProgressProps) => void;
     onProgressPlayer2: (e: PlayerOnProgressProps) => void;
+    onStartedPlayer1: (player: ReactPlayer) => void;
+    onStartedPlayer2: (player: ReactPlayer) => void;
     playerNum: number;
     playerRef: RefObject<null | WebPlayerEngineHandle>;
     playerStatus: PlayerStatus;
@@ -52,6 +54,8 @@ export const WebPlayerEngine = (props: WebPlayerEngineProps) => {
         onEndedPlayer2,
         onProgressPlayer1,
         onProgressPlayer2,
+        onStartedPlayer1,
+        onStartedPlayer2,
         playerNum,
         playerRef,
         playerStatus,
@@ -158,6 +162,7 @@ export const WebPlayerEngine = (props: WebPlayerEngineProps) => {
                 onEnded={src1 ? () => onEndedPlayer1() : undefined}
                 onError={handleOnError(player1Ref, () => onEndedPlayer1())}
                 onProgress={onProgressPlayer1}
+                onReady={onStartedPlayer1}
                 playbackRate={speed || 1}
                 playing={playerNum === 1 && playerStatus === PlayerStatus.PLAYING}
                 progressInterval={isTransitioning ? 10 : 250}
@@ -177,6 +182,7 @@ export const WebPlayerEngine = (props: WebPlayerEngineProps) => {
                 onEnded={src2 ? () => onEndedPlayer2() : undefined}
                 onError={handleOnError(player2Ref, () => onEndedPlayer2())}
                 onProgress={onProgressPlayer2}
+                onReady={onStartedPlayer2}
                 playbackRate={speed || 1}
                 playing={playerNum === 2 && playerStatus === PlayerStatus.PLAYING}
                 progressInterval={isTransitioning ? 10 : 250}

src/renderer/features/player/audio-player/web-player.tsx

@@ -10,7 +10,9 @@ import {
 import { usePlayerEvents } from '/@/renderer/features/player/audio-player/hooks/use-player-events';
 import { useSongUrl } from '/@/renderer/features/player/audio-player/hooks/use-stream-url';
 import { PlayerOnProgressProps } from '/@/renderer/features/player/audio-player/types';
+import { useWebAudio } from '/@/renderer/features/player/hooks/use-webaudio';
 import {
+    useMpvSettings,
     usePlaybackSettings,
     usePlayerActions,
     usePlayerData,
@@ -18,6 +20,7 @@ import {
     usePlayerProperties,
     usePlayerVolume,
 } from '/@/renderer/store';
+import { QueueSong } from '/@/shared/types/domain-types';
 import { CrossfadeStyle, PlayerStatus, PlayerStyle } from '/@/shared/types/types';
 
 const PLAY_PAUSE_FADE_DURATION = 300;
@@ -27,6 +30,9 @@ export function WebPlayer() {
     const playerRef = useRef<null | WebPlayerEngineHandle>(null);
     const { num, player1, player2, status } = usePlayerData();
     const { mediaAutoNext, setTimestamp } = usePlayerActions();
+    const playback = useMpvSettings();
+    const { webAudio } = useWebAudio();
+
     const { crossfadeDuration, crossfadeStyle, speed, transitionType } = usePlayerProperties();
     const isMuted = usePlayerMuted();
     const volume = usePlayerVolume();
@@ -35,6 +41,9 @@ export function WebPlayer() {
     const [localPlayerStatus, setLocalPlayerStatus] = useState<PlayerStatus>(status);
     const [isTransitioning, setIsTransitioning] = useState<boolean | string>(false);
 
+    const [player1Source, setPlayer1Source] = useState<MediaElementAudioSourceNode | null>(null);
+    const [player2Source, setPlayer2Source] = useState<MediaElementAudioSourceNode | null>(null);
+
     const fadeAndSetStatus = useCallback(
         async (startVolume: number, endVolume: number, duration: number, status: PlayerStatus) => {
             if (isTransitioning) {
@@ -261,9 +270,134 @@ export function WebPlayer() {
         return () => clearInterval(interval);
     }, [localPlayerStatus, num, setTimestamp, transitionType]);
 
+    const calculateReplayGain = useCallback(
+        (song: QueueSong): number => {
+            if (playback.replayGainMode === 'no') {
+                return 1;
+            }
+
+            let gain: number | undefined;
+            let peak: number | undefined;
+
+            if (playback.replayGainMode === 'track') {
+                gain = song.gain?.track ?? song.gain?.album;
+                peak = song.peak?.track ?? song.peak?.album;
+            } else {
+                gain = song.gain?.album ?? song.gain?.track;
+                peak = song.peak?.album ?? song.peak?.track;
+            }
+
+            if (gain === undefined) {
+                gain = playback.replayGainFallbackDB;
+
+                if (!gain) {
+                    return 1;
+                }
+            }
+
+            if (peak === undefined) {
+                peak = 1;
+            }
+
+            const preAmp = playback.replayGainPreampDB ?? 0;
+
+            // https://wiki.hydrogenaud.io/index.php?title=ReplayGain_1.0_specification&section=19
+            // Normalized to max gain
+            let expectedGain = 10 ** ((gain + preAmp) / 20);
+
+            // Nothing in the system should allow this. But, in the case that preAmp is a
+            // bad value (not a number, for example), a NaN gain will cause the entire system to panic
+            if (isNaN(expectedGain)) {
+                expectedGain = 1;
+            }
+
+            if (playback.replayGainClip) {
+                return Math.min(expectedGain, 1 / peak);
+            }
+            return expectedGain;
+        },
+        [
+            playback.replayGainClip,
+            playback.replayGainFallbackDB,
+            playback.replayGainMode,
+            playback.replayGainPreampDB,
+        ],
+    );
+
+    useEffect(() => {
+        if (!webAudio) return;
+
+        if (player1 && player1Source && num === 1) {
+            const newGain = calculateReplayGain(player1);
+
+            // This error SHOULD never happen, as calculateReplayGain is expected to
+            // always return a real value. However, to prevent app crash, check this just in case
+            try {
+                webAudio.gains[0].gain.setValueAtTime(Math.max(0, newGain), 0);
+            } catch (error) {
+                console.error('Error setting gain', error);
+            }
+        }
+    }, [calculateReplayGain, num, player1, player1Source, volume, webAudio]);
+
+    useEffect(() => {
+        if (!webAudio) return;
+
+        if (player2 && player2Source && num === 2) {
+            const newGain = calculateReplayGain(player2);
+            try {
+                webAudio.gains[1].gain.setValueAtTime(Math.max(0, newGain), 0);
+            } catch (error) {
+                console.error('Error setting gain', error);
+            }
+        }
+    }, [calculateReplayGain, num, player1, player2Source, player2, volume, webAudio]);
+
     const player1Url = useSongUrl(player1, num === 1, transcode);
     const player2Url = useSongUrl(player2, num === 2, transcode);
 
+    const handlePlayer1Start = useCallback(
+        async (player: ReactPlayer) => {
+            if (!webAudio || player1Source) return;
+            if (player1Url) {
+                // This should fire once, only if the source is real (meaning we
+                // saw the dummy source) and the context is not ready
+                if (webAudio.context.state !== 'running') {
+                    await webAudio.context.resume();
+                }
+            }
+
+            const internal = player.getInternalPlayer() as HTMLMediaElement | undefined;
+            if (internal) {
+                const { context, gains } = webAudio;
+                const source = context.createMediaElementSource(internal);
+                source.connect(gains[0]);
+                setPlayer1Source(source);
+            }
+        },
+        [player1Source, player1Url, webAudio],
+    );
+
+    const handlePlayer2Start = useCallback(
+        async (player: ReactPlayer) => {
+            if (!webAudio || player2Source) return;
+            if (player2Url) {
+                if (webAudio.context.state !== 'running') {
+                    await webAudio.context.resume();
+                }
+            }
+
+            const internal = player.getInternalPlayer() as HTMLMediaElement | undefined;
+            if (internal) {
+                const { context, gains } = webAudio;
+                const source = context.createMediaElementSource(internal);
+                source.connect(gains[1]);
+                setPlayer2Source(source);
+            }
+        },
+        [player2Source, player2Url, webAudio],
+    );
+
     return (
         <WebPlayerEngine
             isMuted={isMuted}
@@ -272,6 +406,8 @@ export function WebPlayer() {
             onEndedPlayer2={handleOnEndedPlayer2}
             onProgressPlayer1={onProgressPlayer1}
             onProgressPlayer2={onProgressPlayer2}
+            onStartedPlayer1={handlePlayer1Start}
+            onStartedPlayer2={handlePlayer2Start}
             playerNum={num}
             playerRef={playerRef}
             playerStatus={localPlayerStatus}

src/renderer/features/player/components/audio-players.tsx

@@ -1,3 +1,4 @@
+import isElectron from 'is-electron';
 import { useEffect } from 'react';
 
 import { eventEmitter } from '/@/renderer/events/event-emitter';
@@ -11,18 +12,30 @@ import { useMPRIS } from '/@/renderer/features/player/hooks/use-mpris';
 import { usePlaybackHotkeys } from '/@/renderer/features/player/hooks/use-playback-hotkeys';
 import { usePowerSaveBlocker } from '/@/renderer/features/player/hooks/use-power-save-blocker';
 import { useScrobble } from '/@/renderer/features/player/hooks/use-scrobble';
+import { useWebAudio } from '/@/renderer/features/player/hooks/use-webaudio';
 import {
     updateQueueFavorites,
     updateQueueRatings,
     useCurrentServerId,
+    usePlaybackSettings,
     usePlaybackType,
+    useSettingsStoreActions,
 } from '/@/renderer/store';
+import { toast } from '/@/shared/components/toast/toast';
 import { LibraryItem } from '/@/shared/types/domain-types';
 import { PlayerType } from '/@/shared/types/types';
 
 export const AudioPlayers = () => {
     const playbackType = usePlaybackType();
     const serverId = useCurrentServerId();
+    const { resetSampleRate } = useSettingsStoreActions();
+
+    const {
+        audioDeviceId,
+        mpvProperties: { audioSampleRateHz },
+        webAudio,
+    } = usePlaybackSettings();
+    const { setWebAudio, webAudio: audioContext } = useWebAudio();
 
     useScrobble();
     usePowerSaveBlocker();
@@ -32,6 +45,54 @@ export const AudioPlayers = () => {
     useMediaSession();
     usePlaybackHotkeys();
 
+    useEffect(() => {
+        if (webAudio && 'AudioContext' in window) {
+            let context: AudioContext;
+
+            try {
+                context = new AudioContext({
+                    latencyHint: 'playback',
+                    sampleRate: audioSampleRateHz || undefined,
+                });
+            } catch (error) {
+                // In practice, this should never be hit because the UI should validate
+                // the range. However, the actual supported range is not guaranteed
+                toast.error({ message: (error as Error).message });
+                context = new AudioContext({ latencyHint: 'playback' });
+                resetSampleRate();
+            }
+
+            const gains = [context.createGain(), context.createGain()];
+            for (const gain of gains) {
+                gain.connect(context.destination);
+            }
+
+            setWebAudio!({ context, gains });
+        }
+
+        // Intentionally ignore the sample rate dependency, as it makes things really messy
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, []);
+
+    useEffect(() => {
+        // Not standard, just used in chromium-based browsers. See
+        // https://developer.chrome.com/blog/audiocontext-setsinkid/.
+        // If the isElectron() check is every removed, fix this.
+        if (isElectron() && audioContext && 'setSinkId' in audioContext.context && audioDeviceId) {
+            const setSink = async () => {
+                try {
+                    if (audioContext.context.state !== 'closed') {
+                        await (audioContext.context as any).setSinkId(audioDeviceId);
+                    }
+                } catch (error) {
+                    toast.error({ message: `Error setting sink: ${(error as Error).message}` });
+                }
+            };
+
+            setSink();
+        }
+    }, [audioContext, audioDeviceId]);
+
     // Listen to favorite and rating events to update queue songs
     useEffect(() => {
         const handleFavorite = (payload: UserFavoriteEventPayload) => {

src/renderer/features/player/components/visualizer.tsx

@@ -13,8 +13,8 @@ export const Visualizer = () => {
     const [motion, setMotion] = useState<AudioMotionAnalyzer>();
 
     useEffect(() => {
-        const { context, gain } = webAudio || {};
+        const { context, gains } = webAudio || {};
-        if (gain && context && canvasRef.current && !motion) {
+        if (gains && context && canvasRef.current && !motion) {
             const audioMotion = new AudioMotionAnalyzer(canvasRef.current, {
                 ansiBands: true,
                 audioCtx: context,
@@ -27,7 +27,7 @@ export const Visualizer = () => {
                 smoothing: 0.8,
             });
             setMotion(audioMotion);
-            audioMotion.connectInput(gain);
+            for (const gain of gains) audioMotion.connectInput(gain);
         }
 
         return () => {};

src/renderer/features/settings/components/playback/mpv-settings.tsx

@@ -360,7 +360,7 @@ export const MpvSettings = () => {
             control: (
                 <NumberInput
                     defaultValue={settings.mpvProperties.replayGainPreampDB}
-                    onChange={(e) => handleSetMpvProperty('replayGainPreampDB', e)}
+                    onChange={(e) => handleSetMpvProperty('replayGainPreampDB', Number(e) || 0)}
                     width={75}
                 />
             ),

src/shared/components/number-input/number-input.tsx

@@ -16,7 +16,9 @@ export const NumberInput = forwardRef<HTMLInputElement, NumberInputProps>(
         {
             children,
             classNames,
+            defaultValue,
             maxWidth,
+            onChange,
             size = 'sm',
             style,
             variant = 'default',
@@ -38,6 +40,11 @@ export const NumberInput = forwardRef<HTMLInputElement, NumberInputProps>(
                     ...classNames,
                 }}
                 hideControls
+                onChange={
+                    onChange
+                        ? (e) => onChange(typeof e === 'number' ? e : defaultValue || e)
+                        : undefined
+                }
                 ref={ref}
                 size={size}
                 style={{ maxWidth, width, ...style }}

src/shared/types/types.ts

@@ -271,5 +271,5 @@ export interface UniqueId {
 
 export type WebAudio = {
     context: AudioContext;
-    gain: GainNode;
+    gains: GainNode[];
 };