fix: Improve 2-way audio detection (#2293)

- For Frigate cameras, you must be running at least [integration
v5.12.0](https://github.com/blakeblackshear/frigate-hass-integration/releases/tag/v5.12.0).
 - Closes: #2191 

Includes a significant refactor of cameras, and the introduction of a
`2-way-audio` capability that is dynamically fetched from `go2rtc`. One
gotcha is if you previously had a substream with 2-way audio, you may
need to modify

```yaml
 - camera_entity: camera.foo
    capabilities:
      disable_except:
        - substream
```

... to ...

```yaml
 - camera_entity: camera.foo
    capabilities:
      disable_except:
        - substream
        - 2-way-audio
```
This commit is contained in:
Dermot Duffy
2025-12-20 13:06:09 -08:00
committed by GitHub
parent 588ed6c98a
commit 1e821f09c6
47 changed files with 2240 additions and 958 deletions
+107
View File
@@ -22,3 +22,110 @@ export const mayHaveAudio = (video: HTMLVideoElement & AudioProperties): boolean
}
return true;
};
/**
* Determine if audio is available for a go2rtc stream.
* @param pc The RTCPeerConnection (for WebRTC streams).
* @param mseCodecs The negotiated MSE codecs string (for MSE streams).
* @param video The video element (fallback for browser-based detection).
* @returns True if audio is available.
*/
export const hasAudio = (
video: HTMLVideoElement & AudioProperties,
pc?: RTCPeerConnection | null,
mseCodecs?: string,
): boolean => {
// For WebRTC: Check if there's an audio receiver with an active track.
// We check that the track is not muted because muted means no media data
// is flowing (e.g., the source isn't producing audio). It is not related to
// the audio being muted by the user on the receiving end.
if (pc) {
const receivers = pc.getReceivers();
// Only trust receivers if they're populated (connection established)
if (receivers.length > 0) {
return receivers.some(
(receiver) => receiver.track?.kind === 'audio' && !receiver.track?.muted,
);
}
}
// For MSE: Check negotiated codecs for audio codecs
if (mseCodecs) {
return (
mseCodecs.includes('mp4a') ||
mseCodecs.includes('opus') ||
mseCodecs.includes('flac')
);
}
// Fallback to browser-based detection (unreliable in Chrome)
return mayHaveAudio(video);
};
/**
* Check if a WebRTC peer connection has an outbound audio channel (i.e. 2-way
* audio / microphone support).
* @param pc The RTCPeerConnection to check.
* @returns True if the connection has an audio transceiver configured to send.
*/
export const has2WayAudio = (pc: RTCPeerConnection | null): boolean => {
return !!pc
?.getTransceivers()
.some(
(tr) =>
tr.sender.track?.kind === 'audio' &&
(tr.direction === 'sendonly' || tr.direction === 'sendrecv'),
);
};
export type AudioTracksMuteStateCleanup = (() => void) | null;
/**
* Add listeners for mute/unmute events on all audio tracks in a WebRTC connection.
* The callback is fired when the aggregate mute state changes between:
* - All tracks unmuted (hasAudio = true)
* - All tracks muted (hasAudio = false)
* Mixed states (some muted, some unmuted) do not trigger the callback.
* @param pc The RTCPeerConnection to monitor.
* @param handler Callback fired with `true` when ALL tracks become unmuted,
* `false` when ALL tracks become muted.
* @returns A cleanup function to remove listeners, or null if no audio tracks.
*/
export const addAudioTracksMuteStateListener = (
pc: RTCPeerConnection | null,
handler: (hasAudio: boolean) => void,
): AudioTracksMuteStateCleanup => {
if (!pc) {
return null;
}
const audioTracks = pc
.getReceivers()
.map((r) => r.track)
.filter((t): t is MediaStreamTrack => t?.kind === 'audio');
if (audioTracks.length === 0) {
return null;
}
const hasAnyUnmuted = () => audioTracks.some((t) => !t.muted);
let lastHasAudio = hasAnyUnmuted();
const _handler = () => {
const nowHasAudio = hasAnyUnmuted();
if (nowHasAudio !== lastHasAudio) {
lastHasAudio = nowHasAudio;
handler(nowHasAudio);
}
};
audioTracks.forEach((track) => {
track.addEventListener('unmute', _handler);
track.addEventListener('mute', _handler);
});
return () =>
audioTracks.forEach((track) => {
track.removeEventListener('unmute', _handler);
track.removeEventListener('mute', _handler);
});
};
+36
View File
@@ -0,0 +1,36 @@
import { CameraProxyConfig } from '../camera-manager/types';
import { supports2WayAudio as gortcSupports2WayAudio } from '../camera-manager/utils/go2rtc/audio';
import { CameraConfig } from '../config/schema/cameras';
import { LiveProvider } from '../config/schema/cameras.js';
import { HomeAssistant } from '../ha/types';
import { Endpoint } from '../types';
export const getResolvedLiveProvider = (
config: CameraConfig | undefined,
): Exclude<LiveProvider, 'auto'> => {
if (config?.live_provider === 'auto') {
if (config.webrtc_card?.entity || config.webrtc_card?.url) {
return 'webrtc-card';
} else if (config.camera_entity) {
return 'ha';
} else if (config.frigate?.camera_name) {
return 'jsmpeg';
}
// Default for auto is 'image'
return 'image';
}
return config?.live_provider ?? 'image';
};
export const liveProviderSupports2WayAudio = async (
hass: HomeAssistant,
config: CameraConfig,
go2rtcMetadataEndpoint?: Endpoint | null,
proxyConfig?: CameraProxyConfig,
): Promise<boolean> => {
if (getResolvedLiveProvider(config) !== 'go2rtc') {
return false;
}
return gortcSupports2WayAudio(hass, go2rtcMetadataEndpoint, proxyConfig);
};