fix: Hold the 2-way audio backchannel open for the shortest possible time (#2697)

The card claimed a camera's ONVIF audio backchannel in two places that
had
nothing to do with a call: the capability probe at camera init
(`&microphone`),
and a pre-armed `sendonly` audio transceiver on every live WebRTC offer.
Merely
looking at a dashboard occupied the camera's speaker line. Outbound
audio now
travels on its own audio-only WebRTC connection, opened when a call is
answered
and closed when it ends.

- The backchannel is claimed only for the duration of a call. Idle
viewing
  claims nothing.
- Two-way audio now works in `mse`, `mp4` and `mjpeg` modes (note: the
outbound
  audio still traverses WebRTC).
- No renegotiation and no video blink at call start or end.
- A call that cannot carry audio now reports it and ends, instead of
showing a
  live microphone that goes nowhere.
- `live.microphone.always_connected` is now purely about the browser
microphone
  permission prompt.
- Call setup measured at 66ms (LAN) and ~260ms (cellular) for ICE and
DTLS, plus
  ~300ms for `go2rtc` to open an RTSP backchannel.

Verified against a live Frigate + `go2rtc` instance, and by unit tests
at 100%
coverage.

 - Closes #2691
 - Closes #2039
 - Closes #2178

Ref #2299 -- the probe no longer opens a backchannel, but it still runs
per
camera on every load and reconnect, and still dials the camera on the
direct-`go2rtc` path. Caching remains to be done.

Ref AlexxIT/go2rtc#1860 -- once a call has opened a backchannel,
`go2rtc` keeps
that media set up on the camera's RTSP session for the life of the
producer.

Diagnoses #2678
This commit is contained in:
Dermot Duffy
2026-08-21 20:13:58 -07:00
committed by GitHub
parent 2d7c86c93c
commit 11cc543406
65 changed files with 2251 additions and 1045 deletions
+129
View File
@@ -0,0 +1,129 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { mock } from 'vitest-mock-extended';
import { supports2WayAudio } from '../../src/go2rtc/audio';
import { homeAssistantSignAndFetch } from '../../src/ha/fetch';
import type { HomeAssistant } from '../../src/ha/types';
import { createProxiedEndpointIfNecessary } from '../../src/ha/web-proxy';
import type { Endpoint } from '../../src/types';
vi.mock('../../src/ha/fetch');
vi.mock('../../src/ha/web-proxy');
describe('supports2WayAudio', () => {
const hass = mock<HomeAssistant>();
const endpoint: Endpoint = { endpoint: 'http://go2rtc', sign: true };
beforeEach(() => {
vi.clearAllMocks();
});
it('should return false if no endpoint provided', async () => {
expect(await supports2WayAudio(hass, 2, null)).toBe(false);
});
it('should return false if fetch fails', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockRejectedValue(new Error('fetch error'));
const spy = vi.spyOn(console, 'warn');
const result = await supports2WayAudio(hass, 2, endpoint);
expect(result).toBe(false);
expect(spy).toHaveBeenCalledWith('fetch error');
spy.mockRestore();
});
it('should return false if stream info has no producers', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockResolvedValue({ producers: undefined });
expect(await supports2WayAudio(hass, 2, endpoint)).toBe(false);
});
it('should use default metadata fetch timeout', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockResolvedValue({ producers: [] });
await supports2WayAudio(hass, 2, endpoint);
expect(homeAssistantSignAndFetch).toHaveBeenCalledWith(
hass,
endpoint,
expect.anything(),
{ timeoutSeconds: 2 },
);
});
it('should use custom metadata fetch timeout when provided', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockResolvedValue({ producers: [] });
await supports2WayAudio(hass, 15, endpoint);
expect(homeAssistantSignAndFetch).toHaveBeenCalledWith(
hass,
endpoint,
expect.anything(),
{ timeoutSeconds: 15 },
);
});
it('should return false if no producer supports audio', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockResolvedValue({
producers: [
{
medias: ['video,sendonly,h264'],
},
],
});
expect(await supports2WayAudio(hass, 2, endpoint)).toBe(false);
});
it('should return true if producer supports audio and sendonly', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockResolvedValue({
producers: [
{
medias: ['audio,sendonly,opus'],
},
],
});
expect(await supports2WayAudio(hass, 2, endpoint)).toBe(true);
});
it('should return true if producer supports audio and sendrecv', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockResolvedValue({
producers: [
{
medias: ['audio,sendrecv,pcmu'],
},
],
});
expect(await supports2WayAudio(hass, 2, endpoint)).toBe(true);
});
it('should handle missing medias in producer', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(endpoint);
vi.mocked(homeAssistantSignAndFetch).mockResolvedValue({
producers: [
{
medias: undefined,
},
],
});
expect(await supports2WayAudio(hass, 2, endpoint)).toBe(false);
});
it('should return false if proxied endpoint is null', async () => {
vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(null);
expect(await supports2WayAudio(hass, 2, endpoint)).toBe(false);
});
});
+116
View File
@@ -0,0 +1,116 @@
import { describe, expect, it } from 'vitest';
import {
getGo2RTCMetadataEndpoint,
getGo2RTCStreamEndpoint,
} from '../../src/go2rtc/endpoint.js';
import { createCameraConfig } from '../config/test-utils';
describe('getGo2RTCStreamEndpoint', () => {
it('with local configuration', () => {
expect(
getGo2RTCStreamEndpoint(
createCameraConfig({
go2rtc: {
stream: 'stream',
url: '/local/path',
},
}),
),
).toEqual({
endpoint: '/local/path/api/ws?src=stream',
sign: true,
});
});
it('with remote configuration', () => {
expect(
getGo2RTCStreamEndpoint(
createCameraConfig({
go2rtc: {
stream: 'stream',
url: 'https://my-custom-go2rtc',
},
}),
),
).toEqual({
endpoint: 'https://my-custom-go2rtc/api/ws?src=stream',
sign: false,
});
});
it('should encode the stream name', () => {
expect(
getGo2RTCStreamEndpoint(
createCameraConfig({
go2rtc: {
stream: 'front door&microphone',
url: '/local/path',
},
}),
),
).toEqual({
endpoint: '/local/path/api/ws?src=front%20door%26microphone',
sign: true,
});
});
it('without configuration', () => {
expect(getGo2RTCStreamEndpoint(createCameraConfig())).toBeNull();
});
});
describe('getGo2RTCMetadataEndpoint', () => {
it('with local configuration', () => {
expect(
getGo2RTCMetadataEndpoint(
createCameraConfig({
go2rtc: {
stream: 'stream',
url: '/local/path',
},
}),
),
).toEqual({
endpoint: '/local/path/api/streams?src=stream&video=all&audio=all',
sign: true,
});
});
it('with remote configuration', () => {
expect(
getGo2RTCMetadataEndpoint(
createCameraConfig({
go2rtc: {
stream: 'stream',
url: 'https://my-custom-go2rtc',
},
}),
),
).toEqual({
endpoint: 'https://my-custom-go2rtc/api/streams?src=stream&video=all&audio=all',
sign: false,
});
});
it('should encode the stream name so it cannot add probe parameters', () => {
expect(
getGo2RTCMetadataEndpoint(
createCameraConfig({
go2rtc: {
stream: 'front door&microphone',
url: '/local/path',
},
}),
),
).toEqual({
endpoint:
'/local/path/api/streams?src=front%20door%26microphone&video=all&audio=all',
sign: true,
});
});
it('without configuration', () => {
expect(getGo2RTCMetadataEndpoint(createCameraConfig())).toBeNull();
});
});
+29
View File
@@ -0,0 +1,29 @@
import { describe, expect, it } from 'vitest';
import { isServerErrorForMode } from '../../src/go2rtc/messages';
describe('isServerErrorForMode', () => {
it('should match an error for the mode', () => {
expect(isServerErrorForMode({ type: 'error', value: 'mse: not found' }, 'mse')).toBe(
true,
);
});
it('should not match an error for another mode', () => {
expect(isServerErrorForMode({ type: 'error', value: 'webrtc: failed' }, 'mse')).toBe(
false,
);
});
it('should not match a non-error message', () => {
expect(isServerErrorForMode({ type: 'mse', value: 'codecs' }, 'mse')).toBe(false);
});
it('should not match an error with a non-string value', () => {
expect(isServerErrorForMode({ type: 'error', value: 42 }, 'mse')).toBe(false);
});
it('should not match an error with no value', () => {
expect(isServerErrorForMode({ type: 'error' }, 'mse')).toBe(false);
});
});
+29
View File
@@ -0,0 +1,29 @@
import { describe, expect, it, vi } from 'vitest';
import {
createBrowserPeerConnection,
GO2RTC_PEER_CONNECTION_CONFIG,
} from '../../src/go2rtc/peer-connection';
describe('peer-connection', () => {
it('should configure two STUN servers with max-bundle', () => {
expect(GO2RTC_PEER_CONNECTION_CONFIG).toEqual({
bundlePolicy: 'max-bundle',
iceServers: [
{
urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'],
},
],
});
});
it('should construct a real peer connection', () => {
const RTCPeerConnectionMock = vi.fn();
vi.stubGlobal('RTCPeerConnection', RTCPeerConnectionMock);
createBrowserPeerConnection(GO2RTC_PEER_CONNECTION_CONFIG);
expect(RTCPeerConnectionMock).toHaveBeenCalledWith(GO2RTC_PEER_CONNECTION_CONFIG);
vi.unstubAllGlobals();
});
});
+254
View File
@@ -0,0 +1,254 @@
import { describe, expect, it, vi } from 'vitest';
import { SignalingChannel } from '../../src/go2rtc/signaling';
import { FakeWebSocket } from './test-utils';
describe('SignalingChannel', () => {
const setup = (options?: {
openCallback?: () => void;
disconnectCallback?: () => void;
}) => {
const websockets: FakeWebSocket[] = [];
const createWebSocket = vi.fn<(url: string) => WebSocket>(() => {
const websocket = new FakeWebSocket();
websockets.push(websocket);
return websocket.asWebSocket();
});
const channel = new SignalingChannel(
'ws://host/api/ws?src=camera',
{
openCallback: options?.openCallback,
disconnectCallback: options?.disconnectCallback,
},
{ createWebSocket },
);
return { channel, createWebSocket, websockets };
};
it('should connect with an arraybuffer binary type', () => {
const { channel, createWebSocket, websockets } = setup();
channel.connect();
expect(createWebSocket).toHaveBeenCalledWith('ws://host/api/ws?src=camera');
expect(websockets[0].binaryType).toBe('arraybuffer');
});
it('should not connect twice', () => {
const { channel, createWebSocket } = setup();
channel.connect();
channel.connect();
expect(createWebSocket).toHaveBeenCalledTimes(1);
});
it('should report open state and call the open callback', () => {
const openCallback = vi.fn();
const { channel, websockets } = setup({ openCallback });
channel.connect();
expect(channel.isOpen()).toBe(false);
websockets[0].fireOpen();
expect(channel.isOpen()).toBe(true);
expect(openCallback).toHaveBeenCalled();
});
it('should tolerate an absent open callback', () => {
const { channel, websockets } = setup();
channel.connect();
expect(() => websockets[0].fireOpen()).not.toThrow();
});
it('should not send before the connection is open', () => {
const { channel, websockets } = setup();
channel.connect();
channel.send({ type: 'mse', value: 'codecs' });
expect(websockets[0].send).not.toHaveBeenCalled();
});
it('should send JSON once open', () => {
const { channel, websockets } = setup();
channel.connect();
websockets[0].fireOpen();
const message = { type: 'mse', value: 'codecs' };
channel.send(message);
expect(websockets[0].sent).toEqual([JSON.stringify(message)]);
});
it('should dispatch parsed messages to subscribers', () => {
const { channel, websockets } = setup();
const callback = vi.fn();
channel.subscribeToMessages(callback);
channel.connect();
websockets[0].fireMessage('{"type":"mse","value":"video/mp4"}');
expect(callback).toHaveBeenCalledWith({ type: 'mse', value: 'video/mp4' });
});
it('should stop dispatching after unsubscribe', () => {
const { channel, websockets } = setup();
const callback = vi.fn();
const unsubscribe = channel.subscribeToMessages(callback);
channel.connect();
unsubscribe();
websockets[0].fireMessage('{"type":"mse"}');
expect(callback).not.toHaveBeenCalled();
});
it('should dispatch to remaining subscribers when one unsubscribes during dispatch', () => {
const { channel, websockets } = setup();
const secondCallback = vi.fn();
const unsubscribeDuringDispatch = vi.fn((): void => {
unsubscribe();
});
const unsubscribe = channel.subscribeToMessages(unsubscribeDuringDispatch);
channel.subscribeToMessages(secondCallback);
channel.connect();
websockets[0].fireMessage('{"type":"mse"}');
expect(unsubscribeDuringDispatch).toHaveBeenCalledTimes(1);
expect(secondCallback).toHaveBeenCalledTimes(1);
});
it('should ignore invalid JSON', () => {
const { channel, websockets } = setup();
const callback = vi.fn();
channel.subscribeToMessages(callback);
channel.connect();
websockets[0].fireMessage('NOT JSON');
expect(callback).not.toHaveBeenCalled();
});
it('should ignore malformed messages', () => {
const { channel, websockets } = setup();
const callback = vi.fn();
channel.subscribeToMessages(callback);
channel.connect();
websockets[0].fireMessage('{"type":6}');
expect(callback).not.toHaveBeenCalled();
});
it('should ignore unexpected data types', () => {
const { channel, websockets } = setup();
const callback = vi.fn();
channel.subscribeToMessages(callback);
channel.connect();
websockets[0].fireMessage(42);
expect(callback).not.toHaveBeenCalled();
});
it('should route binary data to the binary callback', () => {
const { channel, websockets } = setup();
const binaryCallback = vi.fn();
channel.setBinaryCallback(binaryCallback);
channel.connect();
const data = new ArrayBuffer(8);
websockets[0].fireMessage(data);
expect(binaryCallback).toHaveBeenCalledWith(data);
});
it('should drop binary data without a binary callback', () => {
const { channel, websockets } = setup();
channel.connect();
expect(() => websockets[0].fireMessage(new ArrayBuffer(8))).not.toThrow();
});
it('should drop binary data after the binary callback is cleared', () => {
const { channel, websockets } = setup();
const binaryCallback = vi.fn();
channel.setBinaryCallback(binaryCallback);
channel.setBinaryCallback(null);
channel.connect();
websockets[0].fireMessage(new ArrayBuffer(8));
expect(binaryCallback).not.toHaveBeenCalled();
});
it('should close the underlying websocket without firing the disconnect callback', () => {
const disconnectCallback = vi.fn();
const { channel, websockets } = setup({ disconnectCallback });
channel.connect();
websockets[0].fireOpen();
channel.close();
expect(websockets[0].close).toHaveBeenCalled();
expect(channel.isOpen()).toBe(false);
expect(disconnectCallback).not.toHaveBeenCalled();
});
it('should tolerate closing when never connected', () => {
const { channel } = setup();
expect(() => channel.close()).not.toThrow();
});
it('should ignore websocket events delivered after close', () => {
const openCallback = vi.fn();
const disconnectCallback = vi.fn();
const messageCallback = vi.fn();
const { channel, websockets } = setup({ openCallback, disconnectCallback });
channel.subscribeToMessages(messageCallback);
channel.connect();
channel.close();
websockets[0].fireOpen();
websockets[0].fireMessage('{"type":"mse"}');
websockets[0].fireClose();
expect(openCallback).not.toHaveBeenCalled();
expect(messageCallback).not.toHaveBeenCalled();
expect(disconnectCallback).not.toHaveBeenCalled();
});
it('should fire the disconnect callback on unexpected closure', () => {
const disconnectCallback = vi.fn();
const { channel, websockets } = setup({ disconnectCallback });
channel.connect();
websockets[0].fireOpen();
websockets[0].fireClose();
expect(disconnectCallback).toHaveBeenCalledTimes(1);
expect(channel.isOpen()).toBe(false);
});
it('should tolerate an absent disconnect callback on unexpected closure', () => {
const { channel, websockets } = setup();
channel.connect();
expect(() => websockets[0].fireClose()).not.toThrow();
});
it('should allow reconnecting after unexpected closure', () => {
const { channel, createWebSocket, websockets } = setup();
channel.connect();
websockets[0].fireClose();
channel.connect();
expect(createWebSocket).toHaveBeenCalledTimes(2);
});
it('should construct a real websocket by default', () => {
// A mock implementation must be callable with `new`, so it cannot be an
// arrow function.
const webSocketConstructor = vi.fn(function () {
return new FakeWebSocket().asWebSocket();
});
vi.stubGlobal('WebSocket', webSocketConstructor);
const channel = new SignalingChannel('ws://host/api/ws', {});
channel.connect();
channel.close();
expect(webSocketConstructor).toHaveBeenCalledWith('ws://host/api/ws');
vi.unstubAllGlobals();
});
});
+155
View File
@@ -0,0 +1,155 @@
import { vi, type Mock } from 'vitest';
// ===========================================================================
// Fakes for browser APIs jsdom does not provide.
// ===========================================================================
export class FakeWebSocket extends EventTarget {
public binaryType = '';
public sent: string[] = [];
public close = vi.fn();
public send = vi.fn((data: string): void => {
this.sent.push(data);
});
public asWebSocket(): WebSocket {
return this as unknown as WebSocket;
}
public fireOpen(): void {
this.dispatchEvent(new Event('open'));
}
public fireClose(): void {
this.dispatchEvent(new Event('close'));
}
public fireMessage(data: unknown): void {
this.dispatchEvent(new MessageEvent('message', { data }));
}
}
export class FakeMediaStreamTrack extends EventTarget {
public muted = false;
public readyState: MediaStreamTrackState = 'live';
public kind: string;
constructor(kind: string) {
super();
this.kind = kind;
}
public asTrack(): MediaStreamTrack {
return this as unknown as MediaStreamTrack;
}
public setMuted(muted: boolean): void {
this.muted = muted;
this.dispatchEvent(new Event(muted ? 'mute' : 'unmute'));
}
}
export class FakeMediaStream {
private _tracks: FakeMediaStreamTrack[];
constructor(tracks: FakeMediaStreamTrack[] = []) {
this._tracks = tracks;
}
public getTracks(): FakeMediaStreamTrack[] {
return this._tracks;
}
public getVideoTracks(): FakeMediaStreamTrack[] {
return this._tracks.filter((track) => track.kind === 'video');
}
public getAudioTracks(): FakeMediaStreamTrack[] {
return this._tracks.filter((track) => track.kind === 'audio');
}
public asMediaStream(): MediaStream {
return this as unknown as MediaStream;
}
}
class FakeRTCTransceiver {
public direction: string;
public currentDirection: string;
public sender: {
track: FakeMediaStreamTrack | null;
replaceTrack: Mock<(track: MediaStreamTrack | null) => Promise<void>>;
};
public receiver: { track: FakeMediaStreamTrack };
constructor(direction: string, kind: string, track: FakeMediaStreamTrack | null) {
this.direction = direction;
this.currentDirection = direction;
this.sender = {
track,
replaceTrack: vi.fn<(track: MediaStreamTrack | null) => Promise<void>>(() =>
Promise.resolve(),
),
};
this.receiver = { track: new FakeMediaStreamTrack(kind) };
}
}
export class FakeRTCPeerConnection extends EventTarget {
public connectionState: RTCPeerConnectionState = 'new';
public remoteDescription: { sdp: string } | null = null;
public transceivers: FakeRTCTransceiver[] = [];
public createOffer = vi.fn(
(): Promise<{ type: string; sdp?: string }> =>
Promise.resolve({ type: 'offer', sdp: 'v=0\r\noffer' }),
);
public setLocalDescription = vi.fn(() => Promise.resolve());
public setRemoteDescription = vi.fn((description: { sdp: string }) => {
this.remoteDescription = description;
return Promise.resolve();
});
public addIceCandidate = vi.fn(() => Promise.resolve());
public close = vi.fn();
public addTransceiver(
trackOrKind: FakeMediaStreamTrack | string,
init: { direction: string },
): FakeRTCTransceiver {
const kind = typeof trackOrKind === 'string' ? trackOrKind : trackOrKind.kind;
const track = typeof trackOrKind === 'string' ? null : trackOrKind;
const transceiver = new FakeRTCTransceiver(init.direction, kind, track);
this.transceivers.push(transceiver);
return transceiver;
}
public getTransceivers(): FakeRTCTransceiver[] {
return this.transceivers;
}
public getReceivers(): { track: FakeMediaStreamTrack }[] {
return this.transceivers.map((transceiver) => transceiver.receiver);
}
public asPeerConnection(): RTCPeerConnection {
return this as unknown as RTCPeerConnection;
}
public getMicrophoneTransceiver(): FakeRTCTransceiver {
return this.transceivers[0];
}
public fireConnectionStateChange(state: RTCPeerConnectionState): void {
this.connectionState = state;
this.dispatchEvent(new Event('connectionstatechange'));
}
public fireIceCandidate(candidate: string | null): void {
const event = new Event('icecandidate');
Object.assign(event, {
candidate: candidate === null ? null : { candidate },
});
this.dispatchEvent(event);
}
}