feat: Add microphone audio processing constraints (#2708)

## Summary

- add optional microphone constraints for echo cancellation, noise
suppression, automatic gain control, and channel count
- request configured values as non-mandatory `ideal` constraints
- expose privacy-safe microphone capabilities, requested constraints,
and applied settings in card diagnostics
- document the new configuration and add schema, microphone manager, and
diagnostics tests

## Motivation

The card currently calls `getUserMedia()` with `audio: true`. This
leaves echo cancellation, noise suppression, automatic gain control, and
channel count implicit.

Browser and device behavior differs. Explicit processing defaults can
regress microphone gain or amplify noise on some devices. This change
therefore keeps all processing constraints optional and configurable.

## Configuration

```yaml
live:
  microphone:
    constraints:
      echo_cancellation: true
      noise_suppression: true
      auto_gain_control: false
      channel_count: 1
```

Configured values use `ideal` constraints. A browser can ignore
unsupported values. Card diagnostics show the browser capabilities, the
requested constraints, and the reported applied settings.

## Backward compatibility

- existing configurations still use `audio: true`
- no audio-processing defaults are added
- explicit `false` values are preserved
- diagnostic output excludes device and group identifiers

## Validation

- focused microphone, schema, and diagnostics tests: 46 passed
- full test suite: 7,177 passed
- lint passed
- format check passed
- typecheck passed
- unused-code check passed
- production build passed

The optional constraints were also tested successfully with an iOS Home
Assistant Companion client and a go2rtc-based full-duplex intercom. This
is a client microphone-processing change only. It does not add backend
audio denoise.

---------

Co-authored-by: dermotduffy <dermot.duffy@gmail.com>
This commit is contained in:
Filip Pytloun
2026-08-24 07:38:33 -07:00
committed by GitHub
co-authored by dermotduffy
parent 543e5d0fcf
commit 3ee6b6059e
13 changed files with 377 additions and 25 deletions
@@ -40,6 +40,7 @@ describe('MicrophoneManager', () => {
const track = mock<MediaStreamTrack>();
track.enabled = !mute;
stream.getTracks.mockImplementation(() => [track]);
stream.getAudioTracks.mockImplementation(() => [track]);
return stream;
};
@@ -89,6 +90,150 @@ describe('MicrophoneManager', () => {
expect(manager.getStream()).toBe(stream);
expect(manager.isMuted()).toBeTruthy();
expect(api.getCardElementManager().update).toHaveBeenCalled();
expect(navigatorMock.mediaDevices.getUserMedia).toHaveBeenCalledWith({
audio: true,
video: false,
});
});
it('should request configured ideal audio constraints', async () => {
const api = createCardAPI();
vi.mocked(api.getConfigManager().getConfig).mockReturnValue(
createConfig({
live: {
microphone: {
audio_processing: {
echo_cancellation: true,
noise_suppression: false,
auto_gain_control: false,
channel_count: 1,
},
},
},
}),
);
vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(
createMockStream(),
);
const manager = new MicrophoneManager(api);
manager.setTransmissionActive(true);
await manager.connect();
expect(navigatorMock.mediaDevices.getUserMedia).toHaveBeenCalledWith({
audio: {
autoGainControl: { ideal: false },
channelCount: { ideal: 1 },
echoCancellation: { ideal: true },
noiseSuppression: { ideal: false },
},
video: false,
});
});
it('should request no constraint for an audio processing option left to the browser', async () => {
const api = createCardAPI();
vi.mocked(api.getConfigManager().getConfig).mockReturnValue(
createConfig({
live: {
microphone: {
audio_processing: {
auto_gain_control: 'auto',
echo_cancellation: 'auto',
noise_suppression: true,
},
},
},
}),
);
vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(
createMockStream(),
);
const manager = new MicrophoneManager(api);
manager.setTransmissionActive(true);
await manager.connect();
expect(navigatorMock.mediaDevices.getUserMedia).toHaveBeenCalledWith({
audio: {
noiseSuppression: { ideal: true },
},
video: false,
});
});
it('should expose microphone diagnostics without device identifiers', async () => {
const api = createCardAPI();
const stream = createMockStream();
const track = getTrack(stream);
vi.mocked(track.getCapabilities).mockReturnValue({
echoCancellation: [true, false],
deviceId: 'secret-device',
});
vi.mocked(track.getSettings).mockReturnValue({
echoCancellation: true,
deviceId: 'secret-device',
groupId: 'secret-group',
});
vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(stream);
const manager = new MicrophoneManager(api);
manager.setTransmissionActive(true);
await manager.connect();
manager.setTransmissionActive(false);
expect(manager.getDiagnostics()).toEqual({
capabilities: {
echoCancellation: [true, false],
},
settings: {
echoCancellation: true,
},
});
});
it('should omit a diagnostic that holds nothing but device identifiers', async () => {
const api = createCardAPI();
const stream = createMockStream();
const track = getTrack(stream);
vi.mocked(track.getCapabilities).mockReturnValue({ deviceId: 'secret-device' });
vi.mocked(track.getSettings).mockReturnValue({ echoCancellation: true });
vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(stream);
const manager = new MicrophoneManager(api);
manager.setTransmissionActive(true);
await manager.connect();
expect(manager.getDiagnostics()).toEqual({
settings: { echoCancellation: true },
});
});
it('should support a browser without the capabilities method', async () => {
const api = createCardAPI();
const stream = createMockStream();
const track = getTrack(stream);
track.getCapabilities = undefined as unknown as MediaStreamTrack['getCapabilities'];
vi.mocked(track.getSettings).mockReturnValue({});
vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(stream);
const manager = new MicrophoneManager(api);
manager.setTransmissionActive(true);
expect(await manager.connect()).toBeTruthy();
expect(manager.getDiagnostics()).toBeNull();
});
it('should report no diagnostics for a stream without an audio track', async () => {
const api = createCardAPI();
const stream = createMockStream();
vi.mocked(stream.getAudioTracks).mockImplementation(() => []);
vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(stream);
const manager = new MicrophoneManager(api);
manager.setTransmissionActive(true);
expect(await manager.connect()).toBeTruthy();
expect(manager.getDiagnostics()).toBeNull();
});
it('should release a stream that connects without active transmission', async () => {
+5
View File
@@ -147,6 +147,11 @@ describe('config defaults', () => {
lazy_unload: [],
microphone: {
always_connected: false,
audio_processing: {
auto_gain_control: 'auto',
echo_cancellation: 'auto',
noise_suppression: 'auto',
},
auto_mute: [],
auto_unmute: [],
mute_after_microphone_mute_seconds: 60,
+29 -8
View File
@@ -67,8 +67,10 @@ describe('getDiagnostics', () => {
});
expect(
await getDiagnostics(hass, deviceRegistryManager, {
cameras: [{ camera_entity: 'camera.office' }],
await getDiagnostics({
hass,
deviceRegistryManager,
rawConfig: { cameras: [{ camera_entity: 'camera.office' }] },
}),
).toEqual({
browser: 'AdvancedCameraCardTest/1.0',
@@ -107,8 +109,10 @@ describe('getDiagnostics', () => {
const deviceRegistryManager = mock<DeviceRegistryManager>();
deviceRegistryManager.getMatchingDevices.mockResolvedValue([]);
await getDiagnostics(hass, deviceRegistryManager, {
cameras: [{ camera_entity: 'camera.office' }],
await getDiagnostics({
hass,
deviceRegistryManager,
rawConfig: { cameras: [{ camera_entity: 'camera.office' }] },
});
// Verify the matcher passed into the deviceRegistryManager correctly filters
@@ -163,21 +167,38 @@ describe('getDiagnostics', () => {
],
]);
const result = await getDiagnostics(
const result = await getDiagnostics({
hass,
deviceRegistryManager,
{ cameras: [{ camera_entity: 'camera.office' }] },
rawConfig: { cameras: [{ camera_entity: 'camera.office' }] },
issues,
);
});
expect(result.issues).toEqual(['config_upgrade']);
});
it('should include microphone diagnostics', async () => {
const microphoneDiagnostics = {
capabilities: {
echoCancellation: [true, false],
},
settings: {
echoCancellation: true,
},
};
expect(await getDiagnostics({ microphoneDiagnostics })).toEqual(
expect.objectContaining({
microphone: microphoneDiagnostics,
}),
);
});
it('should fetch diagnostics without device model', async () => {
const deviceRegistryManager = mock<DeviceRegistryManager>();
deviceRegistryManager.getMatchingDevices.mockResolvedValue([]);
expect(await getDiagnostics(hass, deviceRegistryManager)).toEqual({
expect(await getDiagnostics({ hass, deviceRegistryManager })).toEqual({
browser: 'AdvancedCameraCardTest/1.0',
card_version: '1.2.3',
git: {