Files
advanced-camera-card/src/config/schema/live.ts
T
Filip Pytlounanddermotduffy 3ee6b6059e feat: Add microphone audio processing constraints (#2708)
## Summary

- add optional microphone constraints for echo cancellation, noise
suppression, automatic gain control, and channel count
- request configured values as non-mandatory `ideal` constraints
- expose privacy-safe microphone capabilities, requested constraints,
and applied settings in card diagnostics
- document the new configuration and add schema, microphone manager, and
diagnostics tests

## Motivation

The card currently calls `getUserMedia()` with `audio: true`. This
leaves echo cancellation, noise suppression, automatic gain control, and
channel count implicit.

Browser and device behavior differs. Explicit processing defaults can
regress microphone gain or amplify noise on some devices. This change
therefore keeps all processing constraints optional and configurable.

## Configuration

```yaml
live:
  microphone:
    constraints:
      echo_cancellation: true
      noise_suppression: true
      auto_gain_control: false
      channel_count: 1
```

Configured values use `ideal` constraints. A browser can ignore
unsupported values. Card diagnostics show the browser capabilities, the
requested constraints, and the reported applied settings.

## Backward compatibility

- existing configurations still use `audio: true`
- no audio-processing defaults are added
- explicit `false` values are preserved
- diagnostic output excludes device and group identifiers

## Validation

- focused microphone, schema, and diagnostics tests: 46 passed
- full test suite: 7,177 passed
- lint passed
- format check passed
- typecheck passed
- unused-code check passed
- production build passed

The optional constraints were also tested successfully with an iOS Home
Assistant Companion client and a go2rtc-based full-duplex intercom. This
is a client microphone-processing change only. It does not add backend
audio denoise.

---------

Co-authored-by: dermotduffy <dermot.duffy@gmail.com>
2026-08-24 07:38:33 -07:00

210 lines
7.0 KiB
TypeScript

import { z } from 'zod';
import { actionsSchema } from './actions/types';
import { BUTTON_SIZE_MIN } from './common/const';
import { nextPreviousControlConfigSchema } from './common/controls/next-previous';
import { ptzControlsConfigSchema, ptzControlsDefaults } from './common/controls/ptz';
import {
thumbnailControlsDefaults,
thumbnailsControlSchema,
} from './common/controls/thumbnails';
import {
miniTimelineConfigDefault,
miniTimelineConfigSchema,
} from './common/controls/timeline';
import { viewDisplaySchema } from './common/display';
import {
MEDIA_ACTION_NEGATIVE_CONDITIONS,
MEDIA_ACTION_POSITIVE_CONDITIONS,
MEDIA_MUTE_CONDITIONS,
MEDIA_UNMUTE_CONDITIONS,
MICROPHONE_MUTE_CONDITIONS,
MICROPHONE_UNMUTE_CONDITIONS,
} from './common/media-actions';
import { transitionEffectConfigSchema } from './common/transition-effect';
const microphoneAudioProcessingDefault = {
auto_gain_control: 'auto' as const,
echo_cancellation: 'auto' as const,
noise_suppression: 'auto' as const,
};
const microphoneConfigDefault = {
always_connected: false,
audio_processing: { ...microphoneAudioProcessingDefault },
auto_mute: [],
auto_unmute: [],
mute_after_microphone_mute_seconds: 60,
};
// `auto` sends no constraint for the option and leaves the choice to the
// browser, which behaves differently from an explicit `false`.
const audioProcessingModeSchema = z.boolean().or(z.literal('auto'));
const microphoneAudioProcessingSchema = z.object({
auto_gain_control: audioProcessingModeSchema.default(
microphoneAudioProcessingDefault.auto_gain_control,
),
channel_count: z.number().int().positive().optional(),
echo_cancellation: audioProcessingModeSchema.default(
microphoneAudioProcessingDefault.echo_cancellation,
),
noise_suppression: audioProcessingModeSchema.default(
microphoneAudioProcessingDefault.noise_suppression,
),
});
const ringtoneConfigDefault = {
type: 'chime' as const,
repeat: 0,
};
const ringtoneConfigSchema = z.object({
type: z
.enum(['none', 'chime', 'westminster', 'arpeggio', 'melody', 'custom'])
.default(ringtoneConfigDefault.type),
// For `type` is `custom`, path to an audio file.
url: z.string().optional(),
// Number of times the ringtone plays per inbound call. `0` is indefinitely.
repeat: z.number().int().min(0).default(ringtoneConfigDefault.repeat),
});
export type RingtoneConfig = z.infer<typeof ringtoneConfigSchema>;
const callConfigDefault = {
button_size: 40,
enabled: true,
lock: true,
ringtone: { ...ringtoneConfigDefault },
unanswered_timeout_seconds: 60,
};
const callConfigSchema = z.object({
button_size: z.number().min(BUTTON_SIZE_MIN).default(callConfigDefault.button_size),
enabled: z.boolean().default(callConfigDefault.enabled),
lock: z.boolean().default(callConfigDefault.lock),
ringtone: ringtoneConfigSchema.default(callConfigDefault.ringtone),
// Seconds an inbound call may ring unanswered before it is auto-ended.
unanswered_timeout_seconds: z
.number()
.min(0)
.default(callConfigDefault.unanswered_timeout_seconds),
});
const microphoneConfigSchema = z
.object({
always_connected: z.boolean().default(microphoneConfigDefault.always_connected),
audio_processing: microphoneAudioProcessingSchema.default(
microphoneConfigDefault.audio_processing,
),
auto_mute: z
.enum(MICROPHONE_MUTE_CONDITIONS)
.array()
.default(microphoneConfigDefault.auto_mute),
auto_unmute: z
.enum(MICROPHONE_UNMUTE_CONDITIONS)
.array()
.default(microphoneConfigDefault.auto_unmute),
mute_after_microphone_mute_seconds: z
.number()
.min(0)
.default(microphoneConfigDefault.mute_after_microphone_mute_seconds),
})
.default(microphoneConfigDefault);
export const liveConfigDefault = {
auto_play: [...MEDIA_ACTION_POSITIVE_CONDITIONS],
auto_pause: [],
auto_mute: [...MEDIA_MUTE_CONDITIONS],
auto_unmute: ['microphone' as const, 'call' as const],
preload: false,
lazy_load: true,
lazy_unload: [],
draggable: true,
zoomable: true,
transition_effect: 'slide' as const,
show_image_during_load: true,
controls: {
builtin: true,
call: { ...callConfigDefault },
next_previous: {
auto_hide: ['call' as const, 'casting' as const],
size: 48,
style: 'chevrons' as const,
},
ptz: ptzControlsDefaults,
thumbnails: thumbnailControlsDefaults,
timeline: miniTimelineConfigDefault,
wheel: true,
},
microphone: {
...microphoneConfigDefault,
},
};
export const liveConfigSchema = z
.object({
auto_pause: z
.enum(MEDIA_ACTION_NEGATIVE_CONDITIONS)
.array()
.default(liveConfigDefault.auto_pause),
auto_play: z
.enum(MEDIA_ACTION_POSITIVE_CONDITIONS)
.array()
.default(liveConfigDefault.auto_play),
auto_mute: z
.enum(MEDIA_MUTE_CONDITIONS)
.array()
.default(liveConfigDefault.auto_mute),
auto_unmute: z
.enum(MEDIA_UNMUTE_CONDITIONS)
.array()
.default(liveConfigDefault.auto_unmute),
controls: z
.object({
builtin: z.boolean().default(liveConfigDefault.controls.builtin),
call: callConfigSchema.default(liveConfigDefault.controls.call),
next_previous: nextPreviousControlConfigSchema
.extend({
auto_hide: nextPreviousControlConfigSchema.shape.auto_hide.default(
liveConfigDefault.controls.next_previous.auto_hide,
),
// Live cannot show thumbnails, remove that option.
style: z
.enum(['none', 'chevrons', 'icons'])
.default(liveConfigDefault.controls.next_previous.style),
size: nextPreviousControlConfigSchema.shape.size.default(
liveConfigDefault.controls.next_previous.size,
),
})
.default(liveConfigDefault.controls.next_previous),
ptz: ptzControlsConfigSchema.default(liveConfigDefault.controls.ptz),
thumbnails: thumbnailsControlSchema.default(
liveConfigDefault.controls.thumbnails,
),
timeline: miniTimelineConfigSchema.default(liveConfigDefault.controls.timeline),
wheel: z.boolean().default(liveConfigDefault.controls.wheel),
})
.default(liveConfigDefault.controls),
display: viewDisplaySchema,
draggable: z.boolean().default(liveConfigDefault.draggable),
lazy_load: z.boolean().default(liveConfigDefault.lazy_load),
lazy_unload: z
.enum(MEDIA_ACTION_NEGATIVE_CONDITIONS)
.array()
.default(liveConfigDefault.lazy_unload),
microphone: microphoneConfigSchema.default(liveConfigDefault.microphone),
preload: z.boolean().default(liveConfigDefault.preload),
show_image_during_load: z
.boolean()
.default(liveConfigDefault.show_image_during_load),
transition_effect: transitionEffectConfigSchema.default(
liveConfigDefault.transition_effect,
),
zoomable: z.boolean().default(liveConfigDefault.zoomable),
})
.extend(actionsSchema.shape)
.default(liveConfigDefault);
export type LiveConfig = z.infer<typeof liveConfigSchema>;