feat: Add microphone audio processing constraints (#2708)

## Summary

- add optional microphone constraints for echo cancellation, noise
suppression, automatic gain control, and channel count
- request configured values as non-mandatory `ideal` constraints
- expose privacy-safe microphone capabilities, requested constraints,
and applied settings in card diagnostics
- document the new configuration and add schema, microphone manager, and
diagnostics tests

## Motivation

The card currently calls `getUserMedia()` with `audio: true`. This
leaves echo cancellation, noise suppression, automatic gain control, and
channel count implicit.

Browser and device behavior differs. Explicit processing defaults can
regress microphone gain or amplify noise on some devices. This change
therefore keeps all processing constraints optional and configurable.

## Configuration

```yaml
live:
  microphone:
    constraints:
      echo_cancellation: true
      noise_suppression: true
      auto_gain_control: false
      channel_count: 1
```

Configured values use `ideal` constraints. A browser can ignore
unsupported values. Card diagnostics show the browser capabilities, the
requested constraints, and the reported applied settings.

## Backward compatibility

- existing configurations still use `audio: true`
- no audio-processing defaults are added
- explicit `false` values are preserved
- diagnostic output excludes device and group identifiers

## Validation

- focused microphone, schema, and diagnostics tests: 46 passed
- full test suite: 7,177 passed
- lint passed
- format check passed
- typecheck passed
- unused-code check passed
- production build passed

The optional constraints were also tested successfully with an iOS Home
Assistant Companion client and a go2rtc-based full-duplex intercom. This
is a client microphone-processing change only. It does not add backend
audio denoise.

---------

Co-authored-by: dermotduffy <dermot.duffy@gmail.com>

src/card-controller/microphone-manager.ts

@@ -1,7 +1,13 @@
+import { omit } from 'lodash-es';
+
 import { localize } from '../localize/localize';
 import { AdvancedCameraCardError } from '../types';
 import { Generation } from '../utils/concurrency/generation';
-import type { CardMicrophoneAPI, MicrophoneState } from './types';
+import type { CardMicrophoneAPI, MicrophoneDiagnostics, MicrophoneState } from './types';
+
+const MICROPHONE_DEVICE_IDENTIFIERS = ['deviceId', 'groupId'] as const;
+
+export type MicrophoneDeviceIdentifier = (typeof MICROPHONE_DEVICE_IDENTIFIERS)[number];
 
 export class MicrophoneNotSupportedError extends AdvancedCameraCardError {
   constructor() {
@@ -13,6 +19,9 @@ export class MicrophoneManager {
   private _api: CardMicrophoneAPI;
   private _stream: MediaStream | null = null;
 
+  // The most recent microphone connection's diagnostics.
+  private _diagnostics: MicrophoneDiagnostics | null = null;
+
   // Whether the browser denied the most recent microphone request. Cleared by
   // a later successful connect.
   private _forbidden = false;
@@ -45,6 +54,10 @@ export class MicrophoneManager {
     return this._state;
   }
 
+  public getDiagnostics(): MicrophoneDiagnostics | null {
+    return this._diagnostics;
+  }
+
   public initialize(): void {
     this._setState();
   }
@@ -79,7 +92,7 @@ export class MicrophoneManager {
     let stream: MediaStream;
     try {
       stream = await navigator.mediaDevices.getUserMedia({
-        audio: true,
+        audio: this._getAudioProcessingConstraints(),
         video: false,
       });
     } catch (e: unknown) {
@@ -104,6 +117,7 @@ export class MicrophoneManager {
     this._removeEndedListeners(this._stream);
     this._stopTracks(this._stream);
     this._stream = stream;
+    this._diagnostics = this._getTrackDiagnostics(stream.getAudioTracks()[0]);
     this._addEndedListeners(stream);
     this._forbidden = false;
     this._reconcile();
@@ -169,6 +183,55 @@ export class MicrophoneManager {
     return !this._stream || this._stream.getTracks().every((track) => !track.enabled);
   }
 
+  private _getAudioProcessingConstraints(): true | MediaTrackConstraints {
+    const audioProcessing = this._api.getConfigManager().getConfig()?.live
+      .microphone?.audio_processing;
+
+    const constraints: MediaTrackConstraints = {};
+    if (typeof audioProcessing?.auto_gain_control === 'boolean') {
+      constraints.autoGainControl = { ideal: audioProcessing.auto_gain_control };
+    }
+    if (audioProcessing?.channel_count !== undefined) {
+      constraints.channelCount = { ideal: audioProcessing.channel_count };
+    }
+    if (typeof audioProcessing?.echo_cancellation === 'boolean') {
+      constraints.echoCancellation = { ideal: audioProcessing.echo_cancellation };
+    }
+    if (typeof audioProcessing?.noise_suppression === 'boolean') {
+      constraints.noiseSuppression = { ideal: audioProcessing.noise_suppression };
+    }
+
+    return Object.keys(constraints).length ? constraints : true;
+  }
+
+  private _getTrackDiagnostics(track?: MediaStreamTrack): MicrophoneDiagnostics | null {
+    if (!track) {
+      return null;
+    }
+
+    // Remove values not suitable for sharing.
+    const getReportableValues = <
+      T extends Partial<Record<MicrophoneDeviceIdentifier, unknown>>,
+    >(
+      values?: T,
+    ): Omit<T, MicrophoneDeviceIdentifier> | null => {
+      if (!values) {
+        return null;
+      }
+
+      const reportable = omit(values, MICROPHONE_DEVICE_IDENTIFIERS);
+      return Object.keys(reportable).length ? reportable : null;
+    };
+
+    const capabilities = getReportableValues(track.getCapabilities?.());
+    const settings = getReportableValues(track.getSettings());
+    const diagnostics = {
+      ...(capabilities && { capabilities }),
+      ...(settings && { settings }),
+    };
+    return Object.keys(diagnostics).length ? diagnostics : null;
+  }
+
   private _stopTracks(stream: MediaStream | null): void {
     stream?.getTracks().forEach((track) => track.stop());
   }

src/card-controller/types.ts

@@ -24,7 +24,10 @@ import type { KeyboardStateManager } from './keyboard-state-manager';
 import type { LockManager } from './lock/manager';
 import type { MediaLoadedInfoManager } from './media-info-manager';
 import type { MediaPlayerManager } from './media-player-manager';
-import type { MicrophoneManager } from './microphone-manager';
+import type {
+  MicrophoneDeviceIdentifier,
+  MicrophoneManager,
+} from './microphone-manager';
 import type { NotificationManager } from './notification-manager';
 import type { PIPManager } from './pip-manager';
 import type { QueryStringManager } from './query-string-manager';
@@ -377,6 +380,14 @@ export interface MicrophoneState {
   forbidden: boolean;
 }
 
+export interface MicrophoneDiagnostics {
+  // What the microphone is able to do.
+  capabilities?: Omit<MediaTrackCapabilities, MicrophoneDeviceIdentifier>;
+
+  // What the browser actually applied.
+  settings?: Omit<MediaTrackSettings, MicrophoneDeviceIdentifier>;
+}
+
 export interface TaggedAutomation extends Automation {
   tag?: unknown;
 }

src/components-lib/editor/schema/live.ts

@@ -1,5 +1,5 @@
 import { BUTTON_SIZE_MIN } from '../../../config/schema/common/const';
-import type { HAFormExpandableSchema } from '../../../ha/types';
+import type { HAFormExpandableSchema, HAFormSelectorSchema } from '../../../ha/types';
 import { localize } from '../../../localize/localize';
 import type { EditorForm } from '../types';
 import { getNextPreviousSchema } from './common/controls/next-previous';
@@ -97,6 +97,30 @@ const getControlsSchema = (): HAFormExpandableSchema => ({
   ],
 });
 
+const AUDIO_PROCESSING_LOCALIZE_PREFIX = 'config.live.microphone.audio_processing';
+
+const getAudioProcessingModeField = (name: string): HAFormSelectorSchema => ({
+  name,
+  selector: createSelectSelector([
+    { value: 'auto', label: localize(`${AUDIO_PROCESSING_LOCALIZE_PREFIX}.modes.auto`) },
+    { value: true, label: localize(`${AUDIO_PROCESSING_LOCALIZE_PREFIX}.modes.true`) },
+    { value: false, label: localize(`${AUDIO_PROCESSING_LOCALIZE_PREFIX}.modes.false`) },
+  ]),
+});
+
+const getAudioProcessingSchema = (): HAFormExpandableSchema => ({
+  name: 'audio_processing',
+  type: 'expandable',
+  title: localize(`${AUDIO_PROCESSING_LOCALIZE_PREFIX}.editor_label`),
+  icon: 'mdi:audio-input-stereo-minijack',
+  schema: [
+    getAudioProcessingModeField('auto_gain_control'),
+    { name: 'channel_count', selector: createNumberSelector({ min: 1 }) },
+    getAudioProcessingModeField('echo_cancellation'),
+    getAudioProcessingModeField('noise_suppression'),
+  ],
+});
+
 const getMicrophoneSchema = (): HAFormExpandableSchema => ({
   name: 'microphone',
   type: 'expandable',
@@ -104,6 +128,7 @@ const getMicrophoneSchema = (): HAFormExpandableSchema => ({
   icon: 'mdi:microphone',
   schema: [
     { name: 'always_connected', selector: { boolean: {} } },
+    getAudioProcessingSchema(),
     {
       name: 'auto_mute',
       selector: createSelectSelector(getMicrophoneMuteOptions(), { multiple: true }),

src/components/diagnostics.ts

@@ -9,6 +9,7 @@ import { customElement, property } from 'lit/decorators.js';
 import { until } from 'lit/directives/until.js';
 
 import type { IssuePresence } from '../card-controller/issues/types';
+import type { MicrophoneDiagnostics } from '../card-controller/types';
 import type { RawAdvancedCameraCardConfig } from '../config/types';
 import type { DeviceRegistryManager } from '../ha/registry/device';
 import type { HomeAssistant } from '../ha/types';
@@ -31,13 +32,17 @@ export class AdvancedCameraCardDiagnostics extends LitElement {
   @property({ attribute: false })
   public issues?: IssuePresence;
 
+  @property({ attribute: false })
+  public microphoneDiagnostics?: MicrophoneDiagnostics;
+
   private async _renderDiagnostics(): Promise<TemplateResult> {
-    const diagnostics = await getDiagnostics(
-      this.hass,
-      this.deviceRegistryManager,
-      this.rawConfig,
-      this.issues,
-    );
+    const diagnostics = await getDiagnostics({
+      hass: this.hass,
+      deviceRegistryManager: this.deviceRegistryManager,
+      rawConfig: this.rawConfig,
+      issues: this.issues,
+      microphoneDiagnostics: this.microphoneDiagnostics,
+    });
 
     return renderNotificationBlockFromText(localize('error.diagnostics'), {
       icon: 'mdi:cogs',

src/components/views.ts

@@ -236,6 +236,7 @@ export class AdvancedCameraCardViews extends LitElement {
             .rawConfig=${this.rawConfig}
             .deviceRegistryManager=${this.deviceRegistryManager}
             .issues=${this.issues}
+            .microphoneDiagnostics=${this.microphoneManager?.getDiagnostics()}
           >
           </advanced-camera-card-diagnostics>`
         : ``}

src/config/schema/live.ts

@@ -23,13 +23,37 @@ import {
 } from './common/media-actions';
 import { transitionEffectConfigSchema } from './common/transition-effect';
 
+const microphoneAudioProcessingDefault = {
+  auto_gain_control: 'auto' as const,
+  echo_cancellation: 'auto' as const,
+  noise_suppression: 'auto' as const,
+};
+
 const microphoneConfigDefault = {
   always_connected: false,
+  audio_processing: { ...microphoneAudioProcessingDefault },
   auto_mute: [],
   auto_unmute: [],
   mute_after_microphone_mute_seconds: 60,
 };
 
+// `auto` sends no constraint for the option and leaves the choice to the
+// browser, which behaves differently from an explicit `false`.
+const audioProcessingModeSchema = z.boolean().or(z.literal('auto'));
+
+const microphoneAudioProcessingSchema = z.object({
+  auto_gain_control: audioProcessingModeSchema.default(
+    microphoneAudioProcessingDefault.auto_gain_control,
+  ),
+  channel_count: z.number().int().positive().optional(),
+  echo_cancellation: audioProcessingModeSchema.default(
+    microphoneAudioProcessingDefault.echo_cancellation,
+  ),
+  noise_suppression: audioProcessingModeSchema.default(
+    microphoneAudioProcessingDefault.noise_suppression,
+  ),
+});
+
 const ringtoneConfigDefault = {
   type: 'chime' as const,
   repeat: 0,
@@ -71,6 +95,9 @@ const callConfigSchema = z.object({
 const microphoneConfigSchema = z
   .object({
     always_connected: z.boolean().default(microphoneConfigDefault.always_connected),
+    audio_processing: microphoneAudioProcessingSchema.default(
+      microphoneConfigDefault.audio_processing,
+    ),
     auto_mute: z
       .enum(MICROPHONE_MUTE_CONDITIONS)
       .array()

src/localize/languages/en.json

@@ -492,6 +492,18 @@
       "lazy_unload": "Live cameras are lazily unloaded",
       "microphone": {
         "always_connected": "Always keep the microphone connected",
+        "audio_processing": {
+          "auto_gain_control": "Automatic gain control",
+          "channel_count": "Channel count (1=mono)",
+          "echo_cancellation": "Echo cancellation",
+          "editor_label": "Audio processing",
+          "modes": {
+            "auto": "Automatic",
+            "false": "Disabled",
+            "true": "Enabled"
+          },
+          "noise_suppression": "Noise suppression"
+        },
         "auto_mute": "Automatically mute the microphone",
         "auto_unmute": "Automatically unmute the microphone",
         "editor_label": "Microphone",

src/utils/diagnostics.ts

@@ -1,4 +1,5 @@
 import type { IssueKey, IssuePresence } from '../card-controller/issues/types';
+import type { MicrophoneDiagnostics } from '../card-controller/types';
 import type { RawAdvancedCameraCardConfig } from '../config/types';
 import { getIntegrationManifest } from '../ha/integration';
 import type { IntegrationManifest } from '../ha/integration/types';
@@ -32,6 +33,7 @@ interface Diagnostics {
   ha_version?: string;
   config?: RawAdvancedCameraCardConfig;
   issues?: IssueKey[];
+  microphone?: MicrophoneDiagnostics;
 
   custom_integrations: {
     frigate: IntegrationDiagnostics & {
@@ -62,12 +64,16 @@ const getIntegrationDiagnostics = async (
   };
 };
 
-export const getDiagnostics = async (
-  hass?: HomeAssistant,
-  deviceRegistryManager?: DeviceRegistryManager,
-  rawConfig?: RawAdvancedCameraCardConfig,
-  issues?: IssuePresence,
-): Promise<Diagnostics> => {
+export const getDiagnostics = async (options?: {
+  hass?: HomeAssistant;
+  deviceRegistryManager?: DeviceRegistryManager;
+  rawConfig?: RawAdvancedCameraCardConfig;
+  issues?: IssuePresence;
+  microphoneDiagnostics?: MicrophoneDiagnostics;
+}): Promise<Diagnostics> => {
+  const { hass, deviceRegistryManager, rawConfig, issues, microphoneDiagnostics } =
+    options ?? {};
+
   // Get the Frigate devices in order to extract the Frigate integration and
   // server version numbers.
   const frigateDevices =
@@ -112,5 +118,6 @@ export const getDiagnostics = async (
     },
     issues: issues ? [...issues.keys()] : [],
     ...(rawConfig && { config: rawConfig }),
+    ...(microphoneDiagnostics && { microphone: microphoneDiagnostics }),
   };
 };