Add experimental 2-way audio support.

src/components/live/live-go2rtc.ts

@@ -12,6 +12,7 @@ import {
   CameraConfig,
   ExtendedHomeAssistant,
   FrigateCardMediaPlayer,
+  MicrophoneConfig,
 } from '../../types.js';
 import '../image.js';
 import {
@@ -34,10 +35,40 @@ const GO2RTC_URL_SIGN_EXPIRY_SECONDS = 24 * 60 * 60;
 
 @customElement('frigate-card-live-go2rtc-player')
 class FrigateCardGo2RTCPlayer extends VideoRTC {
+  protected _microphoneStream?: MediaStream;
+
+  constructor(microphoneStream?: MediaStream) {
+    super();
+    if (microphoneStream) {
+      this._microphoneStream = this._microphoneStream;
+    }
+  }
+
   public play(): void {
     // Let Frigate card control auto playing.
   }
 
+  protected reconnect(): void {
+    if (this.wsState !== WebSocket.CLOSED) {
+      // The websocket has onclose handlers, so don't call onconnect directly,
+      // wait until the eventhandlers are finished..
+      this.ws?.addEventListener('close', () => this.onconnect());
+      this.ondisconnect();
+    } else {
+      // Still call ondisconnect() as there may be an RTC connection to
+      // terminate even if the websocket is closed.
+      this.ondisconnect();
+      this.onconnect();
+    }
+  }
+
+  public async setMicrophoneStream(microphoneStream?: MediaStream): Promise<void> {
+    if (this._microphoneStream !== microphoneStream) {
+      this._microphoneStream = microphoneStream;
+      this.reconnect();
+    }
+  }
+
   public oninit(): void {
     super.oninit();
 
@@ -56,6 +87,85 @@ class FrigateCardGo2RTCPlayer extends VideoRTC {
       this.video.muted = true;
     }
   }
+
+  // This is a modified version of onwebrtc() to support 2-way audio.
+  override onwebrtc() {
+    const pc = new RTCPeerConnection(this.pcConfig);
+
+    const video2: HTMLVideoElement = document.createElement('video');
+    video2.addEventListener('loadeddata', (ev) => this.onpcvideo(ev), { once: true });
+
+    pc.addEventListener('icecandidate', (ev) => {
+      const candidate = ev.candidate ? ev.candidate.toJSON().candidate : '';
+      this.send({ type: 'webrtc/candidate', value: candidate });
+    });
+
+    pc.addEventListener('track', (ev) => {
+      // when stream already init
+      if (video2.srcObject !== null) return;
+
+      // when audio track not exist in Chrome
+      if (ev.streams.length === 0) return;
+
+      // when audio track not exist in Firefox
+      if (ev.streams[0].id[0] === '{') return;
+
+      // Filter out tracks that are not video related.
+      if (ev.track.kind !== 'video') return;
+
+      video2.srcObject = ev.streams[0];
+    });
+
+    pc.addEventListener('connectionstatechange', () => {
+      if (pc.connectionState === 'failed' || pc.connectionState === 'disconnected') {
+        pc.close(); // stop next events
+
+        this.pcState = WebSocket.CLOSED;
+        this.pc = null;
+
+        this.onconnect();
+      }
+    });
+
+    this.onmessage['webrtc'] = (msg) => {
+      switch (msg.type) {
+        case 'webrtc/candidate':
+          pc.addIceCandidate({
+            candidate: msg.value,
+            sdpMid: '0',
+          }).catch(() => console.debug);
+          break;
+        case 'webrtc/answer':
+          pc.setRemoteDescription({
+            type: 'answer',
+            sdp: msg.value,
+          }).catch(() => console.debug);
+          break;
+        case 'error':
+          if (msg.value.indexOf('webrtc/offer') < 0) return;
+          pc.close();
+      }
+    };
+
+    // Safari doesn't support "offerToReceiveVideo"
+    pc.addTransceiver('video', { direction: 'recvonly' });
+    pc.addTransceiver('audio', { direction: 'recvonly' });
+
+    // Must add microphone tracks prior to making the offer.
+    this._microphoneStream?.getTracks().forEach((track) => {
+      console.info('Adding microphone track', track);
+      pc.addTransceiver(track, { direction: 'sendonly' });
+    });
+
+    pc.createOffer().then((offer) => {
+      pc.setLocalDescription(offer).then(() => {
+        this.send({ type: 'webrtc/offer', value: offer.sdp });
+      });
+    });
+
+    this.pcState = WebSocket.CONNECTING;
+    this.pc = pc;
+  }
 }
 
 @customElement('frigate-card-live-go2rtc')
@@ -69,6 +179,12 @@ export class FrigateCardGo2RTC extends LitElement implements FrigateCardMediaPla
   @property({ attribute: false })
   public cameraEndpoints?: CameraEndpoints;
 
+  @property({ attribute: false })
+  public microphoneStream?: MediaStream;
+
+  @property({ attribute: false })
+  public microphoneConfig?: MicrophoneConfig;
+
   protected _player?: FrigateCardGo2RTCPlayer;
 
   public async play(): Promise<void> {
@@ -141,7 +257,7 @@ export class FrigateCardGo2RTC extends LitElement implements FrigateCardMediaPla
       return;
     }
 
-    this._player = new FrigateCardGo2RTCPlayer();
+    this._player = new FrigateCardGo2RTCPlayer(this.microphoneStream);
     this._player.src = address;
     this._player.visibilityCheck = false;
 
@@ -156,6 +272,9 @@ export class FrigateCardGo2RTC extends LitElement implements FrigateCardMediaPla
     if (!this._player || changedProps.has('cameraEndpoints')) {
       this._createPlayer();
     }
+    if (changedProps.has('microphoneStream')) {
+      this._player?.setMicrophoneStream(this.microphoneStream);
+    }
   }
 
   protected render(): TemplateResult | void {

src/components/live/live.ts

@@ -138,6 +138,9 @@ export class FrigateCardLive extends LitElement {
   @property({ attribute: false })
   public cardWideConfig?: CardWideConfig;
 
+  @property({ attribute: false })
+  public microphoneStream?: MediaStream;
+
   // Whether or not the live view is currently in the background (i.e. preloaded
   // but not visible)
   @state()
@@ -251,6 +254,7 @@ export class FrigateCardLive extends LitElement {
           .liveOverrides=${this.liveOverrides}
           .cardWideConfig=${this.cardWideConfig}
           .cameraManager=${this.cameraManager}
+          .microphoneStream=${this.microphoneStream}
           @frigate-card:message=${(ev: CustomEvent<Message>) => {
             this._renderKey++;
             this._messageReceivedPostRender = true;
@@ -315,6 +319,9 @@ export class FrigateCardLiveCarousel extends LitElement {
   @property({ attribute: false })
   public cameraManager?: CameraManager;
 
+  @property({ attribute: false })
+  public microphoneStream?: MediaStream;
+
   // Index between camera name and slide number.
   protected _cameraToSlide: Record<string, number> = {};
   protected _refMediaCarousel: Ref<FrigateCardMediaCarousel> = createRef();
@@ -541,6 +548,7 @@ export class FrigateCardLiveCarousel extends LitElement {
       <div class="embla__slide">
         <frigate-card-live-provider
           ?disabled=${this.liveConfig.lazy_load}
+          .microphoneStream=${this.view?.camera === cameraID ? this.microphoneStream : undefined}
           .cameraConfig=${cameraConfig}
           .cameraEndpoints=${guard(
             [this.cameraManager, cameraID],
@@ -722,6 +730,9 @@ export class FrigateCardLiveProvider
   @property({ attribute: false })
   public cardWideConfig?: CardWideConfig;
 
+  @property({ attribute: false })
+  public microphoneStream?: MediaStream;
+
   @state()
   protected _isVideoMediaLoaded = false;
 
@@ -933,6 +944,8 @@ export class FrigateCardLiveProvider
               .hass=${this.hass}
               .cameraConfig=${this.cameraConfig}
               .cameraEndpoints=${this.cameraEndpoints}
+              .microphoneStream=${this.microphoneStream}
+              .microphoneConfig=${this.liveConfig.microphone}
               @frigate-card:media:loaded=${this._videoMediaShowHandler.bind(this)}
             >
             </frigate-card-live-webrtc-card>`

src/components/views.ts

@@ -48,6 +48,9 @@ export class FrigateCardViews extends LitElement {
   @property({ attribute: false })
   public hide?: boolean;
 
+  @property({ attribute: false })
+  public microphoneStream?: MediaStream;
+
   protected willUpdate(changedProps: PropertyValues): void {
     if (changedProps.has('view') || changedProps.has('config')) {
       if (this.view?.is('live') || this._shouldLivePreload()) {
@@ -204,6 +207,7 @@ export class FrigateCardViews extends LitElement {
                 .liveOverrides=${getOverridesByKey(this.config.overrides, 'live')}
                 .cameraManager=${this.cameraManager}
                 .cardWideConfig=${this.cardWideConfig}
+                .microphoneStream=${this.microphoneStream}
                 class="${classMap(liveClasses)}"
               >
               </frigate-card-live>