diff --git a/docs/configuration/cameras/live-provider.md b/docs/configuration/cameras/live-provider.md index 8a2ef337..05d9ff96 100644 --- a/docs/configuration/cameras/live-provider.md +++ b/docs/configuration/cameras/live-provider.md @@ -42,7 +42,12 @@ cameras: > microphone button is intermittently missing on load, try increasing > `metadata_fetch_timeout_seconds` or use > [`capabilities.force`](./README.md?id=capabilities) to skip metadata -> detection entirely. +> detection entirely. If it is _never_ present, check the `go2rtc` stream +> itself: Frigate [recommends](https://docs.frigate.video/configuration/live/) +> adding `#backchannel=0` to the stream it restreams, which stops that stream +> offering 2-way audio at all. Point `stream` at a talk-capable stream instead. +> Note that _any_ `#` option on an RTSP source has this effect unless it also +> includes `backchannel=1`. ## `go2rtc (experimental)` diff --git a/docs/images/call-sequence.svg b/docs/images/call-sequence.svg index 6ba7cab2..6b3d19dd 100644 --- a/docs/images/call-sequence.svg +++ b/docs/images/call-sequence.svg @@ -1 +1 @@ -UserCallMicrophoneGuestUserUserCallCallMicrophoneMicrophoneGuestGuestopt[live.microphone.always_connectedis enabled]Microphone connection attempted (muted)Connected at card load, before (and independent of) any callMicrophone access grantedalt[Outbound call]Start a callVia Call menu button orcall_startaction.Needs a camera that supports 2-way audio[Inbound call]RingsVia aview.triggers.actions.trigger: calltrigger, e.g. a doorbell.The card rings (live.controls.call.ringtone) until answered,rejected, orlive.controls.call.unanswered_timeout_secondselapsesAnswer the callopt[live.microphone.always_connectednot enabled]Microphone connection attempted (muted)Stays muted by default (live.microphone.auto_unmute)Microphone access grantedThe microphone is connected when the call is answered -- immediatelyfor an outbound call. An inbound call rings without it, so a doorbellcan ring even where microphone access needs a tap to be grantedYou hear the callerInbound audio unmutes if configured (live.auto_unmute)Menu, status bar & next/previous hide (auto_hide)Camera navigation is locked (live.controls.call.lock)Unmute to talkList 'call' inlive.microphone.auto_unmuteto unmute automaticallyloop[Until the call ends]TalkingRespondingEnd the callVia Hang-up button,call_endaction, or -- whenlive.controls.call.lockis disabled -- navigating awayInbound audio mutes (live.auto_mute)Microphone mutes and disconnectsThe browser stops reporting the microphone as in use, unlesslive.microphone.always_connectedholds it open \ No newline at end of file +UserCallMicrophoneGuestUserUserCallCallMicrophoneMicrophoneGuestGuestopt[live.microphone.always_connectedis enabled]Microphone connection attempted (muted)Connected at card load, before (and independent of) any callMicrophone access grantedalt[Outbound call]Start a callVia Call menu button orcall_startaction.Needs a camera that supports 2-way audio[Inbound call]RingsVia aview.triggers.actions.trigger: calltrigger, e.g. a doorbell.The card rings (live.controls.call.ringtone) until answered,rejected, orlive.controls.call.unanswered_timeout_secondselapsesAnswer the callopt[live.microphone.always_connectednot enabled]Microphone connection attempted (muted)Stays muted by default (live.microphone.auto_unmute)Microphone access grantedThe microphone is connected when the call is answered -- immediatelyfor an outbound call. An inbound call rings without it, so a doorbellcan ring even where microphone access needs a tap to be grantedThe card opens an audio connection to the camera and holds it only while thecall is in progress. If the camera has no 2-way audio, or another applicationis already using it, the call fails with an errorYou hear the callerInbound audio unmutes if configured (live.auto_unmute)Menu, status bar & next/previous hide (auto_hide)Camera navigation is locked (live.controls.call.lock)Unmute to talkList 'call' inlive.microphone.auto_unmuteto unmute automaticallyloop[Until the call ends]TalkingRespondingEnd the callVia Hang-up button,call_endaction, or -- whenlive.controls.call.lockis disabled -- navigating awayThe audio connection to the camera closes, releasing it for other applicationsInbound audio mutes (live.auto_mute)Microphone mutes and disconnectsThe browser stops reporting the microphone as in use, unlesslive.microphone.always_connectedholds it open \ No newline at end of file diff --git a/docs/uml/call-sequence.puml b/docs/uml/call-sequence.puml index 06be1ccd..3cf2e24a 100644 --- a/docs/uml/call-sequence.puml +++ b/docs/uml/call-sequence.puml @@ -40,6 +40,12 @@ note over User, Mic can ring even where microphone access needs a tap to be granted end note +note over Card + The card opens an audio connection to the camera and holds it only while the + call is in progress. If the camera has no 2-way audio, or another application + is already using it, the call fails with an error +end note + Card --> User : You hear the caller note over Card Inbound audio unmutes if configured (**""live.auto_unmute""**) @@ -61,6 +67,7 @@ note over User, Card **""live.controls.call.lock""** is disabled -- navigating away end note +note over Card : The audio connection to the camera closes, releasing it for other applications Card --> User : Inbound audio mutes (**""live.auto_mute""**) Card -> Mic : Microphone mutes and disconnects note over Mic diff --git a/docs/usage/2-way-audio.md b/docs/usage/2-way-audio.md index 650ecca0..9f6d07a2 100644 --- a/docs/usage/2-way-audio.md +++ b/docs/usage/2-way-audio.md @@ -17,13 +17,19 @@ challenging. - Only Frigate cameras are supported. - Only the `go2rtc` and `go2rtc-experimental` live providers are supported. -- Only the `webrtc` mode supports 2-way audio. +- The browser must be able to reach `go2rtc` over WebRTC. Outbound audio always + travels on its own WebRTC connection, regardless of what mode is carrying the + video. If your setup supports 2-way audio but detection is intermittent on load: - Increase `cameras[].go2rtc.metadata_fetch_timeout_seconds`. - Or force the capability with `cameras[].capabilities.force: ['2-way-audio']`. +If detection never succeeds for a camera, the `go2rtc` stream itself may not +offer 2-way audio -- see +[`go2rtc` live provider configuration](../configuration/cameras/live-provider.md?id=go2rtc). + ## Example configuration ```yaml @@ -65,10 +71,14 @@ enabled by default and appears in the `live` view whenever the selected camera in the overlay to speak. Both behaviors are configurable via [`live.microphone.auto_unmute`](../configuration/live.md?id=microphone) and [`live.auto_unmute`](../configuration/live.md). -- The camera will always load _without_ the microphone connected, unless the +- The camera loads _without_ the microphone connected, unless the [`always_connected`](../configuration/live.md?id=microphone) microphone option - is set to `true`. On the first call there may be a brief `webrtc` connection - reset to include 2-way audio. + is set to `true`. Starting a call opens a separate connection that carries your + voice to the camera; ending the call closes it. The camera's audio input is + therefore occupied only while a call is in progress, leaving it free for other + applications the rest of the time. Expect under half a second between starting + a call and being audible on a local network, and a little more remotely. The + video keeps playing throughout. - The browser asks for microphone permission when a call needs it. How often it asks depends on the browser: Chrome remembers the choice for the site, Safari asks once per page load, and Firefox asks for every call unless _Remember this diff --git a/src/camera-manager/camera.ts b/src/camera-manager/camera.ts index 8c89e5ab..a48667d5 100644 --- a/src/camera-manager/camera.ts +++ b/src/camera-manager/camera.ts @@ -10,6 +10,7 @@ import { resolveProxyConfig, type EnabledProxyConfig, } from '../config/schema/common/proxy'; +import { getGo2RTCMetadataEndpoint, getGo2RTCStreamEndpoint } from '../go2rtc/endpoint'; import { computeDomain } from '../ha/compute-domain'; import { matchesEventContext, matchesEventData } from '../ha/event-match'; import { getTriggerEventType } from '../ha/get-trigger-event-type'; @@ -32,10 +33,6 @@ import type { CameraProxyConfig, } from './types'; import { getCameraEntityFromConfig } from './utils/camera-entity-from-config'; -import { - getGo2RTCMetadataEndpoint, - getGo2RTCStreamEndpoint, -} from './utils/go2rtc/endpoint'; import { getConfiguredPTZAction } from './utils/ptz'; interface CapabilityOptions { diff --git a/src/camera-manager/frigate/camera.ts b/src/camera-manager/frigate/camera.ts index 3554efac..ea44ecb8 100644 --- a/src/camera-manager/frigate/camera.ts +++ b/src/camera-manager/frigate/camera.ts @@ -3,6 +3,10 @@ import { format } from 'date-fns'; import type { ActionsExecutor } from '../../card-controller/actions/types'; import type { PTZAction, PTZActionPhase } from '../../config/schema/actions/custom/ptz'; import type { CameraConfig } from '../../config/schema/cameras'; +import { + getGo2RTCMetadataEndpoint, + getGo2RTCStreamEndpoint, +} from '../../go2rtc/endpoint'; import type { Entity, EntityRegistryManager } from '../../ha/registry/entity/types'; import type { HomeAssistant } from '../../ha/types'; import { @@ -16,10 +20,6 @@ import { Camera, type CameraInitializationOptions } from '../camera'; import { CameraNoEntityError } from '../error'; import type { CameraEndpoints, CameraEndpointsContext } from '../types'; import { getCameraEntityFromConfig } from '../utils/camera-entity-from-config'; -import { - getGo2RTCMetadataEndpoint, - getGo2RTCStreamEndpoint, -} from '../utils/go2rtc/endpoint'; import { getPTZCapabilitiesFromCameraConfig, mergePTZCapabilities } from '../utils/ptz'; import { getPTZInfo } from './requests'; import { diff --git a/src/card-controller/call/manager.ts b/src/card-controller/call/manager.ts index c658b139..89dcc347 100644 --- a/src/card-controller/call/manager.ts +++ b/src/card-controller/call/manager.ts @@ -1,3 +1,8 @@ +import { createBackchannel } from '../../components-lib/live/backchannel/factory'; +import { + BackchannelError, + type Backchannel, +} from '../../components-lib/live/backchannel/types'; import { createNotificationFromText } from '../../components-lib/notification/factory'; import type { ConditionStateChange } from '../../condition-trigger/conditions/types'; import { localize } from '../../localize/localize'; @@ -16,6 +21,8 @@ export class CallManager { private _ringtone = new Ringtone(); private _unansweredTimer = new Timer(); + private _backchannel: Backchannel | null = null; + // Identifies the current init/uninit cycle so an in-flight `start()` or // `answer()` resuming from its microphone-connect await can detect that its // CallManager was torn down -- or torn down and re-initialized -- while it @@ -200,7 +207,8 @@ export class CallManager { if (call.inbound && !call.answered && timeoutSeconds > 0) { this._unansweredTimer.start(timeoutSeconds, () => this.end()); } - return true; + + return call.answered ? await this._openBackchannel(call, inbound) : true; } // Ends the call and returns to the pre-call view. Returns true iff a call was @@ -242,10 +250,12 @@ export class CallManager { // Replace (don't mutate) so Lit identity checks downstream pick up the // change. The `update()` below forces card.ts to re-render and re-read // `getCall()`, propagating the new session to the carousel. - this._call = { ...call, answered: true }; + const answeredCall = { ...call, answered: true }; + this._call = answeredCall; this._api.getConditionStateManager().setState({ call: 'answered' }); this._api.getCardElementManager().update(); - return true; + + return await this._openBackchannel(answeredCall, false); } // Ends the active call iff every supplied predicate matches the session. @@ -270,20 +280,81 @@ export class CallManager { return this.end(); } - // The microphone could not be used for the call, so it is connected but the - // user cannot be heard. `description` is what the reporting layer knows about - // the failure, when it knows anything. - public reportCallMicrophoneError(targetID: string, description?: string): void { - const call = this._call; + // Opens the backchannel for an answered call. Failing to open it ends + // the call, so the user is never left with call controls when they cannot be + // heard. Returns true iff the call is still running. + private async _openBackchannel(call: CallSession, inbound: boolean): Promise { + const targetID = call.callCameraID ?? call.cameraID; + const hass = this._api.getHASSManager().getHASS(); + const camera = this._api.getCameraManager().getStore().getCamera(targetID); + const stream = this._api.getMicrophoneManager().getStream(); - // A report that no longer matches the call in progress describes an attempt - // the user has already moved past, e.g. the call ended before the provider - // finished reporting. - if (!call || !call.answered || call.cameraID !== targetID) { + const backchannel = + hass && camera + ? createBackchannel(hass, camera, (error) => + this._reportBackchannelLoss(call, error), + ) + : null; + if (!backchannel || !stream) { + this._notifyError('error.call_no_two_way_audio', { inbound }); + this._end(true); + return false; + } + + this._backchannel = backchannel; + + try { + await backchannel.start(stream); + } catch (error: unknown) { + if (this._call !== call) { + return false; + } + this._closeBackchannel(); + this._notifyBackchannelError(error, inbound); + this._end(true); + return false; + } + + if (this._call !== call) { + backchannel.stop(); + return false; + } + return true; + } + + private _closeBackchannel(): void { + this._backchannel?.stop(); + this._backchannel = null; + } + + private _reportBackchannelLoss(call: CallSession, error: BackchannelError): void { + if (this._call !== call) { + return; + } + this._notifyBackchannelError(error, call.inbound); + } + + private _notifyBackchannelError(error: unknown, inbound: boolean): void { + const reason = error instanceof BackchannelError ? error.reason : 'failed'; + + // Abandonment means this manager closed the backchannel itself, because the + // call ended or was replaced. Nothing failed, so there is nothing to report. + if (reason === 'abandoned') { return; } - this._notifyError('error.call_microphone_failed', { context: description }); + const messageKey = + reason === 'no_two_way_audio' + ? 'error.call_no_two_way_audio' + : reason === 'no_microphone' + ? 'error.call_microphone_failed' + : 'error.call_camera_unreachable'; + + this._notifyError(messageKey, { + inbound, + ...(error instanceof BackchannelError && + error.description && { context: error.description }), + }); } // Tears down everything `initialize()` set up: stops any in-flight ringtone @@ -296,6 +367,7 @@ export class CallManager { this._initGeneration.invalidate(); this._ringtone.stop(); this._unansweredTimer.stop(); + this._closeBackchannel(); this._api.getMicrophoneManager().setTransmissionActive(false); if (this._call) { this._call = null; @@ -324,6 +396,8 @@ export class CallManager { this._ringtone.stop(); this._unansweredTimer.stop(); + this._closeBackchannel(); + // Clear the session first: ending the call dispatches a view change, and // the resulting condition-state change must not see this (now-ending) call // and recurse. diff --git a/src/card-controller/microphone-manager.ts b/src/card-controller/microphone-manager.ts index 24bcf06c..b596f9fa 100644 --- a/src/card-controller/microphone-manager.ts +++ b/src/card-controller/microphone-manager.ts @@ -101,8 +101,10 @@ export class MicrophoneManager { // A connect over an existing stream must not leak the tracks of the // stream it replaces. + this._removeEndedListeners(this._stream); this._stopTracks(this._stream); this._stream = stream; + this._addEndedListeners(stream); this._forbidden = false; this._reconcile(); this._setState(); @@ -171,8 +173,31 @@ export class MicrophoneManager { stream?.getTracks().forEach((track) => track.stop()); } + // A device that disappears -- unplugged, or its permission revoked -- ends + // its tracks. Nothing can revive them, so the stream is dropped and the new + // state published, rather than leaving the card reporting a connected + // microphone that captures nothing. `stop()` does not fire this event, so + // releasing the stream cannot re-enter. + private _handleTrackEnded = (): void => { + this._releaseStream(); + this._setState(); + }; + + private _addEndedListeners(stream: MediaStream): void { + stream + .getTracks() + .forEach((track) => track.addEventListener('ended', this._handleTrackEnded)); + } + + private _removeEndedListeners(stream: MediaStream | null): void { + stream + ?.getTracks() + .forEach((track) => track.removeEventListener('ended', this._handleTrackEnded)); + } + private _releaseStream(): void { this._connectGeneration.invalidate(); + this._removeEndedListeners(this._stream); this._stopTracks(this._stream); this._stream = null; } diff --git a/src/card-controller/types.ts b/src/card-controller/types.ts index 826fdd92..0bea5272 100644 --- a/src/card-controller/types.ts +++ b/src/card-controller/types.ts @@ -85,6 +85,7 @@ export interface CardCallAPI { getCardElementManager(): CardElementManager; getConditionStateManager(): ConditionStateManager; getConfigManager(): ConfigManager; + getHASSManager(): HASSManager; getMicrophoneManager(): MicrophoneManager; getNotificationManager(): NotificationManager; getViewManager(): ViewManager; diff --git a/src/card.ts b/src/card.ts index 6212d9fc..f5177c70 100644 --- a/src/card.ts +++ b/src/card.ts @@ -22,7 +22,6 @@ import type { IssueTriggerEventData, } from './card-controller/issues/types.js'; import { resolveAutoHideState, type AutoHideState } from './components-lib/auto-hide.js'; -import type { MicrophoneError } from './components-lib/live/utils/dispatch-microphone-error.js'; import { MenuButtonController } from './components-lib/menu-button-controller'; import './components/effects/effects'; @@ -442,12 +441,6 @@ export class AdvancedCameraCard extends LitElement { detail: { key, ...context }, }: CustomEvent) => this._controller.getIssueManager().resolve(key, context)} - @advanced-camera-card:microphone:error=${({ - detail, - }: CustomEvent) => - this._controller - .getCallManager() - .reportCallMicrophoneError(detail.targetID, detail.description)} @advanced-camera-card:media:loaded=${( ev: CustomEvent, ) => this._controller.getMediaLoadedInfoManager().handleLoadEvent(ev)} diff --git a/src/components-lib/live/backchannel/factory.ts b/src/components-lib/live/backchannel/factory.ts new file mode 100644 index 00000000..6437cf0e --- /dev/null +++ b/src/components-lib/live/backchannel/factory.ts @@ -0,0 +1,25 @@ +import type { Camera } from '../../../camera-manager/camera'; +import type { HomeAssistant } from '../../../ha/types'; +import { + getResolvedLiveProvider, + isGo2RTCLiveProvider, +} from '../../../utils/live-provider'; +import { Go2RTCBackchannel } from './go2rtc'; +import type { Backchannel, BackchannelErrorCallback } from './types'; + +export const createBackchannel = ( + hass: HomeAssistant, + camera: Camera, + errorCallback?: BackchannelErrorCallback, +): Backchannel | null => { + if (!isGo2RTCLiveProvider(getResolvedLiveProvider(camera.getConfig()))) { + return null; + } + + const endpoint = camera.getEndpoints()?.go2rtc; + return endpoint + ? new Go2RTCBackchannel(hass, endpoint, camera.getLiveProxyConfig(), { + errorCallback, + }) + : null; +}; diff --git a/src/components-lib/live/backchannel/go2rtc.ts b/src/components-lib/live/backchannel/go2rtc.ts new file mode 100644 index 00000000..46b05313 --- /dev/null +++ b/src/components-lib/live/backchannel/go2rtc.ts @@ -0,0 +1,337 @@ +import type { EnabledProxyConfig } from '../../../config/schema/common/proxy'; +import { isServerErrorForMode, type Go2RTCMessage } from '../../../go2rtc/messages'; +import { + createBrowserPeerConnection, + GO2RTC_PEER_CONNECTION_CONFIG, + type PeerConnectionFactory, +} from '../../../go2rtc/peer-connection'; +import { SignalingChannel, type WebSocketFactory } from '../../../go2rtc/signaling'; +import { resolveEndpointURL } from '../../../ha/resolve-endpoint'; +import type { HomeAssistant } from '../../../ha/types'; +import type { Endpoint, UnsubscribeCallback } from '../../../types'; +import { getErrorDescription } from '../../../utils/basic'; +import { Generation } from '../../../utils/concurrency/generation'; +import { Timer } from '../../../utils/timer'; +import { convertToWebSocketURL } from '../../../utils/websocket-url'; +import { + BackchannelError, + type Backchannel, + type BackchannelErrorCallback, +} from './types'; + +const BACKCHANNEL_CONNECT_TIMEOUT_SECONDS = 10; + +const toBackchannelError = (error: unknown): BackchannelError => + error instanceof BackchannelError + ? error + : new BackchannelError('failed', getErrorDescription(error) ?? undefined); + +interface PendingStart { + resolve: () => void; + reject: (error: BackchannelError) => void; +} + +export interface Go2RTCBackchannelOptions { + createWebSocket?: WebSocketFactory; + createPeerConnection?: PeerConnectionFactory; + errorCallback?: BackchannelErrorCallback; +} + +// Carries microphone audio to a camera over its own WebRTC connection to +// go2rtc, separate from whatever is carrying video. go2rtc claims the camera's +// audio backchannel when this connection's offer arrives and releases it when +// the connection closes, so the camera is occupied only for the duration of a +// call. +// See: https://github.com/dermotduffy/advanced-camera-card/discussions/2678 +// See: https://github.com/dermotduffy/advanced-camera-card/issues/2691 +export class Go2RTCBackchannel implements Backchannel { + private _hass: HomeAssistant; + private _endpoint: Endpoint; + private _proxyConfig: EnabledProxyConfig | null; + private _options: Go2RTCBackchannelOptions | null; + + private _pc: RTCPeerConnection | null = null; + private _channel: SignalingChannel | null = null; + private _transceiver: RTCRtpTransceiver | null = null; + private _unsubscribeCallbacks: UnsubscribeCallback[] = []; + private _connectTimer = new Timer(); + private _generation = new Generation(); + + private _outboundTrack: MediaStreamTrack | null = null; + + // `start()` cannot determine its own outcome: the camera is only known to be + // reachable once the peer connection reports `connected`, and go2rtc may + // refuse in a message that arrives even later. Whichever handler learns the + // outcome completes `start()` through these. + private _pendingStart: PendingStart | null = null; + + constructor( + hass: HomeAssistant, + endpoint: Endpoint, + proxyConfig?: EnabledProxyConfig, + options?: Go2RTCBackchannelOptions, + ) { + this._hass = hass; + this._endpoint = endpoint; + this._proxyConfig = proxyConfig ?? null; + this._options = options ?? null; + } + + public start(stream: MediaStream): Promise { + this.stop(); + + const generation = this._generation.next(); + + // Both exist before the first await: a timeout cannot end a wait unless the + // promise it rejects has already been created. + return new Promise((resolve, reject) => { + this._pendingStart = { resolve, reject }; + + this._connectTimer.start(BACKCHANNEL_CONNECT_TIMEOUT_SECONDS, () => + this._failStart(generation, new BackchannelError('failed')), + ); + + this._start(stream, generation).catch((error: unknown) => + this._failStart(generation, toBackchannelError(error)), + ); + }); + } + + private async _start(stream: MediaStream, generation: number): Promise { + // A microphone stream carries exactly one audio track. + const track = stream.getAudioTracks()[0] ?? null; + if (!track || track.readyState === 'ended') { + throw new BackchannelError('no_microphone'); + } + + const resolvedURL = await resolveEndpointURL(this._hass, this._endpoint, { + proxyConfig: this._proxyConfig, + proxyEndpointOptions: { websocket: true }, + }); + if (!this._generation.isCurrent(generation)) { + throw new BackchannelError('abandoned'); + } + if (!resolvedURL.success) { + throw new BackchannelError('failed', resolvedURL.error); + } + + const pc = (this._options?.createPeerConnection ?? createBrowserPeerConnection)( + GO2RTC_PEER_CONNECTION_CONFIG, + ); + this._pc = pc; + + // This connection only sends audio. + this._transceiver = pc.addTransceiver(track, { direction: 'sendonly' }); + this._watchTrack(track); + + const channel = new SignalingChannel( + convertToWebSocketURL(resolvedURL.url), + { + openCallback: () => this._negotiate(pc, channel, generation), + disconnectCallback: () => this._handleDisconnect(generation), + }, + { createWebSocket: this._options?.createWebSocket }, + ); + this._channel = channel; + + this._unsubscribeCallbacks.push( + channel.subscribeToMessages((message) => + this._handleMessage(pc, message, generation), + ), + ); + pc.addEventListener('icecandidate', (ev) => { + if (!this._generation.isCurrent(generation)) { + return; + } + // An empty value signals end-of-candidates. + channel.send({ + type: 'webrtc/candidate', + value: ev.candidate ? ev.candidate.candidate : '', + }); + }); + pc.addEventListener('connectionstatechange', () => + this._handleConnectionStateChange(pc, generation), + ); + + channel.connect(); + } + + public async setStream(stream: MediaStream): Promise { + const transceiver = this._transceiver; + + // Nothing to swap onto: the call this belonged to has already ended. + if (!transceiver) { + return; + } + + // A microphone stream carries exactly one audio track. Detaching the sender + // instead would leave the user believing they can be heard. + const track = stream.getAudioTracks()[0] ?? null; + if (!track || track.readyState === 'ended') { + throw new BackchannelError('no_microphone'); + } + + await transceiver.sender.replaceTrack(track); + this._watchTrack(track); + } + + public stop(): void { + this._generation.invalidate(); + this._connectTimer.stop(); + + const pendingStart = this._pendingStart; + this._pendingStart = null; + + this._unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe()); + this._unsubscribeCallbacks = []; + + this._channel?.close(); + this._channel = null; + + this._unwatchTrack(); + + // Closing the peer connection is what makes go2rtc release the camera's + // backchannel. The outbound track keeps running: MicrophoneManager owns the + // microphone and shares it across cameras. + // See https://github.com/dermotduffy/advanced-camera-card/issues/1810 + this._pc?.close(); + this._pc = null; + this._transceiver = null; + + pendingStart?.reject(new BackchannelError('abandoned')); + } + + private _handleTrackEnded = (): void => + this._reportLost(this._generation.current(), new BackchannelError('no_microphone')); + + private _watchTrack(track: MediaStreamTrack): void { + this._unwatchTrack(); + track.addEventListener('ended', this._handleTrackEnded); + this._outboundTrack = track; + } + + private _unwatchTrack(): void { + this._outboundTrack?.removeEventListener('ended', this._handleTrackEnded); + this._outboundTrack = null; + } + + private _reportLost(generation: number, error: BackchannelError): void { + if (this._pendingStart) { + this._failStart(generation, error); + return; + } + this.stop(); + this._options?.errorCallback?.(error); + } + + private async _negotiate( + pc: RTCPeerConnection, + channel: SignalingChannel, + generation: number, + ): Promise { + try { + const offer = await pc.createOffer(); + if (!this._generation.isCurrent(generation)) { + return; + } + await pc.setLocalDescription(offer); + if (!this._generation.isCurrent(generation)) { + return; + } + channel.send({ type: 'webrtc/offer', value: offer.sdp ?? '' }); + } catch (error: unknown) { + this._failStart( + generation, + new BackchannelError('failed', getErrorDescription(error) ?? undefined), + ); + } + } + + private _handleMessage( + pc: RTCPeerConnection, + message: Go2RTCMessage, + generation: number, + ): void { + // go2rtc refuses a stream it cannot send audio to with an error frame + // rather than an answer. + if (isServerErrorForMode(message, 'webrtc')) { + this._failStart( + generation, + new BackchannelError('no_two_way_audio', message.value), + ); + return; + } + + if (typeof message.value !== 'string') { + return; + } + + switch (message.type) { + case 'webrtc/answer': + pc.setRemoteDescription({ type: 'answer', sdp: message.value }).catch( + (error: unknown) => + this._failStart( + generation, + new BackchannelError('failed', getErrorDescription(error) ?? undefined), + ), + ); + break; + + case 'webrtc/candidate': + if (message.value) { + // The server sends no sdpMid; max-bundle puts every track on m-line 0. + pc.addIceCandidate({ candidate: message.value, sdpMid: '0' }).catch(() => {}); + } + break; + } + } + + private _handleDisconnect(generation: number): void { + // This socket is closed deliberately once the peer connection is + // established, so a close arriving here is always premature: the offer and + // answer travel over it and cannot complete without it. + this._failStart(generation, new BackchannelError('failed')); + } + + private _handleConnectionStateChange(pc: RTCPeerConnection, generation: number): void { + if (!this._generation.isCurrent(generation)) { + return; + } + + if (pc.connectionState === 'connected') { + const direction = this._transceiver?.currentDirection; + if (direction !== 'sendonly' && direction !== 'sendrecv') { + this._failStart(generation, new BackchannelError('no_two_way_audio')); + return; + } + + this._connectTimer.stop(); + + // Signaling is finished, and the camera's backchannel belongs to the peer + // connection rather than to this socket. Leaving it open would let a + // proxy drop it as idle much later, which `_handleDisconnect` reports as + // a failure. + this._channel?.close(); + this._channel = null; + + this._pendingStart?.resolve(); + this._pendingStart = null; + } else if (pc.connectionState === 'failed') { + this._reportLost(generation, new BackchannelError('failed')); + } + } + + // Aborts an in-progress `start()`. Every failure before `start()` resolves + // arrives here; afterwards there is no request left to fail. + private _failStart(generation: number, error: BackchannelError): void { + if (!this._generation.isCurrent(generation)) { + return; + } + // Taken before the teardown so this reason is reported rather than the + // abandonment `stop()` would otherwise report. + const pendingStart = this._pendingStart; + this._pendingStart = null; + this.stop(); + pendingStart?.reject(error); + } +} diff --git a/src/components-lib/live/backchannel/types.ts b/src/components-lib/live/backchannel/types.ts new file mode 100644 index 00000000..0a1b89da --- /dev/null +++ b/src/components-lib/live/backchannel/types.ts @@ -0,0 +1,33 @@ +export type BackchannelFailureReason = + | 'no_two_way_audio' + | 'no_microphone' + | 'failed' + | 'abandoned'; + +export class BackchannelError extends Error { + public readonly reason: BackchannelFailureReason; + public readonly description: string | null; + + constructor(reason: BackchannelFailureReason, description?: string) { + super(description ?? reason); + this.reason = reason; + this.description = description ?? null; + } +} + +// Carries audio from the browser microphone to a camera, held for the +// duration of a call. Implementations are per live provider. +export interface Backchannel { + // Opens the backchannel carrying the given microphone stream. Resolves once the + // camera can actually be spoken to. + start(stream: MediaStream): Promise; + + // Swaps the microphone stream being carried, leaving the backchannel open. + setStream(stream: MediaStream): Promise; + + // Closes the backchannel, releasing the camera (the microphone itself belongs to + // MicrophoneManager and is untouched). + stop(): void; +} + +export type BackchannelErrorCallback = (error: BackchannelError) => void; diff --git a/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg b/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg index df3bfa93..3a7412b0 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg +++ b/src/components-lib/live/providers/go2rtc-experimental/architecture.drawio.svg @@ -1,4 +1,4 @@ -go2rtc-experimental provider -- architecture
<advanced-camera-card-live-go2rtc-experimental>
(components/live/providers/go2rtc-experimental)
Web component -- renders a <video> and an <img> (two surfaces), owns the session
Go2RTCSessionController (session-controller.ts)
Owns the signaling channel; starts both lanes; arbitrates the race
and commits the winner to the live surface (<video> or <img>)
via surfaceCommittedCallback. RetryTimer -- bounded reconnects.
BINARY lane
(binary frames on the WebSocket)
WEBRTC lane
(media over an RTCPeerConnection)
sources/factory.ts
createBinarySource(mode)
MSEStreamSource (sources/mse.ts)
MP4StreamSource (sources/mp4.ts)
MJPEGStreamSource (sources/mjpeg.ts)
MP4 & MJPEG extend ImageFrameStreamSource
sources/factory.ts
createWebRTCSource()
WebRTCStreamSource (sources/webrtc.ts)
negotiates SDP / ICE over the socket;
media rides an RTCPeerConnection
SignalingChannel (signaling.ts) -- one WebSocket to go2rtc
JSON text frames to message subscribers
binary frames to the single binary consumer
go2rtc server
adapters/media-source.ts
createBrowserMediaSource
(MSE / ManagedMediaSource)
utils/ (pure helpers)
codecs · live-edge-tracker · bounded-buffer-queue
base64 · user-agent · messages (binary sources)
webrtc-sdp · messages (WebRTC)
source-priority (the race)
<video> surface
(the live video element)
<img> surface
(ImageSurfaceController)
getPreferredSource (utils/source-priority.ts)
video > audio(AAC) > H.265, else WebRTC
OffscreenVideo (offscreen-video.ts)
off-screen <video> decode / probe
adapters/peer-connection.ts
createPeerConnection
(RTCPeerConnection)
connect(url, surfaces, modes)media-loadedinfo / error (escalate)start (sequential)start (parallel)fallbackfallbacknegotiate (JSON) + media (binary)signaling (JSON) onlyWebSocketmedia (RTP / SRTP)rendersrendersMP4 / MJPEGframesbinary profiledecodes intowebrtc profilewinner: MSE orWebRTC \ No newline at end of file +go2rtc-experimental provider -- architecture
<advanced-camera-card-live-go2rtc-experimental>
(components/live/providers/go2rtc-experimental)
Web component -- renders a <video> and an <img> (two surfaces), owns the session
Go2RTCSessionController (session-controller.ts)
Owns the signaling channel; starts both lanes; arbitrates the race
and commits the winner to the live surface (<video> or <img>)
via surfaceCommittedCallback. RetryTimer -- bounded reconnects.
BINARY lane
(binary frames on the WebSocket)
WEBRTC lane
(media over an RTCPeerConnection)
sources/factory.ts
createBinarySource(mode)
MSEStreamSource (sources/mse.ts)
MP4StreamSource (sources/mp4.ts)
MJPEGStreamSource (sources/mjpeg.ts)
MP4 & MJPEG extend ImageFrameStreamSource
sources/factory.ts
createWebRTCSource()
WebRTCStreamSource (sources/webrtc.ts)
negotiates SDP / ICE over the socket;
media rides an RTCPeerConnection
SignalingChannel (go2rtc/signaling.ts) -- one WebSocket to go2rtc
JSON text frames to message subscribers
binary frames to the single binary consumer
go2rtc server
adapters/media-source.ts
createBrowserMediaSource
(MSE / ManagedMediaSource)
utils/ (pure helpers)
codecs · live-edge-tracker · bounded-buffer-queue
base64 · user-agent (binary sources)
webrtc-sdp (WebRTC)
source-priority (the race)
<video> surface
(the live video element)
<img> surface
(ImageSurfaceController)
getPreferredSource (utils/source-priority.ts)
video > audio(AAC) > H.265, else WebRTC
OffscreenVideo (offscreen-video.ts)
off-screen <video> decode / probe
go2rtc/peer-connection.ts
createPeerConnection
(RTCPeerConnection)
connect(url, surfaces, modes)media-loadedinfo / error (escalate)start (sequential)start (parallel)fallbackfallbacknegotiate (JSON) + media (binary)signaling (JSON) onlyWebSocketmedia (RTP / SRTP)rendersrendersMP4 / MJPEGframesbinary profiledecodes intowebrtc profilewinner: MSE orWebRTC \ No newline at end of file diff --git a/src/components-lib/live/providers/go2rtc-experimental/session-controller.ts b/src/components-lib/live/providers/go2rtc-experimental/session-controller.ts index 3d839e67..efca5c23 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/session-controller.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/session-controller.ts @@ -2,6 +2,8 @@ import { isEqual } from 'lodash-es'; import { GO2RTC_MODES, type Go2RTCMode } from '../../../../config/schema/cameras'; import type { CardWideConfig } from '../../../../config/schema/types'; +import type { PeerConnectionFactory } from '../../../../go2rtc/peer-connection'; +import { SignalingChannel, type WebSocketFactory } from '../../../../go2rtc/signaling'; import type { MediaPlayerController, UntargetedMediaLoadedInfo, @@ -19,9 +21,7 @@ import { createMediaLoadedInfo } from '../../../../utils/media-info'; import { RetryTimer } from '../../../../utils/retry-timer'; import { convertToWebSocketURL } from '../../../../utils/websocket-url'; import type { MediaSourceFactory } from './adapters/media-source'; -import type { PeerConnectionFactory } from './adapters/peer-connection'; import { OffscreenVideo, type VideoElementFactory } from './offscreen-video'; -import { SignalingChannel, type WebSocketFactory } from './signaling'; import { createBinarySource, createWebRTCSource, @@ -97,11 +97,6 @@ interface Go2RTCSessionCallbacks { // The reason is the most recent source failure, or null when there is none // (e.g. the socket dropped with no source having reported a cause). streamErrorCallback: (reason: StreamSourceFailureReason | null) => void; - - // The outbound microphone could not be used, so the camera cannot be talked - // to. The inbound video is unaffected. `error` is what the source knows about - // the failure, when it knows anything. - microphoneErrorCallback: (error?: string) => void; } // Injectable platform and factory seams for tests. Every field defaults to @@ -141,7 +136,6 @@ export class Go2RTCSessionController { private _url: string | null = null; private _surfaces: SessionSurfaces | null = null; private _modes: readonly Go2RTCMode[] = GO2RTC_MODES; - private _microphoneStream: MediaStream | null = null; // Binary lane: the active source paired with the surface it renders on (mse // -> video, MP4/MJPEG -> image). Kept as one unit because the factory returns @@ -249,11 +243,6 @@ export class Go2RTCSessionController { this._surfaces = null; } - public setMicrophoneStream(stream: MediaStream | null): void { - this._microphoneStream = stream; - this._webRTCSource?.setMicrophoneStream(stream).catch(() => {}); - } - // =========================================================================== // Session teardown. // =========================================================================== @@ -462,8 +451,6 @@ export class Go2RTCSessionController { }; source = (this._options?.createWebRTCSource ?? createWebRTCSource)(sourceContext, { - microphoneStream: this._microphoneStream, - microphoneErrorCallback: (error) => this._callbacks.microphoneErrorCallback(error), createPeerConnection: this._options?.createPeerConnection, createMediaStream: this._options?.createMediaStream, }); diff --git a/src/components-lib/live/providers/go2rtc-experimental/sources/factory.ts b/src/components-lib/live/providers/go2rtc-experimental/sources/factory.ts index 6d082216..e11f2a28 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/sources/factory.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/sources/factory.ts @@ -1,6 +1,6 @@ import type { Go2RTCMode } from '../../../../../config/schema/cameras'; +import type { PeerConnectionFactory } from '../../../../../go2rtc/peer-connection'; import type { MediaSourceFactory } from '../adapters/media-source'; -import type { PeerConnectionFactory } from '../adapters/peer-connection'; import type { ImageStreamTarget, StreamSource, @@ -88,8 +88,6 @@ export const createBinarySource: BinarySourceFactory = ( export interface CreateWebRTCSourceOptions { createPeerConnection?: PeerConnectionFactory; createMediaStream?: MediaStreamFactory; - microphoneStream?: MediaStream | null; - microphoneErrorCallback?: (error?: string) => void; } export type WebRTCSourceFactory = ( diff --git a/src/components-lib/live/providers/go2rtc-experimental/sources/image-frame.ts b/src/components-lib/live/providers/go2rtc-experimental/sources/image-frame.ts index 899bd77c..84a45825 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/sources/image-frame.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/sources/image-frame.ts @@ -1,4 +1,8 @@ import type { Go2RTCMode } from '../../../../../config/schema/cameras'; +import { + isServerErrorForMode, + type Go2RTCMessage, +} from '../../../../../go2rtc/messages'; import type { MediaLoadedCapabilities, MediaTechnology, @@ -6,13 +10,11 @@ import type { } from '../../../../../types'; import { Timer } from '../../../../../utils/timer'; import type { - Go2RTCMessage, ImageStreamTarget, StreamProfile, StreamSource, StreamSourceContext, } from '../types'; -import { isServerErrorForMode } from '../utils/messages'; // Fail if no frame arrives within this window. The channel is open and the mode // was requested, but the server may send neither a frame nor an error, so diff --git a/src/components-lib/live/providers/go2rtc-experimental/sources/mjpeg.ts b/src/components-lib/live/providers/go2rtc-experimental/sources/mjpeg.ts index 51ac1807..c7316e16 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/sources/mjpeg.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/sources/mjpeg.ts @@ -1,4 +1,4 @@ -import type { Go2RTCMessage } from '../types'; +import type { Go2RTCMessage } from '../../../../../go2rtc/messages'; import { ImageFrameStreamSource } from './image-frame'; // Each binary frame is a complete JPEG, shown directly as an image frame. diff --git a/src/components-lib/live/providers/go2rtc-experimental/sources/mp4.ts b/src/components-lib/live/providers/go2rtc-experimental/sources/mp4.ts index 7a56e744..30e1c32d 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/sources/mp4.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/sources/mp4.ts @@ -1,5 +1,6 @@ +import type { Go2RTCMessage } from '../../../../../go2rtc/messages'; import { OffscreenVideo, type VideoElementFactory } from '../offscreen-video'; -import type { Go2RTCMessage, ImageStreamTarget, StreamSourceContext } from '../types'; +import type { ImageStreamTarget, StreamSourceContext } from '../types'; import { arrayBufferToBase64 } from '../utils/base64'; import { convertToCodecString, diff --git a/src/components-lib/live/providers/go2rtc-experimental/sources/mse.ts b/src/components-lib/live/providers/go2rtc-experimental/sources/mse.ts index 8178741a..d6482a73 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/sources/mse.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/sources/mse.ts @@ -1,3 +1,7 @@ +import { + isServerErrorForMode, + type Go2RTCMessage, +} from '../../../../../go2rtc/messages'; import type { MediaLoadedCapabilities, MediaTechnology, @@ -11,7 +15,6 @@ import { type MediaSourceInterface, } from '../adapters/media-source'; import type { - Go2RTCMessage, StreamProfile, StreamSource, StreamSourceContext, @@ -25,7 +28,6 @@ import { } from '../utils/codecs'; import { LiveEdgeTracker } from '../utils/live-edge-tracker'; import type { LiveEdgeAction } from '../utils/live-edge-tracker/types'; -import { isServerErrorForMode } from '../utils/messages'; import { isWebKitUserAgent } from '../utils/user-agent'; // =========================================================================== @@ -165,7 +167,6 @@ export class MSEStreamSource implements StreamSource { return { supportsPause: true, hasAudio: hasAudio(this._context.target.video, { mseCodecs: this._codecs }), - has2WayAudio: false, }; } diff --git a/src/components-lib/live/providers/go2rtc-experimental/sources/webrtc.ts b/src/components-lib/live/providers/go2rtc-experimental/sources/webrtc.ts index 0cc0ee52..bb859621 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/sources/webrtc.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/sources/webrtc.ts @@ -1,24 +1,25 @@ +import { + isServerErrorForMode, + type Go2RTCMessage, +} from '../../../../../go2rtc/messages'; +import { + createBrowserPeerConnection, + GO2RTC_PEER_CONNECTION_CONFIG, + type PeerConnectionFactory, +} from '../../../../../go2rtc/peer-connection'; import type { MediaLoadedCapabilities, MediaTechnology, UnsubscribeCallback, } from '../../../../../types'; -import { has2WayAudio, hasAudio } from '../../../../../utils/audio'; -import { isRecord } from '../../../../../utils/basic'; +import { hasAudio } from '../../../../../utils/audio'; import { Timer } from '../../../../../utils/timer'; -import { - createBrowserPeerConnection, - GO2RTC_PEER_CONNECTION_CONFIG, - type PeerConnectionFactory, -} from '../adapters/peer-connection'; import type { - Go2RTCMessage, StreamProfile, StreamSource, StreamSourceContext, VideoStreamTarget, } from '../types'; -import { isServerErrorForMode } from '../utils/messages'; import { sdpHasH265 } from '../utils/webrtc-sdp'; // =========================================================================== @@ -39,32 +40,9 @@ const WEBRTC_CONNECT_TIMEOUT_SECONDS = 5; export type MediaStreamFactory = (tracks: MediaStreamTrack[]) => MediaStream; -// What a thrown value has to say for itself, preferring the browser's sentence -// ("The peer connection is closed") over the bare type name -// ("InvalidStateError"), which means nothing to the person reading it. -// -// DOMException may not inherit from Error, and catch blocks may be handed -// anything, so extract details structurally rather than using `instanceof -// Error`. -const getErrorDescription = (error: unknown): string | null => { - if (!isRecord(error)) { - return null; - } - const message = typeof error.message === 'string' ? error.message : ''; - const name = typeof error.name === 'string' ? error.name : ''; - return message || name || null; -}; - interface WebRTCStreamSourceOptions { createPeerConnection?: PeerConnectionFactory; createMediaStream?: MediaStreamFactory; - microphoneStream?: MediaStream | null; - - // The outbound microphone track could not be attached. Separate from the - // stream-source failure channel: a microphone that cannot attach says nothing - // about the inbound video which keeps playing. `error` is what the browser - // said went wrong, when it said anything. - microphoneErrorCallback?: (error?: string) => void; } export class WebRTCStreamSource implements StreamSource { @@ -74,10 +52,6 @@ export class WebRTCStreamSource implements StreamSource { private _createPeerConnection: PeerConnectionFactory; private _createMediaStream: MediaStreamFactory; - private _microphoneStream: MediaStream | null; - private _microphoneErrorCallback: ((error?: string) => void) | null; - - private _microphoneTransceiver: RTCRtpTransceiver | null = null; private _connectTimer = new Timer(); @@ -97,23 +71,12 @@ export class WebRTCStreamSource implements StreamSource { options?.createPeerConnection ?? createBrowserPeerConnection; this._createMediaStream = options?.createMediaStream ?? ((tracks) => new MediaStream(tracks)); - - this._microphoneStream = options?.microphoneStream ?? null; - this._microphoneErrorCallback = options?.microphoneErrorCallback ?? null; } public start(): void { const pc = this._createPeerConnection(GO2RTC_PEER_CONNECTION_CONFIG); this._pc = pc; - // Always pre-arm exactly one outbound audio slot so the microphone track - // can be attached later via `replaceTrack` with no renegotiation. The - // kind-only `addTransceiver('audio', ...)` form never calls getUserMedia, - // so it never raises permission prompt for users. - const microphoneTrack = this._microphoneStream?.getAudioTracks()[0] ?? null; - this._microphoneTransceiver = pc.addTransceiver(microphoneTrack ?? 'audio', { - direction: 'sendonly', - }); pc.addTransceiver('video', { direction: 'recvonly' }); pc.addTransceiver('audio', { direction: 'recvonly' }); @@ -162,17 +125,10 @@ export class WebRTCStreamSource implements StreamSource { this._context.target.video.removeEventListener('loadeddata', this._loadedHandler); if (this._pc) { - // pc.close() does not stop the sender's tracks, so the outbound microphone - // track keeps running. That is deliberate: MicrophoneManager owns the mic - // (it is shared across cameras), so stopping it here would break it - // elsewhere -- do not add a track.stop() here. See - // https://github.com/dermotduffy/advanced-camera-card/issues/1810 this._pc.close(); this._pc = null; } - // The transceiver belonged to the now-closed peer connection. - this._microphoneTransceiver = null; this._context.target.video.srcObject = null; this._stream = null; } @@ -189,7 +145,6 @@ export class WebRTCStreamSource implements StreamSource { return { supportsPause: true, hasAudio: hasAudio(this._context.target.video, { pc: this._pc }), - has2WayAudio: has2WayAudio(this._pc), }; } @@ -208,45 +163,6 @@ export class WebRTCStreamSource implements StreamSource { }; } - // Swap the outbound microphone track without renegotiating. - public async setMicrophoneStream(stream: MediaStream | null): Promise { - if (this._microphoneStream === stream) { - return; - } - this._microphoneStream = stream; - - const transceiver = this._microphoneTransceiver; - if (!transceiver) { - // No peer connection yet; the next `start()` reads the current stream and - // pre-arms the transceiver with it. - return; - } - - // Whether the awaited microphone request is still the one in effect: a newer - // stream, or teardown, retires it, and reporting a retired outcome would - // describe something that is no longer being attempted. - const isCurrentRequest = ( - transceiver: RTCRtpTransceiver, - stream: MediaStream | null, - ): boolean => - transceiver === this._microphoneTransceiver && - this._microphoneStream === stream && - this._pc !== null; - - // A microphone stream carries a single audio track; null detaches the sender. - const desiredTrack = stream?.getAudioTracks()[0] ?? null; - try { - await transceiver.sender.replaceTrack(desiredTrack); - } catch (error) { - // Only a failed attach is reported. A failed detach leaves nothing for - // the user to act on: the track stops being transmitted when the peer - // connection closes. - if (desiredTrack && isCurrentRequest(transceiver, stream)) { - this._microphoneErrorCallback?.(getErrorDescription(error) ?? undefined); - } - } - } - private async _negotiate(pc: RTCPeerConnection): Promise { const offer = await pc.createOffer(); if (this._pc !== pc) { diff --git a/src/components-lib/live/providers/go2rtc-experimental/types.ts b/src/components-lib/live/providers/go2rtc-experimental/types.ts index e3be13f0..f7d55ad0 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/types.ts +++ b/src/components-lib/live/providers/go2rtc-experimental/types.ts @@ -1,30 +1,14 @@ -import { z } from 'zod'; - +import type { + BinaryCallback, + Go2RTCMessage, + MessageCallback, +} from '../../../../go2rtc/messages'; import type { MediaLoadedCapabilities, MediaTechnology, UnsubscribeCallback, } from '../../../../types'; -// =========================================================================== -// Control messages -// =========================================================================== - -// go2rtc control messages are JSON text frames of this shape; media flows as -// separate binary frames. -export const go2RTCMessageSchema = z.object({ - type: z.string(), - - // Per-type payload (a codec list, an SDP, an ICE candidate, error text, ...): - // absent for some types (e.g. mjpeg) and not always a string, so it is typed - // `unknown` and each handler narrows it before use. - value: z.unknown().optional(), -}); -export type Go2RTCMessage = z.infer; - -export type MessageCallback = (message: Go2RTCMessage) => void; -export type BinaryCallback = (data: ArrayBuffer) => void; - // =========================================================================== // Signaling channel // =========================================================================== diff --git a/src/components-lib/live/providers/go2rtc-experimental/utils/messages.ts b/src/components-lib/live/providers/go2rtc-experimental/utils/messages.ts deleted file mode 100644 index 6a117dca..00000000 --- a/src/components-lib/live/providers/go2rtc-experimental/utils/messages.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { Go2RTCMessage } from '../types'; - -// The go2rtc server reports a mode failure as `{ type: 'error', value: ': ...' }` -// (e.g. `mse: stream not found`), so an error is for a given mode when its value -// starts with that mode's name. -export const isServerErrorForMode = (message: Go2RTCMessage, mode: string): boolean => - message.type === 'error' && - typeof message.value === 'string' && - message.value.startsWith(mode); diff --git a/src/components-lib/live/utils/dispatch-microphone-error.ts b/src/components-lib/live/utils/dispatch-microphone-error.ts deleted file mode 100644 index 4d1f7c02..00000000 --- a/src/components-lib/live/utils/dispatch-microphone-error.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { fireAdvancedCameraCardEvent } from '../../../utils/fire-advanced-camera-card-event'; - -// What a provider knows about its own microphone related failure. Stream -// otherwise not impacted (contrast with `live:error`: which marks the whole -// stream not live). -export interface MicrophoneError { - // The base camera the provider is rendering. Carried because this event is - // handled once for the whole card, unlike `live:error` which is caught and - // stopped on the camera's own provider wrapper and so needs no camera named. - targetID: string; - - // Free text naming the specific failure, when the provider has one. - description?: string; -} - -declare global { - interface HTMLElementEventMap { - 'advanced-camera-card:microphone:error': CustomEvent; - } -} - -export function dispatchMicrophoneErrorEvent( - element: EventTarget, - error: MicrophoneError, -): void { - fireAdvancedCameraCardEvent(element, 'microphone:error', error); -} diff --git a/src/components-lib/signed-url-controller.ts b/src/components-lib/signed-url-controller.ts index 734ad17c..31f2affd 100644 --- a/src/components-lib/signed-url-controller.ts +++ b/src/components-lib/signed-url-controller.ts @@ -2,19 +2,16 @@ import type { ReactiveController, ReactiveControllerHost } from 'lit'; import { isEqual } from 'lodash-es'; import type { EnabledProxyConfig } from '../config/schema/common/proxy.js'; -import { homeAssistantGetSignedURLIfNecessary } from '../ha/sign-path.js'; -import type { HomeAssistant } from '../ha/types.js'; import { - createProxiedEndpointIfNecessary, - type CreateProxiedEndpointOptions, -} from '../ha/web-proxy.js'; + PROXY_URL_SIGN_EXPIRY_SECONDS, + resolveEndpointURL, +} from '../ha/resolve-endpoint.js'; +import type { HomeAssistant } from '../ha/types.js'; +import type { CreateProxiedEndpointOptions } from '../ha/web-proxy.js'; import { localize } from '../localize/localize.js'; import type { Endpoint } from '../types.js'; -import { errorToConsole } from '../utils/basic.js'; import { Generation } from '../utils/concurrency/generation.js'; -const PROXY_URL_SIGN_EXPIRY_SECONDS = 24 * 60 * 60; - // Re-register and re-sign well before the signed URL expires. const PROXY_CACHE_TTL_SECONDS = PROXY_URL_SIGN_EXPIRY_SECONDS / 2; @@ -146,78 +143,20 @@ export class SignedURLController implements ReactiveController { this._cachedAt = null; const requestID = this._requestGeneration.next(); - const resolvedEndpoint = await this._proxy( + const resolved = await resolveEndpointURL( hass, - targetURL, - endpoint, - proxyConfig, - proxyEndpointOptions, + { endpoint: targetURL, sign: endpoint.sign }, + { proxyConfig, proxyEndpointOptions }, ); if (!this._requestGeneration.isCurrent(requestID)) { return; } - if (!resolvedEndpoint) { - this._applyError('proxy'); + if (!resolved.success) { + this._applyError(resolved.error); return; } - const signedURL = await this._sign(hass, resolvedEndpoint); - if (!this._requestGeneration.isCurrent(requestID)) { - return; - } - if (!signedURL) { - this._applyError('sign'); - return; - } - - this._applySuccess(signedURL); - } - - /** - * Proxy the endpoint if proxying is enabled, otherwise return it as-is. - */ - private async _proxy( - hass: HomeAssistant, - targetURL: string, - endpoint: Endpoint, - proxyConfig: EnabledProxyConfig | null | undefined, - proxyEndpointOptions: CreateProxiedEndpointOptions | undefined, - ): Promise { - if (!proxyConfig?.enabled) { - return { endpoint: targetURL, sign: endpoint.sign }; - } - - try { - return await createProxiedEndpointIfNecessary( - hass, - { endpoint: targetURL, sign: false }, - proxyConfig, - { - ttl: PROXY_URL_SIGN_EXPIRY_SECONDS, - openLimit: 0, - ...proxyEndpointOptions, - }, - ); - } catch (e: unknown) { - errorToConsole(e); - return null; - } - } - - /** - * Sign the endpoint if it requires signing, otherwise return the URL as-is. - */ - private async _sign(hass: HomeAssistant, endpoint: Endpoint): Promise { - try { - return await homeAssistantGetSignedURLIfNecessary( - hass, - endpoint, - PROXY_URL_SIGN_EXPIRY_SECONDS, - ); - } catch (e: unknown) { - errorToConsole(e); - return null; - } + this._applySuccess(resolved.url); } private _applySuccess(url: string): void { diff --git a/src/components/live/carousel.ts b/src/components/live/carousel.ts index fb281364..116ff682 100644 --- a/src/components/live/carousel.ts +++ b/src/components/live/carousel.ts @@ -265,14 +265,11 @@ export class AdvancedCameraCardLiveCarousel extends LitElement { const mediaEpoch = view?.context?.mediaEpoch?.[cameraID] ?? 0; const isSelectedSlide = !!view?.camera && cameraID === view.camera; - const microphoneStream = this._getRelevantMicrophoneStream(cameraID, view); - return html`
${keyed( mediaEpoch, html` ` @@ -443,8 +438,6 @@ export class AdvancedCameraCardLiveProvider extends LitElement implements MediaP .camera=${this.camera} .targetID=${this.targetID} .cameraTitle=${this.cameraTitle} - .microphoneStream=${this.microphoneStream} - .microphoneConfig=${this.liveConfig.microphone} .cardWideConfig=${this.cardWideConfig} ?controls=${this._getEffectiveBuiltinControls()} > diff --git a/src/components/live/providers/go2rtc-experimental/index.ts b/src/components/live/providers/go2rtc-experimental/index.ts index fb31042b..3a3a145e 100644 --- a/src/components/live/providers/go2rtc-experimental/index.ts +++ b/src/components/live/providers/go2rtc-experimental/index.ts @@ -23,14 +23,12 @@ import { import type { SurfaceKind } from '../../../../components-lib/live/providers/go2rtc-experimental/types.js'; import { mapStreamFailureReasonToIssueReason } from '../../../../components-lib/live/providers/go2rtc-experimental/utils/stream-failure-reason.js'; import { dispatchLiveErrorEvent } from '../../../../components-lib/live/utils/dispatch-live-error.js'; -import { dispatchMicrophoneErrorEvent } from '../../../../components-lib/live/utils/dispatch-microphone-error.js'; import { MediaLoadedInfoSourceController } from '../../../../components-lib/media-loaded-info-source-controller.js'; import { VideoMediaPlayerController } from '../../../../components-lib/media-player/video.js'; import { getSignedURLErrorText, SignedURLController, } from '../../../../components-lib/signed-url-controller.js'; -import type { MicrophoneConfig } from '../../../../config/schema/live.js'; import type { CardWideConfig } from '../../../../config/schema/types.js'; import type { HomeAssistant } from '../../../../ha/types.js'; import { localize } from '../../../../localize/localize.js'; @@ -58,12 +56,6 @@ export class AdvancedCameraCardGo2RTCExperimental @property({ attribute: false }) public targetID?: string; - @property({ attribute: false }) - public microphoneStream?: MediaStream | null; - - @property({ attribute: false }) - public microphoneConfig?: MicrophoneConfig; - @property({ attribute: false }) public cardWideConfig?: CardWideConfig; @@ -168,20 +160,8 @@ export class AdvancedCameraCardGo2RTCExperimental this._streamError = mapStreamFailureReasonToIssueReason(reason); dispatchLiveErrorEvent(this, { reason: this._streamError }); }, - - microphoneErrorCallback: (error) => this._reportMicrophoneError(error), }); - private _reportMicrophoneError(error?: string): void { - if (!this.targetID) { - return; - } - dispatchMicrophoneErrorEvent(this, { - targetID: this.targetID, - description: error, - }); - } - public async getMediaPlayerController(): Promise { return this._activeSurface === 'image' ? this._imageSurface.getMediaPlayer() @@ -237,11 +217,6 @@ export class AdvancedCameraCardGo2RTCExperimental // Only the video surface has native controls; the image surface has none. this._videoMediaPlayerController.setControls(this.controls).catch(() => {}); } - - if (changedProps.has('microphoneStream')) { - // The WebRTC lane swaps the outbound track in place; no visible reload. - this._session.setMicrophoneStream(this.microphoneStream ?? null); - } } protected updated(): void { diff --git a/src/components/live/providers/go2rtc/index.ts b/src/components/live/providers/go2rtc/index.ts index 975da358..e84b4f3b 100644 --- a/src/components/live/providers/go2rtc/index.ts +++ b/src/components/live/providers/go2rtc/index.ts @@ -15,7 +15,6 @@ import { getSignedURLErrorText, SignedURLController, } from '../../../../components-lib/signed-url-controller.js'; -import type { MicrophoneConfig } from '../../../../config/schema/live.js'; import type { HomeAssistant } from '../../../../ha/types.js'; import { localize } from '../../../../localize/localize.js'; import liveGo2RTCStyle from '../../../../scss/live-go2rtc.scss?inline'; @@ -37,12 +36,6 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer @property({ attribute: false }) public targetID?: string; - @property({ attribute: false }) - public microphoneStream?: MediaStream | null; - - @property({ attribute: false }) - public microphoneConfig?: MicrophoneConfig; - // The camera's title, shown in error messages to identify the camera. @property({ attribute: false }) public cameraTitle?: string; @@ -110,7 +103,6 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer this._player = new VideoRTC(); this._player.targetID = this.targetID ?? null; this._player.mediaPlayerController = this._mediaPlayerController; - this._player.microphoneStream = this.microphoneStream ?? null; this._player.src = src; this._player.visibilityCheck = false; this._player.setControls(this.controls); @@ -143,13 +135,6 @@ export class AdvancedCameraCardGo2RTC extends LitElement implements MediaPlayer if (changedProps.has('controls') && this._player) { this._player.setControls(this.controls); } - - if (this._player && changedProps.has('microphoneStream')) { - // VideoRTC owns the transition: it updates microphoneStream, swaps the - // track on the pre-armed transceiver, and validates against stale async - // completions before any reconnect fallback. Fire-and-forget is fine. - void this._player.setMicrophoneStream(this.microphoneStream ?? null); - } } protected render(): TemplateResult | void { diff --git a/src/components/live/providers/go2rtc/video-rtc.d.ts b/src/components/live/providers/go2rtc/video-rtc.d.ts index 4fc6a012..aa998db7 100644 --- a/src/components/live/providers/go2rtc/video-rtc.d.ts +++ b/src/components/live/providers/go2rtc/video-rtc.d.ts @@ -32,10 +32,8 @@ export class VideoRTC extends HTMLElement { // Custom methods/members. mediaPlayerController: MediaPlayerController | null; - microphoneStream: MediaStream | null; targetID: string | null; reconnect(); reset(): void; setControls(controls: boolean): void; - setMicrophoneStream(stream: MediaStream | null): Promise; } diff --git a/src/components/live/providers/go2rtc/video-rtc.js b/src/components/live/providers/go2rtc/video-rtc.js index 04689e42..59b6aa3e 100644 --- a/src/components/live/providers/go2rtc/video-rtc.js +++ b/src/components/live/providers/go2rtc/video-rtc.js @@ -1,9 +1,5 @@ import { getTechnologyForVideoRTC } from '../../../../components-lib/live/utils/get-technology-for-video-rtc.js'; -import { - addAudioTracksMuteStateListener, - has2WayAudio, - hasAudio, -} from '../../../../utils/audio'; +import { addAudioTracksMuteStateListener, hasAudio } from '../../../../utils/audio'; import { hideMediaControlsTemporarily, MEDIA_LOAD_CONTROLS_HIDE_SECONDS, @@ -159,21 +155,6 @@ export class VideoRTC extends HTMLElement { */ this.onmessage = null; - /** - * A microphone stream to attach to a WebRTC connection. - * @type {MediaStream}} - */ - this.microphoneStream = null; - - /** - * The outbound audio transceiver pre-armed during createOffer. Holds a - * reference so `setMicrophoneStream` can swap the track via - * `replaceTrack` without renegotiating the SDP. Cleared on disconnect - * because transceivers belong to the closed peer connection. - * @type {RTCRtpTransceiver | null} - */ - this._microphoneTransceiver = null; - /** * A reference to a MediaPlayerController for this video * @type {MediaPlayerController | null} @@ -216,7 +197,6 @@ export class VideoRTC extends HTMLElement { mediaPlayerController: this.mediaPlayerController, }), capabilities: { - has2WayAudio: has2WayAudio(this.pc), hasAudio: hasAudio(this.video, this.pc, this.mseCodecs), supportsPause: true, }, @@ -268,50 +248,6 @@ export class VideoRTC extends HTMLElement { } } - /** - * Owns the microphone stream transition end-to-end: updates the property, - * extracts the outbound audio track, and swaps it onto the pre-armed audio - * transceiver via `replaceTrack` -- no SDP renegotiation, no visible reload. - * - * Falls back to a full reconnect if `replaceTrack` rejects, but only when - * the rejection still describes the current desired state. The transceiver - * and the requested stream are captured before awaiting so a late rejection - * from a stale operation (e.g. after `reset()` cleared the connection, or - * after a newer `setMicrophoneStream` superseded this one) cannot bring the - * player back online or overwrite a fresher request. - * - * @param {MediaStream | null} stream - */ - async setMicrophoneStream(stream) { - if (this.microphoneStream === stream) { - return; - } - this.microphoneStream = stream; - - const transceiver = this._microphoneTransceiver; - if (!transceiver) { - // No live peer connection yet (or createOffer hasn't run). The next - // createOffer will read `this.microphoneStream` and pre-arm the - // transceiver with the current track, so no separate fix-up is needed. - return; - } - - const desiredTrack = stream?.getAudioTracks()[0] ?? null; - try { - await transceiver.sender.replaceTrack(desiredTrack); - } catch (er) { - const stillCurrent = - transceiver === this._microphoneTransceiver && - this.microphoneStream === stream && - this.pc !== null; - if (!stillCurrent) { - return; - } - console.warn(er); - this.reconnect(); - } - } - /** * Reconnect the stream. */ @@ -552,15 +488,9 @@ export class VideoRTC extends HTMLElement { this.pcState = WebSocket.CLOSED; if (this.pc) { - // Do not close the (microphone) track attached to the peer connection as - // that is controlled by MicrophoneManager. - // See: https://github.com/dermotduffy/advanced-camera-card/issues/1810 - this.pc.close(); this.pc = null; } - // Transceivers belong to the now-closed peer connection. - this._microphoneTransceiver = null; this.video.src = ''; this.video.srcObject = null; @@ -646,7 +576,6 @@ export class VideoRTC extends HTMLElement { this.pc.close(); this.pc = null; this.pcState = WebSocket.CLOSED; - this._microphoneTransceiver = null; } // reconnect no more than once every X seconds @@ -861,25 +790,16 @@ export class VideoRTC extends HTMLElement { * @return {Promise} */ async createOffer(pc) { - // Always pre-arm a single outbound audio transceiver so the SDP advertises - // the slot from the start. With the slot in place, the mic track can be - // attached/detached later via `setMicrophoneStream` (replaceTrack) without - // renegotiating -- avoiding a visible reload of this cell each time grid - // selection moves the mic between cameras. - // - // Pure SDP allocation: the kind-only `addTransceiver('audio', ...)` form - // never calls `getUserMedia`, so users who don't grant mic access see no - // browser permission prompt from this path. - // - // The upstream `media.includes('microphone')` branch (which would have - // performed its own `getUserMedia` and added a second outbound audio - // transceiver) is intentionally omitted: the card drives mic acquisition - // through `MicrophoneManager`, and a second sender would race the one - // owned by `setMicrophoneStream`. - const micTrack = this.microphoneStream?.getAudioTracks()[0] ?? null; - this._microphoneTransceiver = pc.addTransceiver(micTrack ?? 'audio', { - direction: 'sendonly', - }); + try { + if (this.media.includes('microphone')) { + const media = await navigator.mediaDevices.getUserMedia({ audio: true }); + media.getTracks().forEach((track) => { + pc.addTransceiver(track, { direction: 'sendonly' }); + }); + } + } catch (e) { + console.warn(e); + } for (const kind of ['video', 'audio']) { if (this.media.indexOf(kind) >= 0) { diff --git a/src/config/schema/live.ts b/src/config/schema/live.ts index f6503301..cd1c8f0b 100644 --- a/src/config/schema/live.ts +++ b/src/config/schema/live.ts @@ -85,7 +85,6 @@ const microphoneConfigSchema = z .default(microphoneConfigDefault.mute_after_microphone_mute_seconds), }) .default(microphoneConfigDefault); -export type MicrophoneConfig = z.infer; export const liveConfigDefault = { auto_play: [...MEDIA_ACTION_POSITIVE_CONDITIONS], diff --git a/src/camera-manager/utils/go2rtc/audio.ts b/src/go2rtc/audio.ts similarity index 83% rename from src/camera-manager/utils/go2rtc/audio.ts rename to src/go2rtc/audio.ts index d8370403..bf45df46 100644 --- a/src/camera-manager/utils/go2rtc/audio.ts +++ b/src/go2rtc/audio.ts @@ -1,9 +1,9 @@ -import type { EnabledProxyConfig } from '../../../config/schema/common/proxy'; -import { homeAssistantSignAndFetch } from '../../../ha/fetch'; -import type { HomeAssistant } from '../../../ha/types'; -import { createProxiedEndpointIfNecessary } from '../../../ha/web-proxy'; -import type { Endpoint } from '../../../types'; -import { errorToConsole } from '../../../utils/basic'; +import type { EnabledProxyConfig } from '../config/schema/common/proxy'; +import { homeAssistantSignAndFetch } from '../ha/fetch'; +import type { HomeAssistant } from '../ha/types'; +import { createProxiedEndpointIfNecessary } from '../ha/web-proxy'; +import type { Endpoint } from '../types'; +import { errorToConsole } from '../utils/basic'; import { go2RTCStreamInfoSchema, type Go2RTCStreamInfo } from './types'; const getGo2RTCStreamMetadata = async ( diff --git a/src/camera-manager/utils/go2rtc/endpoint.ts b/src/go2rtc/endpoint.ts similarity index 75% rename from src/camera-manager/utils/go2rtc/endpoint.ts rename to src/go2rtc/endpoint.ts index f1a24bc2..cf609fca 100644 --- a/src/camera-manager/utils/go2rtc/endpoint.ts +++ b/src/go2rtc/endpoint.ts @@ -1,5 +1,5 @@ -import type { CameraConfig } from '../../../config/schema/cameras'; -import type { Endpoint } from '../../../types'; +import type { CameraConfig } from '../config/schema/cameras'; +import type { Endpoint } from '../types'; interface EndpointOptions { url?: string; @@ -18,7 +18,7 @@ const buildGo2RTCEndpoint = ( return null; } - const endpoint = pathBuilder(url, stream); + const endpoint = pathBuilder(url, encodeURIComponent(stream)); return { endpoint, // Only sign the endpoint if it's local to HA. @@ -43,9 +43,9 @@ export const getGo2RTCMetadataEndpoint = ( ): Endpoint | null => { return buildGo2RTCEndpoint( cameraConfig, - // Use probe parameters to trigger active stream detection. - // Without these, go2rtc only returns static config without producer medias. - (url, stream) => `${url}/api/streams?src=${stream}&video=all&audio=allµphone`, + // The `video` and `audio` parameters make go2rtc connect to the camera and + // report what it finds vs just reporting its own configuration. + (url, stream) => `${url}/api/streams?src=${stream}&video=all&audio=all`, options, ); }; diff --git a/src/go2rtc/messages.ts b/src/go2rtc/messages.ts new file mode 100644 index 00000000..8e486b29 --- /dev/null +++ b/src/go2rtc/messages.ts @@ -0,0 +1,27 @@ +import { z } from 'zod'; + +// go2rtc control messages are JSON text frames of this shape; media flows as +// separate binary frames. +export const go2RTCMessageSchema = z.object({ + type: z.string(), + + // Per-type payload (a codec list, an SDP, an ICE candidate, error text, ...): + // absent for some types (e.g. mjpeg) and not always a string, so it is typed + // `unknown` and each handler narrows it before use. + value: z.unknown().optional(), +}); +export type Go2RTCMessage = z.infer; + +export type MessageCallback = (message: Go2RTCMessage) => void; +export type BinaryCallback = (data: ArrayBuffer) => void; + +// The go2rtc server reports a mode failure as `{ type: 'error', value: ': +// ...' }` (e.g. `mse: stream not found`), so an error is for a given mode when +// its value starts with that mode's name. +export const isServerErrorForMode = ( + message: Go2RTCMessage, + mode: string, +): message is Go2RTCMessage & { value: string } => + message.type === 'error' && + typeof message.value === 'string' && + message.value.startsWith(mode); diff --git a/src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts b/src/go2rtc/peer-connection.ts similarity index 100% rename from src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.ts rename to src/go2rtc/peer-connection.ts diff --git a/src/components-lib/live/providers/go2rtc-experimental/signaling.ts b/src/go2rtc/signaling.ts similarity index 97% rename from src/components-lib/live/providers/go2rtc-experimental/signaling.ts rename to src/go2rtc/signaling.ts index 28e30e1e..4c6031fa 100644 --- a/src/components-lib/live/providers/go2rtc-experimental/signaling.ts +++ b/src/go2rtc/signaling.ts @@ -1,10 +1,10 @@ -import type { UnsubscribeCallback } from '../../../../types'; +import type { UnsubscribeCallback } from '../types'; import { go2RTCMessageSchema, type BinaryCallback, type Go2RTCMessage, type MessageCallback, -} from './types'; +} from './messages'; export type WebSocketFactory = (url: string) => WebSocket; diff --git a/src/camera-manager/utils/go2rtc/types.ts b/src/go2rtc/types.ts similarity index 97% rename from src/camera-manager/utils/go2rtc/types.ts rename to src/go2rtc/types.ts index 2bfaf2af..105c78de 100644 --- a/src/camera-manager/utils/go2rtc/types.ts +++ b/src/go2rtc/types.ts @@ -7,7 +7,7 @@ const go2RTCProducerSchema = z.object({ /** * Zod schema for Go2RTC stream information. Schema only covers the minimum * required by the card. - * Response from `/api/streams?src=${stream}&video=all&audio=allµphone` + * Response from `/api/streams?src=${stream}&video=all&audio=all` */ export const go2RTCStreamInfoSchema = z.object({ producers: z.array(go2RTCProducerSchema).optional(), diff --git a/src/ha/resolve-endpoint.ts b/src/ha/resolve-endpoint.ts new file mode 100644 index 00000000..9f763831 --- /dev/null +++ b/src/ha/resolve-endpoint.ts @@ -0,0 +1,70 @@ +import type { EnabledProxyConfig } from '../config/schema/common/proxy'; +import type { Endpoint } from '../types'; +import { errorToConsole } from '../utils/basic'; +import { homeAssistantGetSignedURLIfNecessary } from './sign-path'; +import type { HomeAssistant } from './types'; +import { + createProxiedEndpointIfNecessary, + type CreateProxiedEndpointOptions, +} from './web-proxy'; + +export const PROXY_URL_SIGN_EXPIRY_SECONDS = 24 * 60 * 60; + +export type ResolvedEndpoint = + | { success: true; url: string } + | { success: false; error: 'proxy' | 'sign' }; + +// Turns an endpoint into a URL that can be fetched or connected to: proxied +// through Home Assistant when the proxy configuration calls for it, then signed +// when the result needs Home Assistant authentication. +export const resolveEndpointURL = async ( + hass: HomeAssistant, + endpoint: Endpoint, + options?: { + proxyConfig?: EnabledProxyConfig | null; + proxyEndpointOptions?: CreateProxiedEndpointOptions; + }, +): Promise => { + // Proxy registration and signing both need an absolute URL. + const absolute: Endpoint = { + endpoint: new URL(endpoint.endpoint, document.baseURI).toString(), + sign: endpoint.sign, + }; + + let proxied: Endpoint | null; + if (!options?.proxyConfig?.enabled) { + proxied = absolute; + } else { + try { + proxied = await createProxiedEndpointIfNecessary( + hass, + { endpoint: absolute.endpoint, sign: false }, + options.proxyConfig, + { + ttl: PROXY_URL_SIGN_EXPIRY_SECONDS, + openLimit: 0, + ...options.proxyEndpointOptions, + }, + ); + } catch (e: unknown) { + errorToConsole(e); + proxied = null; + } + } + if (!proxied) { + return { success: false, error: 'proxy' }; + } + + let signed: string | null; + try { + signed = await homeAssistantGetSignedURLIfNecessary( + hass, + proxied, + PROXY_URL_SIGN_EXPIRY_SECONDS, + ); + } catch (e: unknown) { + errorToConsole(e); + signed = null; + } + return signed ? { success: true, url: signed } : { success: false, error: 'sign' }; +}; diff --git a/src/localize/languages/en.json b/src/localize/languages/en.json index 838e8a75..4386195c 100644 --- a/src/localize/languages/en.json +++ b/src/localize/languages/en.json @@ -805,6 +805,7 @@ "error": { "awaiting_live": "Waiting for live stream to load", "awaiting_media": "Waiting for media to load", + "call_camera_unreachable": "The camera could not be reached for two-way audio.", "call_invalid_target": "The requested camera or stream is not available to call.", "call_microphone_failed": "Your microphone could not be connected.", "call_microphone_forbidden": "Microphone access has been denied for this page. Update your browser permissions and try again.", diff --git a/src/types.ts b/src/types.ts index eba2bbfe..43c4e2e9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -20,11 +20,6 @@ export interface MediaLoadedCapabilities { supportsPause?: boolean; hasAudio?: boolean; - - // Note: This is whether the current stream already _has_ 2-way audio, not - // whether the underlying camera _could_ establish 2 way audio. For the - // latter, consult the camera's capabilities ('2-way-audio'). - has2WayAudio?: boolean; } export type MediaTechnology = diff --git a/src/utils/audio.ts b/src/utils/audio.ts index 6bd4d782..3805d932 100644 --- a/src/utils/audio.ts +++ b/src/utils/audio.ts @@ -68,22 +68,6 @@ export const hasAudio = ( return mayHaveAudio(video); }; -/** - * Check if a WebRTC peer connection has an outbound audio channel (i.e. 2-way - * audio / microphone support). - * @param pc The RTCPeerConnection to check. - * @returns True if the connection has an audio transceiver configured to send. - */ -export const has2WayAudio = (pc: RTCPeerConnection | null): boolean => { - return !!pc - ?.getTransceivers() - .some( - (tr) => - tr.sender.track?.kind === 'audio' && - (tr.direction === 'sendonly' || tr.direction === 'sendrecv'), - ); -}; - export type AudioTracksMuteStateCleanup = (() => void) | null; /** diff --git a/src/utils/basic.ts b/src/utils/basic.ts index 0a6d94c4..57e5cd65 100644 --- a/src/utils/basic.ts +++ b/src/utils/basic.ts @@ -88,6 +88,21 @@ export function contentsChanged( return !isEqualWith(n, o, customizer); } +// Get a description from thrown value preferring the browser's sentence ("The +// peer connection is closed") over the bare type name ("InvalidStateError"). +// +// DOMException may not inherit from Error, and catch blocks may be handed +// anything, so details are extracted structurally rather than via `instanceof +// Error`. +export const getErrorDescription = (error: unknown): string | null => { + if (!isRecord(error)) { + return null; + } + const message = typeof error.message === 'string' ? error.message : ''; + const name = typeof error.name === 'string' ? error.name : ''; + return message || name || null; +}; + /** * Log an error as a warning to the console. * @param e The caught error or error-like value. diff --git a/src/utils/live-provider.ts b/src/utils/live-provider.ts index b669ef9f..71dbff8f 100644 --- a/src/utils/live-provider.ts +++ b/src/utils/live-provider.ts @@ -1,7 +1,7 @@ -import { supports2WayAudio as gortcSupports2WayAudio } from '../camera-manager/utils/go2rtc/audio'; import type { CameraConfig } from '../config/schema/cameras'; import type { LiveProvider } from '../config/schema/cameras.js'; import type { EnabledProxyConfig } from '../config/schema/common/proxy'; +import { supports2WayAudio as gortcSupports2WayAudio } from '../go2rtc/audio'; import type { HomeAssistant } from '../ha/types'; import type { Endpoint } from '../types'; diff --git a/tests/camera-manager/camera.test.ts b/tests/camera-manager/camera.test.ts index 9fcca3e4..af9fed02 100644 --- a/tests/camera-manager/camera.test.ts +++ b/tests/camera-manager/camera.test.ts @@ -160,8 +160,7 @@ describe('Camera', () => { expect.anything(), 2, { - endpoint: - 'http://go2rtc/api/streams?src=stream&video=all&audio=allµphone', + endpoint: 'http://go2rtc/api/streams?src=stream&video=all&audio=all', sign: false, }, expect.objectContaining({ diff --git a/tests/card-controller/call/manager.test.ts b/tests/card-controller/call/manager.test.ts index 88c9c5d0..4859ad02 100644 --- a/tests/card-controller/call/manager.test.ts +++ b/tests/card-controller/call/manager.test.ts @@ -9,6 +9,11 @@ import { CallManager } from '../../../src/card-controller/call/manager'; import { Ringtone } from '../../../src/card-controller/call/ringtone'; import type { CardController } from '../../../src/card-controller/controller'; import { SubstreamViewModifier } from '../../../src/card-controller/view/modifiers/substream'; +import { createBackchannel } from '../../../src/components-lib/live/backchannel/factory'; +import { + BackchannelError, + type Backchannel, +} from '../../../src/components-lib/live/backchannel/types'; import { ConditionStateManager } from '../../../src/condition-trigger/conditions/state-manager'; import type { ConditionStateChange } from '../../../src/condition-trigger/conditions/types'; import { CallTrigger } from '../../../src/condition-trigger/triggers/triggers/call'; @@ -23,7 +28,7 @@ import { } from '../../camera-manager/test-utils'; import { createTriggerEvaluatorContext } from '../../condition-trigger/triggers/triggers/test-utils'; import { createCameraConfig, createConfig } from '../../config/test-utils'; -import { createCardAPI } from '../../test-utils'; +import { createCardAPI, createHASS } from '../../test-utils'; import { createView } from '../../view/test-utils'; // Replace Ringtone with a fresh `mock()` per construction so each @@ -37,6 +42,20 @@ vi.mock('../../../src/card-controller/call/ringtone', () => ({ }), })); +vi.mock('../../../src/components-lib/live/backchannel/factory'); + +const getBackchannel = (): Backchannel => { + const results = vi.mocked(createBackchannel).mock.results; + const last = results.at(-1); + assert(last); + return last.value; +}; + +beforeEach(() => { + vi.mocked(createBackchannel).mockReset(); + vi.mocked(createBackchannel).mockImplementation(() => mock()); +}); + // Each test creates a new CallManager which constructs a new Ringtone, so the // most recent constructor result is always this test's mock. const getRingtone = (): Ringtone => { @@ -65,6 +84,8 @@ const createAPI = (options?: { config?: PartialDeep; }): CardController => { const api = createCardAPI(); + vi.mocked(api.getHASSManager().getHASS).mockReturnValue(createHASS()); + vi.mocked(api.getMicrophoneManager().getStream).mockReturnValue(mock()); vi.mocked(api.getViewManager().getView).mockReturnValue(options?.view ?? null); vi.mocked(api.getCameraManager).mockReturnValue( createCameraManager(options?.store ?? createCallableStore()), @@ -945,65 +966,6 @@ describe('endIf', () => { }); }); -describe('reportCallMicrophoneError', () => { - it('should report a microphone that could not be attached', async () => { - const api = createAPI({ view: createView({ camera: 'camera.office' }) }); - const manager = new CallManager(api); - expect(await manager.start()).toBe(true); - vi.mocked(api.getNotificationManager().setNotification).mockClear(); - - manager.reportCallMicrophoneError('camera.office', 'The peer connection is closed'); - - expect(api.getNotificationManager().setNotification).toHaveBeenCalledWith({ - heading: { text: 'Two-way audio unavailable' }, - body: { text: 'Your microphone could not be connected.' }, - context: ['The peer connection is closed'], - }); - }); - - it('should omit the context when the browser provided none', async () => { - const api = createAPI({ view: createView({ camera: 'camera.office' }) }); - const manager = new CallManager(api); - expect(await manager.start()).toBe(true); - vi.mocked(api.getNotificationManager().setNotification).mockClear(); - - manager.reportCallMicrophoneError('camera.office'); - - expect(api.getNotificationManager().setNotification).toHaveBeenCalledWith( - expect.not.objectContaining({ context: expect.anything() }), - ); - }); - - it('should not report without a call', () => { - const api = createAPI({ view: createView({ camera: 'camera.office' }) }); - - new CallManager(api).reportCallMicrophoneError('camera.office'); - - expect(api.getNotificationManager().setNotification).not.toHaveBeenCalled(); - }); - - it('should not report while an inbound call is still ringing', async () => { - const api = createAPI({ view: createView({ camera: 'camera.office' }) }); - const manager = new CallManager(api); - expect(await manager.start({ inbound: true })).toBe(true); - - manager.reportCallMicrophoneError('camera.office'); - - expect(api.getNotificationManager().setNotification).not.toHaveBeenCalled(); - }); - - it('should not report for a camera the call is not on', async () => { - const api = createAPI({ view: createView({ camera: 'camera.office' }) }); - const manager = new CallManager(api); - expect(await manager.start()).toBe(true); - vi.mocked(api.getNotificationManager().setNotification).mockClear(); - - manager.reportCallMicrophoneError('camera.other'); - - expect(api.getNotificationManager().setNotification).not.toHaveBeenCalled(); - }); -}); - describe('condition state changes', () => { it('should end the call when the selected camera changes away', async () => { const api = createAPI({ view: createView({ camera: 'camera.office' }) }); @@ -2227,7 +2189,7 @@ describe('state changes during in-flight start', () => { const second = manager.start({ cameraID: 'camera.garage' }); resolveConnect(true); - expect(await first).toBe(true); + expect(await first).toBe(false); expect(await second).toBe(true); // The second request must end the first request's session rather than @@ -2344,6 +2306,244 @@ describe('state changes during in-flight answer', () => { // these drive a real ConditionStateManager and a real CallTrigger and assert // the transitions an automation would fire on, rather than that `setState` was // called. +describe('the backchannel', () => { + const startAnsweredCall = async ( + api: CardController, + ): Promise<{ manager: CallManager; started: boolean }> => { + const manager = new CallManager(api); + manager.initialize(); + const started = await manager.start(); + return { manager, started }; + }; + + it('should open the backchannel when an outbound call starts', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + + const { started } = await startAnsweredCall(api); + + expect(started).toBe(true); + expect(getBackchannel().start).toHaveBeenCalled(); + }); + + it('should not open the backchannel while an inbound call is only ringing', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + const manager = new CallManager(api); + manager.initialize(); + + await manager.start({ inbound: true }); + + expect(createBackchannel).not.toHaveBeenCalled(); + }); + + it('should open the backchannel when an inbound call is answered', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + const manager = new CallManager(api); + manager.initialize(); + await manager.start({ inbound: true }); + + expect(await manager.answer()).toBe(true); + expect(getBackchannel().start).toHaveBeenCalled(); + }); + + it('should release the backchannel when the call ends', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + const { manager } = await startAnsweredCall(api); + const backchannel = getBackchannel(); + + manager.end(); + + expect(backchannel.stop).toHaveBeenCalled(); + }); + + it('should release the backchannel when the manager is torn down', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + const { manager } = await startAnsweredCall(api); + const backchannel = getBackchannel(); + + manager.uninitialize(); + + expect(backchannel.stop).toHaveBeenCalled(); + }); + + it('should end the call and report when the backchannel cannot be opened', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + vi.mocked(createBackchannel).mockImplementation(() => { + const backchannel = mock(); + backchannel.start.mockRejectedValue(new BackchannelError('no_two_way_audio')); + return backchannel; + }); + + const { manager, started } = await startAnsweredCall(api); + + expect(started).toBe(false); + expect(manager.getCall()).toBeNull(); + expect(api.getNotificationManager().setNotification).toHaveBeenCalled(); + }); + + it('should not report a backchannel abandoned by this manager', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + vi.mocked(createBackchannel).mockImplementation(() => { + const backchannel = mock(); + backchannel.start.mockRejectedValue(new BackchannelError('abandoned')); + return backchannel; + }); + + await startAnsweredCall(api); + + expect(api.getNotificationManager().setNotification).not.toHaveBeenCalled(); + }); + + it('should release the previous backchannel before opening the next', async () => { + const api = createAPI({ + view: createView({ camera: 'camera.office' }), + store: createStore([ + { + cameraID: 'camera.office', + capabilities: createCapabilities({ live: true, '2-way-audio': true }), + }, + { + cameraID: 'camera.garage', + capabilities: createCapabilities({ live: true, '2-way-audio': true }), + }, + ]), + }); + const order: string[] = []; + vi.mocked(createBackchannel).mockImplementation(() => { + const backchannel = mock(); + backchannel.start.mockImplementation(async () => void order.push('start')); + backchannel.stop.mockImplementation(() => void order.push('stop')); + return backchannel; + }); + + const manager = new CallManager(api); + manager.initialize(); + await manager.start(); + await manager.start({ cameraID: 'camera.garage' }); + + // A camera that permits a single backchannel must never see two at once. + expect(order).toEqual(['start', 'stop', 'start']); + }); + + it('should end the call when the live provider has no backchannel to offer', async () => { + // This is reachable through `capabilities.force` which can force 2-way + // audio on a live provider that does not support it. + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + vi.mocked(createBackchannel).mockReturnValue(null); + + const { manager, started } = await startAnsweredCall(api); + + expect(started).toBe(false); + expect(manager.getCall()).toBeNull(); + expect(api.getNotificationManager().setNotification).toHaveBeenCalled(); + }); + + it('should still report a failure that has no reason', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + vi.mocked(createBackchannel).mockImplementation(() => { + const backchannel = mock(); + backchannel.start.mockRejectedValue(new Error('boom')); + return backchannel; + }); + + await startAnsweredCall(api); + + expect(api.getNotificationManager().setNotification).toHaveBeenCalledWith( + expect.objectContaining({ + body: { text: 'The camera could not be reached for two-way audio.' }, + }), + ); + }); + + it('should end the call when Home Assistant is not available', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + vi.mocked(api.getHASSManager().getHASS).mockReturnValue(null); + + const { manager, started } = await startAnsweredCall(api); + + expect(started).toBe(false); + expect(manager.getCall()).toBeNull(); + }); + + it('should distinguish a microphone failure from an unreachable camera', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + vi.mocked(createBackchannel).mockImplementation(() => { + const backchannel = mock(); + backchannel.start.mockRejectedValue(new BackchannelError('no_microphone')); + return backchannel; + }); + + await startAnsweredCall(api); + + expect(api.getNotificationManager().setNotification).toHaveBeenCalledWith( + expect.objectContaining({ + body: { text: 'Your microphone could not be connected.' }, + }), + ); + }); + + it('should not report a backchannel that failed for a replaced call', async () => { + const api = createAPI({ + view: createView({ camera: 'camera.office' }), + store: createStore([ + { + cameraID: 'camera.office', + capabilities: createCapabilities({ live: true, '2-way-audio': true }), + }, + { + cameraID: 'camera.garage', + capabilities: createCapabilities({ live: true, '2-way-audio': true }), + }, + ]), + }); + + let rejectFirst: (error: unknown) => void = () => {}; + let call = 0; + vi.mocked(createBackchannel).mockImplementation(() => { + const backchannel = mock(); + if (call++ === 0) { + backchannel.start.mockReturnValue( + new Promise((_resolve, reject) => (rejectFirst = reject)), + ); + } + return backchannel; + }); + + const manager = new CallManager(api); + manager.initialize(); + const first = manager.start(); + await manager.start({ cameraID: 'camera.garage' }); + + rejectFirst(new BackchannelError('failed')); + + expect(await first).toBe(false); + expect(api.getNotificationManager().setNotification).not.toHaveBeenCalled(); + expect(manager.getCall()?.cameraID).toBe('camera.garage'); + }); + + it('should not report a backchannel lost after its call ended', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + const { manager } = await startAnsweredCall(api); + const errorCallback = vi.mocked(createBackchannel).mock.calls[0][2]; + assert(errorCallback); + + manager.end(); + errorCallback(new BackchannelError('failed')); + + expect(api.getNotificationManager().setNotification).not.toHaveBeenCalled(); + }); + + it('should report the backchannel being lost mid-call', async () => { + const api = createAPI({ view: createView({ camera: 'camera.office' }) }); + await startAnsweredCall(api); + + const errorCallback = vi.mocked(createBackchannel).mock.calls[0][2]; + assert(errorCallback); + errorCallback(new BackchannelError('failed', 'the sky fell')); + + expect(api.getNotificationManager().setNotification).toHaveBeenCalled(); + }); +}); + describe('published phase transitions in condition state', () => { const createAPIWithRealStateManager = (options?: { config?: PartialDeep; diff --git a/tests/card-controller/microphone-manager.test.ts b/tests/card-controller/microphone-manager.test.ts index 25241d25..fd6724c2 100644 --- a/tests/card-controller/microphone-manager.test.ts +++ b/tests/card-controller/microphone-manager.test.ts @@ -518,10 +518,63 @@ describe('MicrophoneManager', () => { }); }); + describe('should handle the device disappearing', () => { + const endTrack = (stream: MediaStream): void => { + const track = getTrack(stream); + vi.mocked(track.addEventListener) + .mock.calls.filter(([type]) => type === 'ended') + .forEach(([, listener]) => (listener as EventListener)(new Event('ended'))); + }; + + it('should release a stream whose track ends', async () => { + const stream = createMockStream(); + vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(stream); + + const api = createCardAPI(); + const manager = new MicrophoneManager(api); + manager.setTransmissionActive(true); + await manager.connect(); + expect(manager.isConnected()).toBeTruthy(); + + endTrack(stream); + + expect(manager.isConnected()).toBeFalsy(); + expect(manager.getStream()).toBeNull(); + }); + + it('should stop listening to a stream that has been replaced', async () => { + const streamA = createMockStream(); + vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(streamA); + const api = createCardAPI(); + const manager = new MicrophoneManager(api); + manager.setTransmissionActive(true); + await manager.connect(); + + const streamB = createMockStream(); + vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue(streamB); + await manager.connect(); + + const track = getTrack(streamA); + const added = vi + .mocked(track.addEventListener) + .mock.calls.find(([type]) => type === 'ended'); + const removed = vi + .mocked(track.removeEventListener) + .mock.calls.find(([type]) => type === 'ended'); + + expect(added).toBeDefined(); + expect(removed?.[1]).toBe(added?.[1]); + expect(manager.getStream()).toBe(streamB); + }); + }); + describe('should require initialization', async () => { it('should require when configured and supported', async () => { const api = createCardAPI(); const manager = new MicrophoneManager(api); + vi.mocked(navigatorMock.mediaDevices.getUserMedia).mockResolvedValue( + createMockStream(), + ); vi.mocked(api.getConfigManager().getConfig).mockReturnValue( createConfig({ live: { diff --git a/tests/components-lib/live/backchannel/factory.test.ts b/tests/components-lib/live/backchannel/factory.test.ts new file mode 100644 index 00000000..307e4db4 --- /dev/null +++ b/tests/components-lib/live/backchannel/factory.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest'; +import { mock } from 'vitest-mock-extended'; + +import { Camera } from '../../../../src/camera-manager/camera'; +import type { CameraManagerEngine } from '../../../../src/camera-manager/engine'; +import { FrigateCamera } from '../../../../src/camera-manager/frigate/camera'; +import { createBackchannel } from '../../../../src/components-lib/live/backchannel/factory'; +import { Go2RTCBackchannel } from '../../../../src/components-lib/live/backchannel/go2rtc'; +import type { HomeAssistant } from '../../../../src/ha/types'; +import { createCameraConfig } from '../../../config/test-utils'; + +describe('createBackchannel', () => { + it('should create a backchannel for a go2rtc camera', () => { + const camera = new Camera( + createCameraConfig({ + live_provider: 'go2rtc', + go2rtc: { url: 'https://go2rtc', stream: 'office' }, + }), + mock(), + ); + + expect(createBackchannel(mock(), camera)).toBeInstanceOf( + Go2RTCBackchannel, + ); + }); + + it('should create a backchannel for a go2rtc-experimental camera', () => { + const camera = new Camera( + createCameraConfig({ + live_provider: 'go2rtc-experimental', + go2rtc: { url: 'https://go2rtc', stream: 'office' }, + }), + mock(), + ); + + expect(createBackchannel(mock(), camera)).toBeInstanceOf( + Go2RTCBackchannel, + ); + }); + + it('should not create a backchannel for a non-go2rtc camera', () => { + const camera = new Camera( + createCameraConfig({ + camera_entity: 'camera.office', + live_provider: 'ha', + }), + mock(), + ); + + expect(createBackchannel(mock(), camera)).toBeNull(); + }); + + it('should not create a backchannel without a go2rtc endpoint', () => { + const camera = new Camera( + createCameraConfig({ live_provider: 'go2rtc' }), + mock(), + ); + + expect(createBackchannel(mock(), camera)).toBeNull(); + }); + + it('should use the endpoint the camera engine resolves', () => { + // A Frigate camera serves go2rtc through the Frigate integration's proxy + // rather than at a directly-configured URL, so the endpoint must come from + // the camera rather than being rebuilt from its configuration. + const camera = new FrigateCamera( + createCameraConfig({ + live_provider: 'go2rtc', + frigate: { client_id: 'frigate', camera_name: 'office' }, + }), + mock(), + ); + + expect(camera.getEndpoints()?.go2rtc?.endpoint).toBe( + '/api/frigate/frigate/mse/api/ws?src=office', + ); + expect(createBackchannel(mock(), camera)).toBeInstanceOf( + Go2RTCBackchannel, + ); + }); +}); diff --git a/tests/components-lib/live/backchannel/go2rtc.test.ts b/tests/components-lib/live/backchannel/go2rtc.test.ts new file mode 100644 index 00000000..2060cd87 --- /dev/null +++ b/tests/components-lib/live/backchannel/go2rtc.test.ts @@ -0,0 +1,726 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mock } from 'vitest-mock-extended'; + +import { Go2RTCBackchannel } from '../../../../src/components-lib/live/backchannel/go2rtc'; +import type { BackchannelErrorCallback } from '../../../../src/components-lib/live/backchannel/types'; +import { + resolveEndpointURL, + type ResolvedEndpoint, +} from '../../../../src/ha/resolve-endpoint'; +import type { HomeAssistant } from '../../../../src/ha/types'; +import { + FakeMediaStream, + FakeMediaStreamTrack, + FakeRTCPeerConnection, + FakeWebSocket, +} from '../../../go2rtc/test-utils'; +import { flushPromises } from '../../../test-utils'; + +vi.mock('../../../../src/ha/resolve-endpoint'); + +const createStream = (): FakeMediaStream => + new FakeMediaStream([new FakeMediaStreamTrack('audio')]); + +const setup = (options?: { errorCallback?: BackchannelErrorCallback }) => { + const pc = new FakeRTCPeerConnection(); + const websocket = new FakeWebSocket(); + const backchannel = new Go2RTCBackchannel( + mock(), + { endpoint: '/local/api/ws?src=camera', sign: true }, + undefined, + { + createPeerConnection: () => pc.asPeerConnection(), + createWebSocket: () => websocket.asWebSocket(), + ...(options?.errorCallback && { errorCallback: options.errorCallback }), + }, + ); + return { backchannel, pc, websocket }; +}; + +// Drives a successful negotiation up to (but not including) the point the +// caller chooses to complete or fail it. +const negotiate = async (websocket: FakeWebSocket) => { + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage(JSON.stringify({ type: 'webrtc/answer', value: 'v=0\r\n' })); + await flushPromises(); +}; + +const connect = async (pc: FakeRTCPeerConnection, websocket: FakeWebSocket) => { + await negotiate(websocket); + pc.fireConnectionStateChange('connected'); + await flushPromises(); +}; + +// @vitest-environment jsdom +describe('Go2RTCBackchannel', () => { + beforeEach(() => { + vi.mocked(resolveEndpointURL).mockResolvedValue({ + success: true, + url: 'http://go2rtc/api/ws', + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('start', () => { + it('should offer exactly one outbound audio slot and no video', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + expect(pc.transceivers).toHaveLength(1); + expect(pc.transceivers[0].direction).toBe('sendonly'); + expect(pc.transceivers[0].sender.track?.kind).toBe('audio'); + }); + + it('should resolve only once the camera is reachable', async () => { + const { backchannel, pc, websocket } = setup(); + let resolved = false; + const started = backchannel.start(createStream().asMediaStream()).then(() => { + resolved = true; + }); + + await negotiate(websocket); + expect(resolved).toBe(false); + + pc.fireConnectionStateChange('connected'); + await started; + expect(resolved).toBe(true); + }); + + it('should send the offer over the signaling channel', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + expect(websocket.sent.map((message) => JSON.parse(message).type)).toContain( + 'webrtc/offer', + ); + }); + + it('should reject when the microphone has no audio track', async () => { + const { backchannel } = setup(); + await expect( + backchannel.start(new FakeMediaStream().asMediaStream()), + ).rejects.toMatchObject({ reason: 'no_microphone' }); + }); + + it('should reject when the microphone track has already ended', async () => { + const { backchannel } = setup(); + const track = new FakeMediaStreamTrack('audio'); + track.readyState = 'ended'; + await expect( + backchannel.start(new FakeMediaStream([track]).asMediaStream()), + ).rejects.toMatchObject({ reason: 'no_microphone' }); + }); + + it('should reject when the address cannot be resolved', async () => { + vi.mocked(resolveEndpointURL).mockResolvedValue({ + success: false, + error: 'proxy', + }); + const { backchannel } = setup(); + await expect( + backchannel.start(createStream().asMediaStream()), + ).rejects.toMatchObject({ reason: 'failed', description: 'proxy' }); + }); + + it('should reject when the server reports the stream cannot take audio', async () => { + const { backchannel, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage( + JSON.stringify({ type: 'error', value: 'webrtc: no backchannel' }), + ); + await expect(started).rejects.toMatchObject({ + reason: 'no_two_way_audio', + description: 'webrtc: no backchannel', + }); + }); + + it('should reject when the camera declines to receive audio', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await negotiate(websocket); + pc.getMicrophoneTransceiver().currentDirection = 'inactive'; + pc.fireConnectionStateChange('connected'); + await expect(started).rejects.toMatchObject({ reason: 'no_two_way_audio' }); + }); + + it('should reject when the signaling channel closes before connecting', async () => { + const { backchannel, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireClose(); + await expect(started).rejects.toMatchObject({ reason: 'failed' }); + }); + + it('should reject when the peer connection fails', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await negotiate(websocket); + pc.fireConnectionStateChange('failed'); + await expect(started).rejects.toMatchObject({ reason: 'failed' }); + }); + + it('should reject when the camera is not reached in time', async () => { + vi.useFakeTimers(); + const { backchannel, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + vi.advanceTimersByTime(10 * 1000); + await expect(started).rejects.toMatchObject({ reason: 'failed' }); + vi.useRealTimers(); + }); + + it('should reject when the offer cannot be created', async () => { + const { backchannel, pc, websocket } = setup(); + pc.createOffer.mockRejectedValue(new Error('no media')); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + await expect(started).rejects.toMatchObject({ + reason: 'failed', + description: 'no media', + }); + }); + + it('should close the signaling channel once connected', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + expect(websocket.close).toHaveBeenCalled(); + expect(pc.close).not.toHaveBeenCalled(); + }); + + it('should send ICE candidates and signal the end of them', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + + pc.fireIceCandidate('candidate:1'); + pc.fireIceCandidate(null); + + const candidates = websocket.sent + .map((message) => JSON.parse(message)) + .filter((message) => message.type === 'webrtc/candidate') + .map((message) => message.value); + expect(candidates).toEqual(['candidate:1', '']); + + await connect(pc, websocket); + await started; + }); + + it('should apply candidates from the server', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage( + JSON.stringify({ type: 'webrtc/candidate', value: 'candidate:2' }), + ); + await flushPromises(); + + expect(pc.addIceCandidate).toHaveBeenCalledWith({ + candidate: 'candidate:2', + sdpMid: '0', + }); + + await connect(pc, websocket); + await started; + }); + }); + + describe('after connecting', () => { + it('should report a peer connection that later fails', async () => { + const errorCallback = vi.fn(); + const { backchannel, pc, websocket } = setup({ errorCallback }); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + pc.fireConnectionStateChange('failed'); + expect(errorCallback).toHaveBeenCalledWith( + expect.objectContaining({ reason: 'failed' }), + ); + }); + + it('should release the camera when the peer connection later fails', async () => { + const stream = createStream(); + const { backchannel, pc, websocket } = setup({ errorCallback: vi.fn() }); + const started = backchannel.start(stream.asMediaStream()); + await connect(pc, websocket); + await started; + + pc.fireConnectionStateChange('failed'); + + expect(pc.close).toHaveBeenCalled(); + expect(stream.getAudioTracks()[0].readyState).toBe('live'); + }); + + it('should ignore the signaling channel closing', async () => { + const errorCallback = vi.fn(); + const { backchannel, pc, websocket } = setup({ errorCallback }); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + websocket.fireClose(); + expect(errorCallback).not.toHaveBeenCalled(); + }); + }); + + describe('setStream', () => { + it('should swap the outbound track without reconnecting', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + const replacement = createStream(); + await backchannel.setStream(replacement.asMediaStream()); + + expect(pc.getMicrophoneTransceiver().sender.replaceTrack).toHaveBeenCalledWith( + replacement.getAudioTracks()[0], + ); + expect(pc.close).not.toHaveBeenCalled(); + }); + + it('should do nothing without an established path', async () => { + const { backchannel } = setup(); + await expect( + backchannel.setStream(createStream().asMediaStream()), + ).resolves.toBeUndefined(); + }); + }); + + describe('stop', () => { + it('should close the connection but leave the microphone running', async () => { + const { backchannel, pc, websocket } = setup(); + const stream = createStream(); + const started = backchannel.start(stream.asMediaStream()); + await connect(pc, websocket); + await started; + + backchannel.stop(); + + expect(pc.close).toHaveBeenCalled(); + expect(stream.getAudioTracks()[0].readyState).toBe('live'); + }); + + it('should not report a peer connection that fails after being stopped', async () => { + const errorCallback = vi.fn(); + const { backchannel, pc, websocket } = setup({ errorCallback }); + const started = backchannel.start(createStream().asMediaStream()); + await negotiate(websocket); + + backchannel.stop(); + pc.fireConnectionStateChange('failed'); + await flushPromises(); + + await expect(started).rejects.toMatchObject({ reason: 'abandoned' }); + expect(errorCallback).not.toHaveBeenCalled(); + }); + }); + describe('losing the microphone', () => { + const endTrack = (stream: FakeMediaStream): void => { + const track = stream.getAudioTracks()[0]; + track.readyState = 'ended'; + track.dispatchEvent(new Event('ended')); + }; + + it('should fail a start whose microphone ends before connecting', async () => { + const stream = createStream(); + const { backchannel, websocket } = setup(); + const started = backchannel.start(stream.asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + + endTrack(stream); + + await expect(started).rejects.toMatchObject({ reason: 'no_microphone' }); + }); + + it('should release the camera and report when the microphone ends mid-call', async () => { + const errorCallback = vi.fn(); + const stream = createStream(); + const { backchannel, pc, websocket } = setup({ errorCallback }); + const started = backchannel.start(stream.asMediaStream()); + await connect(pc, websocket); + await started; + + endTrack(stream); + + expect(pc.close).toHaveBeenCalled(); + expect(errorCallback).toHaveBeenCalledWith( + expect.objectContaining({ reason: 'no_microphone' }), + ); + }); + + it('should stop watching a microphone that has been replaced', async () => { + const stream = createStream(); + const errorCallback = vi.fn(); + const { backchannel, pc, websocket } = setup({ errorCallback }); + const started = backchannel.start(stream.asMediaStream()); + await connect(pc, websocket); + await started; + + await backchannel.setStream(createStream().asMediaStream()); + endTrack(stream); + + expect(errorCallback).not.toHaveBeenCalled(); + expect(pc.close).not.toHaveBeenCalled(); + }); + + it('should stop watching the microphone once stopped', async () => { + const stream = createStream(); + const errorCallback = vi.fn(); + const { backchannel, pc, websocket } = setup({ errorCallback }); + const started = backchannel.start(stream.asMediaStream()); + await connect(pc, websocket); + await started; + + backchannel.stop(); + endTrack(stream); + + expect(errorCallback).not.toHaveBeenCalled(); + }); + }); + + describe('stale and defensive paths', () => { + it('should use the browser factories when none are supplied', async () => { + const pc = new FakeRTCPeerConnection(); + const websocket = new FakeWebSocket(); + vi.stubGlobal('RTCPeerConnection', function () { + return pc.asPeerConnection(); + }); + vi.stubGlobal('WebSocket', function () { + return websocket.asWebSocket(); + }); + + const backchannel = new Go2RTCBackchannel(mock(), { + endpoint: '/local/api/ws?src=camera', + }); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + expect(pc.transceivers).toHaveLength(1); + vi.unstubAllGlobals(); + }); + + it('should time out an address resolution that never returns', async () => { + vi.useFakeTimers(); + vi.mocked(resolveEndpointURL).mockReturnValue(new Promise(() => {})); + + const { backchannel } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + vi.advanceTimersByTime(10 * 1000); + + await expect(started).rejects.toMatchObject({ reason: 'failed' }); + vi.useRealTimers(); + }); + + it('should reject with a reason and release the connection when setup throws', async () => { + const { backchannel, pc } = setup(); + pc.addTransceiver = () => { + throw new Error('bad track'); + }; + + await expect( + backchannel.start(createStream().asMediaStream()), + ).rejects.toMatchObject({ reason: 'failed', description: 'bad track' }); + expect(pc.close).toHaveBeenCalled(); + }); + + it('should reject without a description when setup throws something not error-like', async () => { + const { backchannel, pc } = setup(); + pc.addTransceiver = () => { + throw 'a bare string'; + }; + + await expect( + backchannel.start(createStream().asMediaStream()), + ).rejects.toMatchObject({ reason: 'failed', description: null }); + }); + + it('should abandon a start stopped while the address resolves', async () => { + let release: (value: ResolvedEndpoint) => void = () => {}; + vi.mocked(resolveEndpointURL).mockReturnValue( + new Promise((resolve) => { + release = resolve; + }), + ); + + const { backchannel, pc } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + backchannel.stop(); + release({ success: true, url: 'http://go2rtc/api/ws' }); + await flushPromises(); + + await expect(started).rejects.toMatchObject({ reason: 'abandoned' }); + expect(pc.transceivers).toHaveLength(0); + }); + + it('should not send candidates after being stopped', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + const before = websocket.sent.length; + + backchannel.stop(); + pc.fireIceCandidate('candidate:late'); + + expect(websocket.sent).toHaveLength(before); + await expect(started).rejects.toThrow(); + }); + + it('should not negotiate after being stopped', async () => { + const { backchannel, pc, websocket } = setup(); + let releaseOffer: (value: { type: string; sdp?: string }) => void = () => {}; + pc.createOffer.mockReturnValue( + new Promise((resolve) => { + releaseOffer = resolve; + }), + ); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + + backchannel.stop(); + releaseOffer({ type: 'offer', sdp: 'v=0' }); + await flushPromises(); + + expect(pc.setLocalDescription).not.toHaveBeenCalled(); + await expect(started).rejects.toThrow(); + }); + + it('should not send an offer when the local description is stopped mid-flight', async () => { + const { backchannel, pc, websocket } = setup(); + let releaseLocal: () => void = () => {}; + pc.setLocalDescription.mockReturnValue( + new Promise((resolve) => { + releaseLocal = resolve; + }), + ); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + + backchannel.stop(); + releaseLocal(); + await flushPromises(); + + expect( + websocket.sent.filter((m) => JSON.parse(m).type === 'webrtc/offer'), + ).toHaveLength(0); + await expect(started).rejects.toThrow(); + }); + + it('should send an empty offer when the browser produces no SDP', async () => { + const { backchannel, pc, websocket } = setup(); + pc.createOffer.mockResolvedValue({ type: 'offer' }); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + const offer = websocket.sent + .map((m) => JSON.parse(m)) + .find((m) => m.type === 'webrtc/offer'); + expect(offer.value).toBe(''); + }); + + it('should reject without a description when a negotiation failure is not error-like', async () => { + const { backchannel, pc, websocket } = setup(); + pc.createOffer.mockRejectedValue('a bare string'); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + await expect(started).rejects.toMatchObject({ + reason: 'failed', + description: null, + }); + }); + + it('should describe a negotiation failure by its name when it carries no message', async () => { + const { backchannel, pc, websocket } = setup(); + pc.createOffer.mockRejectedValue({ name: 'InvalidStateError' }); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + await expect(started).rejects.toMatchObject({ + reason: 'failed', + description: 'InvalidStateError', + }); + }); + + it('should reject when the answer cannot be applied', async () => { + const { backchannel, pc, websocket } = setup(); + pc.setRemoteDescription.mockRejectedValue(new Error('bad sdp')); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage(JSON.stringify({ type: 'webrtc/answer', value: 'v=0' })); + await expect(started).rejects.toMatchObject({ + reason: 'failed', + description: 'bad sdp', + }); + }); + + it('should ignore messages without a string payload', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage(JSON.stringify({ type: 'webrtc/answer', value: 42 })); + await flushPromises(); + + expect(pc.setRemoteDescription).not.toHaveBeenCalled(); + await connect(pc, websocket); + await started; + }); + + it('should tolerate a candidate the browser rejects', async () => { + const { backchannel, pc, websocket } = setup(); + pc.addIceCandidate.mockRejectedValue(new Error('bad candidate')); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage( + JSON.stringify({ type: 'webrtc/candidate', value: 'candidate:3' }), + ); + await flushPromises(); + + await connect(pc, websocket); + await expect(started).resolves.toBeUndefined(); + }); + + it('should ignore an empty candidate from the server', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage(JSON.stringify({ type: 'webrtc/candidate', value: '' })); + await flushPromises(); + + expect(pc.addIceCandidate).not.toHaveBeenCalled(); + await connect(pc, websocket); + await started; + }); + + it('should ignore messages after being stopped', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + + backchannel.stop(); + websocket.fireMessage(JSON.stringify({ type: 'webrtc/answer', value: 'v=0' })); + await flushPromises(); + + expect(pc.setRemoteDescription).not.toHaveBeenCalled(); + await expect(started).rejects.toThrow(); + }); + + it('should ignore intermediate connection states', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await negotiate(websocket); + + pc.fireConnectionStateChange('connecting'); + await flushPromises(); + expect(websocket.close).not.toHaveBeenCalled(); + + pc.fireConnectionStateChange('connected'); + await started; + }); + + it('should reject a replacement carrying no audio', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + await expect( + backchannel.setStream(new FakeMediaStream().asMediaStream()), + ).rejects.toMatchObject({ reason: 'no_microphone' }); + expect(pc.getMicrophoneTransceiver().sender.replaceTrack).not.toHaveBeenCalled(); + }); + + it('should reject a replacement whose track has ended', async () => { + const { backchannel, pc, websocket } = setup(); + const started = backchannel.start(createStream().asMediaStream()); + await connect(pc, websocket); + await started; + + const track = new FakeMediaStreamTrack('audio'); + track.readyState = 'ended'; + await expect( + backchannel.setStream(new FakeMediaStream([track]).asMediaStream()), + ).rejects.toMatchObject({ reason: 'no_microphone' }); + }); + it('should reject without a description when an answer failure names nothing', async () => { + const { backchannel, pc, websocket } = setup(); + pc.setRemoteDescription.mockRejectedValue({}); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + websocket.fireMessage(JSON.stringify({ type: 'webrtc/answer', value: 'v=0' })); + await expect(started).rejects.toMatchObject({ + reason: 'failed', + description: null, + }); + }); + + it('should ignore a negotiation failure that arrives after being stopped', async () => { + const { backchannel, pc, websocket } = setup(); + let rejectOffer: (error: unknown) => void = () => {}; + pc.createOffer.mockReturnValue( + new Promise((_resolve, reject) => { + rejectOffer = reject; + }), + ); + const started = backchannel.start(createStream().asMediaStream()); + await flushPromises(); + websocket.fireOpen(); + await flushPromises(); + + backchannel.stop(); + await expect(started).rejects.toMatchObject({ reason: 'abandoned' }); + + rejectOffer(new Error('too late')); + await flushPromises(); + }); + }); +}); diff --git a/tests/components-lib/live/providers/go2rtc-experimental/offscreen-video.test.ts b/tests/components-lib/live/providers/go2rtc-experimental/offscreen-video.test.ts index ec8ca5bb..7b70f953 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/offscreen-video.test.ts +++ b/tests/components-lib/live/providers/go2rtc-experimental/offscreen-video.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { OffscreenVideo } from '../../../../../src/components-lib/live/providers/go2rtc-experimental/offscreen-video'; -import { FakeMediaStream, FakeMediaStreamTrack } from './test-utils'; +import { FakeMediaStream, FakeMediaStreamTrack } from '../../../../go2rtc/test-utils'; // @vitest-environment jsdom describe('OffscreenVideo', () => { diff --git a/tests/components-lib/live/providers/go2rtc-experimental/session-controller.test.ts b/tests/components-lib/live/providers/go2rtc-experimental/session-controller.test.ts index 3a1387f0..159ef7a7 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/session-controller.test.ts +++ b/tests/components-lib/live/providers/go2rtc-experimental/session-controller.test.ts @@ -26,7 +26,7 @@ import { FakeMediaStreamTrack, FakeRTCPeerConnection, FakeWebSocket, -} from './test-utils'; +} from '../../../../go2rtc/test-utils'; const H264_PROFILE: StreamProfile = { hasVideo: true, @@ -128,7 +128,6 @@ describe('Go2RTCSessionController', () => { source.getCapabilities.mockReturnValue({ supportsPause: true, hasAudio: true, - has2WayAudio: false, }); source.getTechnology.mockReturnValue([mode]); binarySources.push(source); @@ -154,15 +153,12 @@ describe('Go2RTCSessionController', () => { source.getCapabilities.mockReturnValue({ supportsPause: true, hasAudio: true, - has2WayAudio: false, }); source.getTechnology.mockReturnValue(['webrtc']); source.getMediaStream.mockReturnValue(webRTCStream.asMediaStream()); source.getPeerConnection.mockReturnValue( options?.webRTCPeerConnection?.asPeerConnection() ?? null, ); - source.setMicrophoneStream.mockResolvedValue(undefined); - webRTCSources.push(source); return source; }, @@ -179,7 +175,6 @@ describe('Go2RTCSessionController', () => { const mediaLoadedCallback = vi.fn(); const surfaceCommittedCallback = vi.fn(); const streamErrorCallback = vi.fn(); - const microphoneErrorCallback = vi.fn(); const session = new Go2RTCSessionController( { @@ -188,7 +183,6 @@ describe('Go2RTCSessionController', () => { mediaLoadedCallback, surfaceCommittedCallback, streamErrorCallback, - microphoneErrorCallback, }, { createWebSocket, createBinarySource, createWebRTCSource, createVideoElement }, ); @@ -204,7 +198,6 @@ describe('Go2RTCSessionController', () => { createWebRTCSource, createWebSocket, streamErrorCallback, - microphoneErrorCallback, mediaLoadedCallback, offscreenVideos, session, @@ -285,7 +278,6 @@ describe('Go2RTCSessionController', () => { getCardWideConfig: () => null, mediaLoadedCallback: vi.fn(), streamErrorCallback: vi.fn(), - microphoneErrorCallback: vi.fn(), }); session.connect('ws://localhost:1/api/ws', createSurfaces().surfaces, ['mse']); session.reset(); @@ -528,42 +520,6 @@ describe('Go2RTCSessionController', () => { expect(createWebSocket).toHaveBeenCalledTimes(2); }); - it('should pre-arm the WebRTC source with the current microphone stream', () => { - const { session, surfaces, websockets, webRTCOptions } = setup(); - const micStream = new FakeMediaStream([ - new FakeMediaStreamTrack('audio'), - ]).asMediaStream(); - session.setMicrophoneStream(micStream); - session.connect('http://host/api/ws?src=camera', surfaces, ['webrtc']); - websockets[0].fireOpen(); - - expect(webRTCOptions[0]?.microphoneStream).toBe(micStream); - }); - - it('should report a microphone error without disturbing the stream', () => { - const { - session, - surfaces, - websockets, - webRTCOptions, - webRTCSources, - microphoneErrorCallback, - streamErrorCallback, - } = setup(); - session.connect('http://host/api/ws?src=camera', surfaces, ['webrtc']); - websockets[0].fireOpen(); - - webRTCOptions[0]?.microphoneErrorCallback?.('InvalidStateError'); - - expect(microphoneErrorCallback).toHaveBeenCalledWith('InvalidStateError'); - - // The inbound video is unaffected by an outbound audio failure, so the - // source keeps running and the session neither escalates nor reconnects. - expect(streamErrorCallback).not.toHaveBeenCalled(); - expect(webRTCSources[0].stop).not.toHaveBeenCalled(); - expect(websockets).toHaveLength(1); - }); - it('should re-dispatch loaded media on an audio mute transition', () => { const peerConnection = new FakeRTCPeerConnection(); const audioTransceiver = peerConnection.addTransceiver('audio', { @@ -858,26 +814,6 @@ describe('Go2RTCSessionController', () => { }); }); - describe('microphone', () => { - it('should forward a microphone change to the WebRTC source', () => { - const { session, surfaces, websockets, webRTCSources } = setup(); - session.connect('http://host/api/ws?src=camera', surfaces, ['webrtc']); - websockets[0].fireOpen(); - const micStream = new FakeMediaStream([ - new FakeMediaStreamTrack('audio'), - ]).asMediaStream(); - session.setMicrophoneStream(micStream); - - expect(webRTCSources[0].setMicrophoneStream).toHaveBeenCalledWith(micStream); - }); - - it('should tolerate a microphone change with no WebRTC source', () => { - const { session } = setup(); - - expect(() => session.setMicrophoneStream(null)).not.toThrow(); - }); - }); - describe('lifecycle', () => { it('should stop the source and reconnect on unexpected closure', () => { const { session, surfaces, websockets, binarySources, createWebSocket } = setup(); @@ -1078,16 +1014,6 @@ describe('Go2RTCSessionController', () => { expect(mediaLoadedCallback).not.toHaveBeenCalled(); }); - it('should swallow a rejected microphone update', async () => { - const { session, surfaces, websockets, webRTCSources } = setup(); - session.connect('http://host/api/ws?src=camera', surfaces, ['webrtc']); - websockets[0].fireOpen(); - webRTCSources[0].setMicrophoneStream.mockRejectedValue(new Error('replace')); - - expect(() => session.setMicrophoneStream(null)).not.toThrow(); - await Promise.resolve(); - }); - it('should ignore callbacks fired while a binary source is constructed', () => { const websockets: FakeWebSocket[] = []; const createWebSocket = vi.fn<(url: string) => WebSocket>(() => { @@ -1112,7 +1038,6 @@ describe('Go2RTCSessionController', () => { getCardWideConfig: () => null, mediaLoadedCallback, streamErrorCallback: vi.fn(), - microphoneErrorCallback: vi.fn(), }, { createWebSocket, createBinarySource }, ); @@ -1147,7 +1072,6 @@ describe('Go2RTCSessionController', () => { getCardWideConfig: () => null, mediaLoadedCallback, streamErrorCallback: vi.fn(), - microphoneErrorCallback: vi.fn(), }, { createWebSocket, createWebRTCSource }, ); @@ -1174,7 +1098,6 @@ describe('Go2RTCSessionController', () => { getCardWideConfig: () => null, mediaLoadedCallback: vi.fn(), streamErrorCallback: vi.fn(), - microphoneErrorCallback: vi.fn(), }, { createWebSocket }, ); @@ -1206,7 +1129,6 @@ describe('Go2RTCSessionController', () => { getCardWideConfig: () => null, mediaLoadedCallback: vi.fn(), streamErrorCallback: vi.fn(), - microphoneErrorCallback: vi.fn(), }, { createWebSocket }, ); @@ -1247,7 +1169,6 @@ describe('Go2RTCSessionController', () => { getCardWideConfig: () => null, mediaLoadedCallback: vi.fn(), streamErrorCallback: vi.fn(), - microphoneErrorCallback: vi.fn(), }, { createWebSocket, createBinarySource, createWebRTCSource }, ); diff --git a/tests/components-lib/live/providers/go2rtc-experimental/sources/mse.test.ts b/tests/components-lib/live/providers/go2rtc-experimental/sources/mse.test.ts index 6ce8de6b..1d243de6 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/sources/mse.test.ts +++ b/tests/components-lib/live/providers/go2rtc-experimental/sources/mse.test.ts @@ -568,7 +568,6 @@ describe('MSEStreamSource', () => { expect(source.getCapabilities()).toEqual({ supportsPause: true, hasAudio: false, - has2WayAudio: false, }); }); @@ -579,7 +578,6 @@ describe('MSEStreamSource', () => { expect(setupResult.source.getCapabilities()).toEqual({ supportsPause: true, hasAudio: true, - has2WayAudio: false, }); }); @@ -595,7 +593,6 @@ describe('MSEStreamSource', () => { expect(setupResult.source.getCapabilities()).toEqual({ supportsPause: true, hasAudio: false, - has2WayAudio: false, }); }); diff --git a/tests/components-lib/live/providers/go2rtc-experimental/sources/webrtc.test.ts b/tests/components-lib/live/providers/go2rtc-experimental/sources/webrtc.test.ts index 0f69294b..2a121eab 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/sources/webrtc.test.ts +++ b/tests/components-lib/live/providers/go2rtc-experimental/sources/webrtc.test.ts @@ -5,17 +5,17 @@ import type { StreamSourceContext, VideoStreamTarget, } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/types'; -import { flushPromises } from '../../../../../test-utils'; import { FakeMediaStream, - FakeMediaStreamTrack, FakeRTCPeerConnection, - FakeStreamSourceChannel, -} from '../test-utils'; + type FakeMediaStreamTrack, +} from '../../../../../go2rtc/test-utils'; +import { flushPromises } from '../../../../../test-utils'; +import { FakeStreamSourceChannel } from '../test-utils'; // @vitest-environment jsdom describe('WebRTCStreamSource', () => { - const setup = (options?: { microphoneStream?: FakeMediaStream | null }) => { + const setup = () => { const video = document.createElement('video'); const channel = new FakeStreamSourceChannel(); const loadedCallback = vi.fn(); @@ -29,13 +29,10 @@ describe('WebRTCStreamSource', () => { const pc = new FakeRTCPeerConnection(); const createPeerConnection = vi.fn(() => pc.asPeerConnection()); - const microphoneErrorCallback = vi.fn(); const source = new WebRTCStreamSource(context, { createPeerConnection, createMediaStream: (tracks) => new FakeMediaStream(tracks as unknown as FakeMediaStreamTrack[]).asMediaStream(), - microphoneStream: options?.microphoneStream?.asMediaStream() ?? null, - microphoneErrorCallback, }); return { @@ -44,7 +41,6 @@ describe('WebRTCStreamSource', () => { createPeerConnection, failedCallback, loadedCallback, - microphoneErrorCallback, pc, source, video, @@ -60,27 +56,14 @@ describe('WebRTCStreamSource', () => { }); describe('transceivers', () => { - it('should pre-arm a sendonly audio transceiver and recvonly video and audio', () => { + it('should offer inbound video and audio only', () => { const { source, pc } = setup(); source.start(); - expect(pc.transceivers).toHaveLength(3); - expect(pc.transceivers[0].direction).toBe('sendonly'); - expect(pc.transceivers[1].direction).toBe('recvonly'); - expect(pc.transceivers[2].direction).toBe('recvonly'); - - // Kind-only pre-arm: no track, so no getUserMedia and no permission prompt. - expect(pc.transceivers[0].sender.track).toBeNull(); - }); - - it('should pre-arm with the current microphone track', () => { - const micTrack = new FakeMediaStreamTrack('audio'); - const { source, pc } = setup({ - microphoneStream: new FakeMediaStream([micTrack]), - }); - source.start(); - - expect(pc.transceivers[0].sender.track).toBe(micTrack); + expect(pc.transceivers.map((transceiver) => transceiver.direction)).toEqual([ + 'recvonly', + 'recvonly', + ]); }); }); @@ -431,6 +414,17 @@ describe('WebRTCStreamSource', () => { expect(source.getPeerConnection()).toBe(pc.asPeerConnection()); }); + it('should report its media capabilities', () => { + const { source, pc } = setup(); + source.start(); + pc.fireConnectionStateChange('connected'); + + expect(source.getCapabilities()).toEqual({ + supportsPause: true, + hasAudio: expect.any(Boolean), + }); + }); + it('should report webrtc technology', () => { const { source } = setup(); @@ -462,139 +456,5 @@ describe('WebRTCStreamSource', () => { hasAACAudio: false, }); }); - - it('should report 2-way audio capability once a mic track is armed', () => { - const micTrack = new FakeMediaStreamTrack('audio'); - const { source, pc } = setup({ - microphoneStream: new FakeMediaStream([micTrack]), - }); - source.start(); - pc.fireConnectionStateChange('connected'); - - expect(source.getCapabilities().has2WayAudio).toBe(true); - expect(source.getCapabilities().supportsPause).toBe(true); - }); - }); - - describe('setMicrophoneStream', () => { - it('should do nothing for an unchanged stream', async () => { - const stream = new FakeMediaStream([new FakeMediaStreamTrack('audio')]); - const { source, pc } = setup({ microphoneStream: stream }); - source.start(); - await source.setMicrophoneStream(stream.asMediaStream()); - - expect(pc.getMicrophoneTransceiver().sender.replaceTrack).not.toHaveBeenCalled(); - }); - - it('should replace the outbound track without renegotiating', async () => { - const { source, pc } = setup(); - source.start(); - const newTrack = new FakeMediaStreamTrack('audio'); - await source.setMicrophoneStream(new FakeMediaStream([newTrack]).asMediaStream()); - - expect(pc.getMicrophoneTransceiver().sender.replaceTrack).toHaveBeenCalledWith( - newTrack, - ); - }); - - it('should clear the outbound track for a null stream', async () => { - const stream = new FakeMediaStream([new FakeMediaStreamTrack('audio')]); - const { source, pc } = setup({ microphoneStream: stream }); - source.start(); - await source.setMicrophoneStream(null); - - expect(pc.getMicrophoneTransceiver().sender.replaceTrack).toHaveBeenCalledWith( - null, - ); - }); - - it('should do nothing before there is a peer connection', async () => { - const { source, microphoneErrorCallback } = setup(); - await source.setMicrophoneStream( - new FakeMediaStream([new FakeMediaStreamTrack('audio')]).asMediaStream(), - ); - - expect(microphoneErrorCallback).not.toHaveBeenCalled(); - }); - - it.each([ - [ - 'what the browser said when the rejection has a message', - new DOMException('The peer connection is closed', 'InvalidStateError'), - 'The peer connection is closed', - ], - [ - 'the rejection type when there is no message to quote', - new DOMException('', 'InvalidStateError'), - 'InvalidStateError', - ], - ['nothing when the rejection is not an object', 'nope', undefined], - [ - 'nothing when the rejection describes itself with neither', - { message: 5, name: 7 }, - undefined, - ], - ] as const)( - 'should report %s when a current replaceTrack rejects', - async (_summary, rejection, expected) => { - const { source, pc, microphoneErrorCallback } = setup(); - source.start(); - pc.getMicrophoneTransceiver().sender.replaceTrack.mockRejectedValue(rejection); - await source.setMicrophoneStream( - new FakeMediaStream([new FakeMediaStreamTrack('audio')]).asMediaStream(), - ); - - expect(microphoneErrorCallback).toHaveBeenCalledWith(expected); - }, - ); - - it('should not fail the stream source when the microphone cannot attach', async () => { - const { source, pc, failedCallback } = setup(); - source.start(); - pc.getMicrophoneTransceiver().sender.replaceTrack.mockRejectedValue( - new DOMException('replace failed', 'InvalidStateError'), - ); - await source.setMicrophoneStream( - new FakeMediaStream([new FakeMediaStreamTrack('audio')]).asMediaStream(), - ); - - // The inbound video is unaffected by an outbound audio failure, so the - // source must keep running rather than failing over to another one. - expect(failedCallback).not.toHaveBeenCalled(); - }); - - it('should not report a rejection when detaching the microphone', async () => { - const stream = new FakeMediaStream([new FakeMediaStreamTrack('audio')]); - const { source, pc, microphoneErrorCallback } = setup({ - microphoneStream: stream, - }); - source.start(); - pc.getMicrophoneTransceiver().sender.replaceTrack.mockRejectedValue( - new DOMException('The peer connection is closed', 'InvalidStateError'), - ); - await source.setMicrophoneStream(null); - - // Ignore the error, the user is not trying to be heard anyway. - expect(microphoneErrorCallback).not.toHaveBeenCalled(); - }); - - it('should ignore a stale replaceTrack rejection after stop', async () => { - const { source, pc, microphoneErrorCallback } = setup(); - source.start(); - let rejectReplace: (reason: Error) => void = () => {}; - pc.getMicrophoneTransceiver().sender.replaceTrack.mockReturnValue( - new Promise((_resolve, reject) => { - rejectReplace = reject; - }), - ); - const promise = source.setMicrophoneStream( - new FakeMediaStream([new FakeMediaStreamTrack('audio')]).asMediaStream(), - ); - source.stop(); - rejectReplace(new Error('replace failed')); - await promise; - - expect(microphoneErrorCallback).not.toHaveBeenCalled(); - }); }); }); diff --git a/tests/components-lib/live/providers/go2rtc-experimental/test-utils.ts b/tests/components-lib/live/providers/go2rtc-experimental/test-utils.ts index ae9a53cf..dc8669b3 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/test-utils.ts +++ b/tests/components-lib/live/providers/go2rtc-experimental/test-utils.ts @@ -1,15 +1,15 @@ -import { vi, type Mock } from 'vitest'; +import { vi } from 'vitest'; import type { MediaSourceFactory, MediaSourceInterface, } from '../../../../../src/components-lib/live/providers/go2rtc-experimental/adapters/media-source'; +import type { StreamSourceChannel } from '../../../../../src/components-lib/live/providers/go2rtc-experimental/types'; import type { BinaryCallback, Go2RTCMessage, MessageCallback, - StreamSourceChannel, -} from '../../../../../src/components-lib/live/providers/go2rtc-experimental/types'; +} from '../../../../../src/go2rtc/messages'; import type { UnsubscribeCallback } from '../../../../../src/types'; // =========================================================================== @@ -28,32 +28,6 @@ export const SAFARI_17_USER_AGENT = // Fakes for browser APIs jsdom does not provide. // =========================================================================== -export class FakeWebSocket extends EventTarget { - public binaryType = ''; - public sent: string[] = []; - - public close = vi.fn(); - public send = vi.fn((data: string): void => { - this.sent.push(data); - }); - - public asWebSocket(): WebSocket { - return this as unknown as WebSocket; - } - - public fireOpen(): void { - this.dispatchEvent(new Event('open')); - } - - public fireClose(): void { - this.dispatchEvent(new Event('close')); - } - - public fireMessage(data: unknown): void { - this.dispatchEvent(new MessageEvent('message', { data })); - } -} - export const createTimeRanges = (ranges: [number, number][]): TimeRanges => ({ length: ranges.length, start: (index: number) => ranges[index][0], @@ -77,129 +51,6 @@ class FakeSourceBuffer extends EventTarget { } } -export class FakeMediaStreamTrack extends EventTarget { - public muted = false; - public kind: string; - - constructor(kind: string) { - super(); - this.kind = kind; - } - - public asTrack(): MediaStreamTrack { - return this as unknown as MediaStreamTrack; - } - - public setMuted(muted: boolean): void { - this.muted = muted; - this.dispatchEvent(new Event(muted ? 'mute' : 'unmute')); - } -} - -export class FakeMediaStream { - private _tracks: FakeMediaStreamTrack[]; - - constructor(tracks: FakeMediaStreamTrack[] = []) { - this._tracks = tracks; - } - - public getTracks(): FakeMediaStreamTrack[] { - return this._tracks; - } - - public getVideoTracks(): FakeMediaStreamTrack[] { - return this._tracks.filter((track) => track.kind === 'video'); - } - - public getAudioTracks(): FakeMediaStreamTrack[] { - return this._tracks.filter((track) => track.kind === 'audio'); - } - - public asMediaStream(): MediaStream { - return this as unknown as MediaStream; - } -} - -class FakeRTCTransceiver { - public direction: string; - public currentDirection: string; - public sender: { - track: FakeMediaStreamTrack | null; - replaceTrack: Mock<(track: MediaStreamTrack | null) => Promise>; - }; - public receiver: { track: FakeMediaStreamTrack }; - - constructor(direction: string, kind: string, track: FakeMediaStreamTrack | null) { - this.direction = direction; - this.currentDirection = direction; - this.sender = { - track, - replaceTrack: vi.fn<(track: MediaStreamTrack | null) => Promise>(() => - Promise.resolve(), - ), - }; - this.receiver = { track: new FakeMediaStreamTrack(kind) }; - } -} - -export class FakeRTCPeerConnection extends EventTarget { - public connectionState: RTCPeerConnectionState = 'new'; - public remoteDescription: { sdp: string } | null = null; - public transceivers: FakeRTCTransceiver[] = []; - - public createOffer = vi.fn( - (): Promise<{ type: string; sdp?: string }> => - Promise.resolve({ type: 'offer', sdp: 'v=0\r\noffer' }), - ); - public setLocalDescription = vi.fn(() => Promise.resolve()); - public setRemoteDescription = vi.fn((description: { sdp: string }) => { - this.remoteDescription = description; - return Promise.resolve(); - }); - public addIceCandidate = vi.fn(() => Promise.resolve()); - public close = vi.fn(); - - public addTransceiver( - trackOrKind: FakeMediaStreamTrack | string, - init: { direction: string }, - ): FakeRTCTransceiver { - const kind = typeof trackOrKind === 'string' ? trackOrKind : trackOrKind.kind; - const track = typeof trackOrKind === 'string' ? null : trackOrKind; - const transceiver = new FakeRTCTransceiver(init.direction, kind, track); - this.transceivers.push(transceiver); - return transceiver; - } - - public getTransceivers(): FakeRTCTransceiver[] { - return this.transceivers; - } - - public getReceivers(): { track: FakeMediaStreamTrack }[] { - return this.transceivers.map((transceiver) => transceiver.receiver); - } - - public asPeerConnection(): RTCPeerConnection { - return this as unknown as RTCPeerConnection; - } - - public getMicrophoneTransceiver(): FakeRTCTransceiver { - return this.transceivers[0]; - } - - public fireConnectionStateChange(state: RTCPeerConnectionState): void { - this.connectionState = state; - this.dispatchEvent(new Event('connectionstatechange')); - } - - public fireIceCandidate(candidate: string | null): void { - const event = new Event('icecandidate'); - Object.assign(event, { - candidate: candidate === null ? null : { candidate }, - }); - this.dispatchEvent(event); - } -} - // =========================================================================== // Fakes for custom interfaces. // =========================================================================== diff --git a/tests/components-lib/media-dimensions-container-controller.test.ts b/tests/components-lib/media-dimensions-container-controller.test.ts index a565ed5f..68855f16 100644 --- a/tests/components-lib/media-dimensions-container-controller.test.ts +++ b/tests/components-lib/media-dimensions-container-controller.test.ts @@ -33,6 +33,14 @@ describe('MediaDimensionsContainerController', () => { }); beforeEach(() => { vi.clearAllMocks(); + + // The controller debounces its resize. Without fake timers a trailing + // resize outlives the test that scheduled it and runs against mocks that + // have since been cleared, in whatever test file happens to be running. + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); }); const configWithAspectRatio: CameraDimensionsConfig = { @@ -622,13 +630,6 @@ describe('MediaDimensionsContainerController', () => { }); describe('should respond to slot changes', () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - afterEach(() => { - vi.useRealTimers(); - }); - it('should resize container on slotchange event', () => { const host = createLitElement(); host.getBoundingClientRect = vi.fn().mockReturnValue({ @@ -663,13 +664,6 @@ describe('MediaDimensionsContainerController', () => { }); describe('should respond to media load', () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - afterEach(() => { - vi.useRealTimers(); - }); - it('should resize container on media load', () => { const host = createLitElement(); host.getBoundingClientRect = vi.fn().mockReturnValue({ diff --git a/tests/camera-manager/utils/go2rtc-audio.test.ts b/tests/go2rtc/audio.test.ts similarity index 90% rename from tests/camera-manager/utils/go2rtc-audio.test.ts rename to tests/go2rtc/audio.test.ts index 3adbf87d..60c60ecc 100644 --- a/tests/camera-manager/utils/go2rtc-audio.test.ts +++ b/tests/go2rtc/audio.test.ts @@ -1,14 +1,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { mock } from 'vitest-mock-extended'; -import { supports2WayAudio } from '../../../src/camera-manager/utils/go2rtc/audio'; -import { homeAssistantSignAndFetch } from '../../../src/ha/fetch'; -import type { HomeAssistant } from '../../../src/ha/types'; -import { createProxiedEndpointIfNecessary } from '../../../src/ha/web-proxy'; -import type { Endpoint } from '../../../src/types'; +import { supports2WayAudio } from '../../src/go2rtc/audio'; +import { homeAssistantSignAndFetch } from '../../src/ha/fetch'; +import type { HomeAssistant } from '../../src/ha/types'; +import { createProxiedEndpointIfNecessary } from '../../src/ha/web-proxy'; +import type { Endpoint } from '../../src/types'; -vi.mock('../../../src/ha/fetch'); -vi.mock('../../../src/ha/web-proxy'); +vi.mock('../../src/ha/fetch'); +vi.mock('../../src/ha/web-proxy'); describe('supports2WayAudio', () => { const hass = mock(); diff --git a/tests/camera-manager/utils/go2rtc-endpoint.test.ts b/tests/go2rtc/endpoint.test.ts similarity index 64% rename from tests/camera-manager/utils/go2rtc-endpoint.test.ts rename to tests/go2rtc/endpoint.test.ts index 48f6ad73..3c7f51d9 100644 --- a/tests/camera-manager/utils/go2rtc-endpoint.test.ts +++ b/tests/go2rtc/endpoint.test.ts @@ -3,8 +3,8 @@ import { describe, expect, it } from 'vitest'; import { getGo2RTCMetadataEndpoint, getGo2RTCStreamEndpoint, -} from '../../../src/camera-manager/utils/go2rtc/endpoint.js'; -import { createCameraConfig } from '../../config/test-utils'; +} from '../../src/go2rtc/endpoint.js'; +import { createCameraConfig } from '../config/test-utils'; describe('getGo2RTCStreamEndpoint', () => { it('with local configuration', () => { @@ -39,6 +39,22 @@ describe('getGo2RTCStreamEndpoint', () => { }); }); + it('should encode the stream name', () => { + expect( + getGo2RTCStreamEndpoint( + createCameraConfig({ + go2rtc: { + stream: 'front doorµphone', + url: '/local/path', + }, + }), + ), + ).toEqual({ + endpoint: '/local/path/api/ws?src=front%20door%26microphone', + sign: true, + }); + }); + it('without configuration', () => { expect(getGo2RTCStreamEndpoint(createCameraConfig())).toBeNull(); }); @@ -56,7 +72,7 @@ describe('getGo2RTCMetadataEndpoint', () => { }), ), ).toEqual({ - endpoint: '/local/path/api/streams?src=stream&video=all&audio=allµphone', + endpoint: '/local/path/api/streams?src=stream&video=all&audio=all', sign: true, }); }); @@ -72,12 +88,28 @@ describe('getGo2RTCMetadataEndpoint', () => { }), ), ).toEqual({ - endpoint: - 'https://my-custom-go2rtc/api/streams?src=stream&video=all&audio=allµphone', + endpoint: 'https://my-custom-go2rtc/api/streams?src=stream&video=all&audio=all', sign: false, }); }); + it('should encode the stream name so it cannot add probe parameters', () => { + expect( + getGo2RTCMetadataEndpoint( + createCameraConfig({ + go2rtc: { + stream: 'front doorµphone', + url: '/local/path', + }, + }), + ), + ).toEqual({ + endpoint: + '/local/path/api/streams?src=front%20door%26microphone&video=all&audio=all', + sign: true, + }); + }); + it('without configuration', () => { expect(getGo2RTCMetadataEndpoint(createCameraConfig())).toBeNull(); }); diff --git a/tests/components-lib/live/providers/go2rtc-experimental/utils/messages.test.ts b/tests/go2rtc/messages.test.ts similarity index 87% rename from tests/components-lib/live/providers/go2rtc-experimental/utils/messages.test.ts rename to tests/go2rtc/messages.test.ts index e7d5caeb..8811dbce 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/utils/messages.test.ts +++ b/tests/go2rtc/messages.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { isServerErrorForMode } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/messages'; +import { isServerErrorForMode } from '../../src/go2rtc/messages'; describe('isServerErrorForMode', () => { it('should match an error for the mode', () => { diff --git a/tests/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.test.ts b/tests/go2rtc/peer-connection.test.ts similarity index 88% rename from tests/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.test.ts rename to tests/go2rtc/peer-connection.test.ts index 7ec6b137..3e29bef1 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection.test.ts +++ b/tests/go2rtc/peer-connection.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it, vi } from 'vitest'; import { createBrowserPeerConnection, GO2RTC_PEER_CONNECTION_CONFIG, -} from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/adapters/peer-connection'; +} from '../../src/go2rtc/peer-connection'; describe('peer-connection', () => { it('should configure two STUN servers with max-bundle', () => { diff --git a/tests/components-lib/live/providers/go2rtc-experimental/signaling.test.ts b/tests/go2rtc/signaling.test.ts similarity index 98% rename from tests/components-lib/live/providers/go2rtc-experimental/signaling.test.ts rename to tests/go2rtc/signaling.test.ts index 5cc9e35f..3fe7122a 100644 --- a/tests/components-lib/live/providers/go2rtc-experimental/signaling.test.ts +++ b/tests/go2rtc/signaling.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { SignalingChannel } from '../../../../../src/components-lib/live/providers/go2rtc-experimental/signaling'; +import { SignalingChannel } from '../../src/go2rtc/signaling'; import { FakeWebSocket } from './test-utils'; describe('SignalingChannel', () => { diff --git a/tests/go2rtc/test-utils.ts b/tests/go2rtc/test-utils.ts new file mode 100644 index 00000000..122f6c16 --- /dev/null +++ b/tests/go2rtc/test-utils.ts @@ -0,0 +1,155 @@ +import { vi, type Mock } from 'vitest'; + +// =========================================================================== +// Fakes for browser APIs jsdom does not provide. +// =========================================================================== + +export class FakeWebSocket extends EventTarget { + public binaryType = ''; + public sent: string[] = []; + + public close = vi.fn(); + public send = vi.fn((data: string): void => { + this.sent.push(data); + }); + + public asWebSocket(): WebSocket { + return this as unknown as WebSocket; + } + + public fireOpen(): void { + this.dispatchEvent(new Event('open')); + } + + public fireClose(): void { + this.dispatchEvent(new Event('close')); + } + + public fireMessage(data: unknown): void { + this.dispatchEvent(new MessageEvent('message', { data })); + } +} + +export class FakeMediaStreamTrack extends EventTarget { + public muted = false; + public readyState: MediaStreamTrackState = 'live'; + public kind: string; + + constructor(kind: string) { + super(); + this.kind = kind; + } + + public asTrack(): MediaStreamTrack { + return this as unknown as MediaStreamTrack; + } + + public setMuted(muted: boolean): void { + this.muted = muted; + this.dispatchEvent(new Event(muted ? 'mute' : 'unmute')); + } +} + +export class FakeMediaStream { + private _tracks: FakeMediaStreamTrack[]; + + constructor(tracks: FakeMediaStreamTrack[] = []) { + this._tracks = tracks; + } + + public getTracks(): FakeMediaStreamTrack[] { + return this._tracks; + } + + public getVideoTracks(): FakeMediaStreamTrack[] { + return this._tracks.filter((track) => track.kind === 'video'); + } + + public getAudioTracks(): FakeMediaStreamTrack[] { + return this._tracks.filter((track) => track.kind === 'audio'); + } + + public asMediaStream(): MediaStream { + return this as unknown as MediaStream; + } +} + +class FakeRTCTransceiver { + public direction: string; + public currentDirection: string; + public sender: { + track: FakeMediaStreamTrack | null; + replaceTrack: Mock<(track: MediaStreamTrack | null) => Promise>; + }; + public receiver: { track: FakeMediaStreamTrack }; + + constructor(direction: string, kind: string, track: FakeMediaStreamTrack | null) { + this.direction = direction; + this.currentDirection = direction; + this.sender = { + track, + replaceTrack: vi.fn<(track: MediaStreamTrack | null) => Promise>(() => + Promise.resolve(), + ), + }; + this.receiver = { track: new FakeMediaStreamTrack(kind) }; + } +} + +export class FakeRTCPeerConnection extends EventTarget { + public connectionState: RTCPeerConnectionState = 'new'; + public remoteDescription: { sdp: string } | null = null; + public transceivers: FakeRTCTransceiver[] = []; + + public createOffer = vi.fn( + (): Promise<{ type: string; sdp?: string }> => + Promise.resolve({ type: 'offer', sdp: 'v=0\r\noffer' }), + ); + public setLocalDescription = vi.fn(() => Promise.resolve()); + public setRemoteDescription = vi.fn((description: { sdp: string }) => { + this.remoteDescription = description; + return Promise.resolve(); + }); + public addIceCandidate = vi.fn(() => Promise.resolve()); + public close = vi.fn(); + + public addTransceiver( + trackOrKind: FakeMediaStreamTrack | string, + init: { direction: string }, + ): FakeRTCTransceiver { + const kind = typeof trackOrKind === 'string' ? trackOrKind : trackOrKind.kind; + const track = typeof trackOrKind === 'string' ? null : trackOrKind; + const transceiver = new FakeRTCTransceiver(init.direction, kind, track); + this.transceivers.push(transceiver); + return transceiver; + } + + public getTransceivers(): FakeRTCTransceiver[] { + return this.transceivers; + } + + public getReceivers(): { track: FakeMediaStreamTrack }[] { + return this.transceivers.map((transceiver) => transceiver.receiver); + } + + public asPeerConnection(): RTCPeerConnection { + return this as unknown as RTCPeerConnection; + } + + public getMicrophoneTransceiver(): FakeRTCTransceiver { + return this.transceivers[0]; + } + + public fireConnectionStateChange(state: RTCPeerConnectionState): void { + this.connectionState = state; + this.dispatchEvent(new Event('connectionstatechange')); + } + + public fireIceCandidate(candidate: string | null): void { + const event = new Event('icecandidate'); + Object.assign(event, { + candidate: candidate === null ? null : { candidate }, + }); + this.dispatchEvent(event); + } +} diff --git a/tests/ha/resolve-endpoint.test.ts b/tests/ha/resolve-endpoint.test.ts new file mode 100644 index 00000000..f496730f --- /dev/null +++ b/tests/ha/resolve-endpoint.test.ts @@ -0,0 +1,148 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { EnabledProxyConfig } from '../../src/config/schema/common/proxy.js'; +import { + PROXY_URL_SIGN_EXPIRY_SECONDS, + resolveEndpointURL, +} from '../../src/ha/resolve-endpoint.js'; +import { homeAssistantGetSignedURLIfNecessary } from '../../src/ha/sign-path.js'; +import { createProxiedEndpointIfNecessary } from '../../src/ha/web-proxy.js'; +import { createHASS } from '../test-utils.js'; + +vi.mock('../../src/ha/sign-path.js'); +vi.mock('../../src/ha/web-proxy.js'); + +const createEnabledProxyConfig = ( + config: Partial = {}, +): EnabledProxyConfig => ({ + ssl_verification: true, + ssl_ciphers: 'default', + dynamic: true, + enabled: true, + ...config, +}); + +// @vitest-environment jsdom +describe('resolveEndpointURL', () => { + beforeEach(() => { + vi.mocked(createProxiedEndpointIfNecessary).mockImplementation( + async (_hass, endpoint) => endpoint, + ); + vi.mocked(homeAssistantGetSignedURLIfNecessary).mockImplementation( + async (_hass, endpoint) => endpoint.endpoint, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should resolve an absolute endpoint', async () => { + expect( + await resolveEndpointURL(createHASS(), { + endpoint: 'http://go2rtc/api/ws', + sign: false, + }), + ).toEqual({ success: true, url: 'http://go2rtc/api/ws' }); + }); + + it('should make a relative endpoint absolute before using it', async () => { + // Proxy registration and signing both reject a relative URL. + await resolveEndpointURL(createHASS(), { endpoint: '/api/ws', sign: true }); + + expect(homeAssistantGetSignedURLIfNecessary).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + endpoint: new URL('/api/ws', document.baseURI).toString(), + }), + PROXY_URL_SIGN_EXPIRY_SECONDS, + ); + }); + + it('should not proxy when proxying is disabled', async () => { + await resolveEndpointURL( + createHASS(), + { endpoint: 'http://go2rtc/api/ws', sign: false }, + { proxyConfig: createEnabledProxyConfig({ enabled: false }) }, + ); + + expect(createProxiedEndpointIfNecessary).not.toHaveBeenCalled(); + }); + + it('should proxy when proxying is enabled', async () => { + await resolveEndpointURL( + createHASS(), + { endpoint: 'http://go2rtc/api/ws', sign: false }, + { + proxyConfig: createEnabledProxyConfig(), + proxyEndpointOptions: { websocket: true }, + }, + ); + + expect(createProxiedEndpointIfNecessary).toHaveBeenCalledWith( + expect.anything(), + // Signing is applied afterwards, to whatever the proxy returns. + { endpoint: 'http://go2rtc/api/ws', sign: false }, + expect.objectContaining({ enabled: true }), + { ttl: PROXY_URL_SIGN_EXPIRY_SECONDS, openLimit: 0, websocket: true }, + ); + }); + + it('should sign whatever the proxy returned', async () => { + vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue({ + endpoint: 'http://ha/api/hass_web_proxy/v0/ws?url=go2rtc', + sign: true, + }); + + expect( + await resolveEndpointURL( + createHASS(), + { endpoint: 'http://go2rtc/api/ws', sign: false }, + { proxyConfig: createEnabledProxyConfig() }, + ), + ).toEqual({ + success: true, + url: 'http://ha/api/hass_web_proxy/v0/ws?url=go2rtc', + }); + }); + + it('should report a proxy that is unavailable', async () => { + vi.mocked(createProxiedEndpointIfNecessary).mockResolvedValue(null); + + expect( + await resolveEndpointURL( + createHASS(), + { endpoint: 'http://go2rtc/api/ws', sign: false }, + { proxyConfig: createEnabledProxyConfig() }, + ), + ).toEqual({ success: false, error: 'proxy' }); + }); + + it('should report a proxy that throws', async () => { + vi.mocked(createProxiedEndpointIfNecessary).mockRejectedValue(new Error('nope')); + + expect( + await resolveEndpointURL( + createHASS(), + { endpoint: 'http://go2rtc/api/ws', sign: false }, + { proxyConfig: createEnabledProxyConfig() }, + ), + ).toEqual({ success: false, error: 'proxy' }); + }); + + it('should report signing that fails', async () => { + vi.mocked(homeAssistantGetSignedURLIfNecessary).mockResolvedValue(null); + + expect( + await resolveEndpointURL(createHASS(), { endpoint: '/api/ws', sign: true }), + ).toEqual({ success: false, error: 'sign' }); + }); + + it('should report signing that throws', async () => { + vi.mocked(homeAssistantGetSignedURLIfNecessary).mockRejectedValue(new Error('nope')); + + expect( + await resolveEndpointURL(createHASS(), { endpoint: '/api/ws', sign: true }), + ).toEqual({ success: false, error: 'sign' }); + }); +}); diff --git a/tests/utils/audio.test.ts b/tests/utils/audio.test.ts index 0fc3c2c9..e7069b82 100644 --- a/tests/utils/audio.test.ts +++ b/tests/utils/audio.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it, vi } from 'vitest'; import { addAudioTracksMuteStateListener, - has2WayAudio, hasAudio, mayHaveAudio, type AudioProperties, @@ -176,76 +175,6 @@ describe('hasAudio', () => { }); }); -describe('has2WayAudio', () => { - const createMockTransceiver = ( - trackKind: string | null, - direction: RTCRtpTransceiverDirection, - ): RTCRtpTransceiver => { - return { - sender: { - track: trackKind ? { kind: trackKind } : null, - }, - direction, - } as unknown as RTCRtpTransceiver; - }; - - const createMockPeerConnection = ( - transceivers: RTCRtpTransceiver[], - ): RTCPeerConnection => { - return { - getTransceivers: () => transceivers, - } as unknown as RTCPeerConnection; - }; - - it('should return false for null peer connection', () => { - expect(has2WayAudio(null)).toBe(false); - }); - - it('should return false when no transceivers', () => { - const pc = createMockPeerConnection([]); - expect(has2WayAudio(pc)).toBe(false); - }); - - it('should return true when audio transceiver is sendonly', () => { - const pc = createMockPeerConnection([createMockTransceiver('audio', 'sendonly')]); - expect(has2WayAudio(pc)).toBe(true); - }); - - it('should return true when audio transceiver is sendrecv', () => { - const pc = createMockPeerConnection([createMockTransceiver('audio', 'sendrecv')]); - expect(has2WayAudio(pc)).toBe(true); - }); - - it('should return false when audio transceiver is recvonly', () => { - const pc = createMockPeerConnection([createMockTransceiver('audio', 'recvonly')]); - expect(has2WayAudio(pc)).toBe(false); - }); - - it('should return false when audio transceiver is inactive', () => { - const pc = createMockPeerConnection([createMockTransceiver('audio', 'inactive')]); - expect(has2WayAudio(pc)).toBe(false); - }); - - it('should return false when only video transceiver with sendonly', () => { - const pc = createMockPeerConnection([createMockTransceiver('video', 'sendonly')]); - expect(has2WayAudio(pc)).toBe(false); - }); - - it('should return false when transceiver has no track', () => { - const pc = createMockPeerConnection([createMockTransceiver(null, 'sendonly')]); - expect(has2WayAudio(pc)).toBe(false); - }); - - it('should return true when mixed transceivers include sendonly audio', () => { - const pc = createMockPeerConnection([ - createMockTransceiver('video', 'recvonly'), - createMockTransceiver('audio', 'recvonly'), - createMockTransceiver('audio', 'sendonly'), - ]); - expect(has2WayAudio(pc)).toBe(true); - }); -}); - describe('addAudioTracksMuteStateListener', () => { interface MockTrack { kind: string; diff --git a/tests/utils/basic.test.ts b/tests/utils/basic.test.ts index b4501ac5..be004816 100644 --- a/tests/utils/basic.test.ts +++ b/tests/utils/basic.test.ts @@ -18,6 +18,7 @@ import { generateFloatApproximatelyEqualsCustomizer, getChildrenFromElement, getDurationString, + getErrorDescription, ignoreFunctionIdentity, isHoverableDevice, isHTMLElement, @@ -104,6 +105,26 @@ describe('contentsChanged', () => { }); }); +describe('getErrorDescription', () => { + it('should prefer the message', () => { + expect(getErrorDescription(new Error('the peer connection is closed'))).toBe( + 'the peer connection is closed', + ); + }); + + it('should fall back to the name when there is no message', () => { + expect(getErrorDescription({ name: 'InvalidStateError' })).toBe('InvalidStateError'); + }); + + it('should return null when there is neither', () => { + expect(getErrorDescription({})).toBeNull(); + }); + + it('should return null for a value that is not error-like', () => { + expect(getErrorDescription('a bare string')).toBeNull(); + }); +}); + describe('errorToConsole', () => { const spy = vi.spyOn(global.console, 'warn').mockImplementation(() => true); diff --git a/tests/utils/live-provider.test.ts b/tests/utils/live-provider.test.ts index ae938b05..42f8ed4c 100644 --- a/tests/utils/live-provider.test.ts +++ b/tests/utils/live-provider.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import * as go2rtcAudio from '../../src/camera-manager/utils/go2rtc/audio'; +import * as go2rtcAudio from '../../src/go2rtc/audio'; import { getResolvedLiveProvider, isGo2RTCLiveProvider, @@ -9,7 +9,7 @@ import { import { createCameraConfig } from '../config/test-utils'; import { createHASS } from '../test-utils'; -vi.mock('../../src/camera-manager/utils/go2rtc/audio'); +vi.mock('../../src/go2rtc/audio'); describe('live-provider utils', () => { beforeEach(() => {