feat: Add experimental rewrite of go2rtc live provider (MSE/WebRTC/MP4/MJPEG) (#2580)
- Closes #2556 - Closes #2450 **Key intended features:** - go2rtc compatible - 100% test coverage to significantly improve ability to test, maintain and work around browser weirdnesses (e.g. Safari). - Written from the ground up in the style of the rest of the project. **To use:** - Change `live_provider` from `go2rtc` to `go2rtc-experimental`.
This commit is contained in:
Dermot Duffy
@@ -1,3 +1,4 @@
|
||||
import type { MediaUnavailableIssueReason } from '../../../../card-controller/issues/issues/media-unavailable';
|
||||
import type { LivenessDetector, LivenessVerdict } from '../stream-liveness-controller';
|
||||
|
||||
const LIVE_ERROR_EVENT = 'advanced-camera-card:live:error';
|
||||
@@ -38,12 +39,19 @@ export class ProviderErrorDetector implements LivenessDetector {
|
||||
return this._verdict;
|
||||
}
|
||||
|
||||
private _handler = (ev: Event): void => {
|
||||
private _handler = (
|
||||
ev: CustomEvent<MediaUnavailableIssueReason | undefined>,
|
||||
): void => {
|
||||
ev.stopPropagation();
|
||||
if (this._verdict.state !== 'not_live') {
|
||||
// Authoritative: an explicit provider error overrides even direct frame
|
||||
// evidence. No placeholder -- the provider renders its own error.
|
||||
this._verdict = { state: 'not_live', authority: 'hard', reason: 'playback_error' };
|
||||
// evidence. No placeholder -- the provider renders its own error. The
|
||||
// provider may name the cause; otherwise it is a generic playback error.
|
||||
this._verdict = {
|
||||
state: 'not_live',
|
||||
authority: 'hard',
|
||||
reason: ev.detail ?? 'playback_error',
|
||||
};
|
||||
this._onChange();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -41,10 +41,19 @@ export type LivenessVerdict =
|
||||
renderPlaceholder?: boolean;
|
||||
};
|
||||
|
||||
// The reconnecting placeholder the wrapper renders in place of a frozen
|
||||
// provider, carrying the cause so it can show a cause-specific message.
|
||||
interface LivenessPlaceholder {
|
||||
// The wrapper-facing projection of the internal LivenessVerdict: it keeps the
|
||||
// detector-internal `authority` from leaking out and collapses `unknown` into
|
||||
// "no failure", so a consumer sees only a confirmed failure (with its cause) or
|
||||
// nothing. Modeled as the failure rather than a `live` flag because a stream
|
||||
// that is merely still connecting is not a failure yet is not playing either, so
|
||||
// a positive `live` boolean would misleadingly read as "media is playing".
|
||||
interface StreamFailure {
|
||||
reason: MediaUnavailableIssueReason;
|
||||
|
||||
// Whether the wrapper should replace the provider with a reconnecting
|
||||
// placeholder (a silent freeze, e.g. an unavailable camera). False when the
|
||||
// provider renders its own error and should stay mounted.
|
||||
renderPlaceholder: boolean;
|
||||
}
|
||||
|
||||
export interface LivenessDetector {
|
||||
@@ -116,17 +125,15 @@ export class StreamLivenessController implements ReactiveController {
|
||||
}
|
||||
|
||||
public isLive(): boolean {
|
||||
return this._getVerdict().state !== 'not_live';
|
||||
return !this.getFailure();
|
||||
}
|
||||
|
||||
// The reconnecting placeholder to render in place of the (frozen) provider,
|
||||
// carrying the cause so the wrapper can show a cause-specific message. Null
|
||||
// when the provider should stay mounted (live, or a provider error that
|
||||
// renders its own error).
|
||||
public getPlaceholder(): LivenessPlaceholder | null {
|
||||
// The stream's confirmed liveness failure, or null when there is none (the
|
||||
// stream is live, or still connecting).
|
||||
public getFailure(): StreamFailure | null {
|
||||
const verdict = this._getVerdict();
|
||||
return verdict.state === 'not_live' && verdict.renderPlaceholder
|
||||
? { reason: verdict.reason }
|
||||
return verdict.state === 'not_live'
|
||||
? { reason: verdict.reason, renderPlaceholder: !!verdict.renderPlaceholder }
|
||||
: null;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# go2rtc-experimental provider
|
||||
|
||||
|
||||
|
||||
The SVG is also its own editable source: it embeds the draw.io diagram, so open
|
||||
`architecture.drawio.svg` directly in draw.io / diagrams.net to change it.
|
||||
|
||||
## Maintaining the diagram
|
||||
|
||||
`architecture.drawio.svg` is a dual file: the **rendered SVG** (shown above /
|
||||
in GitHub) plus the **editable mxGraph XML** embedded in its `content="..."`
|
||||
attribute. Keep them in sync -- never commit an SVG whose embedded XML changed
|
||||
but whose rendered picture did not, or the image above goes stale.
|
||||
@@ -0,0 +1,148 @@
|
||||
// Wraps the browser's two incompatible MediaSource "flavors" behind one
|
||||
// interface, so mse.ts stays flavor-agnostic (and mockable in tests):
|
||||
// - classic MediaSource (Chrome/Firefox/desktop): attach via an object URL
|
||||
// on `video.src`.
|
||||
// - ManagedMediaSource (Safari/iOS 17+; iOS had no MediaSource at all before
|
||||
// then): attach via `video.srcObject`, and needs `disableRemotePlayback`.
|
||||
import type { UnsubscribeCallback } from '../../../../../types';
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
// Safari 17+ managed variant of MediaSource; API-compatible for the
|
||||
// subset used here.
|
||||
ManagedMediaSource?: typeof MediaSource;
|
||||
}
|
||||
}
|
||||
|
||||
// The unified MediaSource surface mse.ts programs against; the flavor-specific
|
||||
// attach/detach lives in the implementations below.
|
||||
export interface MediaSourceInterface {
|
||||
attach(video: HTMLVideoElement): void;
|
||||
detach(video: HTMLVideoElement): void;
|
||||
|
||||
// Fires once the source is ready to accept SourceBuffers; nothing can be
|
||||
// appended before it.
|
||||
subscribeToSourceOpen(callback: () => void): UnsubscribeCallback;
|
||||
|
||||
// Creates the SourceBuffer that media chunks are appended to, configured for
|
||||
// the given codec MIME string.
|
||||
addSourceBuffer(codecs: string): SourceBuffer;
|
||||
|
||||
// Declares the seekable live window to the browser, which cannot infer it for
|
||||
// an open-ended live source.
|
||||
setLiveSeekableRange(startSeconds: number, endSeconds: number): void;
|
||||
|
||||
// Whether the source is still attached and accepting SourceBuffer operations.
|
||||
isOpen(): boolean;
|
||||
|
||||
// Whether a codec MIME string is playable.
|
||||
isTypeSupported(mimeType: string): boolean;
|
||||
}
|
||||
|
||||
// Creates a wrapped MediaSource, or null when the browser supports no variant.
|
||||
export type MediaSourceFactory = () => MediaSourceInterface | null;
|
||||
|
||||
abstract class MediaSourceInstanceBase implements MediaSourceInterface {
|
||||
protected _mediaSource: MediaSource;
|
||||
|
||||
constructor(mediaSource: MediaSource) {
|
||||
this._mediaSource = mediaSource;
|
||||
}
|
||||
|
||||
public abstract attach(video: HTMLVideoElement): void;
|
||||
public abstract detach(video: HTMLVideoElement): void;
|
||||
public abstract isTypeSupported(mimeType: string): boolean;
|
||||
|
||||
public subscribeToSourceOpen(callback: () => void): UnsubscribeCallback {
|
||||
this._mediaSource.addEventListener('sourceopen', callback);
|
||||
return () => this._mediaSource.removeEventListener('sourceopen', callback);
|
||||
}
|
||||
|
||||
public addSourceBuffer(codecs: string): SourceBuffer {
|
||||
return this._mediaSource.addSourceBuffer(codecs);
|
||||
}
|
||||
|
||||
public setLiveSeekableRange(startSeconds: number, endSeconds: number): void {
|
||||
this._mediaSource.setLiveSeekableRange(startSeconds, endSeconds);
|
||||
}
|
||||
|
||||
public isOpen(): boolean {
|
||||
return this._mediaSource.readyState === 'open';
|
||||
}
|
||||
}
|
||||
|
||||
class ManagedMediaSourceInstance extends MediaSourceInstanceBase {
|
||||
private _mediaSourceConstructor: typeof MediaSource;
|
||||
|
||||
constructor(mediaSourceConstructor: typeof MediaSource) {
|
||||
super(new mediaSourceConstructor());
|
||||
this._mediaSourceConstructor = mediaSourceConstructor;
|
||||
}
|
||||
|
||||
public attach(video: HTMLVideoElement): void {
|
||||
// ManagedMediaSource does not deliver data while remote playback (e.g.
|
||||
// AirPlay) is possible.
|
||||
video.disableRemotePlayback = true;
|
||||
video.srcObject = this._mediaSource;
|
||||
}
|
||||
|
||||
public detach(video: HTMLVideoElement): void {
|
||||
video.srcObject = null;
|
||||
}
|
||||
|
||||
public isTypeSupported(mimeType: string): boolean {
|
||||
return this._mediaSourceConstructor.isTypeSupported(mimeType);
|
||||
}
|
||||
}
|
||||
|
||||
class ClassicMediaSourceInstance extends MediaSourceInstanceBase {
|
||||
private _objectURL: string | null = null;
|
||||
|
||||
constructor() {
|
||||
super(new MediaSource());
|
||||
}
|
||||
|
||||
public attach(video: HTMLVideoElement): void {
|
||||
this._objectURL = URL.createObjectURL(this._mediaSource);
|
||||
|
||||
// The object URL is only needed until the browser has opened the media
|
||||
// source.
|
||||
this._mediaSource.addEventListener('sourceopen', () => this._revokeObjectURL(), {
|
||||
once: true,
|
||||
});
|
||||
|
||||
video.src = this._objectURL;
|
||||
video.srcObject = null;
|
||||
}
|
||||
|
||||
public detach(video: HTMLVideoElement): void {
|
||||
video.src = '';
|
||||
this._revokeObjectURL();
|
||||
}
|
||||
|
||||
public isTypeSupported(mimeType: string): boolean {
|
||||
return MediaSource.isTypeSupported(mimeType);
|
||||
}
|
||||
|
||||
private _revokeObjectURL(): void {
|
||||
if (this._objectURL) {
|
||||
URL.revokeObjectURL(this._objectURL);
|
||||
this._objectURL = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const createBrowserMediaSource: MediaSourceFactory = () => {
|
||||
// Safari exposes both flavors, so check Managed first and prefer it there: it
|
||||
// lets the browser throttle buffering to save battery and memory and hands
|
||||
// off cleanly to AirPlay, which classic MediaSource does not. Classic is the
|
||||
// fallback for other browsers.
|
||||
const managedMediaSource = window.ManagedMediaSource;
|
||||
if (managedMediaSource) {
|
||||
return new ManagedMediaSourceInstance(managedMediaSource);
|
||||
}
|
||||
if ('MediaSource' in window) {
|
||||
return new ClassicMediaSourceInstance();
|
||||
}
|
||||
return null;
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
export type PeerConnectionFactory = (config: RTCConfiguration) => RTCPeerConnection;
|
||||
|
||||
export const GO2RTC_PEER_CONNECTION_CONFIG: RTCConfiguration = {
|
||||
bundlePolicy: 'max-bundle',
|
||||
iceServers: [
|
||||
{
|
||||
// Two public STUN servers so connectivity survives one being unreachable.
|
||||
urls: ['stun:stun.l.google.com:19302', 'stun:stun.cloudflare.com:3478'],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const createBrowserPeerConnection: PeerConnectionFactory = (config) =>
|
||||
new RTCPeerConnection(config);
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 924 KiB |
@@ -0,0 +1,144 @@
|
||||
import type { LitElement } from 'lit';
|
||||
|
||||
import { Generation } from '../../../../utils/concurrency/generation';
|
||||
import { LatestValueRunner } from '../../../../utils/concurrency/latest-value-runner';
|
||||
import { ImageMediaPlayerController } from '../../../media-player/image';
|
||||
import { OffscreenImage } from './offscreen-image';
|
||||
import type { ImageSurface } from './session-controller';
|
||||
|
||||
// Liveness: while frames are expected, a gap beyond the window is a stall.
|
||||
// Omitted -> the surface reports no liveness. `stallWindowSeconds` defaults to
|
||||
// the standard frame-stall window.
|
||||
interface ImageSurfaceLivenessOptions {
|
||||
isFrameExpected: () => boolean;
|
||||
stallWindowSeconds?: number;
|
||||
}
|
||||
|
||||
interface ImageSurfaceOptions {
|
||||
livenessOptions?: ImageSurfaceLivenessOptions;
|
||||
|
||||
// Factory for the detached image loader used to decode a frame.
|
||||
createImage?: () => HTMLImageElement;
|
||||
}
|
||||
|
||||
// Presents a go2rtc MJPEG/MP4 stream as a sequence of frames on an <img>: each
|
||||
// frame is decoded off-DOM, then shown via an object URL (the previous one
|
||||
// revoked), and an ImageMediaPlayerController exposes it to the card as a media
|
||||
// player. The renderer owns the <img> element; this owns what is shown on it.
|
||||
export class ImageSurfaceController implements ImageSurface {
|
||||
private _getImageCallback: () => HTMLImageElement | null;
|
||||
private _mediaPlayerController: ImageMediaPlayerController;
|
||||
|
||||
// An off-screen <img> used to decode each frame before it is shown on the
|
||||
// visible element (reused: only one frame decodes at a time).
|
||||
private _decoder: OffscreenImage;
|
||||
|
||||
// Current frame's object URL.
|
||||
private _currentObjectURL: string | null = null;
|
||||
|
||||
// Invalidated by reset() so a frame whose decode was still in flight when the
|
||||
// surface was switched or torn down does not paint a stale frame afterwards.
|
||||
private _generation = new Generation();
|
||||
|
||||
// Frames can arrive faster than the browser decodes one. Present at most one
|
||||
// at a time and keep only the newest frame while busy, so the display
|
||||
// converges on the latest without unbounded decode work.
|
||||
private _frameRunner = new LatestValueRunner<Blob>((frame) =>
|
||||
this._presentFrame(frame),
|
||||
);
|
||||
|
||||
constructor(
|
||||
host: LitElement,
|
||||
getImageCallback: () => HTMLImageElement | null,
|
||||
options?: ImageSurfaceOptions,
|
||||
) {
|
||||
this._getImageCallback = getImageCallback;
|
||||
this._decoder = new OffscreenImage(options?.createImage);
|
||||
this._mediaPlayerController = new ImageMediaPlayerController(
|
||||
host,
|
||||
getImageCallback,
|
||||
{
|
||||
livenessOptions: options?.livenessOptions,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
public getElement(): HTMLImageElement | null {
|
||||
return this._getImageCallback();
|
||||
}
|
||||
|
||||
public getMediaPlayer(): ImageMediaPlayerController {
|
||||
return this._mediaPlayerController;
|
||||
}
|
||||
|
||||
// Show a new frame. A newer frame supersedes any still waiting to show; the
|
||||
// returned promise resolves once *a* frame has been presented.
|
||||
public showFrame(blob: Blob): Promise<void> {
|
||||
return this._frameRunner.submit(blob);
|
||||
}
|
||||
|
||||
private async _presentFrame(blob: Blob): Promise<void> {
|
||||
const image = this._getImageCallback();
|
||||
|
||||
// Drop frames for a detached element (e.g. a surface superseded
|
||||
// mid-teardown) so a retired surface never paints.
|
||||
if (!image || !image.isConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
const generation = this._generation.current();
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
// Decode the frame on the off-screen loader before showing it. Assigning an
|
||||
// undecoded object URL to the visible <img> makes WebKit repaint the
|
||||
// element empty until the decode completes, flashing the media background
|
||||
// between frames; decoding first lets the swap paint from cache (as it's
|
||||
// the same url) with no empty state.
|
||||
const decoder = this._decoder.get();
|
||||
decoder.src = url;
|
||||
try {
|
||||
await decoder.decode();
|
||||
} catch {
|
||||
// An undecodable frame is not fatal: drop it and keep the current one.
|
||||
URL.revokeObjectURL(url);
|
||||
return;
|
||||
}
|
||||
|
||||
// The surface may have been switched (reset) or detached during the decode;
|
||||
// painting now would show a stale frame on a retired surface.
|
||||
if (!this._generation.isCurrent(generation) || !image.isConnected) {
|
||||
URL.revokeObjectURL(url);
|
||||
return;
|
||||
}
|
||||
|
||||
image.src = url;
|
||||
|
||||
const previousURL = this._currentObjectURL;
|
||||
this._currentObjectURL = url;
|
||||
if (previousURL) {
|
||||
URL.revokeObjectURL(previousURL);
|
||||
}
|
||||
}
|
||||
|
||||
// Drop the current frame and its object URL (e.g. on a surface switch or
|
||||
// disconnect).
|
||||
public reset(): void {
|
||||
// Invalidate any frame still decoding so it cannot paint onto a surface
|
||||
// that has since been switched or disconnected.
|
||||
this._generation.invalidate();
|
||||
|
||||
// Drop any frame still waiting to show, for the same reason.
|
||||
this._frameRunner.clear();
|
||||
|
||||
if (this._currentObjectURL) {
|
||||
URL.revokeObjectURL(this._currentObjectURL);
|
||||
this._currentObjectURL = null;
|
||||
}
|
||||
this._decoder.clear();
|
||||
|
||||
const image = this._getImageCallback();
|
||||
if (image) {
|
||||
image.removeAttribute('src');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
type ImageElementFactory = () => HTMLImageElement;
|
||||
|
||||
// A held, injectable off-screen <img> for decoding a frame separate from the
|
||||
// visible display element. Created lazily on first `get()` and reused until
|
||||
// `clear()` detaches its source and forgets it.
|
||||
export class OffscreenImage {
|
||||
private _create: ImageElementFactory;
|
||||
private _image: HTMLImageElement | null = null;
|
||||
|
||||
constructor(create?: ImageElementFactory) {
|
||||
this._create = create ?? (() => new Image());
|
||||
}
|
||||
|
||||
// Return the held image, creating it on first use.
|
||||
public get(): HTMLImageElement {
|
||||
return (this._image ??= this._create());
|
||||
}
|
||||
|
||||
// Detach any source and forget the image. Safe to call when none is held.
|
||||
public clear(): void {
|
||||
if (this._image) {
|
||||
this._image.removeAttribute('src');
|
||||
this._image = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
export type VideoElementFactory = () => HTMLVideoElement;
|
||||
|
||||
// A held, injectable off-screen <video> for decoding a stream separate from the
|
||||
// real display element. Created lazily on first `get()` and reused until
|
||||
// `clear()` detaches its media and forgets it.
|
||||
export class OffscreenVideo {
|
||||
private _create: VideoElementFactory;
|
||||
private _video: HTMLVideoElement | null = null;
|
||||
|
||||
constructor(create?: VideoElementFactory) {
|
||||
this._create = create ?? (() => document.createElement('video'));
|
||||
}
|
||||
|
||||
// Return the held video, creating it on first use.
|
||||
public get(): HTMLVideoElement {
|
||||
return (this._video ??= this._create());
|
||||
}
|
||||
|
||||
// Detach any media (whether attached via `src` or `srcObject`) and forget the
|
||||
// video. Safe to call when none is held.
|
||||
public clear(): void {
|
||||
if (this._video) {
|
||||
this._video.removeAttribute('src');
|
||||
this._video.srcObject = null;
|
||||
this._video = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,721 @@
|
||||
import { isEqual } from 'lodash-es';
|
||||
|
||||
import { GO2RTC_MODES, type Go2RTCMode } from '../../../../config/schema/cameras';
|
||||
import type { CardWideConfig } from '../../../../config/schema/types';
|
||||
import type {
|
||||
MediaPlayerController,
|
||||
UntargetedMediaLoadedInfo,
|
||||
} from '../../../../types';
|
||||
import {
|
||||
addAudioTracksMuteStateListener,
|
||||
type AudioTracksMuteStateCleanup,
|
||||
} from '../../../../utils/audio';
|
||||
import {
|
||||
hideMediaControlsTemporarily,
|
||||
MEDIA_LOAD_CONTROLS_HIDE_SECONDS,
|
||||
} from '../../../../utils/controls';
|
||||
import { log } from '../../../../utils/debug';
|
||||
import { createMediaLoadedInfo } from '../../../../utils/media-info';
|
||||
import { RetryTimer } from '../../../../utils/retry-timer';
|
||||
import { convertToWebSocketURL } from '../../../../utils/websocket-url';
|
||||
import type { MediaSourceFactory } from './adapters/media-source';
|
||||
import type { PeerConnectionFactory } from './adapters/peer-connection';
|
||||
import { OffscreenVideo, type VideoElementFactory } from './offscreen-video';
|
||||
import { SignalingChannel, type WebSocketFactory } from './signaling';
|
||||
import {
|
||||
createBinarySource,
|
||||
createWebRTCSource,
|
||||
type BinarySource,
|
||||
type BinarySourceFactory,
|
||||
type WebRTCSourceFactory,
|
||||
} from './sources/factory';
|
||||
import type { MediaStreamFactory, WebRTCStreamSource } from './sources/webrtc';
|
||||
import type {
|
||||
Lane,
|
||||
StreamSource,
|
||||
StreamSourceContext,
|
||||
StreamSourceFailureReason,
|
||||
SurfaceKind,
|
||||
VideoStreamTarget,
|
||||
} from './types';
|
||||
import { getPreferredSource } from './utils/source-priority';
|
||||
|
||||
// Preference order for modes that stream binary media over the WebSocket; the
|
||||
// protocol permits only one such mode per connection, tried in this order with
|
||||
// fallback (real video first with MSE, then the MP4/MJPEG image fallbacks).
|
||||
// WebRTC is not here: it carries no WebSocket binary and runs in parallel with
|
||||
// the chosen binary mode. The configured modes select membership, not order.
|
||||
const BINARY_MODE_PRECEDENCE: readonly Go2RTCMode[] = ['mse', 'mp4', 'mjpeg'];
|
||||
|
||||
// A dropped connection is retried in place a few times, quickly, so a transient
|
||||
// blip (e.g. a brief network drop on an MSE stream) heals with no full player
|
||||
// remount (e.g. without the card IssueManager). After a short run of failures
|
||||
// the stream is treated as down: the failure is reported upward, handing
|
||||
// recovery to the card's managed retry (which shows a reconnecting indicator,
|
||||
// backs off, and can give up) instead of looping here silently. The attempt cap
|
||||
// keeps a persistently-failing stream from hammering the server.
|
||||
const RECONNECT_INTERVAL_SECONDS = 2;
|
||||
const RECONNECT_MAX_ATTEMPTS = 3;
|
||||
|
||||
// The two render surfaces the session commits onto. The component owns the
|
||||
// elements and the media-player controllers; the session drives which one is
|
||||
// live and reports it back via surfaceCommittedCallback.
|
||||
export interface VideoSurface {
|
||||
getElement(): HTMLVideoElement | null;
|
||||
getMediaPlayer(): MediaPlayerController;
|
||||
}
|
||||
|
||||
export interface ImageSurface {
|
||||
getElement(): HTMLImageElement | null;
|
||||
getMediaPlayer(): MediaPlayerController;
|
||||
|
||||
// Frames are handed over as Blobs through this callback rather than the
|
||||
// session setting `img.src` itself. The implementation (ImageSurfaceController)
|
||||
// creates each frame's object URL and revokes the previous one; the session
|
||||
// only ever holds Blobs, so it cannot leak URLs. The returned promise resolves
|
||||
// once the frame has decoded (real dimensions are then available).
|
||||
showFrame(frame: Blob): Promise<void>;
|
||||
reset(): void;
|
||||
}
|
||||
|
||||
export interface SessionSurfaces {
|
||||
video: VideoSurface;
|
||||
image: ImageSurface;
|
||||
}
|
||||
|
||||
interface Go2RTCSessionCallbacks {
|
||||
getControls: () => boolean;
|
||||
getCardWideConfig: () => CardWideConfig | null;
|
||||
mediaLoadedCallback: (info: UntargetedMediaLoadedInfo) => void;
|
||||
|
||||
// The lane that just committed is live on this surface (the component shows
|
||||
// it and hides the other).
|
||||
surfaceCommittedCallback: (surface: SurfaceKind) => void;
|
||||
|
||||
// The session has exhausted its own quick reconnects and cannot recover the
|
||||
// stream; a higher level should take over (e.g. the card's media-load retry).
|
||||
// The reason is the most recent source failure, or null when there is none
|
||||
// (e.g. the socket dropped with no source having reported a cause).
|
||||
errorCallback: (reason: StreamSourceFailureReason | null) => void;
|
||||
}
|
||||
|
||||
// Injectable platform and factory seams for tests. Every field defaults to
|
||||
// the real browser implementation; production passes none of them.
|
||||
interface Go2RTCSessionOptions {
|
||||
createBinarySource?: BinarySourceFactory;
|
||||
createWebRTCSource?: WebRTCSourceFactory;
|
||||
createWebSocket?: WebSocketFactory;
|
||||
createPeerConnection?: PeerConnectionFactory;
|
||||
createMediaStream?: MediaStreamFactory;
|
||||
createVideoElement?: VideoElementFactory;
|
||||
createMediaSource?: MediaSourceFactory;
|
||||
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
// One WebSocket connection attempt: its channel and URL, plus a reference to
|
||||
// the render surfaces to draw onto (the surfaces outlive any single attempt).
|
||||
// Handlers capture this rather than reading session fields so a stale
|
||||
// connection's events cannot act on a newer connection.
|
||||
interface ConnectionContext {
|
||||
channel: SignalingChannel;
|
||||
url: string;
|
||||
surfaces: SessionSurfaces;
|
||||
}
|
||||
|
||||
// Coordinates one go2rtc streaming session. A "binary" lane (MSE, MP4/MJPEG)
|
||||
// plays on the real video with sequential fallback; a parallel WebRTC lane
|
||||
// decodes into an off-screen video element and, if its stream scores
|
||||
// higher, takes over the real video. The session owns the signaling channel, the
|
||||
// bounded fast reconnects, and reporting loaded media.
|
||||
export class Go2RTCSessionController {
|
||||
private _callbacks: Go2RTCSessionCallbacks;
|
||||
private _options: Go2RTCSessionOptions | null;
|
||||
|
||||
private _channel: SignalingChannel | null = null;
|
||||
private _url: string | null = null;
|
||||
private _surfaces: SessionSurfaces | null = null;
|
||||
private _modes: readonly Go2RTCMode[] = GO2RTC_MODES;
|
||||
private _microphoneStream: MediaStream | null = null;
|
||||
|
||||
// Binary lane: the active source paired with the surface it renders on (mse
|
||||
// -> video, MP4/MJPEG -> image). Kept as one unit because the factory returns
|
||||
// them together and they are read together when the source commits. Null when
|
||||
// no binary source is running.
|
||||
private _binary: BinarySource | null = null;
|
||||
|
||||
// Queue of fallback modes not yet tried on this connection, in precedence
|
||||
// order. Starting a binary source consumes the head; teardown empties the
|
||||
// queue so no further fallback is attempted.
|
||||
private _binaryModes: Go2RTCMode[] = [];
|
||||
|
||||
// WebRTC lane.
|
||||
private _webRTCSource: WebRTCStreamSource | null = null;
|
||||
|
||||
// Holds the off-screen video element the WebRTC stream decodes into while the
|
||||
// WebRTC lane races a binary lane.
|
||||
private _offscreenVideo: OffscreenVideo;
|
||||
|
||||
// The committed lane: the one whose media is live (on the video or image
|
||||
// surface) and has reported as loaded; null before anything loads or once the
|
||||
// lanes are torn down. Failure handling reads it to distinguish a committed
|
||||
// (live) stream dying from a losing racer being dropped.
|
||||
private _committedLane: Lane | null = null;
|
||||
|
||||
// The surface currently showing committed media; null before anything commits
|
||||
// or once torn down. On a surface switch (e.g. MSE falls back to MP4, or a
|
||||
// WebRTC win over an MJPEG image) the outgoing surface is reset.
|
||||
private _committedSurface: SurfaceKind | null = null;
|
||||
|
||||
// The source whose media is committed; null before anything commits or once
|
||||
// torn down. A source can re-report loaded (e.g. a <video> re-fires
|
||||
// loadeddata on a mid-stream resolution change), so this distinguishes a
|
||||
// fresh commit from a repeated load.
|
||||
private _committedSource: StreamSource | null = null;
|
||||
|
||||
// Unsubscriber for the listener on the committed WebRTC stream's audio
|
||||
// tracks. Should a WebRTC track's mute state change after load, the listener
|
||||
// sends the card a fresh media-loaded info so its audio capabilities reflect
|
||||
// the stream's current audio, not the state at load time. Null when no WebRTC
|
||||
// stream is committed; invoked and cleared on teardown and before re-arming
|
||||
// so a retired stream's listener can never fire.
|
||||
private _audioTracksMuteStateUnsubscribeCallback: AudioTracksMuteStateCleanup = null;
|
||||
|
||||
// The most recent source failure on this connection, handed to the error
|
||||
// callback when the session finally gives up so the card can name the cause.
|
||||
// Null before any failure and after a healthy commit.
|
||||
private _lastFailureReason: StreamSourceFailureReason | null = null;
|
||||
|
||||
private _retryTimer = new RetryTimer(RECONNECT_INTERVAL_SECONDS);
|
||||
|
||||
constructor(callbacks: Go2RTCSessionCallbacks, options?: Go2RTCSessionOptions) {
|
||||
this._callbacks = callbacks;
|
||||
this._options = options ?? null;
|
||||
|
||||
this._offscreenVideo = new OffscreenVideo(this._options?.createVideoElement);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Public API.
|
||||
// ===========================================================================
|
||||
|
||||
// The `surfaces` object identifies the session target: the same object means
|
||||
// "keep the established session" (so callers may invoke this on every render),
|
||||
// a new object means "reset and reconnect". Callers must therefore hold one
|
||||
// stable surfaces object for as long as the target is unchanged, and hand over
|
||||
// a fresh one (or `reset()` first) if they ever remount the underlying
|
||||
// elements.
|
||||
public connect(url: string, surfaces: SessionSurfaces, modes?: Go2RTCMode[]): void {
|
||||
const normalizedModes: readonly Go2RTCMode[] = modes?.length ? modes : GO2RTC_MODES;
|
||||
|
||||
if (
|
||||
this._url === url &&
|
||||
this._surfaces === surfaces &&
|
||||
isEqual(this._modes, normalizedModes)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.reset();
|
||||
|
||||
this._url = url;
|
||||
this._surfaces = surfaces;
|
||||
this._modes = normalizedModes;
|
||||
|
||||
this._connectChannel(url, surfaces);
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this._retryTimer.reset();
|
||||
this._lastFailureReason = null;
|
||||
|
||||
this._teardownLanes();
|
||||
this._channel?.close();
|
||||
this._channel = null;
|
||||
|
||||
const video = this._surfaces?.video.getElement();
|
||||
if (video) {
|
||||
video.srcObject = null;
|
||||
video.src = '';
|
||||
}
|
||||
this._surfaces?.image.reset();
|
||||
|
||||
this._url = null;
|
||||
this._surfaces = null;
|
||||
}
|
||||
|
||||
public setMicrophoneStream(stream: MediaStream | null): void {
|
||||
this._microphoneStream = stream;
|
||||
this._webRTCSource?.setMicrophoneStream(stream).catch(() => {});
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Session teardown.
|
||||
// ===========================================================================
|
||||
|
||||
private _teardownLanes(): void {
|
||||
this._teardownBinaryLane();
|
||||
this._teardownWebRTCLane();
|
||||
this._committedLane = null;
|
||||
this._committedSurface = null;
|
||||
this._committedSource = null;
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Signaling channel lifecycle.
|
||||
// ===========================================================================
|
||||
|
||||
private _connectChannel(url: string, surfaces: SessionSurfaces): void {
|
||||
// The open/close callbacks reference `channel`, the very constant this
|
||||
// `new` expression is being assigned to. That is safe because the callbacks
|
||||
// fire on asynchronous WebSocket events, which cannot happen before the
|
||||
// assignment completes.
|
||||
const channel: SignalingChannel = new SignalingChannel(
|
||||
convertToWebSocketURL(url),
|
||||
{
|
||||
openCallback: () => this._handleOpen({ channel, url, surfaces }),
|
||||
disconnectCallback: () => this._handleClose({ channel, url, surfaces }),
|
||||
},
|
||||
{ createWebSocket: this._options?.createWebSocket },
|
||||
);
|
||||
this._channel = channel;
|
||||
channel.connect();
|
||||
}
|
||||
|
||||
private _handleOpen(context: ConnectionContext): void {
|
||||
this._binaryModes = BINARY_MODE_PRECEDENCE.filter((mode) =>
|
||||
this._modes.includes(mode),
|
||||
);
|
||||
|
||||
// While a binary lane races, WebRTC decodes into a separate off-screen video
|
||||
// element (see OffscreenVideo) where its stream can be evaluated without
|
||||
// contending for the real <video> that an MSE binary lane plays on. With no
|
||||
// binary lane the real video element is free, so WebRTC attaches to it
|
||||
// directly; that also avoids decoding into an element outside the document,
|
||||
// which has been implicated in Firefox WebRTC failures. See
|
||||
// https://github.com/dermotduffy/advanced-camera-card/issues/2222
|
||||
//
|
||||
// Every configured mode is a binary mode or webrtc, and empty modes default
|
||||
// to all, so at least one lane always starts.
|
||||
//
|
||||
// Start WebRTC before the binary lane: starting a binary source can fail
|
||||
// synchronously (e.g. an unsupported codec drains the whole mode queue),
|
||||
// which would trigger a premature reconnect while no lane is live yet and
|
||||
// leave a stale WebRTC source running on a closed channel. Starting WebRTC
|
||||
// first means that reconnect check always sees a live lane.
|
||||
const hasBinaryModes = this._binaryModes.length > 0;
|
||||
if (this._modes.includes('webrtc')) {
|
||||
this._startWebRTCSource(context, hasBinaryModes);
|
||||
}
|
||||
if (hasBinaryModes) {
|
||||
this._startNextBinarySource(context);
|
||||
}
|
||||
}
|
||||
|
||||
private _handleClose(context: ConnectionContext): void {
|
||||
this._teardownLanes();
|
||||
this._channel = null;
|
||||
this._reconnectOrEscalateError(context);
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Binary lane.
|
||||
// ===========================================================================
|
||||
|
||||
private _startNextBinarySource(context: ConnectionContext): void {
|
||||
const mode = this._binaryModes.shift();
|
||||
if (!mode) {
|
||||
this._maybeReconnectIfLanesDead(context);
|
||||
return;
|
||||
}
|
||||
|
||||
// No element to render onto (the surface was detached); abandon the lane.
|
||||
const video = context.surfaces.video.getElement();
|
||||
if (!video) {
|
||||
return;
|
||||
}
|
||||
|
||||
// The callbacks capture the source's own identity so a retired source
|
||||
// (stopped, replaced or reset) cannot act on the session. The variable is
|
||||
// declared before the factory call so callbacks fired during construction
|
||||
// see null and are ignored.
|
||||
let source: StreamSource | null = null;
|
||||
|
||||
const binarySource = (this._options?.createBinarySource ?? createBinarySource)(
|
||||
mode,
|
||||
{
|
||||
video: { kind: 'video', video },
|
||||
image: {
|
||||
kind: 'image',
|
||||
showFrame: (frame) => context.surfaces.image.showFrame(frame),
|
||||
},
|
||||
},
|
||||
context.channel,
|
||||
{
|
||||
loadedCallback: () => {
|
||||
if (source) {
|
||||
this._handleBinaryLoaded(context, source);
|
||||
}
|
||||
},
|
||||
failedCallback: (reason: StreamSourceFailureReason) => {
|
||||
if (source) {
|
||||
this._lastFailureReason = reason;
|
||||
this._logSourceFailure('binary', reason, mode);
|
||||
this._handleBinaryFailed(context, source);
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
createMediaSource: this._options?.createMediaSource,
|
||||
userAgent: this._options?.userAgent,
|
||||
},
|
||||
);
|
||||
if (!binarySource) {
|
||||
this._startNextBinarySource(context);
|
||||
return;
|
||||
}
|
||||
|
||||
source = binarySource.source;
|
||||
this._binary = binarySource;
|
||||
source.start();
|
||||
}
|
||||
|
||||
private _handleBinaryLoaded(context: ConnectionContext, source: StreamSource): void {
|
||||
const binary = this._binary;
|
||||
if (!binary || source !== binary.source) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._committedLane = 'binary';
|
||||
this._reportCommittedLoad(context, binary.surface, source);
|
||||
}
|
||||
|
||||
private _handleBinaryFailed(context: ConnectionContext, source: StreamSource): void {
|
||||
if (source !== this._binary?.source) {
|
||||
return;
|
||||
}
|
||||
|
||||
source.stop();
|
||||
this._binary = null;
|
||||
|
||||
if (this._committedLane === 'binary') {
|
||||
this._committedLane = null;
|
||||
}
|
||||
|
||||
this._startNextBinarySource(context);
|
||||
}
|
||||
|
||||
private _teardownBinaryLane(): void {
|
||||
// Clear the current-source identity before stopping, so any callback fired
|
||||
// synchronously by stop() fails its `!== this._binary?.source` guard.
|
||||
const source = this._binary?.source;
|
||||
this._binary = null;
|
||||
this._binaryModes = [];
|
||||
|
||||
source?.stop();
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// WebRTC lane.
|
||||
// ===========================================================================
|
||||
|
||||
private _startWebRTCSource(
|
||||
context: ConnectionContext,
|
||||
useOffscreenVideo: boolean,
|
||||
): void {
|
||||
// While racing a binary lane, decode into the off-screen element; otherwise
|
||||
// decode straight onto the real video (absent it, there is nothing to do).
|
||||
let decodeVideo: HTMLVideoElement;
|
||||
if (useOffscreenVideo) {
|
||||
decodeVideo = this._offscreenVideo.get();
|
||||
} else {
|
||||
const video = context.surfaces.video.getElement();
|
||||
if (!video) {
|
||||
return;
|
||||
}
|
||||
decodeVideo = video;
|
||||
}
|
||||
|
||||
let source: WebRTCStreamSource | null = null;
|
||||
const sourceContext: StreamSourceContext<VideoStreamTarget> = {
|
||||
target: { kind: 'video', video: decodeVideo },
|
||||
channel: context.channel,
|
||||
callbacks: {
|
||||
loadedCallback: () => {
|
||||
if (source) {
|
||||
this._handleWebRTCLoaded(context, source, useOffscreenVideo, decodeVideo);
|
||||
}
|
||||
},
|
||||
failedCallback: (reason) => {
|
||||
if (source) {
|
||||
this._lastFailureReason = reason;
|
||||
this._logSourceFailure('webrtc', reason);
|
||||
this._handleWebRTCFailed(context, source);
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
source = (this._options?.createWebRTCSource ?? createWebRTCSource)(sourceContext, {
|
||||
microphoneStream: this._microphoneStream,
|
||||
createPeerConnection: this._options?.createPeerConnection,
|
||||
createMediaStream: this._options?.createMediaStream,
|
||||
});
|
||||
|
||||
this._webRTCSource = source;
|
||||
source.start();
|
||||
}
|
||||
|
||||
private _handleWebRTCLoaded(
|
||||
context: ConnectionContext,
|
||||
source: WebRTCStreamSource,
|
||||
useOffscreenVideo: boolean,
|
||||
decodeVideo: HTMLVideoElement,
|
||||
): void {
|
||||
if (source !== this._webRTCSource) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!useOffscreenVideo) {
|
||||
// There is nothing in the 'binary' lane, this controller is WebRTC-only:
|
||||
// the stream already decoded on the real video element.
|
||||
this._commitWebRTC(context, source, decodeVideo);
|
||||
return;
|
||||
}
|
||||
|
||||
// There is video on both the 'binary' and 'webrtc' lane -- choose a winner.
|
||||
const binary = this._binary?.source ?? null;
|
||||
const winner = binary
|
||||
? getPreferredSource(source.getStreamProfile(), binary.getStreamProfile())
|
||||
: 'webrtc';
|
||||
|
||||
if (winner === 'binary') {
|
||||
// The binary source scored higher: keep it on the real video element and
|
||||
// tear down the WebRTC lane.
|
||||
this._teardownWebRTCLane();
|
||||
return;
|
||||
}
|
||||
|
||||
// WebRTC wins. Tear down the binary lane first: stopping the binary source
|
||||
// clears the real video element it was playing on, so the WebRTC stream is
|
||||
// attached to that element (below) only after, never before.
|
||||
this._teardownBinaryLane();
|
||||
const stream = source.getMediaStream();
|
||||
const video = context.surfaces.video.getElement();
|
||||
if (video && stream) {
|
||||
video.srcObject = stream;
|
||||
}
|
||||
|
||||
// Report the media-loaded frame size from the off-screen video element
|
||||
// (decodeVideo): it has been decoding this stream throughout the
|
||||
// lane-choice so it already knows the size, whereas the real video element
|
||||
// was only handed the stream on the line above and has not decoded a frame
|
||||
// yet (so it would report 0x0).
|
||||
this._commitWebRTC(context, source, decodeVideo);
|
||||
this._offscreenVideo.clear();
|
||||
}
|
||||
|
||||
private _handleWebRTCFailed(
|
||||
context: ConnectionContext,
|
||||
source: WebRTCStreamSource,
|
||||
): void {
|
||||
if (source !== this._webRTCSource) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wasCommitted = this._committedLane === 'webrtc';
|
||||
this._teardownWebRTCLane();
|
||||
|
||||
if (wasCommitted) {
|
||||
// WebRTC was the committed (live) lane, so losing it leaves the view with
|
||||
// nothing playing: clear the real video element and force a full
|
||||
// reconnect (which restarts every configured mode, letting a binary lane
|
||||
// take over) or escalate via the error callback.
|
||||
this._committedLane = null;
|
||||
const video = context.surfaces.video.getElement();
|
||||
if (video) {
|
||||
video.srcObject = null;
|
||||
}
|
||||
|
||||
this._closeChannelAndReconnect(context);
|
||||
} else {
|
||||
// WebRTC lost a race it was still contesting; the binary lane carries on
|
||||
// if present, otherwise reconnect.
|
||||
this._maybeReconnectIfLanesDead(context);
|
||||
}
|
||||
}
|
||||
|
||||
private _commitWebRTC(
|
||||
context: ConnectionContext,
|
||||
source: WebRTCStreamSource,
|
||||
dimensionsVideo: HTMLVideoElement,
|
||||
): void {
|
||||
this._committedLane = 'webrtc';
|
||||
|
||||
// WebRTC now carries the media over the peer connection, so the socket is
|
||||
// done: signaling (offer/answer/ICE) is complete and any binary lane's
|
||||
// local consumer is torn down. go2rtc has no "stop" message, though --
|
||||
// tearing a lane down only detaches our side, so the server keeps muxing
|
||||
// that mode's binary frames down the socket (onto a now-discarded consumer)
|
||||
// until the socket itself closes. Closing it is what actually ends that
|
||||
// server stream.
|
||||
this._channel?.close();
|
||||
this._channel = null;
|
||||
|
||||
this._setupAudioMuteRedispatch(context, source);
|
||||
this._reportCommittedLoad(context, 'video', source, dimensionsVideo);
|
||||
}
|
||||
|
||||
private _setupAudioMuteRedispatch(
|
||||
context: ConnectionContext,
|
||||
source: WebRTCStreamSource,
|
||||
): void {
|
||||
this._audioTracksMuteStateUnsubscribeCallback?.();
|
||||
|
||||
// The listener is removed on teardown, so a stale source can never fire it.
|
||||
// Audio mute/unmute re-dispatches read the real video, which holds the
|
||||
// stream (and its dimensions) by the time any such change occurs.
|
||||
this._audioTracksMuteStateUnsubscribeCallback = addAudioTracksMuteStateListener(
|
||||
source.getPeerConnection(),
|
||||
() => this._dispatchMediaLoaded(context, 'video', source),
|
||||
);
|
||||
}
|
||||
|
||||
private _teardownWebRTCLane(): void {
|
||||
// Clear the current-source identity and the audio listener before stopping,
|
||||
// so any callback fired synchronously by stop() fails its guard or is gone.
|
||||
const source = this._webRTCSource;
|
||||
this._webRTCSource = null;
|
||||
|
||||
this._audioTracksMuteStateUnsubscribeCallback?.();
|
||||
this._audioTracksMuteStateUnsubscribeCallback = null;
|
||||
|
||||
this._offscreenVideo.clear();
|
||||
|
||||
source?.stop();
|
||||
}
|
||||
|
||||
// A source in one of the lanes failed.
|
||||
private _logSourceFailure(
|
||||
lane: Lane,
|
||||
reason: StreamSourceFailureReason,
|
||||
mode?: Go2RTCMode,
|
||||
): void {
|
||||
log(this._callbacks.getCardWideConfig(), 'go2rtc-experimental source failed', {
|
||||
lane,
|
||||
...(mode ? { mode } : {}),
|
||||
reason,
|
||||
});
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Reconnect & escalation.
|
||||
// ===========================================================================
|
||||
|
||||
private _maybeReconnectIfLanesDead(context: ConnectionContext): void {
|
||||
if (!this._binary && !this._webRTCSource && !this._binaryModes.length) {
|
||||
this._closeChannelAndReconnect(context);
|
||||
}
|
||||
}
|
||||
|
||||
// Abandon this connection's channel and start the reconnect-or-escalate flow.
|
||||
private _closeChannelAndReconnect(context: ConnectionContext): void {
|
||||
context.channel.close();
|
||||
this._channel = null;
|
||||
|
||||
this._reconnectOrEscalateError(context);
|
||||
}
|
||||
|
||||
private _reconnectOrEscalateError(context: ConnectionContext): void {
|
||||
if (this._retryTimer.getAttempts() >= RECONNECT_MAX_ATTEMPTS) {
|
||||
this._callbacks.errorCallback(this._lastFailureReason);
|
||||
return;
|
||||
}
|
||||
this._retryTimer.schedule(() => this._connectChannel(context.url, context.surfaces));
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Media-loaded reporting.
|
||||
// ===========================================================================
|
||||
|
||||
// A lane just committed with live media: the connection has proven healthy, so
|
||||
// reset the fast-reconnect budget, hand the newly live surface to the
|
||||
// component (resetting the one being left behind), briefly hide the video
|
||||
// controls so they do not flash over the freshly loaded media, then report the
|
||||
// load with the committed surface's controller.
|
||||
private _reportCommittedLoad(
|
||||
context: ConnectionContext,
|
||||
surface: SurfaceKind,
|
||||
source: StreamSource,
|
||||
dimensionsVideo?: HTMLVideoElement,
|
||||
): void {
|
||||
if (source === this._committedSource) {
|
||||
// The same source re-reporting (e.g. a mid-stream resolution change
|
||||
// re-fires the video's loadeddata): the surface and controls are already
|
||||
// set up, so only refresh the reported media dimensions. Re-running the
|
||||
// control hide would leak its loadstart listener on each repeat.
|
||||
this._dispatchMediaLoaded(context, surface, source, dimensionsVideo);
|
||||
return;
|
||||
}
|
||||
this._committedSource = source;
|
||||
|
||||
this._retryTimer.reset();
|
||||
this._lastFailureReason = null;
|
||||
|
||||
if (this._committedSurface && this._committedSurface !== surface) {
|
||||
this._resetSurface(context, this._committedSurface);
|
||||
}
|
||||
this._committedSurface = surface;
|
||||
this._callbacks.surfaceCommittedCallback(surface);
|
||||
|
||||
const video = context.surfaces.video.getElement();
|
||||
|
||||
if (surface === 'video' && this._callbacks.getControls() && video) {
|
||||
hideMediaControlsTemporarily(video, MEDIA_LOAD_CONTROLS_HIDE_SECONDS);
|
||||
}
|
||||
|
||||
this._dispatchMediaLoaded(context, surface, source, dimensionsVideo);
|
||||
}
|
||||
|
||||
private _resetSurface(context: ConnectionContext, surface: SurfaceKind): void {
|
||||
if (surface === 'image') {
|
||||
context.surfaces.image.reset();
|
||||
return;
|
||||
}
|
||||
const video = context.surfaces.video.getElement();
|
||||
if (video) {
|
||||
video.srcObject = null;
|
||||
video.src = '';
|
||||
}
|
||||
}
|
||||
|
||||
private _dispatchMediaLoaded(
|
||||
context: ConnectionContext,
|
||||
surface: SurfaceKind,
|
||||
source: StreamSource,
|
||||
// Element to read the frame size from; used only for the video surface and
|
||||
// defaults to the video-surface element. Differs when that element was just
|
||||
// handed a stream and has not decoded a frame yet (WebRTC committed from the
|
||||
// off-screen video), so its size must come from that off-screen element.
|
||||
dimensionsVideo?: HTMLVideoElement,
|
||||
): void {
|
||||
const targetSurface =
|
||||
surface === 'video' ? context.surfaces.video : context.surfaces.image;
|
||||
const dimensionsElement =
|
||||
surface === 'video'
|
||||
? dimensionsVideo ?? context.surfaces.video.getElement()
|
||||
: context.surfaces.image.getElement();
|
||||
if (!dimensionsElement) {
|
||||
return;
|
||||
}
|
||||
|
||||
const info = createMediaLoadedInfo(dimensionsElement, {
|
||||
mediaPlayerController: targetSurface.getMediaPlayer(),
|
||||
capabilities: source.getCapabilities(),
|
||||
technology: source.getTechnology(),
|
||||
});
|
||||
if (info) {
|
||||
this._callbacks.mediaLoadedCallback(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { UnsubscribeCallback } from '../../../../types';
|
||||
import {
|
||||
go2RTCMessageSchema,
|
||||
type BinaryCallback,
|
||||
type Go2RTCMessage,
|
||||
type MessageCallback,
|
||||
} from './types';
|
||||
|
||||
export type WebSocketFactory = (url: string) => WebSocket;
|
||||
|
||||
interface SignalingChannelCallbacks {
|
||||
openCallback?: () => void;
|
||||
|
||||
// Fires when the socket disconnects (i.e. not intentional `close()`).
|
||||
disconnectCallback?: () => void;
|
||||
}
|
||||
|
||||
// Injectable test seam. Defaults to the real `WebSocket`; production passes
|
||||
// nothing here.
|
||||
interface SignalingChannelOptions {
|
||||
createWebSocket?: WebSocketFactory;
|
||||
}
|
||||
|
||||
// Owns the go2rtc WebSocket: JSON text frames are parsed and fanned out to
|
||||
// message subscribers, binary frames go to the single binary consumer. No
|
||||
// callbacks of any kind are delivered after `close()`.
|
||||
export class SignalingChannel {
|
||||
private _url: string;
|
||||
|
||||
private _ws: WebSocket | null = null;
|
||||
|
||||
private _open = false;
|
||||
|
||||
private _callbacks: SignalingChannelCallbacks;
|
||||
private _createWebSocket: WebSocketFactory;
|
||||
private _binaryCallback: BinaryCallback | null = null;
|
||||
|
||||
private _messageCallbacks = new Set<MessageCallback>();
|
||||
|
||||
constructor(
|
||||
url: string,
|
||||
callbacks: SignalingChannelCallbacks,
|
||||
options?: SignalingChannelOptions,
|
||||
) {
|
||||
this._url = url;
|
||||
this._callbacks = callbacks;
|
||||
this._createWebSocket =
|
||||
options?.createWebSocket ?? ((wsURL: string) => new WebSocket(wsURL));
|
||||
}
|
||||
|
||||
public connect(): void {
|
||||
if (this._ws) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ws = this._createWebSocket(this._url);
|
||||
this._ws = ws;
|
||||
|
||||
ws.binaryType = 'arraybuffer';
|
||||
|
||||
// Each listener ignores events once this websocket is no longer current
|
||||
// (e.g. events already queued when `close()` was called).
|
||||
ws.addEventListener('open', () => {
|
||||
if (this._ws !== ws) {
|
||||
return;
|
||||
}
|
||||
this._open = true;
|
||||
this._callbacks.openCallback?.();
|
||||
});
|
||||
|
||||
ws.addEventListener('close', () => {
|
||||
if (this._ws !== ws) {
|
||||
return;
|
||||
}
|
||||
this._ws = null;
|
||||
this._open = false;
|
||||
|
||||
// This will not be called when we intentionally call close(), as the
|
||||
// guard above will have nulled out this._ws.
|
||||
this._callbacks.disconnectCallback?.();
|
||||
});
|
||||
|
||||
ws.addEventListener('message', (ev) => {
|
||||
if (this._ws !== ws) {
|
||||
return;
|
||||
}
|
||||
this._handleMessage(ev.data);
|
||||
});
|
||||
}
|
||||
|
||||
public close(): void {
|
||||
const ws = this._ws;
|
||||
this._ws = null;
|
||||
this._open = false;
|
||||
|
||||
ws?.close();
|
||||
}
|
||||
|
||||
public isOpen(): boolean {
|
||||
return this._open;
|
||||
}
|
||||
|
||||
public send(message: Go2RTCMessage): void {
|
||||
if (this._ws && this._open) {
|
||||
this._ws.send(JSON.stringify(message));
|
||||
}
|
||||
}
|
||||
|
||||
public subscribeToMessages(callback: MessageCallback): UnsubscribeCallback {
|
||||
this._messageCallbacks.add(callback);
|
||||
|
||||
return () => {
|
||||
this._messageCallbacks.delete(callback);
|
||||
};
|
||||
}
|
||||
|
||||
public setBinaryCallback(callback: BinaryCallback | null): void {
|
||||
this._binaryCallback = callback;
|
||||
}
|
||||
|
||||
private _handleMessage(data: unknown): void {
|
||||
if (data instanceof ArrayBuffer) {
|
||||
this._binaryCallback?.(data);
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof data !== 'string') {
|
||||
return;
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(data);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = go2RTCMessageSchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Iterate over a copy: a callback may subscribe or unsubscribe during
|
||||
// dispatch.
|
||||
for (const callback of [...this._messageCallbacks]) {
|
||||
callback(result.data);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { Go2RTCMode } from '../../../../../config/schema/cameras';
|
||||
import type { MediaSourceFactory } from '../adapters/media-source';
|
||||
import type { PeerConnectionFactory } from '../adapters/peer-connection';
|
||||
import type {
|
||||
ImageStreamTarget,
|
||||
StreamSource,
|
||||
StreamSourceCallbacks,
|
||||
StreamSourceChannel,
|
||||
StreamSourceContext,
|
||||
SurfaceKind,
|
||||
VideoStreamTarget,
|
||||
} from '../types';
|
||||
import { MJPEGStreamSource } from './mjpeg';
|
||||
import { MP4StreamSource } from './mp4';
|
||||
import { MSEStreamSource } from './mse';
|
||||
import { WebRTCStreamSource, type MediaStreamFactory } from './webrtc';
|
||||
|
||||
// ===========================================================================
|
||||
// Binary lane
|
||||
// ===========================================================================
|
||||
|
||||
interface CreateBinarySourceOptions {
|
||||
createMediaSource?: MediaSourceFactory;
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
// The available render targets, one of which each mode is wired to. The factory
|
||||
// is the single place that maps a mode to its surface, so it reports back which
|
||||
// one it chose.
|
||||
export interface BinaryStreamTargets {
|
||||
video: VideoStreamTarget;
|
||||
image: ImageStreamTarget;
|
||||
}
|
||||
|
||||
export interface BinarySource {
|
||||
source: StreamSource;
|
||||
surface: SurfaceKind;
|
||||
}
|
||||
|
||||
// Builds a source for a mode that streams binary media over the WebSocket (only
|
||||
// one such mode runs per connection). WebRTC is created separately via
|
||||
// `createWebRTCSource` because it runs in parallel and has its own interface.
|
||||
export type BinarySourceFactory = (
|
||||
mode: Go2RTCMode,
|
||||
targets: BinaryStreamTargets,
|
||||
channel: StreamSourceChannel,
|
||||
callbacks: StreamSourceCallbacks,
|
||||
options?: CreateBinarySourceOptions,
|
||||
) => BinarySource | null;
|
||||
|
||||
// Returns null for modes with no binary source (i.e. `webrtc`, handled
|
||||
// separately via `createWebRTCSource`).
|
||||
export const createBinarySource: BinarySourceFactory = (
|
||||
mode: Go2RTCMode,
|
||||
targets: BinaryStreamTargets,
|
||||
channel: StreamSourceChannel,
|
||||
callbacks: StreamSourceCallbacks,
|
||||
options?: CreateBinarySourceOptions,
|
||||
): BinarySource | null => {
|
||||
switch (mode) {
|
||||
case 'mse':
|
||||
return {
|
||||
source: new MSEStreamSource(
|
||||
{ target: targets.video, channel, callbacks },
|
||||
options,
|
||||
),
|
||||
surface: 'video',
|
||||
};
|
||||
case 'mp4':
|
||||
return {
|
||||
source: new MP4StreamSource({ target: targets.image, channel, callbacks }),
|
||||
surface: 'image',
|
||||
};
|
||||
case 'mjpeg':
|
||||
return {
|
||||
source: new MJPEGStreamSource({ target: targets.image, channel, callbacks }),
|
||||
surface: 'image',
|
||||
};
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
// ===========================================================================
|
||||
// WebRTC lane
|
||||
// ===========================================================================
|
||||
|
||||
export interface CreateWebRTCSourceOptions {
|
||||
createPeerConnection?: PeerConnectionFactory;
|
||||
createMediaStream?: MediaStreamFactory;
|
||||
microphoneStream?: MediaStream | null;
|
||||
}
|
||||
|
||||
export type WebRTCSourceFactory = (
|
||||
context: StreamSourceContext<VideoStreamTarget>,
|
||||
options?: CreateWebRTCSourceOptions,
|
||||
) => WebRTCStreamSource;
|
||||
|
||||
export const createWebRTCSource: WebRTCSourceFactory = (
|
||||
context: StreamSourceContext<VideoStreamTarget>,
|
||||
options?: CreateWebRTCSourceOptions,
|
||||
): WebRTCStreamSource => new WebRTCStreamSource(context, options);
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { Go2RTCMode } from '../../../../../config/schema/cameras';
|
||||
import type {
|
||||
MediaLoadedCapabilities,
|
||||
MediaTechnology,
|
||||
UnsubscribeCallback,
|
||||
} from '../../../../../types';
|
||||
import { Timer } from '../../../../../utils/timer';
|
||||
import type {
|
||||
Go2RTCMessage,
|
||||
ImageStreamTarget,
|
||||
StreamProfile,
|
||||
StreamSource,
|
||||
StreamSourceContext,
|
||||
} from '../types';
|
||||
import { isServerErrorForMode } from '../utils/messages';
|
||||
|
||||
// Fail if no frame arrives within this window. The channel is open and the mode
|
||||
// was requested, but the server may send neither a frame nor an error, so
|
||||
// without this the source would hang silently instead of failing over. Mirrors
|
||||
// the MSE negotiation and WebRTC connect timeouts.
|
||||
const FIRST_FRAME_TIMEOUT_SECONDS = 5;
|
||||
|
||||
// Base for the modes that present a stream as a sequence of still images fed to
|
||||
// the image surface (MJPEG, MP4): both receive binary frames and turn each into
|
||||
// a single-image Blob, differing only in the request message and the per-frame
|
||||
// conversion. These are last-resort fallbacks: no audio, no "real" playback.
|
||||
export abstract class ImageFrameStreamSource implements StreamSource {
|
||||
protected _context: StreamSourceContext<ImageStreamTarget>;
|
||||
|
||||
private _loaded = false;
|
||||
private _stopped = false;
|
||||
|
||||
private _firstFrameTimer = new Timer();
|
||||
|
||||
private _unsubscribe: UnsubscribeCallback | null = null;
|
||||
|
||||
constructor(context: StreamSourceContext<ImageStreamTarget>) {
|
||||
this._context = context;
|
||||
}
|
||||
|
||||
protected abstract _mode: Go2RTCMode;
|
||||
protected abstract _getRequestMessage(): Go2RTCMessage;
|
||||
protected abstract _handleFrame(data: ArrayBuffer): void;
|
||||
|
||||
public start(): void {
|
||||
// Arm before wiring up the channel: a synchronous channel could deliver the
|
||||
// first frame during send(), which must be able to stop an already-armed
|
||||
// timer rather than have start() arm one that never stops.
|
||||
this._firstFrameTimer.start(FIRST_FRAME_TIMEOUT_SECONDS, () =>
|
||||
this._context.callbacks.failedCallback('connect_timeout'),
|
||||
);
|
||||
|
||||
this._unsubscribe = this._context.channel.subscribeToMessages((message) => {
|
||||
if (isServerErrorForMode(message, this._mode)) {
|
||||
this._context.callbacks.failedCallback('server_error');
|
||||
}
|
||||
});
|
||||
this._context.channel.setBinaryCallback((data) => this._handleFrame(data));
|
||||
this._context.channel.send(this._getRequestMessage());
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this._stopped = true;
|
||||
this._firstFrameTimer.stop();
|
||||
this._unsubscribe?.();
|
||||
this._unsubscribe = null;
|
||||
this._context.channel.setBinaryCallback(null);
|
||||
|
||||
this._teardown();
|
||||
}
|
||||
|
||||
public getCapabilities(): MediaLoadedCapabilities {
|
||||
return { supportsPause: false };
|
||||
}
|
||||
|
||||
public getTechnology(): MediaTechnology[] {
|
||||
return [this._mode];
|
||||
}
|
||||
|
||||
public getStreamProfile(): StreamProfile {
|
||||
return { hasVideo: true, hasH265Video: false, hasAudio: false, hasAACAudio: false };
|
||||
}
|
||||
|
||||
// Show a rendered frame and, on the first one, report the stream loaded.
|
||||
protected async _showFrame(frame: Blob): Promise<void> {
|
||||
// MP4 turns each frame into a Blob asynchronously (canvas.toBlob), so a
|
||||
// frame can arrive here after stop(). Drop it so a stopped source never
|
||||
// writes to the shared image surface or reports a stale load.
|
||||
if (this._stopped) {
|
||||
return;
|
||||
}
|
||||
|
||||
const decoded = this._context.target.showFrame(frame);
|
||||
|
||||
if (this._loaded) {
|
||||
return;
|
||||
}
|
||||
this._loaded = true;
|
||||
this._firstFrameTimer.stop();
|
||||
|
||||
// Report loaded only once the frame has decoded: the media-loaded info is
|
||||
// built by reading the <img> dimensions, and an <img> whose src was just
|
||||
// set still reports 0x0 (which is rejected as invalid), so reporting
|
||||
// synchronously would leave the media stuck "not loading".
|
||||
await decoded;
|
||||
if (!this._stopped) {
|
||||
this._context.callbacks.loadedCallback();
|
||||
}
|
||||
}
|
||||
|
||||
protected _teardown(): void {}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { Go2RTCMessage } from '../types';
|
||||
import { ImageFrameStreamSource } from './image-frame';
|
||||
|
||||
// Each binary frame is a complete JPEG, shown directly as an image frame.
|
||||
export class MJPEGStreamSource extends ImageFrameStreamSource {
|
||||
protected _mode = 'mjpeg' as const;
|
||||
|
||||
protected _getRequestMessage(): Go2RTCMessage {
|
||||
return { type: 'mjpeg' };
|
||||
}
|
||||
|
||||
protected _handleFrame(data: ArrayBuffer): void {
|
||||
/* istanbul ignore next: This never rejects; the catch satisfies the no-floating-promises -- @preserve */
|
||||
this._showFrame(new Blob([data], { type: 'image/jpeg' })).catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import { OffscreenVideo, type VideoElementFactory } from '../offscreen-video';
|
||||
import type { Go2RTCMessage, ImageStreamTarget, StreamSourceContext } from '../types';
|
||||
import { arrayBufferToBase64 } from '../utils/base64';
|
||||
import {
|
||||
convertToCodecString,
|
||||
GO2RTC_CODECS,
|
||||
selectSupportedCodecs,
|
||||
} from '../utils/codecs';
|
||||
import { ImageFrameStreamSource } from './image-frame';
|
||||
|
||||
type CanvasElementFactory = () => HTMLCanvasElement;
|
||||
|
||||
interface MP4StreamSourceOptions {
|
||||
createVideoElement?: VideoElementFactory;
|
||||
createCanvasElement?: CanvasElementFactory;
|
||||
}
|
||||
|
||||
// Each binary frame is a standalone MP4 holding one keyframe. Unlike JPEG it
|
||||
// cannot be shown directly, so it is decoded in an off-screen video and drawn
|
||||
// to a canvas to produce the image.
|
||||
export class MP4StreamSource extends ImageFrameStreamSource {
|
||||
protected _mode = 'mp4' as const;
|
||||
|
||||
private _decoder: OffscreenVideo;
|
||||
private _createCanvasElement: CanvasElementFactory;
|
||||
private _canvas: HTMLCanvasElement | null = null;
|
||||
|
||||
constructor(
|
||||
context: StreamSourceContext<ImageStreamTarget>,
|
||||
options?: MP4StreamSourceOptions,
|
||||
) {
|
||||
super(context);
|
||||
|
||||
const createVideo =
|
||||
options?.createVideoElement ?? (() => document.createElement('video'));
|
||||
this._decoder = new OffscreenVideo(() => {
|
||||
const video = createVideo();
|
||||
video.autoplay = true;
|
||||
video.playsInline = true;
|
||||
video.muted = true;
|
||||
video.addEventListener('loadeddata', () => this._drawFrame(video));
|
||||
return video;
|
||||
});
|
||||
|
||||
this._createCanvasElement =
|
||||
options?.createCanvasElement ?? (() => document.createElement('canvas'));
|
||||
}
|
||||
|
||||
protected _getRequestMessage(): Go2RTCMessage {
|
||||
// Codec support is probed on the decoder video, since that is what plays
|
||||
// the incoming MP4 frames; the image surface onto which this is rendered
|
||||
// has has no video element of its own.
|
||||
const decoder = this._decoder.get();
|
||||
return {
|
||||
type: 'mp4',
|
||||
value: convertToCodecString(
|
||||
selectSupportedCodecs(
|
||||
GO2RTC_CODECS,
|
||||
{ audio: false, video: true },
|
||||
(mimeType) => !!decoder.canPlayType(mimeType),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
protected _handleFrame(data: ArrayBuffer): void {
|
||||
this._decoder.get().src = 'data:video/mp4;base64,' + arrayBufferToBase64(data);
|
||||
}
|
||||
|
||||
protected _teardown(): void {
|
||||
this._decoder.clear();
|
||||
this._canvas = null;
|
||||
}
|
||||
|
||||
private _drawFrame(decoder: HTMLVideoElement): void {
|
||||
const canvas = (this._canvas ??= this._createCanvasElement());
|
||||
|
||||
// Reassigning a canvas dimension reallocates its backing buffer, so skip it
|
||||
// when the video size is unchanged.
|
||||
if (canvas.width !== decoder.videoWidth || canvas.height !== decoder.videoHeight) {
|
||||
canvas.width = decoder.videoWidth;
|
||||
canvas.height = decoder.videoHeight;
|
||||
}
|
||||
|
||||
const context = canvas.getContext('2d');
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
|
||||
context.drawImage(decoder, 0, 0, canvas.width, canvas.height);
|
||||
canvas.toBlob((frame) => {
|
||||
if (frame) {
|
||||
/* istanbul ignore next: This never rejects; the catch satisfies the no-floating-promises -- @preserve */
|
||||
this._showFrame(frame).catch(() => {});
|
||||
}
|
||||
}, 'image/jpeg');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
import type {
|
||||
MediaLoadedCapabilities,
|
||||
MediaTechnology,
|
||||
UnsubscribeCallback,
|
||||
} from '../../../../../types';
|
||||
import { hasAudio } from '../../../../../utils/audio';
|
||||
import { Timer } from '../../../../../utils/timer';
|
||||
import {
|
||||
createBrowserMediaSource,
|
||||
type MediaSourceFactory,
|
||||
type MediaSourceInterface,
|
||||
} from '../adapters/media-source';
|
||||
import type {
|
||||
Go2RTCMessage,
|
||||
StreamProfile,
|
||||
StreamSource,
|
||||
StreamSourceContext,
|
||||
VideoStreamTarget,
|
||||
} from '../types';
|
||||
import { BoundedBufferQueue } from '../utils/bounded-buffer-queue';
|
||||
import {
|
||||
convertToCodecString,
|
||||
getCodecsForUserAgent,
|
||||
selectSupportedCodecs,
|
||||
} from '../utils/codecs';
|
||||
import { LiveEdgeTracker } from '../utils/live-edge-tracker';
|
||||
import type { LiveEdgeAction } from '../utils/live-edge-tracker/types';
|
||||
import { isServerErrorForMode } from '../utils/messages';
|
||||
import { isWebKitUserAgent } from '../utils/user-agent';
|
||||
|
||||
// ===========================================================================
|
||||
// MSE Tuning
|
||||
// ===========================================================================
|
||||
|
||||
// These are judgement-based budgets (time, buffered seconds, staged bytes)
|
||||
// balancing latency and resilience, not derived values. The retained-buffer
|
||||
// window and the staged-bytes cap follow go2rtc's reference web client; the
|
||||
// negotiation timeout has no counterpart there and is added by this
|
||||
// implementation.
|
||||
|
||||
// Seconds of media retained behind the live edge; older media is trimmed. This
|
||||
// bounds memory even across a long pause (the buffer never grows past this
|
||||
// window), so a paused stream can hold its frame indefinitely.
|
||||
const RETAINED_BUFFER_SECONDS = 15;
|
||||
|
||||
// On resume, if the playhead is more than this far behind the live edge, jump
|
||||
// forward to rejoin live rather than replaying the buffered-behind media.
|
||||
const LIVE_EDGE_RESUME_LAG_SECONDS = 1;
|
||||
|
||||
// Where a live-edge jump lands, as a gap behind the buffered end (a small
|
||||
// cushion so playback does not immediately starve at the very edge).
|
||||
const LIVE_EDGE_JUMP_OFFSET_SECONDS = 0.75;
|
||||
|
||||
// Bound on media staged while the SourceBuffer is busy; a stream that outruns
|
||||
// this is unrecoverable without a reconnect.
|
||||
const MAX_PENDING_BUFFER_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
// How long to wait for the server's codec reply before failing the source.
|
||||
const MSE_NEGOTIATION_TIMEOUT_SECONDS = 5;
|
||||
|
||||
// ===========================================================================
|
||||
// MSEStreamSource
|
||||
// ===========================================================================
|
||||
|
||||
interface MSEStreamSourceOptions {
|
||||
createMediaSource?: MediaSourceFactory;
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
export class MSEStreamSource implements StreamSource {
|
||||
private _context: StreamSourceContext<VideoStreamTarget>;
|
||||
|
||||
private _createMediaSource: MediaSourceFactory;
|
||||
private _mediaSource: MediaSourceInterface | null = null;
|
||||
|
||||
private _sourceBuffer: SourceBuffer | null = null;
|
||||
private _pendingBuffer = new BoundedBufferQueue(MAX_PENDING_BUFFER_BYTES);
|
||||
|
||||
// The codec string the server settled on during negotiation; null until then.
|
||||
private _codecs: string | null = null;
|
||||
|
||||
// The codecs offered to the server, derived once from the user agent.
|
||||
private _codecCandidates: readonly string[];
|
||||
|
||||
private _negotiationTimer = new Timer();
|
||||
private _liveEdge: LiveEdgeTracker;
|
||||
|
||||
private _unsubscribeCallbacks: UnsubscribeCallback[] = [];
|
||||
|
||||
private _loadedHandler = (): void => {
|
||||
this._context.callbacks.loadedCallback();
|
||||
};
|
||||
|
||||
private _errorHandler = (): void => {
|
||||
this._context.callbacks.failedCallback('media_error');
|
||||
};
|
||||
|
||||
// On resume, rejoin the very live edge.
|
||||
private _playHandler = (): void => {
|
||||
const sourceBuffer = this._sourceBuffer;
|
||||
if (!sourceBuffer?.buffered.length) {
|
||||
return;
|
||||
}
|
||||
const video = this._context.target.video;
|
||||
const end = sourceBuffer.buffered.end(sourceBuffer.buffered.length - 1);
|
||||
if (end - video.currentTime > LIVE_EDGE_RESUME_LAG_SECONDS) {
|
||||
video.currentTime = end - LIVE_EDGE_JUMP_OFFSET_SECONDS;
|
||||
}
|
||||
};
|
||||
|
||||
constructor(
|
||||
context: StreamSourceContext<VideoStreamTarget>,
|
||||
options?: MSEStreamSourceOptions,
|
||||
) {
|
||||
this._context = context;
|
||||
this._createMediaSource = options?.createMediaSource ?? createBrowserMediaSource;
|
||||
|
||||
const userAgent = options?.userAgent ?? navigator.userAgent;
|
||||
this._liveEdge = new LiveEdgeTracker({ webkit: isWebKitUserAgent(userAgent) });
|
||||
this._codecCandidates = getCodecsForUserAgent(userAgent);
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
const mediaSource = this._createMediaSource();
|
||||
if (!mediaSource) {
|
||||
this._context.callbacks.failedCallback('unsupported');
|
||||
return;
|
||||
}
|
||||
this._mediaSource = mediaSource;
|
||||
|
||||
this._unsubscribeCallbacks.push(
|
||||
mediaSource.subscribeToSourceOpen(() => this._negotiate(mediaSource)),
|
||||
this._context.channel.subscribeToMessages((message) =>
|
||||
this._handleMessage(mediaSource, message),
|
||||
),
|
||||
);
|
||||
this._context.target.video.addEventListener('loadeddata', this._loadedHandler);
|
||||
this._context.target.video.addEventListener('error', this._errorHandler);
|
||||
this._context.target.video.addEventListener('play', this._playHandler);
|
||||
|
||||
mediaSource.attach(this._context.target.video);
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this._negotiationTimer.stop();
|
||||
|
||||
this._unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe());
|
||||
this._unsubscribeCallbacks = [];
|
||||
|
||||
this._context.target.video.removeEventListener('loadeddata', this._loadedHandler);
|
||||
this._context.target.video.removeEventListener('error', this._errorHandler);
|
||||
this._context.target.video.removeEventListener('play', this._playHandler);
|
||||
|
||||
this._context.channel.setBinaryCallback(null);
|
||||
|
||||
this._mediaSource?.detach(this._context.target.video);
|
||||
this._mediaSource = null;
|
||||
|
||||
this._sourceBuffer = null;
|
||||
|
||||
this._pendingBuffer.clear();
|
||||
}
|
||||
|
||||
public getCapabilities(): MediaLoadedCapabilities {
|
||||
return {
|
||||
supportsPause: true,
|
||||
hasAudio: hasAudio(this._context.target.video, { mseCodecs: this._codecs }),
|
||||
has2WayAudio: false,
|
||||
};
|
||||
}
|
||||
|
||||
public getTechnology(): MediaTechnology[] {
|
||||
return ['mse'];
|
||||
}
|
||||
|
||||
public getStreamProfile(): StreamProfile {
|
||||
const codecs = this._codecs ?? '';
|
||||
return {
|
||||
hasVideo: codecs.includes('avc1') || codecs.includes('hvc1'),
|
||||
hasH265Video: codecs.includes('hvc1'),
|
||||
hasAudio:
|
||||
codecs.includes('mp4a') || codecs.includes('opus') || codecs.includes('flac'),
|
||||
hasAACAudio: codecs.includes('mp4a'),
|
||||
};
|
||||
}
|
||||
|
||||
private _negotiate(mediaSource: MediaSourceInterface): void {
|
||||
const codecs = selectSupportedCodecs(
|
||||
this._codecCandidates,
|
||||
{ audio: true, video: true },
|
||||
(mimeType) => mediaSource.isTypeSupported(mimeType),
|
||||
);
|
||||
|
||||
this._context.channel.send({ type: 'mse', value: convertToCodecString(codecs) });
|
||||
this._negotiationTimer.start(MSE_NEGOTIATION_TIMEOUT_SECONDS, () =>
|
||||
this._context.callbacks.failedCallback('negotiation_timeout'),
|
||||
);
|
||||
}
|
||||
|
||||
private _handleMessage(
|
||||
mediaSource: MediaSourceInterface,
|
||||
message: Go2RTCMessage,
|
||||
): void {
|
||||
if (isServerErrorForMode(message, 'mse')) {
|
||||
this._negotiationTimer.stop();
|
||||
this._context.callbacks.failedCallback('server_error');
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
message.type !== 'mse' ||
|
||||
typeof message.value !== 'string' ||
|
||||
this._sourceBuffer
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._negotiationTimer.stop();
|
||||
|
||||
// go2rtc answers the codec offer with an `mse` text message whose value is
|
||||
// the codec string for addSourceBuffer. It arrives once, before any
|
||||
// SourceBuffer exists (the guard above returns for every later message), so
|
||||
// this value is the negotiated codecs.
|
||||
this._codecs = message.value;
|
||||
|
||||
let sourceBuffer: SourceBuffer;
|
||||
try {
|
||||
sourceBuffer = mediaSource.addSourceBuffer(message.value);
|
||||
} catch {
|
||||
this._context.callbacks.failedCallback('media_error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Segments mode: order media by its fMP4 timestamps rather than by
|
||||
// arrival order.
|
||||
sourceBuffer.mode = 'segments';
|
||||
|
||||
const updateEndListener = (): void =>
|
||||
this._handleUpdateEnd(mediaSource, sourceBuffer);
|
||||
sourceBuffer.addEventListener('updateend', updateEndListener);
|
||||
this._unsubscribeCallbacks.push(() =>
|
||||
sourceBuffer.removeEventListener('updateend', updateEndListener),
|
||||
);
|
||||
|
||||
this._sourceBuffer = sourceBuffer;
|
||||
|
||||
this._context.channel.setBinaryCallback((data) =>
|
||||
this._handleData(sourceBuffer, data),
|
||||
);
|
||||
}
|
||||
|
||||
private _handleData(sourceBuffer: SourceBuffer, data: ArrayBuffer): void {
|
||||
if (sourceBuffer.updating || !this._pendingBuffer.isEmpty) {
|
||||
if (!this._pendingBuffer.push(data)) {
|
||||
this._context.callbacks.failedCallback('buffer_overflow');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
this._append(sourceBuffer, data);
|
||||
}
|
||||
|
||||
private _append(sourceBuffer: SourceBuffer, data: ArrayBuffer): void {
|
||||
try {
|
||||
sourceBuffer.appendBuffer(data);
|
||||
} catch {
|
||||
// Append failures during teardown races are recoverable by later appends;
|
||||
// fatal conditions surface through the video element's error event
|
||||
// instead.
|
||||
}
|
||||
}
|
||||
|
||||
private _handleUpdateEnd(
|
||||
mediaSource: MediaSourceInterface,
|
||||
sourceBuffer: SourceBuffer,
|
||||
): void {
|
||||
if (sourceBuffer.updating) {
|
||||
return;
|
||||
}
|
||||
|
||||
// A queued updateend can fire after a reconnect or teardown has closed the
|
||||
// MediaSource (detached from the video).
|
||||
if (!mediaSource.isOpen()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pending = this._pendingBuffer.shift();
|
||||
if (pending) {
|
||||
this._append(sourceBuffer, pending);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!sourceBuffer.buffered.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const video = this._context.target.video;
|
||||
const end = sourceBuffer.buffered.end(sourceBuffer.buffered.length - 1);
|
||||
this._trimSourceBuffer(mediaSource, sourceBuffer, end);
|
||||
|
||||
// While paused the user is holding a frame: leave the playhead alone (the
|
||||
// trim above still bounds memory) and do not sample the growing lag, which
|
||||
// would otherwise bias the catch-up rate once playback resumes. Resuming
|
||||
// jumps back to the live edge (see the play handler).
|
||||
if (video.paused) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Keep playback tracking the live edge as new media arrives.
|
||||
this._applyLiveEdgeAction(
|
||||
video,
|
||||
this._liveEdge.next({
|
||||
bufferedEndSeconds: end,
|
||||
currentTimeSeconds: video.currentTime,
|
||||
playbackRate: video.playbackRate,
|
||||
now: new Date(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private _applyLiveEdgeAction(video: HTMLVideoElement, action: LiveEdgeAction): void {
|
||||
if (action.action === 'seek') {
|
||||
video.currentTime = action.seconds;
|
||||
} else if (action.action === 'rate' && video.playbackRate !== action.rate) {
|
||||
video.playbackRate = action.rate;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep only the most recent RETAINED_BUFFER_SECONDS of media, trimming older
|
||||
// media and re-declaring the seekable range. The playhead is never snapped
|
||||
// here: while playing the live-edge tracker keeps it near the edge, and while
|
||||
// paused it is left holding its frame (a playhead that falls past the lag
|
||||
// bound is handled by a reconnect, not by dragging it forward).
|
||||
private _trimSourceBuffer(
|
||||
mediaSource: MediaSourceInterface,
|
||||
sourceBuffer: SourceBuffer,
|
||||
end: number,
|
||||
): void {
|
||||
const retainedStart = end - RETAINED_BUFFER_SECONDS;
|
||||
const bufferedStart = sourceBuffer.buffered.start(0);
|
||||
|
||||
if (retainedStart > bufferedStart) {
|
||||
sourceBuffer.remove(bufferedStart, retainedStart);
|
||||
mediaSource.setLiveSeekableRange(retainedStart, end);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
import type {
|
||||
MediaLoadedCapabilities,
|
||||
MediaTechnology,
|
||||
UnsubscribeCallback,
|
||||
} from '../../../../../types';
|
||||
import { has2WayAudio, hasAudio } from '../../../../../utils/audio';
|
||||
import { Timer } from '../../../../../utils/timer';
|
||||
import {
|
||||
createBrowserPeerConnection,
|
||||
GO2RTC_PEER_CONNECTION_CONFIG,
|
||||
type PeerConnectionFactory,
|
||||
} from '../adapters/peer-connection';
|
||||
import type {
|
||||
Go2RTCMessage,
|
||||
StreamProfile,
|
||||
StreamSource,
|
||||
StreamSourceContext,
|
||||
VideoStreamTarget,
|
||||
} from '../types';
|
||||
import { isServerErrorForMode } from '../utils/messages';
|
||||
import { sdpHasH265 } from '../utils/webrtc-sdp';
|
||||
|
||||
// ===========================================================================
|
||||
// WebRTC Tuning
|
||||
// ===========================================================================
|
||||
|
||||
// Fail if no decoded frame arrives within this window. Covers the case where
|
||||
// ICE connects and packets flow but frames never decode (the connection looks
|
||||
// healthy yet nothing plays), so the session can fall back instead of hanging.
|
||||
// See https://github.com/dermotduffy/advanced-camera-card/issues/1699
|
||||
// The window length is a tuning value, not derived: long enough to avoid
|
||||
// failing a slow-but-healthy start, short enough to fall back promptly.
|
||||
const WEBRTC_CONNECT_TIMEOUT_SECONDS = 5;
|
||||
|
||||
// ===========================================================================
|
||||
// WebRTCStreamSource
|
||||
// ===========================================================================
|
||||
|
||||
export type MediaStreamFactory = (tracks: MediaStreamTrack[]) => MediaStream;
|
||||
|
||||
interface WebRTCStreamSourceOptions {
|
||||
createPeerConnection?: PeerConnectionFactory;
|
||||
createMediaStream?: MediaStreamFactory;
|
||||
microphoneStream?: MediaStream | null;
|
||||
}
|
||||
|
||||
export class WebRTCStreamSource implements StreamSource {
|
||||
private _context: StreamSourceContext<VideoStreamTarget>;
|
||||
private _stream: MediaStream | null = null;
|
||||
private _pc: RTCPeerConnection | null = null;
|
||||
|
||||
private _createPeerConnection: PeerConnectionFactory;
|
||||
private _createMediaStream: MediaStreamFactory;
|
||||
private _microphoneStream: MediaStream | null;
|
||||
|
||||
private _microphoneTransceiver: RTCRtpTransceiver | null = null;
|
||||
|
||||
private _connectTimer = new Timer();
|
||||
|
||||
private _unsubscribeCallbacks: UnsubscribeCallback[] = [];
|
||||
|
||||
private _loadedHandler = (): void => {
|
||||
this._connectTimer.stop();
|
||||
this._context.callbacks.loadedCallback();
|
||||
};
|
||||
|
||||
constructor(
|
||||
context: StreamSourceContext<VideoStreamTarget>,
|
||||
options?: WebRTCStreamSourceOptions,
|
||||
) {
|
||||
this._context = context;
|
||||
this._createPeerConnection =
|
||||
options?.createPeerConnection ?? createBrowserPeerConnection;
|
||||
this._createMediaStream =
|
||||
options?.createMediaStream ?? ((tracks) => new MediaStream(tracks));
|
||||
|
||||
this._microphoneStream = options?.microphoneStream ?? null;
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
const pc = this._createPeerConnection(GO2RTC_PEER_CONNECTION_CONFIG);
|
||||
this._pc = pc;
|
||||
|
||||
// Always pre-arm exactly one outbound audio slot so the microphone track
|
||||
// can be attached later via `replaceTrack` with no renegotiation. The
|
||||
// kind-only `addTransceiver('audio', ...)` form never calls getUserMedia,
|
||||
// so it never raises permission prompt for users.
|
||||
const microphoneTrack = this._microphoneStream?.getAudioTracks()[0] ?? null;
|
||||
this._microphoneTransceiver = pc.addTransceiver(microphoneTrack ?? 'audio', {
|
||||
direction: 'sendonly',
|
||||
});
|
||||
pc.addTransceiver('video', { direction: 'recvonly' });
|
||||
pc.addTransceiver('audio', { direction: 'recvonly' });
|
||||
|
||||
pc.addEventListener('icecandidate', (ev) => {
|
||||
// A late candidate from a superseded connection (e.g. after stop(), if
|
||||
// WebRTC has lost the race) must not send on the channel a binary lane
|
||||
// may still be using.
|
||||
if (this._pc !== pc) {
|
||||
return;
|
||||
}
|
||||
|
||||
// An empty value signals end-of-candidates, which the server accepts.
|
||||
this._context.channel.send({
|
||||
type: 'webrtc/candidate',
|
||||
value: ev.candidate ? ev.candidate.candidate : '',
|
||||
});
|
||||
});
|
||||
pc.addEventListener('connectionstatechange', () =>
|
||||
this._handleConnectionStateChange(pc),
|
||||
);
|
||||
|
||||
this._unsubscribeCallbacks.push(
|
||||
this._context.channel.subscribeToMessages((message) =>
|
||||
this._handleMessage(pc, message),
|
||||
),
|
||||
);
|
||||
|
||||
this._connectTimer.start(WEBRTC_CONNECT_TIMEOUT_SECONDS, () =>
|
||||
this._context.callbacks.failedCallback('connect_timeout'),
|
||||
);
|
||||
|
||||
this._negotiate(pc).catch(() => {
|
||||
// A rejection from a superseded connection (e.g. after `stop`) is not
|
||||
// this source's concern.
|
||||
if (this._pc === pc) {
|
||||
this._context.callbacks.failedCallback('media_error');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this._connectTimer.stop();
|
||||
|
||||
this._unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe());
|
||||
this._unsubscribeCallbacks = [];
|
||||
|
||||
this._context.target.video.removeEventListener('loadeddata', this._loadedHandler);
|
||||
if (this._pc) {
|
||||
// pc.close() does not stop the sender's tracks, so the outbound microphone
|
||||
// track keeps running. That is deliberate: MicrophoneManager owns the mic
|
||||
// (it is shared across cameras), so stopping it here would break it
|
||||
// elsewhere -- do not add a track.stop() here. See
|
||||
// https://github.com/dermotduffy/advanced-camera-card/issues/1810
|
||||
this._pc.close();
|
||||
this._pc = null;
|
||||
}
|
||||
|
||||
// The transceiver belonged to the now-closed peer connection.
|
||||
this._microphoneTransceiver = null;
|
||||
this._context.target.video.srcObject = null;
|
||||
this._stream = null;
|
||||
}
|
||||
|
||||
public getMediaStream(): MediaStream | null {
|
||||
return this._stream;
|
||||
}
|
||||
|
||||
public getPeerConnection(): RTCPeerConnection | null {
|
||||
return this._pc;
|
||||
}
|
||||
|
||||
public getCapabilities(): MediaLoadedCapabilities {
|
||||
return {
|
||||
supportsPause: true,
|
||||
hasAudio: hasAudio(this._context.target.video, { pc: this._pc }),
|
||||
has2WayAudio: has2WayAudio(this._pc),
|
||||
};
|
||||
}
|
||||
|
||||
public getTechnology(): MediaTechnology[] {
|
||||
return ['webrtc'];
|
||||
}
|
||||
|
||||
public getStreamProfile(): StreamProfile {
|
||||
const sdp = this._pc?.remoteDescription?.sdp ?? null;
|
||||
|
||||
return {
|
||||
hasVideo: (this._stream?.getVideoTracks().length ?? 0) > 0,
|
||||
hasH265Video: sdp ? sdpHasH265(sdp) : false,
|
||||
hasAudio: (this._stream?.getAudioTracks().length ?? 0) > 0,
|
||||
hasAACAudio: false,
|
||||
};
|
||||
}
|
||||
|
||||
// Swap the outbound microphone track without renegotiating. Guards against a
|
||||
// late rejection from a superseded call (a newer stream, or teardown)
|
||||
// bringing a retired connection back or overwriting a fresher request.
|
||||
public async setMicrophoneStream(stream: MediaStream | null): Promise<void> {
|
||||
if (this._microphoneStream === stream) {
|
||||
return;
|
||||
}
|
||||
this._microphoneStream = stream;
|
||||
|
||||
const transceiver = this._microphoneTransceiver;
|
||||
if (!transceiver) {
|
||||
// No peer connection yet; the next `start()` reads the current stream and
|
||||
// pre-arms the transceiver with it.
|
||||
return;
|
||||
}
|
||||
|
||||
// A microphone stream carries a single audio track; null detaches the sender.
|
||||
const desiredTrack = stream?.getAudioTracks()[0] ?? null;
|
||||
try {
|
||||
await transceiver.sender.replaceTrack(desiredTrack);
|
||||
} catch {
|
||||
const stillCurrent =
|
||||
transceiver === this._microphoneTransceiver &&
|
||||
this._microphoneStream === stream &&
|
||||
this._pc !== null;
|
||||
if (stillCurrent) {
|
||||
this._context.callbacks.failedCallback('two_way_audio_error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async _negotiate(pc: RTCPeerConnection): Promise<void> {
|
||||
const offer = await pc.createOffer();
|
||||
if (this._pc !== pc) {
|
||||
return;
|
||||
}
|
||||
|
||||
await pc.setLocalDescription(offer);
|
||||
if (this._pc !== pc) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._context.channel.send({ type: 'webrtc/offer', value: offer.sdp ?? '' });
|
||||
}
|
||||
|
||||
private _handleMessage(pc: RTCPeerConnection, message: Go2RTCMessage): void {
|
||||
// Every message type handled here carries a string value.
|
||||
if (this._pc !== pc || typeof message.value !== 'string') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (isServerErrorForMode(message, 'webrtc')) {
|
||||
this._context.callbacks.failedCallback('server_error');
|
||||
return;
|
||||
}
|
||||
|
||||
switch (message.type) {
|
||||
case 'webrtc/answer':
|
||||
pc.setRemoteDescription({ type: 'answer', sdp: message.value }).catch(() => {});
|
||||
break;
|
||||
|
||||
case 'webrtc/candidate':
|
||||
if (message.value) {
|
||||
// The server sends no sdpMid; max-bundle puts every track on m-line 0.
|
||||
pc.addIceCandidate({ candidate: message.value, sdpMid: '0' }).catch(() => {});
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private _handleConnectionStateChange(pc: RTCPeerConnection): void {
|
||||
if (this._pc !== pc) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (pc.connectionState === 'connected') {
|
||||
this._attachStream(pc);
|
||||
} else if (pc.connectionState === 'failed') {
|
||||
// Only 'failed' is terminal. 'disconnected' is a recoverable ICE blip
|
||||
// that usually returns to 'connected' on its own, so failing on it would
|
||||
// turn a brief network hiccup into a needless reconnect; a disconnect
|
||||
// that never recovers stops delivering frames and is caught by the
|
||||
// frame-stall watchdog instead.
|
||||
this._context.callbacks.failedCallback('media_error');
|
||||
}
|
||||
}
|
||||
|
||||
private _attachStream(pc: RTCPeerConnection): void {
|
||||
if (this._stream) {
|
||||
return;
|
||||
}
|
||||
|
||||
const tracks = pc
|
||||
.getTransceivers()
|
||||
.filter((transceiver) => transceiver.currentDirection === 'recvonly')
|
||||
.map((transceiver) => transceiver.receiver.track);
|
||||
|
||||
const stream = this._createMediaStream(tracks);
|
||||
this._stream = stream;
|
||||
|
||||
this._context.target.video.addEventListener('loadeddata', this._loadedHandler, {
|
||||
once: true,
|
||||
});
|
||||
this._context.target.video.srcObject = stream;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
import type {
|
||||
MediaLoadedCapabilities,
|
||||
MediaTechnology,
|
||||
UnsubscribeCallback,
|
||||
} from '../../../../types';
|
||||
|
||||
// ===========================================================================
|
||||
// Control messages
|
||||
// ===========================================================================
|
||||
|
||||
// go2rtc control messages are JSON text frames of this shape; media flows as
|
||||
// separate binary frames.
|
||||
export const go2RTCMessageSchema = z.object({
|
||||
type: z.string(),
|
||||
|
||||
// Per-type payload (a codec list, an SDP, an ICE candidate, error text, ...):
|
||||
// absent for some types (e.g. mjpeg) and not always a string, so it is typed
|
||||
// `unknown` and each handler narrows it before use.
|
||||
value: z.unknown().optional(),
|
||||
});
|
||||
export type Go2RTCMessage = z.infer<typeof go2RTCMessageSchema>;
|
||||
|
||||
export type MessageCallback = (message: Go2RTCMessage) => void;
|
||||
export type BinaryCallback = (data: ArrayBuffer) => void;
|
||||
|
||||
// ===========================================================================
|
||||
// Signaling channel
|
||||
// ===========================================================================
|
||||
|
||||
// A session's two parallel delivery paths: 'binary' (MSE/MP4/MJPEG media over
|
||||
// the WebSocket) and 'webrtc' (media over the peer connection).
|
||||
export type Lane = 'binary' | 'webrtc';
|
||||
|
||||
// The narrow view of the signaling channel that stream sources receive: they
|
||||
// may exchange messages but not manage the connection.
|
||||
export interface StreamSourceChannel {
|
||||
send(message: Go2RTCMessage): void;
|
||||
subscribeToMessages(callback: MessageCallback): UnsubscribeCallback;
|
||||
setBinaryCallback(callback: BinaryCallback | null): void;
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Render targets
|
||||
// ===========================================================================
|
||||
|
||||
// Surface vs Target: A "target" is where a source (binary vs webrtc) puts its
|
||||
// frames for one mode: a <video> element, or an <img> element (via a showFrame
|
||||
// callback). The component adds a media-player controller to a target to make a
|
||||
// fuller "surface" (see SessionSurfaces in session-controller.ts); it hands a
|
||||
// source only the target, never the surface, so a source cannot reach the
|
||||
// controller.
|
||||
|
||||
// What a source renders onto: the real <video> for modes the browser plays
|
||||
// (MSE, WebRTC), or an image sink fed decoded frames for modes presented as a
|
||||
// sequence of still images (MP4, MJPEG).
|
||||
export interface VideoStreamTarget {
|
||||
kind: 'video';
|
||||
video: HTMLVideoElement;
|
||||
}
|
||||
|
||||
// A frame is handed over as a Blob rather than exposing the raw <img> so the
|
||||
// object-URL lifecycle (create, revoke the previous) stays owned by the image
|
||||
// surface, not each source. The returned promise resolves once the frame has
|
||||
// decoded, so a source can defer reporting itself loaded until the surface has
|
||||
// real dimensions.
|
||||
export interface ImageStreamTarget {
|
||||
kind: 'image';
|
||||
showFrame(frame: Blob): Promise<void>;
|
||||
}
|
||||
|
||||
type StreamSourceTarget = VideoStreamTarget | ImageStreamTarget;
|
||||
|
||||
// Which of the two render surfaces media is shown on: the real <video> or the
|
||||
// <img>. Matches the `kind` of the corresponding stream target.
|
||||
export type SurfaceKind = 'video' | 'image';
|
||||
|
||||
// ===========================================================================
|
||||
// Stream sources
|
||||
// ===========================================================================
|
||||
|
||||
// Describes a source's negotiated media, used to arbitrate the MSE-vs-WebRTC
|
||||
// race. The bit-weighted comparison lives in `utils/source-priority.ts`.
|
||||
export interface StreamProfile {
|
||||
hasVideo: boolean;
|
||||
hasH265Video: boolean;
|
||||
hasAudio: boolean;
|
||||
|
||||
// The MSE audio score is inferred from the negotiated codec string, not an
|
||||
// observed track, so it counts only AAC -- the audio codec browsers decode
|
||||
// reliably via MSE. Opus and FLAC over MSE are inconsistently supported, so
|
||||
// crediting them could prefer a stream whose audio never plays.
|
||||
hasAACAudio: boolean;
|
||||
}
|
||||
|
||||
export type StreamSourceFailureReason =
|
||||
| 'two_way_audio_error'
|
||||
| 'buffer_overflow'
|
||||
| 'connect_timeout'
|
||||
| 'media_error'
|
||||
| 'negotiation_timeout'
|
||||
| 'server_error'
|
||||
| 'unsupported';
|
||||
|
||||
export interface StreamSourceCallbacks {
|
||||
loadedCallback: () => void;
|
||||
failedCallback: (reason: StreamSourceFailureReason) => void;
|
||||
}
|
||||
|
||||
export interface StreamSourceContext<T extends StreamSourceTarget = StreamSourceTarget> {
|
||||
target: T;
|
||||
channel: StreamSourceChannel;
|
||||
callbacks: StreamSourceCallbacks;
|
||||
}
|
||||
|
||||
// A single streaming mode implementation (e.g. MSE). Sources negotiate over the
|
||||
// shared channel, attach media to the provided video element, and report
|
||||
// readiness or failure via callbacks; connection management and failure
|
||||
// recovery belong to the session not the source.
|
||||
export interface StreamSource {
|
||||
// Single-use: start() begins the source and stop() tears it down for good.
|
||||
start(): void;
|
||||
stop(): void;
|
||||
|
||||
getCapabilities(): MediaLoadedCapabilities;
|
||||
getTechnology(): MediaTechnology[];
|
||||
getStreamProfile(): StreamProfile;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
export const arrayBufferToBase64 = (buffer: ArrayBuffer): string => {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = '';
|
||||
|
||||
// `btoa` needs bytes as a binary string, build char-by-char.
|
||||
for (let i = 0; i < bytes.byteLength; ++i) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
|
||||
return window.btoa(binary);
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
// A byte-bounded FIFO queue of ArrayBuffer chunks: a push that would exceed the
|
||||
// byte cap is rejected.
|
||||
export class BoundedBufferQueue {
|
||||
private _buffers: ArrayBuffer[] = [];
|
||||
private _bytes = 0;
|
||||
|
||||
private _maxBytes: number;
|
||||
|
||||
constructor(maxBytes: number) {
|
||||
this._maxBytes = maxBytes;
|
||||
}
|
||||
|
||||
public get isEmpty(): boolean {
|
||||
return this._buffers.length === 0;
|
||||
}
|
||||
|
||||
// Stage a chunk at the back, or return false (staging nothing) if it would
|
||||
// exceed the byte cap.
|
||||
public push(data: ArrayBuffer): boolean {
|
||||
if (this._bytes + data.byteLength > this._maxBytes) {
|
||||
return false;
|
||||
}
|
||||
this._buffers.push(data);
|
||||
this._bytes += data.byteLength;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Remove and return the oldest staged chunk, or null when empty.
|
||||
public shift(): ArrayBuffer | null {
|
||||
const data = this._buffers.shift();
|
||||
if (!data) {
|
||||
return null;
|
||||
}
|
||||
this._bytes -= data.byteLength;
|
||||
return data;
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
this._buffers = [];
|
||||
this._bytes = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { getSafariMajorVersion } from './user-agent';
|
||||
|
||||
// Codec strings offered to the go2rtc server when negotiating an MSE or MP4
|
||||
// stream. The go2rtc server recognizes only a fixed set of exact strings (one
|
||||
// canonical spelling per codec, e.g. `avc1.640029` for all H.264) and silently
|
||||
// ignores the rest; it then matches the camera's tracks by codec name alone,
|
||||
// never comparing profile or level.
|
||||
//
|
||||
// So membership here is not correctness-critical: an unrecognized entry (a
|
||||
// different H.264 level, AAC-HE) has no effect, and a camera streaming any
|
||||
// level still matches. The list can stay a broad superset rather than tracking
|
||||
// one server version's exact set.
|
||||
//
|
||||
// Order matters as the offer's preference order: video before audio, and audio
|
||||
// most to least reliably supported.
|
||||
//
|
||||
// H.264/H.265 video, most-compatible first. Offered to every browser.
|
||||
const GO2RTC_VIDEO_CODECS: readonly string[] = [
|
||||
// H.264 high 4.1
|
||||
'avc1.640029',
|
||||
|
||||
// H.264 high 4.2
|
||||
'avc1.64002A',
|
||||
|
||||
// H.264 high 5.1
|
||||
'avc1.640033',
|
||||
|
||||
// H.265 main 5.1
|
||||
'hvc1.1.6.L153.B0',
|
||||
];
|
||||
|
||||
// Audio is layered by the oldest Safari major version that can decode each
|
||||
// codec via MSE (see `getCodecsForUserAgent`). Each tier extends the previous
|
||||
// one with the next-most-reliable audio codec.
|
||||
const GO2RTC_SAFARI_13_CODECS: readonly string[] = [
|
||||
...GO2RTC_VIDEO_CODECS,
|
||||
// AAC LC
|
||||
'mp4a.40.2',
|
||||
|
||||
// AAC HE
|
||||
'mp4a.40.5',
|
||||
];
|
||||
|
||||
const GO2RTC_SAFARI_14_CODECS: readonly string[] = [
|
||||
...GO2RTC_SAFARI_13_CODECS,
|
||||
|
||||
// FLAC
|
||||
'flac',
|
||||
];
|
||||
|
||||
// Full codec set: offered to non-Safari browsers.
|
||||
export const GO2RTC_CODECS: readonly string[] = [
|
||||
...GO2RTC_SAFARI_14_CODECS,
|
||||
|
||||
// Opus
|
||||
'opus',
|
||||
];
|
||||
|
||||
// Safari's `isTypeSupported` over-reports audio codec support, so offer only
|
||||
// the codec set the running major version can *actually* play via MSE. The
|
||||
// version boundaries (AAC from 13, FLAC from 14, OPUS never) are empirical
|
||||
// browser-compatibility observations, not derived values -- treat them as a
|
||||
// best-known table that may need revising for future Safari versions.
|
||||
export const getCodecsForUserAgent = (userAgent: string): readonly string[] => {
|
||||
const version = getSafariMajorVersion(userAgent);
|
||||
if (version === null) {
|
||||
return GO2RTC_CODECS;
|
||||
}
|
||||
if (version < 13) {
|
||||
return GO2RTC_VIDEO_CODECS;
|
||||
}
|
||||
if (version < 14) {
|
||||
return GO2RTC_SAFARI_13_CODECS;
|
||||
}
|
||||
return GO2RTC_SAFARI_14_CODECS;
|
||||
};
|
||||
|
||||
interface CodecMediaSelection {
|
||||
audio: boolean;
|
||||
video: boolean;
|
||||
}
|
||||
|
||||
// A codec is a video codec if it contains 'vc1' beyond position zero (avc1.*,
|
||||
// hvc1.*); everything else in the go2rtc codec list is audio.
|
||||
const isVideoCodec = (codec: string): boolean => codec.indexOf('vc1') > 0;
|
||||
|
||||
// Filter the codec list to the requested media kinds and to what the given
|
||||
// support callback accepts.
|
||||
export const selectSupportedCodecs = (
|
||||
codecs: readonly string[],
|
||||
media: CodecMediaSelection,
|
||||
isSupported: (mimeType: string) => boolean,
|
||||
): readonly string[] =>
|
||||
codecs
|
||||
.filter((codec) => (isVideoCodec(codec) ? media.video : media.audio))
|
||||
.filter((codec) => isSupported(`video/mp4; codecs="${codec}"`));
|
||||
|
||||
// Encode a codec list as the comma-joined string go2rtc expects on the wire.
|
||||
export const convertToCodecString = (codecs: readonly string[]): string =>
|
||||
codecs.join(',');
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { MediaUnavailableIssueReason } from '../../../../../card-controller/issues/issues/media-unavailable';
|
||||
import type { StreamSourceFailureReason } from '../types';
|
||||
|
||||
// The card's media-unavailable causes are user-facing; a source's failure
|
||||
// reasons are technical. Only a media error and a buffer overflow have no
|
||||
// distinct user story, so they map to a generic playback error; the rest each
|
||||
// keep a meaningful cause -- a server rejection, an unsupported stream, a failed
|
||||
// two-way-audio call, or a timeout that means the stream never got going.
|
||||
const FAILURE_TO_ISSUE_REASON: Record<
|
||||
StreamSourceFailureReason,
|
||||
MediaUnavailableIssueReason
|
||||
> = {
|
||||
two_way_audio_error: 'two_way_audio_error',
|
||||
buffer_overflow: 'playback_error',
|
||||
connect_timeout: 'not_loading',
|
||||
media_error: 'playback_error',
|
||||
negotiation_timeout: 'not_loading',
|
||||
server_error: 'server_error',
|
||||
unsupported: 'unsupported',
|
||||
};
|
||||
|
||||
// A null reason is a connection-level failure with no source detail (e.g. the
|
||||
// socket dropped), which reads as a generic playback error.
|
||||
export const mapFailureReasonToIssueReason = (
|
||||
reason: StreamSourceFailureReason | null,
|
||||
): MediaUnavailableIssueReason =>
|
||||
reason === null ? 'playback_error' : FAILURE_TO_ISSUE_REASON[reason];
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
// A GOP ("group of pictures") is the span from one keyframe to just before the
|
||||
// next: a full keyframe followed by the smaller frames that only encode what
|
||||
// changed since it. Streamed video arrives one GOP at a time, so a camera with
|
||||
// a 2s keyframe interval delivers media in roughly 2s bursts. That cadence sets
|
||||
// how far behind the live edge playback must sit to ride over the bursts
|
||||
// without starving, so this file estimates the GOP length by timing the
|
||||
// wall-clock interval between buffer advances (each advance is about one GOP of
|
||||
// new media) and averaging it over a short rolling window.
|
||||
//
|
||||
// Measuring, rather than assuming a fixed length, is necessary here: the stream
|
||||
// carries no declared GOP length (it's raw fragments arriving over a
|
||||
// WebSocket), and camera GOPs vary widely -- a doorbell at ~1s, an NVR tuned
|
||||
// for bandwidth at 4s+ -- so any single constant is either too small (high-GOP
|
||||
// cameras stall) or needlessly laggy for everyone else. Measuring per stream
|
||||
// right-sizes it without a configuration knob.
|
||||
//
|
||||
// This is necessary to adaptively calculate the buffer (and thus inflicted
|
||||
// latency) to maintain for Safari, in order to avoid it "hitting the end" and
|
||||
// thus pausing, requiring human intervention to play again.
|
||||
|
||||
// The assumed GOP length (seconds) until enough delivery cadence is measured,
|
||||
// and how many recent cadence samples the rolling average spans.
|
||||
const DEFAULT_GOP_SECONDS = 1;
|
||||
export const GOP_SAMPLE_WINDOW_SIZE = 5;
|
||||
|
||||
// Estimates the GOP length (the delivery cadence) as a rolling average of the
|
||||
// interval between buffer advances. A trim updates the buffer without growing
|
||||
// it, so only advances are sampled.
|
||||
export class GOPCadenceEstimator {
|
||||
private _samples: number[] = [];
|
||||
private _lastBufferedEnd: number | null = null;
|
||||
private _lastAdvanceTime: Date | null = null;
|
||||
|
||||
public sample(bufferedEndSeconds: number, now: Date): void {
|
||||
const advanced =
|
||||
this._lastBufferedEnd === null || bufferedEndSeconds > this._lastBufferedEnd;
|
||||
if (!advanced) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this._lastAdvanceTime !== null) {
|
||||
const interval = (now.getTime() - this._lastAdvanceTime.getTime()) / 1000;
|
||||
if (interval > 0) {
|
||||
if (this._samples.length >= GOP_SAMPLE_WINDOW_SIZE) {
|
||||
this._samples.shift();
|
||||
}
|
||||
this._samples.push(interval);
|
||||
}
|
||||
}
|
||||
|
||||
this._lastAdvanceTime = now;
|
||||
this._lastBufferedEnd = bufferedEndSeconds;
|
||||
}
|
||||
|
||||
public estimateSeconds(): number {
|
||||
if (!this._samples.length) {
|
||||
return DEFAULT_GOP_SECONDS;
|
||||
}
|
||||
return this._samples.reduce((sum, sample) => sum + sample, 0) / this._samples.length;
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
// Keeps MSE playback near the live edge. The strategy is browser-specific: see
|
||||
// ./webkit.ts (seek-based, because WebKit stutters on playbackRate changes) and
|
||||
// ./non-webkit.ts (playback-rate-based).
|
||||
|
||||
import { NonWebKitLiveEdgeStrategy } from './non-webkit';
|
||||
import type { LiveEdgeAction, LiveEdgeStatus, LiveEdgeStrategy } from './types';
|
||||
import { WebKitLiveEdgeStrategy } from './webkit';
|
||||
|
||||
// Picks the browser-appropriate live-edge strategy and delegates to it.
|
||||
export class LiveEdgeTracker {
|
||||
private _strategy: LiveEdgeStrategy;
|
||||
|
||||
constructor(options: { webkit: boolean }) {
|
||||
this._strategy = options.webkit
|
||||
? new WebKitLiveEdgeStrategy()
|
||||
: new NonWebKitLiveEdgeStrategy();
|
||||
}
|
||||
|
||||
public next(status: LiveEdgeStatus): LiveEdgeAction {
|
||||
return this._strategy.next(status);
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
// Everything that is not WebKit: keep playback near the live edge by nudging the
|
||||
// playback rate (the approach WebKit cannot use; see ./webkit.ts).
|
||||
|
||||
import type { LiveEdgeAction, LiveEdgeStatus, LiveEdgeStrategy } from './types';
|
||||
|
||||
// The lag (seconds) below which playback stays at realtime, how much the
|
||||
// adaptive threshold scales with the stream's normal lag, the exponential rate
|
||||
// curve's scale and steepness, and the rate ceiling.
|
||||
const CATCH_UP_MIN_LAG_SECONDS = 3;
|
||||
const CATCH_UP_AVERAGE_LAG_MULTIPLIER = 1.5;
|
||||
const CATCH_UP_RATE_SCALE = 0.2;
|
||||
const CATCH_UP_RATE_STEEPNESS = 0.5;
|
||||
const CATCH_UP_MAX_RATE = 2;
|
||||
|
||||
// How many recent lag samples the rolling average spans; a tuning value trading
|
||||
// responsiveness against noise, not a derived one.
|
||||
export const LAG_SAMPLE_WINDOW_SIZE = 10;
|
||||
|
||||
// Nudge the playback rate up on an exponential curve that stays gentle for
|
||||
// ordinary drift and only accelerates when playback is far behind, clamped so it
|
||||
// never drops below realtime.
|
||||
export class NonWebKitLiveEdgeStrategy implements LiveEdgeStrategy {
|
||||
private _samples: number[] = [];
|
||||
private _nextSampleIndex = 0;
|
||||
|
||||
public next(status: LiveEdgeStatus): LiveEdgeAction {
|
||||
const lag = status.bufferedEndSeconds - status.currentTimeSeconds;
|
||||
this._sampleLag(lag, status.playbackRate);
|
||||
|
||||
if (lag < CATCH_UP_MIN_LAG_SECONDS) {
|
||||
return { action: 'rate', rate: 1 };
|
||||
}
|
||||
|
||||
// The threshold adapts to the stream's normal lag (e.g. long keyframe
|
||||
// intervals make a constant multi-second lag healthy), so the rate only
|
||||
// rises when playback is genuinely falling behind.
|
||||
const threshold = (this._averageLag() ?? 0) * CATCH_UP_AVERAGE_LAG_MULTIPLIER;
|
||||
const rate = Math.min(
|
||||
1 + CATCH_UP_RATE_SCALE * Math.exp(CATCH_UP_RATE_STEEPNESS * lag - threshold),
|
||||
CATCH_UP_MAX_RATE,
|
||||
);
|
||||
return { action: 'rate', rate };
|
||||
}
|
||||
|
||||
private _sampleLag(lagSeconds: number, playbackRate: number): void {
|
||||
// Samples taken while deliberately catching up (rate above realtime and
|
||||
// still far behind) would inflate the average the threshold derives from,
|
||||
// so they are excluded.
|
||||
if (playbackRate !== 1 && lagSeconds >= CATCH_UP_MIN_LAG_SECONDS) {
|
||||
return;
|
||||
}
|
||||
if (this._samples.length < LAG_SAMPLE_WINDOW_SIZE) {
|
||||
this._samples.push(lagSeconds);
|
||||
} else {
|
||||
this._samples[this._nextSampleIndex] = lagSeconds;
|
||||
this._nextSampleIndex = (this._nextSampleIndex + 1) % LAG_SAMPLE_WINDOW_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
private _averageLag(): number | null {
|
||||
if (!this._samples.length) {
|
||||
return null;
|
||||
}
|
||||
return this._samples.reduce((sum, sample) => sum + sample, 0) / this._samples.length;
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
export type LiveEdgeAction =
|
||||
| { action: 'none' }
|
||||
| { action: 'rate'; rate: number }
|
||||
| { action: 'seek'; seconds: number };
|
||||
|
||||
export interface LiveEdgeStatus {
|
||||
bufferedEndSeconds: number;
|
||||
currentTimeSeconds: number;
|
||||
playbackRate: number;
|
||||
now: Date;
|
||||
}
|
||||
|
||||
// Given the current live-edge lag, returns the action needed to stay near the
|
||||
// edge (a seek, a playback-rate nudge, or nothing).
|
||||
export interface LiveEdgeStrategy {
|
||||
next(status: LiveEdgeStatus): LiveEdgeAction;
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
// WebKit re-buffers whenever `playbackRate` changes, which in practice locks
|
||||
// playback into 1fps slow-motion stutter (see
|
||||
// https://github.com/dermotduffy/advanced-camera-card/issues/2450), so this
|
||||
// strategy never touches the rate and manages position with seeks instead.
|
||||
|
||||
import { GOPCadenceEstimator } from './gop-cadence-estimator';
|
||||
import type { LiveEdgeAction, LiveEdgeStatus, LiveEdgeStrategy } from './types';
|
||||
|
||||
// How many GOPs behind the live edge to hold playback, and the band (seconds)
|
||||
// that hold-back is clamped to so an anomalous GOP estimate cannot pin playback
|
||||
// at the edge or strand it far behind live.
|
||||
const HOLDBACK_GOP_MULTIPLIER = 3;
|
||||
const MIN_HOLDBACK_SECONDS = 1.5;
|
||||
const MAX_HOLDBACK_SECONDS = 8;
|
||||
|
||||
// How far past the hold-back (in GOPs) playback must fall before a forward
|
||||
// catch-up seek, and the minimum gap between such seeks.
|
||||
const CATCHUP_EXCESS_GOP_MULTIPLIER = 2;
|
||||
const JUMP_COOLDOWN_SECONDS = 5;
|
||||
|
||||
// Hold playback a few GOPs behind the live edge. Sitting too close lets the
|
||||
// playhead outrun the bursty per-keyframe delivery and reach the buffered end,
|
||||
// at which point WebKit stalls into a pause a muted stream cannot auto-resume.
|
||||
// The hold-back is sized to the measured GOP cadence, since staying under
|
||||
// roughly three GOPs behind live lets a delivery gap drain the buffer and
|
||||
// stall. Playback is seeked forward only when it has fallen well behind the
|
||||
// hold-back, and back toward it when it drifts within one GOP of the edge.
|
||||
export class WebKitLiveEdgeStrategy implements LiveEdgeStrategy {
|
||||
private _cadence = new GOPCadenceEstimator();
|
||||
private _lastJumpTime: Date | null = null;
|
||||
|
||||
public next(status: LiveEdgeStatus): LiveEdgeAction {
|
||||
this._cadence.sample(status.bufferedEndSeconds, status.now);
|
||||
|
||||
const lag = status.bufferedEndSeconds - status.currentTimeSeconds;
|
||||
const gop = this._cadence.estimateSeconds();
|
||||
const holdback = this._holdbackSeconds(gop);
|
||||
const target = status.bufferedEndSeconds - holdback;
|
||||
|
||||
// Playback has drifted within one GOP of the edge and is about to starve.
|
||||
// WebKit cannot slow down with playbackRate to avoid it, so seek back to the
|
||||
// hold-back to restore runway before it stalls into a pause. Not
|
||||
// cooldown-gated: avoiding the stall takes priority over jump spacing.
|
||||
if (lag < gop) {
|
||||
return { action: 'seek', seconds: target };
|
||||
}
|
||||
|
||||
// Playback has fallen well behind the hold-back (e.g. after a background
|
||||
// tab): jump forward to the hold-back, rate-limited by a cooldown.
|
||||
const secondsSinceLastJump =
|
||||
this._lastJumpTime === null
|
||||
? null
|
||||
: (status.now.getTime() - this._lastJumpTime.getTime()) / 1000;
|
||||
|
||||
if (
|
||||
lag > holdback + CATCHUP_EXCESS_GOP_MULTIPLIER * gop &&
|
||||
(secondsSinceLastJump === null || secondsSinceLastJump >= JUMP_COOLDOWN_SECONDS)
|
||||
) {
|
||||
this._lastJumpTime = status.now;
|
||||
return { action: 'seek', seconds: target };
|
||||
}
|
||||
|
||||
return { action: 'none' };
|
||||
}
|
||||
|
||||
private _holdbackSeconds(gopSeconds: number): number {
|
||||
return Math.min(
|
||||
Math.max(HOLDBACK_GOP_MULTIPLIER * gopSeconds, MIN_HOLDBACK_SECONDS),
|
||||
MAX_HOLDBACK_SECONDS,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import type { Go2RTCMessage } from '../types';
|
||||
|
||||
// The go2rtc server reports a mode failure as `{ type: 'error', value: '<mode>: ...' }`
|
||||
// (e.g. `mse: stream not found`), so an error is for a given mode when its value
|
||||
// starts with that mode's name.
|
||||
export const isServerErrorForMode = (message: Go2RTCMessage, mode: string): boolean =>
|
||||
message.type === 'error' &&
|
||||
typeof message.value === 'string' &&
|
||||
message.value.startsWith(mode);
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { Lane, StreamProfile } from '../types';
|
||||
|
||||
// Choose which lane to present when both a WebRTC stream and a binary-lane
|
||||
// stream (MSE, MP4, MJPEG) are available for the same camera. The factors are
|
||||
// compared in order of decreasing significance -- the first one that differs
|
||||
// decides, regardless of the rest:
|
||||
// 1. Has video -- a camera view needs a picture before anything else.
|
||||
// 2. Has audio -- a complete stream over a silent one.
|
||||
// 3. H.265 over H.264 -- better quality per bitrate.
|
||||
// If all three are equal the tie goes to WebRTC, which plays as it arrives
|
||||
// (lower latency for a live view) rather than from a buffer.
|
||||
export const getPreferredSource = (
|
||||
webrtc: StreamProfile,
|
||||
binary: StreamProfile,
|
||||
): Lane => {
|
||||
// WebRTC audio is observed from live tracks; the binary side's audio comes
|
||||
// from MSE's negotiated codec string, where only AAC plays reliably, so only
|
||||
// AAC counts as audio on the binary side here (see StreamProfile).
|
||||
const factors: [boolean, boolean][] = [
|
||||
[webrtc.hasVideo, binary.hasVideo],
|
||||
[webrtc.hasAudio, binary.hasAACAudio],
|
||||
[webrtc.hasH265Video, binary.hasH265Video],
|
||||
];
|
||||
|
||||
for (const [webRTCHas, binaryHas] of factors) {
|
||||
if (webRTCHas !== binaryHas) {
|
||||
return webRTCHas ? 'webrtc' : 'binary';
|
||||
}
|
||||
}
|
||||
return 'webrtc';
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
// Whether the user agent runs the WebKit engine (Safari, and every browser or
|
||||
// WebView on iOS regardless of branding, e.g. Chrome on iOS is `CriOS`).
|
||||
// Blink-based browsers also advertise `AppleWebKit`, but are distinguishable by
|
||||
// their `Chrome/`, `Chromium` or `Android` tokens.
|
||||
export const isWebKitUserAgent = (userAgent: string): boolean =>
|
||||
userAgent.includes('AppleWebKit') && !/Chrome\/|Chromium|Android/.test(userAgent);
|
||||
|
||||
// The Safari major version (from the `Version/<n>` token), or null when the user
|
||||
// agent is not Safari.
|
||||
export const getSafariMajorVersion = (userAgent: string): number | null => {
|
||||
const match = userAgent.match(/Version\/(\d+).+Safari/);
|
||||
return match ? Number(match[1]) : null;
|
||||
};
|
||||
@@ -0,0 +1,6 @@
|
||||
// The SDP (Session Description Protocol) is the text blob describing a WebRTC
|
||||
// session's media. go2rtc advertises an H.265 track with this rtpmap encoding
|
||||
// name, so its presence in the answer SDP means the browser negotiated H.265 --
|
||||
// preferred over H.264 when choosing between the WebRTC and MSE streams (see
|
||||
// https://github.com/dermotduffy/advanced-camera-card/issues/2200).
|
||||
export const sdpHasH265 = (sdp: string): boolean => sdp.includes('H265/90000');
|
||||
@@ -1,5 +1,20 @@
|
||||
import type { MediaUnavailableIssueReason } from '../../../card-controller/issues/issues/media-unavailable';
|
||||
import { fireAdvancedCameraCardEvent } from '../../../utils/fire-advanced-camera-card-event';
|
||||
|
||||
export function dispatchLiveErrorEvent(element: EventTarget): void {
|
||||
fireAdvancedCameraCardEvent(element, 'live:error');
|
||||
declare global {
|
||||
interface HTMLElementEventMap {
|
||||
'advanced-camera-card:live:error': CustomEvent<
|
||||
MediaUnavailableIssueReason | undefined
|
||||
>;
|
||||
}
|
||||
}
|
||||
|
||||
// The optional reason lets a provider that knows why it failed drive a specific
|
||||
// media-unavailable message; absent, the liveness detector falls back to a
|
||||
// generic playback error.
|
||||
export function dispatchLiveErrorEvent(
|
||||
element: EventTarget,
|
||||
reason?: MediaUnavailableIssueReason,
|
||||
): void {
|
||||
fireAdvancedCameraCardEvent(element, 'live:error', reason);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user