feat: Add experimental rewrite of go2rtc live provider (MSE/WebRTC/MP4/MJPEG) (#2580)

- Closes #2556
- Closes #2450

**Key intended features:**
 - go2rtc compatible
- 100% test coverage to significantly improve ability to test, maintain
and work around browser weirdnesses (e.g. Safari).
 - Written from the ground up in the style of the rest of the project.

**To use:**
 - Change `live_provider` from `go2rtc` to `go2rtc-experimental`.
This commit is contained in:
Dermot Duffy
2026-07-14 14:22:41 -07:00
committed by GitHub
parent 5662e48c22
commit c02c692f68
125 changed files with 9608 additions and 807 deletions
@@ -0,0 +1,15 @@
import { describe, expect, it } from 'vitest';
import { arrayBufferToBase64 } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/base64';
// @vitest-environment jsdom
describe('arrayBufferToBase64', () => {
it('should base64-encode the bytes', () => {
const bytes = new TextEncoder().encode('Hi');
expect(arrayBufferToBase64(bytes.buffer)).toBe('SGk=');
});
it('should encode an empty buffer as an empty string', () => {
expect(arrayBufferToBase64(new ArrayBuffer(0))).toBe('');
});
});
@@ -0,0 +1,64 @@
import { describe, expect, it } from 'vitest';
import { BoundedBufferQueue } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/bounded-buffer-queue';
describe('BoundedBufferQueue', () => {
it('should start empty', () => {
expect(new BoundedBufferQueue(10).isEmpty).toBe(true);
});
it('should accept a chunk within the byte cap', () => {
const queue = new BoundedBufferQueue(10);
expect(queue.push(new ArrayBuffer(4))).toBe(true);
expect(queue.isEmpty).toBe(false);
});
it('should accept a chunk that fills the cap exactly', () => {
expect(new BoundedBufferQueue(10).push(new ArrayBuffer(10))).toBe(true);
});
it('should reject a chunk that would exceed the byte cap and stage nothing', () => {
const queue = new BoundedBufferQueue(10);
expect(queue.push(new ArrayBuffer(8))).toBe(true);
expect(queue.push(new ArrayBuffer(3))).toBe(false);
// The rejected chunk left the byte total unchanged, so a smaller one fits.
expect(queue.push(new ArrayBuffer(2))).toBe(true);
});
it('should return staged chunks oldest first', () => {
const queue = new BoundedBufferQueue(10);
const first = new ArrayBuffer(2);
const second = new ArrayBuffer(3);
queue.push(first);
queue.push(second);
expect(queue.shift()).toBe(first);
expect(queue.shift()).toBe(second);
expect(queue.isEmpty).toBe(true);
});
it('should return null when shifting an empty queue', () => {
expect(new BoundedBufferQueue(10).shift()).toBeNull();
});
it('should free the shifted chunk bytes back toward the cap', () => {
const queue = new BoundedBufferQueue(10);
queue.push(new ArrayBuffer(8));
expect(queue.push(new ArrayBuffer(4))).toBe(false);
queue.shift();
expect(queue.push(new ArrayBuffer(4))).toBe(true);
});
it('should reset the chunks and the byte total on clear', () => {
const queue = new BoundedBufferQueue(10);
queue.push(new ArrayBuffer(8));
queue.clear();
expect(queue.isEmpty).toBe(true);
// The byte total reset too, so the full cap is available again.
expect(queue.push(new ArrayBuffer(10))).toBe(true);
});
});
@@ -0,0 +1,102 @@
import { describe, expect, it } from 'vitest';
import {
convertToCodecString,
getCodecsForUserAgent,
GO2RTC_CODECS,
selectSupportedCodecs,
} from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/codecs';
import { CHROME_USER_AGENT, SAFARI_17_USER_AGENT } from '../test-utils';
const safariUserAgent = (version: number): string =>
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 ' +
`(KHTML, like Gecko) Version/${version}.0 Safari/605.1.15`;
describe('getCodecsForUserAgent', () => {
it('should return all codecs for non-Safari browsers', () => {
expect(getCodecsForUserAgent(CHROME_USER_AGENT)).toEqual([...GO2RTC_CODECS]);
});
it('should exclude AAC and later for Safari before version 13', () => {
expect(getCodecsForUserAgent(safariUserAgent(12))).toEqual([
'avc1.640029',
'avc1.64002A',
'avc1.640033',
'hvc1.1.6.L153.B0',
]);
});
it('should exclude FLAC and later for Safari before version 14', () => {
expect(getCodecsForUserAgent(safariUserAgent(13))).toEqual([
'avc1.640029',
'avc1.64002A',
'avc1.640033',
'hvc1.1.6.L153.B0',
'mp4a.40.2',
'mp4a.40.5',
]);
});
it('should exclude OPUS for modern Safari', () => {
expect(getCodecsForUserAgent(SAFARI_17_USER_AGENT)).toEqual([
'avc1.640029',
'avc1.64002A',
'avc1.640033',
'hvc1.1.6.L153.B0',
'mp4a.40.2',
'mp4a.40.5',
'flac',
]);
});
});
describe('selectSupportedCodecs', () => {
it('should include all supported codecs for video and audio', () => {
expect(
selectSupportedCodecs(GO2RTC_CODECS, { audio: true, video: true }, () => true),
).toEqual([
'avc1.640029',
'avc1.64002A',
'avc1.640033',
'hvc1.1.6.L153.B0',
'mp4a.40.2',
'mp4a.40.5',
'flac',
'opus',
]);
});
it('should exclude audio codecs when audio is not requested', () => {
expect(
selectSupportedCodecs(GO2RTC_CODECS, { audio: false, video: true }, () => true),
).toEqual(['avc1.640029', 'avc1.64002A', 'avc1.640033', 'hvc1.1.6.L153.B0']);
});
it('should exclude video codecs when video is not requested', () => {
expect(
selectSupportedCodecs(GO2RTC_CODECS, { audio: true, video: false }, () => true),
).toEqual(['mp4a.40.2', 'mp4a.40.5', 'flac', 'opus']);
});
it('should exclude codecs the support callback rejects', () => {
expect(
selectSupportedCodecs(
GO2RTC_CODECS,
{ audio: true, video: true },
(mimeType) => mimeType === 'video/mp4; codecs="avc1.640029"',
),
).toEqual(['avc1.640029']);
});
});
describe('convertToCodecString', () => {
it('should join codecs with commas', () => {
expect(convertToCodecString(['avc1.640029', 'mp4a.40.2'])).toBe(
'avc1.640029,mp4a.40.2',
);
});
it('should return an empty string for no codecs', () => {
expect(convertToCodecString([])).toBe('');
});
});
@@ -0,0 +1,21 @@
import { describe, expect, it } from 'vitest';
import { mapFailureReasonToIssueReason } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/failure-reason';
describe('mapFailureReasonToIssueReason', () => {
it.each([
['connect_timeout', 'not_loading'],
['negotiation_timeout', 'not_loading'],
['media_error', 'playback_error'],
['buffer_overflow', 'playback_error'],
['two_way_audio_error', 'two_way_audio_error'],
['server_error', 'server_error'],
['unsupported', 'unsupported'],
] as const)('should map %s to the %s cause', (reason, expected) => {
expect(mapFailureReasonToIssueReason(reason)).toBe(expected);
});
it('should map a null reason to a generic playback error', () => {
expect(mapFailureReasonToIssueReason(null)).toBe('playback_error');
});
});
@@ -0,0 +1,54 @@
import { describe, expect, it } from 'vitest';
import {
GOP_SAMPLE_WINDOW_SIZE,
GOPCadenceEstimator,
} from '../../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/gop-cadence-estimator';
describe('GOPCadenceEstimator', () => {
it('should return the default GOP before any samples', () => {
expect(new GOPCadenceEstimator().estimateSeconds()).toBe(1);
});
it('should average the interval between buffer advances', () => {
const estimator = new GOPCadenceEstimator();
estimator.sample(10, new Date(0));
estimator.sample(12, new Date(2000));
estimator.sample(14, new Date(4000));
expect(estimator.estimateSeconds()).toBe(2);
});
it('should ignore updates that do not advance the buffer', () => {
const estimator = new GOPCadenceEstimator();
estimator.sample(10, new Date(0));
estimator.sample(12, new Date(2000));
// A trim: same buffered end, later time -> not a delivery interval, and it
// must not reset the last-advance timestamp.
estimator.sample(12, new Date(5000));
estimator.sample(14, new Date(6000));
// Intervals are 2s (0 -> 2000) and 4s (2000 -> 6000) -> average 3s.
expect(estimator.estimateSeconds()).toBe(3);
});
it('should ignore advances with no elapsed time', () => {
const estimator = new GOPCadenceEstimator();
estimator.sample(10, new Date(1000));
estimator.sample(12, new Date(1000));
expect(estimator.estimateSeconds()).toBe(1);
});
it('should evict the oldest sample beyond the window', () => {
const estimator = new GOPCadenceEstimator();
// A slow 5s interval, then a full window of 1s intervals evicts it.
estimator.sample(0, new Date(0));
estimator.sample(5, new Date(5000));
let time = 5000;
let end = 5;
for (let i = 0; i < GOP_SAMPLE_WINDOW_SIZE; ++i) {
time += 1000;
end += 1;
estimator.sample(end, new Date(time));
}
expect(estimator.estimateSeconds()).toBe(1);
});
});
@@ -0,0 +1,17 @@
import { describe, expect, it } from 'vitest';
import { LiveEdgeTracker } from '../../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker';
import { createStatus } from './test-utils';
describe('LiveEdgeTracker', () => {
it('should use the seek strategy on WebKit', () => {
const tracker = new LiveEdgeTracker({ webkit: true });
// Far behind: the WebKit strategy seeks to the default 3s hold-back.
expect(tracker.next(createStatus(20, 13))).toEqual({ action: 'seek', seconds: 17 });
});
it('should use the playback-rate strategy on other browsers', () => {
const tracker = new LiveEdgeTracker({ webkit: false });
expect(tracker.next(createStatus(20, 18))).toEqual({ action: 'rate', rate: 1 });
});
});
@@ -0,0 +1,75 @@
import { assert, describe, expect, it } from 'vitest';
import {
LAG_SAMPLE_WINDOW_SIZE,
NonWebKitLiveEdgeStrategy,
} from '../../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/non-webkit';
import { createStatus } from './test-utils';
describe('NonWebKitLiveEdgeStrategy', () => {
it('should play at realtime when close to the live edge', () => {
const strategy = new NonWebKitLiveEdgeStrategy();
expect(strategy.next(createStatus(20, 18))).toEqual({ action: 'rate', rate: 1 });
});
it('should nudge the rate up when lag exceeds the stream norm', () => {
const strategy = new NonWebKitLiveEdgeStrategy();
for (let i = 0; i < LAG_SAMPLE_WINDOW_SIZE; ++i) {
strategy.next(createStatus(20, 19));
}
const action = strategy.next(createStatus(20, 15));
assert(action.action === 'rate');
expect(action.rate).toBeGreaterThan(1);
expect(action.rate).toBeLessThanOrEqual(2);
});
it('should cap the catch-up rate', () => {
const strategy = new NonWebKitLiveEdgeStrategy();
for (let i = 0; i < LAG_SAMPLE_WINDOW_SIZE; ++i) {
strategy.next(createStatus(20, 20));
}
expect(strategy.next(createStatus(60, 10))).toEqual({ action: 'rate', rate: 2 });
});
it('should stay near realtime within the stream normal lag', () => {
const strategy = new NonWebKitLiveEdgeStrategy();
for (let i = 0; i < LAG_SAMPLE_WINDOW_SIZE; ++i) {
strategy.next(createStatus(24, 20));
}
const action = strategy.next(createStatus(24, 20));
assert(action.action === 'rate');
expect(action.rate).toBeCloseTo(1, 2);
});
it('should catch up hard before any baseline samples exist', () => {
const strategy = new NonWebKitLiveEdgeStrategy();
// The very first sample is taken while already catching up, so it is
// excluded and there is no average to temper the threshold.
expect(strategy.next(createStatus(20, 15, { playbackRate: 2 }))).toEqual({
action: 'rate',
rate: 2,
});
});
it('should drop stale lag samples as the stream recovers', () => {
const strategy = new NonWebKitLiveEdgeStrategy();
for (let i = 0; i < LAG_SAMPLE_WINDOW_SIZE; ++i) {
strategy.next(createStatus(26, 20));
}
// A full window of low lag evicts the earlier high-lag samples.
for (let i = 0; i < LAG_SAMPLE_WINDOW_SIZE; ++i) {
strategy.next(createStatus(21, 20));
}
const action = strategy.next(createStatus(25, 20));
assert(action.action === 'rate');
expect(action.rate).toBeGreaterThan(1.1);
});
});
@@ -0,0 +1,10 @@
export const createStatus = (
bufferedEndSeconds: number,
currentTimeSeconds: number,
options?: { playbackRate?: number; now?: Date },
) => ({
bufferedEndSeconds,
currentTimeSeconds,
playbackRate: options?.playbackRate ?? 1,
now: options?.now ?? new Date(0),
});
@@ -0,0 +1,100 @@
import { describe, expect, it } from 'vitest';
import { GOP_SAMPLE_WINDOW_SIZE } from '../../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/gop-cadence-estimator';
import { WebKitLiveEdgeStrategy } from '../../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/live-edge-tracker/webkit';
import { createStatus } from './test-utils';
describe('WebKitLiveEdgeStrategy', () => {
// With no measured cadence the GOP defaults to 1s, so the hold-back is 3s.
it('should do nothing within the hold-back band', () => {
const strategy = new WebKitLiveEdgeStrategy();
expect(strategy.next(createStatus(20, 18))).toEqual({ action: 'none' });
});
it('should seek back to the hold-back when starving within a GOP of the edge', () => {
const strategy = new WebKitLiveEdgeStrategy();
expect(strategy.next(createStatus(20, 19.5))).toEqual({
action: 'seek',
seconds: 17,
});
});
it('should seek forward to the hold-back when far behind', () => {
const strategy = new WebKitLiveEdgeStrategy();
expect(strategy.next(createStatus(20, 13))).toEqual({ action: 'seek', seconds: 17 });
});
it('should not seek forward again within the cooldown', () => {
const strategy = new WebKitLiveEdgeStrategy();
strategy.next(createStatus(20, 13, { now: new Date(0) }));
expect(strategy.next(createStatus(20, 13, { now: new Date(2000) }))).toEqual({
action: 'none',
});
});
it('should seek forward again after the cooldown', () => {
const strategy = new WebKitLiveEdgeStrategy();
strategy.next(createStatus(20, 13, { now: new Date(0) }));
expect(strategy.next(createStatus(20, 13, { now: new Date(6000) }))).toEqual({
action: 'seek',
seconds: 17,
});
});
it('should widen the hold-back to the measured GOP cadence', () => {
const strategy = new WebKitLiveEdgeStrategy();
// Buffer advances 2s apart -> GOP ~2s -> hold-back 6s. Runs past the sample
// window so the oldest samples are evicted. (Sampling happens before the
// action, so intermediate seeks do not affect the estimate.)
let end = 20;
for (let i = 1; i <= GOP_SAMPLE_WINDOW_SIZE + 2; ++i) {
end = 20 + i * 2;
strategy.next(createStatus(end, end - 4, { now: new Date(i * 2000) }));
}
// A far-behind sample now seeks to bufferedEnd - 6, not - 3.
expect(
strategy.next(createStatus(end, end - 12, { now: new Date(100000) })),
).toEqual({
action: 'seek',
seconds: end - 6,
});
});
it('should clamp the widened hold-back to the maximum', () => {
const strategy = new WebKitLiveEdgeStrategy();
// Buffer advances 5s apart -> GOP 5s -> 15s, clamped to 8s.
let end = 0;
for (let i = 1; i <= GOP_SAMPLE_WINDOW_SIZE + 1; ++i) {
end = i * 5;
strategy.next(createStatus(end, end - 2, { now: new Date(i * 5000) }));
}
expect(
strategy.next(createStatus(end, end - 19, { now: new Date(100000) })),
).toEqual({
action: 'seek',
seconds: end - 8,
});
});
it('should clamp the shrunken hold-back to the minimum', () => {
const strategy = new WebKitLiveEdgeStrategy();
// Buffer advances 0.3s apart -> GOP 0.3s -> 0.9s, clamped to 1.5s.
let end = 20;
for (let i = 1; i <= GOP_SAMPLE_WINDOW_SIZE + 1; ++i) {
end = 20 + i * 0.3;
strategy.next(createStatus(end, end - 0.1, { now: new Date(i * 300) }));
}
expect(strategy.next(createStatus(end, end - 3, { now: new Date(100000) }))).toEqual(
{
action: 'seek',
seconds: end - 1.5,
},
);
});
});
@@ -0,0 +1,29 @@
import { describe, expect, it } from 'vitest';
import { isServerErrorForMode } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/messages';
describe('isServerErrorForMode', () => {
it('should match an error for the mode', () => {
expect(isServerErrorForMode({ type: 'error', value: 'mse: not found' }, 'mse')).toBe(
true,
);
});
it('should not match an error for another mode', () => {
expect(isServerErrorForMode({ type: 'error', value: 'webrtc: failed' }, 'mse')).toBe(
false,
);
});
it('should not match a non-error message', () => {
expect(isServerErrorForMode({ type: 'mse', value: 'codecs' }, 'mse')).toBe(false);
});
it('should not match an error with a non-string value', () => {
expect(isServerErrorForMode({ type: 'error', value: 42 }, 'mse')).toBe(false);
});
it('should not match an error with no value', () => {
expect(isServerErrorForMode({ type: 'error' }, 'mse')).toBe(false);
});
});
@@ -0,0 +1,76 @@
import { describe, expect, it } from 'vitest';
import type { StreamProfile } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/types';
import { getPreferredSource } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/source-priority';
const createProfile = (overrides: Partial<StreamProfile>): StreamProfile => ({
hasVideo: false,
hasH265Video: false,
hasAudio: false,
hasAACAudio: false,
...overrides,
});
describe('getPreferredSource', () => {
it('should prefer WebRTC when both offer equal H.264 video and audio', () => {
expect(
getPreferredSource(
createProfile({ hasVideo: true, hasAudio: true }),
createProfile({ hasVideo: true, hasAACAudio: true }),
),
).toBe('webrtc');
});
it('should prefer WebRTC H.265 over binary-source H.265', () => {
expect(
getPreferredSource(
createProfile({ hasVideo: true, hasH265Video: true }),
createProfile({ hasVideo: true, hasH265Video: true }),
),
).toBe('webrtc');
});
it('should prefer binary-source H.265 with audio over WebRTC H.264 with audio', () => {
expect(
getPreferredSource(
createProfile({ hasVideo: true, hasAudio: true }),
createProfile({ hasVideo: true, hasH265Video: true, hasAACAudio: true }),
),
).toBe('binary');
});
it('should prefer WebRTC when the binary source has no media', () => {
expect(
getPreferredSource(createProfile({ hasVideo: true }), createProfile({})),
).toBe('webrtc');
});
it('should prefer the binary source when WebRTC has no video', () => {
expect(
getPreferredSource(
createProfile({ hasAudio: true }),
createProfile({ hasVideo: true }),
),
).toBe('binary');
});
it('should prefer the stream with audio when video is otherwise equal', () => {
expect(
getPreferredSource(
createProfile({ hasVideo: true }),
createProfile({ hasVideo: true, hasAACAudio: true }),
),
).toBe('binary');
});
it('should not count non-AAC MSE audio', () => {
// MSE opus audio (hasAudio true, hasAACAudio false) does not raise the
// binary-side score, so WebRTC video wins.
expect(
getPreferredSource(
createProfile({ hasVideo: true }),
createProfile({ hasVideo: true, hasAudio: true }),
),
).toBe('webrtc');
});
});
@@ -0,0 +1,86 @@
import { describe, expect, it } from 'vitest';
import {
getSafariMajorVersion,
isWebKitUserAgent,
} from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/user-agent';
describe('isWebKitUserAgent', () => {
it.each([
[
'Safari on macOS',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 ' +
'(KHTML, like Gecko) Version/17.4 Safari/605.1.15',
true,
],
[
'Safari on iOS',
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 ' +
'(KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1',
true,
],
[
'iOS WebView without a Safari token',
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 ' +
'(KHTML, like Gecko) Mobile/21A329',
true,
],
[
'Chrome on iOS',
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 ' +
'(KHTML, like Gecko) CriOS/123.0.6312.52 Mobile/15E148 Safari/604.1',
true,
],
[
'Firefox on iOS',
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 ' +
'(KHTML, like Gecko) FxiOS/124.0 Mobile/15E148 Safari/605.1.15',
true,
],
[
'Chrome on Linux',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
false,
],
[
'Chrome on Android',
'Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/126.0.6478.71 Mobile Safari/537.36',
false,
],
[
'Edge on Windows',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0',
false,
],
[
'Firefox on Linux',
'Mozilla/5.0 (X11; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0',
false,
],
])('should detect %s', (_name: string, userAgent: string, expected: boolean) => {
expect(isWebKitUserAgent(userAgent)).toBe(expected);
});
});
describe('getSafariMajorVersion', () => {
it('should return the major version for Safari', () => {
expect(
getSafariMajorVersion(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 ' +
'(KHTML, like Gecko) Version/17.4 Safari/605.1.15',
),
).toBe(17);
});
it('should return null for a non-Safari user agent', () => {
expect(
getSafariMajorVersion(
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
),
).toBe(null);
});
});
@@ -0,0 +1,13 @@
import { describe, expect, it } from 'vitest';
import { sdpHasH265 } from '../../../../../../src/components-lib/live/providers/go2rtc-experimental/utils/webrtc-sdp';
describe('sdpHasH265', () => {
it('should detect an H.265 rtpmap', () => {
expect(sdpHasH265('a=rtpmap:98 H265/90000\r\n')).toBe(true);
});
it('should return false without an H.265 rtpmap', () => {
expect(sdpHasH265('a=rtpmap:96 H264/90000\r\n')).toBe(false);
});
});