diff --git a/src/components/timeline-core.ts b/src/components/timeline-core.ts
index a02a1771..c4e51c5d 100644
--- a/src/components/timeline-core.ts
+++ b/src/components/timeline-core.ts
@@ -455,6 +455,10 @@ export class FrigateCardTimelineCore extends LitElement {
    * @param properties
    */
   protected _timelineRangeChangeHandler(properties: TimelineRangeChange): void {
+    if (this._pointerHeld) {
+      this._ignoreClick = true;
+    }
+
     if (
       this._timeline &&
       properties.byUser &&
@@ -463,9 +467,6 @@ export class FrigateCardTimelineCore extends LitElement {
       properties.event.additionalEvent !== 'pinchin' &&
       properties.event.additionalEvent !== 'pinchout'
     ) {
-      if (this._pointerHeld) {
-        this._ignoreClick = true;
-      }
 
       const targetTime = this._pointerHeld?.window
         ? add(properties.start, {
diff --git a/src/components/timeline.ts b/src/components/timeline.ts
index 756ad4d5..e2333bca 100644
--- a/src/components/timeline.ts
+++ b/src/components/timeline.ts
@@ -1,6 +1,4 @@
 // TODO: When a media viewer is first loaded the selected child won't work (because the underlying carousel has not yet rendered)
-
-// TODO: delete segments if not in summary? is this actually necessary? could it create gaps in data? better off stopping access via summary?
 // TODO: Make minitimeline configurable in the editor
 
 import { CSSResultGroup, html, LitElement, TemplateResult, unsafeCSS } from 'lit';
diff --git a/src/utils/basic.ts b/src/utils/basic.ts
index 694a5e41..d44142fd 100644
--- a/src/utils/basic.ts
+++ b/src/utils/basic.ts
@@ -94,4 +94,20 @@ export const isHoverableDevice = (): boolean => window.matchMedia(
  */
 export const formatDateAndTime = (date: Date): string => {
   return format(date, 'yyyy-MM-dd HH:mm');
+}
+
+/**
+ * Run a function in idle periods. If idle callbacks are not supported (e.g.
+ * Safari) the callback is run immediately.
+ * @param func The function to call.
+ * @param timeout The maximum number of seconds to wait.
+ */
+export const runWhenIdleIfSupported = (func: () => void, timeout?: number): void => {
+  if (window.requestIdleCallback) {
+    window.requestIdleCallback(func, {
+      ...(timeout && { timeout: timeout})
+    });
+  } else {
+    func();
+  }
 }
\ No newline at end of file
diff --git a/src/utils/frigate.ts b/src/utils/frigate.ts
index c5ef776a..ade6cf3a 100644
--- a/src/utils/frigate.ts
+++ b/src/utils/frigate.ts
@@ -49,7 +49,7 @@ const recordingSummarySchema = z
   .object({
     day: z.preprocess((arg) => {
       // Must provide the hour:minute:second on parsing or Javascript will
-      // assume UTC midnight.
+      // assume *UTC* midnight.
       return typeof arg === 'string' ? new Date(`${arg}T00:00:00`) : arg;
     }, z.date()),
     events: z.number(),
diff --git a/src/utils/timeline-data-manager.ts b/src/utils/timeline-data-manager.ts
index 489e490d..680c351b 100644
--- a/src/utils/timeline-data-manager.ts
+++ b/src/utils/timeline-data-manager.ts
@@ -9,7 +9,7 @@ import {
   FrigateEvent,
   FrigateEvents,
 } from '../types.js';
-import { errorToConsole } from '../utils/basic.js';
+import { errorToConsole, runWhenIdleIfSupported } from '../utils/basic.js';
 import {
   FrigateGetEventsParameters,
   getEventsMultiple,
@@ -19,6 +19,8 @@ import {
   RecordingSummary,
 } from './frigate.js';
 import { dispatchFrigateCardErrorEvent } from '../components/message.js';
+import fromUnixTime from 'date-fns/fromUnixTime';
+import { throttle } from 'lodash-es';
 
 const RECORDING_SEGMENT_TOLERANCE = 60;
 const TIMELINE_DATA_MANAGER_MAX_AGE_SECONDS = 10;
@@ -104,6 +106,15 @@ export class TimelineDataManager {
   protected _cameras: Map<string, CameraConfig>;
   protected _mediaType: TimelineMediaType;
 
+  // Garbage collect segments at most once an hour.
+  protected _throttledSegmentGarbageCollector = throttle(
+    () => {
+      runWhenIdleIfSupported(this._garbageCollectSegments.bind(this));
+    },
+    60 * 60 * 1000,
+    { trailing: true },
+  );
+
   constructor(cameras: Map<string, CameraConfig>, mediaType: TimelineMediaType) {
     this._cameras = cameras;
     this._mediaType = mediaType;
@@ -131,10 +142,14 @@ export class TimelineDataManager {
   ): DataView<FrigateCardTimelineItem> {
     return new DataView(this._dataset, {
       filter: (item: FrigateCardTimelineItem) =>
+        // Only return items for the given cameras.
         !!item.group &&
         cameraIDs.has(String(item.group)) &&
+        // Don't return recordings if the user does not want them.
         (showRecordings || item.type !== 'background') &&
-        (mediaType === 'all' ||
+        // Don't return events that are the wrong media type.
+        (item.type === 'background' ||
+          mediaType === 'all' ||
           (mediaType === 'clips' && !!item.event?.has_clip) ||
           (mediaType === 'snapshots' && !!item.event?.has_snapshot)),
     });
@@ -290,9 +305,50 @@ export class TimelineDataManager {
         ? [this._fetchRecordingSegments(hass, segmentStart, segmentEnd)]
         : []),
     ]);
+
+    this._throttledSegmentGarbageCollector();
     return true;
   }
 
+  /**
+   * Garbage collect recording segments that no longer feature in the summary.
+   */
+  protected _garbageCollectSegments(): void {
+    if (!this._recordingSegments || !this._recordingSummary) {
+      return;
+    }
+
+    // Performance: _recordingSegments is potentially very large (e.g. 10K - 1M
+    // items) and each item must be examined, so care required here to stick to
+    // nothing worse than O(n) performance.
+    const getHourID = (cameraID: string, day: number, hour: number): string => {
+      return `${cameraID}/${day}/${hour}`;
+    };
+
+    const goodHours: Set<string> = new Set();
+    for (const cameraID of this._recordingSummary.keys()) {
+      for (const summaryDay of this._recordingSummary?.get(cameraID) ?? []) {
+        for (const summaryHour of summaryDay.hours) {
+          goodHours.add(getHourID(cameraID, summaryDay.day.getDate(), summaryHour.hour));
+        }
+      }
+    }
+
+    const deleteIDs: string[] = [];
+    this._recordingSegments.forEach((item, id) => {
+      const startDate = fromUnixTime(item.start / 1000);
+      const hourID = getHourID(item.cameraID, startDate.getDate(), startDate.getHours());
+
+      // ~O(1) lookup time for a JS set.
+      if (!goodHours.has(hourID)) {
+        deleteIDs.push(String(id));
+      }
+    });
+
+    this._recordingSegments.remove(deleteIDs);
+    this._compressRecordingSegmentsOntoTimeline();
+  }
+
   /**
    * Fetch recording segments for cameras.
    * @param hass The HomeAssistant object.