playback: support concatenating segments with long gaps (#5172)

Thanks to the new mtxi MP4 box, it's possible to check whether two
segments are consecutive without involving dates or timestamps.

When the new mtxi box is present in both segments, do not check if
the end of the first segment corresponds to the start of the
second segment.

internal/playback/segment_fmp4.go

@@ -58,24 +58,13 @@ func findMtxi(userData []amp4.IBox) *recordstore.Mtxi {
 	return nil
 }
 
-func segmentFMP4AreConsecutive(init1 *fmp4.Init, init2 *fmp4.Init) bool {
+func segmentFMP4TracksAreEqual(tracks1 []*fmp4.InitTrack, tracks2 []*fmp4.InitTrack) bool {
-	mtxi1 := findMtxi(init1.UserData)
+	if len(tracks1) != len(tracks2) {
-	mtxi2 := findMtxi(init2.UserData)
-
-	switch {
-	case mtxi1 == nil && mtxi2 != nil:
-		return false
-
-	case mtxi1 != nil && mtxi2 == nil:
-		return false
-
-	case mtxi1 == nil && mtxi2 == nil: // legacy method: compare tracks
-		if len(init1.Tracks) != len(init2.Tracks) {
 		return false
 	}
 
-		for i, track1 := range init1.Tracks {
+	for i, track1 := range tracks1 {
-			track2 := init2.Tracks[i]
+		track2 := tracks2[i]
 
 		if track1.ID != track2.ID ||
 			track1.TimeScale != track2.TimeScale ||
@@ -85,11 +74,6 @@ func segmentFMP4AreConsecutive(init1 *fmp4.Init, init2 *fmp4.Init) bool {
 	}
 
 	return true
-
-	default:
-		return bytes.Equal(mtxi1.StreamID[:], mtxi2.StreamID[:]) &&
-			(mtxi1.SegmentNumber+1) == mtxi2.SegmentNumber
-	}
 }
 
 func segmentFMP4CanBeConcatenated(
@@ -98,9 +82,25 @@ func segmentFMP4CanBeConcatenated(
 	curInit *fmp4.Init,
 	curStart time.Time,
 ) bool {
-	return segmentFMP4AreConsecutive(prevInit, curInit) &&
+	mtxi1 := findMtxi(prevInit.UserData)
+	mtxi2 := findMtxi(curInit.UserData)
+
+	switch {
+	case mtxi1 == nil && mtxi2 != nil:
+		return false
+
+	case mtxi1 != nil && mtxi2 == nil:
+		return false
+
+	case mtxi1 == nil && mtxi2 == nil: // legacy method
+		return segmentFMP4TracksAreEqual(prevInit.Tracks, curInit.Tracks) &&
 			!curStart.Before(prevEnd.Add(-concatenationTolerance)) &&
 			!curStart.After(prevEnd.Add(concatenationTolerance))
+
+	default:
+		return bytes.Equal(mtxi1.StreamID[:], mtxi2.StreamID[:]) &&
+			(mtxi1.SegmentNumber+1) == mtxi2.SegmentNumber
+	}
 }
 
 func segmentFMP4ReadHeader(r io.ReadSeeker) (*fmp4.Init, time.Duration, error) {

internal/playback/segment_fmp4_test.go

@@ -4,11 +4,15 @@ import (
 	"io"
 	"os"
 	"testing"
+	"time"
 
+	amp4 "github.com/abema/go-mp4"
 	"github.com/bluenviron/mediacommon/v2/pkg/codecs/mpeg4audio"
 	"github.com/bluenviron/mediacommon/v2/pkg/formats/fmp4"
 	"github.com/bluenviron/mediacommon/v2/pkg/formats/mp4"
+	"github.com/bluenviron/mediamtx/internal/recordstore"
 	"github.com/bluenviron/mediamtx/internal/test"
+	"github.com/stretchr/testify/require"
 )
 
 func writeBenchInit(f io.WriteSeeker) {
@@ -74,3 +78,350 @@ func BenchmarkFMP4ReadHeader(b *testing.B) {
 		}()
 	}
 }
+
+func TestSegmentFMP4CanBeConcatenated(t *testing.T) {
+	baseTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+	streamID1 := [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+	streamID2 := [16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17}
+
+	baseTracks := []*fmp4.InitTrack{
+		{
+			ID:        1,
+			TimeScale: 90000,
+			Codec: &mp4.CodecH264{
+				SPS: test.FormatH264.SPS,
+				PPS: test.FormatH264.PPS,
+			},
+		},
+		{
+			ID:        2,
+			TimeScale: 48000,
+			Codec: &mp4.CodecMPEG4Audio{
+				Config: mpeg4audio.AudioSpecificConfig{
+					Type:         mpeg4audio.ObjectTypeAACLC,
+					SampleRate:   48000,
+					ChannelCount: 2,
+				},
+			},
+		},
+	}
+
+	differentTracks := []*fmp4.InitTrack{
+		{
+			ID:        1,
+			TimeScale: 90000,
+			Codec: &mp4.CodecH264{
+				SPS: test.FormatH264.SPS,
+				PPS: test.FormatH264.PPS,
+			},
+		},
+	}
+
+	for _, tt := range []struct {
+		name     string
+		prevInit *fmp4.Init
+		prevEnd  time.Time
+		curInit  *fmp4.Init
+		curStart time.Time
+		want     bool
+	}{
+		{
+			name: "with mtxi - consecutive segments, same stream",
+			prevInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID1,
+						SegmentNumber: 1,
+					},
+				},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID1,
+						SegmentNumber: 2,
+					},
+				},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     true,
+		},
+		{
+			name: "with mtxi - non-consecutive segments, same stream",
+			prevInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID1,
+						SegmentNumber: 1,
+					},
+				},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID1,
+						SegmentNumber: 3,
+					},
+				},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+		{
+			name: "with mtxi - consecutive segments, different streams",
+			prevInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID1,
+						SegmentNumber: 1,
+					},
+				},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID2,
+						SegmentNumber: 2,
+					},
+				},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+		{
+			name: "prev has mtxi, current does not",
+			prevInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID1,
+						SegmentNumber: 1,
+					},
+				},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+		{
+			name: "prev does not have mtxi, current has mtxi",
+			prevInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks: baseTracks,
+				UserData: []amp4.IBox{
+					&recordstore.Mtxi{
+						StreamID:      streamID1,
+						SegmentNumber: 1,
+					},
+				},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+		{
+			name: "legacy mode - same tracks, within time tolerance",
+			prevInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime.Add(10 * time.Second),
+			curInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(10 * time.Second),
+			want:     true,
+		},
+		{
+			name: "legacy mode - same tracks, exactly at tolerance boundary (before)",
+			prevInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime.Add(10 * time.Second),
+			curInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(9 * time.Second),
+			want:     true,
+		},
+		{
+			name: "legacy mode - same tracks, exactly at tolerance boundary (after)",
+			prevInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime.Add(10 * time.Second),
+			curInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(11 * time.Second),
+			want:     true,
+		},
+		{
+			name: "legacy mode - same tracks, outside time tolerance (too early)",
+			prevInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime.Add(10 * time.Second),
+			curInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(8*time.Second + 999*time.Millisecond),
+			want:     false,
+		},
+		{
+			name: "legacy mode - same tracks, outside time tolerance (too late)",
+			prevInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime.Add(10 * time.Second),
+			curInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(11*time.Second + 1*time.Millisecond),
+			want:     false,
+		},
+		{
+			name: "legacy mode - different number of tracks",
+			prevInit: &fmp4.Init{
+				Tracks:   baseTracks,
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks:   differentTracks,
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+		{
+			name: "legacy mode - different track IDs",
+			prevInit: &fmp4.Init{
+				Tracks: []*fmp4.InitTrack{
+					{
+						ID:        1,
+						TimeScale: 90000,
+						Codec: &mp4.CodecH264{
+							SPS: test.FormatH264.SPS,
+							PPS: test.FormatH264.PPS,
+						},
+					},
+				},
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks: []*fmp4.InitTrack{
+					{
+						ID:        2,
+						TimeScale: 90000,
+						Codec: &mp4.CodecH264{
+							SPS: test.FormatH264.SPS,
+							PPS: test.FormatH264.PPS,
+						},
+					},
+				},
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+		{
+			name: "legacy mode - different time scales",
+			prevInit: &fmp4.Init{
+				Tracks: []*fmp4.InitTrack{
+					{
+						ID:        1,
+						TimeScale: 90000,
+						Codec: &mp4.CodecH264{
+							SPS: test.FormatH264.SPS,
+							PPS: test.FormatH264.PPS,
+						},
+					},
+				},
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks: []*fmp4.InitTrack{
+					{
+						ID:        1,
+						TimeScale: 48000,
+						Codec: &mp4.CodecH264{
+							SPS: test.FormatH264.SPS,
+							PPS: test.FormatH264.PPS,
+						},
+					},
+				},
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+		{
+			name: "legacy mode - different codec types",
+			prevInit: &fmp4.Init{
+				Tracks: []*fmp4.InitTrack{
+					{
+						ID:        1,
+						TimeScale: 90000,
+						Codec: &mp4.CodecH264{
+							SPS: test.FormatH264.SPS,
+							PPS: test.FormatH264.PPS,
+						},
+					},
+				},
+				UserData: []amp4.IBox{},
+			},
+			prevEnd: baseTime,
+			curInit: &fmp4.Init{
+				Tracks: []*fmp4.InitTrack{
+					{
+						ID:        1,
+						TimeScale: 90000,
+						Codec: &mp4.CodecMPEG4Audio{
+							Config: mpeg4audio.AudioSpecificConfig{
+								Type:         mpeg4audio.ObjectTypeAACLC,
+								SampleRate:   48000,
+								ChannelCount: 2,
+							},
+						},
+					},
+				},
+				UserData: []amp4.IBox{},
+			},
+			curStart: baseTime.Add(5 * time.Second),
+			want:     false,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			got := segmentFMP4CanBeConcatenated(tt.prevInit, tt.prevEnd, tt.curInit, tt.curStart)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}