rtmp: support additional enhanced RTMP features (#4168) (#4321) (#4954)

* support reading AV1, VP9, H265, Opus, AC-3, G711, LPCM
* support reading multiple video or audio tracks at once

internal/formatprocessor/av1.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
 	"github.com/bluenviron/gortsplib/v4/pkg/format/rtpav1"
+	mcav1 "github.com/bluenviron/mediacommon/v2/pkg/codecs/av1"
 	"github.com/pion/rtp"
 
 	"github.com/bluenviron/mediamtx/internal/logger"
@@ -48,9 +49,44 @@ func (t *av1) createEncoder() error {
 	return t.encoder.Init()
 }
 
+func (t *av1) remuxTemporalUnit(tu [][]byte) [][]byte {
+	n := 0
+
+	for _, obu := range tu {
+		typ := mcav1.OBUType((obu[0] >> 3) & 0b1111)
+
+		if typ == mcav1.OBUTypeTemporalDelimiter {
+			continue
+		}
+		n++
+	}
+
+	if n == 0 {
+		return nil
+	}
+
+	filteredTU := make([][]byte, n)
+	i := 0
+
+	for _, obu := range tu {
+		typ := mcav1.OBUType((obu[0] >> 3) & 0b1111)
+
+		if typ == mcav1.OBUTypeTemporalDelimiter {
+			continue
+		}
+
+		filteredTU[i] = obu
+		i++
+	}
+
+	return filteredTU
+}
+
 func (t *av1) ProcessUnit(uu unit.Unit) error { //nolint:dupl
 	u := uu.(*unit.AV1)
 
+	u.TU = t.remuxTemporalUnit(u.TU)
+
 	pkts, err := t.encoder.Encode(u.TU)
 	if err != nil {
 		return err
@@ -106,7 +142,7 @@ func (t *av1) ProcessRTPPacket( //nolint:dupl
 			return nil, err
 		}
 
-		u.TU = tu
+		u.TU = t.remuxTemporalUnit(tu)
 	}
 
 	// route packet as is

internal/formatprocessor/av1_test.go

@@ -0,0 +1,34 @@
+package formatprocessor
+
+import (
+	"testing"
+
+	"github.com/bluenviron/gortsplib/v4/pkg/format"
+	mcav1 "github.com/bluenviron/mediacommon/v2/pkg/codecs/av1"
+	"github.com/bluenviron/mediamtx/internal/unit"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAV1RemoveTUD(t *testing.T) {
+	forma := &format.AV1{}
+
+	p, err := New(1450, forma, true, nil)
+	require.NoError(t, err)
+
+	u := &unit.AV1{
+		Base: unit.Base{
+			PTS: 30000,
+		},
+		TU: [][]byte{
+			{byte(mcav1.OBUTypeTemporalDelimiter) << 3},
+			{5},
+		},
+	}
+
+	err = p.ProcessUnit(u)
+	require.NoError(t, err)
+
+	require.Equal(t, [][]byte{
+		{5},
+	}, u.TU)
+}

internal/formatprocessor/h264.go

@@ -174,13 +174,13 @@ func (t *h264) remuxAccessUnit(au [][]byte) [][]byte {
 		typ := mch264.NALUType(nalu[0] & 0x1F)
 
 		switch typ {
-		case mch264.NALUTypeSPS, mch264.NALUTypePPS: // parameters: remove
+		case mch264.NALUTypeSPS, mch264.NALUTypePPS:
 			continue
 
-		case mch264.NALUTypeAccessUnitDelimiter: // AUD: remove
+		case mch264.NALUTypeAccessUnitDelimiter:
 			continue
 
-		case mch264.NALUTypeIDR: // key frame
+		case mch264.NALUTypeIDR:
 			if !isKeyFrame {
 				isKeyFrame = true
 
@@ -197,12 +197,12 @@ func (t *h264) remuxAccessUnit(au [][]byte) [][]byte {
 		return nil
 	}
 
-	filteredNALUs := make([][]byte, n)
+	filteredAU := make([][]byte, n)
 	i := 0
 
 	if isKeyFrame && t.Format.SPS != nil && t.Format.PPS != nil {
-		filteredNALUs[0] = t.Format.SPS
-		filteredNALUs[1] = t.Format.PPS
+		filteredAU[0] = t.Format.SPS
+		filteredAU[1] = t.Format.PPS
 		i = 2
 	}
 
@@ -217,11 +217,11 @@ func (t *h264) remuxAccessUnit(au [][]byte) [][]byte {
 			continue
 		}
 
-		filteredNALUs[i] = nalu
+		filteredAU[i] = nalu
 		i++
 	}
 
-	return filteredNALUs
+	return filteredAU
 }
 
 func (t *h264) ProcessUnit(uu unit.Unit) error {

internal/formatprocessor/h264_test.go

@@ -28,7 +28,31 @@ func Logger(cb func(logger.Level, string, ...interface{})) logger.Writer {
 	return &testLogger{cb: cb}
 }
 
-func TestH264ProcessUnit(t *testing.T) {
+func TestH264RemoveAUD(t *testing.T) {
+	forma := &format.H264{}
+
+	p, err := New(1450, forma, true, nil)
+	require.NoError(t, err)
+
+	u := &unit.H264{
+		Base: unit.Base{
+			PTS: 30000,
+		},
+		AU: [][]byte{
+			{9, 24}, // AUD
+			{5, 1},  // IDR
+		},
+	}
+
+	err = p.ProcessUnit(u)
+	require.NoError(t, err)
+
+	require.Equal(t, [][]byte{
+		{5, 1}, // IDR
+	}, u.AU)
+}
+
+func TestH264AddParams(t *testing.T) {
 	forma := &format.H264{}
 
 	p, err := New(1450, forma, true, nil)
@@ -77,11 +101,11 @@ func TestH264ProcessUnit(t *testing.T) {
 		{5, 2},       // IDR
 	}, u2.AU)
 
-	// test that timestamp had increased
+	// test that timestamp has increased
 	require.Equal(t, u1.RTPPackets[0].Timestamp+30000, u2.RTPPackets[0].Timestamp)
 }
 
-func TestH264ProcessUnitEmpty(t *testing.T) {
+func TestH264ProcessEmptyUnit(t *testing.T) {
 	forma := &format.H264{
 		PayloadTyp:        96,
 		PacketizationMode: 1,
@@ -104,7 +128,7 @@ func TestH264ProcessUnitEmpty(t *testing.T) {
 	require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets)
 }
 
-func TestH264ProcessRTPPacketUpdateParams(t *testing.T) {
+func TestH264RTPExtractParams(t *testing.T) {
 	for _, ca := range []string{"standard", "aggregated"} {
 		t.Run(ca, func(t *testing.T) {
 			forma := &format.H264{
@@ -169,7 +193,7 @@ func TestH264ProcessRTPPacketUpdateParams(t *testing.T) {
 	}
 }
 
-func TestH264ProcessRTPPacketOversized(t *testing.T) {
+func TestH264RTPOversized(t *testing.T) {
 	forma := &format.H264{
 		PayloadTyp:        96,
 		SPS:               []byte{0x01, 0x02, 0x03, 0x04},

internal/formatprocessor/h265.go

@@ -205,13 +205,13 @@ func (t *h265) remuxAccessUnit(au [][]byte) [][]byte {
 		typ := mch265.NALUType((nalu[0] >> 1) & 0b111111)
 
 		switch typ {
-		case mch265.NALUType_VPS_NUT, mch265.NALUType_SPS_NUT, mch265.NALUType_PPS_NUT: // parameters: remove
+		case mch265.NALUType_VPS_NUT, mch265.NALUType_SPS_NUT, mch265.NALUType_PPS_NUT:
 			continue
 
-		case mch265.NALUType_AUD_NUT: // AUD: remove
+		case mch265.NALUType_AUD_NUT:
 			continue
 
-		case mch265.NALUType_IDR_W_RADL, mch265.NALUType_IDR_N_LP, mch265.NALUType_CRA_NUT: // key frame
+		case mch265.NALUType_IDR_W_RADL, mch265.NALUType_IDR_N_LP, mch265.NALUType_CRA_NUT:
 			if !isKeyFrame {
 				isKeyFrame = true
 
@@ -228,13 +228,13 @@ func (t *h265) remuxAccessUnit(au [][]byte) [][]byte {
 		return nil
 	}
 
-	filteredNALUs := make([][]byte, n)
+	filteredAU := make([][]byte, n)
 	i := 0
 
 	if isKeyFrame && t.Format.VPS != nil && t.Format.SPS != nil && t.Format.PPS != nil {
-		filteredNALUs[0] = t.Format.VPS
-		filteredNALUs[1] = t.Format.SPS
-		filteredNALUs[2] = t.Format.PPS
+		filteredAU[0] = t.Format.VPS
+		filteredAU[1] = t.Format.SPS
+		filteredAU[2] = t.Format.PPS
 		i = 3
 	}
 
@@ -249,11 +249,11 @@ func (t *h265) remuxAccessUnit(au [][]byte) [][]byte {
 			continue
 		}
 
-		filteredNALUs[i] = nalu
+		filteredAU[i] = nalu
 		i++
 	}
 
-	return filteredNALUs
+	return filteredAU
 }
 
 func (t *h265) ProcessUnit(uu unit.Unit) error { //nolint:dupl

internal/formatprocessor/h265_test.go

@@ -15,7 +15,31 @@ import (
 	"github.com/bluenviron/mediamtx/internal/unit"
 )
 
-func TestH265ProcessUnit(t *testing.T) {
+func TestH265RemoveAUD(t *testing.T) {
+	forma := &format.H265{}
+
+	p, err := New(1450, forma, true, nil)
+	require.NoError(t, err)
+
+	u := &unit.H265{
+		Base: unit.Base{
+			PTS: 30000,
+		},
+		AU: [][]byte{
+			{byte(mch265.NALUType_AUD_NUT) << 1, 0},
+			{byte(mch265.NALUType_CRA_NUT) << 1, 0},
+		},
+	}
+
+	err = p.ProcessUnit(u)
+	require.NoError(t, err)
+
+	require.Equal(t, [][]byte{
+		{byte(mch265.NALUType_CRA_NUT) << 1, 0},
+	}, u.AU)
+}
+
+func TestH265AddParams(t *testing.T) {
 	forma := &format.H265{}
 
 	p, err := New(1450, forma, true, nil)
@@ -68,11 +92,11 @@ func TestH265ProcessUnit(t *testing.T) {
 		{byte(mch265.NALUType_CRA_NUT) << 1, 1},
 	}, u2.AU)
 
-	// test that timestamp had increased
+	// test that timestamp has increased
 	require.Equal(t, u1.RTPPackets[0].Timestamp+30000, u2.RTPPackets[0].Timestamp)
 }
 
-func TestH265ProcessUnitEmpty(t *testing.T) {
+func TestH265ProcessEmptyUnit(t *testing.T) {
 	forma := &format.H265{
 		PayloadTyp: 96,
 	}
@@ -95,7 +119,7 @@ func TestH265ProcessUnitEmpty(t *testing.T) {
 	require.Equal(t, []*rtp.Packet(nil), unit.RTPPackets)
 }
 
-func TestH265ProcessRTPPacketUpdateParams(t *testing.T) {
+func TestH265RTPExtractParams(t *testing.T) {
 	for _, ca := range []string{"standard", "aggregated"} {
 		t.Run(ca, func(t *testing.T) {
 			forma := &format.H265{
@@ -168,7 +192,7 @@ func TestH265ProcessRTPPacketUpdateParams(t *testing.T) {
 	}
 }
 
-func TestH265ProcessRTPPacketOversized(t *testing.T) {
+func TestH265RTPOversized(t *testing.T) {
 	forma := &format.H265{
 		PayloadTyp: 96,
 		VPS:        []byte{byte(mch265.NALUType_VPS_NUT) << 1, 10, 11, 12},

internal/formatprocessor/mpeg4_video_test.go

@@ -66,6 +66,6 @@ func TestMPEG4VideoProcessUnit(t *testing.T) {
 		0, 0, 1, 0xF1,
 	}, u2.Frame)
 
-	// test that timestamp had increased
+	// test that timestamp has increased
 	require.Equal(t, u1.RTPPackets[0].Timestamp+30000, u2.RTPPackets[0].Timestamp)
 }