webrtc: set fmtp of outgoing VP9 and multiopus tracks (#3446)

internal/protocols/webrtc/outgoing_track.go

@@ -8,6 +8,15 @@ import (
 	"github.com/pion/webrtc/v3"
 )
 
+var multichannelOpusSDP = map[int]string{
+	3: "channel_mapping=0,2,1;num_streams=2;coupled_streams=1",
+	4: "channel_mapping=0,1,2,3;num_streams=2;coupled_streams=2",
+	5: "channel_mapping=0,4,1,2,3;num_streams=3;coupled_streams=2",
+	6: "channel_mapping=0,4,1,2,3,5;num_streams=4;coupled_streams=2",
+	7: "channel_mapping=0,4,1,2,3,5,6;num_streams=4;coupled_streams=4",
+	8: "channel_mapping=0,6,1,4,5,2,3,7;num_streams=5;coupled_streams=4",
+}
+
 // OutgoingTrack is a WebRTC outgoing track
 type OutgoingTrack struct {
 	Format format.Format
@@ -29,8 +38,9 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) {
 	case *format.VP9:
 		return webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:  webrtc.MimeTypeVP9,
+				MimeType:    webrtc.MimeTypeVP9,
-				ClockRate: 90000,
+				ClockRate:   90000,
+				SDPFmtpLine: "profile-id=0",
 			},
 			PayloadType: 96,
 		}, nil
@@ -55,32 +65,38 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) {
 		}, nil
 
 	case *format.Opus:
-		if forma.ChannelCount > 2 {
+		switch forma.ChannelCount {
+		case 1, 2:
 			return webrtc.RTPCodecParameters{
 				RTPCodecCapability: webrtc.RTPCodecCapability{
-					MimeType:  mimeTypeMultiopus,
+					MimeType:  webrtc.MimeTypeOpus,
 					ClockRate: 48000,
-					Channels:  uint16(forma.ChannelCount),
+					Channels:  2,
+					SDPFmtpLine: func() string {
+						s := "minptime=10;useinbandfec=1"
+						if forma.ChannelCount == 2 {
+							s += ";stereo=1;sprop-stereo=1"
+						}
+						return s
+					}(),
 				},
 				PayloadType: 96,
 			}, nil
-		}
 
-		return webrtc.RTPCodecParameters{
+		case 3, 4, 5, 6, 7, 8:
-			RTPCodecCapability: webrtc.RTPCodecCapability{
+			return webrtc.RTPCodecParameters{
-				MimeType:  webrtc.MimeTypeOpus,
+				RTPCodecCapability: webrtc.RTPCodecCapability{
-				ClockRate: 48000,
+					MimeType:    mimeTypeMultiopus,
-				Channels:  2,
+					ClockRate:   48000,
-				SDPFmtpLine: func() string {
+					Channels:    uint16(forma.ChannelCount),
-					s := "minptime=10;useinbandfec=1"
+					SDPFmtpLine: multichannelOpusSDP[forma.ChannelCount],
-					if forma.ChannelCount == 2 {
+				},
-						s += ";stereo=1;sprop-stereo=1"
+				PayloadType: 96,
-					}
+			}, nil
-					return s
+
-				}(),
+		default:
-			},
+			return webrtc.RTPCodecParameters{}, fmt.Errorf("unsupported channel count: %d", forma.ChannelCount)
-			PayloadType: 96,
+		}
-		}, nil
 
 	case *format.G722:
 		return webrtc.RTPCodecParameters{

internal/protocols/webrtc/peer_connection_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/bluenviron/mediamtx/internal/conf"
 	"github.com/bluenviron/mediamtx/internal/test"
 	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v3"
 	"github.com/stretchr/testify/require"
 )
 
@@ -36,15 +37,20 @@ func TestPeerConnectionCloseImmediately(t *testing.T) {
 
 func TestPeerConnectionPublishRead(t *testing.T) {
 	for _, ca := range []struct {
-		name string
+		name      string
-		in   format.Format
+		in        format.Format
-		out  format.Format
+		webrtcOut webrtc.RTPCodecCapability
+		out       format.Format
 	}{
 		{
 			"av1",
 			&format.AV1{
 				PayloadTyp: 96,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "video/AV1",
+				ClockRate: 90000,
+			},
 			&format.AV1{
 				PayloadTyp: 96,
 			},
@@ -54,6 +60,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 			&format.VP9{
 				PayloadTyp: 96,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:    "video/VP9",
+				ClockRate:   90000,
+				SDPFmtpLine: "profile-id=0",
+			},
 			&format.VP9{
 				PayloadTyp: 96,
 			},
@@ -63,6 +74,10 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 			&format.VP8{
 				PayloadTyp: 96,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "video/VP8",
+				ClockRate: 90000,
+			},
 			&format.VP8{
 				PayloadTyp: 96,
 			},
@@ -70,6 +85,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 		{
 			"h264",
 			test.FormatH264,
+			webrtc.RTPCodecCapability{
+				MimeType:    "video/H264",
+				ClockRate:   90000,
+				SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+			},
 			&format.H264{
 				PayloadTyp:        96,
 				PacketizationMode: 1,
@@ -81,6 +101,12 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				PayloadTyp:   112,
 				ChannelCount: 6,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:    "audio/multiopus",
+				ClockRate:   48000,
+				Channels:    6,
+				SDPFmtpLine: "channel_mapping=0,4,1,2,3,5;num_streams=4;coupled_streams=2",
+			},
 			&format.Opus{
 				PayloadTyp:   96,
 				ChannelCount: 6,
@@ -92,6 +118,12 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				PayloadTyp:   111,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:    "audio/opus",
+				ClockRate:   48000,
+				Channels:    2,
+				SDPFmtpLine: "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1",
+			},
 			&format.Opus{
 				PayloadTyp:   96,
 				ChannelCount: 2,
@@ -103,6 +135,12 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				PayloadTyp:   111,
 				ChannelCount: 1,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:    "audio/opus",
+				ClockRate:   48000,
+				Channels:    2,
+				SDPFmtpLine: "minptime=10;useinbandfec=1",
+			},
 			&format.Opus{
 				PayloadTyp:   96,
 				ChannelCount: 1,
@@ -111,6 +149,10 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 		{
 			"g722",
 			&format.G722{},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/G722",
+				ClockRate: 8000,
+			},
 			&format.G722{},
 		},
 		{
@@ -120,6 +162,10 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   8000,
 				ChannelCount: 1,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/PCMA",
+				ClockRate: 8000,
+			},
 			&format.G711{
 				PayloadTyp:   8,
 				SampleRate:   8000,
@@ -134,6 +180,10 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   8000,
 				ChannelCount: 1,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/PCMU",
+				ClockRate: 8000,
+			},
 			&format.G711{
 				MULaw:        true,
 				PayloadTyp:   0,
@@ -148,6 +198,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   8000,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/PCMA",
+				ClockRate: 8000,
+				Channels:  2,
+			},
 			&format.G711{
 				PayloadTyp:   119,
 				SampleRate:   8000,
@@ -162,6 +217,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   8000,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/PCMU",
+				ClockRate: 8000,
+				Channels:  2,
+			},
 			&format.G711{
 				MULaw:        true,
 				PayloadTyp:   118,
@@ -176,6 +236,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   16000,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/L16",
+				ClockRate: 16000,
+				Channels:  2,
+			},
 			&format.LPCM{
 				PayloadTyp:   96,
 				BitDepth:     16,
@@ -191,6 +256,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   16000,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/L16",
+				ClockRate: 16000,
+				Channels:  2,
+			},
 			&format.LPCM{
 				PayloadTyp:   96,
 				BitDepth:     16,
@@ -206,6 +276,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   8000,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/L16",
+				ClockRate: 8000,
+				Channels:  2,
+			},
 			&format.LPCM{
 				PayloadTyp:   96,
 				BitDepth:     16,
@@ -221,6 +296,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   16000,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/L16",
+				ClockRate: 16000,
+				Channels:  2,
+			},
 			&format.LPCM{
 				PayloadTyp:   96,
 				BitDepth:     16,
@@ -236,6 +316,11 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				SampleRate:   48000,
 				ChannelCount: 2,
 			},
+			webrtc.RTPCodecCapability{
+				MimeType:  "audio/L16",
+				ClockRate: 48000,
+				Channels:  2,
+			},
 			&format.LPCM{
 				PayloadTyp:   96,
 				BitDepth:     16,
@@ -316,6 +401,10 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 			inc, err := pc2.GatherIncomingTracks(context.Background())
 			require.NoError(t, err)
 
+			exp := ca.webrtcOut
+			exp.RTCPFeedback = inc[0].track.Codec().RTPCodecCapability.RTCPFeedback
+			require.Equal(t, exp, inc[0].track.Codec().RTPCodecCapability)
+
 			require.Equal(t, ca.out, inc[0].Format())
 		})
 	}