webrtc: support publishing H265 tracks (#3435) (#3492)

IMPORTANT NOTE: this doesn't allow to read H265 tracks with WebRTC,
just to publish them. The inability to read H265 tracks with WebRTC is
not in any way related to the server but depends on browsers and on the
fact that they are not legally entitled to embed a H265 decoder inside
them.

README.md

@@ -22,8 +22,8 @@ Live streams can be published to the server with:
 |--------|--------|------------|------------|
 |[SRT clients](#srt-clients)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3|
 |[SRT cameras and servers](#srt-cameras-and-servers)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3|
-|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)|
+|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)|
-|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)|
+|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)|
 |[RTSP clients](#rtsp-clients)|UDP, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
 |[RTSP cameras and servers](#rtsp-cameras-and-servers)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec|
 |[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM|

internal/protocols/webrtc/incoming_track.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bluenviron/gortsplib/v4/pkg/description"
 	"github.com/bluenviron/gortsplib/v4/pkg/format"
 	"github.com/bluenviron/gortsplib/v4/pkg/liberrors"
 	"github.com/bluenviron/gortsplib/v4/pkg/rtpreorderer"
@@ -81,9 +82,8 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{
 	},
 	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
-			MimeType:    webrtc.MimeTypeH264,
+			MimeType:  webrtc.MimeTypeH265,
 			ClockRate: 90000,
-			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
 		},
 		PayloadType: 103,
 	},
@@ -91,10 +91,18 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:    webrtc.MimeTypeH264,
 			ClockRate:   90000,
-			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
 		},
 		PayloadType: 104,
 	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    webrtc.MimeTypeH264,
+			ClockRate:   90000,
+			SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+		},
+		PayloadType: 105,
+	},
 }
 
 var incomingAudioCodecs = []webrtc.RTPCodecParameters{
@@ -229,6 +237,7 @@ type IncomingTrack struct {
 	track *webrtc.TrackRemote
 	log   logger.Writer
 
+	typ       description.MediaType
 	format    format.Format
 	reorderer *rtpreorderer.Reorderer
 	pkts      []*rtp.Packet
@@ -246,41 +255,47 @@ func newIncomingTrack(
 		reorderer: rtpreorderer.New(),
 	}
 
-	isVideo := false
-
 	switch strings.ToLower(track.Codec().MimeType) {
 	case strings.ToLower(webrtc.MimeTypeAV1):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.AV1{
 			PayloadTyp: uint8(track.PayloadType()),
 		}
 
 	case strings.ToLower(webrtc.MimeTypeVP9):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.VP9{
 			PayloadTyp: uint8(track.PayloadType()),
 		}
 
 	case strings.ToLower(webrtc.MimeTypeVP8):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.VP8{
 			PayloadTyp: uint8(track.PayloadType()),
 		}
 
+	case strings.ToLower(webrtc.MimeTypeH265):
+		t.typ = description.MediaTypeVideo
+		t.format = &format.H265{
+			PayloadTyp: uint8(track.PayloadType()),
+		}
+
 	case strings.ToLower(webrtc.MimeTypeH264):
-		isVideo = true
+		t.typ = description.MediaTypeVideo
 		t.format = &format.H264{
 			PayloadTyp:        uint8(track.PayloadType()),
 			PacketizationMode: 1,
 		}
 
 	case strings.ToLower(mimeTypeMultiopus):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.Opus{
 			PayloadTyp:   uint8(track.PayloadType()),
 			ChannelCount: int(track.Codec().Channels),
 		}
 
 	case strings.ToLower(webrtc.MimeTypeOpus):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.Opus{
 			PayloadTyp: uint8(track.PayloadType()),
 			ChannelCount: func() int {
@@ -292,9 +307,12 @@ func newIncomingTrack(
 		}
 
 	case strings.ToLower(webrtc.MimeTypeG722):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.G722{}
 
 	case strings.ToLower(webrtc.MimeTypePCMU):
+		t.typ = description.MediaTypeAudio
+
 		channels := track.Codec().Channels
 		if channels == 0 {
 			channels = 1
@@ -313,6 +331,8 @@ func newIncomingTrack(
 		}
 
 	case strings.ToLower(webrtc.MimeTypePCMA):
+		t.typ = description.MediaTypeAudio
+
 		channels := track.Codec().Channels
 		if channels == 0 {
 			channels = 1
@@ -331,6 +351,7 @@ func newIncomingTrack(
 		}
 
 	case strings.ToLower(mimeTypeL16):
+		t.typ = description.MediaTypeAudio
 		t.format = &format.LPCM{
 			PayloadTyp:   uint8(track.PayloadType()),
 			BitDepth:     16,
@@ -339,7 +360,7 @@ func newIncomingTrack(
 		}
 
 	default:
-		return nil, fmt.Errorf("unsupported codec: %+v", track.Codec())
+		return nil, fmt.Errorf("unsupported codec: %+v", track.Codec().RTPCodecCapability)
 	}
 
 	// read incoming RTCP packets to make interceptors work
@@ -354,7 +375,7 @@ func newIncomingTrack(
 	}()
 
 	// send period key frame requests
-	if isVideo {
+	if t.typ == description.MediaTypeVideo {
 		go func() {
 			keyframeTicker := time.NewTicker(keyFrameInterval)
 			defer keyframeTicker.Stop()

internal/protocols/webrtc/outgoing_track.go

@@ -54,6 +54,15 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) {
 			PayloadType: 96,
 		}, nil
 
+	case *format.H265:
+		return webrtc.RTPCodecParameters{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:  webrtc.MimeTypeH265,
+				ClockRate: 90000,
+			},
+			PayloadType: 96,
+		}, nil
+
 	case *format.H264:
 		return webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
@@ -205,6 +214,7 @@ func (t *OutgoingTrack) isVideo() bool {
 	case *format.AV1,
 		*format.VP9,
 		*format.VP8,
+		*format.H265,
 		*format.H264:
 		return true
 	}

internal/protocols/webrtc/peer_connection_test.go

@@ -83,6 +83,17 @@ func TestPeerConnectionPublishRead(t *testing.T) {
 				PayloadTyp: 96,
 			},
 		},
+		{
+			"h265",
+			test.FormatH265,
+			webrtc.RTPCodecCapability{
+				MimeType:  "video/H265",
+				ClockRate: 90000,
+			},
+			&format.H265{
+				PayloadTyp: 96,
+			},
+		},
 		{
 			"h264",
 			test.FormatH264,

internal/protocols/webrtc/tracks_to_medias.go

@@ -10,21 +10,9 @@ func TracksToMedias(tracks []*IncomingTrack) []*description.Media {
 	ret := make([]*description.Media, len(tracks))
 
 	for i, track := range tracks {
-		forma := track.Format()
-
-		var mediaType description.MediaType
-
-		switch forma.(type) {
-		case *format.AV1, *format.VP9, *format.VP8, *format.H264:
-			mediaType = description.MediaTypeVideo
-
-		default:
-			mediaType = description.MediaTypeAudio
-		}
-
 		ret[i] = &description.Media{
-			Type:    mediaType,
+			Type:    track.typ,
-			Formats: []format.Format{forma},
+			Formats: []format.Format{track.format},
 		}
 	}