From a1dc9f45f5e2a03084274bea139ecaa2e805d659 Mon Sep 17 00:00:00 2001 From: Alessandro Ros Date: Wed, 19 Jun 2024 21:02:08 +0200 Subject: [PATCH] webrtc: support publishing H265 tracks (#3435) (#3492) IMPORTANT NOTE: this doesn't allow to read H265 tracks with WebRTC, just to publish them. The inability to read H265 tracks with WebRTC is not in any way related to the server but depends on browsers and on the fact that they are not legally entitled to embed a H265 decoder inside them. --- README.md | 4 +- internal/protocols/webrtc/incoming_track.go | 45 ++++++++++++++----- internal/protocols/webrtc/outgoing_track.go | 10 +++++ .../protocols/webrtc/peer_connection_test.go | 11 +++++ internal/protocols/webrtc/tracks_to_medias.go | 16 +------ 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 25ec0c46..3e09e429 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ Live streams can be published to the server with: |--------|--------|------------|------------| |[SRT clients](#srt-clients)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3| |[SRT cameras and servers](#srt-cameras-and-servers)||H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3| -|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)| -|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H264|Opus, G722, G711 (PCMA, PCMU)| +|[WebRTC clients](#webrtc-clients)|Browser-based, WHIP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)| +|[WebRTC servers](#webrtc-servers)|WHEP|AV1, VP9, VP8, H265, H264|Opus, G722, G711 (PCMA, PCMU)| |[RTSP clients](#rtsp-clients)|UDP, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec| |[RTSP cameras and servers](#rtsp-cameras-and-servers)|UDP, UDP-Multicast, TCP, RTSPS|AV1, VP9, VP8, H265, H264, MPEG-4 Video (H263, Xvid), MPEG-1/2 Video, M-JPEG and any RTP-compatible codec|Opus, MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), AC-3, G726, G722, G711 (PCMA, PCMU), LPCM and any RTP-compatible codec| |[RTMP clients](#rtmp-clients)|RTMP, RTMPS, Enhanced RTMP|AV1, VP9, H265, H264|MPEG-4 Audio (AAC), MPEG-1/2 Audio (MP3), G711 (PCMA, PCMU), LPCM| diff --git a/internal/protocols/webrtc/incoming_track.go b/internal/protocols/webrtc/incoming_track.go index 96944031..71016e96 100644 --- a/internal/protocols/webrtc/incoming_track.go +++ b/internal/protocols/webrtc/incoming_track.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "github.com/bluenviron/gortsplib/v4/pkg/description" "github.com/bluenviron/gortsplib/v4/pkg/format" "github.com/bluenviron/gortsplib/v4/pkg/liberrors" "github.com/bluenviron/gortsplib/v4/pkg/rtpreorderer" @@ -81,9 +82,8 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{ }, { RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypeH264, - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", + MimeType: webrtc.MimeTypeH265, + ClockRate: 90000, }, PayloadType: 103, }, @@ -91,10 +91,18 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{ RTPCodecCapability: webrtc.RTPCodecCapability{ MimeType: webrtc.MimeTypeH264, ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", }, PayloadType: 104, }, + { + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + }, + PayloadType: 105, + }, } var incomingAudioCodecs = []webrtc.RTPCodecParameters{ @@ -229,6 +237,7 @@ type IncomingTrack struct { track *webrtc.TrackRemote log logger.Writer + typ description.MediaType format format.Format reorderer *rtpreorderer.Reorderer pkts []*rtp.Packet @@ -246,41 +255,47 @@ func newIncomingTrack( reorderer: rtpreorderer.New(), } - isVideo := false - switch strings.ToLower(track.Codec().MimeType) { case strings.ToLower(webrtc.MimeTypeAV1): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.AV1{ PayloadTyp: uint8(track.PayloadType()), } case strings.ToLower(webrtc.MimeTypeVP9): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.VP9{ PayloadTyp: uint8(track.PayloadType()), } case strings.ToLower(webrtc.MimeTypeVP8): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.VP8{ PayloadTyp: uint8(track.PayloadType()), } + case strings.ToLower(webrtc.MimeTypeH265): + t.typ = description.MediaTypeVideo + t.format = &format.H265{ + PayloadTyp: uint8(track.PayloadType()), + } + case strings.ToLower(webrtc.MimeTypeH264): - isVideo = true + t.typ = description.MediaTypeVideo t.format = &format.H264{ PayloadTyp: uint8(track.PayloadType()), PacketizationMode: 1, } case strings.ToLower(mimeTypeMultiopus): + t.typ = description.MediaTypeAudio t.format = &format.Opus{ PayloadTyp: uint8(track.PayloadType()), ChannelCount: int(track.Codec().Channels), } case strings.ToLower(webrtc.MimeTypeOpus): + t.typ = description.MediaTypeAudio t.format = &format.Opus{ PayloadTyp: uint8(track.PayloadType()), ChannelCount: func() int { @@ -292,9 +307,12 @@ func newIncomingTrack( } case strings.ToLower(webrtc.MimeTypeG722): + t.typ = description.MediaTypeAudio t.format = &format.G722{} case strings.ToLower(webrtc.MimeTypePCMU): + t.typ = description.MediaTypeAudio + channels := track.Codec().Channels if channels == 0 { channels = 1 @@ -313,6 +331,8 @@ func newIncomingTrack( } case strings.ToLower(webrtc.MimeTypePCMA): + t.typ = description.MediaTypeAudio + channels := track.Codec().Channels if channels == 0 { channels = 1 @@ -331,6 +351,7 @@ func newIncomingTrack( } case strings.ToLower(mimeTypeL16): + t.typ = description.MediaTypeAudio t.format = &format.LPCM{ PayloadTyp: uint8(track.PayloadType()), BitDepth: 16, @@ -339,7 +360,7 @@ func newIncomingTrack( } default: - return nil, fmt.Errorf("unsupported codec: %+v", track.Codec()) + return nil, fmt.Errorf("unsupported codec: %+v", track.Codec().RTPCodecCapability) } // read incoming RTCP packets to make interceptors work @@ -354,7 +375,7 @@ func newIncomingTrack( }() // send period key frame requests - if isVideo { + if t.typ == description.MediaTypeVideo { go func() { keyframeTicker := time.NewTicker(keyFrameInterval) defer keyframeTicker.Stop() diff --git a/internal/protocols/webrtc/outgoing_track.go b/internal/protocols/webrtc/outgoing_track.go index 3f72c1cc..adc60e51 100644 --- a/internal/protocols/webrtc/outgoing_track.go +++ b/internal/protocols/webrtc/outgoing_track.go @@ -54,6 +54,15 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) { PayloadType: 96, }, nil + case *format.H265: + return webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH265, + ClockRate: 90000, + }, + PayloadType: 96, + }, nil + case *format.H264: return webrtc.RTPCodecParameters{ RTPCodecCapability: webrtc.RTPCodecCapability{ @@ -205,6 +214,7 @@ func (t *OutgoingTrack) isVideo() bool { case *format.AV1, *format.VP9, *format.VP8, + *format.H265, *format.H264: return true } diff --git a/internal/protocols/webrtc/peer_connection_test.go b/internal/protocols/webrtc/peer_connection_test.go index 1927786f..60454133 100644 --- a/internal/protocols/webrtc/peer_connection_test.go +++ b/internal/protocols/webrtc/peer_connection_test.go @@ -83,6 +83,17 @@ func TestPeerConnectionPublishRead(t *testing.T) { PayloadTyp: 96, }, }, + { + "h265", + test.FormatH265, + webrtc.RTPCodecCapability{ + MimeType: "video/H265", + ClockRate: 90000, + }, + &format.H265{ + PayloadTyp: 96, + }, + }, { "h264", test.FormatH264, diff --git a/internal/protocols/webrtc/tracks_to_medias.go b/internal/protocols/webrtc/tracks_to_medias.go index 809eeec4..81ea0850 100644 --- a/internal/protocols/webrtc/tracks_to_medias.go +++ b/internal/protocols/webrtc/tracks_to_medias.go @@ -10,21 +10,9 @@ func TracksToMedias(tracks []*IncomingTrack) []*description.Media { ret := make([]*description.Media, len(tracks)) for i, track := range tracks { - forma := track.Format() - - var mediaType description.MediaType - - switch forma.(type) { - case *format.AV1, *format.VP9, *format.VP8, *format.H264: - mediaType = description.MediaTypeVideo - - default: - mediaType = description.MediaTypeAudio - } - ret[i] = &description.Media{ - Type: mediaType, - Formats: []format.Format{forma}, + Type: track.typ, + Formats: []format.Format{track.format}, } }