From acd788d6323de2007013505421ef8276efa22f19 Mon Sep 17 00:00:00 2001 From: aler9 <46489434+aler9@users.noreply.github.com> Date: Thu, 2 Jun 2022 19:03:08 +0200 Subject: [PATCH] update gortsplib --- go.mod | 2 +- go.sum | 4 +- internal/core/rtmp_conn.go | 34 ++++---- internal/hls/muxer_test.go | 81 ++++++++++--------- internal/hls/muxer_variant_fmp4_segmenter.go | 56 +++++-------- internal/hls/muxer_variant_mpegts_segment.go | 12 +-- .../hls/muxer_variant_mpegts_segmenter.go | 25 ++---- 7 files changed, 96 insertions(+), 118 deletions(-) diff --git a/go.mod b/go.mod index 153ee856..91ee1d53 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 github.com/abema/go-mp4 v0.7.2 - github.com/aler9/gortsplib v0.0.0-20220602092441-d19c37025dc4 + github.com/aler9/gortsplib v0.0.0-20220602183811-46253a74b083 github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757 github.com/fsnotify/fsnotify v1.4.9 github.com/gin-gonic/gin v1.7.2 diff --git a/go.sum b/go.sum index 3ed4ec7e..f681357b 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aler9/gortsplib v0.0.0-20220602092441-d19c37025dc4 h1:0xhKBSxHasMGLUgdE/kTw5QmywTcil4oQwh0+MmK+6o= -github.com/aler9/gortsplib v0.0.0-20220602092441-d19c37025dc4/go.mod h1:i1e4CEs42IrbidMUNTSNOKmeGPCOHVX9P3BvPxzyMtI= +github.com/aler9/gortsplib v0.0.0-20220602183811-46253a74b083 h1:VmY5jw7jBl4PTX7J9UIrslRRco1ELt9oDteq9CmecqU= +github.com/aler9/gortsplib v0.0.0-20220602183811-46253a74b083/go.mod h1:i1e4CEs42IrbidMUNTSNOKmeGPCOHVX9P3BvPxzyMtI= github.com/aler9/rtmp v0.0.0-20210403095203-3be4a5535927 h1:95mXJ5fUCYpBRdSOnLAQAdJHHKxxxJrVCiaqDi965YQ= github.com/aler9/rtmp v0.0.0-20210403095203-3be4a5535927/go.mod h1:vzuE21rowz+lT1NGsWbreIvYulgBpCGnQyeTyFblUHc= github.com/aler9/writerseeker v0.0.0-20220601075008-6f0e685b9c82 h1:9WgSzBLo3a9ToSVV7sRTBYZ1GGOZUpq4+5H3SN0UZq4= diff --git a/internal/core/rtmp_conn.go b/internal/core/rtmp_conn.go index 90441d0a..c877b69c 100644 --- a/internal/core/rtmp_conn.go +++ b/internal/core/rtmp_conn.go @@ -331,8 +331,7 @@ func (c *rtmpConn) runRead(ctx context.Context) error { var videoInitialPTS *time.Duration videoFirstIDRFound := false var videoFirstIDRPTS time.Duration - videoDTSExtractor := h264.NewDTSExtractor() - var videoSPS *h264.SPS + var videoDTSExtractor *h264.DTSExtractor for { item, ok := c.ringBuffer.Pull() @@ -355,17 +354,28 @@ func (c *rtmpConn) runRead(ctx context.Context) error { } pts := data.h264PTS - *videoInitialPTS + idrPresent := h264.IDRPresent(data.h264NALUs) + // wait until we receive an IDR if !videoFirstIDRFound { - if !h264.IDRPresent(data.h264NALUs) { + if !idrPresent { continue } videoFirstIDRFound = true videoFirstIDRPTS = pts + videoDTSExtractor = h264.NewDTSExtractor() } - if h264.IDRPresent(data.h264NALUs) { + pts -= videoFirstIDRPTS + + dts, err := videoDTSExtractor.Extract(data.h264NALUs, pts) + if err != nil { + return err + } + + // insert a H264DecoderConfig before every IDR + if idrPresent { sps := videoTrack.SPS() pps := videoTrack.PPS() @@ -389,13 +399,6 @@ func (c *rtmpConn) runRead(ctx context.Context) error { if err != nil { return err } - - var psps h264.SPS - err := psps.Unmarshal(sps) - if err != nil { - return err - } - videoSPS = &psps } avcc, err := h264.AVCCEncode(data.h264NALUs) @@ -403,13 +406,6 @@ func (c *rtmpConn) runRead(ctx context.Context) error { return err } - pts -= videoFirstIDRPTS - dts, err := videoDTSExtractor.Extract( - data.h264NALUs, h264.IDRPresent(data.h264NALUs), pts, videoSPS) - if err != nil { - return err - } - c.conn.SetWriteDeadline(time.Now().Add(time.Duration(c.writeTimeout))) err = c.conn.WritePacket(av.Packet{ Type: av.H264, @@ -568,7 +564,7 @@ func (c *rtmpConn) runPublish(ctx context.Context) error { rres.stream.writeData(&data{ trackID: videoTrackID, rtp: pkt, - ptsEqualsDTS: h264.IDRPresent(nalus), + ptsEqualsDTS: false, h264NALUs: nalus, h264PTS: pts, }) diff --git a/internal/hls/muxer_test.go b/internal/hls/muxer_test.go index 772c3626..59e3e70a 100644 --- a/internal/hls/muxer_test.go +++ b/internal/hls/muxer_test.go @@ -13,8 +13,16 @@ import ( "github.com/stretchr/testify/require" ) +// baseline profile without POC +var testSPS = []byte{ + 0x67, 0x42, 0xc0, 0x28, 0xd9, 0x00, 0x78, 0x02, + 0x27, 0xe5, 0x84, 0x00, 0x00, 0x03, 0x00, 0x04, + 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, 0x60, 0xc9, + 0x20, +} + func TestMuxerVideoAudio(t *testing.T) { - videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil) + videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil) require.NoError(t, err) audioTrack, err := gortsplib.NewTrackAAC(97, 2, 44100, 2, nil, 13, 3, 3) @@ -33,9 +41,9 @@ func TestMuxerVideoAudio(t *testing.T) { // group with IDR err = m.WriteH264(2*time.Second, [][]byte{ - {7}, // SPS - {8}, // PPS - {5}, // IDR + testSPS, // SPS + {8}, // PPS + {5}, // IDR }) require.NoError(t, err) @@ -47,8 +55,7 @@ func TestMuxerVideoAudio(t *testing.T) { // group without IDR err = m.WriteH264(4*time.Second, [][]byte{ - {6}, - {7}, + {1}, // non-IDR }) require.NoError(t, err) @@ -67,7 +74,7 @@ func TestMuxerVideoAudio(t *testing.T) { "#EXT-X-VERSION:3\n"+ "#EXT-X-INDEPENDENT-SEGMENTS\n"+ "\n"+ - "#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.010203,mp4a.40.2\"\n"+ + "#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.42c028,mp4a.40.2\"\n"+ "stream.m3u8\n", string(byts)) byts, err = ioutil.ReadAll(m.File("stream.m3u8", "", "", "").Body) @@ -126,8 +133,8 @@ func TestMuxerVideoAudio(t *testing.T) { require.NoError(t, err) require.Equal(t, &astits.Packet{ AdaptationField: &astits.PacketAdaptationField{ - Length: 148, - StuffingLength: 141, + Length: 119, + StuffingLength: 112, HasPCR: true, PCR: &astits.ClockReference{}, RandomAccessIndicator: true, @@ -138,14 +145,16 @@ func TestMuxerVideoAudio(t *testing.T) { PayloadUnitStartIndicator: true, PID: 256, }, - Payload: append([]byte{ - 0x00, 0x00, 0x01, 0xe0, 0x00, 0x00, 0x80, 0x80, - 0x05, 0x21, 0x00, 0x03, 0x5f, 0x91, - 0, 0, 0, 1, 9, 240, // AUD - 0, 0, 0, 1, 7, // SPS - 0, 0, 0, 1, 8, // PPS - 0, 0, 0, 1, 5, // IDR - }, bytes.Repeat([]byte{0xff}, 0)...), + Payload: []byte{ + 0x00, 0x00, 0x01, 0xe0, 0x00, 0x00, 0x80, 0xc0, + 0x0a, 0x31, 0x00, 0x05, 0x32, 0x81, 0x11, 0x00, + 0x03, 0x19, 0x41, 0x00, 0x00, 0x00, 0x01, 0x09, + 0xf0, 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, + 0x28, 0xd9, 0x00, 0x78, 0x02, 0x27, 0xe5, 0x84, + 0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, + 0x00, 0xf0, 0x3c, 0x60, 0xc9, 0x20, 0x00, 0x00, + 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x01, 0x05, + }, }, pkt) // PES (AAC) @@ -163,18 +172,18 @@ func TestMuxerVideoAudio(t *testing.T) { PayloadUnitStartIndicator: true, PID: 257, }, - Payload: append([]byte{ + Payload: []byte{ 0x00, 0x00, 0x01, 0xc0, 0x00, 0x1e, 0x80, 0x80, - 0x05, 0x21, 0x00, 0x09, 0x1e, 0xb1, 0xff, 0xf1, + 0x05, 0x21, 0x00, 0x09, 0xf1, 0xa1, 0xff, 0xf1, 0x50, 0x80, 0x01, 0x7f, 0xfc, 0x01, 0x02, 0x03, 0x04, 0xff, 0xf1, 0x50, 0x80, 0x01, 0x7f, 0xfc, 0x05, 0x06, 0x07, 0x08, - }, bytes.Repeat([]byte{0xff}, 0)...), + }, }, pkt) } func TestMuxerVideoOnly(t *testing.T) { - videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil) + videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil) require.NoError(t, err) m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil) @@ -183,10 +192,9 @@ func TestMuxerVideoOnly(t *testing.T) { // group with IDR err = m.WriteH264(2*time.Second, [][]byte{ - {5}, // IDR - {9}, // AUD - {8}, // PPS - {7}, // SPS + testSPS, // SPS + {8}, // PPS + {5}, // IDR }) require.NoError(t, err) @@ -203,7 +211,7 @@ func TestMuxerVideoOnly(t *testing.T) { "#EXT-X-VERSION:3\n"+ "#EXT-X-INDEPENDENT-SEGMENTS\n"+ "\n"+ - "#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.010203\"\n"+ + "#EXT-X-STREAM-INF:BANDWIDTH=200000,CODECS=\"avc1.42c028\"\n"+ "stream.m3u8\n", string(byts)) byts, err = ioutil.ReadAll(m.File("stream.m3u8", "", "", "").Body) @@ -346,7 +354,7 @@ func TestMuxerAudioOnly(t *testing.T) { } func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) { - videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil) + videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil) require.NoError(t, err) m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil) @@ -354,10 +362,9 @@ func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) { // group with IDR err = m.WriteH264(2*time.Second, [][]byte{ - {5}, // IDR - {9}, // AUD - {8}, // PPS - {7}, // SPS + testSPS, // SPS + {8}, // PPS + {5}, // IDR }) require.NoError(t, err) @@ -368,7 +375,7 @@ func TestMuxerCloseBeforeFirstSegmentReader(t *testing.T) { } func TestMuxerMaxSegmentSize(t *testing.T) { - videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil) + videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil) require.NoError(t, err) m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 0, videoTrack, nil) @@ -376,13 +383,14 @@ func TestMuxerMaxSegmentSize(t *testing.T) { defer m.Close() err = m.WriteH264(2*time.Second, [][]byte{ - {5}, + testSPS, + {5}, // IDR }) require.EqualError(t, err, "reached maximum segment size") } func TestMuxerDoubleRead(t *testing.T) { - videoTrack, err := gortsplib.NewTrackH264(96, []byte{0x07, 0x01, 0x02, 0x03}, []byte{0x08}, nil) + videoTrack, err := gortsplib.NewTrackH264(96, testSPS, []byte{0x08}, nil) require.NoError(t, err) m, err := NewMuxer(MuxerVariantMPEGTS, 3, 1*time.Second, 0, 50*1024*1024, videoTrack, nil) @@ -390,13 +398,14 @@ func TestMuxerDoubleRead(t *testing.T) { defer m.Close() err = m.WriteH264(0, [][]byte{ - {5}, + testSPS, + {5}, // IDR {1}, }) require.NoError(t, err) err = m.WriteH264(2*time.Second, [][]byte{ - {5}, + {5}, // IDR {2}, }) require.NoError(t, err) diff --git a/internal/hls/muxer_variant_fmp4_segmenter.go b/internal/hls/muxer_variant_fmp4_segmenter.go index 6df487c2..aa39e7e0 100644 --- a/internal/hls/muxer_variant_fmp4_segmenter.go +++ b/internal/hls/muxer_variant_fmp4_segmenter.go @@ -54,11 +54,11 @@ type muxerVariantFMP4Segmenter struct { onSegmentFinalized func(*muxerVariantFMP4Segment) onPartFinalized func(*muxerVariantFMP4Part) - currentSegment *muxerVariantFMP4Segment - startPTS time.Duration - videoSPS []byte - videoSPSP *h264.SPS + videoFirstIDRReceived bool videoDTSExtractor *h264.DTSExtractor + videoSPS []byte + startPTS time.Duration + currentSegment *muxerVariantFMP4Segment nextSegmentID uint64 nextPartID uint64 nextVideoSample *fmp4VideoSample @@ -88,7 +88,6 @@ func newMuxerVariantFMP4Segmenter( audioTrack: audioTrack, onSegmentFinalized: onSegmentFinalized, onPartFinalized: onPartFinalized, - videoDTSExtractor: h264.NewDTSExtractor(), nextSegmentID: uint64(segmentCount), sampleDurations: make(map[time.Duration]struct{}), } @@ -139,34 +138,24 @@ func (m *muxerVariantFMP4Segmenter) writeH264(pts time.Duration, nalus [][]byte) } func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) error { - // parse SPS - spsChanged := false - if sample.idrPresent { - videoSPS := m.videoTrack.SPS() - if m.videoSPS == nil || !bytes.Equal(m.videoSPS, videoSPS) { - spsChanged = true - - var videoSPSP h264.SPS - err := videoSPSP.Unmarshal(videoSPS) - if err != nil { - return err - } - - m.videoSPS = videoSPS - m.videoSPSP = &videoSPSP + if !m.videoFirstIDRReceived { + // skip sample silently until we find one with an IDR + if !sample.idrPresent { + return nil } + + m.videoFirstIDRReceived = true + m.videoDTSExtractor = h264.NewDTSExtractor() + m.videoSPS = append([]byte(nil), m.videoTrack.SPS()...) } // fill DTS - if m.videoSPSP != nil { - var err error - sample.dts, err = m.videoDTSExtractor.Extract( - sample.nalus, sample.idrPresent, sample.pts, m.videoSPSP) - if err != nil { - return err - } - sample.nalus = nil + var err error + sample.dts, err = m.videoDTSExtractor.Extract(sample.nalus, sample.pts) + if err != nil { + return err } + sample.nalus = nil sample.pts -= m.startPTS sample.dts -= m.startPTS @@ -183,11 +172,6 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) erro now := time.Now() if m.currentSegment == nil { - // skip groups silently until we find one with a IDR - if !sample.idrPresent { - return nil - } - // create first segment m.currentSegment = newMuxerVariantFMP4Segment( m.lowLatency, @@ -210,13 +194,16 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) erro m.adjustPartDuration(sample.duration()) - err := m.currentSegment.writeH264(sample, m.adjustedPartDuration) + err = m.currentSegment.writeH264(sample, m.adjustedPartDuration) if err != nil { return err } // switch segment if sample.next.idrPresent { + sps := m.videoTrack.SPS() + spsChanged := !bytes.Equal(m.videoSPS, sps) + if (sample.next.pts-m.currentSegment.startDTS) >= m.segmentDuration || spsChanged { err := m.currentSegment.finalize(sample.next, nil) @@ -241,6 +228,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(sample *fmp4VideoSample) erro // if SPS changed, reset adjusted part duration if spsChanged { + m.videoSPS = append([]byte(nil), sps...) m.firstSegmentFinalized = false m.sampleDurations = make(map[time.Duration]struct{}) } diff --git a/internal/hls/muxer_variant_mpegts_segment.go b/internal/hls/muxer_variant_mpegts_segment.go index ec1e0f92..1068bcde 100644 --- a/internal/hls/muxer_variant_mpegts_segment.go +++ b/internal/hls/muxer_variant_mpegts_segment.go @@ -110,15 +110,13 @@ func (t *muxerVariantMPEGTSSegment) writeH264( MarkerBits: 2, } - pts += mpegtsPTSDTSOffset - - if dts == pts { + if dts == (pts + mpegtsPTSDTSOffset) { oh.PTSDTSIndicator = astits.PTSDTSIndicatorOnlyPTS - oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPCROffset).Seconds() * 90000)} + oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPTSDTSOffset + mpegtsPCROffset).Seconds() * 90000)} } else { oh.PTSDTSIndicator = astits.PTSDTSIndicatorBothPresent oh.DTS = &astits.ClockReference{Base: int64((dts + mpegtsPCROffset).Seconds() * 90000)} - oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPCROffset).Seconds() * 90000)} + oh.PTS = &astits.ClockReference{Base: int64((pts + mpegtsPTSDTSOffset + mpegtsPCROffset).Seconds() * 90000)} } _, err = t.writeData(&astits.MuxerData{ @@ -182,8 +180,6 @@ func (t *muxerVariantMPEGTSSegment) writeAAC( t.pcrSendCounter-- } - pts += mpegtsPTSDTSOffset - _, err = t.writeData(&astits.MuxerData{ PID: 257, AdaptationField: af, @@ -192,7 +188,7 @@ func (t *muxerVariantMPEGTSSegment) writeAAC( OptionalHeader: &astits.PESOptionalHeader{ MarkerBits: 2, PTSDTSIndicator: astits.PTSDTSIndicatorOnlyPTS, - PTS: &astits.ClockReference{Base: int64((pts + mpegtsPCROffset).Seconds() * 90000)}, + PTS: &astits.ClockReference{Base: int64((pts + mpegtsPTSDTSOffset + mpegtsPCROffset).Seconds() * 90000)}, }, PacketLength: uint16(len(enc) + 8), StreamID: 192, // audio diff --git a/internal/hls/muxer_variant_mpegts_segmenter.go b/internal/hls/muxer_variant_mpegts_segmenter.go index 8536e763..511b859e 100644 --- a/internal/hls/muxer_variant_mpegts_segmenter.go +++ b/internal/hls/muxer_variant_mpegts_segmenter.go @@ -28,7 +28,6 @@ type muxerVariantMPEGTSSegmenter struct { writer *astits.Muxer currentSegment *muxerVariantMPEGTSSegment - videoSPS *h264.SPS videoDTSExtractor *h264.DTSExtractor startPCR time.Time startPTS time.Duration @@ -42,12 +41,11 @@ func newMuxerVariantMPEGTSSegmenter( onSegmentReady func(*muxerVariantMPEGTSSegment), ) *muxerVariantMPEGTSSegmenter { m := &muxerVariantMPEGTSSegmenter{ - segmentDuration: segmentDuration, - segmentMaxSize: segmentMaxSize, - videoTrack: videoTrack, - audioTrack: audioTrack, - onSegmentReady: onSegmentReady, - videoDTSExtractor: h264.NewDTSExtractor(), + segmentDuration: segmentDuration, + segmentMaxSize: segmentMaxSize, + videoTrack: videoTrack, + audioTrack: audioTrack, + onSegmentReady: onSegmentReady, } m.writer = astits.NewMuxer( @@ -94,6 +92,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(pts time.Duration, nalus [][]byt m.videoTrack, m.audioTrack, m.writer.WriteData) m.startPCR = now m.startPTS = pts + m.videoDTSExtractor = h264.NewDTSExtractor() pts = 0 } else { pts -= m.startPTS @@ -109,17 +108,7 @@ func (m *muxerVariantMPEGTSSegmenter) writeH264(pts time.Duration, nalus [][]byt } } - if idrPresent { - sps := m.videoTrack.SPS() - var psps h264.SPS - err := psps.Unmarshal(sps) - if err != nil { - return err - } - m.videoSPS = &psps - } - - dts, err := m.videoDTSExtractor.Extract(nalus, idrPresent, pts, m.videoSPS) + dts, err := m.videoDTSExtractor.Extract(nalus, pts) if err != nil { return err }