From e5ab731d1475925a4fec7607f6446fcb085768fe Mon Sep 17 00:00:00 2001 From: Alessandro Ros Date: Sun, 23 Oct 2022 14:04:33 +0200 Subject: [PATCH] Improve HLS client (#1179) * hls source: support fMP4s video streams * hls source: start reading live streams from (end of playlist - starting point) * hls client: wait processing of current fMP4 segment before downloading another one * hls client: support fmp4 trun boxes with default sample duration, flags and size * hls client: merge fmp4 init file reader and writer * hls client: merge fmp4 part reader and writer * hls client: improve precision of go <-> mp4 time conversion * hls client: fix esds generation in go-mp4 * hls client: support audio in separate playlist * hls client: support an arbitrary number of tracks in fmp4 init files * hls client: support EXT-X-BYTERANGE * hls client: support fmp4 segments with multiple parts at once * hls client: support an arbitrary number of mpeg-ts tracks * hls client: synchronize tracks around a primary track * update go-mp4 * hls: synchronize track reproduction around a leading one * hls client: reset stream if playback is too late * hls client: add limit on DTS-RTC difference * hls client: support again streams that don't provide codecs in master playlist --- go.mod | 2 +- go.sum | 4 +- internal/core/hls_source_test.go | 1 + internal/hls/client.go | 48 +- internal/hls/client_downloader.go | 278 ------- internal/hls/client_downloader_primary.go | 325 +++++++++ internal/hls/client_downloader_stream.go | 258 +++++++ internal/hls/client_processor_fmp4.go | 217 ++++++ internal/hls/client_processor_fmp4_track.go | 70 ++ internal/hls/client_processor_mpegts.go | 294 +++----- internal/hls/client_processor_mpegts_track.go | 73 +- internal/hls/client_routine_pool.go | 16 +- internal/hls/client_test.go | 4 +- internal/hls/client_timesync_fmp4.go | 59 ++ internal/hls/client_timesync_mpegts.go | 47 ++ internal/hls/fmp4/audiosample.go | 17 - internal/hls/fmp4/init.go | 261 +++++++ internal/hls/fmp4/init_read.go | 104 --- internal/hls/fmp4/init_test.go | 689 ++++++++++++++++++ internal/hls/fmp4/init_track.go | 382 ++++++++++ internal/hls/fmp4/init_write.go | 618 ---------------- internal/hls/fmp4/init_write_test.go | 318 -------- internal/hls/fmp4/part.go | 259 +++++++ internal/hls/fmp4/part_read.go | 96 --- internal/hls/fmp4/part_test.go | 249 +++++++ internal/hls/fmp4/part_track.go | 106 +++ internal/hls/fmp4/part_write.go | 300 -------- internal/hls/fmp4/part_write_test.go | 143 ---- internal/hls/fmp4/videosample.go | 25 - internal/hls/m3u8/m3u8.go | 106 +++ .../decoder.go => mpegts/timedecoder.go} | 22 +- .../timedecoder_test.go} | 14 +- internal/hls/mpegts/tracks.go | 101 +++ internal/hls/mpegts/writer.go | 2 +- internal/hls/muxer_variant_fmp4.go | 22 +- internal/hls/muxer_variant_fmp4_part.go | 75 +- internal/hls/muxer_variant_fmp4_segment.go | 18 +- internal/hls/muxer_variant_fmp4_segmenter.go | 108 ++- 38 files changed, 3479 insertions(+), 2252 deletions(-) delete mode 100644 internal/hls/client_downloader.go create mode 100644 internal/hls/client_downloader_primary.go create mode 100644 internal/hls/client_downloader_stream.go create mode 100644 internal/hls/client_processor_fmp4.go create mode 100644 internal/hls/client_processor_fmp4_track.go create mode 100644 internal/hls/client_timesync_fmp4.go create mode 100644 internal/hls/client_timesync_mpegts.go delete mode 100644 internal/hls/fmp4/audiosample.go create mode 100644 internal/hls/fmp4/init.go delete mode 100644 internal/hls/fmp4/init_read.go create mode 100644 internal/hls/fmp4/init_test.go create mode 100644 internal/hls/fmp4/init_track.go delete mode 100644 internal/hls/fmp4/init_write.go delete mode 100644 internal/hls/fmp4/init_write_test.go create mode 100644 internal/hls/fmp4/part.go delete mode 100644 internal/hls/fmp4/part_read.go create mode 100644 internal/hls/fmp4/part_test.go create mode 100644 internal/hls/fmp4/part_track.go delete mode 100644 internal/hls/fmp4/part_write.go delete mode 100644 internal/hls/fmp4/part_write_test.go delete mode 100644 internal/hls/fmp4/videosample.go create mode 100644 internal/hls/m3u8/m3u8.go rename internal/hls/{mpegtstimedec/decoder.go => mpegts/timedecoder.go} (69%) rename internal/hls/{mpegtstimedec/decoder_test.go => mpegts/timedecoder_test.go} (85%) create mode 100644 internal/hls/mpegts/tracks.go diff --git a/go.mod b/go.mod index 504fe6f3..9c1b39bd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 - github.com/abema/go-mp4 v0.7.2 + github.com/abema/go-mp4 v0.8.0 github.com/aler9/gortsplib v0.0.0-20221009091420-74f941be7166 github.com/asticode/go-astits v1.10.1-0.20220319093903-4abe66a9b757 github.com/fsnotify/fsnotify v1.4.9 diff --git a/go.sum b/go.sum index 06839acb..c92fc7f8 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 h1:tM5+dn2C9xZw1RzgI6WTQW1rGqdUimKB3RFbyu4h6Hc= code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB6oBMz/c9fRY6vZrwr5xKRWOH5NPDjQZlPk0Gbs= -github.com/abema/go-mp4 v0.7.2 h1:ugTC8gfEmjyaDKpXs3vi2QzgJbDu9B8m6UMMIpbYbGg= -github.com/abema/go-mp4 v0.7.2/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws= +github.com/abema/go-mp4 v0.8.0 h1:JHYkOvTfBpTnqJHiFFOXe8d6wiFy5MtDnA10fgccNqY= +github.com/abema/go-mp4 v0.8.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= diff --git a/internal/core/hls_source_test.go b/internal/core/hls_source_test.go index 27a2efa6..04217da3 100644 --- a/internal/core/hls_source_test.go +++ b/internal/core/hls_source_test.go @@ -52,6 +52,7 @@ func (ts *testHLSServer) onPlaylist(ctx *gin.Context) { #EXT-X-MEDIA-SEQUENCE:0 #EXTINF:2, segment.ts +#EXT-X-ENDLIST ` ctx.Writer.Header().Set("Content-Type", `application/x-mpegURL`) diff --git a/internal/hls/client.go b/internal/hls/client.go index 8f09645f..a31c6084 100644 --- a/internal/hls/client.go +++ b/internal/hls/client.go @@ -12,12 +12,14 @@ import ( ) const ( - clientMinDownloadPause = 5 * time.Second - clientQueueSize = 100 - clientMinSegmentsBeforeDownloading = 2 + clientMPEGTSEntryQueueSize = 100 + clientFMP4MaxPartTracksPerSegment = 50 + clientLiveStartingInvPosition = 3 + clientLiveMaxInvPosition = 5 + clientMaxDTSRTCDiff = 10 * time.Second ) -func clientURLAbsolute(base *url.URL, relative string) (*url.URL, error) { +func clientAbsoluteURL(base *url.URL, relative string) (*url.URL, error) { u, err := url.Parse(relative) if err != nil { return nil, err @@ -38,9 +40,9 @@ type Client struct { onAudioData func(time.Duration, []byte) logger ClientLogger - ctx context.Context - ctxCancel func() - primaryPlaylistURL *url.URL + ctx context.Context + ctxCancel func() + playlistURL *url.URL // out outErr chan error @@ -48,14 +50,14 @@ type Client struct { // NewClient allocates a Client. func NewClient( - primaryPlaylistURLStr string, + playlistURLStr string, fingerprint string, onTracks func(*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio) error, onVideoData func(time.Duration, [][]byte), onAudioData func(time.Duration, []byte), logger ClientLogger, ) (*Client, error) { - primaryPlaylistURL, err := url.Parse(primaryPlaylistURLStr) + playlistURL, err := url.Parse(playlistURLStr) if err != nil { return nil, err } @@ -63,15 +65,15 @@ func NewClient( ctx, ctxCancel := context.WithCancel(context.Background()) c := &Client{ - fingerprint: fingerprint, - onTracks: onTracks, - onVideoData: onVideoData, - onAudioData: onAudioData, - logger: logger, - ctx: ctx, - ctxCancel: ctxCancel, - primaryPlaylistURL: primaryPlaylistURL, - outErr: make(chan error, 1), + fingerprint: fingerprint, + onTracks: onTracks, + onVideoData: onVideoData, + onAudioData: onAudioData, + logger: logger, + ctx: ctx, + ctxCancel: ctxCancel, + playlistURL: playlistURL, + outErr: make(chan error, 1), } go c.run() @@ -95,19 +97,17 @@ func (c *Client) run() { func (c *Client) runInner() error { rp := newClientRoutinePool() - segmentQueue := newClientSegmentQueue() - dl := newClientDownloader( - c.primaryPlaylistURL, + dl := newClientDownloaderPrimary( + c.playlistURL, c.fingerprint, - segmentQueue, c.logger, + rp, c.onTracks, c.onVideoData, c.onAudioData, - rp, ) - rp.add(dl.run) + rp.add(dl) select { case err := <-rp.errorChan(): diff --git a/internal/hls/client_downloader.go b/internal/hls/client_downloader.go deleted file mode 100644 index e49d2981..00000000 --- a/internal/hls/client_downloader.go +++ /dev/null @@ -1,278 +0,0 @@ -package hls - -import ( - "context" - "crypto/sha256" - "crypto/tls" - "encoding/hex" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/aler9/gortsplib" - "github.com/grafov/m3u8" - - "github.com/aler9/rtsp-simple-server/internal/logger" -) - -type clientDownloader struct { - primaryPlaylistURL *url.URL - segmentQueue *clientSegmentQueue - logger ClientLogger - onTracks func(*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio) error - onVideoData func(time.Duration, [][]byte) - onAudioData func(time.Duration, []byte) - rp *clientRoutinePool - - streamPlaylistURL *url.URL - downloadedSegmentURIs []string - httpClient *http.Client - lastDownloadTime time.Time - firstPlaylistReceived bool -} - -func newClientDownloader( - primaryPlaylistURL *url.URL, - fingerprint string, - segmentQueue *clientSegmentQueue, - logger ClientLogger, - onTracks func(*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio) error, - onVideoData func(time.Duration, [][]byte), - onAudioData func(time.Duration, []byte), - rp *clientRoutinePool, -) *clientDownloader { - var tlsConfig *tls.Config - if fingerprint != "" { - tlsConfig = &tls.Config{ - InsecureSkipVerify: true, - VerifyConnection: func(cs tls.ConnectionState) error { - h := sha256.New() - h.Write(cs.PeerCertificates[0].Raw) - hstr := hex.EncodeToString(h.Sum(nil)) - fingerprintLower := strings.ToLower(fingerprint) - - if hstr != fingerprintLower { - return fmt.Errorf("server fingerprint do not match: expected %s, got %s", - fingerprintLower, hstr) - } - - return nil - }, - } - } - - return &clientDownloader{ - primaryPlaylistURL: primaryPlaylistURL, - segmentQueue: segmentQueue, - logger: logger, - onTracks: onTracks, - onVideoData: onVideoData, - onAudioData: onAudioData, - rp: rp, - httpClient: &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: tlsConfig, - }, - }, - } -} - -func (d *clientDownloader) run(ctx context.Context) error { - for { - ok := d.segmentQueue.waitUntilSizeIsBelow(ctx, clientMinSegmentsBeforeDownloading) - if !ok { - return fmt.Errorf("terminated") - } - - _, err := d.fillSegmentQueue(ctx) - if err != nil { - return err - } - } -} - -func (d *clientDownloader) fillSegmentQueue(ctx context.Context) (bool, error) { - minTime := d.lastDownloadTime.Add(clientMinDownloadPause) - now := time.Now() - if now.Before(minTime) { - select { - case <-time.After(minTime.Sub(now)): - case <-ctx.Done(): - return false, fmt.Errorf("terminated") - } - } - - d.lastDownloadTime = now - - pl, err := func() (*m3u8.MediaPlaylist, error) { - if d.streamPlaylistURL == nil { - return d.downloadPrimaryPlaylist(ctx) - } - return d.downloadStreamPlaylist(ctx) - }() - if err != nil { - return false, err - } - - if !d.firstPlaylistReceived { - d.firstPlaylistReceived = true - - if pl.Map != nil && pl.Map.URI != "" { - return false, fmt.Errorf("fMP4 streams are not supported yet") - } - - proc := newClientProcessorMPEGTS( - d.segmentQueue, - d.logger, - d.rp, - d.onTracks, - d.onVideoData, - d.onAudioData, - ) - d.rp.add(proc.run) - } - - added := false - - for _, seg := range pl.Segments { - if seg == nil { - break - } - - if !d.segmentWasDownloaded(seg.URI) { - d.downloadedSegmentURIs = append(d.downloadedSegmentURIs, seg.URI) - byts, err := d.downloadSegment(ctx, seg.URI) - if err != nil { - return false, err - } - - d.segmentQueue.push(byts) - added = true - } - } - - return added, nil -} - -func (d *clientDownloader) segmentWasDownloaded(ur string) bool { - for _, q := range d.downloadedSegmentURIs { - if q == ur { - return true - } - } - return false -} - -func (d *clientDownloader) downloadPrimaryPlaylist(ctx context.Context) (*m3u8.MediaPlaylist, error) { - d.logger.Log(logger.Debug, "downloading primary playlist %s", d.primaryPlaylistURL) - - pl, err := d.downloadPlaylist(ctx, d.primaryPlaylistURL) - if err != nil { - return nil, err - } - - switch plt := pl.(type) { - case *m3u8.MediaPlaylist: - d.streamPlaylistURL = d.primaryPlaylistURL - return plt, nil - - case *m3u8.MasterPlaylist: - // choose the variant with the highest bandwidth - var chosenVariant *m3u8.Variant - for _, v := range plt.Variants { - if chosenVariant == nil || - v.VariantParams.Bandwidth > chosenVariant.VariantParams.Bandwidth { - chosenVariant = v - } - } - - if chosenVariant == nil { - return nil, fmt.Errorf("no variants found") - } - - u, err := clientURLAbsolute(d.primaryPlaylistURL, chosenVariant.URI) - if err != nil { - return nil, err - } - - d.streamPlaylistURL = u - - return d.downloadStreamPlaylist(ctx) - - default: - return nil, fmt.Errorf("invalid playlist") - } -} - -func (d *clientDownloader) downloadStreamPlaylist(ctx context.Context) (*m3u8.MediaPlaylist, error) { - d.logger.Log(logger.Debug, "downloading stream playlist %s", d.streamPlaylistURL.String()) - - pl, err := d.downloadPlaylist(ctx, d.streamPlaylistURL) - if err != nil { - return nil, err - } - - plt, ok := pl.(*m3u8.MediaPlaylist) - if !ok { - return nil, fmt.Errorf("invalid playlist") - } - - return plt, nil -} - -func (d *clientDownloader) downloadPlaylist(ctx context.Context, ur *url.URL) (m3u8.Playlist, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, ur.String(), nil) - if err != nil { - return nil, err - } - - res, err := d.httpClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("bad status code: %d", res.StatusCode) - } - - pl, _, err := m3u8.DecodeFrom(res.Body, true) - if err != nil { - return nil, err - } - - return pl, nil -} - -func (d *clientDownloader) downloadSegment(ctx context.Context, segmentURI string) ([]byte, error) { - u, err := clientURLAbsolute(d.streamPlaylistURL, segmentURI) - if err != nil { - return nil, err - } - - d.logger.Log(logger.Debug, "downloading segment %s", u) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - - res, err := d.httpClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("bad status code: %d", res.StatusCode) - } - - byts, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - - return byts, nil -} diff --git a/internal/hls/client_downloader_primary.go b/internal/hls/client_downloader_primary.go new file mode 100644 index 00000000..21eccd82 --- /dev/null +++ b/internal/hls/client_downloader_primary.go @@ -0,0 +1,325 @@ +package hls + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/aler9/gortsplib" + gm3u8 "github.com/grafov/m3u8" + + "github.com/aler9/rtsp-simple-server/internal/hls/m3u8" + "github.com/aler9/rtsp-simple-server/internal/logger" +) + +func clientDownloadPlaylist(ctx context.Context, httpClient *http.Client, ur *url.URL) (m3u8.Playlist, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, ur.String(), nil) + if err != nil { + return nil, err + } + + res, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad status code: %d", res.StatusCode) + } + + byts, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + + return m3u8.Unmarshal(byts) +} + +func allCodecsAreSupported(codecs string) bool { + for _, codec := range strings.Split(codecs, ",") { + if !strings.HasPrefix(codec, "avc1") && + !strings.HasPrefix(codec, "mp4a") { + return false + } + } + return true +} + +func pickLeadingPlaylist(variants []*gm3u8.Variant) *gm3u8.Variant { + var candidates []*gm3u8.Variant //nolint:prealloc + for _, v := range variants { + if v.Codecs != "" && !allCodecsAreSupported(v.Codecs) { + continue + } + candidates = append(candidates, v) + } + if candidates == nil { + return nil + } + + // pick the variant with the greatest bandwidth + var leadingPlaylist *gm3u8.Variant + for _, v := range candidates { + if leadingPlaylist == nil || + v.VariantParams.Bandwidth > leadingPlaylist.VariantParams.Bandwidth { + leadingPlaylist = v + } + } + return leadingPlaylist +} + +func pickAudioPlaylist(alternatives []*gm3u8.Alternative, groupID string) *gm3u8.Alternative { + candidates := func() []*gm3u8.Alternative { + var ret []*gm3u8.Alternative + for _, alt := range alternatives { + if alt.GroupId == groupID { + ret = append(ret, alt) + } + } + return ret + }() + if candidates == nil { + return nil + } + + // pick the default audio playlist + for _, alt := range candidates { + if alt.Default { + return alt + } + } + + // alternatively, pick the first one + return candidates[0] +} + +type clientTimeSync interface{} + +type clientDownloaderPrimary struct { + primaryPlaylistURL *url.URL + logger ClientLogger + onTracks func(*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio) error + onVideoData func(time.Duration, [][]byte) + onAudioData func(time.Duration, []byte) + rp *clientRoutinePool + + httpClient *http.Client + leadingTimeSync clientTimeSync + + // in + streamTracks chan []gortsplib.Track + + // out + startStreaming chan struct{} + leadingTimeSyncReady chan struct{} +} + +func newClientDownloaderPrimary( + primaryPlaylistURL *url.URL, + fingerprint string, + logger ClientLogger, + rp *clientRoutinePool, + onTracks func(*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio) error, + onVideoData func(time.Duration, [][]byte), + onAudioData func(time.Duration, []byte), +) *clientDownloaderPrimary { + var tlsConfig *tls.Config + if fingerprint != "" { + tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + VerifyConnection: func(cs tls.ConnectionState) error { + h := sha256.New() + h.Write(cs.PeerCertificates[0].Raw) + hstr := hex.EncodeToString(h.Sum(nil)) + fingerprintLower := strings.ToLower(fingerprint) + + if hstr != fingerprintLower { + return fmt.Errorf("server fingerprint do not match: expected %s, got %s", + fingerprintLower, hstr) + } + + return nil + }, + } + } + + return &clientDownloaderPrimary{ + primaryPlaylistURL: primaryPlaylistURL, + logger: logger, + onTracks: onTracks, + onVideoData: onVideoData, + onAudioData: onAudioData, + rp: rp, + httpClient: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + }, + streamTracks: make(chan []gortsplib.Track), + startStreaming: make(chan struct{}), + leadingTimeSyncReady: make(chan struct{}), + } +} + +func (d *clientDownloaderPrimary) run(ctx context.Context) error { + d.logger.Log(logger.Debug, "downloading primary playlist %s", d.primaryPlaylistURL) + + pl, err := clientDownloadPlaylist(ctx, d.httpClient, d.primaryPlaylistURL) + if err != nil { + return err + } + + streamCount := 0 + + switch plt := pl.(type) { + case *m3u8.MediaPlaylist: + d.logger.Log(logger.Debug, "primary playlist is a stream playlist") + ds := newClientDownloaderStream( + true, + d.httpClient, + d.primaryPlaylistURL, + plt, + d.logger, + d.rp, + d.onStreamTracks, + d.onSetLeadingTimeSync, + d.onGetLeadingTimeSync, + d.onVideoData, + d.onAudioData) + d.rp.add(ds) + streamCount++ + + case *m3u8.MasterPlaylist: + leadingPlaylist := pickLeadingPlaylist(plt.Variants) + if leadingPlaylist == nil { + return fmt.Errorf("no variants with supported codecs found") + } + + u, err := clientAbsoluteURL(d.primaryPlaylistURL, leadingPlaylist.URI) + if err != nil { + return err + } + + ds := newClientDownloaderStream( + true, + d.httpClient, + u, + nil, + d.logger, + d.rp, + d.onStreamTracks, + d.onSetLeadingTimeSync, + d.onGetLeadingTimeSync, + d.onVideoData, + d.onAudioData) + d.rp.add(ds) + streamCount++ + + if leadingPlaylist.Audio != "" { + audioPlaylist := pickAudioPlaylist(plt.Alternatives, leadingPlaylist.Audio) + if audioPlaylist == nil { + return fmt.Errorf("audio playlist with id \"%s\" not found", leadingPlaylist.Audio) + } + + u, err := clientAbsoluteURL(d.primaryPlaylistURL, audioPlaylist.URI) + if err != nil { + return err + } + + ds := newClientDownloaderStream( + false, + d.httpClient, + u, + nil, + d.logger, + d.rp, + d.onStreamTracks, + d.onSetLeadingTimeSync, + d.onGetLeadingTimeSync, + d.onVideoData, + d.onAudioData) + d.rp.add(ds) + streamCount++ + } + + default: + return fmt.Errorf("invalid playlist") + } + + var tracks []gortsplib.Track + + for i := 0; i < streamCount; i++ { + select { + case streamTracks := <-d.streamTracks: + tracks = append(tracks, streamTracks...) + case <-ctx.Done(): + return fmt.Errorf("terminated") + } + } + + var videoTrack *gortsplib.TrackH264 + var audioTrack *gortsplib.TrackMPEG4Audio + + for _, track := range tracks { + switch ttrack := track.(type) { + case *gortsplib.TrackH264: + if videoTrack != nil { + return fmt.Errorf("multiple video tracks are not supported") + } + videoTrack = ttrack + + case *gortsplib.TrackMPEG4Audio: + if audioTrack != nil { + return fmt.Errorf("multiple audio tracks are not supported") + } + audioTrack = ttrack + } + } + + err = d.onTracks(videoTrack, audioTrack) + if err != nil { + return err + } + + close(d.startStreaming) + + return nil +} + +func (d *clientDownloaderPrimary) onStreamTracks(ctx context.Context, tracks []gortsplib.Track) bool { + select { + case d.streamTracks <- tracks: + case <-ctx.Done(): + return false + } + + select { + case <-d.startStreaming: + case <-ctx.Done(): + return false + } + + return true +} + +func (d *clientDownloaderPrimary) onSetLeadingTimeSync(ts clientTimeSync) { + d.leadingTimeSync = ts + close(d.leadingTimeSyncReady) +} + +func (d *clientDownloaderPrimary) onGetLeadingTimeSync(ctx context.Context) (clientTimeSync, bool) { + select { + case <-d.leadingTimeSyncReady: + case <-ctx.Done(): + return nil, false + } + return d.leadingTimeSync, true +} diff --git a/internal/hls/client_downloader_stream.go b/internal/hls/client_downloader_stream.go new file mode 100644 index 00000000..c4435163 --- /dev/null +++ b/internal/hls/client_downloader_stream.go @@ -0,0 +1,258 @@ +package hls + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/aler9/gortsplib" + gm3u8 "github.com/grafov/m3u8" + + "github.com/aler9/rtsp-simple-server/internal/hls/m3u8" + "github.com/aler9/rtsp-simple-server/internal/logger" +) + +func segmentsLen(segments []*gm3u8.MediaSegment) int { + for i, seg := range segments { + if seg == nil { + return i + } + } + return 0 +} + +func findSegmentWithInvPosition(segments []*gm3u8.MediaSegment, pos int) *gm3u8.MediaSegment { + index := len(segments) - pos + if index < 0 { + return nil + } + + return segments[index] +} + +func findSegmentWithID(seqNo uint64, segments []*gm3u8.MediaSegment, id uint64) (*gm3u8.MediaSegment, int) { + index := int(int64(id) - int64(seqNo)) + if (index) >= len(segments) { + return nil, 0 + } + + return segments[index], len(segments) - index +} + +type clientDownloaderStream struct { + isLeading bool + httpClient *http.Client + playlistURL *url.URL + initialPlaylist *m3u8.MediaPlaylist + logger ClientLogger + rp *clientRoutinePool + onStreamTracks func(context.Context, []gortsplib.Track) bool + onSetLeadingTimeSync func(clientTimeSync) + onGetLeadingTimeSync func(context.Context) (clientTimeSync, bool) + onVideoData func(time.Duration, [][]byte) + onAudioData func(time.Duration, []byte) + + curSegmentID *uint64 +} + +func newClientDownloaderStream( + isLeading bool, + httpClient *http.Client, + playlistURL *url.URL, + initialPlaylist *m3u8.MediaPlaylist, + logger ClientLogger, + rp *clientRoutinePool, + onStreamTracks func(context.Context, []gortsplib.Track) bool, + onSetLeadingTimeSync func(clientTimeSync), + onGetLeadingTimeSync func(context.Context) (clientTimeSync, bool), + onVideoData func(time.Duration, [][]byte), + onAudioData func(time.Duration, []byte), +) *clientDownloaderStream { + return &clientDownloaderStream{ + isLeading: isLeading, + httpClient: httpClient, + playlistURL: playlistURL, + initialPlaylist: initialPlaylist, + logger: logger, + rp: rp, + onStreamTracks: onStreamTracks, + onSetLeadingTimeSync: onSetLeadingTimeSync, + onGetLeadingTimeSync: onGetLeadingTimeSync, + onVideoData: onVideoData, + onAudioData: onAudioData, + } +} + +func (d *clientDownloaderStream) run(ctx context.Context) error { + initialPlaylist := d.initialPlaylist + d.initialPlaylist = nil + if initialPlaylist == nil { + var err error + initialPlaylist, err = d.downloadPlaylist(ctx) + if err != nil { + return err + } + } + + segmentQueue := newClientSegmentQueue() + + if initialPlaylist.Map != nil && initialPlaylist.Map.URI != "" { + byts, err := d.downloadSegment(ctx, initialPlaylist.Map.URI, initialPlaylist.Map.Offset, initialPlaylist.Map.Limit) + if err != nil { + return err + } + + proc, err := newClientProcessorFMP4( + ctx, + d.isLeading, + byts, + segmentQueue, + d.logger, + d.rp, + d.onStreamTracks, + d.onSetLeadingTimeSync, + d.onGetLeadingTimeSync, + d.onVideoData, + d.onAudioData, + ) + if err != nil { + return err + } + + d.rp.add(proc) + } else { + proc := newClientProcessorMPEGTS( + d.isLeading, + segmentQueue, + d.logger, + d.rp, + d.onStreamTracks, + d.onSetLeadingTimeSync, + d.onGetLeadingTimeSync, + d.onVideoData, + d.onAudioData, + ) + d.rp.add(proc) + } + + for { + ok := segmentQueue.waitUntilSizeIsBelow(ctx, 1) + if !ok { + return fmt.Errorf("terminated") + } + + err := d.fillSegmentQueue(ctx, segmentQueue) + if err != nil { + return err + } + } +} + +func (d *clientDownloaderStream) downloadPlaylist(ctx context.Context) (*m3u8.MediaPlaylist, error) { + d.logger.Log(logger.Debug, "downloading stream playlist %s", d.playlistURL.String()) + + pl, err := clientDownloadPlaylist(ctx, d.httpClient, d.playlistURL) + if err != nil { + return nil, err + } + + plt, ok := pl.(*m3u8.MediaPlaylist) + if !ok { + return nil, fmt.Errorf("invalid playlist") + } + + return plt, nil +} + +func (d *clientDownloaderStream) downloadSegment(ctx context.Context, + uri string, offset int64, limit int64, +) ([]byte, error) { + u, err := clientAbsoluteURL(d.playlistURL, uri) + if err != nil { + return nil, err + } + + d.logger.Log(logger.Debug, "downloading segment %s", u) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + + if limit != 0 { + req.Header.Add("Range", "bytes="+strconv.FormatInt(offset, 10)+"-"+strconv.FormatInt(offset+limit-1, 10)) + } + + res, err := d.httpClient.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusPartialContent { + return nil, fmt.Errorf("bad status code: %d", res.StatusCode) + } + + byts, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + + return byts, nil +} + +func (d *clientDownloaderStream) fillSegmentQueue(ctx context.Context, segmentQueue *clientSegmentQueue) error { + pl, err := d.downloadPlaylist(ctx) + if err != nil { + return err + } + + pl.Segments = pl.Segments[:segmentsLen(pl.Segments)] + var seg *gm3u8.MediaSegment + + if d.curSegmentID == nil { + if !pl.Closed { // live stream: start from clientLiveStartingInvPosition + seg = findSegmentWithInvPosition(pl.Segments, clientLiveStartingInvPosition) + if seg == nil { + return fmt.Errorf("there aren't enough segments to fill the buffer") + } + } else { // VOD stream: start from beginning + if len(pl.Segments) == 0 { + return fmt.Errorf("no segments found") + } + seg = pl.Segments[0] + } + } else { + var invPos int + seg, invPos = findSegmentWithID(pl.SeqNo, pl.Segments, *d.curSegmentID+1) + if seg == nil { + return fmt.Errorf("following segment not found or not ready yet") + } + + d.logger.Log(logger.Debug, "segment inverse position: %d", invPos) + + if !pl.Closed && invPos > clientLiveMaxInvPosition { + return fmt.Errorf("playback is too late") + } + } + + v := seg.SeqId + d.curSegmentID = &v + + byts, err := d.downloadSegment(ctx, seg.URI, seg.Offset, seg.Limit) + if err != nil { + return err + } + + segmentQueue.push(byts) + + if pl.Closed && pl.Segments[len(pl.Segments)-1] == seg { + <-ctx.Done() + return fmt.Errorf("stream has ended") + } + + return nil +} diff --git a/internal/hls/client_processor_fmp4.go b/internal/hls/client_processor_fmp4.go new file mode 100644 index 00000000..d2bddce3 --- /dev/null +++ b/internal/hls/client_processor_fmp4.go @@ -0,0 +1,217 @@ +package hls + +import ( + "context" + "fmt" + "time" + + "github.com/aler9/gortsplib" + "github.com/aler9/gortsplib/pkg/h264" + + "github.com/aler9/rtsp-simple-server/internal/hls/fmp4" +) + +func fmp4PickLeadingTrack(init *fmp4.Init) int { + // pick first video track + for _, track := range init.Tracks { + if _, ok := track.Track.(*gortsplib.TrackH264); ok { + return track.ID + } + } + + // otherwise, pick first track + return init.Tracks[0].ID +} + +type clientProcessorFMP4 struct { + isLeading bool + segmentQueue *clientSegmentQueue + logger ClientLogger + rp *clientRoutinePool + onSetLeadingTimeSync func(clientTimeSync) + onGetLeadingTimeSync func(context.Context) (clientTimeSync, bool) + onVideoData func(time.Duration, [][]byte) + onAudioData func(time.Duration, []byte) + + init fmp4.Init + leadingTrackID int + trackProcs map[int]*clientProcessorFMP4Track + + // in + subpartProcessed chan struct{} +} + +func newClientProcessorFMP4( + ctx context.Context, + isLeading bool, + initFile []byte, + segmentQueue *clientSegmentQueue, + logger ClientLogger, + rp *clientRoutinePool, + onStreamTracks func(context.Context, []gortsplib.Track) bool, + onSetLeadingTimeSync func(clientTimeSync), + onGetLeadingTimeSync func(context.Context) (clientTimeSync, bool), + onVideoData func(time.Duration, [][]byte), + onAudioData func(time.Duration, []byte), +) (*clientProcessorFMP4, error) { + p := &clientProcessorFMP4{ + isLeading: isLeading, + segmentQueue: segmentQueue, + logger: logger, + rp: rp, + onSetLeadingTimeSync: onSetLeadingTimeSync, + onGetLeadingTimeSync: onGetLeadingTimeSync, + onVideoData: onVideoData, + onAudioData: onAudioData, + subpartProcessed: make(chan struct{}, clientFMP4MaxPartTracksPerSegment), + } + + err := p.init.Unmarshal(initFile) + if err != nil { + return nil, err + } + + p.leadingTrackID = fmp4PickLeadingTrack(&p.init) + + tracks := make([]gortsplib.Track, len(p.init.Tracks)) + for i, track := range p.init.Tracks { + tracks[i] = track.Track + } + + ok := onStreamTracks(ctx, tracks) + if !ok { + return nil, fmt.Errorf("terminated") + } + + return p, nil +} + +func (p *clientProcessorFMP4) run(ctx context.Context) error { + for { + seg, ok := p.segmentQueue.pull(ctx) + if !ok { + return fmt.Errorf("terminated") + } + + err := p.processSegment(ctx, seg) + if err != nil { + return err + } + } +} + +func (p *clientProcessorFMP4) processSegment(ctx context.Context, byts []byte) error { + var parts fmp4.Parts + err := parts.Unmarshal(byts) + if err != nil { + return err + } + + processingCount := 0 + + for _, part := range parts { + for _, track := range part.Tracks { + if p.trackProcs == nil { + var ts *clientTimeSyncFMP4 + + if p.isLeading { + if track.ID != p.leadingTrackID { + continue + } + + timeScale := func() uint32 { + for _, track := range p.init.Tracks { + if track.ID == p.leadingTrackID { + return track.TimeScale + } + } + return 0 + }() + ts = newClientTimeSyncFMP4(timeScale, track.BaseTime) + p.onSetLeadingTimeSync(ts) + } else { + rawTS, ok := p.onGetLeadingTimeSync(ctx) + if !ok { + return fmt.Errorf("terminated") + } + + ts, ok = rawTS.(*clientTimeSyncFMP4) + if !ok { + return fmt.Errorf("stream playlists are mixed MPEGTS/FMP4") + } + } + + p.initializeTrackProcs(ts) + } + + proc, ok := p.trackProcs[track.ID] + if !ok { + return fmt.Errorf("track ID %d not present in init file", track.ID) + } + + if processingCount >= (clientFMP4MaxPartTracksPerSegment - 1) { + return fmt.Errorf("too many part tracks at once") + } + + select { + case proc.queue <- track: + case <-ctx.Done(): + return fmt.Errorf("terminated") + } + processingCount++ + } + } + + for i := 0; i < processingCount; i++ { + select { + case <-p.subpartProcessed: + case <-ctx.Done(): + return fmt.Errorf("terminated") + } + } + + return nil +} + +func (p *clientProcessorFMP4) onPartTrackProcessed(ctx context.Context) { + select { + case p.subpartProcessed <- struct{}{}: + case <-ctx.Done(): + } +} + +func (p *clientProcessorFMP4) initializeTrackProcs(ts *clientTimeSyncFMP4) { + p.trackProcs = make(map[int]*clientProcessorFMP4Track) + + for _, track := range p.init.Tracks { + var cb func(time.Duration, []byte) error + + switch track.Track.(type) { + case *gortsplib.TrackH264: + cb = func(pts time.Duration, payload []byte) error { + nalus, err := h264.AVCCUnmarshal(payload) + if err != nil { + return err + } + + p.onVideoData(pts, nalus) + return nil + } + + case *gortsplib.TrackMPEG4Audio: + cb = func(pts time.Duration, payload []byte) error { + p.onAudioData(pts, payload) + return nil + } + } + + proc := newClientProcessorFMP4Track( + track.TimeScale, + ts, + p.onPartTrackProcessed, + cb, + ) + p.rp.add(proc) + p.trackProcs[track.ID] = proc + } +} diff --git a/internal/hls/client_processor_fmp4_track.go b/internal/hls/client_processor_fmp4_track.go new file mode 100644 index 00000000..8cfb5c96 --- /dev/null +++ b/internal/hls/client_processor_fmp4_track.go @@ -0,0 +1,70 @@ +package hls + +import ( + "context" + "time" + + "github.com/aler9/rtsp-simple-server/internal/hls/fmp4" +) + +type clientProcessorFMP4Track struct { + timeScale uint32 + ts *clientTimeSyncFMP4 + onPartTrackProcessed func(context.Context) + onEntry func(time.Duration, []byte) error + + // in + queue chan *fmp4.PartTrack +} + +func newClientProcessorFMP4Track( + timeScale uint32, + ts *clientTimeSyncFMP4, + onPartTrackProcessed func(context.Context), + onEntry func(time.Duration, []byte) error, +) *clientProcessorFMP4Track { + return &clientProcessorFMP4Track{ + timeScale: timeScale, + ts: ts, + onPartTrackProcessed: onPartTrackProcessed, + onEntry: onEntry, + queue: make(chan *fmp4.PartTrack, clientFMP4MaxPartTracksPerSegment), + } +} + +func (t *clientProcessorFMP4Track) run(ctx context.Context) error { + for { + select { + case entry := <-t.queue: + err := t.processPartTrack(ctx, entry) + if err != nil { + return err + } + + t.onPartTrackProcessed(ctx) + + case <-ctx.Done(): + return nil + } + } +} + +func (t *clientProcessorFMP4Track) processPartTrack(ctx context.Context, pt *fmp4.PartTrack) error { + rawDTS := pt.BaseTime + + for _, sample := range pt.Samples { + pts, err := t.ts.convertAndSync(ctx, t.timeScale, rawDTS, sample.PTSOffset) + if err != nil { + return err + } + + err = t.onEntry(pts, sample.Payload) + if err != nil { + return err + } + + rawDTS += uint64(sample.Duration) + } + + return nil +} diff --git a/internal/hls/client_processor_mpegts.go b/internal/hls/client_processor_mpegts.go index 64ca7258..9078b35e 100644 --- a/internal/hls/client_processor_mpegts.go +++ b/internal/hls/client_processor_mpegts.go @@ -12,46 +12,59 @@ import ( "github.com/aler9/gortsplib/pkg/mpeg4audio" "github.com/asticode/go-astits" - "github.com/aler9/rtsp-simple-server/internal/hls/mpegtstimedec" + "github.com/aler9/rtsp-simple-server/internal/hls/mpegts" "github.com/aler9/rtsp-simple-server/internal/logger" ) -type clientProcessorMPEGTS struct { - segmentQueue *clientSegmentQueue - logger ClientLogger - rp *clientRoutinePool - onTracks func(*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio) error - onVideoData func(time.Duration, [][]byte) - onAudioData func(time.Duration, []byte) +func mpegtsPickLeadingTrack(mpegtsTracks []*mpegts.Track) uint16 { + // pick first video track + for _, mt := range mpegtsTracks { + if _, ok := mt.Track.(*gortsplib.TrackH264); ok { + return mt.ES.ElementaryPID + } + } - tracksParsed bool - clockInitialized bool - timeDec *mpegtstimedec.Decoder - startDTS time.Duration - videoPID *uint16 - audioPID *uint16 - videoTrack *gortsplib.TrackH264 - audioTrack *gortsplib.TrackMPEG4Audio - videoProc *clientProcessorMPEGTSTrack - audioProc *clientProcessorMPEGTSTrack + // otherwise, pick first track + return mpegtsTracks[0].ES.ElementaryPID +} + +type clientProcessorMPEGTS struct { + isLeading bool + segmentQueue *clientSegmentQueue + logger ClientLogger + rp *clientRoutinePool + onStreamTracks func(context.Context, []gortsplib.Track) bool + onSetLeadingTimeSync func(clientTimeSync) + onGetLeadingTimeSync func(context.Context) (clientTimeSync, bool) + onVideoData func(time.Duration, [][]byte) + onAudioData func(time.Duration, []byte) + + mpegtsTracks []*mpegts.Track + leadingTrackPID uint16 + trackProcs map[uint16]*clientProcessorMPEGTSTrack } func newClientProcessorMPEGTS( + isLeading bool, segmentQueue *clientSegmentQueue, logger ClientLogger, rp *clientRoutinePool, - onTracks func(*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio) error, + onStreamTracks func(context.Context, []gortsplib.Track) bool, + onSetLeadingTimeSync func(clientTimeSync), + onGetLeadingTimeSync func(context.Context) (clientTimeSync, bool), onVideoData func(time.Duration, [][]byte), onAudioData func(time.Duration, []byte), ) *clientProcessorMPEGTS { return &clientProcessorMPEGTS{ - segmentQueue: segmentQueue, - logger: logger, - rp: rp, - timeDec: mpegtstimedec.New(), - onTracks: onTracks, - onVideoData: onVideoData, - onAudioData: onAudioData, + isLeading: isLeading, + segmentQueue: segmentQueue, + logger: logger, + rp: rp, + onStreamTracks: onStreamTracks, + onSetLeadingTimeSync: onSetLeadingTimeSync, + onGetLeadingTimeSync: onGetLeadingTimeSync, + onVideoData: onVideoData, + onAudioData: onAudioData, } } @@ -70,24 +83,28 @@ func (p *clientProcessorMPEGTS) run(ctx context.Context) error { } func (p *clientProcessorMPEGTS) processSegment(ctx context.Context, byts []byte) error { - p.logger.Log(logger.Debug, "processing segment") - - dem := astits.NewDemuxer(context.Background(), bytes.NewReader(byts)) - - if !p.tracksParsed { - p.tracksParsed = true - - err := p.parseTracks(dem) + if p.mpegtsTracks == nil { + var err error + p.mpegtsTracks, err = mpegts.FindTracks(byts) if err != nil { return err } - // rewind demuxer in order to read again the audio packet that was used to create the track - if p.audioTrack != nil { - dem = astits.NewDemuxer(context.Background(), bytes.NewReader(byts)) + p.leadingTrackPID = mpegtsPickLeadingTrack(p.mpegtsTracks) + + tracks := make([]gortsplib.Track, len(p.mpegtsTracks)) + for i, mt := range p.mpegtsTracks { + tracks[i] = mt.Track + } + + ok := p.onStreamTracks(ctx, tracks) + if !ok { + return fmt.Errorf("terminated") } } + dem := astits.NewDemuxer(context.Background(), bytes.NewReader(byts)) + for { data, err := dem.NextData() if err != nil { @@ -110,185 +127,92 @@ func (p *clientProcessorMPEGTS) processSegment(ctx context.Context, byts []byte) return fmt.Errorf("PTS is missing") } - pts := p.timeDec.Decode(data.PES.Header.OptionalHeader.PTS.Base) + if p.trackProcs == nil { + var ts *clientTimeSyncMPEGTS - if p.videoPID != nil && data.PID == *p.videoPID { - var dts time.Duration - if data.PES.Header.OptionalHeader.PTSDTSIndicator == astits.PTSDTSIndicatorBothPresent { - diff := time.Duration((data.PES.Header.OptionalHeader.PTS.Base- - data.PES.Header.OptionalHeader.DTS.Base)&0x1FFFFFFFF) * - time.Second / 90000 - dts = pts - diff + if p.isLeading { + if data.PID != p.leadingTrackPID { + continue + } + + var dts int64 + if data.PES.Header.OptionalHeader.PTSDTSIndicator == astits.PTSDTSIndicatorBothPresent { + dts = data.PES.Header.OptionalHeader.DTS.Base + } else { + dts = data.PES.Header.OptionalHeader.PTS.Base + } + + ts = newClientTimeSyncMPEGTS(dts) + p.onSetLeadingTimeSync(ts) } else { - dts = pts - } + rawTS, ok := p.onGetLeadingTimeSync(ctx) + if !ok { + return fmt.Errorf("terminated") + } - if !p.clockInitialized { - p.clockInitialized = true - p.startDTS = dts - now := time.Now() - p.initializeTrackProcs(now) - } - - pts -= p.startDTS - dts -= p.startDTS - - p.videoProc.push(ctx, &clientProcessorMPEGTSTrackEntryVideo{ - data: data.PES.Data, - pts: pts, - dts: dts, - }) - } else if p.audioPID != nil && data.PID == *p.audioPID { - if !p.clockInitialized { - p.clockInitialized = true - p.startDTS = pts - now := time.Now() - p.initializeTrackProcs(now) - } - - pts -= p.startDTS - - p.audioProc.push(ctx, &clientProcessorMPEGTSTrackEntryAudio{ - data: data.PES.Data, - pts: pts, - }) - } - } -} - -func (p *clientProcessorMPEGTS) parseTracks(dem *astits.Demuxer) error { - // find and parse PMT - for { - data, err := dem.NextData() - if err != nil { - return err - } - - if data.PMT != nil { - for _, e := range data.PMT.ElementaryStreams { - switch e.StreamType { - case astits.StreamTypeH264Video: - if p.videoPID != nil { - return fmt.Errorf("multiple video/audio tracks are not supported") - } - - v := e.ElementaryPID - p.videoPID = &v - - case astits.StreamTypeAACAudio: - if p.audioPID != nil { - return fmt.Errorf("multiple video/audio tracks are not supported") - } - - v := e.ElementaryPID - p.audioPID = &v + ts, ok = rawTS.(*clientTimeSyncMPEGTS) + if !ok { + return fmt.Errorf("stream playlists are mixed MPEGTS/FMP4") } } - break - } - } - if p.videoPID == nil && p.audioPID == nil { - return fmt.Errorf("stream doesn't contain tracks with supported codecs (H264 or AAC)") - } - - if p.videoPID != nil { - p.videoTrack = &gortsplib.TrackH264{ - PayloadType: 96, + p.initializeTrackProcs(ts) } - if p.audioPID == nil { - err := p.onTracks(p.videoTrack, nil) - if err != nil { - return err - } + proc, ok := p.trackProcs[data.PID] + if !ok { + return fmt.Errorf("received data from track not present into PMT (%d)", data.PID) + } + + select { + case proc.queue <- data.PES: + case <-ctx.Done(): } } - - // find and parse first audio packet - if p.audioPID != nil { - for { - data, err := dem.NextData() - if err != nil { - return err - } - - if data.PES == nil || data.PID != *p.audioPID { - continue - } - - var adtsPkts mpeg4audio.ADTSPackets - err = adtsPkts.Unmarshal(data.PES.Data) - if err != nil { - return fmt.Errorf("unable to decode ADTS: %s", err) - } - - pkt := adtsPkts[0] - p.audioTrack = &gortsplib.TrackMPEG4Audio{ - PayloadType: 96, - Config: &mpeg4audio.Config{ - Type: pkt.Type, - SampleRate: pkt.SampleRate, - ChannelCount: pkt.ChannelCount, - }, - SizeLength: 13, - IndexLength: 3, - IndexDeltaLength: 3, - } - - err = p.onTracks(p.videoTrack, p.audioTrack) - if err != nil { - return err - } - - break - } - } - - return nil } -func (p *clientProcessorMPEGTS) initializeTrackProcs(clockStartRTC time.Time) { - if p.videoTrack != nil { - p.videoProc = newClientProcessorMPEGTSTrack( - clockStartRTC, - func(e clientProcessorMPEGTSTrackEntry) error { - vd := e.(*clientProcessorMPEGTSTrackEntryVideo) +func (p *clientProcessorMPEGTS) initializeTrackProcs(ts *clientTimeSyncMPEGTS) { + p.trackProcs = make(map[uint16]*clientProcessorMPEGTSTrack) - nalus, err := h264.AnnexBUnmarshal(vd.data) + for _, mt := range p.mpegtsTracks { + var cb func(time.Duration, []byte) error + + switch mt.Track.(type) { + case *gortsplib.TrackH264: + cb = func(pts time.Duration, payload []byte) error { + nalus, err := h264.AnnexBUnmarshal(payload) if err != nil { p.logger.Log(logger.Warn, "unable to decode Annex-B: %s", err) return nil } - p.onVideoData(vd.pts, nalus) + p.onVideoData(pts, nalus) return nil - }, - ) - p.rp.add(p.videoProc.run) - } - - if p.audioTrack != nil { - p.audioProc = newClientProcessorMPEGTSTrack( - clockStartRTC, - func(e clientProcessorMPEGTSTrackEntry) error { - ad := e.(*clientProcessorMPEGTSTrackEntryAudio) + } + case *gortsplib.TrackMPEG4Audio: + cb = func(pts time.Duration, payload []byte) error { var adtsPkts mpeg4audio.ADTSPackets - err := adtsPkts.Unmarshal(ad.data) + err := adtsPkts.Unmarshal(payload) if err != nil { return fmt.Errorf("unable to decode ADTS: %s", err) } for i, pkt := range adtsPkts { p.onAudioData( - ad.pts+time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*time.Second/time.Duration(pkt.SampleRate), + pts+time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*time.Second/time.Duration(pkt.SampleRate), pkt.AU) } return nil - }, + } + } + + proc := newClientProcessorMPEGTSTrack( + ts, + cb, ) - p.rp.add(p.audioProc.run) + p.rp.add(proc) + p.trackProcs[mt.ES.ElementaryPID] = proc } } diff --git a/internal/hls/client_processor_mpegts_track.go b/internal/hls/client_processor_mpegts_track.go index 11a6dfa2..32de38f3 100644 --- a/internal/hls/client_processor_mpegts_track.go +++ b/internal/hls/client_processor_mpegts_track.go @@ -2,56 +2,34 @@ package hls import ( "context" - "fmt" "time" + + "github.com/asticode/go-astits" ) -type clientProcessorMPEGTSTrackEntry interface { - DTS() time.Duration -} - -type clientProcessorMPEGTSTrackEntryVideo struct { - data []byte - pts time.Duration - dts time.Duration -} - -func (e clientProcessorMPEGTSTrackEntryVideo) DTS() time.Duration { - return e.dts -} - -type clientProcessorMPEGTSTrackEntryAudio struct { - data []byte - pts time.Duration -} - -func (e clientProcessorMPEGTSTrackEntryAudio) DTS() time.Duration { - return e.pts -} - type clientProcessorMPEGTSTrack struct { - clockStartRTC time.Time - onEntry func(e clientProcessorMPEGTSTrackEntry) error + ts *clientTimeSyncMPEGTS + onEntry func(time.Duration, []byte) error - queue chan clientProcessorMPEGTSTrackEntry + queue chan *astits.PESData } func newClientProcessorMPEGTSTrack( - clockStartRTC time.Time, - onEntry func(e clientProcessorMPEGTSTrackEntry) error, + ts *clientTimeSyncMPEGTS, + onEntry func(time.Duration, []byte) error, ) *clientProcessorMPEGTSTrack { return &clientProcessorMPEGTSTrack{ - clockStartRTC: clockStartRTC, - onEntry: onEntry, - queue: make(chan clientProcessorMPEGTSTrackEntry, clientQueueSize), + ts: ts, + onEntry: onEntry, + queue: make(chan *astits.PESData, clientMPEGTSEntryQueueSize), } } func (t *clientProcessorMPEGTSTrack) run(ctx context.Context) error { for { select { - case entry := <-t.queue: - err := t.processEntry(ctx, entry) + case pes := <-t.queue: + err := t.processEntry(ctx, pes) if err != nil { return err } @@ -62,22 +40,19 @@ func (t *clientProcessorMPEGTSTrack) run(ctx context.Context) error { } } -func (t *clientProcessorMPEGTSTrack) processEntry(ctx context.Context, entry clientProcessorMPEGTSTrackEntry) error { - elapsed := time.Since(t.clockStartRTC) - if entry.DTS() > elapsed { - select { - case <-ctx.Done(): - return fmt.Errorf("terminated") - case <-time.After(entry.DTS() - elapsed): - } +func (t *clientProcessorMPEGTSTrack) processEntry(ctx context.Context, pes *astits.PESData) error { + rawPTS := pes.Header.OptionalHeader.PTS.Base + var rawDTS int64 + if pes.Header.OptionalHeader.PTSDTSIndicator == astits.PTSDTSIndicatorBothPresent { + rawDTS = pes.Header.OptionalHeader.DTS.Base + } else { + rawDTS = rawPTS } - return t.onEntry(entry) -} - -func (t *clientProcessorMPEGTSTrack) push(ctx context.Context, entry clientProcessorMPEGTSTrackEntry) { - select { - case t.queue <- entry: - case <-ctx.Done(): + pts, err := t.ts.convertAndSync(ctx, rawDTS, rawPTS) + if err != nil { + return err } + + return t.onEntry(pts, pes.Data) } diff --git a/internal/hls/client_routine_pool.go b/internal/hls/client_routine_pool.go index e9a0f90c..ca7f5944 100644 --- a/internal/hls/client_routine_pool.go +++ b/internal/hls/client_routine_pool.go @@ -5,6 +5,10 @@ import ( "sync" ) +type clientRoutinePoolRunnable interface { + run(context.Context) error +} + type clientRoutinePool struct { ctx context.Context ctxCancel func() @@ -32,13 +36,17 @@ func (rp *clientRoutinePool) errorChan() chan error { return rp.err } -func (rp *clientRoutinePool) add(cb func(context.Context) error) { +func (rp *clientRoutinePool) add(r clientRoutinePoolRunnable) { rp.wg.Add(1) go func() { defer rp.wg.Done() - select { - case rp.err <- cb(rp.ctx): - case <-rp.ctx.Done(): + + err := r.run(rp.ctx) + if err != nil { + select { + case rp.err <- err: + case <-rp.ctx.Done(): + } } }() } diff --git a/internal/hls/client_test.go b/internal/hls/client_test.go index c07ed3a4..6b9c833b 100644 --- a/internal/hls/client_test.go +++ b/internal/hls/client_test.go @@ -120,7 +120,9 @@ func newTestHLSServer(ca string) (*testHLSServer, error) { #EXT-X-TARGETDURATION:2 #EXT-X-MEDIA-SEQUENCE:0 #EXTINF:2, -` + segment + "\n" +` + segment + ` +#EXT-X-ENDLIST +` ctx.Writer.Header().Set("Content-Type", `application/x-mpegURL`) io.Copy(ctx.Writer, bytes.NewReader([]byte(cnt))) diff --git a/internal/hls/client_timesync_fmp4.go b/internal/hls/client_timesync_fmp4.go new file mode 100644 index 00000000..5289c689 --- /dev/null +++ b/internal/hls/client_timesync_fmp4.go @@ -0,0 +1,59 @@ +package hls + +import ( + "context" + "fmt" + "time" +) + +func durationGoToMp4(v time.Duration, timeScale uint32) uint64 { + timeScale64 := uint64(timeScale) + secs := v / time.Second + dec := v % time.Second + return uint64(secs)*timeScale64 + uint64(dec)*timeScale64/uint64(time.Second) +} + +func durationMp4ToGo(v uint64, timeScale uint32) time.Duration { + timeScale64 := uint64(timeScale) + secs := v / timeScale64 + dec := v % timeScale64 + return time.Duration(secs)*time.Second + time.Duration(dec)*time.Second/time.Duration(timeScale64) +} + +type clientTimeSyncFMP4 struct { + startRTC time.Time + startDTS time.Duration +} + +func newClientTimeSyncFMP4(timeScale uint32, baseTime uint64) *clientTimeSyncFMP4 { + return &clientTimeSyncFMP4{ + startRTC: time.Now(), + startDTS: durationMp4ToGo(baseTime, timeScale), + } +} + +func (ts *clientTimeSyncFMP4) convertAndSync(ctx context.Context, timeScale uint32, + rawDTS uint64, ptsOffset int32, +) (time.Duration, error) { + pts := durationMp4ToGo(rawDTS+uint64(ptsOffset), timeScale) + dts := durationMp4ToGo(rawDTS, timeScale) + + pts -= ts.startDTS + dts -= ts.startDTS + + elapsed := time.Since(ts.startRTC) + if dts > elapsed { + diff := dts - elapsed + if diff > clientMaxDTSRTCDiff { + return 0, fmt.Errorf("difference between DTS and RTC is too big") + } + + select { + case <-time.After(diff): + case <-ctx.Done(): + return 0, fmt.Errorf("terminated") + } + } + + return pts, nil +} diff --git a/internal/hls/client_timesync_mpegts.go b/internal/hls/client_timesync_mpegts.go new file mode 100644 index 00000000..f49e0ffa --- /dev/null +++ b/internal/hls/client_timesync_mpegts.go @@ -0,0 +1,47 @@ +package hls + +import ( + "context" + "fmt" + "time" + + "github.com/aler9/rtsp-simple-server/internal/hls/mpegts" +) + +type clientTimeSyncMPEGTS struct { + startRTC time.Time + startDTS int64 + td *mpegts.TimeDecoder +} + +func newClientTimeSyncMPEGTS(startDTS int64) *clientTimeSyncMPEGTS { + return &clientTimeSyncMPEGTS{ + startRTC: time.Now(), + startDTS: startDTS, + td: mpegts.NewTimeDecoder(), + } +} + +func (ts *clientTimeSyncMPEGTS) convertAndSync(ctx context.Context, rawDTS int64, rawPTS int64) (time.Duration, error) { + rawDTS = (rawDTS - ts.startDTS) & 0x1FFFFFFFF + rawPTS = (rawPTS - ts.startDTS) & 0x1FFFFFFFF + + dts := ts.td.Decode(rawDTS) + pts := ts.td.Decode(rawPTS) + + elapsed := time.Since(ts.startRTC) + if dts > elapsed { + diff := dts - elapsed + if diff > clientMaxDTSRTCDiff { + return 0, fmt.Errorf("difference between DTS and RTC is too big") + } + + select { + case <-time.After(diff): + case <-ctx.Done(): + return 0, fmt.Errorf("terminated") + } + } + + return pts, nil +} diff --git a/internal/hls/fmp4/audiosample.go b/internal/hls/fmp4/audiosample.go deleted file mode 100644 index 323683bd..00000000 --- a/internal/hls/fmp4/audiosample.go +++ /dev/null @@ -1,17 +0,0 @@ -package fmp4 - -import ( - "time" -) - -// AudioSample is an audio sample. -type AudioSample struct { - AU []byte - PTS time.Duration - Next *AudioSample -} - -// Duration returns the sample duration. -func (s AudioSample) Duration() time.Duration { - return s.Next.PTS - s.PTS -} diff --git a/internal/hls/fmp4/init.go b/internal/hls/fmp4/init.go new file mode 100644 index 00000000..6261e870 --- /dev/null +++ b/internal/hls/fmp4/init.go @@ -0,0 +1,261 @@ +package fmp4 + +import ( + "bytes" + "fmt" + + gomp4 "github.com/abema/go-mp4" + "github.com/aler9/gortsplib" + "github.com/aler9/gortsplib/pkg/mpeg4audio" +) + +// Init is a FMP4 initialization file. +type Init struct { + Tracks []*InitTrack +} + +// Unmarshal decodes a FMP4 initialization file. +func (i *Init) Unmarshal(byts []byte) error { + type readState int + + const ( + waitingTrak readState = iota + waitingTkhd + waitingMdhd + waitingCodec + waitingAvcc + waitingEsds + ) + + state := waitingTrak + var curTrack *InitTrack + + _, err := gomp4.ReadBoxStructure(bytes.NewReader(byts), func(h *gomp4.ReadHandle) (interface{}, error) { + switch h.BoxInfo.Type.String() { + case "trak": + if state != waitingTrak { + return nil, fmt.Errorf("parse error") + } + + curTrack = &InitTrack{} + i.Tracks = append(i.Tracks, curTrack) + state = waitingTkhd + + case "tkhd": + if state != waitingTkhd { + return nil, fmt.Errorf("parse error") + } + + box, _, err := h.ReadPayload() + if err != nil { + return nil, err + } + tkhd := box.(*gomp4.Tkhd) + + curTrack.ID = int(tkhd.TrackID) + state = waitingMdhd + + case "mdhd": + if state != waitingMdhd { + return nil, fmt.Errorf("parse error") + } + + box, _, err := h.ReadPayload() + if err != nil { + return nil, err + } + mdhd := box.(*gomp4.Mdhd) + + curTrack.TimeScale = mdhd.Timescale + state = waitingCodec + + case "avc1": + if state != waitingCodec { + return nil, fmt.Errorf("parse error") + } + + state = waitingAvcc + + case "avcC": + if state != waitingAvcc { + return nil, fmt.Errorf("parse error") + } + + box, _, err := h.ReadPayload() + if err != nil { + return nil, err + } + conf := box.(*gomp4.AVCDecoderConfiguration) + + if len(conf.SequenceParameterSets) > 1 { + return nil, fmt.Errorf("multiple SPS are not supported") + } + + var sps []byte + if len(conf.SequenceParameterSets) == 1 { + sps = conf.SequenceParameterSets[0].NALUnit + } + + if len(conf.PictureParameterSets) > 1 { + return nil, fmt.Errorf("multiple PPS are not supported") + } + + var pps []byte + if len(conf.PictureParameterSets) == 1 { + pps = conf.PictureParameterSets[0].NALUnit + } + + curTrack.Track = &gortsplib.TrackH264{ + PayloadType: 96, + SPS: sps, + PPS: pps, + } + state = waitingTrak + + case "mp4a": + if state != waitingCodec { + return nil, fmt.Errorf("parse error") + } + + state = waitingEsds + + case "esds": + if state != waitingEsds { + return nil, fmt.Errorf("parse error") + } + + box, _, err := h.ReadPayload() + if err != nil { + return nil, err + } + esds := box.(*gomp4.Esds) + + encodedConf := func() []byte { + for _, desc := range esds.Descriptors { + if desc.Tag == gomp4.DecSpecificInfoTag { + return desc.Data + } + } + return nil + }() + if encodedConf == nil { + return nil, fmt.Errorf("unable to find MPEG4-audio configuration") + } + + var c mpeg4audio.Config + err = c.Unmarshal(encodedConf) + if err != nil { + return nil, fmt.Errorf("invalid MPEG4-audio configuration: %s", err) + } + + curTrack.Track = &gortsplib.TrackMPEG4Audio{ + PayloadType: 96, + Config: &c, + SizeLength: 13, + IndexLength: 3, + IndexDeltaLength: 3, + } + state = waitingTrak + + case "ac-3": + return nil, fmt.Errorf("AC-3 codec is not supported (yet)") + } + + return h.Expand() + }) + if err != nil { + return err + } + + if state != waitingTrak { + return fmt.Errorf("parse error") + } + + if i.Tracks == nil { + return fmt.Errorf("no tracks found") + } + + return nil +} + +// Marshal encodes a FMP4 initialization file. +func (i *Init) Marshal() ([]byte, error) { + /* + - ftyp + - moov + - mvhd + - trak + - trak + - ... + - mvex + - trex + - trex + - ... + */ + + w := newMP4Writer() + + _, err := w.WriteBox(&gomp4.Ftyp{ // + MajorBrand: [4]byte{'m', 'p', '4', '2'}, + MinorVersion: 1, + CompatibleBrands: []gomp4.CompatibleBrandElem{ + {CompatibleBrand: [4]byte{'m', 'p', '4', '1'}}, + {CompatibleBrand: [4]byte{'m', 'p', '4', '2'}}, + {CompatibleBrand: [4]byte{'i', 's', 'o', 'm'}}, + {CompatibleBrand: [4]byte{'h', 'l', 's', 'f'}}, + }, + }) + if err != nil { + return nil, err + } + + _, err = w.writeBoxStart(&gomp4.Moov{}) // + if err != nil { + return nil, err + } + + _, err = w.WriteBox(&gomp4.Mvhd{ // + Timescale: 1000, + Rate: 65536, + Volume: 256, + Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000}, + NextTrackID: 4294967295, + }) + if err != nil { + return nil, err + } + + for _, track := range i.Tracks { + err := track.marshal(w) + if err != nil { + return nil, err + } + } + + _, err = w.writeBoxStart(&gomp4.Mvex{}) // + if err != nil { + return nil, err + } + + for _, track := range i.Tracks { + _, err = w.WriteBox(&gomp4.Trex{ // + TrackID: uint32(track.ID), + DefaultSampleDescriptionIndex: 1, + }) + if err != nil { + return nil, err + } + } + + err = w.writeBoxEnd() // + if err != nil { + return nil, err + } + + err = w.writeBoxEnd() // + if err != nil { + return nil, err + } + + return w.bytes(), nil +} diff --git a/internal/hls/fmp4/init_read.go b/internal/hls/fmp4/init_read.go deleted file mode 100644 index bf311c4b..00000000 --- a/internal/hls/fmp4/init_read.go +++ /dev/null @@ -1,104 +0,0 @@ -package fmp4 - -import ( - "bytes" - "fmt" - - gomp4 "github.com/abema/go-mp4" - "github.com/aler9/gortsplib" -) - -type initReadState int - -const ( - waitingTrak initReadState = iota - waitingCodec - waitingAVCC -) - -// InitRead reads a FMP4 initialization file. -func InitRead(byts []byte) (*gortsplib.TrackH264, *gortsplib.TrackMPEG4Audio, error) { - state := waitingTrak - var videoTrack *gortsplib.TrackH264 - var audioTrack *gortsplib.TrackMPEG4Audio - - _, err := gomp4.ReadBoxStructure(bytes.NewReader(byts), func(h *gomp4.ReadHandle) (interface{}, error) { - switch h.BoxInfo.Type.String() { - case "trak": - if state != waitingTrak { - return nil, fmt.Errorf("parse error") - } - state = waitingCodec - - case "avc1": - if state != waitingCodec { - return nil, fmt.Errorf("parse error") - } - - if videoTrack != nil { - return nil, fmt.Errorf("multiple video tracks are not supported") - } - - state = waitingAVCC - - case "avcC": - if state != waitingAVCC { - return nil, fmt.Errorf("parse error") - } - - box, _, err := h.ReadPayload() - if err != nil { - return nil, err - } - conf := box.(*gomp4.AVCDecoderConfiguration) - - if len(conf.SequenceParameterSets) > 1 { - return nil, fmt.Errorf("multiple SPS are not supported") - } - - var sps []byte - if len(conf.SequenceParameterSets) == 1 { - sps = conf.SequenceParameterSets[0].NALUnit - } - - if len(conf.PictureParameterSets) > 1 { - return nil, fmt.Errorf("multiple PPS are not supported") - } - - var pps []byte - if len(conf.PictureParameterSets) == 1 { - pps = conf.PictureParameterSets[0].NALUnit - } - - videoTrack = &gortsplib.TrackH264{ - PayloadType: 96, - SPS: sps, - PPS: pps, - } - - state = waitingTrak - - case "mp4a": - if state != waitingCodec { - return nil, fmt.Errorf("parse error") - } - - if audioTrack != nil { - return nil, fmt.Errorf("multiple audio tracks are not supported") - } - - return nil, fmt.Errorf("TODO: MP4a") - } - - return h.Expand() - }) - if err != nil { - return nil, nil, err - } - - if state != waitingTrak { - return nil, nil, fmt.Errorf("parse error") - } - - return videoTrack, audioTrack, nil -} diff --git a/internal/hls/fmp4/init_test.go b/internal/hls/fmp4/init_test.go new file mode 100644 index 00000000..a13d5041 --- /dev/null +++ b/internal/hls/fmp4/init_test.go @@ -0,0 +1,689 @@ +//nolint:dupl +package fmp4 + +import ( + "testing" + + "github.com/aler9/gortsplib" + "github.com/aler9/gortsplib/pkg/mpeg4audio" + "github.com/stretchr/testify/require" +) + +var testSPS = []byte{ + 0x67, 0x42, 0xc0, 0x28, 0xd9, 0x00, 0x78, 0x02, + 0x27, 0xe5, 0x84, 0x00, 0x00, 0x03, 0x00, 0x04, + 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, 0x60, 0xc9, + 0x20, +} + +var testVideoTrack = &gortsplib.TrackH264{ + PayloadType: 96, + SPS: testSPS, + PPS: []byte{0x08}, +} + +var testAudioTrack = &gortsplib.TrackMPEG4Audio{ + PayloadType: 97, + Config: &mpeg4audio.Config{ + Type: 2, + SampleRate: 44100, + ChannelCount: 2, + }, + SizeLength: 13, + IndexLength: 3, + IndexDeltaLength: 3, +} + +func TestInitMarshal(t *testing.T) { + t.Run("video + audio", func(t *testing.T) { + init := Init{ + Tracks: []*InitTrack{ + { + ID: 1, + TimeScale: 90000, + Track: testVideoTrack, + }, + { + ID: 2, + TimeScale: uint32(testAudioTrack.ClockRate()), + Track: testAudioTrack, + }, + }, + } + + byts, err := init.Marshal() + require.NoError(t, err) + + require.Equal(t, []byte{ + 0x00, 0x00, 0x00, 0x20, + 'f', 't', 'y', 'p', + 0x6d, 0x70, 0x34, 0x32, 0x00, 0x00, 0x00, 0x01, + 0x6d, 0x70, 0x34, 0x31, 0x6d, 0x70, 0x34, 0x32, + 0x69, 0x73, 0x6f, 0x6d, 0x68, 0x6c, 0x73, 0x66, + 0x00, 0x00, 0x04, 0x64, + 'm', 'o', 'o', 'v', + 0x00, 0x00, 0x00, 0x6c, + 'm', 'v', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xe8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x01, 0xec, + 't', 'r', 'a', 'k', + 0x00, 0x00, 0x00, 0x5c, + 't', 'k', 'h', 'd', + 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x07, 0x80, 0x00, 0x00, 0x04, 0x38, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x88, 0x6d, 0x64, 0x69, 0x61, + 0x00, 0x00, 0x00, 0x20, + 'm', 'd', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x5f, 0x90, + 0x00, 0x00, 0x00, 0x00, 0x55, 0xc4, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2d, 0x68, 0x64, 0x6c, 0x72, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x76, 0x69, 0x64, 0x65, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x56, 0x69, 0x64, 0x65, 0x6f, 0x48, 0x61, 0x6e, + 0x64, 0x6c, 0x65, 0x72, 0x00, 0x00, 0x00, 0x01, + 0x33, + 'm', 'i', 'n', 'f', + 0x00, 0x00, 0x00, 0x14, + 'v', 'm', 'h', 'd', + 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x24, 0x64, 0x69, 0x6e, + 0x66, 0x00, 0x00, 0x00, 0x1c, 0x64, 0x72, 0x65, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x0c, 0x75, 0x72, 0x6c, + 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xf3, 0x73, 0x74, 0x62, 0x6c, 0x00, 0x00, 0x00, + 0xa7, 0x73, 0x74, 0x73, 0x64, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x97, 0x61, 0x76, 0x63, 0x31, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x80, 0x04, + 0x38, 0x00, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x18, 0xff, 0xff, 0x00, 0x00, 0x00, 0x2d, 0x61, + 0x76, 0x63, 0x43, 0x01, 0x42, 0xc0, 0x28, 0x03, + 0x01, 0x00, 0x19, 0x67, 0x42, 0xc0, 0x28, 0xd9, + 0x00, 0x78, 0x02, 0x27, 0xe5, 0x84, 0x00, 0x00, + 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, + 0x3c, 0x60, 0xc9, 0x20, 0x01, 0x00, 0x01, 0x08, + 0x00, 0x00, 0x00, 0x14, 0x62, 0x74, 0x72, 0x74, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, + 0x00, 0x0f, 0x42, 0x40, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x74, 0x74, 0x73, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x74, 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, + 0x73, 0x74, 0x73, 0x7a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x73, 0x74, 0x63, 0x6f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0xbc, + 't', 'r', 'a', 'k', + 0x00, 0x00, 0x00, 0x5c, + 't', 'k', 'h', 'd', + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x58, + 0x6d, 0x64, 0x69, 0x61, 0x00, 0x00, 0x00, 0x20, + 'm', 'd', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xac, 0x44, 0x00, 0x00, 0x00, 0x00, + 0x55, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, + 0x68, 0x64, 0x6c, 0x72, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x73, 0x6f, 0x75, 0x6e, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x53, 0x6f, 0x75, 0x6e, + 0x64, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, + 0x00, 0x00, 0x00, 0x01, 0x03, 0x6d, 0x69, 0x6e, + 0x66, 0x00, 0x00, 0x00, 0x10, + 's', 'm', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x24, 0x64, 0x69, 0x6e, + 0x66, 0x00, 0x00, 0x00, 0x1c, 0x64, 0x72, 0x65, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x0c, 0x75, 0x72, 0x6c, + 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xc7, 0x73, 0x74, 0x62, 0x6c, 0x00, 0x00, 0x00, + 0x7b, 0x73, 0x74, 0x73, 0x64, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x6b, 0x6d, 0x70, 0x34, 0x61, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x00, 0xac, 0x44, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x33, 0x65, 0x73, 0x64, + 0x73, 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x80, + 0x80, 0x22, 0x00, 0x02, 0x00, 0x04, 0x80, 0x80, + 0x80, 0x14, 0x40, 0x15, 0x00, 0x00, 0x00, 0x00, + 0x01, 0xf7, 0x39, 0x00, 0x01, 0xf7, 0x39, 0x05, + 0x80, 0x80, 0x80, 0x02, 0x12, 0x10, 0x06, 0x80, + 0x80, 0x80, 0x01, 0x02, 0x00, 0x00, 0x00, 0x14, + 0x62, 0x74, 0x72, 0x74, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0xf7, 0x39, 0x00, 0x01, 0xf7, 0x39, + 0x00, 0x00, 0x00, 0x10, 0x73, 0x74, 0x74, 0x73, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x73, 0x74, 0x73, 0x63, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x14, 0x73, 0x74, 0x73, 0x7a, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x74, 0x63, 0x6f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, + 0x6d, 0x76, 0x65, 0x78, 0x00, 0x00, 0x00, 0x20, + 0x74, 0x72, 0x65, 0x78, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + 0x74, 0x72, 0x65, 0x78, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }, byts) + }) + + t.Run("video only", func(t *testing.T) { + init := Init{ + Tracks: []*InitTrack{ + { + ID: 1, + TimeScale: 90000, + Track: testVideoTrack, + }, + }, + } + + byts, err := init.Marshal() + require.NoError(t, err) + + require.Equal(t, []byte{ + 0x00, 0x00, 0x00, 0x20, + 'f', 't', 'y', 'p', + 0x6d, 0x70, 0x34, 0x32, 0x00, 0x00, 0x00, 0x01, + 0x6d, 0x70, 0x34, 0x31, 0x6d, 0x70, 0x34, 0x32, + 0x69, 0x73, 0x6f, 0x6d, 0x68, 0x6c, 0x73, 0x66, + 0x00, 0x00, 0x02, 0x88, + 'm', 'o', 'o', 'v', + 0x00, 0x00, 0x00, 0x6c, + 'm', 'v', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xe8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x01, 0xec, + 't', 'r', 'a', 'k', + 0x00, 0x00, 0x00, 0x5c, + 't', 'k', 'h', 'd', + 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x07, 0x80, 0x00, 0x00, 0x04, 0x38, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x88, 0x6d, 0x64, 0x69, 0x61, + 0x00, 0x00, 0x00, 0x20, + 'm', 'd', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x5f, 0x90, + 0x00, 0x00, 0x00, 0x00, 0x55, 0xc4, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2d, 0x68, 0x64, 0x6c, 0x72, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x76, 0x69, 0x64, 0x65, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x56, 0x69, 0x64, 0x65, 0x6f, 0x48, 0x61, 0x6e, + 0x64, 0x6c, 0x65, 0x72, 0x00, 0x00, 0x00, 0x01, + 0x33, + 'm', 'i', 'n', 'f', + 0x00, 0x00, 0x00, + 0x14, + 'v', 'm', 'h', 'd', + 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x24, + 'd', 'i', 'n', 'f', + 0x00, 0x00, 0x00, 0x1c, 0x64, 0x72, 0x65, + 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x0c, 0x75, 0x72, 0x6c, + 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xf3, 0x73, 0x74, 0x62, 0x6c, 0x00, 0x00, 0x00, + 0xa7, 0x73, 0x74, 0x73, 0x64, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x97, 0x61, 0x76, 0x63, 0x31, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x80, 0x04, + 0x38, 0x00, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x18, 0xff, 0xff, 0x00, 0x00, 0x00, 0x2d, 0x61, + 0x76, 0x63, 0x43, 0x01, 0x42, 0xc0, 0x28, 0x03, + 0x01, 0x00, 0x19, 0x67, 0x42, 0xc0, 0x28, 0xd9, + 0x00, 0x78, 0x02, 0x27, 0xe5, 0x84, 0x00, 0x00, + 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, + 0x3c, 0x60, 0xc9, 0x20, 0x01, 0x00, 0x01, 0x08, + 0x00, 0x00, 0x00, 0x14, 0x62, 0x74, 0x72, 0x74, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, + 0x00, 0x0f, 0x42, 0x40, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x74, 0x74, 0x73, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x74, 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, + 0x73, 0x74, 0x73, 0x7a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x73, 0x74, 0x63, 0x6f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x28, 0x6d, 0x76, 0x65, 0x78, + 0x00, 0x00, 0x00, 0x20, 0x74, 0x72, 0x65, 0x78, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, byts) + }) + + t.Run("audio only", func(t *testing.T) { + init := &Init{ + Tracks: []*InitTrack{ + { + ID: 1, + TimeScale: uint32(testAudioTrack.ClockRate()), + Track: testAudioTrack, + }, + }, + } + + byts, err := init.Marshal() + require.NoError(t, err) + + require.Equal(t, []byte{ + 0x00, 0x00, 0x00, 0x20, + 'f', 't', 'y', 'p', + 0x6d, 0x70, 0x34, 0x32, 0x00, 0x00, 0x00, 0x01, + 0x6d, 0x70, 0x34, 0x31, 0x6d, 0x70, 0x34, 0x32, + 0x69, 0x73, 0x6f, 0x6d, 0x68, 0x6c, 0x73, 0x66, + 0x00, 0x00, 0x02, 0x58, + 'm', 'o', 'o', 'v', + 0x00, 0x00, 0x00, 0x6c, + 'm', 'v', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xe8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x01, 0xbc, + 't', 'r', 'a', 'k', + 0x00, 0x00, 0x00, 0x5c, + 't', 'k', 'h', 'd', + 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x58, + 'm', 'd', 'i', 'a', + 0x00, 0x00, 0x00, 0x20, + 'm', 'd', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x44, + 0x00, 0x00, 0x00, 0x00, 0x55, 0xc4, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x2d, + 'h', 'd', 'l', 'r', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x6f, 0x75, 0x6e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x6f, 0x75, 0x6e, 0x64, 0x48, 0x61, 0x6e, + 0x64, 0x6c, 0x65, 0x72, 0x00, 0x00, 0x00, 0x01, + 0x03, + 'm', 'i', 'n', 'f', + 0x00, 0x00, 0x00, 0x10, + 's', 'm', 'h', 'd', + 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x24, + 'd', 'i', 'n', 'f', + 0x00, 0x00, 0x00, + 0x1c, 0x64, 0x72, 0x65, 0x66, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0c, 0x75, 0x72, 0x6c, 0x20, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xc7, 0x73, 0x74, 0x62, + 0x6c, 0x00, 0x00, 0x00, 0x7b, 0x73, 0x74, 0x73, + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x6b, + 'm', 'p', '4', 'a', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0xac, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, + 'e', 's', 'd', 's', + 0x00, 0x00, 0x00, + 0x00, 0x03, 0x80, 0x80, 0x80, 0x22, 0x00, 0x01, + 0x00, 0x04, 0x80, 0x80, 0x80, 0x14, 0x40, 0x15, + 0x00, 0x00, 0x00, 0x00, 0x01, 0xf7, 0x39, 0x00, + 0x01, 0xf7, 0x39, 0x05, 0x80, 0x80, 0x80, 0x02, + 0x12, 0x10, 0x06, 0x80, 0x80, 0x80, 0x01, 0x02, + 0x00, 0x00, 0x00, 0x14, + 'b', 't', 'r', 't', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf7, 0x39, + 0x00, 0x01, 0xf7, 0x39, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x74, 0x74, 0x73, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + 0x73, 0x74, 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, + 0x73, 0x74, 0x73, 0x7a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x73, 0x74, 0x63, 0x6f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x28, + 'm', 'v', 'e', 'x', + 0x00, 0x00, 0x00, 0x20, + 't', 'r', 'e', 'x', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, byts) + }) +} + +func TestInitUnmarshal(t *testing.T) { + t.Run("video", func(t *testing.T) { + byts := []byte{ + 0x00, 0x00, 0x00, 0x1c, + 'f', 't', 'y', 'p', + 0x64, 0x61, 0x73, 0x68, 0x00, 0x00, 0x00, 0x01, + 0x69, 0x73, 0x6f, 0x6d, 0x61, 0x76, 0x63, 0x31, + 0x64, 0x61, 0x73, 0x68, 0x00, 0x00, 0x02, 0x92, + 'm', 'o', 'o', 'v', + 0x00, 0x00, 0x00, 0x6c, + 'm', 'v', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x98, 0x96, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x01, 0xf6, + 't', 'r', 'a', 'k', + 0x00, 0x00, 0x00, 0x5c, + 't', 'k', 'h', 'd', + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x03, 0xc0, 0x00, 0x00, + 0x02, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, + 0x6d, 0x64, 0x69, 0x61, 0x00, 0x00, 0x00, 0x20, + 'm', 'd', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x98, 0x96, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x55, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, + 0x68, 0x64, 0x6c, 0x72, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x76, 0x69, 0x64, 0x65, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x42, 0x72, 0x6f, 0x61, + 0x64, 0x70, 0x65, 0x61, 0x6b, 0x20, 0x56, 0x69, + 0x64, 0x65, 0x6f, 0x20, 0x48, 0x61, 0x6e, 0x64, + 0x6c, 0x65, 0x72, 0x00, 0x00, 0x00, 0x01, 0x32, + 'm', 'i', 'n', 'f', + 0x00, 0x00, 0x00, 0x14, + 'v', 'm', 'h', 'd', + 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x24, + 'd', 'i', 'n', 'f', + 0x00, 0x00, 0x00, 0x1c, 0x64, 0x72, 0x65, 0x66, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x0c, 0x75, 0x72, 0x6c, 0x20, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xf2, + 0x73, 0x74, 0x62, 0x6c, 0x00, 0x00, 0x00, 0xa6, + 0x73, 0x74, 0x73, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x96, + 0x61, 0x76, 0x63, 0x31, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0xc0, 0x02, 0x1c, + 0x00, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x68, + 0x32, 0x36, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x30, 0x61, 0x76, + 0x63, 0x43, 0x01, 0x42, 0xc0, 0x1f, 0xff, 0xe1, + 0x00, 0x19, 0x67, 0x42, 0xc0, 0x1f, 0xd9, 0x00, + 0xf0, 0x11, 0x7e, 0xf0, 0x11, 0x00, 0x00, 0x03, + 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, 0x30, 0x8f, + 0x18, 0x32, 0x48, 0x01, 0x00, 0x04, 0x68, 0xcb, + 0x8c, 0xb2, 0x00, 0x00, 0x00, 0x10, 0x70, 0x61, + 0x73, 0x70, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x73, 0x74, + 0x74, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x73, 0x74, + 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x73, 0x74, + 0x73, 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x73, 0x74, 0x63, 0x6f, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x28, 0x6d, 0x76, 0x65, 0x78, 0x00, 0x00, + 0x00, 0x20, 0x74, 0x72, 0x65, 0x78, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + } + + var init Init + err := init.Unmarshal(byts) + require.NoError(t, err) + + require.Equal(t, Init{ + Tracks: []*InitTrack{ + { + ID: 256, + TimeScale: 10000000, + Track: &gortsplib.TrackH264{ + PayloadType: 96, + SPS: []byte{ + 0x67, 0x42, 0xc0, 0x1f, 0xd9, 0x00, 0xf0, 0x11, + 0x7e, 0xf0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, + 0x00, 0x00, 0x03, 0x00, 0x30, 0x8f, 0x18, 0x32, + 0x48, + }, + PPS: []byte{ + 0x68, 0xcb, 0x8c, 0xb2, + }, + }, + }, + }, + }, init) + }) + + t.Run("audio", func(t *testing.T) { + byts := []byte{ + 0x00, 0x00, 0x00, 0x18, + 'f', 't', 'y', 'p', + 0x69, 0x73, 0x6f, 0x35, 0x00, 0x00, 0x00, 0x01, + 0x69, 0x73, 0x6f, 0x35, 0x64, 0x61, 0x73, 0x68, + 0x00, 0x00, 0x02, 0x43, + 'm', 'o', 'o', 'v', + 0x00, 0x00, 0x00, 0x6c, + 'm', 'v', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x96, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x01, 0xa7, + 't', 'r', 'a', 'k', + 0x00, 0x00, 0x00, 0x5c, + 't', 'k', 'h', 'd', + 0x00, 0x00, 0x00, 0x07, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x43, 0x6d, 0x64, 0x69, 0x61, + 0x00, 0x00, 0x00, 0x20, + 'm', 'd', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x96, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x55, 0xc4, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x38, 0x68, 0x64, 0x6c, 0x72, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x6f, 0x75, 0x6e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x42, 0x72, 0x6f, 0x61, 0x64, 0x70, 0x65, 0x61, + 0x6b, 0x20, 0x53, 0x6f, 0x75, 0x6e, 0x64, 0x20, + 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x00, + 0x00, 0x00, 0x00, 0xe3, + 'm', 'i', 'n', 'f', + 0x00, 0x00, 0x00, 0x10, + 's', 'm', 'h', 'd', + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x24, + 'd', 'i', 'n', 'f', + 0x00, 0x00, 0x00, 0x1c, 0x64, 0x72, 0x65, 0x66, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x0c, 0x75, 0x72, 0x6c, 0x20, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xa7, + 0x73, 0x74, 0x62, 0x6c, 0x00, 0x00, 0x00, 0x5b, + 0x73, 0x74, 0x73, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x4b, + 0x6d, 0x70, 0x34, 0x61, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x10, + 0x00, 0x00, 0x00, 0x00, 0xbb, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x27, 0x65, 0x73, 0x64, 0x73, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x19, 0x00, 0x00, + 0x00, 0x04, 0x11, 0x40, 0x15, 0x00, 0x30, 0x00, + 0x00, 0x11, 0x94, 0x00, 0x00, 0x11, 0x94, 0x00, + 0x05, 0x02, 0x11, 0x90, 0x06, 0x01, 0x02, 0x00, + 0x00, 0x00, 0x10, 0x73, 0x74, 0x74, 0x73, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x73, 0x74, 0x73, 0x63, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x14, 0x73, 0x74, 0x73, 0x7a, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x73, + 0x74, 0x63, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x6d, + 0x76, 0x65, 0x78, 0x00, 0x00, 0x00, 0x20, 0x74, + 0x72, 0x65, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, + } + + var init Init + err := init.Unmarshal(byts) + require.NoError(t, err) + + require.Equal(t, Init{ + Tracks: []*InitTrack{ + { + ID: 257, + TimeScale: 10000000, + Track: &gortsplib.TrackMPEG4Audio{ + PayloadType: 96, + Config: &mpeg4audio.Config{ + Type: mpeg4audio.ObjectTypeAACLC, + SampleRate: 48000, + ChannelCount: 2, + }, + SizeLength: 13, + IndexLength: 3, + IndexDeltaLength: 3, + }, + }, + }, + }, init) + }) +} diff --git a/internal/hls/fmp4/init_track.go b/internal/hls/fmp4/init_track.go new file mode 100644 index 00000000..fff89452 --- /dev/null +++ b/internal/hls/fmp4/init_track.go @@ -0,0 +1,382 @@ +package fmp4 + +import ( + gomp4 "github.com/abema/go-mp4" + "github.com/aler9/gortsplib" + + "github.com/aler9/gortsplib/pkg/h264" +) + +// InitTrack is a track of Init. +type InitTrack struct { + ID int + TimeScale uint32 + Track gortsplib.Track +} + +func (track *InitTrack) marshal(w *mp4Writer) error { + /* + trak + - tkhd + - mdia + - mdhd + - hdlr + - minf + - vmhd (video only) + - smhd (audio only) + - dinf + - dref + - url + - stbl + - stsd + - avc1 (h264 only) + - avcC + - pasp + - btrt + - mp4a (mpeg4audio only) + - esds + - btrt + - stts + - stsc + - stsz + - stco + */ + + _, err := w.writeBoxStart(&gomp4.Trak{}) // + if err != nil { + return err + } + + var sps []byte + var pps []byte + var spsp h264.SPS + var width int + var height int + + switch ttrack := track.Track.(type) { + case *gortsplib.TrackH264: + sps = ttrack.SafeSPS() + pps = ttrack.SafePPS() + + err = spsp.Unmarshal(sps) + if err != nil { + return err + } + + width = spsp.Width() + height = spsp.Height() + + _, err = w.WriteBox(&gomp4.Tkhd{ // + FullBox: gomp4.FullBox{ + Flags: [3]byte{0, 0, 3}, + }, + TrackID: uint32(track.ID), + Width: uint32(width * 65536), + Height: uint32(height * 65536), + Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000}, + }) + if err != nil { + return err + } + + case *gortsplib.TrackMPEG4Audio: + _, err = w.WriteBox(&gomp4.Tkhd{ // + FullBox: gomp4.FullBox{ + Flags: [3]byte{0, 0, 3}, + }, + TrackID: uint32(track.ID), + AlternateGroup: 1, + Volume: 256, + Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000}, + }) + if err != nil { + return err + } + } + + _, err = w.writeBoxStart(&gomp4.Mdia{}) // + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Mdhd{ // + Timescale: track.TimeScale, + Language: [3]byte{'u', 'n', 'd'}, + }) + if err != nil { + return err + } + + switch track.Track.(type) { + case *gortsplib.TrackH264: + _, err = w.WriteBox(&gomp4.Hdlr{ // + HandlerType: [4]byte{'v', 'i', 'd', 'e'}, + Name: "VideoHandler", + }) + if err != nil { + return err + } + + case *gortsplib.TrackMPEG4Audio: + _, err = w.WriteBox(&gomp4.Hdlr{ // + HandlerType: [4]byte{'s', 'o', 'u', 'n'}, + Name: "SoundHandler", + }) + if err != nil { + return err + } + } + + _, err = w.writeBoxStart(&gomp4.Minf{}) // + if err != nil { + return err + } + + switch track.Track.(type) { + case *gortsplib.TrackH264: + _, err = w.WriteBox(&gomp4.Vmhd{ // + FullBox: gomp4.FullBox{ + Flags: [3]byte{0, 0, 1}, + }, + }) + if err != nil { + return err + } + + case *gortsplib.TrackMPEG4Audio: + _, err = w.WriteBox(&gomp4.Smhd{ // + }) + if err != nil { + return err + } + } + + _, err = w.writeBoxStart(&gomp4.Dinf{}) // + if err != nil { + return err + } + + _, err = w.writeBoxStart(&gomp4.Dref{ // + EntryCount: 1, + }) + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Url{ // + FullBox: gomp4.FullBox{ + Flags: [3]byte{0, 0, 1}, + }, + }) + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + _, err = w.writeBoxStart(&gomp4.Stbl{}) // + if err != nil { + return err + } + + _, err = w.writeBoxStart(&gomp4.Stsd{ // + EntryCount: 1, + }) + if err != nil { + return err + } + + switch ttrack := track.Track.(type) { + case *gortsplib.TrackH264: + _, err = w.writeBoxStart(&gomp4.VisualSampleEntry{ // + SampleEntry: gomp4.SampleEntry{ + AnyTypeBox: gomp4.AnyTypeBox{ + Type: gomp4.BoxTypeAvc1(), + }, + DataReferenceIndex: 1, + }, + Width: uint16(width), + Height: uint16(height), + Horizresolution: 4718592, + Vertresolution: 4718592, + FrameCount: 1, + Depth: 24, + PreDefined3: -1, + }) + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.AVCDecoderConfiguration{ // + AnyTypeBox: gomp4.AnyTypeBox{ + Type: gomp4.BoxTypeAvcC(), + }, + ConfigurationVersion: 1, + Profile: spsp.ProfileIdc, + ProfileCompatibility: sps[2], + Level: spsp.LevelIdc, + LengthSizeMinusOne: 3, + NumOfSequenceParameterSets: 1, + SequenceParameterSets: []gomp4.AVCParameterSet{ + { + Length: uint16(len(sps)), + NALUnit: sps, + }, + }, + NumOfPictureParameterSets: 1, + PictureParameterSets: []gomp4.AVCParameterSet{ + { + Length: uint16(len(pps)), + NALUnit: pps, + }, + }, + }) + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Btrt{ // + MaxBitrate: 1000000, + AvgBitrate: 1000000, + }) + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + case *gortsplib.TrackMPEG4Audio: + _, err = w.writeBoxStart(&gomp4.AudioSampleEntry{ // + SampleEntry: gomp4.SampleEntry{ + AnyTypeBox: gomp4.AnyTypeBox{ + Type: gomp4.BoxTypeMp4a(), + }, + DataReferenceIndex: 1, + }, + ChannelCount: uint16(ttrack.Config.ChannelCount), + SampleSize: 16, + SampleRate: uint32(ttrack.ClockRate() * 65536), + }) + if err != nil { + return err + } + + enc, _ := ttrack.Config.Marshal() + + _, err = w.WriteBox(&gomp4.Esds{ // + FullBox: gomp4.FullBox{ + Version: 0, + Flags: [3]byte{0x00, 0x00, 0x00}, + }, + Descriptors: []gomp4.Descriptor{ + { + Tag: gomp4.ESDescrTag, + Size: 32 + uint32(len(enc)), + ESDescriptor: &gomp4.ESDescriptor{ + ESID: uint16(track.ID), + }, + }, + { + Tag: gomp4.DecoderConfigDescrTag, + Size: 18 + uint32(len(enc)), + DecoderConfigDescriptor: &gomp4.DecoderConfigDescriptor{ + ObjectTypeIndication: 0x40, + StreamType: 0x05, + UpStream: false, + Reserved: true, + MaxBitrate: 128825, + AvgBitrate: 128825, + }, + }, + { + Tag: gomp4.DecSpecificInfoTag, + Size: uint32(len(enc)), + Data: enc, + }, + { + Tag: gomp4.SLConfigDescrTag, + Size: 1, + Data: []byte{0x02}, + }, + }, + }) + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Btrt{ // + MaxBitrate: 128825, + AvgBitrate: 128825, + }) + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Stts{ // + }) + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Stsc{ // + }) + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Stsz{ // + }) + if err != nil { + return err + } + + _, err = w.WriteBox(&gomp4.Stco{ // + }) + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + err = w.writeBoxEnd() // + if err != nil { + return err + } + + return nil +} diff --git a/internal/hls/fmp4/init_write.go b/internal/hls/fmp4/init_write.go deleted file mode 100644 index b574305e..00000000 --- a/internal/hls/fmp4/init_write.go +++ /dev/null @@ -1,618 +0,0 @@ -package fmp4 - -import ( - gomp4 "github.com/abema/go-mp4" - "github.com/aler9/gortsplib" - "github.com/aler9/gortsplib/pkg/h264" -) - -type myEsds struct { - gomp4.FullBox `mp4:"0,extend"` - Data []byte `mp4:"1,size=8"` -} - -func (*myEsds) GetType() gomp4.BoxType { - return gomp4.StrToBoxType("esds") -} - -func init() { //nolint:gochecknoinits - gomp4.AddBoxDef(&myEsds{}, 0) -} - -func initWriteVideoTrack(w *mp4Writer, trackID int, videoTrack *gortsplib.TrackH264) error { - /* - trak - - tkhd - - mdia - - mdhd - - hdlr - - minf - - vmhd - - dinf - - dref - - url - - stbl - - stsd - - avc1 - - avcC - - pasp - - btrt - - stts - - stsc - - stsz - - stco - */ - - _, err := w.writeBoxStart(&gomp4.Trak{}) // - if err != nil { - return err - } - - sps := videoTrack.SafeSPS() - pps := videoTrack.SafePPS() - - var spsp h264.SPS - err = spsp.Unmarshal(sps) - if err != nil { - return err - } - - width := spsp.Width() - height := spsp.Height() - - _, err = w.WriteBox(&gomp4.Tkhd{ // - FullBox: gomp4.FullBox{ - Flags: [3]byte{0, 0, 3}, - }, - TrackID: uint32(trackID), - Width: uint32(width * 65536), - Height: uint32(height * 65536), - Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000}, - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Mdia{}) // - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Mdhd{ // - Timescale: videoTimescale, // the number of time units that pass per second - Language: [3]byte{'u', 'n', 'd'}, - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Hdlr{ // - HandlerType: [4]byte{'v', 'i', 'd', 'e'}, - Name: "VideoHandler", - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Minf{}) // - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Vmhd{ // - FullBox: gomp4.FullBox{ - Flags: [3]byte{0, 0, 1}, - }, - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Dinf{}) // - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Dref{ // - EntryCount: 1, - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Url{ // - FullBox: gomp4.FullBox{ - Flags: [3]byte{0, 0, 1}, - }, - }) - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Stbl{}) // - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Stsd{ // - EntryCount: 1, - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.VisualSampleEntry{ // - SampleEntry: gomp4.SampleEntry{ - AnyTypeBox: gomp4.AnyTypeBox{ - Type: gomp4.BoxTypeAvc1(), - }, - DataReferenceIndex: 1, - }, - Width: uint16(width), - Height: uint16(height), - Horizresolution: 4718592, - Vertresolution: 4718592, - FrameCount: 1, - Depth: 24, - PreDefined3: -1, - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.AVCDecoderConfiguration{ // - AnyTypeBox: gomp4.AnyTypeBox{ - Type: gomp4.BoxTypeAvcC(), - }, - ConfigurationVersion: 1, - Profile: spsp.ProfileIdc, - ProfileCompatibility: sps[2], - Level: spsp.LevelIdc, - LengthSizeMinusOne: 3, - NumOfSequenceParameterSets: 1, - SequenceParameterSets: []gomp4.AVCParameterSet{ - { - Length: uint16(len(sps)), - NALUnit: sps, - }, - }, - NumOfPictureParameterSets: 1, - PictureParameterSets: []gomp4.AVCParameterSet{ - { - Length: uint16(len(pps)), - NALUnit: pps, - }, - }, - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Btrt{ // - MaxBitrate: 1000000, - AvgBitrate: 1000000, - }) - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stts{ // - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stsc{ // - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stsz{ // - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stco{ // - }) - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - return nil -} - -func initWriteAudioTrack(w *mp4Writer, trackID int, audioTrack *gortsplib.TrackMPEG4Audio) error { - /* - trak - - tkhd - - mdia - - mdhd - - hdlr - - minf - - smhd - - dinf - - dref - - url - - stbl - - stsd - - mp4a - - esds - - btrt - - stts - - stsc - - stsz - - stco - */ - - _, err := w.writeBoxStart(&gomp4.Trak{}) // - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Tkhd{ // - FullBox: gomp4.FullBox{ - Flags: [3]byte{0, 0, 3}, - }, - TrackID: uint32(trackID), - AlternateGroup: 1, - Volume: 256, - Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000}, - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Mdia{}) // - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Mdhd{ // - Timescale: uint32(audioTrack.ClockRate()), - Language: [3]byte{'u', 'n', 'd'}, - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Hdlr{ // - HandlerType: [4]byte{'s', 'o', 'u', 'n'}, - Name: "SoundHandler", - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Minf{}) // - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Smhd{ // - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Dinf{}) // - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Dref{ // - EntryCount: 1, - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Url{ // - FullBox: gomp4.FullBox{ - Flags: [3]byte{0, 0, 1}, - }, - }) - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Stbl{}) // - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.Stsd{ // - EntryCount: 1, - }) - if err != nil { - return err - } - - _, err = w.writeBoxStart(&gomp4.AudioSampleEntry{ // - SampleEntry: gomp4.SampleEntry{ - AnyTypeBox: gomp4.AnyTypeBox{ - Type: gomp4.BoxTypeMp4a(), - }, - DataReferenceIndex: 1, - }, - ChannelCount: uint16(audioTrack.Config.ChannelCount), - SampleSize: 16, - SampleRate: uint32(audioTrack.ClockRate() * 65536), - }) - if err != nil { - return err - } - - enc, _ := audioTrack.Config.Marshal() - - decSpecificInfoTagSize := uint8(len(enc)) - decSpecificInfoTag := append( - []byte{ - gomp4.DecSpecificInfoTag, - 0x80, 0x80, 0x80, decSpecificInfoTagSize, // size - }, - enc..., - ) - - esDescrTag := []byte{ - gomp4.ESDescrTag, - 0x80, 0x80, 0x80, 32 + decSpecificInfoTagSize, // size - 0x00, - byte(trackID), // ES_ID - 0x00, - } - - decoderConfigDescrTag := []byte{ - gomp4.DecoderConfigDescrTag, - 0x80, 0x80, 0x80, 18 + decSpecificInfoTagSize, // size - 0x40, // object type indicator (MPEG-4 Audio) - 0x15, 0x00, - 0x00, 0x00, 0x00, 0x01, - 0xf7, 0x39, 0x00, 0x01, - 0xf7, 0x39, - } - - slConfigDescrTag := []byte{ - gomp4.SLConfigDescrTag, - 0x80, 0x80, 0x80, 0x01, // size (1) - 0x02, - } - - data := make([]byte, len(esDescrTag)+len(decoderConfigDescrTag)+len(decSpecificInfoTag)+len(slConfigDescrTag)) - pos := 0 - - pos += copy(data[pos:], esDescrTag) - pos += copy(data[pos:], decoderConfigDescrTag) - pos += copy(data[pos:], decSpecificInfoTag) - copy(data[pos:], slConfigDescrTag) - - _, err = w.WriteBox(&myEsds{ // - Data: data, - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Btrt{ // - MaxBitrate: 128825, - AvgBitrate: 128825, - }) - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stts{ // - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stsc{ // - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stsz{ // - }) - if err != nil { - return err - } - - _, err = w.WriteBox(&gomp4.Stco{ // - }) - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - err = w.writeBoxEnd() // - if err != nil { - return err - } - - return nil -} - -// InitWrite generates a FMP4 initialization file. -func InitWrite( - videoTrack *gortsplib.TrackH264, - audioTrack *gortsplib.TrackMPEG4Audio, -) ([]byte, error) { - /* - - ftyp - - moov - - mvhd - - trak (video) - - trak (audio) - - mvex - - trex (video) - - trex (audio) - */ - - w := newMP4Writer() - - _, err := w.WriteBox(&gomp4.Ftyp{ // - MajorBrand: [4]byte{'m', 'p', '4', '2'}, - MinorVersion: 1, - CompatibleBrands: []gomp4.CompatibleBrandElem{ - {CompatibleBrand: [4]byte{'m', 'p', '4', '1'}}, - {CompatibleBrand: [4]byte{'m', 'p', '4', '2'}}, - {CompatibleBrand: [4]byte{'i', 's', 'o', 'm'}}, - {CompatibleBrand: [4]byte{'h', 'l', 's', 'f'}}, - }, - }) - if err != nil { - return nil, err - } - - _, err = w.writeBoxStart(&gomp4.Moov{}) // - if err != nil { - return nil, err - } - - _, err = w.WriteBox(&gomp4.Mvhd{ // - Timescale: 1000, - Rate: 65536, - Volume: 256, - Matrix: [9]int32{0x00010000, 0, 0, 0, 0x00010000, 0, 0, 0, 0x40000000}, - NextTrackID: 2, - }) - if err != nil { - return nil, err - } - - trackID := 1 - - if videoTrack != nil { - err := initWriteVideoTrack(w, trackID, videoTrack) - if err != nil { - return nil, err - } - - trackID++ - } - - if audioTrack != nil { - err := initWriteAudioTrack(w, trackID, audioTrack) - if err != nil { - return nil, err - } - } - - _, err = w.writeBoxStart(&gomp4.Mvex{}) // - if err != nil { - return nil, err - } - - trackID = 1 - - if videoTrack != nil { - _, err = w.WriteBox(&gomp4.Trex{ // - TrackID: uint32(trackID), - DefaultSampleDescriptionIndex: 1, - }) - if err != nil { - return nil, err - } - - trackID++ - } - - if audioTrack != nil { - _, err = w.WriteBox(&gomp4.Trex{ // - TrackID: uint32(trackID), - DefaultSampleDescriptionIndex: 1, - }) - if err != nil { - return nil, err - } - } - - err = w.writeBoxEnd() // - if err != nil { - return nil, err - } - - err = w.writeBoxEnd() // - if err != nil { - return nil, err - } - - return w.bytes(), nil -} diff --git a/internal/hls/fmp4/init_write_test.go b/internal/hls/fmp4/init_write_test.go deleted file mode 100644 index d362c15e..00000000 --- a/internal/hls/fmp4/init_write_test.go +++ /dev/null @@ -1,318 +0,0 @@ -//nolint:dupl -package fmp4 - -import ( - "bytes" - "testing" - - gomp4 "github.com/abema/go-mp4" - "github.com/aler9/gortsplib" - "github.com/aler9/gortsplib/pkg/mpeg4audio" - "github.com/stretchr/testify/require" -) - -func testMP4(t *testing.T, byts []byte, boxes []gomp4.BoxPath) { - i := 0 - _, err := gomp4.ReadBoxStructure(bytes.NewReader(byts), func(h *gomp4.ReadHandle) (interface{}, error) { - require.Equal(t, boxes[i], h.Path) - i++ - return h.Expand() - }) - require.NoError(t, err) -} - -var testSPS = []byte{ - 0x67, 0x42, 0xc0, 0x28, 0xd9, 0x00, 0x78, 0x02, - 0x27, 0xe5, 0x84, 0x00, 0x00, 0x03, 0x00, 0x04, - 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, 0x60, 0xc9, - 0x20, -} - -var testVideoTrack = &gortsplib.TrackH264{ - PayloadType: 96, - SPS: testSPS, - PPS: []byte{0x08}, -} - -var testAudioTrack = &gortsplib.TrackMPEG4Audio{ - PayloadType: 97, - Config: &mpeg4audio.Config{ - Type: 2, - SampleRate: 44100, - ChannelCount: 2, - }, - SizeLength: 13, - IndexLength: 3, - IndexDeltaLength: 3, -} - -func TestInitWrite(t *testing.T) { - t.Run("video + audio", func(t *testing.T) { - byts, err := InitWrite(testVideoTrack, testAudioTrack) - require.NoError(t, err) - - boxes := []gomp4.BoxPath{ - {gomp4.BoxTypeFtyp()}, - {gomp4.BoxTypeMoov()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeTkhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMdhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeHdlr()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), gomp4.BoxTypeVmhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), gomp4.BoxTypeDinf()}, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), gomp4.BoxTypeUrl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeAvc1(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeAvc1(), gomp4.BoxTypeAvcC(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeAvc1(), gomp4.BoxTypeBtrt(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStts(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsc(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsz(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStco(), - }, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeTkhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMdhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeHdlr()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf()}, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeSmhd(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), gomp4.BoxTypeUrl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeMp4a(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeMp4a(), gomp4.BoxTypeEsds(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeMp4a(), gomp4.BoxTypeBtrt(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStts(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsc(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsz(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStco(), - }, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvex()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvex(), gomp4.BoxTypeTrex()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvex(), gomp4.BoxTypeTrex()}, - } - testMP4(t, byts, boxes) - }) - - t.Run("video only", func(t *testing.T) { - byts, err := InitWrite(testVideoTrack, nil) - require.NoError(t, err) - - boxes := []gomp4.BoxPath{ - {gomp4.BoxTypeFtyp()}, - {gomp4.BoxTypeMoov()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeTkhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMdhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeHdlr()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf()}, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeVmhd(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), gomp4.BoxTypeUrl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeAvc1(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeAvc1(), gomp4.BoxTypeAvcC(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeAvc1(), gomp4.BoxTypeBtrt(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStts(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsc(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsz(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStco(), - }, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvex()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvex(), gomp4.BoxTypeTrex()}, - } - testMP4(t, byts, boxes) - }) - - t.Run("audio only", func(t *testing.T) { - byts, err := InitWrite(nil, testAudioTrack) - require.NoError(t, err) - - boxes := []gomp4.BoxPath{ - {gomp4.BoxTypeFtyp()}, - {gomp4.BoxTypeMoov()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeTkhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMdhd()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeHdlr()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf()}, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeSmhd(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeDinf(), gomp4.BoxTypeDref(), gomp4.BoxTypeUrl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeMp4a(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeMp4a(), gomp4.BoxTypeEsds(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsd(), gomp4.BoxTypeMp4a(), gomp4.BoxTypeBtrt(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStts(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsc(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStsz(), - }, - { - gomp4.BoxTypeMoov(), gomp4.BoxTypeTrak(), gomp4.BoxTypeMdia(), gomp4.BoxTypeMinf(), - gomp4.BoxTypeStbl(), gomp4.BoxTypeStco(), - }, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvex()}, - {gomp4.BoxTypeMoov(), gomp4.BoxTypeMvex(), gomp4.BoxTypeTrex()}, - } - testMP4(t, byts, boxes) - }) -} diff --git a/internal/hls/fmp4/part.go b/internal/hls/fmp4/part.go new file mode 100644 index 00000000..ac727eff --- /dev/null +++ b/internal/hls/fmp4/part.go @@ -0,0 +1,259 @@ +package fmp4 + +import ( + "bytes" + "fmt" + + gomp4 "github.com/abema/go-mp4" +) + +const ( + trunFlagDataOffsetPreset = 0x01 + trunFlagSampleDurationPresent = 0x100 + trunFlagSampleSizePresent = 0x200 + trunFlagSampleFlagsPresent = 0x400 + trunFlagSampleCompositionTimeOffsetPresentOrV1 = 0x800 +) + +// Part is a FMP4 part file. +type Part struct { + Tracks []*PartTrack +} + +// Parts is a sequence of FMP4 parts. +type Parts []*Part + +// Unmarshal decodes one or more FMP4 parts. +func (ps *Parts) Unmarshal(byts []byte) error { + type readState int + + const ( + waitingMoof readState = iota + waitingTraf + waitingTfdtTfhdTrun + ) + + state := waitingMoof + var curPart *Part + var moofOffset uint64 + var curTrack *PartTrack + var tfdt *gomp4.Tfdt + var tfhd *gomp4.Tfhd + + _, err := gomp4.ReadBoxStructure(bytes.NewReader(byts), func(h *gomp4.ReadHandle) (interface{}, error) { + switch h.BoxInfo.Type.String() { + case "moof": + if state != waitingMoof { + return nil, fmt.Errorf("unexpected moof") + } + + curPart = &Part{} + *ps = append(*ps, curPart) + moofOffset = h.BoxInfo.Offset + state = waitingTraf + + case "traf": + if state != waitingTraf && state != waitingTfdtTfhdTrun { + return nil, fmt.Errorf("unexpected traf") + } + + if curTrack != nil { + if tfdt == nil || tfhd == nil || curTrack.Samples == nil { + return nil, fmt.Errorf("parse error") + } + } + + curTrack = &PartTrack{} + curPart.Tracks = append(curPart.Tracks, curTrack) + tfdt = nil + tfhd = nil + state = waitingTfdtTfhdTrun + + case "tfhd": + if state != waitingTfdtTfhdTrun || tfhd != nil { + return nil, fmt.Errorf("unexpected tfhd") + } + + box, _, err := h.ReadPayload() + if err != nil { + return nil, err + } + + tfhd = box.(*gomp4.Tfhd) + curTrack.ID = int(tfhd.TrackID) + + case "tfdt": + if state != waitingTfdtTfhdTrun || tfdt != nil { + return nil, fmt.Errorf("unexpected tfdt") + } + + box, _, err := h.ReadPayload() + if err != nil { + return nil, err + } + + tfdt = box.(*gomp4.Tfdt) + + if tfdt.FullBox.Version != 1 { + return nil, fmt.Errorf("unsupported tfdt version") + } + + curTrack.BaseTime = tfdt.BaseMediaDecodeTimeV1 + + case "trun": + if state != waitingTfdtTfhdTrun || tfhd == nil { + return nil, fmt.Errorf("unexpected trun") + } + + box, _, err := h.ReadPayload() + if err != nil { + return nil, err + } + trun := box.(*gomp4.Trun) + + flags := uint16(trun.Flags[1])<<8 | uint16(trun.Flags[2]) + if (flags & trunFlagDataOffsetPreset) == 0 { + return nil, fmt.Errorf("unsupported flags") + } + + existing := len(curTrack.Samples) + tmp := make([]*PartSample, existing+len(trun.Entries)) + copy(tmp, curTrack.Samples) + curTrack.Samples = tmp + + ptr := byts[uint64(trun.DataOffset)+moofOffset:] + + for i, e := range trun.Entries { + s := &PartSample{} + + if (flags & trunFlagSampleDurationPresent) != 0 { + s.Duration = e.SampleDuration + } else { + s.Duration = tfhd.DefaultSampleDuration + } + + s.PTSOffset = e.SampleCompositionTimeOffsetV1 + + if (flags & trunFlagSampleFlagsPresent) != 0 { + s.Flags = e.SampleFlags + } else { + s.Flags = tfhd.DefaultSampleFlags + } + + var size uint32 + if (flags & trunFlagSampleSizePresent) != 0 { + size = e.SampleSize + } else { + size = tfhd.DefaultSampleSize + } + + s.Payload = ptr[:size] + ptr = ptr[size:] + + curTrack.Samples[existing+i] = s + } + + case "mdat": + if state != waitingTraf && state != waitingTfdtTfhdTrun { + return nil, fmt.Errorf("unexpected mdat") + } + + if curTrack != nil { + if tfdt == nil || tfhd == nil || curTrack.Samples == nil { + return nil, fmt.Errorf("parse error") + } + } + + state = waitingMoof + return nil, nil + } + + return h.Expand() + }) + if err != nil { + return err + } + + if state != waitingMoof { + return fmt.Errorf("decode error") + } + + return nil +} + +// Marshal encodes a FMP4 part file. +func (p *Part) Marshal() ([]byte, error) { + /* + moof + - mfhd + - traf (video) + - traf (audio) + mdat + */ + + w := newMP4Writer() + + moofOffset, err := w.writeBoxStart(&gomp4.Moof{}) // + if err != nil { + return nil, err + } + + _, err = w.WriteBox(&gomp4.Mfhd{ // + SequenceNumber: 0, + }) + if err != nil { + return nil, err + } + + trackLen := len(p.Tracks) + truns := make([]*gomp4.Trun, trackLen) + trunOffsets := make([]int, trackLen) + dataOffsets := make([]int, trackLen) + dataSize := 0 + + for i, track := range p.Tracks { + trun, trunOffset, err := track.marshal(w) + if err != nil { + return nil, err + } + + dataOffsets[i] = dataSize + + for _, sample := range track.Samples { + dataSize += len(sample.Payload) + } + + truns[i] = trun + trunOffsets[i] = trunOffset + } + + err = w.writeBoxEnd() // + if err != nil { + return nil, err + } + + mdat := &gomp4.Mdat{} // + mdat.Data = make([]byte, dataSize) + pos := 0 + + for _, track := range p.Tracks { + for _, sample := range track.Samples { + pos += copy(mdat.Data[pos:], sample.Payload) + } + } + + mdatOffset, err := w.WriteBox(mdat) + if err != nil { + return nil, err + } + + for i := range p.Tracks { + truns[i].DataOffset = int32(dataOffsets[i] + mdatOffset - moofOffset + 8) + err = w.rewriteBox(trunOffsets[i], truns[i]) + if err != nil { + return nil, err + } + } + + return w.bytes(), nil +} diff --git a/internal/hls/fmp4/part_read.go b/internal/hls/fmp4/part_read.go deleted file mode 100644 index 91a4dda0..00000000 --- a/internal/hls/fmp4/part_read.go +++ /dev/null @@ -1,96 +0,0 @@ -package fmp4 - -import ( - "bytes" - "fmt" - - gomp4 "github.com/abema/go-mp4" -) - -type partReadState int - -const ( - waitingTraf partReadState = iota - waitingTfhd - waitingTfdt - waitingTrun -) - -// PartRead reads a FMP4 part file. -func PartRead( - byts []byte, - cb func(), -) error { - state := waitingTraf - var trackID uint32 - var baseTime uint64 - var entries []gomp4.TrunEntry - - _, err := gomp4.ReadBoxStructure(bytes.NewReader(byts), func(h *gomp4.ReadHandle) (interface{}, error) { - switch h.BoxInfo.Type.String() { - case "traf": - if state != waitingTraf { - return nil, fmt.Errorf("decode error") - } - state = waitingTfhd - - case "tfhd": - if state != waitingTfhd { - return nil, fmt.Errorf("decode error") - } - - box, _, err := h.ReadPayload() - if err != nil { - return nil, err - } - trackID = box.(*gomp4.Tfhd).TrackID - - state = waitingTfdt - - case "tfdt": - if state != waitingTfdt { - return nil, fmt.Errorf("decode error") - } - - box, _, err := h.ReadPayload() - if err != nil { - return nil, err - } - t := box.(*gomp4.Tfdt) - - if t.FullBox.Version != 1 { - return nil, fmt.Errorf("unsupported tfdt version") - } - - baseTime = t.BaseMediaDecodeTimeV1 - state = waitingTrun - - case "trun": - if state != waitingTrun { - return nil, fmt.Errorf("decode error") - } - - box, _, err := h.ReadPayload() - if err != nil { - return nil, err - } - t := box.(*gomp4.Trun) - - entries = t.Entries - state = waitingTraf - } - - return h.Expand() - }) - if err != nil { - return err - } - - if state != waitingTraf { - return fmt.Errorf("parse error") - } - - fmt.Println("TODO", trackID, baseTime, entries) - - return nil -} diff --git a/internal/hls/fmp4/part_test.go b/internal/hls/fmp4/part_test.go new file mode 100644 index 00000000..ed0d36f5 --- /dev/null +++ b/internal/hls/fmp4/part_test.go @@ -0,0 +1,249 @@ +package fmp4 + +import ( + "bytes" + "testing" + + gomp4 "github.com/abema/go-mp4" + "github.com/stretchr/testify/require" +) + +func testMP4(t *testing.T, byts []byte, boxes []gomp4.BoxPath) { + i := 0 + _, err := gomp4.ReadBoxStructure(bytes.NewReader(byts), func(h *gomp4.ReadHandle) (interface{}, error) { + require.Equal(t, boxes[i], h.Path) + i++ + return h.Expand() + }) + require.NoError(t, err) +} + +func TestPartMarshal(t *testing.T) { + testVideoSamples := []*PartSample{ + { + Duration: 2 * 90000, + Payload: []byte{ + 0x00, 0x00, 0x00, 0x04, + 0x01, 0x02, 0x03, 0x04, // SPS + 0x00, 0x00, 0x00, 0x01, + 0x08, // PPS + 0x00, 0x00, 0x00, 0x01, + 0x05, // IDR + }, + }, + { + Duration: 2 * 90000, + Payload: []byte{ + 0x00, 0x00, 0x00, 0x01, + 0x01, // non-IDR + }, + Flags: 1 << 16, + }, + { + Duration: 1 * 90000, + Payload: []byte{ + 0x00, 0x00, 0x00, 0x01, + 0x01, // non-IDR + }, + Flags: 1 << 16, + }, + } + + testAudioSamples := []*PartSample{ + { + Duration: 500 * 48000 / 1000, + Payload: []byte{ + 0x01, 0x02, 0x03, 0x04, + }, + }, + { + Duration: 1 * 48000, + Payload: []byte{ + 0x01, 0x02, 0x03, 0x04, + }, + }, + } + + t.Run("video + audio", func(t *testing.T) { + part := Part{ + Tracks: []*PartTrack{ + { + ID: 1, + Samples: testVideoSamples, + IsVideo: true, + }, + { + ID: 2, + BaseTime: 3 * 48000, + Samples: testAudioSamples, + }, + }, + } + + byts, err := part.Marshal() + require.NoError(t, err) + + boxes := []gomp4.BoxPath{ + {gomp4.BoxTypeMoof()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeMfhd()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, + {gomp4.BoxTypeMdat()}, + } + testMP4(t, byts, boxes) + }) + + t.Run("video only", func(t *testing.T) { + part := Part{ + Tracks: []*PartTrack{ + { + ID: 1, + Samples: testVideoSamples, + IsVideo: true, + }, + }, + } + + byts, err := part.Marshal() + require.NoError(t, err) + + boxes := []gomp4.BoxPath{ + {gomp4.BoxTypeMoof()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeMfhd()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, + {gomp4.BoxTypeMdat()}, + } + testMP4(t, byts, boxes) + }) + + t.Run("audio only", func(t *testing.T) { + part := Part{ + Tracks: []*PartTrack{ + { + ID: 1, + Samples: testAudioSamples, + }, + }, + } + + byts, err := part.Marshal() + require.NoError(t, err) + + boxes := []gomp4.BoxPath{ + {gomp4.BoxTypeMoof()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeMfhd()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, + {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, + {gomp4.BoxTypeMdat()}, + } + testMP4(t, byts, boxes) + }) +} + +func TestPartUnmarshal(t *testing.T) { + byts := []byte{ + 0x00, 0x00, 0x00, 0xd8, 0x6d, 0x6f, 0x6f, 0x66, + 0x00, 0x00, 0x00, 0x10, 0x6d, 0x66, 0x68, 0x64, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x70, 0x74, 0x72, 0x61, 0x66, + 0x00, 0x00, 0x00, 0x10, 0x74, 0x66, 0x68, 0x64, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x14, 0x74, 0x66, 0x64, 0x74, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, + 0x74, 0x72, 0x75, 0x6e, 0x01, 0x00, 0x0f, 0x01, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xe0, + 0x00, 0x02, 0xbf, 0x20, 0x00, 0x00, 0x00, 0x12, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0xbf, 0x20, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x5f, 0x90, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x50, 0x74, 0x72, 0x61, 0x66, + 0x00, 0x00, 0x00, 0x10, 0x74, 0x66, 0x68, 0x64, + 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x14, 0x74, 0x66, 0x64, 0x74, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x32, 0x80, 0x00, 0x00, 0x00, 0x24, + 0x74, 0x72, 0x75, 0x6e, 0x01, 0x00, 0x03, 0x01, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0xfc, + 0x00, 0x00, 0x5d, 0xc0, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0xbb, 0x80, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x2c, 0x6d, 0x64, 0x61, 0x74, + 0x00, 0x00, 0x00, 0x04, 0x01, 0x02, 0x03, 0x04, + 0x00, 0x00, 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, + 0x01, 0x05, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x01, 0x01, 0x02, 0x03, 0x04, + 0x01, 0x02, 0x03, 0x04, + } + + var parts Parts + err := parts.Unmarshal(byts) + require.NoError(t, err) + + require.Equal(t, Parts{{ + Tracks: []*PartTrack{ + { + ID: 1, + Samples: []*PartSample{ + { + Duration: 2 * 90000, + Payload: []byte{ + 0x00, 0x00, 0x00, 0x04, + 0x01, 0x02, 0x03, 0x04, // SPS + 0x00, 0x00, 0x00, 0x01, + 0x08, // PPS + 0x00, 0x00, 0x00, 0x01, + 0x05, // IDR + }, + }, + { + Duration: 2 * 90000, + Payload: []byte{ + 0x00, 0x00, 0x00, 0x01, + 0x01, // non-IDR + }, + Flags: 1 << 16, + }, + { + Duration: 1 * 90000, + Payload: []byte{ + 0x00, 0x00, 0x00, 0x01, + 0x01, // non-IDR + }, + Flags: 1 << 16, + }, + }, + }, + { + ID: 2, + BaseTime: 3 * 48000, + Samples: []*PartSample{ + { + Duration: 500 * 48000 / 1000, + Payload: []byte{ + 0x01, 0x02, 0x03, 0x04, + }, + }, + { + Duration: 1 * 48000, + Payload: []byte{ + 0x01, 0x02, 0x03, 0x04, + }, + }, + }, + }, + }, + }}, parts) +} diff --git a/internal/hls/fmp4/part_track.go b/internal/hls/fmp4/part_track.go new file mode 100644 index 00000000..45361330 --- /dev/null +++ b/internal/hls/fmp4/part_track.go @@ -0,0 +1,106 @@ +package fmp4 + +import ( + gomp4 "github.com/abema/go-mp4" +) + +// PartSample is a sample of a PartTrack. +type PartSample struct { + Duration uint32 + PTSOffset int32 + Flags uint32 + Payload []byte +} + +// PartTrack is a track of Part. +type PartTrack struct { + ID int + BaseTime uint64 + Samples []*PartSample + IsVideo bool // marshal only +} + +func (pt *PartTrack) marshal(w *mp4Writer) (*gomp4.Trun, int, error) { + /* + traf + - tfhd + - tfdt + - trun + */ + + _, err := w.writeBoxStart(&gomp4.Traf{}) // + if err != nil { + return nil, 0, err + } + + flags := 0 + + _, err = w.WriteBox(&gomp4.Tfhd{ // + FullBox: gomp4.FullBox{ + Flags: [3]byte{2, byte(flags >> 8), byte(flags)}, + }, + TrackID: uint32(pt.ID), + }) + if err != nil { + return nil, 0, err + } + + _, err = w.WriteBox(&gomp4.Tfdt{ // + FullBox: gomp4.FullBox{ + Version: 1, + }, + // sum of decode durations of all earlier samples + BaseMediaDecodeTimeV1: pt.BaseTime, + }) + if err != nil { + return nil, 0, err + } + + if pt.IsVideo { + flags = trunFlagDataOffsetPreset | + trunFlagSampleDurationPresent | + trunFlagSampleSizePresent | + trunFlagSampleFlagsPresent | + trunFlagSampleCompositionTimeOffsetPresentOrV1 + } else { + flags = trunFlagDataOffsetPreset | + trunFlagSampleDurationPresent | + trunFlagSampleSizePresent + } + + trun := &gomp4.Trun{ // + FullBox: gomp4.FullBox{ + Version: 1, + Flags: [3]byte{0, byte(flags >> 8), byte(flags)}, + }, + SampleCount: uint32(len(pt.Samples)), + } + + for _, sample := range pt.Samples { + if pt.IsVideo { + trun.Entries = append(trun.Entries, gomp4.TrunEntry{ + SampleDuration: sample.Duration, + SampleSize: uint32(len(sample.Payload)), + SampleFlags: sample.Flags, + SampleCompositionTimeOffsetV1: sample.PTSOffset, + }) + } else { + trun.Entries = append(trun.Entries, gomp4.TrunEntry{ + SampleDuration: sample.Duration, + SampleSize: uint32(len(sample.Payload)), + }) + } + } + + trunOffset, err := w.WriteBox(trun) + if err != nil { + return nil, 0, err + } + + err = w.writeBoxEnd() // + if err != nil { + return nil, 0, err + } + + return trun, trunOffset, nil +} diff --git a/internal/hls/fmp4/part_write.go b/internal/hls/fmp4/part_write.go deleted file mode 100644 index f311821b..00000000 --- a/internal/hls/fmp4/part_write.go +++ /dev/null @@ -1,300 +0,0 @@ -package fmp4 - -import ( - "math" - "time" - - gomp4 "github.com/abema/go-mp4" - "github.com/aler9/gortsplib" - "github.com/aler9/gortsplib/pkg/h264" -) - -func durationGoToMp4(v time.Duration, timescale time.Duration) int64 { - return int64(math.Round(float64(v*timescale) / float64(time.Second))) -} - -func partWriteVideoInfo( - w *mp4Writer, - trackID int, - videoSamples []*VideoSample, -) (*gomp4.Trun, int, error) { - /* - traf - - tfhd - - tfdt - - trun - */ - - _, err := w.writeBoxStart(&gomp4.Traf{}) // - if err != nil { - return nil, 0, err - } - - flags := 0 - - _, err = w.WriteBox(&gomp4.Tfhd{ // - FullBox: gomp4.FullBox{ - Flags: [3]byte{2, byte(flags >> 8), byte(flags)}, - }, - TrackID: uint32(trackID), - }) - if err != nil { - return nil, 0, err - } - - _, err = w.WriteBox(&gomp4.Tfdt{ // - FullBox: gomp4.FullBox{ - Version: 1, - }, - // sum of decode durations of all earlier samples - BaseMediaDecodeTimeV1: uint64(durationGoToMp4(videoSamples[0].DTS, videoTimescale)), - }) - if err != nil { - return nil, 0, err - } - - flags = 0 - flags |= 0x01 // data offset present - flags |= 0x100 // sample duration present - flags |= 0x200 // sample size present - flags |= 0x400 // sample flags present - flags |= 0x800 // sample composition time offset present or v1 - - trun := &gomp4.Trun{ // - FullBox: gomp4.FullBox{ - Version: 1, - Flags: [3]byte{0, byte(flags >> 8), byte(flags)}, - }, - SampleCount: uint32(len(videoSamples)), - } - - for _, e := range videoSamples { - off := e.PTS - e.DTS - - flags := uint32(0) - if !e.IDRPresent { - flags |= 1 << 16 // sample_is_non_sync_sample - } - - trun.Entries = append(trun.Entries, gomp4.TrunEntry{ - SampleDuration: uint32(durationGoToMp4(e.Duration(), videoTimescale)), - SampleSize: uint32(len(e.avcc)), - SampleFlags: flags, - SampleCompositionTimeOffsetV1: int32(durationGoToMp4(off, videoTimescale)), - }) - } - - trunOffset, err := w.WriteBox(trun) - if err != nil { - return nil, 0, err - } - - err = w.writeBoxEnd() // - if err != nil { - return nil, 0, err - } - - return trun, trunOffset, nil -} - -func partWriteAudioInfo( - w *mp4Writer, - trackID int, - audioTrack *gortsplib.TrackMPEG4Audio, - audioSamples []*AudioSample, -) (*gomp4.Trun, int, error) { - /* - traf - - tfhd - - tfdt - - trun - */ - - if len(audioSamples) == 0 { - return nil, 0, nil - } - - _, err := w.writeBoxStart(&gomp4.Traf{}) // - if err != nil { - return nil, 0, err - } - - flags := 0 - - _, err = w.WriteBox(&gomp4.Tfhd{ // - FullBox: gomp4.FullBox{ - Flags: [3]byte{2, byte(flags >> 8), byte(flags)}, - }, - TrackID: uint32(trackID), - }) - if err != nil { - return nil, 0, err - } - - _, err = w.WriteBox(&gomp4.Tfdt{ // - FullBox: gomp4.FullBox{ - Version: 1, - }, - // sum of decode durations of all earlier samples - BaseMediaDecodeTimeV1: uint64(durationGoToMp4(audioSamples[0].PTS, time.Duration(audioTrack.ClockRate()))), - }) - if err != nil { - return nil, 0, err - } - - flags = 0 - flags |= 0x01 // data offset present - flags |= 0x100 // sample duration present - flags |= 0x200 // sample size present - - trun := &gomp4.Trun{ // - FullBox: gomp4.FullBox{ - Version: 0, - Flags: [3]byte{0, byte(flags >> 8), byte(flags)}, - }, - SampleCount: uint32(len(audioSamples)), - } - - for _, e := range audioSamples { - trun.Entries = append(trun.Entries, gomp4.TrunEntry{ - SampleDuration: uint32(durationGoToMp4(e.Duration(), time.Duration(audioTrack.ClockRate()))), - SampleSize: uint32(len(e.AU)), - }) - } - - trunOffset, err := w.WriteBox(trun) - if err != nil { - return nil, 0, err - } - - err = w.writeBoxEnd() // - if err != nil { - return nil, 0, err - } - - return trun, trunOffset, nil -} - -// PartWrite generates a FMP4 part file. -func PartWrite( - videoTrack *gortsplib.TrackH264, - audioTrack *gortsplib.TrackMPEG4Audio, - videoSamples []*VideoSample, - audioSamples []*AudioSample, -) ([]byte, error) { - /* - moof - - mfhd - - traf (video) - - traf (audio) - mdat - */ - - w := newMP4Writer() - - moofOffset, err := w.writeBoxStart(&gomp4.Moof{}) // - if err != nil { - return nil, err - } - - _, err = w.WriteBox(&gomp4.Mfhd{ // - SequenceNumber: 0, - }) - if err != nil { - return nil, err - } - - trackID := 1 - - var videoTrun *gomp4.Trun - var videoTrunOffset int - if videoTrack != nil { - for _, e := range videoSamples { - var err error - e.avcc, err = h264.AVCCMarshal(e.NALUs) - if err != nil { - return nil, err - } - } - - var err error - videoTrun, videoTrunOffset, err = partWriteVideoInfo( - w, trackID, videoSamples) - if err != nil { - return nil, err - } - - trackID++ - } - - var audioTrun *gomp4.Trun - var audioTrunOffset int - if audioTrack != nil { - var err error - audioTrun, audioTrunOffset, err = partWriteAudioInfo(w, trackID, audioTrack, audioSamples) - if err != nil { - return nil, err - } - } - - err = w.writeBoxEnd() // - if err != nil { - return nil, err - } - - mdat := &gomp4.Mdat{} // - - dataSize := 0 - videoDataSize := 0 - - if videoTrack != nil { - for _, e := range videoSamples { - dataSize += len(e.avcc) - } - videoDataSize = dataSize - } - - if audioTrack != nil { - for _, e := range audioSamples { - dataSize += len(e.AU) - } - } - - mdat.Data = make([]byte, dataSize) - pos := 0 - - if videoTrack != nil { - for _, e := range videoSamples { - pos += copy(mdat.Data[pos:], e.avcc) - } - } - - if audioTrack != nil { - for _, e := range audioSamples { - pos += copy(mdat.Data[pos:], e.AU) - } - } - - mdatOffset, err := w.WriteBox(mdat) - if err != nil { - return nil, err - } - - if videoTrack != nil { - videoTrun.DataOffset = int32(mdatOffset - moofOffset + 8) - err = w.rewriteBox(videoTrunOffset, videoTrun) - if err != nil { - return nil, err - } - } - - if audioTrack != nil && audioTrun != nil { - audioTrun.DataOffset = int32(videoDataSize + mdatOffset - moofOffset + 8) - err = w.rewriteBox(audioTrunOffset, audioTrun) - if err != nil { - return nil, err - } - } - - return w.bytes(), nil -} diff --git a/internal/hls/fmp4/part_write_test.go b/internal/hls/fmp4/part_write_test.go deleted file mode 100644 index ab72af90..00000000 --- a/internal/hls/fmp4/part_write_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package fmp4 - -import ( - "testing" - "time" - - gomp4 "github.com/abema/go-mp4" - "github.com/aler9/gortsplib/pkg/h264" - "github.com/stretchr/testify/require" -) - -func TestPartWrite(t *testing.T) { - testVideoSamples := []*VideoSample{ - { - NALUs: [][]byte{ - {0x06}, - {0x07}, - }, - PTS: 0, - DTS: 0, - }, - { - NALUs: [][]byte{ - testSPS, // SPS - {8}, // PPS - {5}, // IDR - }, - PTS: 2 * time.Second, - DTS: 2 * time.Second, - }, - - { - NALUs: [][]byte{ - {1}, // non-IDR - }, - PTS: 4 * time.Second, - DTS: 4 * time.Second, - }, - - { - NALUs: [][]byte{ - {1}, // non-IDR - }, - PTS: 6 * time.Second, - DTS: 6 * time.Second, - }, - { - NALUs: [][]byte{ - {5}, // IDR - }, - PTS: 7 * time.Second, - DTS: 7 * time.Second, - }, - } - - testAudioSamples := []*AudioSample{ - { - AU: []byte{ - 0x01, 0x02, 0x03, 0x04, - }, - PTS: 3 * time.Second, - }, - { - AU: []byte{ - 0x01, 0x02, 0x03, 0x04, - }, - PTS: 3500 * time.Millisecond, - }, - { - AU: []byte{ - 0x01, 0x02, 0x03, 0x04, - }, - PTS: 4500 * time.Millisecond, - }, - } - - for i, sample := range testVideoSamples { - sample.IDRPresent = h264.IDRPresent(sample.NALUs) - if i != len(testVideoSamples)-1 { - sample.Next = testVideoSamples[i+1] - } - } - testVideoSamples = testVideoSamples[:len(testVideoSamples)-1] - - for i, sample := range testAudioSamples { - if i != len(testAudioSamples)-1 { - sample.Next = testAudioSamples[i+1] - } - } - testAudioSamples = testAudioSamples[:len(testAudioSamples)-1] - - t.Run("video + audio", func(t *testing.T) { - byts, err := PartWrite(testVideoTrack, testAudioTrack, testVideoSamples, testAudioSamples) - require.NoError(t, err) - - boxes := []gomp4.BoxPath{ - {gomp4.BoxTypeMoof()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeMfhd()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, - {gomp4.BoxTypeMdat()}, - } - testMP4(t, byts, boxes) - }) - - t.Run("video only", func(t *testing.T) { - byts, err := PartWrite(testVideoTrack, nil, testVideoSamples, nil) - require.NoError(t, err) - - boxes := []gomp4.BoxPath{ - {gomp4.BoxTypeMoof()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeMfhd()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, - {gomp4.BoxTypeMdat()}, - } - testMP4(t, byts, boxes) - }) - - t.Run("audio only", func(t *testing.T) { - byts, err := PartWrite(nil, testAudioTrack, nil, testAudioSamples) - require.NoError(t, err) - - boxes := []gomp4.BoxPath{ - {gomp4.BoxTypeMoof()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeMfhd()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfhd()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTfdt()}, - {gomp4.BoxTypeMoof(), gomp4.BoxTypeTraf(), gomp4.BoxTypeTrun()}, - {gomp4.BoxTypeMdat()}, - } - testMP4(t, byts, boxes) - }) -} diff --git a/internal/hls/fmp4/videosample.go b/internal/hls/fmp4/videosample.go deleted file mode 100644 index 4f7416da..00000000 --- a/internal/hls/fmp4/videosample.go +++ /dev/null @@ -1,25 +0,0 @@ -package fmp4 - -import ( - "time" -) - -const ( - videoTimescale = 90000 -) - -// VideoSample is a video sample. -type VideoSample struct { - NALUs [][]byte - PTS time.Duration - DTS time.Duration - IDRPresent bool - Next *VideoSample - - avcc []byte -} - -// Duration returns the sample duration. -func (s VideoSample) Duration() time.Duration { - return s.Next.DTS - s.DTS -} diff --git a/internal/hls/m3u8/m3u8.go b/internal/hls/m3u8/m3u8.go new file mode 100644 index 00000000..4955a871 --- /dev/null +++ b/internal/hls/m3u8/m3u8.go @@ -0,0 +1,106 @@ +// Package m3u8 contains a M3U8 parser. +package m3u8 + +import ( + "bytes" + "errors" + "regexp" + "strings" + + gm3u8 "github.com/grafov/m3u8" +) + +var reKeyValue = regexp.MustCompile(`([a-zA-Z0-9_-]+)=("[^"]+"|[^",]+)`) + +func decodeParamsLine(line string) map[string]string { + out := make(map[string]string) + for _, kv := range reKeyValue.FindAllStringSubmatch(line, -1) { + k, v := kv[1], kv[2] + out[k] = strings.Trim(v, ` "`) + } + return out +} + +// MasterPlaylist is a master playlist. +type MasterPlaylist struct { + gm3u8.MasterPlaylist + Alternatives []*gm3u8.Alternative +} + +func (MasterPlaylist) isPlaylist() {} + +func newMasterPlaylist(byts []byte, mpl *gm3u8.MasterPlaylist) (*MasterPlaylist, error) { + var alternatives []*gm3u8.Alternative + + // https://github.com/grafov/m3u8/blob/036100c52a87e26c62be56df85450e9c703201a6/reader.go#L301 + for _, line := range strings.Split(string(byts), "\n") { + if strings.HasPrefix(line, "#EXT-X-MEDIA:") { + var alt gm3u8.Alternative + for k, v := range decodeParamsLine(line[13:]) { + switch k { + case "TYPE": + alt.Type = v + case "GROUP-ID": + alt.GroupId = v + case "LANGUAGE": + alt.Language = v + case "NAME": + alt.Name = v + case "DEFAULT": + switch { + case strings.ToUpper(v) == "YES": + alt.Default = true + case strings.ToUpper(v) == "NO": + alt.Default = false + default: + return nil, errors.New("value must be YES or NO") + } + case "AUTOSELECT": + alt.Autoselect = v + case "FORCED": + alt.Forced = v + case "CHARACTERISTICS": + alt.Characteristics = v + case "SUBTITLES": + alt.Subtitles = v + case "URI": + alt.URI = v + } + } + alternatives = append(alternatives, &alt) + } + } + + return &MasterPlaylist{ + MasterPlaylist: *mpl, + Alternatives: alternatives, + }, nil +} + +// MediaPlaylist is a media playlist. +type MediaPlaylist gm3u8.MediaPlaylist + +func (MediaPlaylist) isPlaylist() {} + +// Playlist is a M3U8 playlist. +type Playlist interface { + isPlaylist() +} + +// Unmarshal decodes a M3U8 Playlist. +func Unmarshal(byts []byte) (Playlist, error) { + pl, _, err := gm3u8.Decode(*(bytes.NewBuffer(byts)), true) + if err != nil { + return nil, err + } + + switch tpl := pl.(type) { + case *gm3u8.MasterPlaylist: + return newMasterPlaylist(byts, tpl) + + case *gm3u8.MediaPlaylist: + return (*MediaPlaylist)(tpl), nil + } + + panic("unexpected playlist type") +} diff --git a/internal/hls/mpegtstimedec/decoder.go b/internal/hls/mpegts/timedecoder.go similarity index 69% rename from internal/hls/mpegtstimedec/decoder.go rename to internal/hls/mpegts/timedecoder.go index 97ee68db..0879cddd 100644 --- a/internal/hls/mpegtstimedec/decoder.go +++ b/internal/hls/mpegts/timedecoder.go @@ -1,30 +1,34 @@ -// Package mpegtstimedec contains a MPEG-TS timestamp decoder. -package mpegtstimedec +package mpegts import ( + "sync" "time" ) const ( maximum = 0x1FFFFFFFF // 33 bits - negativeThreshold = 0xFFFFFFF + negativeThreshold = 0x1FFFFFFFF / 2 clockRate = 90000 ) -// Decoder is a MPEG-TS timestamp decoder. -type Decoder struct { +// TimeDecoder is a MPEG-TS timestamp decoder. +type TimeDecoder struct { initialized bool tsOverall time.Duration tsPrev int64 + mutex sync.Mutex } -// New allocates a Decoder. -func New() *Decoder { - return &Decoder{} +// NewTimeDecoder allocates a TimeDecoder. +func NewTimeDecoder() *TimeDecoder { + return &TimeDecoder{} } // Decode decodes a MPEG-TS timestamp. -func (d *Decoder) Decode(ts int64) time.Duration { +func (d *TimeDecoder) Decode(ts int64) time.Duration { + d.mutex.Lock() + defer d.mutex.Unlock() + if !d.initialized { d.initialized = true d.tsPrev = ts diff --git a/internal/hls/mpegtstimedec/decoder_test.go b/internal/hls/mpegts/timedecoder_test.go similarity index 85% rename from internal/hls/mpegtstimedec/decoder_test.go rename to internal/hls/mpegts/timedecoder_test.go index f3e6838c..8bf48fac 100644 --- a/internal/hls/mpegtstimedec/decoder_test.go +++ b/internal/hls/mpegts/timedecoder_test.go @@ -1,4 +1,4 @@ -package mpegtstimedec +package mpegts import ( "testing" @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" ) -func TestNegativeDiff(t *testing.T) { - d := New() +func TestTimeDecoderNegativeDiff(t *testing.T) { + d := NewTimeDecoder() i := int64(0) pts := d.Decode(i) @@ -27,8 +27,8 @@ func TestNegativeDiff(t *testing.T) { require.Equal(t, 3*time.Second, pts) } -func TestOverflow(t *testing.T) { - d := New() +func TestTimeDecoderOverflow(t *testing.T) { + d := NewTimeDecoder() i := int64(0x1FFFFFFFF - 20) secs := time.Duration(0) @@ -56,8 +56,8 @@ func TestOverflow(t *testing.T) { } } -func TestOverflowAndBack(t *testing.T) { - d := New() +func TestTimeDecoderOverflowAndBack(t *testing.T) { + d := NewTimeDecoder() pts := d.Decode(0x1FFFFFFFF - 90000 + 1) require.Equal(t, time.Duration(0), pts) diff --git a/internal/hls/mpegts/tracks.go b/internal/hls/mpegts/tracks.go new file mode 100644 index 00000000..443771f7 --- /dev/null +++ b/internal/hls/mpegts/tracks.go @@ -0,0 +1,101 @@ +package mpegts + +import ( + "bytes" + "context" + "fmt" + + "github.com/aler9/gortsplib" + "github.com/aler9/gortsplib/pkg/mpeg4audio" + "github.com/asticode/go-astits" +) + +func findMPEG4AudioConfig(dem *astits.Demuxer, pid uint16) (*mpeg4audio.Config, error) { + for { + data, err := dem.NextData() + if err != nil { + return nil, err + } + + if data.PES == nil || data.PID != pid { + continue + } + + var adtsPkts mpeg4audio.ADTSPackets + err = adtsPkts.Unmarshal(data.PES.Data) + if err != nil { + return nil, fmt.Errorf("unable to decode ADTS: %s", err) + } + + pkt := adtsPkts[0] + return &mpeg4audio.Config{ + Type: pkt.Type, + SampleRate: pkt.SampleRate, + ChannelCount: pkt.ChannelCount, + }, nil + } +} + +// Track is a MPEG-TS track. +type Track struct { + ES *astits.PMTElementaryStream + Track gortsplib.Track +} + +// FindTracks finds the tracks in a MPEG-TS stream. +func FindTracks(byts []byte) ([]*Track, error) { + var tracks []*Track + dem := astits.NewDemuxer(context.Background(), bytes.NewReader(byts)) + + for { + data, err := dem.NextData() + if err != nil { + return nil, err + } + + if data.PMT != nil { + for _, es := range data.PMT.ElementaryStreams { + switch es.StreamType { + case astits.StreamTypeH264Video, + astits.StreamTypeAACAudio: + default: + return nil, fmt.Errorf("track type %d not supported (yet)", es.StreamType) + } + + tracks = append(tracks, &Track{ + ES: es, + }) + } + break + } + } + + if tracks == nil { + return nil, fmt.Errorf("no tracks found") + } + + for _, t := range tracks { + switch t.ES.StreamType { + case astits.StreamTypeH264Video: + t.Track = &gortsplib.TrackH264{ + PayloadType: 96, + } + + case astits.StreamTypeAACAudio: + conf, err := findMPEG4AudioConfig(dem, t.ES.ElementaryPID) + if err != nil { + return nil, err + } + + t.Track = &gortsplib.TrackMPEG4Audio{ + PayloadType: 96, + Config: conf, + SizeLength: 13, + IndexLength: 3, + IndexDeltaLength: 3, + } + } + } + + return tracks, nil +} diff --git a/internal/hls/mpegts/writer.go b/internal/hls/mpegts/writer.go index 674ac592..b7512c7e 100644 --- a/internal/hls/mpegts/writer.go +++ b/internal/hls/mpegts/writer.go @@ -1,4 +1,4 @@ -// Package mpegts contains a MPEG-TS writer. +// Package mpegts contains a MPEG-TS reader and writer. package mpegts import ( diff --git a/internal/hls/muxer_variant_fmp4.go b/internal/hls/muxer_variant_fmp4.go index ca3b982b..1108a1f4 100644 --- a/internal/hls/muxer_variant_fmp4.go +++ b/internal/hls/muxer_variant_fmp4.go @@ -85,7 +85,27 @@ func (v *muxerVariantFMP4) file(name string, msn string, part string, skip strin if v.initContent == nil || (v.videoTrack != nil && (!bytes.Equal(v.videoLastSPS, sps) || !bytes.Equal(v.videoLastPPS, pps))) { - initContent, err := fmp4.InitWrite(v.videoTrack, v.audioTrack) + init := fmp4.Init{} + trackID := 1 + + if v.videoTrack != nil { + init.Tracks = append(init.Tracks, &fmp4.InitTrack{ + ID: trackID, + TimeScale: 90000, + Track: v.videoTrack, + }) + trackID++ + } + + if v.audioTrack != nil { + init.Tracks = append(init.Tracks, &fmp4.InitTrack{ + ID: trackID, + TimeScale: uint32(v.audioTrack.ClockRate()), + Track: v.audioTrack, + }) + } + + initContent, err := init.Marshal() if err != nil { return &MuxerFileResponse{Status: http.StatusInternalServerError} } diff --git a/internal/hls/muxer_variant_fmp4_part.go b/internal/hls/muxer_variant_fmp4_part.go index 41838e62..f11109bf 100644 --- a/internal/hls/muxer_variant_fmp4_part.go +++ b/internal/hls/muxer_variant_fmp4_part.go @@ -21,11 +21,15 @@ type muxerVariantFMP4Part struct { audioTrack *gortsplib.TrackMPEG4Audio id uint64 - isIndependent bool - videoSamples []*fmp4.VideoSample - audioSamples []*fmp4.AudioSample - content []byte - renderedDuration time.Duration + isIndependent bool + videoSamples []*fmp4.PartSample + audioSamples []*fmp4.PartSample + content []byte + renderedDuration time.Duration + videoStartDTSFilled bool + videoStartDTS time.Duration + audioStartDTSFilled bool + audioStartDTS time.Duration } func newMuxerVariantFMP4Part( @@ -56,11 +60,11 @@ func (p *muxerVariantFMP4Part) reader() io.Reader { func (p *muxerVariantFMP4Part) duration() time.Duration { if p.videoTrack != nil { - ret := time.Duration(0) + ret := uint64(0) for _, e := range p.videoSamples { - ret += e.Duration() + ret += uint64(e.Duration) } - return ret + return durationMp4ToGo(ret, 90000) } // use the sum of the default duration of all samples, @@ -71,13 +75,35 @@ func (p *muxerVariantFMP4Part) duration() time.Duration { } func (p *muxerVariantFMP4Part) finalize() error { - if len(p.videoSamples) > 0 || len(p.audioSamples) > 0 { + if p.videoSamples != nil || p.audioSamples != nil { + part := fmp4.Part{} + + if p.videoSamples != nil { + part.Tracks = append(part.Tracks, &fmp4.PartTrack{ + ID: 1, + BaseTime: durationGoToMp4(p.videoStartDTS, 90000), + Samples: p.videoSamples, + IsVideo: true, + }) + } + + if p.audioSamples != nil { + var id int + if p.videoTrack != nil { + id = 2 + } else { + id = 1 + } + + part.Tracks = append(part.Tracks, &fmp4.PartTrack{ + ID: id, + BaseTime: durationGoToMp4(p.audioStartDTS, uint32(p.audioTrack.ClockRate())), + Samples: p.audioSamples, + }) + } + var err error - p.content, err = fmp4.PartWrite( - p.videoTrack, - p.audioTrack, - p.videoSamples, - p.audioSamples) + p.content, err = part.Marshal() if err != nil { return err } @@ -91,13 +117,24 @@ func (p *muxerVariantFMP4Part) finalize() error { return nil } -func (p *muxerVariantFMP4Part) writeH264(sample *fmp4.VideoSample) { - if sample.IDRPresent { +func (p *muxerVariantFMP4Part) writeH264(sample *augmentedVideoSample) { + if !p.videoStartDTSFilled { + p.videoStartDTSFilled = true + p.videoStartDTS = sample.dts + } + + if (sample.Flags & (1 << 16)) == 0 { p.isIndependent = true } - p.videoSamples = append(p.videoSamples, sample) + + p.videoSamples = append(p.videoSamples, &sample.PartSample) } -func (p *muxerVariantFMP4Part) writeAAC(sample *fmp4.AudioSample) { - p.audioSamples = append(p.audioSamples, sample) +func (p *muxerVariantFMP4Part) writeAAC(sample *augmentedAudioSample) { + if !p.audioStartDTSFilled { + p.audioStartDTSFilled = true + p.audioStartDTS = sample.dts + } + + p.audioSamples = append(p.audioSamples, &sample.PartSample) } diff --git a/internal/hls/muxer_variant_fmp4_segment.go b/internal/hls/muxer_variant_fmp4_segment.go index 926644f4..ebde925e 100644 --- a/internal/hls/muxer_variant_fmp4_segment.go +++ b/internal/hls/muxer_variant_fmp4_segment.go @@ -7,8 +7,6 @@ import ( "time" "github.com/aler9/gortsplib" - - "github.com/aler9/rtsp-simple-server/internal/hls/fmp4" ) type partsReader struct { @@ -101,8 +99,7 @@ func (s *muxerVariantFMP4Segment) getRenderedDuration() time.Duration { } func (s *muxerVariantFMP4Segment) finalize( - nextVideoSample *fmp4.VideoSample, - nextAudioSample *fmp4.AudioSample, + nextVideoSampleDTS time.Duration, ) error { err := s.currentPart.finalize() if err != nil { @@ -117,7 +114,7 @@ func (s *muxerVariantFMP4Segment) finalize( s.currentPart = nil if s.videoTrack != nil { - s.renderedDuration = nextVideoSample.DTS - s.startDTS + s.renderedDuration = nextVideoSampleDTS - s.startDTS } else { s.renderedDuration = 0 for _, pa := range s.parts { @@ -128,11 +125,8 @@ func (s *muxerVariantFMP4Segment) finalize( return nil } -func (s *muxerVariantFMP4Segment) writeH264(sample *fmp4.VideoSample, adjustedPartDuration time.Duration) error { - size := uint64(0) - for _, nalu := range sample.NALUs { - size += uint64(len(nalu)) - } +func (s *muxerVariantFMP4Segment) writeH264(sample *augmentedVideoSample, adjustedPartDuration time.Duration) error { + size := uint64(len(sample.Payload)) if (s.size + size) > s.segmentMaxSize { return fmt.Errorf("reached maximum segment size") } @@ -161,8 +155,8 @@ func (s *muxerVariantFMP4Segment) writeH264(sample *fmp4.VideoSample, adjustedPa return nil } -func (s *muxerVariantFMP4Segment) writeAAC(sample *fmp4.AudioSample, adjustedPartDuration time.Duration) error { - size := uint64(len(sample.AU)) +func (s *muxerVariantFMP4Segment) writeAAC(sample *augmentedAudioSample, adjustedPartDuration time.Duration) error { + size := uint64(len(sample.Payload)) if (s.size + size) > s.segmentMaxSize { return fmt.Errorf("reached maximum segment size") } diff --git a/internal/hls/muxer_variant_fmp4_segmenter.go b/internal/hls/muxer_variant_fmp4_segmenter.go index 0cfccab3..5e663ee8 100644 --- a/internal/hls/muxer_variant_fmp4_segmenter.go +++ b/internal/hls/muxer_variant_fmp4_segmenter.go @@ -45,6 +45,16 @@ func findCompatiblePartDuration( return i } +type augmentedVideoSample struct { + fmp4.PartSample + dts time.Duration +} + +type augmentedAudioSample struct { + fmp4.PartSample + dts time.Duration +} + type muxerVariantFMP4Segmenter struct { lowLatency bool segmentDuration time.Duration @@ -62,8 +72,8 @@ type muxerVariantFMP4Segmenter struct { currentSegment *muxerVariantFMP4Segment nextSegmentID uint64 nextPartID uint64 - nextVideoSample *fmp4.VideoSample - nextAudioSample *fmp4.AudioSample + nextVideoSample *augmentedVideoSample + nextAudioSample *augmentedAudioSample firstSegmentFinalized bool sampleDurations map[time.Duration]struct{} adjustedPartDuration time.Duration @@ -147,17 +157,20 @@ func (m *muxerVariantFMP4Segmenter) writeH264(now time.Time, pts time.Duration, return nil } - return m.writeH264Entry(now, &fmp4.VideoSample{ - PTS: pts, - NALUs: nalus, - IDRPresent: idrPresent, - }) + return m.writeH264Entry(now, pts, nalus, idrPresent) } -func (m *muxerVariantFMP4Segmenter) writeH264Entry(now time.Time, sample *fmp4.VideoSample) error { +func (m *muxerVariantFMP4Segmenter) writeH264Entry( + now time.Time, + pts time.Duration, + nalus [][]byte, + idrPresent bool, +) error { + var dts time.Duration + if !m.videoFirstIDRReceived { // skip sample silently until we find one with an IDR - if !sample.IDRPresent { + if !idrPresent { return nil } @@ -166,23 +179,42 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(now time.Time, sample *fmp4.V m.videoSPS = m.videoTrack.SafeSPS() var err error - sample.DTS, err = m.videoDTSExtractor.Extract(sample.NALUs, sample.PTS) + dts, err = m.videoDTSExtractor.Extract(nalus, pts) if err != nil { return err } - m.startDTS = sample.DTS - sample.DTS = 0 - sample.PTS -= m.startDTS + m.startDTS = dts + dts = 0 + pts -= m.startDTS } else { var err error - sample.DTS, err = m.videoDTSExtractor.Extract(sample.NALUs, sample.PTS) + dts, err = m.videoDTSExtractor.Extract(nalus, pts) if err != nil { return err } - sample.DTS -= m.startDTS - sample.PTS -= m.startDTS + dts -= m.startDTS + pts -= m.startDTS + } + + avcc, err := h264.AVCCMarshal(nalus) + if err != nil { + return err + } + + var flags uint32 + if !idrPresent { + flags |= 1 << 16 + } + + sample := &augmentedVideoSample{ + PartSample: fmp4.PartSample{ + PTSOffset: int32(durationGoToMp4(pts-dts, 90000)), + Flags: flags, + Payload: avcc, + }, + dts: dts, } // put samples into a queue in order to @@ -192,7 +224,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(now time.Time, sample *fmp4.V if sample == nil { return nil } - sample.Next = m.nextVideoSample + sample.Duration = uint32(durationGoToMp4(m.nextVideoSample.dts-sample.dts, 90000)) if m.currentSegment == nil { // create first segment @@ -200,7 +232,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(now time.Time, sample *fmp4.V m.lowLatency, m.genSegmentID(), now, - sample.DTS, + sample.dts, m.segmentMaxSize, m.videoTrack, m.audioTrack, @@ -209,21 +241,21 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(now time.Time, sample *fmp4.V ) } - m.adjustPartDuration(sample.Duration()) + m.adjustPartDuration(durationMp4ToGo(uint64(sample.Duration), 90000)) - err := m.currentSegment.writeH264(sample, m.adjustedPartDuration) + err = m.currentSegment.writeH264(sample, m.adjustedPartDuration) if err != nil { return err } // switch segment - if sample.Next.IDRPresent { + if idrPresent { sps := m.videoTrack.SafeSPS() spsChanged := !bytes.Equal(m.videoSPS, sps) - if (sample.Next.DTS-m.currentSegment.startDTS) >= m.segmentDuration || + if (m.nextVideoSample.dts-m.currentSegment.startDTS) >= m.segmentDuration || spsChanged { - err := m.currentSegment.finalize(sample.Next, nil) + err := m.currentSegment.finalize(m.nextVideoSample.dts) if err != nil { return err } @@ -235,7 +267,7 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(now time.Time, sample *fmp4.V m.lowLatency, m.genSegmentID(), now, - sample.Next.DTS, + m.nextVideoSample.dts, m.segmentMaxSize, m.videoTrack, m.audioTrack, @@ -255,21 +287,21 @@ func (m *muxerVariantFMP4Segmenter) writeH264Entry(now time.Time, sample *fmp4.V return nil } -func (m *muxerVariantFMP4Segmenter) writeAAC(now time.Time, pts time.Duration, au []byte) error { - return m.writeAACEntry(now, &fmp4.AudioSample{ - PTS: pts, - AU: au, - }) -} - -func (m *muxerVariantFMP4Segmenter) writeAACEntry(now time.Time, sample *fmp4.AudioSample) error { +func (m *muxerVariantFMP4Segmenter) writeAAC(now time.Time, dts time.Duration, au []byte) error { if m.videoTrack != nil { // wait for the video track if !m.videoFirstIDRReceived { return nil } - sample.PTS -= m.startDTS + dts -= m.startDTS + } + + sample := &augmentedAudioSample{ + PartSample: fmp4.PartSample{ + Payload: au, + }, + dts: dts, } // put samples into a queue in order to @@ -278,7 +310,7 @@ func (m *muxerVariantFMP4Segmenter) writeAACEntry(now time.Time, sample *fmp4.Au if sample == nil { return nil } - sample.Next = m.nextAudioSample + sample.Duration = uint32(durationGoToMp4(m.nextAudioSample.dts-sample.dts, uint32(m.audioTrack.ClockRate()))) if m.videoTrack == nil { if m.currentSegment == nil { @@ -287,7 +319,7 @@ func (m *muxerVariantFMP4Segmenter) writeAACEntry(now time.Time, sample *fmp4.Au m.lowLatency, m.genSegmentID(), now, - sample.PTS, + sample.dts, m.segmentMaxSize, m.videoTrack, m.audioTrack, @@ -309,8 +341,8 @@ func (m *muxerVariantFMP4Segmenter) writeAACEntry(now time.Time, sample *fmp4.Au // switch segment if m.videoTrack == nil && - (sample.Next.PTS-m.currentSegment.startDTS) >= m.segmentDuration { - err := m.currentSegment.finalize(nil, sample.Next) + (m.nextAudioSample.dts-m.currentSegment.startDTS) >= m.segmentDuration { + err := m.currentSegment.finalize(0) if err != nil { return err } @@ -322,7 +354,7 @@ func (m *muxerVariantFMP4Segmenter) writeAACEntry(now time.Time, sample *fmp4.Au m.lowLatency, m.genSegmentID(), now, - sample.Next.PTS, + m.nextAudioSample.dts, m.segmentMaxSize, m.videoTrack, m.audioTrack,