Repository: Eyevinn/mp4ff
Pull request #220 (2023-02-27 10:22:31)

Emsg in fragments
When parsing fragmented MP4 files, emsg boxes may appear before the first moof box, or in between if identical.
This PR adds them to the start of fragments in the structure

      [init segment] [media segment [fragment][fragment]...] [media segment [fragment][fragment]...]...
 
 The media segment may start with an styp box or not, but the fragments never start with an styp box.


Patch:
diff --git a/mp4/audiosamplentry.go b/mp4/audiosamplentry.go
index d1a381f..10b529d 100644
--- a/mp4/audiosamplentry.go
+++ b/mp4/audiosamplentry.go
@@ -108,7 +108,7 @@ func DecodeAudioSampleEntry(hdr BoxHeader, startPos uint64, r io.Reader) (Box, e
 		if pos == startPos+hdr.Size {
 			break
 		} else if pos > startPos+hdr.Size {
-			return nil, fmt.Errorf("Bad size when decoding %s", hdr.Name)
+			return nil, fmt.Errorf("bad size when decoding %s", hdr.Name)
 		}
 	}
 	return a, nil
diff --git a/mp4/file.go b/mp4/file.go
index 2c498b1..0b35163 100644
--- a/mp4/file.go
+++ b/mp4/file.go
@@ -206,19 +206,19 @@ func (f *File) Size() uint64 {
 }
 
 // AddChild - add child with start position
-func (f *File) AddChild(box Box, boxStartPos uint64) {
-	switch box.Type() {
-	case "ftyp":
-		f.Ftyp = box.(*FtypBox)
-	case "moov":
-		f.Moov = box.(*MoovBox)
+func (f *File) AddChild(child Box, boxStartPos uint64) {
+	switch box := child.(type) {
+	case *FtypBox:
+		f.Ftyp = box
+	case *MoovBox:
+		f.Moov = box
 		if len(f.Moov.Trak.Mdia.Minf.Stbl.Stts.SampleCount) == 0 {
 			f.isFragmented = true
 			f.Init = NewMP4Init()
 			f.Init.AddChild(f.Ftyp)
 			f.Init.AddChild(f.Moov)
 		}
-	case "sidx":
+	case *SidxBox:
 		// sidx boxes are either added to the File or to the current media segment.
 		// Since sidx boxes for a segment come before the moof, it is important that a new
 		// segment is started with a styp box for the sidx to be associated with the
@@ -230,43 +230,61 @@ func (f *File) AddChild(box Box, boxStartPos uint64) {
 		// and testing such a solution, that track is not deemed worth the effort for now.
 		if len(f.Segments) == 0 {
 			// Add sidx to top level until we know that a segment has started
-			f.AddSidx(box.(*SidxBox))
+			f.AddSidx(box)
 		} else {
 			currSeg := f.Segments[len(f.Segments)-1]
-			currSeg.AddSidx(box.(*SidxBox))
+			currSeg.AddSidx(box)
 		}
-	case "styp":
+	case *StypBox:
+		// Starts a new segment
 		f.isFragmented = true
-		newSeg := NewMediaSegment()
-		newSeg.Styp = box.(*StypBox)
-		f.AddMediaSegment(newSeg)
-	case "moof":
+		f.AddMediaSegment(NewMediaSegmentWithStyp(box))
+	case *EmsgBox:
+		// emsg box is only added at the start of a fragment (inside a segment).
+		// The case that a segment starts without an emsg is also handled.
+		f.startSegmentIfNeeded(box)
+		lastSeg := f.LastSegment()
+		if len(lastSeg.Fragments) == 0 {
+			lastSeg.AddFragment(NewFragment())
+		}
+		frag := lastSeg.LastFragment()
+		frag.AddChild(box)
+	case *MoofBox:
 		f.isFragmented = true
-		moof := box.(*MoofBox)
+		moof := box
 		moof.StartPos = boxStartPos
-
-		var currentSegment *MediaSegment
-
-		if len(f.Segments) == 0 || f.Segments[0].Styp == nil {
-			// No styp present, so one fragment per segment
-			currentSegment = NewMediaSegment()
-			f.AddMediaSegment(currentSegment)
-		} else {
-			currentSegment = f.LastSegment()
+		f.startSegmentIfNeeded(moof)
+		currSeg := f.LastSegment()
+		if currSeg.LastFragment() == nil {
+			currSeg.AddFragment(NewFragment())
 		}
-		newFragment := NewFragment()
-		currentSegment.AddFragment(newFragment)
-		newFragment.AddChild(moof)
-	case "mdat":
-		mdat := box.(*MdatBox)
+		frag := currSeg.LastFragment()
+		frag.AddChild(moof)
+	case *MdatBox:
 		if !f.isFragmented {
-			f.Mdat = mdat
+			f.Mdat = box
 		} else {
 			currentFragment := f.LastSegment().LastFragment()
-			currentFragment.AddChild(mdat)
+			currentFragment.AddChild(box)
+		}
+	}
+	f.Children = append(f.Children, child)
+}
+
+// startSegmentIfNeeded starts a new segment if there is none, or the latest has a moof box.
+func (f *File) startSegmentIfNeeded(b Box) {
+	if len(f.Segments) == 0 {
+		f.isFragmented = true
+		f.AddMediaSegment(NewMediaSegmentWithoutStyp())
+		return
+	}
+	lastSeg := f.LastSegment()
+	lastFrag := lastSeg.LastFragment()
+	if lastFrag != nil {
+		if lastFrag.Moof != nil {
+			f.AddMediaSegment(NewMediaSegmentWithoutStyp())
 		}
 	}
-	f.Children = append(f.Children, box)
 }
 
 // AddSidx adds a sidx box to the File and not a MediaSegment.
diff --git a/mp4/fragment.go b/mp4/fragment.go
index 5409f04..d5b2394 100644
--- a/mp4/fragment.go
+++ b/mp4/fragment.go
@@ -10,6 +10,7 @@ import (
 
 // Fragment - MP4 Fragment ([prft] + moof + mdat)
 type Fragment struct {
+	Emsgs       []*EmsgBox
 	Prft        *PrftBox
 	Moof        *MoofBox
 	Mdat        *MdatBox
@@ -18,12 +19,12 @@ type Fragment struct {
 	EncOptimize EncOptimize // Bit field with optimizations being done at encoding
 }
 
-// NewFragment - New empty one-track MP4 Fragment
+// NewFragment creates an empty MP4 Fragment.
 func NewFragment() *Fragment {
 	return &Fragment{}
 }
 
-// CreateFragment - create single track empty fragment
+// CreateFragment creates a single track fragment
 func CreateFragment(seqNumber uint32, trackID uint32) (*Fragment, error) {
 	f := Fragment{Children: make([]Box, 0, 2)}
 	moof := &MoofBox{}
@@ -45,7 +46,7 @@ func CreateFragment(seqNumber uint32, trackID uint32) (*Fragment, error) {
 	return &f, nil
 }
 
-// CreateMultiTrackFragment - create multi-track empty fragment without trun
+// CreateMultiTrackFragment creates a multi-track fragment without trun boxes.
 func CreateMultiTrackFragment(seqNumber uint32, trackIDs []uint32) (*Fragment, error) {
 	f := NewFragment()
 	moof := &MoofBox{}
@@ -67,19 +68,34 @@ func CreateMultiTrackFragment(seqNumber uint32, trackIDs []uint32) (*Fragment, e
 	return f, nil
 }
 
-// AddChild - Add a top-level box to Fragment
+// AddChild adds a top-level box to Fragment. Add in proper order.
 func (f *Fragment) AddChild(b Box) {
-	switch b.Type() {
-	case "prft":
-		f.Prft = b.(*PrftBox)
-	case "moof":
-		f.Moof = b.(*MoofBox)
-	case "mdat":
-		f.Mdat = b.(*MdatBox)
+	switch box := b.(type) {
+	case *EmsgBox:
+		f.Emsgs = append(f.Emsgs, box)
+	case *PrftBox:
+		f.Prft = box
+	case *MoofBox:
+		f.Moof = box
+	case *MdatBox:
+		f.Mdat = box
 	}
 	f.Children = append(f.Children, b)
 }
 
+// AddEmsg inserts an emsg box at the end of a sequence of emsg boxes at the start of the fragment.
+func (f *Fragment) AddEmsg(emsg *EmsgBox) {
+	prevEmsg := -1
+	for i, c := range f.Children {
+		if _, ok := c.(*EmsgBox); ok {
+			prevEmsg = i
+		}
+	}
+	newIdx := prevEmsg + 1
+	f.Children = append(f.Children[:newIdx+1], f.Children[newIdx:]...)
+	f.Children[newIdx] = emsg
+}
+
 // Size - return size of fragment including all boxes.
 // Be aware that TrafBox.OptimizeTfhdTrun() can change size
 func (f *Fragment) Size() uint64 {
diff --git a/mp4/mediasegment.go b/mp4/mediasegment.go
index 5bc7373..c46a107 100644
--- a/mp4/mediasegment.go
+++ b/mp4/mediasegment.go
@@ -24,6 +24,15 @@ func NewMediaSegment() *MediaSegment {
 	}
 }
 
+// NewMediaSegmentWithStyp - create empty MediaSegment with styp box
+func NewMediaSegmentWithStyp(styp *StypBox) *MediaSegment {
+	return &MediaSegment{
+		Styp:        styp,
+		Fragments:   nil,
+		EncOptimize: OptimizeNone,
+	}
+}
+
 // NewMediaSegmentWithoutStyp - create empty media segment with no styp box
 func NewMediaSegmentWithoutStyp() *MediaSegment {
 	return &MediaSegment{
@@ -46,8 +55,11 @@ func (s *MediaSegment) AddFragment(f *Fragment) {
 	s.Fragments = append(s.Fragments, f)
 }
 
-// LastFragment - Currently last fragment
+// LastFragment returns the currently last fragment, or nil if no fragments.
 func (s *MediaSegment) LastFragment() *Fragment {
+	if len(s.Fragments) == 0 {
+		return nil
+	}
 	return s.Fragments[len(s.Fragments)-1]
 }
 


Test patch:
diff --git a/examples/decrypt-cenc/decrypt_cenc_test.go b/examples/decrypt-cenc/decrypt_cenc_test.go
index 6cabe54..67c4cc4 100644
--- a/examples/decrypt-cenc/decrypt_cenc_test.go
+++ b/examples/decrypt-cenc/decrypt_cenc_test.go
@@ -56,7 +56,6 @@ func TestDecryptFiles(t *testing.T) {
 			t.Errorf("Mismatch for case %s: %s", tc.name, diff)
 		}
 	}
-
 }
 
 func BenchmarkDecodeCenc(b *testing.B) {
diff --git a/mp4/file_test.go b/mp4/file_test.go
index d4e7b64..b3e168e 100644
--- a/mp4/file_test.go
+++ b/mp4/file_test.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/Eyevinn/mp4ff/aac"
 	"github.com/Eyevinn/mp4ff/bits"
 )
 
@@ -156,3 +157,73 @@ func TestDecodeEncode(t *testing.T) {
 		}
 	}
 }
+
+func TestFilesWithEmsg(t *testing.T) {
+	// File with ftyp, moov, styp, emsg, emsg, moof, mdat, moof, mdat
+	init := CreateEmptyInit()
+	init.AddEmptyTrack(uint32(48000), "audio", "en")
+	trak := init.Moov.Trak
+	err := trak.SetAACDescriptor(aac.AAClc, 48000)
+	if err != nil {
+		t.Error(err)
+	}
+	data := make([]byte, 0, init.Size())
+	buf := bytes.NewBuffer(data)
+	err = init.Encode(buf)
+	if err != nil {
+		t.Error(err)
+	}
+	seg := NewMediaSegment()
+	frag, err := CreateFragment(1, 1)
+	if err != nil {
+		t.Error(err)
+	}
+	frag.AddFullSample(FullSample{
+		Sample: Sample{
+			Flags:                 0x0,
+			Dur:                   1024,
+			Size:                  6,
+			CompositionTimeOffset: 0,
+		},
+		DecodeTime: 0,
+		Data:       []byte{0, 1, 2, 3, 4, 5},
+	})
+	frag.AddEmsg(&EmsgBox{ID: 1})
+	frag.AddEmsg(&EmsgBox{ID: 2})
+	seg.AddFragment(frag)
+	err = seg.Encode(buf)
+	if err != nil {
+		t.Error(err)
+	}
+	encData := buf.Bytes()
+	sr := bits.NewFixedSliceReader(encData)
+	decFile, err := DecodeFileSR(sr)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(decFile.Segments) != 1 {
+		t.Error("not 1 segment in file")
+	}
+	if len(decFile.Segments[0].Fragments) != 1 {
+		t.Error("not 1 fragment in segment")
+	}
+	dFrag := decFile.Segments[0].Fragments[0]
+	if len(dFrag.Emsgs) != 2 {
+		t.Error("not 2 emsg boxes in fragment")
+	}
+	if dFrag.Emsgs[0].ID != 1 {
+		t.Error("first emsg box does not have index 1")
+	}
+	if dFrag.Emsgs[1].ID != 2 {
+		t.Error("second emsg box does not have index 2")
+	}
+	sw := bits.NewFixedSliceWriter(int(decFile.Size()))
+	err = decFile.EncodeSW(sw)
+	if err != nil {
+		t.Error(err)
+	}
+	reEncData := sw.Bytes()
+	if !bytes.Equal(reEncData, encData) {
+		t.Errorf("re-encoded bytes differ from encoded bytes")
+	}
+}