Repository: pion/mediadevices
Pull request #186 (2020-06-09 13:32:36)

Make raw audio decoder more practical
Unfortunately, in audio world, there's no clear standard for defining formats like fourcc for video. While defining our own consts for audio is fine, it's impractical when we're trying to integrate frameworks like avfoundation.

lherman-cs: > LGTM
> Sorry for the delay!

No worries. Thanks!

Patch:
diff --git a/pkg/wave/decoder.go b/pkg/wave/decoder.go
index f33debe..6861e35 100644
--- a/pkg/wave/decoder.go
+++ b/pkg/wave/decoder.go
@@ -9,16 +9,30 @@ import (
 )
 
 // Format represents how audio is formatted in memory
-type Format string
+type Format fmt.Stringer
 
-const (
-	FormatInt16Interleaved      Format = "Int16Interleaved"
-	FormatInt16NonInterleaved          = "Int16NonInterleaved"
-	FormatFloat32Interleaved           = "Float32Interleaved"
-	FormatFloat32NonInterleaved        = "Float32NonInterleaved"
-)
+type RawFormat struct {
+	SampleSize  int
+	IsFloat     bool
+	Interleaved bool
+}
+
+func (f *RawFormat) String() string {
+	sampleSizeInBits := f.SampleSize * 8
+	dataTypeStr := "Int"
+	if f.IsFloat {
+		dataTypeStr = "Float"
+	}
+	interleavedStr := "NonInterleaved"
+	if f.Interleaved {
+		interleavedStr = "Interleaved"
+	}
+
+	return fmt.Sprintf("%s%d%s", dataTypeStr, sampleSizeInBits, interleavedStr)
+}
 
 var hostEndian binary.ByteOrder
+var registeredDecoders = map[string]Decoder{}
 
 func init() {
 	switch v := *(*uint16)(unsafe.Pointer(&([]byte{0x12, 0x34}[0]))); v {
@@ -29,6 +43,20 @@ func init() {
 	default:
 		panic(fmt.Sprintf("failed to determine host endianness: %x", v))
 	}
+
+	decoderBuilders := []DecoderBuilderFunc{
+		newInt16InterleavedDecoder,
+		newInt16NonInterleavedDecoder,
+		newFloat32InterleavedDecoder,
+		newFloat32NonInterleavedDecoder,
+	}
+
+	for _, decoderBuilder := range decoderBuilders {
+		err := RegisterDecoder(decoderBuilder)
+		if err != nil {
+			panic(err)
+		}
+	}
 }
 
 // Decoder decodes raw chunk to Audio
@@ -44,21 +72,35 @@ func (f DecoderFunc) Decode(endian binary.ByteOrder, chunk []byte, channels int)
 	return f(endian, chunk, channels)
 }
 
+// DecoderBuilder builds raw audio decoder
+type DecoderBuilder interface {
+	// NewDecoder creates a new decoder for specified format
+	NewDecoder() (Decoder, Format)
+}
+
+// DecoderBuilderFunc is a proxy type for DecoderBuilder
+type DecoderBuilderFunc func() (Decoder, Format)
+
+func (builderFunc DecoderBuilderFunc) NewDecoder() (Decoder, Format) {
+	return builderFunc()
+}
+
+func RegisterDecoder(builder DecoderBuilder) error {
+	decoder, format := builder.NewDecoder()
+	formatStr := format.String()
+	if _, ok := registeredDecoders[formatStr]; ok {
+		return fmt.Errorf("%v has already been registered", format)
+	}
+
+	registeredDecoders[formatStr] = decoder
+	return nil
+}
+
 // NewDecoder creates a decoder to decode raw audio data in the given format
-func NewDecoder(f Format) (Decoder, error) {
-	var decoder DecoderFunc
-
-	switch f {
-	case FormatInt16Interleaved:
-		decoder = decodeInt16Interleaved
-	case FormatInt16NonInterleaved:
-		decoder = decodeInt16NonInterleaved
-	case FormatFloat32Interleaved:
-		decoder = decodeFloat32Interleaved
-	case FormatFloat32NonInterleaved:
-		decoder = decodeFloat32NonInterleaved
-	default:
-		return nil, fmt.Errorf("%s is not supported", f)
+func NewDecoder(format Format) (Decoder, error) {
+	decoder, ok := registeredDecoders[format.String()]
+	if !ok {
+		return nil, fmt.Errorf("%s format is not supported", format)
 	}
 
 	return decoder, nil
@@ -85,130 +127,171 @@ func calculateChunkInfo(chunk []byte, channels int, sampleSize int) (ChunkInfo,
 	}, nil
 }
 
-func decodeInt16Interleaved(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
-	sampleSize := 2
-	chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
-	if err != nil {
-		return nil, err
+func newInt16InterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  2,
+		IsFloat:     false,
+		Interleaved: true,
 	}
 
-	container := NewInt16Interleaved(chunkInfo)
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
 
-	if endian == hostEndian {
-		n := len(chunk)
-		h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[0])), Len: n, Cap: n}
-		dst := *(*[]byte)(unsafe.Pointer(&h))
-		copy(dst, chunk)
-		return container, nil
-	}
+		container := NewInt16Interleaved(chunkInfo)
 
-	sampleLen := sampleSize * channels
-	var i int
-	for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
-		for ch := 0; ch < channels; ch++ {
-			flatOffset := offset + ch*sampleSize
-			sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
-			container.SetInt16(i, ch, Int16Sample(sample))
+		if endian == hostEndian {
+			n := len(chunk)
+			h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[0])), Len: n, Cap: n}
+			dst := *(*[]byte)(unsafe.Pointer(&h))
+			copy(dst, chunk)
+			return container, nil
 		}
-		i++
-	}
 
-	return container, nil
+		sampleLen := sampleSize * channels
+		var i int
+		for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
+			for ch := 0; ch < channels; ch++ {
+				flatOffset := offset + ch*sampleSize
+				sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
+				container.SetInt16(i, ch, Int16Sample(sample))
+			}
+			i++
+		}
+
+		return container, nil
+
+	})
+
+	return decoder, format
 }
 
-func decodeInt16NonInterleaved(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
-	sampleSize := 2
-	chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
-	if err != nil {
-		return nil, err
+func newInt16NonInterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  2,
+		IsFloat:     false,
+		Interleaved: false,
 	}
 
-	container := NewInt16NonInterleaved(chunkInfo)
-	chunkLen := len(chunk) / channels
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
+
+		container := NewInt16NonInterleaved(chunkInfo)
+		chunkLen := len(chunk) / channels
+
+		if endian == hostEndian {
+			for ch := 0; ch < channels; ch++ {
+				offset := ch * chunkLen
+				h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[ch][0])), Len: chunkLen, Cap: chunkLen}
+				dst := *(*[]byte)(unsafe.Pointer(&h))
+				copy(dst, chunk[offset:offset+chunkLen])
+			}
+			return container, nil
+		}
 
-	if endian == hostEndian {
 		for ch := 0; ch < channels; ch++ {
 			offset := ch * chunkLen
-			h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[ch][0])), Len: chunkLen, Cap: chunkLen}
-			dst := *(*[]byte)(unsafe.Pointer(&h))
-			copy(dst, chunk[offset:offset+chunkLen])
+			for i := 0; i < chunkInfo.Len; i++ {
+				flatOffset := offset + i*sampleSize
+				sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
+				container.SetInt16(i, ch, Int16Sample(sample))
+			}
 		}
-		return container, nil
-	}
 
-	for ch := 0; ch < channels; ch++ {
-		offset := ch * chunkLen
-		for i := 0; i < chunkInfo.Len; i++ {
-			flatOffset := offset + i*sampleSize
-			sample := endian.Uint16(chunk[flatOffset : flatOffset+sampleSize])
-			container.SetInt16(i, ch, Int16Sample(sample))
-		}
-	}
+		return container, nil
+	})
 
-	return container, nil
+	return decoder, format
 }
 
-func decodeFloat32Interleaved(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
-	sampleSize := 4
-	chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
-	if err != nil {
-		return nil, err
+func newFloat32InterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  4,
+		IsFloat:     true,
+		Interleaved: true,
 	}
 
-	container := NewFloat32Interleaved(chunkInfo)
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
 
-	if endian == hostEndian {
-		n := len(chunk)
-		h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[0])), Len: n, Cap: n}
-		dst := *(*[]byte)(unsafe.Pointer(&h))
-		copy(dst, chunk)
-		return container, nil
-	}
+		container := NewFloat32Interleaved(chunkInfo)
 
-	sampleLen := sampleSize * channels
-	var i int
-	for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
-		for ch := 0; ch < channels; ch++ {
-			flatOffset := offset + ch*sampleSize
-			sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
-			sampleF := math.Float32frombits(sample)
-			container.SetFloat32(i, ch, Float32Sample(sampleF))
+		if endian == hostEndian {
+			n := len(chunk)
+			h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[0])), Len: n, Cap: n}
+			dst := *(*[]byte)(unsafe.Pointer(&h))
+			copy(dst, chunk)
+			return container, nil
+		}
+
+		sampleLen := sampleSize * channels
+		var i int
+		for offset := 0; offset+sampleLen <= len(chunk); offset += sampleLen {
+			for ch := 0; ch < channels; ch++ {
+				flatOffset := offset + ch*sampleSize
+				sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
+				sampleF := math.Float32frombits(sample)
+				container.SetFloat32(i, ch, Float32Sample(sampleF))
+			}
+			i++
 		}
-		i++
-	}
 
-	return container, nil
+		return container, nil
+	})
+
+	return decoder, format
 }
 
-func decodeFloat32NonInterleaved(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
-	sampleSize := 4
-	chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
-	if err != nil {
-		return nil, err
+func newFloat32NonInterleavedDecoder() (Decoder, Format) {
+	format := &RawFormat{
+		SampleSize:  4,
+		IsFloat:     true,
+		Interleaved: false,
 	}
 
-	container := NewFloat32NonInterleaved(chunkInfo)
-	chunkLen := len(chunk) / channels
+	decoder := DecoderFunc(func(endian binary.ByteOrder, chunk []byte, channels int) (Audio, error) {
+		sampleSize := format.SampleSize
+		chunkInfo, err := calculateChunkInfo(chunk, channels, sampleSize)
+		if err != nil {
+			return nil, err
+		}
+
+		container := NewFloat32NonInterleaved(chunkInfo)
+		chunkLen := len(chunk) / channels
+
+		if endian == hostEndian {
+			for ch := 0; ch < channels; ch++ {
+				offset := ch * chunkLen
+				h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[ch][0])), Len: chunkLen, Cap: chunkLen}
+				dst := *(*[]byte)(unsafe.Pointer(&h))
+				copy(dst, chunk[offset:offset+chunkLen])
+			}
+			return container, nil
+		}
 
-	if endian == hostEndian {
 		for ch := 0; ch < channels; ch++ {
 			offset := ch * chunkLen
-			h := reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&container.Data[ch][0])), Len: chunkLen, Cap: chunkLen}
-			dst := *(*[]byte)(unsafe.Pointer(&h))
-			copy(dst, chunk[offset:offset+chunkLen])
+			for i := 0; i < chunkInfo.Len; i++ {
+				flatOffset := offset + i*sampleSize
+				sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
+				sampleF := math.Float32frombits(sample)
+				container.SetFloat32(i, ch, Float32Sample(sampleF))
+			}
 		}
-		return container, nil
-	}
 
-	for ch := 0; ch < channels; ch++ {
-		offset := ch * chunkLen
-		for i := 0; i < chunkInfo.Len; i++ {
-			flatOffset := offset + i*sampleSize
-			sample := endian.Uint32(chunk[flatOffset : flatOffset+sampleSize])
-			sampleF := math.Float32frombits(sample)
-			container.SetFloat32(i, ch, Float32Sample(sampleF))
-		}
-	}
+		return container, nil
+	})
 
-	return container, nil
+	return decoder, format
 }


Test patch:
diff --git a/pkg/wave/decoder_bench_test.go b/pkg/wave/decoder_bench_test.go
index b86c936..2f5f81f 100644
--- a/pkg/wave/decoder_bench_test.go
+++ b/pkg/wave/decoder_bench_test.go
@@ -7,13 +7,6 @@ import (
 )
 
 func BenchmarkDecoder(b *testing.B) {
-	formats := []Format{
-		FormatInt16Interleaved,
-		FormatInt16NonInterleaved,
-		FormatFloat32Interleaved,
-		FormatFloat32NonInterleaved,
-	}
-
 	var nonHostEndian binary.ByteOrder
 	if hostEndian == binary.BigEndian {
 		nonHostEndian = binary.LittleEndian
@@ -21,12 +14,9 @@ func BenchmarkDecoder(b *testing.B) {
 		nonHostEndian = binary.BigEndian
 	}
 
-	for _, format := range formats {
+	for format, decoder := range registeredDecoders {
 		format := format
-		decoder, err := NewDecoder(format)
-		if err != nil {
-			b.Fatal(err)
-		}
+		decoder := decoder
 
 		b.Run(fmt.Sprintf("%sHostEndian", format), func(b *testing.B) {
 			for i := 0; i < b.N; i++ {
diff --git a/pkg/wave/decoder_test.go b/pkg/wave/decoder_test.go
index bda52db..413d2f5 100644
--- a/pkg/wave/decoder_test.go
+++ b/pkg/wave/decoder_test.go
@@ -93,12 +93,45 @@ func TestCalculateChunkInfo(t *testing.T) {
 	}
 }
 
+func TestNewDecoder(t *testing.T) {
+	rawFormats := []RawFormat{
+		{
+			SampleSize:  2,
+			IsFloat:     false,
+			Interleaved: false,
+		},
+		{
+			SampleSize:  4,
+			IsFloat:     true,
+			Interleaved: false,
+		},
+		{
+			SampleSize:  2,
+			IsFloat:     false,
+			Interleaved: true,
+		},
+		{
+			SampleSize:  4,
+			IsFloat:     true,
+			Interleaved: true,
+		},
+	}
+
+	for _, rawFormat := range rawFormats {
+		_, err := NewDecoder(&rawFormat)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
 func TestDecodeInt16Interleaved(t *testing.T) {
 	raw := []byte{
 		// 16 bits per channel
 		0x01, 0x02, 0x03, 0x04,
 		0x05, 0x06, 0x07, 0x08,
 	}
+	decoder, _ := newInt16InterleavedDecoder()
 
 	t.Run("BigEndian", func(t *testing.T) {
 		expected := &Int16Interleaved{
@@ -113,7 +146,7 @@ func TestDecodeInt16Interleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeInt16Interleaved(binary.BigEndian, raw, 2)
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -136,7 +169,7 @@ func TestDecodeInt16Interleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeInt16Interleaved(binary.LittleEndian, raw, 2)
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -154,6 +187,8 @@ func TestDecodeInt16NonInterleaved(t *testing.T) {
 		0x05, 0x06, 0x07, 0x08,
 	}
 
+	decoder, _ := newInt16NonInterleavedDecoder()
+
 	t.Run("BigEndian", func(t *testing.T) {
 		expected := &Int16NonInterleaved{
 			Data: [][]int16{
@@ -165,7 +200,7 @@ func TestDecodeInt16NonInterleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeInt16NonInterleaved(binary.BigEndian, raw, 2)
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -186,7 +221,7 @@ func TestDecodeInt16NonInterleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeInt16NonInterleaved(binary.LittleEndian, raw, 2)
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -204,6 +239,8 @@ func TestDecodeFloat32Interleaved(t *testing.T) {
 		0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
 	}
 
+	decoder, _ := newFloat32InterleavedDecoder()
+
 	t.Run("BigEndian", func(t *testing.T) {
 		expected := &Float32Interleaved{
 			Data: []float32{
@@ -217,7 +254,7 @@ func TestDecodeFloat32Interleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeFloat32Interleaved(binary.BigEndian, raw, 2)
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -240,7 +277,7 @@ func TestDecodeFloat32Interleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeFloat32Interleaved(binary.LittleEndian, raw, 2)
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -258,6 +295,8 @@ func TestDecodeFloat32NonInterleaved(t *testing.T) {
 		0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
 	}
 
+	decoder, _ := newFloat32NonInterleavedDecoder()
+
 	t.Run("BigEndian", func(t *testing.T) {
 		expected := &Float32NonInterleaved{
 			Data: [][]float32{
@@ -275,7 +314,7 @@ func TestDecodeFloat32NonInterleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeFloat32NonInterleaved(binary.BigEndian, raw, 2)
+		actual, err := decoder.Decode(binary.BigEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -302,7 +341,7 @@ func TestDecodeFloat32NonInterleaved(t *testing.T) {
 				Channels: 2,
 			},
 		}
-		actual, err := decodeFloat32NonInterleaved(binary.LittleEndian, raw, 2)
+		actual, err := decoder.Decode(binary.LittleEndian, raw, 2)
 		if err != nil {
 			t.Fatal(err)
 		}