Repository: pion/mediadevices
Pull request #209 (2020-09-03 01:08:53)

Add frame Copier
This is an incremental effort to redesign the project. Frame copy is necessary for supporting a video broadcaster.

Benchmark:

```
goos: darwin
goarch: amd64
pkg: github.com/pion/mediadevices/pkg/io/video
BenchmarkCopyOptimized
BenchmarkCopyOptimized-4            9290            132430 ns/op             670 B/op          0 allocs/op
BenchmarkCopyNaive
BenchmarkCopyNaive-4                2613            446601 ns/op         3130664 B/op          4 allocs/op
PASS
ok      github.com/pion/mediadevices/pkg/io/video       5.769s
```

lherman-cs: @at-wat I think the naming is really confusing. I've updated by renaming it to `FrameBuffer` instead, and changed the API to be `Load` and `Store` style. This way, it'll let the users aware the buffer is not meant to be shared across multiple goroutines unless there's a synchronization in place.

The design is inspired by atomic.Value, https://golang.org/pkg/sync/atomic/#Value.Load.

What do you think about this approach?
at-wat: `FrameBuffer` API sounds good to me!
lherman-cs: @at-wat updated the PR, PTAL.

Patch:
diff --git a/pkg/io/video/framebuffer.go b/pkg/io/video/framebuffer.go
new file mode 100644
index 0000000..1b909fd
--- /dev/null
+++ b/pkg/io/video/framebuffer.go
@@ -0,0 +1,214 @@
+package video
+
+import (
+	"image"
+)
+
+// FrameBuffer is a buffer that can store any image format.
+type FrameBuffer struct {
+	buffer []uint8
+	tmp    image.Image
+}
+
+// NewFrameBuffer creates a new FrameBuffer instance and initialize internal buffer
+// with initialSize
+func NewFrameBuffer(initialSize int) *FrameBuffer {
+	return &FrameBuffer{
+		buffer: make([]uint8, initialSize),
+	}
+}
+
+func (buff *FrameBuffer) storeInOrder(srcs ...[]uint8) {
+	var neededSize int
+
+	for _, src := range srcs {
+		neededSize += len(src)
+	}
+
+	if len(buff.buffer) < neededSize {
+		if cap(buff.buffer) >= neededSize {
+			buff.buffer = buff.buffer[:neededSize]
+		} else {
+			buff.buffer = make([]uint8, neededSize)
+		}
+	}
+
+	var currentLen int
+	for _, src := range srcs {
+		copy(buff.buffer[currentLen:], src)
+		currentLen += len(src)
+	}
+}
+
+// Load loads the current owned image
+func (buff *FrameBuffer) Load() image.Image {
+	return buff.tmp
+}
+
+// StoreCopy makes a copy of src and store its copy. StoreCopy will reuse as much memory as it can
+// from the previous copies. For example, if StoreCopy is given an image that has the same resolution
+// and format from the previous call, StoreCopy will not allocate extra memory and only copy the content
+// from src to the previous buffer.
+func (buff *FrameBuffer) StoreCopy(src image.Image) {
+	switch src := src.(type) {
+	case *image.Alpha:
+		clone, ok := buff.tmp.(*image.Alpha)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.Alpha16:
+		clone, ok := buff.tmp.(*image.Alpha16)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.CMYK:
+		clone, ok := buff.tmp.(*image.CMYK)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.Gray:
+		clone, ok := buff.tmp.(*image.Gray)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.Gray16:
+		clone, ok := buff.tmp.(*image.Gray16)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.NRGBA:
+		clone, ok := buff.tmp.(*image.NRGBA)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.NRGBA64:
+		clone, ok := buff.tmp.(*image.NRGBA64)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.RGBA:
+		clone, ok := buff.tmp.(*image.RGBA)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.RGBA64:
+		clone, ok := buff.tmp.(*image.RGBA64)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		buff.storeInOrder(src.Pix)
+		clone.Pix = buff.buffer[:len(src.Pix)]
+
+		buff.tmp = clone
+	case *image.NYCbCrA:
+		clone, ok := buff.tmp.(*image.NYCbCrA)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		var currentLen int
+		buff.storeInOrder(src.Y, src.Cb, src.Cr, src.A)
+		clone.Y = buff.buffer[currentLen : currentLen+len(src.Y) : currentLen+len(src.Y)]
+		currentLen += len(src.Y)
+		clone.Cb = buff.buffer[currentLen : currentLen+len(src.Cb) : currentLen+len(src.Cb)]
+		currentLen += len(src.Cb)
+		clone.Cr = buff.buffer[currentLen : currentLen+len(src.Cr) : currentLen+len(src.Cr)]
+		currentLen += len(src.Cr)
+		clone.A = buff.buffer[currentLen : currentLen+len(src.A) : currentLen+len(src.A)]
+
+		buff.tmp = clone
+	case *image.YCbCr:
+		clone, ok := buff.tmp.(*image.YCbCr)
+		if ok {
+			*clone = *src
+		} else {
+			copied := *src
+			clone = &copied
+		}
+
+		var currentLen int
+		buff.storeInOrder(src.Y, src.Cb, src.Cr)
+		clone.Y = buff.buffer[currentLen : currentLen+len(src.Y) : currentLen+len(src.Y)]
+		currentLen += len(src.Y)
+		clone.Cb = buff.buffer[currentLen : currentLen+len(src.Cb) : currentLen+len(src.Cb)]
+		currentLen += len(src.Cb)
+		clone.Cr = buff.buffer[currentLen : currentLen+len(src.Cr) : currentLen+len(src.Cr)]
+
+		buff.tmp = clone
+	default:
+		var converted image.RGBA
+		imageToRGBA(&converted, src)
+		buff.StoreCopy(&converted)
+	}
+}


Test patch:
diff --git a/pkg/io/video/framebuffer_test.go b/pkg/io/video/framebuffer_test.go
new file mode 100644
index 0000000..de63d0b
--- /dev/null
+++ b/pkg/io/video/framebuffer_test.go
@@ -0,0 +1,195 @@
+package video
+
+import (
+	"image"
+	"math/rand"
+	"reflect"
+	"testing"
+)
+
+func randomize(arr []uint8) {
+	for i := range arr {
+		arr[i] = uint8(rand.Uint32())
+	}
+}
+
+func BenchmarkFrameBufferCopyOptimized(b *testing.B) {
+	frameBuffer := NewFrameBuffer(0)
+	resolution := image.Rect(0, 0, 1920, 1080)
+	src := image.NewYCbCr(resolution, image.YCbCrSubsampleRatio420)
+
+	for i := 0; i < b.N; i++ {
+		frameBuffer.StoreCopy(src)
+	}
+}
+
+func BenchmarkFrameBufferCopyNaive(b *testing.B) {
+	resolution := image.Rect(0, 0, 1920, 1080)
+	src := image.NewYCbCr(resolution, image.YCbCrSubsampleRatio420)
+	var dst image.Image
+
+	for i := 0; i < b.N; i++ {
+		clone := *src
+		clone.Cb = make([]uint8, len(src.Cb))
+		clone.Cr = make([]uint8, len(src.Cr))
+		clone.Y = make([]uint8, len(src.Y))
+
+		copy(clone.Cb, src.Cb)
+		copy(clone.Cr, src.Cr)
+		copy(clone.Y, src.Y)
+		dst = &clone
+		_ = dst
+	}
+}
+
+func TestFrameBufferStoreCopyAndLoad(t *testing.T) {
+	resolution := image.Rect(0, 0, 16, 8)
+	rgbaLike := image.NewRGBA64(resolution)
+	randomize(rgbaLike.Pix)
+	testCases := map[string]struct {
+		New    func() image.Image
+		Update func(image.Image)
+	}{
+		"Alpha": {
+			New: func() image.Image {
+				return (*image.Alpha)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Alpha)
+				randomize(img.Pix)
+			},
+		},
+		"Alpha16": {
+			New: func() image.Image {
+				return (*image.Alpha16)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Alpha16)
+				randomize(img.Pix)
+			},
+		},
+		"CMYK": {
+			New: func() image.Image {
+				return (*image.CMYK)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.CMYK)
+				randomize(img.Pix)
+			},
+		},
+		"Gray": {
+			New: func() image.Image {
+				return (*image.Gray)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Gray)
+				randomize(img.Pix)
+			},
+		},
+		"Gray16": {
+			New: func() image.Image {
+				return (*image.Gray16)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.Gray16)
+				randomize(img.Pix)
+			},
+		},
+		"NRGBA": {
+			New: func() image.Image {
+				return (*image.NRGBA)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.NRGBA)
+				randomize(img.Pix)
+			},
+		},
+		"NRGBA64": {
+			New: func() image.Image {
+				return (*image.NRGBA64)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.NRGBA64)
+				randomize(img.Pix)
+			},
+		},
+		"RGBA": {
+			New: func() image.Image {
+				return (*image.RGBA)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.RGBA)
+				randomize(img.Pix)
+			},
+		},
+		"RGBA64": {
+			New: func() image.Image {
+				return (*image.RGBA64)(rgbaLike)
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.RGBA64)
+				randomize(img.Pix)
+			},
+		},
+		"NYCbCrA": {
+			New: func() image.Image {
+				img := image.NewNYCbCrA(resolution, image.YCbCrSubsampleRatio420)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				randomize(img.A)
+				img.CStride = 10
+				img.YStride = 5
+				return img
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.NYCbCrA)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				randomize(img.A)
+				img.CStride = 3
+				img.YStride = 2
+			},
+		},
+		"YCbCr": {
+			New: func() image.Image {
+				img := image.NewYCbCr(resolution, image.YCbCrSubsampleRatio420)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				img.CStride = 10
+				img.YStride = 5
+				return img
+			},
+			Update: func(src image.Image) {
+				img := src.(*image.YCbCr)
+				randomize(img.Y)
+				randomize(img.Cb)
+				randomize(img.Cr)
+				img.CStride = 3
+				img.YStride = 2
+			},
+		},
+	}
+
+	frameBuffer := NewFrameBuffer(0)
+
+	for name, testCase := range testCases {
+		// Since the test also wants to make sure that Copier can convert from 1 type to another,
+		// t.Run is not ideal since it'll run the tests separately
+		t.Log("Testing", name)
+
+		src := testCase.New()
+		frameBuffer.StoreCopy(src)
+		if !reflect.DeepEqual(frameBuffer.Load(), src) {
+			t.Fatal("Expected the copied image to be identical with the source")
+		}
+
+		testCase.Update(src)
+		frameBuffer.StoreCopy(src)
+		if !reflect.DeepEqual(frameBuffer.Load(), src) {
+			t.Fatal("Expected the copied image to be identical with the source after an update in source")
+		}
+	}
+}