Repository: pion/mediadevices
Pull request #196 (2020-07-05 00:08:56)

Add property change detection for video pipeline
#### Description
This will flexibly detect changes whenever the source transforms.

#### Reference issue
Solve the video side, https://github.com/pion/mediadevices/issues/171



Patch:
diff --git a/pkg/io/video/detect.go b/pkg/io/video/detect.go
new file mode 100644
index 0000000..078f7da
--- /dev/null
+++ b/pkg/io/video/detect.go
@@ -0,0 +1,58 @@
+package video
+
+import (
+	"image"
+	"time"
+
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+// DetectChanges will detect frame and video property changes. For video property detection,
+// since it's time related, interval will be used to determine the sample rate.
+func DetectChanges(interval time.Duration, onChange func(prop.Media)) TransformFunc {
+	return func(r Reader) Reader {
+		var currentProp prop.Media
+		var lastTaken time.Time
+		var frames uint
+		return ReaderFunc(func() (image.Image, error) {
+			var dirty bool
+
+			img, err := r.Read()
+			if err != nil {
+				return nil, err
+			}
+
+			bounds := img.Bounds()
+			if currentProp.Width != bounds.Dx() {
+				currentProp.Width = bounds.Dx()
+				dirty = true
+			}
+
+			if currentProp.Height != bounds.Dy() {
+				currentProp.Height = bounds.Dy()
+				dirty = true
+			}
+
+			// TODO: maybe detect frame format? It probably doesn't make sense since some
+			// formats only are about memory layout, e.g. YUV2 vs NV12.
+
+			now := time.Now()
+			elapsed := now.Sub(lastTaken)
+			if elapsed >= interval {
+				fps := float32(float64(frames) / elapsed.Seconds())
+				// TODO: maybe add some epsilon so that small changes will not mark as dirty
+				currentProp.FrameRate = fps
+				frames = 0
+				lastTaken = now
+				dirty = true
+			}
+
+			if dirty {
+				onChange(currentProp)
+			}
+
+			frames++
+			return img, nil
+		})
+	}
+}


Test patch:
diff --git a/pkg/io/video/detect_test.go b/pkg/io/video/detect_test.go
new file mode 100644
index 0000000..bc7b78a
--- /dev/null
+++ b/pkg/io/video/detect_test.go
@@ -0,0 +1,152 @@
+package video
+
+import (
+	"fmt"
+	"image"
+	"testing"
+	"time"
+
+	"github.com/pion/mediadevices/pkg/prop"
+)
+
+func BenchmarkDetectChanges(b *testing.B) {
+	var src Reader
+	src = ReaderFunc(func() (image.Image, error) {
+		return image.NewRGBA(image.Rect(0, 0, 1920, 1080)), nil
+	})
+
+	b.Run("WithoutDetectChanges", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			src.Read()
+		}
+	})
+
+	ns := []int{1, 8, 64, 256}
+	for _, n := range ns {
+		n := n
+		src := src
+		b.Run(fmt.Sprintf("WithDetectChanges%d", n), func(b *testing.B) {
+			for i := 0; i < n; i++ {
+				src = DetectChanges(time.Microsecond, func(p prop.Media) {})(src)
+			}
+
+			for i := 0; i < b.N; i++ {
+				src.Read()
+			}
+		})
+	}
+}
+
+func TestDetectChanges(t *testing.T) {
+	buildSource := func(p prop.Media) (Reader, func(prop.Media)) {
+		return ReaderFunc(func() (image.Image, error) {
+				return image.NewRGBA(image.Rect(0, 0, p.Width, p.Height)), nil
+			}), func(newProp prop.Media) {
+				p = newProp
+			}
+	}
+
+	assertEq := func(t *testing.T, actual prop.Media, expected prop.Media, output image.Image, assertFrameRate bool) {
+		if actual.Height != expected.Height {
+			t.Fatalf("expected height from to be %d but got %d", expected.Height, actual.Height)
+		}
+
+		if actual.Width != expected.Width {
+			t.Fatalf("expected width from to be %d but got %d", expected.Width, actual.Width)
+		}
+
+		if assertFrameRate {
+			diff := actual.FrameRate - expected.FrameRate
+			// TODO: reduce this eps. Darwin CI keeps failing if we use a lower value
+			var eps float32 = 1.5
+			if diff < -eps || diff > eps {
+				t.Fatalf("expected frame rate to be %f (+-%f) but got %f", expected.FrameRate, eps, actual.FrameRate)
+			}
+		}
+
+		if output.Bounds().Dy() != expected.Height {
+			t.Fatalf("expected output height from to be %d but got %d", expected.Height, output.Bounds().Dy())
+		}
+
+		if output.Bounds().Dx() != expected.Width {
+			t.Fatalf("expected output width from to be %d but got %d", expected.Width, output.Bounds().Dx())
+		}
+	}
+
+	t.Run("OnChangeCalledBeforeFirstFrame", func(t *testing.T) {
+		var detectBeforeFirstFrame bool
+		var expected prop.Media
+		var actual prop.Media
+		expected.Width = 1920
+		expected.Height = 1080
+		src, _ := buildSource(expected)
+		src = DetectChanges(time.Second, func(p prop.Media) {
+			actual = p
+			detectBeforeFirstFrame = true
+		})(src)
+
+		frame, err := src.Read()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !detectBeforeFirstFrame {
+			t.Fatal("on change callback should have called before first frame")
+		}
+
+		assertEq(t, actual, expected, frame, false)
+	})
+
+	t.Run("DetectChangesOnEveryUpdate", func(t *testing.T) {
+		var expected prop.Media
+		var actual prop.Media
+		expected.Width = 1920
+		expected.Height = 1080
+		src, update := buildSource(expected)
+		src = DetectChanges(time.Second, func(p prop.Media) {
+			actual = p
+		})(src)
+
+		for width := 1920; width < 4000; width += 100 {
+			for height := 1080; height < 2000; height += 100 {
+				expected.Width = width
+				expected.Height = height
+				update(expected)
+				frame, err := src.Read()
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				assertEq(t, actual, expected, frame, false)
+			}
+		}
+	})
+
+	t.Run("FrameRateAccuracy", func(t *testing.T) {
+		var expected prop.Media
+		var actual prop.Media
+		var count int
+		expected.Width = 1920
+		expected.Height = 1080
+		expected.FrameRate = 30
+		src, _ := buildSource(expected)
+		src = Throttle(expected.FrameRate)(src)
+		src = DetectChanges(time.Second*5, func(p prop.Media) {
+			actual = p
+			count++
+		})(src)
+
+		for count < 3 {
+			frame, err := src.Read()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			checkFrameRate := false
+			if actual.FrameRate != 0.0 {
+				checkFrameRate = true
+			}
+			assertEq(t, actual, expected, frame, checkFrameRate)
+		}
+	})
+}