Repository: pion/mediadevices
Pull request #56 (2020-02-15 10:14:59)

Fix color conversions
- Fix R-B byte order in screen capture
- Set image metadata everytime in screen capture
- Fix pixel format conversion from I444

Fixed version:
![Screenshot_2020-02-15 Edit fiddle - JSFiddle](https://user-images.githubusercontent.com/8390204/74586097-6106be80-5027-11ea-8e87-179ca8864e0a.png)



Patch:
diff --git a/pkg/driver/screen/x11_linux.go b/pkg/driver/screen/x11_linux.go
index 729cfb4..e942116 100644
--- a/pkg/driver/screen/x11_linux.go
+++ b/pkg/driver/screen/x11_linux.go
@@ -62,8 +62,8 @@ func (s *screen) Close() error {
 
 func (s *screen) VideoRecord(p prop.Media) (video.Reader, error) {
 	rect := s.reader.img.Bounds()
-	w := rect.Max.X - rect.Min.X
-	h := rect.Max.Y - rect.Min.Y
+	w := rect.Dx()
+	h := rect.Dy()
 	imgI444 := image.NewYCbCr(rect, image.YCbCrSubsampleRatio444)
 
 	if p.FrameRate == 0 {
@@ -75,6 +75,10 @@ func (s *screen) VideoRecord(p prop.Media) (video.Reader, error) {
 		<-s.tick.C
 		img := s.reader.Read()
 		// Convert it to I444
+		imgI444.SubsampleRatio = image.YCbCrSubsampleRatio444
+		imgI444.Rect = rect
+		imgI444.YStride = w
+		imgI444.CStride = w
 		for y := 0; y < h; y++ {
 			iyBase := y * imgI444.YStride
 			icBase := y * imgI444.CStride
@@ -95,8 +99,8 @@ func (s *screen) VideoRecord(p prop.Media) (video.Reader, error) {
 
 func (s *screen) Properties() []prop.Media {
 	rect := s.reader.img.Bounds()
-	w := rect.Max.X - rect.Min.X
-	h := rect.Max.Y - rect.Min.Y
+	w := rect.Dx()
+	h := rect.Dy()
 	return []prop.Media{
 		{
 			DeviceID: deviceID(s.num),
diff --git a/pkg/driver/screen/x11capture_linux.go b/pkg/driver/screen/x11capture_linux.go
index aac976a..056d559 100644
--- a/pkg/driver/screen/x11capture_linux.go
+++ b/pkg/driver/screen/x11capture_linux.go
@@ -75,9 +75,9 @@ func (c colorFunc) RGBA() (r, g, b, a uint32) {
 
 func (s *shmImage) At(x, y int) color.Color {
 	addr := (x + y*int(s.img.width)) * 4
-	r := uint32(s.b[addr]) * 0x100
+	b := uint32(s.b[addr]) * 0x100
 	g := uint32(s.b[addr+1]) * 0x100
-	b := uint32(s.b[addr+2]) * 0x100
+	r := uint32(s.b[addr+2]) * 0x100
 	a := uint32(s.b[addr+3]) * 0x100
 	return colorFunc(func() (_, _, _, _ uint32) {
 		return r, g, b, a
diff --git a/pkg/io/video/convert.go b/pkg/io/video/convert.go
index 51d4809..b297a39 100644
--- a/pkg/io/video/convert.go
+++ b/pkg/io/video/convert.go
@@ -15,7 +15,7 @@ func ToI420(r Reader) Reader {
 
 		// TODO: Not sure how to handle this when it's not YCbCr, maybe try to convert it to YCvCr?
 		yuvImg := img.(*image.YCbCr)
-		h := yuvImg.Rect.Max.Y - yuvImg.Rect.Min.Y
+		h := yuvImg.Rect.Dy()
 
 		// Covert pixel format to I420
 		switch yuvImg.SubsampleRatio {
@@ -24,15 +24,19 @@ func ToI420(r Reader) Reader {
 				addrSrc := i * 2 * yuvImg.CStride
 				addrDst := i * yuvImg.CStride / 2
 				for j := 0; j < yuvImg.CStride/2; j++ {
-					cb := uint16(yuvImg.Cb[addrSrc+j]) + uint16(yuvImg.Cb[addrSrc+yuvImg.CStride+j]) +
-						uint16(yuvImg.Cb[addrSrc+j+1]) + uint16(yuvImg.Cb[addrSrc+yuvImg.CStride+j+1])
-					cr := uint16(yuvImg.Cr[addrSrc+j]) + uint16(yuvImg.Cr[addrSrc+yuvImg.CStride+j]) +
-						uint16(yuvImg.Cr[addrSrc+j+1]) + uint16(yuvImg.Cr[addrSrc+yuvImg.CStride+j+1])
+					j2 := j * 2
+					cb := uint16(yuvImg.Cb[addrSrc+j2]) + uint16(yuvImg.Cb[addrSrc+yuvImg.CStride+j2]) +
+						uint16(yuvImg.Cb[addrSrc+j2+1]) + uint16(yuvImg.Cb[addrSrc+yuvImg.CStride+j2+1])
+					cr := uint16(yuvImg.Cr[addrSrc+j2]) + uint16(yuvImg.Cr[addrSrc+yuvImg.CStride+j2]) +
+						uint16(yuvImg.Cr[addrSrc+j2+1]) + uint16(yuvImg.Cr[addrSrc+yuvImg.CStride+j2+1])
 					yuvImg.Cb[addrDst+j] = uint8(cb / 4)
 					yuvImg.Cr[addrDst+j] = uint8(cr / 4)
 				}
 			}
 			yuvImg.CStride = yuvImg.CStride / 2
+			cLen := yuvImg.CStride * (h / 2)
+			yuvImg.Cb = yuvImg.Cb[:cLen:cLen]
+			yuvImg.Cr = yuvImg.Cr[:cLen:cLen]
 		case image.YCbCrSubsampleRatio422:
 			for i := 0; i < h/2; i++ {
 				addrSrc := i * 2 * yuvImg.CStride
@@ -44,11 +48,15 @@ func ToI420(r Reader) Reader {
 					yuvImg.Cr[addrDst+j] = uint8(cr / 2)
 				}
 			}
+			cLen := yuvImg.CStride * (h / 2)
+			yuvImg.Cb = yuvImg.Cb[:cLen:cLen]
+			yuvImg.Cr = yuvImg.Cr[:cLen:cLen]
 		case image.YCbCrSubsampleRatio420:
 		default:
 			return nil, fmt.Errorf("unsupported pixel format: %s", yuvImg.SubsampleRatio)
 		}
 
+		yuvImg.SubsampleRatio = image.YCbCrSubsampleRatio420
 		return yuvImg, nil
 	})
 }


Test patch:
diff --git a/pkg/io/video/convert_test.go b/pkg/io/video/convert_test.go
new file mode 100644
index 0000000..8af92ac
--- /dev/null
+++ b/pkg/io/video/convert_test.go
@@ -0,0 +1,122 @@
+package video
+
+import (
+	"image"
+	"reflect"
+	"testing"
+)
+
+func TestToI420(t *testing.T) {
+	cases := map[string]struct {
+		src      image.Image
+		expected image.Image
+	}{
+		"I444": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio444,
+				Y: []uint8{
+					0xF0, 0x10, 0x00, 0x00,
+					0x00, 0x00, 0x40, 0x00,
+					0x00, 0x00, 0x00, 0x00,
+					0x00, 0x80, 0x30, 0x00,
+				},
+				Cb: []uint8{
+					0xF0, 0xF0, 0x80, 0x80,
+					0xF0, 0xF0, 0x80, 0x80,
+					0x80, 0x80, 0x30, 0x30,
+					0x80, 0x80, 0x30, 0x30,
+				},
+				Cr: []uint8{
+					0x10, 0x10, 0x40, 0x40,
+					0x10, 0x10, 0x40, 0x40,
+					0x80, 0x80, 0x80, 0x80,
+					0x80, 0x80, 0x80, 0x80,
+				},
+				YStride: 4,
+				CStride: 4,
+				Rect:    image.Rect(0, 0, 4, 4),
+			},
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0x10, 0x00, 0x00,
+					0x00, 0x00, 0x40, 0x00,
+					0x00, 0x00, 0x00, 0x00,
+					0x00, 0x80, 0x30, 0x00,
+				},
+				Cb: []uint8{
+					0xF0, 0x80,
+					0x80, 0x30,
+				},
+				Cr: []uint8{
+					0x10, 0x40,
+					0x80, 0x80,
+				},
+				YStride: 4,
+				CStride: 2,
+				Rect:    image.Rect(0, 0, 4, 4),
+			},
+		},
+		"I422": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio422,
+				Y: []uint8{
+					0xF0, 0x10, 0x00, 0x00,
+					0x00, 0x00, 0x40, 0x00,
+					0x00, 0x00, 0x00, 0x00,
+					0x00, 0x80, 0x30, 0x00,
+				},
+				Cb: []uint8{
+					0xF0, 0x80,
+					0xF0, 0x80,
+					0x80, 0x30,
+					0x80, 0x30,
+				},
+				Cr: []uint8{
+					0x10, 0x40,
+					0x10, 0x40,
+					0x80, 0x80,
+					0x80, 0x80,
+				},
+				YStride: 4,
+				CStride: 2,
+				Rect:    image.Rect(0, 0, 4, 4),
+			},
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0x10, 0x00, 0x00,
+					0x00, 0x00, 0x40, 0x00,
+					0x00, 0x00, 0x00, 0x00,
+					0x00, 0x80, 0x30, 0x00,
+				},
+				Cb: []uint8{
+					0xF0, 0x80,
+					0x80, 0x30,
+				},
+				Cr: []uint8{
+					0x10, 0x40,
+					0x80, 0x80,
+				},
+				YStride: 4,
+				CStride: 2,
+				Rect:    image.Rect(0, 0, 4, 4),
+			},
+		},
+	}
+	for name, c := range cases {
+		c := c
+		t.Run(name, func(t *testing.T) {
+			r := ToI420(ReaderFunc(func() (image.Image, error) {
+				return c.src, nil
+			}))
+			out, err := r.Read()
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			if !reflect.DeepEqual(c.expected, out) {
+				t.Errorf("Expected output image:\n%v\ngot:\n%v\n", c.expected, out)
+			}
+		})
+	}
+}