Playing video in a Golang game
Tags: gamedev, go
Published: 2021-01-04 04:33:16

Please proceed [here](https://medium.com/@maximgradan/playing-videos-with-golang-83e67447b111) for a new version of the article. The method described below doesn't work anymore (and it's not really good either).

In videogames, we often need to show the player beautiful cinematics, either for ending or in the middle of the playthrough. To do it, we need to somehow load video frames and render them on the screen. Here's how to do that using [Pixel game library](https://github.com/faiface/pixel) and [goav](https://github.com/giorgisio/goav), **Golang** bindings for **FFmpeg**.

## Initial setup

First we need to create a **GLFW** window for rendering **OpenGL** stuff:

```go
package main

import (
	"fmt"

	"github.com/faiface/pixel"
	"github.com/faiface/pixel/pixelgl"
	colors "golang.org/x/image/colornames"
)

const (
	// WindowWidth is the width of the window.
	WindowWidth = 1280
	// WindowHeight is the height of the window.
	WindowHeight = 720
)

func run() {
	// Create a new window.
	cfg := pixelgl.WindowConfig{
		Title:  "Pixel Rocks!",
		Bounds: pixel.R(0, 0,
			float64(WindowWidth), float64(WindowHeight)),
		VSync:  false,
	}
	win, err := pixelgl.NewWindow(cfg)
	handleError(err)

	fps := 0
	perSecond := time.Tick(time.Second)

	for !win.Closed() {
		win.Clear(colors.White)

		win.Update()

		// Show FPS in the window title.
		fps++

		select {
		case <-perSecond:
			win.SetTitle(fmt.Sprintf("%s | FPS: %d",
				cfg.Title, fps))
            fps = 0

		default:
		}
	}
}

func main() {
	pixelgl.Run(run)
}

func handleError(err error) {
	if err != nil {
		panic(err)
	}
}
```

## Obtaining video frames

Now we need to obtain frames from the video stream of the file. **goav** provides an [example code](https://github.com/giorgisio/goav/blob/master/example/tutorial01.go) on how to do that. To make it work with **Pixel** game library, we need to set the frame decoding format to `avcodec.AV_PIX_FMT_RGBA` which is used by **Pixel**.

We also need a way to send the decoded frames to the renderer. For this task we will use a thread-safe frame buffer channel. First we should specify its size:

```go
const (
	FrameBufferSize = 1024
)
```

The greater the buffer size, the faster the frame transfer from the decoder to the renderer. Now to the channel creation:

```go
frameBuffer := make(chan *pixel.PictureData, FrameBufferSize)
```

When the frame transfer is complete, we need to close the channel, but we also have to make sure the renderer got all the frames from the buffer. So here's the code for closing the frame buffer:

```go
go func() {
    for {
        if len(frameBuffer) <= 0 {
            close(frameBuffer)
            break
        }
    }
}()
```

The complete code for reading video frames is presented below:

```go
func readVideoFrames(videoPath string) <-chan *pixel.PictureData {
	// Create a frame buffer.
	frameBuffer := make(chan *pixel.PictureData, FrameBufferSize)

	go func() {
		// Open a video file.
		pFormatContext := avformat.AvformatAllocContext()

		if avformat.AvformatOpenInput(&pFormatContext, videoPath, nil, nil) != 0 {
			fmt.Printf("Unable to open file %s\n", videoPath)
			os.Exit(1)
		}

		// Retrieve the stream information.
		if pFormatContext.AvformatFindStreamInfo(nil) < 0 {
			fmt.Println("Couldn't find stream information")
			os.Exit(1)
		}

		// Dump information about the video to stderr.
		pFormatContext.AvDumpFormat(0, videoPath, 0)

		// Find the first video stream
		for i := 0; i < int(pFormatContext.NbStreams()); i++ {
			switch pFormatContext.Streams()[i].
				CodecParameters().AvCodecGetType() {
			case avformat.AVMEDIA_TYPE_VIDEO:

				// Get a pointer to the codec context for the video stream
				pCodecCtxOrig := pFormatContext.Streams()[i].Codec()
				// Find the decoder for the video stream
				pCodec := avcodec.AvcodecFindDecoder(avcodec.
					CodecId(pCodecCtxOrig.GetCodecId()))

				if pCodec == nil {
					fmt.Println("Unsupported codec!")
					os.Exit(1)
				}

				// Copy context
				pCodecCtx := pCodec.AvcodecAllocContext3()

				if pCodecCtx.AvcodecCopyContext((*avcodec.
					Context)(unsafe.Pointer(pCodecCtxOrig))) != 0 {
					fmt.Println("Couldn't copy codec context")
					os.Exit(1)
				}

				// Open codec
				if pCodecCtx.AvcodecOpen2(pCodec, nil) < 0 {
					fmt.Println("Could not open codec")
					os.Exit(1)
				}

				// Allocate video frame
				pFrame := avutil.AvFrameAlloc()

				// Allocate an AVFrame structure
				pFrameRGB := avutil.AvFrameAlloc()

				if pFrameRGB == nil {
					fmt.Println("Unable to allocate RGB Frame")
					os.Exit(1)
				}

				// Determine required buffer size and allocate buffer
				numBytes := uintptr(avcodec.AvpictureGetSize(
					avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(),
					pCodecCtx.Height()))
				buffer := avutil.AvMalloc(numBytes)

				// Assign appropriate parts of buffer to image planes in pFrameRGB
				// Note that pFrameRGB is an AVFrame, but AVFrame is a superset
				// of AVPicture
				avp := (*avcodec.Picture)(unsafe.Pointer(pFrameRGB))
				avp.AvpictureFill((*uint8)(buffer),
					avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(), pCodecCtx.Height())

				// initialize SWS context for software scaling
				swsCtx := swscale.SwsGetcontext(
					pCodecCtx.Width(),
					pCodecCtx.Height(),
					(swscale.PixelFormat)(pCodecCtx.PixFmt()),
					pCodecCtx.Width(),
					pCodecCtx.Height(),
					avcodec.AV_PIX_FMT_RGBA,
					avcodec.SWS_BILINEAR,
					nil,
					nil,
					nil,
				)

				// Read frames and save first five frames to disk
				packet := avcodec.AvPacketAlloc()

				for pFormatContext.AvReadFrame(packet) >= 0 {
					// Is this a packet from the video stream?
					if packet.StreamIndex() == i {
						// Decode video frame
						response := pCodecCtx.AvcodecSendPacket(packet)

						if response < 0 {
							fmt.Printf("Error while sending a packet to the decoder: %s\n",
								avutil.ErrorFromCode(response))
						}

						for response >= 0 {
							response = pCodecCtx.AvcodecReceiveFrame(
								(*avcodec.Frame)(unsafe.Pointer(pFrame)))

							if response == avutil.AvErrorEAGAIN ||
								response == avutil.AvErrorEOF {
								break
							} else if response < 0 {
								//fmt.Printf("Error while receiving a frame from the decoder: %s\n",
								//avutil.ErrorFromCode(response))

								//return
							}

							// Convert the image from its native format to RGB
							swscale.SwsScale2(swsCtx, avutil.Data(pFrame),
								avutil.Linesize(pFrame), 0, pCodecCtx.Height(),
								avutil.Data(pFrameRGB), avutil.Linesize(pFrameRGB))

							// Save the frame to the frame buffer.
							frame := getFrameRGBA(pFrameRGB,
								pCodecCtx.Width(), pCodecCtx.Height())
							frameBuffer <- frame
						}
					}

					// Free the packet that was allocated by av_read_frame
					packet.AvFreePacket()
				}

				go func() {
					for {
						if len(frameBuffer) <= 0 {
							close(frameBuffer)
							break
						}
					}
				}()

				// Free the RGB image
				avutil.AvFree(buffer)
				avutil.AvFrameFree(pFrameRGB)

				// Free the YUV frame
				avutil.AvFrameFree(pFrame)

				// Close the codecs
				pCodecCtx.AvcodecClose()
				(*avcodec.Context)(unsafe.Pointer(pCodecCtxOrig)).AvcodecClose()

				// Close the video file
				pFormatContext.AvformatCloseInput()

				// Stop after saving frames of first video straem
				break

			default:
				fmt.Println("Didn't find a video stream")
				os.Exit(1)
			}
		}
	}()

	return frameBuffer
}
```

We allocate a separate goroutine for the frame decoder so it can do its work in its own pace and just put all the results in the frame buffer.

## Converting video frames

Now we need to bring the extracted video frame to the form appropriate for **Pixel**. First we should extract raw **RGBA** bytes:

```go
func getFrameRGBA(frame *avutil.Frame, width, height int) *pixel.PictureData {
	pix := []byte{}

	for y := 0; y < height; y++ {
		data0 := avutil.Data(frame)[0]
		buf := make([]byte, width*4)
		startPos := uintptr(unsafe.Pointer(data0)) +
			uintptr(y)*uintptr(avutil.Linesize(frame)[0])

		for i := 0; i < width*4; i++ {
			element := *(*uint8)(unsafe.Pointer(startPos + uintptr(i)))
			buf[i] = element
		}

		pix = append(pix, buf...)
	}

	return pixToPictureData(pix, width, height)
}
```

To create `*pixel.PictureData` out of these bytes, we'll use this simple code:

```go
func pixToPictureData(pixels []byte, width, height int) *pixel.PictureData {
	picData := pixel.MakePictureData(pixel.
		R(0, 0, float64(width), float64(height)))

	for y := height - 1; y >= 0; y-- {
		for x := 0; x < width; x++ {
			picData.Pix[(height-y-1)*width+x].R = pixels[y*width*4+x*4+0]
			picData.Pix[(height-y-1)*width+x].G = pixels[y*width*4+x*4+1]
			picData.Pix[(height-y-1)*width+x].B = pixels[y*width*4+x*4+2]
			picData.Pix[(height-y-1)*width+x].A = pixels[y*width*4+x*4+3]
		}
	}

	return picData
}
```

We need to fill the `Pix` array vice versa because this is how **Pixel** treats picture data.

## Rendering video frames

Now let's create a new animated sprite to output the video frames:

```go
videoSprite := pixel.NewSprite(nil, pixel.Rect{})
videoTransform := pixel.IM.Moved(pixel.V(
    float64(WindowWidth)/2, float64(WindowHeight)/2))
frameBuffer := readVideoFrames(os.Args[1])
```

The path to the video file is specified as a command line argument.

Then let's start rendering the video frames:

```go
select {
    case frame, ok: = <-frameBuffer:
        if !ok {
            os.Exit(0)
        }

        if frame != nil {
            videoSprite.Set(frame, frame.Rect)
        }

    default:
}

videoSprite.Draw(win, videoTransform)
```

Here's the result:

![Alt Text](https://dev-to-uploads.s3.amazonaws.com/i/nlb1qi9re9lb5p6gqplx.png)

This method can be adapted for other **Golang** game engines like [Ebiten](https://github.com/hajimehoshi/ebiten) if you know the way to convert **RGBA** bytes to the appropriate form.

Enjoy the full [source code](https://github.com/zergon321/pixel-video).