Repository: pion/rtp
Pull request #112 (2021-01-08 14:16:18)

Pack SPS and PPS in one STAP-A packet
Chrome need SPS and PPS in one STAP-A, and timestamp of STAP-A should be same with next I-Frame. Otherwise Chrome will discard SPS and PPS before receiving I frame.

#### Reference issue
Fixes https://github.com/pion/webrtc/issues/1625

codecov[bot]: # [Codecov](https://codecov.io/gh/pion/rtp/pull/112?src=pr&el=h1) Report
> Merging [#112](https://codecov.io/gh/pion/rtp/pull/112?src=pr&el=desc) (71fde37) into [master](https://codecov.io/gh/pion/rtp/commit/b58e1cba07f8cf32c0b42167e3c9df8f8635d6c2?el=desc) (b58e1cb) will **decrease** coverage by `3.10%`.
> The diff coverage is `0.00%`.

[![Impacted file tree graph](https://codecov.io/gh/pion/rtp/pull/112/graphs/tree.svg?width=650&height=150&src=pr&token=4KrbiBsspj)](https://codecov.io/gh/pion/rtp/pull/112?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #112      +/-   ##
==========================================
- Coverage   91.37%   88.27%   -3.11%     
==========================================
  Files          13       13              
  Lines         626      648      +22     
==========================================
  Hits          572      572              
- Misses         37       56      +19     
- Partials       17       20       +3     
```

| Flag | Coverage Δ | |
|---|---|---|
| go | `88.27% <0.00%> (-3.11%)` | :arrow_down: |
| wasm | `87.65% <0.00%> (-3.09%)` | :arrow_down: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Impacted Files](https://codecov.io/gh/pion/rtp/pull/112?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [codecs/h264\_packet.go](https://codecov.io/gh/pion/rtp/pull/112/diff?src=pr&el=tree#diff-Y29kZWNzL2gyNjRfcGFja2V0Lmdv) | `79.31% <0.00%> (-18.57%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/pion/rtp/pull/112?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/pion/rtp/pull/112?src=pr&el=footer). Last update [b58e1cb...71fde37](https://codecov.io/gh/pion/rtp/pull/112?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).

caiiiycuk: Hi @baiyufei,  you fixes work fine with chrome. But FF shows error and does not start streaming.
Error:
```
Answer tried to set recv when offer did not set send
```

It also reproduced in your test repo.
--
Firefox 89.0.2 Ubuntu

Sean-Der: This is great @baiyufei thank you so much! Sorry for the delay, I am finally getting around to a `v3.1.0`

What do you think of emitting the STAP-A [here](https://github.com/pion/rtp/pull/112/files#diff-5eca440e933baa4c5491dcd9ccc7c1f4d807994e6c5b54c304228c34fcc3c963R92) instead? Do we need to wait for the next call to packetize?
Sean-Der: @baiyufei I just made the change, if you agree I will merge that. If it needs to stay as it was before I will rollback.

thanks!
Sean-Der: Confirmed your fix is needed as is, nice work @baiyufei !

Patch:
diff --git a/codecs/h264_packet.go b/codecs/h264_packet.go
index a1e0381..40d207f 100644
--- a/codecs/h264_packet.go
+++ b/codecs/h264_packet.go
@@ -6,12 +6,18 @@ import (
 )
 
 // H264Payloader payloads H264 packets
-type H264Payloader struct{}
+type H264Payloader struct {
+	spsNalu, ppsNalu []byte
+}
 
 const (
-	stapaNALUType = 24
-	fuaNALUType   = 28
-	fubNALUType   = 29
+	stapaNALUType  = 24
+	fuaNALUType    = 28
+	fubNALUType    = 29
+	spsNALUType    = 7
+	ppsNALUType    = 8
+	audNALUType    = 9
+	fillerNALUType = 12
 
 	fuaHeaderSize       = 2
 	stapaHeaderSize     = 1
@@ -21,6 +27,8 @@ const (
 	naluRefIdcBitmask = 0x60
 	fuStartBitmask    = 0x80
 	fuEndBitmask      = 0x40
+
+	outputStapAHeader = 0x78
 )
 
 func annexbNALUStartCode() []byte { return []byte{0x00, 0x00, 0x00, 0x01} }
@@ -75,8 +83,36 @@ func (p *H264Payloader) Payload(mtu uint16, payload []byte) [][]byte {
 		naluType := nalu[0] & naluTypeBitmask
 		naluRefIdc := nalu[0] & naluRefIdcBitmask
 
-		if naluType == 9 || naluType == 12 {
+		switch {
+		case naluType == audNALUType || naluType == fillerNALUType:
+			return
+		case naluType == spsNALUType:
+			p.spsNalu = nalu
 			return
+		case naluType == ppsNALUType:
+			p.ppsNalu = nalu
+			return
+		case p.spsNalu != nil && p.ppsNalu != nil:
+			// Pack current NALU with SPS and PPS as STAP-A
+			spsLen := make([]byte, 2)
+			binary.BigEndian.PutUint16(spsLen, uint16(len(p.spsNalu)))
+
+			ppsLen := make([]byte, 2)
+			binary.BigEndian.PutUint16(ppsLen, uint16(len(p.ppsNalu)))
+
+			stapANalu := []byte{outputStapAHeader}
+			stapANalu = append(stapANalu, spsLen...)
+			stapANalu = append(stapANalu, p.spsNalu...)
+			stapANalu = append(stapANalu, ppsLen...)
+			stapANalu = append(stapANalu, p.ppsNalu...)
+			if len(stapANalu) <= int(mtu) {
+				out := make([]byte, len(stapANalu))
+				copy(out, stapANalu)
+				payloads = append(payloads, out)
+			}
+
+			p.spsNalu = nil
+			p.ppsNalu = nil
 		}
 
 		// Single NALU


Test patch:
diff --git a/codecs/h264_packet_test.go b/codecs/h264_packet_test.go
index 4ee7fd2..608b020 100644
--- a/codecs/h264_packet_test.go
+++ b/codecs/h264_packet_test.go
@@ -222,3 +222,26 @@ func TestH264PartitionHeadChecker_IsPartitionHead(t *testing.T) {
 		t.Fatal("fub end nalu must not be a partition head")
 	}
 }
+
+func TestH264Payloader_Payload_SPS_and_PPS_handling(t *testing.T) {
+	pck := H264Payloader{}
+	expected := [][]byte{
+		{0x78, 0x00, 0x03, 0x07, 0x00, 0x01, 0x00, 0x03, 0x08, 0x02, 0x03},
+		{0x05, 0x04, 0x05},
+	}
+
+	// When packetizing SPS and PPS are emitted with following NALU
+	res := pck.Payload(1500, []byte{0x07, 0x00, 0x01})
+	if len(res) != 0 {
+		t.Fatal("Generated payload should be empty")
+	}
+
+	res = pck.Payload(1500, []byte{0x08, 0x02, 0x03})
+	if len(res) != 0 {
+		t.Fatal("Generated payload should be empty")
+	}
+
+	if !reflect.DeepEqual(pck.Payload(1500, []byte{0x05, 0x04, 0x05}), expected) {
+		t.Fatal("SPS and PPS aren't packed together")
+	}
+}