Repository: ytdl-org/youtube-dl

Issue:
Add support for gbnews.uk
## Checklist

- [x] I'm reporting a new site support request
- [x] I've verified that I'm running youtube-dl version **2021.06.06**
- [x] I've checked that all provided URLs are alive and playable in a browser
- [x] I've checked that none of provided URLs violate any copyrights
- [x] I've searched the bugtracker for similar site support requests including closed ones

## Example URLs

- Video: https://www.gbnews.uk/shows/andrew-neils-message-to-companies-choosing-to-boycott-gb-news/106889
- Live: https://www.gbnews.uk/watchlive

## Description

GB News uses simplestreamcdn for videos. Despite mention of widevine and fairplay in the js, they don't seem to serve drm-protected content.

I was able to make a shell script to negotiate with the simplestreamcdn api using jq and httpie. Might help with adding support. I'm unsure if sending "heartbeats" is required to keep the livestream working

```bash
function gbnews-dl () {
    local SSMP="https://mm-dev.simplestream.com/ssmp/api.php"
    local PLATFORM="safari" # Stored as a global "platform". Likely could effect if drm is served

    local article=$(http -b "$1")
    local element=$(echo "$article" | xmllint --html --xpath "//*[@class='simplestream']" - 2>/dev/null)

    if [ -z "$element" ]; then
        echo "gbnews-dl: Can't find simplestream element!" 1>&2
        return 1
    fi

    local datasetId=$(echo "$element" | xmllint --xpath "string(/div/@data-id)" -)
    local datasetEnv=$(echo "$element" | xmllint --xpath "string(/div/@data-env)" -)
    local datasetKey=$(echo "$element" | xmllint --xpath "string(/div/@data-key)" -)
    local datasetType=$(echo "$element" | xmllint --xpath "string(/div/@data-type)" -)
    local datasetUvid=$(echo "$element" | xmllint --xpath "string(/div/@data-uvid)" -)
    local datasetToken=$(echo "$element" | xmllint --xpath "string(/div/@data-token)" -)
    local datasetExpiry=$(echo "$element" | xmllint --xpath "string(/div/@data-expiry)" -)

    if [ "$datasetType" = "vod" ]; then
        datasetType="show"
    fi

    echo "gbnews-dl: Attempt lookup for $datasetId" 1>&2

    local sSMeta=$(http -b "$SSMP" "id==$datasetId" "env==$datasetEnv")
    echo "$sSMeta" | jq 2>&1 1>/dev/null
    if [ "$?" != 0 ]; then
        echo "gbnews-dl: Failed to parse metadata server response. Not valid JSON" 1>&2
        echo "$sSMeta" 1>&2
        return 1
    fi

    local sSOrigin=$(echo "$sSMeta" | jq -r '.response?.api_hostname')

    if [ -z "$sSOrigin" ]; then
        echo "gbnews-dl: No API hostname returned. Blocked by server?" 1>&2
        echo "$sSMeta" | jq '.' 1>&2
        return 1
    fi

    local streamMeta=$(http -b "$sSOrigin/api/$datasetType/stream/$datasetUvid" \
                       "key==$datasetKey" "platform==$PLATFORM" "Token:$datasetToken" \
                       "Token-Expiry:$datasetExpiry" "Uvid:$datasetUvid")
    echo "$streamMeta" | jq 2>&1 1>/dev/null
    if [ "$?" != 0 ]; then
        echo "gbnews-dl: Failed to parse negotiation server response. Not valid JSON" 1>&2
        echo "$streamMeta" 1>&2
        return 1
    fi

    local streamUsesDRM=$(echo "$streamMeta" | jq -r '.response?.drm != null')

    if [ "$streamUsesDRM" = "true" ]; then
        echo "gbnews-dl: Stream is requesting DRM playback. Unsupported" 1>&2
        echo "$streamMeta" | jq '.' 1>&2
        return 1
    fi

    # TODO: Videos play fine without messing with heartbeats however sending them might be required to keep
    #       the livestream url working. Needs more looking into
    local streamUrl=$(echo "$streamMeta" | jq -r '.response.stream')

    if [ -z "$streamUrl" ]; then
        echo "gbnews-dl: Successful negotiation but no stream URL provided!" 1>&2
        echo "$streamMeta" | jq '.' 1>&2
        return 1
    fi

    echo "$streamUrl"
}
```


Discussion:
Is your script still working for the test video? Using a preliminary extractor, I get `api-streams` returned instead of the JSON that you store in `streamMeta`.
@dirkf Yeap, still working for me. `api-streams` is what the api returns as a 404, you might be sending `data-type` as `vod` instead of `show`
```sh
$ gbnews-dl 'https://www.gbnews.uk/shows/andrew-neils-message-to-companies-choosing-to-boycott-gb-news/106889'
gbnews-dl: Attempt lookup for GB001
https://vod-gbnews.simplestreamcdn.com/gbnews/encoded/334361.ism/manifest.m3u8
```
If I comment out the if check to change `vod` to `show`, I'll get the same error you're getting
```bash
#if [ "$datasetType" = "vod" ]; then
#    datasetType="show"
#fi
```
```sh
$ gbnews-dl 'https://www.gbnews.uk/shows/andrew-neils-message-to-companies-choosing-to-boycott-gb-news/106889'
gbnews-dl: Attempt lookup for GB001
parse error: Invalid numeric literal at line 2, column 0
gbnews-dl: Failed to parse negotiation server response. Not valid JSON
api-streams
```

Resolving patch:
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -426,6 +426,7 @@
 from .gamestar import GameStarIE
 from .gaskrank import GaskrankIE
 from .gazeta import GazetaIE
+from .gbnews import GBNewsIE
 from .gdcvault import GDCVaultIE
 from .gedidigital import GediDigitalIE
 from .generic import GenericIE
diff --git a/youtube_dl/extractor/gbnews.py b/youtube_dl/extractor/gbnews.py
new file mode 100644
--- /dev/null
+++ b/youtube_dl/extractor/gbnews.py
@@ -0,0 +1,139 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    ExtractorError,
+    T,
+    traverse_obj,
+    txt_or_none,
+    url_or_none,
+)
+
+
+class GBNewsIE(InfoExtractor):
+    IE_DESC = 'GB News clips, features and live stream'
+
+    # \w+ is normally shows or news, but apparently any word redirects to the correct URL
+    _VALID_URL = r'https?://(?:www\.)?gbnews\.(?:uk|com)/(?:\w+/)?(?P<id>[^#?]+)'
+
+    _PLATFORM = 'safari'
+    _SSMP_URL = 'https://mm-v2.simplestream.com/ssmp/api.php'
+    _TESTS = [{
+        'url': 'https://www.gbnews.uk/shows/andrew-neils-message-to-companies-choosing-to-boycott-gb-news/106889',
+        'info_dict': {
+            'id': '106889',
+            'ext': 'mp4',
+            'title': "Andrew Neil's message to companies choosing to boycott GB News",
+            'description': 'md5:b281f5d22fd6d5eda64a4e3ba771b351',
+        },
+        'skip': '404 not found',
+    }, {
+        'url': 'https://www.gbnews.com/news/bbc-claudine-gay-harvard-university-antisemitism-row',
+        'info_dict': {
+            'id': '52264136',
+            'display_id': 'bbc-claudine-gay-harvard-university-antisemitism-row',
+            'ext': 'mp4',
+            'title': 'BBC deletes post after furious backlash over headline downplaying antisemitism',
+            'description': 'The post was criticised by former employers of the broadcaster',
+        },
+    }, {
+        'url': 'https://www.gbnews.uk/watchlive',
+        'info_dict': {
+            'id': '1069',
+            'display_id': 'watchlive',
+            'ext': 'mp4',
+            'title': 'GB News Live',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url).split('/')[-1]
+
+        webpage = self._download_webpage(url, display_id)
+        # extraction based on https://github.com/ytdl-org/youtube-dl/issues/29341
+        '''
+        <div id="video-106908"
+            class="simplestream"
+            data-id="GB001"
+            data-type="vod"
+            data-key="3Li3Nt2Qs8Ct3Xq9Fi5Uy0Mb2Bj0Qs"
+            data-token="f9c317c727dc07f515b20036c8ef14a6"
+            data-expiry="1624300052"
+            data-uvid="37900558"
+            data-poster="https://thumbnails.simplestreamcdn.com/gbnews/ondemand/37900558.jpg?width=700&"
+            data-npaw="false"
+            data-env="production">
+        '''
+        # exception if no match
+        video_data = self._search_regex(
+            r'(<div\s[^>]*\bclass\s*=\s*(\'|")(?!.*sidebar\b)simplestream(?:\s[\s\w$-]*)?\2[^>]*>)',
+            webpage, 'video data')
+
+        video_data = extract_attributes(video_data)
+        ss_id = video_data.get('data-id')
+        if not ss_id:
+            raise ExtractorError('Simplestream ID not found')
+
+        json_data = self._download_json(
+            self._SSMP_URL, display_id,
+            note='Downloading Simplestream JSON metadata',
+            errnote='Unable to download Simplestream JSON metadata',
+            query={
+                'id': ss_id,
+                'env': video_data.get('data-env', 'production'),
+            }, fatal=False)
+
+        meta_url = traverse_obj(json_data, ('response', 'api_hostname'))
+        if not meta_url:
+            raise ExtractorError('No API host found')
+
+        uvid = video_data['data-uvid']
+        dtype = video_data.get('data-type')
+        stream_data = self._download_json(
+            '%s/api/%s/stream/%s' % (meta_url, 'show' if dtype == 'vod' else dtype, uvid),
+            uvid,
+            query={
+                'key': video_data.get('data-key'),
+                'platform': self._PLATFORM,
+            },
+            headers={
+                'Token': video_data.get('data-token'),
+                'Token-Expiry': video_data.get('data-expiry'),
+                'Uvid': uvid,
+            }, fatal=False)
+
+        stream_url = traverse_obj(stream_data, (
+            'response', 'stream', T(url_or_none)))
+        if not stream_url:
+            raise ExtractorError('No stream data/URL')
+
+        # now known to be a dict
+        stream_data = stream_data['response']
+        drm = stream_data.get('drm')
+        if drm:
+            self.report_drm(uvid)
+
+        formats = self._extract_m3u8_formats(
+            stream_url, uvid, ext='mp4', entry_protocol='m3u8_native',
+            fatal=False)
+        # exception if no formats
+        self._sort_formats(formats)
+
+        return {
+            'id': uvid,
+            'display_id': display_id,
+            'title': (traverse_obj(stream_data, ('title', T(txt_or_none)))
+                      or self._og_search_title(webpage, default=None)
+                      or display_id.replace('-', ' ').capitalize()),
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': (traverse_obj(video_data, ('data-poster', T(url_or_none)))
+                          or self._og_search_thumbnail(webpage)),
+            'formats': formats,
+            'is_live': (dtype == 'live') or None,
+        }


Test patch: