Repository: yt-dlp/yt-dlp
Pull request #1020 (2021-09-18 12:47:36)

Add aes-128-gcm decrypt and verify
### Before submitting a *pull request* make sure you have:
- [ ] At least skimmed through [adding new extractor tutorial](https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/ytdl-org/youtube-dl#youtube-dl-coding-conventions) sections
- [x] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
- [x] Checked the code with [flake8](https://pypi.python.org/pypi/flake8)

### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
- [x] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)

### What is the purpose of your *pull request*?
- [ ] Bug fix
- [x] Improvement
- [ ] New extractor
- [ ] New feature

---

### Description of your *pull request* and other information

This is based on #938, w/o compatibility.


Patch:
diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py
index 57caae069..f52b992df 100644
--- a/yt_dlp/aes.py
+++ b/yt_dlp/aes.py
@@ -11,39 +11,59 @@ def aes_cbc_decrypt_bytes(data, key, iv):
         """ Decrypt bytes with AES-CBC using pycryptodome """
         return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_CBC, iv).decrypt(data)
 
+    def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
+        """ Decrypt bytes with AES-GCM using pycryptodome """
+        return compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag)
+
 else:
     def aes_cbc_decrypt_bytes(data, key, iv):
         """ Decrypt bytes with AES-CBC using native implementation since pycryptodome is unavailable """
         return intlist_to_bytes(aes_cbc_decrypt(*map(bytes_to_intlist, (data, key, iv))))
 
+    def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce):
+        """ Decrypt bytes with AES-GCM using native implementation since pycryptodome is unavailable """
+        return intlist_to_bytes(aes_gcm_decrypt_and_verify(*map(bytes_to_intlist, (data, key, tag, nonce))))
+
 
 BLOCK_SIZE_BYTES = 16
 
 
-def aes_ctr_decrypt(data, key, counter):
+def aes_ctr_decrypt(data, key, iv):
     """
     Decrypt with aes in counter mode
 
     @param {int[]} data        cipher
     @param {int[]} key         16/24/32-Byte cipher key
-    @param {instance} counter  Instance whose next_value function (@returns {int[]}  16-Byte block)
-                               returns the next counter block
+    @param {int[]} iv          16-Byte initialization vector
     @returns {int[]}           decrypted data
     """
+    return aes_ctr_encrypt(data, key, iv)
+
+
+def aes_ctr_encrypt(data, key, iv):
+    """
+    Encrypt with aes in counter mode
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @param {int[]} iv          16-Byte initialization vector
+    @returns {int[]}           encrypted data
+    """
     expanded_key = key_expansion(key)
     block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+    counter = iter_vector(iv)
 
-    decrypted_data = []
+    encrypted_data = []
     for i in range(block_count):
-        counter_block = counter.next_value()
+        counter_block = next(counter)
         block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
         block += [0] * (BLOCK_SIZE_BYTES - len(block))
 
         cipher_counter_block = aes_encrypt(counter_block, expanded_key)
-        decrypted_data += xor(block, cipher_counter_block)
-    decrypted_data = decrypted_data[:len(data)]
+        encrypted_data += xor(block, cipher_counter_block)
+    encrypted_data = encrypted_data[:len(data)]
 
-    return decrypted_data
+    return encrypted_data
 
 
 def aes_cbc_decrypt(data, key, iv):
@@ -100,39 +120,47 @@ def aes_cbc_encrypt(data, key, iv):
     return encrypted_data
 
 
-def key_expansion(data):
+def aes_gcm_decrypt_and_verify(data, key, tag, nonce):
     """
-    Generate key schedule
+    Decrypt with aes in GBM mode and checks authenticity using tag
 
-    @param {int[]} data  16/24/32-Byte cipher key
-    @returns {int[]}     176/208/240-Byte expanded key
+    @param {int[]} data        cipher
+    @param {int[]} key         16-Byte cipher key
+    @param {int[]} tag         authentication tag
+    @param {int[]} nonce       IV (recommended 12-Byte)
+    @returns {int[]}           decrypted data
     """
-    data = data[:]  # copy
-    rcon_iteration = 1
-    key_size_bytes = len(data)
-    expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES
 
-    while len(data) < expanded_key_size_bytes:
-        temp = data[-4:]
-        temp = key_schedule_core(temp, rcon_iteration)
-        rcon_iteration += 1
-        data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+    # XXX: check aes, gcm param
 
-        for _ in range(3):
-            temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+    hash_subkey = aes_encrypt([0] * BLOCK_SIZE_BYTES, key_expansion(key))
 
-        if key_size_bytes == 32:
-            temp = data[-4:]
-            temp = sub_bytes(temp)
-            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+    if len(nonce) == 12:
+        j0 = nonce + [0, 0, 0, 1]
+    else:
+        fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES + 8
+        ghash_in = nonce + [0] * fill + bytes_to_intlist((8 * len(nonce)).to_bytes(8, 'big'))
+        j0 = ghash(hash_subkey, ghash_in)
 
-        for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
-            temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
-    data = data[:expanded_key_size_bytes]
+    # TODO: add nonce support to aes_ctr_decrypt
 
-    return data
+    # nonce_ctr = j0[:12]
+    iv_ctr = inc(j0)
+
+    decrypted_data = aes_ctr_decrypt(data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr)))
+    pad_len = len(data) // 16 * 16
+    s_tag = ghash(
+        hash_subkey,
+        data
+        + [0] * (BLOCK_SIZE_BYTES - len(data) + pad_len)        # pad
+        + bytes_to_intlist((0 * 8).to_bytes(8, 'big')           # length of associated data
+                           + ((len(data) * 8).to_bytes(8, 'big')))  # length of data
+    )
+
+    if tag != aes_ctr_encrypt(s_tag, key, j0):
+        raise ValueError("Mismatching authentication tag")
+
+    return decrypted_data
 
 
 def aes_encrypt(data, expanded_key):
@@ -201,15 +229,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     nonce = data[:NONCE_LENGTH_BYTES]
     cipher = data[NONCE_LENGTH_BYTES:]
 
-    class Counter(object):
-        __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
-
-        def next_value(self):
-            temp = self.__value
-            self.__value = inc(self.__value)
-            return temp
-
-    decrypted_data = aes_ctr_decrypt(cipher, key, Counter())
+    decrypted_data = aes_ctr_decrypt(cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES))
     plaintext = intlist_to_bytes(decrypted_data)
 
     return plaintext
@@ -290,6 +310,47 @@ def next_value(self):
                       0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)
 
 
+def key_expansion(data):
+    """
+    Generate key schedule
+
+    @param {int[]} data  16/24/32-Byte cipher key
+    @returns {int[]}     176/208/240-Byte expanded key
+    """
+    data = data[:]  # copy
+    rcon_iteration = 1
+    key_size_bytes = len(data)
+    expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES
+
+    while len(data) < expanded_key_size_bytes:
+        temp = data[-4:]
+        temp = key_schedule_core(temp, rcon_iteration)
+        rcon_iteration += 1
+        data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+        for _ in range(3):
+            temp = data[-4:]
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+        if key_size_bytes == 32:
+            temp = data[-4:]
+            temp = sub_bytes(temp)
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+        for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
+            temp = data[-4:]
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+    data = data[:expanded_key_size_bytes]
+
+    return data
+
+
+def iter_vector(iv):
+    while True:
+        yield iv
+        iv = inc(iv)
+
+
 def sub_bytes(data):
     return [SBOX[x] for x in data]
 
@@ -315,7 +376,7 @@ def xor(data1, data2):
 
 
 def rijndael_mul(a, b):
-    if(a == 0 or b == 0):
+    if a == 0 or b == 0:
         return 0
     return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
 
@@ -359,6 +420,20 @@ def shift_rows_inv(data):
     return data_shifted
 
 
+def shift_block(data):
+    data_shifted = []
+
+    bit = 0
+    for n in data:
+        if bit:
+            n |= 0x100
+        bit = n & 1
+        n >>= 1
+        data_shifted.append(n)
+
+    return data_shifted
+
+
 def inc(data):
     data = data[:]  # copy
     for i in range(len(data) - 1, -1, -1):
@@ -370,4 +445,50 @@ def inc(data):
     return data
 
 
-__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']
+def block_product(block_x, block_y):
+    # NIST SP 800-38D, Algorithm 1
+
+    if len(block_x) != BLOCK_SIZE_BYTES or len(block_y) != BLOCK_SIZE_BYTES:
+        raise ValueError("Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES)
+
+    block_r = [0xE1] + [0] * (BLOCK_SIZE_BYTES - 1)
+    block_v = block_y[:]
+    block_z = [0] * BLOCK_SIZE_BYTES
+
+    for i in block_x:
+        for bit in range(7, -1, -1):
+            if i & (1 << bit):
+                block_z = xor(block_z, block_v)
+
+            do_xor = block_v[-1] & 1
+            block_v = shift_block(block_v)
+            if do_xor:
+                block_v = xor(block_v, block_r)
+
+    return block_z
+
+
+def ghash(subkey, data):
+    # NIST SP 800-38D, Algorithm 2
+
+    if len(data) % BLOCK_SIZE_BYTES:
+        raise ValueError("Length of data should be %d bytes" % BLOCK_SIZE_BYTES)
+
+    last_y = [0] * BLOCK_SIZE_BYTES
+    for i in range(0, len(data), BLOCK_SIZE_BYTES):
+        block = data[i : i + BLOCK_SIZE_BYTES]  # noqa: E203
+        last_y = block_product(xor(last_y, block), subkey)
+
+    return last_y
+
+
+__all__ = [
+    'aes_ctr_decrypt',
+    'aes_cbc_decrypt',
+    'aes_cbc_decrypt_bytes',
+    'aes_decrypt_text',
+    'aes_encrypt',
+    'aes_gcm_decrypt_and_verify',
+    'aes_gcm_decrypt_and_verify_bytes',
+    'key_expansion'
+]
diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py
index 4f582f4e1..1409e6799 100644
--- a/yt_dlp/cookies.py
+++ b/yt_dlp/cookies.py
@@ -9,17 +9,14 @@
 from datetime import datetime, timedelta, timezone
 from hashlib import pbkdf2_hmac
 
-from yt_dlp.aes import aes_cbc_decrypt
-from yt_dlp.compat import (
+from .aes import aes_cbc_decrypt_bytes, aes_gcm_decrypt_and_verify_bytes
+from .compat import (
     compat_b64decode,
     compat_cookiejar_Cookie,
-    compat_pycrypto_AES
 )
-from yt_dlp.utils import (
+from .utils import (
     bug_reports_message,
-    bytes_to_intlist,
     expand_path,
-    intlist_to_bytes,
     process_communicate_or_kill,
     YoutubeDLCookieJar,
 )
@@ -395,11 +392,6 @@ def decrypt(self, encrypted_value):
             if self._v10_key is None:
                 self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True)
                 return None
-            elif not compat_pycrypto_AES:
-                self._logger.warning('cannot decrypt cookie as the `pycryptodome` module is not installed. '
-                                     'Please install by running `python3 -m pip install pycryptodome`',
-                                     only_once=True)
-                return None
 
             # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc
             #   kNonceLength
@@ -643,21 +635,18 @@ def pbkdf2_sha1(password, salt, iterations, key_length):
 
 
 def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16):
-    plaintext = aes_cbc_decrypt(bytes_to_intlist(ciphertext),
-                                bytes_to_intlist(key),
-                                bytes_to_intlist(initialization_vector))
+    plaintext = aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)
     padding_length = plaintext[-1]
     try:
-        return intlist_to_bytes(plaintext[:-padding_length]).decode('utf-8')
+        return plaintext[:-padding_length].decode('utf-8')
     except UnicodeDecodeError:
         logger.warning('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?', only_once=True)
         return None
 
 
 def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger):
-    cipher = compat_pycrypto_AES.new(key, compat_pycrypto_AES.MODE_GCM, nonce)
     try:
-        plaintext = cipher.decrypt_and_verify(ciphertext, authentication_tag)
+        plaintext = aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce)
     except ValueError:
         logger.warning('failed to decrypt cookie because the MAC check failed. Possibly the key is wrong?', only_once=True)
         return None
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index bea286604..e0dc1def7 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -56,8 +56,6 @@ def can_download(manifest, info_dict, allow_unplayable_formats=False):
 
         def check_results():
             yield not info_dict.get('is_live')
-            is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
-            yield not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest)
             for feature in UNSUPPORTED_FEATURES:
                 yield not re.search(feature, manifest)
         return all(check_results())
diff --git a/yt_dlp/extractor/cgtn.py b/yt_dlp/extractor/cgtn.py
new file mode 100644
index 000000000..89f173887
--- /dev/null
+++ b/yt_dlp/extractor/cgtn.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    try_get,
+    unified_timestamp,
+)
+
+
+class CGTNIE(InfoExtractor):
+    _VALID_URL = r'https?://news\.cgtn\.com/news/[0-9]{4}-[0-9]{2}-[0-9]{2}/[a-zA-Z0-9-]+-(?P<id>[a-zA-Z0-9-]+)/index\.html'
+    _TESTS = [
+        {
+            'url': 'https://news.cgtn.com/news/2021-03-09/Up-and-Out-of-Poverty-Ep-1-A-solemn-promise-YuOUaOzGQU/index.html',
+            'info_dict': {
+                'id': 'YuOUaOzGQU',
+                'ext': 'mp4',
+                'title': 'Up and Out of Poverty Ep. 1: A solemn promise',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'timestamp': 1615295940,
+                'upload_date': '20210309',
+            },
+            'params': {
+                'skip_download': True
+            }
+        }, {
+            'url': 'https://news.cgtn.com/news/2021-06-06/China-Indonesia-vow-to-further-deepen-maritime-cooperation-10REvJCewCY/index.html',
+            'info_dict': {
+                'id': '10REvJCewCY',
+                'ext': 'mp4',
+                'title': 'China, Indonesia vow to further deepen maritime cooperation',
+                'thumbnail': r're:^https?://.*\.png$',
+                'description': 'China and Indonesia vowed to upgrade their cooperation into the maritime sector and also for political security, economy, and cultural and people-to-people exchanges.',
+                'author': 'CGTN',
+                'category': 'China',
+                'timestamp': 1622950200,
+                'upload_date': '20210606',
+            },
+            'params': {
+                'skip_download': False
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        download_url = self._html_search_regex(r'data-video ="(?P<url>.+m3u8)"', webpage, 'download_url')
+        datetime_str = self._html_search_regex(r'<span class="date">\s*(.+?)\s*</span>', webpage, 'datetime_str', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': self._extract_m3u8_formats(download_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls'),
+            'category': self._html_search_regex(r'<span class="section">\s*(.+?)\s*</span>',
+                                                webpage, 'category', fatal=False),
+            'author': self._html_search_regex(r'<div class="news-author-name">\s*(.+?)\s*</div>',
+                                              webpage, 'author', default=None, fatal=False),
+            'timestamp': try_get(unified_timestamp(datetime_str), lambda x: x - 8 * 3600),
+        }
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index e79684231..f6ca686a3 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -2622,8 +2622,10 @@ def extract_Initialization(source):
                             base_url = base_url_e.text + base_url
                             if re.match(r'^https?://', base_url):
                                 break
-                    if mpd_base_url and not re.match(r'^https?://', base_url):
-                        if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
+                    if mpd_base_url and base_url.startswith('/'):
+                        base_url = compat_urlparse.urljoin(mpd_base_url, base_url)
+                    elif mpd_base_url and not re.match(r'^https?://', base_url):
+                        if not mpd_base_url.endswith('/'):
                             mpd_base_url += '/'
                         base_url = mpd_base_url + base_url
                     representation_id = representation_attrib.get('id')
diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py
index 3dd56c65a..ddae1d7cc 100644
--- a/yt_dlp/extractor/extractors.py
+++ b/yt_dlp/extractor/extractors.py
@@ -239,6 +239,7 @@
     CeskaTelevizeIE,
     CeskaTelevizePoradyIE,
 )
+from .cgtn import CGTNIE
 from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
 from .chaturbate import ChaturbateIE
diff --git a/yt_dlp/extractor/safari.py b/yt_dlp/extractor/safari.py
index fbbbc7e77..cca4464ca 100644
--- a/yt_dlp/extractor/safari.py
+++ b/yt_dlp/extractor/safari.py
@@ -197,7 +197,7 @@ def _real_extract(self, url):
         if 'library/view' in web_url:
             web_url = web_url.replace('library/view', 'videos')
             natural_keys = part['natural_key']
-            web_url = f'{web_url.rsplit("/")[0]}/{natural_keys[0]}-{natural_keys[1][:-5]}'
+            web_url = f'{web_url.rsplit("/", 1)[0]}/{natural_keys[0]}-{natural_keys[1][:-5]}'
         return self.url_result(web_url, SafariIE.ie_key())
 
 
diff --git a/yt_dlp/extractor/videa.py b/yt_dlp/extractor/videa.py
index da0212bb2..512ade7af 100644
--- a/yt_dlp/extractor/videa.py
+++ b/yt_dlp/extractor/videa.py
@@ -47,10 +47,24 @@ class VideaIE(InfoExtractor):
         },
     }, {
         'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
-        'only_matching': True,
+        'md5': 'd57ccd8812c7fd491d33b1eab8c99975',
+        'info_dict': {
+            'id': 'jAHDWfWSJH5XuFhH',
+            'ext': 'mp4',
+            'title': 'Supercars előzés',
+            'thumbnail': r're:^https?://.*',
+            'duration': 64,
+        },
     }, {
         'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
-        'only_matching': True,
+        'md5': '97a7af41faeaffd9f1fc864a7c7e7603',
+        'info_dict': {
+            'id': '8YfIAjxwWGwT8HVQ',
+            'ext': 'mp4',
+            'title': 'Az őrült kígyász 285 kígyót enged szabadon',
+            'thumbnail': r're:^https?://.*',
+            'duration': 21,
+        },
     }, {
         'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
         'only_matching': True,
@@ -100,10 +114,14 @@ def _real_extract(self, url):
 
         video_page = self._download_webpage(url, video_id)
 
-        player_url = self._search_regex(
-            r'<iframe.*?src="(/player\?[^"]+)"', video_page, 'player url')
-        player_url = urljoin(url, player_url)
-        player_page = self._download_webpage(player_url, video_id)
+        if 'videa.hu/player' in url:
+            player_url = url
+            player_page = video_page
+        else:
+            player_url = self._search_regex(
+                r'<iframe.*?src="(/player\?[^"]+)"', video_page, 'player url')
+            player_url = urljoin(url, player_url)
+            player_page = self._download_webpage(player_url, video_id)
 
         nonce = self._search_regex(
             r'_xt\s*=\s*"([^"]+)"', player_page, 'nonce')
@@ -134,7 +152,7 @@ def _real_extract(self, url):
         sources = xpath_element(
             info, './video_sources', 'sources', fatal=True)
         hash_values = xpath_element(
-            info, './hash_values', 'hash values', fatal=True)
+            info, './hash_values', 'hash values', fatal=False)
 
         title = xpath_text(video, './title', fatal=True)
 
@@ -143,15 +161,16 @@ def _real_extract(self, url):
             source_url = source.text
             source_name = source.get('name')
             source_exp = source.get('exp')
-            if not (source_url and source_name and source_exp):
+            if not (source_url and source_name):
                 continue
-            hash_value = xpath_text(hash_values, 'hash_value_' + source_name)
-            if not hash_value:
-                continue
-            source_url = update_url_query(source_url, {
-                'md5': hash_value,
-                'expires': source_exp,
-            })
+            hash_value = None
+            if hash_values:
+                hash_value = xpath_text(hash_values, 'hash_value_' + source_name)
+            if hash_value and source_exp:
+                source_url = update_url_query(source_url, {
+                    'md5': hash_value,
+                    'expires': source_exp,
+                })
             f = parse_codecs(source.get('codecs'))
             f.update({
                 'url': self._proto_relative_url(source_url),
diff --git a/yt_dlp/postprocessor/modify_chapters.py b/yt_dlp/postprocessor/modify_chapters.py
index 9a7ba8eff..2871e16d5 100644
--- a/yt_dlp/postprocessor/modify_chapters.py
+++ b/yt_dlp/postprocessor/modify_chapters.py
@@ -15,7 +15,7 @@
 )
 
 
-_TINY_SPONSOR_OVERLAP_DURATION = 1
+_TINY_CHAPTER_DURATION = 1
 DEFAULT_SPONSORBLOCK_CHAPTER_TITLE = '[SponsorBlock]: %(category_names)l'
 
 
@@ -50,7 +50,6 @@ def run(self, info):
             if not info.get('__real_download'):
                 raise PostProcessingError('Cannot cut video since the real and expected durations mismatch. '
                                           'Different chapters may have already been removed')
-                return [], info
             else:
                 self.write_debug('Expected and actual durations mismatch')
 
@@ -145,38 +144,15 @@ def excess_duration(c):
 
         new_chapters = []
 
-        def chapter_length(c):
-            return c['end_time'] - c['start_time']
-
-        def original_uncut_chapter(c):
-            return '_was_cut' not in c and '_categories' not in c
-
         def append_chapter(c):
             assert 'remove' not in c
-            length = chapter_length(c) - excess_duration(c)
+            length = c['end_time'] - c['start_time'] - excess_duration(c)
             # Chapter is completely covered by cuts or sponsors.
             if length <= 0:
                 return
             start = new_chapters[-1]['end_time'] if new_chapters else 0
             c.update(start_time=start, end_time=start + length)
-            # Append without checking for tininess to prevent having
-            # a completely empty chapter list.
-            if not new_chapters:
-                new_chapters.append(c)
-                return
-            old_c = new_chapters[-1]
-            # Merge with the previous if the chapter is tiny.
-            # Only tiny chapters resulting from a cut can be skipped.
-            # Chapters that were already tiny in the original list will be preserved.
-            if not original_uncut_chapter(c) and length < _TINY_SPONSOR_OVERLAP_DURATION:
-                old_c['end_time'] = c['end_time']
-            # Previous tiny chapter was appended for the sake of preventing an empty chapter list.
-            # Replace it with the current one.
-            elif not original_uncut_chapter(old_c) and chapter_length(old_c) < _TINY_SPONSOR_OVERLAP_DURATION:
-                c['start_time'] = old_c['start_time']
-                new_chapters[-1] = c
-            else:
-                new_chapters.append(c)
+            new_chapters.append(c)
 
         # Turn into a priority queue, index is a tie breaker.
         # Plain stack sorted by start_time is not enough: after splitting the chapter,
@@ -275,10 +251,36 @@ def append_chapter(c):
                 append_chapter(cur_chapter)
                 cur_i, cur_chapter = i, c
         (append_chapter if 'remove' not in cur_chapter else append_cut)(cur_chapter)
+        return self._remove_tiny_rename_sponsors(new_chapters), cuts
+
+    def _remove_tiny_rename_sponsors(self, chapters):
+        new_chapters = []
+        for i, c in enumerate(chapters):
+            # Merge with the previous/next if the chapter is tiny.
+            # Only tiny chapters resulting from a cut can be skipped.
+            # Chapters that were already tiny in the original list will be preserved.
+            if (('_was_cut' in c or '_categories' in c)
+                    and c['end_time'] - c['start_time'] < _TINY_CHAPTER_DURATION):
+                if not new_chapters:
+                    # Prepend tiny chapter to the next one if possible.
+                    if i < len(chapters) - 1:
+                        chapters[i + 1]['start_time'] = c['start_time']
+                        continue
+                else:
+                    old_c = new_chapters[-1]
+                    if i < len(chapters) - 1:
+                        next_c = chapters[i + 1]
+                        # Not a typo: key names in old_c and next_c are really different.
+                        prev_is_sponsor = 'categories' in old_c
+                        next_is_sponsor = '_categories' in next_c
+                        # Preferentially prepend tiny normals to normals and sponsors to sponsors.
+                        if (('_categories' not in c and prev_is_sponsor and not next_is_sponsor)
+                                or ('_categories' in c and not prev_is_sponsor and next_is_sponsor)):
+                            next_c['start_time'] = c['start_time']
+                            continue
+                    old_c['end_time'] = c['end_time']
+                    continue
 
-        i = -1
-        for c in new_chapters.copy():
-            i += 1
             c.pop('_was_cut', None)
             cats = c.pop('_categories', None)
             if cats:
@@ -292,12 +294,13 @@ def append_chapter(c):
                 })
                 outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(self._sponsorblock_chapter_title, c)
                 c['title'] = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
-            if i > 0 and c['title'] == new_chapters[i - 1]['title']:
-                new_chapters[i - 1]['end_time'] = c['end_time']
-                new_chapters.pop(i)
-                i -= 1
-
-        return new_chapters, cuts
+                # Merge identically named sponsors.
+                if (new_chapters and 'categories' in new_chapters[-1]
+                        and new_chapters[-1]['title'] == c['title']):
+                    new_chapters[-1]['end_time'] = c['end_time']
+                    continue
+            new_chapters.append(c)
+        return new_chapters
 
     def remove_chapters(self, filename, ranges_to_cut, concat_opts, force_keyframes=False):
         in_file = filename
diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 983ca6ced..de0213b14 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1762,6 +1762,7 @@ def random_user_agent():
     '%b %d %Y at %H:%M:%S',
     '%B %d %Y at %H:%M',
     '%B %d %Y at %H:%M:%S',
+    '%H:%M %d-%b-%Y',
 )
 
 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
@@ -3034,8 +3035,16 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
 
 def extract_timezone(date_str):
     m = re.search(
-        r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-        date_str)
+        r'''(?x)
+            ^.{8,}?                                              # >=8 char non-TZ prefix, if present
+            (?P<tz>Z|                                            # just the UTC Z, or
+                (?:(?<=.\b\d{4}|\b\d{2}:\d\d)|                   # preceded by 4 digits or hh:mm or
+                   (?<!.\b[a-zA-Z]{3}|[a-zA-Z]{4}|..\b\d\d))     # not preceded by 3 alpha word or >= 4 alpha or 2 digits
+                   [ ]?                                          # optional space
+                (?P<sign>\+|-)                                   # +/-
+                (?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})       # hh[:]mm
+            $)
+        ''', date_str)
     if not m:
         timezone = datetime.timedelta()
     else:


Test patch:
diff --git a/test/test_aes.py b/test/test_aes.py
index d2e51af29..46db59e57 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -7,7 +7,19 @@
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from yt_dlp.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text
+from yt_dlp.aes import (
+    aes_decrypt,
+    aes_encrypt,
+    aes_cbc_decrypt,
+    aes_cbc_decrypt_bytes,
+    aes_cbc_encrypt,
+    aes_ctr_decrypt,
+    aes_ctr_encrypt,
+    aes_gcm_decrypt_and_verify,
+    aes_gcm_decrypt_and_verify_bytes,
+    aes_decrypt_text
+)
+from yt_dlp.compat import compat_pycrypto_AES
 from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes
 import base64
 
@@ -27,18 +39,43 @@ def test_encrypt(self):
         self.assertEqual(decrypted, msg)
 
     def test_cbc_decrypt(self):
-        data = bytes_to_intlist(
-            b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd"
-        )
-        decrypted = intlist_to_bytes(aes_cbc_decrypt(data, self.key, self.iv))
+        data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd'
+        decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
+        if compat_pycrypto_AES:
+            decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv))
+            self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
     def test_cbc_encrypt(self):
         data = bytes_to_intlist(self.secret_msg)
         encrypted = intlist_to_bytes(aes_cbc_encrypt(data, self.key, self.iv))
         self.assertEqual(
             encrypted,
-            b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd")
+            b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd')
+
+    def test_ctr_decrypt(self):
+        data = bytes_to_intlist(b'\x03\xc7\xdd\xd4\x8e\xb3\xbc\x1a*O\xdc1\x12+8Aio\xd1z\xb5#\xaf\x08')
+        decrypted = intlist_to_bytes(aes_ctr_decrypt(data, self.key, self.iv))
+        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
+
+    def test_ctr_encrypt(self):
+        data = bytes_to_intlist(self.secret_msg)
+        encrypted = intlist_to_bytes(aes_ctr_encrypt(data, self.key, self.iv))
+        self.assertEqual(
+            encrypted,
+            b'\x03\xc7\xdd\xd4\x8e\xb3\xbc\x1a*O\xdc1\x12+8Aio\xd1z\xb5#\xaf\x08')
+
+    def test_gcm_decrypt(self):
+        data = b'\x159Y\xcf5eud\x90\x9c\x85&]\x14\x1d\x0f.\x08\xb4T\xe4/\x17\xbd'
+        authentication_tag = b'\xe8&I\x80rI\x07\x9d}YWuU@:e'
+
+        decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify(
+            bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12]))
+        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
+        if compat_pycrypto_AES:
+            decrypted = aes_gcm_decrypt_and_verify_bytes(
+                data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12]))
+            self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
     def test_decrypt_text(self):
         password = intlist_to_bytes(self.key).decode('utf-8')
diff --git a/test/test_cookies.py b/test/test_cookies.py
index 6053ebb4e..15afb6627 100644
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@@ -2,7 +2,6 @@
 from datetime import datetime, timezone
 
 from yt_dlp import cookies
-from yt_dlp.compat import compat_pycrypto_AES
 from yt_dlp.cookies import (
     LinuxChromeCookieDecryptor,
     MacChromeCookieDecryptor,
@@ -53,7 +52,6 @@ def test_chrome_cookie_decryptor_linux_v11(self):
             decryptor = LinuxChromeCookieDecryptor('Chrome', YDLLogger())
             self.assertEqual(decryptor.decrypt(encrypted_value), value)
 
-    @unittest.skipIf(not compat_pycrypto_AES, 'cryptography library not available')
     def test_chrome_cookie_decryptor_windows_v10(self):
         with MonkeyPatch(cookies, {
             '_get_windows_v10_key': lambda *args, **kwargs: b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2<z\x16]\n\xbb\xb8\xcb\xd7\x9bA\xc3\x14e\x99{\xd6\xf4&'
diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py
index 7d1368769..090c7b47b 100644
--- a/test/test_postprocessors.py
+++ b/test/test_postprocessors.py
@@ -461,11 +461,23 @@ def test_remove_marked_arrange_sponsors_TinyChaptersResultingFromCutsAreIgnored(
         self._remove_marked_arrange_sponsors_test_impl(
             chapters, self._chapters([2, 2.5], ['c1', 'c3']), cuts)
 
+    def test_remove_marked_arrange_sponsors_SingleTinyChapterIsPreserved(self):
+        cuts = [self._chapter(0.5, 2, remove=True)]
+        chapters = self._chapters([2], ['c']) + cuts
+        self._remove_marked_arrange_sponsors_test_impl(
+            chapters, self._chapters([0.5], ['c']), cuts)
+
+    def test_remove_marked_arrange_sponsors_TinyChapterAtTheStartPrependedToTheNext(self):
+        cuts = [self._chapter(0.5, 2, remove=True)]
+        chapters = self._chapters([2, 4], ['c1', 'c2']) + cuts
+        self._remove_marked_arrange_sponsors_test_impl(
+            chapters, self._chapters([2.5], ['c2']), cuts)
+
     def test_remove_marked_arrange_sponsors_TinyChaptersResultingFromSponsorOverlapAreIgnored(self):
         chapters = self._chapters([1, 3, 4], ['c1', 'c2', 'c3']) + [
             self._sponsor_chapter(1.5, 2.5, 'sponsor')]
         self._remove_marked_arrange_sponsors_test_impl(
-            chapters, self._chapters([1.5, 3, 4], ['c1', '[SponsorBlock]: Sponsor', 'c3']), [])
+            chapters, self._chapters([1.5, 2.5, 4], ['c1', '[SponsorBlock]: Sponsor', 'c3']), [])
 
     def test_remove_marked_arrange_sponsors_TinySponsorsOverlapsAreIgnored(self):
         chapters = self._chapters([2, 3, 5], ['c1', 'c2', 'c3']) + [
@@ -476,6 +488,26 @@ def test_remove_marked_arrange_sponsors_TinySponsorsOverlapsAreIgnored(self):
             chapters, self._chapters([1, 3, 4, 5], [
                 'c1', '[SponsorBlock]: Sponsor', '[SponsorBlock]: Unpaid/Self Promotion', 'c3']), [])
 
+    def test_remove_marked_arrange_sponsors_TinySponsorsPrependedToTheNextSponsor(self):
+        chapters = self._chapters([4], ['c']) + [
+            self._sponsor_chapter(1.5, 2, 'sponsor'),
+            self._sponsor_chapter(2, 4, 'selfpromo')
+        ]
+        self._remove_marked_arrange_sponsors_test_impl(
+            chapters, self._chapters([1.5, 4], ['c', '[SponsorBlock]: Unpaid/Self Promotion']), [])
+
+    def test_remove_marked_arrange_sponsors_SmallestSponsorInTheOverlapGetsNamed(self):
+        self._pp._sponsorblock_chapter_title = '[SponsorBlock]: %(name)s'
+        chapters = self._chapters([10], ['c']) + [
+            self._sponsor_chapter(2, 8, 'sponsor'),
+            self._sponsor_chapter(4, 6, 'selfpromo')
+        ]
+        self._remove_marked_arrange_sponsors_test_impl(
+            chapters, self._chapters([2, 4, 6, 8, 10], [
+                'c', '[SponsorBlock]: Sponsor', '[SponsorBlock]: Unpaid/Self Promotion',
+                '[SponsorBlock]: Sponsor', 'c'
+            ]), [])
+
     def test_make_concat_opts_CommonCase(self):
         sponsor_chapters = [self._chapter(1, 2, 's1'), self._chapter(10, 20, 's2')]
         expected = '''ffconcat version 1.0