Repository: yt-dlp/yt-dlp

Issue:
[nitter] wrong uploader
<!--

######################################################################
  WARNING!
  IGNORING THE FOLLOWING TEMPLATE WILL RESULT IN ISSUE CLOSED AS INCOMPLETE
######################################################################

-->


## Checklist

<!--
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of yt-dlp:
- First of, make sure you are using the latest version of yt-dlp. Run `yt-dlp --version` and ensure your version is 2021.04.03. If it's not, see https://github.com/yt-dlp/yt-dlp on how to update. Issues with outdated version will be REJECTED.
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
- Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in https://github.com/yt-dlp/yt-dlp.
- Search the bugtracker for similar issues: https://github.com/yt-dlp/yt-dlp. DO NOT post duplicates.
- Read bugs section in FAQ: https://github.com/yt-dlp/yt-dlp
- Finally, put x into all relevant boxes like this [x] (Dont forget to delete the empty space)
-->

- [x] I'm reporting a broken site support issue
- [x] I've verified that I'm running yt-dlp version **2021.04.03**
- [x] I've checked that all provided URLs are alive and playable in a browser
- [x] I've checked that all URLs and arguments with special characters are properly quoted or escaped
- [x] I've searched the bugtracker for similar bug reports including closed ones
- [x] I've read bugs section in FAQ


## Verbose log
`youtube-dl -J https://nitter.pussthecat.org/manential/status/1379007759811772419#m -v`

```
[debug] Command-line config: ['-J', 'https://nitter.pussthecat.org/manential/status/1379007759811772419#m', '-v']
[debug] Loading archive file None
[debug] Encodings: locale UTF-8, fs utf-8, out utf-8, pref UTF-8
[debug] yt-dlp version 2021.04.03 (zip)
[debug] Plugin Extractors: ['SamplePlugin']
[debug] Python version 3.8.6 (CPython 64bit) - Linux-5.11.0-7612-generic-x86_64-with-glibc2.32
[debug] exe versions: ffmpeg 4.3.1, ffprobe 4.3.1, rtmpdump 2.4
[debug] Proxy map: {}
{"id": "1379007759811772419", "title": "franceinfo - #OnVeutLesNoms #chalencon #MangeonsLesRiches #restaurantclandestin ", "description": "#OnVeutLesNoms #chalencon #MangeonsLesRiches #restaurantclandestin ", "uploader": "franceinfo", "timestamp": 1617613023, "uploader_id": "manential", "uploader_url": "https://nitter.pussthecat.org/manential", "view_count": 883, "like_count": 41, "repost_count": 21, "comment_count": 40, "formats": [{"format_id": "256", "format_index": null, "url": "https://nitter.pussthecat.org/video/5B85E8AC863C4/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2F320x180%2FrHf8P7IawktnLB4H.m3u8", "manifest_url": "https://nitter.pussthecat.org/video/F766930E58019/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2FFaGFrWPuuNDv7k2f.m3u8%3Ftag%3D6", "tbr": 256.0, "ext": "mp4", "fps": null, "protocol": "m3u8", "preference": null, "quality": null, "width": 320, "height": 180, "vcodec": "avc1.4d0015", "acodec": "mp4a.40.2", "format": "256 - 320x180", "http_headers": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3649.1 Safari/537.36", "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en-us,en;q=0.5", "Cookie": "hlsPlayback=on"}}, {"format_id": "832", "format_index": null, "url": "https://nitter.pussthecat.org/video/622EA0CE58406/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2F640x360%2FYwCMb4BqAeCcCcLG.m3u8", "manifest_url": "https://nitter.pussthecat.org/video/F766930E58019/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2FFaGFrWPuuNDv7k2f.m3u8%3Ftag%3D6", "tbr": 832.0, "ext": "mp4", "fps": null, "protocol": "m3u8", "preference": null, "quality": null, "width": 640, "height": 360, "vcodec": "avc1.4d001f", "acodec": "mp4a.40.2", "format": "832 - 640x360", "http_headers": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3649.1 Safari/537.36", "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en-us,en;q=0.5", "Cookie": "hlsPlayback=on"}}, {"format_id": "2176", "format_index": null, "url": "https://nitter.pussthecat.org/video/AAEEBA690FA91/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2F1280x720%2Ft-IdaHQfnAVVvmJw.m3u8", "manifest_url": "https://nitter.pussthecat.org/video/F766930E58019/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2FFaGFrWPuuNDv7k2f.m3u8%3Ftag%3D6", "tbr": 2176.0, "ext": "mp4", "fps": null, "protocol": "m3u8", "preference": null, "quality": null, "width": 1280, "height": 720, "vcodec": "avc1.640020", "acodec": "mp4a.40.2", "format": "2176 - 1280x720", "http_headers": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3649.1 Safari/537.36", "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en-us,en;q=0.5", "Cookie": "hlsPlayback=on"}}], "thumbnails": [{"id": "large", "url": "https://nitter.pussthecat.orghttps://nitter.pussthecat.org/pic/ext_tw_video_thumb%2F1080208155735400450%2Fpu%2Fimg%2Fhm6uLLKu6P1Bm29A.jpg%3Alarge"}, {"id": "medium", "url": "https://nitter.pussthecat.orghttps://nitter.pussthecat.org/pic/ext_tw_video_thumb%2F1080208155735400450%2Fpu%2Fimg%2Fhm6uLLKu6P1Bm29A.jpg%3Amedium"}, {"id": "orig", "url": "https://nitter.pussthecat.orghttps://nitter.pussthecat.org/pic/ext_tw_video_thumb%2F1080208155735400450%2Fpu%2Fimg%2Fhm6uLLKu6P1Bm29A.jpg%3Aorig"}, {"id": "small", "url": "https://nitter.pussthecat.orghttps://nitter.pussthecat.org/pic/ext_tw_video_thumb%2F1080208155735400450%2Fpu%2Fimg%2Fhm6uLLKu6P1Bm29A.jpg%3Asmall"}, {"id": "thumb", "url": "https://nitter.pussthecat.orghttps://nitter.pussthecat.org/pic/ext_tw_video_thumb%2F1080208155735400450%2Fpu%2Fimg%2Fhm6uLLKu6P1Bm29A.jpg%3Athumb"}], "thumbnail": "https://nitter.pussthecat.orghttps://nitter.pussthecat.org/pic/ext_tw_video_thumb%2F1080208155735400450%2Fpu%2Fimg%2Fhm6uLLKu6P1Bm29A.jpg", "upload_date": "20210405", "extractor": "Nitter", "webpage_url": "https://nitter.pussthecat.org/manential/status/1379007759811772419#m", "webpage_url_basename": "1379007759811772419", "extractor_key": "Nitter", "playlist": null, "playlist_index": null, "display_id": "1379007759811772419", "requested_subtitles": null, "format_id": "2176", "format_index": null, "url": "https://nitter.pussthecat.org/video/AAEEBA690FA91/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2F1280x720%2Ft-IdaHQfnAVVvmJw.m3u8", "manifest_url": "https://nitter.pussthecat.org/video/F766930E58019/https%3A%2F%2Fvideo.twimg.com%2Fext_tw_video%2F1080208155735400450%2Fpu%2Fpl%2FFaGFrWPuuNDv7k2f.m3u8%3Ftag%3D6", "tbr": 2176.0, "ext": "mp4", "fps": null, "protocol": "m3u8", "preference": null, "quality": null, "width": 1280, "height": 720, "vcodec": "avc1.640020", "acodec": "mp4a.40.2", "format": "2176 - 1280x720", "http_headers": {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3649.1 Safari/537.36", "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en-us,en;q=0.5", "Cookie": "hlsPlayback=on"}}
```
<!--
Do not remove the above ```
-->

## Description

With this video, there is a bug: it says author is franceinfo while the real author is @manential
this is because it picks the first author with in fullname


Discussion:
also wrong like, view, repost, comment count for the same reason: it picks first occurrence but not the good one if the tweet in second in thread, timeline…
I think a similar issue would happen if there are two videos and we want the same.
I know we can look after id="m" or class="main-tweet" to get info about the tweet in url, not the first in thread

Resolving patch:
diff --git a/yt_dlp/extractor/nitter.py b/yt_dlp/extractor/nitter.py
--- a/yt_dlp/extractor/nitter.py
+++ b/yt_dlp/extractor/nitter.py
@@ -11,60 +11,100 @@
     determine_ext,
 )
 import re
+import random
 
 
 class NitterIE(InfoExtractor):
     # Taken from https://github.com/zedeus/nitter/wiki/Instances
-    INSTANCES = ('nitter.net',
-                 'nitter.snopyta.org',
-                 'nitter.42l.fr',
-                 'nitter.nixnet.services',
-                 'nitter.13ad.de',
-                 'nitter.pussthecat.org',
-                 'nitter.mastodont.cat',
-                 'nitter.dark.fail',
-                 'nitter.tedomum.net',
-                 'nitter.cattube.org',
-                 'nitter.fdn.fr',
-                 'nitter.1d4.us',
-                 'nitter.kavin.rocks',
-                 'tweet.lambda.dance',
-                 'nitter.cc',
-                 'nitter.weaponizedhumiliation.com',
-                 '3nzoldnxplag42gqjs23xvghtzf6t6yzssrtytnntc6ppc7xxuoneoad.onion',
-                 'nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion',
-                 'nitterlgj3n5fgwesu3vxc5h67ruku33nqaoeoocae2mvlzhsu6k7fqd.onion')
+
+    NON_HTTP_INSTANCES = (
+        '3nzoldnxplag42gqjs23xvghtzf6t6yzssrtytnntc6ppc7xxuoneoad.onion',
+        'nitter.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd.onion',
+        'nitter7bryz3jv7e3uekphigvmoyoem4al3fynerxkj22dmoxoq553qd.onion',
+        'npf37k3mtzwxreiw52ccs5ay4e6qt2fkcs2ndieurdyn2cuzzsfyfvid.onion',
+        'nitter.v6vgyqpa7yefkorazmg5d5fimstmvm2vtbirt6676mt7qmllrcnwycqd.onion',
+        'i23nv6w3juvzlw32xzoxcqzktegd4i4fu3nmnc2ewv4ggiu4ledwklad.onion',
+        '26oq3gioiwcmfojub37nz5gzbkdiqp7fue5kvye7d4txv4ny6fb4wwid.onion',
+
+        'nitter.i2p',
+        'u6ikd6zndl3c4dsdq4mmujpntgeevdk5qzkfb57r4tnfeccrn2qa.b32.i2p',
+
+        'nitterlgj3n5fgwesu3vxc5h67ruku33nqaoeoocae2mvlzhsu6k7fqd.onion',
+    )
+
+    HTTP_INSTANCES = (
+        'nitter.42l.fr',
+        'nitter.pussthecat.org',
+        'nitter.nixnet.services',
+        'nitter.mastodont.cat',
+        'nitter.tedomum.net',
+        'nitter.fdn.fr',
+        'nitter.1d4.us',
+        'nitter.kavin.rocks',
+        'tweet.lambda.dance',
+        'nitter.cc',
+        'nitter.vxempire.xyz',
+        'nitter.unixfox.eu',
+        'nitter.domain.glass',
+        'nitter.himiko.cloud',
+        'nitter.eu',
+        'nitter.namazso.eu',
+        'nitter.mailstation.de',
+        'nitter.actionsack.com',
+        'nitter.cattube.org',
+        'nitter.dark.fail',
+        'birdsite.xanny.family',
+        'nitter.40two.app',
+        'nitter.skrep.in',
+
+        # not in the list anymore
+        'nitter.snopyta.org',
+    )
+
+    DEAD_INSTANCES = (
+        # maintenance
+        'nitter.ethibox.fr',
+
+        # official, rate limited
+        'nitter.net',
+        # offline
+        'nitter.13ad.de',
+        'nitter.weaponizedhumiliation.com',
+    )
+
+    INSTANCES = NON_HTTP_INSTANCES + HTTP_INSTANCES + DEAD_INSTANCES
 
     _INSTANCES_RE = '(?:' + '|'.join([re.escape(instance) for instance in INSTANCES]) + ')'
     _VALID_URL = r'https?://%(instance)s/(?P<uploader_id>.+)/status/(?P<id>[0-9]+)(#.)?' % {'instance': _INSTANCES_RE}
-    current_instance = INSTANCES[0]  # the test and official instance
+    current_instance = random.choice(HTTP_INSTANCES)
+
     _TESTS = [
         {
             # GIF (wrapped in mp4)
-            'url': 'https://' + current_instance + '/firefox/status/1314279897502629888#m',
+            'url': 'https://%s/firefox/status/1314279897502629888#m' % current_instance,
             'info_dict': {
                 'id': '1314279897502629888',
                 'ext': 'mp4',
-                'title': 'Firefox 🔥 - You know the old saying, if you see something say something. Now you actually can with the YouTube regrets extension.   Report harmful YouTube recommendations so others can avoid watching them. ➡️ https://mzl.la/3iFIiyg  #UnfckTheInternet',
-                'description': 'You know the old saying, if you see something say something. Now you actually can with the YouTube regrets extension.   Report harmful YouTube recommendations so others can avoid watching them. ➡️ https://mzl.la/3iFIiyg  #UnfckTheInternet',
+                'title': 'Firefox 🔥 - You know the old saying, if you see something say something. Now you actually can with the YouTube regrets extension. \n\nReport harmful YouTube recommendations so others can avoid watching them. ➡️ https://mzl.la/3iFIiyg\n\n#UnfckTheInternet',
+                'description': 'You know the old saying, if you see something say something. Now you actually can with the YouTube regrets extension. \n\nReport harmful YouTube recommendations so others can avoid watching them. ➡️ https://mzl.la/3iFIiyg\n\n#UnfckTheInternet',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'uploader': 'Firefox 🔥',
                 'uploader_id': 'firefox',
-                'uploader_url': 'https://' + current_instance + '/firefox',
+                'uploader_url': 'https://%s/firefox' % current_instance,
                 'upload_date': '20201008',
                 'timestamp': 1602183720,
             },
         }, {  # normal video
-            'url': 'https://' + current_instance + '/Le___Doc/status/1299715685392756737#m',
+            'url': 'https://%s/Le___Doc/status/1299715685392756737#m' % current_instance,
             'info_dict': {
                 'id': '1299715685392756737',
                 'ext': 'mp4',
-                'title': 'Le Doc - "Je ne prédis jamais rien" D Raoult, Août 2020...',
-                'description': '"Je ne prédis jamais rien" D Raoult, Août 2020...',
+                'title': 'Le Doc - "Je ne prédis jamais rien"\nD Raoult, Août 2020...',
+                'description': '"Je ne prédis jamais rien"\nD Raoult, Août 2020...',
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'uploader': 'Le Doc',
                 'uploader_id': 'Le___Doc',
-                'uploader_url': 'https://' + current_instance + '/Le___Doc',
+                'uploader_url': 'https://%s/Le___Doc' % current_instance,
                 'upload_date': '20200829',
                 'timestamp': 1598711341,
                 'view_count': int,
@@ -73,31 +113,51 @@ class NitterIE(InfoExtractor):
                 'comment_count': int,
             },
         }, {  # video embed in a "Streaming Political Ads" box
-            'url': 'https://' + current_instance + '/mozilla/status/1321147074491092994#m',
+            'url': 'https://%s/mozilla/status/1321147074491092994#m' % current_instance,
             'info_dict': {
                 'id': '1321147074491092994',
                 'ext': 'mp4',
-                'title': "Mozilla - Are you being targeted with weird, ominous or just plain annoying political ads while streaming your favorite shows?  This isn't a real political ad, but if you're watching streaming TV in the U.S., chances are you've seen quite a few.   Learn more ➡️ https://mzl.la/StreamingAds",
-                'description': "Are you being targeted with weird, ominous or just plain annoying political ads while streaming your favorite shows?  This isn't a real political ad, but if you're watching streaming TV in the U.S., chances are you've seen quite a few.   Learn more ➡️ https://mzl.la/StreamingAds",
+                'title': "Mozilla - Are you being targeted with weird, ominous or just plain annoying political ads while streaming your favorite shows?\n\nThis isn't a real political ad, but if you're watching streaming TV in the U.S., chances are you've seen quite a few. \n\nLearn more ➡️ https://mzl.la/StreamingAds",
+                'description': "Are you being targeted with weird, ominous or just plain annoying political ads while streaming your favorite shows?\n\nThis isn't a real political ad, but if you're watching streaming TV in the U.S., chances are you've seen quite a few. \n\nLearn more ➡️ https://mzl.la/StreamingAds",
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'uploader': 'Mozilla',
                 'uploader_id': 'mozilla',
-                'uploader_url': 'https://' + current_instance + '/mozilla',
+                'uploader_url': 'https://%s/mozilla' % current_instance,
                 'upload_date': '20201027',
                 'timestamp': 1603820982
             },
-        },
+        }, {  # not the first tweet but main-tweet
+            'url': 'https://%s/TheNaturalNu/status/1379050895539724290#m' % current_instance,
+            'info_dict': {
+                'id': '1379050895539724290',
+                'ext': 'mp4',
+                'title': 'Dorothy Zbornak - This had me hollering!!',
+                'description': 'This had me hollering!!',
+                'thumbnail': r're:^https?://.*\.jpg$',
+                'uploader': 'Dorothy Zbornak',
+                'uploader_id': 'TheNaturalNu',
+                'uploader_url': 'https://%s/TheNaturalNu' % current_instance,
+                'timestamp': 1617626329,
+                'upload_date': '20210405'
+            }
+        }
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         parsed_url = compat_urlparse.urlparse(url)
-        base_url = parsed_url.scheme + '://' + parsed_url.netloc
+        base_url = '%s://%s' % (parsed_url.scheme, parsed_url.netloc)
 
         self._set_cookie(parsed_url.netloc, 'hlsPlayback', 'on')
-        webpage = self._download_webpage(url, video_id)
+        full_webpage = self._download_webpage(url, video_id)
+
+        main_tweet_start = full_webpage.find('class="main-tweet"')
+        if main_tweet_start > 0:
+            webpage = full_webpage[main_tweet_start:]
+        if not webpage:
+            webpage = full_webpage
 
-        video_url = base_url + self._html_search_regex(r'(?:<video[^>]+data-url|<source[^>]+src)="([^"]+)"', webpage, 'video url')
+        video_url = '%s%s' % (base_url, self._html_search_regex(r'(?:<video[^>]+data-url|<source[^>]+src)="([^"]+)"', webpage, 'video url'))
         ext = determine_ext(video_url)
 
         if ext == 'unknown_video':
@@ -108,32 +168,33 @@ def _real_extract(self, url):
                 'ext': ext
             }]
 
-        title = (
-            self._og_search_description(webpage).replace('\n', ' ')
-            or self._html_search_regex(r'<div class="tweet-content[^>]+>([^<]+)</div>', webpage, 'title'))
+        title = self._og_search_description(full_webpage)
+        if not title:
+            title = self._html_search_regex(r'<div class="tweet-content[^>]+>([^<]+)</div>', webpage, 'title')
         description = title
 
         mobj = re.match(self._VALID_URL, url)
         uploader_id = (
             mobj.group('uploader_id')
-            or self._html_search_regex(r'<a class="fullname"[^>]+title="([^"]+)"', webpage, 'uploader name', fatal=False))
+            or self._html_search_regex(r'<a class="fullname"[^>]+title="([^"]+)"', webpage, 'uploader name', fatal=False)
+        )
 
         if uploader_id:
-            uploader_url = base_url + '/' + uploader_id
+            uploader_url = '%s/%s' % (base_url, uploader_id)
 
         uploader = self._html_search_regex(r'<a class="fullname"[^>]+title="([^"]+)"', webpage, 'uploader name', fatal=False)
 
         if uploader:
-            title = uploader + ' - ' + title
+            title = '%s - %s' % (uploader, title)
 
         view_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-play[^>]*></span>\s([^<]+)</div>', webpage, 'view count', fatal=False))
         like_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-heart[^>]*></span>\s([^<]+)</div>', webpage, 'like count', fatal=False))
         repost_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-retweet[^>]*></span>\s([^<]+)</div>', webpage, 'repost count', fatal=False))
         comment_count = parse_count(self._html_search_regex(r'<span[^>]+class="icon-comment[^>]*></span>\s([^<]+)</div>', webpage, 'repost count', fatal=False))
 
-        thumbnail = self._html_search_meta('og:image', webpage, 'thumbnail url')
+        thumbnail = self._html_search_meta('og:image', full_webpage, 'thumbnail url')
         if not thumbnail:
-            thumbnail = base_url + self._html_search_regex(r'<video[^>]+poster="([^"]+)"', webpage, 'thumbnail url', fatal=False)
+            thumbnail = '%s%s' % (base_url, self._html_search_regex(r'<video[^>]+poster="([^"]+)"', webpage, 'thumbnail url', fatal=False))
             thumbnail = remove_end(thumbnail, '%3Asmall')
 
         thumbnails = []


Test patch: