Repository: Flexget/Flexget

Issue:
The torrentleech plugin is not working with their V5 api 
### Expected behaviour:
The first episode of the series is downloaded

### Actual behaviour:
No results are found from the torrentleech search plugin


### Steps to reproduce:
- Step 1: Run flexget with the config supplied in the issue

#### Config:
```
tasks:
  test task:
    series:
      - Rick and Morty:
          begin: S02E01
    discover:
      what:
        - next_series_episodes:
            backfill: yes
      from:
        - torrentleech:
            rss_key: ***********
            username: ***********
            password: ***********
    download: ~/downloads

schedules:
  # Run every 30 minutes
  - tasks: '*'
    schedule:
      minute: "*/20"
```
  
### Additional information:
This seems to be a result of the upgrade to the Torrentleech V5 api. I modified the plugin to use the V4 api and this is working for now as long as they don't remove support for it. Would it be reasonable to put a PR with a Torrentleech plugin that uses the V4 api until support for the V5 api has been implemented? 


- FlexGet version: 2.13.8.dev
- Python version: Python 2.7.14
- Installation method: Latest git version
- Using daemon (yes/no): no
- OS and version: MacOS and Ubuntu Server



Discussion:

Resolving patch:
diff --git a/flexget/plugins/output/download.py b/flexget/plugins/output/download.py
--- a/flexget/plugins/output/download.py
+++ b/flexget/plugins/output/download.py
@@ -232,8 +232,13 @@ def download_entry(self, task, entry, url, tmp_path):
             auth = entry['download_auth']
             log.debug('Custom auth enabled for %s download: %s', entry['title'], entry['download_auth'])
 
+        headers = task.requests.headers
+        if 'download_headers' in entry:
+            headers.update(entry['download_headers'])
+            log.debug('Custom headers enabled for %s download: %s', entry['title'], entry['download_headers'])
+
         try:
-            response = task.requests.get(url, auth=auth, raise_status=False)
+            response = task.requests.get(url, auth=auth, raise_status=False, headers=headers)
         except UnicodeError:
             log.error('Unicode error while encoding url %s', url)
             return
diff --git a/flexget/plugins/sites/torrentleech.py b/flexget/plugins/sites/torrentleech.py
--- a/flexget/plugins/sites/torrentleech.py
+++ b/flexget/plugins/sites/torrentleech.py
@@ -2,7 +2,6 @@
 from builtins import *  # noqa pylint: disable=unused-import, redefined-builtin
 from future.moves.urllib.parse import quote
 
-import re
 import logging
 
 from requests.exceptions import RequestException
@@ -13,7 +12,6 @@
 from flexget.event import event
 from flexget.plugin import PluginError
 from flexget.plugins.internal.urlrewriting import UrlRewritingError
-from flexget.utils.soup import get_soup
 from flexget.utils.search import torrent_availability, normalize_unicode
 from flexget.utils.tools import parse_filesize
 
@@ -77,7 +75,7 @@ def url_rewritable(self, task, entry):
         url = entry['url']
         if url.endswith('.torrent'):
             return False
-        if url.startswith('https://v4.torrentleech.org/'):
+        if url.startswith('https://www.torrentleech.org/'):
             return True
         return False
 
@@ -87,7 +85,7 @@ def url_rewrite(self, task, entry):
             log.error("Didn't actually get a URL...")
         else:
             log.debug("Got the URL: %s" % entry['url'])
-        if entry['url'].startswith('https://v4.torrentleech.org/torrents/browse/index/query/'):
+        if entry['url'].startswith('https://www.torrentleech.org/torrents/browse/list/query/'):
             # use search
             results = self.search(task, entry)
             if not results:
@@ -100,13 +98,15 @@ def search(self, task, entry, config=None):
         """
         Search for name from torrentleech.
         """
+        request_headers = {'User-Agent': 'curl/7.54.0'}
         rss_key = config['rss_key']
 
         # build the form request:
         data = {'username': config['username'], 'password': config['password']}
         # POST the login form:
         try:
-            login = task.requests.post('https://v4.torrentleech.org/user/account/login/', data=data)
+            login = task.requests.post('https://www.torrentleech.org/user/account/login/', data=data,
+                                       headers=request_headers, allow_redirects=True)
         except RequestException as e:
             raise PluginError('Could not connect to torrentleech: %s', str(e))
 
@@ -121,48 +121,35 @@ def search(self, task, entry, config=None):
             categories = [categories]
         # If there are any text categories, turn them into their id number
         categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
-        filter_url = '/categories/%s' % ','.join(str(c) for c in categories)
+        filter_url = '/categories/{}'.format(','.join(str(c) for c in categories))
         entries = set()
         for search_string in entry.get('search_strings', [entry['title']]):
             query = normalize_unicode(search_string).replace(":", "")
-            # urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
-            url = ('https://v4.torrentleech.org/torrents/browse/index/query/' +
+            # urllib.quote will crash if the unicode string has non ascii characters,
+            # so encode in utf-8 beforehand
+
+            url = ('https://www.torrentleech.org/torrents/browse/list/query/' +
                    quote(query.encode('utf-8')) + filter_url)
-            log.debug('Using %s as torrentleech search url' % url)
+            log.debug('Using %s as torrentleech search url', url)
 
-            page = task.requests.get(url, cookies=login.cookies).content
-            soup = get_soup(page)
+            results = task.requests.get(url, headers=request_headers, cookies=login.cookies).json()
 
-            for tr in soup.find_all("tr", ["even", "odd"]):
-                # within each even or odd row, find the torrent names
-                link = tr.find("a", attrs={'href': re.compile('/torrent/\d+')})
-                log.debug('link phase: %s' % link.contents[0])
+            for torrent in results['torrentList']:
                 entry = Entry()
-                # extracts the contents of the <a>titlename/<a> tag
-                entry['title'] = link.contents[0]
-
-                # find download link
-                torrent_url = tr.find("a", attrs={'href': re.compile('/download/\d+/.*')}).get('href')
-                # parse link and split along /download/12345 and /name.torrent
-                download_url = re.search('(/download/\d+)/(.+\.torrent)', torrent_url)
-                # change link to rss and splice in rss_key
-                torrent_url = 'https://v4.torrentleech.org/rss' + download_url.group(1) + '/' \
-                              + rss_key + '/' + download_url.group(2)
-                log.debug('RSS-ified download link: %s' % torrent_url)
+                entry['download_headers'] = request_headers
+                entry['title'] = torrent['name']
+
+                # construct download URL
+                torrent_url = 'https://www.torrentleech.org/rss/download/{}/{}/{}'.format(torrent['fid'], rss_key,
+                                                                                          torrent['filename'])
+                log.debug('RSS-ified download link: %s', torrent_url)
                 entry['url'] = torrent_url
 
-                # us tr object for seeders/leechers
-                seeders, leechers = tr.find_all('td', ["seeders", "leechers"])
-                entry['torrent_seeds'] = int(seeders.contents[0])
-                entry['torrent_leeches'] = int(leechers.contents[0])
+                # seeders/leechers
+                entry['torrent_seeds'] = torrent['seeders']
+                entry['torrent_leeches'] = torrent['leechers']
                 entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
-
-                # use tr object for size
-                size = tr.find("td", text=re.compile('([\.\d]+) ([TGMK]?)B')).contents[0]
-                size = re.search('([\.\d]+) ([TGMK]?)B', size)
-
-                entry['content_size'] = parse_filesize(size.group(0))
-
+                entry['content_size'] = parse_filesize(str(torrent['size']) + ' b')
                 entries.add(entry)
 
         return sorted(entries, reverse=True, key=lambda x: x.get('search_sort'))


Test patch: