Repository: kmille/deezer-downloader (language: python)

Issue:
Spotify downloader isn't downloading the full playlist
Hello,

I tried downloading a playlist from a Spotify playlist. It didn't seem to retrieve the full list. Logs say it has 100 `We got 100 songs from the Spotify playlist`. Retrying it doesn't work as it just says all the items already exist. Is there any way to download the rest?

Thanks


Discussion / hints:

Fix patch:
diff --git a/app/spotify.py b/app/spotify.py
index cf356b7..0d08c77 100644
--- a/app/spotify.py
+++ b/app/spotify.py
@@ -1,17 +1,60 @@
-import requests
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
-import json
 import re
+from time import sleep
+from urllib.parse import urlparse, parse_qs
+
+import requests
+
+token_url = 'https://open.spotify.com/get_access_token?reason=transport&productType=web_player'
+playlist_base_url = 'https://api.spotify.com/v1/playlists/{}/tracks?limit=100&additional_types=track&market=GB'  # todo figure out market
+track_base_url = 'https://api.spotify.com/v1/tracks/{}'
+album_base_url = 'https://api.spotify.com/v1/albums/{}/tracks'
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}
 
-base_url = "https://open.spotify.com/embed/playlist/{}"
-headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}
+
+class SpotifyInvalidUrlException(Exception):
+    pass
 
 
 class SpotifyWebsiteParserException(Exception):
     pass
 
 
+def parse_uri(uri):
+    u = urlparse(uri)
+    if u.netloc == "embed.spotify.com":
+        if not u.query:
+            raise SpotifyInvalidUrlException("ERROR: url {} is not supported".format(uri))
+
+        qs = parse_qs(u.query)
+        return parse_uri(qs['uri'][0])
+
+    # backwards compatibility
+    if not u.scheme and not u.netloc:
+        return {"type": "playlist", "id": u.path}
+
+    if u.scheme == "spotify":
+        parts = uri.split(":")
+    else:
+        if u.netloc != "open.spotify.com" and u.netloc != "play.spotify.com":
+            raise SpotifyInvalidUrlException("ERROR: url {} is not supported".format(uri))
+
+        parts = u.path.split("/")
+
+    if parts[1] == "embed":
+        parts = parts[1:]
+
+    l = len(parts)
+    if l == 3 and parts[1] in ["album", "track", "playlist"]:
+        return {"type": parts[1], "id": parts[2]}
+    if l == 5 and parts[3] == "playlist":
+        return {"type": parts[3], "id": parts[4]}
+
+    # todo add support for other types; artists, searches, users
+
+    raise SpotifyInvalidUrlException("ERROR: unable to determine Spotify URL type or type is unsupported.")
+
+
 def get_songs_from_spotify_website(playlist):
     # parses Spotify Playlist from Spotify website
     # playlist: playlist url or playlist id as string
@@ -22,34 +65,67 @@ def get_songs_from_spotify_website(playlist):
     # raises SpotifyWebsiteParserException if parsing the website goes wrong
 
     return_data = []
-    if playlist.startswith("http"):
-        url = playlist.replace("spotify.com/playlist", "spotify.com/embed/playlist")
-    else:
-        url = base_url.format(playlist)
+    url_info = parse_uri(playlist)
+
+    req = requests.get(token_url)
+    if req.status_code != 200:
+        raise SpotifyWebsiteParserException(
+            "ERROR: {} gave us not a 200. Instead: {}".format(token_url, req.status_code))
+    token = req.json()
+
+    if url_info['type'] == "playlist":
+        url = playlist_base_url.format(url_info["id"])
+        while True:
+            resp = get_json_from_api(url, token["accessToken"])
+            for track in resp['items']:
+                return_data.append(parse_track(track["track"]))
+
+            if resp['next'] is None:
+                break
+            url = resp['next']
+    elif url_info["type"] == "track":
+        resp = get_json_from_api(track_base_url.format(url_info["id"]), token["accessToken"])
+        if resp is None:  # try again in case of rate limit
+            resp = get_json_from_api(track_base_url.format(url_info["id"]), token["accessToken"])
+
+        return_data.append(parse_track(resp))
+    elif url_info["type"] == "album":
+        resp = get_json_from_api(album_base_url.format(url_info["id"]), token["accessToken"])
+        if resp is None:  # try again in case of rate limit
+            resp = get_json_from_api(album_base_url.format(url_info["id"]), token["accessToken"])
+
+        for track in resp['items']:
+            return_data.append(parse_track(track))
+
+    return [track for track in return_data if track]
+
+
+def parse_track(track):
+    artist = track['artists'][0]['name']
+    song = track['name']
+    full = "{} {}".format(artist, song)
+    # remove everything in brackets to get better search results later on Deezer
+    # e.g. (Radio  Version) or (Remastered)
+    return re.sub(r'\([^)]*\)', '', full)
+
+
+def get_json_from_api(api_url, access_token):
+    req = requests.get(api_url, headers={'Authorization': 'Bearer {}'.format(access_token)})
+
+    if req.status_code == 429:
+        seconds = int(req.headers.get("Retry-After")) + 1
+        print("INFO: rate limited! Sleeping for {} seconds".format(seconds))
+        sleep(seconds)
+        return None
 
-    req = requests.get(url)
     if req.status_code != 200:
-        raise SpotifyWebsiteParserException("ERROR: {} gave us not a 200. Instead: {}".format(url, req.status_code))
-
-    bs = BeautifulSoup(req.text, 'html.parser')
-    try:
-        songs_txt = bs.find('script', {'id': 'resource'}).string.strip()
-    except AttributeError:
-        raise SpotifyWebsiteParserException("ERROR: Could not get songs from Spotify website. Wrong Playlist id? Tried {}".format(url))
-    songs_json = json.loads(unquote(songs_txt))
-
-    for track in songs_json['tracks']['items']:
-        artist = track['track']['artists'][0]['name']
-        song = track['track']['name']
-        full = "{} {}".format(artist, song)
-        # remove everything in brackets to get better search results later on Deezer
-        # e.g. (Radio  Version) or (Remastered)
-        full = re.sub(r'\([^)]*\)', '', full)
-        return_data.append(full)
-    return return_data
+        raise SpotifyWebsiteParserException("ERROR: {} gave us not a 200. Instead: {}".format(api_url, req.status_code))
+    return req.json()
 
 
 if __name__ == '__main__':
-    #playlist = "21wZXvtrERELL0bVtKtuUh"
-    playlist = "0wl9Q3oedquNlBAJ4MGZtS?si=jmdvnQSyRYCxDTWzrZARJg"
-    get_songs_from_spotify_website(playlist)
+    # playlist = "21wZXvtrERELL0bVtKtuUh"
+    playlist = "0wl9Q3oedquNlBAJ4MGZtS"
+    album = "spotify:album:7zCODUHkfuRxsUjtuzNqbd"
+    song = "https://open.spotify.com/track/6piFKF6WvM6ZZLmi2Vz8Vt"
+    print(get_songs_from_spotify_website(playlist))
diff --git a/requirements.txt b/requirements.txt
index 9512592..5e26e1c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,4 @@
 attrs==20.3.0
-beautifulsoup4==4.9.3
-bs4==0.0.1
 certifi==2020.12.5
 chardet==4.0.0
 click==7.1.2
@@ -27,7 +25,6 @@ pytest==6.2.2
 python-magic==0.4.22
 python-mpd2==3.0.4
 requests==2.25.1
-soupsieve==2.2.1
 toml==0.10.2
 urllib3==1.26.4
 Werkzeug==1.0.1


Test patch:
diff --git a/app/tests.py b/app/tests.py
index b2f87fa..212499d 100644
--- a/app/tests.py
+++ b/app/tests.py
@@ -5,7 +5,7 @@
 from deezer import deezer_search, get_song_infos_from_deezer_website, parse_deezer_playlist, download_song, get_deezer_favorites
 from deezer import TYPE_TRACK, TYPE_ALBUM
 from deezer import Deezer404Exception, DeezerApiException
-from spotify import get_songs_from_spotify_website, SpotifyWebsiteParserException
+from spotify import get_songs_from_spotify_website, SpotifyWebsiteParserException, parse_uri, SpotifyInvalidUrlException
 from youtubedl import youtubedl_download, YoutubeDLFailedException
 
 from configuration import config
@@ -236,6 +236,33 @@ def test_download_song_invalid_song_type(self):
 
 
 class TestSpotifyMethods(unittest.TestCase):
+    def test_parse_url_spotify(self):
+        res = parse_uri("spotify:album:Hksdhfaif23ffushef9823")
+        self.assertEqual(res['type'], "album")
+        self.assertEqual(res['id'], "Hksdhfaif23ffushef9823")
+
+        res = parse_uri("spotify:playlist:Hksdhfaif23ffushef9823")
+        self.assertEqual(res['type'], "playlist")
+        self.assertEqual(res['id'], "Hksdhfaif23ffushef9823")
+
+        res = parse_uri("spotify:track:Hksdhfaif23ffushef9823")
+        self.assertEqual(res['type'], "track")
+        self.assertEqual(res['id'], "Hksdhfaif23ffushef9823")
+
+    def test_parse_url_open_domain(self):
+        res = parse_uri("https://open.spotify.com/track/Hksdhfaif23ffushef9823")
+        self.assertEqual(res['type'], "track")
+        self.assertEqual(res['id'], "Hksdhfaif23ffushef9823")
+
+    def test_parse_url_play_domain(self):
+        res = parse_uri("https://play.spotify.com/track/Hksdhfaif23ffushef9823")
+        self.assertEqual(res['type'], "track")
+        self.assertEqual(res['id'], "Hksdhfaif23ffushef9823")
+
+    def test_parse_url_embed_domain(self):
+        res = parse_uri("https://embed.spotify.com/?uri=spotify:track:Hksdhfaif23ffushef9823")
+        self.assertEqual(res['type'], "track")
+        self.assertEqual(res['id'], "Hksdhfaif23ffushef9823")
 
     def _test_parse_spotify_playlist_website(self, playlist):
         songs = get_songs_from_spotify_website(playlist)
@@ -260,7 +287,7 @@ def test_spotify_parser_invalid_playlist_id(self):
 
     def test_spotify_parser_invalid_playlist_url(self):
         playlist_url = "https://www.heise.de"
-        with self.assertRaises(SpotifyWebsiteParserException):
+        with self.assertRaises(SpotifyInvalidUrlException):
             get_songs_from_spotify_website(playlist_url)