Upload 2 files

app.py

@@ -0,0 +1,364 @@
+import gradio as gr
+import spotipy
+from spotipy import oauth2
+
+from transformers import ViTForImageClassification, ViTImageProcessor
+import torch
+from torch.nn import functional as F
+from torchvision.io import read_image
+
+import tensorflow as tf
+
+from fastapi import FastAPI
+from starlette.middleware.sessions import SessionMiddleware
+from starlette.responses import HTMLResponse, RedirectResponse
+from starlette.requests import Request
+import gradio as gr
+import uvicorn
+from fastapi.responses import HTMLResponse
+from fastapi.responses import RedirectResponse
+
+import numpy as np
+import base64
+from io import BytesIO
+from PIL import Image
+
+import shred_model
+
+# Xception fine tuned from pretrained imagenet weights for identifying Sraddha
+SRADDHA_MODEL_PATH = "shred_model"
+SHRED_MODEL = tf.keras.models.load_model(SRADDHA_MODEL_PATH)
+
+SPOTIPY_TOKEN = None # Set in the homepage function
+
+def main(img, playlist_length, privacy, gen_mode, genre_choice, request: gr.Request):
+    if img is None:
+        return None
+
+    mood_dict = get_image_mood_dict_from_transformer(img)
+    sraddha_found = get_sraddha(img)
+    
+    playlist = get_playlist(mood_dict, img, playlist_length, privacy, gen_mode, genre_choice, request)
+    if playlist is None:
+        playlist = "Spotipy account token not set"
+    
+    ret = playlist
+    if sraddha_found:
+        sraddha_msg = """Sraddha, you are the love of my life and seeing you always lifts my spirits. Hopefully these tunes can do the same for you.
+        <p>
+        </p>
+        - With Love, Scoob"""
+        return gr.update(value=ret, visible=True), gr.update(value=sraddha_msg, visible=True)
+    return gr.update(value=ret, visible=True), gr.update(visible=False)
+
+def get_image_mood_dict_from_transformer(img):
+    img = read_image(img)
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    
+    model = ViTForImageClassification.from_pretrained("jayanta/google-vit-base-patch16-224-cartoon-emotion-detection")
+    model.eval()
+    model.to(device)
+
+    feature_extractor = ViTImageProcessor.from_pretrained("jayanta/google-vit-base-patch16-224-cartoon-emotion-detection")
+    encoding = feature_extractor(images=img, return_tensors="pt")
+    pixel_values = encoding['pixel_values'].to(device)
+    outputs = model(pixel_values)
+
+    logits = outputs.logits
+    probabilities = F.softmax(logits, dim = -1).detach().numpy()[0]
+    mood_dict = dict(zip(model.config.id2label.values(), probabilities))
+    return mood_dict
+
+def get_sraddha(img):
+    fixed_img = shred_model.prepare_image(img)
+    prob = SHRED_MODEL.predict(fixed_img)[0]
+    if prob >= .5:
+        return True
+
+def compute_mood(mood_dict):
+    print(mood_dict)
+    return mood_dict['happy'] + mood_dict['angry'] * .5 + mood_dict['sad'] * .1
+
+def get_playlist(mood_dict, img, playlist_length, privacy, gen_mode, genre_choice, request: gr.Request):
+    token = request.request.session.get('token')
+    genre_map = {'Rock': ['alt-rock', 'alternative', 'indie', 'r-n-b', 'rock'], 'Hip-hop': ['hip-hop'], 'Party': ['club', 'dance', 'house', 'pop', 'party'], 'Mellow': ['blues', 'chill', 'classical', 'jazz', 'happy'], 'Indian': ['idm', 'indian'], 'Pop': ['pop', 'new-age'], 'Study': ['study', 'classical', 'jazz', 'happy', 'chill'], 'Romance': ['romance', 'happy', 'pop']}
+        
+    if token:
+        playlist_name = "Mood " + str(round(compute_mood(mood_dict) * 100, 1))
+        sp = spotipy.Spotify(token)
+
+        if gen_mode == 'Recently Played':
+            top_tracks_uri = set(sp.current_user_recently_played(limit=50))
+            # I honestly don't know if this errors for people with not enough saved tracks
+            # Shouldn't be a problem for Sraddha
+            first_few = [x['track']['uri'] for x in sp.current_user_saved_tracks(limit=50)['items']]
+            top_tracks_uri.update(first_few)
+            top_tracks_uri.update([x['track']['uri'] for x in sp.current_user_saved_tracks(limit=50, offset=50)['items']])
+            top_tracks_uri.update([x['track']['uri'] for x in sp.current_user_saved_tracks(limit=50, offset=100)['items']])
+            top_tracks_uri.update([x['track']['uri'] for x in sp.current_user_saved_tracks(limit=50, offset=150)['items']])
+            top_tracks_uri.update([x['uri'] for x in sp.recommendations(seed_tracks=first_few[:5], limit=50)['tracks']])
+            top_tracks_uri.update([x['uri'] for x in sp.recommendations(seed_tracks=first_few[5:10], limit=50)['tracks']])
+            top_tracks_uri = list(top_tracks_uri)
+        elif gen_mode == 'By a Chosen Genre':
+            genres = genre_map[genre_choice]
+            mood = compute_mood(mood_dict)
+            final_track_list = [x['uri'] for x in sp.recommendations(
+                seed_genres=genres, limit=playlist_length, max_valence=mood+.15, 
+                min_valence=mood-.15, min_danceability=mood/1.75, max_danceability=mood*8, 
+                min_energy=mood/2)['tracks']]
+        else:
+            top_artists_uri = aggregate_favorite_artists(sp)
+            top_tracks_uri = aggregate_top_tracks(sp, top_artists_uri)
+
+        if gen_mode != 'By a Chosen Genre':
+            final_track_list = filter_tracks(sp, top_tracks_uri, mood_dict, playlist_length)
+        
+        # If no tracks fit the filter: generate some results anyways
+        if len(final_track_list) == 0:
+            if gen_mode == 'By a Chosen Genre':
+                final_track_list = [x['uri'] for x in sp.recommendations(
+                    seed_genres=genres, limit=playlist_length,
+                    min_valence=mood-.3, min_energy=mood/3)['tracks']]
+            else:
+                seed = sp.current_user_recently_played(limit=5)
+                final_track_list = [x['uri'] for x in sp.recommendations(
+                    seed_tracks=seed, limit=playlist_length,
+                    min_valence=mood-.3, min_energy=mood/3)['tracks']]
+
+        iframe_embedding = create_playlist(sp, img, final_track_list, playlist_name,
+                                            privacy)
+        return iframe_embedding
+    return None
+
+def create_playlist(sp, img, tracks, playlist_name, privacy): 
+    privacy = privacy == "Public"
+    user_id = sp.current_user()['id']
+    playlist_description = "This playlist was created using the img-to-music application built by the best boyfriend there ever was and ever will be"
+    playlist_data = sp.user_playlist_create(user_id, playlist_name, public=privacy, 
+                                            description=playlist_description)
+    playlist_id = playlist_data['id']
+    if len(tracks) == 0:
+        return """No tracks could be generated from this image"""
+    sp.user_playlist_add_tracks(user_id, playlist_id, tracks)
+    try: 
+        with Image.open(img) as im_file:
+            im_file.thumbnail((300, 300))
+            buffered = BytesIO()
+            im_file.save(buffered, format="JPEG")
+            img_str = base64.b64encode(buffered.getvalue())
+        
+            sp.playlist_upload_cover_image(playlist_id, img_str)
+    except spotipy.exceptions.SpotifyException:
+        print("Image file too large, couldn't upload")
+
+    iframe_embedding = f"""<iframe style="border-radius:12px" src="https://open.spotify.com/embed/playlist/{playlist_id}" width="100%" height="352" frameBorder="0" allowfullscreen="" allow="autoplay; clipboard-write; encrypted-media; fullscreen; picture-in-picture" loading="lazy"></iframe>"""
+    return iframe_embedding
+
+def aggregate_favorite_artists(sp):
+    top_artists_name = set()
+    top_artists_uri = []
+
+    ranges = ['short_term', 'medium_term', 'long_term']
+    for r in ranges:
+        top_artists_all_data = sp.current_user_top_artists(limit=50, time_range=r)
+        top_artists_data = top_artists_all_data['items']
+        for artist_data in top_artists_data:
+            if artist_data["name"] not in top_artists_name:
+                top_artists_name.add(artist_data['name'])
+                top_artists_uri.append(artist_data['uri'])
+    
+    followed_artists_all_data = sp.current_user_followed_artists(limit=50)
+    followed_artsits_data = followed_artists_all_data['artists']
+    for artist_data in followed_artsits_data['items']:
+        if artist_data["name"] not in top_artists_name:
+            top_artists_name.add(artist_data['name'])
+            top_artists_uri.append(artist_data['uri'])
+
+    # attempt to garauntee 200 artists
+    i = 0
+    while len(top_artists_uri) < 200:
+        related_artists_all_data = sp.artist_related_artists(top_artists_uri[i])
+        i += 1
+        related_artists_data = related_artists_all_data['artists']
+        for artist_data in related_artists_data:
+            if artist_data["name"] not in top_artists_name:
+                top_artists_name.add(artist_data['name'])
+                top_artists_uri.append(artist_data['uri'])
+        if i == len(top_artists_uri):
+            # could build in a deeper artist recommendation finder here 
+            # would do this if it was going to production but Sraddha follows lots of artists
+            break
+
+    return top_artists_uri
+
+def aggregate_top_tracks(sp, top_artists_uri):
+    top_tracks_uri = []
+    for artist in top_artists_uri:
+        top_tracks_all_data = sp.artist_top_tracks(artist)
+        top_tracks_data = top_tracks_all_data['tracks']
+        for track_data in top_tracks_data:
+            top_tracks_uri.append(track_data['uri'])
+    return top_tracks_uri
+
+def filter_tracks(sp, top_tracks_uri, mood_dict, playlist_length):
+    mood = compute_mood(mood_dict)
+    selected_tracks_uri = []
+    
+    np.random.shuffle(top_tracks_uri)
+    # Batch network requests
+    BATCH_SIZE = 100
+    i = 0
+    all_track_data = []
+    while i + BATCH_SIZE < len(top_tracks_uri):
+        all_track_data += sp.audio_features(top_tracks_uri[i:i+BATCH_SIZE])
+        i += BATCH_SIZE
+    all_track_data += sp.audio_features(top_tracks_uri[i:])
+
+    for i, track in enumerate(top_tracks_uri):
+        track_data = all_track_data[i]
+        if track_data is None:
+            continue
+
+        valence = track_data['valence']
+        danceability = track_data['danceability']
+        energy = track_data['energy']
+        if mood < .1:
+            if valence <= mood + .15 and \
+               danceability <= mood * 8 and \
+               energy <= mood * 10:
+                selected_tracks_uri.append(track)
+        elif mood < .25:
+            if (mood - .1) <= valence <= (mood + .1) and \
+               danceability <= mood * 4 and \
+               energy <= mood * 5:
+                selected_tracks_uri.append(track)
+        elif mood < .5:
+            if mood - .05 <= valence <= mood + .05 and \
+               danceability <= mood * 1.75 and \
+               energy <= mood * 1.75:
+                selected_tracks_uri.append(track)
+        elif mood < .75:
+            if mood - .1 <= valence <= mood + .1 and \
+               danceability >= mood / 2.5 and \
+               energy >= mood / 2:
+                selected_tracks_uri.append(track)
+        elif mood < .9:
+            if mood - .1 <= valence <= mood + .1 and \
+               danceability >= mood / 2 and \
+               energy >= mood / 1.75:
+                selected_tracks_uri.append(track)
+        else:
+            if mood - .15 <= valence <= 1 and \
+               danceability >= mood / 1.75 and \
+               energy >= mood / 1.5:
+                selected_tracks_uri.append(track)
+        
+        if len(selected_tracks_uri) >= playlist_length:
+            break
+    return selected_tracks_uri
+
+# Define login and frontend
+PORT_NUMBER = 8080
+SPOTIPY_CLIENT_ID = '2320153024d042c8ba138a108066246c'
+SPOTIPY_CLIENT_SECRET = 'da2746490f6542a3b0cfcff50893e8e8'
+#SPOTIPY_REDIRECT_URI = 'http://localhost:7860'
+SPOTIPY_REDIRECT_URI = "https://huggingface.co/spaces/Bokanovskii/Image-to-music"
+SCOPE = 'ugc-image-upload playlist-read-private playlist-read-collaborative playlist-modify-private playlist-modify-public user-top-read user-read-recently-played user-library-modify user-library-read user-read-email user-read-private'
+
+sp_oauth = oauth2.SpotifyOAuth(SPOTIPY_CLIENT_ID, SPOTIPY_CLIENT_SECRET, SPOTIPY_REDIRECT_URI, scope=SCOPE)
+
+app = FastAPI()
+app.add_middleware(SessionMiddleware, secret_key="w.o.w")
+
+@app.get('/', response_class=HTMLResponse)
+async def homepage(request: Request):
+    url = str(request.url)
+    code = sp_oauth.parse_response_code(url)
+    if code != url:
+        request.session['token'] = sp_oauth.get_access_token(code, as_dict=False)
+        return RedirectResponse("/gradio")
+
+    auth_url = sp_oauth.get_authorize_url()
+    return """<div style="text-align: center; max-width: 1000px; margin: 0 auto;">
+                <div
+                style="
+                    align-items: center;
+                    gap: 0.8rem;
+                    font-size: 1.75rem;
+                "
+                >
+                <h3 style="font-weight: 900; margin-bottom: 30px; margin-top: 20px;">
+                    Image to Music Generator
+                </h3>\n""" + \
+                f"<a href='" + auth_url + "'>Login to Spotify</a>\n" + \
+                """<p>
+                </p>
+                <div
+                style="
+                    align-items: center;
+                    gap: 0.8rem;
+                    font-size: 1rem;
+                "
+                >
+                <small> 
+                    This applet requires a whitelisted Spotify account (contact Charlie Ward)
+                </small>"""
+
+with gr.Blocks(css="style.css") as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.HTML("""<div style="text-align: center; max-width: 700px; margin: 0 auto;">
+                <div
+                style="
+                    display: inline-flex;
+                    align-items: center;
+                    gap: 0.8rem;
+                    font-size: 1.75rem;
+                "
+                >
+                <h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;">
+                    Image to Music Generator
+                </h1>""")
+    
+        input_img = gr.Image(type="filepath", elem_id="input-img")
+        sraddhas_box = gr.HTML(label="Sraddha's Box", elem_id="sraddhas-box", visible=False)
+        playlist_output = gr.HTML(label="Generated Playlist", elem_id="app-output", visible=True)
+
+        with gr.Accordion(label="Playlist Generation Options", open=False):
+            playlist_length = gr.Slider(minimum=5, maximum=100, value=30, step=5, 
+                                        label="Playlist Length", elem_id="playlist-length")
+            with gr.Row():
+                privacy = gr.Radio(label="Playlist Privacy Level", choices=["Public", "Private"],
+                                    value="Private")
+                gen_mode = gr.Radio(label="Recommendation Base", choices=["Favorites", "Recently Played", "By a Chosen Genre"], value="By a Chosen Genre")
+            with gr.Row(visible=True) as genre_choice_row:
+                genre_choice = gr.Dropdown(label='Choose a Genre', choices=['Rock', 'Pop', 'Hip-hop', 'Party', 'Mellow', 'Indian', 'Study', 'Romance'], value='Pop')
+        
+        def sraddha_box_hide():
+            return {sraddhas_box: gr.update(visible=False)}
+
+        def genre_dropdown_toggle(gen_mode):
+            if gen_mode == 'By a Chosen Genre':
+                return {genre_choice_row: gr.update(visible=True)}
+            else:
+                return {genre_choice_row: gr.update(visible=False)}
+
+        generate = gr.Button("Generate Playlist from Image")
+
+        article = """
+                <div class="footer">
+                    <p>
+                    Built for Sraddha: playlist generation from image inference
+                    </p>
+                    <p>
+                    Sending Love 🤗
+                    </p>
+                </div>
+            """
+        gr.HTML(article)
+    gen_mode.change(genre_dropdown_toggle, inputs=[gen_mode], outputs=[genre_choice_row])
+    generate.click(sraddha_box_hide, outputs=[sraddhas_box])
+    generate.click(main, inputs=[input_img, playlist_length, privacy, gen_mode, genre_choice], 
+                   outputs=[playlist_output, sraddhas_box], api_name="img-to-music")
+
+gradio_app = gr.mount_gradio_app(app, demo, "/gradio")
+uvicorn.run(app, host="localhost", port=7860)

style.css

@@ -0,0 +1,42 @@
+#col-container {
+  max-width: 510px;
+  margin-left: auto;
+  margin-right: auto;
+}
+a {
+  text-decoration-line: underline;
+  font-weight: 600;
+}
+div#app-output .h-full {
+  min-height: 5rem;
+}
+.footer {
+  margin-bottom: 45px;
+  margin-top: 10px;
+  text-align: center;
+  border-bottom: 1px solid #e5e5e5;
+}
+.footer > p {
+  font-size: 0.8rem;
+  display: inline-block;
+  padding: 0 10px;
+  transform: translateY(10px);
+  background: white;
+}
+.dark .footer {
+  border-color: #303030;
+}
+.dark .footer > p {
+  background: #0b0f19;
+}
+.animate-spin {
+  animation: spin 1s linear infinite;
+}
+@keyframes spin {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}