I have the following code and it works well in detecting the cords. but it I wonder when transposing to -1 I get when i Expect a G chord i get an E and when expecting an Am Chord displayed is a G. Have i understood the program wrong or is there a bug in the code?


import tkinter as tk
from tkinter import filedialog, messagebox, scrolledtext
import pygame
import librosa
import numpy as np
import threading
import time
import whisper
from pydub import AudioSegment
import os
import tempfile
import re
from tqdm import tqdm

# Define chord templates (Major and Minor)
CHORD_TEMPLATES = {
    'C':    [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1],
    'C#':   [1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0],
    'D':    [0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0],
    'D#':   [0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0],
    'E':    [0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1],
    'F':    [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0],
    'F#':   [0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0],
    'G':    [0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0],
    'G#':   [0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0],
    'A':    [0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],
    'A#':   [0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1],
    'B':    [0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0],
    'Cm':   [1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1],
    'C#m':  [0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0],
    'Dm':   [0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0],
    'D#m':  [0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0],
    'Em':   [0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1],
    'Fm':   [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1],
    'F#m':  [0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1],
    'Gm':   [0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0],
    'G#m':  [0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0],
    'Am':   [0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],
    'A#m':  [0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0],
    'Bm':   [0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0],
}

# Normalize chord templates
for chord in CHORD_TEMPLATES:
    template = np.array(CHORD_TEMPLATES[chord], dtype=float)
    CHORD_TEMPLATES[chord] = template / np.linalg.norm(template) if np.linalg.norm(template) != 0 else template

def transpose_chord(chord, semitones):
    """Transpose a chord by a number of semitones."""
    base_chords = ['C', 'C#', 'D', 'D#', 'E', 'F',
                  'F#', 'G', 'G#', 'A', 'A#', 'B']

    # Handle minor chords
    is_minor = False
    if chord.endswith('m'):
        is_minor = True
        base = chord[:-1]
    else:
        base = chord

    if base not in base_chords:
        return chord  # Return original if chord not recognized

    idx = base_chords.index(base)
    transposed_idx = (idx + semitones) % 12
    transposed_base = base_chords[transposed_idx]

    transposed_chord = transposed_base + ('m' if is_minor else '')
    return transposed_chord

class ChordLyricsPlayer:
    def __init__(self, root):
        self.root = root
        self.root.title("Chord and Lyrics Player")

        # Initialize pygame mixer
        pygame.mixer.init()

        # Define the status label
        self.status = tk.Label(root, text="Loading Whisper model...", relief=tk.SUNKEN, anchor='w')
        self.status.pack(fill=tk.X, side=tk.BOTTOM, ipady=2)

        # Load the Whisper model (use 'base' for faster performance or 'large' for better accuracy)
        self.update_status("Loading Whisper model...")
        try:
            self.model = whisper.load_model("base")
            self.update_status("Whisper model loaded. Load an MP3 to start.")
        except Exception as e:
            messagebox.showerror("Error", f"Failed to load Whisper model: {e}")
            self.update_status("Error loading Whisper model.")

        # Variables
        self.mp3_path = ""
        self.y = None
        self.sr = None
        self.chords = []  # List of {'time': float, 'chord': str}
        self.lyrics = ""  # Extracted lyrics
        self.lyrics_timed = []  # List of {'time': float, 'lyric': str}
        self.playing = False
        self.start_time = 0
        self.transposition = 0
        self.current_chord_index = -1
        self.current_lyric_index = -1
        self.processed = False

        # UI Elements Frames
        self.control_frame = tk.Frame(root)
        self.control_frame.pack(pady=10)

        self.display_frame = tk.Frame(root)
        self.display_frame.pack(pady=10, padx=10, fill=tk.BOTH, expand=True)

        # Control Buttons
        self.load_mp3_button = tk.Button(self.control_frame, text="Load MP3", command=self.load_mp3)
        self.load_mp3_button.pack(side=tk.LEFT, padx=5)

        self.play_button = tk.Button(self.control_frame, text="Play", command=self.play_pause, state=tk.DISABLED)
        self.play_button.pack(side=tk.LEFT, padx=5)

        # Transpose Buttons
        transpose_frame = tk.Frame(self.control_frame)
        transpose_frame.pack(side=tk.LEFT, padx=5)

        self.transpose_up = tk.Button(transpose_frame, text="Transpose +1", command=lambda: self.transpose(1),
                                      state=tk.DISABLED)
        self.transpose_up.pack(side=tk.LEFT, padx=2)

        self.transpose_down = tk.Button(transpose_frame, text="Transpose -1", command=lambda: self.transpose(-1),
                                        state=tk.DISABLED)
        self.transpose_down.pack(side=tk.LEFT, padx=2)

        # Current Chord Display
        self.current_chord_frame = tk.Frame(self.display_frame)
        self.current_chord_frame.pack(pady=10)

        self.current_chord_label = tk.Label(self.current_chord_frame, text="Current Chord: None",
                                           font=("Helvetica", 36), fg="red")
        self.current_chord_label.pack()

        # Lyrics Display
        self.lyrics_frame = tk.Frame(self.display_frame)
        self.lyrics_frame.pack(pady=10)

        self.lyrics_text = scrolledtext.ScrolledText(self.lyrics_frame, font=("Courier", 12), wrap=tk.WORD,
                                                     state=tk.DISABLED)
        self.lyrics_text.pack(fill=tk.BOTH, expand=True)

        # Upcoming Chords Listbox
        self.upcoming_chords_frame = tk.Frame(self.display_frame)
        self.upcoming_chords_frame.pack(pady=10)

        self.upcoming_chords_label = tk.Label(self.upcoming_chords_frame, text="Upcoming Chords:", font=("Helvetica", 14))
        self.upcoming_chords_label.pack()

        self.upcoming_chords_listbox = tk.Listbox(self.upcoming_chords_frame, font=("Helvetica", 12), height=10)
        self.upcoming_chords_listbox.pack(fill=tk.X, padx=20, pady=5)

    def update_status(self, message):
        """Update the status label."""
        self.status.config(text=message)
        self.status.update_idletasks()

    def load_mp3(self):
        """Load an MP3 file and process it for chord detection and lyrics extraction."""
        filepath = filedialog.askopenfilename(filetypes=[("MP3 Files", "*.mp3")])
        if not filepath:
            return
        self.mp3_path = filepath
        self.update_status("Processing audio for chord detection and lyrics extraction...")
        self.play_button.config(state=tk.DISABLED)
        self.transpose_up.config(state=tk.DISABLED)
        self.transpose_down.config(state=tk.DISABLED)
        self.current_chord_index = -1
        self.current_lyric_index = -1
        self.chords = []
        self.lyrics = ""
        self.lyrics_timed = []
        self.processed = False

        # Clear displays
        self.current_chord_label.config(text="Current Chord: None", font=("Helvetica", 36), fg="red")
        self.lyrics_text.config(state=tk.NORMAL)
        self.lyrics_text.delete(1.0, tk.END)
        self.lyrics_text.config(state=tk.DISABLED)
        self.upcoming_chords_listbox.delete(0, tk.END)

        # Start a new thread for processing to keep GUI responsive
        threading.Thread(target=self.process_audio, daemon=True).start()

    def process_audio(self):
        """Process the loaded audio file to detect chords and extract lyrics."""
        try:
            # Load audio using librosa
            y, sr = librosa.load(self.mp3_path, sr=None, mono=True)
            self.y = y
            self.sr = sr

            # Tempo and beat tracking
            tempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr)
            beat_times = librosa.frames_to_time(beat_frames, sr=sr)
            self.update_status(f"Detected Tempo: {int(tempo)} BPM. Extracting chords...")

            # Calculate takt duration based on tempo
            # Target: takt duration between 1-3 seconds
            beats_per_takt = 4
            takt_duration = beats_per_takt * (60 / tempo)
            if takt_duration < 1:
                beats_per_takt = max(1, int(round(tempo / 60)))  # More beats per takt for very high BPM
            elif takt_duration > 3:
                beats_per_takt = 2  # Fewer beats per takt for very low BPM
            takt_duration = beats_per_takt * (60 / tempo)

            # Group beats into takts
            takts = []
            num_beats = len(beat_times)
            for i in range(0, num_beats, beats_per_takt):
                takt_beats = beat_times[i:i + beats_per_takt]
                if len(takt_beats) == 0:
                    continue
                start = takt_beats[0]
                if i + beats_per_takt < num_beats:
                    if len(takt_beats) > 1:
                        end = takt_beats[-1] + (takt_beats[-1] - takt_beats[-2])
                    else:
                        end = takt_beats[-1] + (60 / tempo)
                else:
                    end = librosa.get_duration(y=y, sr=sr)
                takts.append((start, end))

            # Extract chroma features
            chroma = librosa.feature.chroma_cqt(y=y, sr=sr)

            # Get timing for each chroma frame
            chroma_times = librosa.frames_to_time(np.arange(chroma.shape[1]), sr=sr)

            # Detect chords for each takt
            for takt in tqdm(takts, desc="Detecting Chords", unit="takt"):
                start, end = takt
                # Find chroma frames within the takt
                indices = np.where((chroma_times >= start) & (chroma_times < end))[0]
                if len(indices) == 0:
                    continue
                # Average chroma features within the takt
                mean_chroma = np.mean(chroma[:, indices], axis=1)
                # Normalize
                norm = np.linalg.norm(mean_chroma)
                if norm != 0:
                    mean_chroma /= norm
                # Compare with chord templates
                similarity = {}
                for chord, template in CHORD_TEMPLATES.items():
                    similarity[chord] = np.dot(mean_chroma, template)
                # Select the chord with the highest similarity
                detected_chord = max(similarity, key=similarity.get)
                self.chords.append({'time': start, 'chord': detected_chord})

            self.update_status("Chords detected. Extracting lyrics using Whisper...")
            # Extract lyrics using Whisper
            self.extract_lyrics()

            self.processed = True
            self.update_status("Chord and lyrics extraction complete. Ready to play.")
            self.play_button.config(state=tk.NORMAL)
            self.transpose_up.config(state=tk.NORMAL)
            self.transpose_down.config(state=tk.NORMAL)

            # Load the mp3 into pygame
            pygame.mixer.music.load(self.mp3_path)

            # Print synchronized chords and lyrics to console
            self.print_output()

        except Exception as e:
            messagebox.showerror("Error", f"Failed to process audio: {e}")
            self.update_status("Error processing audio.")

    def extract_lyrics(self):
        """Extract lyrics from the audio using Whisper."""
        try:
            # Use Whisper to transcribe the audio and get timestamped segments
            result = self.model.transcribe(self.mp3_path, verbose=False, language='en')

            # Extract segments with timestamps
            self.lyrics_timed = []
            for segment in result['segments']:
                start = segment['start']
                end = segment['end']
                text = segment['text'].strip()
                if text:
                    self.lyrics_timed.append({'time': start, 'lyric': text})

            # Extract full lyrics text for display
            self.lyrics = "\n".join([segment['lyric'] for segment in self.lyrics_timed])

            # Update the GUI with the extracted lyrics
            self.update_lyrics_display()

        except Exception as e:
            messagebox.showerror("Error", f"Failed to extract lyrics: {e}")
            self.update_status("Error extracting lyrics.")

    def play_pause(self):
        """Toggle playback of the song."""
        if not self.playing:
            pygame.mixer.music.play()
            self.start_time = time.time()
            self.playing = True
            self.play_button.config(text="Pause")
            self.update_status("Playing...")
            # Start a new thread to update lyrics and chords
            threading.Thread(target=self.update_display, daemon=True).start()
        else:
            pygame.mixer.music.pause()
            self.playing = False
            self.play_button.config(text="Play")
            self.update_status("Paused.")

    def transpose(self, semitones):
        """Transpose all detected chords by a number of semitones."""
        self.transposition += semitones
        # Update all chords with transposition
        transposed_chords = [transpose_chord(chord['chord'], semitones) for chord in self.chords]
        # Update the chords list with transposed chords
        for i in range(len(self.chords)):
            self.chords[i]['chord'] = transposed_chords[i]
        # Update display immediately
        self.update_displayed_chords()
        self.update_lyrics_display()
        # Reprint the synchronized chords and lyrics
        self.print_output()

    def update_display(self):
        """Continuously update the displayed lyrics and chords based on playback time."""
        while self.playing and pygame.mixer.music.get_busy():
            elapsed_time = time.time() - self.start_time

            # Update current chord
            current_chord_index = -1
            for i, chord in enumerate(self.chords):
                if elapsed_time >= chord['time']:
                    current_chord_index = i
                else:
                    break
            if current_chord_index != self.current_chord_index:
                self.current_chord_index = current_chord_index
                self.update_displayed_chords()

            # Update current lyric
            current_lyric_index = -1
            for i, lyric in enumerate(self.lyrics_timed):
                if elapsed_time >= lyric['time']:
                    current_lyric_index = i
                else:
                    break
            if current_lyric_index != self.current_lyric_index:
                self.current_lyric_index = current_lyric_index
                self.update_lyrics_display()

            time.sleep(0.5)  # Update every 0.5 seconds

        if self.playing:
            # Song ended
            self.playing = False
            self.play_button.config(text="Play")
            self.update_status("Finished.")

    def update_displayed_chords(self):
        """Update the GUI with the current and upcoming chords."""
        if self.current_chord_index >= 0 and self.current_chord_index < len(self.chords):
            current_chord = self.chords[self.current_chord_index]['chord']
            transposed = transpose_chord(current_chord, self.transposition)
            self.current_chord_label.config(text=f"Current Chord: {transposed}", font=("Helvetica", 36), fg="red")
        else:
            self.current_chord_label.config(text="Current Chord: None", font=("Helvetica", 36), fg="red")

        # Update upcoming chords
        self.upcoming_chords_listbox.delete(0, tk.END)
        # Show next 10 upcoming chords
        for chord_entry in self.chords[self.current_chord_index + 1:self.current_chord_index + 11]:
            transposed = transpose_chord(chord_entry['chord'], self.transposition)
            self.upcoming_chords_listbox.insert(tk.END, transposed)

    def update_lyrics_display(self):
        """Update the GUI with the current lyric."""
        if self.current_lyric_index >= 0 and self.current_lyric_index < len(self.lyrics_timed):
            current_lyric = self.lyrics_timed[self.current_lyric_index]['lyric']
            # Highlight the current lyric
            self.lyrics_text.config(state=tk.NORMAL)
            self.lyrics_text.delete(1.0, tk.END)
            for i, lyric in enumerate(self.lyrics_timed):
                if i == self.current_lyric_index:
                    self.lyrics_text.insert(tk.END, f"> {lyric['lyric']}\n", "current")
                else:
                    self.lyrics_text.insert(tk.END, f"{lyric['lyric']}\n")
            self.lyrics_text.tag_config("current", foreground="green", font=("Courier", 12, "bold"))
            self.lyrics_text.config(state=tk.DISABLED)
        elif self.lyrics:
            self.lyrics_text.config(state=tk.NORMAL)
            self.lyrics_text.delete(1.0, tk.END)
            self.lyrics_text.insert(tk.END, self.lyrics)
            self.lyrics_text.config(state=tk.DISABLED)

    def print_output(self):
        """Print the synchronized chords and lyrics to the console."""
        combined = []
        chord_index = 0
        lyric_index = 0
        total_duration = librosa.get_duration(y=self.y, sr=self.sr)

        # Prepare a sorted list of events based on time
        events = []
        for chord in self.chords:
            events.append({'time': chord['time'], 'type': 'chord', 'content': chord['chord']})
        for lyric in self.lyrics_timed:
            events.append({'time': lyric['time'], 'type': 'lyric', 'content': lyric['lyric']})
        events = sorted(events, key=lambda x: x['time'])

        # Combine chords and lyrics based on timing
        print("\n--- Synchronized Chords and Lyrics ---\n")
        for event in events:
            time_stamp = self.format_time(event['time'])
            if event['type'] == 'chord':
                print(f"[{time_stamp}] {event['content']}")
            elif event['type'] == 'lyric':
                print(f"[{time_stamp}] {event['content']}")
        print("\n--- End of Output ---\n")

    def format_time(self, seconds):
        """Format time in seconds to MM:SS format."""
        minutes = int(seconds // 60)
        secs = int(seconds % 60)
        return f"{minutes}:{secs:02d}"

def main():
    root = tk.Tk()
    app = ChordLyricsPlayer(root)
    root.geometry("800x600")
    root.mainloop()

if __name__ == "__main__":
    main()