diff --git "a/app.py" "b/app.py" --- "a/app.py" +++ "b/app.py" @@ -19,10 +19,8 @@ from utils import ( load_audio, extract_audio_duration, extract_mfcc_features, - calculate_lyrics_length, format_genre_results, - ensure_cuda_availability, - preprocess_audio_for_model + ensure_cuda_availability ) from emotionanalysis import MusicAnalyzer import librosa @@ -34,7 +32,7 @@ if "HF_TOKEN" in os.environ: # Constants GENRE_MODEL_NAME = "dima806/music_genres_classification" MUSIC_DETECTION_MODEL = "MIT/ast-finetuned-audioset-10-10-0.4593" -LLM_MODEL_NAME = "Qwen/Qwen3-14B" +LLM_MODEL_NAME = "Qwen/Qwen3-32B" SAMPLE_RATE = 22050 # Standard sample rate for audio processing # Check CUDA availability (for informational purposes) @@ -106,6 +104,75 @@ llm_pipeline = pipeline( # Initialize music emotion analyzer music_analyzer = MusicAnalyzer() +# New global function moved outside of verify_flexible_syllable_counts +@functools.lru_cache(maxsize=512) +def cached_phones_for_word(word): + """Get word pronunciations with caching for better performance.""" + return pronouncing.phones_for_word(word) + +@functools.lru_cache(maxsize=512) +def count_syllables_for_word(word): + """Count syllables in a single word with caching for performance.""" + # Try using pronouncing library first + pronunciations = cached_phones_for_word(word.lower()) + if pronunciations: + return pronouncing.syllable_count(pronunciations[0]) + + # Fallback method for words not in the pronouncing dictionary + vowels = "aeiouy" + word = word.lower() + count = 0 + prev_is_vowel = False + + for char in word: + is_vowel = char in vowels + if is_vowel and not prev_is_vowel: + count += 1 + prev_is_vowel = is_vowel + + # Handle special cases + if word.endswith('e') and not word.endswith('le'): + count -= 1 + if word.endswith('le') and len(word) > 2 and word[-3] not in vowels: + count += 1 + if count == 0: + count = 1 + + return count + +@functools.lru_cache(maxsize=512) +def get_word_stress(word): + """Get the stress pattern for a word with improved fallback handling.""" + pronunciations = cached_phones_for_word(word.lower()) + if pronunciations: + return pronouncing.stresses(pronunciations[0]) + + # Enhanced fallback for words not in the dictionary + syllables = count_syllables_for_word(word) + + # Common English stress patterns by word length + if syllables == 1: + return "1" # Single syllable words are stressed + elif syllables == 2: + # Most 2-syllable nouns and adjectives stress first syllable + # Common endings that indicate second-syllable stress + second_syllable_stress = ["ing", "er", "or", "ize", "ise", "ate", "ect", "end", "ure"] + if any(word.endswith(ending) for ending in second_syllable_stress): + return "01" + else: + return "10" # Default for 2-syllable words + elif syllables == 3: + # Common endings for specific stress patterns in 3-syllable words + if any(word.endswith(ending) for ending in ["ity", "ety", "ify", "ogy", "graphy"]): + return "100" # First syllable stress + elif any(word.endswith(ending) for ending in ["ation", "ious", "itis"]): + return "010" # Middle syllable stress + else: + return "100" # Default for 3-syllable words + else: + # For longer words, use common English patterns + return "1" + "0" * (syllables - 1) + # New function: Count syllables in text def count_syllables(text): """Count syllables in a given text using the pronouncing library.""" @@ -113,31 +180,7 @@ def count_syllables(text): syllable_count = 0 for word in words: - # Get pronunciations for the word - pronunciations = pronouncing.phones_for_word(word) - if pronunciations: - # Count syllables in the first pronunciation - syllable_count += pronouncing.syllable_count(pronunciations[0]) - else: - # Fallback: estimate syllables based on vowel groups - vowels = "aeiouy" - count = 0 - prev_is_vowel = False - - for char in word: - is_vowel = char.lower() in vowels - if is_vowel and not prev_is_vowel: - count += 1 - prev_is_vowel = is_vowel - - if word.endswith('e'): - count -= 1 - if word.endswith('le') and len(word) > 2 and word[-3] not in vowels: - count += 1 - if count == 0: - count = 1 - - syllable_count += count + syllable_count += count_syllables_for_word(word) return syllable_count @@ -267,7 +310,7 @@ def detect_music(audio_data): return False, [] def detect_beats(y, sr): - """Enhanced beat detection with adaptive threshold analysis and improved time signature detection.""" + """Enhanced beat detection with adaptive threshold analysis, improved time signature detection and scientific confidence metrics.""" # STEP 1: Improved pre-processing with robustness for quiet sections # Apply a small floor to avoid division-by-zero issues y = np.clip(y, 1e-10, None) # Prevent extreme quiet sections from causing NaN @@ -286,9 +329,10 @@ def detect_beats(y, sr): # Create weighted combination combined_onset = onset_env_full * 0.3 + onset_env_perc * 0.7 - # STEP 2: Multi-strategy tempo and beat detection + # STEP 2: Multi-strategy tempo and beat detection with confidence tracking tempo_candidates = [] beat_candidates = [] + consistency_metrics = [] # Strategy 1: Standard detection tempo1, beats1 = librosa.beat.beat_track( @@ -299,6 +343,20 @@ def detect_beats(y, sr): tempo_candidates.append(tempo1) beat_candidates.append(beats1) + # Calculate autocorrelation-based confidence for this tempo + ac = librosa.autocorrelate(combined_onset) + estimated_period = int(sr * 60.0 / (tempo1 * librosa.get_duration(y=y, sr=sr) / len(combined_onset))) + if estimated_period < len(ac) and estimated_period > 0: + # Measure peak height relative to surroundings + local_ac = ac[max(0, estimated_period-5):min(len(ac), estimated_period+6)] + if np.max(local_ac) > 0: + tempo1_confidence = ac[estimated_period] / np.max(local_ac) + else: + tempo1_confidence = 0.5 + else: + tempo1_confidence = 0.5 + consistency_metrics.append(tempo1_confidence) + # Strategy 2: Try with different tempo range for complex signatures tempo2, beats2 = librosa.beat.beat_track( onset_envelope=combined_onset, @@ -309,9 +367,44 @@ def detect_beats(y, sr): tempo_candidates.append(tempo2) beat_candidates.append(beats2) - # Select the best strategy based on consistency + # Calculate confidence for the second tempo estimate + estimated_period2 = int(sr * 60.0 / (tempo2 * librosa.get_duration(y=y, sr=sr) / len(combined_onset))) + if estimated_period2 < len(ac) and estimated_period2 > 0: + local_ac2 = ac[max(0, estimated_period2-5):min(len(ac), estimated_period2+6)] + if np.max(local_ac2) > 0: + tempo2_confidence = ac[estimated_period2] / np.max(local_ac2) + else: + tempo2_confidence = 0.5 + else: + tempo2_confidence = 0.5 + consistency_metrics.append(tempo2_confidence) + + # Strategy 3: Use dynamic programming for beat tracking + try: + tempo3, beats3 = librosa.beat.beat_track( + onset_envelope=combined_onset, + sr=sr, + tightness=300, # Higher tightness for more structured detection + trim=False + ) + tempo_candidates.append(tempo3) + beat_candidates.append(beats3) + + # Calculate DP-based confidence + if len(beats3) > 1: + beat_times3 = librosa.frames_to_time(beats3, sr=sr) + intervals3 = np.diff(beat_times3) + tempo3_consistency = 1.0 / (1.0 + np.std(intervals3)/np.mean(intervals3)) if np.mean(intervals3) > 0 else 0.5 + else: + tempo3_consistency = 0.5 + consistency_metrics.append(tempo3_consistency) + except Exception: + # Skip if this approach fails + pass + + # Select the best strategy based on improved consistency measurement beat_consistency = [] - for beats in beat_candidates: + for i, beats in enumerate(beat_candidates): if len(beats) <= 1: beat_consistency.append(0) continue @@ -319,32 +412,79 @@ def detect_beats(y, sr): times = librosa.frames_to_time(beats, sr=sr) intervals = np.diff(times) - # More consistent beats have lower variance in intervals + # Comprehensive consistency metrics with better statistical justification if np.mean(intervals) > 0: - consistency = 1.0 / (1.0 + np.std(intervals)/np.mean(intervals)) + # Combine coefficient of variation with autocorrelation confidence + cv = np.std(intervals)/np.mean(intervals) # Lower is better + + # Add adjustments for beat count reasonability + duration = librosa.get_duration(y=y, sr=sr) + expected_beats = duration * tempo_candidates[i] / 60 + beats_ratio = min(len(beats) / expected_beats, expected_beats / len(beats)) if expected_beats > 0 else 0.5 + + # Combine metrics with scientific weighting + consistency = (0.7 * (1.0 / (1.0 + cv))) + (0.3 * consistency_metrics[i]) + (0.2 * beats_ratio) beat_consistency.append(consistency) else: beat_consistency.append(0) - best_idx = np.argmax(beat_consistency) if beat_consistency else 0 + # Select best model with scientific confidence calculation + if beat_consistency: + best_idx = np.argmax(beat_consistency) + best_confidence = beat_consistency[best_idx] * 100 # Convert to percentage + else: + best_idx = 0 + best_confidence = 50.0 # Default 50% confidence if no good metrics + tempo = tempo_candidates[best_idx] beat_frames = beat_candidates[best_idx] - # STEP 3: Performance optimization with vectorized operations + # Calculate beat entropy - scientific measure of beat pattern predictability + beat_entropy = 0.0 + if len(beat_frames) > 2: + times = librosa.frames_to_time(beat_frames, sr=sr) + intervals = np.diff(times) + + # Quantize intervals to detect patterns + if len(intervals) > 0 and np.std(intervals) > 0: + quantized = np.round(intervals / np.min(intervals)) + # Count frequencies of each interval type + unique, counts = np.unique(quantized, return_counts=True) + probs = counts / np.sum(counts) + # Calculate Shannon entropy + beat_entropy = -np.sum(probs * np.log2(probs)) + + # STEP 3: Improved beat strength extraction beat_times = librosa.frames_to_time(beat_frames, sr=sr) - # Vectorized extraction of beat strengths instead of loop + # Vectorized extraction of beat strengths with improved error handling beat_strengths = [] if len(beat_frames) > 0: # Filter out beat frames that exceed the onset envelope length valid_frames = [frame for frame in beat_frames if frame < len(combined_onset)] if valid_frames: - # Vectorized extraction of valid beat strengths - beat_strengths = combined_onset[valid_frames].tolist() + # Vectorized extraction with normalization for consistency + raw_strengths = combined_onset[valid_frames] - # Handle any remaining beats - avg_strength = np.mean(beat_strengths) if beat_strengths else 1.0 - beat_strengths.extend([avg_strength] * (len(beat_times) - len(beat_strengths))) + # Normalize strengths to [0,1] for scientific consistency + if np.max(raw_strengths) > 0: + normalized_strengths = raw_strengths / np.max(raw_strengths) + else: + normalized_strengths = np.ones_like(raw_strengths) + + beat_strengths = normalized_strengths.tolist() + + # Handle remaining beats with interpolation instead of constant values + if len(beat_times) > len(beat_strengths): + missing_count = len(beat_times) - len(beat_strengths) + # Use linear interpolation for more scientific approach + if beat_strengths: + last_strength = beat_strengths[-1] + decay_factor = 0.9 # Gradual decay for trailing beats + beat_strengths.extend([last_strength * (decay_factor ** (i+1)) + for i in range(missing_count)]) + else: + beat_strengths = [1.0] * len(beat_times) else: beat_strengths = [1.0] * len(beat_times) else: @@ -353,9 +493,10 @@ def detect_beats(y, sr): # STEP 4: Calculate intervals between beats intervals = np.diff(beat_times).tolist() if len(beat_times) > 1 else [] - # STEP 5: Improved time signature detection for various patterns + # STEP 5: Improved time signature detection with scientific confidence # Start with default assumption time_signature = 4 + time_sig_confidence = 70.0 # Default confidence if len(beat_strengths) > 8: # Use autocorrelation to find periodicity in beat strengths @@ -376,46 +517,47 @@ def detect_beats(y, sr): if len(peaks) > 0: # Get the first significant peak position (cycle length N) - N = peaks[0] + peak_idx = peaks[0] + N = peak_idx + + # Calculate confidence based on peak prominence + if peak_idx < len(ac): + peak_height = ac[peak_idx] + local_prominence = peak_height / np.mean(ac[max(0, peak_idx-2):min(len(ac), peak_idx+3)]) + time_sig_confidence = min(95, 60 + 35 * local_prominence) # Scale between 60-95% - # Map common cycle lengths to time signatures - if 2 <= N <= 3: - time_signature = N # Direct mapping for simple cases + # Map common cycle lengths to time signatures with improved musical theory + if N == 2: + time_signature = 2 # Clear binary meter (2/4, 2/2, etc.) + time_sig_confidence += 5 # Boost for simple meter + elif N == 3: + time_signature = 3 # Clear triple meter (3/4, 3/8, etc.) + time_sig_confidence += 5 # Boost for simple meter + elif 4 <= N <= 5: + time_signature = N # Direct mapping for common cases (4/4 or 5/4) elif N == 6: - time_signature = 3 # Could be 6/8 or 3/4 with subdivisions + # Could be 6/8 (compound duple) or 3/4 with subdivisions + # Further analyze to distinguish + group_3_count = 0 + for i in range(0, len(beat_strengths) - 6, 3): + if i + 2 < len(beat_strengths): + if beat_strengths[i] > beat_strengths[i+1] and beat_strengths[i] > beat_strengths[i+2]: + group_3_count += 1 + + group_2_count = 0 + for i in range(0, len(beat_strengths) - 4, 2): + if i + 1 < len(beat_strengths): + if beat_strengths[i] > beat_strengths[i+1]: + group_2_count += 1 + + # Determine if it's grouped in 2s or 3s + time_signature = 3 if group_3_count > group_2_count else 6 elif N == 8: - time_signature = 4 # Could be 4/4 with subdivisions + time_signature = 4 # 4/4 with embellishments elif N == 5 or N == 7: time_signature = N # Odd time signatures like 5/4 or 7/8 - # Otherwise, keep default 4 - - # Use adaptive thresholds for pattern detection instead of fixed values - if len(beat_strengths) > 3: - # Calculate z-scores to identify statistically significant strong beats - strengths_array = np.array(beat_strengths) - mean_strength = np.mean(strengths_array) - std_strength = np.std(strengths_array) - - if std_strength > 0: - z_scores = (strengths_array - mean_strength) / std_strength - - # Count beats with z-score > 1 in groups of 3 (for 3/4 time) - strong_beat_pattern = [] - for i in range(0, len(z_scores) - 2, 3): - # First beat should be significantly stronger (z > 1) - # Second and third beats should be weaker - if z_scores[i] > 1 and z_scores[i+1] < 0.5 and z_scores[i+2] < 0.5: - strong_beat_pattern.append(1) - else: - strong_beat_pattern.append(0) - - # Check if we have a clear 3/4 pattern - if strong_beat_pattern and len(strong_beat_pattern) >= 3: - three_pattern_probability = sum(strong_beat_pattern) / len(strong_beat_pattern) - if three_pattern_probability > 0.6: - time_signature = 3 - # STEP 6: Enhanced phrase detection with adaptive thresholds + # STEP 6: Enhanced phrase detection with adaptive thresholds and scientific justification phrases = [] current_phrase = [] @@ -428,7 +570,7 @@ def detect_beats(y, sr): if intervals: mean_interval = np.mean(intervals) std_interval = np.std(intervals) - # A significant gap is > 1.5 standard deviations above mean + # A significant gap is > 1.5 standard deviations above mean (95th percentile) significant_gap = mean_interval + (1.5 * std_interval) if std_interval > 0 else mean_interval * 1.3 else: significant_gap = 0 @@ -437,7 +579,7 @@ def detect_beats(y, sr): strong_threshold = np.max(beat_strengths) * 0.8 if beat_strengths else 1.0 significant_gap = 0 - # Identify phrase boundaries + # Identify phrase boundaries with improved musical heuristics for i in range(len(beat_times)): current_phrase.append(i) @@ -456,8 +598,21 @@ def detect_beats(y, sr): # Measure boundary based on time signature is_measure_boundary = (i + 1) % time_signature == 0 and i > 0 - # Combined decision for phrase boundary - if ((is_stronger_next or is_longer_gap) and len(current_phrase) >= 2) or \ + # Check for significant dip in onset strength (phrase boundary often has reduced energy) + is_energy_dip = False + if i < len(beat_strengths) - 1: + onset_ratio = beat_strengths[i+1] / max(beat_strengths[i], 0.001) + is_energy_dip = onset_ratio < 0.6 + + # Combined decision for phrase boundary with scientific weighting + phrase_boundary_score = ( + (1.5 if is_stronger_next else 0) + + (2.0 if is_longer_gap else 0) + + (1.0 if is_measure_boundary else 0) + + (0.5 if is_energy_dip else 0) + ) + + if (phrase_boundary_score >= 1.5 and len(current_phrase) >= 2) or \ (is_measure_boundary and len(current_phrase) >= time_signature): phrases.append(current_phrase) current_phrase = [] @@ -474,28 +629,310 @@ def detect_beats(y, sr): if end - i >= 2: # Ensure at least 2 beats per phrase phrases.append(list(range(i, end))) - # Return in the original format for compatibility + # Calculate beat periodicity (average time between beats) + beat_periodicity = np.mean(intervals) if intervals else (60 / tempo) + + # Return enhanced results with scientific confidence metrics return { "tempo": tempo, + "tempo_confidence": best_confidence, # New scientific confidence metric + "time_signature": time_signature, + "time_sig_confidence": time_sig_confidence, # New scientific confidence metric "beat_frames": beat_frames, "beat_times": beat_times, "beat_count": len(beat_times), "beat_strengths": beat_strengths, "intervals": intervals, - "time_signature": time_signature, - "phrases": phrases + "phrases": phrases, + "beat_periodicity": beat_periodicity, + "beat_entropy": beat_entropy # New scientific measure of rhythm complexity + } + +def detect_beats_and_subbeats(y, sr, subdivision=4): + """ + Detect main beats and interpolate subbeats between consecutive beats. + + Parameters: + y: Audio time series + sr: Sample rate + subdivision: Number of subdivisions between beats (default: 4 for quarter beats) + + Returns: + Dictionary containing beat times, subbeat times, and tempo information + """ + # Detect main beats using librosa + try: + tempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr) + beat_times = librosa.frames_to_time(beat_frames, sr=sr) + + # Convert numpy values to native Python types + if isinstance(tempo, np.ndarray) or isinstance(tempo, np.number): + tempo = float(tempo) + + # Convert beat_times to a list of floats + if isinstance(beat_times, np.ndarray): + beat_times = [float(t) for t in beat_times] + except Exception as e: + print(f"Error in beat detection: {e}") + # Default fallbacks + tempo = 120.0 + beat_times = [] + + # Create subbeats by interpolating between main beats + subbeat_times = [] + + # Early return if no beats detected + if not beat_times or len(beat_times) < 2: + return { + "tempo": float(tempo) if tempo is not None else 120.0, + "beat_times": beat_times, + "subbeat_times": [] + } + + for i in range(len(beat_times) - 1): + # Get current and next beat time + try: + current_beat = float(beat_times[i]) + next_beat = float(beat_times[i + 1]) + except (IndexError, ValueError, TypeError): + continue + + # Calculate time interval between beats + interval = (next_beat - current_beat) / subdivision + + # Add the main beat + subbeat_times.append({ + "time": float(current_beat), + "type": "main", + "strength": 1.0, + "beat_index": i + }) + + # Add subbeats + for j in range(1, subdivision): + subbeat_time = current_beat + j * interval + # Calculate strength based on position + # For 4/4 time, beat 3 is stronger than beats 2 and 4 + if j == subdivision // 2 and subdivision == 4: + strength = 0.8 # Stronger subbeat (e.g., beat 3 in 4/4) + else: + strength = 0.5 # Weaker subbeat + + subbeat_times.append({ + "time": float(subbeat_time), + "type": "sub", + "strength": float(strength), + "beat_index": i, + "subbeat_index": j + }) + + # Add the last main beat + if beat_times: + try: + subbeat_times.append({ + "time": float(beat_times[-1]), + "type": "main", + "strength": 1.0, + "beat_index": len(beat_times) - 1 + }) + except (ValueError, TypeError): + # Skip if conversion fails + pass + + return { + "tempo": float(tempo) if tempo is not None else 120.0, + "beat_times": beat_times, + "subbeat_times": subbeat_times } +def map_beats_to_seconds(subbeat_times, duration, fps=1.0): + """ + Map beats and subbeats to second-level intervals. + + Parameters: + subbeat_times: List of dictionaries containing beat and subbeat information + duration: Total duration of the audio in seconds + fps: Frames per second (default: 1.0 for one-second intervals) + + Returns: + List of dictionaries, each containing beats within a time window + """ + # Safety check for input parameters + if not isinstance(subbeat_times, list): + print("Warning: subbeat_times is not a list") + subbeat_times = [] + + try: + duration = float(duration) + except (ValueError, TypeError): + print("Warning: duration is not convertible to float, defaulting to 30") + duration = 30.0 + + # Calculate number of time windows + num_windows = int(duration * fps) + 1 + + # Initialize time windows + time_windows = [] + + for i in range(num_windows): + # Calculate window boundaries + start_time = i / fps + end_time = (i + 1) / fps + + # Find beats and subbeats within this window + window_beats = [] + + for beat in subbeat_times: + # Safety check for beat object + if not isinstance(beat, dict): + continue + + # Safely access beat time + try: + beat_time = float(beat.get("time", 0)) + except (ValueError, TypeError): + continue + + if start_time <= beat_time < end_time: + # Safely extract beat properties with defaults + beat_type = beat.get("type", "sub") + if not isinstance(beat_type, str): + beat_type = "sub" + + # Safely handle strength + try: + strength = float(beat.get("strength", 0.5)) + except (ValueError, TypeError): + strength = 0.5 + + # Add beat to this window + window_beats.append({ + "time": beat_time, + "type": beat_type, + "strength": strength, + "relative_pos": (beat_time - start_time) / (1/fps) # Position within window (0-1) + }) + + # Add window to list + time_windows.append({ + "second": i, + "start": start_time, + "end": end_time, + "beats": window_beats + }) + + return time_windows + +def create_second_level_templates(sec_map, tempo, genre=None): + """ + Create syllable templates for each second-level window. + + Parameters: + sec_map: List of second-level time windows with beat information + tempo: Tempo in BPM + genre: Optional genre for genre-specific adjustments + + Returns: + List of template strings, one for each second + """ + # Helper function to map tempo to base syllable count + def tempo_to_syllable_base(tempo): + """Continuous function mapping tempo to syllable base count""" + # Sigmoid-like function that smoothly transitions between syllable counts + if tempo > 180: + return 1.0 + elif tempo > 140: + return 1.0 + (180 - tempo) * 0.02 # Gradual increase 1.0 → 1.8 + elif tempo > 100: + return 1.8 + (140 - tempo) * 0.01 # Gradual increase 1.8 → 2.2 + elif tempo > 70: + return 2.2 + (100 - tempo) * 0.02 # Gradual increase 2.2 → 2.8 + else: + return 2.8 + max(0, (70 - tempo) * 0.04) # Continue increasing for very slow tempos + + # Calculate base syllable count from tempo + base_syllables = tempo_to_syllable_base(tempo) + + # Apply genre-specific adjustments + genre_factor = 1.0 + if genre: + genre_lower = genre.lower() + if any(term in genre_lower for term in ["rap", "hip hop", "hip-hop"]): + genre_factor = 1.4 # Much higher syllable density for rap + elif any(term in genre_lower for term in ["folk", "country", "ballad"]): + genre_factor = 0.8 # Lower density for folk styles + + # Create templates for each second + templates = [] + + for window in sec_map: + beats = window["beats"] + + # If no beats in this second, create a default template + if not beats: + templates.append("w(0.5):1") + continue + + # Create beat patterns for this second + beat_patterns = [] + + for beat in beats: + # Ensure we're dealing with a dictionary and that it has a "strength" key + if not isinstance(beat, dict): + continue # Skip this beat if it's not a dictionary + + # Safely get beat type and strength + if "type" not in beat or not isinstance(beat["type"], str): + beat_type = "w" # Default to weak if type is missing or not a string + else: + beat_type = "S" if beat["type"] == "main" else "m" if beat.get("strength", 0) >= 0.7 else "w" + + # Safely get strength value with fallback + try: + strength = float(beat.get("strength", 0.5)) + except (ValueError, TypeError): + strength = 0.5 # Default if conversion fails + + # Adjust syllable count based on beat type and strength + if beat_type == "S": + syllable_factor = 1.2 # More syllables for strong beats + elif beat_type == "m": + syllable_factor = 1.0 # Normal for medium beats + else: + syllable_factor = 0.8 # Fewer for weak beats + + # Calculate final syllable count + syllable_count = base_syllables * syllable_factor * genre_factor + + # Round to half-syllable precision + syllable_count = round(syllable_count * 2) / 2 + + # Ensure reasonable limits + syllable_count = max(0.5, min(4, syllable_count)) + + # Format with embedded strength value + strength_pct = round(strength * 100) / 100 + beat_patterns.append(f"{beat_type}({strength_pct}):{syllable_count}") + + # Join patterns with dashes - ensure we have at least one pattern + if not beat_patterns: + templates.append("w(0.5):1") # Default if no valid patterns were created + else: + second_template = "-".join(beat_patterns) + templates.append(second_template) + + return templates + def detect_sections(y, sr): """ - Advanced detection of musical sections with adaptive segmentation and improved classification. + Detect musical segments without classifying them by type (verse, chorus, etc.). Parameters: y: Audio time series sr: Sample rate Returns: - A list of section dictionaries with type, start time, end time, and duration + A list of section dictionaries with start time, end time, and duration """ # Step 1: Extract rich feature set for comprehensive analysis # ---------------------------------------------------------------------- @@ -516,7 +953,6 @@ def detect_sections(y, sr): # Harmonic-percussive source separation for better rhythm analysis y_harmonic, y_percussive = librosa.effects.hpss(y) - percussive_rms = librosa.feature.rms(y=y_percussive, hop_length=hop_length) # Step 2: Adaptive determination of segment count based on song complexity # ---------------------------------------------------------------------- @@ -534,10 +970,8 @@ def detect_sections(y, sr): # Transpose to get time as first dimension feature_matrix = feature_stack.T - # Step 3: Feature fusion using dimensionality reduction (addressing simple summation issue) + # Step 3: Feature fusion using dimensionality reduction # ---------------------------------------------------------------------- - - # Apply PCA to reduce dimensionality while preserving relationships from sklearn.decomposition import PCA # Handle very short audio files @@ -620,28 +1054,7 @@ def detect_sections(y, sr): # Fallback to librosa's agglomerative clustering on original features bounds_frames = librosa.segment.agglomerative(feature_stack, n_segments) - # Step 6: Detect harmonic changes for better bridge identification - # ---------------------------------------------------------------------- - - # Calculate tonal centroids to identify key changes - tonnetz = librosa.feature.tonnetz(y=y_harmonic, sr=sr) - - # Look for significant changes in harmonic content - harmonic_changes = [] - - if tonnetz.shape[1] > 1: - tonnetz_diff = np.sum(np.abs(np.diff(tonnetz, axis=1)), axis=0) - # Normalize - if np.max(tonnetz_diff) > 0: - tonnetz_diff = tonnetz_diff / np.max(tonnetz_diff) - - # Identify significant harmonic changes (potential bridges or section changes) - threshold = np.percentile(tonnetz_diff, 90) # Top 10% most significant changes - for i in range(len(tonnetz_diff)): - if tonnetz_diff[i] > threshold: - harmonic_changes.append(i) - - # Step 7: Convert boundaries to time and create sections + # Step 6: Convert boundaries to time and create sections # ---------------------------------------------------------------------- bounds_times = librosa.frames_to_time(bounds_frames, sr=sr, hop_length=hop_length) @@ -656,99 +1069,17 @@ def detect_sections(y, sr): # Skip extremely short sections if duration < 4 and i > 0 and i < len(bounds_times) - 2: continue - - # Step 8: Section type classification with improved musical features - # ---------------------------------------------------------------------- - - # Get indices for this section - start_idx = bounds_frames[i] - end_idx = bounds_frames[i+1] - - # Basic section type based on position - if i == 0: - section_type = "intro" - elif i == len(bounds_times) - 2: - section_type = "outro" - else: - # Default to alternating verse/chorus - section_type = "chorus" if i % 2 == 1 else "verse" - - # Only analyze characteristics if we have enough frames - if end_idx > start_idx: - # Calculate musical characteristics for this section - - # 1. Energy profile - energy = np.mean(rms[0, start_idx:end_idx]) - - # 2. Rhythm intensity (percussive content) - rhythm_intensity = np.mean(percussive_rms[0, start_idx:end_idx]) - - # 3. Harmonic complexity - if chroma.shape[1] > 0: - chroma_var = np.var(chroma[:, start_idx:end_idx]) - else: - chroma_var = 0 - - # 4. Timbral characteristics - if mfcc.shape[1] > 0: - mfcc_mean = np.mean(mfcc[:, start_idx:end_idx], axis=1) - mfcc_var = np.var(mfcc[:, start_idx:end_idx], axis=1) - else: - mfcc_mean = np.zeros(mfcc.shape[0]) - mfcc_var = np.zeros(mfcc.shape[0]) - - # 5. Check for harmonic changes within this section (for bridge detection) - has_harmonic_change = False - for change_idx in harmonic_changes: - if start_idx <= change_idx < end_idx: - has_harmonic_change = True - break - # Calculate relative metrics by comparing to the entire song - relative_energy = energy / np.mean(rms) - relative_rhythm = rhythm_intensity / np.mean(percussive_rms) - - # Improved section type classification: - - # Chorus: High energy, strong rhythm, less harmonic variation - if (relative_energy > 1.1 and relative_rhythm > 1.1 and - section_type != "intro" and section_type != "outro"): - section_type = "chorus" - - # Verse: Moderate energy, moderate rhythm, more harmonic variation - elif (0.8 <= relative_energy <= 1.1 and chroma_var > np.mean(np.var(chroma, axis=1)) and - section_type != "intro" and section_type != "outro"): - section_type = "verse" - - # Bridge: Often has harmonic changes, energy drop, or unique timbral characteristics - if (section_type not in ["intro", "outro"] and - (has_harmonic_change or - (0.5 <= relative_energy <= 0.9 and duration < 30) or - np.any(mfcc_var > np.percentile(np.var(mfcc, axis=1), 75)))): - section_type = "bridge" - - # Add section to the list + # Add section to the list (without classifying as verse/chorus/etc) sections.append({ - "type": section_type, + "type": "segment", # Generic type instead of verse/chorus/etc "start": start, "end": end, "duration": duration }) - # Post-processing: Ensure reasonable section sequence and durations - for i in range(1, len(sections) - 1): - # Check for unreasonably short sections and merge them - if sections[i]["duration"] < 8 and sections[i]["type"] not in ["intro", "outro", "bridge"]: - # Either merge with previous or next section based on similarity - prev_type = sections[i-1]["type"] - next_type = sections[i+1]["type"] if i+1 < len(sections) else "outro" - - # Default to merging with the previous section - sections[i]["type"] = prev_type - # Filter out any remaining extremely short sections - sections = [s for s in sections if s["duration"] >= 5 or - s["type"] == "intro" or s["type"] == "outro"] + sections = [s for s in sections if s["duration"] >= 5] return sections @@ -767,6 +1098,24 @@ def create_flexible_syllable_templates(beats_info, genre=None, phrase_mode='defa import numpy as np from sklearn.cluster import KMeans + # Convert any numpy values to native Python types for safety - directly handle conversions + # Process the dictionary to convert numpy values to Python native types + if isinstance(beats_info, dict): + processed_beats_info = {} + for k, v in beats_info.items(): + if isinstance(v, np.ndarray): + if v.size == 1: + processed_beats_info[k] = float(v.item()) + else: + processed_beats_info[k] = [float(x) if isinstance(x, np.number) else x for x in v] + elif isinstance(v, np.number): + processed_beats_info[k] = float(v) + elif isinstance(v, list): + processed_beats_info[k] = [float(x) if isinstance(x, np.number) else x for x in v] + else: + processed_beats_info[k] = v + beats_info = processed_beats_info + # Extract basic beat information beat_times = beats_info.get("beat_times", []) beat_strengths = beats_info.get("beat_strengths", [1.0] * len(beat_times)) @@ -777,7 +1126,7 @@ def create_flexible_syllable_templates(beats_info, genre=None, phrase_mode='defa if len(beat_times) < 2: return "S(1.0):1-w(0.5):1|S(1.0):1-w(0.5):1" # Default fallback pattern - # Step 1: Adaptive thresholding using k-means clustering + # Step 1: Improved adaptive thresholding using k-means clustering # ---------------------------------------------------------------------- if len(beat_strengths) >= 6: # Need enough data points for clustering # Reshape for k-means @@ -824,21 +1173,22 @@ def create_flexible_syllable_templates(beats_info, genre=None, phrase_mode='defa if current_phrase and len(current_phrase) >= 2: phrases.append(current_phrase) - # Step 3: Calculate continuous tempo-to-syllable mapping function + # Step 3: Improved continuous tempo-to-syllable mapping function # ---------------------------------------------------------------------- def tempo_to_syllable_base(tempo): - """Continuous function mapping tempo to syllable base count""" - # Sigmoid-like function that smoothly transitions between syllable counts - if tempo > 180: - return 1.0 - elif tempo > 140: - return 1.0 + (180 - tempo) * 0.02 # Gradual increase 1.0 → 1.8 - elif tempo > 100: - return 1.8 + (140 - tempo) * 0.01 # Gradual increase 1.8 → 2.2 - elif tempo > 70: - return 2.2 + (100 - tempo) * 0.02 # Gradual increase 2.2 → 2.8 + """Continuous function mapping tempo to syllable base count with scientific curve""" + # Sigmoid-like function with more scientific parameters + # Using logistic function: L/(1+e^(-k(x-x0))) to create smooth transitions + if tempo < 40: # Very slow tempos + return 3.5 # Maximum syllables for extremely slow tempos + elif tempo > 200: # Very fast tempos + return 0.8 # Minimum syllables for extremely fast tempos else: - return 2.8 + max(0, (70 - tempo) * 0.04) # Continue increasing for very slow tempos + # Scientific logistic function for middle range (40-200 BPM) + L = 3.5 # Upper limit + k = 0.04 # Steepness of curve + x0 = 120 # Midpoint (inflection point at normal tempo) + return L / (1 + np.exp(k * (tempo - x0))) # Step 4: Generate enhanced templates with flexible timing # ---------------------------------------------------------------------- @@ -854,17 +1204,23 @@ def create_flexible_syllable_templates(beats_info, genre=None, phrase_mode='defa if not phrase_strengths: phrase_strengths = [1.0] * len(phrase) - # Apply adaptive thresholding for stress pattern detection + # Apply improved adaptive thresholding for stress pattern detection stress_pattern = [] for i, strength in enumerate(phrase_strengths): - # Consider both strength and metrical position + # Consider both strength and metrical position with improved weighting metrical_position = i % time_signature - # Apply positional boost for strong metrical positions - position_boost = 0.15 if metrical_position == 0 else 0 - # Secondary stress on beat 3 in 4/4 time - if time_signature == 4 and metrical_position == 2: - position_boost = 0.08 + # Apply improved position boosting based on musical theory + # In common time signatures, first beat gets strong emphasis, + # third beat gets moderate emphasis (in 4/4) + if metrical_position == 0: # Downbeat (first beat) + position_boost = 0.18 # Stronger boost for downbeats + elif time_signature == 4 and metrical_position == 2: # Third beat in 4/4 + position_boost = 0.1 # Moderate boost for third beat + elif time_signature == 3 and metrical_position == 1: # Second beat in 3/4 + position_boost = 0.05 # Slight boost for second beat in 3/4 + else: + position_boost = 0 # No boost for other beats effective_strength = strength + position_boost @@ -875,57 +1231,69 @@ def create_flexible_syllable_templates(beats_info, genre=None, phrase_mode='defa else: stress_pattern.append(("w", effective_strength)) # Weak beat with strength - # Step 5: Calculate syllable counts using continuous function + # Step 5: Calculate syllable counts using improved continuous function # ---------------------------------------------------------------------- detailed_template = [] for i, (stress_type, strength) in enumerate(stress_pattern): - # Get base syllable count from tempo + # Get base syllable count from tempo with more nuanced mapping base_syllables = tempo_to_syllable_base(tempo) - # Adjust based on stress type + # Adjust based on both stress type AND metrical position + metrical_position = i % time_signature + position_factor = 1.2 if metrical_position == 0 else 1.0 + + # More nuanced adjustment based on stress type if stress_type == "S": - syllable_factor = 1.2 # More syllables for strong beats + syllable_factor = 1.2 * position_factor # Emphasize strong beats more elif stress_type == "m": - syllable_factor = 1.0 # Normal for medium beats + syllable_factor = 1.0 * position_factor # Medium beats else: - syllable_factor = 0.8 # Fewer for weak beats + syllable_factor = 0.8 # Weak beats - # Apply genre-specific adjustments + # Apply improved genre-specific adjustments with more granular factors genre_factor = 1.0 if genre: genre = genre.lower() - if any(term in genre for term in ["rap", "hip hop", "hip-hop"]): - genre_factor = 1.4 # Much higher syllable density for rap - elif any(term in genre for term in ["folk", "country", "ballad"]): - genre_factor = 0.8 # Lower density for folk styles - - # Calculate adjusted syllable count + if "rap" in genre or "hip" in genre: + genre_factor = 1.5 # Significantly higher syllable density for rap + elif "folk" in genre or "country" in genre or "ballad" in genre: + genre_factor = 0.7 # Lower density for folk styles + elif "metal" in genre or "rock" in genre: + genre_factor = 1.1 # Slightly higher density for rock/metal + elif "jazz" in genre: + genre_factor = 1.2 # Higher density for jazz (complex rhythms) + elif "classical" in genre: + genre_factor = 0.9 # More moderate for classical + + # Calculate adjusted syllable count with scientific weighting raw_count = base_syllables * syllable_factor * genre_factor - # Allow for more flexible syllable counts with non-integer values - # Round to multiples of 0.5 for half-syllable precision - rounded_count = round(raw_count * 2) / 2 + # Use more precise rounding that preserves subtle differences + # Round to quarters rather than halves for more precision + rounded_count = round(raw_count * 4) / 4 - # Limit to reasonable range (0.5 to 4) + # Limit to reasonable range (0.5 to 4) with improved bounds syllable_count = max(0.5, min(4, rounded_count)) # Format with embedded strength value for reversibility # Convert strength to 2-decimal precision percentage - strength_pct = int(strength * 100) / 100 + strength_pct = round(strength * 100) / 100 detailed_template.append(f"{stress_type}({strength_pct}):{syllable_count}") # Join beat templates for this phrase phrase_template = "-".join(detailed_template) syllable_templates.append(phrase_template) - # Step 6: Ensure valid output with reasonable defaults + # Step 6: Ensure valid output with improved defaults # ---------------------------------------------------------------------- if not syllable_templates: - # Create a sensible default based on time signature - if time_signature == 3: + # Create sensible defaults based on time signature that reflect musical theory + if time_signature == 3: # 3/4 time - waltz pattern syllable_templates = ["S(0.95):2-w(0.4):1-w(0.35):1"] # 3/4 default - else: + elif time_signature == 2: # 2/4 time - march pattern + syllable_templates = ["S(0.95):1.5-w(0.4):1"] # 2/4 default + else: # 4/4 time - common time syllable_templates = ["S(0.95):2-w(0.4):1-m(0.7):1.5-w(0.35):1"] # 4/4 default # Join all phrase templates with the original separator for compatibility @@ -1168,10 +1536,10 @@ def format_syllable_templates_for_prompt(syllable_templates, arrow="→", line_w return "\n".join(output) -def verify_flexible_syllable_counts(lyrics, templates): +def verify_flexible_syllable_counts(lyrics, templates, second_level_templates=None): """ Enhanced verification of syllable counts and stress patterns with precise alignment analysis - and detailed feedback for all phrases in a template. + for both phrase-level and second-level templates. """ import re import pronouncing @@ -1179,73 +1547,27 @@ def verify_flexible_syllable_counts(lyrics, templates): import functools from itertools import chain - # Apply caching to improve performance for repeated word lookups - @functools.lru_cache(maxsize=512) - def cached_phones_for_word(word): - return pronouncing.phones_for_word(word) - - @functools.lru_cache(maxsize=512) - def count_syllables_for_word(word): - """Count syllables in a single word with caching for performance.""" - # Try using pronouncing library first - pronunciations = cached_phones_for_word(word.lower()) - if pronunciations: - return pronouncing.syllable_count(pronunciations[0]) - - # Fallback method for words not in the pronouncing dictionary - vowels = "aeiouy" - word = word.lower() - count = 0 - prev_is_vowel = False - - for char in word: - is_vowel = char in vowels - if is_vowel and not prev_is_vowel: - count += 1 - prev_is_vowel = is_vowel - - # Handle special cases - if word.endswith('e') and not word.endswith('le'): - count -= 1 - if word.endswith('le') and len(word) > 2 and word[-3] not in vowels: - count += 1 - if count == 0: - count = 1 - - return count - - @functools.lru_cache(maxsize=512) - def get_word_stress(word): - """Get the stress pattern for a word with improved fallback handling.""" - pronunciations = cached_phones_for_word(word.lower()) - if pronunciations: - return pronouncing.stresses(pronunciations[0]) - - # Enhanced fallback for words not in the dictionary - syllables = count_syllables_for_word(word) - - # Common English stress patterns by word length - if syllables == 1: - return "1" # Single syllable words are stressed - elif syllables == 2: - # Most 2-syllable nouns and adjectives stress first syllable - # Common endings that indicate second-syllable stress - second_syllable_stress = ["ing", "er", "or", "ize", "ise", "ate", "ect", "end", "ure"] - if any(word.endswith(ending) for ending in second_syllable_stress): - return "01" - else: - return "10" # Default for 2-syllable words - elif syllables == 3: - # Common endings for specific stress patterns in 3-syllable words - if any(word.endswith(ending) for ending in ["ity", "ety", "ify", "ogy", "graphy"]): - return "100" # First syllable stress - elif any(word.endswith(ending) for ending in ["ation", "ious", "itis"]): - return "010" # Middle syllable stress - else: - return "100" # Default for 3-syllable words + print(f"DEBUG: In verify_flexible_syllable_counts, type of lyrics={type(lyrics)}") + print(f"DEBUG: Type of templates={type(templates)}") + + # Ensure lyrics is a string + if not isinstance(lyrics, str): + print(f"DEBUG: lyrics is not a string, it's {type(lyrics)}") + # Convert to string if possible + try: + lyrics = str(lyrics) + except Exception as e: + print(f"DEBUG: Cannot convert lyrics to string: {str(e)}") + return "Error: Cannot process non-string lyrics" + + # Ensure templates is a list + if not isinstance(templates, list): + print(f"DEBUG: templates is not a list, it's {type(templates)}") + # If it's not a list, create a single-item list + if templates is not None: + templates = [templates] else: - # For longer words, use common English patterns - return "1" + "0" * (syllables - 1) + templates = [] # Split lyrics into lines lines = [line.strip() for line in lyrics.split("\n") if line.strip()] @@ -1262,15 +1584,22 @@ def verify_flexible_syllable_counts(lyrics, templates): break template = templates[i] + print(f"DEBUG: Processing template {i+1}, type={type(template)}") # Extract the template string from different possible formats + template_str = None if isinstance(template, dict) and "syllable_template" in template: template_str = template["syllable_template"] elif isinstance(template, str): template_str = template else: + print(f"DEBUG: Skipping template {i+1}, not a string or dict with syllable_template") continue + if not isinstance(template_str, str): + print(f"DEBUG: template_str is not a string, it's {type(template_str)}") + continue + # Handle multiple phrases in template - process ALL phrases, not just the first template_phrases = [template_str] if "|" in template_str: @@ -1462,6 +1791,97 @@ def verify_flexible_syllable_counts(lyrics, templates): # If no matching template was found verification_notes.append(f"Line {i+1}: Unable to find matching template pattern") + # Add second-level verification if templates are provided + if second_level_templates: + verification_notes.append("\n=== SECOND-LEVEL VERIFICATION ===\n") + + # Check each second against corresponding line + for i, template in enumerate(second_level_templates): + if i >= len(lines): + break + + line = lines[i] + + # Skip section headers + if line.startswith('[') and ']' in line: + continue + + actual_count = count_syllables(line) + + # Parse template to get expected syllable count + total_expected = 0 + beat_patterns = [] + + # Handle templates with beat patterns like "S(0.95):2-w(0.4):1" + if isinstance(template, str) and "-" in template: + for beat in template.split("-"): + if ":" in beat: + try: + count_part = beat.split(":")[1] + count = float(count_part) + total_expected += count + + # Extract beat type for alignment check + beat_type = beat.split("(")[0] if "(" in beat else beat[0] + beat_patterns.append((beat_type, count)) + except (IndexError, ValueError): + pass + + # Compare actual vs expected count + if total_expected > 0: + # Calculate adaptive threshold based on expected syllables + expected_ratio = 0.2 # More strict at second level + threshold = max(0.5, round(total_expected * expected_ratio)) + + difference = abs(actual_count - total_expected) + + if difference > threshold: + verification_notes.append(f"Second {i+1}: Expected {total_expected} syllables, got {actual_count}") + total_mismatch_count += 1 + + # Check for stress misalignment in this second + words = re.findall(r'\b[a-zA-Z]+\b', line.lower()) + word_analysis = [] + cumulative_syllables = 0 + + for word in words: + syllable_count = count_syllables_for_word(word) + stress_pattern = get_word_stress(word) + + word_analysis.append({ + "word": word, + "syllables": syllable_count, + "stress_pattern": stress_pattern, + "position": cumulative_syllables + }) + + cumulative_syllables += syllable_count + + # Check if stressed syllables align with strong beats + if beat_patterns: + strong_positions = [] + current_pos = 0 + + for beat_type, count in beat_patterns: + if beat_type == "S": + strong_positions.append(current_pos) + current_pos += count + + # Look for misalignments + for pos in strong_positions: + for word_info in word_analysis: + word_start = word_info["position"] + word_end = word_start + word_info["syllables"] + + if word_start <= pos < word_end: + # Check if a stressed syllable falls on this position + syllable_in_word = int(pos - word_start) + stress = word_info["stress_pattern"] + + if stress and syllable_in_word < len(stress) and stress[syllable_in_word] != '1': + verification_notes.append(f" → In second {i+1}, '{word_info['word']}' has unstressed syllable on strong beat") + break + # Only add detailed analysis if we have rhythm mismatches if verification_notes: lyrics += "\n\n[Note: Potential rhythm mismatches detected in these lines:]\n" @@ -1643,7 +2063,7 @@ def get_stress_aligned_alternatives(word, position_to_stress): # For other cases, just provide general guidance return f"a word with stress on syllable {position_to_stress + 1}" -def generate_lyrics(genre, duration, emotion_results, song_structure=None): +def generate_lyrics(genre, duration, emotion_results, song_structure=None, lyrics_requirements=None): """ Generate lyrics based on the genre, emotion, and structure analysis with enhanced rhythmic alignment. @@ -1655,22 +2075,58 @@ def generate_lyrics(genre, duration, emotion_results, song_structure=None): duration: Duration of the audio in seconds emotion_results: Dictionary containing emotional analysis results song_structure: Optional dictionary containing song structure analysis + lyrics_requirements: Optional user-provided requirements for the lyrics Returns: Generated lyrics aligned with the rhythm patterns of the music """ - # Extract emotion and theme data from analysis results - primary_emotion = emotion_results["emotion_analysis"]["primary_emotion"] - primary_theme = emotion_results["theme_analysis"]["primary_theme"] + # Safety check for strings + def is_safe_dict_access(obj, key): + """Safe dictionary key access with type checking""" + if not isinstance(obj, dict): + print(f"WARNING: Attempted to access key '{key}' on non-dictionary object of type {type(obj)}") + return False + return key in obj + + # Ensure emotion_results is a dictionary with the expected structure + if not isinstance(emotion_results, dict): + emotion_results = { + "emotion_analysis": {"primary_emotion": "Unknown"}, + "theme_analysis": {"primary_theme": "Unknown"}, + "rhythm_analysis": {"tempo": 0}, + "tonal_analysis": {"key": "Unknown", "mode": ""}, + "summary": {"tempo": 0, "key": "Unknown", "mode": "", "primary_emotion": "Unknown", "primary_theme": "Unknown"} + } + + # Ensure song_structure is properly structured + if song_structure is not None and not isinstance(song_structure, dict): + print(f"WARNING: song_structure is not a dict, it's {type(song_structure)}") + song_structure = None + + print(f"DEBUG: Starting generate_lyrics with genre={genre}, duration={duration}") + print(f"DEBUG: Type of song_structure={type(song_structure)}") + print(f"DEBUG: Type of emotion_results={type(emotion_results)}") + + # Helper function to safely access dictionary with string keys + def safe_dict_get(d, key, default=None): + """Safely get a value from a dictionary, handling non-dictionary objects.""" + if not isinstance(d, dict): + print(f"WARNING: Attempted to access key '{key}' in non-dictionary object of type {type(d)}") + return default + return d.get(key, default) + + # Extract emotion and theme data with safe defaults + primary_emotion = safe_dict_get(safe_dict_get(emotion_results, "emotion_analysis", {}), "primary_emotion", "Unknown") + primary_theme = safe_dict_get(safe_dict_get(emotion_results, "theme_analysis", {}), "primary_theme", "Unknown") # Extract numeric values safely with fallbacks try: - tempo = float(emotion_results["rhythm_analysis"]["tempo"]) - except (KeyError, ValueError, TypeError): + tempo = float(safe_dict_get(safe_dict_get(emotion_results, "rhythm_analysis", {}), "tempo", 0.0)) + except (ValueError, TypeError): tempo = 0.0 - key = emotion_results["tonal_analysis"]["key"] - mode = emotion_results["tonal_analysis"]["mode"] + key = safe_dict_get(safe_dict_get(emotion_results, "tonal_analysis", {}), "key", "Unknown") + mode = safe_dict_get(safe_dict_get(emotion_results, "tonal_analysis", {}), "mode", "") # Format syllable templates for the prompt syllable_guidance = "" @@ -1681,34 +2137,71 @@ def generate_lyrics(genre, duration, emotion_results, song_structure=None): structure_visualization += f"Song Duration: {duration:.1f} seconds\n" structure_visualization += f"Tempo: {tempo:.1f} BPM\n\n" - if song_structure: + # Add second-level template guidance if available + if song_structure and is_safe_dict_access(song_structure, "second_level") and is_safe_dict_access(song_structure.get("second_level", {}), "templates"): + print(f"DEBUG: Using second-level templates") + second_level_templates = song_structure.get("second_level", {}).get("templates", []) + + # Create second-level guidance + second_level_guidance = "\nSECOND-BY-SECOND RHYTHM INSTRUCTIONS:\n" + second_level_guidance += "Each line below corresponds to ONE SECOND of audio. Follow these rhythm patterns EXACTLY:\n\n" + + # Format each second's template + formatted_second_templates = [] + for i, template in enumerate(second_level_templates): + if i < min(60, len(second_level_templates)): # Limit to 60 seconds to avoid overwhelming the LLM + formatted_template = format_syllable_templates_for_prompt(template, arrow="→", line_wrap=0) + formatted_second_templates.append(f"Second {i+1}: {formatted_template}") + + second_level_guidance += "\n".join(formatted_second_templates) + + # Add critical instructions for second-level alignment + second_level_guidance += "\n\nCRITICAL: Create ONE LINE of lyrics for EACH SECOND, following the exact rhythm pattern." + second_level_guidance += "\nIf a second has no beats, use it for a breath or pause in the lyrics." + second_level_guidance += "\nThe first line of your lyrics MUST match Second 1, the second line matches Second 2, and so on." + + # Add to syllable guidance + syllable_guidance = second_level_guidance + + # Store templates for verification + templates_for_verification = second_level_templates + + elif song_structure: + print(f"DEBUG: Checking flexible structure") # Try to use flexible structure if available - if "flexible_structure" in song_structure and song_structure["flexible_structure"]: - flexible = song_structure["flexible_structure"] - if "segments" in flexible and flexible["segments"]: + if is_safe_dict_access(song_structure, "flexible_structure"): + print(f"DEBUG: Using flexible structure") + flexible = song_structure.get("flexible_structure", {}) + if is_safe_dict_access(flexible, "segments") and len(flexible.get("segments", [])) > 0: + print(f"DEBUG: Found segments in flexible structure") # Get the segments - segments = flexible["segments"] + segments = flexible.get("segments", []) + + # Add structure visualization + structure_visualization += f"Total segments: {len(segments)}\n" + structure_visualization += "Each segment represents one musical phrase for which you should write ONE line of lyrics.\n\n" + + # Process each segment to create enhanced rhythmic templates + enhanced_templates = [] + + for i, segment in enumerate(segments): + if i < 30: # Extend limit to 30 lines to handle longer songs + # Get the beat information for this segment + segment_start = segment["start"] + segment_end = segment["end"] - # Add structure visualization - structure_visualization += f"Total segments: {len(segments)}\n" - structure_visualization += "Each segment represents one musical phrase for which you should write ONE line of lyrics.\n\n" + # Add segment info to visualization + structure_visualization += f"Segment {i+1}: {segment_start:.1f}s - {segment_end:.1f}s (duration: {segment_end-segment_start:.1f}s)\n" - # Process each segment to create enhanced rhythmic templates - enhanced_templates = [] + # Find beats within this segment + segment_beats = [] - for i, segment in enumerate(segments): - if i < 30: # Extend limit to 30 lines to handle longer songs - # Get the beat information for this segment - segment_start = segment["start"] - segment_end = segment["end"] - - # Add segment info to visualization - structure_visualization += f"Segment {i+1}: {segment_start:.1f}s - {segment_end:.1f}s (duration: {segment_end-segment_start:.1f}s)\n" - - # Find beats within this segment - segment_beats = [] - beat_times = flexible["beats"]["beat_times"] - beat_strengths = flexible["beats"].get("beat_strengths", []) + # Add type checking for beat_times access + print(f"DEBUG: Checking beat_times in flexible structure") + if is_safe_dict_access(flexible, "beats") and is_safe_dict_access(flexible.get("beats", {}), "beat_times"): + beat_times = flexible.get("beats", {}).get("beat_times", []) + if isinstance(beat_times, list): + beat_strengths = flexible.get("beats", {}).get("beat_strengths", []) for j, beat_time in enumerate(beat_times): if segment_start <= beat_time < segment_end: @@ -1717,11 +2210,11 @@ def generate_lyrics(genre, duration, emotion_results, song_structure=None): # Create segment-specific beat info segment_beats_info = { - "beat_times": [beat_times[j] for j in segment_beats], - "tempo": flexible["beats"].get("tempo", 120) + "beat_times": [beat_times[j] for j in segment_beats if j < len(beat_times)], + "tempo": flexible.get("beats", {}).get("tempo", 120) } - if beat_strengths: + if beat_strengths and isinstance(beat_strengths, list): segment_beats_info["beat_strengths"] = [ beat_strengths[j] for j in segment_beats if j < len(beat_strengths) @@ -1731,6 +2224,7 @@ def generate_lyrics(genre, duration, emotion_results, song_structure=None): segment_beats_info["phrases"] = [segment_beats] # Generate enhanced template with genre awareness and auto phrasing + print(f"DEBUG: Creating flexible syllable template for segment {i+1}") enhanced_template = create_flexible_syllable_templates( segment_beats_info, genre=genre, @@ -1741,174 +2235,131 @@ def generate_lyrics(genre, duration, emotion_results, song_structure=None): # Add template to visualization structure_visualization += f" Template: {enhanced_template}\n" - - # Use these templates to determine verse/chorus structure based on similar patterns - # This is a simple version - could be enhanced with more sophisticated pattern detection - section_types = [] - pattern_groups = {} - - for i, template in enumerate(enhanced_templates): - # Create simplified version for pattern matching - simple_pattern = template.replace("(", "").replace(")", "").replace(":", "") - - # Check if this pattern is similar to any we've seen - found_match = False - for group, patterns in pattern_groups.items(): - if any(simple_pattern == p.replace("(", "").replace(")", "").replace(":", "") for p in patterns): - pattern_groups[group].append(template) - section_types.append(group) - found_match = True - break - - if not found_match: - # New pattern type - group_name = f"Group_{len(pattern_groups) + 1}" - pattern_groups[group_name] = [template] - section_types.append(group_name) - - # Map pattern groups to verse/chorus/bridge based on common structures - section_mapping = {} - if len(pattern_groups) >= 1: - # Assume the most common pattern is the verse - most_common = max(pattern_groups.items(), key=lambda x: len(x[1]))[0] - section_mapping[most_common] = "verse" - - if len(pattern_groups) >= 2: - # Second most common might be chorus - sorted_groups = sorted(pattern_groups.items(), key=lambda x: len(x[1]), reverse=True) - if len(sorted_groups) > 1: - section_mapping[sorted_groups[1][0]] = "chorus" - - if len(pattern_groups) >= 3: - # Third pattern could be bridge - sorted_groups = sorted(pattern_groups.items(), key=lambda x: len(x[1]), reverse=True) - if len(sorted_groups) > 2: - section_mapping[sorted_groups[2][0]] = "bridge" - - # Update section types using the mapping - mapped_section_types = [] - for section_type in section_types: - if section_type in section_mapping: - mapped_section_types.append(section_mapping[section_type]) else: - mapped_section_types.append("verse") # Default to verse - - # Add structure visualization with section types - structure_visualization += "\nPredicted Song Structure:\n" - for i, section_type in enumerate(mapped_section_types): - if i < len(enhanced_templates): - structure_visualization += f"Line {i+1}: [{section_type.upper()}] {enhanced_templates[i]}\n" - - # Calculate total line count - total_lines = len(enhanced_templates) - verse_lines = mapped_section_types.count("verse") - chorus_lines = mapped_section_types.count("chorus") - bridge_lines = mapped_section_types.count("bridge") - - # Add summary - structure_visualization += f"\nTotal Lines Required: {total_lines}\n" - structure_visualization += f"Verse Lines: {verse_lines}\n" - structure_visualization += f"Chorus Lines: {chorus_lines}\n" - structure_visualization += f"Bridge Lines: {bridge_lines}\n" - - # Format templates with improved formatting for the prompt - syllable_guidance = "CRITICAL RHYTHM INSTRUCTIONS:\n" - syllable_guidance += "Each line of lyrics MUST match exactly with one musical phrase/segment.\n" - syllable_guidance += "Follow these rhythm patterns for each line (STRONG beats need stressed syllables):\n\n" - - # Add section headers to formatted templates - formatted_templates = [] - for i, template in enumerate(enhanced_templates): - if i < len(mapped_section_types): - section_type = mapped_section_types[i].upper() - if i > 0 and mapped_section_types[i] != mapped_section_types[i-1]: - # New section - formatted_templates.append(f"\n[{section_type}]") - elif i == 0: - # First section - formatted_templates.append(f"[{section_type}]") - formatted_templates.append(format_syllable_templates_for_prompt([template], arrow="→", line_wrap=8)) - - syllable_guidance += "\n".join(formatted_templates) - - # Store info for later use in traditional sections approach - use_sections = True - - # Use the detected section structure for traditional approach - if verse_lines > 0: - verse_lines = min(verse_lines, total_lines // 2) # Ensure reasonable limits - else: - verse_lines = total_lines // 2 - - if chorus_lines > 0: - chorus_lines = min(chorus_lines, total_lines // 3) - else: - chorus_lines = total_lines // 3 - - if bridge_lines > 0: - bridge_lines = min(bridge_lines, total_lines // 6) + print(f"DEBUG: beat_times is not a list, it's {type(beat_times)}") else: - bridge_lines = 0 + print(f"DEBUG: beats or beat_times not found in flexible structure") + # Skip segment if we don't have beat information + continue + + # Use these templates to determine rhythm patterns, without classifying as verse/chorus + pattern_groups = {} + + for i, template in enumerate(enhanced_templates): + # Create simplified version for pattern matching + simple_pattern = template.replace("(", "").replace(")", "").replace(":", "") - # Fallback to traditional sections if needed - elif "syllables" in song_structure and song_structure["syllables"]: - syllable_guidance = "RHYTHM PATTERN INSTRUCTIONS:\n" - syllable_guidance += "Follow these syllable patterns for each section. Each line should match ONE phrase:\n\n" + # Check if this pattern is similar to any we've seen + found_match = False + for group, patterns in pattern_groups.items(): + if any(simple_pattern == p.replace("(", "").replace(")", "").replace(":", "") for p in patterns): + pattern_groups[group].append(template) + found_match = True + break + + if not found_match: + # New pattern type + group_name = f"Group_{len(pattern_groups) + 1}" + pattern_groups[group_name] = [template] + + # Format templates with improved formatting for the prompt + syllable_guidance = "CRITICAL RHYTHM INSTRUCTIONS:\n" + syllable_guidance += "Each line of lyrics MUST match exactly with one musical phrase/segment.\n" + syllable_guidance += "Follow these rhythm patterns for each line (STRONG beats need stressed syllables):\n\n" - # Count sections for visualization - section_counts = {"verse": 0, "chorus": 0, "bridge": 0, "intro": 0, "outro": 0} + # Add formatted templates without section labels + formatted_templates = [] + for i, template in enumerate(enhanced_templates): + formatted_templates.append(format_syllable_templates_for_prompt([template], arrow="→", line_wrap=8)) - for section in song_structure["syllables"]: - section_counts[section["type"]] = section_counts.get(section["type"], 0) + 1 + syllable_guidance += "\n".join(formatted_templates) + + # Store info for later use in traditional sections approach + use_sections = True + + # Use the detected section structure for traditional approach + if verse_lines > 0: + verse_lines = min(verse_lines, total_lines // 2) # Ensure reasonable limits + else: + verse_lines = total_lines // 2 - if "syllable_template" in section: - # Process to create enhanced template + if chorus_lines > 0: + chorus_lines = min(chorus_lines, total_lines // 3) + else: + chorus_lines = total_lines // 3 + + if bridge_lines > 0: + bridge_lines = min(bridge_lines, total_lines // 6) + else: + bridge_lines = 0 + + # Fallback to traditional sections if needed + elif song_structure and is_safe_dict_access(song_structure, "syllables") and song_structure.get("syllables"): + syllable_guidance = "RHYTHM PATTERN INSTRUCTIONS:\n" + syllable_guidance += "Follow these syllable patterns for each section. Each line should match ONE phrase:\n\n" + + # Count sections for visualization + section_counts = {"verse": 0, "chorus": 0, "bridge": 0, "intro": 0, "outro": 0} + + for section in song_structure.get("syllables", []): + if not isinstance(section, dict): + continue + + section_type = section.get("type", "verse") + section_counts[section_type] = section_counts.get(section_type, 0) + 1 + + if is_safe_dict_access(section, "syllable_template"): + # Process to create enhanced template + if is_safe_dict_access(song_structure, "beats") and is_safe_dict_access(song_structure.get("beats", {}), "beat_times"): section_beats_info = { - "beat_times": [beat for beat in song_structure["beats"]["beat_times"] - if section["start"] <= beat < section["end"]], - "tempo": song_structure["beats"].get("tempo", 120) + "beat_times": [beat for beat in song_structure.get("beats", {}).get("beat_times", []) + if section.get("start", 0) <= beat < section.get("end", 0)], + "tempo": song_structure.get("beats", {}).get("tempo", 120) } - if "beat_strengths" in song_structure["beats"]: + if is_safe_dict_access(song_structure.get("beats", {}), "beat_strengths"): section_beats_info["beat_strengths"] = [ - strength for i, strength in enumerate(song_structure["beats"]["beat_strengths"]) - if i < len(song_structure["beats"]["beat_times"]) and - section["start"] <= song_structure["beats"]["beat_times"][i] < section["end"] + strength for i, strength in enumerate(song_structure.get("beats", {}).get("beat_strengths", [])) + if i < len(song_structure.get("beats", {}).get("beat_times", [])) and + section.get("start", 0) <= song_structure.get("beats", {}).get("beat_times", [])[i] < section.get("end", 0) ] # Create a phrase structure for this section section_beats_info["phrases"] = [list(range(len(section_beats_info["beat_times"])))] - - # Generate enhanced template with genre awareness - enhanced_template = create_flexible_syllable_templates( - section_beats_info, - genre=genre, - phrase_mode='auto' if section['type'] == 'verse' else 'default' - ) - - syllable_guidance += f"[{section['type'].capitalize()}]:\n" - syllable_guidance += format_syllable_templates_for_prompt( - enhanced_template, - arrow="→", - line_wrap=6 - ) + "\n\n" - templates_for_verification.append(section) - elif "syllable_count" in section: - syllable_guidance += f"[{section['type'].capitalize()}]: ~{section['syllable_count']} syllables total\n" - - # Create structure visualization - structure_visualization += "Using traditional section-based structure:\n" - for section_type, count in section_counts.items(): - if count > 0: - structure_visualization += f"{section_type.capitalize()}: {count} sections\n" - - # Set traditional section counts - verse_lines = max(2, section_counts.get("verse", 0) * 4) - chorus_lines = max(2, section_counts.get("chorus", 0) * 4) - bridge_lines = max(0, section_counts.get("bridge", 0) * 2) - - # Use sections approach - use_sections = True + + # Create a phrase structure for this section + section_beats_info["phrases"] = [list(range(len(section_beats_info["beat_times"])))] + + # Generate enhanced template with genre awareness + enhanced_template = create_flexible_syllable_templates( + section_beats_info, + genre=genre, + phrase_mode='auto' if section['type'] == 'verse' else 'default' + ) + + syllable_guidance += f"[{section['type'].capitalize()}]:\n" + syllable_guidance += format_syllable_templates_for_prompt( + enhanced_template, + arrow="→", + line_wrap=6 + ) + "\n\n" + templates_for_verification.append(section) + elif "syllable_count" in section: + syllable_guidance += f"[{section['type'].capitalize()}]: ~{section['syllable_count']} syllables total\n" + + # Create structure visualization + structure_visualization += "Using traditional section-based structure:\n" + for section_type, count in section_counts.items(): + if count > 0: + structure_visualization += f"{section_type.capitalize()}: {count} sections\n" + + # Set traditional section counts + verse_lines = max(2, section_counts.get("verse", 0) * 4) + chorus_lines = max(2, section_counts.get("chorus", 0) * 4) + bridge_lines = max(0, section_counts.get("bridge", 0) * 2) + + # Use sections approach + use_sections = True # If we couldn't get specific templates, use general guidance if not syllable_guidance: @@ -1981,8 +2432,18 @@ def generate_lyrics(genre, duration, emotion_results, song_structure=None): # Store the syllable guidance for later use syllable_guidance_text = syllable_guidance - # Determine if we should use traditional sections or not based on structure - if song_structure and "flexible_structure" in song_structure and song_structure["flexible_structure"]: + # Determine if we should use traditional sections or second-level alignment + use_sections = True + use_second_level = False + + if song_structure and "second_level" in song_structure and song_structure["second_level"]: + use_second_level = True + # If we have second-level templates, prioritize those over traditional sections + if isinstance(song_structure["second_level"], dict) and "templates" in song_structure["second_level"]: + templates = song_structure["second_level"]["templates"] + if isinstance(templates, list) and len(templates) > 0: + use_sections = False + elif song_structure and "flexible_structure" in song_structure and song_structure["flexible_structure"]: # If we have more than 4 segments, it's likely not a traditional song structure if "segments" in song_structure["flexible_structure"]: segments = song_structure["flexible_structure"]["segments"] @@ -1990,12 +2451,83 @@ def generate_lyrics(genre, duration, emotion_results, song_structure=None): use_sections = False # Create enhanced prompt with better rhythm alignment instructions - if use_sections: + if use_second_level: + # Second-level approach with per-second alignment + content = f""" +You are a talented songwriter who specializes in {genre} music. +Write original lyrics that match the rhythm of a {genre} music segment that is {duration:.1f} seconds long. + +IMPORTANT: DO NOT include any thinking process, explanations, or analysis before the lyrics. Start directly with the song lyrics. + +Music analysis has detected the following qualities: +- Tempo: {tempo:.1f} BPM +- Key: {key} {mode} +- Primary emotion: {primary_emotion} +- Primary theme: {primary_theme} + +{syllable_guidance} + +CRITICAL PRINCIPLES FOR RHYTHMIC ALIGNMENT: +1. STRESSED syllables MUST fall on STRONG beats (marked with STRONG in the pattern) +2. Natural word stress patterns must match the beat strength (strong words on strong beats) +3. Line breaks should occur at phrase endings for natural breathing +4. Consonant clusters should be avoided on fast notes and strong beats +5. Open vowels (a, e, o) work better for sustained notes and syllables +6. Pay attention to strength values in the pattern (higher values like 0.95 need stronger emphasis) +7. For half-syllable positions (like S1.5 or m2.5), use short, quick syllables or words with weak vowels + +The lyrics should: +- Perfectly capture the essence and style of {genre} music +- Express the {primary_emotion} emotion and {primary_theme} theme +- Be completely original +- Maintain a consistent theme throughout +- Match the audio segment duration of {duration:.1f} seconds + +Each line of lyrics must follow the corresponding segment's rhythm pattern EXACTLY. + +IMPORTANT: Start immediately with the lyrics. DO NOT include any thinking process, analysis, or explanation before presenting the lyrics. + +IMPORTANT: Your generated lyrics must be followed by a section titled "[RHYTHM_ANALYSIS_SECTION]" +where you analyze how well the lyrics align with the musical rhythm. This section MUST appear +even if there are no rhythm issues. Include the following in your analysis: +1. Syllable counts for each line and how they match the rhythm pattern +2. Where stressed syllables align with strong beats +3. Any potential misalignments or improvements + +Your lyrics: +""" + + # Add user requirements if provided + if lyrics_requirements and lyrics_requirements.strip(): + content += f""" +USER REQUIREMENTS: +{lyrics_requirements.strip()} + +The lyrics MUST incorporate these user requirements while still following the rhythm patterns. +""" + + content += """ +Each line of lyrics must follow the corresponding segment's rhythm pattern EXACTLY. + +IMPORTANT: Start immediately with the lyrics. DO NOT include any thinking process, analysis, or explanation before presenting the lyrics. + +IMPORTANT: Your generated lyrics must be followed by a section titled "[RHYTHM_ANALYSIS_SECTION]" +where you analyze how well the lyrics align with the musical rhythm. This section MUST appear +even if there are no rhythm issues. Include the following in your analysis: +1. Syllable counts for each line and how they match the rhythm pattern +2. Where stressed syllables align with strong beats +3. Any potential misalignments or improvements + +Your lyrics: +""" + elif use_sections: # Traditional approach with sections content = f""" You are a talented songwriter who specializes in {genre} music. Write original {genre} song lyrics for a song that is {duration:.1f} seconds long. +IMPORTANT: DO NOT include any thinking process, explanations, or analysis before the lyrics. Start directly with the song lyrics. + Music analysis has detected the following qualities in the music: - Tempo: {tempo:.1f} BPM - Key: {key} {mode} @@ -2013,20 +2545,25 @@ CRITICAL PRINCIPLES FOR RHYTHMIC ALIGNMENT: 6. Pay attention to strength values in the pattern (higher values like 0.95 need stronger emphasis) 7. For half-syllable positions (like S1.5 or m2.5), use short, quick syllables or words with weak vowels -Think step by step about how to match words to the rhythm pattern: -1. First, identify the strong beats in each line pattern -2. Choose words where stressed syllables naturally fall on strong beats -3. Count syllables carefully to ensure they match the pattern precisely -4. Test your line against the pattern by mapping each syllable - -IMPORTANT: Each line of lyrics must match exactly to ONE musical phrase/segment. - The lyrics should: - Perfectly capture the essence and style of {genre} music - Express the {primary_emotion} emotion and {primary_theme} theme - Follow the structure patterns provided above - Be completely original - Match the song duration of {duration:.1f} seconds +""" + + # Add user requirements if provided + if lyrics_requirements and lyrics_requirements.strip(): + content += f""" +USER REQUIREMENTS: +{lyrics_requirements.strip()} + +The lyrics MUST incorporate these user requirements while still following the rhythm patterns. +""" + + content += """ +IMPORTANT: Start immediately with the lyrics. DO NOT include any thinking process, analysis, or explanation before presenting the lyrics. IMPORTANT: Your generated lyrics must be followed by a section titled "[RHYTHM_ANALYSIS_SECTION]" where you analyze how well the lyrics align with the musical rhythm. This section MUST appear @@ -2043,6 +2580,8 @@ Your lyrics: You are a talented songwriter who specializes in {genre} music. Write original lyrics that match the rhythm of a {genre} music segment that is {duration:.1f} seconds long. +IMPORTANT: DO NOT include any thinking process, explanations, or analysis before the lyrics. Start directly with the song lyrics. + Music analysis has detected the following qualities: - Tempo: {tempo:.1f} BPM - Key: {key} {mode} @@ -2060,29 +2599,29 @@ CRITICAL PRINCIPLES FOR RHYTHMIC ALIGNMENT: 6. Pay attention to strength values in the pattern (higher values like 0.95 need stronger emphasis) 7. For half-syllable positions (like S1.5 or m2.5), use short, quick syllables or words with weak vowels -Think step by step about how to match words to the rhythm pattern: -1. First, identify the strong beats in each line pattern -2. Choose words where stressed syllables naturally fall on strong beats -3. Count syllables carefully to ensure they match the pattern precisely -4. Test your line against the pattern by mapping each syllable - -CRITICAL: Each line of lyrics must match exactly to ONE musical phrase/segment. - -For perfect alignment examples: -- "FEEL the RHY-thm in your SOUL" – stressed syllables on strong beats -- "to-DAY we DANCE a-LONG" – natural speech stress matches musical stress -- "WAIT-ing FOR the SUN to RISE" – syllable emphasis aligns with beat emphasis - The lyrics should: - Perfectly capture the essence and style of {genre} music - Express the {primary_emotion} emotion and {primary_theme} theme - Be completely original - Maintain a consistent theme throughout - Match the audio segment duration of {duration:.1f} seconds +""" + # Add user requirements if provided + if lyrics_requirements and lyrics_requirements.strip(): + content += f""" +USER REQUIREMENTS: +{lyrics_requirements.strip()} + +The lyrics MUST incorporate these user requirements while still following the rhythm patterns. +""" + + content += """ Include any section labels like [Verse] or [Chorus] as indicated in the rhythm patterns above. Each line of lyrics must follow the corresponding segment's rhythm pattern EXACTLY. +IMPORTANT: Start immediately with the lyrics. DO NOT include any thinking process, analysis, or explanation before presenting the lyrics. + IMPORTANT: Your generated lyrics must be followed by a section titled "[RHYTHM_ANALYSIS_SECTION]" where you analyze how well the lyrics align with the musical rhythm. This section MUST appear even if there are no rhythm issues. Include the following in your analysis: @@ -2095,6 +2634,7 @@ Your lyrics: # Format as a chat message for the LLM messages = [ + {"role": "system", "content": "You are a professional songwriter. Create lyrics that match the specified rhythm patterns exactly. Start with the lyrics immediately without any explanation or thinking. Be concise and direct."}, {"role": "user", "content": content} ] @@ -2111,13 +2651,21 @@ Your lyrics: # Configure generation parameters based on model capability generation_params = { "do_sample": True, - "temperature": 0.6, # Lower for more consistent rhythm alignment - "top_p": 0.95, - "top_k": 50, # Increased from 20 for more diversity + "temperature": 0.5, # Lower for more consistent and direct output + "top_p": 0.85, # Slightly lower for more predictable responses + "top_k": 50, "repetition_penalty": 1.2, - "max_new_tokens": 2048 # Doubled from 1024 for more comprehensive lyrics + "max_new_tokens": 2048, + "num_return_sequences": 1 } + # Add specific stop sequences to prevent excessive explanation + if hasattr(llm_model.generation_config, "stopping_criteria"): + thinking_stops = ["Let me think", "First, I need to", "Let's analyze", "I'll approach this", "Step 1:", "To start,"] + for stop in thinking_stops: + if stop not in llm_model.generation_config.stopping_criteria: + llm_model.generation_config.stopping_criteria.append(stop) + # Generate output generated_ids = llm_model.generate( **model_inputs, @@ -2127,32 +2675,151 @@ Your lyrics: # Extract output tokens output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist() - # Skip the thinking process completely and just get the raw output + # Get the raw output and strip any thinking process lyrics = llm_tokenizer.decode(output_ids, skip_special_tokens=True).strip() - # If we find tags, extract only the content after + # Enhanced thinking process removal - handle multiple formats + # First check for standard thinking tags if "" in lyrics and "" in lyrics: lyrics = lyrics.split("")[1].strip() - # Remove any other thinking indicators that might be present - thinking_markers = ["", "", "[thinking]", "[/thinking]", "I'll think step by step:"] + # Check for alternative thinking indicators with improved detection + thinking_markers = [ + "", "", + "[thinking]", "[/thinking]", + "I'll think step by step:", + "First, I need to understand", + "Let me think about", + "Let's tackle this query", + "Okay, let's tackle this query", + "First, I need to understand the requirements", + "Looking at the rhythm patterns" + ] + + # First try to find clear section breaks for marker in thinking_markers: if marker in lyrics: parts = lyrics.split(marker) if len(parts) > 1: lyrics = parts[-1].strip() # Take the last part after any thinking marker - # Verify syllable counts with enhanced verification + # Look for long analytical sections followed by clear lyrics + analytical_patterns = [ + "Let me analyze", + "I need to understand", + "The tempo is", + "First, let's look at", + "Wait, maybe", + "Considering the emotional tone", + "Starting with the first line", + "Let me check the examples" + ] + + # Check if lyrics begin with any analytical patterns + for pattern in analytical_patterns: + if lyrics.startswith(pattern): + # Try to find where the actual lyrics start - look for common lyrics markers + lyrics_markers = [ + "\n\n[Verse", + "\n\n[Chorus", + "\n\nVerse", + "\n\nChorus", + "\n\n[Verse 1]", + "\n\n[Intro]" + ] + + for marker in lyrics_markers: + if marker in lyrics: + lyrics = lyrics[lyrics.index(marker):].strip() + break + + # One last effort to clean up - if the text is very long and contains obvious thinking + # before getting to actual lyrics, try to find a clear starting point + if len(lyrics.split()) > 100 and "\n\n" in lyrics: + paragraphs = lyrics.split("\n\n") + for i, paragraph in enumerate(paragraphs): + # Look for typical song structure indicators in a paragraph + if any(marker in paragraph for marker in ["[Verse", "[Chorus", "Verse 1", "Chorus:"]): + lyrics = "\n\n".join(paragraphs[i:]) + break + + # Clean up any remaining thinking artifacts at the beginning + lines = lyrics.split('\n') + clean_lines = [] + lyrics_started = False + + for line in lines: + # Skip initial commentary/thinking lines until we hit what looks like lyrics + if not lyrics_started: + if (line.strip().startswith('[') and ']' in line) or not any(thinking in line.lower() for thinking in ["i think", "let me", "maybe", "perhaps", "alternatively", "checking"]): + lyrics_started = True + + if lyrics_started: + clean_lines.append(line) + + # Only use the cleaning logic if we found some actual lyrics + if clean_lines: + lyrics = '\n'.join(clean_lines) + + # Special handling for second-level templates + second_level_verification = None + if song_structure and "second_level" in song_structure and song_structure["second_level"]: + if isinstance(song_structure["second_level"], dict) and "templates" in song_structure["second_level"]: + second_level_verification = song_structure["second_level"]["templates"] + if not isinstance(second_level_verification, list): + second_level_verification = None + + # Verify syllable counts with enhanced verification - pass second-level templates if available if templates_for_verification: - verified_lyrics = verify_flexible_syllable_counts(lyrics, templates_for_verification) + # Convert any NumPy values to native types before verification - directly handle conversions + # Simple conversion for basic templates (non-recursive) + if isinstance(templates_for_verification, list): + safe_templates = [] + for template in templates_for_verification: + if isinstance(template, dict): + processed_template = {} + for k, v in template.items(): + if isinstance(v, np.ndarray): + if v.size == 1: + processed_template[k] = float(v.item()) + else: + processed_template[k] = [float(x) if isinstance(x, np.number) else x for x in v] + elif isinstance(v, np.number): + processed_template[k] = float(v) + else: + processed_template[k] = v + safe_templates.append(processed_template) + else: + safe_templates.append(template) + else: + safe_templates = templates_for_verification + + # Wrap verification in try-except to handle any potential string indices errors + try: + print(f"DEBUG: Calling verify_flexible_syllable_counts") + print(f"DEBUG: Type of lyrics: {type(lyrics)}") + print(f"DEBUG: Type of safe_templates: {type(safe_templates)}") + print(f"DEBUG: Type of second_level_verification: {type(second_level_verification)}") + + verified_lyrics = verify_flexible_syllable_counts(lyrics, safe_templates, second_level_verification) + print(f"DEBUG: Type of verified_lyrics: {type(verified_lyrics)}") + + except Exception as e: + print(f"ERROR in verify_flexible_syllable_counts: {str(e)}") + # Return the original lyrics if verification fails + return { + "lyrics": lyrics if isinstance(lyrics, str) else str(lyrics), + "rhythm_analysis": f"Error in rhythm analysis: {str(e)}", + "syllable_analysis": "Error performing syllable analysis", + "prompt_template": "Error generating prompt template" + } - # Check if significant issues were detected - if "[Note: Potential rhythm mismatches" in verified_lyrics and "Detailed Alignment Analysis" in verified_lyrics: + if isinstance(verified_lyrics, str) and "[Note: Potential rhythm mismatches" in verified_lyrics and "Detailed Alignment Analysis" in verified_lyrics: # Extract the original lyrics (before the notes section) - original_lyrics = lyrics.split("[Note:")[0].strip() + original_lyrics = lyrics.split("[Note:")[0].strip() if isinstance(lyrics, str) else str(lyrics) # Extract the analysis - analysis = verified_lyrics.split("[Note:")[1] + analysis = verified_lyrics.split("[Note:")[1] if "[Note:" in verified_lyrics else "" # If we have serious alignment issues, consider a refinement step if "stress misalignments" in analysis and len(templates_for_verification) > 0: @@ -2205,14 +2872,18 @@ Improved lyrics with fixed rhythm: refined_lyrics = llm_tokenizer.decode(refined_output_ids, skip_special_tokens=True).strip() # Verify the refined lyrics - refined_verified_lyrics = verify_flexible_syllable_counts(refined_lyrics, templates_for_verification) - - # Only use refined lyrics if they're better (fewer notes) - if "[Note: Potential rhythm mismatches" not in refined_verified_lyrics: - lyrics = refined_lyrics - elif refined_verified_lyrics.count("misalignments") < verified_lyrics.count("misalignments"): - lyrics = refined_verified_lyrics - else: + try: + refined_verified_lyrics = verify_flexible_syllable_counts(refined_lyrics, safe_templates, second_level_verification) + + # Only use refined lyrics if they're better (fewer notes) + if "[Note: Potential rhythm mismatches" not in refined_verified_lyrics: + lyrics = refined_lyrics + elif refined_verified_lyrics.count("misalignments") < verified_lyrics.count("misalignments"): + lyrics = refined_verified_lyrics + else: + lyrics = verified_lyrics + except Exception as e: + print(f"Error in refined lyrics verification: {str(e)}") lyrics = verified_lyrics except Exception as e: print(f"Error in lyrics refinement: {str(e)}") @@ -2273,6 +2944,16 @@ Improved lyrics with fixed rhythm: if len(templates_for_verification) > 30: syllable_analysis += f"... and {len(templates_for_verification) - 30} more lines\n\n" + + # Add second-level analysis if available + if second_level_verification: + syllable_analysis += "\nSecond-Level Template Analysis:\n" + for i, template in enumerate(second_level_verification): + if i < min(len(second_level_verification), 30): # Limit to 30 seconds + syllable_analysis += f"Second {i+1}: {template}\n" + + if len(second_level_verification) > 30: + syllable_analysis += f"... and {len(second_level_verification) - 30} more seconds\n" # Add structure visualization to syllable analysis syllable_analysis += "\n" + structure_visualization @@ -2295,9 +2976,14 @@ Improved lyrics with fixed rhythm: "prompt_template": prompt_template } - return lyrics + return { + "lyrics": lyrics, + "rhythm_analysis": "No rhythm analysis available", + "syllable_analysis": "No syllable analysis available", + "prompt_template": "No prompt template available" + } -def process_audio(audio_file): +def process_audio(audio_file, lyrics_requirements=None): """Main function to process audio file, classify genre, and generate lyrics with enhanced rhythm analysis.""" if audio_file is None: return "Please upload an audio file.", None, None @@ -2324,28 +3010,36 @@ def process_audio(audio_file): top_genres = classify_genre(audio_data) # Format genre results using utility function genre_results = format_genre_results(top_genres) + if not isinstance(top_genres, list) or len(top_genres) == 0: + # Fallback if we don't have valid top_genres + top_genres = [("rock", 1.0)] except Exception as e: print(f"Error in genre classification: {str(e)}") + top_genres = [("rock", 1.0)] # Ensure we have a default even when exception happens return f"Error in genre classification: {str(e)}", None, ast_results + # Initialize default values + ast_results = ast_results if ast_results else [] + song_structure = None + emotion_results = { + "emotion_analysis": {"primary_emotion": "Unknown"}, + "theme_analysis": {"primary_theme": "Unknown"}, + "rhythm_analysis": {"tempo": 0}, + "tonal_analysis": {"key": "Unknown", "mode": ""}, + "summary": {"tempo": 0, "key": "Unknown", "mode": "", "primary_emotion": "Unknown", "primary_theme": "Unknown"} + } + print("Step 4/5: Analyzing music emotions, themes, and structure...") # Analyze music emotions and themes try: emotion_results = music_analyzer.analyze_music(audio_file) except Exception as e: print(f"Error in emotion analysis: {str(e)}") - # Continue even if emotion analysis fails - emotion_results = { - "emotion_analysis": {"primary_emotion": "Unknown"}, - "theme_analysis": {"primary_theme": "Unknown"}, - "rhythm_analysis": {"tempo": 0}, - "tonal_analysis": {"key": "Unknown", "mode": ""}, - "summary": {"tempo": 0, "key": "Unknown", "mode": "", "primary_emotion": "Unknown", "primary_theme": "Unknown"} - } + # Continue with default emotion_results # Calculate detailed song structure for better lyrics alignment try: - # Enhanced song structure calculation for precise lyrics matching + # Load audio data y, sr = load_audio(audio_file, SAMPLE_RATE) # Analyze beats and phrases for music-aligned lyrics @@ -2426,21 +3120,21 @@ def process_audio(audio_file): "end": segment_end }) - # Create a flexible structure with the segments + # Create flexible structure with the segments flexible_structure = { "beats": beats_info, "segments": segments } - # Add to song structure + # Create song structure object song_structure = { "beats": beats_info, "sections": sections_info, - "flexible_structure": flexible_structure + "flexible_structure": flexible_structure, + "syllables": [] } # Add syllable counts to each section - song_structure["syllables"] = [] for section in sections_info: # Create syllable templates for sections section_beats_info = { @@ -2469,45 +3163,142 @@ def process_audio(audio_file): # Try to create a more detailed syllable template if len(section_beats_info["beat_times"]) >= 2: + # Ensure top_genres is a list with at least one element + if isinstance(top_genres, list) and len(top_genres) > 0 and isinstance(top_genres[0], tuple): + genre_name = top_genres[0][0] + else: + genre_name = "unknown" # Default genre if top_genres is invalid + section_info["syllable_template"] = create_flexible_syllable_templates( section_beats_info, - genre=top_genres[0][0] + genre=genre_name ) song_structure["syllables"].append(section_info) - print(f"Successfully analyzed song structure with {len(segments)} segments") - + # Add second-level beat analysis + try: + # Get enhanced beat information with subbeats + subbeat_info = detect_beats_and_subbeats(y, sr, subdivision=4) + + # Map beats to second-level windows + sec_map = map_beats_to_seconds( + subbeat_info["subbeat_times"], + audio_data["duration"] + ) + + # Create second-level templates + # Ensure top_genres is a list with at least one element + genre_name = "unknown" + if isinstance(top_genres, list) and len(top_genres) > 0 and isinstance(top_genres[0], tuple): + genre_name = top_genres[0][0] + + second_level_templates = create_second_level_templates( + sec_map, + subbeat_info["tempo"], + genre_name # Use top genre with safety check + ) + + # Add to song structure + song_structure["second_level"] = { + "sec_map": sec_map, + "templates": second_level_templates + } + + except Exception as e: + print(f"Error in second-level beat analysis: {str(e)}") + # Continue without second-level data + except Exception as e: print(f"Error analyzing song structure: {str(e)}") - # Continue with a simpler approach if this fails - song_structure = None + # Continue without song structure print("Step 5/5: Generating rhythmically aligned lyrics...") # Generate lyrics based on top genre, emotion analysis, and song structure try: - primary_genre, _ = top_genres[0] - lyrics_result = generate_lyrics(primary_genre, audio_data["duration"], emotion_results, song_structure) - - # Handle both old and new return formats - if isinstance(lyrics_result, dict): - lyrics = lyrics_result["lyrics"] - rhythm_analysis = lyrics_result["rhythm_analysis"] - syllable_analysis = lyrics_result["syllable_analysis"] - prompt_template = lyrics_result["prompt_template"] - else: - lyrics = lyrics_result - rhythm_analysis = "No detailed rhythm analysis available" - syllable_analysis = "No syllable analysis available" - prompt_template = "No prompt template available" + # Ensure top_genres is a list with at least one element before accessing + primary_genre = "unknown" + if isinstance(top_genres, list) and len(top_genres) > 0 and isinstance(top_genres[0], tuple): + primary_genre, _ = top_genres[0] + + # CRITICAL FIX: Create a sanitized version of song_structure to prevent string indices error + sanitized_song_structure = None + if song_structure: + sanitized_song_structure = {} + + # Safely copy beats data + if "beats" in song_structure and isinstance(song_structure["beats"], dict): + sanitized_song_structure["beats"] = song_structure["beats"] + + # Safely copy sections data + if "sections" in song_structure and isinstance(song_structure["sections"], list): + sanitized_song_structure["sections"] = song_structure["sections"] + + # Safely handle flexible structure + if "flexible_structure" in song_structure and isinstance(song_structure["flexible_structure"], dict): + flex_struct = song_structure["flexible_structure"] + sanitized_flex = {} + + # Safely handle segments + if "segments" in flex_struct and isinstance(flex_struct["segments"], list): + sanitized_flex["segments"] = flex_struct["segments"] + + # Safely handle beats + if "beats" in flex_struct and isinstance(flex_struct["beats"], dict): + sanitized_flex["beats"] = flex_struct["beats"] + + sanitized_song_structure["flexible_structure"] = sanitized_flex + + # Safely handle syllables + if "syllables" in song_structure and isinstance(song_structure["syllables"], list): + sanitized_song_structure["syllables"] = song_structure["syllables"] + + # Safely handle second-level + if "second_level" in song_structure and isinstance(song_structure["second_level"], dict): + second_level = song_structure["second_level"] + sanitized_second = {} + + if "templates" in second_level and isinstance(second_level["templates"], list): + sanitized_second["templates"] = second_level["templates"] + + if "sec_map" in second_level and isinstance(second_level["sec_map"], list): + sanitized_second["sec_map"] = second_level["sec_map"] + + sanitized_song_structure["second_level"] = sanitized_second + + try: + print("Calling generate_lyrics function...") + # Pass lyrics_requirements to generate_lyrics function + lyrics_result = generate_lyrics(primary_genre, audio_data["duration"], emotion_results, + sanitized_song_structure, lyrics_requirements) + print(f"Type of lyrics_result: {type(lyrics_result)}") + + # Handle both old and new return formats with robust type checking + if isinstance(lyrics_result, dict) and all(k in lyrics_result for k in ["lyrics"]): + lyrics = lyrics_result.get("lyrics", "No lyrics generated") + rhythm_analysis = lyrics_result.get("rhythm_analysis", "No rhythm analysis available") + syllable_analysis = lyrics_result.get("syllable_analysis", "No syllable analysis available") + prompt_template = lyrics_result.get("prompt_template", "No prompt template available") + else: + # Convert to string regardless of the type + lyrics = str(lyrics_result) if lyrics_result is not None else "No lyrics generated" + rhythm_analysis = "No detailed rhythm analysis available" + syllable_analysis = "No syllable analysis available" + prompt_template = "No prompt template available" + except Exception as inner_e: + print(f"Inner error in lyrics generation: {str(inner_e)}") + # Create a simplified fallback result with just the error message + lyrics = f"Error generating lyrics: {str(inner_e)}" + rhythm_analysis = "Error in rhythm analysis" + syllable_analysis = "Error in syllable analysis" + prompt_template = "Error in prompt template generation" except Exception as e: - print(f"Error generating lyrics: {str(e)}") + print(f"Outer error in lyrics generation: {str(e)}") lyrics = f"Error generating lyrics: {str(e)}" rhythm_analysis = "No rhythm analysis available" syllable_analysis = "No syllable analysis available" prompt_template = "No prompt template available" - # Prepare results dictionary with additional rhythm analysis results = { "genre_results": genre_results, @@ -2525,8 +3316,8 @@ def process_audio(audio_file): print(error_msg) return error_msg, None, [] -def format_beat_timeline(audio_file, lyrics=None): - """Creates a formatted timeline showing beat timings and their syllable patterns""" +def format_complete_beat_timeline(audio_file, lyrics=None): + """Creates a complete formatted timeline showing all beat timings and their syllable patterns without truncation""" if audio_file is None: return "Please upload an audio file to see beat timeline." @@ -2537,47 +3328,86 @@ def format_beat_timeline(audio_file, lyrics=None): # Get beat information beats_info = detect_beats(y, sr) - # Format the timeline - timeline = "=== BEAT & SYLLABLE TIMELINE ===\n\n" - # Convert tempo to float before formatting if it's a numpy array - tempo = float(beats_info['tempo']) if isinstance(beats_info['tempo'], np.ndarray) else beats_info['tempo'] - timeline += f"Tempo: {tempo:.1f} BPM\n" - timeline += f"Time Signature: {beats_info['time_signature']}/4\n" - timeline += f"Total Beats: {beats_info['beat_count']}\n\n" + # Helper function to convert numpy values to floats - FIXED + def ensure_float(value): + if isinstance(value, np.ndarray) or isinstance(value, np.number): + return float(value) + return value - # Create a table header + # Format the timeline with enhanced scientific headers + timeline = "=== BEAT & SYLLABLE TIMELINE ===\n\n" + + tempo = ensure_float(beats_info['tempo']) + tempo_confidence = ensure_float(beats_info.get('tempo_confidence', 90.0)) + time_sig_confidence = ensure_float(beats_info.get('time_sig_confidence', 85.0)) + beat_periodicity = ensure_float(beats_info.get('beat_periodicity', 60 / tempo)) + + timeline += f"Tempo: {tempo:.1f} BPM (±{tempo_confidence:.1f}%)\n" + timeline += f"Time Signature: {beats_info['time_signature']}/4 (Confidence: {time_sig_confidence:.1f}%)\n" + timeline += f"Beat Periodicity: {beat_periodicity:.3f}s\n" + timeline += f"Beat Entropy: {beats_info.get('beat_entropy', 'N/A')}\n" + timeline += f"Total Beats: {beats_info['beat_count']}\n" + + # Add musicological context based on tempo classification + if tempo < 60: + tempo_class = "Largo (very slow)" + elif tempo < 76: + tempo_class = "Adagio (slow)" + elif tempo < 108: + tempo_class = "Andante (walking pace)" + elif tempo < 132: + tempo_class = "Moderato (moderate)" + elif tempo < 168: + tempo_class = "Allegro (fast)" + else: + tempo_class = "Presto (very fast)" + + timeline += f"Tempo Classification: {tempo_class}\n\n" + + # Create an enhanced table header with better column descriptions timeline += "| Beat # | Time (s) | Beat Strength | Syllable Pattern |\n" timeline += "|--------|----------|--------------|------------------|\n" - # Add beat-by-beat information + # Add beat-by-beat information with improved classification for i, (time, strength) in enumerate(zip(beats_info['beat_times'], beats_info['beat_strengths'])): # Convert numpy values to Python float if needed - time = float(time) if isinstance(time, np.ndarray) else time - strength = float(strength) if isinstance(strength, np.ndarray) else strength + time = ensure_float(time) + strength = ensure_float(strength) + + # More scientific determination of beat type based on both strength and metrical position + metrical_position = i % beats_info['time_signature'] - # Determine beat type based on strength - if strength >= 0.8: + if metrical_position == 0: # Downbeat (first beat of measure) beat_type = "STRONG" - elif strength >= 0.5: - beat_type = "medium" + syllable_value = 1.5 + elif metrical_position == beats_info['time_signature'] // 2 and beats_info['time_signature'] > 2: + # Secondary strong beat (e.g., beat 3 in 4/4 time) + beat_type = "MEDIUM" if strength < 0.8 else "STRONG" + syllable_value = 1.0 if strength < 0.8 else 1.5 else: - beat_type = "weak" - - # Create beat pattern indicator - if i % beats_info['time_signature'] == 0: - pattern = "S" # Strong beat at start of measure - elif i % beats_info['time_signature'] == beats_info['time_signature'] // 2 and beats_info['time_signature'] > 3: - pattern = "m" # Medium beat (3rd beat in 4/4) + # Other beats - classified by actual strength value + if strength >= 0.8: + beat_type = "STRONG" + syllable_value = 1.5 + elif strength >= 0.5: + beat_type = "MEDIUM" + syllable_value = 1.0 + else: + beat_type = "WEAK" + syllable_value = 1.0 + + # Determine pattern letter based on beat type for consistency + if beat_type == "STRONG": + pattern = "S" + elif beat_type == "MEDIUM": + pattern = "m" else: - pattern = "w" # Weak beat - - # Add row to table - timeline += f"| {i+1:<6} | {time:.2f}s | {beat_type:<12} | {pattern}:{1.5 if pattern=='S' else 1.0} |\n" + pattern = "w" - # Keep table to a reasonable size - if i >= 29: - timeline += f"... and {beats_info['beat_count'] - 30} more beats ...\n" - break + # Add row to table with the correct beat classification + timeline += f"| {i+1:<6} | {time:.2f}s | {beat_type:<12} | {pattern}:{syllable_value} |\n" + + # No truncation - show all beats # Add a visual timeline of beats timeline += "\n=== VISUAL BEAT TIMELINE ===\n\n" @@ -2585,9 +3415,9 @@ def format_beat_timeline(audio_file, lyrics=None): timeline += "S = Strong beat | m = Medium beat | w = Weak beat | · = No beat\n\n" # Calculate total duration and create time markers - if beats_info['beat_times'] and len(beats_info['beat_times']) > 0: + if 'beat_times' in beats_info and len(beats_info['beat_times']) > 0: # Get the max value safely - max_beat_time = max([float(t) if isinstance(t, np.ndarray) else t for t in beats_info['beat_times']]) + max_beat_time = max([ensure_float(t) for t in beats_info['beat_times']]) total_duration = max_beat_time + 2 # Add 2 seconds of padding else: total_duration = 30 # Default duration if no beats found @@ -2615,7 +3445,7 @@ def format_beat_timeline(audio_file, lyrics=None): break # Convert to float if it's a numpy array - time_val = float(time) if isinstance(time, np.ndarray) else time + time_val = ensure_float(time) # Determine position in the timeline pos = int(time_val * 2) # Convert to position in the beat_line @@ -2625,7 +3455,7 @@ def format_beat_timeline(audio_file, lyrics=None): # Determine beat type based on strength and position strength = beats_info['beat_strengths'][i] # Convert to float if it's a numpy array - strength = float(strength) if isinstance(strength, np.ndarray) else strength + strength = ensure_float(strength) if i % beats_info['time_signature'] == 0: beat_line[pos] = "S" # Strong beat at start of measure @@ -2654,7 +3484,7 @@ def format_beat_timeline(audio_file, lyrics=None): for i, time in enumerate(beats_info['beat_times']): if i % beats_info['time_signature'] == 0: # Start of measure # Convert to float if it's a numpy array - time_val = float(time) if isinstance(time, np.ndarray) else time + time_val = ensure_float(time) measure_starts.append((i // beats_info['time_signature'] + 1, time_val)) # Format measure information @@ -2668,10 +3498,10 @@ def format_beat_timeline(audio_file, lyrics=None): # Calculate end time (start of next measure or end of song) if i < len(measure_starts) - 1: end_time = measure_starts[i+1][1] - elif beats_info['beat_times'] and len(beats_info['beat_times']) > 0: + elif 'beat_times' in beats_info and len(beats_info['beat_times']) > 0: # Get the last beat time and convert to float if needed last_beat = beats_info['beat_times'][-1] - end_time = float(last_beat) if isinstance(last_beat, np.ndarray) else last_beat + end_time = ensure_float(last_beat) else: end_time = start_time + 2.0 # Default 2 seconds if no next measure @@ -2679,93 +3509,157 @@ def format_beat_timeline(audio_file, lyrics=None): timeline += f"| {measure_num:<9} | {start_time:.2f}s | {duration:.2f}s |\n" - # Limit to reasonable size - if i >= 9: - timeline += f"... and {len(measure_starts) - 10} more measures ...\n" - break + # No truncation - show all measures # Add phrase information if 'phrases' in beats_info and beats_info['phrases']: timeline += "\n=== MUSICAL PHRASES ===\n\n" for i, phrase in enumerate(beats_info['phrases']): - if i < 10: # Limit to first 10 phrases - if not phrase: - continue - - # Safely check phrase indices - if not (len(phrase) > 0 and len(beats_info['beat_times']) > 0): - continue - - start_beat = min(phrase[0], len(beats_info['beat_times'])-1) - end_beat = min(phrase[-1], len(beats_info['beat_times'])-1) + # Show all phrases, not just the first 10 + if not phrase: + continue + + # Safely check phrase indices + if not (len(phrase) > 0 and len(beats_info['beat_times']) > 0): + continue - # Convert to float if needed - phrase_start = beats_info['beat_times'][start_beat] - phrase_start = float(phrase_start) if isinstance(phrase_start, np.ndarray) else phrase_start + start_beat = min(phrase[0], len(beats_info['beat_times'])-1) + end_beat = min(phrase[-1], len(beats_info['beat_times'])-1) + + # Convert to float if needed + phrase_start = ensure_float(beats_info['beat_times'][start_beat]) + phrase_end = ensure_float(beats_info['beat_times'][end_beat]) + + timeline += f"Phrase {i+1}: Beats {start_beat+1}-{end_beat+1} ({phrase_start:.2f}s - {phrase_end:.2f}s)\n" + + # Create syllable template for this phrase with simplified numpy handling + phrase_beats = { + "beat_times": [ensure_float(beats_info['beat_times'][j]) + for j in phrase if j < len(beats_info['beat_times'])], + "beat_strengths": [ensure_float(beats_info['beat_strengths'][j]) + for j in phrase if j < len(beats_info['beat_strengths'])], + "tempo": ensure_float(beats_info['tempo']), + "time_signature": beats_info['time_signature'], + "phrases": [list(range(len(phrase)))] + } + + template = create_flexible_syllable_templates(phrase_beats) + timeline += f" Syllable Template: {template}\n" + + # Create a visual representation of this phrase + if phrase_start < total_duration and phrase_end < total_duration: + # Create a timeline for this phrase + phrase_visualization = ["·"] * int(total_duration * 2) - phrase_end = beats_info['beat_times'][end_beat] - phrase_end = float(phrase_end) if isinstance(phrase_end, np.ndarray) else phrase_end + # Mark the phrase boundaries + start_pos = int(phrase_start * 2) + end_pos = int(phrase_end * 2) - timeline += f"Phrase {i+1}: Beats {start_beat+1}-{end_beat+1} ({phrase_start:.2f}s - {phrase_end:.2f}s)\n" + if start_pos < len(phrase_visualization): + phrase_visualization[start_pos] = "[" - # Create syllable template for this phrase - phrase_beats = { - "beat_times": [float(beats_info['beat_times'][j]) if isinstance(beats_info['beat_times'][j], np.ndarray) - else beats_info['beat_times'][j] - for j in phrase if j < len(beats_info['beat_times'])], - "beat_strengths": [float(beats_info['beat_strengths'][j]) if isinstance(beats_info['beat_strengths'][j], np.ndarray) - else beats_info['beat_strengths'][j] - for j in phrase if j < len(beats_info['beat_strengths'])], - "tempo": float(beats_info['tempo']) if isinstance(beats_info['tempo'], np.ndarray) else beats_info['tempo'], - "time_signature": beats_info['time_signature'], - "phrases": [list(range(len(phrase)))] - } + if end_pos < len(phrase_visualization): + phrase_visualization[end_pos] = "]" - template = create_flexible_syllable_templates(phrase_beats) - timeline += f" Syllable Template: {template}\n" + # Mark the beats in this phrase + for j in phrase: + if j < len(beats_info['beat_times']): + beat_time = ensure_float(beats_info['beat_times'][j]) + beat_pos = int(beat_time * 2) + + if beat_pos < len(phrase_visualization) and beat_pos != start_pos and beat_pos != end_pos: + # Determine beat type + if j % beats_info['time_signature'] == 0: + phrase_visualization[beat_pos] = "S" + elif j % beats_info['time_signature'] == beats_info['time_signature'] // 2: + phrase_visualization[beat_pos] = "m" + else: + phrase_visualization[beat_pos] = "w" - # Create a visual representation of this phrase - if phrase_start < total_duration and phrase_end < total_duration: - # Create a timeline for this phrase - phrase_visualization = ["·"] * int(total_duration * 2) - - # Mark the phrase boundaries - start_pos = int(phrase_start * 2) - end_pos = int(phrase_end * 2) - - if start_pos < len(phrase_visualization): - phrase_visualization[start_pos] = "[" - - if end_pos < len(phrase_visualization): - phrase_visualization[end_pos] = "]" - - # Mark the beats in this phrase - for j in phrase: - if j < len(beats_info['beat_times']): - beat_time = beats_info['beat_times'][j] - beat_time = float(beat_time) if isinstance(beat_time, np.ndarray) else beat_time - beat_pos = int(beat_time * 2) - - if beat_pos < len(phrase_visualization) and beat_pos != start_pos and beat_pos != end_pos: - # Determine beat type - if j % beats_info['time_signature'] == 0: - phrase_visualization[beat_pos] = "S" - elif j % beats_info['time_signature'] == beats_info['time_signature'] // 2: - phrase_visualization[beat_pos] = "m" - else: - phrase_visualization[beat_pos] = "w" - - # Format and add visualization - phrase_visual = "" - for k in range(0, len(phrase_visualization), 10): - phrase_visual += "".join(phrase_visualization[k:k+10]) - if k + 10 < len(phrase_visualization): - phrase_visual += " " - - timeline += f" Timeline: {phrase_visual}\n\n" + # Format and add visualization + phrase_visual = "" + for k in range(0, len(phrase_visualization), 10): + phrase_visual += "".join(phrase_visualization[k:k+10]) + if k + 10 < len(phrase_visualization): + phrase_visual += " " + + timeline += f" Timeline: {phrase_visual}\n\n" + + # Add second-level script display + try: + # Get second-level beat information + subbeat_info = detect_beats_and_subbeats(y, sr, subdivision=4) + duration = librosa.get_duration(y=y, sr=sr) + + # Map to seconds + sec_map = map_beats_to_seconds(subbeat_info["subbeat_times"], duration) - if len(beats_info['phrases']) > 10: - timeline += f"... and {len(beats_info['phrases']) - 10} more phrases ...\n" + # Create templates + templates = create_second_level_templates(sec_map, subbeat_info["tempo"]) + + # Add to timeline + timeline += "\n=== SECOND-LEVEL SCRIPT ===\n\n" + timeline += "Each line below represents ONE SECOND of audio with matching lyric content.\n" + timeline += "| Second | Beat Pattern | Lyric Content |\n" + timeline += "|--------|-------------|---------------|\n" + + # Get clean lyrics (without analysis notes) + clean_lyrics = lyrics + if isinstance(lyrics, str): + if "[Note: Rhythm Analysis]" in lyrics: + clean_lyrics = lyrics.split("[Note: Rhythm Analysis]")[0].strip() + elif "[Note: Potential rhythm mismatches" in lyrics: + clean_lyrics = lyrics.split("[Note:")[0].strip() + + # Get lyric lines + lines = clean_lyrics.strip().split('\n') if clean_lyrics else [] + + for i, template in enumerate(templates): + # Get corresponding lyric line if available + lyric = lines[i] if i < len(lines) else "" + if lyric.startswith('[') and ']' in lyric: + lyric = "" # Skip section headers + + # Format nicely for display + timeline += f"| {i+1:<6} | {template:<30} | {lyric[:40]} |\n" + + # Add ASCII visualization of second-level beats + timeline += "\n=== SECOND-LEVEL VISUALIZATION ===\n\n" + timeline += "Each row represents ONE SECOND. Beat types:\n" + timeline += "S = Strong beat | m = Medium beat | w = Weak beat | · = No beat\n\n" + + for i, window in enumerate(sec_map): + beats = window["beats"] + + # Create ASCII visualization + beat_viz = ["·"] * 20 # 20 columns for visualization + + for beat in beats: + # Calculate position in visualization + pos = int(beat["relative_pos"] * 19) # Map 0-1 to 0-19 + if 0 <= pos < len(beat_viz): + # Set marker based on beat type + if beat["type"] == "main": + beat_viz[pos] = "S" + elif beat["strength"] >= 0.7: + beat_viz[pos] = "m" + else: + beat_viz[pos] = "w" + + # Get corresponding lyric + lyric = lines[i] if i < len(lines) else "" + if lyric.startswith('[') and ']' in lyric: + lyric = "" + + # Format visualization line + viz_line = f"Second {i+1:2d}: [" + "".join(beat_viz) + "]" + if lyric: + viz_line += f" → {lyric[:40]}" + + timeline += viz_line + "\n" + + except Exception as e: + timeline += f"\n[Error generating second-level analysis: {str(e)}]" # Add a section showing alignment if lyrics were generated if lyrics and isinstance(lyrics, str): @@ -2775,63 +3669,97 @@ def format_beat_timeline(audio_file, lyrics=None): clean_lyrics = lyrics.split("[Note:")[0].strip() else: clean_lyrics = lyrics - + lines = clean_lyrics.strip().split('\n') - # Show alignment for first few lines - for i, line in enumerate(lines[:10]): + # Show alignment for ALL lines, not just the first 10 + for i, line in enumerate(lines): if not line.strip() or line.startswith('['): continue - + timeline += f"Line: \"{line}\"\n" # Count syllables syllable_count = count_syllables(line) timeline += f" Syllables: {syllable_count}\n" - # Show ideal timing (if we have enough phrases) - if 'phrases' in beats_info and beats_info['phrases'] and i < len(beats_info['phrases']): - phrase = beats_info['phrases'][i] + # Create adaptive phrase matching - if we don't have a direct phrase match, + # try to find the closest matching phrase by time or measure + matching_phrase = None + if 'phrases' in beats_info and beats_info['phrases']: + # First try direct index matching + if i < len(beats_info['phrases']) and beats_info['phrases'][i]: + matching_phrase = beats_info['phrases'][i] + else: + # If no direct match, try to find a phrase by musical position + # Calculate which section of the song we're in + if len(beats_info['phrases']) > 0: + section_size = max(1, len(beats_info['phrases']) // 4) + section_index = min(i // section_size, 3) # Limit to 4 sections + section_start = section_index * section_size + section_end = min(section_start + section_size, len(beats_info['phrases'])) + + # Try to find a phrase within this section + candidate_phrases = [phrase for j, phrase in enumerate(beats_info['phrases']) + if section_start <= j < section_end and phrase] + + if candidate_phrases: + matching_phrase = candidate_phrases[min(i % section_size, len(candidate_phrases)-1)] + elif beats_info['phrases']: + # Fallback to cycling through available phrases + phrase_index = i % len(beats_info['phrases']) + if beats_info['phrases'][phrase_index]: + matching_phrase = beats_info['phrases'][phrase_index] + + # Show timing and detailed alignment if we found a matching phrase + if matching_phrase and len(matching_phrase) > 0 and len(beats_info['beat_times']) > 0: # Safely check if phrase has elements and indices are valid - if phrase and len(phrase) > 0 and len(beats_info['beat_times']) > 0: - start_beat = min(phrase[0], len(beats_info['beat_times'])-1) - end_beat = min(phrase[-1], len(beats_info['beat_times'])-1) + if len(matching_phrase) > 0 and len(beats_info['beat_times']) > 0: + start_beat = min(matching_phrase[0], len(beats_info['beat_times'])-1) + end_beat = min(matching_phrase[-1], len(beats_info['beat_times'])-1) - start_time = beats_info['beat_times'][start_beat] - start_time = float(start_time) if isinstance(start_time, np.ndarray) else start_time - - end_time = beats_info['beat_times'][end_beat] - end_time = float(end_time) if isinstance(end_time, np.ndarray) else end_time + start_time = ensure_float(beats_info['beat_times'][start_beat]) + end_time = ensure_float(beats_info['beat_times'][end_beat]) timeline += f" Timing: {start_time:.2f}s - {end_time:.2f}s\n" - # Create a visualization of syllable alignment + # Create an enhanced visualization of syllable alignment timeline += " Alignment: " # Create a timeline focused on just this phrase phrase_duration = end_time - start_time syllable_viz = [] - # Initialize with beat markers for this phrase - for j in phrase: - if j < len(beats_info['beat_times']): - beat_time = beats_info['beat_times'][j] - beat_time = float(beat_time) if isinstance(beat_time, np.ndarray) else beat_time + # Initialize with beat markers for this phrase using improved algorithm + for j, beat_idx in enumerate(matching_phrase): + if beat_idx < len(beats_info['beat_times']): + beat_time = ensure_float(beats_info['beat_times'][beat_idx]) + # Handle edge case where phrase_duration is very small if phrase_duration > 0.001: # Avoid division by very small numbers - relative_pos = int((beat_time - start_time) / phrase_duration * syllable_count) + # Use non-linear mapping for more musical alignment + # This accounts for natural speech rhythms not being strictly linear + normalized_pos = (beat_time - start_time) / phrase_duration + # Apply slight curve to map syllable positions more naturally + curved_pos = min(1.0, normalized_pos * (1.0 + 0.1 * (normalized_pos - 0.5))) + relative_pos = int(curved_pos * syllable_count) else: - relative_pos = 0 + relative_pos = j # Default to sequential if duration is too small + # Ensure we have enough space while len(syllable_viz) <= relative_pos: syllable_viz.append("·") - if j % beats_info['time_signature'] == 0: - syllable_viz[relative_pos] = "S" - elif j % beats_info['time_signature'] == beats_info['time_signature'] // 2: - syllable_viz[relative_pos] = "m" + # Determine beat type with metrical context + metrical_pos = beat_idx % beats_info['time_signature'] + beat_strength = beats_info['beat_strengths'][beat_idx] if beat_idx < len(beats_info['beat_strengths']) else 0 + + if metrical_pos == 0 or beat_strength >= 0.8: + syllable_viz[relative_pos] = "S" # Strong beat + elif metrical_pos == beats_info['time_signature'] // 2 or beat_strength >= 0.5: + syllable_viz[relative_pos] = "m" # Medium beat else: - syllable_viz[relative_pos] = "w" + syllable_viz[relative_pos] = "w" # Weak beat # Fill in any gaps while len(syllable_viz) < syllable_count: @@ -2840,19 +3768,140 @@ def format_beat_timeline(audio_file, lyrics=None): # Trim if too long syllable_viz = syllable_viz[:syllable_count] - # Now map to the line + # Add alignment visualization with word stress analysis timeline += "".join(syllable_viz) + "\n" + + # Add word stress analysis + words = re.findall(r'\b[a-zA-Z]+\b', line.lower()) + if words: + word_stresses = [] + cumulative_syllables = 0 + + for word in words: + syllable_count_word = count_syllables_for_word(word) + stress_pattern = get_word_stress(word) + + # Ensure stress pattern is as long as syllable count + while len(stress_pattern) < syllable_count_word: + stress_pattern += "0" + + for j in range(syllable_count_word): + stress_char = "S" if j < len(stress_pattern) and stress_pattern[j] == "1" else "_" + word_stresses.append(stress_char) + + cumulative_syllables += syllable_count_word + + # Add word stress information + timeline += " Word stress: " + "".join(word_stresses) + "\n" + + # Check if stressed syllables align with strong beats + alignment_score = 0 + alignment_issues = [] + + for j, (stress, beat) in enumerate(zip(word_stresses, syllable_viz)): + if (stress == "S" and beat == "S") or (stress != "S" and beat != "S"): + alignment_score += 1 + elif stress == "S" and beat != "S": + alignment_issues.append(f"Syllable {j+1} has stress but weak beat") + elif stress != "S" and beat == "S": + alignment_issues.append(f"Syllable {j+1} has no stress but strong beat") + + if word_stresses: + alignment_percent = (alignment_score / len(word_stresses)) * 100 + timeline += f" Stress alignment: {alignment_percent:.1f}% match\n" + + if alignment_issues and len(alignment_issues) <= 3: + timeline += " Issues: " + "; ".join(alignment_issues) + "\n" + else: + timeline += " No matching phrase found for alignment\n" timeline += "\n" - - if len(lines) > 10: - timeline += f"... and {len(lines) - 10} more lines ...\n" - + return timeline except Exception as e: - print(f"Error generating beat timeline: {str(e)}") - return f"Error generating beat timeline: {str(e)}" + print(f"Error generating complete beat timeline: {str(e)}") + return f"Error generating complete beat timeline: {str(e)}" + +def display_results(audio_file, lyrics_requirements=None): + """Process audio file and return formatted results for display in the UI.""" + # Default error response + error_response = ("Please upload an audio file.", + "No emotion analysis available.", + "No audio classification available.", + "No lyrics generated.", + "No beat timeline available.") + + if audio_file is None: + return error_response + + try: + # Process audio and get results - pass user requirements + results = process_audio(audio_file, lyrics_requirements) + + # Check if we got an error message + if isinstance(results, str) and "Error" in results: + return results, *error_response[1:] + elif isinstance(results, tuple) and isinstance(results[0], str) and "Error" in results[0]: + return results[0], *error_response[1:] + + # Extract results + if isinstance(results, dict): + # New format + genre_results = results.get("genre_results", "Genre classification failed") + lyrics = results.get("lyrics", "Lyrics generation failed") + ast_results = results.get("ast_results", []) + else: + # Old tuple format + genre_results, lyrics, ast_results = results + + # Get clean lyrics (without analysis notes) + clean_lyrics = lyrics + if isinstance(lyrics, str): + if "[Note: Rhythm Analysis]" in lyrics: + clean_lyrics = lyrics.split("[Note: Rhythm Analysis]")[0].strip() + elif "[Note: Potential rhythm mismatches" in lyrics: + clean_lyrics = lyrics.split("[Note:")[0].strip() + + # Generate beat timeline - use the complete timeline function that shows all beats + beat_timeline = format_complete_beat_timeline(audio_file, clean_lyrics) + + # Format emotion analysis results + emotion_text = "No emotion analysis available." + try: + emotion_results = music_analyzer.analyze_music(audio_file) + emotion_text = (f"Tempo: {emotion_results['summary']['tempo']:.1f} BPM\n" + f"Key: {emotion_results['summary']['key']} {emotion_results['summary']['mode']}\n" + f"Primary Emotion: {emotion_results['summary']['primary_emotion']}\n" + f"Primary Theme: {emotion_results['summary']['primary_theme']}") + + # Keep basic beat analysis without section information + y, sr = load_audio(audio_file, SAMPLE_RATE) + beats_info = detect_beats(y, sr) + + # Add beat analysis info + emotion_text += f"\n\nBeat Analysis:\n" + emotion_text += f"- Tempo: {beats_info.get('tempo', 0):.1f} BPM\n" + emotion_text += f"- Time Signature: {beats_info.get('time_signature', 4)}/4\n" + emotion_text += f"- Total Beats: {beats_info.get('beat_count', 0)}\n" + + except Exception as e: + print(f"Error in emotion analysis: {str(e)}") + + # Format audio classification results + ast_text = "No valid audio classification results available." + if ast_results and isinstance(ast_results, list): + ast_text = "Audio Classification Results:\n" + for result in ast_results[:5]: # Show top 5 results + ast_text += f"{result['label']}: {result['score']*100:.2f}%\n" + + # Return all results + return genre_results, emotion_text, ast_text, clean_lyrics, beat_timeline + + except Exception as e: + error_msg = f"Error: {str(e)}" + print(error_msg) + return error_msg, *error_response[1:] # Create enhanced Gradio interface with tabs for better organization with gr.Blocks(title="Music Genre Classifier & Lyrics Generator") as demo: @@ -2862,6 +3911,14 @@ with gr.Blocks(title="Music Genre Classifier & Lyrics Generator") as demo: with gr.Row(): with gr.Column(scale=1): audio_input = gr.Audio(label="Upload Music", type="filepath") + + # Add the new lyrics requirements input + lyrics_requirements_input = gr.Textbox( + label="Lyrics Requirements (optional)", + placeholder="Enter specific themes, topics, words, or styles you want in the lyrics", + lines=3 + ) + submit_btn = gr.Button("Analyze & Generate", variant="primary") # Add genre info box @@ -2894,177 +3951,14 @@ with gr.Blocks(title="Music Genre Classifier & Lyrics Generator") as demo: with gr.TabItem("Generated Lyrics"): lyrics_output = gr.Textbox(label="Lyrics", lines=18) - with gr.TabItem("Rhythm Analysis"): - rhythm_analysis_output = gr.Textbox(label="Syllable-Beat Alignment Analysis", lines=16) - - with gr.TabItem("Syllable Analysis"): - syllable_analysis_output = gr.Textbox(label="Detailed Syllable Analysis", lines=16) - prompt_template_output = gr.Textbox(label="Prompt Template", lines=16) - with gr.TabItem("Beat & Syllable Timeline"): - beat_timeline_output = gr.Textbox(label="Beat Timings & Syllable Patterns", lines=16) - - # Processing function with better handling of results - def display_results(audio_file): - if audio_file is None: - return "Please upload an audio file.", "No emotion analysis available.", "No audio classification available.", "No lyrics generated.", "No rhythm analysis available.", "No syllable analysis available.", "No prompt template available.", "No beat timeline available." - - try: - # Process audio and get results - results = process_audio(audio_file) - - # Check if we got an error message instead of results - if isinstance(results, str) and "Error" in results: - return results, "Error in analysis", "Error in classification", "No lyrics generated", "No rhythm analysis available", "No syllable analysis available", "No prompt template available", "No beat timeline available" - elif isinstance(results, tuple) and isinstance(results[0], str) and "Error" in results[0]: - return results[0], "Error in analysis", "Error in classification", "No lyrics generated", "No rhythm analysis available", "No syllable analysis available", "No prompt template available", "No beat timeline available" - - # For backwards compatibility, handle both dictionary and tuple returns - if isinstance(results, dict): - genre_results = results.get("genre_results", "Genre classification failed") - lyrics = results.get("lyrics", "Lyrics generation failed") - ast_results = results.get("ast_results", []) - - # Use clean lyrics if available - clean_lyrics = results.get("clean_lyrics", lyrics) - rhythm_analysis = results.get("rhythm_analysis", "No detailed rhythm analysis available") - - # Extract syllable analysis and prompt template - syllable_analysis = results.get("syllable_analysis", "No syllable analysis available") - prompt_template = results.get("prompt_template", "No prompt template available") - else: - # Handle the old tuple return format - genre_results, lyrics, ast_results = results - clean_lyrics = lyrics - - # Extract rhythm analysis if present - rhythm_analysis = "No detailed rhythm analysis available" - if isinstance(lyrics, str): - # First check for new format - if "[Note: Rhythm Analysis]" in lyrics: - clean_lyrics = lyrics.split("[Note: Rhythm Analysis]")[0].strip() - rhythm_analysis = lyrics.split("[Note: Rhythm Analysis]")[1] - # Check for old format - elif "[Note: Potential rhythm mismatches" in lyrics: - clean_lyrics = lyrics.split("[Note:")[0].strip() - rhythm_analysis = "[Note:" + lyrics.split("[Note:")[1] - - # Default values for new fields - syllable_analysis = "No syllable analysis available" - prompt_template = "No prompt template available" - - # Generate beat timeline data - beat_timeline = format_beat_timeline(audio_file, clean_lyrics) - - # Format emotion analysis results - try: - emotion_results = music_analyzer.analyze_music(audio_file) - emotion_text = f"Tempo: {emotion_results['summary']['tempo']:.1f} BPM\n" - emotion_text += f"Key: {emotion_results['summary']['key']} {emotion_results['summary']['mode']}\n" - emotion_text += f"Primary Emotion: {emotion_results['summary']['primary_emotion']}\n" - emotion_text += f"Primary Theme: {emotion_results['summary']['primary_theme']}" - - # Add detailed song structure information if available - try: - audio_data = extract_audio_features(audio_file) - # Use the existing information rather than a function that doesn't exist - y, sr = load_audio(audio_file, SAMPLE_RATE) - beats_info = detect_beats(y, sr) - sections_info = detect_sections(y, sr) - - # Create a simple song structure from the available data - song_structure = { - "beats": beats_info, - "sections": sections_info, - "syllables": [] - } - - # Add syllable counts to each section - for section in sections_info: - # Create syllable templates for sections - section_beats_info = { - "beat_times": [beat for beat in beats_info["beat_times"] - if section["start"] <= beat < section["end"]], - "tempo": beats_info.get("tempo", 120) - } - if "beat_strengths" in beats_info: - section_beats_info["beat_strengths"] = [ - strength for i, strength in enumerate(beats_info["beat_strengths"]) - if i < len(beats_info["beat_times"]) and - section["start"] <= beats_info["beat_times"][i] < section["end"] - ] - - # Get a syllable count based on section duration and tempo - syllable_count = int(section["duration"] * (beats_info.get("tempo", 120) / 60) * 1.5) - - section_info = { - "type": section["type"], - "start": section["start"], - "end": section["end"], - "duration": section["duration"], - "syllable_count": syllable_count, - "beat_count": len(section_beats_info["beat_times"]) - } - - # Try to create a more detailed syllable template - if len(section_beats_info["beat_times"]) >= 2: - section_info["syllable_template"] = create_flexible_syllable_templates( - section_beats_info - ) - - song_structure["syllables"].append(section_info) - - emotion_text += "\n\nSong Structure:\n" - for section in song_structure["syllables"]: - emotion_text += f"- {section['type'].capitalize()}: {section['start']:.1f}s to {section['end']:.1f}s " - emotion_text += f"({section['duration']:.1f}s, {section['beat_count']} beats, " - - if "syllable_template" in section: - emotion_text += f"template: {section['syllable_template']})\n" - else: - emotion_text += f"~{section['syllable_count']} syllables)\n" - - # Add flexible structure info if available - if "flexible_structure" in song_structure and song_structure["flexible_structure"]: - flexible = song_structure["flexible_structure"] - if "segments" in flexible and flexible["segments"]: - emotion_text += "\nDetailed Rhythm Analysis:\n" - for i, segment in enumerate(flexible["segments"][:5]): # Show first 5 segments - emotion_text += f"- Segment {i+1}: {segment['start']:.1f}s to {segment['end']:.1f}s, " - emotion_text += f"pattern: {segment.get('syllable_template', 'N/A')}\n" - - if len(flexible["segments"]) > 5: - emotion_text += f" (+ {len(flexible['segments']) - 5} more segments)\n" - - except Exception as e: - print(f"Error displaying song structure: {str(e)}") - # Continue without showing structure details - - except Exception as e: - print(f"Error in emotion analysis: {str(e)}") - emotion_text = f"Error in emotion analysis: {str(e)}" - - # Format AST classification results - if ast_results and isinstance(ast_results, list): - ast_text = "Audio Classification Results:\n" - for result in ast_results[:5]: # Show top 5 results - ast_text += f"{result['label']}: {result['score']*100:.2f}%\n" - else: - ast_text = "No valid audio classification results available." - - # Return all results including new fields - return genre_results, emotion_text, ast_text, clean_lyrics, rhythm_analysis, syllable_analysis, prompt_template, beat_timeline - - except Exception as e: - error_msg = f"Error: {str(e)}" - print(error_msg) - return error_msg, "Error in emotion analysis", "Error in audio classification", "No lyrics generated", "No rhythm analysis available", "No syllable analysis available", "No prompt template available", "No beat timeline available" + beat_timeline_output = gr.Textbox(label="Beat Timings & Syllable Patterns", lines=40) - # Connect the button to the display function with updated outputs + # Connect the button to the display function with updated inputs submit_btn.click( fn=display_results, - inputs=[audio_input], - outputs=[genre_output, emotion_output, ast_output, lyrics_output, rhythm_analysis_output, syllable_analysis_output, prompt_template_output, beat_timeline_output] + inputs=[audio_input, lyrics_requirements_input], + outputs=[genre_output, emotion_output, ast_output, lyrics_output, beat_timeline_output] ) # Enhanced explanation of how the system works @@ -3082,24 +3976,30 @@ with gr.Blocks(title="Music Genre Classifier & Lyrics Generator") as demo: - Strong and weak beats - Natural phrase boundaries - Time signature and tempo variations + - Beat subdivisions (half and quarter beats) + + 5. **Second-Level Alignment**: The system maps beats and subbeats to each second of audio, creating precise templates for perfect alignment. - 5. **Syllable Template Creation**: For each musical phrase, the system generates precise syllable templates that reflect: + 6. **Syllable Template Creation**: For each second of audio, the system generates precise syllable templates that reflect: - Beat stress patterns (strong, medium, weak) - Appropriate syllable counts based on tempo - Genre-specific rhythmic qualities + - Half-beat and quarter-beat subdivisions - 6. **Lyrics Generation**: Using the detected genre, emotion, and rhythm patterns, a large language model generates lyrics that: + 7. **Lyrics Generation**: Using the detected genre, emotion, rhythm patterns, and your custom requirements, a large language model generates lyrics that: - Match the emotional quality of the music - - Follow the precise syllable templates + - Follow the precise syllable templates for each second - Align stressed syllables with strong beats - Maintain genre-appropriate style and themes + - Incorporate your specific requirements and preferences - 7. **Rhythm Verification**: The system verifies the generated lyrics, analyzing: + 8. **Rhythm Verification**: The system verifies the generated lyrics, analyzing: - Syllable count accuracy - Stress alignment with strong beats - Word stress patterns + - Second-by-second alignment precision - 8. **Refinement**: If significant rhythm mismatches are detected, the system can automatically refine the lyrics for better alignment. + 9. **Refinement**: If significant rhythm mismatches are detected, the system can automatically refine the lyrics for better alignment. This multi-step process creates lyrics that feel naturally connected to the music, as if they were written specifically for it. """)