Enhance DataFrame optimization and vectorized calculations in app.py. Refine category conversion logic to exclude specific player columns and improve memory efficiency. Introduce a safe mapping function to handle NaN values for salary, median, and ownership calculations, ensuring better performance across various scenarios.

app.py

@@ -133,8 +133,10 @@ def optimize_dataframe_dtypes(df):
     """Optimize DataFrame data types for memory efficiency"""
     for col in df.columns:
         if df[col].dtype == 'object':
-            # Try to convert to category if many duplicates
-            if df[col].nunique() / len(df) < 0.5:
+            # Only convert to category if there are many duplicates AND it's not a player column
+            # Player columns need to stay as object for mapping operations
+            excluded_cols = ['salary', 'median', 'Own', 'Finish_percentile', 'Dupes', 'Stack', 'Size', 'Win%', 'Lineup Edge', 'Weighted Own', 'Geomean', 'Diversity']
+            if col not in excluded_cols and df[col].nunique() / len(df) < 0.3:
                 df[col] = df[col].astype('category')
     return df
 
@@ -206,53 +208,77 @@ def create_memory_efficient_mappings(projections_df, site_var, type_var, sport_v
 
 def calculate_salary_vectorized(df, player_columns, map_dict, type_var, sport_var):
     """Vectorized salary calculation to replace expensive apply operations"""
+    def safe_map_and_fill(series, mapping, fill_value=0):
+        """Safely map values and fill NaN, handling categorical columns"""
+        mapped = series.map(mapping)
+        if hasattr(series, 'cat'):
+            # Handle categorical columns by converting to object first
+            mapped = mapped.astype('object')
+        return mapped.fillna(fill_value)
+    
     if type_var == 'Classic' and (sport_var == 'CS2' or sport_var == 'LOL'):
         # Captain + flex calculations
-        cpt_salaries = df.iloc[:, 0].map(map_dict['cpt_salary_map']).fillna(0)
-        flex_salaries = sum(df.iloc[:, i].map(map_dict['salary_map']).fillna(0) for i in range(1, len(player_columns)))
+        cpt_salaries = safe_map_and_fill(df.iloc[:, 0], map_dict['cpt_salary_map'])
+        flex_salaries = sum(safe_map_and_fill(df.iloc[:, i], map_dict['salary_map']) for i in range(1, len(player_columns)))
         return cpt_salaries + flex_salaries
     elif type_var == 'Showdown':
         if sport_var == 'GOLF':
-            return sum(df[col].map(map_dict['salary_map']).fillna(0) for col in player_columns)
+            return sum(safe_map_and_fill(df[col], map_dict['salary_map']) for col in player_columns)
         else:
-            cpt_salaries = df.iloc[:, 0].map(map_dict['cpt_salary_map']).fillna(0)
-            flex_salaries = sum(df.iloc[:, i].map(map_dict['salary_map']).fillna(0) for i in range(1, len(player_columns)))
+            cpt_salaries = safe_map_and_fill(df.iloc[:, 0], map_dict['cpt_salary_map'])
+            flex_salaries = sum(safe_map_and_fill(df.iloc[:, i], map_dict['salary_map']) for i in range(1, len(player_columns)))
             return cpt_salaries + flex_salaries
     else:
         # Classic non-CS2/LOL
-        return sum(df[col].map(map_dict['salary_map']).fillna(0) for col in player_columns)
+        return sum(safe_map_and_fill(df[col], map_dict['salary_map']) for col in player_columns)
 
 def calculate_median_vectorized(df, player_columns, map_dict, type_var, sport_var):
     """Vectorized median calculation to replace expensive apply operations"""
+    def safe_map_and_fill(series, mapping, fill_value=0):
+        """Safely map values and fill NaN, handling categorical columns"""
+        mapped = series.map(mapping)
+        if hasattr(series, 'cat'):
+            # Handle categorical columns by converting to object first
+            mapped = mapped.astype('object')
+        return mapped.fillna(fill_value)
+    
     if type_var == 'Classic' and (sport_var == 'CS2' or sport_var == 'LOL'):
-        cpt_medians = df.iloc[:, 0].map(map_dict['cpt_proj_map']).fillna(0)
-        flex_medians = sum(df.iloc[:, i].map(map_dict['proj_map']).fillna(0) for i in range(1, len(player_columns)))
+        cpt_medians = safe_map_and_fill(df.iloc[:, 0], map_dict['cpt_proj_map'])
+        flex_medians = sum(safe_map_and_fill(df.iloc[:, i], map_dict['proj_map']) for i in range(1, len(player_columns)))
         return cpt_medians + flex_medians
     elif type_var == 'Showdown':
         if sport_var == 'GOLF':
-            return sum(df[col].map(map_dict['proj_map']).fillna(0) for col in player_columns)
+            return sum(safe_map_and_fill(df[col], map_dict['proj_map']) for col in player_columns)
         else:
-            cpt_medians = df.iloc[:, 0].map(map_dict['cpt_proj_map']).fillna(0)
-            flex_medians = sum(df.iloc[:, i].map(map_dict['proj_map']).fillna(0) for i in range(1, len(player_columns)))
+            cpt_medians = safe_map_and_fill(df.iloc[:, 0], map_dict['cpt_proj_map'])
+            flex_medians = sum(safe_map_and_fill(df.iloc[:, i], map_dict['proj_map']) for i in range(1, len(player_columns)))
             return cpt_medians + flex_medians
     else:
-        return sum(df[col].map(map_dict['proj_map']).fillna(0) for col in player_columns)
+        return sum(safe_map_and_fill(df[col], map_dict['proj_map']) for col in player_columns)
 
 def calculate_ownership_vectorized(df, player_columns, map_dict, type_var, sport_var):
     """Vectorized ownership calculation to replace expensive apply operations"""
+    def safe_map_and_fill(series, mapping, fill_value=0):
+        """Safely map values and fill NaN, handling categorical columns"""
+        mapped = series.map(mapping)
+        if hasattr(series, 'cat'):
+            # Handle categorical columns by converting to object first
+            mapped = mapped.astype('object')
+        return mapped.fillna(fill_value)
+    
     if type_var == 'Classic' and (sport_var == 'CS2' or sport_var == 'LOL'):
-        cpt_own = df.iloc[:, 0].map(map_dict['cpt_own_map']).fillna(0)
-        flex_own = sum(df.iloc[:, i].map(map_dict['own_map']).fillna(0) for i in range(1, len(player_columns)))
+        cpt_own = safe_map_and_fill(df.iloc[:, 0], map_dict['cpt_own_map'])
+        flex_own = sum(safe_map_and_fill(df.iloc[:, i], map_dict['own_map']) for i in range(1, len(player_columns)))
         return cpt_own + flex_own
     elif type_var == 'Showdown':
         if sport_var == 'GOLF':
-            return sum(df[col].map(map_dict['own_map']).fillna(0) for col in player_columns)
+            return sum(safe_map_and_fill(df[col], map_dict['own_map']) for col in player_columns)
         else:
-            cpt_own = df.iloc[:, 0].map(map_dict['cpt_own_map']).fillna(0)
-            flex_own = sum(df.iloc[:, i].map(map_dict['own_map']).fillna(0) for i in range(1, len(player_columns)))
+            cpt_own = safe_map_and_fill(df.iloc[:, 0], map_dict['cpt_own_map'])
+            flex_own = sum(safe_map_and_fill(df.iloc[:, i], map_dict['own_map']) for i in range(1, len(player_columns)))
             return cpt_own + flex_own
     else:
-        return sum(df[col].map(map_dict['own_map']).fillna(0) for col in player_columns)
+        return sum(safe_map_and_fill(df[col], map_dict['own_map']) for col in player_columns)
 
 def calculate_lineup_metrics(df, player_columns, map_dict, type_var, sport_var, projections_df=None):
     """Centralized function to calculate salary, median, and ownership efficiently"""