Update part1_data.py

part1_data.py

@@ -66,7 +66,7 @@ class TobaccoAnalyzer:
                     data = response.json()
                     weather_data = {
                         'date': date,
-                        'temperature': float(data['main']['temp']),  # Ensure numeric
+                        'temperature': float(data['main']['temp']),
                         'humidity': float(data['main']['humidity']),
                         'rainfall': float(data.get('rain', {}).get('1h', 0)) * 24,
                         'type': 'historical',
@@ -85,7 +85,6 @@ class TobaccoAnalyzer:
             response = requests.get(forecast_url)
             if response.status_code == 200:
                 data = response.json()
-                # Group forecast data by day
                 daily_forecasts = {}
                 
                 for item in data['list']:
@@ -97,21 +96,25 @@ class TobaccoAnalyzer:
                             'temps': [],
                             'humidity': [],
                             'rainfall': 0,
-                            'descriptions': []
+                            'descriptions': [],
+                            'temp_mins': [],
+                            'temp_maxs': []
                         }
                     
                     daily_forecasts[day_key]['temps'].append(float(item['main']['temp']))
                     daily_forecasts[day_key]['humidity'].append(float(item['main']['humidity']))
                     daily_forecasts[day_key]['rainfall'] += float(item.get('rain', {}).get('3h', 0))
                     daily_forecasts[day_key]['descriptions'].append(item['weather'][0]['description'])
+                    daily_forecasts[day_key]['temp_mins'].append(float(item['main']['temp_min']))
+                    daily_forecasts[day_key]['temp_maxs'].append(float(item['main']['temp_max']))
 
                 # Create daily forecast entries
                 for day_key, day_data in daily_forecasts.items():
                     forecast = {
                         'date': datetime.combine(day_key, datetime.min.time()),
                         'temperature': np.mean(day_data['temps']),
-                        'temp_min': min(day_data['temps']),
-                        'temp_max': max(day_data['temps']),
+                        'temp_min': min(day_data['temp_mins']),
+                        'temp_max': max(day_data['temp_maxs']),
                         'humidity': np.mean(day_data['humidity']),
                         'rainfall': day_data['rainfall'],
                         'type': 'forecast',
@@ -119,10 +122,39 @@ class TobaccoAnalyzer:
                     }
                     forecast_data.append(forecast)
 
+                # Generate extended forecast using trends
+                if forecast_data:
+                    last_date = max(d['date'] for d in forecast_data)
+                    temp_trend = 0
+                    humidity_trend = 0
+                    rainfall_trend = 0
+
+                    if len(historical_data) > 1:
+                        historical_df = pd.DataFrame(historical_data)
+                        temp_trend = stats.linregress(range(len(historical_df)), historical_df['temperature'])[0]
+                        humidity_trend = stats.linregress(range(len(historical_df)), historical_df['humidity'])[0]
+                        rainfall_trend = stats.linregress(range(len(historical_df)), historical_df['rainfall'])[0]
+
+                    for day in range(1, forecast_days - len(forecast_data)):
+                        base_forecast = forecast_data[-1]
+                        date = last_date + timedelta(days=day)
+                        
+                        extended_forecast = {
+                            'date': date,
+                            'temperature': base_forecast['temperature'] + temp_trend * day,
+                            'temp_min': base_forecast['temp_min'] + temp_trend * day,
+                            'temp_max': base_forecast['temp_max'] + temp_trend * day,
+                            'humidity': base_forecast['humidity'] + humidity_trend * day,
+                            'rainfall': max(0, base_forecast['rainfall'] + rainfall_trend * day),
+                            'type': 'forecast_extended',
+                            'description': 'Extended Forecast'
+                        }
+                        forecast_data.append(extended_forecast)
+
         except Exception as e:
             print(f"Error fetching forecast data: {e}")
 
-        # Combine all data
+        # Combine and process all data
         all_data = pd.DataFrame(historical_data + forecast_data)
         
         if not all_data.empty:
@@ -134,7 +166,10 @@ class TobaccoAnalyzer:
             # Sort by date
             all_data = all_data.sort_values('date')
             
-            # Add additional columns
+            # Calculate temperature range
+            all_data['temp_range'] = all_data['temp_max'] - all_data['temp_min']
+            
+            # Add analysis columns
             all_data['month'] = all_data['date'].dt.month
             all_data['season'] = all_data['month'].map(self.tanzania_seasons)
             
@@ -143,119 +178,77 @@ class TobaccoAnalyzer:
             all_data['humidity_7day_avg'] = all_data['humidity'].rolling(window=7, min_periods=1).mean()
             all_data['rainfall_7day_avg'] = all_data['rainfall'].rolling(window=7, min_periods=1).mean()
             
-            # Calculate daily suitability
+            # Calculate daily suitability and NDVI
             all_data['daily_suitability'] = self.calculate_daily_suitability(all_data)
-            
-            # Calculate NDVI
             all_data['estimated_ndvi'] = self.estimate_ndvi(all_data)
 
-            # Group by date to get daily values while preserving types
-            daily_data = pd.DataFrame()
-            daily_data['date'] = all_data['date'].dt.date.unique()
-            daily_data = daily_data.set_index('date')
+            return all_data
+
+        return pd.DataFrame()
+
+    def calculate_daily_suitability(self, df):
+        """Calculate daily growing suitability"""
+        try:
+            # Temperature suitability
+            temp_suit = 1 - np.clip(abs(df['temperature'] - 25) / 10, 0, 1)
             
-            # Aggregate numeric columns
-            numeric_aggs = {
-                'temperature': 'mean',
-                'humidity': 'mean',
-                'rainfall': 'sum',
-                'temp_min': 'min',
-                'temp_max': 'max',
-                'temp_7day_avg': 'last',
-                'humidity_7day_avg': 'last',
-                'rainfall_7day_avg': 'last',
-                'daily_suitability': 'mean',
-                'estimated_ndvi': 'mean'
-            }
+            # Temperature range suitability
+            temp_range_suit = 1 - np.clip(df['temp_range'] / 15, 0, 1)
             
-            # Aggregate categoric columns
-            categoric_aggs = {
-                'type': 'first',
-                'description': 'first',
-                'season': 'first'
-            }
+            # Humidity suitability
+            humidity_suit = 1 - np.clip(abs(df['humidity'] - 70) / 30, 0, 1)
             
-            # Perform aggregations separately
-            numeric_data = all_data.groupby(all_data['date'].dt.date).agg(numeric_aggs)
-            categoric_data = all_data.groupby(all_data['date'].dt.date).agg(categoric_aggs)
+            # Rainfall suitability
+            daily_rainfall_target = (self.optimal_conditions['rainfall']['min'] + 
+                                   self.optimal_conditions['rainfall']['max']) / 2
+            rainfall_suit = 1 - np.clip(abs(df['rainfall'] - daily_rainfall_target) / 
+                                      daily_rainfall_target, 0, 1)
             
-            # Combine the results
-            daily_data = pd.concat([numeric_data, categoric_data], axis=1)
-            daily_data = daily_data.reset_index()
-            daily_data['date'] = pd.to_datetime(daily_data['date'])
-
-            return daily_data
-
-        return pd.DataFrame()
-
-    def get_weather_description(self, temp, humidity, rainfall):
-        """Generate weather description based on conditions"""
-        if rainfall > 5:
-            return "Heavy Rain"
-        elif rainfall > 0:
-            return "Light Rain"
-        elif humidity > 80:
-            return "Humid"
-        elif temp > 30:
-            return "Hot"
-        elif temp < 20:
-            return "Cool"
-        else:
-            return "Fair"
+            # Combine scores with weights
+            suitability = (
+                0.35 * temp_suit +
+                0.15 * temp_range_suit +
+                0.25 * humidity_suit +
+                0.25 * rainfall_suit
+            )
+            
+            return np.clip(suitability, 0, 1)
+            
+        except Exception as e:
+            print(f"Error calculating suitability: {e}")
+            return pd.Series(0.5, index=df.index)
 
     def estimate_ndvi(self, weather_data):
-        """Estimate NDVI based on weather conditions with patterns"""
-        # Base calculation
-        normalized_temp = (weather_data['temperature'] - 15) / (30 - 15)
-        normalized_humidity = (weather_data['humidity'] - 50) / (80 - 50)
-        normalized_rainfall = weather_data['rainfall'] / 5
-
-        # Season adjustment factors
-        season_factors = {
-            'Main': 1.0,
-            'Early': 0.8,
-            'Late': 0.7,
-            'Dry': 0.5
-        }
-        
-        # Apply season adjustments with smooth transitions
-        season_multiplier = weather_data['season'].map(season_factors)
-        
-        # Calculate base NDVI
-        base_ndvi = (
-            0.4 * normalized_temp +
-            0.3 * normalized_humidity +
-            0.3 * normalized_rainfall
-        ) * season_multiplier
-        
-        # Add slight random variation to make it more realistic
-        variation = np.random.normal(0, 0.05, size=len(base_ndvi))
-        
-        # Combine and clip to valid range
-        return np.clip(base_ndvi + variation, -1, 1)
+        """Estimate NDVI based on weather conditions"""
+        try:
+            # Normalize weather parameters
+            normalized_temp = (weather_data['temperature'] - 15) / (30 - 15)
+            normalized_humidity = (weather_data['humidity'] - 50) / (80 - 50)
+            normalized_rainfall = weather_data['rainfall'] / 5
 
-    def calculate_daily_suitability(self, df):
-        """Calculate daily growing suitability with patterns"""
-        # Temperature suitability
-        temp_suit = 1 - np.abs((df['temperature'] - 25) / 10)  # Optimal at 25°C
-        
-        # Humidity suitability
-        humidity_suit = 1 - np.abs((df['humidity'] - 70) / 30)  # Optimal at 70%
-        
-        # Rainfall suitability with diminishing returns
-        rainfall_suit = 1 - np.exp(-df['rainfall'] / 2)
-        
-        # Combine with weights and add slight variation
-        base_suit = (
-            0.4 * temp_suit +
-            0.3 * humidity_suit +
-            0.3 * rainfall_suit
-        )
-        
-        # Add small random variation
-        variation = np.random.normal(0, 0.05, size=len(base_suit))
-        
-        return np.clip(base_suit + variation, 0, 1)
+            # Season adjustment factors
+            season_factors = {
+                'Main': 1.0,
+                'Early': 0.8,
+                'Late': 0.7,
+                'Dry': 0.5
+            }
+            
+            # Apply season adjustments
+            season_multiplier = weather_data['season'].map(season_factors)
+            
+            # Calculate estimated NDVI
+            estimated_ndvi = (
+                0.4 * normalized_temp +
+                0.3 * normalized_humidity +
+                0.3 * normalized_rainfall
+            ) * season_multiplier
+            
+            return np.clip(estimated_ndvi, -1, 1)
+            
+        except Exception as e:
+            print(f"Error estimating NDVI: {e}")
+            return pd.Series(0, index=weather_data.index)
 
     def analyze_trends(self, df):
         """Analyze weather trends and patterns"""
@@ -271,24 +264,26 @@ class TobaccoAnalyzer:
                     'temperature': {
                         'mean': historical['temperature'].mean(),
                         'std': historical['temperature'].std(),
-                        'trend': stats.linregress(range(len(historical)), historical['temperature'])[0],
-                        'daily_range': (historical['temp_max'] - historical['temp_min']).mean()
+                        'trend': stats.linregress(range(len(historical)), 
+                                                historical['temperature'])[0]
                     },
                     'humidity': {
                         'mean': historical['humidity'].mean(),
                         'std': historical['humidity'].std(),
-                        'trend': stats.linregress(range(len(historical)), historical['humidity'])[0]
+                        'trend': stats.linregress(range(len(historical)), 
+                                                historical['humidity'])[0]
                     },
                     'rainfall': {
                         'mean': historical['rainfall'].mean(),
                         'std': historical['rainfall'].std(),
-                        'trend': stats.linregress(range(len(historical)), historical['rainfall'])[0],
-                        'rainy_days': (historical['rainfall'] > 0).sum()
+                        'trend': stats.linregress(range(len(historical)), 
+                                                historical['rainfall'])[0]
                     },
                     'ndvi': {
                         'mean': historical['estimated_ndvi'].mean(),
                         'std': historical['estimated_ndvi'].std(),
-                        'trend': stats.linregress(range(len(historical)), historical['estimated_ndvi'])[0]
+                        'trend': stats.linregress(range(len(historical)), 
+                                                historical['estimated_ndvi'])[0]
                     }
                 }
             }
@@ -297,158 +292,24 @@ class TobaccoAnalyzer:
                 analysis['forecast'] = {
                     'temperature': {
                         'mean': forecast['temperature'].mean(),
-                        'std': forecast['temperature'].std(),
-                        'daily_range': (forecast['temp_max'] - forecast['temp_min']).mean()
+                        'std': forecast['temperature'].std()
                     },
                     'humidity': {
                         'mean': forecast['humidity'].mean(),
                         'std': forecast['humidity'].std()
-                                            },
+                    },
                     'rainfall': {
                         'mean': forecast['rainfall'].mean(),
-                        'std': forecast['rainfall'].std(),
-                        'rainy_days': (forecast['rainfall'] > 0).sum()
+                        'std': forecast['rainfall'].std()
                     },
                     'ndvi': {
                         'mean': forecast['estimated_ndvi'].mean(),
                         'std': forecast['estimated_ndvi'].std()
-                    },
-                    'confidence': {
-                        'short_term': 0.9,  # First 5 days
-                        'medium_term': 0.7,  # 6-15 days
-                        'long_term': 0.5   # Beyond 15 days
                     }
                 }
-
+            
             return analysis
+            
         except Exception as e:
             print(f"Error in trend analysis: {e}")
-            return None
-
-    def calculate_season_factor(self, date):
-        """Calculate seasonal influence factor"""
-        day_of_year = date.timetuple().tm_yday
-        season_phase = 2 * np.pi * day_of_year / 365
-        
-        # Base seasonal factor
-        base_factor = np.sin(season_phase)
-        
-        # Adjust for Tanzania's specific seasons
-        month = date.month
-        if month in [12, 1, 2]:  # Main growing season
-            season_modifier = 1.2
-        elif month in [3, 4, 5]:  # Late season
-            season_modifier = 0.8
-        elif month in [6, 7, 8]:  # Dry season
-            season_modifier = 0.5
-        else:  # Early season
-            season_modifier = 0.9
-            
-        return base_factor * season_modifier
-
-    def calculate_daily_pattern(self, hour, base_value, amplitude=1.0):
-        """Calculate daily cyclic pattern"""
-        hour_phase = 2 * np.pi * hour / 24
-        return base_value + amplitude * np.sin(hour_phase - np.pi/2)
-
-    def get_weather_risk_factors(self, df):
-        """Analyze weather-related risk factors"""
-        risks = []
-        
-        # Temperature risks
-        temp_mean = df['temperature'].mean()
-        temp_std = df['temperature'].std()
-        if temp_mean > self.optimal_conditions['temperature']['max']:
-            risks.append(('High Temperature Risk', 'Average temperature above optimal range'))
-        elif temp_mean < self.optimal_conditions['temperature']['min']:
-            risks.append(('Low Temperature Risk', 'Average temperature below optimal range'))
-        if temp_std > 5:
-            risks.append(('Temperature Volatility Risk', 'High temperature variations observed'))
-
-        # Humidity risks
-        humidity_mean = df['humidity'].mean()
-        if humidity_mean > self.optimal_conditions['humidity']['max']:
-            risks.append(('High Humidity Risk', 'Average humidity above optimal range'))
-        elif humidity_mean < self.optimal_conditions['humidity']['min']:
-            risks.append(('Low Humidity Risk', 'Average humidity below optimal range'))
-
-        # Rainfall risks
-        daily_rainfall = df.groupby(df['date'].dt.date)['rainfall'].sum()
-        rainy_days = (daily_rainfall > 0).sum()
-        total_rainfall = daily_rainfall.sum()
-        
-        if total_rainfall < self.optimal_conditions['rainfall']['min'] * len(daily_rainfall):
-            risks.append(('Drought Risk', 'Insufficient rainfall observed'))
-        elif total_rainfall > self.optimal_conditions['rainfall']['max'] * len(daily_rainfall):
-            risks.append(('Flood Risk', 'Excessive rainfall observed'))
-        
-        if rainy_days < len(daily_rainfall) * 0.2:
-            risks.append(('Rainfall Distribution Risk', 'Too few rainy days'))
-
-        # NDVI risks
-        ndvi_mean = df['estimated_ndvi'].mean()
-        if ndvi_mean < self.optimal_conditions['ndvi']['min']:
-            risks.append(('Vegetation Health Risk', 'Low vegetation health indicated by NDVI'))
-
-        # Season-specific risks
-        current_season = df['season'].iloc[-1]
-        if current_season == 'Dry':
-            risks.append(('Seasonal Risk', 'Currently in dry season'))
-
-        return risks
-
-    def calculate_risk_score(self, df):
-        """Calculate overall risk score based on all factors"""
-        risk_score = 0
-        weights = {
-            'temperature': 0.3,
-            'humidity': 0.2,
-            'rainfall': 0.2,
-            'ndvi': 0.2,
-            'season': 0.1
-        }
-
-        # Temperature component
-        temp_mean = df['temperature'].mean()
-        temp_optimal_range = self.optimal_conditions['temperature']
-        temp_score = 1 - min(abs(temp_mean - np.mean([temp_optimal_range['min'], 
-                                                     temp_optimal_range['max']])) / 10, 1)
-        
-        # Humidity component
-        humidity_mean = df['humidity'].mean()
-        humidity_optimal_range = self.optimal_conditions['humidity']
-        humidity_score = 1 - min(abs(humidity_mean - np.mean([humidity_optimal_range['min'], 
-                                                            humidity_optimal_range['max']])) / 20, 1)
-        
-        # Rainfall component
-        daily_rainfall = df.groupby(df['date'].dt.date)['rainfall'].sum()
-        rainfall_optimal_range = self.optimal_conditions['rainfall']
-        rainfall_score = 1 - min(abs(daily_rainfall.mean() - np.mean([rainfall_optimal_range['min'], 
-                                                                     rainfall_optimal_range['max']])) / 5, 1)
-        
-        # NDVI component
-        ndvi_mean = df['estimated_ndvi'].mean()
-        ndvi_optimal_range = self.optimal_conditions['ndvi']
-        ndvi_score = 1 - min(abs(ndvi_mean - np.mean([ndvi_optimal_range['min'], 
-                                                     ndvi_optimal_range['max']])) / 0.3, 1)
-        
-        # Season component
-        current_season = df['season'].iloc[-1]
-        season_scores = {
-            'Main': 1.0,
-            'Early': 0.8,
-            'Late': 0.6,
-            'Dry': 0.4
-        }
-        season_score = season_scores.get(current_season, 0.5)
-
-        # Calculate weighted score
-        risk_score = (
-            weights['temperature'] * temp_score +
-            weights['humidity'] * humidity_score +
-            weights['rainfall'] * rainfall_score +
-            weights['ndvi'] * ndvi_score +
-            weights['season'] * season_score
-        )
-
-        return np.clip(risk_score, 0, 1)
+            return None