feat(conus): add tiled CONUS nowcast pipeline, dynamic heatmap sampling, and UI coverage option; fix fallback lat/lon and overlay threshold

app.py

@@ -344,7 +344,14 @@ def fetch_real_time_radar_data(location="Toronto, ON"):
     except Exception as e:
         print(f"❌ Failed to fetch real-time radar data: {e}")
         print("🔄 Falling back to temporal patterns...")
-        return create_temporal_weather_patterns(lat, lon)
+        # Ensure we have sane defaults if geocoding/requests failed before lat/lon assignment
+        try:
+            _lat = lat  # may be undefined
+            _lon = lon
+        except Exception:
+            # Default to central US to keep map + forecast sensible
+            _lat, _lon = 39.0, -98.0
+        return create_temporal_weather_patterns(_lat, _lon)
 
 def fetch_canadian_radar_data(lat, lon):
     """
@@ -1023,6 +1030,208 @@ def fetch_iowa_nexrad_composites(lat, lon):
         print(f"❌ Iowa Mesonet composite fetch failed: {e}")
         return None
 
+def fetch_iowa_conus_composites():
+    """Fetch CONUS-wide NEXRAD composites (entire US) from Iowa Mesonet."""
+    try:
+        print("🗺️ Fetching Iowa Mesonet CONUS-wide NEXRAD composites...")
+        radar_images = []
+        current_time = datetime.utcnow()
+
+        for i in range(4):
+            target_time = current_time - timedelta(minutes=i * 5)
+            time_str = target_time.strftime('%Y%m%d%H%M')
+            composite_urls = [
+                f"https://mesonet.agron.iastate.edu/data/gis/images/4326/USCOMP/n0q_{time_str}.png",
+                f"https://mesonet.agron.iastate.edu/data/gis/images/4326/USCOMP/n0r_{time_str}.png"
+            ]
+
+            got = False
+            for url in composite_urls:
+                try:
+                    headers = {
+                        'User-Agent': 'DeepNowcast/1.0 (Research Application)',
+                        'Accept': 'image/png,image/*,*/*'
+                    }
+                    resp = requests.get(url, timeout=15, headers=headers)
+                    if resp.status_code == 200 and len(resp.content) > 5000:
+                        comp = Image.open(io.BytesIO(resp.content))
+                        # Convert to grayscale, resize to model input size
+                        comp = comp.convert('L').resize((256, 256), Image.Resampling.BILINEAR)
+                        radar_images.append(comp)
+                        print(f"✅ CONUS composite fetched for {target_time.strftime('%H:%M')}")
+                        got = True
+                        break
+                except Exception as e:
+                    print(f"⚠️ Error fetching CONUS composite: {e}")
+            if not got:
+                print(f"⚠️ Missing CONUS composite for {target_time.strftime('%H:%M')}")
+
+        radar_images.reverse()
+
+        if len(radar_images) >= 2:
+            while len(radar_images) < 4:
+                radar_images.append(radar_images[-1])
+
+            # Set CONUS geospatial metadata
+            globals()['_current_geo_metadata'] = {
+                'center_lat': 39.0,
+                'center_lon': -98.0,
+                'data_source': 'Iowa_Mesonet_CONUS',
+                'product_type': 'NEXRAD_CONUS_Composite',
+                'bbox': {
+                    'north': 50.0,
+                    'south': 20.0,
+                    'east': -65.0,
+                    'west': -125.0
+                },
+                'resolution': '1-4km',
+                'update_frequency': '5 minutes',
+                'coverage': 'CONUS'
+            }
+            print(f"✅ Successfully prepared {len(radar_images)} CONUS composites")
+            return radar_images
+        else:
+            print("❌ Could not fetch sufficient CONUS composites")
+            return None
+    except Exception as e:
+        print(f"❌ CONUS composite fetch failed: {e}")
+        return None
+
+def fetch_iowa_conus_composites_full():
+    """Fetch full-resolution CONUS NEXRAD composites without resizing (for tiling)."""
+    try:
+        print("🗺️ Fetching full-resolution Iowa Mesonet CONUS composites…")
+        images = []
+        current_time = datetime.utcnow()
+        sizes = []
+        for i in range(4):
+            target_time = current_time - timedelta(minutes=i * 5)
+            time_str = target_time.strftime('%Y%m%d%H%M')
+            urls = [
+                f"https://mesonet.agron.iastate.edu/data/gis/images/4326/USCOMP/n0q_{time_str}.png",
+                f"https://mesonet.agron.iastate.edu/data/gis/images/4326/USCOMP/n0r_{time_str}.png"
+            ]
+            got = False
+            for url in urls:
+                try:
+                    headers = {
+                        'User-Agent': 'DeepNowcast/1.0 (Research Application)',
+                        'Accept': 'image/png,image/*,*/*'
+                    }
+                    r = requests.get(url, timeout=15, headers=headers)
+                    if r.status_code == 200 and len(r.content) > 5000:
+                        img = Image.open(io.BytesIO(r.content)).convert('L')
+                        images.append(img)
+                        sizes.append(img.size)
+                        print(f"✅ CONUS full composite {i+1}/4 at {time_str}")
+                        got = True
+                        break
+                except Exception as e:
+                    print(f"⚠️ Error fetching CONUS full composite: {e}")
+            if not got:
+                print(f"⚠️ Missing CONUS full composite for {time_str}")
+
+        images.reverse()
+        if len(images) >= 2:
+            # Normalize sizes by resizing to the smallest common size if needed
+            if len(set(sizes)) > 1:
+                min_w = min(w for (w, h) in sizes)
+                min_h = min(h for (w, h) in sizes)
+                images = [im.resize((min_w, min_h), Image.Resampling.BILINEAR) for im in images]
+                common_size = (min_w, min_h)
+            else:
+                common_size = sizes[0]
+
+            # Update CONUS metadata
+            globals()['_current_geo_metadata'] = {
+                'center_lat': 39.0,
+                'center_lon': -98.0,
+                'data_source': 'Iowa_Mesonet_CONUS',
+                'product_type': 'NEXRAD_CONUS_Composite',
+                'bbox': {
+                    'north': 50.0,
+                    'south': 20.0,
+                    'east': -65.0,
+                    'west': -125.0
+                },
+                'resolution': '1-4km',
+                'update_frequency': '5 minutes',
+                'coverage': 'CONUS',
+                'conus_image_size': {'width': common_size[0], 'height': common_size[1]}
+            }
+            print(f"✅ Prepared {len(images)} CONUS composites at size {common_size}")
+            return images
+        else:
+            print("❌ Not enough CONUS composites (full) fetched")
+            return None
+    except Exception as e:
+        print(f"❌ CONUS full composite fetch failed: {e}")
+        return None
+
+def run_conus_tiled_inference(conus_images, system: RadarNowcastingSystem, tile_size=256, stride=192):
+    """Run tiled inference over full CONUS composites and assemble a full forecast tensor.
+
+    Returns: (forecast_tensor_np, info_dict)
+      - forecast_tensor_np shape: (1, 4, H_full, W_full) with values in [0,1]
+    """
+    assert len(conus_images) >= 4
+    # Use the first image to get full size
+    full_w, full_h = conus_images[0].size
+    print(f"📐 CONUS full size: {full_w}x{full_h}, tile={tile_size}, stride={stride}")
+
+    # Compute tile positions
+    x_positions = list(range(0, max(1, full_w - tile_size + 1), stride))
+    y_positions = list(range(0, max(1, full_h - tile_size + 1), stride))
+    if x_positions[-1] != full_w - tile_size:
+        x_positions.append(max(0, full_w - tile_size))
+    if y_positions[-1] != full_h - tile_size:
+        y_positions.append(max(0, full_h - tile_size))
+
+    # Accumulators for blending
+    T = 4
+    sum_arr = np.zeros((T, full_h, full_w), dtype=np.float32)
+    weight = np.zeros((full_h, full_w), dtype=np.float32)
+
+    # Prepare overlap weighting kernel (cosine window)
+    wx = np.hanning(tile_size)
+    wy = np.hanning(tile_size)
+    win = np.outer(wy, wx).astype(np.float32)
+    win = win / (win.max() + 1e-6)
+
+    tiles_done = 0
+    for y0 in y_positions:
+        for x0 in x_positions:
+            # Extract tile sequence
+            tile_seq_imgs = [im.crop((x0, y0, x0 + tile_size, y0 + tile_size)) for im in conus_images[:4]]
+            # Preprocess to tensor
+            tile_input = system.preprocess_radar_data(tile_seq_imgs)  # (1,4,256,256)
+            # Predict
+            tile_forecast = system.generate_forecast(tile_input)  # numpy (1,4,256,256)
+            tile_forecast = tile_forecast[0]  # (4,256,256)
+
+            # Blend into full arrays
+            for t in range(T):
+                sum_arr[t, y0:y0+tile_size, x0:x0+tile_size] += tile_forecast[t] * win
+            weight[y0:y0+tile_size, x0:x0+tile_size] += win
+
+            tiles_done += 1
+            if tiles_done % 20 == 0:
+                print(f"… processed {tiles_done} tiles")
+
+    # Avoid divide by zero
+    weight = np.clip(weight, 1e-6, None)
+    assembled = sum_arr / weight[None, :, :]
+    assembled = np.clip(assembled, 0.0, 1.0)
+
+    print(f"🧩 Tiling complete: {tiles_done} tiles blended")
+    # Return with batch dim
+    return assembled[None, ...], {
+        'num_tiles': tiles_done,
+        'tile_size': tile_size,
+        'stride': stride,
+        'full_size': (full_h, full_w)
+    }
+
 def crop_iowa_composite(composite_img, target_lat, target_lon):
     """
     Crop Iowa Mesonet composite image to region around target location
@@ -1289,7 +1498,7 @@ def create_radar_map(forecast_data=None, input_data=None):
         if data_source == 'Canadian_MSC_GeoMet':
             # Add Canadian radar WMS overlay
             add_canadian_radar_overlay(m, geo_metadata)
-        elif data_source == 'Iowa_Mesonet_NEXRAD':
+        elif data_source in ['Iowa_Mesonet_NEXRAD', 'Iowa_Mesonet_CONUS']:
             # Add Iowa Mesonet NEXRAD composite overlay
             add_iowa_mesonet_overlay(m, geo_metadata)
         elif data_source == 'NOAA_NEXRAD_Images':
@@ -1350,6 +1559,7 @@ def create_radar_map(forecast_data=None, input_data=None):
         data_source_name = {
             'Canadian_MSC_GeoMet': '🇨🇦 Canadian MSC GeoMet',
             'Iowa_Mesonet_NEXRAD': '🌽 Iowa Mesonet NEXRAD',
+            'Iowa_Mesonet_CONUS': '🌽 Iowa Mesonet CONUS',
             'NOAA_NEXRAD_Images': '🇺🇸 NOAA NEXRAD',
             'RainViewer': '🌍 RainViewer Global',
             'Temporal_Patterns': '📊 Weather Patterns'
@@ -1575,7 +1785,7 @@ def get_radar_site_coordinates(radar_site):
 # Initialize the system
 nowcast_system = RadarNowcastingSystem()
 
-def run_nowcast_prediction(use_real_time_data=True, location="Toronto, ON"):
+def run_nowcast_prediction(use_real_time_data=True, location="Toronto, ON", coverage_mode="Local"):
     """Main function to run nowcasting prediction with real radar data"""
     
     print(f"===== Nowcast Prediction Started at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} =====")
@@ -1583,7 +1793,50 @@ def run_nowcast_prediction(use_real_time_data=True, location="Toronto, ON"):
     if use_real_time_data:
         # Fetch real NEXRAD radar data
         print(f"🚀 Fetching real-time NEXRAD radar data for {location}...")
-        radar_images = fetch_real_time_radar_data(location)
+        if isinstance(coverage_mode, str) and coverage_mode.lower().startswith('conus'):
+            print("🗺️ Coverage mode: CONUS (entire US)")
+            # Full-resolution CONUS workflow with tiling
+            conus_images = fetch_iowa_conus_composites_full()
+            if conus_images and len(conus_images) >= 2:
+                # Ensure we have 4 frames
+                while len(conus_images) < 4:
+                    conus_images.append(conus_images[-1])
+
+                # Tile-based inference
+                assembled_forecast, tiles_info = run_conus_tiled_inference(conus_images, nowcast_system)
+
+                # Store forecast and metadata
+                forecast_tensor = torch.from_numpy(assembled_forecast)
+                globals()['_current_forecast_data'] = forecast_tensor
+
+                geo_meta = globals().get('_current_geo_metadata', {})
+                data_source = geo_meta.get('data_source', 'Iowa_Mesonet_CONUS')
+
+                # Prepare 256×256 quicklook for plots
+                input_quick = [img.resize((256, 256), Image.Resampling.BILINEAR) for img in conus_images[:4]]
+                input_sequence = nowcast_system.preprocess_radar_data(input_quick)
+                forecast_quick = torch.nn.functional.interpolate(
+                    forecast_tensor, size=(256, 256), mode='bilinear', align_corners=False
+                )
+
+                input_titles = ['T-15min', 'T-10min', 'T-5min', 'T-0min (Now)']
+                forecast_titles = ['T+5min', 'T+10min', 'T+15min', 'T+20min']
+                input_fig = nowcast_system.visualize_sequence(input_sequence.cpu(), input_titles, "Input Radar Sequence (Past 20 minutes)")
+                forecast_fig = nowcast_system.visualize_sequence(forecast_quick, forecast_titles, "Nowcast Forecast (Next 20 minutes)")
+
+                success_info = (
+                    f"✅ REAL Iowa Mesonet CONUS composites (entire US)\n"
+                    f"🧩 Tiled inference: {tiles_info['num_tiles']} tiles, tile={tiles_info['tile_size']}, stride={tiles_info['stride']}\n"
+                    f"🗺️ Assembled forecast size: {assembled_forecast.shape[-2]}x{assembled_forecast.shape[-1]}\n"
+                    f"🎯 Model: {'DGMR' if nowcast_system.model else 'Mock'} | Device: {nowcast_system.device}"
+                )
+                print("🎉 CONUS tiled nowcast completed successfully!")
+                return input_fig, forecast_fig, success_info
+            else:
+                print("⚠️ CONUS composites unavailable, falling back to local workflow…")
+                radar_images = fetch_real_time_radar_data(location)
+        else:
+            radar_images = fetch_real_time_radar_data(location)
         
         # Get data source info from metadata
         geo_meta = globals().get('_current_geo_metadata', {})
@@ -1602,6 +1855,13 @@ def run_nowcast_prediction(use_real_time_data=True, location="Toronto, ON"):
             data_info = f"✅ REAL NEXRAD radar data from {radar_site} ({location})\n🇺🇸 Level II radar processing\n📡 Direct AWS S3 access"
         elif data_source == 'RainViewer':
             data_info = f"✅ REAL RainViewer global radar composite ({location})\n🌍 Multi-national radar mosaic\n📡 Public weather radar network"
+        elif data_source == 'Iowa_Mesonet_CONUS':
+            data_info = (
+                "✅ REAL Iowa Mesonet CONUS composite (entire US)\n"
+                "🌽 NEXRAD US composite\n"
+                "⏱️ Updates every 5 minutes\n"
+                "📡 Country-wide coverage"
+            )
         else:
             data_info = f"📊 Real weather-informed patterns ({location})\n⏰ Generated at {datetime.now().strftime('%H:%M:%S')}\n🌤️ Based on current weather conditions"
     else:
@@ -1740,6 +2000,13 @@ def create_interface():
                         label="Data Source",
                         info="Real NEXRAD/RainViewer vs patterns"
                     )
+                with gr.Column(scale=1):
+                    coverage_mode = gr.Radio(
+                        choices=["Local around location", "CONUS (entire US)"],
+                        value="Local around location",
+                        label="Coverage",
+                        info="Pick entire US to generate a country-wide nowcast"
+                    )
             
             with gr.Row():
                 predict_btn = gr.Button("🚀 Generate Nowcast", 
@@ -1769,7 +2036,7 @@ def create_interface():
             
             predict_btn.click(
                 fn=run_nowcast_prediction,
-                inputs=[data_mode, location_input],
+                inputs=[data_mode, location_input, coverage_mode],
                 outputs=[input_plot, forecast_plot, status_text]
             )
             
@@ -1936,12 +2203,18 @@ def add_forecast_overlay(m, forecast_data, geo_metadata):
             for t, frame in enumerate(forecast_frames):
                 lat_range = np.linspace(bbox['south'], bbox['north'], frame.shape[0])
                 lon_range = np.linspace(bbox['west'], bbox['east'], frame.shape[1])
-                
+
+                # Dynamic sampling step to cap total points (~20k)
+                H, W = frame.shape
+                target_points = 20000
+                step = int(max(4, np.sqrt((H * W) / max(1, target_points))))
+                step = int(min(max(step, 4), 32))
+
                 # Create heatmap points for significant precipitation areas
                 heat_data = []
-                for i in range(0, frame.shape[0], 4):  # Sample every 4th point for performance
-                    for j in range(0, frame.shape[1], 4):
-                        if frame[i, j] > 0.15:  # Only show significant precipitation
+                for i in range(0, H, step):
+                    for j in range(0, W, step):
+                        if frame[i, j] > 0.05:
                             intensity = float(frame[i, j])
                             heat_data.append([lat_range[i], lon_range[j], intensity])
                 
@@ -2025,4 +2298,4 @@ def add_forecast_overlay(m, forecast_data, geo_metadata):
 if __name__ == "__main__":
     demo = create_interface()
     demo.launch(server_name="0.0.0.0", server_port=7860, share=True)
-    
+