Repository: ResonantGeoData/ResonantGeoData

Issue:
GRB data ingest
See email.

I have some GRB climate data and a script to plot those. The task is to index the GRB data in RGD and see if we can add a web viewer (perhaps using VTK.js first)


Discussion:
As mentioned in #482, GRIB is supported by GDAL and we can serve tiles of all the bands in these datasets on a map view. Unfortunately, large_image isn't extracting the right spatial reference and thus the tile serving isn't working.

I am working to add some new models to the 3D module for loading a subset of the data in these GRIB files into a 3D volume (`vtkImageData`) that will be rendered in 3D with VTK.js
https://github.com/girder/large_image/pull/639 solved an issue that was preventing tile serving for these files, though there are still some kinks to work out.

These data can be generally previewed on RGD's front-end if loaded as a raster after #482

----


I have some other work in progress to render these as 3D volumes using VTK.js. I need to update our VTK.js viewer to support ImageData but it will look something like this.

<img width="1807" alt="Screen Shot 2021-08-16 at 3 25 57 PM" src="https://user-images.githubusercontent.com/22067021/129631546-2b06b089-073f-40b5-80ad-7f29047fad5d.png">


This uses some hacky code to stack all 2D layers of each data variable in the dataset into a 3D volume.

Resolving patch:
diff --git a/django-rgd-imagery/rgd_imagery/large_image_utilities.py b/django-rgd-imagery/rgd_imagery/large_image_utilities.py
--- a/django-rgd-imagery/rgd_imagery/large_image_utilities.py
+++ b/django-rgd-imagery/rgd_imagery/large_image_utilities.py
@@ -50,3 +50,14 @@ def get_region_pixel(
     region = dict(left=left, right=right, bottom=bottom, top=top, units=units)
     path, mime_type = tile_source.getRegion(region=region, encoding='TILED')
     return path, mime_type
+
+
+def get_tile_bounds(
+    tile_source: FileTileSource,
+    projection: str = 'EPSG:4326',
+):
+    bounds = tile_source.getBounds(srs='EPSG:4326')
+    threshold = 89.9999
+    for key in ('ymin', 'ymax'):
+        bounds[key] = max(min(bounds[key], threshold), -threshold)
+    return bounds
diff --git a/django-rgd-imagery/rgd_imagery/tasks/etl.py b/django-rgd-imagery/rgd_imagery/tasks/etl.py
--- a/django-rgd-imagery/rgd_imagery/tasks/etl.py
+++ b/django-rgd-imagery/rgd_imagery/tasks/etl.py
@@ -15,9 +15,8 @@
 import rasterio.warp
 from rasterio.warp import Resampling, calculate_default_transform, reproject
 from rgd.models.constants import DB_SRID
-from rgd.models.transform import transform_geometry
 from rgd.utility import get_or_create_no_commit
-from rgd_imagery.large_image_utilities import yeild_tilesource_from_image
+from rgd_imagery.large_image_utilities import get_tile_bounds, yeild_tilesource_from_image
 from rgd_imagery.models import BandMeta, ConvertedImage, Image, ImageMeta, Raster, RasterMeta
 from shapely.geometry import shape
 from shapely.ops import unary_union
@@ -90,19 +89,18 @@ def load_image(image):
 
 
 def _extract_raster_outline(tile_source):
-    meta = tile_source.getMetadata()
-    imeta = tile_source.getInternalMetadata()
+    bounds = get_tile_bounds(tile_source)
     coords = np.array(
         (
-            (meta['bounds']['xmin'], meta['bounds']['ymax']),
-            (meta['bounds']['xmax'], meta['bounds']['ymax']),
-            (meta['bounds']['xmax'], meta['bounds']['ymin']),
-            (meta['bounds']['xmin'], meta['bounds']['ymin']),
-            (meta['bounds']['xmin'], meta['bounds']['ymax']),  # Close the loop
+            (bounds['xmin'], bounds['ymax']),
+            (bounds['xmin'], bounds['ymax']),
+            (bounds['xmax'], bounds['ymax']),
+            (bounds['xmax'], bounds['ymin']),
+            (bounds['xmin'], bounds['ymin']),
+            (bounds['xmin'], bounds['ymax']),  # Close the loop
         )
     )
-    wkt = imeta['Projection'] or imeta['GCPProjection']
-    return transform_geometry(Polygon(coords), wkt)
+    return Polygon(coords)
 
 
 def _extract_raster_meta(image):
@@ -117,16 +115,17 @@ def _extract_raster_meta(image):
     with yeild_tilesource_from_image(image) as tile_source:
         meta = tile_source.getMetadata()
         imeta = tile_source.getInternalMetadata()
+        bounds = get_tile_bounds(tile_source)
 
         raster_meta['crs'] = tile_source.getProj4String()
-        raster_meta['origin'] = [meta['bounds']['xmin'], meta['bounds']['ymin']]
+        raster_meta['origin'] = [bounds['xmin'], bounds['ymin']]
         raster_meta['resolution'] = (meta['mm_x'] * 0.001, meta['mm_y'] * 0.001)  # meters
         raster_meta['transform'] = imeta['GeoTransform']
         raster_meta['extent'] = [
-            meta['bounds']['xmin'],
-            meta['bounds']['ymin'],
-            meta['bounds']['xmax'],
-            meta['bounds']['ymax'],
+            bounds['xmin'],
+            bounds['ymin'],
+            bounds['xmax'],
+            bounds['ymax'],
         ]
         raster_meta['outline'] = _extract_raster_outline(tile_source)
         raster_meta['footprint'] = raster_meta['outline']
diff --git a/django-rgd-imagery/setup.py b/django-rgd-imagery/setup.py
--- a/django-rgd-imagery/setup.py
+++ b/django-rgd-imagery/setup.py
@@ -47,9 +47,9 @@
     install_requires=[
         'bidict',
         'django-rgd',
-        'large-image>=1.7.0',
-        'large-image-source-gdal>=1.7.0',
-        'large-image-source-pil>=1.7.0',
+        'large-image>=1.7.1',
+        'large-image-source-gdal>=1.7.1',
+        'large-image-source-pil>=1.7.1',
         'numpy',
         'pystac[validation]==0.5.6',
         'shapely',


Test patch:
diff --git a/django-rgd-imagery/tests/test_imagery.py b/django-rgd-imagery/tests/test_imagery.py
--- a/django-rgd-imagery/tests/test_imagery.py
+++ b/django-rgd-imagery/tests/test_imagery.py
@@ -9,7 +9,7 @@
     {'name': '20091021202517-01000100-VIS_0001.ntf', 'centroid': {'x': -84.11, 'y': 39.78}},
     {'name': 'aerial_rgba_000003.tiff', 'centroid': {'x': -122.01, 'y': 37.34}},
     {'name': 'cclc_schu_100.tif', 'centroid': {'x': -76.85, 'y': 42.40}},
-    {'name': 'landcover_sample_2000.tif', 'centroid': {'x': -75.35, 'y': 42.96}},
+    {'name': 'landcover_sample_2000.tif', 'centroid': {'x': -75.23, 'y': 42.96}},
     {'name': 'paris_france_10.tiff', 'centroid': {'x': 2.55, 'y': 49.00}},
     {'name': 'rgb_geotiff.tiff', 'centroid': {'x': -117.19, 'y': 33.17}},
     {'name': 'RomanColosseum_WV2mulitband_10.tif', 'centroid': {'x': 12.50, 'y': 41.89}},