Upload 2 files

app.py

@@ -1,212 +1,67 @@
-import cftime
-import fsspec
-
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import panel as pn
 import xarray as xr
-from ipywidgets import (
-    Checkbox,
-#    FloatRangeSlider,
-    FloatSlider,
-    IntRangeSlider,
-    IntSlider,
-#    interactive,
-)
-
-
 import param
 
-pn.extension("tabulator")
-
-def greg_0h(jourjul):
-    import math
-    # Julian days start and end at noon.
-    # Julian day 2440000 begins at 00 hours, May 23, 1968.
-
-    # Round the input Julian day number to avoid precision errors
-    fac = 10**9
-    jourjul = np.round(fac * jourjul + 0.5) / fac
-
-    # Calculate seconds in the day
-    secs = (jourjul % 1) * 24 * 3600
-
-    # Round seconds to avoid precision errors
-    secs = np.round(fac * secs + 0.5) / fac
-
-    # Gregorian calendar conversion
-    j = math.floor(jourjul) - 1721119
-    in_ = 4 * j - 1
-    y = math.floor(in_ / 146097)
-    j = in_ - 146097 * y
-    in_ = math.floor(j / 4)
-    in_ = 4 * in_ + 3
-    j = math.floor(in_ / 1461)
-    d = math.floor(((in_ - 1461 * j) + 4) / 4)
-    in_ = 5 * d - 3
-    m = math.floor(in_ / 153)
-    d = math.floor(((in_ - 153 * m) + 5) / 5)
-    y = y * 100 + j
-    if m < 10:
-        mo = m + 3
-        yr = y
-    else:
-        mo = m - 9
-        yr = y + 1
-
-    hour = math.floor(secs / 3600)
-    mins = math.floor((secs % 3600) / 60)
-    sec = int(secs % 60)
-
-    gtime = [yr, mo, d, hour, mins, sec]
-
-    return gtime
-
-
-def fix_time(ds):
-    from datetime import datetime, timedelta
-
-    time = ds["TIME"]
-    time2 = time.dropna(dim="MAXT")
-    time2 = time2.reindex_like(time, method="nearest")
-    jourjul = [greg_0h(jourjul) for jourjul in time2.values]
-    date = [
-        datetime(jourjul[0], jourjul[1], jourjul[2], jourjul[3], jourjul[4], jourjul[5])
-        for jourjul in jourjul
-    ]
-    date = xr.DataArray(date, dims="MAXT")
-    return ds.assign(TIME=date)
-
-@pn.cache(max_items=32,policy='LRU',per_session=True)
-def load_csv():
-    df = pd.read_csv('./data/zarr_table.csv',index_col=None)
-    df.sort_values(by="year") #inplace=True)
-    return df
-
-@pn.cache(max_items=4,policy='LRU',per_session=True)
-def load_bathymetry():
-    #url="https://data-eurogoship.ifremer.fr/bathymetrie/bathy6min.nc"
-    #fs = fsspec.filesystem("https")
-    return xr.open_dataset('./data/bathy6min.nc', decode_times=False, use_cftime=True)
-
-@pn.cache(max_items=16,policy='LRU',per_session=True)
-def load_zarr(selected_file):
-    from datatree import open_datatree
-    tree= open_datatree('./data/1H_file.zarr', engine='zarr')
-    return tree[selected_file+"/"].ds
-
-def filter_df(sorted_df,selected_file):
-    dataframe = sorted_df[sorted_df["file_name"] == selected_file].drop(
-                columns=[
-                    "file_name",
-                    "title",
-                    "Conventions",
-                    "featureType",
-                    "date_update",
-                    "ADCP_beam_angle",
-                    "ADCP_ship_angle",
-                    "middle_bin1_depth",
-                    "heading_corr",
-                    "pitch_corr",
-                    "ampli_corr",
-                    "pitch_roll_used",
-                    "date_creation",
-                    "ADCP_type",
-                    "data_type",
-                ]
-            )
-    dataframe2 = sorted_df[sorted_df["file_name"] == selected_file].drop(
-                columns=[
-                    "file_name",
-                    "date_start",
-                    "date_end",
-                    "ADCP_frequency(kHz)",
-                    "bin_length(meter)",
-                    "year",
-                ]
-            )
-    return dataframe.transpose(), dataframe2.transpose()
+import hvplot.xarray
+import matplotlib.pyplot as plt
+import cartopy.crs as ccrs
+import cartopy.feature as cfeature
+from datatree import open_datatree
+from xsadcp import get_range, load_csv, load_bathymetry, load_zarr, load_file, filter_df, filter_data, quiver_depth_filtered, bathy_uship_vship_bottom_depth, corsen_data, vectors_plot, fix_time, transform_netCDF, get_info, get_file_names, open_ds
 
-def quiver_depth_filterd(ax,ds_filtered, depth1, depth_range_slider, scale_factor_slider, color="blue"):
-    import cartopy.crs as ccrs
-    depth_filtered = ds_filtered.where(
-        (depth1 > depth_range_slider.value[0])
-        & (depth1 <= depth_range_slider.value[1])
-    )
 
-    lon = ds_filtered.coords["LONGITUDE"].values
-    lat = ds_filtered.coords["LATITUDE"].values
 
-    # Moyenne des vecteurs de courant sur la plage de profondeur sélectionnée
-    u_mean = depth_filtered.UCUR.mean(dim="MAXZ", skipna=True)
-    v_mean = depth_filtered.VCUR.mean(dim="MAXZ", skipna=True)
+class SADCP_Viewer(param.Parameterized):
+    """
+    A parameterized class for viewing SADCP data.
     
-    return ax.quiver(
-        lon,
-        lat,
-        u_mean * scale_factor_slider.value,
-        v_mean * scale_factor_slider.value,
-        color=color,
-        scale=2,
-        width=0.001,
-        headwidth=3,
-        transform=ccrs.PlateCarree(),
-    )
+    This class provides widgets for selecting data parameters, updating data based on selections,
+    and generating plots to visualize the SADCP data.
+    
+    Available functions:
+        - update_name_options: Update dropdown options and slider ranges based on selected years and file.
+        - update_plots: Update plots based on selected data and parameters.
+    """
 
-class SADCP_Viewer(param.Parameterized):
+    # Load data and initialize widgets
     df = load_csv()
     bathy = load_bathymetry()
+    tree=load_zarr()
     file_names = df["file_name"].tolist()
     years = sorted(df["year"].unique())
 
-    year_slider = pn.widgets.IntRangeSlider(
-        name="Year Range", start=df["year"].min(), end=df["year"].max()
-    )
+    # Widgets for selecting data parameters
+    year_slider = pn.widgets.IntRangeSlider(name="Year Range", start=df["year"].min(), end=df["year"].max())
     file_dropdown = pn.widgets.Select(name="File Selector")
-    data_table = pn.widgets.Tabulator(df, name="metadata", height=200, width=300)
-    # data_table2 = pn.widgets.Tabulator(df2, name="metadata", height=900, width=400)
-
-    longitude_slider = pn.widgets.RangeSlider(
-        name="Longitude Range", start=-180, end=180, step=1
-    )
-    latitude_slider = pn.widgets.RangeSlider(
-        name="Latitude Range", start=-90, end=90, step=1
-    )
-
-    depth_range_slider = pn.widgets.IntRangeSlider(
-        start=100, end=300, value=(100, 300), step=1, name="Depth Range"
-    )
+    longitude_slider = pn.widgets.RangeSlider(name="Longitude Range", start=-180, end=180, step=1)
+    latitude_slider = pn.widgets.RangeSlider(name="Latitude Range", start=-90, end=90, step=1)
+    depth_range_slider = pn.widgets.IntRangeSlider(start=100, end=300, value=(100, 300), step=1, name="Depth Range")
     depth_2_checkbox = pn.widgets.Checkbox(value=False, name="Depth 2 Checkbox")
     depth_3_checkbox = pn.widgets.Checkbox(value=False, name="Depth 3 Checkbox")
-
-    depth2_range_slider = pn.widgets.IntRangeSlider(
-        start=100, end=300, value=(100, 300), step=1, name="Depth 2 Range"
-    )
-    depth3_range_slider = pn.widgets.IntRangeSlider(
-        start=100, end=300, value=(100, 300), step=1, name="Depth 3 Range"
-    )
-
-    num_vectors_slider = pn.widgets.IntSlider(
-        start=40, end=800, step=1, value=100, name="Number of Vectors"
-    )
-    scale_factor_slider = pn.widgets.FloatSlider(
-        start=0.1, end=1, step=0.1, value=0.5, name="Scale Factor"
-    )
+    depth_2_range_slider = pn.widgets.IntRangeSlider(start=100, end=300, value=(100, 300), step=1, name="Depth 2 Range")
+    depth_3_range_slider = pn.widgets.IntRangeSlider(start=100, end=300, value=(100, 300), step=1, name="Depth 3 Range")
+    num_vectors_slider = pn.widgets.IntSlider(start=40, end=800, step=1, value=100, name="Number of Vectors")
+    scale_factor_slider = pn.widgets.FloatSlider(start=0.1, end=1, step=0.1, value=0.5, name="Scale Factor")
     bathy_checkbox = pn.widgets.Checkbox(value=False, name="Bathy Checkbox")
 
-    plot = pn.pane.HoloViews()
-    plot_map = pn.pane.Matplotlib(width=800, height=600, sizing_mode="fixed")
-
+ 
     data_table = pn.widgets.Tabulator(width=400, height=200)
-
     metadata_table = pn.widgets.Tabulator(width=600, height=800)
+    # Download button is not working : TODO
     download_button = pn.widgets.Button(name="Download", button_type="primary")
-    # plot = pn.Column()
 
     def __init__(self, **params):
+        """
+        Initialize the SADCP_Viewer class.
+        
+        Parameters:
+            **params: Additional parameters to be passed to the superclass.
+        """
+
         super(SADCP_Viewer, self).__init__(**params)
-        self.file_dropdown.objects = self.get_file_list()
+        self.file_dropdown.objects = self.file_names
         self.file_dropdown.value = (
             self.file_dropdown.objects[0] if self.file_dropdown.objects else None
         )
@@ -214,54 +69,50 @@ class SADCP_Viewer(param.Parameterized):
 
     @param.depends("year_slider.value", "file_dropdown.value", watch=True)
     def update_name_options(self):
+        """
+        Update dropdown options and slider ranges based on selected years and file.
+
+        This function updates the dropdown options and slider ranges based on the selected years
+        and file. It also loads the selected file's data and adjusts slider ranges accordingly.
+
+        """
+        # Extract selected start and end years
         start_year, end_year = self.year_slider.value
+        
+        # Filter DataFrame based on selected years and sort by year
         mask = (self.df["year"] >= start_year) & (self.df["year"] <= end_year)
         sorted_df = self.df[mask].sort_values(by="year")
+        
+        # Get unique file names
         files = sorted_df["file_name"].unique().tolist()
-
+        
+        # Update file dropdown options
         self.file_dropdown.options = files
+        
         if files:
             selected_file = self.file_dropdown.value
+            
+            # Set default selected file if not selected or not in options
             if not selected_file or selected_file not in files:
                 selected_file = files[0]
                 self.file_dropdown.value = selected_file
-            self.data_table.value, self.metadata_table.value  = filter_df(sorted_df,selected_file)
-
-            self.ds=load_zarr(selected_file)
-
-            lon_range = (
-                int(self.ds["LONGITUDE"].min().round() - 1),
-                int(self.ds["LONGITUDE"].max().round() + 1),
-            )
-            lat_range = (
-                int(self.ds["LATITUDE"].min().round() - 1),
-                int(self.ds["LATITUDE"].max().round() + 1),
-            )
-            self.depth1 = abs(self.ds.coords["PROFZ"])
-            deph_range = (int(self.depth1.min()), int(self.depth1.max()))
-
-            self.longitude_slider.start = lon_range[0]
-            self.longitude_slider.end = lon_range[1]
-            self.longitude_slider.value = lon_range
-
-            self.latitude_slider.start = lat_range[0]
-            self.latitude_slider.end = lat_range[1]
-            self.latitude_slider.value = lat_range
-
-            self.depth_range_slider.start = deph_range[0]
-            self.depth_range_slider.end = deph_range[1]
-            self.depth_range_slider.value = deph_range
-
-            self.depth2_range_slider.start = deph_range[0]
-            self.depth2_range_slider.end = deph_range[1]
-            self.depth2_range_slider.value = deph_range
-
-            self.depth3_range_slider.start = deph_range[0]
-            self.depth3_range_slider.end = deph_range[1]
-            self.depth3_range_slider.value = deph_range
-
-            # self.update_plots()
-            self.ds.close()
+            
+            # Update data table and metadata table based on selected file
+            self.data_table.value, self.metadata_table.value = filter_df(sorted_df, selected_file)
+            
+            # Load selected file's data
+            self.ds = load_file(self.tree,selected_file)
+            
+            # Update slider ranges for longitude, latitude, and depth
+            for slider, coord in zip([self.longitude_slider, self.latitude_slider, self.depth_range_slider,
+                          self.depth_2_range_slider, self.depth_3_range_slider],
+                         [self.ds.LONGITUDE, self.ds.LATITUDE, self.ds.PROFZ,self.ds.PROFZ,self.ds.PROFZ]):
+                coord_range = get_range(coord)
+                slider.start, slider.end, slider.value = coord_range[0], coord_range[1], coord_range
+
+
+            # Close dataset to free up resources
+            # self.ds.close()
 
     @param.depends(
         "year_slider.value",
@@ -269,117 +120,60 @@ class SADCP_Viewer(param.Parameterized):
         "depth_range_slider.value",
         "depth_2_checkbox.value",
         "depth_3_checkbox.value",
-        "depth2_range_slider.value",
-        "depth3_range_slider.value",
+        "depth_2_range_slider.value",
+        "depth_3_range_slider.value",
         "longitude_slider.value",
         "latitude_slider.value",
         "num_vectors_slider.value",
         "scale_factor_slider.value",
         "bathy_checkbox.value",
-        watch=False,
-    )
+        watch=False,)
     def update_plots(self):
-        self.ds_filtered = self.filter_data()
-        # vector_plot = self.vectors_plot()
-        other_plots = self.hvplot_plots()
-
-        # self.plot_map = pn.pane.Matplotlib(vector_plot, width=800, height=600, sizing_mode="fixed")
-        self.plot = pn.Column(
+        """
+        This function updates the plots based on the selected data and parameters.
+
+        The function filters the data, generates additional plots, and updates the main vector plot based on the selected parameters.
+
+        Returns:
+            pn.Row: A Panel row containing the updated map plot and additional plots.
+
+        """
+        # Filter the data
+        self.ds_filtered = filter_data(self.ds,self.longitude_slider.value,self.latitude_slider.value)
+
+        # Prepare the plots shown in left
+        # Update vector plots
+        vector_plot = vectors_plot(self.ds_filtered, self.bathy, 
+                                   self.longitude_slider.value, self.latitude_slider.value,
+                                   self.depth_range_slider.value, self.depth_2_range_slider.value, self.depth_3_range_slider.value,
+                                   self.scale_factor_slider.value,self.num_vectors_slider.value,
+                                   depth_2_checkbox= self.depth_2_checkbox.value,
+                                   depth_3_checkbox= self.depth_3_checkbox.value,
+                                   bathy_checkbox=self.bathy_checkbox.value,
+                                   )
+
+
+        # Generate plots which will be plotted on the left row.
+        self.plot_left = pn.Column(
+                              # Here adjust the style option later TODO
+                              # https://panel.holoviz.org/how_to/styling/matplotlib.html
+                               pn.pane.Matplotlib(vector_plot, dpi=144),
+                              # Add here the hvplot block of contour TODO
+                                sizing_mode="stretch_both")
+
+        # Generate additional plots which will be plotted on the right row.
+        other_plots = bathy_uship_vship_bottom_depth(self.ds_filtered)
+        self.plot_right = pn.Column(
             *(pn.pane.HoloViews(plot, width=400, height=200) for plot in other_plots),
             sizing_mode="stretch_width"
         )
-        vector_plot = self.vectors_plot()  # Update vector plot
-        self.plot_map.object = vector_plot
-        # other_plots = self.plots()  # Update other plots
-        # self.plot.objects = [*(pn.pane.HoloViews(p, width=400, height=200) for p in other_plots)]
-        # self.plot.object=plot
-
-        return pn.Row(self.plot_map, self.plot, sizing_mode="stretch_both")
-
-    def filter_data(self):
-        return self.ds.where(
-            (self.ds.LONGITUDE >= self.longitude_slider.start)
-            & (self.ds.LONGITUDE <= self.longitude_slider.end)
-            & (self.ds.LATITUDE >= self.latitude_slider.start)
-            & (self.ds.LATITUDE <= self.latitude_slider.end),
-            drop=True,
-        )
-
-    def vectors_plot(self):
-        import cartopy.crs as ccrs
-        import cartopy.feature as cfeature
 
-        self.ds_filtered = self.filter_data()
+        # Return a Panel row containing the updated map plot and additional plots
+        return pn.Row(self.plot_left, self.plot_right, sizing_mode="stretch_both")
 
-        # return self.ds_filtered['VSHIP'].hvplot(x='TIME',width=400, height=200) #if 'BATHY' in self.ds_filtered else hvplot.show(hvplot.text(0, 0, "No data available", fontsize=12))
-
-        fig, ax = plt.subplots(
-            figsize=(8, 7), subplot_kw={"projection": ccrs.Mercator()}
-        )
-        coords = ["LATITUDE", "LONGITUDE"]
-        corsen = max(1, self.ds_filtered.MAXT.size // self.num_vectors_slider.value)
-        self.ds_filtered = (
-            self.ds_filtered.reset_coords(coords)
-            .coarsen({"MAXT": corsen}, boundary="trim")
-            .mean()
-            .set_coords(coords)
-        )
-
-        # self.ds_filtered =self.ds_filtered.coarsen(MAXT = corsen,  side = "center", boundary = "trim").mean()[["LONGITUDE", "LONGITUDE", "TIME"]].isel(MAXZ=0)
-
-        quiver_depth_filterd(ax,self.ds_filtered, self.depth1, self.depth_range_slider,self.scale_factor_slider,color="blue")
-
-        if self.depth_2_checkbox.value:
-            quiver_depth_filterd(ax,self.ds_filtered, self.depth1, self.depth2_range_slider,self.scale_factor_slider,color="green")
-
-        if self.depth_3_checkbox.value:
-            quiver_depth_filterd(ax,self.ds_filtered, self.depth1, self.depth3_range_slider,self.scale_factor_slider,color="red")
-
-        ax.add_feature(cfeature.COASTLINE)
-        ax.add_feature(cfeature.BORDERS, linestyle=":")
-        ax.add_feature(cfeature.LAND, color="lightgray")
-
-        if self.bathy_checkbox.value:
-            contour_levels = [-1000]
-            ax.contour(
-                self.bathy.longitude,
-                self.bathy.latitude,
-                self.bathy.z,
-                levels=contour_levels,
-                colors="black",
-                transform=ccrs.PlateCarree(),
-            )
-
-        ax.set_extent(
-            [
-                self.longitude_slider.value[0],
-                self.longitude_slider.value[1],
-                self.latitude_slider.value[0],
-                self.latitude_slider.value[1],
-            ]
-        )
-        ax.gridlines(draw_labels=True)
-        plt.ylabel("Latitude", fontsize=15, labelpad=35)
-        plt.xlabel("Longitude", fontsize=15, labelpad=20)
-        plt.close()
-        return fig
-
-        # pn.pane.Matplotlib(fig,width=800, height=600, sizing_mode="fixed", name="Plot")
-
-    # @param.depends( 'file_dropdown.value', 'longitude_slider.value', 'latitude_slider.value',  watch=True)
-    def hvplot_plots(self):
-        import hvplot.xarray
-        return [
-            self.ds_filtered["BATHY"].max(dim="MAXZ").hvplot(x="TIME", width=400, height=200),
-            self.ds_filtered["USHIP"].max(dim="MAXZ").hvplot(x="TIME", width=400, height=200),
-            self.ds_filtered["VSHIP"].max(dim="MAXZ").hvplot(x="TIME", width=400, height=200),
-            self.ds_filtered["BOTTOM_DEPTH"].max(dim="MAXZ").hvplot(x="TIME", width=400, height=200),
-          
-        ]
-
-    def get_file_list(self):
-        return self.file_names
 
+pn.extension("tabulator")
+pn.config.theme = 'dark'
 
 explorer = SADCP_Viewer()
 # Instantiate the SADCP_Viewer class and create a template
@@ -394,6 +188,7 @@ tabs = pn.Tabs(
 )
 
 sidebar = [
+    pn.panel('./EuroGO-SHIP_logo_wide_tagline_1.2.png',width=300 ),
     """This application, developed in the frame of Euro Go Shop, helps to interactively visualise and download ship ADCP data.""",
     explorer.year_slider,
     explorer.file_dropdown,
@@ -403,14 +198,21 @@ sidebar = [
     explorer.depth_range_slider,
     explorer.depth_2_checkbox,
     explorer.depth_3_checkbox,
-    explorer.depth2_range_slider,
-    explorer.depth3_range_slider,
+    explorer.depth_2_range_slider,
+    explorer.depth_3_range_slider,
     explorer.num_vectors_slider,
     explorer.scale_factor_slider,
     explorer.data_table,
+    """You can consult detailed information on this data in the metadata tab shown on the right.
+       To download full dataset, please go to https://cdi.seadatanet.org/search 
+      and search with LOCAL_CDI_ID indicated above.""",
+    #pn.panel('https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png', ),
+#width=10)
 ]
+
 template = pn.template.FastListTemplate(
-    title="SADCP data Viewer", sidebar=sidebar, main=[tabs]
+    title="SADCP data Viewer", logo='https://avatars.githubusercontent.com/u/123177533?s=200&v=4',
+    sidebar=sidebar, main=[tabs]
+    
 )
 template.servable()
-

requirements.txt

@@ -1,9 +1,7 @@
 panel
 jupyter
-transformers
 numpy
-torch
-aiohttp
+fsspec
 cartopy
 cftime
 hvplot
@@ -11,4 +9,5 @@ xarray==2024.2.0
 scipy
 matplotlib
 xarray-datatree
-zarr
+zarr
+lxml