Repository: MET-OM/metocean-stats (language: python)

Issue:
update dev_ml



Discussion / hints:

Fix patch:
diff --git a/environment.yml b/environment.yml
index 024c07e..5137b2f 100644
--- a/environment.yml
+++ b/environment.yml
@@ -12,12 +12,7 @@ dependencies:
   - pytest
   - windrose
   - seaborn>=0.12.2
-  - metocean-api
   - pyextremes
-  - virocon
-  - scikit-learn
-  - keras
-  - tensorflow
   - cartopy
   - python-docx
 
diff --git a/examples/example_ml.py b/examples/example_ml.py
deleted file mode 100644
index 925891b..0000000
--- a/examples/example_ml.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import matplotlib.pyplot as plt
-import sklearn
-from metocean_api import ts
-from metocean_stats.stats import ml
-import numpy as np
-
-# Import norac data
-df_norac = ts.TimeSeries(lon=6.08, lat=62.4,start_time='2018-01-01', end_time='2019-12-31' ,product='NORAC_wave')
-#df_norac.import_data(save_csv=True)
-df_norac.load_data(local_file=df_norac.datafile)
-# Import nora3 data
-df_nora3 = ts.TimeSeries(lon=6.08, lat=62.4,start_time='2015-01-01', end_time='2019-12-31' ,product='NORA3_wave_sub')
-#df_nora3.import_data(save_csv=True)
-df_nora3.load_data(local_file=df_nora3.datafile)
-
-# Define training and validation period:
-start_training = '2019-01-01'
-end_training   = '2019-12-31'
-start_valid    = '2018-01-01'
-end_valid      = '2018-12-31'
-
-# Select method and variables for ML model:
-model='GBR' # 'SVR_RBF', 'LSTM', GBR
-var_origin = ['hs','tp','Pdir']
-var_train  = ['hs']
-# Run ML model:
-ts_pred = ml.predict_ts(ts_origin=df_nora3.data,var_origin=var_origin,ts_train=df_norac.data.loc[start_training:end_training],var_train=var_train, model=model)
-# Plotting a month of data:
-fig, ax = plt.subplots(nrows=1, ncols=1,figsize=(12, 6),gridspec_kw={'top': 0.95,'bottom': 0.150,'left': 0.05,'right': 0.990,'hspace': 0.2,'wspace': 0.2})
-plt.title('Model: '+model+',Training Variables: '+','.join(var_origin))
-plt.plot(df_nora3.data['hs'].loc['2017-12-30':'2018-01-30'],'o',label='NORA3')
-plt.plot(ts_pred.loc['2017-12-30':'2018-01-30'],'x',label='NORAC_pred')
-plt.ylabel('Hs[m]',fontsize=20)
-plt.plot(df_norac.data['hs'].loc['2017-12-30':'2018-01-30'].asfreq('h'),'.',label='NORAC')
-plt.grid()
-plt.legend()
-plt.savefig(model+'-'+'_'.join(var_origin)+'ts.png')
-plt.close()
-
-#Plot all the data:
-fig, ax = plt.subplots(nrows=1, ncols=1,figsize=(12, 6),gridspec_kw={'top': 0.95,'bottom': 0.150,'left': 0.05,'right': 0.990,'hspace': 0.2,'wspace': 0.2})
-plt.title('Model: '+model+',Training Variables: '+','.join(var_origin))
-plt.plot(df_nora3.data['hs'],'o',label='NORA3')
-plt.plot(ts_pred,'x',label='NORAC_pred')
-plt.ylabel('Hs[m]',fontsize=20)
-plt.plot(df_norac.data['hs'],'.',label='NORAC')
-plt.grid()
-plt.legend()
-plt.savefig(model+'-'+'_'.join(var_origin)+'ts_all.png')
-plt.close()
-
-# Scatter plot and metrics:
-plt.scatter(df_norac.data['hs'].loc[start_valid:end_valid], ts_pred.loc[start_valid:end_valid], color='black')
-plt.title('scatter:'+model+'-'+'_'.join(var_origin))
-plt.text(0, 1.0,'ΜΑΕ:'+str(np.round(sklearn.metrics.mean_absolute_error(df_norac.data['hs'].loc[start_valid:end_valid], ts_pred.loc[start_valid:end_valid]),3)))
-plt.text(0, 0.8,'$R²$:'+str(np.round(sklearn.metrics.r2_score(df_norac.data['hs'].loc[start_valid:end_valid], ts_pred.loc[start_valid:end_valid]),3)))
-plt.text(0, 0.6,'RMSE:'+str(np.round(sklearn.metrics.mean_squared_error(df_norac.data['hs'].loc[start_valid:end_valid], ts_pred.loc[start_valid:end_valid])**0.5,3)))
-plt.xlabel('Hs from NORAC')
-plt.ylabel('Hs from NORAC_pred')
-plt.savefig(model+'-'+'_'.join(var_origin)+'scatter.png')
-plt.close()
diff --git a/metocean_stats/stats/aux_funcs.py b/metocean_stats/stats/aux_funcs.py
index 95a9120..da621aa 100644
--- a/metocean_stats/stats/aux_funcs.py
+++ b/metocean_stats/stats/aux_funcs.py
@@ -504,4 +504,3 @@ def degminsec_to_decimals(degrees,minutes,seconds):
     else:
         loc_decimals=degrees+(minutes/60)+(seconds/3600)
     return loc_decimals
-
diff --git a/metocean_stats/stats/extreme.py b/metocean_stats/stats/extreme.py
index e65b33e..861d990 100644
--- a/metocean_stats/stats/extreme.py
+++ b/metocean_stats/stats/extreme.py
@@ -344,40 +344,6 @@ def threshold_sensitivity(data, var, thresholds,
 
 
 
-def get_joint_2D_contour(data=pd.DataFrame,var1='hs', var2='tp', periods=[50,100]) -> Sequence[Dict]:
-    """Compute a joint contour for the given return periods. 
-    Input:
-        data: pd.DataFrame
-        var1: e.g., 'hs'
-        var2: e.g., 'tp'
-        return_periods: A list of return periods in years, default [50,100]
-    Output:
-         contours : list of joint contours, 
-         i.e.,  number of contours based on given return_periods 
-    """
-    from virocon import get_OMAE2020_Hs_Tz, get_DNVGL_Hs_U,get_OMAE2020_Hs_Tz,get_OMAE2020_V_Hs, GlobalHierarchicalModel,IFORMContour, ISORMContour     
-    # Define 2D joint distribution model
-    dist_descriptions, fit_descriptions, _ = get_OMAE2020_Hs_Tz()
-    model = GlobalHierarchicalModel(dist_descriptions)
-    dt = (data.index[1]-data.index[0]).total_seconds()/3600  # duration in hours
-    data_2D =  np.transpose(np.array([data[var1],data[var2]]))
-    model.fit(data_2D, fit_descriptions=fit_descriptions)
-    contours = []
-    for rp in periods:
-        alpha = 1 / (rp * 365.25 * 24 / dt)
-        contour = IFORMContour(model, alpha)
-        coords = contour.coordinates
-        x = coords[:, 1].tolist()
-        y = coords[:, 0].tolist()
-        contours.append({
-            'return_period': rp,
-            'x': x,
-            'y': y
-        })
-    return contours, data_2D
-
-
-
     
 def return_levels_weibull_2p(data, var,
                              periods=[50, 100, 1000], 
diff --git a/metocean_stats/stats/ml.py b/metocean_stats/stats/ml.py
deleted file mode 100644
index 36fcc53..0000000
--- a/metocean_stats/stats/ml.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import pandas as pd
-from sklearn.model_selection import train_test_split
-import matplotlib.pyplot as plt
-import numpy as np
-
-def predict_ts(ts_origin,var_origin, ts_train,var_train, model='GBR'):
-    """
-    Input:
-    ts_origin: pandas DataFrame
-    var_origin: variable name (str) e.g., ['hs','tp','Pdir','hs_swell']
-    ts_train: pandas DataFrame
-    var_train: variable name (str) e.g., ['hs']
-    model = 'GBR', 'SVR_RBF', LSTM
-    Output:
-    ts_pred: pandas DataFrame
-    """
-    Y_pred = pd.DataFrame(columns=[var_train], index=ts_origin.index)
-    
-    # Add extension _x, _y
-    ts_origin.columns = [col + '_x' for col in ts_origin.columns]
-    ts_train.columns  = [col + '_y' for col in ts_train.columns]
-    
-    var_origin = [var + '_x' for var in var_origin]
-    var_train = [var + '_y' for var in var_train]
-
-    # Merge or join the dataframes based on time
-    merged_data = pd.merge(ts_origin[var_origin], ts_train[var_train], how='inner', left_on='time', right_on='time')
-
-    # Handling missing values if any
-    merged_data = merged_data.dropna()
-    # Extracting features and target variables
-    X = merged_data[var_origin]
-    Y = merged_data[var_train]
-
-    # Splitting the data into training and testing sets
-    X_train, X_test, Y_train, Y_test = train_test_split(merged_data[var_origin],merged_data[var_train], test_size=0.1)
-
-    # Creating and fitting the linear regression model
-    from sklearn.linear_model import LinearRegression
-    from sklearn.svm import SVR
-    from sklearn.ensemble import GradientBoostingRegressor
-    if model == 'LinearRegression':
-        model = LinearRegression()
-        model.fit(X_train, Y_train)
-        Y_pred[:] = model.predict(ts_origin[var_origin].values).reshape(-1, 1)    
-    elif model == 'SVR_RBF':    
-        model = SVR(kernel='rbf', C=100, gamma=0.1, epsilon=0.1)
-        model.fit(X_train, Y_train)
-        Y_pred[:] = model.predict(ts_origin[var_origin].values).reshape(-1, 1)    
-    elif model == 'SVR_LINEAR':            
-        model = SVR(kernel="linear", C=100, gamma="auto")
-        model.fit(X_train, Y_train)
-        Y_pred[:] = model.predict(ts_origin[var_origin].values).reshape(-1, 1)    
-    elif model == 'SVR_POLY':    
-        model = SVR(kernel="poly", C=100, gamma="auto", degree=3, epsilon=0.1, coef0=1)
-        model.fit(X_train, Y_train)
-        Y_pred[:] = model.predict(ts_origin[var_origin].values).reshape(-1, 1)    
-    elif model == 'GBR':
-        model = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, max_depth=3)
-        model.fit(X_train, Y_train)
-        Y_pred[:] = model.predict(ts_origin[var_origin].values).reshape(-1, 1)    
-    elif model == 'LSTM':
-        from keras.models import Sequential
-        from keras.layers import LSTM, Dense
-        from keras.optimizers import Adam
-        optimizer = Adam(learning_rate=0.001)
-        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)
-        model = Sequential()
-        model.add(LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], 1)))
-        model.add(LSTM(units=50, return_sequences=False))
-        model.add(Dense(units=1))
-        # Compile the model
-        model.compile(optimizer=optimizer, loss='mean_squared_error')
-        model.fit(X_train, Y_train, epochs=40, batch_size=32, validation_data=(X_test, Y_test), verbose=1)
-        Y_pred[:] = model.predict(ts_origin[var_origin].values).reshape(-1, 1) 
-
-    # remove back extension _x, _y
-    ts_origin.columns = [col.replace('_x', '') for col in ts_origin.columns]
-    ts_train.columns = [col.replace('_y', '') for col in ts_train.columns]
-    
-    return Y_pred
\ No newline at end of file
diff --git a/setup.py b/setup.py
index f22e65a..7b2e948 100644
--- a/setup.py
+++ b/setup.py
@@ -16,21 +16,16 @@
     license = 'GPLv3',
     install_requires = [
         'numpy>=1.17',
+        'xarray',
         'matplotlib>=3.1',
         'pandas',
         'pip',
         'scipy',
-        'pip',
         'windrose',
         'seaborn>=0.12.2',
-        'metocean-api',
-        'xarray',
-        'metocean-api',
         'pyextremes',
-        'scikit-learn',
-        'keras',
-        'tensorflow',
         'cartopy',
+        'python-docx',
     ],
     packages = setuptools.find_packages(),
     include_package_data = True,
diff --git a/version.py b/version.py
index b390071..36dadf6 100644
--- a/version.py
+++ b/version.py
@@ -1,4 +1,4 @@
-__version__ = "1.0.0"
+__version__ = "1.0.1"
 
 
 def git_describe():


Test patch:
diff --git a/tests/test_ml.py b/tests/test_ml.py
deleted file mode 100644
index 06cec15..0000000
--- a/tests/test_ml.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from metocean_api import ts
-from metocean_stats import plots, tables, stats
-from metocean_stats.stats.aux_funcs import *
-from metocean_stats.stats import ml
-import pandas as pd
-import os
-
-# Define TimeSeries-object for NORA3
-ds = readNora10File('tests/data/NORA_test.txt')
-#ds_ocean = pd.read_csv('tests/data/NorkystDA_test.csv',comment='#',index_col=0, parse_dates=True)
-#depth = ['0m', '1m', '2.5m', '5m', '10m', '15m', '20m', '25m', '30m', '40m', '50m', '75m', '100m', '150m', '200m']
-
-
-def test_predict_ts_GBR(ds=ds):
-    ml.predict_ts(ts_origin=ds,var_origin=['W10','TP','DIRP'],ts_train=ds.loc['1960-01-01':'1960-12-31'],var_train=['HS'], model='GBR')
-
-def test_predict_ts_SVR(ds=ds):
-    ml.predict_ts(ts_origin=ds,var_origin=['W10','TP','DIRP'],ts_train=ds.loc['1960-01-01':'1960-12-31'],var_train=['HS'], model='SVR_RBF')
-
-#def test_predict_ts_LSTM(ds=ds):
-#    ml.predict_ts(ts_origin=ds,var_origin=['W10','TP','DIRP'],ts_train=ds.loc['1960-01-01':'1960-12-31'],var_train=['HS'], model='LSTM')
-
-  
\ No newline at end of file
diff --git a/tests/test_plots.py b/tests/test_plots.py
index d92e088..a99c262 100644
--- a/tests/test_plots.py
+++ b/tests/test_plots.py
@@ -1,4 +1,3 @@
-from metocean_api import ts
 from metocean_stats import plots, tables, stats, maps
 from metocean_stats.stats.aux_funcs import *
 import pandas as pd
diff --git a/tests/test_tables.py b/tests/test_tables.py
index fb0c716..3a0cc36 100644
--- a/tests/test_tables.py
+++ b/tests/test_tables.py
@@ -1,4 +1,3 @@
-from metocean_api import ts
 from metocean_stats import plots, tables, stats, maps
 from metocean_stats.stats.aux_funcs import *
 import pandas as pd