Kaggle competition: How Much Did It Rain? II
def check_missing_data(df):
flag=df.isna().sum().any()
if flag==True:
total = df.isnull().sum()
percent =(df.isnull().sum())/(df.isnull().count() *100)
 output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 data_type = []
for col in df.columns:
dtype = str(df[col].dtype)
 data_type.append(dtype)
 output['Types'] = data_type
return(np.transpose(output))
else:
return(False )<count_missing_values>
col_list = ['minutes_past', 'radardist_km', 'Ref', 'Ref_5x5_10th',
'Ref_5x5_50th', 'Ref_5x5_90th', 'RefComposite', 'RefComposite_5x5_10th',
'RefComposite_5x5_50th', 'RefComposite_5x5_90th', 'RhoHV',
'RhoHV_5x5_10th', 'RhoHV_5x5_50th', 'RhoHV_5x5_90th', 'Zdr',
'Zdr_5x5_10th', 'Zdr_5x5_50th', 'Zdr_5x5_90th', 'Kdp', 'Kdp_5x5_10th',
'Kdp_5x5_50th', 'Kdp_5x5_90th', 'Expected']
d = {c : np.float32 for c in col_list}
check_missing_data(train )<count_missing_values>
train = pd.read_csv(".. /input/how-much-did-it-rain-ii/train.zip", dtype=d)
 train
check_missing_data(test )<count_values>
train.loc[train["Id"] == 862571]
train['target'].value_counts()<count_values>
train.loc[train["Id"] == 5]
def check_balance(df,target):
check=[]
print('size of data is:',df.shape[0])
 for i in [0,1]:
print('for target {} ='.format(i))
print(df[target].value_counts() [i]/df.shape[0]*100,'%')
<compute_test_metric>
train.isna().sum()
check_balance(train,'target' )<prepare_x_and_y>
pd.set_option('display.float_format', lambda x: '%.3f' % x )
cols=["target","ID_code"]
X = train.drop(cols,axis=1)
 y = train["target"] 
<drop_column>
print(stats.percentileofscore(train["Expected"], 106))
X_test = test.drop("ID_code",axis=1 )<split>
train.drop(train[train["Expected"] >= 106].index, inplace=True)
 train
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 rfc_model = RandomForestClassifier(random_state=0 ).fit(train_X, train_y )<train_model>
train_grouped = train.groupby('Id')
 target = pd.DataFrame(train_grouped['Expected'].mean() )
perm = PermutationImportance(rfc_model, random_state=1 ).fit(val_X, val_y )<load_pretrained>
target.reset_index(inplace=True)
 target = target["Expected"]
target
eli5.show_weights(perm, feature_names = val_X.columns.tolist() , top=150 )<train_model>
def pad_series(X, target_len=19):
seq_len = X.shape[0]
pad_size = target_len-seq_len
if(pad_size > 0):
X = np.pad(X,(( 0,pad_size),(0,0)) , 'constant', constant_values=0.)
return X, seq_len
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
 tree_model = DecisionTreeClassifier(random_state=0, max_depth=5, min_samples_split=5 ).fit(train_X, train_y )<define_variables>
INPUT_WIDTH = 19
data_size = len(train_grouped)
 X_train = np.empty(( data_size, INPUT_WIDTH, 22))
seq_lengths = np.zeros(data_size)
 y_train = np.zeros(data_size)
 i = 0
for _, group in train_grouped:
X = group.values
seq_len = X.shape[0]
X_train[i,:seq_len,:] = X[:,1:23]
y_train[i] = X[0,23]
i += 1
del X
del train_grouped
features = [c for c in train.columns if c not in ['ID_code', 'target']]<save_to_csv>
x_train, x_valid, y_train, y_valid = train_test_split(X_train, target, random_state=42, shuffle=True )
tree_graph = tree.export_graphviz(tree_model, out_file=None, feature_names=features )<predict_on_test>
del X_train
del target
train_data = tf.data.Dataset.from_tensor_slices(( x_train, y_train))
valid_data = tf.data.Dataset.from_tensor_slices(( x_valid, y_valid))
train_data = train_data.batch(32)
 valid_data = valid_data.batch(32)
 del x_train
del x_valid
del y_train
del y_valid
row_to_show = 5
data_for_prediction = val_X.iloc[row_to_show]
data_for_prediction_array = data_for_prediction.values.reshape(1, -1)
 rfc_model.predict_proba(data_for_prediction_array);<init_hyperparams>
def create_model(shape=(19, 22)) :
tfkl = tf.keras.layers
model = tf.keras.Sequential([
tfkl.Bidirectional(( tfkl.LSTM(128, return_sequences=True)) , input_shape=shape),
tfkl.Bidirectional(tfkl.LSTM(64)) ,
tfkl.Dense(64, activation="linear"),
tfkl.Dense(1, activation="linear")
 ])
 model.compile(loss='mean_absolute_error', optimizer="adam")
 return model
model = create_model()
model.summary()
params = {'objective' : "binary",
'boost':"gbdt",
'metric':"auc",
'boost_from_average':"false",
'num_threads':8,
'learning_rate' : 0.01,
'num_leaves' : 13,
'max_depth':-1,
'tree_learner' : "serial",
'feature_fraction' : 0.05,
'bagging_freq' : 5,
'bagging_fraction' : 0.4,
'min_data_in_leaf' : 80,
'min_sum_hessian_in_leaf' : 10.0,
'verbosity' : 1}<train_model>
model.fit(train_data, epochs=100, validation_data=valid_data,
callbacks=[tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience = 10),
tf.keras.callbacks.ModelCheckpoint("model.h5", save_best_only=True)] )
%%time
y_pred_lgb = np.zeros(len(X_test))
num_round = 1000000
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params,train_data,num_round,
valid_sets = [train_data, valid_data],verbose_eval=1000,early_stopping_rounds = 3500)
 y_pred_lgb += lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration)/5<predict_on_test>
col_list.pop()
d = {c: np.float32 for c in col_list}
test = pd.read_csv(".. /input/how-much-did-it-rain-ii/test.zip", dtype=d)
 test[test.columns[1:]] = test[test.columns[1:]].astype(np.float32)
 test_ids = test['Id'].unique()
test = test.reset_index(drop=True)
 test.fillna(0.0, inplace=True)
 test_groups = test.groupby("Id")
 test_size = len(test_groups)
 X_test = np.zeros(( test_size, INPUT_WIDTH, 22), dtype=np.float32)
 i = 0
for _, group in test_groups:
X = group.values
seq_len = X.shape[0]
X_test[i,:seq_len,:] = X[:,1:23]
i += 1
del X
del test_groups
X_test.shape
y_pred_rfc = rfc_model.predict(X_test )<predict_on_test>
submission = pd.read_csv(".. /input/how-much-did-it-rain-ii/sample_solution.csv.zip")
 submission
<train_on_grid><EOS> 
model.load_weights("model.h5")
 predictions = model.predict(X_test, batch_size=32)
 submission["Expected"] = predictions
submission.to_csv("submission.csv", index=False )