Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<compute_test_metric>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_sample = pd.read_csv('.. /input/sample_submission.csv')
print(f'Local CV is {accuracy_score(y_oof, y_train):.4f}' )<save_to_csv>
print(len(df_train.columns))
new_cont_ord_cols = [c for c in df_train.columns if not c.endswith('bin')]
no_bin_cat_cols = [c for c in new_cont_ord_cols if not c.endswith('cat')][2:]
y_test = np.argmax(y_test, axis=1)
 sub['surface'] = y_test
sub['surface'] = sub['surface'].map(decode_dic)
 sub.head()
sub.to_csv('submission.csv', index=False )<set_options>
gc.collect()<set_options>
%matplotlib inline
le = preprocessing.LabelEncoder()
warnings.filterwarnings('ignore')
 gc.enable() 
<define_variables>
target = df_train['target']
PATH = ".. /input/career-con-2019"<load_from_csv>
print(df_train.dtypes.unique())
 print(df_train.dtypes.nunique())
 print(df_test.dtypes.unique())
 print(df_test.dtypes.nunique() )
testing = True
X_train = pd.read_csv(PATH+'/X_train.csv')
 y_train = pd.read_csv(PATH+'/y_train.csv')
 X_train = pd.merge(X_train,y_train,on='series_id')
 X_train['surface'] = le.fit_transform(X_train['surface'])
 if(testing):
X_test = pd.read_csv(PATH+'/X_test.csv')
 X_test['series_id'] = X_test['series_id']+3810
X_test['group_id'] = 0
X_test['surface'] = 0
frames = [X_train,X_test]
X_train = pd.concat(frames)
 X_train.reset_index(drop=True,inplace=True )<drop_column>
print(df_train.isnull().values.any())
 print(df_test.isnull().values.any() )
cols = list(X_train.columns.values)
 cols.remove('orientation_W')
 cols.insert(3,'orientation_W')
 X_train = X_train[cols]
print("New train data")
 X_train.head()<count_unique_values>
df_train[(df_train == -1)] = np.nan
df_test[(df_test == -1)] = np.nan
print('done' )
num_meas = 128
num_series = X_train['series_id'].nunique()
print("Number of measurements = ", num_meas)
 print("Number of total sereis = ", num_series )<prepare_x_and_y>
print(df_train.isnull().values.any())
 print(df_test.isnull().values.any() )
def q_to_angle(q_val):
roll = np.arctan2(2*(q_val[0]*q_val[1] + q_val[2]*q_val[3]),1 - 2*(q_val[1]*q_val[1] + q_val[2]*q_val[2]))
pitch = np.arcsin(2*(q_val[0]*q_val[2] - q_val[3]*q_val[1]))
yaw = np.arctan2(2*(q_val[0]*q_val[3] + q_val[1]*q_val[2]),1 - 2*(q_val[2]*q_val[2] + q_val[3]*q_val[3]))
return np.array([roll, pitch, yaw] )<feature_engineering>
cols_missing_val_train = df_train.columns[df_train.isnull().any() ].tolist()
print(cols_missing_val_train)
 print('
')
 cols_missing_val_test = df_test.columns[df_test.isnull().any() ].tolist()
print(cols_missing_val_test )
quat_arr = np.array(X_train[['orientation_W','orientation_X','orientation_Y','orientation_Z']])
 euler_arr = np.zeros([quat_arr.shape[0],3])
 for n,arr in enumerate(quat_arr):
euler_arr[n] = q_to_angle(arr )<prepare_x_and_y>
df_train.replace(np.nan, -1, inplace=True)
 df_test.replace(np.nan, -1, inplace=True )
X_train['roll'] = euler_arr[:,0]
X_train['pitch'] = euler_arr[:,1]
X_train['yaw'] = euler_arr[:,2]<drop_column>
def change_datatype(df):
float_cols = list(df.select_dtypes(include=['int'] ).columns)
 for col in float_cols:
if(( np.max(df[col])<= 127)and(np.min(df[col] >= -128))):
df[col] = df[col].astype(np.int8)
 elif(( np.max(df[col])<= 32767)and(np.min(df[col] >= -32768))):
df[col] = df[col].astype(np.int16)
 elif(( np.max(df[col])<= 2147483647)and(np.min(df[col] >= -2147483648))):
df[col] = df[col].astype(np.int32)
 else:
df[col] = df[col].astype(np.int64)
 change_datatype(df_train)
 change_datatype(df_test )
cols = list(X_train.columns.values)
 cols.remove('group_id')
 cols.append('group_id')
 cols.remove('surface')
 cols.append('surface')
 X_train = X_train[cols]
print("Train updated")
 X_train.head()<prepare_x_and_y>
def change_datatype_float(df):
float_cols = list(df.select_dtypes(include=['float'] ).columns)
 for col in float_cols:
df[col] = df[col].astype(np.float32)
 change_datatype_float(df_train)
 change_datatype_float(df_test )
feat_cols = ['roll','pitch','yaw','angular_velocity_X','angular_velocity_Y','angular_velocity_Z','linear_acceleration_X','linear_acceleration_Y','linear_acceleration_Z']
feat_array = np.array(X_train[feat_cols])
 feat_array = np.reshape(feat_array,[num_series,128,len(feat_cols)])
 group_array = np.array(X_train['group_id'])
 group_array = np.reshape(group_array,[num_series,128])
 group_array = group_array[:,0]
target_array = np.array(X_train['surface'])
 target_array = np.reshape(target_array,[num_series,128])
 target_array = target_array[:,0]<feature_engineering>
len(set(df_test.columns)and set(df_train.columns))
delta_cols = ['roll','pitch','yaw']
for dc in delta_cols:
iia = feat_cols.index(dc)
 np_arr = feat_array[:,:,iia]
roll_arr = np.copy(np_arr)
 roll_arr[:,1:] = roll_arr[:,:-1]
np_arr = np_arr - roll_arr
feat_array[:,:,iia] = np_arr<feature_engineering>
df_train = df_train.replace(-1, np.NaN)
 d_median = df_train.median(axis=0)
 d_mean = df_train.mean(axis=0)
 df_train = df_train.fillna(-1)
 dcol = [c for c in df_train.columns if c not in ['id','target']]
df_train['ps_car_13_x_ps_reg_03'] = df_train['ps_car_13'] * df_train['ps_reg_03']
for c in dcol:
if '_bin' not in c:
df_train[c+str('_median_range')] =(df_train[c].values > d_median[c] ).astype(np.int)
 df_train[c+str('_mean_range')] =(df_train[c].values > d_mean[c] ).astype(np.int)
 df_train[c+str('_sq')] = np.power(df_train[c].values,2 ).astype(np.float32)
 df_train[c+str('_log')] = np.log(np.abs(df_train[c].values)+ 1)
 df_train[c+str('_exp')] = np.exp(df_train[c].values)- 1
norm_cols = ['linear_acceleration_X','linear_acceleration_Y','linear_acceleration_Z','angular_velocity_X','angular_velocity_Y','angular_velocity_Z']
for norm in norm_cols:
iia = feat_cols.index(norm)
 np_arr = feat_array[:,:,iia]
mean_arr = np.mean(np_arr,1)
 mean_arr = np.expand_dims(mean_arr,1)
 mean_arr = np.repeat(mean_arr,num_meas,1)
 np_arr = np_arr - mean_arr
feat_array[:,:,iia] = np_arr 
<concatenate>
change_datatype(df_train )
def absfft(x):
return np.abs(np.fft.rfft(x))
feat_fft_array = np.copy(feat_array[:,:,3:])
 feat_fft_array = np.apply_along_axis(absfft,1,feat_fft_array )<feature_engineering>
features= [c for c in df_train.columns.values if c not in ['id', 'target']]
X_train, X_test, y_train, y_test = train_test_split(df_train[features], df_train['target'], test_size=0.33, random_state=42)
 X_train.head()
num_sensor = feat_array.shape[2]
for i in range(num_sensor):
mean_s = np.mean(feat_array[:,:,i])
 sd_s = np.std(feat_array[:,:,i])
 feat_array[:,:,i] =(feat_array[:,:,i]-mean_s)/sd_s
num_sensor_fft = feat_fft_array.shape[2]
for i in range(num_sensor_fft):
mean_s = np.mean(feat_fft_array[:,:,i])
 sd_s = np.std(feat_fft_array[:,:,i])
 feat_fft_array[:,:,i] =(feat_fft_array[:,:,i]-mean_s)/sd_s<import_modules>
pipeline = Pipeline([ 
('classifier', RandomForestClassifier(random_state = 42))
])
 pipeline.fit(X_train, y_train)
 preds = pipeline.predict(X_test)
 np.mean(preds == y_test )
from keras.layers import Input,Dense, Dropout, BatchNormalization, SeparableConv1D, Reshape, LSTM, DepthwiseConv2D,AveragePooling2D, CuDNNLSTM, Concatenate
from keras.models import Model
from keras.backend import squeeze
from keras.regularizers import l2<define_search_model>
hyperparameters = {
'classifier__n_estimators': [20, 30, 50],
'classifier__max_depth': [2, 4],
'classifier__min_samples_leaf': [2, 4]
}
clf = GridSearchCV(pipeline, hyperparameters, cv = 3)
 clf.fit(X_train, y_train )
kr = None
num_groups = np.unique(group_array ).shape[0]
num_surfaces = np.unique(target_array ).shape[0]
def get_net_with_fft_mag_only(dp):
inputs_t = Input(shape=(128,len(feat_cols)))
 x = SeparableConv1D(32,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(inputs_t)
 x = Dropout(dp )(x)
 x = SeparableConv1D(64,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = SeparableConv1D(256,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Reshape(( 256,))(x)
 x = Dropout(dp )(x)
 x = Dense(64, activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = Dense(64, activation='relu' )(x)
 inputs_f = Input(shape=(feat_fft_array.shape[1],feat_fft_array.shape[2]))
y = SeparableConv1D(32,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(inputs_f)
 y = Dropout(dp )(y)
 y = SeparableConv1D(64,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(256,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Reshape(( 256,))(y)
 y = Dropout(dp )(y)
 y = Dense(64, activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = Dense(64, activation='relu' )(y)
 inputs = [inputs_t,inputs_f]
z = Concatenate()([x,y])
 z = Dense(64, activation='relu' )(z)
 predictions = Dense(num_surfaces, activation='softmax' )(z)
 model = Model(inputs=inputs, outputs=predictions)
 model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return model<split>
clf.best_params_
depthwise = False
fft_net = True
if(not(testing)) :
gkf = GroupKFold(3)
 train_gen = gkf.split(X=feat_array,groups=group_array)
 preds = np.zeros_like(target_array)
 for train_idx,test_idx in train_gen:
t_feats = feat_array[train_idx]
t_feats_fft = feat_fft_array[train_idx]
v_feats = feat_array[test_idx]
v_feats_fft = feat_fft_array[test_idx]
t_vals = target_array[train_idx]
v_vals = target_array[test_idx]
pred_classes = np.zeros([v_vals.shape[0],num_surfaces,5])
 for k in range(5):
nnet = get_net_with_fft_mag_only(0.5)
 nnet.fit(x=[t_feats,t_feats_fft],y=t_vals,batch_size=256,epochs=3000,validation_data=([v_feats,v_feats_fft],v_vals),verbose=2)
 pred_classes[:,:,k] = nnet.predict([v_feats,v_feats_fft])
 pred_classes = np.mean(pred_classes,axis=2)
 pred_classes = np.argmax(pred_classes,axis=1)
 preds[test_idx] = pred_classes
print('Val accuracy: ',accuracy_score(v_vals,pred_classes))
pred_classes = nnet.predict([t_feats,t_feats_fft])
 pred_classes = np.argmax(pred_classes,axis=1)
 print('Train accuracy: ',accuracy_score(t_vals,pred_classes))
print('5 Fold accuracy: ', accuracy_score(target_array,preds))
else:
t_feats = feat_array[:3810]
t_feats_fft = feat_fft_array[:3810]
t_vals = target_array[:3810]
v_feats = feat_array[3810:]
v_feats_fft = feat_fft_array[3810:]
pred_classes = np.zeros([v_feats.shape[0],num_surfaces,3])
 for k in range(3):
nnet = get_net_with_fft_mag_only(0.5)
 nnet.fit(x=[t_feats,t_feats_fft],y=t_vals,batch_size=256,epochs=3000,verbose=0)
 pred_classes[:,:,k] = nnet.predict([v_feats,v_feats_fft])
 pred_classes = np.mean(pred_classes,axis=2)
 pred_classes = list(np.argmax(pred_classes,axis=1))
pred_classes = [le.inverse_transform([i])[0] for i in pred_classes]
sub_df = pd.read_csv(PATH+'/sample_submission.csv')
 sub_df['surface'] = pred_classes
sub_df.to_csv('submission.csv',index=False )<set_options>
clf.refit
preds = clf.predict(X_test)
 probs = clf.predict_proba(X_test)
 np.mean(preds == y_test )
gc.collect()<import_modules>
df_test = df_test.replace(-1, np.NaN)
 dt_median = df_test.median(axis=0)
 dt_mean = df_test.mean(axis=0)
 df_test = df_test.fillna(-1)
 dtcol = [c for c in df_test.columns if c not in ['id']]
df_test['ps_car_13_x_ps_reg_03'] = df_test['ps_car_13'] * df_test['ps_reg_03']
for c in dtcol:
if '_bin' not in c:
df_test[c+str('_median_range')] =(df_test[c].values > dt_median[c] ).astype(np.int)
 df_test[c+str('_mean_range')] =(df_test[c].values > dt_mean[c] ).astype(np.int)
 df_test[c+str('_sq')] = np.power(df_test[c].values,2 ).astype(np.float32)
 df_test[c+str('_log')] = np.log(np.abs(df_test[c].values)+ 1)
 df_test[c+str('_exp')] = np.exp(df_test[c].values)- 1
from collections import Counter
from sklearn.neighbors import KNeighborsClassifier
from keras.layers import LSTM,Input,Dense,Flatten,SpatialDropout1D,Dropout,CuDNNLSTM,Reshape,Concatenate
from keras.layers import Lambda,concatenate,BatchNormalization
from keras.models import Model
from keras.utils import to_categorical
from keras.optimizers import Adam
from keras import backend as K
from sklearn.preprocessing import LabelEncoder<load_from_csv>
change_datatype(df_test )
directory = ".. /input/career-con-2019/"
X_test_path = os.path.join(directory,"X_test.csv")
 X_train_path = os.path.join(directory,"X_train.csv")
 X_test_data = pd.read_csv(X_test_path)
 X_train_data = pd.read_csv(X_train_path)
 y_train_data = pd.read_csv(os.path.join(directory,"y_train.csv"))
sample_submission = pd.read_csv(os.path.join(directory,"sample_submission.csv"))<train_model>
submission = pd.read_csv('.. /input/test.csv')
 test_features= [c for c in df_test.columns.values if c not in ['id']]
predictions = clf.predict_proba(df_test[test_features])
 preds = pd.DataFrame(data = predictions, columns = clf.best_estimator_.named_steps['classifier'].classes_)
 result = pd.concat([submission[['id']], preds], axis=1)
 result = result.drop(0, axis=1)
 result.columns = ['id', 'target']
result.head()
result.to_csv('random_forest.csv', index=False )
print("Train data: measurements = %d , features=%d" %X_train_data.shape)
 X_train_data.head()<categorify>
le = LabelEncoder()
le.fit(list(y_train_data["surface"]))
y_train_dataset_for_nn = to_categorical(le.transform(list(y_train_data["surface"])) )<categorify>
train_cols = df_train.columns.tolist()
bin_cols = []
cat_cols = []
cont_ord_cols = []
ind_cols = []
reg_cols = []
car_cols = []
calc_cols = []
for col in train_cols:
if(( 'ps' in str(col)) &('bin' not in str(col)) &('cat' not in str(col))):
cont_ord_cols.append(col)
 for col in train_cols:
if('bin' in str(col)) :
bin_cols.append(col)
 if('cat' in str(col)) :
cat_cols.append(col)
 if('ind' in str(col)) :
ind_cols.append(col)
 if('reg' in str(col)) :
reg_cols.append(col)
 if('car' in str(col)) :
car_cols.append(col)
 if('calc' in str(col)) :
calc_cols.append(col)
def dataset_for_nn(X_dataset):
num_samples = X_dataset.shape[0]//128
X_dataset_for_nn = np.zeros(( num_samples,128,10))
for i in range(num_samples):
subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 X_dataset_for_nn[i,:,:] = subset
return X_dataset_for_nn<categorify>
float_cols = []
int_cols = []
for col in cont_ord_cols:
if(df_train[col].dtype == np.float32):
float_cols.append(col)
 elif(( df_train[col].dtype == np.int8)or(df_train[col].dtype == np.int16)) :
int_cols.append(col )
X_train_for_nn = dataset_for_nn(X_train_data)
 X_test_for_nn = dataset_for_nn(X_test_data )<train_model>
for col in int_cols:
print(df_train[col].nunique() )
print("Train data for NN")
 X_train_for_nn[0]<feature_engineering>
for col in cols_to_delete:
df_train.drop([col], axis=1, inplace=True)
 df_test.drop([col], axis=1, inplace=True)
def freqs(dataset,width):
X = np.abs(fftpack.fft(dataset))
squeezed_dataset = []
for i in range(64//width):
squeezed_dataset.append(np.mean(X[i*width:(i+1)*width]))
return squeezed_dataset<feature_engineering>
ind_cols_no_bin_cat = []
reg_cols_no_bin_cat = []
car_cols_no_bin_cat = []
calc_cols_no_bin_cat = []
for col in train_cols:
if(( 'ind' in str(col)) and('bin' not in str(col)) and('cat' not in str(col))):
ind_cols_no_bin_cat.append(col)
 if(( 'reg' in str(col)) and('bin' not in str(col)) and('cat' not in str(col))):
reg_cols_no_bin_cat.append(col)
 if(( 'car' in str(col)) and('bin' not in str(col)) and('cat' not in str(col))):
car_cols_no_bin_cat.append(col)
 if(( 'calc' in str(col)) and('bin' not in str(col)) and('cat' not in str(col))):
calc_cols_no_bin_cat.append(col )
def X_features(X_dataset,width=3):
num_samples = len(list(set(X_dataset["series_id"])))
 num_cols = 64//width
features = np.zeros(( num_samples,40+10*num_cols))
for i in range(num_samples):
X_train_subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 features[i,:10] = np.mean(X_train_subset,axis=0)
 features[i,10:20] = np.std(X_train_subset,axis=0)
 features[i,20:30] = np.max(X_train_subset,axis=0)-np.min(X_train_subset,axis=0)
 features[i,30:40] = X_train_subset[-1,:]-X_train_subset[0,:]
for j in range(X_train_subset.shape[1]):
features[i,40+j*num_cols:40+(j+1)*num_cols] = freqs(X_train_subset[:,j],width)
 return features<feature_engineering>
train_cols = df_train.columns
bin_cols = df_train.columns[df_train.columns.str.endswith('bin')]
X_train_features = X_features(X_train_data)
 X_test_features = X_features(X_test_data )<choose_model_class>
def LSTM_NN(drop):
inp = Input(shape=(128,10))
x = SpatialDropout1D(0.1 )(inp)
 inp_2 = Input(shape=(250,))
x_2 = Dense(250, input_shape=(250,), activation="sigmoid" )(inp_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(120, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(60, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = BatchNormalization()(x_2)
 x = CuDNNLSTM(units=200, return_sequences=True, return_state=False, go_backwards=False )(x)
 x = Dropout(drop )(x)
 x = CuDNNLSTM(units=100, return_sequences=False, return_state=False, go_backwards=False )(x)
 x = concatenate([x,x_2])
 x = Dropout(drop )(x)
 outp = Dense(9, activation="sigmoid" )(x)
 model = Model(inputs=[inp,inp_2], outputs=outp)
 model.compile(loss='categorical_crossentropy', optimizer=Adam())
 return model<split>
%%time
n_splits=5
kfold = StratifiedKFold(n_splits=n_splits, random_state=10, shuffle=True)
 y_test = np.zeros(( X_test_for_nn.shape[0],9*n_splits))
train_preds = np.zeros(( X_train_for_nn.shape[0],9))
X = X_train_for_nn
X_test = X_test_for_nn
Y = np.array(list(y_train_data["surface"]))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val = X[list(train_index),:,:],X[list(valid_index),:,:]
X_train_feat, X_val_feat = X_train_features[list(train_index),:],X_train_features[list(valid_index),:]
Y_train, Y_val = Y[list(train_index)], Y[list(valid_index)]
Y_train = to_categorical(le.transform(Y_train))
Y_val = to_categorical(le.transform(Y_val))
model = LSTM_NN(0.15)
 model.fit([X_train,X_train_feat], Y_train, epochs=120, validation_data=([X_val,X_val_feat], Y_val), verbose=2)
 y_pred = model.predict([X_val,X_val_feat], verbose=2)
 y_test[:,i*9:(i+1)*9] = model.predict([X_test,X_test_features])
 train_preds[list(valid_index),:] = np.squeeze(y_pred )<compute_test_metric>
res_train = np.argmax(train_preds,axis=1)
 ans_train = np.argmax(to_categorical(le.transform(Y)) ,axis=1)
 print("CV score",round(accuracy_score(res_train,ans_train),4)) 
<feature_engineering>
df_train.replace(np.nan, -1, inplace=True)
 df_test.replace(np.nan, -1, inplace=True)
 print('Done' )
def most_frequent(List):
occurence_count = Counter(List)
 return occurence_count.most_common(1)[0][0]
res_test_inter = np.zeros(( y_test.shape[0],n_splits))
res_test=[]
for i in range(n_splits):
inter_arr = y_test[:,i*9:(i+1)*9]
res_test_inter[:,i] = np.argmax(inter_arr,axis=1)
 for j in range(y_test.shape[0]):
res_test.append(int(most_frequent(res_test_inter[j,:])) )<save_to_csv>
test_for_sub=le.inverse_transform(res_test)
 print(test_for_sub[:5])
 test_size = len(list(set(X_test_data["series_id"])))
 Y_test_pred_array = np.zeros(( test_size,2))
Y_test_for_submission = pd.DataFrame(Y_test_pred_array,columns = ["series_id","surface"])
 Y_test_for_submission.iloc[:,0] = list(range(test_size))
Y_test_for_submission.iloc[:,1] = test_for_sub
Y_test_for_submission.to_csv("submission.csv",index=None)
 Y_test_for_submission.head()<set_options>
gc.collect()<load_from_csv>
data_target = pd.read_csv(PATH+"/y_train.csv")
 train_X = pd.read_csv(PATH+'/X_train.csv' ).iloc[:,3:]
test_X = pd.read_csv(PATH+'/X_test.csv' ).iloc[:,3:]<concatenate>
train_X = train_X.append(test_X)
 train_X = train_X.values.reshape(-1,128,10 )<find_best_params>
import xgboost as xgb
train_left_edges, train_left_linked = find_run_edges(train_X, edge='left')
 train_right_edges, train_right_linked = find_run_edges(train_X, edge='right')
 print('Found', len(train_left_edges), 'left edges and', len(train_right_edges), 'right edges.' )<compute_test_metric>
def gini(actual, pred, cmpcol = 0, sortcol = 1):
assert(len(actual)== len(pred))
all = np.asarray(np.c_[ actual, pred, np.arange(len(actual)) ], dtype=np.float)
 all = all[ np.lexsort(( all[:,2], -1*all[:,1])) ]
totalLosses = all[:,0].sum()
giniSum = all[:,0].cumsum().sum() / totalLosses
giniSum -=(len(actual)+ 1)/ 2.
 return giniSum / len(actual)
 def gini_normalized(a, p):
return gini(a, p)/ gini(a, a)
 def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = gini_normalized(labels, preds)
 return 'gini', gini_score
train_runs = find_runs(train_X, train_left_edges, train_right_edges )<define_variables>
params = {
'min_child_weight': 10.0,
'objective': 'binary:logistic',
'max_depth': 7,
'max_delta_step': 1.8,
'colsample_bytree': 0.4,
'subsample': 0.8,
'eta': 0.025,
'gamma': 0.65,
'num_boost_round' : 700
}
lost_samples = np.array([ i for i in range(len(train_X)) if i not in np.concatenate(train_runs)])
 print(lost_samples)
 print(len(lost_samples)) 
<define_variables>
<drop_column><EOS>