Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<compute_test_metric>
print(classification_report(flat_predictions,flat_true_labels))<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
comments1 = df_test.text.values
indices1=tokenizer.batch_encode_plus(comments1,max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
 input_ids1=indices1["input_ids"]
attention_masks1=indices1["attention_mask"]
prediction_inputs1= torch.tensor(input_ids1)
 prediction_masks1 = torch.tensor(attention_masks1)
 batch_size = 32
prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
 prediction_sampler1 = SequentialSampler(prediction_data1)
 prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
<predict_on_test>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
 model.eval()
predictions = []
for batch in prediction_dataloader1:
batch = tuple(t.to(device)for t in batch)
 b_input_ids1, b_input_mask1 = batch
with torch.no_grad() :
outputs1 = model(b_input_ids1, token_type_ids=None,
attention_mask=b_input_mask1)
 logits1 = outputs1[0]
logits1 = logits1.detach().cpu().numpy()
predictions.append(logits1)
 flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()<save_to_csv>
all_features = [c for c in test.columns if c not in ['ID']]
def has_ugly(row):
for v in row.values[row.values > 0]:
if str(v)[::-1].find('.')> 2:
return True
return False
test['has_ugly'] = test[all_features].apply(has_ugly, axis=1)
 test_og = test[['ID']].copy()
test_og['nonzero_mean'] = test[[c for c in test.columns if c not in ['ID', 'has_ugly']]].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 test = test[test.has_ugly == False]
sample_sub=pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submit=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':flat_predictions})
<data_type_conversions>
train_t = train.drop(['target'], axis = 1, inplace=False)
 train_t.set_index('ID', inplace=True)
 train_t = train_t.T
test_t = test.set_index('ID', inplace=False)
 test_t = test_t.T
df_leak = pd.read_csv('/kaggle/input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv', encoding ='ISO-8859-1')[['choose_one', 'text']]
df_leak['target'] =(df_leak['choose_one'] == 'Relevant' ).astype(np.int8)
 df_leak['id'] = df_leak.index.astype(np.int16)
 df_leak.drop(columns=['choose_one', 'text'], inplace=True)
 df_test = df_test.merge(df_leak, on=['id'], how='left')
 print('Leaked Data Set Shape = {}'.format(df_leak.shape))
print('Leaked Data Set Memory Usage = {:.2f} MB'.format(df_leak.memory_usage().sum() / 1024**2))
perfect_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 perfect_submission['target'] = df_test['target'].values
perfect_submission.to_csv('submission.csv', index=False)
 perfect_submission.describe()<load_from_csv>
gc.collect()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape)
 print(test.shape)
 test_ids = test.id
levels=train.species
train.drop(['species', 'id'], axis=1,inplace=True)
 test.drop(['id'],axis=1,inplace=True)
 print("after ")
 print(levels.shape)
 print(train.shape)
 print(test.shape)
 print("number of classes =",levels.unique().shape )<categorify>
features = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1',
'15ace8c9f', 'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9',
'd6bb78916', 'b43a7cfd5', '58232a6fb', '1702b5bf0', '324921c7b',
'62e59a501', '2ec5b290f', '241f0f867', 'fb49e4212', '66ace2992',
'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7', '1931ccfdd',
'703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2',
'190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98']
le=LabelEncoder().fit(levels)
 levels=le.transform(levels )<train_on_grid>
extra_features = []
Model=RandomForestClassifier(n_estimators=1000)
 Model = cc(Model, cv=3, method='isotonic')
 Model.fit(train, levels )<save_to_csv>
def find_new_ordered_features(ordered_ids, data_t):
data = data_t.copy()
f1 = ordered_ids[0][:-1]
f2 = ordered_ids[0][1:]
for ef in ordered_ids[1:]:
f1 += ef[:-1]
f2 += ef[1:]
d1 = data[f1].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d1['ID'] = data.index
gc.collect()
d2 = data[f2].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2['ID'] = data.index
gc.collect()
d3 = d2[~d2.duplicated(['key'], keep=False)]
d4 = d1[~d1.duplicated(['key'], keep=False)]
d5 = d4.merge(d3, how='inner', on='key')
 d_feat = d1.merge(d5, how='left', on='key')
 d_feat.fillna(0, inplace=True)
 ordered_features = list(d_feat[['ID_x', 'ID_y']][d_feat.ID_x != 0].apply(list, axis=1))
del d1,d2,d3,d4,d5,d_feat
gc.collect()
links_found = 1
while links_found > 0:
links_found, ordered_features = chain_pairs(ordered_features)
 ordered_features = prune_chain(ordered_features)
 found = {}
for ef in extra_features:
found[ef[0]] = ef
found [features[0]] = features
new_feature_sets = []
for of in ordered_features:
if len(of)>= 40:
if of[0] not in found:
new_feature_sets.append(of)
 return new_feature_sets
predictions = Model.predict_proba(test)
 print(predictions.shape)
 sub = pd.DataFrame(predictions, columns=list(le.classes_))
sub.insert(0, 'id', test_ids)
 sub.reset_index()
sub.to_csv('submit.csv', index = False)
 sub.head()<set_options>
def add_new_feature_sets(data, data_t):
print('
Data Shape:', data.shape)
 f1 = features[:-1]
f2 = features[1:]
for ef in extra_features:
f1 += ef[:-1]
f2 += ef[1:]
d1 = data[f1].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d1['ID'] = data['ID']
gc.collect()
d2 = data[f2].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2['ID'] = data['ID']
gc.collect()
d3 = d2[~d2.duplicated(['key'], keep=False)]
del d2
d4 = d1[~d1.duplicated(['key'], keep=False)]
d5 = d4.merge(d3, how='inner', on='key')
 del d4
d = d1.merge(d5, how='left', on='key')
 d.fillna(0, inplace=True)
 ordered_ids = list(d[['ID_x', 'ID_y']][d.ID_x != 0].apply(list, axis=1))
del d1,d3,d5,d
gc.collect()
links_found = 1
while links_found > 0:
links_found, ordered_ids = chain_pairs(ordered_ids)
 print('OrderedIds:', len(ordered_ids))
ordered_ids = prune_chain(ordered_ids)
 print('OrderedIds Pruned:', len(ordered_ids))
new_feature_sets = find_new_ordered_features(ordered_ids, data_t)
 extra_features.extend(new_feature_sets)
 print('New Feature Count:', len(new_feature_sets))
print('Extra Feature Count:', len(extra_features))
%matplotlib inline 
<load_from_csv>
%%time
add_new_feature_sets(train,train_t)
 add_new_feature_sets(test,test_t)
 add_new_feature_sets(train,train_t)
 add_new_feature_sets(test,test_t)
 add_new_feature_sets(train,train_t )
train_df = pd.read_csv('.. /input/train.csv')
 train_df.info()
copy_df = train_df.copy()<data_type_conversions>
with open("extra_features_{}.txt".format(len(extra_features)) , "w")as text_file:
for ef in extra_features:
text_file.write(','.join(ef)+ '
' )
species = train_df['species'].unique()
species.sort()
spe_dict = dict(enumerate(species))
inv_spe_dict = {v: k for k,v in spe_dict.items() }
train_df['species_index'] = train_df['species'].map(inv_spe_dict ).astype(int)
 train_df = train_df.drop(['species','id'],axis = 1 )<create_dataframe>
del train_t, test_t, test
gc.collect()
train_data = train_df.values<randomize_order>
test = pd.read_csv('.. /input/test.csv')
 test['has_ugly'] = test[all_features].apply(has_ugly, axis=1)
 test[test.has_ugly == True] = 0
a = np.random.permutation(train_data)
 b = np.random.permutation(train_data)
 train_data = np.vstack(( a,b))
train_data.shape<normalization>
def get_log_pred(data, feats, extra_feats, offset = 2):
f1 = feats[:(offset * -1)]
f2 = feats[offset:]
for ef in extra_feats:
f1 += ef[:(offset * -1)]
f2 += ef[offset:]
d1 = data[f1].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2 = data[f2].apply(tuple, axis=1 ).apply(hash ).to_frame().rename(columns={0: 'key'})
 d2['pred'] = data[feats[offset-2]]
d2 = d2[d2['pred'] != 0]
d3 = d2[~d2.duplicated(['key'], keep=False)]
d4 = d1[~d1.duplicated(['key'], keep=False)]
d5 = d4.merge(d3, how='inner', on='key')
 d = d1.merge(d5, how='left', on='key')
 return np.log1p(d.pred ).fillna(0)
X,y = train_data[:,:-1],train_data[:,-1]
scaler = StandardScaler().fit(X)
 X = scaler.transform(X )<train_model>
end_offset = 39
pred_test = []
pred_train = []
efs = extra_features
for o in tqdm_notebook(list(range(2, end_offset))):
print('Offset:', o)
 log_pred = get_log_pred(train, features, extra_features, o)
 pred_train.append(np.expm1(log_pred))
have_data = log_pred != 0
train_count = have_data.sum()
score = sqrt(mean_squared_error(np.log1p(train.target[have_data]), log_pred[have_data]))
print(f'Score = {score} on {have_data.sum() } out of {train.shape[0]} training samples')
 log_pred_test = get_log_pred(test, features, efs, o)
 pred_test.append(np.expm1(log_pred_test))
have_data = log_pred_test != 0
test_count = have_data.sum()
print(f'Have predictions for {have_data.sum() } out of {test.shape[0]} test samples')
model = MLPClassifier(hidden_layer_sizes=(150,),activation='logistic',solver='lbfgs',alpha=0.001
,max_iter=200,early_stopping=True,validation_fraction=0.2,
learning_rate='adaptive',tol=1e-8,random_state=1 ).fit(X,y )<compute_train_metric>
pred_train_final = pred_train[0].copy()
for r in range(1,len(pred_train)) :
pred_train_final[pred_train_final == 0] = pred_train[r][pred_train_final == 0]
train_leak_match_count =(pred_train_final!=0 ).sum() ;
no_match_count =(pred_train_final==0 ).sum() ;
print("Train leak count: ", train_leak_match_count, "Train no leak count: ", no_match_count)
 pred_train_temp = pred_train_final.copy()
train["nonzero_mean"] = train[[f for f in train.columns if f not in ["ID", "target","nonzero_mean"]]].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 pred_train_temp[pred_train_temp==0] = train['nonzero_mean'][pred_train_temp==0]
print(f'Baseline Train Score = {sqrt(mean_squared_error(np.log1p(train.target), np.log1p(pred_train_temp)))}' )
def cv(a,b,model):
cv_X = train_data[a:b,:-1]
cv_X = scaler.transform(cv_X)
 cv_y = train_data[a:b,-1]
cv_p1_y = model.predict(cv_X)
 cv_p2_y = model.predict_log_proba(cv_X)
 print(accuracy_score(cv_y,cv_p1_y))
print(cv_y)
 print(cv_p1_y)
<compute_train_metric>
pred_test_final = pred_test[0].copy()
for r in range(1,len(pred_test)) :
pred_test_final[pred_test_final == 0] = pred_test[r][pred_test_final == 0]
d = np.random.permutation(train_df.values)
 Xp = d[:,:-1]
Xp = scaler.transform(Xp)
 yp = d[:,-1]
ypp = model.predict(Xp)
 print(accuracy_score(yp,ypp)) 
<load_from_csv>
pred_test_final[(4e+07 < pred_test_final)] = 4e+07
pred_test_final[(( pred_test_final < 29000)&(pred_test_final > 0)) ] = 30000
pred_test_final[test.ID == 'd72fad286'] = 1560000
pred_test_final[test.ID == 'a304cde42'] = 320000.0
test_leak_match_count =(pred_test_final!=0 ).sum() ;
no_match_count =(pred_test_final==0 ).sum() ;
print("Test leak count: ", test_leak_match_count, "Test no leak count: ", no_match_count )
test_df = pd.read_csv('.. /input/test.csv')
 index = test_df.pop('id')
 test_data = test_df.values
test_X = test_data
test_X = scaler.transform(test_X )<predict_on_test>
pred_test_temp = pred_test_final.copy()
test_og["nonzero_mean"] = test_og[[f for f in test_og.columns if f not in ["ID", "target", "nonzero_mean", "has_ugly"]]].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 pred_test_temp[pred_test_temp==0] = test_og['nonzero_mean'][pred_test_temp==0]
test_og['target']=pred_test_temp
test_og[['ID', 'target']].to_csv('leak_baseline_{}.csv'.format(test_leak_match_count), index=False)
predict = model.predict(test_X)
 predict_proba = model.predict_proba(test_X)
 predict_proba<create_dataframe>
test_leaks = pd.read_csv('.. /input/sample_submission.csv')
 del test_leaks['target']
test_leaks['target']=pred_test_final
test_leaks.to_csv('leak_only_{}.csv'.format(test_leak_match_count), index=False)
species_list = species.tolist()
result = pd.DataFrame(predict_proba,index = index, columns = species_list)
 result<save_to_csv>
extra_features_list = []
for ef in extra_features:
extra_features_list.extend(ef)
 extra_features_list.extend(features)
 len(extra_features_list )
fp = open('submission.csv','w')
 fp.write(result.to_csv() )<load_from_csv>
feats = pd.DataFrame(extra_features)
 time_features = []
for c in feats.columns[:]:
time_features.append([f for f in feats[c].values if f is not None])
 agg_features = []
all_cols = train.columns.drop(['ID', 'target', 'nonzero_mean'])
 agg_features.append(all_cols)
 agg_features.append([c for c in all_cols if c not in extra_features_list])
 agg_features.append(extra_features_list)
 agg_features.extend(time_features)
 agg_features.extend(extra_features)
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()<save_to_csv>
def add_new_features(source, dest, feats):
dest['mean_log_{}_{}'.format(feats[0], len(feats)) ] = np.log1p(source[feats].replace(0, np.nan ).mean(axis=1))
dest['first_nonZero_{}_{}'.format(feats[0], len(feats)) ] = np.log1p(source[feats].replace(0, np.nan ).bfill(1 ).iloc[:, 0])
 dest['last_nonZero_{}_{}'.format(feats[0], len(feats)) ] = np.log1p(source[feats[::-1]].replace(0, np.nan ).bfill(1 ).iloc[:, 0])
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()<load_from_csv>
del test
gc.collect
test = pd.read_csv('.. /input/test.csv' )
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()<define_variables>
train_feats = pd.DataFrame()
test_feats =pd.DataFrame()
for i, ef in tqdm_notebook(list(enumerate(agg_features))):
add_new_features(train, train_feats, ef)
 add_new_features(test, test_feats, ef )
print('Total number of question pairs for testing: {}'.format(len(df_test)) )<feature_engineering>
class MyModel() :
def __init__(self, X_tr, y_tr, X_val, y_val, X_test):
self.X_tr = X_tr
self.y_tr = y_tr
self.X_val = X_val
self.y_val = y_val
self.X_test = X_test
self.params = {}
def predict_val(self):
return self.model.predict(self.X_val)
 def predict_test(self):
return self.model.predict(self.X_test)
 class LgbBoostModel(MyModel):
def train(self):
self.params = { 'objective': 'regression', 'metric': 'rmse', 'boosting': 'gbdt', 'seed':seed, 'is_training_metric': True
,'max_bin': 350
,'learning_rate':.005
,'max_depth': -1
,'num_leaves': 48
,'feature_fraction': 0.1
,'reg_alpha': 0
,'reg_lambda': 0.2
,'min_child_weight': 10}
self.model = lgb.train(self.params, lgb.Dataset(self.X_tr, label=self.y_tr), 30000,
[lgb.Dataset(self.X_tr, label=self.y_tr), lgb.Dataset(self.X_val, label=self.y_val)],
verbose_eval=200, early_stopping_rounds=200)
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))<string_transform>
cols = train_feats.columns
train_feat_final = pd.concat([train_feats[cols], test_feats[cols][test_leaks.target != 0]], axis = 0)
 train_feat_id = pd.concat([train['ID'], test['ID'][test_leaks.target != 0]], axis = 0)
 test_feat_final = test_feats[cols]
y = np.array(list(np.log1p(train.target.values)) + list(np.log1p(test_leaks['target'][test_leaks.target != 0])))
 X = train_feat_final.values
X_test = test_feat_final.values
print(X.shape)
 print(X_test.shape )
<init_hyperparams><EOS> 
def mean(values):
return float(sum(values)) / max(len(values), 1)
 def sum_of_square_deviation(values, mean):
return float(1/len(values)* sum(( x - mean)** 2 for x in values))
def export_results(model_type):
subm = pd.read_csv('.. /input/sample_submission.csv')
 subm['target'] = np.expm1(sub_preds[model_type])
 oof = pd.DataFrame(train_feat_id.copy())
 oof['target'] = np.expm1(y)
 oof['prediction'] = np.expm1(oof_preds[model_type])
 mean_rmse = mean(rmse_scores[model_type])
 standard_deviation_rmse = math.sqrt(sum_of_square_deviation(rmse_scores[model_type],mean_rmse))
key = '{}'.format(model_type)
 print('{} Mean Squared Error {}'.format(model_type ,mean_rmse))
print('{} Stdev Squared Error {}'.format(model_type, standard_deviation_rmse))
file_name = 'subm_{}_ml_base.csv'.format(key)
 subm.to_csv(file_name, index=False, float_format="%.8f")
 file_name = 'submission.csv'.format(key)
 subm['target'][test_leaks.target != 0] = test_leaks['target'][test_leaks.target != 0]
subm.to_csv(file_name, index=False, float_format="%.8f")
 file_name = 'subm_{}_oof.csv'.format(key)
 oof.to_csv(file_name, index=False, float_format="%.8f")
 model_params[model_type]['cv_score'] = int(mean_rmse * 10000)
 model_params[model_type]['cv_stddev'] = int(standard_deviation_rmse * 10000)
 model_params[model_type]['train_row_count'] = X.shape[0]
model_params[model_type]['train_feature_count'] = X.shape[1]
model_params[model_type]['test_leak_count'] =(test_leaks.target != 0 ).sum()
with open('subm_{}_params.txt'.format(key), "w")as text_file:
params = str(model_params[model_type])
 print(f"{params}", file=text_file )