Kaggle competition: DM-Assignment 1
x_train, x_test, y_train, y_test = train_test_split(df_test_under.drop(["ID_code","target"],axis=1),
df_test_under["target"], test_size=0.3, random_state=42)
<train_model>
%matplotlib inline
X_train = np.reshape(x_train.values,(-1, 200, 1))
X_valid = np.reshape(x_test.values,(-1, 200, 1))
y_valid =y_test
model = _Model1()
history1 = model.fit(X_train,
y_train,
epochs=40,
verbose=2,batch_size=256,
validation_data=(X_valid, y_valid))<train_model>
import pandas as pd
import numpy as np
model = _Model2()
history2 = model.fit(X_train,
y_train,
epochs=50,
verbose=2,batch_size=256,
validation_data=(X_valid, y_valid))<train_model>
import pandas as pd
import numpy as np
model = _Model3()
history4 = model.fit(X_train,
y_train,
epochs=20,
verbose=2,batch_size=256,
validation_data=(X_valid, y_valid))<install_modules>
def create_download_link(df, title = "Download CSV file",count=[0]):
count[0] = count[0]+1
filename = "data"+str(count[0])+".csv"
csv = df.to_csv(index=False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html )
! pip install git+https://www.github.com/keras-team/keras-contrib.git
! wget https://github.com/jganzabal/santander_kaggle_solutions_tests/raw/master/santander_helper.py<set_options>
df = pd.read_csv("/kaggle/input/dmassign1/data.csv")
 df_test = df.iloc[1300:]
df_train = df.iloc[:1300]
warnings.filterwarnings("ignore" )<load_from_csv>
class_labels={}
class_id = {}
for i,j in df_train.iterrows() :
class_labels[j["ID"]] = j["Class"]
class_id[i]=j["ID"]
labels = df_train["Class"]
df_train1 = df_train.drop(["ID","Class"],axis=1)
 df_test1 = df_test.drop(["ID","Class"],axis=1 )
test_path = '.. /input/test.csv'
train_path = '.. /input/train.csv'
df_test = pd.read_csv(test_path)
 df_test.drop(['ID_code'], axis=1, inplace=True)
 df_test = df_test.values
unique_samples = []
unique_count = np.zeros_like(df_test)
 for feature in range(df_test.shape[1]):
_, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)
 unique_count[index_[count_ == 1], feature] += 1
real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
print('Found',len(real_samples_indexes),'real test')
 print('Found',len(synthetic_samples_indexes),'fake test')
 d = {}
for i in range(200): d['var_'+str(i)] = 'float32'
d['target'] = 'uint8'
d['ID_code'] = 'object'
train = pd.read_csv('.. /input/train.csv', dtype=d)
 test = pd.read_csv('.. /input/test.csv', dtype=d)
 print('Loaded',len(train),'rows of train')
 print('Loaded',len(test),'rows of test')
 print('Found',len(real_samples_indexes),'real test')
 print('Found',len(synthetic_samples_indexes),'fake test')
 d = {}
for i in range(200): d['var_'+str(i)] = 'float32'
d['target'] = 'uint8'
d['ID_code'] = 'object'
train = pd.read_csv(train_path, dtype=d)
 test = pd.read_csv(test_path, dtype=d)
 print('Loaded',len(train),'rows of train')
 print('Loaded',len(test),'rows of test' )<data_type_conversions>
df_full = df_train1.append(df_test1)
 df_full.replace("?",np.nan,inplace=True )
def encode_FE(df,col,test):
cv = df[col].value_counts()
nm = col+'_FE'
df[nm] = df[col].map(cv)
 test[nm] = test[col].map(cv)
 test[nm].fillna(0,inplace=True)
 if cv.max() <=255:
df[nm] = df[nm].astype('uint8')
 test[nm] = test[nm].astype('uint8')
 else:
df[nm] = df[nm].astype('uint16')
 test[nm] = test[nm].astype('uint16')
 return
test['target'] = -1
comb = pd.concat([train,test.loc[real_samples_indexes]],axis=0,sort=True)
 for i in range(200):
encode_FE(comb,'var_'+str(i),test)
 train = comb[:len(train)]; del comb
print('Added 200 new magic features!' )<drop_column>
a = df_full.nunique()
print(a)
 a = dict(a)
 for i,j in a.items() :
a[i] = [j,df_full[i].dtype]
del df_test, real_samples_indexes, synthetic_samples_indexes, unique_count, unique_samples, d
gc.collect()<prepare_x_and_y>
for i,j in a.items() :
t,q = j
if q == np.dtype('O'):
if t <100:
print(Counter(df_full[i]))
df_train_data = train.drop(columns=['ID_code'])
 y = df_train_data['target'].values
df_train_X = df_train_data.drop(columns=['target'] )<define_variables>
def myconverter(string):
if string=="me":return "me"
elif string=="ME":return "me"
elif string=="M.E.":return "me"
else:
try:
return string.lower()
except:
return string
reverse_columns = True
if reverse_columns:
reverse_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 15, 16, 18, 19, 22, 24, 25, 26,
27, 29, 32, 35, 37, 40, 41, 47, 48, 49, 51, 52, 53, 55, 60, 61,
62, 65, 66, 67, 69, 70, 71, 74, 78, 79, 82, 84, 89, 90, 91, 94,
95, 96, 97, 99, 103, 105, 106, 110, 111, 112, 118, 119, 125, 128,
130, 133, 134, 135, 137, 138, 140, 144, 145, 147, 151, 155, 157,
159, 161, 162, 163, 164, 167, 168, 170, 171, 173, 175, 176, 179,
180, 181, 184, 185, 187, 189, 190, 191, 195, 196, 199,
]
for j in reverse_list:
df_train_X[f'var_{j}'] *= -1<feature_engineering>
le = preprocessing.LabelEncoder()
scaler = StandardScaler()
ab=[]
for i,j in a.items() :
t,q = j
if q == np.dtype('O'):
if t <100:
df_full[i] = df_full[i].apply(lambda x:myconverter(x))
df_full[i].fillna(df_full[i].mode() [0], inplace=True)
 ab.append(i)
 else:
df_full[i]= df_full[i].astype("float64")
 df_full[i].fillna(df_full[i].mean() ,inplace=True)
 df_full[i] = scaler.fit_transform(df_full[i].values.reshape(-1,1))
else:
df_full[i] = scaler.fit_transform(df_full[i].values.reshape(-1,1))
df_full = pd.get_dummies(df_full, prefix=ab)
means = df_train_X.mean(axis=0)
 stds = df_train_X.std(axis=0)
 df_train_X_normalized =(df_train_X - means)/stds<prepare_x_and_y>
modified_PCA = PCA(n_components = 0.99,svd_solver="full" ).fit_transform(df_full.values )
X_train_normalized = np.zeros(( df_train_X_normalized.shape[0], 400, 1))
for i in range(200):
X_train_normalized[:, 2*i] = df_train_X_normalized[[f'var_{i}']].values
X_train_normalized[:, 2*i+1] = df_train_X_normalized[[f'var_{i}_FE']].values<choose_model_class>
PCA_df = pd.DataFrame(data=modified_PCA )
def get_model(N_units = 600, kernel_size=2, strides=2):
model = Sequential()
model.add(Conv1D(N_units, kernel_size=kernel_size, strides=strides, padding='valid',
activation='relu', input_shape=(X_train_normalized.shape[1], 1)))
 model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
return model<set_options>
n_clusters=144
model = KMeans(n_clusters=n_clusters,random_state=42)
 pred = model.fit_predict(PCA_df.values)
 cntr = Counter(pred)
 mapping={}
res={0:0,1:0,2:0,3:0,4:0,5:0}
score = [defaultdict(int)for i in range(n_clusters)]
for i,j in enumerate(pred[:1300]):
score[j][labels[i]]+=1
for j,i in enumerate(score):
t =[(j,k)for k,j in i.items() ]
t.sort(reverse=True)
 t = [(j,k)for k,j in t]
try:
mapping[j]=int(t[0][0])
 res[int(t[0][0])] +=cntr[j]
except:
random_int = 0
mapping[j]=random_int
res[random_int]+=cntr[j]
print(res )
del df_train_data, df_train_X
gc.collect()<prepare_x_and_y>
PCA_df["pred"] = pred
df_test = test.drop(columns=['ID_code', 'target'])
 if reverse_columns:
for j in reverse_list:
df_test[f'var_{j}'] *= -1
df_test_X_normalized =(df_test - means)/stds<feature_engineering>
def mapper(x,mapping):
y = mapping[x]
if y==0:return np.random.randint(1,6)
 else:return y
X_test_normalized = np.zeros(( df_test_X_normalized.shape[0], 400, 1))
for i in range(200):
X_test_normalized[:, 2*i] = df_test_X_normalized[[f'var_{i}']].values
X_test_normalized[:, 2*i+1] = df_test_X_normalized[[f'var_{i}_FE']].values<load_pretrained>
PCA_df["pred"]=PCA_df["pred"].apply(lambda x:mapper(x,mapping))
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 full_val_preds = np.zeros(( len(df_train_X_normalized), 1))
model = get_model(N_units)
 test_predictions = 0
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train_X_normalized, y)) :
print('
print(f'
print('
X_train = X_train_normalized[trn_idx]
X_val = X_train_normalized[val_idx]
filename = f'CNN_generator_fold_{fold}_cl1_{class_1_aug}_cl0_{class_0_aug}_{N_units}.hdf5'
model.load_weights(filename)
 full_val_preds[val_idx] = model.predict(X_val, verbose=1)
 print(roc_auc_score(y[val_idx], full_val_preds[val_idx]))
test_predictions = test_predictions + model.predict(X_test_normalized, verbose=1)/5<compute_test_metric>
Counter(PCA_df["pred"] )
roc_auc_score(y, full_val_preds )<save_to_csv>
df_submission = PCA_df.iloc[1300:]
def save_submit_file(predictions, filename, test_filename=test_path, index_column='ID_code', target_column = 'target'):
df_test_submit = pd.read_csv(test_filename ).set_index(index_column)
 df_test_submit[target_column] = predictions
df_test_submit[[target_column]].to_csv(filename)
 return<save_to_csv>
predictions = df_submission["pred"]
save_submit_file(test_predictions,
f'submit_CNN_generator_cl1_{class_1_aug}_cl0_{class_0_aug}_{N_units}.csv' )<load_from_csv>
submission = pd.DataFrame(data={"ID": df["ID"][1300:], "Class": predictions } )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 features = [x for x in train_df.columns if x.startswith("var")]<feature_engineering>
Counter(submission["Class"] )
for var in features:
if np.corrcoef(train_df['target'], train_df[var])[1][0] < 0:
train_df[var] = train_df[var] * -1
test_df[var] = test_df[var] * -1<count_values>
create_download_link(submission )
<feature_engineering><EOS> 
submission.to_csv("Final.csv",index=False )