Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
train_identity_data = pd.read_csv('.. /input/train_identity.csv')
 train_transaction_data = pd.read_csv('.. /input/train_transaction.csv' )<set_options>
train_data["Name"] = train_data["Name"].str.split(',' ).str[1]
train_data["Name"] = train_data["Name"].str.split('.' ).str[0]
train_data["Name"] = train_data["Name"].str.strip()
pd.set_option('display.max_columns', 500)
 train_transaction_data.head(5 )<feature_engineering>
x = train_data.groupby('Name' ).agg(['count'] ).index.get_level_values('Name')
 x
train_transaction_data['Transaction_dow'] = np.floor(( train_transaction_data['TransactionDT'] /(3600 * 24)- 1)% 7)
 train_transaction_data['Transaction_hour'] = np.floor(train_transaction_data['TransactionDT'] / 3600)% 24<count_missing_values>
train_data["Age"] = train_data.groupby("Name" ).transform(lambda x: x.fillna(x.mean())) ['Age']
train_data['Sex'].replace({'female':0,'male':1},inplace=True)
 train_data.head()
na_columns = train_identity_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0]/train_identity_data.shape[0] )<import_modules>
train_data_log = train_data.iloc[:,[False,False,True, False,True,True,True,True,False,True,False,False]]
normalized_data_train=(train_data_log-train_data_log.min())/(train_data_log.max() -train_data_log.min())
 train_labels_log = train_data.iloc[:,1]
normalized_data_train.head()
import seaborn as sns
import matplotlib.pyplot as plt<set_options>
def log_odds(data,coefficients,intercept):

 return np.dot(data,coefficients)+ intercept
l_o= log_odds(normalized_data_train,initial_coefs, initial_intercept)
pd.set_option('display.max_rows', 500)
 na_columns = train_transaction_data.isna().sum()
print(na_columns[na_columns==0])
 print(na_columns[na_columns>0]/train_transaction_data.shape[0] )<feature_engineering>
def sigmoid(log_odds_vars):

 sigmoid_values = 1/(1+np.exp(-log_odds_vars))
return sigmoid_values
sigmoid_vals = sigmoid(l_o)
identity_data_columns = train_identity_data.columns
numericCols = train_identity_data._get_numeric_data().columns
categoricalCols = list(set(identity_data_columns)- set(numericCols))
print('The categorical columns in identity data are: ',categoricalCols)
 train_identity_data[categoricalCols] = train_identity_data[categoricalCols].replace({ np.nan:'missing'})
 train_identity_data[numericCols] = train_identity_data[numericCols].replace({ np.nan:-1})
 colstoplot = ['id_01','id_02','id_03','id_04','id_05','id_06','id_07','id_08','id_09','id_10',
'id_11','id_13','id_14','id_17','id_18','id_19','id_20','id_21','id_22','id_24','id_25','id_26','id_32']
for i in colstoplot:
plt.figure(figsize=(16, 16))
ax = sns.distplot(eval('train_identity_data["'+i+'"]'))
ax.set(ylabel="Distribution")
 ax.set_title([i] )<feature_engineering>
def log_loss(probabilities, labels):

 data_length = len(labels)
 labels = np.array(labels)
 left_half = np.dot(labels,np.log(probabilities+.0001))
right_half = np.dot(1-labels,np.log(1-probabilities+.0001))
loss =(-1/data_length)*(left_half + right_half)
 return loss
train_transaction_data[categoricalCols] = train_transaction_data[categoricalCols].replace({ np.nan:'missing'})
 train_transaction_data[numericCols] = train_transaction_data[numericCols].replace({ np.nan:-1} )<feature_engineering>
def find_coefficients(data, coefficients, intercept,labels,learning_rate, iterations):
coefs = coefficients
for i in range(iterations):
l_odds = log_odds(data,coefs,intercept)
 sig_vals = sigmoid(l_odds)
 data_transpose = np.transpose(learning_rate * data)
 coefs = np.dot(data_transpose,(labels-sig_vals)* sig_vals*(1-sig_vals)) + coefs
intercept = intercept + learning_rate * np.dot(( labels-sig_vals),(sig_vals*(1-sig_vals)))
 print(coefs, intercept)
 return coefs, intercept
best_coefs= find_coefficients(normalized_data_train,initial_coefs, initial_intercept,train_labels_log,.0005, 50000 )
print('The average transaction amount for non fraudulent transactions is: ',
np.mean(train_transaction_data.loc[train_transaction_data["isFraud"]==0]['TransactionAmt']))
print('The average transaction amount for fraudulent transactions is: ',
np.mean(train_transaction_data.loc[train_transaction_data["isFraud"]==1]['TransactionAmt']))<feature_engineering>
best_coef = best_coefs[0]
best_int = best_coefs[1]
v = sigmoid(log_odds(normalized_data_train,best_coef,best_int))
maxCardData = {}
minCardData = {}
meanCardData = {}
for i,i_card in enumerate(cardTypes):
cardData = eval('train_transaction_data.loc[train_transaction_data["card4"]=="'+i_card+'"]')
 maxCardData[i_card] = np.max(cardData['TransactionAmt'])
 minCardData[i_card] = np.min(cardData['TransactionAmt'])
 meanCardData[i_card] = np.mean(cardData['TransactionAmt'] )<drop_column>
def find_threshold(sigmoid_vals):

 predictions = []
vals = []
accuracies = []
for num in range(1000):
vals.append(num/1000)
 accuracy = 0
for i in v:
if i > num/1000:
predictions.append(1)
 else:
predictions.append(0)
 for j in range(len(predictions)) :
if predictions[j] == train_labels_log[j]:
accuracy += 1
accuracies.append(accuracy/len(predictions))
accuracy = 0
predictions = []
indx = accuracies.index(max(accuracies))
print("Best accuracy on training set:")
 print(max(accuracies))
best_threshold = vals[indx]
return best_threshold
best_thresh = find_threshold(v)
 print(best_thresh )
del fraudDataTransaction,normalDataTransaction,productData,cardData<feature_engineering>
def calculate_precision(sigmoid_vals, threshold, labels):
"Precision is True Positives /(True Positives + False Positives)"
predictions = []
true_positives = 0
false_positives = 0
for i in sigmoid_vals:
if i > threshold:
predictions.append(1)
 else:
predictions.append(0)
 for i in range(len(labels)) :
if labels[i] == 1 and labels[i] == predictions[i]:
true_positives += 1
elif labels[i] == 0 and labels[i] != predictions[i]:
false_positives += 1
return true_positives/(true_positives + false_positives)
 print("Precision:")
 print(calculate_precision(v, best_thresh, train_labels_log))
train_identity_data.loc[train_identity_data['id_30'].str.contains('Mac', na=False), 'id_30'] = 'mac'
train_identity_data.loc[train_identity_data['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
train_identity_data.loc[train_identity_data['id_30'].str.contains('Android', na=False), 'id_30'] = 'android'
train_identity_data.loc[train_identity_data['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
train_identity_data.loc[train_identity_data['id_30'].str.contains('Linux', na=False), 'id_30'] = 'Linux'<feature_engineering>
def calculate_recall(sigmoid_vals, threshold, labels):
"Precision is True Positives /(True Positives + False Negatives)"
predictions = []
true_positives = 0
false_negatives = 0
for i in sigmoid_vals:
if i > threshold:
predictions.append(1)
 else:
predictions.append(0)
 for i in range(len(labels)) :
if labels[i] == 1 and labels[i] == predictions[i]:
true_positives += 1
elif labels[i] == 1 and labels[i] != predictions[i]:
false_negatives += 1
return true_positives/(true_positives + false_negatives)
 print("Recall")
 print(calculate_recall(v, best_thresh, train_labels_log))
train_identity_data['device_name'] = train_identity_data['DeviceInfo'].str.split('/', expand=True)[0]
train_identity_data.loc[train_identity_data['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
train_identity_data.loc[train_identity_data['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
train_identity_data.loc[train_identity_data['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
train_identity_data.loc[train_identity_data['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
train_identity_data.loc[train_identity_data['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
train_identity_data.loc[train_identity_data['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
train_identity_data.loc[train_identity_data['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
train_identity_data.loc[train_identity_data['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
train_identity_data.loc[train_identity_data['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
train_identity_data.loc[train_identity_data['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
train_identity_data.loc[train_identity_data['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
train_identity_data.loc[train_identity_data['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
train_identity_data.loc[train_identity_data['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
train_identity_data.loc[train_identity_data['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
train_identity_data.loc[train_identity_data.device_name.isin(train_identity_data.device_name.value_counts() [train_identity_data.device_name.value_counts() < 200].index), 'device_name'] = "Others"<merge>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data["Name"] = test_data["Name"].str.split(',' ).str[1]
test_data["Name"] = test_data["Name"].str.split('.' ).str[0]
test_data["Name"] = test_data["Name"].str.strip()
test_data['Sex'].replace({'female':0,'male':1},inplace=True)
 x = test_data.groupby('Name' ).agg(['count'] ).index.get_level_values('Name')
 test_data["Age"] = test_data.groupby("Name" ).transform(lambda x: x.fillna(x.mean())) ['Age']
test_data_log = test_data.iloc[:,[False,True,False,True,True,True,True,False,True,False,False]]
normalized_data_test=(test_data_log-test_data_log.min())/(test_data_log.max() -test_data_log.min())
raw_train_data = pd.merge(train_transaction_data, train_identity_data, on='TransactionID', how='left')
 del train_transaction_data,train_identity_data<feature_engineering>
pred_test = sigmoid(log_odds(normalized_data_test,best_coef,best_int))
raw_train_data_columns = raw_train_data.columns
numericCols = raw_train_data._get_numeric_data().columns
categoricalCols = list(set(raw_train_data_columns)- set(numericCols))
print('The categorical columns in training data are: ',categoricalCols)
 raw_train_data[categoricalCols] = raw_train_data[categoricalCols].replace({ np.nan:'missing'})
 raw_train_data[numericCols] = raw_train_data[numericCols].replace({ np.nan:-1} )<drop_column>
classifier = []
for i in range(len(pred_test)) :
if pred_test[i] > best_thresh:
classifier.append(1)
 else:
classifier.append(0 )
<drop_column><EOS> 
data = {'PassengerId': test_data["PassengerId"].values, 'Survived':classifier}
df_submission = pd.DataFrame(data)
 df_submission.to_csv("submission_log_regression2.csv",index=False)