Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%matplotlib inline
train = pd.read_csv('.. /input/train.csv', header = 0, dtype={'Age': np.float64})
 test = pd.read_csv('.. /input/test.csv' , header = 0, dtype={'Age': np.float64})
 full_data = [train, test]
print(train.info())
 train.head()
DATA = Path('.. /input')
 CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = DATA/'train_noisy.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
TRN_CURATED = DATA/'train_curated'
TRN_NOISY = DATA/'train_noisy'
TEST = DATA/'test'
WORK = Path('work')
 IMG_TRN_CURATED = WORK/'image/trn_curated'
IMG_TRN_NOISY = WORK/'image/train_noisy'
IMG_TEST = WORK/'image/test'
for folder in [WORK, IMG_TRN_CURATED, IMG_TRN_NOISY, IMG_TEST]:
Path(folder ).mkdir(exist_ok=True, parents=True)
 df = pd.read_csv(CSV_TRN_CURATED)
 test_df = pd.read_csv(CSV_SUBMISSION )<normalization>
print(train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean())
 print(train[['Pclass', 'Survived']].groupby(['Pclass'] ).mean() )
def read_audio(conf, pathname, trim_long_data):
y, sr = librosa.load(pathname, sr=conf.sampling_rate)
 if 0 < len(y):
y, _ = librosa.effects.trim(y)
 if len(y)> conf.samples:
if trim_long_data:
y = y[0:0+conf.samples]
else:
padding = conf.samples - len(y)
 offset = padding // 2
y = np.pad(y,(offset, conf.samples - len(y)- offset), 'constant')
 return y
def audio_to_melspectrogram(conf, audio):
spectrogram = librosa.feature.melspectrogram(audio,
sr=conf.sampling_rate,
n_mels=conf.n_mels,
hop_length=conf.hop_length,
n_fft=conf.n_fft,
fmin=conf.fmin,
fmax=conf.fmax)
 spectrogram = librosa.power_to_db(spectrogram)
 spectrogram = spectrogram.astype(np.float32)
 return spectrogram
def show_melspectrogram(conf, mels, title='Log-frequency power spectrogram'):
librosa.display.specshow(mels, x_axis='time', y_axis='mel',
sr=conf.sampling_rate, hop_length=conf.hop_length,
fmin=conf.fmin, fmax=conf.fmax)
 plt.colorbar(format='%+2.0f dB')
 plt.title(title)
 plt.show()
def read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):
x = read_audio(conf, pathname, trim_long_data)
 mels = audio_to_melspectrogram(conf, x)
 if debug_display:
IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))
show_melspectrogram(conf, mels)
 return mels
class conf:
sampling_rate = 44100
duration = 2
hop_length = 347*duration
fmin = 20
fmax = sampling_rate // 2
n_mels = 128
n_fft = n_mels * 20
samples = sampling_rate * duration
x = read_as_melspectrogram(conf, TRN_CURATED/'0006ae4e.wav', trim_long_data=False, debug_display=True )<categorify>
print(train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean() )
def mono_to_color(X, mean=None, std=None, norm_max=None, norm_min=None, eps=1e-6):
X = np.stack([X, X, X], axis=-1)
 mean = mean or X.mean()
std = std or X.std()
Xstd =(X - mean)/(std + eps)
 _min, _max = Xstd.min() , Xstd.max()
norm_max = norm_max or _max
norm_min = norm_min or _min
if(_max - _min)> eps:
V = Xstd
V[V < norm_min] = norm_min
V[V > norm_max] = norm_max
V = 255 *(V - norm_min)/(norm_max - norm_min)
 V = V.astype(np.uint8)
 else:
V = np.zeros_like(Xstd, dtype=np.uint8)
 return V
def convert_wav_to_image(df, source, img_dest):
X = []
for i, row in tqdm_notebook(df.iterrows()):
x = read_as_melspectrogram(conf, source/str(row.fname), trim_long_data=False)
 x_color = mono_to_color(x)
 X.append(x_color)
 return X
X_train = convert_wav_to_image(df, source=TRN_CURATED, img_dest=IMG_TRN_CURATED)
 X_test = convert_wav_to_image(test_df, source=TEST, img_dest=IMG_TEST )<define_variables>
print(train[['SibSp', 'Survived']].groupby(['SibSp'], as_index=False ).mean())
 print(train[['Parch', 'Survived']].groupby(['Parch'], as_index=False ).mean() )
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
def lwlrap(scores, truth, **kwargs):
score, weight = calculate_per_class_lwlrap(to_np(truth), to_np(scores))
return torch.Tensor([(score * weight ).sum() ])
<define_variables>
for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
print(train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean() )
data.show_batch(3 )<choose_model_class>
for dataset in full_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
print(train[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean() )
learn = cnn_learner(data, models.resnet18, pretrained=False, metrics=[lwlrap])
 learn.unfreeze()
learn.lr_find() ; learn.recorder.plot()<train_model>
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S')
 print(train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean() )
learn.fit_one_cycle(5, 1e-1)
 learn.fit_one_cycle(10, 1e-2 )<train_model>
for dataset in full_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median())
 dataset['CategoricalFare'] = pd.qcut(dataset['Fare'], 4)
 print(type(train['CategoricalFare'][0]))
print(train[['CategoricalFare', 'Survived']].groupby(['CategoricalFare'], as_index=False ).mean() )
learn.fit_one_cycle(20, 3e-3 )<train_model>
for dataset in full_data:
age_avg 	 = dataset['Age'].mean()
age_std 	 = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 dataset['CategoricalAge'] = pd.cut(dataset['Age'], 5)
 print(train[['CategoricalAge', 'Survived']].groupby(['CategoricalAge'], as_index=False ).mean() )
learn.fit_one_cycle(20, 1e-3 )<train_model>
def get_title(name):
	title_search = re.search('([A-Za-z]+)\.', name)
 	
	if title_search:
		return title_search.group(1)
 	return ""
for dataset in full_data:
dataset['Title'] = dataset['Name'].apply(get_title)
 print(pd.crosstab(train['Title'], train['Sex']))
learn.fit_one_cycle(50, slice(1e-3, 3e-3))<train_model>
for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 print(train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean() )
learn.fit_one_cycle(10, slice(1e-4, 1e-3))<save_model>
for dataset in full_data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] 						 = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] 							 = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 dataset.loc[ dataset['Age'] <= 16, 'Age'] 					 = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
learn.save('fat2019_fastai_cnn2d_stage-2')
 learn.export()<load_from_csv>
train = pd.get_dummies(train,columns=['Sex','Title','Embarked','Fare','Age'])
 test = pd.get_dummies(test,columns=['Sex','Title','Embarked','Fare','Age'] )
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK/'image', Path('.. /.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK/'image', test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test )<save_to_csv>
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp',\
'Parch', 'FamilySize']
train = train.drop(drop_elements, axis = 1)
 train = train.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)
 test = test.drop(drop_elements, axis = 1)
 test = test.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)
 print(train.columns)
 print(full_data[0].columns )
test_df[learn.data.classes] = preds
test_df.to_csv('submission.csv', index=False)
 test_df.head()<choose_model_class>
train = train.values
test = test.values
CUR_X_FILES, CUR_X = list(df.fname.values), X_train
learn = cnn_learner(data, models.resnet18, pretrained=False, metrics=[lwlrap])
 learn.load('fat2019_fastai_cnn2d_stage-2');<load_from_csv>
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.metrics import accuracy_score, log_loss,f1_score
from xgboost import XGBClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier,ExtraTreesClassifier,VotingClassifier
from sklearn.naive_bayes import GaussianNB,BernoulliNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
train = pd.read_csv('.. /input/train.csv',
dtype={'is_booking':bool,'srch_destination_id':np.int32, 'hotel_cluster':np.int32},
usecols=['srch_destination_id','is_booking','hotel_cluster'],
chunksize=1000000)
 aggs = []
print('-'*38)
 for chunk in train:
agg = chunk.groupby(['srch_destination_id',
'hotel_cluster'])['is_booking'].agg(['sum','count'])
 agg.reset_index(inplace=True)
 aggs.append(agg)
 print('.',end='')
 print('')
 aggs = pd.concat(aggs, axis=0)
 aggs.head()<count_unique_values>
classifiers = [
KNeighborsClassifier(n_neighbors=5,weights='distance',p=2,n_jobs=-1),
SVC(probability=True,random_state=666,tol=1e-6),
DecisionTreeClassifier(min_samples_split=10,random_state=666),
RandomForestClassifier(n_estimators=500,random_state=666,n_jobs=-1),
	AdaBoostClassifier(n_estimators=500,random_state=666),
GradientBoostingClassifier(n_estimators=500,random_state=666,min_samples_split=10),
GaussianNB() ,
BernoulliNB() ,
LinearDiscriminantAnalysis(tol=1e-6),
QuadraticDiscriminantAnalysis(tol=1e-6),
LogisticRegression(tol=1e-6,random_state=666,class_weight='balanced',n_jobs=-1),
XGBClassifier(n_estimators=500,n_jobs=-1,random_state=666)]
log_cols = ["Classifier", "Accuracy","F1"]
log 	 = pd.DataFrame(columns=log_cols)
 sss = StratifiedShuffleSplit(n_splits=10, test_size=0.1, random_state=0)
 X = train[0::, 1::]
y = train[0::, 0]
acc_dict = {}
f1_dict = {}
for train_index, test_index in sss.split(X, y):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
for clf in classifiers:
name = clf.__class__.__name__
clf.fit(X_train, y_train)
 train_predictions = clf.predict(X_test)
 acc = accuracy_score(y_test, train_predictions)
 f1 = f1_score(y_test, train_predictions)
 if name in acc_dict:
acc_dict[name] += acc
else:
acc_dict[name] = acc
if name in f1_dict:
f1_dict[name] += f1
else:
f1_dict[name] = f1
for clf in acc_dict:
acc_dict[clf] = acc_dict[clf] / 10.0
f1_dict[clf] = f1_dict[clf] / 10.0
log_entry = pd.DataFrame([[clf, acc_dict[clf],f1_dict[clf]]], columns=log_cols)
 log = log.append(log_entry )
def most_popular(group, n_max=5):
relevance = group['relevance'].values
hotel_cluster = group['hotel_cluster'].values
most_popular = hotel_cluster[np.argsort(relevance)[::-1]][:n_max]
return np.array_str(most_popular)[1:-1]<prepare_output>
clf1 = KNeighborsClassifier(n_neighbors=5,weights='distance',p=2,n_jobs=-1)
 clf2 = SVC(probability=True,random_state=666,tol=1e-6)
 clf3 = DecisionTreeClassifier(min_samples_split=10,random_state=666)
 clf4 = RandomForestClassifier(n_estimators=500,random_state=666,n_jobs=-1)
 clf5 = AdaBoostClassifier(n_estimators=500,random_state=666)
 clf6 = GradientBoostingClassifier(n_estimators=500,random_state=666,min_samples_split=10)
 clf7 = BernoulliNB()
clf8 = LinearDiscriminantAnalysis(tol=1e-6)
 clf9 = LogisticRegression(tol=1e-6,random_state=666,class_weight='balanced',n_jobs=-1)
 clf10 = XGBClassifier(n_estimators=500,n_jobs=-1,random_state=666)
 optimal = VotingClassifier(estimators=[('1',clf1),('4',clf4),('5',clf5),('7',clf7),\ 
('9',clf9),('10',clf10)],n_jobs=-1,voting='hard')
 optimal = optimal.fit(train[:,1:],train[:,0])
 result = optimal.predict(test)
most_pop = agg.groupby(['srch_destination_id'] ).apply(most_popular)
 most_pop = pd.DataFrame(most_pop ).rename(columns={0:'hotel_cluster'})
 most_pop.head()<load_from_csv>
final_file = pd.DataFrame({'PassengerId':full_data[1].PassengerId,'Survived':result.astype(int)})
 final_file.to_csv('Titanic best working Classifier.csv',index=False )