Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<create_dataframe>
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings("ignore")
 pd.set_option('max_columns',100 )
genres_predict = []
ids = []
for i in range(y_predict.shape[0]):
y_predict_max = np.argmax(y_predict[i,:])
 genres_predict.extend([lb.classes_[y_predict_max]])
 submission = pd.DataFrame({'id':predict_data_np['id'].values,
'genre':genres_predict,
'title':predict_data_np['title'].values},
columns=['id', 'genre','title'])
 submission.to_csv('submission.csv', index=False,columns=['id', 'genre'])
 print('Save submit')
 submission.head() 
<load_from_csv>
traindf = pd.read_csv('.. /input/titanic/train.csv' ).set_index('PassengerId')
 testdf = pd.read_csv('.. /input/titanic/test.csv' ).set_index('PassengerId')
 submission = pd.read_csv('.. /input/titanic/gender_submission.csv' )
train=pd.read_csv('.. /input/kaggle18011884/train_cabbage_price.csv')
 train.avgPrice.plot()<load_from_csv>
df = pd.concat([traindf, testdf], axis=0, sort=False)
 df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['Title'] = df.Name.str.split(',' ).str[1].str.split('.' ).str[0].str.strip()
df['IsWomanOrBoy'] =(( df.Title == 'Master')|(df.Sex == 'female'))
df['LastName'] = df.Name.str.split(',' ).str[0]
family = df.groupby(df.LastName ).Survived
df['WomanOrBoyCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).count())
 df['WomanOrBoyCount'] = df.mask(df.IsWomanOrBoy, df.WomanOrBoyCount - 1, axis=0)
 df['FamilySurvivedCount'] = family.transform(lambda s: s[df.IsWomanOrBoy].fillna(0 ).sum())
 df['FamilySurvivedCount'] = df.mask(df.IsWomanOrBoy, df.FamilySurvivedCount - \
df.Survived.fillna(0), axis=0)
 df['WomanOrBoySurvived'] = df.FamilySurvivedCount / df.WomanOrBoyCount.replace(0, np.nan)
 df.WomanOrBoyCount = df.WomanOrBoyCount.replace(np.nan, 0)
 df['Alone'] =(df.WomanOrBoyCount == 0)
 df['Title'] = df['Title'].replace('Ms','Miss')
 df['Title'] = df['Title'].replace('Mlle','Miss')
 df['Title'] = df['Title'].replace('Mme','Mrs')
 df['Embarked'] = df['Embarked'].fillna('S')
 df['Deck'] = df['Cabin'].apply(lambda s: s[0] if pd.notnull(s)else 'M')
 med_fare = df.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df['Fare'] = df['Fare'].fillna(med_fare)
 df['Age'] = df.groupby(['Sex', 'Pclass', 'Title'])['Age'].apply(lambda x: x.fillna(x.median()))
df['Family_Size'] = df['SibSp'] + df['Parch'] + 1
cols_to_drop = ['Name','Ticket','Cabin', 'IsWomanOrBoy', 'WomanOrBoyCount', 'FamilySurvivedCount']
df = df.drop(cols_to_drop, axis=1)
 df.WomanOrBoySurvived = df.WomanOrBoySurvived.fillna(0)
 df.Alone = df.Alone.fillna(0)
 target = df.Survived.loc[traindf.index]
df = df.drop(['Survived'], axis=1)
 train, test = df.loc[traindf.index], df.loc[testdf.index]
test=pd.read_csv('.. /input/kaggle18011884/test_cabbage_price.csv')
 test<data_type_conversions>
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = train.columns.values.tolist()
for col in features:
if train[col].dtype in numerics: continue
categorical_columns.append(col)
 for col in categorical_columns:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
train['yeargb']=[y[:6] for y in train['year'].astype('str')]
test['yeargb']=test['year']
train['month']=[np.int(y[4:6])for y in train['year'].astype('str')]
test['month']=[np.int(y[-2:])for y in test['year'].astype('str')]
train['year2']=[np.int(y[:4])for y in train['year'].astype('str')]
test['year2']=[np.int(y[:4])for y in test['year'].astype('str')]
traingb=train.groupby('yeargb' ).mean().reset_index()
sns.heatmap(traingb.corr() )<concatenate>
Xtrain, Xval, Ztrain, Zval = train_test_split(train, target, test_size=0.2, random_state=0)
 train_set = lgbm.Dataset(Xtrain, Ztrain, silent=False)
 valid_set = lgbm.Dataset(Xval, Zval, silent=False )
traingb.reset_index().append(test.reset_index() )<create_dataframe>
params = {
'boosting_type':'gbdt',
'objective': 'binary',
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 50 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'binary',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 2,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
}
modelL = lgbm.train(params, train_set = train_set, num_boost_round=2000,
early_stopping_rounds=10, verbose_eval=10, valid_sets=valid_set )
pd.DataFrame(np.dot(np.array(ymean.avgPrice ).reshape(-1,1), np.array(mmean.avgPrice ).reshape(1,-1)))/(traingb.avgPrice.mean() **2 )<prepare_x_and_y>
feature_score = pd.DataFrame(train.columns, columns = ['feature'])
 feature_score['LGB'] = modelL.feature_importance()
def clustertechniques2(dtrain,label,indexv):
print('
cols=[ci for ci in dtrain.columns if ci not in [indexv,'index',label]]
dtest=dtrain[dtrain[label].isnull() ==True][[indexv,label]]
print(dtest)
 print('encodings after shape',dtrain.shape)
 X_train=dtrain[dtrain[label].isnull() ==False].drop([indexv,label],axis=1 ).fillna(0)
 Y_train=dtrain[dtrain[label].isnull() ==False][label]
X_test=dtrain[dtrain[label].isnull() ==True].drop([indexv,label],axis=1 ).fillna(0)
 Y_test=dtrain[dtrain[label].isnull() ==True][label].fillna(0)
 print(Y_test)
 for xi in range(len(Y_test)) :
Y_test.iloc[xi]=np.random.random(( 1,1)) [0]
print(Y_test)
 if len(X_test)==0:
X_train,X_test,Y_train,Y_test = train_test_split(dtrain.drop(label,axis=1 ).fillna(0),dtrain[label],test_size=0.25,random_state=0)
 lenxtr=len(X_train)
 print('splitting data train test X-y',X_train.shape,Y_train.shape,X_test.shape,Y_test.shape)
 scale = preprocessing.MinMaxScaler().fit(X_train)
 X_train = scale.transform(X_train)
 X_test = scale.transform(X_test)
 clusters = [Dummy(1),
PCA(n_components=0.7,random_state=0,whiten=True),
FastICA(n_components=7,random_state=0),
TruncatedSVD(n_components=5, n_iter=7, random_state=42),
NMF(n_components=10,random_state=0),
UMAP(n_neighbors=5,n_components=10, min_dist=0.3,metric='minkowski'),
TSNE(n_components=2,random_state=0)
 ]
clunaam=["raw",'PCA','tSVD','UMAP','tSNE']
classifiers = [
ExtraTreesRegressor() ,
RandomForestRegressor(random_state=1, n_estimators=10),
BayesianRidge() ,
RANSACRegressor() ,
KNeighborsRegressor() ,
ElasticNetCV(cv=5, random_state=0),
HuberRegressor() ,
LinearRegression() ,
]
clanaam= ['xTreer','rFor','BaysR','Ransac','KNNr','elast','huber','linear',]
results=[]
for clu in clusters:
clunm=clunaam[clusters.index(clu)]
X_total_clu = clu.fit_transform(np.concatenate(( X_train,X_test),axis=0))
X_total_clu=np.concatenate(( X_total_clu,np.concatenate(( X_train,X_test),axis=0)) ,axis=1)
 print(X_total_clu.shape)
 plt.scatter(X_total_clu[:lenxtr,0],X_total_clu[:lenxtr,1],c=Y_train.values,cmap='prism')
 plt.title(clu)
 plt.show()
for cla in classifiers:
start = datetime.datetime.now()
clanm=clanaam[classifiers.index(cla)]
print(' ',cla)
 cla.fit(X_total_clu,np.concatenate(( Y_train,Y_test)))
 cla.fit(X_total_clu[:lenxtr],Y_train)
 trainpredi=cla.predict(X_total_clu[:lenxtr])
 testpredi=cla.predict(X_total_clu[lenxtr:])
 if classifiers.index(cla)in [0,2,3,4,5,7,8,9,10,11,12,13]:
trainprediprob=cla.predict(X_total_clu[:lenxtr])
 testprediprob=cla.predict(X_total_clu[lenxtr:])
 plt.scatter(x=testprediprob, y=testpredi, marker='.', alpha=0.53)
 plt.show()
if len(dtest)==0:
test_score=cla.score(X_total_clu[lenxtr:],Y_test)
 mse = mean_squared_error(testpredi,Y_test)
 train_score=cla.score(X_total_clu[:lenxtr],Y_train)
 li = [clunm,clanm,train_score,test_score,mse]
results.append(li)
 r2s=r2_score(testpredi,Y_test)
 print(r2s)
 plt.title(clanm+'test corr & mse:'+np.str(test_score)+' '+np.str(mse)+' and test confusionmatrix')
 plt.scatter(x=Y_test, y=testpredi, marker='.', alpha=1)
 plt.scatter(x=[np.mean(Y_test)], y=[np.mean(testpredi)], marker='o', color='red')
 plt.xlabel('Real test'); plt.ylabel('Pred.test')
 plt.show()
else:
testpredlabel=testpredi
print('train correl',r2_score(trainpredi,Y_train),'mse ',mean_squared_error(trainpredi,Y_train))
submit = pd.DataFrame({'Id': dtest[indexv],'Expected': testpredlabel})
 submit['Expected']=submit['Expected'].astype('int')
 filenaam='subm_'+clunm+'_'+clanm+'.csv'
submit.to_csv(path_or_buf =filenaam, index=False)
 print(clanm,'0 classifier time',datetime.datetime.now() -start)
 if len(dtest)==0:
print(pd.DataFrame(results ).sort_values(3))
submit=[]
return submit
class Dummy() :
def __init__(self, feature_names):
self._feature_names = feature_names
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
clustertechniques2(traingb.reset_index().append(test.reset_index() ).drop('year',axis=1),'avgPrice','index' )<load_from_csv>
y_preds_lgb = modelL.predict(test, num_iteration=modelL.best_iteration )
train=pd.read_csv('.. /input/utkml/train_final.csv')
 test=pd.read_csv('.. /input/utkml/test_final.csv')
 total=train.append(test,ignore_index=True)
<sort_values>
data_tr = xgb.DMatrix(Xtrain, label=Ztrain)
 data_cv = xgb.DMatrix(Xval , label=Zval)
 data_train = xgb.DMatrix(train)
 data_test = xgb.DMatrix(test)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]
total.sort_values('user_id' )<feature_engineering>
parms = {'max_depth':5,
'objective':'reg:logistic',
'eval_metric':'error',
'learning_rate':0.01,
'subsample':0.8,
'colsample_bylevel':0.9,
'min_child_weight': 2,
'seed': 0}
modelx = xgb.train(parms, data_tr, num_boost_round=2000, evals = evallist,
early_stopping_rounds=300, maximize=False,
verbose_eval=100)
 print('score = %1.5f, n_boost_round =%d.'%(modelx.best_score,modelx.best_iteration))
datacol=total[['user_id','JOKE:5']]
datacol.columns=['user_id','rating']
datacol['item_id']=0
data=datacol.dropna()
for ci in range(2,141):
colnm=train.columns[ci]
datacol=total[['user_id',colnm]]
datacol.columns=['user_id','rating']
datacol['item_id']=ci-1
data=data.append(datacol.dropna())
 data
datacoo=coo_matrix(( data.rating,(data.item_id,data.user_id)))
 datacoo 
<import_modules>
feature_score['XGB'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))
ratings=datacoo
TFIDFRecommender, bm25_weight)
 log = logging.getLogger("implicit")
<choose_model_class>
y_preds_xgb = modelx.predict(data_test )
start = time.time()
output_filename='output.txt'
model_name='bpr'
min_rating=-10.0,
titles=train.columns[1:]
ratings.data[ratings.data < min_rating] = 0
ratings.eliminate_zeros()
ratings.data = np.ones(len(ratings.data))
log.info("read data file in %s", time.time() - start)
 if model_name == "als":
model = AlternatingLeastSquares()
print("weighting matrix by bm25_weight")
 ratings =(bm25_weight(ratings, B=0.9)* 5 ).tocsr()
elif model_name == "bpr":
model = BayesianPersonalizedRanking()
elif model_name == "lmf":
model = LogisticMatrixFactorization()
elif model_name == "tfidf":
model = TFIDFRecommender()
elif model_name == "cosine":
model = CosineRecommender()
elif model_name == "bm25":
model = BM25Recommender(B=0.2)
 else:
raise NotImplementedError("TODO: model %s" % model_name)
 print("training model %s", model_name)
 start = time.time()
model.fit(ratings)
 print("trained model '%s' in %s", model_name, time.time() - start)
 log.debug("calculating top movies")
 user_count = np.ediff1d(ratings.indptr)
 to_generate = sorted(np.arange(len(titles)) , key=lambda x: -user_count[x])
 print("calculating similar movies")
 with tqdm.tqdm(total=len(to_generate)) as progress:
with codecs.open(output_filename, "w", "utf8")as o:
for movieid in to_generate:
print(movieid,model.similar_items(movieid))
if ratings.indptr[movieid] != ratings.indptr[movieid + 1]:
title = titles[movieid]
for other, score in model.similar_items(movieid):
" %(title, titles[other], score))
try:
print(title,titles[other],score)
 print()
except:
print(title,other,score)
 progress.update(1)
<sort_values>
Scaler_train = preprocessing.MinMaxScaler().fit(train)
 train = pd.DataFrame(Scaler_train.transform(train), columns=train.columns, index=train.index)
 test = pd.DataFrame(Scaler_train.transform(test), columns=test.columns, index=test.index )
for xi in range(10):
recommendations = model.recommend(xi, user_items)
 print('USER',xi,train.iloc[xi].sort_values(ascending=False)[:3])
 for ri,prob in recommendations:
print('recommended ',ri,titles[ri],prob)
<categorify>
linreg = LinearRegression()
linreg.fit(train, target )
test['predictions']='np.nan'
for xi in range(len(train),len(total)) :
testxi=xi-len(train)
 testuserid=test.iloc[testxi]['user_id']
recommendations = model.recommend(testuserid, user_items)
 test.iat[testxi,141]=titles[recommendations[0][0]]
if xi/1000==int(xi/1000):
print('USER',testxi,total.iloc[xi].sort_values(ascending=False)[:3])
 for ri,prob in recommendations:
print('recommended ',ri,titles[ri],prob)
<save_to_csv>
eli5.show_weights(linreg )
test[['user_id','predictions']].to_csv('submit.csv',index=False )<load_from_csv>
coeff_linreg["LinRegress"] = coeff_linreg["LinRegress"].abs()
feature_score = pd.merge(feature_score, coeff_linreg, on='feature')
 feature_score = feature_score.fillna(0)
 feature_score = feature_score.set_index('feature')
 feature_score
train=pd.read_csv('.. /input/utkml/train_final.csv')
 test=pd.read_csv('.. /input/utkml/test_final.csv')
 total=train.append(test,ignore_index=True)
<sort_values>
y_preds_linreg = linreg.predict(test )
total.sort_values('user_id' )<feature_engineering>
feature_score = pd.DataFrame(
 preprocessing.MinMaxScaler().fit_transform(feature_score),
columns=feature_score.columns,
index=feature_score.index 
)
 feature_score['Mean'] = feature_score.mean(axis=1 )
datacol=total[['user_id','JOKE:5']]
datacol.columns=['user_id','rating']
datacol['item_id']=0
data=datacol.dropna()
for ci in range(2,141):
colnm=train.columns[ci]
datacol=total[['user_id',colnm]]
datacol.columns=['user_id','rating']
datacol['item_id']=ci-1
data=data.append(datacol.dropna())
 data
datacoo=coo_matrix(( data.rating,(data.item_id,data.user_id)))
 datacoo 
<import_modules>
w_lgb = 0.4
w_xgb = 0.5
w_linreg = 1 - w_lgb - w_xgb
w_linreg
feature_score['Merging'] = w_lgb*feature_score['LGB'] + w_xgb*feature_score['XGB'] + w_linreg*feature_score['LinRegress']
feature_score.sort_values('Merging', ascending=False )
ratings=datacoo
TFIDFRecommender, bm25_weight)
 log = logging.getLogger("implicit")
<choose_model_class>
def features_selection_by_weights(df, threshold):
features_list = df.feature.tolist()
features_best = []
for i in range(len(df)) :
feature_name = features_list[i]
feature_is_best = False
for col in feature_score_columns:
if df.loc[i, col] > threshold:
feature_is_best = True
if feature_is_best:
features_best.append(feature_name)
 return df[df['feature'].isin(features_best)].reset_index(drop=True )
start = time.time()
output_filename='output.txt'
model_name='bpr'
min_rating=-10.0,
titles=train.columns[1:]
ratings.data[ratings.data < min_rating] = 0
ratings.eliminate_zeros()
ratings.data = np.ones(len(ratings.data))
log.info("read data file in %s", time.time() - start)
 if model_name == "als":
model = AlternatingLeastSquares()
print("weighting matrix by bm25_weight")
 ratings =(bm25_weight(ratings, B=0.9)* 5 ).tocsr()
elif model_name == "bpr":
model = BayesianPersonalizedRanking()
print("weighting matrix by bm25_weight")
 ratings =(bm25_weight(ratings, B=0.9)* 5 ).tocsr()
elif model_name == "lmf":
model = LogisticMatrixFactorization()
elif model_name == "tfidf":
model = TFIDFRecommender()
elif model_name == "cosine":
model = CosineRecommender()
elif model_name == "bm25":
model = BM25Recommender(B=0.2)
 else:
raise NotImplementedError("TODO: model %s" % model_name)
 print("training model %s", model_name)
 start = time.time()
model.fit(ratings)
 print("trained model '%s' in %s", model_name, time.time() - start)
 log.debug("calculating top movies")
 user_count = np.ediff1d(ratings.indptr)
 to_generate = sorted(np.arange(len(titles)) , key=lambda x: -user_count[x])
 print("calculating similar movies")
 with tqdm.tqdm(total=len(to_generate)) as progress:
with codecs.open(output_filename, "w", "utf8")as o:
for movieid in to_generate:
print(movieid,model.similar_items(movieid))
if ratings.indptr[movieid] != ratings.indptr[movieid + 1]:
title = titles[movieid]
for other, score in model.similar_items(movieid):
" %(title, titles[other], score))
try:
print(title,titles[other],score)
 print()
except:
print(title,other,score)
 progress.update(1)
<sort_values>
threshold_fi = 0.25
feature_score_best = features_selection_by_weights(feature_score, threshold_fi)
 feature_score_best
for xi in range(10):
recommendations = model.recommend(xi, user_items)
 print('USER',xi,train.iloc[xi].sort_values(ascending=False)[:3])
 for ri,prob in recommendations:
print('recommended ',ri,titles[ri],prob)
<categorify>
y_preds = w_lgb*y_preds_lgb + w_xgb*y_preds_xgb + w_linreg*y_preds_linreg
submission['Survived'] = [1 if x>0.5 else 0 for x in y_preds]
submission.head()
test['predictions']='np.nan'
for xi in range(len(train),len(total)) :
testxi=xi-len(train)
 testuserid=test.iloc[testxi]['user_id']
recommendations = model.recommend(testuserid, user_items)
 test.iat[testxi,141]=titles[recommendations[0][0]]
if xi/1000==int(xi/1000):
print('USER',testxi,total.iloc[xi].sort_values(ascending=False)[:3])
 for ri,prob in recommendations:
print('recommended ',ri,titles[ri],prob)
<save_to_csv>
submission.to_csv('submission.csv', index=False )
<categorify><EOS> 
submission.to_csv('submission.csv', index=False )