Kaggle competition: Titanic - Machine Learning from Disaster
class SentimentAnalyzer(nn.Module):
def __init__(self):
super(SentimentAnalyzer, self ).__init__()
self.reset_structure(1,1)
 def reset_structure(self,vocab_size, num_labels):
self.W = nn.Linear(vocab_size, num_labels)
 def forward(self, text_vec):
return torch.sigmoid(self.W(text_vec))
def train(self,train_set,learning_rate,epochs):
self.w2idx = make_w2idx(train_set)
 self.reset_structure(len(self.w2idx),1)
 loss_func = nn.BCELoss()
optimizer = optim.SGD(self.parameters() , lr=learning_rate)
 train_dataset, dev_dataset = random_split(train_set, [20000, 5000])
 data_loader = DataLoader(train_dataset, batch_size=len(train_set), shuffle=True)
 max_acc = 0
for epoch in range(epochs):
global_logloss = 0.0
for Xbatch,Ybatch in data_loader:
for X, Y in zip(Xbatch,Ybatch):
self.zero_grad()
xvec = vectorize_text(X,self.w2idx)
 yvec = vectorize_target(Y)
 prob = self(xvec ).squeeze()
loss = loss_func(prob,yvec)
 loss.backward()
optimizer.step()
global_logloss += loss.item()
validation_acc = self.eval_test(dev_dataset)
 print("Epoch %d, mean cross entropy = %f, Validation accurracy : %f"%(epoch,global_logloss/len(train_set),validation_acc))
if validation_acc >= max_acc:
torch.save(self.state_dict() , 'sentiment_model.wt')
 max_acc = validation_acc
self.load_state_dict(torch.load('sentiment_model.wt'))
def eval_test(self,dev_set):
with torch.no_grad() :
data_loader = DataLoader(dev_set, batch_size=len(dev_set), shuffle=False)
 ncorrect = 0
N = 0
for Xbatch,Ybatch in data_loader:
for X,Y in zip(Xbatch,Ybatch):
xvec = vectorize_text(X,self.w2idx)
 prob = self(xvec ).squeeze()
if int(prob > 0.5)== int(Y):
ncorrect += 1
N += 1
return float(ncorrect)/float(N)
 def run_test(self,test_set,pred_filename):
with torch.no_grad() :
data_loader = DataLoader(test_set, batch_size=len(train_set), shuffle=False)
 idxList = []
sentList = []
for Xbatch,idxbatch in data_loader:
for X,idx in zip(Xbatch,idxbatch):
xvec = vectorize_text(X,self.w2idx)
 prob = self(xvec ).squeeze()
idxList.append(idx)
 sentList.append(int(prob > 0.5))
df = pa.DataFrame({'idx':idxList,'sentY':sentList})
 df.to_csv(pred_filename,index=False)
 print('done.' )<train_on_grid>
warnings.filterwarnings('ignore')
 %matplotlib inline
sent = SentimentAnalyzer()
sent.train(train_set,0.01,50 )<import_modules>
train_data = pd.read_csv(".. /input/train.csv")
 train_data.columns
import pandas as pd
import numpy as np<import_modules>
test = pd.read_csv(".. /input/test.csv")
 IDtest = test["PassengerId"]
import pandas as pd
import numpy as np<define_variables>
train_data.drop(['PassengerId','Ticket'], axis=1, inplace = True)
t = 732
m=8
n=8<load_from_csv>
train_data.isnull().sum()
df = pd.read_csv('.. /input/data.txt',skiprows=2,sep=' ',names=list(map(str,(list(range(n))))))<define_variables>
train_data['Age'].fillna(train_data['Age'].mean() , inplace = True)
 test['Age'].fillna(test['Age'].mean() , inplace = True)
 train_data["Embarked"] = train_data["Embarked"].fillna("C")
 test["Embarked"] = test["Embarked"].fillna("C")
 train_data['Fare'].fillna(train_data['Fare'].median() , inplace = True)
 test['Fare'].fillna(test['Fare'].median() , inplace = True)
 train_data.Cabin.fillna("N", inplace=True)
 test.Cabin.fillna("N", inplace=True )
arr = np.array(df )<prepare_x_and_y>
train_data.isnull().sum()
df_r = pd.DataFrame(columns=['id','m1','m2','m3','m4'])
 for i in range(m):
for j in range(n):
for k in range(732):
temp = td[::24,i,j]
temp = temp[temp!=-1]
temp2 = td[k,:,:]
temp2 = temp2[temp2!=-1]
temp3 = td[::24*7,i,j]
temp3 = temp3[temp3!=-1]
x = i//4
y = j//4
temp4 = td[k,x*4:(x+1)*4,y*4:(y+1)*4]
temp4 = temp4[temp4!=-1]
st = str(k)+':'+str(i)+':'+str(j)
 df_r = df_r.append({'id':st,'m1':temp.mean() ,'m2':temp2.mean() ,'m3':temp3.mean() ,'m4':temp4.mean() },ignore_index=True )<concatenate>
train_title = [i.split(",")[1].split(".")[0].strip() for i in train_data["Name"]]
train_data["Title"] = pd.Series(train_title)
 train_data["Title"].head()
new_df = df.stack()<create_dataframe>
test_title = [i.split(",")[1].split(".")[0].strip() for i in test["Name"]]
test["Title"] = pd.Series(test_title)
 test["Title"].head()
new_df = pd.DataFrame(new_df )<rename_columns>
train_data["Title"] = train_data["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train_data["Title"] = train_data["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train_data["Title"] = train_data["Title"].astype(int )
new_df.columns=['value']<rename_columns>
test["Title"] = test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test["Title"] = test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test["Title"] = test["Title"].astype(int )
new_df.columns= ['t','n','val']<feature_engineering>
train_data["Family_size"] = train_data["SibSp"] + train_data["Parch"] + 1
test["Family_size"] = test["SibSp"] + test["Parch"] + 1
new_df['m'] = new_df['t'].apply(lambda x : int(x)%8 )<feature_engineering>
train_data['Single'] = train_data['Family_size'].map(lambda s: 1 if s == 1 else 0)
 train_data['Small_family'] = train_data['Family_size'].map(lambda s: 1 if s == 2 else 0)
 train_data['Med_family'] = train_data['Family_size'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 train_data['Large_family'] = train_data['Family_size'].map(lambda s: 1 if s >= 5 else 0)
new_df['t'] = new_df['t'].apply(lambda x:x//8 )<feature_engineering>
test['Single'] = test['Family_size'].map(lambda s: 1 if s == 1 else 0)
 test['Small_family'] = test['Family_size'].map(lambda s: 1 if s == 2 else 0)
 test['Med_family'] = test['Family_size'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 test['Large_family'] = test['Family_size'].map(lambda s: 1 if s >= 5 else 0 )
new_df['hour'] = new_df['t'].apply(lambda x : x%24 )<feature_engineering>
train_data['survived_dead'] = train_data['Survived'].apply(lambda x : 'Survived' if x == 1 else 'Dead' )
new_df['day'] = new_df['t'].apply(lambda x : x//24 )<feature_engineering>
new_df['mm']=new_df['m']
new_df['nn']= new_df['n']<feature_engineering>
lbl = LabelEncoder()
lbl.fit(list(train_data['Embarked'].values))
train_data['Embarked'] = lbl.transform(list(train_data['Embarked'].values))
lbl.fit(list(test['Embarked'].values))
test['Embarked'] = lbl.transform(list(test['Embarked'].values))
new_df['id'] = new_df['t'].map(str)+':'+new_df['mm'].map(str)+':'+new_df['nn'].map(str )<merge>
train_data['FareBin'] = pd.qcut(train_data['Fare'], 4)
 train_data['AgeBin'] = pd.cut(train_data['Age'].astype(int), 5)
 test['FareBin'] = pd.qcut(test['Fare'], 4)
 test['AgeBin'] = pd.cut(test['Age'].astype(int), 5 )
new_df = df_r.set_index('id' ).join(new_df.set_index('id'))<normalization>
train_data['AgeBin_Code'] = lbl.fit_transform(train_data['AgeBin'])
 train_data['FareBin_Code'] = lbl.fit_transform(train_data['FareBin'])
 test['AgeBin_Code'] = lbl.fit_transform(test['AgeBin'])
 test['FareBin_Code'] = lbl.fit_transform(test['FareBin'] )
scaler = StandardScaler()
scaler.fit(new_df.drop(['val','t','mm','nn'],axis=1))<normalization>
def encode(x): return 1 if x == 'female' else 0
train_data['enc_sex'] = train_data.Sex.apply(encode)
 test['enc_sex'] = test.Sex.apply(encode )
scaler.transform(new_df.drop(['val','t','mm','nn'],axis=1))<normalization>
train_data["has_cabin"] = [0 if i == 'N'else 1 for i in train_data.Cabin]
test["has_cabin"] = [0 if i == 'N'else 1 for i in test.Cabin]
new_df[['m1','m2','m3','m4','n','m','hour','day']]=scaler.transform(new_df.drop(['val','t','mm','nn'],axis=1))<filter>
def detect_outliers(train_data,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(train_data[col], 25)
 Q3 = np.percentile(train_data[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = train_data[(train_data[col] < Q1 - outlier_step)|(train_data[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train_data,2,["Age","SibSp","Parch","Fare"])
train = new_df[new_df['val'] !=-1]<prepare_x_and_y>
train_data.loc[Outliers_to_drop]
X_train = train.drop(['val','t','mm','nn'], axis=1)
 y_train = train['val'].values<filter>
train_data = train_data.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
test = new_df[new_df['val'] == -1]<drop_column>
y_train = train_data["Survived"]
X_train = data.drop(labels = ["Survived"],axis = 1 )
X_test = test.drop(['val','t','mm','nn'], axis=1 )<import_modules>
test = test.select_dtypes(include=[np.number] ).interpolate().dropna()
test = test[X_train.columns]
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import adam<choose_model_class>
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 test = sc.transform(test )
model = Sequential()
model.add(Dense(128, activation='relu', input_shape=(8,)))
 model.add(Dense(256, activation='relu'))
model.add(Dense(256, activation='relu'))
model.add(Dense(1))
model.summary()<compute_test_metric>
kfold = StratifiedKFold(n_splits=10 )
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))<choose_model_class>
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [4],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsExtC.fit(X_train,y_train)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_
model.compile(loss=root_mean_squared_error, optimizer='adam' )<train_model>
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X_train,y_train)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_
model.fit(X_train, y_train, batch_size=128, epochs=240, verbose=1,validation_split=0.2 )<train_model>
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[30],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsadaDTC.fit(X_train,y_train)
 ada_best = gsadaDTC.best_estimator_
gsadaDTC.best_score_
model_XGB = XGBRegressor()
model_XGB.fit(X_train,y_train )<feature_engineering>
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_
def make_positive(x):
if x<0:
return 0
else:
return x<predict_on_test>
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_train,y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_
predict = model.predict(X_test )<feature_engineering>
votingC = VotingClassifier(estimators=[('rfc', RFC_best),('extc', ExtC_best),('svm',SVMC_best), 
('gbc',GBC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(X_train, y_train)
<feature_engineering><EOS> 
test_Survived = pd.Series(votingC.predict(test), name="Survived")
 Submission = pd.concat([IDtest,test_Survived],axis=1)
 Submission.to_csv("submission.csv",index=False )