Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv' )
%%time
df_category=pd.DataFrame(train.category_name.unique() ,columns=['category_name'])
 df_category['count']=df_category.category_name.apply(lambda x: len(train.category_name[train.category_name==x]))
df_category['category']=df_category.category_name.apply(lambda x:(x.split('/')[0]+'/'+x.split('/')[1]))
for i,cat in enumerate(df_category.category):
if df_category.iloc[i,1]<5000:
df_category.iloc[i,2]=(cat.split('/')[0]+'/'+cat.split('/')[0])
 else:
df_category.iloc[i,2]=cat
df_category['countF']=df_category.category.apply(lambda x: sum(df_category['count'][df_category.category==x]))
for i,cat in enumerate(df_category.category):
if df_category.iloc[i,3]<5000:
df_category.iloc[i,2]='other/other'
else:
df_category.iloc[i,2]=cat
df_category['countF']=df_category.category.apply(lambda x: sum(df_category['count'][df_category.category==x]))
df_category=df_category.drop(['count','countF'],axis=1)
 df_train=pd.merge(train,df_category,on='category_name',how='left')
 df_test=pd.merge(test,df_category,on='category_name',how='left')
 df_test.category.fillna('other/other',inplace=True)
 df_train=df_train.drop(['category_name'],axis=1)
 df_test=df_test.drop(['category_name'],axis=1 )<feature_engineering>
train_data.drop(['PassengerId'],axis=1,inplace=True)
 test_data.drop(['PassengerId'],axis=1,inplace=True )
def get_sparse(df,df1,df2,max_feature=300000):
Cvect=CountVectorizer(binary=True)
 Tvect=TfidfVectorizer(stop_words='english',ngram_range=(1,2),max_features=max_feature)
 vect_name=Tvect.fit(df.name)
 vect_name1=vect_name.transform(df.name)
 vect_name2=vect_name.transform(df1.name)
 vect_name3=vect_name.transform(df2.name)
 vect_icond1=sp.csr_matrix(pd.get_dummies(df.item_condition_id,sparse=True ).values)
 vect_icond2=sp.csr_matrix(pd.get_dummies(df1.item_condition_id,sparse=True ).values)
 vect_icond3=sp.csr_matrix(pd.get_dummies(df2.item_condition_id,sparse=True ).values)
 vect_brand=Cvect.fit(df.brand_name)
 vect_brand1=vect_brand.transform(df.brand_name)
 vect_brand2=vect_brand.transform(df1.brand_name)
 vect_brand3=vect_brand.transform(df2.brand_name)
 vect_ship1=sp.csr_matrix(pd.get_dummies(df.shipping,sparse=True ).values)
 vect_ship2=sp.csr_matrix(pd.get_dummies(df1.shipping,sparse=True ).values)
 vect_ship3=sp.csr_matrix(pd.get_dummies(df2.shipping,sparse=True ).values)
 vect_desc=Tvect.fit(df.item_description)
 vect_desc1=vect_desc.transform(df.item_description)
 vect_desc2=vect_desc.transform(df1.item_description)
 vect_desc3=vect_desc.transform(df2.item_description)
 vect_mcat=Cvect.fit(df.main_category)
 vect_mcat1=vect_mcat.transform(df.main_category)
 vect_mcat2=vect_mcat.transform(df1.main_category)
 vect_mcat3=vect_mcat.transform(df2.main_category)
 vect_scat=Cvect.fit(df.sub_category)
 vect_scat1=vect_scat.transform(df.sub_category)
 vect_scat2=vect_scat.transform(df1.sub_category)
 vect_scat3=vect_scat.transform(df2.sub_category)
 vect_itype=Cvect.fit(df.item_type)
 vect_itype1=vect_itype.transform(df.item_type)
 vect_itype2=vect_itype.transform(df1.item_type)
 vect_itype3=vect_itype.transform(df2.item_type)
 sp_mat=sp.hstack(( vect_name1,vect_icond1,vect_brand1,vect_ship1,vect_desc1,vect_mcat1,vect_scat1,vect_itype1)).tocsr()
sp_mat1=sp.hstack(( vect_name2,vect_icond2,vect_brand2,vect_ship2,vect_desc2,vect_mcat2,vect_scat2,vect_itype2)).tocsr()
sp_mat2=sp.hstack(( vect_name3,vect_icond3,vect_brand3,vect_ship3,vect_desc3,vect_mcat3,vect_scat3,vect_itype3)).tocsr()
return(sp_mat,sp_mat1,sp_mat2)
 def get_rmsle(y_true, y_pred):
return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))
<feature_engineering>
print("Train dataset
", train_data.isna().sum())
 print("
Test dataset
", test_data.isna().sum() )
%%time
df_category=pd.DataFrame(df_train.category.unique() ,columns=['category'])
 df_category['count']=df_category.category.apply(lambda x: len(df_train.category[df_train.category==x]))
df_category['rmsle']=0
df_category['skewness']=0
df_category['X_shape']=''
df_category['test_len']=0
df_predictions=pd.DataFrame(columns=['test_id','price'])
 df_predictions
for i,cat in enumerate(df_category.category):
try:
df_cat=df_train[df_train.category==cat]
df_cat=df_cat.drop(['category'],axis=1)
 df_cat_test=df_test[df_test.category==cat]
df_cat_test=df_cat_test.drop(['category'],axis=1)
 df_category.iloc[i,3]=stat.skew(df_cat.price)
 df_cat_y=np.array(np.log1p(df_cat.price))
df_cat_X=df_cat.drop(['price'],axis=1)
 df_pred=pd.DataFrame(df_cat_test.test_id,columns=['test_id','price'])
 df_cat_test=df_cat_test.drop(['test_id'],axis=1)
 for col in df_cat_test.columns.values:
df_cat_test[col]=df_cat_test[col].astype('str')
 for col in df_cat_X.columns.values:
df_cat_X[col]=df_cat_X[col].astype('str')
 df_cat_X=df_cat_X.drop(['train_id'],axis=1)
 j=0
while True:
j+=1
if j==10:
break
try:
train_X,test_X,train_y,test_y=train_test_split(df_cat_X,df_cat_y)
 print('In loop',i,'cat:',cat)
 if df_category.iloc[i,1]>=150000:
sp_train,sp_test,sp_pred=get_sparse(train_X,test_X,df_cat_test)
 model = Ridge(alpha=4,solver='auto',fit_intercept=True,max_iter=1000,normalize=False)
 elif df_category.iloc[i,1]<150000 and df_category.iloc[i,1]>=100000:
sp_train,sp_test,sp_pred=get_sparse(train_X,test_X,df_cat_test)
 model = Ridge(alpha=3.5,solver='auto',fit_intercept=True,max_iter=1000,normalize=False)
 elif df_category.iloc[i,1]<100000 and df_category.iloc[i,1]>=50000:
sp_train,sp_test,sp_pred=get_sparse(train_X,test_X,df_cat_test)
 model = Ridge(alpha=2,solver='auto',fit_intercept=True,max_iter=1000,normalize=False)
 elif df_category.iloc[i,1]<50000 and df_category.iloc[i,1]>=10000:
sp_train,sp_test,sp_pred=get_sparse(train_X,test_X,df_cat_test)
 model=Ridge(alpha=2,solver='auto',fit_intercept=True,max_iter=1000,normalize=False)
 else:
sp_train,sp_test,sp_pred=get_sparse(train_X,test_X,df_cat_test)
 model=Ridge(solver='auto',fit_intercept=True,max_iter=1000,normalize=False)
 if j==8:
if sp_train.shape[1]!=sp_test.shape[1]:
dif=np.abs(sp_train.shape[1]-sp_test.shape[1])
 if sp_train.shape[1]>sp_test.shape[1]:
sp_test=sp.hstack(( sp_test,sp_test[:,:dif])).tocsr()
else:
sp_test=sp_test[:,dif:]
print('train:',sp_train.shape[1],'test:',sp_test.shape[1],'predict:',sp_pred.shape[1])
 if sp_train.shape[1]!=sp_pred.shape[1]:
dif=np.abs(sp_train.shape[1]-sp_pred.shape[1])
 if sp_train.shape[1]>sp_pred.shape[1]:
sp_pred=sp.hstack(( sp_pred,sp_pred[:,:dif])).tocsr()
else:
sp_pred=sp_pred[:,dif:]
print('train:',sp_train.shape[1],'test:',sp_test.shape[1],'predict:',sp_pred.shape[1])
 model.fit(sp_train,train_y)
 preds = model.predict(sp_test)
 df_pred['price']=model.predict(sp_pred)
 df_predictions=df_predictions.append(df_pred)
 break
except:
print('runtime error1')
 continue
df_category.iloc[i,2]=get_rmsle(test_y,preds)
 df_category.iloc[i,5]=len(df_cat_test)
 if i==0:
f_test_y=test_y
f_preds=preds
else:
f_test_y=np.concatenate([f_test_y,test_y])
 f_preds=np.concatenate([f_preds,preds])
 df_category.iloc[i,4]=str(sp_train.shape)
 print('ridge score:',cat,'rmsle:', df_category.iloc[i,2],'len',len(df_cat_test))
del df_cat, df_cat_X, train_X, test_X, train_y, test_y, sp_train, sp_test, preds,df_cat_test,df_pred
except:
print('runtime error2')
 continue
print(get_rmsle(f_test_y,f_preds))
df_predictions['price']=df_predictions.price.apply(lambda x: np.expm1(x))
print('Test set len:',len(df_test),'Predicted values len:',len(df_predictions.price),'difference:',len(df_test)-len(df_predictions.price)) 
<sort_values>
train_age = pd.get_dummies(train_data['Age_Cat'])
 test_age = pd.get_dummies(test_data['Age_Cat'])
 train_data.drop(['Age_Cat'],axis=1,inplace=True)
 test_data.drop(['Age_Cat'],axis=1,inplace=True)
 train_data = pd.concat([train_data,train_age],axis=1)
 test_data = pd.concat([test_data,test_age],axis=1 )
df_predictions=df_predictions.sort_values(by='test_id' ).reset_index()
df_predictions=df_predictions.drop(['index'],axis=1)
 df_predictions<filter>
sex1 = pd.get_dummies(train_data['Sex'])
 sex2 = pd.get_dummies(test_data['Sex'])
 train_data.drop(['Sex'],axis=1,inplace=True)
 test_data.drop(['Sex'],axis=1,inplace=True)
 train_data = pd.concat([train_data,sex1],axis=1)
 test_data = pd.concat([test_data,sex2],axis=1)
df_predictions[df_predictions.test_id==535]<filter>
train_data.drop(['female'],axis=1,inplace=True)
 test_data.drop(['female'],axis=1,inplace=True )
test[test.test_id==535]<sort_values>
train_data.rename(columns={'male':'gender'}, inplace = True)
 test_data.rename(columns={'male':'gender'}, inplace = True )
df_category.sort_values(by='rmsle' )<save_to_csv>
pclass_age_map = {
1: 38,
2: 30,
3: 25,
}
def replace_age_na(x_df, fill_map):
cond=x_df['Age'].isna()
res =x_df.loc[cond,'Pclass'].map(fill_map)
 x_df.loc[cond,'Age']=res
return x_df
df_predictions.to_csv('ridge_submission.csv',index=False )<compute_test_metric>
train_data =(train_data.pipe(replace_age_na, pclass_age_map))
test_data =(test_data.pipe(replace_age_na, pclass_age_map))
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5 
<load_from_csv>
Class_train = pd.get_dummies(train_data['Pclass'].astype(str))
Class_test = pd.get_dummies(test_data['Pclass'].astype(str))
train_data = pd.concat([train_data,Class_train],axis=1)
 test_data = pd.concat([test_data,Class_test],axis=1)
print("Loading data...")
 start_time = time.time()
train = pd.read_csv('.. /input/train.tsv', sep='\t')
 test = pd.read_csv('.. /input/test.tsv', sep='\t')
 train['target'] = np.log1p(train['price'])
 print(train.shape)
 print(test.shape)
 test_len = test.shape[0]<categorify>
train_data.rename(columns = {'1':'Class 1', '2':'Class 2', '3':'Class 3'}, inplace = True)
 test_data.rename(columns = {'1':'Class 1', '2':'Class 2', '3':'Class 3'}, inplace = True )
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="", inplace=True)
 dataset.brand_name.fillna(value="", inplace=True)
 dataset.item_description.fillna(value="", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )<categorify>
Fare_0 = []
Fare_1 = []
for i in range(0,len(train_data["Fare"])) :
if train_data["Survived"][i] == 0:
Fare_0.append(train_data["Fare"][i])
 else:
Fare_1.append(train_data["Fare"][i] )
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train['category'] = le.transform(train.category_name)
 test['category'] = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train['brand'] = le.transform(train.brand_name)
 test['brand'] = le.transform(test.brand_name)
 del le
train.head(3)
<feature_engineering>
test_data.loc[test_data['Fare'].isnull() ]
print("Text to seq process...")
 raw_text = np.hstack([train.category_name.str.lower() ,
train.item_description.str.lower() ,
train.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_category_name"] = tok_raw.texts_to_sequences(train.category_name.str.lower())
 test["seq_category_name"] = tok_raw.texts_to_sequences(test.category_name.str.lower())
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )<split>
test_data.loc[test_data['Name']=='Storey, Mr.Thomas']
dtrain, dvalid = train_test_split(train, random_state=233, train_size=0.99)
 print(dtrain.shape)
 print(dvalid.shape )<define_variables>
embark1 = pd.get_dummies(train_data['Embarked'])
 embark2 = pd.get_dummies(test_data['Embarked'])
 train_data.drop(['Embarked'],axis=1,inplace=True)
 test_data.drop(['Embarked'],axis=1,inplace=True)
 train_data = pd.concat([train_data,embark1],axis=1)
 test_data = pd.concat([test_data,embark2],axis=1 )
MAX_NAME_SEQ = max_name_seq
MAX_ITEM_DESC_SEQ = max_seq_item_description
MAX_CATEGORY_NAME_SEQ = max_seq_category_name
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_category_name.max())
 , np.max(test.seq_category_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category.max() , test.category.max() ])+1
MAX_BRAND = np.max([train.brand.max() , test.brand.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() ,
test.item_condition_id.max() ])+1<string_transform>
train_data.rename(columns = {'C':'Cherbourg', 'Q':'Queenstown', 'S':'Southampton'}, inplace = True)
 test_data.rename(columns = {'C':'Cherbourg', 'Q':'Queenstown', 'S':'Southampton'}, inplace = True )
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description
, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand': np.array(dataset.brand)
 ,'category': np.array(dataset.category)
 ,'category_name': pad_sequences(dataset.seq_category_name
, maxlen=MAX_CATEGORY_NAME_SEQ)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )<define_variables>
def fam(x):
if(x['SibSp'] + x['Parch'])> 0:
return 1
else:
return 0
train_data['Family'] = train_data.apply(fam, axis = 1)
 test_data['Family'] = test_data.apply(fam, axis = 1 )
Activation, concatenate, GRU, Embedding, Flatten
dr = 0.25
def get_model() :
dr_r = dr
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand = Input(shape=[1], name="brand")
 category = Input(shape=[1], name="category")
 category_name = Input(shape=[X_train["category_name"].shape[1]],
name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_size = 64
emb_name = Embedding(MAX_TEXT, emb_size )(name)
 emb_item_desc = Embedding(MAX_TEXT, emb_size )(item_desc)
 emb_category_name = Embedding(MAX_TEXT, emb_size )(category_name)
 emb_brand = Embedding(MAX_BRAND, emb_size )(brand)
 emb_category = Embedding(MAX_CATEGORY, 10 )(category)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_category_name)
 rnn_layer3 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_category)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, rnn_layer3
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128 )(main_l))
main_l = Dropout(dr_r )(Dense(32 )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand
, category, category_name
, item_condition, num_vars], output)
 optimizer = optimizers.Adam()
model.compile(loss="mse",
optimizer=optimizer)
 return model 
<predict_on_test>
train_data.drop(['SibSp','Parch'],axis=1, inplace = True)
 test_data.drop(['SibSp','Parch'],axis=1, inplace = True )
def eval_model(model):
val_preds = model.predict(X_valid)
 val_preds = np.expm1(val_preds)
 y_true = np.array(dvalid.price.values)
 y_pred = val_preds[:, 0]
v_rmsle = rmsle(y_true, y_pred)
 return v_rmsle<find_best_params>
train_data["Cabin"] = pd.Series([i[0]
if not pd.isnull(i)
 else 'X'
for i in train_data['Cabin'] ])
 test_data["Cabin"] = pd.Series([i[0]
if not pd.isnull(i)
 else 'X' for i in test_data['Cabin'] ] )
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
del train
gc.collect()<train_model>
Train_cabin = pd.get_dummies(train_data['Cabin'])
 Test_cabin = pd.get_dummies(test_data['Cabin'])
 train_data = pd.concat([train_data,Train_cabin],axis=1)
 test_data = pd.concat([test_data,Test_cabin],axis=1)
epochs = 1
BATCH_SIZE = 512 * 3
steps = int(len(X_train['name'])/BATCH_SIZE)* epochs
lr_init, lr_fin = 0.013, 0.009
lr_decay = exp_decay(lr_init, lr_fin, steps)
 model = get_model()
K.set_value(model.optimizer.lr, lr_init)
 K.set_value(model.optimizer.decay, lr_decay)
 history = model.fit(X_train, dtrain.target
, epochs=epochs
, batch_size=BATCH_SIZE
, validation_split=0.01
, verbose=1
)<compute_test_metric>
train_title = [i.split(",")[1].split(".")[0].strip()
for i in train_data["Name"]]
train_data["Title"] = pd.Series(train_title)
 test_title = [i.split(",")[1].split(".")[0].strip()
for i in test_data["Name"]]
test_data["Title"] = pd.Series(test_title )
v_rmsle = eval_model(model)
 print(" RMSLE error on dev test: "+str(v_rmsle))
log_subname = '_'.join(['ep', str(epochs),
'bs', str(BATCH_SIZE),
'lrI', str(lr_init),
'lrF', str(lr_fin),
'dr', str(dr)] )<predict_on_test>
print('The list of titles for the passengers names are as follow
',train_data["Title"].value_counts() )
preds = model.predict(X_test, batch_size=BATCH_SIZE)
 preds = np.expm1(preds)
 submission = test[["test_id"]][:test_len]
submission["price"] = preds[:test_len]<save_to_csv>
print('The list of titles for the passengers names are as follow
',test_data["Title"].value_counts() )
submission.to_csv("sample_submission.csv", index=False)
 print('[{}] Finished submission...'.format(time.time() - start_time)) 
<load_from_csv>
Train_title = pd.get_dummies(train_data['Title'])
 Test_title = pd.get_dummies(test_data['Title'])
 train_data = pd.concat([train_data,Train_title],axis=1)
 test_data = pd.concat([test_data,Test_title],axis=1)
start_time = time.time()
df_train = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_test = pd.read_csv('.. /input/test.tsv', sep='\t')
 df_train[df_train.price >= 3.0]
train_rows = len(df_train)
 all_data = pd.concat([df_train, df_test])
 print("[{}] Data concatenated.".format(time.time() - start_time))
def transform_values(data):
data.fillna(value='missing', inplace = True)
 return data
all_data = transform_values(all_data)
 def transform_category_name(category_name):
try:
main, sub1, sub2= category_name.split('/')
 return main, sub1, sub2
except:
return 'missing', 'missing', 'missing'
all_data['category_main'], all_data['category_sub1'], all_data['category_sub2'] = zip(*all_data['category_name'].apply(transform_category_name))
all_data[all_data.item_description != 'missing']
print("[{}] Data transformed.".format(time.time() - start_time))
name_cv = CountVectorizer(ngram_range=(1,2), min_df=7, stop_words='english')
 X_name = name_cv.fit_transform(all_data['name'])
 print("[{}] Name vector created.".format(time.time() - start_time))
cat_cv = CountVectorizer()
X_catmain = cat_cv.fit_transform(all_data['category_main'],)
 X_catsub1 = cat_cv.fit_transform(all_data['category_sub1'])
 X_catsub2 = cat_cv.fit_transform(all_data['category_sub2'])
 print("[{}] Category vectors created.".format(time.time() - start_time))
def descrLength(description):
try:
if description == "missing":
return 0
else:
words = [w for w in description.split(" ")]
return len(words)
 except:
return 0
all_data['descr_length'] = all_data['item_description'].apply(lambda x: descrLength(x))
all_data['descr_length'] = all_data['descr_length'].astype(str)
 X_length = cat_cv.fit_transform(all_data['descr_length'])
 print("[{}] Description length vector created.".format(time.time() - start_time))
vectorizer = TfidfVectorizer(ngram_range=(1,2), max_features=90000, strip_accents="ascii",
stop_words='english')
 X_description = vectorizer.fit_transform(all_data['item_description'])
 print("[{}] Description vector created.".format(time.time() - start_time))
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(all_data['brand_name'])
 print("[{}] Brand vector created.".format(time.time() - start_time))
X_dummies = csr_matrix(pd.get_dummies(all_data[['item_condition_id', 'shipping']],
sparse=True ).values)
 print("[{}] Shipping and condition vector created.".format(time.time() - start_time))
train_X = hstack(( X_name[:train_rows], X_catmain[:train_rows], X_catsub1[:train_rows],
X_catsub2[:train_rows], X_brand[:train_rows], X_length[:train_rows], X_description[:train_rows],
X_dummies[:train_rows]))
print('[{}] Training data stacked'.format(time.time() - start_time))
test_X = hstack(( X_name[train_rows:], X_catmain[train_rows:], X_catsub1[train_rows:],
X_catsub2[train_rows:], X_brand[train_rows:], X_length[train_rows:], X_description[train_rows:],
X_dummies[train_rows:]))
print('[{}] Test data stacked.'.format(time.time() - start_time))
Y_train = np.log1p(df_train['price'])
 print("[{}] Data processing and feature extraction complete.".format(time.time() - start_time))<train_model>
Ticket1 = []
for i in list(train_data.Ticket):
if not i.isdigit() :
Ticket1.append(i.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket1.append("X")
 train_data["Ticket"] = Ticket1
Ticket2 = []
for j in list(test_data.Ticket):
if not j.isdigit() :
Ticket2.append(j.replace(".","" ).replace("/","" ).strip().split(' ')[0])
 else:
Ticket2.append("X")
 test_data["Ticket"] = Ticket2
ridge_time = time.time()
data_train_X, data_val_X, data_train_Y, data_val_Y = train_test_split(train_X, Y_train)
 print('[{}] Data splitted.'.format(time.time() - ridge_time))
clf1 = Ridge(alpha=5.3, fit_intercept=True, normalize=False,
copy_X=True, max_iter=None, tol=0.01, solver='auto', random_state=None)
 clf1.fit(train_X, Y_train)
 print("[{}] Data 1 fitted.".format(time.time() - ridge_time))
clf2 = Ridge(alpha=.75, fit_intercept=True, normalize=False,
copy_X=True, max_iter=None, tol=0.01, solver='auto', random_state=100)
 clf2.fit(train_X, Y_train)
 print("[{}] Data 2 fitted.".format(time.time() - ridge_time)) 
<split>
Train_ticket = pd.get_dummies(train_data['Ticket'], prefix="T")
 Test_ticket = pd.get_dummies(test_data['Ticket'], prefix="T")
 train_data = pd.concat([train_data,Train_ticket],axis=1)
 test_data = pd.concat([test_data,Test_ticket],axis=1)
lgb_time = time.time()
data_train_X, data_val_X, data_train_Y, data_val_Y = train_test_split(train_X, Y_train)
 print('[{}] Data splitted.'.format(time.time() - lgb_time))
dataset_train = lgb.Dataset(data_train_X, label=data_train_Y)
 dataset_val = lgb.Dataset(data_val_X, label=data_val_Y)
 watchlist = [dataset_train, dataset_val]
params = {
'learning_rate': 0.35,
'application': 'regression',
'max_depth': 6,
'num_leaves': 60,
'verbosity': -1,
'metric': 'RMSE',
'data_random_seed': 1,
'bagging_fraction': 0.5,
'nthread': 4
}
lgb_model = lgb.train(params, train_set=dataset_train, num_boost_round=7500, valid_sets=watchlist,
early_stopping_rounds=1000, verbose_eval=500)
 print("[{}] Model completed.".format(time.time() - lgb_time))
print("[{}] Prediction completed.".format(time.time() - lgb_time)) 
<save_to_csv>
train_data.drop(['Age','Cabin', 'Name', 'Title', 'Ticket'],axis=1,inplace=True)
 test_data.drop(['Age','Cabin', 'Name', 'Title', 'Ticket'],axis=1,inplace=True )
final_time = time.time()
final_ridge_pred1 = clf1.predict(test_X)
 print("[{}] First Ridge prediction completed.".format(time.time() - final_time))
final_ridge_pred2 = clf2.predict(test_X)
 print("[{}] Second Ridge prediction completed.".format(time.time() - final_time))
final_lgb_pred = lgb_model.predict(test_X)
 print("[{}] Lgb prediction completed.".format(time.time() - final_time))
final_pred = 0.25*final_ridge_pred1 + 0.25*final_ridge_pred2 + 0.5*final_lgb_pred
submission = pd.DataFrame(data=df_test[['test_id']])
 submission['price'] = np.expm1(final_pred)
 submission.to_csv("submission_ridge_lgbm.csv", index=False )<feature_engineering>
intersect_set = np.intersect1d(train_data.columns, test_data.columns)
 print("Selected Features = ", intersect_set )
<load_from_csv>
train_data_select = train_data[intersect_set]
print("Sum of NaN train dataset = ", train_data_select.isnull().sum().sum())
 test_data_select = test_data[intersect_set]
print("Sum of NaN test dataset = ",test_data_select.isnull().sum().sum())
test_df=pd.read_csv('.. /input/test.tsv',sep='\t',quoting=3)
 train_df=pd.read_csv('.. /input/train.tsv',sep='\t')
<count_missing_values>
X= preprocessing.StandardScaler().fit_transform(train_data_select )
train_df.isnull().any()<split>
y = train_data['Survived']
df_train=train_df
df_test=test_df<categorify>
from sklearn.pipeline import Pipeline
from sklearn import svm
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier, plot_tree
import xgboost as xgb
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn import metrics
from sklearn.model_selection import KFold, cross_val_score
df_train.category_name.fillna(value='missing',inplace=True)
 df_train.brand_name.fillna(value='missing',inplace=True)
 df_train.item_description.fillna(value='missing',inplace=True)
 df_test.category_name.fillna(value='missing',inplace=True)
 df_test.brand_name.fillna(value='missing',inplace=True)
 df_test.item_description.fillna(value='missing',inplace=True )<string_transform>
KFold_Score = pd.DataFrame()
classifiers = ['KNeighborsClassifier','Radial SVM', 'LogisticRegression',
'DecisionTreeClassifier', 'RandomForestClassifier', 'AdaBoostClassifier',
'XGBoostClassifier']
models = [KNeighborsClassifier() ,
svm.SVC(kernel='rbf'),
LogisticRegression(max_iter = 1000),
DecisionTreeClassifier(random_state=42, max_depth = 5),
RandomForestClassifier(n_estimators=200, random_state=42, max_depth = 5),
AdaBoostClassifier(n_estimators=200, random_state = 42),
xgb.XGBClassifier(n_estimators=200, use_label_encoder=False, eval_metric = 'logloss'),
]
j = 0
for i in models:
model = i
cv = KFold(n_splits=5, random_state=42, shuffle=True)
 KFold_Score[classifiers[j]] =(cross_val_score(model, X, np.ravel(y), scoring = 'accuracy', cv=cv))
j = j+1
def split_txt(text):
if text=='missing':
return ['missing']*3
else:
return text.split('/' )<feature_engineering>
mean = pd.DataFrame(KFold_Score.mean() , index= classifiers)
 KFold_Score = pd.concat([KFold_Score,mean.T])
 KFold_Score.index=['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5','Mean']
KFold_Score.T.sort_values(by=['Mean'], ascending = False )
df_train['cat1'],df_train['cat2'],df_train['cat3']=zip(*df_train.category_name.apply(lambda x: split_txt(x)))
 df_test['cat1'],df_test['cat2'],df_test['cat3']=zip(*df_test.category_name.apply(lambda x: split_txt(x)) )<import_modules>
X_test = preprocessing.StandardScaler().fit_transform(test_data_select )
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.text import text_to_word_sequence<feature_engineering>
rfc = RandomForestClassifier(n_estimators=200, random_state=42, max_depth = 5 )
Token=Tokenizer()
text_all=np.hstack([df_train.item_description.str.lower() ,df_train.name.str.lower() ])
 Token.fit_on_texts(text_all)
 df_train['name_sequence']=Token.texts_to_sequences(df_train.name)
 df_train['item_des_sequence']=Token.texts_to_sequences(df_train.item_description)
 df_test['name_sequence']=Token.texts_to_sequences(df_test.name)
 df_test['item_des_sequence']=Token.texts_to_sequences(df_test.item_description )<import_modules>
param_grid = {
'n_estimators': [ 100, 200,300],
'max_features': ['auto', 'sqrt'],
'max_depth' : [5, 6,7,8],
'criterion' :['gini', 'entropy']
}
from keras.models import Model
from keras.layers import Input,Embedding, Dense, Dropout,Recurrent,Flatten,concatenate,GRU<categorify>
CV_rfc = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5)
 CV_rfc.fit(X,y)
 CV_rfc.best_params_
Le_cat1=LabelEncoder()
Le_cat2=LabelEncoder()
Le_cat3=LabelEncoder()
Le_brand=LabelEncoder()
raw_cat1=np.hstack([np.array(df_train.cat1),np.array(df_test.cat1)])
 raw_cat2=np.hstack([np.array(df_train.cat2),np.array(df_test.cat2)])
 raw_cat3=np.hstack([np.array(df_train.cat3),np.array(df_test.cat3)])
 raw_brand=np.hstack([np.array(df_train.brand_name),np.array(df_test.brand_name)])
 Le_cat1.fit(raw_cat1)
 Le_cat2.fit(raw_cat2)
 Le_cat3.fit(raw_cat3)
 Le_brand.fit(raw_brand)
 df_train.cat1=Le_cat1.transform(df_train.cat1)
 df_train.cat2=Le_cat2.transform(df_train.cat2)
 df_train.cat3=Le_cat3.transform(df_train.cat3)
 df_train.brand_name=Le_brand.transform(df_train.brand_name)
 df_test.cat1=Le_cat1.transform(df_test.cat1)
 df_test.cat2=Le_cat2.transform(df_test.cat2)
 df_test.cat3=Le_cat3.transform(df_test.cat3)
 df_test.brand_name=Le_brand.transform(df_test.brand_name )<define_variables>
select_rfc = RandomForestClassifier(random_state=42, n_estimators= 100, criterion = 'entropy',max_features = 'auto',max_depth = 8)
 select_rfc.fit(X, y )
max_name_seq = 10
max_item_desc = 75
max_text = np.max([np.max(df_train.name_sequence.max())
 , np.max(df_test.name_sequence.max())
 , np.max(df_train.item_des_sequence.max())
 , np.max(df_test.item_des_sequence.max())])+5
max_cat1 = np.max([df_train.cat1.max() , df_test.cat1.max() ])+1
max_cat2 = np.max([df_train.cat2.max() , df_test.cat2.max() ])+1
max_cat3 = np.max([df_train.cat3.max() , df_test.cat3.max() ])+1
max_brand = np.max([df_train.brand_name.max() , df_test.brand_name.max() ])+1
def extract_data(d_in):
d_out={
'name':pad_sequences(d_in.name_sequence,maxlen=max_name_seq),
'item_desc':pad_sequences(d_in.item_des_sequence,maxlen=max_item_desc),
'brand_name':np.array(d_in.brand_name ).reshape(-1,1),
'cat1':np.array(d_in.cat1 ).reshape(-1,1),
'cat2':np.array(d_in.cat2 ).reshape(-1,1),
'cat3':np.array(d_in.cat3 ).reshape(-1,1),
'item_condition':np.array(d_in.item_condition_id ).reshape(-1,1),
'shipping':np.array(d_in.shipping ).reshape(-1,1)
 }
return d_out
x_train,x_valid=train_test_split(df_train,test_size=0.2,random_state=1)
 train_keras=extract_data(x_train)
 valid_keras=extract_data(x_valid)
 test_kreas=extract_data(df_test )<define_variables>
y_test = select_rfc.predict(X_test)
 print(y_test )
name=Input(shape=[train_keras['name'].shape[1]],name='name')
 item_desc=Input(shape=[train_keras['item_desc'].shape[1]],name='item_desc')
 brand_name=Input(shape=[train_keras['brand_name'].shape[1]],name='brand_name')
 cat1=Input(shape=[train_keras['cat1'].shape[1]],name='cat1')
 cat2=Input(shape=[train_keras['cat2'].shape[1]],name='cat2')
 cat3=Input(shape=[train_keras['cat3'].shape[1]],name='cat3')
 item_condition=Input(shape=[train_keras['item_condition'].shape[1]],name='item_condition')
 ship=Input(shape=[train_keras['shipping'].shape[1]],name='shipping')
<choose_model_class>
test_data2 = pd.read_csv('.. /input/titanic/test.csv')
 my_submission = pd.DataFrame({'PassengerId': test_data2.PassengerId,'Survived': y_test})
 my_submission.to_csv('submission.csv', index=False)
 my_submission.head()