Kaggle competition: Titanic - Machine Learning from Disaster
<compute_test_metric><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
import pandas as pd
import numpy as np
pickle.dump(model_FM_FTRL, open("model_FM_FTRL.pickle", "wb"),protocol=4)
 model_FM_FTRL = None
del(model_FM_FTRL)
 gc.collect()<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
fselect = SelectKBest(f_regression, k=48000)
 train_kbest_features = fselect.fit_transform(sparse_merge_train, Y_train)
 cv_kbest_features = fselect.transform(sparse_merge_cv )<load_pretrained>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
pickle.dump(fselect, open("fselect.pickle", "wb"),protocol=4)
 fselect = None
del(fselect)
 gc.collect()<drop_column>
test_key = test.PassengerId
sparse_merge_train= None
sparse_merge_cv = None
del(sparse_merge_train,sparse_merge_cv)
 gc.collect()<feature_engineering>
sns.set()
tfidf_desc = TfidfVectorizer(max_features=500000,
ngram_range=(1, 3),
stop_words=None)
 X_desc_train_tfidf = tfidf_desc.fit_transform(X_train['item_description'] )<categorify>
train_test_data = [train, test]
for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )
X_desc_cv_tfidf = tfidf_desc.transform(X_cv['item_description'] )<load_pretrained>
for dataset in train_test_data:
dataset['Family_size'] = dataset["SibSp"] + dataset["Parch"] + 1
pickle.dump(tfidf_desc, open("tfidf_desc.pickle", "wb"),protocol=4)
 tfidf_desc = None
del(tfidf_desc)
 gc.collect()<feature_engineering>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3,
"Master": 5, "Dr": 4, "Rev": 4, "Col": 4, "Major": 4, "Mlle": 4,"Countess": 4,
"Ms": 4, "Lady": 4, "Jonkheer": 4, "Don": 4, "Dona" : 4, "Mme": 4,"Capt": 4,"Sir": 4 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )
tfidf_name = TfidfVectorizer(max_features=250000,
ngram_range=(1, 3),
stop_words=None)
 X_name_train_tfidf = tfidf_name.fit_transform(X_train['name'] )<categorify>
for dataset in train_test_data:
dataset['Last_Name'] = dataset['Name'].apply(lambda x: str.split(x, ",")[0] )
X_name_cv_tfidf =tfidf_name.transform(X_cv['name'] )<load_pretrained>
total_data = train.append(test, sort=False )
pickle.dump(tfidf_name, open("tfidf_name.pickle", "wb"),protocol=4)
 tfidf_name = None
del(tfidf_name)
 gc.collect()<concatenate>
def fill_missing_cabin(data, match_data):
cabin_list = []
for i in range(len(data)) :
if data.Cabin[i] is np.nan:
cabin_mode = match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Cabin.mode()
if len(cabin_mode)> 0:
cabin_list.append(cabin_mode[0])
 else:
cabin_list.append(np.nan)
 else:
cabin_list.append(data.Cabin[i])
 return cabin_list
sparse_merge_train_1 = hstack(( X_name_train_tfidf , X_desc_train_tfidf, X_brand_train, X_main_cat_train,
X_main_sub_cat_1_train, X_main_sub_cat_2_train,X_dummies_train_1)).tocsr()<drop_column>
train['Cabin'] = fill_missing_cabin(train, total_data)
 test['Cabin'] = fill_missing_cabin(test, total_data )
X_dummies_train_1 = None
X_brand_train = None
X_main_cat_train = None
X_main_sub_cat_1_train = None
X_main_sub_cat_2_train = None
X_name_train_tfidf = None
X_desc_train_tfidf = None
del(X_dummies_train_1, X_brand_train, X_main_cat_train,\
X_main_sub_cat_1_train, X_main_sub_cat_2_train, X_name_train_tfidf,X_desc_train_tfidf)
 gc.collect()<concatenate>
for dataset in train_test_data:
dataset['Cabin_letter'] = dataset['Cabin'].str[:1].fillna('Missing_cabin_info' )
sparse_merge_cv_1 = hstack(( X_name_cv_tfidf , X_desc_cv_tfidf, X_brand_cv, X_main_cat_cv,
X_main_sub_cat_1_cv, X_main_sub_cat_2_cv,X_dummies_cv_1)).tocsr()<drop_column>
def get_cabin_count(data, column=''):
cabin_count_list=[]
for i in range(len(data)) :
if data[column][i] is np.nan:
cabin_count_list.append(0)
 else:
cabin_count_list.append(data[column][i].strip().count(' ')+1)
 return cabin_count_list
X_dummies_cv_1 = None
X_brand_cv = None
X_main_cat_cv = None
X_main_sub_cat_1_cv = None
X_main_sub_cat_2_cv = None
X_name_cv_tfidf = None
X_desc_cv_tfidf = None
del(X_dummies_cv_1, X_brand_cv, X_main_cat_cv,\
X_main_sub_cat_1_cv, X_main_sub_cat_2_cv, X_name_cv_tfidf,X_desc_cv_tfidf)
 gc.collect()<train_model>
for dataset in train_test_data:
dataset['Cabin_count'] = get_cabin_count(dataset, column='Cabin' )
model_Ridge_set_2 = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=5)
 model_Ridge_set_2.fit(sparse_merge_train_1, Y_train )<compute_train_metric>
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )
pred_train_4 = model_Ridge_set_2.predict(sparse_merge_train_1)
 pred_cv_4 = model_Ridge_set_2.predict(sparse_merge_cv_1)
 print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_4-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_4-1)) )<load_pretrained>
categorical_features = ['Pclass', 'Sex', 'Cabin_letter', 'Embarked', 'Title', 'Family_size', 'Parch', 'SibSp']
pickle.dump(model_Ridge_set_2, open("model_Ridge_set_2.pickle", "wb"),protocol=4)
 model_Ridge_set_2 = None
del(model_Ridge_set_2)
 gc.collect()<train_model>
for column in categorical_features:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived',\
negative_state='Dead' )
model_MNB_set_2 = MultinomialNB(alpha=1.0, fit_prior=True)
 model_MNB_set_2.fit(sparse_merge_train_1, Y_train >= 4 )<compute_train_metric>
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )
pred_train_5 = model_MNB_set_2.predict(sparse_merge_train_1)
 pred_cv_5 = model_MNB_set_2.predict(sparse_merge_cv_1)
 print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_5-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_5-1)) )<load_pretrained>
test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True )
pickle.dump(model_MNB_set_2, open("model_MNB_set_2.pickle", "wb"),protocol=4)
 model_MNB_set_2 = None
sparse_merge_train_1 = None
sparse_merge_cv_1 = None
del(model_MNB_set_2)
 del(sparse_merge_train_1,sparse_merge_cv_1)
 gc.collect()<define_variables>
continuous_features = [ 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'Family_size']
f_cats = ['brand_name', 'main_category', 'sub_category_1', 'sub_category_2', 'name_first']<categorify>
for column in continuous_features:
get_continuous_feature_info(train, column=column, y_column='Survived' )
targetencoder = TargetEncoder(min_samples_leaf=100, smoothing=10,cols=f_cats,return_df=False )<categorify>
for dataset in train_test_data:
dataset['Cabin_letter'] = dataset['Cabin_letter'].fillna('Missing_cabin_info' )
targetencoder.fit(X_train[f_cats],Y_train )<categorify>
for dataset in train_test_data:
dataset['Cabin_count'] = dataset['Cabin_count'].fillna(0 )
X_train_target_encode = targetencoder.transform(X_train[f_cats] )<categorify>
for column in ['Cabin_letter', 'Cabin_count']:
get_categorical_feature_info(train, column=column, y_column='Survived' )
X_cv_target_encode = targetencoder.transform(X_cv[f_cats] )<feature_engineering>
for dataset in train_test_data:
dataset['Has_a_Cabin'] = dataset["Cabin_letter"].apply(lambda x: 0 if x == 'Missing_cabin_info' else 1)
 dataset.drop(columns='Cabin_letter', inplace=True )
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
<normalization>
get_categorical_feature_info(train, column='Has_a_Cabin', y_column='Survived' )
X_train_target_encode[:,0] = add_noise(X_train_target_encode[:,0],noise_level=0.01)
 X_train_target_encode[:,1] = add_noise(X_train_target_encode[:,1],noise_level=0.01)
 X_train_target_encode[:,2] = add_noise(X_train_target_encode[:,2],noise_level=0.01)
 X_train_target_encode[:,3] = add_noise(X_train_target_encode[:,3],noise_level=0.01)
 X_train_target_encode[:,4] = add_noise(X_train_target_encode[:,4],noise_level=0.01 )<categorify>
outliers_bounds = {}
for column in continuous_features:
outliers_bounds[column] = get_outliers_info(total_data, column=column )
pickle.dump(targetencoder, open("targetencoder.pickle", "wb"),protocol=4)
 targetencoder = None
del(targetencoder)
 gc.collect()<concatenate>
for dataset in train_test_data:
dataset.drop('Parch', axis='columns', inplace=True )
train_features = hstack(( pred_train_1.reshape(-1,1),\
pred_train_2.reshape(-1,1),\
pred_train_3.reshape(-1,1),\
pred_train_4.reshape(-1,1),\
pred_train_5.reshape(-1,1),\
X_train_target_encode,\
train_kbest_features)).tocsr()<concatenate>
del outliers_bounds['Parch']
del outliers_bounds['Pclass']
cv_features = hstack(( pred_cv_1.reshape(-1,1),\
pred_cv_2.reshape(-1,1),\
pred_cv_3.reshape(-1,1),\
pred_cv_4.reshape(-1,1),\
pred_cv_5.reshape(-1,1),\
X_cv_target_encode,\
cv_kbest_features)).tocsr()<load_pretrained>
def normalize_outliers(data, outliers_bounds={}):
for key in outliers_bounds.keys() :
median = data[key].median()
new_values = []
for i in range(len(data)) :
value = data[key][i]
if value < outliers_bounds[key][0] or value > outliers_bounds[key][1]:
new_values.append(outliers_bounds[key][1])
 else:
new_values.append(value)
 data[key] = new_values
return data
pickle.dump(train_features, open("train_features.pickle", "wb"),protocol=4)
 pickle.dump(cv_features, open("cv_features.pickle", "wb"),protocol=4 )<drop_column>
train = normalize_outliers(train, outliers_bounds=outliers_bounds)
 test = normalize_outliers(test, outliers_bounds=outliers_bounds )
pred_train_1 = None
pred_train_2 = None
pred_train_3 = None
pred_train_4 = None
pred_train_5 = None
X_train_target_encode = None
train_kbest_features = None
pred_cv_1 = None
pred_cv_2 = None
pred_cv_3 = None
pred_cv_4 = None
pred_cv_5 = None
X_cv_target_encode = None
cv_kbest_features = None
del(pred_train_1,\
pred_train_2,\
pred_train_3,\
pred_train_4,\
pred_train_5,\
X_train_target_encode,\
train_kbest_features)
 del(pred_cv_1,\
pred_cv_2,\
pred_cv_3,\
pred_cv_4,\
pred_cv_5,\
X_cv_target_encode,\
cv_kbest_features)
 gc.collect()<create_dataframe>
total_data = train.append(test, sort=False )
d_train = lgb.Dataset(train_features, label=Y_train)
 d_valid = lgb.Dataset(cv_features, label=Y_cv)
 watchlist = [d_train, d_valid]<init_hyperparams>
del outliers_bounds['SibSp']
params = {
'colsample_bytree': 0.42799939792816927,
'max_depth': 8,
'min_child_samples': 370,
'min_child_weight': 0.01,
'num_leaves': 29,
'reg_lambda': 5,
'subsample': 0.6739316550896339,
'learning_rate':0.1,
'reg_alpha' :0.5,
'boosting_type': 'gbdt',
'objective' : 'regression',
'metric' : 'RMSE',
'verbosity': -1,
'lambda_l1': 10,
'lambda_l2': 10
} 
<train_model>
for column in outliers_bounds.keys() :
get_continuous_feature_info(train, column=column, y_column='Survived' )
model_lgb_final = lgb.train(params,
train_set=d_train,
num_boost_round=3000,
valid_sets=watchlist,
verbose_eval=200,early_stopping_rounds=100)
<predict_on_test>
for dataset in train_test_data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 16), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 16)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[(dataset['Fare'] > 31)&(dataset['Fare'] <= 50), 'Fare'] = 3
dataset.loc[ dataset['Fare'] > 50, 'Fare'] = 4
dataset['Fare'] = dataset['Fare'].astype(int )
pred_train_6 = model_lgb_final.predict(train_features)
 pred_cv_6 = model_lgb_final.predict(cv_features )<compute_test_metric>
def is_youngest_girl(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Sex[i] == 1):
match =(data.loc[i].Age <= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list
print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_6-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_6-1)) )<load_pretrained>
train['Is_Youngest_lady'] = is_youngest_girl(train, total_data)
 test['Is_Youngest_lady'] = is_youngest_girl(test, total_data )
pickle.dump(model_lgb_final, open("model_lgb_final.pickle", "wb"),protocol=4)
 model_lgb_final = None
del(model_lgb_final)
 gc.collect()<drop_column>
for dataset in train_test_data:
dataset['Is_WifeandRich'] =(( dataset['Name'].str.find('(')> -1)&(dataset['Sex'] == 1)&(dataset['Pclass'] <3)) *1
train_features= None
cv_features = None
d_train = None
d_valid = None
watchlist = None
del(train_features,cv_features,d_train,d_valid,watchlist)
 gc.collect()<prepare_x_and_y>
for column in ['Is_WifeandRich', 'Is_Youngest_lady']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )
X_train = None
X_cv = None
Y_train = None
Y_test = None
del(X_train,X_cv,Y_train,Y_cv)
 gc.collect()<define_variables>
def is_a_rich_Mother(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Pclass[i] < 3)&(data.Sex[i] == 1):
match =(14 >= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list
test_id = []
prediction = []<load_pretrained>
train['Is_a_rich_Mother'] = is_a_rich_Mother(train, total_data)
 test['Is_a_rich_Mother'] = is_a_rich_Mother(test, total_data )
def predict_final(df):
test_id.extend(list(df['test_id']))
df.drop('test_id', axis=1, inplace=True)
 preprocessing(df)
 adding_new_features(df)
 normalize_dataframe(df)
 with open('mask_desc.pickle',mode='rb')as model_f:
mask_desc = pickle.load(model_f)
 with open('wb_desc.pickle',mode='rb')as model_f:
wb_desc= pickle.load(model_f)
 with open('model_desc.pickle',mode='rb')as model_f:
model_desc = pickle.load(model_f)
 X_description_test_wb = wb_desc.transform(df['item_description'])
 X_description_test_wb = X_description_test_wb[:, mask_desc]
print(X_description_test_wb.shape)
 pred_test_1 = model_desc.predict(X_description_test_wb)
 mask_desc = None
wb_desc = None
model_desc = None
del(mask_desc,wb_desc,model_desc)
 gc.collect()
with open('mask_name.pickle',mode='rb')as model_f:
mask_name = pickle.load(model_f)
 with open('wb_name.pickle',mode='rb')as model_f:
wb_name= pickle.load(model_f)
 with open('model_name.pickle',mode='rb')as model_f:
model_name = pickle.load(model_f)
 X_name_test_wb = wb_name.transform(df['name'])
 X_name_test_wb = X_name_test_wb[:, mask_name]
print(X_name_test_wb.shape)
 pred_test_2 = model_name.predict(X_name_test_wb)
 mask_name = None
wb_name = None
model_name = None
del(mask_name,wb_name,model_name)
 gc.collect()
with open('lb_brand_name.pickle',mode='rb')as model_f:
lb_brand_name = pickle.load(model_f)
 with open('lb_main_category.pickle',mode='rb')as model_f:
lb_main_category = pickle.load(model_f)
 with open('lb_sub_category_1.pickle',mode='rb')as model_f:
lb_sub_category_1 = pickle.load(model_f)
 with open('lb_sub_category_2.pickle',mode='rb')as model_f:
lb_sub_category_2 = pickle.load(model_f)
 X_brand_test = lb_brand_name.transform(df['brand_name'])
 X_main_cat_test = lb_main_category.transform(df['main_category'])
 X_main_sub_cat_1_test = lb_sub_category_1.transform(df['sub_category_1'])
 X_main_sub_cat_2_test = lb_sub_category_2.transform(df['sub_category_2'])
 X_dummies_test = csr_matrix(
 pd.get_dummies(df[list(total_cols -(basic_cols)) ],
sparse=True ).values)
 X_dummies_test_1 = csr_matrix(
 pd.get_dummies(df[list({'item_condition_id', 'shipping'})],
sparse=True ).values)
 sparse_merge_test = hstack(( X_name_test_wb , X_description_test_wb, X_brand_test, X_main_cat_test,
X_main_sub_cat_1_test, X_main_sub_cat_2_test,X_dummies_test,X_dummies_test_1)).tocsr()
X_dummies_test = None
X_description_test_wb = None
X_name_test_wb = None
del(X_dummies_test, X_description_test_wb,X_name_test_wb)
 gc.collect()
print(sparse_merge_test.shape)
 with open('model_FM_FTRL.pickle',mode='rb')as model_f:
model_FM_FTRL = pickle.load(model_f)
 pred_test_3 = model_FM_FTRL.predict(sparse_merge_test)
 model_FM_FTRL = None
del(model_FM_FTRL)
 gc.collect()
with open('fselect.pickle',mode='rb')as model_f:
fselect = pickle.load(model_f)
 test_kbest_features = fselect.transform(sparse_merge_test)
 fselect = None
print(test_kbest_features.shape)
 del(fselect)
 gc.collect()
sparse_merge_train= None
sparse_merge_cv = None
del(sparse_merge_train,sparse_merge_cv)
 gc.collect()
with open('tfidf_desc.pickle',mode='rb')as model_f:
tfidf_desc = pickle.load(model_f)
 X_desc_test_tfidf = tfidf_desc.transform(df['item_description'])
 tfidf_desc = None
del(tfidf_desc)
 gc.collect()
with open('tfidf_name.pickle',mode='rb')as model_f:
tfidf_name = pickle.load(model_f)
 X_name_test_tfidf = tfidf_name.transform(df['name'])
 tfidf_name = None
del(tfidf_name)
 gc.collect()
sparse_merge_test_1 = hstack(( X_name_test_tfidf , X_desc_test_tfidf, X_brand_test, X_main_cat_test,
X_main_sub_cat_1_test, X_main_sub_cat_2_test,X_dummies_test_1)).tocsr()
X_dummies_test_1 = None
X_brand_test = None
X_main_cat_test = None
X_main_sub_cat_1_test = None
X_main_sub_cat_2_test = None
X_name_test_tfidf = None
X_desc_test_tfidf = None
del(X_dummies_test_1, X_brand_test, X_main_cat_test,\
X_main_sub_cat_1_test, X_main_sub_cat_2_test, X_name_test_tfidf,X_desc_test_tfidf)
 gc.collect()
with open('model_Ridge_set_2.pickle',mode='rb')as model_f:
model_Ridge_set_2 = pickle.load(model_f)
 pred_test_4 = model_Ridge_set_2.predict(sparse_merge_test_1)
 model_Ridge_set_2 = None
del(model_Ridge_set_2)
 gc.collect()
with open('model_MNB_set_2.pickle',mode='rb')as model_f:
model_MNB_set_2 = pickle.load(model_f)
 pred_test_5 = model_MNB_set_2.predict(sparse_merge_test_1)
 model_MNB_set_2 = None
sparse_merge_train_1 = None
sparse_merge_cv_1 = None
del(model_MNB_set_2)
 del(sparse_merge_train_1,sparse_merge_cv_1)
 gc.collect()
with open('targetencoder.pickle',mode='rb')as model_f:
targetencoder = pickle.load(model_f)
 f_cats = ['brand_name', 'main_category', 'sub_category_1', 'sub_category_2', 'name_first']
X_test_target_encode = targetencoder.transform(df[f_cats])
 targetencoder = None
del(targetencoder)
 gc.collect()
test_features = hstack(( pred_test_1.reshape(-1,1),\
pred_test_2.reshape(-1,1),\
pred_test_3.reshape(-1,1),\
pred_test_4.reshape(-1,1),\
pred_test_5.reshape(-1,1),\
X_test_target_encode,\
test_kbest_features)).tocsr()
pred_test_1 = None
pred_test_2 = None
pred_test_3 = None
pred_test_4 = None
pred_test_5 = None
X_test_target_encode = None
test_kbest_features = None
del(pred_test_1,\
pred_test_2,\
pred_test_3,\
pred_test_4,\
pred_test_5,\
X_test_target_encode,\
test_kbest_features)
 gc.collect()
with open('model_lgb_final.pickle',mode='rb')as model_f:
model_lgb_final = pickle.load(model_f)
 prediction.extend(list(model_lgb_final.predict(test_features)))
 train_features = None
cv_features = None
d_train = None
d_valid = None
watchlist = None
del(train_features,cv_features,d_train,d_valid,watchlist)
 gc.collect()<load_from_csv>
def is_the_only_woman(data, match_data):
match_list = []
for i in range(len(data)) :
if data.Sex[i] == 1:
match =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Sex.sum())
 if match == 1:
match_list.append(match)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
chunksize = 10 ** 6
for chunk in pd.read_csv('.. /input/mercari-price-suggestion-challenge/test_stg2.tsv', delimiter='\t',chunksize=chunksize):
predict_final(chunk )<save_to_csv>
train['Is_the_only_woman'] = is_a_rich_Mother(train, total_data)
 test['Is_the_only_woman'] = is_a_rich_Mother(test, total_data )
submission = pd.DataFrame()
submission['test_id'] = np.asarray(test_id)
 submission['price'] =(10 ** np.asarray(prediction)- 1)
 submission.to_csv('stacked_submission_1.csv', index=False )<set_options>
for column in ['Is_a_rich_Mother', 'Is_the_only_woman']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )
warnings.filterwarnings('ignore')
 plt.style.use('dark_background')
 os.environ["CUDA_VISIBLE_DEVICES"] = "0"
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
 KTF.set_session(sess)
 class Clock:
def __enter__(self):
self.begin_time = time.time()
def __exit__(self, exec_type, exec_value, exec_trace):
print("time:", time.time() - self.begin_time, "s")
 def parallelize_dataframe(df, func, cores=4):

 df_split = np.array_split(df, cores)
 pool = Pool(cores)
 df = pd.concat(pool.map(func, df_split), axis=0)
 pool.close()
pool.join()
return df
def clean_str(text, max_text_length=60):

 try:
text = ' '.join([w for w in text.split() [:max_text_length]])
 text = text.lower()
text = re.sub(u"é", u"e", text)
 text = re.sub(u"ē", u"e", text)
 text = re.sub(u"è", u"e", text)
 text = re.sub(u"ê", u"e", text)
 text = re.sub(u"à", u"a", text)
 text = re.sub(u"â", u"a", text)
 text = re.sub(u"ô", u"o", text)
 text = re.sub(u"ō", u"o", text)
 text = re.sub(u"ü", u"u", text)
 text = re.sub(u"ï", u"i", text)
 text = re.sub(u"ç", u"c", text)
 text = re.sub(u"\u2019", u"'", text)
 text = re.sub(u"\xed", u"i", text)
 text = re.sub(u"w\/", u" with ", text)
 text = re.sub(u"[^a-z0-9]", " ", text)
 text = u" ".join(re.split('(\d+)',text))
text = re.sub(u"\s+", u" ", text ).strip()
except:
text = np.NaN
return text
def take(generator):
try:
for elem in generator:
yield elem
except StopIteration:
return
def fill_brandname(name, brand_name_set):
try:
for size in [4, 3, 2, 1]:
for x in take(ngrams(name.split() , size)) :
if " ".join(x)in brand_name_set:
return " ".join(x)
 return np.NaN
except:
return np.NaN
def clean_str_df(x):
return x.apply(lambda y: clean_str(y))
def fill_brandname_df(x):
global brand_name_set
return x.apply(lambda y: fill_brandname(y, brand_name_set))
def lg(text):
text = [x for x in text.split() if x != '']
return len(text)
 def tokenize(text):
return [w for w in text.split() ]
def word_doc_counts(words, word_doc_count_dic):
words = set(words.split())
 for x in words:
if x in word_doc_count_dic:
word_doc_count_dic[x] = word_doc_count_dic[x] + 1
else:
word_doc_count_dic[x] = 1
def remove_rare_words(words, word_doc_count_dic):
return " ".join([x for x in words.split() if x in word_doc_count_dic])
 def name_to_seq(df):
global vocabulary_one
global MAX_NAME_SEQ
return df.apply(lambda x: [vocabulary_one[y] for y in x.split() [:MAX_NAME_SEQ]])
 def description_to_seq(df):
global vocabulary_one
global MAX_ITEM_DESC_SEQ
ret = df.apply(lambda x: [vocabulary_one[y] for y in x.split() [:MAX_ITEM_DESC_SEQ]])
 return ret
def get_sample(train_file, test_file, max_text_length=60):
print("读取样本")
 with Clock() :
train = pd.read_csv(train_file, sep='\t', encoding='utf-8')
 train.drop(columns=["train_id"], inplace=True)
 print('train shape:', train.shape, "columns:", train.columns)
 test = pd.read_csv(test_file, sep='\t', encoding='utf-8')
 test.drop(columns=["test_id"], inplace=True)
 print('test shape:', test.shape, "columns:", test.columns)
 train['price'] = np.log1p(train['price'].values)
 sample = pd.concat([train.drop(columns=["price"], axis=0), test], axis=0)
 gc.collect()
with Clock() :
print("处理特征")
 sample["item_description"].fillna("", inplace=True)
 sample["item_description"] = sample["item_description"].apply(lambda x :
x.replace('No description yet',''))
sample["item_description"] = parallelize_dataframe(sample["item_description"], clean_str_df)
 sample["name"] = parallelize_dataframe(sample["name"], clean_str_df)
 sample["brand_name"] = parallelize_dataframe(sample["brand_name"], clean_str_df)
 global brand_name_set
brand_name_set = sample.groupby("brand_name" ).size()
sample.loc[sample["brand_name"].isnull() , "brand_name"] = parallelize_dataframe(
 sample.loc[sample["brand_name"].isnull() , "name"],
fill_brandname_df)
 sample["brand_name"].fillna("", inplace=True)
 sample["item_condition_id"].fillna(2, inplace=True)
 sample["category_name"].fillna("//", inplace=True)
 sample["catery_name_first"] = sample["category_name"].apply(lambda x: x.split("/")[0])
 sample["catery_name_second"] = sample["category_name"].apply(lambda x: x.split("/")[1])
 sample["catery_name_third"] = sample["category_name"].apply(lambda x: x.split("/")[2])
 sample['category_name'] = sample['category_name'].apply(lambda x: ' '.join(x.split('/')).strip())
 sample['nb_words_item_description'] = sample['item_description'].apply(lg ).astype(np.uint16)
 sample['nb_words_item_description'] = 1.0 * sample['nb_words_item_description'] / max_text_length
for x in ['brand_name', 'category_name', 'catery_name_first', 'catery_name_second',
'catery_name_third']:
le = preprocessing.LabelEncoder()
sample[x] = le.fit_transform(sample[x])
 sample["item_condition_id"] = 1.0 * sample["item_condition_id"] / 5
sample["old_name"] = sample["name"].copy()
sample["brand_cat"] = "cat1_" + sample["catery_name_first"].astype(str)+ " " + \
"cat2_" + sample["catery_name_second"].astype(str)+ " " + \
"cat3_" + sample["catery_name_third"].astype(str)+ " " + \
"brand_" + sample["brand_name"].astype(str)
 sample["name"] = sample["brand_cat"] + " " + sample["name"]
sample["name_desc"] = sample["name"] + " " + sample["item_description"].apply(lambda x: " ".join(x.split() [:5]))
gc.collect()
return sample, train
def get_statistics_feature(sample, train):
print("构建统计特征:")
 with Clock() :
train_new = sample[:train.shape[0]].copy()
train_new["price"] = train["price"]
for x in ['catery_name_first', 'catery_name_second', 'catery_name_third', 'category_name', 'brand_name' ]:
tmp = train_new.groupby(x)["price"].mean().astype(np.float32)
 sample['mean_price_' + x] = sample[x].map(tmp ).astype(np.float32)
 sample['mean_price_' + x].fillna(tmp.mean() , inplace=True)
 gc.collect()
return sample
def get_bownn1_feature(sample, train):
print("构造文本特征:")
 max_features = 200000
with Clock() :
vectorizer = CountVectorizer(lowercase=True, tokenizer=tokenize, stop_words='english',
ngram_range=(1, 2), min_df=5, max_features=max_features,
binary=True, dtype=np.int8)
 vectorizer.fit(sample['name'] + ' ' + sample['item_description'])
 print("构造样本:")
 with Clock() :
split_num = train.shape[0]
x = {
'sparse_data': vectorizer.transform(sample[:split_num]['name'] +
' ' + sample[:split_num]['item_description']),
'item_condition': np.array(sample['item_condition_id'])[:split_num],
'shipping': np.array(sample["shipping"])[:split_num],
'mean_price_catery_name_second': np.array(sample["mean_price_catery_name_second"])[:split_num],
'nb_words_item_description': np.array(sample["nb_words_item_description"])[:split_num]
}
test = {
'sparse_data': vectorizer.transform(sample[split_num:]['name'] +
' ' + sample[split_num:]['item_description']),
'item_condition': np.array(sample['item_condition_id'])[split_num:],
'shipping': np.array(sample["shipping"])[split_num:],
'mean_price_catery_name_second': np.array(sample["mean_price_catery_name_second"])[split_num:],
'nb_words_item_description': np.array(sample["nb_words_item_description"])[split_num:]
}
y = train.price.values
return x, y, test
def bownn1_model(max_features=200000):
sparse_data = Input(shape=[max_features], name="sparse_data", dtype='float32', sparse=True)
 item_condition = Input(shape=[1], name="item_condition", dtype='float32')
 shipping = Input(shape=[1], name="shipping", dtype='float32')
 mean_price_catery_name_second = Input(shape=[1], name="mean_price_catery_name_second",
dtype='float32')
 nb_words_item_description = Input(shape=[1], name="nb_words_item_description",
dtype='float32')
 x = Dense(200, kernel_initializer=he_uniform(seed=0))(sparse_data)
 x = PReLU()(x)
 x = concatenate([x, item_condition, shipping, mean_price_catery_name_second,
nb_words_item_description], axis=1)
 x = Dense(200, kernel_initializer=he_uniform(seed=0))(x)
 x = PReLU()(x)
 x = Dense(100, kernel_initializer=he_uniform(seed=0))(x)
 x = PReLU()(x)
 x = Dense(1, kernel_initializer=he_uniform(seed=0))(x)
 m = Model([sparse_data, item_condition, shipping, mean_price_catery_name_second,
nb_words_item_description], x)
 optimizer = Adam (.001)
 m.compile(loss="mse", optimizer=optimizer)
 return m
def get_bownn2_feature(sample, train):
print("构造文本特征:")
 split_num = train.shape[0]
with Clock() :
print("brand_cat")
 vectorizer_brandcat = CountVectorizer(lowercase=True, tokenizer=tokenize,
ngram_range=(1, 3), min_df=2, max_features=100000,
binary=True, dtype=np.int8)
 vectorizer_brandcat.fit(sample[:split_num]['brand_cat'])
 sparse_data_brandcat = vectorizer_brandcat.transform(sample['brand_cat'])
 del vectorizer_brandcat
sample.drop(columns=["brand_cat"], inplace=True)
 gc.collect()
print("old_name")
 vectorizer_name = CountVectorizer(lowercase=True, analyzer="char",
ngram_range=(2, 4), min_df=5, max_features=100000,
binary=True, dtype=np.int8)
 vectorizer_name.fit(sample[:train.shape[0]]['old_name'])
 sparse_data_name = vectorizer_name.transform(sample['old_name'])
 del vectorizer_name
sample.drop(columns=["old_name"], inplace=True)
 gc.collect()
print("item_description")
 vectorizer_desc = CountVectorizer(lowercase=True, tokenizer=tokenize, stop_words="english",
ngram_range=(1, 1), min_df=20, max_features=100000,
binary=True, dtype=np.int8)
 vectorizer_desc.fit(sample[:train.shape[0]]["item_description"])
 sparse_data_desc = vectorizer_desc.transform(sample['item_description'])
 del vectorizer_desc
sample.drop(columns=["item_description"], inplace=True)
 gc.collect()
print("构造样本:")
 with Clock() :
x = {
'sparse_data_name': sparse_data_name[:split_num],
'sparse_data_brandcat': sparse_data_brandcat[:split_num],
'sparse_data_desc': sparse_data_desc[:split_num],
'item_condition': np.array(sample['item_condition_id'])[:split_num],
'shipping': np.array(sample["shipping"])[:split_num],
'mean_price_catery_name_second': np.array(sample["mean_price_catery_name_second"])[:split_num],
'nb_words_item_description': np.array(sample["nb_words_item_description"])[:split_num]
}
test = {
'sparse_data_name': sparse_data_name[split_num:],
'sparse_data_brandcat': sparse_data_brandcat[split_num:],
'sparse_data_desc': sparse_data_desc[split_num:],
'item_condition': np.array(sample['item_condition_id'])[split_num:],
'shipping': np.array(sample["shipping"])[split_num:],
'mean_price_catery_name_second': np.array(sample["mean_price_catery_name_second"])[split_num:],
'nb_words_item_description': np.array(sample["nb_words_item_description"])[split_num:]
}
y = train.price.values
return x, y, test
def bownn2_model(len_sparse_data_name, len_sparse_data_brandcat, len_sparse_data_desc):
sparse_data_name = Input(shape=[len_sparse_data_name], name="sparse_data_name", dtype='float32',
sparse=True)
 sparse_data_brandcat = Input(shape=[len_sparse_data_brandcat], name="sparse_data_brandcat",
dtype='float32', sparse=True)
 sparse_data_desc = Input(shape=[len_sparse_data_desc], name="sparse_data_desc",
dtype='float32', sparse=True)
 item_condition = Input(shape=[1], name="item_condition", dtype='float32')
 shipping = Input(shape=[1], name="shipping", dtype='float32')
 mean_price_catery_name_second = Input(shape=[1], name="mean_price_catery_name_second",
dtype='float32')
 nb_words_item_description = Input(shape=[1], name="nb_words_item_description",
dtype='float32')
 x = PReLU()(Dense(50 )(sparse_data_name))
y = PReLU()(Dense(50 )(sparse_data_brandcat))
z = PReLU()(Dense(50 )(sparse_data_desc))
x = concatenate([x, y, z, item_condition, shipping, mean_price_catery_name_second,
nb_words_item_description])
 x = PReLU()(Dense(50 )(x))
x = Dense(1 )(x)
 m = Model([sparse_data_name, sparse_data_brandcat, sparse_data_desc,
item_condition, shipping, mean_price_catery_name_second,
nb_words_item_description], x)
 optimizer = Adam(0.002)
 m.compile(loss="mse", optimizer=optimizer)
 return m
def get_embedding_feature(sample, train):
print("构造文本特征:")
 with Clock() :
word_doc_count_dic = dict()
for x in ['name','item_description']:
sample[x].apply(lambda y: word_doc_counts(y, word_doc_count_dic))
min_df_one = 5
rare_words = [x for x in word_doc_count_dic if word_doc_count_dic[x] < min_df_one]
for x in rare_words:
del word_doc_count_dic[x]
for x in ['name','item_description']:
sample[x] = sample[x].apply(lambda y: remove_rare_words(y, word_doc_count_dic))
global vocabulary_one
vocabulary_one = word_doc_count_dic.copy()
count = 0
for x in word_doc_count_dic:
count += 1
vocabulary_one[x] = count
print("构造样本:")
 global MAX_NAME_SEQ
MAX_NAME_SEQ = 20
global MAX_ITEM_DESC_SEQ
MAX_ITEM_DESC_SEQ = 30
MAX_TEXT = len(vocabulary_one)+ 1
MAX_CATEGORY = np.max(sample['category_name'].max())+ 1
MAX_CATEGORY1 = np.max(sample['catery_name_first'].max())+ 1
MAX_CATEGORY2 = np.max(sample['catery_name_second'].max())+ 1
MAX_CATEGORY3 = np.max(sample['catery_name_third'].max())+ 1
MAX_BRAND = np.max(sample['brand_name'].max())+ 1
split_num = train.shape[0]
with Clock() :
new_train = sample[:split_num]
new_train["seq_name"] = parallelize_dataframe(new_train["name"], name_to_seq)
 new_train["seq_item_description"] = parallelize_dataframe(new_train["item_description"],
description_to_seq)
 seq_name = pad_sequences(new_train["seq_name"], maxlen=MAX_NAME_SEQ)
 seq_item_description = pad_sequences(new_train["seq_item_description"],
maxlen=MAX_ITEM_DESC_SEQ)
 x = {
"name": seq_name,
"item_desc": seq_item_description,
"brand_name": np.array(new_train["brand_name"]),
"category_name": np.array(new_train["category_name"]),
"catery_name_first": np.array(new_train["catery_name_first"]),
"catery_name_second": np.array(new_train["catery_name_second"]),
"catery_name_third": np.array(new_train["catery_name_third"]),
"item_condition": np.array(new_train["item_condition_id"]),
"shipping": np.array(new_train["shipping"])
 }
y = train.price.values
new_test = sample[split_num:]
def test_gen() :
batch = 500000
remain = new_test.shape[0]
current = 0
while remain > 0:
if remain < batch:
tmp = new_test[current:]
else:
tmp = new_test[current:current + batch]
remain = remain - batch
current = current + batch
tmp["seq_name"] = parallelize_dataframe(tmp["name"], name_to_seq)
 tmp["seq_item_description"] = parallelize_dataframe(tmp["item_description"],
description_to_seq)
 seq_name = pad_sequences(tmp["seq_name"], maxlen=MAX_NAME_SEQ)
 seq_item_description = pad_sequences(tmp["seq_item_description"],
maxlen=MAX_ITEM_DESC_SEQ)
 x = {
"name": seq_name,
"item_desc": seq_item_description,
"brand_name": np.array(tmp["brand_name"]),
"category_name": np.array(tmp["category_name"]),
"catery_name_first": np.array(tmp["catery_name_first"]),
"catery_name_second": np.array(tmp["catery_name_second"]),
"catery_name_third": np.array(tmp["catery_name_third"]),
"item_condition": np.array(tmp["item_condition_id"]),
"shipping": np.array(tmp["shipping"])
 }
yield x
return x, y, test_gen, MAX_CATEGORY, MAX_CATEGORY1, MAX_CATEGORY2, MAX_CATEGORY3, MAX_BRAND, MAX_TEXT, MAX_NAME_SEQ, MAX_ITEM_DESC_SEQ
def embeding_model(MAX_CATEGORY, MAX_CATEGORY1, MAX_CATEGORY2, MAX_CATEGORY3, MAX_BRAND, MAX_TEXT,
MAX_NAME_SEQ):
name = Input(shape=[MAX_NAME_SEQ], name="name")
 item_desc = Input(shape=[MAX_ITEM_DESC_SEQ], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 catery_name_first = Input(shape=[1], name="catery_name_first")
 catery_name_second = Input(shape=[1], name="catery_name_second")
 catery_name_third = Input(shape=[1], name="catery_name_third")
 item_condition = Input(shape=[1], name="item_condition")
 shipping = Input(shape=[1], name="shipping")
 share_embedding = Embedding(MAX_TEXT, 50)
 name_pool = GlobalAveragePooling1D()(share_embedding(name))
item_desc_pool = GlobalAveragePooling1D()(share_embedding(item_desc))
brand_name_emb = Flatten()(Embedding(MAX_BRAND, 10 )(brand_name))
category_name_emb = Flatten()(Embedding(MAX_CATEGORY, 10 )(category_name))
catery_name_first_emb = Flatten()(Embedding(MAX_CATEGORY1, 10 )(catery_name_first))
catery_name_second_emb = Flatten()(Embedding(MAX_CATEGORY2, 10 )(catery_name_second))
catery_name_third_emb = Flatten()(Embedding(MAX_CATEGORY3, 10 )(catery_name_third))
x = concatenate([name_pool, item_desc_pool, brand_name_emb, category_name_emb,
catery_name_first_emb, catery_name_second_emb, catery_name_third_emb,
item_condition, shipping])
 x = BatchNormalization()(x)
 x = Dense(1024 )(x)
 x = Activation("relu" )(x)
 x = Dense(1 )(x)
 model = Model([name, item_desc, brand_name, category_name, catery_name_first,
catery_name_second, catery_name_third, item_condition, shipping], x)
 model.compile(loss="mse", optimizer=Adam(0.001))
return model<save_to_csv>
def other_women_in_family(data, match_data):
match_list = []
for i in range(len(data)) :
if data.Sex[i] == 1:
match =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Sex.sum())
 if match > 1:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
path = ".. /input/"
test_id= pd.read_csv(path + "test_stg2.tsv", sep='\t', encoding='utf-8')[["test_id"]]
sample, train = get_sample(path + "train.tsv", path + "test_stg2.tsv", max_text_length=60)
 sample = get_statistics_feature(sample, train)
 x, y, test = get_bownn1_feature(sample, train)
 m = bownn1_model()
m.summary()
m.fit(x, y - np.mean(y), 2048, epochs=3)
 new_feature = pd.DataFrame(m.predict(x, 2048)+ np.mean(y), columns=["price1"])
 test_id["price1"] = m.predict(test, 2048)+ np.mean(y)
 test_id.to_csv("new_test.csv", index=False)
 new_feature.to_csv("new_train.csv", index=False)
 del new_feature
del test_id
del x
del y
del test
del m
gc.collect()
x, y, test_gen, MAX_CATEGORY, MAX_CATEGORY1, MAX_CATEGORY2, MAX_CATEGORY3, MAX_BRAND, MAX_TEXT, MAX_NAME_SEQ, MAX_ITEM_DESC_SEQ = get_embedding_feature(sample, train)
 m = embeding_model(MAX_CATEGORY, MAX_CATEGORY1,
MAX_CATEGORY2, MAX_CATEGORY3,
MAX_BRAND, MAX_TEXT, MAX_NAME_SEQ)
 m.summary()
m.fit(x, y - np.mean(y), 2048, epochs=3)
 new_feature = pd.read_csv("new_train.csv")
 new_feature["price2"] = m.predict(x, 2048)+ np.mean(y)
 new_feature.to_csv("new_train.csv", index=False)
 del new_feature
del x
gc.collect()
pred = []
for test in test_gen() :
tmp = m.predict(test, 2048)+ np.mean(y)
 pred.append(tmp)
 del test
gc.collect()
pred = np.concatenate(pred)
 test_id= pd.read_csv("new_test.csv")
 test_id["price2"] = pred
test_id.to_csv("new_test.csv", index=False)
 del test_id
del y
del m
del pred
gc.collect()
x, y, test = get_bownn2_feature(sample, train)
 m = bownn2_model(x["sparse_data_name"].shape[1], x["sparse_data_brandcat"].shape[1],
x["sparse_data_desc"].shape[1])
 m.summary()
m.fit(x, y - np.mean(y), 2048, epochs=3)
 new_feature = pd.read_csv("new_train.csv")
 new_feature["price3"] = m.predict(x, 2048)+ np.mean(y)
 new_feature.to_csv("new_train.csv", index=False)
 del new_feature
test_id= pd.read_csv("new_test.csv")
 test_id["price3"] = m.predict(test, 2048)+ np.mean(y)
 test_id.to_csv("new_test.csv", index=False)
 del test_id
del x
del test
del m
del sample
del train
gc.collect()<save_to_csv>
train['Other_women_in_family'] = other_women_in_family(train, total_data)
 test['Other_women_in_family'] = other_women_in_family(test, total_data )
new_feature = pd.read_csv("new_train.csv")
 test_id= pd.read_csv("new_test.csv")
 model = LinearRegression()
model.fit(new_feature, y)
 pred = model.predict(test_id.drop(columns=["test_id"]))
pred = np.expm1(pred)
 pred[pred < 3] = 3
pred[pred > 1000] = 1000
test_id.drop(columns=["price1", "price2", "price3"], inplace=True)
 test_id["price"] = pred
test_id.to_csv("ans.csv", index=False )<set_options>
def is_oldest_woman(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Sex[i] == 1):
match =(data.loc[i].Age == max(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list
warnings.filterwarnings('ignore')
 plt.style.use('dark_background')
 os.environ["CUDA_VISIBLE_DEVICES"] = "0"
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
 KTF.set_session(sess)
 class Clock:
def __enter__(self):
self.begin_time = time.time()
def __exit__(self, exec_type, exec_value, exec_trace):
print("time:", time.time() - self.begin_time, "s")
 def parallelize_dataframe(df, func, cores=4):

 df_split = np.array_split(df, cores)
 pool = Pool(cores)
 df = pd.concat(pool.map(func, df_split), axis=0)
 pool.close()
pool.join()
return df
def clean_str(text, max_text_length=60):

 try:
text = ' '.join([w for w in text.split() [:max_text_length]])
 text = text.lower()
text = re.sub(u"é", u"e", text)
 text = re.sub(u"ē", u"e", text)
 text = re.sub(u"è", u"e", text)
 text = re.sub(u"ê", u"e", text)
 text = re.sub(u"à", u"a", text)
 text = re.sub(u"â", u"a", text)
 text = re.sub(u"ô", u"o", text)
 text = re.sub(u"ō", u"o", text)
 text = re.sub(u"ü", u"u", text)
 text = re.sub(u"ï", u"i", text)
 text = re.sub(u"ç", u"c", text)
 text = re.sub(u"\u2019", u"'", text)
 text = re.sub(u"\xed", u"i", text)
 text = re.sub(u"w\/", u" with ", text)
 text = re.sub(u"[^a-z0-9]", " ", text)
 text = u" ".join(re.split('(\d+)',text))
text = re.sub(u"\s+", u" ", text ).strip()
except:
text = np.NaN
return text
def take(generator):
try:
for elem in generator:
yield elem
except StopIteration:
return
def fill_brandname(name, brand_name_set):
try:
for size in [4, 3, 2, 1]:
for x in take(ngrams(name.split() , size)) :
if " ".join(x)in brand_name_set:
return " ".join(x)
 return np.NaN
except:
return np.NaN
def clean_str_df(x):
return x.apply(lambda y: clean_str(y))
def fill_brandname_df(x):
global brand_name_set
return x.apply(lambda y: fill_brandname(y, brand_name_set))
def lg(text):
text = [x for x in text.split() if x != '']
return len(text)
 def tokenize(text):
return [w for w in text.split() ]
def word_doc_counts(words, word_doc_count_dic):
words = set(words.split())
 for x in words:
if x in word_doc_count_dic:
word_doc_count_dic[x] = word_doc_count_dic[x] + 1
else:
word_doc_count_dic[x] = 1
def remove_rare_words(words, word_doc_count_dic):
return " ".join([x for x in words.split() if x in word_doc_count_dic])
 def name_to_seq(df):
global vocabulary_one
global MAX_NAME_SEQ
return df.apply(lambda x: [vocabulary_one[y] for y in x.split() [:MAX_NAME_SEQ]])
 def description_to_seq(df):
global vocabulary_one
global MAX_ITEM_DESC_SEQ
ret = df.apply(lambda x: [vocabulary_one[y] for y in x.split() [:MAX_ITEM_DESC_SEQ]])
 return ret
def get_sample(train_file, test_file, max_text_length=60):
print("读取样本")
 with Clock() :
train = pd.read_csv(train_file, sep='\t', encoding='utf-8')
 train.drop(columns=["train_id"], inplace=True)
 train["price"] = np.log1p(train["price"].values)
 print('train shape:', train.shape, "columns:", train.columns)
 test = pd.read_csv(test_file, sep='\t', encoding='utf-8')
 test.drop(columns=["test_id"], inplace=True)
 print('test shape:', test.shape, "columns:", test.columns)
 sample = pd.concat([train.drop(columns=["price"], axis=0), test], axis=0)
 gc.collect()
with Clock() :
print("处理特征")
 sample["item_description"].fillna("", inplace=True)
 sample["item_description"] = sample["item_description"].apply(lambda x :
x.replace('No description yet',''))
sample["item_description"] = parallelize_dataframe(sample["item_description"], clean_str_df)
 sample["name"] = parallelize_dataframe(sample["name"], clean_str_df)
 sample["brand_name"] = parallelize_dataframe(sample["brand_name"], clean_str_df)
 global brand_name_set
brand_name_set = sample.groupby("brand_name" ).size()
sample.loc[sample["brand_name"].isnull() , "brand_name"] = parallelize_dataframe(
 sample.loc[sample["brand_name"].isnull() , "name"],
fill_brandname_df)
 sample["brand_name"].fillna("", inplace=True)
 sample["item_condition_id"].fillna(2, inplace=True)
 sample["category_name"].fillna("//", inplace=True)
 sample["catery_name_first"] = sample["category_name"].apply(lambda x: x.split("/")[0])
 sample["catery_name_second"] = sample["category_name"].apply(lambda x: x.split("/")[1])
 sample["catery_name_third"] = sample["category_name"].apply(lambda x: x.split("/")[2])
 sample['category_name'] = sample['category_name'].apply(lambda x: ' '.join(x.split('/')).strip())
 sample['nb_words_item_description'] = sample['item_description'].apply(lg ).astype(np.uint16)
 sample['nb_words_item_description'] = 1.0 * sample['nb_words_item_description'] / max_text_length
for x in ['brand_name', 'category_name', 'catery_name_first', 'catery_name_second',
'catery_name_third']:
le = preprocessing.LabelEncoder()
sample[x] = le.fit_transform(sample[x])
 sample["item_condition_id"] = 1.0 * sample["item_condition_id"] / 5
sample["old_name"] = sample["name"].copy()
sample["brand_cat"] = "cat1_" + sample["catery_name_first"].astype(str)+ " " + \
"cat2_" + sample["catery_name_second"].astype(str)+ " " + \
"cat3_" + sample["catery_name_third"].astype(str)+ " " + \
"brand_" + sample["brand_name"].astype(str)
 sample["name"] = sample["brand_cat"] + " " + sample["name"]
sample["name_desc"] = sample["name"] + " " + sample["item_description"].apply(lambda x: " ".join(x.split() [:5]))
gc.collect()
print("构造文本特征:")
 with Clock() :
word_doc_count_dic = dict()
for x in ['name','item_description']:
sample[x].apply(lambda y: word_doc_counts(y, word_doc_count_dic))
min_df_one = 5
rare_words = [x for x in word_doc_count_dic if word_doc_count_dic[x] < min_df_one]
for x in rare_words:
del word_doc_count_dic[x]
for x in ['name','item_description']:
sample[x] = sample[x].apply(lambda y: remove_rare_words(y, word_doc_count_dic))
global vocabulary_one
vocabulary_one = word_doc_count_dic.copy()
count = 0
for x in word_doc_count_dic:
count += 1
vocabulary_one[x] = count
print("构建统计特征:")
 with Clock() :
train_new = sample[:train.shape[0]].copy()
train_new["price"] = train["price"]
for x in ['catery_name_first', 'catery_name_second', 'catery_name_third', 'category_name', 'brand_name' ]:
tmp = train_new.groupby(x)["price"].mean().astype(np.float32)
 sample['mean_price_' + x] = sample[x].map(tmp ).astype(np.float32)
 sample['mean_price_' + x].fillna(tmp.mean() , inplace=True)
 gc.collect()
print("构造样本:")
 global MAX_NAME_SEQ
MAX_NAME_SEQ = 20
global MAX_ITEM_DESC_SEQ
MAX_ITEM_DESC_SEQ = 30
MAX_TEXT = len(vocabulary_one)+ 1
MAX_CATEGORY = np.max(sample['category_name'].max())+ 1
MAX_CATEGORY1 = np.max(sample['catery_name_first'].max())+ 1
MAX_CATEGORY2 = np.max(sample['catery_name_second'].max())+ 1
MAX_CATEGORY3 = np.max(sample['catery_name_third'].max())+ 1
MAX_BRAND = np.max(sample['brand_name'].max())+ 1
split_num = train.shape[0]
with Clock() :
new_train = sample[:split_num]
new_train["seq_name"] = parallelize_dataframe(new_train["name"], name_to_seq)
 new_train["seq_item_description"] = parallelize_dataframe(new_train["item_description"],
description_to_seq)
 seq_name = pad_sequences(new_train["seq_name"], maxlen=MAX_NAME_SEQ)
 seq_item_description = pad_sequences(new_train["seq_item_description"],
maxlen=MAX_ITEM_DESC_SEQ)
 x = {
"name": seq_name,
"item_desc": seq_item_description,
"brand_name": np.array(new_train["brand_name"]),
"category_name": np.array(new_train["category_name"]),
"catery_name_first": np.array(new_train["catery_name_first"]),
"catery_name_second": np.array(new_train["catery_name_second"]),
"catery_name_third": np.array(new_train["catery_name_third"]),
"item_condition": np.array(new_train["item_condition_id"]),
"shipping": np.array(new_train["shipping"])
 }
y = train.price.values
new_test = sample[split_num:]
def test_gen() :
batch = 500000
remain = new_test.shape[0]
current = 0
while remain > 0:
if remain < batch:
tmp = new_test[current:]
else:
tmp = new_test[current:current + batch]
remain = remain - batch
current = current + batch
tmp["seq_name"] = parallelize_dataframe(tmp["name"], name_to_seq)
 tmp["seq_item_description"] = parallelize_dataframe(tmp["item_description"],
description_to_seq)
 seq_name = pad_sequences(tmp["seq_name"], maxlen=MAX_NAME_SEQ)
 seq_item_description = pad_sequences(tmp["seq_item_description"],
maxlen=MAX_ITEM_DESC_SEQ)
 x = {
"name": seq_name,
"item_desc": seq_item_description,
"brand_name": np.array(tmp["brand_name"]),
"category_name": np.array(tmp["category_name"]),
"catery_name_first": np.array(tmp["catery_name_first"]),
"catery_name_second": np.array(tmp["catery_name_second"]),
"catery_name_third": np.array(tmp["catery_name_third"]),
"item_condition": np.array(tmp["item_condition_id"]),
"shipping": np.array(tmp["shipping"])
 }
yield x
return x, y, test_gen, MAX_CATEGORY, MAX_CATEGORY1, MAX_CATEGORY2, MAX_CATEGORY3, MAX_BRAND, MAX_TEXT, MAX_NAME_SEQ, MAX_ITEM_DESC_SEQ
def model(MAX_CATEGORY, MAX_CATEGORY1, MAX_CATEGORY2, MAX_CATEGORY3, MAX_BRAND, MAX_TEXT,
MAX_NAME_SEQ):
name = Input(shape=[MAX_NAME_SEQ], name="name")
 item_desc = Input(shape=[MAX_ITEM_DESC_SEQ], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 catery_name_first = Input(shape=[1], name="catery_name_first")
 catery_name_second = Input(shape=[1], name="catery_name_second")
 catery_name_third = Input(shape=[1], name="catery_name_third")
 item_condition = Input(shape=[1], name="item_condition")
 shipping = Input(shape=[1], name="shipping")
 share_embedding = Embedding(MAX_TEXT, 50)
 name_pool = GlobalAveragePooling1D()(share_embedding(name))
item_desc_pool = GlobalAveragePooling1D()(share_embedding(item_desc))
brand_name_emb = Flatten()(Embedding(MAX_BRAND, 10 )(brand_name))
category_name_emb = Flatten()(Embedding(MAX_CATEGORY, 10 )(category_name))
catery_name_first_emb = Flatten()(Embedding(MAX_CATEGORY1, 10 )(catery_name_first))
catery_name_second_emb = Flatten()(Embedding(MAX_CATEGORY2, 10 )(catery_name_second))
catery_name_third_emb = Flatten()(Embedding(MAX_CATEGORY3, 10 )(catery_name_third))
x = concatenate([name_pool, item_desc_pool, brand_name_emb, category_name_emb,
catery_name_first_emb, catery_name_second_emb, catery_name_third_emb,
item_condition, shipping])
 x = BatchNormalization()(x)
 x = Dense(1024 )(x)
 x = Activation("relu" )(x)
 x = Dense(1 )(x)
 model = Model([name, item_desc, brand_name, category_name, catery_name_first,
catery_name_second, catery_name_third, item_condition, shipping], x)
 model.compile(loss="mse", optimizer=Adam(0.001))
return model<find_best_model_class>
train['Is_oldest_woman'] = is_oldest_woman(train, total_data)
 test['Is_oldest_woman'] = is_oldest_woman(test, total_data )
path = ".. /input/"
x, y, test_gen, MAX_CATEGORY, MAX_CATEGORY1, MAX_CATEGORY2, MAX_CATEGORY3, MAX_BRAND, MAX_TEXT, MAX_NAME_SEQ, MAX_ITEM_DESC_SEQ = get_sample(path + "train.tsv", path + "test_stg2.tsv", max_text_length=60)
 model = model(MAX_CATEGORY, MAX_CATEGORY1,
MAX_CATEGORY2, MAX_CATEGORY3,
MAX_BRAND, MAX_TEXT, MAX_NAME_SEQ)
 model.summary()
model.fit(x, y - np.mean(y), 2048, epochs=3)
 del x
gc.collect()
pred = []
for test in test_gen() :
tmp = model.predict(test, 2048)+ np.mean(y)
 pred.append(tmp)
 del test
gc.collect()
pred = np.concatenate(pred)
 pred = np.expm1(pred)
 pred[pred < 3] = 3
pred[pred > 1000] = 1000
test_id= pd.read_csv(path + "test_stg2.tsv", sep='\t', encoding='utf-8')[["test_id"]]
test_id["price"] = pred
test_id.to_csv("ans.csv", index=False )<compute_test_metric>
for column in ['Is_oldest_woman', 'Other_women_in_family']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5<load_from_csv>
def is_youngest_boy(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Sex[i] == 0):
match =(data.loc[i].Age <= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list
print("Loading data...")
 train = pd.read_table(".. /input/train.tsv")
 test = pd.read_table(".. /input/test.tsv")
 print(train.shape)
 print(test.shape )<categorify>
def is_oldest_man(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Sex[i] == 0):
match =(data.loc[i].Age == max(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list
print("Handling missing values...")
 def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )<categorify>
def man_with_woman(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Sex[i] == 0)&(data.Title[i] != 5):
match =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])].Sex.sum())
 if match > 1:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
print("Handling categorical variables...")
 le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 del le
train.head(3 )<feature_engineering>
def man_with_other_men(data, match_data):
match_list = []
for i in range(len(data)) :
if data.Sex[i] == 0:
match = len(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(match_data['Sex'] == 0)].Sex)
 if match > 1:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
print("Text to seq process...")
 raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 print(" Fitting tokenizer...")
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 print(" Transforming text to seq...")
 train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 train.head(3 )<define_variables>
train['Is_youngest_man'] = is_youngest_boy(train, total_data)
 test['Is_youngest_man'] = is_youngest_boy(test, total_data )
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([np.max(train.seq_name.max())
 , np.max(test.seq_name.max())
 , np.max(train.seq_item_description.max())
 , np.max(test.seq_item_description.max())])+2
MAX_CATEGORY = np.max([train.category_name.max() , test.category_name.max() ])+1
MAX_BRAND = np.max([train.brand_name.max() , test.brand_name.max() ])+1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1<split>
train['Is_oldest_man'] = is_oldest_man(train, total_data)
 test['Is_oldest_man'] = is_oldest_man(test, total_data )
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.99)
 print(dtrain.shape)
 print(dvalid.shape )<categorify>
train['Man_with_woman'] = man_with_woman(train, total_data)
 test['Man_with_woman'] = man_with_woman(test, total_data )
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand_name': np.array(dataset.brand_name)
 ,'category_name': np.array(dataset.category_name)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )<compute_train_metric>
train['Man_with_other_men'] = man_with_woman(train, total_data)
 test['Man_with_other_men'] = man_with_woman(test, total_data )
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.1
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 50 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category_name)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128 )(main_l))
main_l = Dropout(dr_r )(Dense(64 )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name
, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()<train_model>
for column in ['Is_youngest_man', 'Is_oldest_man', 'Man_with_woman', 'Man_with_other_men']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )
BATCH_SIZE = 20000
epochs = 5
model = get_model()
model.fit(X_train, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid, dvalid.target)
 , verbose=1 )<compute_train_metric>
def is_youngest_children(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Title[i] == 5):
match =(data.loc[i].Age <= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(match_data['Title'] == 5)].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list
val_preds = model.predict(X_valid)
 val_preds = target_scaler.inverse_transform(val_preds)
 val_preds = np.exp(val_preds)+1
y_true = np.array(dvalid.price.values)
 y_pred = val_preds[:,0]
v_rmsle = rmsle(y_true, y_pred)
 print(" RMSLE error on dev test: "+str(v_rmsle))<predict_on_test>
def is_oldest_children(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Title[i] == 5):
match =(data.loc[i].Age >= max(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(match_data['Title'] == 5)].Age)) *1
match_list.append(match)
 else:
match_list.append(0)
 return match_list
preds = model.predict(X_test, batch_size=BATCH_SIZE)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1
submission = test[["test_id"]]
submission["price"] = preds<save_to_csv>
def is_the_only_child(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Title[i] == 5):
match = len(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(match_data['Title'] == 5)])
 if match == 1:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
submission.to_csv("./myNNsubmission4.csv", index=False)
 submission.price.hist()<set_options>
train['Is_youngest_children'] = is_youngest_children(train, total_data)
 test['Is_youngest_children'] = is_youngest_children(test, total_data )
%matplotlib inline<feature_engineering>
train['Is_oldest_children'] = is_oldest_children(train, total_data)
 test['Is_oldest_children'] = is_oldest_children(test, total_data )
def handle_missing_inplace(dataset):
dataset['category_name'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category = dataset['category_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['category_name'].isin(pop_category), 'category_name'] = 'missing'
def to_categorical(dataset):
dataset['category_name'] = dataset['category_name'].astype('category')
 dataset['brand_name'] = dataset['brand_name'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category' )<load_from_csv>
train['Is_the_only_child'] = is_the_only_child(train, total_data)
 test['Is_the_only_child'] = is_the_only_child(test, total_data )
train = pd.read_table('.. /input/train.tsv', engine='c')
 test = pd.read_table('.. /input/test.tsv', engine='c')
 nrow_train = train.shape[0]<prepare_x_and_y>
for column in ['Is_youngest_children', 'Is_oldest_children', 'Is_the_only_child']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )
y = np.log1p(train["price"])
 merge: pd.DataFrame = pd.concat([train, test])
 submission: pd.DataFrame = test[['test_id']]<categorify>
def family_survived(data, match_data):
match_list = []
for i in range(len(data)) :
if data.Family_size[i] > 1:
survived =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(data['PassengerId'][i] != match_data['PassengerId'])].Survived ).sum()
if survived > 0:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
NUM_BRANDS = 4000
NUM_CATEGORIES = 1000
NAME_MIN_DF = 10
MAX_FEATURES_ITEM_DESCRIPTION = 50000
handle_missing_inplace(merge)
 cutting(merge)
 to_categorical(merge )<feature_engineering>
def is_a_wife_with_survived_children(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Name[i].find('(')> -1)&(data.Sex[i] == 1):
survived =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(data['PassengerId'][i] != match_data['PassengerId'])\
&(match_data['Title'] == 5)].Survived ).sum()
if survived > 0:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
cv = CountVectorizer(min_df=NAME_MIN_DF)
 X_name = cv.fit_transform(merge['name'])
 cv = CountVectorizer()
X_category = cv.fit_transform(merge['category_name'])
 tv = TfidfVectorizer(max_features=MAX_FEATURES_ITEM_DESCRIPTION,
ngram_range=(1, 3),
stop_words='english')
 X_description = tv.fit_transform(merge['item_description'])
 lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(merge['brand_name'])
 X_dummies = csr_matrix(pd.get_dummies(merge[['item_condition_id', 'shipping']],
sparse=True ).values)
 sparse_merge = hstack(( X_dummies, X_description, X_brand, X_category, X_name)).tocsr()
X = sparse_merge[:nrow_train]
X_test = sparse_merge[nrow_train:]<choose_model_class>
def is_a_child_with_other_candw_survived(data, match_data):
match_list = []
for i in range(len(data)) :
if(data.Title[i] == 5):
survived =(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\
&(data['Family_size'][i] == match_data['Family_size'])\
&(data['Pclass'][i] == match_data['Pclass'])\
&(data['PassengerId'][i] != match_data['PassengerId'])\
&(( match_data['Title'] == 5)|(match_data['Sex'] == 0)) ].Survived ).sum()
if survived > 0:
match_list.append(1)
 else:
match_list.append(0)
 else:
match_list.append(0)
 return match_list
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3)
 model.fit(X, y)
 predsR = model.predict(X=X_test)
 model = Ridge(solver="lsqr", fit_intercept=False, random_state=145, alpha = 3)
 model.fit(X, y)
 predsR2 = model.predict(X=X_test)
 model = Ridge(solver="sag", fit_intercept=False, random_state=205, alpha = 3)
 model.fit(X, y)
 predsR3 = model.predict(X=X_test )<choose_model_class>
train['Family_survived'] = family_survived(train, total_data)
 test['Family_survived'] = family_survived(test, total_data )
model = RandomForestRegressor(max_features='log2', min_weight_fraction_leaf=0.1)
 model.fit(X, y)
 predsL = model.predict(X=X_test)
<save_to_csv>
train['Is_a_wife_with_sc'] = is_a_wife_with_survived_children(train, total_data)
 test['Is_a_wife_with_sc'] = is_a_wife_with_survived_children(test, total_data )
preds1 = np.expm1(predsR3*0.24 + predsR2*0.24 + predsR*0.52)
 preds2 = np.expm1(predsL)
 submission['price'] = preds1*0.75 + preds2*0.25
submission.to_csv('submission.csv', index=False )<set_options>
train['Is_a_child_with_wandc_survived'] = is_a_child_with_other_candw_survived(train, total_data)
 test['Is_a_child_with_wandc_survived'] = is_a_child_with_other_candw_survived(test, total_data )
def timer(start_time=None):
if not start_time:
start_time = datetime.now()
return start_time
elif start_time:
thour, temp_sec = divmod(
(datetime.now() - start_time ).total_seconds() , 3600)
 tmin, tsec = divmod(temp_sec, 60)
 print('
Time taken: %i hours %i minutes and %s seconds.' % 
(thour, tmin, round(tsec, 2)))
 start_time = timer(None)
 def load_test() :
for df in pd.read_csv('.. /input/test.tsv', sep='\t', chunksize=700000):
yield df
def rmse_min_func(weights):
final_prediction = 0
for weight, prediction in zip(weights, blend_train):
final_prediction += weight * prediction
return np.sqrt(mean_squared_error(y_train, final_prediction))
pd.set_option('display.float_format', lambda x: '%.3f' % x)
 njobs = -1
train = pd.read_csv(".. /input/train.tsv", sep='\t')
 test_chunk_count = 0
def DescriptionNotMeaningless(row):
if(row == "No description yet" or row == "No category name given" or row == "No brand name given" or len(row)== 1 or row == u"\uFFFC" or row.isspace() or not row or all(j in string.punctuation for j in row)) :
r = 0
else:
r = 1
return r
for test in load_test() :
if test_chunk_count == 0:
tr_ids = train['train_id'].copy()
train.drop("train_id", axis = 1, inplace = True)
 te_ids = test['test_id'].copy()
test.drop("test_id", axis = 1, inplace = True)
 PRICE = np.log1p(train['price'].copy())
 train.item_description = train.item_description.fillna("No description yet")
 test.item_description = test.item_description.fillna("No description yet")
 train['has_description'] = train.item_description.apply(lambda row : DescriptionNotMeaningless(row))
test['has_description'] = test.item_description.apply(lambda row : DescriptionNotMeaningless(row))
train.brand_name = train.brand_name.fillna("No brand name given")
 test.brand_name = test.brand_name.fillna("No brand name given")
 timer(start_time)
 brand_names = train.brand_name.unique()
test.brand_name.replace([name for name in test.brand_name if name not in brand_names], 'No brand name given', inplace=True)
 timer(start_time)
 train['has_brand'] = train.brand_name.apply(lambda row : DescriptionNotMeaningless(row))
test['has_brand'] = test.brand_name.apply(lambda row : DescriptionNotMeaningless(row))
train.category_name = train.category_name.fillna("No category name given")
 test.category_name = test.category_name.fillna("No category name given")
 category_names = train.category_name.unique()
test.category_name.replace([name for name in test.category_name if name not in category_names], 'No category name given', inplace=True)
 train['has_category'] = train.category_name.apply(lambda row : DescriptionNotMeaningless(row))
test['has_category'] = test.category_name.apply(lambda row : DescriptionNotMeaningless(row))
train['brand_price'] = train.groupby('brand_name')['price'].transform(np.mean)
 mapping = dict(train[['brand_name', 'brand_price']].values)
 test['brand_price'] = test.brand_name.replace(mapping)
 timer(start_time)
 train['category_price'] = train.groupby('category_name')['price'].transform(np.mean)
 mapping2 = dict(train[['category_name', 'category_price']].values)
 test['category_price'] = test.category_name.replace(mapping2)
 timer(start_time)
 train['item_description'] = train['item_description'].astype('category')
 train['brand_name'] = train['brand_name'].astype('category')
 train['has_brand'] = train['has_brand'].astype('category')
 train['has_description'] = train['has_description'].astype('category')
 train['has_category'] = train['has_category'].astype('category')
 train['category_name'] = train['category_name'].astype('category')
 test['item_description'] = test['item_description'].astype('category')
 test['brand_name'] = test['brand_name'].astype('category')
 test['has_brand'] = test['has_brand'].astype('category')
 test['has_description'] = test['has_description'].astype('category')
 test['has_category'] = test['has_category'].astype('category')
 test['category_name'] = test['category_name'].astype('category')
 numerical_features = train.drop(['price'],axis=1 ).select_dtypes(exclude = ["object", "category"] ).columns
stdSc = StandardScaler()
train.loc[:, numerical_features] = stdSc.fit_transform(train.loc[:, numerical_features])
 test.loc[:, numerical_features] = stdSc.transform(test.loc[:, numerical_features])
 timer(start_time)
 df = pd.concat([train, test])
 nrow_train = train.shape[0]
del train,test
gc.collect()
print('Encodings')
 count = CountVectorizer(min_df=10)
 X_name = count.fit_transform(df['name'])
 print('Category Encoders')
 count_category = CountVectorizer()
X_category = count_category.fit_transform(df['category_name'])
 print('Descp encoders')
 count_descp = TfidfVectorizer(max_features=50000,
ngram_range=(1, 3),
stop_words='english')
 X_descp = count_descp.fit_transform(df['item_description'])
 print('Brand encoders')
 vect_brand = LabelBinarizer(sparse_output=True)
 X_brand = vect_brand.fit_transform(df['brand_name'])
 print('Dummy Encoders')
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
'item_condition_id', 'has_description', 'has_brand', 'has_category', 'shipping'
]], sparse=True ).values)
 df_clean = scipy.sparse.hstack(( X_dummies, X_brand, X_category, X_descp, X_name,
scipy.sparse.csr_matrix(df['brand_price'] ).T,
scipy.sparse.csr_matrix(df['category_price'] ).T)).tocsr()
train = df_clean[:nrow_train]
test = df_clean[nrow_train:]
print(train.shape)
 print(test.shape)
 timer(start_time)
 y_train = PRICE
folds = 2
sgd_cv_sum = 0
ridge_cv_sum = 0
lgb_cv_sum = 0
lgb_pred = []
sgd_pred = []
ridge_pred = []
lgb_fpred = []
sgd_fpred = []
ridge_fpred = []
avreal = y_train
lgb_avpred = np.zeros(train.shape[0])
 sgd_avpred = np.zeros(train.shape[0])
 ridge_avpred = np.zeros(train.shape[0])
 idpred = tr_ids
blend_train = []
blend_test = []
model1 = []
model2 = []
model3 = []
train_time = timer(None)
 kf = KFold(n_splits=folds, random_state=1001)
 for i,(train_index, val_index)in enumerate(kf.split(train, y_train)) :
start_time = timer(None)
 Xtrain, Xval = train[train_index], train[val_index]
ytrain, yval = y_train[train_index], y_train[val_index]
model = SGDRegressor(penalty='l2',
loss='squared_epsilon_insensitive',
max_iter=200,
tol=0.00001,
epsilon=0.0001,
learning_rate='invscaling',
fit_intercept=False,
alpha=1e-10,
l1_ratio=0.09,
shuffle=True,
verbose=0,
random_state=1001)
 model.fit(Xtrain, ytrain)
 sgd_scores_val = model.predict(Xval)
 sgd_RMSLE = np.sqrt(mean_squared_error(yval, sgd_scores_val))
print('Fold %02d SGD RMSLE: %.6f' %(( i + 1), sgd_RMSLE))
sgd_y_pred = model.predict(test)
 model1.append(model)
 model = Ridge(alpha=4.75,
solver='sag',
fit_intercept=False,
random_state=1001,
max_iter=1000)
 model.fit(Xtrain, ytrain)
 ridge_scores_val = model.predict(Xval)
 ridge_RMSLE = np.sqrt(mean_squared_error(yval, ridge_scores_val))
print(' Fold %02d Ridge RMSLE: %.6f' %(( i + 1), ridge_RMSLE))
ridge_y_pred = model.predict(test)
 model2.append(model)
 params = {
'boosting': 'gbdt',
'max_bin' :1000,
'max_depth': 7,
'min_data_in_leaf': 80,
'num_leaves': 40,
'learning_rate': 0.75,
'objective': 'regression',
'metric': 'rmse',
'nthread': 4,
'bagging_freq': 1,
'subsample': 0.94,
'colsample_bytree': 0.7,
'min_child_weight': 17,
'is_unbalance': False,
'verbose': -1,
'seed': 1001
}
dtrain = lgb.Dataset(Xtrain, label=ytrain)
 dval = lgb.Dataset(Xval, label=yval)
 watchlist = [dtrain, dval]
watchlist_names = ['train', 'val']
model = lgb.train(params,
train_set=dtrain,
num_boost_round=1800,
valid_sets=watchlist,
valid_names=watchlist_names,
early_stopping_rounds=80,
verbose_eval=80)
 lgb_scores_val = model.predict(Xval)
 lgb_RMSLE = np.sqrt(mean_squared_error(yval, lgb_scores_val))
print(' Fold %02d LightGBM RMSLE: %.6f' %(( i + 1), lgb_RMSLE))
lgb_y_pred = model.predict(test)
 model3.append(model)
 del Xtrain, Xval
gc.collect()
timer(start_time)
 sgd_avpred[val_index] = sgd_scores_val
ridge_avpred[val_index] = ridge_scores_val
lgb_avpred[val_index] = lgb_scores_val
if i > 0:
sgd_fpred = sgd_pred + sgd_y_pred
ridge_fpred = ridge_pred + ridge_y_pred
lgb_fpred = lgb_pred + lgb_y_pred
else:
sgd_fpred = sgd_y_pred
ridge_fpred = ridge_y_pred
lgb_fpred = lgb_y_pred
sgd_pred = sgd_fpred
ridge_pred = ridge_fpred
lgb_pred = lgb_fpred
sgd_cv_sum = sgd_cv_sum + sgd_RMSLE
ridge_cv_sum = ridge_cv_sum + ridge_RMSLE
lgb_cv_sum = lgb_cv_sum + lgb_RMSLE
timer(train_time)
 sgd_cv_score =(sgd_cv_sum / folds)
 ridge_cv_score =(ridge_cv_sum / folds)
 lgb_cv_score =(lgb_cv_sum / folds)
 sgd_oof_RMSLE = np.sqrt(mean_squared_error(avreal, sgd_avpred))
ridge_oof_RMSLE = np.sqrt(mean_squared_error(avreal, ridge_avpred))
lgb_oof_RMSLE = np.sqrt(mean_squared_error(avreal, lgb_avpred))
print('Average SGD RMSLE:	%.6f' % sgd_cv_score)
 print(' Out-of-fold SGD RMSLE:	%.6f' % sgd_oof_RMSLE)
 print('Average Ridge RMSLE:	%.6f' % ridge_cv_score)
 print(' Out-of-fold Ridge RMSLE:	%.6f' % ridge_oof_RMSLE)
 print('Average LightGBM RMSLE:	%.6f' % lgb_cv_score)
 print(' Out-of-fold LightGBM RMSLE:	%.6f' % lgb_oof_RMSLE)
 sgd_score = round(sgd_oof_RMSLE, 6)
 ridge_score = round(ridge_oof_RMSLE, 6)
 lgb_score = round(lgb_oof_RMSLE, 6)
 sgd_mpred = sgd_pred / folds
ridge_mpred = ridge_pred / folds
lgb_mpred = lgb_pred / folds
blend_time = timer(None)
 blend_train.append(sgd_avpred)
 blend_train.append(ridge_avpred)
 blend_train.append(lgb_avpred)
 blend_train = np.array(blend_train)
 blend_test.append(sgd_mpred)
 blend_test.append(ridge_mpred)
 blend_test.append(lgb_mpred)
 blend_test = np.array(blend_test)
 print('
Finding Blending Weights...')
 res_list = []
weights_list = []
for k in range(20):
starting_values = np.random.uniform(size=len(blend_train))
bounds = [(-1, 1)] * len(blend_train)
 res = minimize(rmse_min_func,
starting_values,
method='L-BFGS-B',
bounds=bounds,
options={'disp': False,
'maxiter': 100000})
 res_list.append(res['fun'])
 weights_list.append(res['x'])
 print('{iter}\tScore: {score}\tWeights: {weights}'.format(
 iter=(k + 1),
score=res['fun'],
weights='\t'.join([str(item)for item in res['x']])))
 bestSC = np.min(res_list)
 bestWght = weights_list[np.argmin(res_list)]
weights = bestWght
blend_score = round(bestSC, 6)
 print('
Ensemble Score: {best_score}'.format(best_score=bestSC))
print('
Best Weights: {weights}'.format(weights=bestWght))
train_prices = np.zeros(len(blend_train[0]))
test_prices = np.zeros(len(blend_test[0]))
print('
Your final model:')
 for k in range(len(blend_test)) :
print(' %.6f * model-%d' %(weights[k],(k + 1)))
 test_prices += blend_test[k] * weights[k]
for k in range(len(blend_train)) :
train_prices += blend_train[k] * weights[k]
submission = test_prices
test_chunk_count = 1
del train
gc.collect()
else:
te_ids_new = test['test_id'].copy()
test.drop("test_id", axis = 1, inplace = True)
 test.item_description = test.item_description.fillna("No description yet")
 test['has_description'] = test.item_description.apply(lambda row : DescriptionNotMeaningless(row))
test.brand_name = test.brand_name.fillna("No brand name given")
 timer(start_time)
 test.brand_name.replace([name for name in test.brand_name if name not in brand_names], 'No brand name given', inplace=True)
 timer(start_time)
 test['has_brand'] = test.brand_name.apply(lambda row : DescriptionNotMeaningless(row))
test.category_name = test.category_name.fillna("No category name given")
 test.category_name.replace([name for name in test.category_name if name not in category_names], 'No category name given', inplace=True)
 test['has_category'] = test.category_name.apply(lambda row : DescriptionNotMeaningless(row))
test['brand_price'] = test.brand_name.replace(mapping)
 timer(start_time)
 test['category_price'] = test.category_name.replace(mapping2)
 timer(start_time)
 test['item_description'] = test['item_description'].astype('category')
 test['brand_name'] = test['brand_name'].astype('category')
 test['has_brand'] = test['has_brand'].astype('category')
 test['has_description'] = test['has_description'].astype('category')
 test['has_category'] = test['has_category'].astype('category')
 test['category_name'] = test['category_name'].astype('category')
 test.loc[:, numerical_features] = stdSc.transform(test.loc[:, numerical_features])
 df = test
timer(start_time)
 print('Encodings')
 X_name = count.transform(df['name'])
 print('Category Encoders')
 X_category = count_category.transform(df['category_name'])
 print('Descp encoders')
 X_descp = count_descp.transform(df['item_description'])
 print('Brand encoders')
 X_brand = vect_brand.transform(df['brand_name'])
 print('Dummy Encoders')
 X_dummies = scipy.sparse.csr_matrix(pd.get_dummies(df[[
'item_condition_id', 'has_description', 'has_brand', 'has_category', 'shipping'
]], sparse=True ).values)
 df_clean = scipy.sparse.hstack(( X_dummies, X_brand, X_category, X_descp, X_name,
scipy.sparse.csr_matrix(df['brand_price'] ).T,
scipy.sparse.csr_matrix(df['category_price'] ).T)).tocsr()
test = df_clean
print(test.shape)
 timer(start_time)
 test_chunk_count = test_chunk_count+1
sgd_pred = sum([model.predict(test)for model in model1])
 ridge_pred = sum([model.predict(test)for model in model2])
 lgb_pred = sum([model.predict(test)for model in model3])
 sgd_mpred = sgd_pred / folds
ridge_mpred = ridge_pred / folds
lgb_mpred = lgb_pred / folds
blend_test = []
blend_test.append(sgd_mpred)
 blend_test.append(ridge_mpred)
 blend_test.append(lgb_mpred)
 blend_test = np.array(blend_test)
 test_prices = np.zeros(len(blend_test[0]))
for k in range(len(blend_test)) :
print(' %.6f * model-%d' %(weights[k],(k + 1)))
 test_prices += blend_test[k] * weights[k]
submission = np.concatenate([submission, test_prices], axis=0)
 te_ids = np.concatenate([te_ids, te_ids_new], axis=0)
 submission = np.expm1(submission)
 submission = pd.DataFrame({
"test_id": te_ids,
"price": submission
})
 submission.to_csv('submission.csv', index=False)
 timer(blend_time)
 timer(start_time )<feature_engineering>
for column in ['Family_survived', 'Is_a_wife_with_sc', 'Is_a_child_with_wandc_survived']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )
start_time = time.time()
os.environ['MKL_NUM_THREADS'] = '4'
os.environ['OMP_NUM_THREADS'] = '4'
start_time = time.time()<data_type_conversions>
for value in [(0,15),(15,35),(35,50),(50,65),(65,100)]:
get_continuous_feature_info(train, column='Age', y_column='Survived', xlim=value )
def preprocess_RNN(dataset):
print("Filling Missing Values")
 dataset['category_name'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 print("Casting data types to type Category")
 dataset['category_name'] = dataset['category_name'].astype('category')
 dataset['brand_name'] = dataset['brand_name'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category')
 print("RNN PreProcessing completed")
 return dataset
def fit_RNN_text(dataset):
print("Fit Name and Item description fields for Tokenization")
 raw_text = np.hstack([dataset.item_description.str.lower() , dataset.name.str.lower() ])
 word_token = Tokenizer()
word_token.fit_on_texts(raw_text)
 print("RNN Data fit completed")
 return word_token
def fit_RNN_label(dataset):
print("Fit Categorical variables on full Merged Test and Train data")
 le_name = LabelEncoder()
le_name.fit(dataset.category_name)
 le_brand = LabelEncoder()
le_brand.fit(dataset.brand_name)
 print("Completed Label fitting")
 return le_name, le_brand
def transform_RNN(dataset, le_name, le_brand, word_token):
print("Use Defined Label encoders to Encode brand and category_name")
 dataset['category'] = le_name.transform(dataset.category_name)
 dataset['brand'] = le_brand.transform(dataset.brand_name)
 print("Convert Text to sequences")
 dataset["seq_item_description"] = word_token.texts_to_sequences(dataset.item_description.str.lower())
 dataset["seq_name"] = word_token.texts_to_sequences(dataset.name.str.lower())
 print("Sequence Conversion Completed")
 return dataset
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 ,'item_desc': pad_sequences(dataset.seq_item_description
, maxlen=MAX_ITEM_DESC_SEQ)
 ,'brand': np.array(dataset.brand)
 ,'category': np.array(dataset.category)
 ,'item_condition': np.array(dataset.item_condition_id)
 ,'num_vars': np.array(dataset[["shipping"]])
 }
print("Data ready for Vectorization")
 return X<categorify>
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 14, 'Age'] = 0
dataset.loc[(dataset['Age'] > 14)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
def RNN_model() :
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand = Input(shape=[1], name="brand")
 category = Input(shape=[1], name="category")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_size = 60
emb_name = Embedding(MAX_TEXT, emb_size//3 )(name)
 emb_item_desc = Embedding(MAX_TEXT, emb_size )(item_desc)
 emb_brand = Embedding(MAX_BRAND, 10 )(brand)
 emb_category = Embedding(MAX_CATEGORY, 10 )(category)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(25 )(emb_item_desc)
 rnn_layer2 = GRU(12 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand)
 , Flatten()(emb_category)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(0.1 )(Dense(512,activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(64,activation='relu' )(main_l))
output = Dense(1,activation="linear" )(main_l)
 model = Model([brand, category, item_condition, item_desc, name, num_vars], output)
 optimizer = optimizers.Adam()
model.compile(loss="mse",
optimizer=optimizer)
 return model
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 \
for i, pred in enumerate(y_pred)]
return(sum(to_sum)*(1.0/len(y)))** 0.5
def eval_model(model):
val_preds = model.predict(X_valid)
 val_preds = np.expm1(val_preds)
 y_pred = val_preds[:, 0]
y_true = np.array(valid_prices)
 yt = pd.DataFrame(y_true)
 yp = pd.DataFrame(y_pred)
 print(yt.isnull().any())
 print(yp.isnull().any())
 v_rmsle = rmsle(y_true, y_pred)
 print(" RMSLE error on dev test: "+str(v_rmsle))
return v_rmsle
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1<feature_engineering>
test.isna().sum() , train.isna().sum()
t1 = time.time()
train = pd.read_table('.. /input/mercari-price-suggestion-challenge/train.tsv', sep='\t')
 train = train[train['price'] !=0 ]
ntrain = train.shape[0]
train['target'] = np.log1p(train['price'])
 test = pd.read_table('.. /input/mercari-price-suggestion-challenge/test_stg2.tsv', sep='\t')
 ntest = test.shape[0]
merge = pd.concat([train, test], 0, ignore_index = True)
 del test, train
gc.collect()<categorify>
Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train[train['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train[train['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))
t1 = time.time()
merge = preprocess_RNN(merge)
 t2 = time.time()
print("Time taken for RNN preprocess "+str(t2-t1))
print(merge.shape)
 t1 = time.time()
le_name, le_brand = fit_RNN_label(merge)
 t2 = time.time()
print("Time taken to Fit on RNN "+str(t2-t1))
t1 = time.time()
train = merge[:ntrain]
word_token = fit_RNN_text(train)
 train = transform_RNN(train, le_name, le_brand, word_token)
 t2 = time.time()
print("Time taken to Fit and Transform on RNN "+str(t2-t1))<define_variables>
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S' )
MAX_NAME_SEQ = 20
MAX_ITEM_DESC_SEQ = 60
MAX_CATEGORY_NAME_SEQ = 20
MAX_CATEGORY = np.max(train.category.max())+1
MAX_BRAND = np.max(train.brand.max())+1
MAX_CONDITION = 6
MAX_TEXT = np.max([np.max(train.seq_name.max()), np.max(train.seq_item_description.max())])+2 
<split>
embarked_mapping = {"C": "Cherbourg", "S": "Southampton", "Q": "Queenstown"}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )
dtrain, dvalid = train_test_split(train, random_state=233, train_size=0.99)
 dtrain['target'] =np.log1p(dtrain['price'])
 target = np.array(dtrain.target)
 dvalid['target'] =np.log1p(dvalid['price'])
 valid_prices = np.array(dvalid.target)
 print(dvalid.shape, dtrain.shape )<prepare_x_and_y>
for dataset in train_test_data:
dataset['Is_AloneandPoor'] = 0
dataset.loc[(dataset['Family_size'] == 1)&(dataset['Pclass'] == 3),'Is_AloneandPoor'] = 1
del train
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 del dtrain, dvalid
gc.collect()<choose_model_class>
get_categorical_feature_info(train, column='Is_AloneandPoor', y_column='Survived', positive_state='Survived', negative_state='Dead' )
epochs = 1
BATCH_SIZE = 512 * 3
steps = int(len(X_train['name'])/BATCH_SIZE)* epochs
lr_init, lr_fin = 0.009, 0.0045
lr_decay = exp_decay(lr_init, lr_fin, steps)
 modelRNN = RNN_model()
K.set_value(modelRNN.optimizer.lr, lr_init)
 K.set_value(modelRNN.optimizer.decay, lr_decay)
<train_model>
for dataset in train_test_data:
dataset['IsChildandRich'] = 0
dataset.loc[(dataset['Title'] == 5)&(dataset['Pclass'] < 3),'IsChildandRich'] = 1
os.environ['OMP_NUM_THREADS'] = '4'
os.environ['MKL_NUM_THREADS'] = '4'
for i in range(2):
history = modelRNN.fit(X_train, target
, epochs=epochs
, batch_size=2**(11+i)
 , validation_data =(X_valid, valid_prices)
 , verbose=1 
)
 v_rmsle = eval_model(modelRNN)
 print('[{}] Finished predicting valid set...'.format(time.time() - start_time))
print("Finished Fitting the model" )<drop_column>
get_categorical_feature_info(train, column='IsChildandRich', y_column='Survived', positive_state='Survived', negative_state='Dead' )
del X_train, X_valid
del target, valid_prices
del epochs, lr_init, lr_fin, exp_decay, lr_decay
gc.collect()<load_from_csv>
for data in train_test_data:
data['Cabin'] = data['Cabin'].fillna('X')
 data['Cabin'] = data['Cabin'].apply(lambda x: str(x)[0])
 data['Cabin'] = data['Cabin'].replace(['A', 'D', 'E', 'T'], 'M')
 data['Cabin'] = data['Cabin'].replace(['B', 'C'], 'H')
 data['Cabin'] = data['Cabin'].replace(['F', 'G'], 'L' )
t1 = time.time()
train = pd.read_csv('.. /input/mercari-price-suggestion-challenge/train.tsv', sep='\t')
 train = train[train['price'] != 0]
train['target'] = np.log1p(train['price'])
 n_train = train.shape[0]<data_type_conversions>
get_categorical_feature_info(train, column='Cabin', y_column='Survived', positive_state='Survived', negative_state='Dead' )
def split_cat(text):
try:
s1, s2, s3 = text.split("/")
 return s1, s2, s3
except: return("No Label", "No Label", "No Label")
 def wordbatch_preprocess(df):
df["category_name"] = df["category_name"].fillna(value="missing" ).astype(str)
 df["name"] = df["name"].fillna(value="missing" ).astype(str)
 df["brand_name"] = df["brand_name"].fillna(value="missing" ).astype(str)
 df["item_description"] = df["item_description"].fillna(value="missing" ).astype(str)
 df["item_condition_id"] = df["item_condition_id"].astype(int)
 df["shipping"] = df["shipping"].astype(int)
 print("Got so far")
 print(df.dtypes)
 df['subcat_0'], df['subcat_1'], df['subcat_2'] = zip(*df['category_name'].apply(lambda x: split_cat(x)))
 print(df.dtypes)
 return df
train = wordbatch_preprocess(train)
 print("Pre Process for Word batch is done" )<train_model>
for dataset in train_test_data:
dataset['Family_size_group'] = 'Small'
dataset.loc[dataset['Family_size'] == 1, 'Family_size_group'] = 'Alone'
dataset.loc[dataset['Family_size'] > 2, 'Family_size_group'] = 'Big'
class WordBatchModel(object):
def __init__(self):
self.wb_desc = None
self.desc_indices = None
self.cv_name, self.cv_name2 = None, None
self.cv_cat0, self.cv_cat1, self.cv_cat2 = None, None, None
self.cv_brand = None
self.cv_condition = None
self.cv_cat_brand = None
self.desc3 = None
self.model = None
def train_wbm(self, df):
self.wb_desc = wordbatch.WordBatch(None,
extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28, "norm": "l2", "tf": 1.0,
"idf": None}), procs=8)
 self.wb_desc.dictionary_freeze = True
X_desc = self.wb_desc.fit_transform(df['item_description'])
 self.desc_indices = np.array(np.clip(X_desc.getnnz(axis=0)- 1, 0, 1), dtype=bool)
 X_desc = X_desc[:, self.desc_indices]
self.cv_cat0 = CountVectorizer(min_df=2)
 X_category0 = self.cv_cat0.fit_transform(df['subcat_0'])
 self.cv_cat1 = CountVectorizer(min_df=2)
 X_category1 = self.cv_cat1.fit_transform(df['subcat_1'])
 self.cv_cat2 = CountVectorizer(min_df=2)
 X_category2 = self.cv_cat2.fit_transform(df['subcat_2'])
 self.cv_brand = CountVectorizer(min_df=2, token_pattern=".+")
 X_brand = self.cv_brand.fit_transform(df['brand_name'])
 self.cv_name = CountVectorizer(min_df=2, ngram_range=(1, 1),binary=True, token_pattern="\w+")
 X_name = 2 * self.cv_name.fit_transform(df['name'])
 self.cv_name2 = CountVectorizer(min_df=2, ngram_range=(2, 2), binary=True, token_pattern="\w+")
 X_name2 = 0.5 * self.cv_name2.fit_transform(df['name'])
 df["cat_brand"] = [a + " " + b for a, b in zip(df["category_name"], df["brand_name"])]
self.cv_cat_brand = CountVectorizer(min_df=10, token_pattern=".+")
 X_cat_brand = self.cv_cat_brand.fit_transform(df["cat_brand"])
 self.cv_condition = CountVectorizer(token_pattern=".+")
 X_condition = self.cv_condition.fit_transform(( df['item_condition_id'] + 10 * df["shipping"] ).apply(str))
self.desc3 = CountVectorizer(ngram_range=(3, 3), max_features=1000, binary=True, token_pattern="\w+")
 X_desc3 = self.desc3.fit_transform(df["item_description"])
 X = hstack(( X_condition,
X_desc, X_brand,
X_category0, X_category1, X_category2,
X_name, X_name2,
X_cat_brand,
X_desc3 
)).tocsr()
print("X Reconstructed")
 y = df['target'].values
print("Y created")
 self.model = FM_FTRL(alpha=0.01, beta=0.01, L1=0.00001, L2=0.1, D=X.shape[1], alpha_fm=0.02, L2_fm=0.0,
init_fm=0.01, D_fm=200, e_noise=0.0001, iters=10, inv_link="identity", threads=4)
 print("Model Defined")
 self.model.fit(X, y)
 def predict(self, df):
X_desc = self.wb_desc.transform(df["item_description"])
 X_desc = X_desc[:, self.desc_indices]
X_brand = self.cv_brand.transform(df['brand_name'])
 X_name = 2 * self.cv_name.transform(df["name"])
 X_name2 = 0.5 * self.cv_name2.transform(df["name"])
 X_category0 = self.cv_cat0.transform(df['subcat_0'])
 X_category1 = self.cv_cat1.transform(df['subcat_1'])
 X_category2 = self.cv_cat2.transform(df['subcat_2'])
 X_condition = self.cv_condition.transform(( df['item_condition_id'] + 10 * df["shipping"] ).apply(str))
df["cat_brand"] = [a + " " + b for a, b in zip(df["category_name"], df["brand_name"])]
X_cat_brand = self.cv_cat_brand.transform(df["cat_brand"])
 X_desc3 = self.desc3.transform(df["item_description"])
 X = hstack(( X_condition,
X_desc, X_brand,
X_category0, X_category1, X_category2,
X_name, X_name2,
X_cat_brand,
X_desc3 
)).tocsr()
return self.model.predict(X )<train_model>
train['IsMaleandPoor'] = 0
train.loc[(train['Sex'] == 0)&(train['Pclass'] == 3),'IsMaleandPoor'] = 1
test['IsMaleandPoor'] = 0
test.loc[(test['Sex'] == 0)&(test['Pclass'] == 3),'IsMaleandPoor'] = 1
os.environ['MKL_NUM_THREADS'] = '4'
os.environ['OMP_NUM_THREADS'] = '4'
print(train.shape)
 t1 = time.time()
wbm = WordBatchModel()
wbm.train_wbm(train)
 t2 = time.time()
print("Time taken for Word Batch is "+str(t2-t1))
del train
gc.collect()<load_from_csv>
for column in ['Family_size_group', 'IsMaleandPoor']:
get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead' )
t1 = time.time()
def load_test() :
for df in pd.read_csv('.. /input/mercari-price-suggestion-challenge/test_stg2.tsv', sep='\t', chunksize= 1000000):
yield df
test_ids = np.array([], dtype=np.int32)
 preds= np.array([], dtype=np.float32)
 i = 0
for df in load_test() :
i +=1
print(" Chunk number is "+str(i))
df1 = df2 = df
testRNN = preprocess_RNN(df1)
 print(df1.dtypes)
 print(testRNN.dtypes)
 testRNN = transform_RNN(testRNN, le_name, le_brand, word_token)
 print(testRNN.dtypes)
 X_testRNN = get_keras_data(testRNN)
 predsRNN1 = modelRNN.predict(X_testRNN, batch_size = BATCH_SIZE, verbose = 1)
 test_id = df['test_id']
del testRNN
del df['seq_item_description'], df['seq_name'], df['brand'], df['category']
gc.collect()
print("RNN Prediction is done")
 print(df.isnull().any())
 print(df.dtypes)
 test_WB = wordbatch_preprocess(df2)
 print(df.dtypes)
 print("Word batch preprocess done")
 predsWB1 = wbm.predict(test_WB)
 print("Word Batch Prediction done")
 predsRNN = np.expm1(predsRNN1)
 predsWB = np.expm1(predsWB1)
 predsRNN = predsRNN.reshape(predsRNN.shape[0],1)
 predsWB = predsWB.reshape(predsWB.shape[0],1)
 predsRNN = np.clip(predsRNN, 0, predsRNN.max())
 predsWB = np.clip(predsWB, 0, predsWB.max())
 preds= np.append(preds,(( predsRNN + predsWB)*0.5))
test_ids = np.append(test_ids, test_id)
 print("All chunks done")
 t2 = time.time()
print("Total time for Parallel Batch Prediction is "+str(t2-t1))<save_to_csv>
for dataset in train_test_data:
dataset.drop(['PassengerId', 'Name', 'Ticket', 'Last_Name', 'SibSp', 'Ticket'\
, 'Family_size'], axis=1, inplace=True )
submission = pd.DataFrame(columns = ['test_id', 'price'])
 submission['test_id'] = test_ids
submission['price'] = preds
print("Check Submission NOW!!!!!!!!@")
 submission.to_csv("BatchPrediction_MemoryCoreOptimized.csv", index=False )<train_model>
def get_dummies_from_list(categorical_list):
dummies = pd.get_dummies(pd.Categorical(categorical_list), sparse=True)
 return dummies
t2 = time.time()
print("Total time taken is "+str(t2-start_time))<set_options>
def transform_dummies(data, column_list=[]):
for column in column_list:
dummy_df = get_dummies_from_list(data[column])
 data = data.merge(dummy_df, how='left', left_index=True, right_index=True)
 data.drop(labels=column_list, axis='columns', inplace=True)
 return data
%%time
time_start=datetime.datetime.now()
session_conf = tf.ConfigProto(intra_op_parallelism_threads=10, inter_op_parallelism_threads=1)
 backend.set_session(tf.Session(graph=tf.get_default_graph() , config=session_conf))
os.environ['PYTHONHASHSEED'] = '123'
np.random.seed(123)
 np.random.seed(123)
 random.seed(123)
 set_random_seed(123 )<load_from_csv>
train = transform_dummies(train, column_list=['Embarked', 'Family_size_group', 'Cabin'])
 test = transform_dummies(test, column_list=['Embarked', 'Family_size_group', 'Cabin'] )
%%time
np.random.seed(123)
 train = pd.read_csv('.. /input/train.tsv',sep='\t')
 test = pd.read_csv('.. /input/test.tsv',sep='\t')
 train=train[(train.price>=3)]
full_df=pd.concat([train,test],ignore_index=True,axis=0 )<feature_engineering>
for column in train.columns:
get_indipendent_variables_mix_info(train, x_column=column, y_column='Age', dependent_variable='Survived' )
%%time
all_brands = set(full_df['brand_name'].values)
 full_df.brand_name.fillna('missing',inplace=True)
 premissing = len(full_df.loc[full_df.brand_name == 'missing'])
 def brandfinder(line):
brand = line[0]
name = line[1]
namesplit = name.split(' ')
 if brand == 'missing':
for x in namesplit:
if x in all_brands:
return name
if name in all_brands:
return name
return brand
full_df['brand_name'] = full_df[['brand_name','name']].apply(brandfinder, axis = 1)
 found = premissing-len(full_df.loc[full_df['brand_name'] == 'missing'])
 print(found )<feature_engineering>
abs(corr.Survived ).sort_values(ascending=False )
%%time
def preprocess_data(df):
df.item_description.replace('No description yet',value='missing',inplace=True)
 df.item_description.fillna('missing',inplace=True)
 df.category_name.fillna('missing/missing/missing',inplace=True)
 df['main_category']=df.category_name.apply(lambda x:x.split('/')[0])
 df['sub_category1']=df.category_name.apply(lambda x:x.split('/')[1])
 df['sub_category2']=df.category_name.apply(lambda x:x.split('/')[2])
 preprocess_data(full_df )<categorify>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import numpy as np
%%time
print("Handling categorical variables")
 le=LabelEncoder()
full_df['category_name']=le.fit_transform(full_df.category_name)
 full_df['brand_name'] = le.fit_transform(full_df.brand_name)
 full_df['main_category']=le.fit_transform(full_df.main_category)
 full_df['sub_category1']=le.fit_transform(full_df.sub_category1)
 full_df['sub_category2']=le.fit_transform(full_df.sub_category2)
 del le<split>
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_curve, make_scorer, recall_score, precision_score, average_precision_score, confusion_matrix, accuracy_score
from sklearn.model_selection import GridSearchCV
%%time
train=full_df[:n_train]
test=full_df[n_train:]
dtrain,dvalid=train_test_split(train,random_state=123,test_size=0.01)
<define_variables>
X_train, X_test, y_train, y_test = train_test_split(train_data, train.Survived, test_size=0.10, stratify=train['Survived'].values, random_state=42 )
%%time
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
if NUM_WORDS is None:
MAX_TEXT=len(word_index)+1
else:
MAX_TEXT=NUM_WORDS+1
MAX_CATEGORY_MAIN = np.max([full_df.main_category.max() ])+1
MAX_CATEGORY_SUB1 = np.max([full_df.sub_category1.max() ])+1
MAX_CATEGORY_SUB2 = np.max([full_df.sub_category2.max() ])+1
MAX_BRAND = np.max([full_df.brand_name.max() ])+1
MAX_CONDITION = np.max([full_df.item_condition_id.max() ])+1
MAX_DESC_LEN = np.max([full_df.desc_len.max() ])+1
MAX_NAME_LEN = np.max([full_df.name_len.max() ])+1<categorify>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )
%%time
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(dataset.seq_desc, maxlen=MAX_ITEM_DESC_SEQ),
'brand_name': np.array(dataset[["brand_name"]]),
'item_condition': to_categorical(dataset[["item_condition_id"]]),
'num_vars': np.array(dataset[["shipping"]]),
'desc_len': np.array(dataset[["desc_len"]]),
'name_len': np.array(dataset[["name_len"]]),
'main_category':np.array(dataset[["main_category"]]),
'sub_category1': np.array(dataset[["sub_category1"]]),
'sub_category2':np.array(dataset[["sub_category2"]]),
'category_name':np.array(dataset[["category_name"]]),
'item_condition2':np.array(dataset[["item_condition_id"]]),
}
return X
X_train = get_keras_data(dtrain)
 y_train=np.log1p(dtrain.price)
 X_valid = get_keras_data(dvalid)
 y_valid = np.log1p(dvalid.price)
 X_test = get_keras_data(test)
 test_id=test['test_id'].astype('int32')
 del dtrain
del dvalid
del test
del train
del full_df<define_variables>
clf = SVC(C=1.0, kernel='rbf', degree=3, gamma='auto', probability=True)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score)
 print('The total score is: '+str(score.mean()))
%%time
def get_cnn_model(lr=0.001,decay=0):
dr_r=0
name=Input(shape=[X_train['name'].shape[1]],name='name')
 category_name=Input(shape=[1],name='category_name')
 desc=Input(shape=[X_train['item_desc'].shape[1]],name='item_desc')
 brand=Input(shape=[1],name='brand_name')
 item_condition=Input(shape=[X_train['item_condition'].shape[1]],name='item_condition')
 shipping=Input(shape=[X_train['num_vars'].shape[1]],name='num_vars')
 desc_len=Input(shape=[1],name='desc_len')
 name_len=Input(shape=[1],name='name_len')
 main_category=Input(shape=[1],name='main_category')
 sub_category1=Input(shape=[1],name='sub_category1')
 sub_category2=Input(shape=[1],name='sub_category2')
 emb_name=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(name)
 emb_desc=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(desc)
 emb_brand=Embedding(MAX_BRAND,10,embeddings_initializer='glorot_uniform' )(brand)
 emb_desc_len=Embedding(MAX_DESC_LEN,5,embeddings_initializer='glorot_uniform' )(desc_len)
 emb_name_len=Embedding(MAX_NAME_LEN,5,embeddings_initializer='glorot_uniform' )(name_len)
 emb_main_category=Embedding(MAX_CATEGORY_MAIN,10,embeddings_initializer='glorot_uniform' )(main_category)
 emb_sub_category1=Embedding(MAX_CATEGORY_SUB1,10,embeddings_initializer='glorot_uniform' )(sub_category1)
 emb_sub_category2=Embedding(MAX_CATEGORY_SUB2,10,embeddings_initializer='glorot_uniform' )(sub_category2)
 emb_item_condition =Embedding(MAX_CONDITION, 5,embeddings_initializer='glorot_uniform' )(item_condition)
 conv_name=Conv1D(filters=32,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_name)
 max_pooling_name=GlobalMaxPooling1D()(conv_name)
 conv_desc=Conv1D(filters=64,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_desc)
 max_pooling_desc=GlobalMaxPooling1D()(conv_desc)
 main_l=concatenate([
max_pooling_name,
max_pooling_desc,
Flatten()(emb_main_category),
Flatten()(emb_sub_category1),
Flatten()(emb_sub_category2),
Flatten()(emb_name_len),
Flatten()(emb_desc_len),
Flatten()(emb_brand),
item_condition,
shipping,
])
 main_l=Dropout(dr_r )(Dense(256,activation='relu',kernel_initializer='he_uniform' )(main_l))
main_l=Dropout(dr_r )(Dense(128,activation='relu',kernel_initializer='he_uniform' )(main_l))
output=Dense(1,kernel_initializer='he_uniform' )(main_l)
 cnn_model=Model(inputs=[name,desc,brand,item_condition,shipping,desc_len,name_len,category_name,main_category,sub_category1,sub_category2],outputs=output)
 optimizer=Adam(lr=lr,decay=decay)
 cnn_model.compile(loss="mse", optimizer=optimizer)
 return cnn_model
def new_rnn_model(lr=0.001, decay=0.0):
dr_r=0
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 item_condition = Input(shape=[1], name="item_condition2")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 desc_len = Input(shape=[1], name="desc_len")
 name_len = Input(shape=[1], name="name_len")
 subcat_0 = Input(shape=[1], name="main_category")
 subcat_1 = Input(shape=[1], name="sub_category1")
 subcat_2 = Input(shape=[1], name="sub_category2")
 brand_enc=Input(shape=[1],name='brand_name_enc')
 item_condition_enc=Input(shape=[1],name='item_condition_enc')
 shipping_enc=Input(shape=[1],name='shipping_enc')
 main_category_enc=Input(shape=[1],name='main_category_enc')
 sub_category1_enc=Input(shape=[1],name='sub_category1_enc')
 sub_category2_enc=Input(shape=[1],name='sub_category2_enc')
 emb_name = Embedding(MAX_TEXT, 50,name='emb_name',embeddings_initializer='glorot_uniform' )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50,name='emb_item_desc',embeddings_initializer='glorot_uniform' )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10,name='emb_brand_name',embeddings_initializer='glorot_uniform' )(brand_name)
 emb_item_condition = Embedding(MAX_CONDITION,5,name='emb_item_condition',embeddings_initializer='glorot_uniform' )(item_condition)
 emb_desc_len = Embedding(MAX_DESC_LEN, 5,name='emb_desc_len',embeddings_initializer='glorot_uniform' )(desc_len)
 emb_name_len = Embedding(MAX_NAME_LEN, 5,name='emb_name_len',embeddings_initializer='glorot_uniform' )(name_len)
 emb_subcat_0 = Embedding(MAX_CATEGORY_MAIN, 10,name='emb_sub_cat_0',embeddings_initializer='glorot_uniform' )(subcat_0)
 emb_subcat_1 = Embedding(MAX_CATEGORY_SUB1, 10,name='emb_sub_cat_1',embeddings_initializer='glorot_uniform' )(subcat_1)
 emb_subcat_2 = Embedding(MAX_CATEGORY_SUB2, 10,name='emb_sub_cat_2',embeddings_initializer='glorot_uniform' )(subcat_2)
 rnn_layer1 = GRU(16,kernel_initializer='glorot_uniform' )(emb_item_desc)
 rnn_layer2 = GRU(8,kernel_initializer='glorot_uniform' )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_item_condition)
 , Flatten()(emb_desc_len)
 , Flatten()(emb_name_len)
 , Flatten()(emb_subcat_0)
 , Flatten()(emb_subcat_1)
 , Flatten()(emb_subcat_2)
 , rnn_layer1
, rnn_layer2
, num_vars,
])
 main_l = Dropout(dr_r )(Dense(512,kernel_initializer='he_uniform',activation='relu' )(main_l))
main_l = Dropout(dr_r )(Dense(256,kernel_initializer='he_uniform',activation='relu' )(main_l))
main_l = Dropout(dr_r )(Dense(128,kernel_initializer='he_uniform',activation='relu' )(main_l))
main_l = Dropout(dr_r )(Dense(64,kernel_initializer='he_uniform',activation='relu' )(main_l))
output = Dense(1,activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name , item_condition,
num_vars, desc_len, name_len, subcat_0, subcat_1,
subcat_2], output)
 optimizer = Adam(lr=lr, decay=decay)
 model.compile(loss = 'mse', optimizer = optimizer)
 return model
def get_cnn_model2(lr=0.001,decay=0):
dr_r=0
name=Input(shape=[X_train['name'].shape[1]],name='name')
 category_name=Input(shape=[1],name='category_name')
 desc=Input(shape=[X_train['item_desc'].shape[1]],name='item_desc')
 brand=Input(shape=[1],name='brand_name')
 item_condition=Input(shape=[X_train['item_condition'].shape[1]],name='item_condition')
 shipping=Input(shape=[X_train['num_vars'].shape[1]],name='num_vars')
 desc_len=Input(shape=[1],name='desc_len')
 name_len=Input(shape=[1],name='name_len')
 main_category=Input(shape=[1],name='main_category')
 sub_category1=Input(shape=[1],name='sub_category1')
 sub_category2=Input(shape=[1],name='sub_category2')
 emb_name=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(name)
 emb_desc=Embedding(MAX_TEXT,50,embeddings_initializer='glorot_uniform' )(desc)
 emb_brand=Embedding(MAX_BRAND,10,embeddings_initializer='glorot_uniform' )(brand)
 emb_desc_len=BatchNormalization()(desc_len)
 emb_name_len=BatchNormalization()(name_len)
 emb_main_category=Embedding(MAX_CATEGORY_MAIN,10,embeddings_initializer='glorot_uniform' )(main_category)
 emb_sub_category1=Embedding(MAX_CATEGORY_SUB1,10,embeddings_initializer='glorot_uniform' )(sub_category1)
 emb_sub_category2=Embedding(MAX_CATEGORY_SUB2,10,embeddings_initializer='glorot_uniform' )(sub_category2)
 emb_item_condition = Embedding(MAX_CONDITION, 5,embeddings_initializer='glorot_uniform' )(item_condition)
 conv_name=Conv1D(filters=32,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_name)
 max_pooling_name=GlobalMaxPooling1D()(conv_name)
 conv_desc=Conv1D(filters=64,kernel_size=3,activation=None,kernel_initializer='glorot_uniform' )(emb_desc)
 max_pooling_desc=GlobalMaxPooling1D()(conv_desc)
 main_l=concatenate([
max_pooling_name,
max_pooling_desc,
Flatten()(emb_main_category),
Flatten()(emb_sub_category1),
Flatten()(emb_sub_category2),
emb_name_len,
emb_desc_len,
Flatten()(emb_brand),
item_condition,
shipping
])
 main_l=Dropout(dr_r )(Dense(256,activation='relu',kernel_initializer='he_uniform' )(main_l))
main_l=Dropout(dr_r )(Dense(128,activation='relu',kernel_initializer='he_uniform' )(main_l))
output=Dense(1,kernel_initializer='he_normal' )(main_l)
 cnn_model=Model(inputs=[name,desc,brand,item_condition,shipping,desc_len,name_len,main_category,sub_category1,sub_category2,category_name],outputs=output)
 optimizer=Adam(lr=lr,decay=decay)
 cnn_model.compile(loss="mse", optimizer=optimizer)
 return cnn_model 
<train_model>
clf.fit(train_data, target )
%%time
BATCH_SIZE = 512*3
epochs = 2
np.random.seed(123)
 exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
lr_init, lr_fin = 0.005, 0.0001
lr_decay = exp_decay(lr_init, lr_fin, steps)
 model= get_cnn_model(lr_init,lr_decay)
 model.fit(X_train,y_train, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid,y_valid)
 , verbose=1 )<train_model>
predictions = clf.predict(X_test )
%%time
np.random.seed(123)
 BATCH_SIZE = 512*3
epochs = 2
lr_init, lr_fin = 0.004, 0.001
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
lr_decay = exp_decay(lr_init, lr_fin, steps)
 model2= new_rnn_model(lr_init,lr_decay)
 model2.fit(X_train,y_train, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid,y_valid)
 , verbose=1 )<train_model>
y_score = clf.predict_proba(X_test )
%%time
np.random.seed(123)
 BATCH_SIZE = 512*3
epochs = 2
lr_init, lr_fin = 0.005, 0.001
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps-1)) - 1
lr_decay = exp_decay(lr_init, lr_fin, steps)
 model3= get_cnn_model2(lr_init,lr_decay)
 model3.fit(X_train,y_train, epochs=epochs, batch_size=BATCH_SIZE
, validation_data=(X_valid,y_valid)
 , verbose=1 )<compute_train_metric>
average_precision = average_precision_score(y_test, y_score[:, 0])
 average_precision
%%time
cnn_predict=model.predict(X_valid,batch_size=BATCH_SIZE)
 rnn_predict=model2.predict(X_valid,batch_size=BATCH_SIZE)
 cnn_predict2=model3.predict(X_valid,batch_size=BATCH_SIZE)
 X_meta=np.hstack([cnn_predict,rnn_predict,cnn_predict2])
 lm=LinearRegression()
lm.fit(X_meta,y_valid)
 new_pred=lm.predict(X_meta)
 rmsle=np.sqrt(mean_squared_error(y_valid,new_pred))
print('loss:'+str(rmsle))<save_to_csv>
import scikitplot as skplt
%%time
predictions=lm.coef_[0]*model.predict(X_test,batch_size=BATCH_SIZE)+lm.coef_[1]*model2.predict(X_test,batch_size=BATCH_SIZE)+lm.coef_[2]*model3.predict(X_test,batch_size=BATCH_SIZE)+lm.intercept_
predictions=np.exp(predictions)-1
submission=pd.DataFrame(columns=['test_id','price'])
 submission['test_id']=test_id
submission['price']=predictions
submission.to_csv('EnsembleSubmission.csv',index=False )<train_model>
test_prediction = clf.predict(test )
print('program ended took '+str(datetime.datetime.now() -time_start)+' minutes' )<load_pretrained>
prediction = clf.predict(test )
<load_pretrained><EOS> 
submission = pd.DataFrame({
"PassengerId": test_key,
"Survived": prediction
})
 submission.to_csv('submission.csv', index=False )