Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_x_and_y>
print("pandas version: {}".format(pd.__version__))
print("NumPy version: {}".format(np.__version__))
print("matplotlib version: {}".format(matplotlib.__version__))
print("seaborn version: {}".format(sns.__version__))
print("scikit-learn version: {}".format(sklearn.__version__))
print("statsmodels version: {}".format(statsmodels.__version__))
print("SciPy version: {}".format(sp.__version__))
X_train = df_work[df_work.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = df_work[df_work.date_block_num < 33]['item_cnt_month']
X_valid = df_work[df_work.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_valid = df_work[df_work.date_block_num == 33]['item_cnt_month']
X_test = df_work[df_work.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<drop_column>
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import seaborn as sns
del df_work<init_hyperparams>
data = pd.read_csv(".. /input/titanic/train.csv")
 data_val = pd.read_csv(".. /input/titanic/test.csv")
 data1 = data.copy(deep = True)
 data_cleaner = [data1, data_val]
%%time
feature_names = X_train.columns.tolist()
params = {
'objective': 'mse',
'metric': 'rmse',
'num_leaves': 255,
'learning_rate': 0.005,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'seed': 1,
'verbose': 1,
'force_row_wise' : True
}
categorical_feature_names = [
'item_category_id',
'hl_cat_id',
'city_enc',
'month',
'shop_group'
]
lgb_train = lgb.Dataset(X_train[feature_names], y_train, categorical_feature=None)
 lgb_eval = lgb.Dataset(X_valid[feature_names], y_valid, categorical_feature=None, reference=lgb_train)
 evals_result = {}
gbm = lgb.train(
 params,
lgb_train,
num_boost_round = 4000,
valid_sets =(lgb_train, lgb_eval),
feature_name = feature_names,
categorical_feature = categorical_feature_names,
verbose_eval = 100,
evals_result = evals_result,
early_stopping_rounds = 100 )<save_to_csv>
print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )
sample_submission['item_cnt_month'] = gbm.predict(X_test[feature_names] ).clip(0, 20)
 sample_submission.to_csv('submission_k_l3_1.csv' )<set_options>
for dataset in data_cleaner:
dataset['Age'].fillna(dataset['Age'].median() , inplace = True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace = True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 dataset.drop('Cabin', axis=1, inplace=True )
le = LabelEncoder()
pd.set_option('display.max_rows', 400)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 40)
 warnings.filterwarnings("ignore" )<load_from_csv>
print(data1.isnull().sum())
 print("-"*10)
 print(data_val.isnull().sum() )
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 test.head()<load_from_csv>
varlist = ['Sex']
def binary_map(x):
return x.map({'male': 1, "female": 0})
 for dataset in data_cleaner:
dataset[varlist] = dataset[varlist].apply(binary_map )
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 pd.DataFrame(categories.category_name.values.reshape(-1, 4))<categorify>
dummy1 = pd.get_dummies(data1['Embarked'], prefix='Embarked', drop_first=True)
 data1 = pd.concat([data1, dummy1], axis=1 )
categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
categories['group_id'] = le.fit_transform(categories.group_name.values)
 categories.sample(5 )<load_from_csv>
dummy1 = pd.get_dummies(data_val['Embarked'], prefix='Embarked', drop_first=True)
 data_val = pd.concat([data_val, dummy1], axis=1 )
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
items['item_name'] = items['item_name'].str.replace('.', '')
 for i in [r'[^\w\d\s\.]', r'\bthe\b', r'\bin\b', r'\bis\b',
r'\bfor\b', r'\bof\b', r'\bon\b', r'\band\b',
r'\bto\b', r'\bwith\b' , r'\byo\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(r'\b.\b', ' ')
 items['item_name_no_space'] = items['item_name'].str.replace(' ', '')
 items['item_name_first4'] = [x[:4] for x in items['item_name_no_space']]
items['item_name_first6'] = [x[:6] for x in items['item_name_no_space']]
items['item_name_first11'] = [x[:11] for x in items['item_name_no_space']]
del items['item_name_no_space']
items.item_name_first4 = le.fit_transform(items.item_name_first4.values)
 items.item_name_first6 = le.fit_transform(items.item_name_first6.values)
 items.item_name_first11 = le.fit_transform(items.item_name_first11.values)
 items = items.join(categories.set_index('category_id'), on='category_id')
 items.sample(10 )<remove_duplicates>
dummy1 = pd.get_dummies(data1['Pclass'], prefix='Pclass', drop_first=True)
 data1 = pd.concat([data1, dummy1], axis=1 )
dupes = items[(items.duplicated(subset=['item_name','category_id'],keep=False)) ]
dupes['in_test'] = dupes.item_id.isin(test.item_id.unique())
 dupes = dupes.groupby('item_name' ).agg({'item_id':['first','last'],'in_test':['first','last']})
 dupes = dupes[(dupes[('in_test', 'first')]==False)|(dupes[('in_test', 'last')]==False)]
temp = dupes[dupes[('in_test', 'first')]==True]
keep_first = dict(zip(temp[('item_id', 'last')], temp[('item_id', 'first')]))
temp = dupes[dupes[('in_test', 'first')]==False]
keep_second = dict(zip(temp[('item_id', 'first')], temp[('item_id', 'last')]))
item_map = {**keep_first, **keep_second}<feature_engineering>
dummy1 = pd.get_dummies(data_val['Pclass'], prefix='Pclass', drop_first=True)
 data_val = pd.concat([data_val, dummy1], axis=1 )
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 sales =(sales 
.query('0 < item_price < 50000 and 0 < item_cnt_day < 1001')
.replace({
'shop_id':{0:57, 1:58, 11:10},
'item_id':item_map
})
)
 sales = sales[sales['shop_id'].isin(test.shop_id.unique())]
sales['date'] = pd.to_datetime(sales.date,format='%d.%m.%Y')
 sales['weekday'] = sales.date.dt.dayofweek
sales['first_sale_day'] = sales.date.dt.dayofyear
sales['first_sale_day'] += 365 *(sales.date.dt.year-2013)
 sales['first_sale_day'] = sales.groupby('item_id')['first_sale_day'].transform('min' ).astype('int16')
 sales['revenue'] = sales['item_cnt_day']*sales['item_price']<feature_engineering>
dummy1 = pd.get_dummies(data1['Sex'], prefix='Male', drop_first=True)
 data1 = pd.concat([data1, dummy1], axis=1 )
temp = sales.groupby(['shop_id','weekday'] ).agg({'item_cnt_day':'sum'} ).reset_index()
temp = pd.merge(temp, sales.groupby(['shop_id'] ).agg({'item_cnt_day':'sum'} ).reset_index() , on='shop_id', how='left')
 temp.columns = ['shop_id','weekday', 'shop_day_sales', 'shop_total_sales']
temp['day_quality'] = temp['shop_day_sales']/temp['shop_total_sales']
temp = temp[['shop_id','weekday','day_quality']]
dates = pd.DataFrame(data={'date':pd.date_range(start='2013-01-01',end='2015-11-30')})
 dates['weekday'] = dates.date.dt.dayofweek
dates['month'] = dates.date.dt.month
dates['year'] = dates.date.dt.year - 2013
dates['date_block_num'] = dates['year']*12 + dates['month'] - 1
dates['first_day_of_month'] = dates.date.dt.dayofyear
dates['first_day_of_month'] += 365 * dates['year']
dates = dates.join(temp.set_index('weekday'), on='weekday')
 dates = dates.groupby(['date_block_num','shop_id','month','year'] ).agg({'day_quality':'sum','first_day_of_month':'min'} ).reset_index()
dates.query('shop_id == 28' ).head(15 )<groupby>
dummy1 = pd.get_dummies(data_val['Sex'], prefix='Male', drop_first=True)
 data_val = pd.concat([data_val, dummy1], axis=1 )
sales =(sales 
.groupby(['date_block_num', 'shop_id', 'item_id'])
.agg({
'item_cnt_day':'sum',
'revenue':'sum',
'first_sale_day':'first'
})
.reset_index() 
.rename(columns={'item_cnt_day':'item_cnt'})
)
 sales.sample(5 )<drop_column>
data1['FamilySize'] = data1['SibSp'] + data1['Parch'] + 1
data1.head(2 )
test['date_block_num'] = 34
del test['ID']<concatenate>
data_val['FamilySize'] = data_val['SibSp'] + data_val['Parch'] + 1
data_val.head(2 )
df = pd.concat([df,test] ).fillna(0)
 df = df.reset_index()
del df['index']<merge>
PassengerId = data_val.PassengerId
df = pd.merge(df, sales, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 df = pd.merge(df, dates, on=['date_block_num','shop_id'], how='left')
 df = pd.merge(df, items.drop(columns=['item_name','group_name','category_name']), on='item_id', how='left' )<feature_engineering>
data1= data1.rename(columns={ 'Male_1' : 'Male'})
 data_val= data_val.rename(columns={ 'Male_1' : 'Male'} )
shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops_cats = pd.DataFrame(
 np.array(list(product(*[df['shop_id'].unique() , df['category_id'].unique() ]))),
columns =['shop_id', 'category_id'] 
)
 temp = df.groupby(['category_id', 'shop_id'] ).agg({'item_cnt':'sum'} ).reset_index()
temp2 = temp.groupby('shop_id' ).agg({'item_cnt':'sum'} ).rename(columns={'item_cnt':'shop_total'})
 temp = temp.join(temp2, on='shop_id')
 temp['category_proportion'] = temp['item_cnt']/temp['shop_total']
temp = temp[['shop_id', 'category_id', 'category_proportion']]
shops_cats = pd.merge(shops_cats, temp, on=['shop_id','category_id'], how='left')
 shops_cats = shops_cats.fillna(0)
 shops_cats = shops_cats.pivot(index='shop_id', columns=['category_id'])
 kmeans = KMeans(n_clusters=7, random_state=0 ).fit(shops_cats)
 shops_cats['shop_cluster'] = kmeans.labels_.astype('int8')
 shops = shops.join(shops_cats['shop_cluster'], on='shop_id' )<feature_engineering>
drop_column = ['PassengerId', 'Pclass', 'Name', 'Sex', 'Ticket', 'Fare', 'Embarked']
data1.drop(drop_column, axis=1, inplace = True )
shops.dropna(inplace=True)
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_type'] = 'regular'
shops.loc[shops['shop_name'].str.contains(r'tc'), 'shop_type'] = 'tc'
shops.loc[shops['shop_name'].str.contains(r'mall|center|mega'), 'shop_type'] = 'mall'
shops.loc[shops['shop_id'].isin([9,20]), 'shop_type'] = 'special'
shops.loc[shops['shop_id'].isin([12,55]), 'shop_type'] = 'online'
shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
shops.shop_city = le.fit_transform(shops.shop_city.values)
 shops.shop_type = le.fit_transform(shops.shop_type.values)
 shops.head()<merge>
target = 'Survived'
y = data1[target]
x = data1.drop(columns = target)
 x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state = 42)
 y = data1["Survived"]
features = ["Age", "Male","Pclass_2", "Pclass_3","FamilySize"]
X = pd.get_dummies(data1[features])
 X_test = pd.get_dummies(x_test[features] )
df = pd.merge(df, shops.drop(columns='shop_name'), on='shop_id', how='left')
 df.head()<feature_engineering>
model = RandomForestClassifier(n_estimators=100, max_depth=3, random_state=1)
 model.fit(X, y)
 predictions = model.predict(X_test)
 score = accuracy_score(y_test, predictions)
 print("Score: ",score )
df['first_sale_day'] = df.groupby('item_id')['first_sale_day'].transform('max' ).astype('int16')
 df.loc[df['first_sale_day']==0, 'first_sale_day'] = 1035
df['prev_days_on_sale'] = [max(idx)for idx in zip(df['first_day_of_month']-df['first_sale_day'],[0]*len(df)) ]
del df['first_day_of_month']<drop_column>
X_val = pd.get_dummies(data_val[features])
 predictions_test = model.predict(X_val)
 output = pd.DataFrame({'PassengerId': PassengerId, 'Survived': predictions_test})
 output.to_csv('my_submission_RFC.csv', index=False)
 print("Your submission was successfully saved!" )