Kaggle competition: Titanic - Machine Learning from Disaster
data = add_comparison_stats(data, cols=['rating', 'popularity_mean'])
<data_type_conversions>
plt.style.use('seaborn')
 sns.set(font_scale=2.5)
 warnings.filterwarnings('ignore')
 %matplotlib inline
data.drop(['belongs_to_collection', 'popularity', 'popularity2', 'popularity3'], axis=1, inplace=True)
 data['collection_id'] = data['collection_id'].astype('object' )<drop_column>
df_train=pd.read_csv('.. /input/titanic/train.csv')
 df_test=pd.read_csv('.. /input/titanic/test.csv' )
data.drop(['homepage', 'id', 'imdb_id', 'poster_path', 'release_date', 'budget_cat'], axis=1, inplace=True )<set_options>
for col in df_train.columns:
msg='columns: {:>10}\t Percent of NaN value: {: 2f}%'.format(col,100*(df_train[col].isnull().sum() /df_train[col].shape[0]))
print(msg )
%%HTML
<iframe width="560" height="315" src="https://www.youtube.com/embed/1cRGpDXTJC8?rel=0&amp;controls=1&amp;showinfo=0" frameborder="0" allowfullscreen></iframe><define_variables>
for col in df_test.columns:
msg='columns: {:>10}\t Percent of NaN value: {: 2f}%'.format(col,100*(df_test[col].isnull().sum() /df_test[col].shape[0]))
print(msg )
categorical_columns = []
numerical_columns = []
for col, col_type in zip(data.columns, data.dtypes):
if col == 'dataset':
continue
if col_type==np.dtype('O')or col in final_dummy_cols:
categorical_columns.append(col)
 else:
numerical_columns.append(col )<drop_column>
df_train[['Pclass','Survived']].groupby(['Pclass'],as_index=True ).count()
flag_cols = ['has_homepage',
'isbelongs_to_collectionNA',
'isTaglineNA',
'isOriginalLanguageEng',
'isTitleDifferent',
'isMovieReleased']
numerical_columns = [col for col in numerical_columns if col not in flag_cols]
categorical_columns.extend(flag_cols)
 numerical_columns.remove('revenue' )<feature_engineering>
data_ffm = deepcopy(data)
 for col in numerical_columns:
if col != 'release_year':
data_ffm[col] = pd.qcut(data_ffm[col], q=200, duplicates='drop', labels=False ).factorize() [0] + 1
for col in categorical_columns:
data_ffm[col] = data_ffm[col].factorize() [0] + 1
train_logical_indexing = data_ffm['dataset']=='Train'
test_logical_indexing = data_ffm['dataset']=='Test'
for col in [num_col for num_col in numerical_columns if num_col != 'release_year'] + [cat_col for cat_col in categorical_columns if cat_col != 'dataset']:
train_values = set(data_ffm.loc[train_logical_indexing, col].values)
 test_values = set(data_ffm.loc[test_logical_indexing, col].values)
 values_to_preserve = train_values.intersection(test_values)
 data_ffm.loc[~data_ffm[col].isin(values_to_preserve), col] = 0<feature_engineering>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 cv_folds = [[trn_idx, val_idx] for trn_idx, val_idx in skf.split(list(range(( data_ffm['dataset'] == 'Train' ).sum())) ,
data_ffm.loc[data_ffm['dataset'] == 'Train', 'release_year'])]
data_ffm['release_year'] = data_ffm['release_year'].factorize() [0]
train_ffm = data_ffm[data_ffm['dataset'] == 'Train'].drop('dataset', axis=1)
 test_ffm = data_ffm[data_ffm['dataset'] == 'Test'].drop('dataset', axis=1 )<count_unique_values>
df_train['FamilySize']=df_train['SibSp']+df_train['Parch']+1
df_test['FamilySize'] = df_test['SibSp'] + df_test['Parch'] + 1
accumulated_count = 0
dict_feature = dict()
for col in numerical_columns + categorical_columns:
for val in data_ffm[col].unique() :
dict_feature[(col, val)] = accumulated_count + val
accumulated_count += data_ffm[col].nunique()
def produce_file(file_name, df_to_process, test=False):
noofrows = df_to_process.shape[0]
noofcolumns = df_to_process.shape[1]
currentcode = len(numerical_columns)
 with open(file_name, 'w')as text_file:
for n, r in enumerate(tqdm(range(noofrows), desc=file_name)) :
datastring = ''
datarow = df_to_process.iloc[r]
if not test:
datastring += str(datarow['revenue'])
 else:
datastring += '10'
for col_i, col in enumerate(numerical_columns + categorical_columns):
datastring = f'{datastring} {col_i}:{dict_feature[(col, datarow[col])]}:1'
datastring += '
'
text_file.write(datastring)
 def create_train_val_files(train_ffm, val_ffm, test_ffm=test_ffm, fold_idx=0, dict_feature=dict_feature):
produce_file(f'train_ffm_{fold_idx}', train_ffm)
 produce_file(f'val_ffm_{fold_idx}', val_ffm)
 produce_file('test_ffm', test_ffm, test=True )<load_from_csv>
df_test.loc[df_test.Fare.isnull() , 'Fare'] = df_test['Fare'].mean()
submission_ffm = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_ffm['revenue'] = 0
cv_rmsle_scores = []
for fold_idx,(trn_idx, val_idx)in enumerate(cv_folds):
print('='*40)
 print('='*16 + f' FOLD {fold_idx} ' + '='*16)
 train_ffm_trn = train_ffm.iloc[trn_idx]
train_ffm_val = train_ffm.iloc[val_idx]
create_train_val_files(train_ffm_trn, train_ffm_val, fold_idx=fold_idx)
 val_score = execute(f'libffm-regression/ffm-train -p val_ffm_{fold_idx} --auto-stop -t 1 train_ffm_{fold_idx} model_{fold_idx}', parse_val_result=True)
 print(f'Val score: {val_score}')
 cv_rmsle_scores.append(val_score)
 os.remove(f'val_ffm_{fold_idx}')
 os.remove(f'train_ffm_{fold_idx}')
 execute(f'libffm-regression/ffm-predict test_ffm model_{fold_idx} output_{fold_idx} > /dev/null')
 predictions_current = pd.read_csv(f'output_{fold_idx}',header=None)
 os.remove(f'output_{fold_idx}')
 os.remove(f'model_{fold_idx}')
 predictions_current.columns = ['revenue']
submission_ffm['revenue'] += predictions_current['revenue']/len(cv_folds)
 rmtree('libffm-regression')
 os.remove('test_ffm')
 rmsle_mean = np.mean(cv_rmsle_scores)
 rmsle_mean_std = np.std(cv_rmsle_scores, ddof=1)/np.sqrt(len(cv_rmsle_scores))
tinteval_lb, tinterval_ub = _tconfint_generic(rmsle_mean, rmsle_mean_std, dof=len(cv_rmsle_scores)-1, alpha=0.05, alternative='two-sided')
 print(f"95% confidence interval for RMSLE is:({tinteval_lb:.2f}, {tinterval_ub:.2f} ).")
 submission_ffm['revenue'] = submission_ffm['revenue'].map(np.expm1)
<set_options>
df_train['Fare']=df_train['Fare'].map(lambda i : np.log(i)if i>0 else 0)
 df_test['Fare']=df_test['Fare'].map(lambda i : np.log(i)if i>0 else 0 )
%%HTML
<a id=lgb></a><set_options>
df_train.isnull().sum()
%%HTML
<iframe width="560" height="315" src="https://www.youtube.com/embed/mg7BNAvHeII?rel=0&amp;controls=1&amp;showinfo=0" frameborder="0" allowfullscreen></iframe><prepare_x_and_y>
df_train['Initial']= df_train.Name.str.extract('([A-Za-z]+)\.')
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 train_data = data[data['dataset'] == 'Train']
y_train = train_data['revenue']
cv_folds = [[trn_idx, val_idx] for trn_idx, val_idx in skf.split(list(range(len(train_data))), train_data['release_year'])]
def hide_rare_vals(data, categorical_columns, min_count=2):
for col in categorical_columns:
data['count'] = np.arange(len(data))
value_counts = data.groupby(col, as_index=False)['count'].count()
values_to_discard = set(value_counts.loc[value_counts['count'] < min_count, col].values)
 data.loc[data[col].isin(values_to_discard), col] = np.nan
data.drop('count', axis=1, inplace=True)
 return data
def get_train_lgb(data):
categorical_columns = [col for col, col_type in zip(data.columns, data.dtypes)if col_type==np.dtype('O')and col != 'dataset']
data_for_boosting = hide_rare_vals(data,
[col for col in categorical_columns if 'word' in col],
min_count=10)
 data_one_hot_enc = pd.get_dummies(data_for_boosting.drop(['revenue'], axis=1),
columns=categorical_columns)
 train_one_hot_enc = data_one_hot_enc[data_one_hot_enc['dataset'] == 'Train'].drop(['dataset'], axis=1)
 return train_one_hot_enc, data_one_hot_enc
train_one_hot_enc, data_one_hot_enc = get_train_lgb(data )<train_model>
df_test['Initial']= df_test.Name.str.extract('([A-Za-z]+)\.' )
counter = 0
def onstep(res):
global counter
x0 = res.x_iters
y0 = res.func_vals
print('Current iter: ', counter,
' - Score ', res.fun,
' - Args: ', res.x)
 joblib.dump(( x0, y0), 'checkpoint.pkl')
 counter += 1
def make_objective(model, dimensions, df=data):
model = deepcopy(model)
 @use_named_args(dimensions)
 def objective(**params):
model.set_params(**params)
 score = 0
for trn_idx, val_idx in cv_folds:
X_fit, X_val = df.iloc[trn_idx], df.iloc[val_idx]
y_fit, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
model.fit(X_fit, y_fit,
eval_set=[(X_val, y_val)],
verbose=False, early_stopping_rounds=30)
 score += np.sqrt(mean_squared_error(y_val, model.predict(X_val)))/len(cv_folds)
 return score
return objective<choose_model_class>
df_train['Initial'].replace(['Capt','Col','Countess','Don','Dr','Jonkheer','Lady','Major','Master','Miss','Mlle','Mme','Mr','Mrs','Ms','Rev','Sir','Dona'],
['Mr','Mr','Other','Mr','Mr','Other','Miss','Mr','Master','Miss','Miss','Mrs','Mr','Mrs','Miss','Other','Other','Other'],inplace=True )
skopt_params = {'learning_rate': 0.005, 'num_leaves': 2558, 'max_depth': 0, 'min_child_samples': 15, 'max_bin': 95637,
'subsample': 0.4462883478742228, 'subsample_freq': 0, 'colsample_bytree': 0.12636976300879604, 'min_child_weight': 1, 'subsample_for_bin': 422967, 'reg_lambda': 8.991454050632966e-05, 'reg_alpha': 0.000956883899855844, 'scale_pos_weight': 0.000511057873147196}
def get_skopt_params_lgb(train_one_hot_enc):
clf = LGBMRegressor(boosting_type='gbdt',
objective='rmse',
n_jobs=-1,
verbose=0,
n_estimators=10000)
 dimensions = [Real(0.01, 1.0, 'log-uniform', name='learning_rate'),
Integer(50, 1000, name='num_leaves'),
Integer(0, 500, name='max_depth'),
Integer(0, 200, name='min_child_samples'),
Integer(100, 100000, name='max_bin'),
Real(0.01, 1.0, 'uniform', name='subsample'),
Integer(0, 10, name='subsample_freq'),
Real(0.01, 1.0, 'uniform', name='colsample_bytree'),
Integer(0, 10, name='min_child_weight'),
Integer(100000, 500000, name='subsample_for_bin'),
Real(1e-9, 1000, 'log-uniform', name='reg_lambda'),
Real(1e-9, 1.0, 'log-uniform', name='reg_alpha'),
Real(1e-6, 500, 'log-uniform', name='scale_pos_weight')]
objective = make_objective(clf, dimensions, train_one_hot_enc)
 gp_round = gp_minimize(func=objective,
dimensions=dimensions,
acq_func='gp_hedge',
n_calls=25,
callback=[onstep],
random_state=42)
 best_parameters = gp_round.x
best_result = gp_round.fun
skopt_params = {param.name: value for param, value in zip(dimensions, best_parameters)}
print(f'Best RMSLE: {best_result},
corresponding parameters: {skopt_params}')
 return skopt_params
if skopt_params is None:
skopt_params = get_skopt_params_lgb(train_one_hot_enc )<load_pretrained>
df_test['Initial'].replace(['Capt','Col','Countess','Don','Dr','Jonkheer','Lady','Major','Master','Miss','Mlle','Mme','Mr','Mrs','Ms','Rev','Sir','Dona'],
['Mr','Mr','Other','Mr','Mr','Other','Miss','Mr','Master','Miss','Miss','Mrs','Mr','Mrs','Miss','Other','Other','Other'],inplace=True )
def train_lgb(train_one_hot_enc):
clfs = []
cv_rmsle_scores = []
importances = pd.DataFrame()
for fold_idx,(trn_idx, val_idx)in enumerate(cv_folds):
clf = LGBMRegressor(boosting_type='gbdt',
objective='rmse',
num_leaves=5000,
n_jobs=-1,
verbose=1,
n_estimators=15000,
min_data=50)
 clf.set_params(**skopt_params)
 X_fit, X_val = train_one_hot_enc.iloc[trn_idx], train_one_hot_enc.iloc[val_idx]
y_fit, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
clf.fit(X_fit, y_fit, eval_metric='rmse',
eval_set=[(X_val, y_val)],
verbose=500, early_stopping_rounds=300)
 imp_df = pd.DataFrame({
'feature': X_fit.columns,
'gain': clf.feature_importances_,
'fold': [fold_idx + 1] * len(X_fit.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 clfs.append(clf)
 cv_rmsle_scores.append(np.sqrt(mean_squared_error(y_val, clf.predict(X_val))))
rmsle_mean = np.mean(cv_rmsle_scores)
 rmsle_mean_std = np.std(cv_rmsle_scores, ddof=1)/np.sqrt(len(cv_rmsle_scores))
tinteval_lb, tinterval_ub = _tconfint_generic(rmsle_mean, rmsle_mean_std, dof=len(cv_rmsle_scores)-1, alpha=0.05, alternative='two-sided')
 print(f"95% confidence interval for RMSLE is:({tinteval_lb:.2f}, {tinterval_ub:.2f} ).")
 return clfs, importances
clfs, importances = train_lgb(train_one_hot_enc )<set_options>
df_train.groupby(['Initial'] ).mean().sort_values(by='Survived',ascending=False )
%%HTML
<a id=catboost></a><drop_column>
df_all=pd.concat([df_train,df_test] )
def get_data_cat(data):
data_catboost = data.drop('revenue', axis=1)
 train_logical_idx = data_catboost['dataset'] == 'Train'
test_logical_idx = data_catboost['dataset'] == 'Test'
data_catboost.drop('dataset', axis=1, inplace=True)
 init_cat_features = {'has_homepage',
'isbelongs_to_collectionNA',
'isTaglineNA',
'isOriginalLanguageEng',
'isTitleDifferent',
'isMovieReleased'}
cat_features=[i for i,(col, col_type)in enumerate(zip(data_catboost.columns, data_catboost.dtypes))
if col_type==np.dtype('O')or col in init_cat_features]
for col in cat_features:
data_catboost.iloc[:, col] = data_catboost.iloc[:, col].map(str)
 train_cat = data_catboost[train_logical_idx]
test_cat = data_catboost[test_logical_idx]
return train_cat, test_cat, cat_features
train_data_cat, test_data_cat, cat_features = get_data_cat(data)
 skopt_params_catboost = {'learning_rate': 0.02, 'depth': 4, 'random_strength': 2.6832935037024834e-08, 'bagging_temperature': 0.650888472948853, 'border_count': 15, 'l2_leaf_reg': 22}
def get_skopt_params_cat(train_data_cat):
clf = CatBoostRegressor(loss_function='RMSE',
verbose=False,
cat_features=cat_features,
thread_count=12)
 dimensions = [Real(0.01, 1.0, 'log-uniform', name='learning_rate'),
Integer(1, 10, name='depth'),
Real(1e-9, 10, 'log-uniform', name='random_strength'),
Real(0.0, 1.0, name='bagging_temperature'),
Integer(1, 255, name='border_count'),
Integer(2, 30, name='l2_leaf_reg')]
objective = make_objective(clf, dimensions, train_data_cat)
 gp_round = gp_minimize(func=objective,
dimensions=dimensions,
acq_func='gp_hedge',
n_calls=15,
callback=[onstep],
random_state=42)
 best_parameters = gp_round.x
best_result = gp_round.fun
skopt_params_catboost = {param.name: value for param, value in zip(dimensions, best_parameters)}
print(f'Best RMSLE: {best_result},
corresponding parameters: {skopt_params_catboost}')
 return skopt_params_catboost
if skopt_params_catboost is None:
skopt_params_catboost = get_skopt_params_cat(train_data_cat )<create_dataframe>
df_all.reset_index(drop=True )
def train_catboost(train_data_cat, cat_features):
clfs_cat = []
cv_rmsle_scores = []
importances_cat = pd.DataFrame()
for fold_idx,(trn_idx, val_idx)in enumerate(cv_folds):
clf = CatBoostRegressor(loss_function='RMSE',
verbose=False,
cat_features=cat_features,
thread_count=12,
iterations=15000)
 clf.set_params(**skopt_params_catboost)
 X_fit, X_val = train_data_cat.iloc[trn_idx], train_data_cat.iloc[val_idx]
y_fit, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]
clf.fit(X_fit, y_fit,
eval_set=[(X_fit, y_fit),(X_val, y_val)],
verbose=100, early_stopping_rounds=50)
 imp_df_cat = pd.DataFrame({
'feature': X_fit.columns,
'gain': clf.feature_importances_,
'fold': [fold_idx + 1] * len(X_fit.columns),
})
 importances_cat = pd.concat([importances_cat, imp_df_cat], axis=0, sort=False)
 clfs_cat.append(clf)
 cv_rmsle_scores.append(np.sqrt(mean_squared_error(y_val, clf.predict(X_val))))
rmsle_mean = np.mean(cv_rmsle_scores)
 rmsle_mean_std = np.std(cv_rmsle_scores, ddof=1)/np.sqrt(len(cv_rmsle_scores))
tinteval_lb, tinterval_ub = _tconfint_generic(rmsle_mean, rmsle_mean_std, dof=len(cv_rmsle_scores)-1, alpha=0.05, alternative='two-sided')
 print(f"95% confidence interval for RMSLE is:({tinteval_lb:.2f}, {tinterval_ub:.2f} ).")
 return clfs_cat, importances_cat
clfs_cat, importances_cat = train_catboost(train_data_cat, cat_features )<load_from_csv>
df_all.groupby('Initial' ).mean()
test_one_hot_enc = data_one_hot_enc[data_one_hot_enc['dataset'] == 'Test'].drop(['dataset'], axis=1)
 submission_lgb = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_lgb['revenue'] = 0
for clf in clfs:
submission_lgb['revenue'] += clf.predict(test_one_hot_enc)/len(clfs)
 submission_cat = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_cat['revenue'] = 0
for clf in clfs_cat:
submission_cat['revenue'] += clf.predict(test_data_cat)/len(clfs_cat)
 cat_weight = 3
lgb_weight = 1
submission_lgb_cat = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 submission_lgb_cat['revenue'] =(cat_weight*submission_cat['revenue'] + lgb_weight*submission_lgb['revenue'])/(cat_weight + lgb_weight)
 submission_lgb_cat['revenue'] = submission_lgb_cat['revenue'].map(np.expm1)
 submission_lgb_cat.to_csv('submission_lgb_cat.csv', index=None )<set_options>
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Mr'),'Age']=33
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Mrs'),'Age']=37
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Miss'),'Age']=22
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Master'),'Age']=5
df_train.loc[(df_train.Age.isnull())&(df_train.Initial=='Mr'),'Age']=33
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Mr'),'Age']=33
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Mrs'),'Age']=37
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Miss'),'Age']=22
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Master'),'Age']=5
df_test.loc[(df_test.Age.isnull())&(df_test.Initial=='Mr'),'Age']=33
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
py.init_notebook_mode(connected=True)
<load_from_csv>
df_train.isnull().sum()
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )<define_variables>
df_train['Embarked'].isnull().sum()
for i, e in enumerate(train['belongs_to_collection'][:5]):
print(i, e )<count_values>
df_train['Embarked'].fillna('S',inplace=True )
train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()<feature_engineering>
df_train['Embarked'].value_counts()
train['collection_name'] = train['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 train['has_collection'] = train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 test['collection_name'] = test['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 train = train.drop(['belongs_to_collection'], axis=1)
 test = test.drop(['belongs_to_collection'], axis=1 )<define_variables>
def age_cat(x):
if x<10:
return 0
elif x<20:
return 1
elif x<30:
return 2
elif x<40:
return 3
elif x<50:
return 4
elif x<60:
return 5
elif x<70:
return 6
else:
return 7
df_train['Age_cat']=df_train['Age'].apply(age_cat )
for i, e in enumerate(train['genres'][:5]):
print(i, e )<count_values>
df_test['Age_cat']=df_train['Age'].apply(age_cat )
print('Number of genres in films')
 train['genres'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()<define_variables>
df_train.drop(['Age'],axis=1,inplace=True)
 df_test.drop(['Age'],axis=1,inplace=True )
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )<feature_engineering>
df_train['Initial']=df_train['Initial'].map({'Master':0,'Miss':1,'Mr':2,'Mrs':3,'Other':4} )
train['num_genres'] = train['genres'].apply(lambda x: len(x)if x != {} else 0)
 train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
for g in top_genres:
train['genre_' + g] = train['all_genres'].apply(lambda x: 1 if g in x else 0)
 test['num_genres'] = test['genres'].apply(lambda x: len(x)if x != {} else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_genres:
test['genre_' + g] = test['all_genres'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )<define_variables>
df_test['Initial']=df_test['Initial'].map({'Master':0,'Miss':1,'Mr':2,'Mrs':3,'Other':4} )
for i, e in enumerate(train['production_companies'][:5]):
print(i, e )<count_values>
df_train['Embarked']=df_train['Embarked'].map({'C':0,'Q':1,'S':2} )
print('Number of production companies in films')
 train['production_companies'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()<filter>
df_test['Embarked']=df_test['Embarked'].map({'C':0,'Q':1,'S':2} )
train[train['production_companies'].apply(lambda x: len(x)if x != {} else 0)> 11]<load_pretrained>
df_train.isnull().any()
img = Image.open(urlopen("https://image.tmdb.org/t/p/w600_and_h900_bestv2/y31QB9kn3XSudA15tV7UWQ9XLuW.jpg"))
img<define_variables>
df_train['Sex']=df_train['Sex'].map({'female':0,'male':1})
 df_test['Sex']=df_test['Sex'].map({'female':0,'male':1} )
list_of_companies = list(train['production_companies'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values )<feature_engineering>
df_train=pd.get_dummies(df_train,columns=['Initial'],prefix='Initial' )
train['num_companies'] = train['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 train['all_production_companies'] = train['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_companies = [m[0] for m in Counter([i for j in list_of_companies for i in j] ).most_common(30)]
for g in top_companies:
train['production_company_' + g] = train['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 test['num_companies'] = test['production_companies'].apply(lambda x: len(x)if x != {} else 0)
 test['all_production_companies'] = test['production_companies'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_companies:
test['production_company_' + g] = test['all_production_companies'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_companies', 'all_production_companies'], axis=1)
 test = test.drop(['production_companies', 'all_production_companies'], axis=1 )<define_variables>
df_test=pd.get_dummies(df_test,columns=['Initial'],prefix='Initial' )
for i, e in enumerate(train['production_countries'][:5]):
print(i, e )<count_values>
df_train=pd.get_dummies(df_train,columns=['Embarked'],prefix='Embarked')
 df_test=pd.get_dummies(df_test,columns=['Embarked'],prefix='Embarked' )
print('Number of production countries in films')
 train['production_countries'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()<count_values>
df_train.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin'], axis=1, inplace=True)
 df_test.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin'], axis=1, inplace=True )
list_of_countries = list(train['production_countries'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_countries for i in j] ).most_common(25 )<feature_engineering>
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import confusion_matrix
train['num_countries'] = train['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 train['all_countries'] = train['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_countries = [m[0] for m in Counter([i for j in list_of_countries for i in j] ).most_common(25)]
for g in top_countries:
train['production_country_' + g] = train['all_countries'].apply(lambda x: 1 if g in x else 0)
 test['num_countries'] = test['production_countries'].apply(lambda x: len(x)if x != {} else 0)
 test['all_countries'] = test['production_countries'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_countries:
test['production_country_' + g] = test['all_countries'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['production_countries', 'all_countries'], axis=1)
 test = test.drop(['production_countries', 'all_countries'], axis=1 )<define_variables>
train,valid=train_test_split(df_train,test_size=0.3,random_state=0,stratify=df_train['Survived'])
 train_X=train[train.columns[1:]]
train_Y=train[train.columns[:1]]
vld_X=valid[valid.columns[1:]]
vld_Y=valid[valid.columns[:1]]
X=df_train[df_train.columns[1:]]
Y=df_train['Survived']
for i, e in enumerate(train['spoken_languages'][:5]):
print(i, e )<count_values>
model=svm.SVC(kernel='rbf',C=1,gamma=0.1)
 model.fit(train_X,train_Y)
 prediction1=model.predict(vld_X)
 print('Accuracy for rbf SVM is ',metrics.accuracy_score(prediction1,vld_Y))
print('Number of spoken languages in films')
 train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()<count_values>
model=svm.SVC(kernel='linear',C=0.1,gamma=0.1)
 model.fit(train_X,train_Y)
 prediction2=model.predict(vld_X)
 print('Accuracy for linear SVM is',metrics.accuracy_score(prediction2,vld_Y))
list_of_languages = list(train['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_languages for i in j] ).most_common(15 )<feature_engineering>
model = LogisticRegression()
model.fit(train_X,train_Y)
 prediction3=model.predict(vld_X)
 print('The accuracy of the Logistic Regression is',metrics.accuracy_score(prediction3,vld_Y))
train['num_languages'] = train['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 train['all_languages'] = train['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_languages = [m[0] for m in Counter([i for j in list_of_languages for i in j] ).most_common(30)]
for g in top_languages:
train['language_' + g] = train['all_languages'].apply(lambda x: 1 if g in x else 0)
 test['num_languages'] = test['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 test['all_languages'] = test['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_languages:
test['language_' + g] = test['all_languages'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['spoken_languages', 'all_languages'], axis=1)
 test = test.drop(['spoken_languages', 'all_languages'], axis=1 )<define_variables>
model=DecisionTreeClassifier()
model.fit(train_X,train_Y)
 prediction4=model.predict(vld_X)
 print('The accuract of Decision Tree is' ,metrics.accuracy_score(prediction4,vld_Y))
for i, e in enumerate(train['Keywords'][:5]):
print(i, e )<count_values>
model=KNeighborsClassifier()
model.fit(train_X,train_Y)
 prediction5=model.predict(vld_X)
 print('The accuracy of KNN is ',metrics.accuracy_score(prediction5,vld_Y))
print('Number of Keywords in films')
 train['Keywords'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )<string_transform>
model=GaussianNB()
model.fit(train_X,train_Y)
 prediction6=model.predict(vld_X)
 print('The accuracy of the GaussianNB is', metrics.accuracy_score(prediction6,vld_Y))
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 plt.figure(figsize =(16, 12))
text = ' '.join(['_'.join(i.split(' ')) for j in list_of_keywords for i in j])
 wordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,
width=1200, height=1000 ).generate(text)
 plt.imshow(wordcloud)
 plt.title('Top keywords')
 plt.axis("off")
 plt.show()<feature_engineering>
model=RandomForestClassifier(n_estimators=100)
 model.fit(train_X,train_Y)
 prediction7=model.predict(vld_X)
 print('The accuracy of the Random Forests is',metrics.accuracy_score(prediction7,vld_Y))
train['num_Keywords'] = train['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_' + g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 for g in top_keywords:
test['keyword_' + g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 train = train.drop(['Keywords', 'all_Keywords'], axis=1)
 test = test.drop(['Keywords', 'all_Keywords'], axis=1 )<define_variables>
from pandas import Series
for i, e in enumerate(train['cast'][:1]):
print(i, e )<count_values>
kfold = KFold(n_splits=10, random_state=22)
 xyz=[]
accuracy=[]
std=[]
classifiers=['Linear Svm','Radial Svm','Logistic Regression','KNN','Decision Tree','Naive Bayes','Random Forest']
models=[svm.SVC(kernel='linear'),svm.SVC(kernel='rbf'),LogisticRegression() ,KNeighborsClassifier(n_neighbors=9),DecisionTreeClassifier() ,GaussianNB() ,RandomForestClassifier(n_estimators=100)]
for i in models:
model = i
cv_result = cross_val_score(model,X,Y, cv = kfold,scoring = "accuracy")
 cv_result=cv_result
xyz.append(cv_result.mean())
 std.append(cv_result.std())
 accuracy.append(cv_result)
 new_models_dataframe2=pd.DataFrame({'CV Mean':xyz,'Std':std},index=classifiers)
 new_models_dataframe2
print('Number of casted persons in films')
 train['cast'].apply(lambda x: len(x)if x != {} else 0 ).value_counts().head(10 )<count_values>
model=svm.SVC(kernel='rbf',C=1,gamma=0.1)
 model.fit(train_X,train_Y )
list_of_cast_names = list(train['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_names for i in j] ).most_common(15 )<count_unique_values>
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission.head()
list_of_cast_genders = list(train['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_genders for i in j] ).most_common()<count_values>
prediction_final=model.predict(df_test )
list_of_cast_characters = list(train['cast'].apply(lambda x: [i['character'] for i in x] if x != {} else [] ).values)
 Counter([i for j in list_of_cast_characters for i in j] ).most_common(15 )<feature_engineering>
submission['Survived']=prediction_final
train['num_cast'] = train['cast'].apply(lambda x: len(x)if x != {} else 0)
 top_cast_names = [m[0] for m in Counter([i for j in list_of_cast_names for i in j] ).most_common(15)]
for g in top_cast_names:
train['cast_name_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train['genders_0_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['genders_1_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['genders_2_cast'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
top_cast_characters = [m[0] for m in Counter([i for j in list_of_cast_characters for i in j] ).most_common(15)]
for g in top_cast_characters:
train['cast_character_' + g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['num_cast'] = test['cast'].apply(lambda x: len(x)if x != {} else 0)
 for g in top_cast_names:
test['cast_name_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 test['genders_0_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['genders_1_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['genders_2_cast'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
for g in top_cast_characters:
test['cast_character_' + g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0)
 train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )<count_values>
submission.to_csv('./myfirstsubmission.csv',index=False )