Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<drop_column>
big_and_beautiful_data=big_and_beautiful_data.drop(drop_columns,axis=1)
 train_mean_agg=big_and_beautiful_data.loc[big_and_beautiful_data['dataset']=='train']
test_mean_agg=big_and_beautiful_data.loc[big_and_beautiful_data['dataset']=='test']
train_mean_agg=train_mean_agg.drop('dataset',axis=1)
 test_mean_agg=test_mean_agg.drop(['dataset','revenue'],axis=1)
 train_new=train_new.drop('dataset',axis=1)
 test_new=test_new.drop('dataset',axis=1)
 print("updated train dataset:", train_new.shape,"
","updated test dataset: ",test_new.shape)
 print("updated train agg dataset:", train_mean_agg.shape,"
","updated test agg dataset: ",test_mean_agg.shape)
 print(train_new.columns.difference(test_new.columns))
print(train_mean_agg.columns.difference(test_mean_agg.columns))<prepare_x_and_y>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler, StandardScaler, LabelBinarizer, LabelEncoder
from sklearn.feature_selection import RFECV
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, roc_curve, auc
%%time
X = train_new.drop(['id', 'revenue','release_date'], axis=1)
 y = np.log1p(train_new['revenue'])
 X_test = test_new.drop(['id','release_date'], axis=1)
 dtrain = xgb.DMatrix(X, label=y)
 dtest = xgb.DMatrix(X_test)
 def xgb_evaluate(max_depth, subsample, eta, min_child_weight,colsample_bytree):
params = {'eval_metric': 'rmse',
'max_depth': int(max_depth),
'subsample':subsample,
'eta': eta,
'min_child_weight': int(min_child_weight),
'colsample_bytree': colsample_bytree}
cv_result = xgb.cv(params, dtrain, num_boost_round=5000, nfold=3, early_stopping_rounds=50)
 return -1.0 * cv_result['test-rmse-mean'].iloc[-1]
xgb_bo = BayesianOptimization(xgb_evaluate, {'max_depth':(0, 6),
'subsample':(0.6, 1),
'eta':(0.01, 0.4),
'min_child_weight':(1, 30),
'colsample_bytree':(0.6, 1)})
 xgb_bo.maximize(init_points=10, n_iter=15, acq='ei')
 model_rmse=[]
for model in range(len(xgb_bo.res)) :
model_rmse.append(xgb_bo.res[model]['target'])
 xgb_bo.res[pd.Series(model_rmse ).idxmax() ]['target']
xgb_opt_params = xgb_bo.res[pd.Series(model_rmse ).idxmax() ]['params']
xgb_opt_params['max_depth']= int(round(xgb_opt_params['max_depth']))
xgb_opt_params['objective']='reg:linear'
xgb_opt_params['eval_metric']='rmse' 
<prepare_x_and_y>
train = pd.read_csv('.. /input/train.csv')
 train.head()
%%time
X = train_new.drop(['id', 'revenue','release_date'], axis=1)
 y = np.log1p(train_new['revenue'])
 train_data = lgb.Dataset(data=X, label=y, free_raw_data=False)
 def lgb_eval(learning_rate, num_leaves, bagging_fraction, max_depth, min_data_in_leaf, lambda_l1, lambda_l2):
params = {
'application': 'regression',
'metric': 'rmse',
'learning_rate': learning_rate,
'num_leaves':int(round(num_leaves)) ,
'bagging_fraction':bagging_fraction,
'max_depth': int(round(max_depth)) ,
'min_data_in_leaf':int(round(min_data_in_leaf)) ,
'lambda_l1':lambda_l1,
'lambda_l2':lambda_l2}
cv_result = lgb.cv(params, train_data, num_boost_round=5000, nfold=3,stratified =False,early_stopping_rounds=50)
 return -1.0 * min(cv_result['rmse-mean'])
 lgbBO = BayesianOptimization(lgb_eval, {'learning_rate':(0.01, 0.2),
'num_leaves':(2,31),
'bagging_fraction':(0.8, 1),
'max_depth':(-1, 5),
'min_data_in_leaf':(2, 20),
'lambda_l1':(0, 5),
'lambda_l2':(0, 5)}, random_state=200)
 lgbBO.maximize(init_points=10, n_iter=15, acq='ei')
 model_rmse=[]
for model in range(len(lgbBO.res)) :
model_rmse.append(lgbBO.res[model]['target'])
 lgbBO.res[pd.Series(model_rmse ).idxmax() ]['target']
lgb_opt_params = lgbBO.res[pd.Series(model_rmse ).idxmax() ]['params']
lgb_opt_params['max_depth']= int(round(lgb_opt_params['max_depth']))
lgb_opt_params['num_leaves']= int(round(lgb_opt_params['num_leaves']))
lgb_opt_params['min_data_in_leaf']= int(round(lgb_opt_params['min_data_in_leaf']))
lgb_opt_params['application']='regression'
lgb_opt_params['metric']='rmse' 
<prepare_x_and_y>
test = pd.read_csv('.. /input/test.csv')
 test.head()
n_fold = 10
random_seed=2222
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 X = train_new.drop(['id', 'revenue','release_date'], axis=1)
 y = np.log1p(train_new['revenue'])
 X_test = test_new.drop(['id','release_date'], axis=1)
<split>
print(train['Embarked'].value_counts())
 most_freq = train['Embarked'].value_counts().index[0]
train['Embarked'].fillna(most_freq, inplace=True )
def train_model(X, X_test, y, params=None, folds=folds, model_type='lgb', plot_feature_importance=True, model=None):
oof = np.zeros(X.shape[0])
 prediction = np.zeros(X_test.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print('Fold', fold_n, 'started at', time.ctime())
 if model_type == 'sklearn':
X_train, X_valid = X[train_index], X[valid_index]
else:
X_train, X_valid = X.values[train_index], X.values[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 10000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=100, early_stopping_rounds=100)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=10000, evals=watchlist, early_stopping_rounds=100, verbose_eval=100, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test.values), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = mean_squared_error(y_valid, y_pred_valid)
 y_pred = model.predict(X_test)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=10000, eval_metric='RMSE', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 prediction += y_pred
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction
return oof, prediction
else:
return oof, prediction<train_model>
train["Title"] = train["Name"].str.extract('([A-Za-z]+)\.',expand=False)
 test["Title"] = test["Name"].str.extract('([A-Za-z]+)\.',expand=False)
 for i in [train, test]:
i['Title'] = i['Title'].replace('Mr', 'Mr')
 i['Title'] = i['Title'].replace(( 'Mme', 'Ms'), 'Mrs')
 i['Title'] = i['Title'].replace('Mlle', 'Miss')
 i['Title'] = i['Title'].replace(( 'Capt', 'Col', 'Major', 'Dr','Rev'), 'Officer')
 i['Title'] = i['Title'].replace(( 'Jonkheer', 'Don', 'Sir', 'Countess','Dona', 'Lady'), 'Royalty' )
%%time
oof_xgb, prediction_xgb = train_model(X, X_test, y, params=xgb_opt_params, model_type='xgb' )<train_model>
for df in [train, test]:
for title in df['Title'].unique() :
age = df.loc[df['Title']==title, 'Age'].mean()
df[df['Title']==title].fillna(age, inplace=True )
%%time
oof_lgb, prediction_lgb = train_model(X, X_test, y, params=lgb_opt_params, model_type='lgb' )<prepare_x_and_y>
cat_cols = ['Pclass', 'Sex', 'Embarked']
for i in cat_cols:
train[i] = train[i].astype('category')
 test[i] = test[i].astype('category')
%%time
X = train_mean_agg.drop(['id', 'revenue','release_date'], axis=1)
 y = np.log1p(train_mean_agg['revenue'])
 X_test = test_mean_agg.drop(['id','release_date'], axis=1)
 dtrain = xgb.DMatrix(X, label=y)
 dtest = xgb.DMatrix(X_test)
 def xgb_evaluate(max_depth, subsample, eta, min_child_weight,colsample_bytree):
params = {'eval_metric': 'rmse',
'max_depth': int(max_depth),
'subsample':subsample,
'eta': eta,
'min_child_weight': int(min_child_weight),
'colsample_bytree': colsample_bytree}
cv_result = xgb.cv(params, dtrain, num_boost_round=5000, nfold=3, early_stopping_rounds=50)
 return -1.0 * cv_result['test-rmse-mean'].iloc[-1]
xgb_bo = BayesianOptimization(xgb_evaluate, {'max_depth':(0, 6),
'subsample':(0.6, 1),
'eta':(0.01, 0.4),
'min_child_weight':(1, 30),
'colsample_bytree':(0.6, 1)})
 xgb_bo.maximize(init_points=10, n_iter=15, acq='ei')
 model_rmse=[]
for model in range(len(xgb_bo.res)) :
model_rmse.append(xgb_bo.res[model]['target'])
 xgb_bo.res[pd.Series(model_rmse ).idxmax() ]['target']
xgb_opt_params = xgb_bo.res[pd.Series(model_rmse ).idxmax() ]['params']
xgb_opt_params['max_depth']= int(round(xgb_opt_params['max_depth']))
xgb_opt_params['objective']='reg:linear'
xgb_opt_params['eval_metric']='rmse' 
<prepare_x_and_y>
tr = train[['Age', 'Title']]
ts = test[['Age', 'Title']]
tr_ts = pd.concat([tr, ts])
 print(tr_ts.groupby('Title' ).mean() )
%%time
X = train_mean_agg.drop(['id', 'revenue','release_date'], axis=1)
 y = np.log1p(train_mean_agg['revenue'])
 X_test = test_mean_agg.drop(['id','release_date'], axis=1)
 train_data = lgb.Dataset(data=X, label=y, free_raw_data=False)
 def lgb_eval(learning_rate, num_leaves, bagging_fraction, max_depth, min_data_in_leaf, lambda_l1, lambda_l2):
params = {
'application': 'regression',
'metric': 'rmse',
'learning_rate': learning_rate,
'num_leaves':int(round(num_leaves)) ,
'bagging_fraction':bagging_fraction,
'max_depth': int(round(max_depth)) ,
'min_data_in_leaf':int(round(min_data_in_leaf)) ,
'lambda_l1':lambda_l1,
'lambda_l2':lambda_l2}
cv_result = lgb.cv(params, train_data, num_boost_round=5000, nfold=3,stratified =False,early_stopping_rounds=50)
 return -1.0 * min(cv_result['rmse-mean'])
 lgbBO = BayesianOptimization(lgb_eval, {'learning_rate':(0.01, 0.2),
'num_leaves':(2,31),
'bagging_fraction':(0.8, 1),
'max_depth':(-1, 5),
'min_data_in_leaf':(2, 20),
'lambda_l1':(0, 5),
'lambda_l2':(0, 5)}, random_state=200)
 lgbBO.maximize(init_points=10, n_iter=15, acq='ei')
 model_rmse=[]
for model in range(len(lgbBO.res)) :
model_rmse.append(lgbBO.res[model]['target'])
 lgbBO.res[pd.Series(model_rmse ).idxmax() ]['target']
lgb_opt_params = lgbBO.res[pd.Series(model_rmse ).idxmax() ]['params']
lgb_opt_params['max_depth']= int(round(lgb_opt_params['max_depth']))
lgb_opt_params['num_leaves']= int(round(lgb_opt_params['num_leaves']))
lgb_opt_params['min_data_in_leaf']= int(round(lgb_opt_params['min_data_in_leaf']))
lgb_opt_params['application']='regression'
lgb_opt_params['metric']='rmse' 
<train_model>
c_train_type = train['Cabin'].str[0]
train['c_type'] = c_train_type
train['c_type'] = train['c_type'].fillna('unknown')
 c_test_type = test['Cabin'].str[0]
test['c_type'] = c_test_type
test['c_type'] = test['c_type'].fillna('unknown' )
%%time
oof_xgb_agg, prediction_xgb_agg = train_model(X, X_test, y, params=xgb_opt_params, model_type='xgb' )<train_model>
train['fam_count'] = train['SibSp']+train['Parch']
test['fam_count'] = test['SibSp']+test['Parch']
size = {
0:'alone',
1:'small',
2:'small',
3:'small',
4:'large',
5:'large',
6:'large',
7:'large',
10:'large'
}
train['fam_size'] = train['fam_count'].map(size)
 test['fam_size'] = test['fam_count'].map(size)
 train['is_alone'] = train['fam_size']=='alone'
test['is_alone'] = test['fam_size']=='alone'
%%time
oof_lgb_agg, prediction_lgb_agg = train_model(X, X_test, y, params=lgb_opt_params, model_type='lgb' )<save_to_csv>
mm = MinMaxScaler()
for i in ['Age', 'Fare']:
train[i] = mm.fit_transform(train[i].values.reshape(-1,1))
test[i] = mm.fit_transform(test[i].values.reshape(-1,1))
sam_sub['revenue'] = np.expm1(prediction_lgb)
 sam_sub.to_csv("lgb.csv", index=False)
 sam_sub['revenue'] = np.expm1(prediction_xgb)
 sam_sub.to_csv("xgb.csv", index=False)
 sam_sub['revenue'] = np.expm1(prediction_lgb_agg)
 sam_sub.to_csv("lgb_agg.csv", index=False)
 sam_sub['revenue'] = np.expm1(prediction_xgb_agg)
 sam_sub.to_csv("xgb_agg.csv", index=False)
 sam_sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb)/ 2)
 sam_sub.to_csv("blend_lgb_xgb.csv", index=False)
 sam_sub['revenue'] = np.expm1(( prediction_lgb_agg + prediction_xgb_agg)/ 2)
 sam_sub.to_csv("blend_lgb_xgb_agg.csv", index=False)
 sam_sub['revenue'] = np.expm1(( prediction_lgb + prediction_xgb +prediction_lgb_agg + prediction_xgb_agg)/ 4)
 sam_sub.to_csv("put_them_all.csv", index=False )<set_options>
lb = LabelBinarizer()
for i in ['Sex', 'is_alone']:
train[i] = lb.fit_transform(train[i])
 test[i] = lb.fit_transform(test[i] )
pd.options.display.max_columns = None
warnings.simplefilter(action='ignore')
 stopwords_eng = set(stopwords.words('english'))
%matplotlib inline<load_from_disk>
en = LabelEncoder()
train['Pclass'] = en.fit_transform(train['Pclass'])
 test['Pclass'] = en.fit_transform(test['Pclass'] )
train = pd.read_hdf('.. /input/tmdb-preprocessed/train_kamal.h5')
 test = pd.read_hdf('.. /input/tmdb-preprocessed/test_kamal.h5' )<load_from_csv>
def create_dummies(df, column_name):
dummies = pd.get_dummies(df[column_name], prefix=column_name)
 df = pd.concat([df,dummies],axis=1)
 df.drop(column_name, axis=1, inplace=True)
 return df
def release_date_mapper(str_date):
if str(str_date)== 'nan':
return str_date
month, day, year_candidate = str_date.split('/')
 year = int(year_candidate)
 if year < 100:
if year <= 19:
year += 2000
else:
year += 1900
return month + '-' + day + '-' + str(year)
 test_orig = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test_orig['release_date'] = test_orig['release_date'].map(release_date_mapper)
 train_orig = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train_orig['release_date'] = train_orig['release_date'].map(release_date_mapper)
 train_orig['dataset'] = 'Train'
test_orig['dataset'] = 'Test'
data_orig = pd.concat(( train_orig, test_orig))
latest_release_date_test = pd.to_datetime(train_orig['release_date'] ).max()
print(f"Only {100*sum(pd.to_datetime(train_orig['release_date'])> latest_release_date_test)/len(train_orig):.1f} % of original training movies were release after the latest movie in test set." )<set_options>
train.drop(['Name', 'Ticket', 'Cabin'], axis=1, inplace=True)
 test.drop(['Name', 'Ticket', 'Cabin'], axis=1, inplace=True )
%%HTML
<a id=missing-zeros></a>
<img src=https://i.imgflip.com/2wazn1.jpg width="250" align="left"><define_variables>
features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'fam_count', 'is_alone']
X = train[features]
y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3 )
%%HTML
<img src=https://memegenerator.net/img/instances/84246705/let-make-missing-values-nan-again.jpg width="300"><feature_engineering>
nb = GaussianNB()
nb.fit(X_train, y_train)
 y_pred = nb.predict(X_test)
 accuracy_score(y_pred, y_test)
data.loc[data['budget'] == 0, 'budget'] = np.nan<filter>
perm = PermutationImportance(nb, random_state=1 ).fit(X_test, y_test)
 eli5.show_weights(perm, feature_names = X_test.columns.tolist() )
data[(data['budget'] < 37)&(data['budget'] > 32)]<set_options>
lr = LogisticRegression(C = 1, penalty= 'l2', solver= 'liblinear')
 lr.fit(X_train, y_train)
 y_pred = lr.predict(X_test)
 accuracy_score(y_pred, y_test )
%%HTML
<img src=https://image.tmdb.org/t/p/original/RdsII0tG6vdF6GTaiHePFaAKVM.jpg width="250"><train_on_grid>
perm = PermutationImportance(lr, random_state=1 ).fit(X_test, y_test)
 eli5.show_weights(perm, feature_names = X_test.columns.tolist() )
data_enriched_path = '.. /input/tmdb-preprocessed/data_tmdb_enriched_with_popularities.h5'
if not os.path.exists(data_enriched_path):
tmdb.API_KEY = 'YOUR_KEY_HERE'
search = tmdb.Search()
hits_cout = 0
missing_credits_mask = data['budget'].isnull()
total_number_of_movies_to_fill = sum(missing_credits_mask)
 for index, movie_row in tqdm(data[missing_credits_mask].iterrows() , total=total_number_of_movies_to_fill):
response = search.movie(query=movie_row['title'])if np.isnan(movie_row['release_year'])else search.movie(query=movie_row['title'], year=movie_row['release_year'])
 if response['total_results'] == 1:
movie = tmdb.Movies(response['results'][0]['id'])
 movie_info = movie.info()
if movie_info['budget'] != 0 and np.isnan(data.loc[index, 'budget']):
hits_cout += 1
data.loc[index, 'budget'] = movie_info['budget']
data['budget_adjusted'] =(data['budget'].map(lambda x: [x])+ data['release_year'].map(lambda x: [x])).map(lambda budget_year:
cpi.inflate(budget_year[0], budget_year[1], to=2018))
data['_budget_runtime_ratio'] = data['budget_adjusted']/data['runtime']
data['_budget_popularity_ratio'] = data['budget_adjusted']/data['popularity']
data['_budget_year_ratio'] = data['budget_adjusted']/(data['release_year']*data['release_year'])
 data['_budget_rating_ratio'] = data['budget_adjusted']/data['rating']
data['_budget_totalVotes_ratio'] = data['budget_adjusted']/data['totalVotes']<feature_engineering>
model = SVC()
hyperparameters = {
'C': [0.1, 1, 10, 100],
'gamma': [1, 0.1, 0.01],
'kernel': ['rbf', 'linear']
}
grid = GridSearchCV(model, param_grid=hyperparameters, cv=10)
 grid.fit(X, y)
 best_params = grid.best_params_
best_score = grid.best_score_
svc = grid.best_estimator_
y_pred = svc.predict(X_test)
 print(grid.best_params_)
 print(grid.best_estimator_)
 print(grid.best_score_ )
train_data = data[data['dataset'] == 'Train']
yearly_movie_counts = train_data['release_year'].value_counts()
single_movie_years = set(yearly_movie_counts.index[yearly_movie_counts == 1])
 train_data.loc[train_data['release_year'].isin(single_movie_years), 'release_year'] = 1
trn_indices_set, val_indices_set = [set(indices)for indices in train_test_split(train_data.index, stratify=train_data['release_year'])]
train_data.loc[train_data.index.isin(trn_indices_set), 'dataset'] = 'Train_train'
train_data.loc[train_data.index.isin(val_indices_set), 'dataset'] = 'Train_val'
compare_barplots_categorical(train_data[['budget_cat']], train_data['dataset'] == 'Train_train', train_data['dataset'] == 'Train_val',
n_categories_in_figure=21, binned_categories=True)
 plt.title('Budget train/val distributions, stratified by release year', fontsize=20 )<statistical_test>
perm = PermutationImportance(svc, random_state=1 ).fit(X_test, y_test)
 eli5.show_weights(perm, feature_names = X_test.columns.tolist() )
def nans_proportion(pd_series, logical_index=None):
if logical_index is None:
logical_index = np.ones_like(pd_series, dtype=np.bool)
 return pd_series[logical_index].isnull().sum() /logical_index.sum()
print(f"In train proportion of missing values is: {100*nans_proportion(data['rating'], data['dataset']=='Train'):.2f} %.")
 print(f"In test proportion of missing values is: {100*nans_proportion(data['rating'], data['dataset']=='Test'):.2f} %." )<merge>
model = KNeighborsClassifier()
hyperparameters = {
"n_neighbors" : range(1,20,2),
'weights' : ['uniform', 'distance'],
'p' : [1, 2]
}
grid = GridSearchCV(model, param_grid=hyperparameters, cv=10)
 grid.fit(X, y)
 best_params = grid.best_params_
best_score = grid.best_score_
knn = grid.best_estimator_
y_pred = knn.predict(X_test)
 print(grid.best_params_)
 print(grid.best_estimator_)
 print(grid.best_score_ )
data_orig = data_orig.merge(data[['id', 'rating']], on='id', how='left')
 low_rating_proportion_trn_orig = 100 * data_orig[(data_orig['release_year'] > 1980)& 
(data_orig['dataset'] == 'Train')
 ].groupby('release_year')['rating'].agg(lambda x: sum(x < 3)/len(x))
low_rating_proportion_tst_orig = 100 * data_orig[(data_orig['release_year'] > 1980)& 
(data_orig['dataset'] == 'Test')
 ].groupby('release_year')['rating'].agg(lambda x: sum(x < 3)/len(x))
plt.figure(figsize=(7, 4))
plot_with_dots(plt, low_rating_proportion_trn_orig)
 plot_with_dots(plt, low_rating_proportion_tst_orig)
 plt.xlabel('Year of Movie Release', fontsize=15)
 plt.ylabel('Proportion of yearly produced movies [%]', fontsize=15)
 plt.legend(['Train orig', 'Test orig'])
 plt.ylim(0, low_rating_proportion_trn.max())
 plt.title('Low-rating movies proportion', fontsize=20 )<filter>
perm = PermutationImportance(knn, random_state=1 ).fit(X_test, y_test)
 eli5.show_weights(perm, feature_names = X_test.columns.tolist() )
data.loc[(data['dataset'] == 'Train')&(data['release_year'] == 1990)&(data['rating'] < 3), ['title', 'imdb_id']]<count_duplicates>
dt = DecisionTreeClassifier()
dt.fit(X_train, y_train)
 y_pred = dt.predict(X_test)
 accuracy_score(y_pred, y_test)
print(f"Number of duplicated movies: {len(data)- data['imdb_id'].isnull().sum() - data['imdb_id'].nunique() }.")
 data = data.iloc[data.astype(str ).drop_duplicates().index]
print(f"After the cleaning, number of duplicated movies: {len(data)- data['imdb_id'].isnull().sum() - data['imdb_id'].nunique() }." )<filter>
dot_data = tree.export_graphviz(dt, out_file=None)
 graph = graphviz.Source(dot_data)
 graph.render("iris")
 dot_data = tree.export_graphviz(dt, out_file=None,
feature_names=X_train.columns,
class_names=['Survived', 'Not Survived'],
filled=True, rounded=True,
special_characters=True)
 graph = graphviz.Source(dot_data)
 graph
data.loc[(data['dataset'] == 'Train')&(data['release_year'] == 1999)&(data['rating'] < 3), 'title'].iloc[:5]<feature_engineering>
hyperparameters = {"criterion": ["entropy", "gini"],
"max_depth": [3, 5, 7, 10],
"max_features": ["log2", "sqrt", 'auto'],
'min_samples_leaf' : [2, 3, 4, 5],
'min_samples_split' : [2, 3, 4, 5]
}
grid = GridSearchCV(dt, param_grid=hyperparameters, cv=10)
 grid.fit(X, y)
 best_params = grid.best_params_
best_score = grid.best_score_
dt = grid.best_estimator_
y_pred = dt.predict(X_test)
 print(grid.best_params_)
 print(grid.best_score_)
 plt.figure(figsize=(3,3))
sns.heatmap(confusion_matrix(y_pred, y_test), annot=True, cbar=False, fmt='1d', cmap='Blues')
 perm = PermutationImportance(dt, random_state=1 ).fit(X_test, y_test)
 eli5.show_weights(perm, feature_names = X_test.columns.tolist() )
train_data = data[data['dataset'] == 'Train']
yearly_movie_counts = train_data['release_year'].value_counts()
single_movie_years = set(yearly_movie_counts.index[yearly_movie_counts == 1])
 train_data.loc[train_data['release_year'].isin(single_movie_years), 'release_year'] = 1
trn_indices_set, val_indices_set = [set(indices)for indices in train_test_split(train_data.index, stratify=train_data['release_year'])]
train_data.loc[train_data.index.isin(trn_indices_set), 'dataset'] = 'Train_train'
train_data.loc[train_data.index.isin(val_indices_set), 'dataset'] = 'Train_val'
plt.figure(figsize=(8, 7))
bins = train_data['rating'].quantile(np.arange(0, 1.001, 0.05))
sns.distplot(train_data.loc[train_data['dataset'] == 'Train_train', 'rating'], bins=bins)
 sns.distplot(train_data.loc[train_data['dataset'] == 'Train_val', 'rating'], bins=bins)
 plt.title('Rating, train/val distributions', fontsize=18)
 plt.legend(['Train: train', 'Train: val'] )<create_dataframe>
model = RandomForestClassifier()
hyperparameters = {"criterion": ["entropy", "gini"],
"max_depth": [5, 10],
"max_features": ["log2", "sqrt"],
'min_samples_leaf' : [2, 3, 4, 5],
'min_samples_split' : [2, 3, 4, 5],
"n_estimators": [6, 9]
}
grid = GridSearchCV(model, param_grid=hyperparameters, cv=10)
 grid.fit(X, y)
 best_params = grid.best_params_
best_score = grid.best_score_
rf = grid.best_estimator_
y_pred = rf.predict(X_test)
 print(grid.best_params_)
 print(grid.best_score_)
 plt.figure(figsize=(3,3))
sns.heatmap(confusion_matrix(y_pred, y_test), annot=True, cbar=False, fmt='1d', cmap='Blues')
 perm = PermutationImportance(rf, random_state=1 ).fit(X_test, y_test)
 eli5.show_weights(perm, feature_names = X_test.columns.tolist() )
<statistical_test><EOS> 
holdout_ids = test["PassengerId"]
holdout_features = test[features]
holdout_predictions = lr.predict(holdout_features)
 submission = pd.DataFrame({"PassengerId": holdout_ids,
"Survived": holdout_predictions})
 print(submission.head())
 submission.to_csv("submission.csv",index=False )