Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 print('Shape of train dataset: {}'.format(train.shape))
print('Shape of test dataset: {}'.format(test.shape))
print(f )<categorify>
train['Type'] = 'train'
test['Type'] = 'test'
all = pd.concat([train, test], sort=False ).reset_index(drop=True)
 print('Shape of all dataset: {}'.format(all.shape))
print(f"From the most recent added movies {100 * nans_proportion(data['overview'], data['release_date'] > latest_release_date_test):.1f} % do not have overview.")
 print(f"In the original dataset {100 * nans_proportion(data_orig['overview']):.1f} % of movies do not have overview" )<set_options>
print(all.isnull().values.any())
 train.isnull().sum()
pd.set_option('display.max_colwidth', -1)
 data.loc[(data['release_year'] > 2015)&(data['overview_word_count'] < 20), ['title', 'overview']].sample(5, random_state=2 )<categorify>
all_corr[all_corr['level_0'] == 'Age']
print(f"From originally available movies {100 * nans_proportion(data_orig['tagline']):.1f} % do not have a tagline.")
 print(f"From the most recent added movies {100 * nans_proportion(data['tagline'], data['release_date'] > latest_release_date_test):.1f} % do not have a tagline." )<define_variables>
all['Age'] = all.groupby(['Pclass', 'SibSp'])['Age'].apply(lambda x: x.fillna(x.median()))
%%HTML
<img src=http://wist.info/wp/wp-content/uploads/2016/04/Chekhov-brevity-sister-of-talent-wist_info-quote.jpg width='400'><feature_engineering>
print(all_corr[all_corr['level_0'] == 'Fare'])
 all['Fare'] = all.groupby(['Pclass', 'Parch', 'SibSp'])['Fare'].apply(lambda x: x.fillna(x.median()))
print(f"From originally available movies {100 * nans_proportion(data_orig['crew']):.1f} % do not have information about crew.")
 fraction_of_record_with_missing_crew = data.loc[data['release_date'] > latest_release_date_test, 'crew'].map(lambda x: len(x)== 0 ).sum() /sum(data['release_date'] > latest_release_date_test)
 print(f'From the most recent added movies {100 * fraction_of_record_with_missing_crew:.1f} % do not have information about crew.' )<define_variables>
all['Cabin'].value_counts()
if os.path.exists(data_enriched_path):
data = pd.read_hdf(data_enriched_path, 'data')
 else:
tmdb.API_KEY = 'YOUR_KEY_HERE'
search = tmdb.Search()
hits_cout = 0
attributes_to_fill = ['crew', 'cast', 'Keywords', 'genres', 'production_companies', 'production_countries', 'spoken_languages']
missing_credits_mask =(data[attributes_to_fill[0]].map(lambda x: len(x)== 0))
for col in attributes_to_fill[1:]:
missing_credits_mask = missing_credits_mask |(data[col].map(lambda x: len(x)== 0))
total_number_of_movies_to_fill = sum(missing_credits_mask)
 for col in attributes_to_fill:
data.loc[missing_credits_mask, col] = data.loc[missing_credits_mask, col].map(lambda x: [] if isinstance(x, dict)else x)
 for index, movie_row in tqdm(data[missing_credits_mask].iterrows() , total=total_number_of_movies_to_fill):
response = search.movie(query=movie_row['title'])if np.isnan(movie_row['release_year'])else search.movie(query=movie_row['title'], year=movie_row['release_year'])
 if response['total_results'] == 1:
movie = tmdb.Movies(response['results'][0]['id'])
 movie_info = movie.info(append_to_response='credits,keywords')
 if len(data.loc[index, 'cast'])== 0:
data.loc[index, 'cast'].extend(movie_info['credits']['cast'])
 if len(data.loc[index, 'crew'])== 0:
data.loc[index, 'crew'].extend(movie_info['credits']['crew'])
 if len(data.loc[index, 'Keywords'])== 0:
data.loc[index, 'Keywords'].extend(movie_info['keywords']['keywords'])
 for col in attributes_to_fill[3:]:
if len(data.loc[index, col])== 0:
data.loc[index, col].extend(movie_info[col])
 hits_cout += int(len(movie_info['credits']['cast'])> 0 or len(movie_info['credits']['crew'])> 0)
 print(f'Filling in was successful for {hits_cout} movies out of {total_number_of_movies_to_fill}.')
 data.to_hdf(data_enriched_path, 'data' )<categorify>
all['Cabin'] = all['Cabin'].fillna('N' )
data['genders_0_crew'] = data['crew'].apply(lambda x: sum([i['gender'] == 0 for i in x])if isinstance(x, list)else np.nan)
 data['genders_1_crew'] = data['crew'].apply(lambda x: sum([i['gender'] == 1 for i in x])if isinstance(x, list)else np.nan)
 data['genders_2_crew'] = data['crew'].apply(lambda x: sum([i['gender'] == 2 for i in x])if isinstance(x, list)else np.nan)
 data['_collection_name'] = data['belongs_to_collection'].apply(lambda x: x[0]['name'] if isinstance(x, list)else np.nan)
 le = LabelEncoder()
data['_collection_name'] = le.fit_transform(data['_collection_name'].map(str))
data.loc[data['_collection_name'].map(str)== 'nan', '_collection_name'] = np.nan
data['_num_Keywords'] = data['Keywords'].apply(lambda x: len(x)if isinstance(x, list)else np.nan)
 data['_num_cast'] = data['cast'].apply(lambda x: len(x)if isinstance(x, list)else np.nan)
 data['production_countries_count'] = data['production_countries'].apply(lambda x : len(x)if isinstance(x, list)else np.nan)
 data['production_companies_count'] = data['production_companies'].apply(lambda x : len(x)if isinstance(x, list)else np.nan)
 data['cast_count'] = data['cast'].apply(lambda x : len(x)if isinstance(x, list)else np.nan)
 data['crew_count'] = data['crew'].apply(lambda x : len(x)if isinstance(x, list)else np.nan )<set_options>
print(all['Embarked'].value_counts())
 all['Embarked'] = all['Embarked'].fillna('S' )
%%HTML
<a id='revenue-inflation'></a><merge>
all.isnull().values.any()
dollar_costs_df = pd.DataFrame({'year': mean_yearly_revenues['release_year'], 'dollar_2018_cost': dollar_costs})
 data['dollar_2018_cost'] = data[['release_year']].merge(dollar_costs_df, left_on='release_year', right_on='year')['dollar_2018_cost']<filter>
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
print(all['Name'].value_counts())
 all['Title'] = all['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
all['Family'] = extract_surname(all['Name'])
 all['IsMarried'] = np.where(all['Title'] == 'Mrs', 1, 0)
 all.drop(['Name'], inplace=True, axis=1 )
train.loc[(train['release_date'] >= '2015-01-01')&(train['revenue'] > 2e9), ['title']]<set_options>
all['FamilySize'] = all['SibSp'] + all['Parch'] + 1
%%HTML
<img src=https://media.giphy.com/media/xT0xejJnePNcOWoHOo/giphy.gif width=300><define_variables>
family_map = {1: 'Alone', 2: 'Small', 3: 'Small', 4: 'Small', 5: 'Medium', 6: 'Medium', 7: 'Large', 8: 'Large', 11: 'Large'}
all['FamilySizeGrouped'] = all['FamilySize'].map(family_map )
%%HTML
<img src=http://filmsplusmovies.com/wp-content/uploads/2014/06/317-hollywood.jpg width="500"><feature_engineering>
all['Ticket_count'] = all.Ticket.apply(lambda x: all[all['Ticket']==x].shape[0] )
data.drop('budget', axis=1, inplace=True)
 data['revenue'] = data['revenue'].map(np.log1p)
 data['_budget_runtime_ratio'] = data['budget_adjusted']/data['runtime']
data['_budget_popularity_ratio'] = data['budget_adjusted']/data['popularity_mean']
data['_budget_year_ratio'] = data['budget_adjusted']/(data['release_year']*data['release_year'])
 data['_budget_rating_ratio'] = data['budget_adjusted']/data['rating']
data['_budget_totalVotes_ratio'] = data['budget_adjusted']/data['totalVotes']<count_values>
all.loc[(all['Sex'] == 'male')&(all['Age'] > 18.0), 'Sex'] = 0
all.loc[(all['Sex'] == 'male')&(all['Age'] <= 18.0), 'Sex'] = 1
all.loc[all['Sex'] == 'female', 'Sex'] = 2
jobs_of_interest_set = set(top_frequent_jobs.index)
 def value_counts_in_list(input_list):
counts = defaultdict(int)
 for val in input_list:
counts[val] += 1
return list(counts.items())
 job_person_counts = pd.Series(reduce(lambda list1, list2: list1 + list2,
data['crew'].map(lambda x: value_counts_in_list(
 filter(lambda x: x in jobs_of_interest_set,
map(lambda el: el['job'],
x)
)) ).values))
job_head_counts_df = pd.DataFrame({'job': job_person_counts.map(lambda x: x[0]), 'headcount': job_person_counts.map(lambda x: x[1])})
 percentile_95_job_headcount = job_head_counts_df.groupby('job')['headcount'].quantile(0.95 ).map(int)
 percentile_95_job_headcount[top_frequent_jobs.index]<feature_engineering>
list1 = all[(all['Title'] != 'Mr')&(all['Survived'] == 0)]['Ticket'].tolist()
list2 = all[(all['Title'] == 'Mr')&(all['Survived'] == 1)]['Ticket'].tolist()
all['Ticket_with_FC_dead'] = 0
all['Ticket_with_M_alive'] = 0
all.loc[all['Ticket'].isin(list1), 'Ticket_with_FC_dead'] = 1
all.loc[all['Ticket'].isin(list2), 'Ticket_with_M_alive'] = 1
all.drop(['Ticket'], inplace=True, axis=1 )
columns_len_init = len(data.columns)
<count_missing_values>
all['Cabin'].value_counts()
print(data.isnull().sum().sort_values() [:1])
 count_col = data.isnull().sum().sort_values().index[0]<feature_engineering>
all['Deck'] = all['Cabin'].apply(lambda x: x[0])
 all['Deck'] = all['Deck'].replace(['T'], 'A')
 all.drop(['Cabin'], inplace=True, axis=1 )
def return_joint_entry(crew, job_name, sep='^'):
headcount = percentile_95_job_headcount[job_name]
if not isinstance(crew, list)and np.isnan(crew):
return np.nan
names = [el['name'] for el in crew if el['job'] == job_name][:headcount]
return sep.join(names)
 for crew_col in top_frequent_jobs.index:
data[crew_col] = data['crew'].map(lambda x: return_joint_entry(x, crew_col))
jobs_to_dummify = [job for job in top_frequent_jobs.index if percentile_95_job_headcount[job] > 2]
final_dummy_cols = []
def get_number_of_movies_with_name(pd_series, sep='^'):
names_in_movies = pd.Series(reduce(lambda list1, list2: list1 + list2,
pd_series.map(lambda x: x.split(sep)).values))
number_of_movies_with_name = names_in_movies.value_counts()
return number_of_movies_with_name
for crew_col in jobs_to_dummify:
number_of_movies_with_name = get_number_of_movies_with_name(data[crew_col])
 number_of_movies_with_name_train = get_number_of_movies_with_name(data.loc[data['dataset'] == 'Train', crew_col])
 remaining_names = set(number_of_movies_with_name.index[number_of_movies_with_name >= 10] ).intersection(
 set(number_of_movies_with_name_train.index[number_of_movies_with_name_train >= 5]))
df_added_cols = data[crew_col].map(lambda x: '^'.join([name for name in x.split('^')if name in remaining_names])).str.get_dummies(sep='^')
 df_added_cols.columns = [f'{crew_col}_{value}' for value in df_added_cols.columns]
final_dummy_cols.extend(df_added_cols.columns)
 data = pd.concat(( data, df_added_cols), axis=1)
 data.drop(jobs_to_dummify + ['crew'], axis=1, inplace=True)
 jobs_joint = [job for job in top_frequent_jobs.index if percentile_95_job_headcount[job] <= 2]
def hide_rare_values(data, column, overall_limit=7, train_limit=3):
counts_total = data.groupby(joint_col)[count_col].count()
counts_train = data[data['dataset']=='Train'].groupby(joint_col)[count_col].count()
names_to_forget = set(counts_total[counts_total < overall_limit].index ).union(set(counts_train[counts_train < train_limit].index))
data.loc[data[joint_col].isin(names_to_forget), joint_col] = np.nan
return data
for joint_col in jobs_joint:
data = hide_rare_values(data, joint_col )<feature_engineering>
all.loc[all['Deck'] == 'A', 'Deck'] = 0
all.loc[all['Deck'] == 'B', 'Deck'] = 1
all.loc[all['Deck'] == 'C', 'Deck'] = 2
all.loc[all['Deck'] == 'D', 'Deck'] = 3
all.loc[all['Deck'] == 'E', 'Deck'] = 4
all.loc[all['Deck'] == 'F', 'Deck'] = 5
all.loc[all['Deck'] == 'G', 'Deck'] = 6
all.loc[all['Deck'] == 'N', 'Deck'] = 7
def return_entries(list_dicts, sep='^'):
if not isinstance(list_dicts, list)and np.isnan(list_dicts):
return np.nan
entries = [el['name'] for el in list_dicts]
return sep.join(entries)
 data['genres'] = data['genres'].map(return_entries)
 data = hide_rare_values(data, 'genres', overall_limit=50, train_limit=20)
 for dict_col in ['cast', 'Keywords', 'production_companies', 'production_countries', 'spoken_languages']:
data[dict_col] = data[dict_col].map(return_entries)
 names_in_movies = pd.Series(reduce(lambda list1, list2: list1 + list2,
data[dict_col].map(lambda x: x.split('^')).values))
number_of_movies_with_name = names_in_movies.value_counts()
min_movie_count = 20 if dict_col == 'cast' else 100
remaining_names = number_of_movies_with_name.index[number_of_movies_with_name >= min_movie_count]
df_added_cols = data[dict_col].map(lambda x: '^'.join([name for name in x.split('^')if name in remaining_names])).str.get_dummies(sep='^')
 df_added_cols.columns = [f'{dict_col}_{value}' for value in df_added_cols.columns]
final_dummy_cols.extend(df_added_cols.columns)
 data = pd.concat(( data, df_added_cols), axis=1)
 data.drop(['cast', 'Keywords', 'production_companies', 'production_countries', 'spoken_languages'], axis=1, inplace=True )<feature_engineering>
all.loc[all['Embarked'] == 'S', 'Embarked'] = 0
all.loc[all['Embarked'] == 'Q', 'Embarked'] = 1
all.loc[all['Embarked'] == 'C', 'Embarked'] = 2
porter = PorterStemmer()
def get_listofwords_series(col):
all_list_of_words_series = data[col].map(str ).map(lambda text: [porter.stem(w)if w != 'nan' else np.nan for w in word_tokenize(text.translate(str.maketrans('','',string.punctuation)).lower())])
 if f'{col}_stopwords_count' not in data.columns:
data[f'{col}_stopwords_count'] = all_list_of_words_series.map(lambda x: sum([w in stopwords_eng for w in x]))
data[f'{col}_stopwords_fraction'] = data[f'{col}_stopwords_count']/all_list_of_words_series.map(len)
 return all_list_of_words_series.map(lambda all_words: list(filter(lambda x: x not in stopwords_eng, all_words)) )<set_options>
all['Title'].value_counts()
%%HTML
<iframe width="560" height="315" src="https://www.youtube.com/embed/_wYtG7aQTHA?rel=0&amp;controls=1&amp;showinfo=0" frameborder="0" allowfullscreen></iframe><drop_column>
all['Title'] = all['Title'].replace(['Miss', 'Mrs','Ms', 'Mlle', 'Lady', 'Mme', 'the Countess', 'Dona'], 'Miss/Mrs/Ms')
 all['Title'] = all['Title'].replace(['Dr', 'Col', 'Major', 'Jonkheer', 'Capt', 'Sir', 'Don', 'Rev'], 'Dr/Military/Noble/Clergy')
 all['Title'].value_counts()
data = add_word_columns(data, 'overview' )<set_options>
all.loc[all['Title'] == 'Mr', 'Title'] = 0
all.loc[all['Title'] == 'Miss/Mrs/Ms', 'Title'] = 1
all.loc[all['Title'] == 'Master', 'Title'] = 2
all.loc[all['Title'] == 'Dr/Military/Noble/Clergy', 'Title'] = 3
%%HTML
<img src=https://mindblown.smumn.edu/wp-content/uploads/2018/02/zipf-law-meme.jpg width="200"><drop_column>
list1 = all[(all['Title'] != 0)&(all['Survived'] == 0)]['Family'].tolist()
list2 = all[(all['Title'] == 0)&(all['Survived'] == 1)]['Family'].tolist()
all['Family_with_FC_dead'] = 0
all['Family_with_M_alive'] = 0
all.loc[all['Family'].isin(list1), 'Family_with_FC_dead'] = 1
all.loc[all['Family'].isin(list2), 'Family_with_M_alive'] = 1
all.drop(['Family'], inplace=True, axis=1 )
data = add_word_columns(data, 'tagline' )<drop_column>
all.loc[all['FamilySizeGrouped'] == 'Alone', 'FamilySizeGrouped'] = 0
all.loc[all['FamilySizeGrouped'] == 'Small', 'FamilySizeGrouped'] = 1
all.loc[all['FamilySizeGrouped'] == 'Medium', 'FamilySizeGrouped'] = 2
all.loc[all['FamilySizeGrouped'] == 'Large', 'FamilySizeGrouped'] = 3
data.drop(['title', 'original_title', 'overview', 'tagline'], axis=1, inplace=True )<compute_test_metric>
train_y = train['Survived']
train_x = train.drop(['Survived', 'PassengerId'], axis=1)
 model = LogisticRegression()
model.fit(train_x, train_y)
 pred = model.predict(train_x)
 metrics.accuracy_score(pred, train_y )
def compute_stats(val_date, competition_values_all):
movie_val = val_date[0]
movie_competition_groupy_id_date = val_date[1]
if np.isnan(movie_val)or pd.isnull(movie_competition_groupy_id_date):
return [np.nan] * 6
competition = competition_values_all[movie_competition_groupy_id_date]
zscore =(movie_val - np.nanmean(competition)) /np.nanstd(competition)
 percentile = stats.percentileofscore(competition, movie_val)
 val_to_max = movie_val / np.nanmax(competition)
 min_to_val = np.nanmin(competition)/ movie_val
val_to_mean = movie_val / np.nanmean(competition)
 val_to_median = movie_val / np.nanmedian(competition)
 return [zscore, percentile, val_to_max, min_to_val, val_to_mean, val_to_median]
def add_comparison_stats(data, cols, neighbourhood_in_weeks=1):
def get_next_sunday_date(date):
return date + pd.to_timedelta(6 - date.weekday() % 7, unit='D')
 time_range_groups = None
dummy_release_dates = []
for i in range(2*neighbourhood_in_weeks):
ts_dummy = data.loc[~data['release_date'].isin(dummy_release_dates), 'release_date'].min() - pd.to_timedelta(i * 1, unit='W')
 dummy_row = pd.DataFrame([np.full_like(data.iloc[0], fill_value=np.nan)], columns=data.columns)
 dummy_row['release_date'] = ts_dummy
if i > 0:
dummy_release_dates.append(ts_dummy)
 data = pd.concat(( data, dummy_row))
groups_next =(data[['release_date'] + cols] 
.groupby(pd.Grouper(key='release_date', freq=f'{2*neighbourhood_in_weeks}W', closed='right')) 
.agg(lambda values: list(filter(lambda val: not np.isnan(val), values))))
time_range_groups = groups_next if time_range_groups is None else pd.concat(( time_range_groups, groups_next))
for col in cols:
competition_values = time_range_groups[col]
val_date_series = data[col].map(lambda x: [x])+(data['release_date'] 
.map(lambda x: x + pd.to_timedelta(neighbourhood_in_weeks, unit='W')) 
.map(get_next_sunday_date)
.map(lambda x: [x]))
column_names = [f"{col}_competition_{stat}_±_{neighbourhood_in_weeks}_week{'s' if neighbourhood_in_weeks > 1 else ''}" for stat in ['zscore', 'percentile', 'val_to_max', 'min_to_val', 'val_to_mean', 'val_to_median']]
df_added_cols = pd.DataFrame(np.column_stack(val_date_series.map(lambda val_date: compute_stats(val_date, competition_values)) ).T, columns=column_names, index=data.index)
 data = pd.concat(( data, df_added_cols), axis=1)
 return data<create_dataframe>
test_x = test.drop('PassengerId', axis=1)
 timestamp = int(round(time.time() * 1000))
pred = model.predict(test_x)
 output = pd.DataFrame({"PassengerId":test.PassengerId , "Survived" : pred})
 output = output.astype(int)
 output.to_csv("submission_" + str(timestamp)+ ".csv",index = False )