Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<train_model>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from collections import Counter
from sklearn.model_selection import train_test_split
import lightgbm as lgb
rf3.fit(train[feats], train['Target'])
 accuracy_score(train['Target'], rf3.predict(train[feats]))<choose_model_class>
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 sample_submission = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv' )
rf = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced' )<train_model>
train['has_collection'] = train['belongs_to_collection'].apply(lambda x: 1 if str(x)!= 'nan' else 0)
 train['collection_id'] = train['belongs_to_collection'].apply(lambda x: eval(x)[0]['id'] if str(x)!= 'nan' else 0)
 test['has_collection'] = test['belongs_to_collection'].apply(lambda x: 1 if str(x)!= 'nan' else 0)
 test['collection_id'] = test['belongs_to_collection'].apply(lambda x: eval(x)[0]['id'] if str(x)!= 'nan' else 0 )
rf.fit(train[feats], train['Target'])
 accuracy_score(train['Target'], rf.predict(train[feats]))<train_model>
train = train.drop(['belongs_to_collection'], axis=1)
 test = test.drop(['belongs_to_collection'], axis=1 )
rf.fit(train[feats], train['Target'] )<predict_on_test>
list_of_genres = list(train['genres'].apply(lambda x: [i['name'] for i in eval(x)] if str(x)!= 'nan' else [] ).values )
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
top_genres = [m[0] for m in Counter([i for j in list_of_genres for i in j] ).most_common(15)]
train['all_genres'] = train['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in eval(x)])) if(isinstance(x,int)or isinstance(x,str)) == True else '')
 for gen in top_genres:
train['genre_' + gen] = train['all_genres'].apply(lambda x: 1 if gen in x else 0)
 test['all_genres'] = test['genres'].apply(lambda x: ' '.join(sorted([i['name'] for i in eval(x)])) if(isinstance(x,int)or isinstance(x,str)) == True else '')
 for gen in top_genres:
test['genre_' + gen] = test['all_genres'].apply(lambda x: 1 if gen in x else 0)
 train = train.drop(['genres'], axis=1)
 test = test.drop(['genres'], axis=1 )
test['Target'].value_counts(normalize=True )<save_to_csv>
lang_encoder = LabelEncoder()
train['all_genres'] = lang_encoder.fit_transform(train['all_genres'])
 test['all_genres'] = lang_encoder.fit_transform(test['all_genres'] )
test[['Id', 'Target']].to_csv('submission.csv', index=False )<load_from_csv>
lang_encoder = LabelEncoder()
train['original_language'] = lang_encoder.fit_transform(train['original_language'])
 test['original_language'] = lang_encoder.fit_transform(test['original_language'] )
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
prod_companies = list(train['production_companies'].apply(lambda x: [i['name'] for i in eval(x)] if str(x)!= 'nan' else '' ).values )
df_all = df.append(test)
 df_all.shape<count_values>
train['prod_companies_count'] = train['production_companies'].apply(lambda x: len([i for i in eval(x)])if str(x)!= 'nan' else 0)
 test['prod_companies_count'] = test['production_companies'].apply(lambda x: len([i for i in eval(x)])if str(x)!= 'nan' else 0 )
df_all['dependency'].value_counts()<filter>
pop_production = Counter([i for j in prod_companies for i in j] )
df_all[df_all['idhogar'] == '2b58d945f'][['hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total']]<count_values>
train['production_score'] = train['production_companies'].apply(lambda x: np.tanh(max([pop_production[i['name']] for i in eval(x)])) if str(x)!= 'nan' else 0)
 test['production_score'] = test['production_companies'].apply(lambda x: np.tanh(max([pop_production[i['name']] for i in eval(x)])) if str(x)!= 'nan' else 0 )
df_all['edjefa'].value_counts()<count_values>
train = train.drop(['production_companies'], axis=1)
 test = test.drop(['production_companies'], axis=1 )
df_all['edjefe'].value_counts()<data_type_conversions>
train['production_countries'] = train['production_countries'].apply(lambda x: [i['name'] for i in eval(x)][0] if str(x)!= 'nan' else '')
 test['production_countries'] = test['production_countries'].apply(lambda x: [i['name'] for i in eval(x)][0] if str(x)!= 'nan' else '' )
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )<data_type_conversions>
prod_country_encoder = LabelEncoder()
train['production_countries'] = prod_country_encoder.fit_transform(train['production_countries'])
 test['production_countries'] = prod_country_encoder.fit_transform(test['production_countries'] )
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )<count_missing_values>
train['release_date'] = train['release_date'].apply(lambda x: pd.to_datetime(x))
test['release_date'] = test['release_date'].apply(lambda x: pd.to_datetime(x))
df_all.isnull().sum()<count_missing_values>
train['year'] = train['release_date'].apply(lambda x: x.year)
 train['month'] = train['release_date'].apply(lambda x: x.month)
 train['day_of_week'] = train['release_date'].apply(lambda x: x.weekday())
 test['year'] = test['release_date'].apply(lambda x: x.year)
 test['month'] = test['release_date'].apply(lambda x: x.month)
 test['day_of_week'] = test['release_date'].apply(lambda x: x.weekday() )
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()<count_values>
train = train.drop(['release_date'], axis=1)
 test = test.drop(['release_date'], axis=1 )
df_all['v18q'].value_counts()<count_missing_values>
train['year'] = train['year'].apply(lambda x: x-100 if x>2020 else x)
 test['year'] = test['year'].apply(lambda x: x-100 if x>2020 else x )
df_all.isnull().sum().sort_values()<categorify>
avg_runtime_train = train['runtime'].mean()
train['runtime'] = train['runtime'].apply(lambda x: x if str(x)!= 'nan' else avg_runtime_train)
 avg_runtime_test = test['runtime'].mean()
test['runtime'] = test['runtime'].apply(lambda x: x if str(x)!= 'nan' else avg_runtime_test )
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )<define_variables>
train['spoken_languages_count'] = train['spoken_languages'].apply(lambda x: len(eval(x)) if str(x)!= 'nan' else 0)
 test['spoken_languages_count'] = test['spoken_languages'].apply(lambda x: len(eval(x)) if str(x)!= 'nan' else 0 )
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<choose_model_class>
train = train.drop(['spoken_languages'], axis=1)
 test = test.drop(['spoken_languages'], axis=1 )
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42 )<train_model>
list_of_keywords = list(train['Keywords'].apply(lambda x: [i['name'] for i in eval(x)] if str(x)!= 'nan' else [] ).values )
rf.fit(train[feats], train['Target'] )<predict_on_test>
train['num_Keywords'] = train['Keywords'].apply(lambda x: len(eval(x)) if str(x)!= 'nan' else 0)
 train['all_Keywords'] = train['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in eval(x)])) if str(x)!= 'nan' else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
train['keyword_'+g] = train['all_Keywords'].apply(lambda x: 1 if g in x else 0)
 test['num_Keywords'] = test['Keywords'].apply(lambda x: len(eval(x)) if str(x)!= 'nan' else 0)
 test['all_Keywords'] = test['Keywords'].apply(lambda x: ' '.join(sorted([i['name'] for i in eval(x)])) if str(x)!= 'nan' else '')
 top_keywords = [m[0] for m in Counter([i for j in list_of_keywords for i in j] ).most_common(30)]
for g in top_keywords:
test['keyword_'+g] = test['all_Keywords'].apply(lambda x: 1 if g in x else 0 )
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
keywords_encoder = LabelEncoder()
train['all_Keywords'] = keywords_encoder.fit_transform(train['all_Keywords'])
 test['all_Keywords'] = keywords_encoder.fit_transform(test['all_Keywords'] )
test['Target'].value_counts(normalize=True )<save_to_csv>
train = train.drop(['Keywords'], axis=1)
 test = test.drop(['Keywords'], axis=1 )
test[['Id', 'Target']].to_csv('submission.csv', index=False )<set_options>
train['cast'] = train['cast'].apply(lambda x: eval(x)if str(x)!= 'nan' else [])
 test['cast'] = test['cast'].apply(lambda x: eval(x)if str(x)!= 'nan' else [] )
%matplotlib inline
plt.style.use('fivethirtyeight')
 plt.rcParams['font.size'] = 18
plt.rcParams['patch.edgecolor'] = 'k'<load_from_csv>
train['gender_0_count'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
train['gender_1_count'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
train['gender_2_count'] = train['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
test['gender_0_count'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
test['gender_1_count'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
test['gender_2_count'] = test['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
pd.options.display.max_columns = 150
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<import_modules>
list_of_cast_members = list(train['cast'].apply(lambda x: [i['name'] for i in x] if str(x)!= 'nan' else [] ).values )
from collections import OrderedDict<data_type_conversions>
top_cast_members = [m[0] for m in Counter([i for j in list_of_cast_members for i in j] ).most_common(50)]
map={'yes':1,"no":0}
for df in [train,test]:
df['dependency']=df['dependency'].replace(map ).astype(np.float64)
 df['edjefe']=df['edjefe'].replace(map ).astype(np.float64)
 df['edjefa']=df['edjefa'].replace(map ).astype(np.float64)
 train[['dependency','edjefe','edjefa']].describe()<concatenate>
for g in top_cast_members:
train['cast_member_'+g] = train['cast'].apply(lambda x: 1 if g in str(x)else 0)
 for g in top_cast_members:
test['cast_member_'+g] = test['cast'].apply(lambda x: 1 if g in str(x)else 0 )
test['Target']=np.nan
data=train.append(test,ignore_index=True )<prepare_x_and_y>
train = train.drop(['cast'], axis=1)
 test = test.drop(['cast'], axis=1 )
X = [[0],[1],[2]]
Y = [ 1, 2, 1 ]
dtc = DTC(max_depth=1 )<train_model>
train['crew'] = train['crew'].apply(lambda x: eval(x)if str(x)!= 'nan' else [])
 test['crew'] = test['crew'].apply(lambda x: eval(x)if str(x)!= 'nan' else [] )
dtc.fit(X,Y)
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
<train_model>
list_of_crew_members = list(train['crew'].apply(lambda x: [i['name'] for i in x] if str(x)!= 'nan' else [] ).values )
dtc.fit(X,Y,sample_weight=[1,2,3])
 print(dtc.tree_.threshold)
 print(dtc.tree_.impurity)
<count_values>
top_crew_members = [m[0] for m in Counter([i for j in list_of_crew_members for i in j] ).most_common(50)]
data['Target'].value_counts()<define_variables>
for g in top_crew_members:
train['crew_member_'+g] = train['crew'].apply(lambda x: 1 if g in str(x)else 0)
 for g in top_crew_members:
test['crew_member_'+g] = test['crew'].apply(lambda x: 1 if g in str(x)else 0 )
heads=data.loc[data['parentesco1'] == 1].copy()
train_labels=data.loc[(data['Target'].notnull())&(data['parentesco1']==1),['Target','idhogar']]<count_values>
train = train.drop(['crew'], axis=1)
 test = test.drop(['crew'], axis=1 )
type_counts=train_labels['Target'].value_counts()<create_dataframe>
train['has_homepage'] = train['homepage'].apply(lambda x: 1 if str(x)!= 'nan' else 0)
 test['has_homepage'] = test['homepage'].apply(lambda x: 1 if str(x)!= 'nan' else 0 )
type_counts=pd.DataFrame(type_counts )<feature_engineering>
train = train.drop(['homepage'], axis=1)
 test = test.drop(['homepage'], axis=1 )
type_counts['level']=type_counts.index<feature_engineering>
train = train.drop(['poster_path'], axis=1)
 test = test.drop(['poster_path'], axis=1 )
type_counts['level']=type_counts['level'].replace(poverty_mapping)
<feature_engineering>
train = train.drop(['status'], axis=1)
 test = test.drop(['status'], axis=1 )
type_counts['Count']=type_counts['Target']<groupby>
for col in ['title', 'tagline', 'overview', 'original_title']:
train['len_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x)))
 train['words_' + col] = train[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test['len_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x)))
 test['words_' + col] = test[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
all_equal=train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() ==1 )<filter>
train = train.drop(["imdb_id", "original_title", "overview", "tagline", "title"], axis=1)
 test = test.drop(["imdb_id", "original_title", "overview", "tagline", "title"], axis=1 )
not_equal= all_equal[all_equal!=True]<filter>
X = train.drop(['id', 'revenue'], axis=1)
 Y = np.log1p(train['revenue'])
 X_test = test.drop(['id'], axis=1 )
example=train.loc[train['idhogar']==not_equal.index[0],['idhogar', 'parentesco1', 'Target']]<groupby>
X_train, X_valid, Y_train, Y_valid = train_test_split(X, Y, test_size=0.1 )
households_leader=train.groupby('idhogar')['parentesco1'].sum()<filter>
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model.fit(X_train, Y_train,
eval_set=[(X_train, Y_train),(X_valid, Y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )
households_leader_flase=train.loc[train['idhogar'].isin(households_leader
[households_leader !=1].index),:]<feature_engineering>
y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_ )
<prepare_output><EOS> 
sample_submission['revenue'] = np.expm1(y_pred)
 sample_submission.to_csv("submission.csv", index=False )