Kaggle competition: TMDB Box Office Prediction
<count_values><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<count_values>
import pandas as pd
import numpy as np
df.edjefa.value_counts().head(5 )<data_type_conversions>
le = LabelEncoder()
def get_dictionary(s):
try:
i = eval(s)
 except:
i = {}
return i
def prepare(df):
global json_cols
global train_dict
df[['release_month', 'release_day', 'release_year']] = df['release_date'].str.split('/', expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'].map(lambda x : x if x > 100 else x +(2000 if(x <= 18)else 1900))
releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
df['rating'] = df.rating.fillna(df['rating'].median())
 df['totalVotes'] = df.totalVotes.fillna(df['rating'].median())
 df['weighteRating'] = np.log(df['rating'] * df['totalVotes'])
 df['originalBudget'] = df['budget']
df['inflationBudget'] = df['budget'] + df['budget'] * 1.8 / 100 *(2018 - df['release_year'])
 df['budget'] = np.log1p(df['budget'])
 df['genders_0_crew'] = df['crew'].map(lambda x : sum([1 for i in get_dictionary(x)if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].map(lambda x : sum([1 for i in get_dictionary(x)if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].map(lambda x : sum([1 for i in get_dictionary(x)if i['gender'] == 2]))
df['_collection_name'] = df['belongs_to_collection'].map(lambda x : [i['name'] for i in get_dictionary(x)])
 df['_collection_name'] = le.fit_transform(list(df['_collection_name'].fillna('' ).astype(str)))
 df['_num_Keywords'] = df['Keywords'].map(lambda x : len(eval(x)) if get_dictionary(x)!= {} else 0)
 df['_num_cast'] = df['cast'].map(lambda x : len(eval(x)) if get_dictionary(x)!= {} else 0)
 df['_popularity_mean_year'] = df['popularity'] / df.groupby('release_year')['popularity'].transform('mean')
 df['_budget_runtime_ratio'] = df.budget / df.runtime
df['_release_year_popularity_ratio'] = df.release_year / df.popularity
df['_release_year_polularity_ratio2'] = df.popularity / df.release_year
df['_popularity_totalVote_ratio'] = df.totalVotes / df.popularity
df['_rating_popularity_ratio'] = df.rating / df.popularity
df['_rating_totalVotes_ratio'] = df.totalVotes / df.rating
df['_totalVotes_releaseYear_ratio'] = df.totalVotes / df.release_year
df['has_homepage'] = 1
df.loc[df['homepage'].isnull() , 'has_homepage'] = 0
df['isbelongs_to_collectionNA'] = 0
df.loc[df['belongs_to_collection'].isnull() , 'isbelongs_to_collectionNA'] = 1
df['isTaglineNA'] = 0
df.loc[df['tagline'].isnull() , 'isTaglineNA'] = 1
df['isOriginalLanguage'] = 0
df.loc[df['original_language'] == 'en', 'isOriginalLanguage'] = 1
df['isTitleDifferent'] = 1
df.loc[df['original_title'] == df['title'], 'isTitleDifferent'] = 0
df['isMovieReleased'] = 1
df.loc[df['status'] == 'Released', 'isMovieReleased'] = 0
df['collection_id'] = df['belongs_to_collection'].map(lambda x : np.nan if len(get_dictionary(x)) == 0 else get_dictionary(x)[0]['id'])
 df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].map(lambda x : len(get_dictionary(x)))
 df['cast_count'] = df['cast'].map(lambda x : len(get_dictionary(x)))
 df['crew_count'] = df['crew'].map(lambda x : len(get_dictionary(x)))
 df['meanruntimeByYear'] = df.groupby('release_year')['runtime'].transform('mean' ).round()
df['meanPopularityByYear'] = df.groupby('release_year')['popularity'].transform('mean' ).round()
df['meanBudgetByYear'] = df.groupby('release_year')['budget'].transform('mean')
 df['meanTotalVotesByYear'] = df.groupby('release_year')['totalVotes'].transform('mean' ).round()
df['meanTotalVotesByRating'] = df.groupby('rating')['totalVotes'].transform('mean' ).round()
df['medianBudgetByYear'] = df.groupby('release_year')['budget'].transform('median')
 for col in ['genres', 'production_countries', 'spoken_languages', 'production_companies']:
df[col] = df[col].map(lambda x : sorted(list(set([n if n in train_dict['genres'] else 'genres' + '_etc' for n in [i['name'] for i in get_dictionary(x)]])))).map(lambda x : ','.join(map(str, x)))
 temp = df[col].str.get_dummies(sep=',')
 df = pd.concat([df, temp], axis=1, sort=False)
 df.drop(['genres_etc'], axis=1, inplace=True)
 df = df.drop(['id', 'revenue', 'belongs_to_collection',
'genres', 'homepage', 'imdb_id', 'overview',
'runtime', 'poster_path', 'production_companies',
'production_countries', 'release_date', 'spoken_languages',
'status', 'title', 'Keywords', 'cast', 'crew',
'original_language', 'original_title', 'tagline',
'collection_id'], axis=1)
 df.fillna(value=0.0, inplace=True)
 return df
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test['revenue'] = np.nan
TrainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 TestAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.merge(train, TrainAdditionalFeatures, how='left', on=['imdb_id'])
 test = pd.merge(test, TestAdditionalFeatures, how='left', on=['imdb_id'])
 additionalTrainData = pd.read_csv('.. /input/tmdb-box-office-prediction-more-training-data/additionalTrainData.csv')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].astype('str')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].str.replace('-', '/')
 train = pd.concat([train, additionalTrainData])
 train['revenue'] = np.log1p(train['revenue'])
 y = train['revenue'].values
json_cols = ['genres', 'production_companies', 'production_countries',
'spoken_languages', 'Keywords', 'cast', 'crew']
def get_json_dict(df):
global json_cols
result = {}
for e_col in json_cols:
d = {}
rows = df[e_col].values
for row in rows:
if row is None: continue
if type(row)is str:
for i in get_dictionary(row):
if i['name'] not in d:
d[i['name']] = 0
d[i['name']] += 1
result[e_col] = d
return result
train_dict = get_json_dict(train)
 test_dict = get_json_dict(test)
 for col in json_cols:
remove = []
train_id = set(list(train_dict[col].keys()))
test_id = set(list(test_dict[col].keys()))
remove += list(train_id - test_id)+ list(test_id - train_id)
 for i in train_id.union(test_id)- set(remove):
if train_dict[col][i] < 10 or i == '' :
remove += [i]
for i in remove :
if i in train_dict[col] :
del train_dict[col][i]
if i in test_dict[col] :
del test_dict[col][i]
all_data = prepare(pd.concat([train, test] ).reset_index(drop = True))
train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:]
valores_replace = {'yes': 1, 'no': 0}
df['dependency'] = df['dependency'].replace(valores_replace ).astype(float)
 df['edjefe'] = df['edjefe'].replace(valores_replace ).astype(int)
 df['edjefa'] = df['edjefa'].replace(valores_replace ).astype(int )<sort_values>
from sklearn.model_selection import KFold
df[df['Target'].notnull() ].corr() ['Target'].sort_values(ascending=False )<feature_engineering>
random_seed = 2019
k = 10
fold = list(KFold(k, shuffle=True, random_state=random_seed ).split(train))
np.random.seed(random_seed)
df["m2perpessoa"]=df["tamhog"]/df["tamviv"]
df['rentperpessoa'] = df['v2a1'] / df['tamviv'] 
<count_unique_values>
result_dict = {}
val_pred = np.zeros(train.shape[0])
 test_pred = np.zeros(test.shape[0])
 final_err = 0
verbose = False
print(round(df[df['cielorazo']==0]['idhogar'].nunique() /df[df['cielorazo']==1]['idhogar'].nunique() *100,2),'%' )<count_unique_values>
import xgboost as xgb
print('** Métodos de se jogar lixo fora **')
 print('Tanker truck:',df[df['elimbasu1']==1].groupby(['elimbasu1'])['idhogar'].nunique().iloc[0])
 print('Botan hollow or buried:',df[df['elimbasu2']==1].groupby(['elimbasu2'])['idhogar'].nunique().iloc[0])
 print('Burning:',df[df['elimbasu3']==1].groupby(['elimbasu3'])['idhogar'].nunique().iloc[0])
 print('Throwing in an unoccupied space:',df[df['elimbasu4']==1].groupby(['elimbasu4'])['idhogar'].nunique().iloc[0])
 print('Throwing in river, creek or sea:',df[df['elimbasu5']==1].groupby(['elimbasu5'])['idhogar'].nunique().iloc[0])
 print('Other:',df[df['elimbasu6']==1].groupby(['elimbasu6'])['idhogar'].nunique().iloc[0])
<split>
def xgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective':'reg:linear',
'eta': 0.01,
'max_depth':6,
'subsample':0.6,
'colsample_bytree':0.7,
'eval_metric':'rmse',
'seed':random_seed,
'silent':True,
}
record = dict()
model = xgb.train(params,
xgb.DMatrix(trn_x, trn_y),
100000,
[(xgb.DMatrix(trn_x, trn_y), 'train'), 
(xgb.DMatrix(val_x, val_y), 'valid')],
verbose_eval = verbose,
early_stopping_rounds = 500,
callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
 return {'val' : val_pred,
'test' : test_pred,
'error': record['valid']['rmse'][best_idx],
'importance': [i for k, i in model.get_score().items() ]}
feats = [c for c in df.columns if c not in ['Id', 'idhogar', 'Target']]
train, test = df[~df['Target'].isnull() ], df[df['Target'].isnull() ]
train.shape, test.shape<choose_model_class>
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv' )
param_grid = {
'criterion' : ['gini', 'entropy'],
'class_weight' : ['balanced','balanced_subsample'],
'n_estimators': [100, 200, 300, 400]
}
rf = RandomForestClassifier()
grid_search = GridSearchCV(estimator = rf, param_grid = param_grid,
cv = 3, n_jobs = -1, verbose = 2 )<train_model>
reve = np.expm1(fold_test_pred )
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200 ,random_state=42)
 rf.fit(train[feats], train['Target'] )<predict_on_test>
df_sub = pd.DataFrame()
df_sub['id'] = sub['id']
df_sub['revenue'] = reve[0]
<count_values><EOS> 
df_sub.to_csv('submission_1.csv', index=False )