Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<count_values>
warnings.filterwarnings("ignore" )
heads['Target'].value_counts()<count_values>
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train.info()
preds['Target'].value_counts()<split>
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test.info()
X_train, X_valid, y_train, y_valid = train_test_split(train_selected,
train_labels,
test_size = 1000,
random_state = 10)
 model = lgb.LGBMClassifier(**best_hyp,
class_weight = 'balanced',
random_state = 10)
 model.fit(X_train, y_train);<predict_on_test>
train.isna().sum()
valid_preds = model.predict_proba(X_valid)
 preds_df = pd.DataFrame(valid_preds, columns = [1, 2, 3, 4])
 preds_df['prediction'] = preds_df[[1, 2, 3, 4]].idxmax(axis = 1)
 preds_df['confidence'] = preds_df[[1, 2, 3, 4]].max(axis = 1)
 preds_df.head()<compute_test_metric>
test.isna().sum()
print('F1 score:', round(f1_score(y_valid, preds_df['prediction'], average = 'macro'), 5))<choose_model_class>
train[['release_month','release_day','release_year']]=train['release_date'].str.split('/',expand=True ).replace(np.nan, -1 ).astype(int)
 train.loc[(train['release_year'] <= 19)&(train['release_year'] < 100), "release_year"] += 2000
train.loc[(train['release_year'] > 19)&(train['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(train['release_date'])
 train['release_dayofweek'] = releaseDate.dt.dayofweek
train['release_quarter'] = releaseDate.dt.quarter
n_components = 3
umap = UMAP(n_components=n_components)
 pca = PCA(n_components=n_components)
 ica = FastICA(n_components=n_components)
 tsne = TSNE(n_components=n_components )<categorify>
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
train = train
train['genres'] = train['genres'].map(lambda x: sorted([d['name'] for d in get_dictionary(x)])).map(lambda x: ','.join(map(str, x)))
 genres = train.genres.str.get_dummies(sep=',')
 train = pd.concat([train, genres], axis=1, sort=False)
 print("Action Genres Movie ", train[train.Action == 1].shape[0])
 print("Adventure Genres Movie ", train[train.Adventure == 1].shape[0])
 print("Animation Genres Movie ", train[train.Animation == 1].shape[0])
 print("Comedy Genres Movie ", train[train.Comedy == 1].shape[0])
 print("Crime Genres Movie ", train[train.Crime == 1].shape[0])
 print("Documentary Genres Movie ", train[train.Documentary == 1].shape[0])
 print("Drama Genres Movie ", train[train.Drama == 1].shape[0])
 print("Family Genres Movie ", train[train.Family == 1].shape[0])
 print("Fantasy Genres Movie ", train[train.Fantasy == 1].shape[0])
 print("Foreign Genres Movie ", train[train.Foreign == 1].shape[0])
 print("History Genres Movie ", train[train.History == 1].shape[0])
 print("Music Genres Movie ", train[train.Music == 1].shape[0])
 print("Mystery Genres Movie ", train[train.Mystery == 1].shape[0])
 print("Romance Genres Movie ", train[train.Romance == 1].shape[0])
 print("Science Fiction Genres Movie ", train[train['Science Fiction'] == 1].shape[0])
 print("TV Movie Genres Movie ", train[train['TV Movie'] == 1].shape[0])
 print("Thriller Genres Movie ", train[train.Thriller == 1].shape[0])
 print("War Genres Movie ", train[train.War == 1].shape[0])
 print("Western Genres Movie ", train[train.Western == 1].shape[0] )
train_df = train_selected.copy()
test_df = test_selected.copy()
for method, name in zip([umap, pca, ica, tsne],
['umap', 'pca', 'ica', 'tsne']):
if name == 'tsne':
start = timer()
reduction = method.fit_transform(train_selected)
 end = timer()
else:
start = timer()
reduction = method.fit_transform(train_selected)
 end = timer()
test_reduction = method.transform(test_selected)
 test_df['%s_c1' % name] = test_reduction[:, 0]
test_df['%s_c2' % name] = test_reduction[:, 1]
test_df['%s_c3' % name] = test_reduction[:, 2]
train_df['%s_c1' % name] = reduction[:, 0]
train_df['%s_c2' % name] = reduction[:, 1]
train_df['%s_c3' % name] = reduction[:, 2]
print(f'Method: {name} {round(end - start, 2)} seconds elapsed.' )<concatenate>
test = test
test['genres'] = test['genres'].map(lambda x: sorted([d['name'] for d in get_dictionary(x)])).map(lambda x: ','.join(map(str, x)))
 genres = test.genres.str.get_dummies(sep=',')
 test = pd.concat([test, genres], axis=1, sort=False)
 print("Action Genres Movie ", test[test.Action == 1].shape[0])
 print("Adventure Genres Movie ", test[test.Adventure == 1].shape[0])
 print("Animation Genres Movie ", test[test.Animation == 1].shape[0])
 print("Comedy Genres Movie ", test[test.Comedy == 1].shape[0])
 print("Crime Genres Movie ", test[test.Crime == 1].shape[0])
 print("Documentary Genres Movie ", test[test.Documentary == 1].shape[0])
 print("Drama Genres Movie ", test[test.Drama == 1].shape[0])
 print("Family Genres Movie ", test[test.Family == 1].shape[0])
 print("Fantasy Genres Movie ", test[test.Fantasy == 1].shape[0])
 print("Foreign Genres Movie ", test[test.Foreign == 1].shape[0])
 print("History Genres Movie ", test[test.History == 1].shape[0])
 print("Music Genres Movie ", test[test.Music == 1].shape[0])
 print("Mystery Genres Movie ", test[test.Mystery == 1].shape[0])
 print("Romance Genres Movie ", test[test.Romance == 1].shape[0])
 print("Science Fiction Genres Movie ", test[test['Science Fiction'] == 1].shape[0])
 print("TV Movie Genres Movie 0")
 print("Thriller Genres Movie ", test[test.Thriller == 1].shape[0])
 print("War Genres Movie ", test[test.War == 1].shape[0])
 print("Western Genres Movie ", test[test.Western == 1].shape[0] )
train_df, test_df = train_df.align(test_df, axis = 1, join = 'inner' )<compute_train_metric>
train['status'].value_counts()
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_df, train_labels,
test_df, test_ids, nfolds = 10,
hyp = best_hyp )<save_to_csv>
train.loc[train['status'] == "Rumored"][['status','revenue']]
submission.to_csv('gbm_opt_10fold_dr.csv', index = False )<create_dataframe>
test['status'].value_counts()
model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold_DR"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True )<train_model>
trainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 testAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.merge(train, trainAdditionalFeatures, how='left', on=['imdb_id'])
 test = pd.merge(test, testAdditionalFeatures, how='left', on=['imdb_id'] )
model = RandomForestClassifier(max_depth = 3, n_estimators=10)
 model.fit(train_selected, train_labels)
 estimator_limited = model.estimators_[5]
estimator_limited<save_to_csv>
print("Missing rating in Train set", train['rating'].isna().sum())
 print("Missing total Votes in Train set", train['totalVotes'].isna().sum())
 print("")
 print("Missing rating in Test set", test['rating'].isna().sum())
 print("Missing total Votes in Test set", test['totalVotes'].isna().sum() )
export_graphviz(estimator_limited, out_file='tree_limited.dot', feature_names = train_selected.columns,
class_names = ['extreme', 'moderate' , 'vulnerable', 'non-vulnerable'],
rounded = True, proportion = False, precision = 2, filled = True )<set_options>
train['rating'] = train['rating'].fillna(1.5)
 train['totalVotes'] = train['totalVotes'].fillna(6)
 test['rating'] = test['rating'].fillna(1.5)
 test['totalVotes'] = test['totalVotes'].fillna(6 )
!dot -Tpng tree_limited.dot -o tree_limited.png<set_options>
def prepare(df):
global json_cols
global train_dict
df['rating'] = df['rating'].fillna(1.5)
 df['totalVotes'] = df['totalVotes'].fillna(6)
 df['weightedRating'] =(df['rating']*df['totalVotes'] + 6.367 * 1000)/(df['totalVotes'] + 1000)
 df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'] = df['release_year']
df.loc[(df['release_year'] <= 19)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 19)&(df['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
df['originalBudget'] = df['budget']
df['inflationBudget'] = df['budget'] + df['budget']*1.8/100*(2018-df['release_year'])
 df['budget'] = np.log1p(df['budget'])
 df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df['collection_name'] = df['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['collection_name'].fillna('')))
 df['collection_name'] = le.transform(df['collection_name'].fillna('' ).astype(str))
df['num_Keywords'] = df['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 df['num_cast'] = df['cast'].apply(lambda x: len(x)if x != {} else 0)
 df['_popularity_mean_year'] = df['popularity'] / df.groupby("release_year")["popularity"].transform('mean')
 df['_budget_runtime_ratio'] = df['budget']/df['runtime']
df['_budget_popularity_ratio'] = df['budget']/df['popularity']
df['_budget_year_ratio'] = df['budget']/(df['release_year']*df['release_year'])
 df['_releaseYear_popularity_ratio'] = df['release_year']/df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity']/df['release_year']
df['_popularity_totalVotes_ratio'] = df['totalVotes']/df['popularity']
df['_rating_popularity_ratio'] = df['rating']/df['popularity']
df['_rating_totalVotes_ratio'] = df['totalVotes']/df['rating']
df['_totalVotes_releaseYear_ratio'] = df['totalVotes']/df['release_year']
df['_budget_rating_ratio'] = df['budget']/df['rating']
df['_runtime_rating_ratio'] = df['runtime']/df['rating']
df['_budget_totalVotes_ratio'] = df['budget']/df['totalVotes']
df['has_homepage'] = 0
df.loc[pd.isnull(df['homepage']),"has_homepage"] = 1
df['isbelongs_to_collectionNA'] = 0
df.loc[pd.isnull(df['belongs_to_collection']),"isbelongs_to_collectionNA"] = 1
df['isTaglineNA'] = 0
df.loc[df['tagline'] == 0 ,"isTaglineNA"] = 1
df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'] == "en" ,"isOriginalLanguageEng"] = 1
df['isTitleDifferent'] = 1
df.loc[ df['original_title'] == df['title'] ,"isTitleDifferent"] = 0
df['isMovieReleased'] = 1
df.loc[ df['status'] != "Released" ,"isMovieReleased"] = 0
df['collection_id'] = df['belongs_to_collection'].apply(lambda x : np.nan if len(x)==0 else x[0]['id'])
 df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['cast'].apply(lambda x : len(x))
df['meanruntimeByYear'] = df.groupby("release_year")["runtime"].aggregate('mean')
 df['meanPopularityByYear'] = df.groupby("release_year")["popularity"].aggregate('mean')
 df['meanBudgetByYear'] = df.groupby("release_year")["budget"].aggregate('mean')
 df['meantotalVotesByYear'] = df.groupby("release_year")["totalVotes"].aggregate('mean')
 df['meanTotalVotesByRating'] = df.groupby("rating")["totalVotes"].aggregate('mean')
 for col in ['genres', 'production_countries', 'spoken_languages', 'production_companies'] :
df[col] = df[col].map(lambda x: sorted(list(set([n if n in train_dict[col] else col+'_etc' for n in [d['name'] for d in x]])))).map(lambda x: ','.join(map(str, x)))
 temp = df[col].str.get_dummies(sep=',')
 df = pd.concat([df, temp], axis=1, sort=False)
 df.drop(['genres_etc'], axis = 1, inplace = True)
 df = df.drop(['id', 'revenue','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline', 'collection_id'
],axis=1)
 df.fillna(value=0.0, inplace = True)
 return df
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test['revenue'] = np.nan
train = pd.merge(train, pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 test = pd.merge(test, pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 additionalTrainData = pd.read_csv('.. /input/tmdb-box-office-prediction-more-training-data/additionalTrainData.csv')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].astype('str')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].str.replace('-', '/')
 train = pd.concat([train, additionalTrainData])
 print(train.columns)
 print(train.shape)
 train['revenue'] = np.log1p(train['revenue'])
 y = train['revenue'].values
json_cols = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in tqdm(json_cols + ['belongs_to_collection']):
train[col] = train[col].apply(lambda x : get_dictionary(x))
test[col] = test[col].apply(lambda x : get_dictionary(x))
def get_json_dict(df):
global json_cols
result = dict()
for e_col in json_cols :
d = dict()
rows = df[e_col].values
for row in rows :
if row is None : continue
for i in row :
if i['name'] not in d :
d[i['name']] = 0
d[i['name']] += 1
result[e_col] = d
return result
train_dict = get_json_dict(train)
 test_dict = get_json_dict(test)
 for col in json_cols :
remove = []
train_id = set(list(train_dict[col].keys()))
test_id = set(list(test_dict[col].keys()))
remove += list(train_id - test_id)+ list(test_id - train_id)
 for i in train_id.union(test_id)- set(remove):
if train_dict[col][i] < 10 or i == '' :
remove += [i]
for i in remove :
if i in train_dict[col] :
del train_dict[col][i]
if i in test_dict[col] :
del test_dict[col][i]
all_data = prepare(pd.concat([train, test] ).reset_index(drop = True))
train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:]
Image(filename = 'tree_limited.png' )<train_model>
random_seed = 2019
k = 10
fold = list(KFold(k, shuffle = True, random_state = random_seed ).split(train))
np.random.seed(random_seed )
model = RandomForestClassifier(max_depth = None, n_estimators=10)
 model.fit(train_selected, train_labels)
 estimator_nonlimited = model.estimators_[5]
export_graphviz(estimator_nonlimited, out_file='tree_nonlimited.dot', feature_names = train_selected.columns,
class_names = ['extreme', 'moderate' , 'vulnerable', 'non-vulnerable'],
rounded = True, proportion = False, precision = 2)
 !dot -Tpng tree_nonlimited.dot -o tree_nonlimited.png -Gdpi=600<define_variables>
def xgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 6,
'subsample': 0.6,
'colsample_bytree': 0.7,
'eval_metric': 'rmse',
'seed': random_seed,
'silent': True,
}
record = dict()
model = xgb.train(params
, xgb.DMatrix(trn_x, trn_y)
 , 1000000
, [(xgb.DMatrix(trn_x, trn_y), 'train'),(xgb.DMatrix(val_x, val_y), 'valid')]
, verbose_eval=verbose
, early_stopping_rounds=500
, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid']['rmse'][best_idx], 'importance':[i for k, i in model.get_score().items() ]}
Image(filename = 'tree_nonlimited.png' )<define_variables>
def lgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective':'regression',
'num_leaves' : 40,
'min_data_in_leaf' : 20,
'max_depth' : 4,
'learning_rate': 0.005,
"feature_fraction": 0.8,
"bagging_freq": 1,
"bagging_fraction": 0.8,
"bagging_seed": random_seed,
"metric": 'rmse',
"random_state" : random_seed,
"verbosity": -1}
record = dict()
model = lgb.train(params
, lgb.Dataset(trn_x, trn_y)
 , num_boost_round = 1000000
, valid_sets = [lgb.Dataset(val_x, val_y)]
, verbose_eval = verbose
, early_stopping_rounds = 500
, callbacks = [lgb.record_evaluation(record)] 
)
 best_idx = np.argmin(np.array(record['valid_0']['rmse']))
val_pred = model.predict(val_x, num_iteration = model.best_iteration)
 test_pred = model.predict(test, num_iteration = model.best_iteration)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid_0']['rmse'][best_idx], 'importance':model.feature_importance('gain')}
COMPUTE_LB = False<set_options>
def cat_model(trn_x, trn_y, val_x, val_y, test, verbose):
model = CatBoostRegressor(iterations=10000000,
learning_rate=0.005,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.7,
random_seed = random_seed,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=500 
)
 model.fit(trn_x, trn_y,
eval_set=(val_x, val_y),
use_best_model=True,
verbose=False)
 val_pred = model.predict(val_x)
 test_pred = model.predict(test)
 return {'val':val_pred, 'test':test_pred, 'error':model.get_best_score() ['validation_0']['RMSE'], 'importance':model.get_feature_importance() }
kernel_start_time = time.perf_counter()
starttime = time.time()
warnings.simplefilter("ignore")
<normalization>
result_dict = dict()
val_pred = np.zeros(train.shape[0])
 test_pred = np.zeros(test.shape[0])
 final_err = 0
verbose = False
for i,(trn, val)in enumerate(fold):
print(i+1, "fold.RMSE")
 trn_x = train.loc[trn, :]
trn_y = y[trn]
val_x = train.loc[val, :]
val_y = y[val]
fold_val_pred = []
fold_test_pred = []
fold_err = []
start = datetime.now()
result = xgb_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val'])
 fold_test_pred.append(result['test'])
 fold_err.append(result['error'])
 print("xgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 start = datetime.now()
result = lgb_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val'])
 fold_test_pred.append(result['test'])
 fold_err.append(result['error'])
 print("lgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 start = datetime.now()
result = cat_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val'])
 fold_test_pred.append(result['test'])
 fold_err.append(result['error'])
 print("cat model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 val_pred[val] += np.mean(np.array(fold_val_pred), axis = 0)
 test_pred += np.mean(np.array(fold_test_pred), axis = 0)/ k
final_err +=(sum(fold_err)/ len(fold_err)) / k
print("---------------------------")
 print("avg err.", "{0:.5f}".format(sum(fold_err)/ len(fold_err)))
 print("blend err.", "{0:.5f}".format(np.sqrt(np.mean(( np.mean(np.array(fold_val_pred), axis = 0)- val_y)**2))))
print('')
 print("fianl avg err.", final_err)
 print("fianl blend err.", np.sqrt(np.mean(( val_pred - y)**2)) )
class DogDataset(Dataset):
def __init__(self, img_dir, transform1, transform2=None):
self.img_dir = img_dir
self.img_names = os.listdir(img_dir)
 self.transform1 = transform1
self.transform2 = transform2
self.imgs = []
self.labels = []
for img_name in tqdm(self.img_names):
img_path = os.path.join(img_dir, img_name)
 img = Image.open(img_path)
 annotation_basename = os.path.splitext(os.path.basename(img_path)) [0]
annotation_dirname = next(dirname for dirname in os.listdir(args.root_annots)if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join(args.root_annots, annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 bboxes = []
for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 img_crop = img.crop(bbox)
 self.imgs.append(self.transform1(img_crop))
self.labels.append(annotation_dirname.split('-')[1].lower())
 def __getitem__(self, index):
img = self.imgs[index]
label = self.labels[index]
if self.transform2 is not None:
img = self.transform2(img)
 return img, label
def __len__(self):
return len(self.imgs)
 def get_transforms() :
normalize = transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5))
random_transforms = [transforms.ColorJitter() ,
transforms.RandomRotation(degrees=1)]
random_cropper = [torchvision.transforms.CenterCrop(args.image_size), torchvision.transforms.RandomCrop(args.image_size)]
transform1 = transforms.Compose([
transforms.Resize(args.image_size),
transforms.CenterCrop(args.image_size),
])
 transform2 = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomApply(random_transforms, p=0.3),
transforms.ToTensor() ,
normalize])
 return transform1, transform2
transform1, transform2 = get_transforms()
train_data = DogDataset(img_dir=args.root_images,
transform1=transform1,
transform2=transform2)
 decoded_dog_labels = {i:breed for i, breed in enumerate(sorted(set(train_data.labels)))}
encoded_dog_labels = {breed:i for i, breed in enumerate(sorted(set(train_data.labels)))}
train_data.labels = [encoded_dog_labels[l] for l in train_data.labels]
dataloader = torch.utils.data.DataLoader(train_data,
shuffle=True,
batch_size=args.batch_size,
num_workers=args.num_workers )<normalization>
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 df_sub = pd.DataFrame()
df_sub['id'] = sub['id']
df_sub['revenue'] = np.expm1(test_pred)
 df_sub.to_csv("submission.csv", index=False )