Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<train_model>
output_notebook()
selector.fit(train_set, train_labels )<prepare_output>
train_path = '.. /input/tmdb-box-office-prediction/train.csv'
test_path = '.. /input/tmdb-box-office-prediction/test.csv'
nominations_path='.. /input/nominations/nominations.csv'
train = pd.read_csv(train_path)
 test = pd.read_csv(test_path)
 nominations=pd.read_csv(nominations_path)
rankings = pd.DataFrame({'feature': list(train_set.columns), 'rank': list(selector.ranking_)} ).sort_values('rank')
 rankings.head(10 )<categorify>
colons_in_Json = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew','belongs_to_collection']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in colons_in_Json :
train[col] = train[col].apply(lambda x : get_dictionary(x))
for col in colons_in_Json :
test[col] = test[col].apply(lambda x : get_dictionary(x))
train_selected = selector.transform(train_set)
 test_selected = selector.transform(test_set )<create_dataframe>
%%script false
path = './train.csv'
data = pd.read_csv(path)
 cast_dict = data['cast'].apply(lambda x: ast.literal_eval(x)if pd.notnull(x)else {})
 cast_ids=[[y['id'] for y in x] for x in cast_dict]
unique_cast = pd.DataFrame(cast_ids ).stack().unique()
nominations=pd.DataFrame(columns=['id', 'nominations'])
 nominations['id'] = [x for x in unique_cast]
i=0
with open('nominations.csv', 'a')as csv:
for x in unique_cast[i:]:
start_time = time.time()
print(x)
 try:
with urlopen("https://www.themoviedb.org/person/"+str(x)) as page:
soup = BeautifulSoup(page, 'html.parser')
 name=soup.find('h2' ).text
name = unidecode.unidecode(name)
 name=name.replace(" ", "%20")
 print(name)
 url="https://sg.media-imdb.com/suggests/"+name[0].lower() +"/"+name.lower() +".json"
print(url)
 page = urlopen(url)
 soup = BeautifulSoup(page, 'html.parser')
 id=soup.text.find('"id":"nm')
 id=soup.text[id+6:id+15]
print(id)
 print(i)
 page = urlopen("https://www.imdb.com/name/"+id+"/awards?ref_=nm_ql_2")
 soup = BeautifulSoup(page, 'html.parser')
 a = 0
academy=0
golden=0
if soup.find_all('h3', string="Academy Awards, USA"):
soupCarotte = soup.find_all('table', attrs={'class': 'awards'})[a]
academy=len(soupCarotte.find_all('td', attrs={'class': 'award_outcome'}))
a+=1
if soup.find_all('h3', string="Golden Globes, USA"):
soupSoup = soup.find_all('table', attrs={'class': 'awards'})[a]
golden=len(soupSoup.find_all('td', attrs={'class': 'award_outcome'}))
nominations.loc[i,'nominations']=golden+academy
except urllib.error.HTTPError:
nominations.loc[i,'nominations']=0
print(nominations)
 output = pd.DataFrame({'id': nominations.loc[i, 'id'],
'nominations': nominations.loc[i, 'nominations']}, index=[0])
 output.to_csv(csv, header=False, index=False)
 i+=1
print(time.time() - start_time)
selected_features = train_set.columns[np.where(selector.ranking_==1)]
train_selected = pd.DataFrame(train_selected, columns = selected_features)
 test_selected = pd.DataFrame(test_selected, columns = selected_features )<compute_train_metric>
%%script false
warnings.filterwarnings("ignore")
 path = './train.csv'
data = pd.read_csv(path)
 print(data['belongs_to_collection'].iloc[0])
 colons_in_Json = ['belongs_to_collection']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in tqdm(colons_in_Json):
data[col] = data[col].apply(lambda x : get_dictionary(x))
google_result=data[['id','title','belongs_to_collection']]
google_result['google_result']=""
movie_title=google_result['title']
list_movie_title=list(movie_title)
 google_result['film_belongs_to_collection'] = google_result['belongs_to_collection'].apply(lambda x: 0 if x == {} else 1)
 word_movie=' movie'
with open('google_result.csv', 'a')as csv:
for index, r in google_result.iloc[0:].iterrows() :
if google_result.iloc[index]['film_belongs_to_collection']==0:
search=google_result['title'].iloc[index]
search=search+word_movie
else:
words = ['(Theatrical)','(1958 series)','(Series)','- Коллекция','Collection','(Animation)','(Universal Series)','(Heisei)',': The Original Series','- Collezione','(Original)','(Hammer Series)','(Remake)','(Reboot)','(Series)','Trilogy','(1976 series)','(Original Series)','(Universal Series)','Anthology','(Universal)','() ','The Klapisch ','(Коллекция)']
search = []
belongs_to_collection_line=google_result.iloc[index]['belongs_to_collection']
collection_name = belongs_to_collection_line[0]['name']
for w in words:
collection_name = collection_name.replace(w, '')
 search.append(collection_name)
 search=(search[0])
 search=search+word_movie
try:
print(search)
 r = requests.get("https://www.google.com/search", params={'q':search})
 soup = BeautifulSoup(r.text, "lxml")
 res = soup.find("div", {"id": "resultStats"})
 nb_result = ''.join(x for x in res.text if x.isdigit())
 print(nb_result)
 google_result.loc[index, 'google_result']=nb_result
except urllib.error.HTTPError:
google_result.loc[index,'google_result']=0
output = pd.DataFrame({'id': google_result.loc[index, 'title'],
'google_result': google_result.loc[index, 'google_result']}, index=[0])
 output.to_csv(csv, header=False, index=False)
model_results = cv_model(train_selected, train_labels, model, 'RF-SEL', model_results )<compute_test_metric>
t= train[['id','title','runtime','revenue']].copy()
t.iloc[1335]=t.iloc[1335].replace(np.nan, int(120))
t.iloc[2302]=t.iloc[2302].replace(np.nan, int(90))
t['runtime_cat_min_60'] = t['runtime'].apply(lambda x: 1 if(x <=60)else 0)
 t['runtime_cat_61_80'] = t['runtime'].apply(lambda x: 1 if(x >60)&(x<=80)else 0)
 t['runtime_cat_81_100'] = t['runtime'].apply(lambda x: 1 if(x >80)&(x<=100)else 0)
 t['runtime_cat_101_120'] = t['runtime'].apply(lambda x: 1 if(x >100)&(x<=120)else 0)
 t['runtime_cat_121_140'] = t['runtime'].apply(lambda x: 1 if(x >120)&(x<=140)else 0)
 t['runtime_cat_141_170'] = t['runtime'].apply(lambda x: 1 if(x >140)&(x<=170)else 0)
 t['runtime_cat_171_max'] = t['runtime'].apply(lambda x: 1 if(x >=170)else 0)
 t.loc[t.runtime_cat_min_60 == 1,'runtime_category'] = 'cat_min-60'
t.loc[t.runtime_cat_61_80 == 1,'runtime_category'] = 'cat_61-80'
t.loc[t.runtime_cat_81_100 == 1,'runtime_category'] = 'cat_81-100'
t.loc[t.runtime_cat_101_120 == 1,'runtime_category'] = 'cat_101-120'
t.loc[t.runtime_cat_121_140 == 1,'runtime_category'] = 'cat_121-140'
t.loc[t.runtime_cat_141_170 == 1,'runtime_category'] = 'cat_141-170'
t.loc[t.runtime_cat_171_max == 1,'runtime_category'] = 'cat_171-max'
cat = t['runtime_category']
ctr = Counter(cat)
 cat = [x for x in ctr]
unique_names = pd.Series(cat ).unique()
dic={}
for a in unique_names:
mask = t.runtime_category.apply(lambda x: a in x)
 dic[a] = t[mask]['revenue'].mean()
t = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'runtime_cat'})
 t = t.nlargest(6, 'mean_revenue')
 t['color'] = Category20c[6]
hover1 = HoverTool(tooltips = [ 
('Runtime_category','@runtime_cat'), 
('Revenue','@mean_revenue')
 ])
 p = figure(x_range=t.runtime_cat, plot_width=800,plot_height=400, toolbar_location=None, title="Revenue per runtime category", tools=[hover1])
 p.vbar(x='runtime_cat', top='mean_revenue', width=0.9, source=t, legend='runtime_cat',
line_color='white',fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p )
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True<train_model>
train['runtime_cat_min_60'] = train['runtime'].apply(lambda x: 1 if(x <=60)else 0)
 train['runtime_cat_61_80'] = train['runtime'].apply(lambda x: 1 if(x >60)&(x<=80)else 0)
 train['runtime_cat_81_100'] = train['runtime'].apply(lambda x: 1 if(x >80)&(x<=100)else 0)
 train['runtime_cat_101_120'] = train['runtime'].apply(lambda x: 1 if(x >100)&(x<=120)else 0)
 train['runtime_cat_121_140'] = train['runtime'].apply(lambda x: 1 if(x >120)&(x<=140)else 0)
 train['runtime_cat_141_170'] = train['runtime'].apply(lambda x: 1 if(x >140)&(x<=170)else 0)
 train['runtime_cat_171_max'] = train['runtime'].apply(lambda x: 1 if(x >=170)else 0)
 test['runtime_cat_min_60'] = test['runtime'].apply(lambda x: 1 if(x <=60)else 0)
 test['runtime_cat_61_80'] = test['runtime'].apply(lambda x: 1 if(x >60)&(x<=80)else 0)
 test['runtime_cat_81_100'] = test['runtime'].apply(lambda x: 1 if(x >80)&(x<=100)else 0)
 test['runtime_cat_101_120'] = test['runtime'].apply(lambda x: 1 if(x >100)&(x<=120)else 0)
 test['runtime_cat_121_140'] = test['runtime'].apply(lambda x: 1 if(x >120)&(x<=140)else 0)
 test['runtime_cat_141_170'] = test['runtime'].apply(lambda x: 1 if(x >140)&(x<=170)else 0)
 test['runtime_cat_171_max'] = test['runtime'].apply(lambda x: 1 if(x >=170)else 0 )
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {'boosting_type': 'dart',
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36, 'reg_alpha': 0.76,
'reg_lambda': 0.43,
'subsample_for_bin': 40000,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = -1, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores<compute_train_metric>
train['budget'] = np.log1p(train.budget)
 test['budget'] = np.log1p(test.budget )
%%capture --no-display
predictions, gbm_fi = model_gbm(train_set, train_labels, test_set, test_ids, return_preds=True )<save_to_csv>
train['popularity'] = np.log1p(train.popularity)
 test['popularity'] = np.log1p(test.popularity )
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids, return_preds=False)
 submission.to_csv('gbm_baseline.csv' )<split>
train['film_that_has_homepage'] = train['homepage'].isnull().apply(lambda x: 0 if x==True else 1 ).copy()
test['film_that_has_homepage'] = test['homepage'].isnull().apply(lambda x: 0 if x==True else 1 ).copy()
%%capture --no-display
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels,
test_selected, test_ids )<create_dataframe>
lang = train['original_language']
lang_more_17_samples = [x[0] for x in Counter(pd.DataFrame(lang ).stack() ).most_common(17)]
for col in lang_more_17_samples :
train[col] = train['original_language'].apply(lambda x: 1 if x == col else 0)
 for col in lang_more_17_samples :
test[col] = test['original_language'].apply(lambda x: 1 if x == col else 0)
model_results = model_results.append(pd.DataFrame({'model': ["GBM", "GBM_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )<compute_train_metric>
google_train_path = '.. /input/google-result/google_result_train.csv'
google_test_path = '.. /input/google-result/google_result_test.csv'
google_train = pd.read_csv(google_train_path)
 google_test = pd.read_csv(google_test_path)
 train['google_result'] = google_train['result']
test['google_result'] = google_test['result']
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids,
nfolds=10, return_preds=False )<save_to_csv>
train['google_result']=np.log1p(train.google_result)
 test['google_result']=np.log1p(test.google_result )
submission.to_csv('gbm_10fold.csv', index = False )<compute_train_metric>
train['film_belongs_to_collection'] = train['belongs_to_collection'].apply(lambda x: 0 if x == {} else 1)
 test['film_belongs_to_collection'] = test['belongs_to_collection'].apply(lambda x: 0 if x == {} else 1 )
%%capture
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids,
nfolds=10 )<save_to_csv>
t = train[['id','revenue', 'title', 'genres']].copy()
t['genres'] = [[y['name'] for y in x] for x in t['genres']]
genres = t['genres'].sum()
ctr = Counter(genres)
 df_genres = pd.DataFrame.from_dict(ctr, orient='index' ).reset_index().rename(columns={'index':'genre', 0:'count'})
 df_genres=df_genres.sort_values('count', ascending=False)
 df_genres = df_genres[df_genres['count'] > 1]
df_genres = df_genres.nlargest(20, 'count')
 genres = list(df_genres['genre'])
 dic={}
for a in genres:
mask = t.genres.apply(lambda x: a in x)
 dic[a] = t[mask]['revenue'].mean()
t = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'genre'})
 t['color'] = Category20c[len(t)]
hover1 = HoverTool(tooltips = [ 
('Genre','@genre'), 
('Genre mean revenue','@mean_revenue')
 ])
 p = figure(x_range=t.genre, plot_width=1400,plot_height=400, toolbar_location=None, title="Mean revenue per genre", tools=[hover1])
 p.vbar(x='genre', top='mean_revenue', width=0.9, source=t, legend='genre',
line_color='white', fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p)
submission.to_csv('gmb_10fold_selected.csv', index = False )<create_dataframe>
train['genres_names'] = [[y['name'] for y in x] for x in train['genres']]
genres_list=['Action', 'Adventure', 'Science Fiction', 'Family', 'Fantasy','Animation']
for a in genres_list :
train['genre_'+a]=train['genres_names'].apply(lambda x: 1 if a in x else 0)
 train = train.drop(['genres_names'], axis=1)
 test['genres_names'] = [[y['name'] for y in x] for x in test['genres']]
for a in genres_list :
test['genre_'+a]=test['genres_names'].apply(lambda x: 1 if a in x else 0)
 test = test.drop(['genres_names'], axis=1 )
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )<filter>
def date_features(df):
df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'] = df['release_year']
df.loc[(df['release_year'] <= 18)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 18)&(df['release_year'] < 100), "release_year"] += 1900
df['release_date'] = pd.to_datetime(df['release_date'])
 df['release_month'] = df['release_date'].dt.month
df['release_quarter'] = df['release_date'].dt.quarter
df.drop(columns=['release_date'], inplace=True)
 return df
train=date_features(train)
 test=date_features(test )
print(f"There are {gbm_fi_selected[gbm_fi_selected['importance'] == 0].shape[0]} features with no importance." )<import_modules>
for col in range(1,12):
train['month'+str(col)] = train['release_month'].apply(lambda x: 1 if x == col else 0)
 for col in range(1,12):
test['month'+str(col)] = test['release_month'].apply(lambda x: 1 if x == col else 0)
 for col in range(1,4):
train['quarter'+str(col)] = train['release_quarter'].apply(lambda x: 1 if x == col else 0)
 for col in range(1,4):
test['quarter'+str(col)] = test['release_quarter'].apply(lambda x: 1 if x == col else 0)
from hyperopt import hp, tpe, Trials, fmin, STATUS_OK
from hyperopt.pyll.stochastic import sample<import_modules>
import csv
import ast
from timeit import default_timer as timer<train_on_grid>
train = train.drop(['release_month', 'release_quarter'], axis=1)
 test = test.drop(['release_month', 'release_quarter'], axis=1 )
def objective(hyperparameters, nfolds=5):

 global ITERATION
ITERATION += 1
subsample = hyperparameters['boosting_type'].get('subsample', 1.0)
 subsample_freq = hyperparameters['boosting_type'].get('subsample_freq', 0)
 boosting_type = hyperparameters['boosting_type']['boosting_type']
if boosting_type == 'dart':
hyperparameters['drop_rate'] = hyperparameters['boosting_type']['drop_rate']
hyperparameters['subsample'] = subsample
hyperparameters['subsample_freq'] = subsample_freq
hyperparameters['boosting_type'] = boosting_type
if not hyperparameters['limit_max_depth']:
hyperparameters['max_depth'] = -1
for parameter_name in ['max_depth', 'num_leaves', 'subsample_for_bin',
'min_child_samples', 'subsample_freq']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 if 'n_estimators' in hyperparameters:
del hyperparameters['n_estimators']
strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 features = np.array(train_selected)
 labels = np.array(train_labels ).reshape(( -1))
valid_scores = []
best_estimators = []
run_times = []
model = lgb.LGBMClassifier(**hyperparameters, class_weight = 'balanced',
n_jobs=-1, metric = 'None',
n_estimators=10000)
 for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
start = timer()
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 400)
 end = timer()
valid_scores.append(model.best_score_['valid']['macro_f1'])
 best_estimators.append(model.best_iteration_)
 run_times.append(end - start)
 score = np.mean(valid_scores)
 score_std = np.std(valid_scores)
 loss = 1 - score
run_time = np.mean(run_times)
 run_time_std = np.std(run_times)
 estimators = int(np.mean(best_estimators))
hyperparameters['n_estimators'] = estimators
of_connection = open(OUT_FILE, 'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss, hyperparameters, ITERATION, run_time, score, score_std])
 of_connection.close()
if ITERATION % PROGRESS == 0:
display(f'Iteration: {ITERATION}, Current Score: {round(score, 4)}.')
 return {'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION,
'time': run_time, 'time_std': run_time_std, 'status': STATUS_OK,
'score': score, 'score_std': score_std}<set_options>
df_names = df_names[df_names['count'] > 11]
names_list = list(df_names['actor'])
 train['cast_names']=[[y['name'] for y in x] for x in train['cast']]
train['cast_names'] = train['cast_names'].apply(lambda x: x[:3])
 dic={}
for a in names_list:
mask = train['cast_names'].apply(lambda x: a in x)
 dic[a] = train[mask]['revenue'].mean()
actors_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'actor'})
 names_list = list(actors_mean_revenue.nlargest(40, 'mean_revenue')['actor'])
 train['actors_mean_revenue'] = train['cast_names'].apply(lambda x: actors_mean_revenue[actors_mean_revenue['actor'].isin(x)].mean() ['mean_revenue'])
 train['actors_mean_revenue'].fillna(0,inplace=True)
 train['total_top_actors_revenue']=train['cast_names'].apply(lambda x: sum([1 for i in x if i in names_list]))
train = train.drop(['cast_names'], axis=1)
 test['cast_names']=[[y['name'] for y in x] for x in test['cast']]
test['cast_names'] = test['cast_names'].apply(lambda x: x[:3])
 test['actors_mean_revenue'] = test['cast_names'].apply(lambda x: actors_mean_revenue[actors_mean_revenue['actor'].isin(x)].mean() ['mean_revenue'])
 test['actors_mean_revenue'].fillna(0,inplace=True)
 test['total_top_actors_revenue']=test['cast_names'].apply(lambda x: sum([1 for i in x if i in names_list]))
test = test.drop(['cast_names'], axis=1)
space = {
'boosting_type': hp.choice('boosting_type',
[{'boosting_type': 'gbdt',
'subsample': hp.uniform('gdbt_subsample', 0.5, 1),
'subsample_freq': hp.quniform('gbdt_subsample_freq', 1, 10, 1)},
{'boosting_type': 'dart',
'subsample': hp.uniform('dart_subsample', 0.5, 1),
'subsample_freq': hp.quniform('dart_subsample_freq', 1, 10, 1),
'drop_rate': hp.uniform('dart_drop_rate', 0.1, 0.5)},
{'boosting_type': 'goss',
'subsample': 1.0,
'subsample_freq': 0}]),
'limit_max_depth': hp.choice('limit_max_depth', [True, False]),
'max_depth': hp.quniform('max_depth', 1, 40, 1),
'num_leaves': hp.quniform('num_leaves', 3, 50, 1),
'learning_rate': hp.loguniform('learning_rate',
np.log(0.025),
np.log(0.25)) ,
'subsample_for_bin': hp.quniform('subsample_for_bin', 2000, 100000, 2000),
'min_child_samples': hp.quniform('min_child_samples', 5, 80, 5),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.5, 1.0)
 }<find_best_params>
t = train[['id','revenue', 'title', 'cast']].copy()
t['cast'] = [[y['name'] for y in x] for x in t['cast']]
t['cast'] = t['cast'].apply(lambda x: x[:3])
 names = t['cast'].sum()
ctr = Counter(names)
 names=[n for n in ctr if ctr[n] > 0]
unique_names = pd.Series(names ).unique()
dic={}
for a in unique_names:
mask = t.cast.apply(lambda x: a in x)
 dic[a] = t[mask]['revenue'].mean()
actors_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'actor'})
 t = actors_mean_revenue.nlargest(20, 'mean_revenue')
 t['color'] = Category20c[20]
hover1 = HoverTool(tooltips = [ 
('Actor','@actor'), 
('Revenue','@mean_revenue')
 ])
 p = figure(x_range=t.actor, plot_width=1400,plot_height=400, toolbar_location=None, title="20 actors with highest mean revenue", tools=[hover1])
 p.vbar(x='actor', top='mean_revenue', width=0.9, source=t, legend='actor',
line_color='white', fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p)
algo = tpe.suggest<save_to_csv>
t = train[['id','popularity', 'title', 'cast']].copy()
t['popularity'] = np.expm1(t['popularity'])
 t['cast'] = [[y['name'] for y in x] for x in t['cast']]
t['cast'] = t['cast'].apply(lambda x: x[:3])
 names = t['cast'].sum()
ctr = Counter(names)
 names=[n for n in ctr if ctr[n] > 0]
unique_names = pd.Series(names ).unique()
dic={}
for a in unique_names:
mask = t.cast.apply(lambda x: a in x)
 dic[a] = t[mask]['popularity'].mean()
t = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'actor'})
 t = t.nlargest(20, 'mean_revenue')
 t['color'] = Category20c[20]
hover1 = HoverTool(tooltips = [ 
('Actor','@actor'), 
('Revenue','@mean_revenue')
 ])
 p = figure(x_range=t.actor, plot_width=1400,plot_height=400, toolbar_location=None, title="20 actors with highest mean popularity", tools=[hover1])
 p.vbar(x='actor', top='mean_revenue', width=0.9, source=t, legend='actor',
line_color='white', fill_color='color')
 p.xgrid.grid_line_color = None
p.legend.orientation = "horizontal"
p.legend.location = "top_center"
show(p )
trials = Trials()
OUT_FILE = 'optimization.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 MAX_EVALS = 100
PROGRESS = 10
N_FOLDS = 5
ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score', 'std']
writer.writerow(headers)
 of_connection.close()<train_model>
df_has_nominated_actor['nominations'] = df_has_nominated_actor['nominations'].apply(lambda x: 1 if x == 'True' else 0)
 train['has_nominated_actor'] = df_has_nominated_actor['nominations']
test['cast_ids']=[[y['id'] for y in x] for x in test['cast']]
test['cast_ids'] = test['cast_ids'].apply(lambda x: x[:3])
 test['has_nominated_actor'] = test['cast_ids'].apply(lambda x: 0 if(nominations[nominations['id'].isin(x)]['nominations'] != 0 ).any() else 1)
 test = test.drop(['cast_ids'], axis=1)
 train['cast_ids']=[[y['id'] for y in x] for x in train['cast']]
train['cast_ids'] = train['cast_ids'].apply(lambda x: x[:4])
 train['nominated_actors'] = train['cast_ids'].apply(lambda x:(nominations[nominations['id'].isin(x)]['nominations'] != 0 ).sum())
 test['cast_ids']=[[y['id'] for y in x] for x in test['cast']]
test['cast_ids'] = test['cast_ids'].apply(lambda x: x[:4])
 test['nominated_actors'] = test['cast_ids'].apply(lambda x:(nominations[nominations['id'].isin(x)]['nominations'] != 0 ).sum())
%%capture --no-display
display("Running Optimization for {} Trials.".format(MAX_EVALS))
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials,
max_evals = MAX_EVALS )<save_to_csv>
df_names = df_names[df_names['count'] > 10]
names_list = list(df_names['actor'])
 train['crew_names'] = [[y['name'] for y in x if y['department']=='Directing'] for x in train['crew'] ]
train['crew_names'] = train['crew_names'].apply(lambda x: x[:3])
 dic={}
for a in names_list:
mask = train['crew_names'].apply(lambda x: a in x)
 dic[a] = train[mask]['revenue'].mean()
directors_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'director'})
 names_list = list(directors_mean_revenue.nlargest(40, 'mean_revenue')['director'])
 for a in names_list :
train['director_'+a]=train['crew_names'].apply(lambda x: 1 if a in x else 0)
 train = train.drop(['crew_names'], axis=1)
 test['crew_names'] = [[y['name'] for y in x if y['department']=='Directing'] for x in test['crew'] ]
test['crew_names'] = test['crew_names'].apply(lambda x: x[:3])
 for a in names_list :
test['director_'+a]=test['crew_names'].apply(lambda x: 1 if a in x else 0)
 test = test.drop(['crew_names'], axis=1)
with open('trials.json', 'w')as f:
f.write(json.dumps(str(trials)) )<find_best_params>
df_names = df_names[df_names['count'] > 9]
names_list = list(df_names['actor'])
 train['production_companies'] = [[y['name'] for y in x] for x in train['production_companies'] ]
train['production_companies'] = train['production_companies'].apply(lambda x: x[:3])
 dic={}
for a in names_list:
mask = train['production_companies'].apply(lambda x: a in x)
 dic[a] = train[mask]['revenue'].mean()
companies_mean_revenue = pd.DataFrame.from_dict(dic, orient='index', columns=['mean_revenue'] ).reset_index().rename(columns={'index':'company'})
 names_list = list(companies_mean_revenue.nlargest(20, 'mean_revenue')['company'])
 for a in names_list :
train['production_'+a]=train['production_companies'].apply(lambda x: 1 if a in x else 0)
 train = train.drop(['production_companies'], axis=1)
 test['production_companies'] = [[y['name'] for y in x] for x in test['production_companies'] ]
test['production_companies'] = test['production_companies'].apply(lambda x: x[:3])
 for a in names_list :
test['production_'+a]=test['production_companies'].apply(lambda x: 1 if a in x else 0)
 test = test.drop(['production_companies'], axis=1 )
best_hyp = ast.literal_eval(results.loc[0, 'hyperparameters'])
 best_hyp<create_dataframe>
y = np.log1p(train.revenue )
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_selected, train_labels,
test_selected, test_ids,
nfolds = 10, return_preds=False)
 model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold_SEL"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True ).sort_values('cv_mean', ascending = False )<create_dataframe>
X = train.drop(['id','runtime', 'release_day'], axis=1)
 test_X = test.drop(['id','runtime', 'release_day'], axis=1 ).select_dtypes(exclude=['object'])
 train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1,test_size=0.33)
 train_X=train_X.drop(['revenue'], axis=1 ).select_dtypes(exclude=['object'])
 X=X.drop(['revenue'], axis=1 ).select_dtypes(exclude=['object'])
 val_X_revenue=val_X.pop('revenue')
 val_X_title=val_X.pop('title')
 val_X=val_X.select_dtypes(exclude=['object'])
 xgb_model = XGBRegressor(learning_rate=0.05,
n_estimators=10000,max_depth=4)
 xgb_model.fit(train_X, train_y, early_stopping_rounds=100,
eval_set=[(val_X, val_y)], eval_metric = 'rmse')
 xbg_val_predictions=xgb_model.predict(val_X)
<save_to_csv><EOS> 
xgb_model_full = XGBRegressor(n_estimators=145, learning_rate=0.05,max_depth=4)
 xgb_model_full.fit(X, y)
 test_preds=xgb_model_full.predict(test_X)
 output = pd.DataFrame({'id': test.id,
'revenue': np.expm1(test_preds)})
 output.to_csv('submission.csv', index=False )