Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<load_from_csv>
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams['figure.figsize'] = 12,8
warnings.filterwarnings('ignore')
 %matplotlib inline
submission9470 = pd.read_csv('/kaggle/input/haveatry/submission_train2_9470.csv' )<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('train dataset size:', train.shape)
 print('test dataset size:', test.shape)
 train.sample(4 )
submission_mix9450 = pd.read_csv('/kaggle/input/haveatry/submission_mix123_9450.csv' )<prepare_output>
def proc_json(string, key):
try:
data = eval(string)
 return ",".join([d[key] for d in data])
 except:
return ''
def proc_json_len(string):
try:
data = eval(string)
 return len(data)
 except:
return 0
train.belongs_to_collection = train.belongs_to_collection.apply(lambda x: proc_json(x, 'name'))
test.belongs_to_collection = test.belongs_to_collection.apply(lambda x: proc_json(x, 'name'))
train.genres = train.genres.apply(lambda x: proc_json(x, 'name'))
test.genres = test.genres.apply(lambda x: proc_json(x, 'name'))
train.production_companies = train.production_companies.apply(lambda x: proc_json(x, 'name'))
test.production_companies = test.production_companies.apply(lambda x: proc_json(x, 'name'))
train.production_countries = train.production_countries.apply(lambda x: proc_json(x, 'iso_3166_1'))
test.production_countries = test.production_countries.apply(lambda x: proc_json(x, 'iso_3166_1'))
train.spoken_languages = train.spoken_languages.apply(lambda x: proc_json(x, 'iso_639_1'))
test.spoken_languages = test.spoken_languages.apply(lambda x: proc_json(x, 'iso_639_1'))
train.Keywords = train.Keywords.apply(lambda x: proc_json(x, 'name'))
test.Keywords = test.Keywords.apply(lambda x: proc_json(x, 'name'))
train.cast = train.cast.apply(proc_json_len)
 test.cast = test.cast.apply(proc_json_len)
 train.crew = train.crew.apply(proc_json_len)
 test.crew = test.crew.apply(proc_json_len )
for i in range(len(submission1['toxic'])) :
sub['toxic'][i]=submission9460['toxic'][i]*0.15+submission9487['toxic'][i]*0.85 
<save_to_csv>
train.isnull().sum()
sub.to_csv('submission.csv', index=False )<set_options>
train.isnull().sum()
<define_variables>
columns = ['original_title', 'title', 'overview', 'tagline']
for col in columns:
new_col = col + '_len'
train[new_col] = train[col].apply(lambda x: 0 if x is np.nan else len(x))
test[new_col] = test[col].apply(lambda x: 0 if x is np.nan else len(x))
columns.extend(['homepage', 'imdb_id', 'poster_path'])
 train.drop(columns, axis=1, inplace=True)
 test.drop(columns, axis=1, inplace=True )
interativeEnvironment=False
localEnvironment=False
IN_COLAB=False
try:
except ImportError as e:
try:
IN_COLAB = True
except:
IN_COLAB = False
localEnvironment=True
os.environ['XLA_USE_BF16'] = "1"
localEnvironment
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 300)
 pd.set_option('display.width', None)
 %matplotlib inline 
<set_options>
print('-'*30, '
', train.isnull().sum())
 print('-'*30, '
', test.isnull().sum() )
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive() )<import_modules>
train.runtime = train.runtime.fillna(0)
 test.runtime = test.runtime.fillna(0 )
<define_variables>
test.loc[test.status.isnull() , 'status'] = 'Released'
tf_env="tensorflow"<define_variables>
test.loc[test.release_date.isnull() , 'release_date'] = test.release_date.mode() [0]
fit_verbocity=1
model=tf_env
nrows=None
interativeEnvironment=is_interactive()
interativeEnvironment=False
if localEnvironment:
warnings.warn("Nrows limited")
 pass
elif interativeEnvironment:
pass
else:
fit_verbocity=2
pooling_mode_cls_token="CLS_TOKEN"
pooling_mode_fc="FC"
pooling_mode=pooling_mode_fc
fc_dims=[]
trainable_transformer=False
use_dann=False
dann_lambda=-1
use_parcor=False
parcor_lambda=1
use_augmented_data=False
use_lowercase_data=False
use_finetuning=True
use_validation_during_pretraining=True
use_translated_data=False
use_pretraining=False
load_model=".. /input/ver-22-parcor-output/21_06_2020_05_03_28_zero_shot_pro.h5"
use_english_validation=False
use_label_rounding=True
use_label_filtering=True<feature_engineering>
def expand_release_date(df):
df.release_date = pd.to_datetime(df.release_date)
 df['release_year'] = df.release_date.dt.year
df['release_month'] = df.release_date.dt.month
df['release_day'] = df.release_date.dt.dayofweek
df['release_quarter'] = df.release_date.dt.quarter
return df
train = expand_release_date(train )
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True 
<normalization>
def expand_release_date(df):
df.release_date = pd.to_datetime(df.release_date)
 df['release_year'] = df.release_date.dt.year
df['release_year'] = df.release_year.apply(lambda x: x-100 if x > 2020 else x)
 df['release_month'] = df.release_date.dt.month
df['release_day'] = df.release_date.dt.dayofweek
df['release_quarter'] = df.release_date.dt.quarter
return df
train = expand_release_date(train)
 test = expand_release_date(test )
@tf.custom_gradient
def grad_reverse(x):
y = tf.identity(x)
 def custom_grad(dy):
return tf.math.scalar_mul(-1,dy)
 return y, custom_grad
class GradReverse(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
def call(self, x):
return grad_reverse(x )<set_options>
genres = []
for idx, val in train.genres.iteritems() :
gen_list = val.split(',')
 for gen in gen_list:
if gen == '':
continue
if gen not in genres:
genres.append(gen)
 genre_column_names = []
for gen in genres:
col_name = 'genre_' + gen.replace(' ', '_')
 train[col_name] = train.genres.str.contains(gen ).astype('uint8')
 test[col_name] = test.genres.str.contains(gen ).astype('uint8')
 genre_column_names.append(col_name)
 train.sample(5 )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
train['genre_count'] = train.genres.apply(lambda x: len(x.split(',')))
 test['genre_count'] = test.genres.apply(lambda x: len(x.split(',')) )
AUTO = tf.data.experimental.AUTOTUNE
if not localEnvironment and not IN_COLAB:
GCS_DS_PATH = KaggleDatasets().get_gcs_path("jigsaw-multilingual-toxic-comment-classification")
 NEXAMPLESPEREPOCH=240000
EPOCHS = 10
if strategy.num_replicas_in_sync==1:
BATCH_SIZE = 4 * strategy.num_replicas_in_sync
else:
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
if localEnvironment:
MODEL='jplu/tf-xlm-roberta-base'
if IN_COLAB:
MODEL='jplu/tf-xlm-roberta-base'
print("BATCH_SIZE: ", BATCH_SIZE )<define_variables>
train['country_us'] = train.production_countries.str.contains('US' ).astype('uint8')
 train['country_count'] = train.production_countries.apply(lambda x: len(x.split(',')))
 train['company_count'] = train.production_companies.apply(lambda x: len(x.split(',')))
 test['country_us'] = test.production_countries.str.contains('US' ).astype('uint8')
 test['country_count'] = test.production_countries.apply(lambda x: len(x.split(',')))
 test['company_count'] = test.production_companies.apply(lambda x: len(x.split(',')) )
if IN_COLAB:
project_id="global-sun-279412"<set_options>
train['slc'] = train.spoken_languages.apply(lambda x: len(x.split(',')))
 train['orig_lang_code'] = train.original_language.astype('category' ).cat.codes
test['slc'] = test.spoken_languages.apply(lambda x: len(x.split(',')))
 test['orig_lang_code'] = test.original_language.astype('category' ).cat.codes
!gcloud config set project {project_id}<define_variables>
train['total_staff'] = train.cast + train.crew
test['total_staff'] = test.cast + test.crew
bucket_name="jigsaw-tfrecords"<set_options>
manually_selected_features = ['budget', 'popularity', 'total_staff', 'runtime', 'original_title_len',
'release_year', 'release_day', 'genre_Adventure', 'genre_Action', 'genre_Fantasy',
'genre_Drama', 'genre_Family', 'genre_Animation', 'genre_Science_Fiction',
'genre_count', 'company_count', 'country_us']
if IN_COLAB:
auth.authenticate_user()<define_variables>
pca = PCA(n_components=3)
 fit = pca.fit(X)
 print(pca.explained_variance_ratio_ )
if localEnvironment:
root =".. "
elif IN_COLAB:
root='gs://jigsaw-tfrecords/'
else:
root="/kaggle"
root<define_variables>
model = ExtraTreesRegressor()
model.fit(X, Y)
 feature_df = pd.DataFrame({'feature': features, 'importance': model.feature_importances_})
 feature_df.sort_values(by='importance', ascending=False, inplace=True)
 features4 = feature_df.feature[:20]
features4
validation_fold_index=4
validation_fold_index<feature_engineering>
automatically_selected_features = list(set(features1)& set(features2)& set(features3)& set(features4))
automatically_selected_features
def validationFoldRemoveFilter(validation_fold_index):
def _filter(features):
return tf.math.not_equal(features["ShardIndex"],validation_fold_index)
 return _filter<feature_engineering>
features = train.select_dtypes(include=['int64', 'float64', 'uint8', 'int8'] ).columns.tolist()
features.remove('id')
 features.remove('revenue')
 target = 'revenue'
corr_features = features[:]
corr_features.append(target)
 corrs = abs(train[corr_features].corr() ['revenue'] ).sort_values(ascending=False)
 corr_selected_features = corrs[:20].index.tolist()
corr_selected_features.remove('revenue')
 corr_selected_features
def validationFoldOnlyFilter(validation_fold_index):
def _filter(features):
return tf.math.equal(features["ShardIndex"],validation_fold_index)
 return _filter<define_variables>
train['revenue'] = np.log1p(train['revenue'])
 train['budget'] = np.log1p(train['budget'])
 test['budget'] = np.log1p(test['budget'])
 X = train[corr_selected_features]
y = train['revenue']
if IN_COLAB:
TF_DS_PATH=root+"jigsaw-official"
elif not localEnvironment:
TF_DS_PATH = KaggleDatasets().get_gcs_path("jigsawaugmented")+"/tfrecords"
else:
TF_DS_PATH = ".. /tfrecords"
TF_DS_PATH<define_variables>
def select_model(X, Y):
best_models = {}
models = [
{
'name': 'LinearRegression',
'estimator': LinearRegression() ,
'hyperparameters': {},
},
{
'name': 'KNeighbors',
'estimator': KNeighborsRegressor() ,
'hyperparameters':{
'n_neighbors': range(3,50,3),
'weights': ['distance', 'uniform'],
'algorithm': ['auto'],
'leaf_size': list(range(10,51,10)) ,
}
},
{
'name': 'GradientBoostingRegressor',
'estimator': GradientBoostingRegressor() ,
'hyperparameters':{
'n_estimators': range(70, 150, 10),
'criterion': ['friedman_mse'],
'max_depth': [3, 5, 7, 9],
'max_features': ['log2', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [3, 5, 7]
}
},
{
'name': 'XGBoost',
'estimator': XGBRegressor() ,
'hyperparameters':{
'booster': ['gbtree', 'gblinear', 'dart'],
'max_depth': range(10, 51, 10),
'n_estimators': [200],
'nthread': [4],
'min_child_weight': range(1, 8, 2),
'learning_rate': [.05,.1,.15],
}
},
{
'name': 'Light GBM',
'estimator': LGBMRegressor() ,
'hyperparameters':{
'max_depth': range(20, 85, 15),
'learning_rate': [.01,.05,.1],
'num_leaves': [300, 600, 900, 1200],
'n_estimators': [200]
}
},
{
'name': 'Cat Boost',
'estimator': CatBoostRegressor() ,
'hyperparameters':{
'depth': [4, 7, 10],
'learning_rate': [.03,.06,.1,.15],
'l2_leaf_reg': [1, 4, 7, 9],
'iterations': [300]
}
}
]
for model in tqdm(models):
', '-'*20, '
', model['name'])
 start = time.perf_counter()
grid = GridSearchCV(model['estimator'], param_grid=model['hyperparameters'], cv=5, scoring = "explained_variance", verbose=False, n_jobs=-1)
 grid.fit(X, Y)
 best_models[model['name']] = {'score': grid.best_score_, 'params': grid.best_params_}
run = time.perf_counter() - start
{} --{:.2f} seconds.'.format(str(grid.best_score_), str(grid.best_params_), run))
return best_models
file="en1_train_non_toxic_PropercaseSentences_0_4.tfrecord" 
("Sentences" not in file)or("LowercaseSentences" in file )<define_variables>
def get_accuracy(features):
X, y = train[features], train['revenue']
model = GradientBoostingRegressor(criterion='mse', max_depth=6, max_features='sqrt',
min_samples_leaf=4, min_samples_split=9, n_estimators=110, loss='huber')
 result = cross_validate(model, X, y, cv=10, scoring="explained_variance", verbose=False, n_jobs=-1)
 return np.mean(result['test_score'] )
data_files=tf.io.gfile.glob(TF_DS_PATH + "/*.tfrecord")
 if use_augmented_data:
pass
elif use_lowercase_data:
data_files=list(filter(lambda file:("Sentences" not in file)or("LowercaseSentences" in file),data_files))
else:
data_files=list(filter(lambda file:"Sentences" not in file,data_files))
len(data_files),data_files<define_variables>
all_features = train.select_dtypes(include=['int64', 'float64', 'uint8', 'int8'] ).columns.tolist()
all_features.remove('id')
 all_features.remove('revenue')
 best_features = None
best_accuracy = 0
feature_candidates = [all_features, manually_selected_features, automatically_selected_features, corr_selected_features]
for flist in feature_candidates:
acc = get_accuracy(flist)
 if acc > best_accuracy:
best_accuracy = acc
best_features = flist
print('The best accuracy is', best_accuracy)
 best_features
toxic_en_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non_toxic" not in file and "valid" not in file and "sharded" not in file, data_files))
toxic_en_files<define_variables>
model = GradientBoostingRegressor(criterion='mse', max_depth=6, max_features='sqrt',
min_samples_leaf=4, min_samples_split=9, n_estimators=110, loss='huber')
 model.fit(train[best_features], train['revenue'])
 predict = model.predict(test[best_features])
non_toxic_en_files=list(filter(lambda file:"nen" not in file and "non_toxic" in file and "valid" not in file and "sharded" not in file, data_files))
non_toxic_en_files<define_variables>
submit = pd.DataFrame({'id': test.id, 'revenue':np.expm1(predict)})
 submit.to_csv('submission.csv', index=False )
non_toxic_nen_files=list(filter(lambda file:"non_toxic" in file and "nen1" in file, data_files))
non_toxic_nen_files<define_variables>
def fill_budget(in_train, in_test):
def fb_proc_json_len(string):
try:
data = eval(string)
 return len(data)
 except:
return 0
test = in_test.copy()
test['revenue'] = np.nan
total = pd.concat([in_train, test], axis=0)
 fb_features = ['popularity', 'runtime', 'cast', 'crew', 'genre_count',
'company_count', 'release_year', 'release_month']
fb_target = 'budget'
train = total[total.budget > 0]
pred = total[total.budget == 0]
model = GradientBoostingRegressor(criterion='mse', learning_rate=0.15, loss='huber', max_depth=6,
max_features='sqrt', min_samples_leaf=4, min_samples_split=9, n_estimators=110)
 model.fit(train[fb_features], train[fb_target])
 predict = model.predict(pred[fb_features])
 pred[fb_target] = predict
out_train = in_train.copy()
out_test = in_test.copy()
for idx, row in out_train[out_train.budget == 0].iterrows() :
mid = row.id
out_train.loc[out_train.id == mid, 'budget'] = pred.loc[pred.id == mid, 'budget']
for idx, row in out_test[out_test.budget == 0].iterrows() :
mid = row.id
out_test.loc[out_test.id == mid, 'budget'] = pred.loc[pred.id == mid, 'budget']
return(out_train, out_test )
toxic_nen_files=list(filter(lambda file:"non_toxic" not in file and "nen1" in file, data_files))
toxic_nen_files<define_variables>
candidates = train.select_dtypes(include=['int64', 'float64', 'uint8', 'int8'] ).columns.tolist()
candidates.remove('id')
 candidates.remove('revenue')
 X, y = train[candidates], train['revenue']
model = GradientBoostingRegressor(criterion='mse', max_depth=6, max_features='sqrt',
min_samples_leaf=4, min_samples_split=9, n_estimators=110, loss='huber')
 sfs = SFS(estimator=model,
k_features=(3, 9),
forward=True,
floating=False,
scoring='neg_mean_squared_error',
cv=5)
 sfs.fit(X, y, custom_feature_names=candidates)
 print('best combination(ACC: %.3f): %s
' %(sfs.k_score_, sfs.k_feature_idx_))
print('all subsets:
', sfs.subsets_)
 fig = plot_sfs(sfs.get_metric_dict() , kind='std_err')
 plt.title('Sequential Forward Selection(w.StdErr)')
 plt.grid()
plt.show()
unknown_nen_files=list(filter(lambda file: "test" not in file and "nen2" in file, data_files))
unknown_nen_files<define_variables>
sfs_fatures = ['budget', 'popularity', 'runtime', 'tagline_len', 'release_year', 'genre_Drama', 'genre_Family', 'genre_Thriller', 'genre_Crime']
model = GradientBoostingRegressor(criterion='friedman_mse', max_depth=5, max_features='sqrt',
min_samples_leaf=4, min_samples_split=3, n_estimators=110, random_state=1)
 model.fit(train[sfs_fatures], train['revenue'])
 predict = model.predict(test[sfs_fatures])
 submit = pd.DataFrame({'id': test.id, 'revenue':np.expm1(predict)})
 submit.to_csv('submission.csv', index=False)