Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<import_modules>
%matplotlib inline
plt.style.use('seaborn')
 pd.set_option('max_columns', None)
 warnings.filterwarnings('ignore' )
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_absolute_error<normalization>
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv' )
scaler = StandardScaler()
scaler.fit(x_train)
 x_train_scaled = scaler.transform(x_train)
 y_train_flatten = y_train.values.ravel()<prepare_output>
train.isnull().sum().sort_values(ascending=False )
x_train_dataframe = pd.DataFrame(x_train_scaled)
 x_train_dataframe.head(10 )<split>
test.isnull().sum().sort_values(ascending=False )
x_train1, x_test1, y_train1, y_test1 = train_test_split(x_train, y_train, test_size = 0.2, random_state = 101 )<normalization>
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000
test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30
test.loc[test['release_date'].isnull() == True, 'release_date'] = '01/01/98'
scaler = StandardScaler()
scaler.fit(x_train1)
 x_train_scaled1 = scaler.transform(x_train1)
 x_test_scaled1 = scaler.transform(x_test1)
 y_train_flatten1 = y_train1.values.ravel()
y_test_flatten1 = y_test1.values.ravel()<import_modules>
text_cols = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for col in text_cols:
df[col] = df[col].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )
import keras
from keras.layers import Dense
from keras.layers import BatchNormalization
from keras.layers import Activation
from keras.layers import Dropout
from keras.models import Sequential
from keras import optimizers<choose_model_class>
for i, e in enumerate(train['belongs_to_collection'][:5]):
print(i, e )
kernel_init = 'normal'
def Model(input_dim, activation, classes):
model = Sequential()
model.add(Dense(1024, kernel_initializer = kernel_init, input_dim = input_dim))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(512, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(64, kernel_initializer = kernel_init))
model.add(Activation(activation))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(classes, kernel_initializer = kernel_init))
model.add(Activation('linear'))
return model<choose_model_class>
train['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
input_dim = x_train.shape[1]
activation = 'tanh'
classes = 1
model = Model(input_dim = input_dim, activation = activation, classes = classes)
<choose_model_class>
for i, e in enumerate(train['genres'][:5]):
print(i, e )
optim = optimizers.Adam(lr = 0.001, decay = 0.001 / 32)
 model.compile(loss = 'mean_absolute_error', optimizer = optim, metrics = ['mean_absolute_error'] )<train_model>
genres_per = train['genres'].apply(lambda x: [i['name'] for i in x] if x != {} else [])
 genres_per
h = model.fit(x_train_scaled1, y_train_flatten1, epochs = 100, batch_size = 32, verbose = 1 )<compute_train_metric>
countries_num = train['production_countries'].apply(lambda x: len(x)if x != {} else 0 ).value_counts()
countries_num
prediction_score = []
mae_scores = []
predictions = model.predict(x_test_scaled1, verbose = 1, batch_size = 32)
 prediction_score.append(predictions)
 mae = mean_absolute_error(predictions, y_test_flatten1)
 mae_scores.append(mae )<compute_test_metric>
for i, e in enumerate(train['spoken_languages'][:5]):
print(i, e )
print(np.mean(mae_scores))<compute_test_metric>
train['original_language'].value_counts() [:10]
predictions = np.hstack([p.reshape(-1,1)for p in prediction_score])
 predictions = np.mean(predictions, axis = 1)
 print(mean_absolute_error(predictions, y_test1))<load_from_csv>
train['homepage'].isna().value_counts()
def calc_change_rate(x):
change =(np.diff(x)/ x[:-1] ).values
change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
 submission = pd.read_csv('.. /input/sample_submission.csv', index_col = 'seg_id')
 X_test = pd.DataFrame(columns = x_train.columns, dtype = np.float64, index = submission.index)
 for id in tqdm_notebook(X_test.index):
seg = pd.read_csv('.. /input/test/' + id + '.csv')
 x = pd.Series(seg['acoustic_data'].values)
 X_test.loc[id, 'AVERAGE'] = x.mean()
X_test.loc[id, 'STD'] = x.std()
X_test.loc[id, 'MAX'] = x.max()
X_test.loc[id, 'MIN'] = x.min()
X_test.loc[id, 'SUM'] = x.sum()
X_test.loc[id, 'MEAN_CHANGE_ABS'] = np.mean(np.diff(x))
X_test.loc[id, 'MEAN_CHANGE_RATE'] = calc_change_rate(x)
 X_test.loc[id, 'MAX_TO_MIN'] = x.max() / np.abs(x.min())
 X_test.loc[id, 'MAX_TO_MIN_DIFF'] = x.max() - np.abs(x.min())
 X_test.loc[id, 'COUNT_BIG'] = len(x[np.abs(x)> 500])
 X_test.loc[id, 'AVERAGE_FIRST_10000'] = x[:10000].mean()
X_test.loc[id, 'AVERAGE_LAST_10000'] = x[-10000:].mean()
X_test.loc[id, 'AVERAGE_FIRST_50000'] = x[:50000].mean()
X_test.loc[id, 'AVERAGE_LAST_50000'] = x[-50000:].mean()
X_test.loc[id, 'STD_FIRST_10000'] = x[:10000].std()
X_test.loc[id, 'STD_LAST_10000'] = x[-10000:].std()
X_test.loc[id, 'STD_FIRST_50000'] = x[:50000].std()
X_test.loc[id, 'STD_LAST_50000'] = x[-50000:].std()
X_test.loc[id, 'ABS_AVERAGE'] = np.abs(x ).mean()
X_test.loc[id, 'ABS_STD'] = np.abs(x ).std()
X_test.loc[id, 'ABS_MAX'] = np.abs(x ).max()
X_test.loc[id, 'ABS_MIN'] = np.abs(x ).min()
X_test.loc[id, '10Q'] = np.percentile(x, 0.10)
 X_test.loc[id, '25Q'] = np.percentile(x, 0.25)
 X_test.loc[id, '50Q'] = np.percentile(x, 0.50)
 X_test.loc[id, '75Q'] = np.percentile(x, 0.75)
 X_test.loc[id, '90Q'] = np.percentile(x, 0.90)
 X_test.loc[id, 'ABS_1Q'] = np.percentile(x, np.abs(0.01))
X_test.loc[id, 'ABS_5Q'] = np.percentile(x, np.abs(0.05))
X_test.loc[id, 'ABS_30Q'] = np.percentile(x, np.abs(0.30))
X_test.loc[id, 'ABS_60Q'] = np.percentile(x, np.abs(0.60))
X_test.loc[id, 'ABS_95Q'] = np.percentile(x, np.abs(0.95))
X_test.loc[id, 'ABS_99Q'] = np.percentile(x, np.abs(0.99))
X_test.loc[id, 'KURTOSIS'] = x.kurtosis()
X_test.loc[id, 'SKEW'] = x.skew()
X_test.loc[id, 'MEDIAN'] = x.median()
X_test.loc[id, 'HILBERT_MEAN'] = np.abs(hilbert(x)).mean()
X_test.loc[id, 'HANN_WINDOW_MEAN'] =(convolve(x, hann(150), mode = 'same')/ sum(hann(150)) ).mean()
for windows in [10, 100, 1000]:
x_roll_std = x.rolling(windows ).std().dropna().values
x_roll_mean = x.rolling(windows ).mean().dropna().values
X_test.loc[id, 'AVG_ROLL_STD' + str(windows)] = x_roll_std.mean()
X_test.loc[id, 'STD_ROLL_STD' + str(windows)] = x_roll_std.std()
X_test.loc[id, 'MAX_ROLL_STD' + str(windows)] = x_roll_std.max()
X_test.loc[id, 'MIN_ROLL_STD' + str(windows)] = x_roll_std.min()
X_test.loc[id, '1Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X_test.loc[id, '5Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X_test.loc[id, '95Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X_test.loc[id, '99Q_ROLL_STD' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X_test.loc[id, 'AV_CHANGE_ABS_ROLL_STD' + str(windows)] = np.mean(np.diff(x_roll_std))
X_test.loc[id, 'ABS_MAX_ROLL_STD' + str(windows)] = np.abs(x_roll_std ).max()
X_test.loc[id, 'AVG_ROLL_MEAN' + str(windows)] = x_roll_mean.mean()
X_test.loc[id, 'STD_ROLL_MEAN' + str(windows)] = x_roll_mean.std()
X_test.loc[id, 'MAX_ROLL_MEAN' + str(windows)] = x_roll_mean.max()
X_test.loc[id, 'MIN_ROLL_MEAN' + str(windows)] = x_roll_mean.min()
X_test.loc[id, '1Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X_test.loc[id, '5Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X_test.loc[id, '95Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X_test.loc[id, '99Q_ROLL_MEAN' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X_test.loc[id, 'AV_CHANGE_ABS_ROLL_MEAN' + str(windows)] = np.mean(np.diff(x_roll_mean))
X_test.loc[id, 'ABS_MAX_ROLL_MEAN' + str(windows)] = np.abs(x_roll_mean ).max()<count_missing_values>
train['homepage'].value_counts().sort_values(ascending=False)[:5]
X_test.isnull().sum()<normalization>
train['status'].value_counts()
X_test_scaled = scaler.transform(X_test)
 print(X_test_scaled.shape )<train_on_grid>
train['original_language'].value_counts().head(15 )
input_dim = x_train.shape[1]
activation = 'tanh'
classes = 1
history = dict()
prediction_scores = dict()
for i in np.arange(0, 15):
optim = optimizers.Adam(lr = 0.001, decay = 0.001 / 32)
 ensemble_model = Model(input_dim = input_dim, activation = activation, classes = classes)
 ensemble_model.compile(loss = 'mean_absolute_error', optimizer = optim, metrics = ['mean_absolute_error'])
 print('TRAINING MODEL NO : {}'.format(i))
H = ensemble_model.fit(x_train_scaled, y_train_flatten,
batch_size = 32,
epochs = 200,
verbose = 1)
 history[i] = H
predictions = ensemble_model.predict(X_test_scaled, verbose = 1, batch_size = 32)
 prediction_scores[i] = predictions<prepare_output>
def fix_date(x):
year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
train['release_date'] = train['release_date'].apply(lambda x: fix_date(x))
test['release_date'] = test['release_date'].apply(lambda x: fix_date(x))
def date_features(df):
df['release_date'] = pd.to_datetime(df['release_date'])
 df['release_year'] = df['release_date'].dt.year
df['release_month'] = df['release_date'].dt.month
df['release_day'] = df['release_date'].dt.day
df['release_dayofweek'] = df['release_date'].dt.dayofweek
df.drop(columns=['release_date'], inplace=True)
 return df
train = date_features(train)
 test = date_features(test )
prediction1 = np.hstack([p.reshape(-1,1)for p in prediction_scores.values() ])
 prediction1 = np.mean(prediction1, axis = 1)
 print(prediction1.shape )<save_to_csv>
def prepare_data(df):
df['has_collection'] = df['belongs_to_collection'].apply(lambda x: len(x)if x != {} else 0)
 df['has_homepage'] = 1 - df['homepage'].isna()
df['is_released'] =(df['status'] == 'Released')*1
df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_letter_count'] = df['title'].str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_letter_count'] = df['overview'].str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['has_tagline'] = 1 - df['tagline'].isna()
df['tagline_letter_count'] = df['tagline'].str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['genders_0_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df['log_budget'] = np.log1p(df['budget'])
 df['log_popularity'] = np.log1p(df['popularity'])
 df['ratio_budget_runtime'] =(df['log_budget'] / df['runtime'] ).replace([np.inf,-np.inf,np.nan],0)
 df['ratio_budget_popularity'] = df['log_budget'] / df['log_popularity']
df['ratio_budget_year'] = df['log_budget'] / df['release_year']
df['budget_to_mean_year'] = df['log_budget'] / df.groupby('release_year')['log_budget'].transform('mean')
 df['ratio_popularity_year'] = df['log_popularity'] / df['release_year']
df['popularity_to_mean_year'] = df['log_popularity'] / df.groupby('release_year')['log_popularity'].transform('mean')
 df['ratio_runtime_year'] = df['runtime'] / df['release_year']
df['runtime_to_mean_year'] = df['runtime'] / df.groupby('release_year')['runtime'].transform('mean')
 df.fillna(value=0.0, inplace = True)
 return df
submission['time_to_failure'] = prediction1
submission.to_csv('submission.csv' )<set_options>
train = prepare_data(train)
 test = prepare_data(test )
h2o.init(max_mem_size='16G' )<load_from_csv>
def dist_processing(train, test, col, key):
value_list = train[col].apply(lambda x: [i[key] for i in x] if x != {} else [])
 value_count = Counter([i for j in value_list for i in j] ).most_common()
top_list = [m[0] for m in value_count if m[1] > 10]
train['num_' + col] = train[col].apply(lambda x: len(x)if x != {} else 0)
 train['all_' + col] = train[col].apply(lambda x: ' '.join(sorted([i[key] for i in x])) if x != {} else '')
 test['num_' + col] = test[col].apply(lambda x: len(x)if x != {} else 0)
 test['all_' + col] = test[col].apply(lambda x: ' '.join(sorted([i[key] for i in x])) if x != {} else '')
 for p in top_list:
train[col + '_' + p] = train['all_' + col].apply(lambda x: 1 if p in x else 0)
 test[col + '_' + p] = test['all_' + col].apply(lambda x: 1 if p in x else 0)
 train = train.drop([col, 'all_' + col], axis=1)
 test = test.drop([col, 'all_' + col], axis=1)
 return train, test
text_cols = ['genres', 'production_companies', 'production_countries', 'Keywords', 'cast', 'crew']
for col in text_cols:
train, test = dist_processing(train, test, col, 'name')
 train, test = dist_processing(train, test, 'spoken_languages', 'iso_639_1')
 value_count = Counter(train['original_language'] ).most_common()
top_list = [m[0] for m in value_count if m[1] > 10]
for p in top_list:
train['original_language_' + p] = train['original_language'].apply(lambda x: 1 if p in x else 0)
 test['original_language_' + p] = test['original_language'].apply(lambda x: 1 if p in x else 0 )
train = h2o.import_file(".. /input/andrews-features-only/X_tr.csv")
 test = h2o.import_file(".. /input/andrews-features-only/X_test.csv")
 y_train = h2o.import_file(".. /input/andrews-features-only/y_tr.csv" )<feature_engineering>
train = train.drop(['id', 'belongs_to_collection', 'homepage', 'status', 'original_language', 'original_title', 'title', 'overview',
'tagline', 'imdb_id', 'poster_path', 'budget', 'popularity', 'revenue', 'is_title_different'], axis=1)
 test = test.drop(['id', 'belongs_to_collection', 'homepage', 'status', 'original_language', 'original_title', 'title', 'overview',
'tagline', 'imdb_id', 'poster_path', 'budget', 'popularity'], axis=1 )
train['time_to_failure'] = y_train['time_to_failure']<prepare_x_and_y>
newnames = []
for col in train.columns.values:
encoded_string = col.encode("ascii", "ignore")
 decode_string = encoded_string.decode()
newnames.append(decode_string)
 train.columns = newnames
newnames = []
for col in test.columns.values:
encoded_string = col.encode("ascii", "ignore")
 decode_string = encoded_string.decode()
newnames.append(decode_string)
 test.columns = newnames
x = test.columns
y = 'time_to_failure'<train_model>
X = train.drop(['log_revenue'], axis=1)
 y = train['log_revenue']
X_col = list(X.columns)
 test = test.loc[:,X_col]
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=22 )
aml = H2OAutoML(max_models=50000, seed=55, max_runtime_secs=31000,nfolds=0,sort_metric="MAE",stopping_metric="MAE")
 aml.train(x=x, y=y, training_frame=train )<find_best_params>
params = {'objective': 'regression',
'num_leaves': 30,
'min_data_in_leaf': 20,
'max_depth': 5,
'learning_rate': 0.005,
'boosting': "gbdt",
'feature_fraction': 0.9,
'bagging_freq': 1,
'bagging_fraction': 0.9,
'bagging_seed': 22,
'metric': 'rmse',
'lambda_l1': 0.2,
'verbosity': -1}
model = lgb.LGBMRegressor(**params, n_estimators = 100000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)],
eval_metric='rmse',
verbose=False,
early_stopping_rounds=200 )
aml.leader<predict_on_test>
def new_interactions(df):
df['ratio_budget_year2'] = df['log_budget'].fillna(0)/(df['release_year']*df['release_year'])
 df['ratio_year_budget'] = df['release_year'] / df['log_budget']
df['popularity_to_budget_to_mean_year'] = df['popularity_to_mean_year'] / df['budget_to_mean_year']
df['genders_2_crew_to_budget_to_mean_year'] = df['genders_2_crew'] / df['budget_to_mean_year']
df['num_crew_to_ratio_popularity_year'] = df['num_crew'] / df['ratio_popularity_year']
df['popularity_runtime_to_budget'] = df['log_popularity'] / df['ratio_budget_runtime']
df['mean_budget_to_year'] = df['budget_to_mean_year'] / df['release_year']
df['budget_to_runtime_to_year'] = df['ratio_budget_runtime'] / df['release_year']
df['ratio_year_popularity'] = df['release_year'] / df['log_popularity']
df.fillna(value=0.0, inplace = True)
 return df
X = new_interactions(X)
 test = new_interactions(test)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=22 )
preds = aml.predict(test )<save_to_csv>
""
sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.as_data_frame().values.flatten()
sample_submission.to_csv('h2o_submission_1.csv', index=False )<load_from_csv>
""
warnings.filterwarnings("ignore")
 df_train = pd.read_csv('.. /input/porto-seguro-safe-driver-prediction/train.csv')
 df_test = pd.read_csv('.. /input/porto-seguro-safe-driver-prediction/test.csv')
 print('train data size', df_train.shape)
 print('test data size', df_test.shape)
 display(df_train.head())
 display(df_test.head() )<prepare_x_and_y>
""
X_train = df_train.drop(['id', 'target'], axis=1)
 y_train = df_train['target']<compute_test_metric>
def models(X_train, X_valid, y_train, y_valid, test, model_type, params, plot_feature_importance=False):
pred = np.zeros(test.shape[0])
 val_pred = np.zeros(X_valid.shape[0])
 scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds):
print('Fold', fold_n, 'started at', time.ctime())
 X_train = X.loc[train_index,:]
y_train = y[train_index]
X_valid = X.loc[valid_index,:]
y_valid = y[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = 100000, nthread = 4, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)],
eval_metric='rmse',
verbose=False,
early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid, num_iteration = model.best_iteration_)
 y_pred = model.predict(test, num_iteration = model.best_iteration_)
 if model_type == 'xgb':
model = xgb.XGBRegressor(**params, n_estimators = 10000)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)],
eval_metric='rmse',
verbose=False,
early_stopping_rounds=200)
 y_pred_valid = model.predict(X_valid, ntree_limit = model.best_ntree_limit)
 y_pred = model.predict(test, ntree_limit = model.best_ntree_limit)
 if model_type == 'cat':
model = CatBoostRegressor(**params, iterations=20000, eval_metric = 'RMSE')
 model.fit(X_train, y_train,
eval_set=(X_valid, y_valid),
use_best_model=True,
verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(test)
 scores.append(mean_squared_error(y_valid, y_pred_valid)** 0.5)
 pred += y_pred
fold_importance = pd.DataFrame()
fold_importance['feature'] = X.columns
fold_importance['importance'] = model.feature_importances_
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 pred /= n_fold
print('Mean RMSE: {0:.5f}, std: {1:.5f}.'.format(np.mean(scores), np.std(scores)))
 feature_importance['importance'] /= n_fold
if plot_feature_importance:
cols = feature_importance[['feature', 'importance']].groupby('feature' ).mean().sort_values(
 by='importance', ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12))
sns.barplot(x='importance', y='feature', data=best_features.sort_values(by='importance', ascending=False))
plt.title('Important Features(avg over folds)')
 return pred
def metrics(true, preds):
accuracy = accuracy_score(true, preds)
 recall = recall_score(true, preds)
 f1score = f1_score(true, preds)
 cf = confusion_matrix(true, preds)
 print('accuracy: {}, recall: {}, f1-score: {}'.format(accuracy, recall, f1score))
print('Confusion matrix', cf)
 def gini(true, preds):
res = 2* roc_auc_score(true, preds)- 1
return res
X_tr, X_te, y_tr, y_te = train_test_split(X_train, y_train, random_state=42, test_size=0.2)
 def test_clfs(clfs):
for clf in clfs:
print('------------------------------------------')
 start = time()
clf = clf(random_state=42)
 clf.fit(X_tr, y_tr)
 y1_pred = clf.predict(X_te)
 y1_pred_prob = clf.predict_proba(X_te)
 print(str(clf), '
results:')
 metrics(y_te, y1_pred)
 print('gini score', gini(y_te, y1_pred_prob[:, 1]))
end = time()
print('Processing time', end-start,'s')
 classifiers = [LGBMClassifier, XGBClassifier, RandomForestClassifier]
test_clfs(classifiers )<train_model>
lgb_params = {'objective': 'regression',
'num_leaves': 40,
'min_data_in_leaf': 10,
'max_depth': 7,
'learning_rate': 0.01,
'boosting': "gbdt",
'feature_fraction': 0.9,
'bagging_freq': 1,
'bagging_fraction': 0.7,
'bagging_seed': 22,
'metric': 'rmse',
'lambda_l1': 0.2,
'verbosity': -1}
start = time.time()
lgb_pred = models(X_train, X_valid, y_train, y_valid, test, 'lgb', lgb_params, True)
 end = time.time()
print('Time elapsed: {0:.2f} m'.format(( end-start)/60))
sm_methods = [SMOTE,RandomUnderSampler,RandomOverSampler]
clf = LGBMClassifier(random_state=42)
 for sm in sm_methods:
print('------------------------------------------')
 sm = sm(random_state=42)
 start = time()
X_train_resampled, y_train_resampled = sm.fit_sample(X_tr, y_tr)
 clf.fit(X_train_resampled, y_train_resampled)
 y_pred = clf.predict(X_te)
 y_pred_prob = clf.predict_proba(X_te)
 print(str(sm), 'results:')
 print('training resampled shape', X_train_resampled.shape)
 print('value counts in each class', y_train_resampled.value_counts())
 metrics(y_te, y_pred)
 gini_score = gini(y_te, y_pred_prob[:, 1])
 print('Normalized gini score', gini_score)
 end = time()
print('Processing time', end-start,'s' )<find_best_model_class>
xgb_params = {'objective': 'reg:squarederror',
'eta': 0.01,
'max_depth': 5,
'min_child_weight': 3,
'subsample': 0.8,
'colsample_bytree': 0.7,
'colsample_bylevel': 0.5,
'eval_metric': 'rmse',
'seed': 22}
start = time.time()
xgb_pred = models(X_train, X_valid, y_train, y_valid, test, 'xgb', xgb_params, True)
 end = time.time()
print('Time elapsed: {0:.2f} m'.format(( end-start)/60))
cv = StratifiedKFold(n_splits=5)
 scores_gini = []
for train_index, validation_index in cv.split(X_train, y_train):
train, val = X_train.iloc[train_index], X_train.iloc[validation_index]
target_train, target_val = y_train.iloc[train_index], y_train.iloc[validation_index]
clf = LGBMClassifier(random_state=42)
 clf.fit(train, target_train)
 y_val_pred = clf.predict(val)
 y_val_pred_prob = clf.predict_proba(val)
 gini_score = gini(target_val, y_val_pred_prob[:, 1])
 print('gini score:', gini_score)
 scores_gini.append(gini_score)
 print('mean gini_score: {}'.format(np.mean(scores_gini)))
<prepare_x_and_y>
cat_params = {'learning_rate': 0.01,
'depth': 7,
'colsample_bylevel': 0.7,
'bagging_temperature': 0.2,
'l2_leaf_reg': 1,
'random_seed': 22,
'allow_writing_files': False,
'early_stopping_rounds': 200
}
start = time.time()
cat_pred = models(X_train, X_valid, y_train, y_valid, test, 'cat', cat_params, True)
 end = time.time()
print('Time elapsed: {0:.2f} m'.format(( end-start)/60))
df_train = pd.read_csv('.. /input/porto-seguro-safe-driver-prediction/train.csv')
 X_train = df_train.drop(['id', 'target'], axis=1)
 y_train = df_train['target']
features = df_train.columns.tolist()
cat_features = [c for c in features if 'cat' in c]
X_train = pd.get_dummies(X_train, columns=cat_features, prefix_sep='_', drop_first=True)
<find_best_model_class>
test_pred = lgb_pred*0.4 + xgb_pred*0.2 + cat_pred*0.4
scores_gini = []
for train_index, validation_index in cv.split(X_train, y_train):
train, val = X_train.iloc[train_index], X_train.iloc[validation_index]
target_train, target_val = y_train.iloc[train_index], y_train.iloc[validation_index]
clf = LGBMClassifier(random_state=42, n_jobs=-1)
 clf.set_params(**best_params)
 clf.fit(train, target_train)
 y_val_pred = clf.predict(val)
 y_val_pred_prob = clf.predict_proba(val)
 gini_score = gini(target_val, y_val_pred_prob[:, 1])
 print('gini score:', gini_score)
 scores_gini.append(gini_score)
 print('mean gini_score: {}'.format(np.mean(scores_gini)) )<prepare_output>
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 df_sub = pd.DataFrame()
df_sub['id'] = sub['id']
df_sub['revenue'] = np.expm1(test_pred)
 df_sub.to_csv("submission.csv", index=False )