Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<save_to_csv>
%matplotlib inline
warnings.filterwarnings("ignore")
sub = pd.DataFrame(preds)
 pd.concat([testset["id"],sub], axis=1 ).rename(columns = {0: 'type'} ).to_csv("submission_neural_net.csv", index=False )<import_modules>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.index = train['id']
test.index = test['id']
print(train.shape, test.shape)
 train.head()
import pandas as pd
import tensorflow as tf
from tensorflow.contrib import learn<load_from_csv>
train['release_date'] = pd.to_datetime(train['release_date'], dayfirst=True)
 train['release_year'] = train['release_date'].dt.year
train['release_month'] = train['release_date'].dt.month
train['release_day'] = train['release_date'].dt.day
train['release_dayofweek'] = train['release_date'].dt.dayofweek
train['release_quarter'] = train['release_date'].dt.quarter
df = pd.read_csv(".. /input/train.csv", index_col='id' )<prepare_x_and_y>
train['release_year'].groupby(train['release_year'][train['release_year']>2010] ).count()
features = ["bone_length","rotting_flesh","hair_length","has_soul"]
X = df[features]
y = df["type"]
X['hair_soul'] = X['hair_length'] * X['has_soul']
X['hair_bone'] = X['hair_length'] * X['bone_length']
X['hair_soul_bone'] = X['hair_length'] * X['has_soul'] * X['bone_length']
features+=['hair_soul','hair_bone','hair_soul_bone']
X.head()<categorify>
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else eval(x))
return df
letype = LE()
y = letype.fit_transform(y )<split>
temp = copy.deepcopy(train)
 temp = text_to_dict(temp )
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.0 )<train_model>
temp['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0 ).value_counts() [:10]
x=tf.contrib.learn.infer_real_valued_columns_from_input(X_train)
 tf_clf_dnn = learn.DNNClassifier(hidden_units=[16], n_classes=3, feature_columns=x, activation_fn=tf.sigmoid)
 tf_clf_dnn.fit(X_train, y_train,max_steps=5000)
 print(acc_s(y_train,tf_clf_dnn.predict(X_train)) )<save_to_csv>
temp['genres'].apply(lambda x: x[0]['name'] if x != {} else 0 ).value_counts() [:10]
test_df = pd.read_csv(".. /input/test.csv",index_col='id')
 features = ["bone_length","rotting_flesh","hair_length","has_soul"]
X = test_df[features]
X['hair_soul'] = X['hair_length'] * X['has_soul']
X['hair_bone'] = X['hair_length'] * X['bone_length']
X['hair_soul_bone'] = X['hair_length'] * X['has_soul'] * X['bone_length']
features+=['hair_soul','hair_bone','hair_soul_bone']
pred = tf_clf_dnn.predict(X)
 pred = letype.inverse_transform(pred)
 submission_df = pd.DataFrame({'type':pred})
 submission_df['id'] = test_df.index
submission_df = submission_df[['id', 'type']].set_index('id')
 submission_df.to_csv('tf_pred.csv')
 submission_df.head()<import_modules>
runtime_mean = train['runtime'].mean()
budget_mean =train['budget'].mean()
def prepare(df):
df['runtime'][(df['runtime'].isnull())|(df['runtime'] == 0)] = runtime_mean
df['budget'][df['budget']==0] = budget_mean
df['hasHP']=0
df.loc[df['homepage'].isnull() == False, 'hasHP'] = 1
df['hasTL']=0
df.loc[df['tagline'].isnull() == False, 'hasTL'] = 1
df['hasCollection']=0
df.loc[df['belongs_to_collection'].isnull() == False, 'hasCollection'] = 1
df['isEng']=0
df.loc[df['original_language'] == 'en', 'isEng'] = 1
df['isReleased'] = 1
df.loc[ df['status'] != "Released" ,"isReleased"] = 0
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['release_date'] = pd.to_datetime(df['release_date'], dayfirst=True)
 df['release_year'] = df['release_date'].dt.year
df['release_month'] = df['release_date'].dt.month
df['release_day'] = df['release_date'].dt.day
df['release_year'][df['release_year']>2018] = df['release_year']-100
df['release_dayofweek'] = df['release_date'].dt.dayofweek
df['release_quarter'] = df['release_date'].dt.quarter
df['budget'] = np.log1p(df['budget'])
 df = pd.get_dummies(df, columns=['original_language'], prefix='original_language')
 df = text_to_dict(df)
 df['_collection_name'] = df['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['_collection_name'].fillna('')))
 df['_collection_name'] = le.transform(df['_collection_name'].fillna('' ).astype(str))
df['_genres_name'] = df['genres'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['_genres_name'].fillna('')))
 df['_genres_name'] = le.transform(df['_genres_name'].fillna('' ).astype(str))
df['_num_Keywords'] = df['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 df['_num_cast'] = df['cast'].apply(lambda x: len(x)if x != {} else 0)
 df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['crew'].apply(lambda x : len(x))
df = df.drop(['imdb_id','poster_path','status','homepage','tagline','belongs_to_collection',\
'genres', 'production_companies','production_countries', 'spoken_languages',\
'Keywords', 'cast', 'crew','overview','title','original_title','release_date'],axis=1)
 return df
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.utils.np_utils import to_categorical
from sklearn.cross_validation import train_test_split
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
all_data = pd.concat([train, test] ).reset_index(drop = True)
 all_data = prepare(all_data)
 train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:]
print(train.shape)
 train.head()
data = pd.read_csv(".. /input/train.csv")
 ids = data.pop("id")
 creature = data.pop("type")
 color = data.pop("color")
 data.head()<normalization>
features = list(train.columns)
 features = [i for i in features if i != 'id' and i != 'revenue']
coords = [[0,1], [0,2], [0,3], [1,2], [1,3], [2,3]]
add_names2 = ["bone_flesh", "bone_hair", "bone_soul", "flesh_hair", "flesh_soul", "hair_soul"]
for i in range(len(coords)) :
data[add_names2[i]] = data.values[:, coords[i][0]] *data.values[:, coords[i][1]]
scal = StandardScaler()
data = pd.DataFrame(scal.fit_transform(data))
data.head()<split>
def score(data, y):
validation_res = pd.DataFrame({"id": data["id"].values,
"transactionrevenue": data["revenue"].values,
"predictedrevenue": np.expm1(y)})
 validation_res = validation_res.groupby("id")["transactionrevenue", "predictedrevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionrevenue"].values),
np.log1p(validation_res["predictedrevenue"].values)) )
X_train, X_test, y_train, y_test = train_test_split(data, creature, test_size=0.33, random_state=42 )<categorify>
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['id'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['id'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['id'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 500, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["revenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["revenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score
creature_encoder = LabelEncoder()
creature_code = creature_encoder.fit(creature)
 y_train_code = creature_encoder.transform(y_train)
 y_test_code = creature_encoder.transform(y_test )<categorify>
Kfolder = KFoldValidation(train )
X_train = X_train.values
X_test = X_test.values
y_train_code = to_categorical(y_train_code)
 y_test_code = to_categorical(y_test_code )<choose_model_class>
lgbmodel = lgb.LGBMRegressor(n_estimators=10000,
objective='regression',
metric='rmse',
max_depth = 5,
num_leaves=30,
min_child_samples=100,
learning_rate=0.01,
boosting = 'gbdt',
min_data_in_leaf= 10,
feature_fraction = 0.9,
bagging_freq = 1,
bagging_fraction = 0.9,
importance_type='gain',
lambda_l1 = 0.2,
bagging_seed=42,
subsample=.8,
colsample_bytree=.9,
use_best_model=True)
 Kfolder.validate(train, test, features , lgbmodel, name="lgbfinal", prepare_stacking=True )
def nn_model(in_dim):
model = Sequential()
model.add(Dense(300, input_dim=in_dim, init='uniform', activation='relu'))
model.add(Dropout(0.6))
model.add(Dense(100, init='uniform', activation='sigmoid'))
model.add(Dropout(0.4))
model.add(Dense(3, init='uniform', activation='softmax'))
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
 return model<train_model>
xgbmodel = xgb.XGBRegressor(max_depth=5,
learning_rate=0.01,
n_estimators=10000,
objective='reg:linear',
gamma=1.45,
seed=42,
silent=True,
subsample=0.8,
colsample_bytree=0.7,
colsample_bylevel=0.5)
 Kfolder.validate(train, test, features, xgbmodel, name="xgbfinal", prepare_stacking=True )
np.random.seed(42)
 model = nn_model(10)
 model.fit(X_train, y_train_code, verbose= 0, nb_epoch= 500)
 model.evaluate(X_train, y_train_code), model.evaluate(X_test, y_test_code)
<train_model>
catmodel = cat.CatBoostRegressor(iterations=10000,
learning_rate=0.01,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.8,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=200,
random_seed=42)
 Kfolder.validate(train, test, features , catmodel, name="catfinal", prepare_stacking=True,
fit_params={"use_best_model": True, "verbose": 100} )
np.random.seed(42)
 model = nn_model(10)
 y = creature_code.transform(creature)
 y = to_categorical(y)
 model.fit(data.values, y, verbose= 0, nb_epoch= 1000)
 model.evaluate(data.values, y )<feature_engineering>
train['Revenue_lgb'] = train["lgbfinal"]
print("RMSE model lgb :" ,score(train, train.Revenue_lgb),)
 train['Revenue_xgb'] = train["xgbfinal"]
print("RMSE model xgb :" ,score(train, train.Revenue_xgb))
train['Revenue_cat'] = train["catfinal"]
print("RMSE model cat :" ,score(train, train.Revenue_cat))
train['Revenue_Dragon'] = 0.4 * train["lgbfinal"] + 0.2 * train["xgbfinal"] + 0.4 * train["catfinal"]
print("RMSE model Dragon :" ,score(train, train.Revenue_Dragon))
<predict_on_test><EOS> 
test['revenue'] = np.expm1(test["lgbfinal"])
 test[['id','revenue']].to_csv('submission_lgb.csv', index=False)
 print(test[['id','revenue']].head())
 test['revenue'] = np.expm1(test["xgbfinal"])
 test[['id','revenue']].to_csv('submission_xgb.csv', index=False)
 print(test[['id','revenue']].head())
 test['revenue'] = np.expm1(test["catfinal"])
 test[['id','revenue']].to_csv('submission_cat.csv', index=False)
 print(test[['id','revenue']].head())
 test['revenue'] = np.expm1(0.4 * test["lgbfinal"]+ 0.4 * test["catfinal"] + 0.2 * test["xgbfinal"])
 test[['id','revenue']].to_csv('submission_Dragon1.csv', index=False)
 print(test[['id','revenue']].head() )