Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<load_pretrained>
warnings.filterwarnings("ignore" )
zip_images()<import_modules>
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train.info()
import numpy as np
import pandas as pd
import keras
from keras.layers import Input, Dense, Reshape, Flatten, Dropout,Concatenate
from keras.backend import random_normal,ones_like,zeros_like,mean
from keras.backend import get_session
from keras.layers import BatchNormalization, Activation, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.models import Sequential, Model
from keras.optimizers import Adam
from keras.layers import concatenate
from keras.initializers import TruncatedNormal
from keras.callbacks import LearningRateScheduler, EarlyStopping, History
from PIL import Image
import warnings
import os
import time
from glob import glob
import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np, pandas as pd, os
import xml.etree.ElementTree as ET
import matplotlib.pyplot as plt, zipfile
from PIL import Image
from glob import glob<define_variables>
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test.info()
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<define_variables>
train.isna().sum()
IMG_SIZE = Input(( 12288,))
IMG_SIZE_2 = Input(( 10000,))
NOISE_SIZE = 10000
BATCH_SIZE = 64<choose_model_class>
test.isna().sum()
def discriminatorFunction() :
input_layer = Dense(12288, activation='sigmoid' )(IMG_SIZE_2)
 input_layer = Reshape(( 2,12288,1))(concatenate([IMG_SIZE,input_layer]))
discriminator = Conv2D(filters = 1, kernel_size=[2,1],use_bias=False, name = 'layer_1' )(input_layer)
 out = Flatten()(discriminator)
 return out<choose_model_class>
train[['release_month','release_day','release_year']]=train['release_date'].str.split('/',expand=True ).replace(np.nan, -1 ).astype(int)
 train.loc[(train['release_year'] <= 19)&(train['release_year'] < 100), "release_year"] += 2000
train.loc[(train['release_year'] > 19)&(train['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(train['release_date'])
 train['release_dayofweek'] = releaseDate.dt.dayofweek
train['release_quarter'] = releaseDate.dt.quarter
print("Discriminator")
 model = discriminatorFunction()
model_discriminator = Model([IMG_SIZE,IMG_SIZE_2], model)
 model_discriminator.get_layer('layer_1' ).trainable = False
model_discriminator.get_layer('layer_1' ).set_weights([np.array([[[[-1.0 ]]],[[[1.0]]]])])
 model_discriminator.summary()
model_discriminator.compile(optimizer='adam', loss='binary_crossentropy' )<choose_model_class>
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
train = train
train['genres'] = train['genres'].map(lambda x: sorted([d['name'] for d in get_dictionary(x)])).map(lambda x: ','.join(map(str, x)))
 genres = train.genres.str.get_dummies(sep=',')
 train = pd.concat([train, genres], axis=1, sort=False)
 print("Action Genres Movie ", train[train.Action == 1].shape[0])
 print("Adventure Genres Movie ", train[train.Adventure == 1].shape[0])
 print("Animation Genres Movie ", train[train.Animation == 1].shape[0])
 print("Comedy Genres Movie ", train[train.Comedy == 1].shape[0])
 print("Crime Genres Movie ", train[train.Crime == 1].shape[0])
 print("Documentary Genres Movie ", train[train.Documentary == 1].shape[0])
 print("Drama Genres Movie ", train[train.Drama == 1].shape[0])
 print("Family Genres Movie ", train[train.Family == 1].shape[0])
 print("Fantasy Genres Movie ", train[train.Fantasy == 1].shape[0])
 print("Foreign Genres Movie ", train[train.Foreign == 1].shape[0])
 print("History Genres Movie ", train[train.History == 1].shape[0])
 print("Music Genres Movie ", train[train.Music == 1].shape[0])
 print("Mystery Genres Movie ", train[train.Mystery == 1].shape[0])
 print("Romance Genres Movie ", train[train.Romance == 1].shape[0])
 print("Science Fiction Genres Movie ", train[train['Science Fiction'] == 1].shape[0])
 print("TV Movie Genres Movie ", train[train['TV Movie'] == 1].shape[0])
 print("Thriller Genres Movie ", train[train.Thriller == 1].shape[0])
 print("War Genres Movie ", train[train.War == 1].shape[0])
 print("Western Genres Movie ", train[train.Western == 1].shape[0] )
def GeneratorFunction(noise_shape=(NOISE_SIZE,)) :
input_layer = Input(noise_shape)
 generated = Dense(12288, activation='linear' )(input_layer)
 model = Model(inputs=input_layer,outputs = [generated,Reshape(( 10000,))(input_layer)])
 model.summary()
return model<choose_model_class>
test = test
test['genres'] = test['genres'].map(lambda x: sorted([d['name'] for d in get_dictionary(x)])).map(lambda x: ','.join(map(str, x)))
 genres = test.genres.str.get_dummies(sep=',')
 test = pd.concat([test, genres], axis=1, sort=False)
 print("Action Genres Movie ", test[test.Action == 1].shape[0])
 print("Adventure Genres Movie ", test[test.Adventure == 1].shape[0])
 print("Animation Genres Movie ", test[test.Animation == 1].shape[0])
 print("Comedy Genres Movie ", test[test.Comedy == 1].shape[0])
 print("Crime Genres Movie ", test[test.Crime == 1].shape[0])
 print("Documentary Genres Movie ", test[test.Documentary == 1].shape[0])
 print("Drama Genres Movie ", test[test.Drama == 1].shape[0])
 print("Family Genres Movie ", test[test.Family == 1].shape[0])
 print("Fantasy Genres Movie ", test[test.Fantasy == 1].shape[0])
 print("Foreign Genres Movie ", test[test.Foreign == 1].shape[0])
 print("History Genres Movie ", test[test.History == 1].shape[0])
 print("Music Genres Movie ", test[test.Music == 1].shape[0])
 print("Mystery Genres Movie ", test[test.Mystery == 1].shape[0])
 print("Romance Genres Movie ", test[test.Romance == 1].shape[0])
 print("Science Fiction Genres Movie ", test[test['Science Fiction'] == 1].shape[0])
 print("TV Movie Genres Movie 0")
 print("Thriller Genres Movie ", test[test.Thriller == 1].shape[0])
 print("War Genres Movie ", test[test.War == 1].shape[0])
 print("Western Genres Movie ", test[test.Western == 1].shape[0] )
print("Generator")
 model_generator = GeneratorFunction(noise_shape=(NOISE_SIZE,))<train_model>
train['status'].value_counts()
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
LR_Scheduler = LearningRateScheduler(lambda x: lr)
 h = model_discriminator.fit([zeros,train_X], train_y, epochs = 100,batch_size = BATCH_SIZE, callbacks=[LR_Scheduler], verbose=0)
 print('Epoch',(k+1)*10,'/50 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.533: lr = 0.1<drop_column>
train.loc[train['status'] == "Rumored"][['status','revenue']]
del train_X, train_y, imagesIn<choose_model_class>
test['status'].value_counts()
model_discriminator.trainable = False
z = Input(shape=(NOISE_SIZE,))
img = model_generator(z)
 real = model_discriminator(img)
 gan = Model(z, real)
 gan.get_layer('model_1' ).get_layer('layer_1' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
 print("Model created based on Discriminator and Generator")
 gan.summary()<train_model>
trainAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 testAdditionalFeatures = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train = pd.merge(train, trainAdditionalFeatures, how='left', on=['imdb_id'])
 test = pd.merge(test, testAdditionalFeatures, how='left', on=['imdb_id'] )
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,12288))
Steps_per_epoch = 50
print("Training begins...Total steps per epoch: {}".format(Steps_per_epoch)) 
<train_model>
print("Missing rating in Train set", train['rating'].isna().sum())
 print("Missing total Votes in Train set", train['totalVotes'].isna().sum())
 print("")
 print("Missing rating in Test set", test['rating'].isna().sum())
 print("Missing total Votes in Test set", test['totalVotes'].isna().sum() )
warnings.filterwarnings("ignore")
 lr = 5.
 for step in range(Steps_per_epoch):
LR_Scheduler = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = 1, batch_size=256, callbacks=[LR_Scheduler], verbose=0)
 if(step<10)|(step%5==4):
print("Step: {}/{} [G loss: {:.4f}]".format(
(step+1)*10, Steps_per_epoch*10, h.history['loss'][-1]))
if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.01
if step<10:
plt.figure(figsize=(15,3))
for j in range(5):
zz = np.zeros(( 10000))
zz[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = model_generator.predict(zz.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()<predict_on_test>
train['rating'] = train['rating'].fillna(1.5)
 train['totalVotes'] = train['totalVotes'].fillna(6)
 test['rating'] = test['rating'].fillna(1.5)
 test['totalVotes'] = test['totalVotes'].fillna(6 )
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.999
xx[np.random.randint(10000)] = 0.001
img = model_generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))<import_modules>
def prepare(df):
global json_cols
global train_dict
df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'] = df['release_year']
df.loc[(df['release_year'] <= 19)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 19)&(df['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
rating_na = df.groupby(["release_year","original_language"])['rating'].mean().reset_index()
df[df.rating.isna() ]['rating'] = df.merge(rating_na, how = 'left' ,on = ["release_year","original_language"])
 vote_count_na = df.groupby(["release_year","original_language"])['totalVotes'].mean().reset_index()
df[df.totalVotes.isna() ]['totalVotes'] = df.merge(vote_count_na, how = 'left' ,on = ["release_year","original_language"])
 df['weightedRating'] =(df['rating']*df['totalVotes'] + 6.367 * 1000)/(df['totalVotes'] + 1000)
 df['originalBudget'] = df['budget']
df['inflationBudget'] = df['budget'] + df['budget']*1.8/100*(2018-df['release_year'])
 df['budget'] = np.log1p(df['budget'])
 df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df['_collection_name'] = df['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['_collection_name'].fillna('')))
 df['_collection_name'] = le.transform(df['_collection_name'].fillna('' ).astype(str))
df['_num_Keywords'] = df['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 df['_num_cast'] = df['cast'].apply(lambda x: len(x)if x != {} else 0)
 df['_popularity_mean_year'] = df['popularity'] / df.groupby("release_year")["popularity"].transform('mean')
 df['_budget_runtime_ratio'] = df['budget']/df['runtime']
df['_budget_popularity_ratio'] = df['budget']/df['popularity']
df['_budget_year_ratio'] = df['budget']/(df['release_year']*df['release_year'])
 df['_releaseYear_popularity_ratio'] = df['release_year']/df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity']/df['release_year']
df['_popularity_totalVotes_ratio'] = df['totalVotes']/df['popularity']
df['_rating_popularity_ratio'] = df['rating']/df['popularity']
df['_rating_totalVotes_ratio'] = df['totalVotes']/df['rating']
df['_totalVotes_releaseYear_ratio'] = df['totalVotes']/df['release_year']
df['_budget_rating_ratio'] = df['budget']/df['rating']
df['_runtime_rating_ratio'] = df['runtime']/df['rating']
df['_budget_totalVotes_ratio'] = df['budget']/df['totalVotes']
df['has_homepage'] = 1
df.loc[pd.isnull(df['homepage']),"has_homepage"] = 0
df['isbelongs_to_collectionNA'] = 0
df.loc[pd.isnull(df['belongs_to_collection']),"isbelongs_to_collectionNA"] = 1
df['isTaglineNA'] = 0
df.loc[df['tagline'] == 0 ,"isTaglineNA"] = 1
df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'] == "en" ,"isOriginalLanguageEng"] = 1
df['isTitleDifferent'] = 1
df.loc[ df['original_title'] == df['title'] ,"isTitleDifferent"] = 0
df['isMovieReleased'] = 1
df.loc[ df['status'] != "Released" ,"isMovieReleased"] = 0
df['collection_id'] = df['belongs_to_collection'].apply(lambda x : np.nan if len(x)==0 else x[0]['id'])
 df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['crew'].apply(lambda x : len(x))
df['meanruntimeByYear'] = df.groupby("release_year")["runtime"].aggregate('mean')
 df['meanPopularityByYear'] = df.groupby("release_year")["popularity"].aggregate('mean')
 df['meanBudgetByYear'] = df.groupby("release_year")["budget"].aggregate('mean')
 df['meantotalVotesByYear'] = df.groupby("release_year")["totalVotes"].aggregate('mean')
 df['meanTotalVotesByRating'] = df.groupby("rating")["totalVotes"].aggregate('mean')
 df['medianBudgetByYear'] = df.groupby("release_year")["budget"].aggregate('median')
 for col in ['genres', 'production_countries', 'spoken_languages', 'production_companies'] :
df[col] = df[col].map(lambda x: sorted(list(set([n if n in train_dict[col] else col+'_etc' for n in [d['name'] for d in x]])))).map(lambda x: ','.join(map(str, x)))
 temp = df[col].str.get_dummies(sep=',')
 df = pd.concat([df, temp], axis=1, sort=False)
 df.drop(['genres_etc'], axis = 1, inplace = True)
 df = df.drop(['id', 'revenue','belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline', 'collection_id'
],axis=1)
 df.fillna(value=0.0, inplace = True)
 return df
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000
test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 test.loc[test['id'] == 6733,'budget'] = 5000000
test.loc[test['id'] == 3889,'budget'] = 15000000
test.loc[test['id'] == 6683,'budget'] = 50000000
test.loc[test['id'] == 5704,'budget'] = 4300000
test.loc[test['id'] == 6109,'budget'] = 281756
test.loc[test['id'] == 7242,'budget'] = 10000000
test.loc[test['id'] == 7021,'budget'] = 17540562
test.loc[test['id'] == 5591,'budget'] = 4000000
test.loc[test['id'] == 4282,'budget'] = 20000000
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30
test['revenue'] = np.nan
train = pd.merge(train, pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 test = pd.merge(test, pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv'), how='left', on=['imdb_id'])
 additionalTrainData = pd.read_csv('.. /input/tmdb-box-office-prediction-more-training-data/additionalTrainData.csv')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].astype('str')
 additionalTrainData['release_date'] = additionalTrainData['release_date'].str.replace('-', '/')
 train = pd.concat([train, additionalTrainData])
 print(train.columns)
 print(train.shape)
 train['revenue'] = np.log1p(train['revenue'])
 y = train['revenue'].values
json_cols = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in tqdm(json_cols + ['belongs_to_collection']):
train[col] = train[col].apply(lambda x : get_dictionary(x))
test[col] = test[col].apply(lambda x : get_dictionary(x))
def get_json_dict(df):
global json_cols
result = dict()
for e_col in json_cols :
d = dict()
rows = df[e_col].values
for row in rows :
if row is None : continue
for i in row :
if i['name'] not in d :
d[i['name']] = 0
d[i['name']] += 1
result[e_col] = d
return result
train_dict = get_json_dict(train)
 test_dict = get_json_dict(test)
 for col in json_cols :
remove = []
train_id = set(list(train_dict[col].keys()))
test_id = set(list(test_dict[col].keys()))
remove += list(train_id - test_id)+ list(test_id - train_id)
 for i in train_id.union(test_id)- set(remove):
if train_dict[col][i] < 10 or i == '' :
remove += [i]
for i in remove :
if i in train_dict[col] :
del train_dict[col][i]
if i in test_dict[col] :
del test_dict[col][i]
all_data = prepare(pd.concat([train, test] ).reset_index(drop = True))
train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:]
batch_size = 32<define_variables>
random_seed = 2019
k = 10
fold = list(KFold(k, shuffle = True, random_state = random_seed ).split(train))
np.random.seed(random_seed )
start = time()<set_options>
def xgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 6,
'subsample': 0.6,
'colsample_bytree': 0.7,
'eval_metric': 'rmse',
'seed': random_seed,
'silent': True,
}
record = dict()
model = xgb.train(params
, xgb.DMatrix(trn_x, trn_y)
 , 100000
, [(xgb.DMatrix(trn_x, trn_y), 'train'),(xgb.DMatrix(val_x, val_y), 'valid')]
, verbose_eval=verbose
, early_stopping_rounds=500
, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid']['rmse'][best_idx], 'importance':[i for k, i in model.get_score().items() ]}
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_pretrained>
def lgb_model(trn_x, trn_y, val_x, val_y, test, verbose):
params = {'objective':'regression',
'num_leaves' : 30,
'min_data_in_leaf' : 20,
'max_depth' : 9,
'learning_rate': 0.004,
'feature_fraction':0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
'lambda_l1': 0.2,
"bagging_seed": random_seed,
"metric": 'rmse',
"random_state" : random_seed,
"verbosity": -1}
record = dict()
model = lgb.train(params
, lgb.Dataset(trn_x, trn_y)
 , num_boost_round = 100000
, valid_sets = [lgb.Dataset(val_x, val_y)]
, verbose_eval = verbose
, early_stopping_rounds = 500
, callbacks = [lgb.record_evaluation(record)] 
)
 best_idx = np.argmin(np.array(record['valid_0']['rmse']))
val_pred = model.predict(val_x, num_iteration = model.best_iteration)
 test_pred = model.predict(test, num_iteration = model.best_iteration)
 return {'val':val_pred, 'test':test_pred, 'error':record['valid_0']['rmse'][best_idx], 'importance':model.feature_importance('gain')}
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 random_transforms = [transforms.RandomRotation(degrees=5)]
required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
transforms.RandomApply(random_transforms, p=0.3)
 ])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
<categorify>
def cat_model(trn_x, trn_y, val_x, val_y, test, verbose):
model = CatBoostRegressor(iterations=100000,
learning_rate=0.004,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.8,
random_seed = random_seed,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=200 
)
 model.fit(trn_x, trn_y,
eval_set=(val_x, val_y),
use_best_model=True,
verbose=False)
 val_pred = model.predict(val_x)
 test_pred = model.predict(test)
 return {'val':val_pred,
'test':test_pred,
'error':model.get_best_score() ['validation_0']['RMSE']}
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.1),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4 )<normalization>
result_dict = dict()
val_pred = np.zeros(train.shape[0])
 test_pred = np.zeros(test.shape[0])
 final_err = 0
verbose = False
for i,(trn, val)in enumerate(fold):
print(i+1, "fold.RMSE")
 trn_x = train.loc[trn, :]
trn_y = y[trn]
val_x = train.loc[val, :]
val_y = y[val]
fold_val_pred = []
fold_test_pred = []
fold_err = []
start = datetime.now()
result = xgb_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val']*0.2)
 fold_test_pred.append(result['test']*0.2)
 fold_err.append(result['error'])
 print("xgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 start = datetime.now()
result = lgb_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val']*0.4)
 fold_test_pred.append(result['test']*0.4)
 fold_err.append(result['error'])
 print("lgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 start = datetime.now()
result = cat_model(trn_x, trn_y, val_x, val_y, test, verbose)
 fold_val_pred.append(result['val']*0.4)
 fold_test_pred.append(result['test']*0.4)
 fold_err.append(result['error'])
 print("cat model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)')
 val_pred[val] += np.mean(np.array(fold_val_pred), axis = 0)
 test_pred += np.mean(np.array(fold_test_pred), axis = 0)/ k
final_err +=(sum(fold_err)/ len(fold_err)) / k
print("---------------------------")
 print("avg err.", "{0:.5f}".format(sum(fold_err)/ len(fold_err)))
 print("blend err.", "{0:.5f}".format(np.sqrt(np.mean(( np.mean(np.array(fold_val_pred), axis = 0)- val_y)**2))))
print('')
 print("fianl avg err.", final_err)
 print("fianl blend err.", np.sqrt(np.mean(( val_pred - y)**2)) )
<categorify><EOS> 
sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 df_sub = pd.DataFrame()
df_sub['id'] = sub['id']
df_sub['revenue'] = np.expm1(test_pred*3)
 df_sub.to_csv("submission.csv", index=False )