Kaggle competition: TMDB Box Office Prediction
<SOS> metric: RMSLE Kaggle data source: tmdb-box-office-prediction<import_modules>
pd.set_option('max_columns', None)
 %matplotlib inline
plt.style.use('ggplot')
 stop = set(stopwords.words('english'))
warnings.filterwarnings("ignore" )
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam<train_model>
train=pd.read_csv('.. /input/train.csv',index_col=0)
 test=pd.read_csv('.. /input/test.csv',index_col=0 )
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
annealer = LearningRateScheduler(lambda x: lr)
 h = discriminator.fit([zeros,train_X], train_y, epochs = 10, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1)*10,'/30 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.533: lr = 0.1<drop_column>
test['revenue']=-99
del train_X, train_y, imagesIn<define_search_model>
train=train.drop('belongs_to_collection',axis=1)
 test=test.drop('belongs_to_collection',axis=1 )
BadMemory = False
if BadMemory:
seed = Input(( 10000,))
x = Dense(2048, activation='elu' )(seed)
 x = Reshape(( 8,8,32))(x)
 x = Conv2D(128,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(64,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(32,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(3,(3, 3), activation='linear', padding='same' )(x)
 generated = Flatten()(x)
 else:
seed = Input(( 10000,))
generated = Dense(12288, activation='linear' )(seed)
 generator = Model(seed, [generated,Reshape(( 10000,))(seed)])
 generator.summary()<choose_model_class>
train.loc[train.index == 16,'revenue'] = 192864
train.loc[train.index == 90,'budget'] = 30000000
train.loc[train.index== 118,'budget'] = 60000000
train.loc[train.index== 149,'budget'] = 18000000
train.loc[train.index== 313,'revenue'] = 12000000
train.loc[train.index == 451,'revenue'] = 12000000
train.loc[train.index == 464,'budget'] = 20000000
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index== 513,'budget'] = 930000
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 819,'budget'] = 90000000
train.loc[train.index == 850,'budget'] = 90000000
train.loc[train.index == 1007,'budget'] = 2
train.loc[train.index== 1112,'budget'] = 7500000
train.loc[train.index == 1131,'budget'] = 4300000
train.loc[train.index == 1359,'budget'] = 10000000
train.loc[train.index == 1542,'budget'] = 1
train.loc[train.index == 1570,'budget'] = 15800000
train.loc[train.index== 1571,'budget'] = 4000000
train.loc[train.index == 1714,'budget'] = 46000000
train.loc[train.index == 1721,'budget'] = 17500000
train.loc[train.index== 1865,'revenue'] = 25000000
train.loc[train.index == 1885,'budget'] = 12
train.loc[train.index == 2091,'budget'] = 10
train.loc[train.index == 2268,'budget'] = 17500000
train.loc[train.index == 2491,'budget'] = 6
train.loc[train.index == 2602,'budget'] = 31000000
train.loc[train.index == 2612,'budget'] = 15000000
train.loc[train.index == 2696,'budget'] = 10000000
train.loc[train.index == 2801,'budget'] = 10000000
train.loc[train.index == 335,'budget'] = 2
train.loc[train.index == 348,'budget'] = 12
train.loc[train.index == 470,'budget'] = 13000000
train.loc[train.index == 513,'budget'] = 1100000
train.loc[train.index == 640,'budget'] = 6
train.loc[train.index == 696,'budget'] = 1
train.loc[train.index == 797,'budget'] = 8000000
train.loc[train.index == 850,'budget'] = 1500000
train.loc[train.index == 1199,'budget'] = 5
train.loc[train.index == 1282,'budget'] = 9
train.loc[train.index== 1347,'budget'] = 1
train.loc[train.index== 1755,'budget'] = 2
train.loc[train.index == 1801,'budget'] = 5
train.loc[train.index == 1918,'budget'] = 592
train.loc[train.index == 2033,'budget'] = 4
train.loc[train.index == 2118,'budget'] = 344
train.loc[train.index == 2252,'budget'] = 130
train.loc[train.index == 2256,'budget'] = 1
train.loc[train.index == 2696,'budget'] = 10000000
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
 gan.summary()<train_model>
test.loc[test.index== 3033,'budget'] = 250
test.loc[test.index== 3051,'budget'] = 50
test.loc[test.index == 3084,'budget'] = 337
test.loc[test.index == 3224,'budget'] = 4
test.loc[test.index == 3594,'budget'] = 25
test.loc[test.index == 3619,'budget'] = 500
test.loc[test.index == 3831,'budget'] = 3
test.loc[test.index== 3935,'budget'] = 500
test.loc[test.index == 4049,'budget'] = 995946
test.loc[test.index== 4424,'budget'] = 3
test.loc[test.index == 4460,'budget'] = 8
test.loc[test.index == 4555,'budget'] = 1200000
test.loc[test.index== 4624,'budget'] = 30
test.loc[test.index== 4645,'budget'] = 500
test.loc[test.index == 4709,'budget'] = 450
test.loc[test.index == 4839,'budget'] = 7
test.loc[test.index== 3125,'budget'] = 25
test.loc[test.index== 3142,'budget'] = 1
test.loc[test.index == 3201,'budget'] = 450
test.loc[test.index == 3222,'budget'] = 6
test.loc[test.index== 3545,'budget'] = 38
test.loc[test.index == 3670,'budget'] = 18
test.loc[test.index == 3792,'budget'] = 19
test.loc[test.index == 3881,'budget'] = 7
test.loc[test.index == 3969,'budget'] = 400
test.loc[test.index == 4196,'budget'] = 6
test.loc[test.index == 4221,'budget'] = 11
test.loc[test.index == 4222,'budget'] = 500
test.loc[test.index== 4285,'budget'] = 11
test.loc[test.index == 4319,'budget'] = 1
test.loc[test.index == 4639,'budget'] = 10
test.loc[test.index == 4719,'budget'] = 45
test.loc[test.index == 4822,'budget'] = 22
test.loc[test.index == 4829,'budget'] = 20
test.loc[test.index== 4969,'budget'] = 20
test.loc[test.index== 5021,'budget'] = 40
test.loc[test.index== 5035,'budget'] = 1
test.loc[test.index== 5063,'budget'] = 14
test.loc[test.index == 5119,'budget'] = 2
test.loc[test.index== 5214,'budget'] = 30
test.loc[test.index== 5221,'budget'] = 50
test.loc[test.index== 4903,'budget'] = 15
test.loc[test.index == 4983,'budget'] = 3
test.loc[test.index == 5102,'budget'] = 28
test.loc[test.index== 5217,'budget'] = 75
test.loc[test.index == 5224,'budget'] = 3
test.loc[test.index== 5469,'budget'] = 20
test.loc[test.index == 5840,'budget'] = 1
test.loc[test.index == 5960,'budget'] = 30
test.loc[test.index == 6506,'budget'] = 11
test.loc[test.index== 6553,'budget'] = 280
test.loc[test.index == 6561,'budget'] = 7
test.loc[test.index== 6582,'budget'] = 218
test.loc[test.index == 6638,'budget'] = 5
test.loc[test.index== 6749,'budget'] = 8
test.loc[test.index==6759,'budget'] = 50
test.loc[test.index == 6856,'budget'] = 10
test.loc[test.index== 6858,'budget'] = 100
test.loc[test.index == 6876,'budget'] = 250
test.loc[test.index == 6972,'budget'] = 1
test.loc[test.index== 7079,'budget'] = 8000000
test.loc[test.index == 7150,'budget'] = 118
test.loc[test.index == 6506,'budget'] = 118
test.loc[test.index == 7225,'budget'] = 6
test.loc[test.index == 7231,'budget'] = 85
test.loc[test.index == 5222,'budget'] = 5
test.loc[test.index == 5322,'budget'] = 90
test.loc[test.index == 5350,'budget'] = 70
test.loc[test.index == 5378,'budget'] = 10
test.loc[test.index== 5545,'budget'] = 80
test.loc[test.index == 5810,'budget'] = 8
test.loc[test.index== 5926,'budget'] = 300
test.loc[test.index== 5927,'budget'] = 4
test.loc[test.index== 5986,'budget'] = 1
test.loc[test.index == 6053,'budget'] = 20
test.loc[test.index== 6104,'budget'] = 1
test.loc[test.index == 6130,'budget'] = 30
test.loc[test.index == 6301,'budget'] = 150
test.loc[test.index == 6276,'budget'] = 100
test.loc[test.index == 6473,'budget'] = 100
test.loc[test.index== 6842,'budget'] = 30
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 0.924
zeros = np.zeros(( 10000,12288))
ep = 1; it = 50
if BadMemory: lr = 0.005
else: lr = 5.
 for k in range(it):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = ep, batch_size=256, callbacks=[annealer], verbose=0)
 if(k==it)|(k%10==9):
print('Epoch',(k+1),'/'+str(it)+' - loss =',h.history['loss'][-1])
 plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
if BadMemory:
ep *= 2
if ep>=32: lr = 0.001
if ep>256: ep = 256
else:
if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5<predict_on_test>
new_data=pd.concat([train,test],axis=0)
 new_data.head()
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.902
xx[np.random.randint(10000)] = 0.098
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))<load_from_csv>
new_data.isnull().sum()
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
new_data[new_data['release_date'].isnull() ]
df_all = df.append(test)
 df_all.shape<count_values>
new_data['release_date']=new_data['release_date'].fillna('3/20/01' )
df_all['dependency'].value_counts()<count_values>
new_data['release_year']=pd.to_datetime(new_data['release_date'] ).dt.year
new_data['release_month']=pd.to_datetime(new_data['release_date'] ).dt.month
new_data['release_day']=pd.to_datetime(new_data['release_date'] ).dt.day
df_all['edjefa'].value_counts()<count_values>
new_data['release_year'].loc[new_data['release_year']>=2018]-=100
df_all['edjefe'].value_counts()<data_type_conversions>
new_data=new_data.drop('release_date',axis=1 )
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )<count_values>
weiji=[1929,1930,1930,1931,1932,1933,1934,1935,1936,1937,1938,1939,1940,1941,1942,1943,1944,1945,1950,1951,1952,1953,1961,1962,1963,1964,1965,1966,1967,1968,1969,1971,1973,1974,1975,1980,1981,1982,1983,1984,1985,1986,1987,1988,1989,
1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2007,2008,2009,2010,2011]
df_all['dependency'].value_counts()<data_type_conversions>
for i in weiji:
new_data['is_'+str(i)]=new_data['release_year'].apply(lambda x:1 if x==i else 0 )
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )<count_missing_values>
new_data['homepage_fact']=new_data['homepage'].apply(lambda x: 0 if x is np.nan else 1 )
df_all.isnull().sum()<count_missing_values>
new_data['homepage_end']=new_data[new_data['homepage'].notna() ]['homepage'].str.findall(r'\.( [a-z]+ )(?:\/|$)' ).apply(lambda x:x[0])
 new_data['homepage_end'].head()
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()<count_values>
new_data['homepage_end']=new_data['homepage_end'].fillna('unknow' )
df_all['v18q'].value_counts()<count_missing_values>
page=pd.get_dummies(new_data['homepage_end'])
 page.head()
df_all.isnull().sum().sort_values()<categorify>
new_data=new_data.drop('homepage',axis=1 )
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )<count_values>
new_data=new_data.drop('poster_path',axis=1 )
df_all.Target.value_counts()<define_variables>
new_data['len_overview']=new_data['overview'].fillna('NAN' ).apply(lambda x:len(x))
new_data['len_overview'][new_data['len_overview']==0]=1
new_data['len_overview_budget']=new_data['budget']/(new_data['len_overview']+1 )
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<save_to_csv>
len_rew_sort=new_data['len_overview'].sort_values(ascending=True)
 len_rew_sort.head()
rf = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=701, min_impurity_decrease=0.0007, min_samples_leaf=2, min_samples_split=5, verbose=0, class_weight='balanced')
 rf.fit(train[feats], train['Target'])
 test['Target'] = rf.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )<filter>
length=len(new_data['len_overview'])
 m=0.1
n=0.1
arr_len_ove=[]
for i in range(1,11):
arr_len_ove.append(round(length*m))
m+=n
arr_len_ove
df_a = df_all[df_all['Target'] == 1]
df_b = df_all[df_all['Target'] == 2]
df_c = df_all[df_all['Target'] == 3]
df_d = df_all[df_all['Target'] == 4]<concatenate>
for i in range(10):
qu=qu_arr[i]
if i==0:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])]=qu
else:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])&(new_data['len_overview'] >qu_arr[i-1])]=qu
print(i,qu )
df_a_os = resample(df_a, replace=True, n_samples=len(df_d), random_state=42)
 df_b_os = resample(df_b, replace=True, n_samples=len(df_d), random_state=42)
 df_c_os = resample(df_c, replace=True, n_samples=len(df_d), random_state=42)
 df_os = pd.concat([df_a_os, df_b_os, df_c_os, df_d] )<count_values>
len_ove_agg=new_data.groupby('len_overview' ).revenue.aggregate(['min','max','std'] )
df_os['Target'].value_counts()<predict_on_test>
new_data['geres_name']=new_data['genres'].str.findall(r''name'\s?:\s?'(\w+)'')
 new_data['geres_name'].head()
train, test = train_test_split(df_os, test_size=0.2, random_state=42)
 rf.fit(train[feats], train['Target'])
 preds_test = rf.predict(test[feats])
 accuracy_score(test['Target'], preds_test )<concatenate>
new_data=new_data.drop('genres',axis=1 )
df_b_us = resample(df_b, replace=False, n_samples=len(df_a),random_state=42)
 df_c_us = resample(df_c, replace=False, n_samples=len(df_a),random_state=42)
 df_d_us = resample(df_d, replace=False, n_samples=len(df_a),random_state=42)
 df_us = pd.concat([df_a, df_b_us, df_c_us, df_d_us] )<count_values>
country=new_data['production_countries'].str.findall(r'[A-Z]{2,5}')
df_us['Target'].value_counts()<predict_on_test>
new_data['production_countries']=country
train, test = train_test_split(df_us, test_size=0.2, random_state=42)
 rf.fit(train[feats], train['Target'])
 preds_test = rf.predict(test[feats])
 accuracy_score(test['Target'], preds_test )<import_modules>
new_data['original_and_new']=new_data['original_title']==new_data['title']
import numpy as np
import pandas as pd<load_from_csv>
new_data=new_data.drop('original_title',axis=1)
 new_data.head()
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
new_data=pd.concat([new_data,page],axis=1 )
df_all = df.append(test )<data_type_conversions>
new_data['production_companies']=new_data['production_companies'].str.findall(r''name'?:\s?'([A-Za-z]+)')
 new_data.fillna('Unknow')
 print('接下来就是地图可视化了' )
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int)
 df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float)
 df_all['v2a1'].fillna(-1, inplace=True)
 df_all['v18q1'].fillna(0, inplace=True)
 df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True)
 feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
train, test = df_all[~df_all['Target'].isnull() ], df_all[df_all['Target'].isnull() ] 
<save_to_csv>
new_data=new_data.drop('imdb_id',axis=1 )
forest = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced')
 forest.fit(train[feats], train['Target'])
 test['Target'] = forest.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )<choose_model_class>
new_data['spoken_languages']=new_data['spoken_languages'].str.findall(r''([a-z]{2})'' )
feats2 = [c for c in list(dict(teste.head(10)).keys())if c not in ['Id', 'idhogar', 'Target']]
forest2 = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced' )<save_to_csv>
new_data['production_companies']=new_data['production_companies'].fillna('unknow' )
forest.fit(train[feats2], train['Target'])
 test['Target'] = forest.predict(test[feats2] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )<concatenate>
new_data['spoken_languages']=new_data['spoken_languages'].fillna('unknow' )
retirar.append('Id')
 retirar.append('idhogar')
 retirar.append('Target' )<split>
new_data['Keywords']=new_data['Keywords'].str.findall(r''?:\s?'([a-z]+\s?[a-z]+)'' ).fillna('unkonw' )
feats3 = [c for c in df_all.columns if c not in retirar]
train, test = df_all[~df_all['Target'].isnull() ], df_all[df_all['Target'].isnull() ]<save_to_csv>
d = Counter([i for j in list_of_keywords for i in j] ).most_common(30)
 d
forest = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced')
 forest.fit(train[feats3], train['Target'])
 test['Target'] = forest.predict(test[feats3] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )<define_variables>
for i in d:
m=i[0]
new_data['Keywords_'+m]=new_data['Keywords'].apply(lambda x:1 if m in x else 0 )
retirar = list()
colunas_escolhidas = list()
while len(retirar)< 10:
index_coluna = random.randint(0,140)
 coluna = teste.index[index_coluna]
if coluna not in retirar:
retirar.append(coluna)
 else:
continue
retirar.append('Id')
 retirar.append('idhogar')
 retirar.append('Target' )<split>
new_data=new_data.drop('Keywords',axis=1 )
feats4 = [c for c in df_all.columns if c not in retirar]
train, test = df_all[~df_all['Target'].isnull() ], df_all[df_all['Target'].isnull() ]<save_to_csv>
new_data['cast']=new_data['cast'].fillna('unknow' )
forest = RandomForestClassifier(max_depth=None, random_state=43, n_jobs=5, n_estimators=707,
min_impurity_decrease=1e-3, min_samples_leaf=2, min_samples_split=5 ,
verbose=0, class_weight='balanced')
 forest.fit(train[feats4], train['Target'])
 test['Target'] = forest.predict(test[feats4] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )<load_from_csv>
for i in d:
m=i[0][0]
new_data['cast_name_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
list_cast_gender=list(new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?'))
Counter([i for j in list_cast_gender for i in j] ).most_common()
df_all = df.append(test)
 df_all.shape<count_values>
new_data['cast_gender_0']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('0'))
new_data['cast_gender_1']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('1'))
new_data['cast_gender_2']=new_data['cast'].str.findall(r''gender'\s?:\s?(\d+)\s?' ).apply(lambda x: x.count('2'))
new_data['budget_gender_count']=new_data['budget']/(new_data['cast_gender_0']+new_data['cast_gender_1']+new_data['cast_gender_2']+1 )
df.Target.value_counts() /100<count_values>
for g in top_cast_char:
m=g[0]
new_data['cast_char_'+m]=new_data['cast'].apply(lambda x:1 if m in x else 0 )
df_all['edjefa'].value_counts()<data_type_conversions>
new_data=new_data.drop('cast',axis=1 )
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )<count_values>
new_data['status']=new_data['status'].fillna('Unknow' )
df_all['dependency'].value_counts()<data_type_conversions>
new_data['overview']=new_data['overview'].fillna('' )
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )<count_missing_values>
from sklearn.linear_model import LinearRegression
import eli5
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
df_all.isnull().sum().sort_values()<create_dataframe>
log_re=np.log1p(train['revenue'])
 v1=TfidfVectorizer(sublinear_tf=True,analyzer='word',
token_pattern=r'\w{1,}',
ngram_range=(1, 2),
min_df=5)
 overview_text = v1.fit_transform(train['overview'].fillna(''))
linre=LinearRegression()
linre.fit(overview_text,log_re)
data_na = df_all.isnull().sum().values / df_all.shape[0] *100
df_na = pd.DataFrame(data_na, index=df_all.columns, columns=['Count'])
 df_na = df_na.sort_values(by=['Count'], ascending=False)
 missing_value_count = df_na[df_na['Count']>0].shape[0]
print(f'We got {missing_value_count} rows which have missing value in train set ')
 df_na.head(6)
<count_missing_values>
eli5.show_weights(linre, vec=v1, top=20, feature_filter=lambda x: x !='<BIAS>' )
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()<count_values>
new_data['crew_0']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('0'))
new_data['crew_1']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('1'))
new_data['crew_2']=new_data['crew'].fillna('' ).str.replace(',','' ).str.replace('}','' ).str.findall(''gender\S?'\s?:\s?\S?(\d+)\s?\S?' ).apply(lambda x:x.count('2'))
df_all['v18q'].value_counts()<data_type_conversions>
for i in d:
new_data['crew_depart_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0)
 for i in dd:
new_data['crew_job_is_'+i[0]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0] in x else 0 )
df_all['v2a1'].fillna(0, inplace=True)
 df_all['v18q1'].fillna(0, inplace=True)
 df_all['rez_esc'].fillna(0, inplace=True)
<feature_engineering>
new_data['crew']=new_data['crew'].str.replace(',','' ).str.replace('}','' ).str.replace(r''gender'\s?\S?:\s?\S?\s+'id'\s?:\s?\d+','' ).str.findall(r''department'\s?\S?:\s+\S?(\S+)\S?\s?\S?\s+'job\S?'\s?:\s?'(\D+)\S\s?\S?'name\S?'\s?:\s?'(\D+)'\s?\S?'profile_path\S?':\s+'?(\S+)\S?\s?\s?\S?' )
df_all['meaneduc'].fillna(df_all['meaneduc'].median() , inplace=True)
 df_all['SQBmeaned'].fillna(df_all['SQBmeaned'].median() , inplace=True)
<count_missing_values>
new_data['len_crew']=new_data['crew'].fillna('1' ).apply(lambda x:len(x))
df_all.isnull().sum().sort_values()<correct_missing_values>
new_train=new_data.loc[np.array(train.index)]
df_all.fillna(-1, inplace=True)
<feature_engineering>
new_train['production_countries']=new_train['production_countries'].fillna('' )
df_all['hsize-pc'] = df_all['hhsize'] / df_all['tamviv']
df_all['phone-pc'] = df_all['qmobilephone'] / df_all['tamviv']
df_all['tablets-pc'] = df_all['v18q1'] / df_all['tamviv']
df_all['rooms-pc'] = df_all['rooms'] / df_all['tamviv']
df_all['rent-pc'] = df_all['v2a1'] / df_all['tamviv']<define_variables>
new_train['production_countries']=new_train['production_countries'].apply(lambda x:'_'.join(x))
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<split>
count=new_train['production_countries'].value_counts()
count=count[count>5]
train, test = df_all[df_all['Target'] != -1], df_all[df_all['Target'] == -1]<define_variables>
bud=new_train.groupby('production_countries' ).budget.mean()
heads = train[train['parentesco1'] == 1]<train_model>
bud=bud.sort_values(ascending=False)[:10]
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=700, class_weight='balanced',
colsample_bytree = 0.93, min_child_samples = 95, num_leaves = 14, subsample = 0.96)
 clf.fit(heads[feats], heads['Target'])
 accuracy_score(heads['Target'], clf.predict(heads[feats]))<predict_on_test>
rev=new_train.groupby('production_countries' ).revenue.mean()
test['Target'] = clf.predict(test[feats] ).astype(int)
<count_values>
rev=rev.sort_values(ascending=False)[:10]
test['Target'].value_counts(normalize=True )<train_model>
new_train['production_countries'].loc[new_train['production_countries']=='ET']='Ethiopia'
along_co=new_train[new_train['production_countries'].apply(lambda x:1 if len(x)==2 else 0)==1]
along_co.head()
cbc = CatBoostClassifier(random_state=42)
 cbc.fit(heads[feats], heads['Target'])
 accuracy_score(test['Target'], cbc.predict(test[feats]))<predict_on_test>
count_rev=along_co[['production_countries','revenue']]
test['Target'] = cbc.predict(test[feats] ).astype(int)
 test['Target'].value_counts(normalize=True)
<choose_model_class>
dd=count_rev['production_countries'].unique()
mn=pd.DataFrame(dd,columns=['address'])
 mn.head()
rf = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
<train_model>
import geopandas as ge
rf.fit(heads[feats], heads['Target'] )<predict_on_test>
world = ge.read_file(ge.datasets.get_path('naturalearth_lowres'))
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
from mpl_toolkits.axes_grid1 import make_axes_locatable
test['Target'].value_counts(normalize=True)
<compute_test_metric>
dd=count_rev['production_countries'].value_counts()
mn['values']=dd.values
mn.head()
accuracy_score(heads['Target'], rf.predict(heads[feats]))<save_to_csv>
mn['address'][0]='United States'
mn['address'][2]='Korea'
mn['address'][1]='India'
mn['address'][3]='Serbia'
mn['address'][4]='United Kingdom'
mn['address'][5]='France'
mn['address'][6]='New Zealand'
mn['address'][7]='Italy'
mn['address'][8]='Belgium'
mn['address'][9]='Czech Rep.'
mn['address'][11]='Russia'
mn['address'][12]='Spain'
mn['address'][13]='Turkey'
mn['address'][14]='China'
mn['address'][15]='Canada'
mn['address'][16]='Australia'
mn['address'][17]='Iran'
mn['address'][18]='Japan'
mn['address'][19]='Sweden'
mn['address'][20]='Philippines'
mn['address'][21]='Brazil'
mn['address'][22]='Netherlands'
mn['address'][23]='Ireland'
mn['address'][24]='Mexico'
mn['address'][25]='Germany'
mn['address'][26]='South Africa'
mn['address'][27]='Finland'
mn['address'][28]='Pakistan'
mn['address'][29]='Norway'
mn['address'][30]='Bulgaria'
mn['address'][31]='Ukraine'
mn['address'][32]='Denmark'
mn['address'][33]='Romania'
mn['address'][35]='Hungary'
mn['address'][36]='Chile'
mn['address'][37]='Indonesia'
mn['address'][38]='Poland'
test[['Id', 'Target']].to_csv('submission.csv', index=False)
<load_from_csv>
mn=mn.drop(10)
 mn=mn.drop(34 )
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
mn['geometry']='unknow'
df_all = df.append(test)
 df_all.shape<count_values>
for i in range(39):
if i==10 or i==34:
continue;
d=world[world['name']==mn['address'][i]]['geometry'].values[0]
mn['geometry'][i]=d
df_all['dependency'].value_counts()<count_values>
TS = train.loc[:,["original_title","release_date","budget","runtime","revenue"]]
TS.dropna()
TS.release_date = pd.to_datetime(TS.release_date)
 TS.loc[:,"Year"] = TS["release_date"].dt.year
TS.loc[:,"Month"] = TS["release_date"].dt.month
TS = TS[TS.Year<2018]
df_all['edjefa'].value_counts()<count_values>
top3 = train.sort_values(by='popularity',ascending=False)[:10]
id3=top3[['title','poster_path','revenue']]
df_all['edjefe'].value_counts()<data_type_conversions>
old_title=new_data['title'].value_counts()
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )<count_values>
a=old_title[old_title.values==3].index
b=old_title[old_title==2].index
df_all['dependency'].value_counts()<data_type_conversions>
new_data['fan_pai_2']=new_data['title'].apply(lambda x:1 if x in a else 0)
 new_data['pan_pai_3']=new_data['title'].apply(lambda x:1 if x in b else 0 )
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )<count_missing_values>
new_data.loc[new_data['title'].fillna('un' ).str.contains('Planet of the Apes')]
total = np.product(df_all.shape)
 miss_values = df_all.isnull().sum().sum()
porcentagem_total =(miss_values/total)* 100
print(f'A porcentagem total de valores nulos do dataset é de: {(porcentagem_total)}' )<sort_values>
top10 = train.sort_values(by='revenue',ascending=False)[:10]
id10=top10[['title','poster_path','revenue']]
total_num = df_all.isnull().sum().sort_values(ascending = False)
 perc = df_all.isnull().sum() /df_all.isnull().count() *100
perc1 =(round(perc,2 ).sort_values(ascending = False))
df_miss = pd.concat([total_num, perc1], axis = 1, keys = ['Total de Valores Nulos', 'Porcentagem %'] ).sort_values(by = 'Porcentagem %', ascending = False)
 top_miss = df_miss[df_miss['Porcentagem %'] > 0]
top_miss.reset_index(inplace = True)
 top_miss<count_values>
title_count=train['title'].value_counts()
len(title_count[title_count.values!=1] )
df_all.rez_esc.value_counts()<feature_engineering>
train[train['title'].str.contains('Furious')]['title']
df_all.loc[(( df_all['age'] > 19)|(df_all['age'] < 7)) &(df_all['rez_esc'].isnull()), 'rez_esc'] = 0<feature_engineering>
test[test['title'].fillna('Unknow' ).str.contains('Furious')]['title']
df_all['rez_esc-missing'] = df_all['rez_esc'].isnull()<feature_engineering>
d.remove(d[7] )
df_all.loc[df_all['rez_esc'] > 5, 'rez_esc'] = 5<correct_missing_values>
for i in d :
new_data['crew_contains_'+i[0][2]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0][2] in x else 0)
df_all['rez_esc'].fillna(0, inplace = True )<count_values>
mon_is=[1,6,7,9,10,12]
for i in mon_is:
new_data['mon_is_'+str(i)]=new_data['release_month'].apply(lambda x:1 if x==i else 0 )
df_all.v18q1.value_counts()<count_values>
new_data['spoken_languages_count']=new_data['spoken_languages'].apply(lambda x:len(x))
df_all.v18q.value_counts()<count_values>
new_data.loc[train.index].groupby('spoken_languages_count' ).revenue.median()
df_all.v2a1.value_counts()<count_values>
status_du=pd.get_dummies(new_data['status'] )
df_all.tipovivi3.value_counts().sort_values()<count_values>
new_data=pd.concat([new_data,status_du],axis=1)
df_all.tipovivi1.value_counts().sort_values()<count_values>
new_data=new_data.drop(['status'],axis=1)
 new_data.head()
df_all.tipovivi2.value_counts().sort_values()<count_values>
new_data=new_data.drop(['original_language'],axis=1 )
df_all.tipovivi4.value_counts()<count_values>
sum(new_data['geres_name'].isna() )
df_all.tipovivi5.value_counts()<data_type_conversions>
new_data['geres_name']=new_data['geres_name'].fillna('Unknow' )
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True )<count_missing_values>
new_data['geres_name_count']=new_data['geres_name'].apply(lambda x:len(x))
total = np.product(df_all.shape)
 miss_values = df_all.isnull().sum().sum()
porcentagem_total =(miss_values/total)* 100
print(f'A porcentagem total de valores nulos do dataset é de: {(porcentagem_total)}' )<count_unique_values>
ge_name_count_rev=new_data.loc[train.index].groupby(['geres_name_count'] ).revenue.median().sort_values()
df_all.idhogar.nunique()<count_values>
new_data=new_data.drop('homepage_end',axis=1 )
df_all.parentesco1.value_counts()<count_unique_values>
new_data['original_and_new']=new_data['original_and_new'].apply(lambda x: 1 if x else 0 )
(
df_all.groupby('r4t3' ).idhogar.nunique() 
.reset_index() 
.rename(columns = {'r4t3':'Quantidade de Pessoas na Casa', 'idhogar':'Quantidade de Casas'})
.sort_values(by = ['Quantidade de Casas'], ascending = True)
.reset_index() 
.drop(['index'], axis = 1)
 )<count_unique_values>
list_geres_name=['Comedy','Thriller','Action','Drama','Romance']
(
df_all.groupby('tamviv' ).idhogar.nunique() 
.reset_index() 
.rename(columns={'tamviv':'Quantidade de Pessoas que vivem na Casa', 'idhogar':'Quantidade de Casas'})
.sort_values(by=['Quantidade de Casas'], ascending=True)
.reset_index() 
.drop(['index'], axis = 1)
 )<groupby>
for i in list_geres_name:
new_data['geres_name'+'_'+i]=new_data['geres_name'].apply(lambda x: 1 if i in x else 0 )
(
df_all.groupby('Target' ).idhogar.count()
)<count_unique_values>
new_data=new_data.drop('geres_name',axis=1 )
(
df_all.groupby('r4h1' ).idhogar.nunique() 
.reset_index() 
.rename(columns={'r4h1':'Meninos mais jovens que 12 anos de idade','idhogar':'Quantidade de Casas'})
 )<count_unique_values>
new_data['production_countries']=new_data['production_countries'].fillna('QQ' )
(
df_all.groupby('r4h2' ).idhogar.nunique() 
.reset_index() 
.rename(columns={'r4h2':'Homens com 12 anos de idade ou mais velhos','idhogar':'Quantidade de Casas'})
 )<count_unique_values>
new_data['production_countries'].loc[new_data['production_countries']=='Unknow']['production_countries']='QQ'
(
df_all.groupby('r4h3' ).idhogar.nunique() 
.reset_index() 
.rename(columns={'r4h3':'Total de Homens na Casa','idhogar':'Quantidade de Casas'})
 )<count_unique_values>
list_pro_coun=list(new_data['production_countries'])
 d = Counter([j for i in list_pro_coun for j in i] ).most_common(5)
 d
(
df_all.groupby('r4m1' ).idhogar.nunique() 
.reset_index() 
.rename(columns={'r4m1':'Meninas mais jovens que 12 anos de idade','idhogar':'Quantidade de Casas'})
 )<count_unique_values>
for i in d:
new_data['production_cou_name'+'_'+i[0]]=new_data['production_countries'].apply(lambda x: 1 if i[0] in x else 0 )
(
df_all.groupby('r4m2' ).idhogar.nunique() 
.reset_index() 
.rename(columns={'r4m2':'Meninas de 12 anos de idade ou mais velhas','idhogar':'Quantidade de Casas'})
 )<count_unique_values>
new_data['pro_country_count']=new_data['production_companies'].apply(lambda x:len(x))
(
df_all.groupby('r4m3' ).idhogar.nunique() 
.reset_index() 
.rename(columns={'r4m3':'Total de Mulheres na Casa','idhogar':'Quantidade de Casas'})
 )<create_dataframe>
pro_count_rev=new_data.loc[train.index].groupby('pro_country_count' ).revenue.median()
edjefas_renda = pd.pivot_table(
 df_all,
columns = 'Target',
index = 'edjefa',
values = 'Id',
aggfunc = {'Id':'count'} 
)
 edjefas_renda<create_dataframe>
new_data=new_data.drop('production_countries',axis=1 )
edjefes_renda = pd.pivot_table(
 df_all,
columns = 'Target',
index = 'edjefe',
values = 'Id',
aggfunc = {'Id':'count'} 
)
 edjefes_renda<rename_columns>
new_data['production_companies'].loc[new_data['production_companies']=='unknow']='u'
room_rent =(
 df_all.groupby('rooms' ).v2a1.mean()
).reset_index()
room_rent.rename(columns = {'rooms':'Quantidade de Quartos', 'v2a1':'Média do Aluguel Mensal'} )<drop_column>
d.remove(( 'u',414))
df_all = df_all.drop(['hhsize','r4t3'], axis = 1 )<drop_column>
dd=[]
for i in d:
dd.append(i[0] )
df_all = df_all.drop(['age'], axis = 1 )<define_variables>
dd=[]
for i in d:
dd.append(i[0] )
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar','Target']]<choose_model_class>
new_data=new_data.drop('spoken_languages',axis=1 )
rf = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced' )<train_model>
new_data['runtime'].loc[new_data['runtime']==0]=1
new_data['time_budget']=new_data['budget'].fillna(0)/new_data['runtime'].fillna(1 )
rf.fit(train[feats], train['Target'] )<predict_on_test>
new_data=new_data.drop('production_companies',axis=1 )
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
new_data=new_data.drop('crew',axis=1 )
test['Target'].value_counts(normalize=True )<feature_engineering>
for i in list1:
new_data['title_is'+i]=new_data['title'].fillna('' ).apply(lambda x: 1 if i in x else 0 )
( 650 + 1202 + 929 + 4664)/ 9557<save_to_csv>
for col in ['tagline', 'overview','title']:
new_data['len_' + col] =new_data[col].fillna('' ).apply(lambda x: len(x))
new_data['words_' + col] = new_data[col].fillna('' ).apply(lambda x: len(x.split(' ')))
 new_data=new_data.drop(col,axis=1 )
<import_modules>
new_data['budget_popularity']=new_data['budget']*1.0/new_data['popularity']
from xgboost import XGBClassifier<choose_model_class>
a=new_data.groupby(['release_year'] ).release_month.value_counts()
b=new_data.groupby(['release_year','release_month'] ).release_day.value_counts()
new_data[(new_data['release_year']==2017)&(new_data['release_day']==30)]
xgb = XGBClassifier(n_estimators = 200, learning_rate = 0.09, random_state = 42, class_weight = 'balanced' )<train_model>
new_data['_releaseYear_popularity_ratio'] = new_data['release_year'] / new_data['popularity']
new_data['_releaseYear_popularity_ratio2'] = new_data['popularity'] / new_data['release_year']
new_data['runtime_to_mean_year'] = new_data['runtime'] / new_data.groupby("release_year")["runtime"].transform('mean')
 new_data['popularity_to_mean_year'] = new_data['popularity'] / new_data.groupby("release_year")["popularity"].transform('mean')
 new_data['budget_to_mean_year'] = new_data['budget'] / new_data.groupby("release_year")["budget"].transform('mean')
xgb.fit(train[feats], train['Target'] )<predict_on_test>
test['Target'] = xgb.predict(test[feats] ).astype(int )<compute_test_metric>
accuracy_score(train['Target'], xgb.predict(train[feats]))<save_to_csv>
new_train=new_data.loc[train.index]
new_test=new_data.loc[test.index]
<import_modules>
x=new_train.drop('revenue',axis=1)
 y=new_train['revenue']
x=x.fillna(0)
 decision_tree = DecisionTreeRegressor(max_depth = 6)
 decision_tree.fit(x, y)
from sklearn.ensemble import AdaBoostClassifier<choose_model_class>
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)
 model_XG = xgboost.XGBRegressor()
model_XG.fit(x_train,y_train)
 y_predict_rf = model_XG.predict(x_test)
 print(mean_squared_error(y_test, y_predict_rf))
abc = AdaBoostClassifier(n_estimators = 200, learning_rate = 1.0, random_state = 42 )<train_model>
X_test=new_test.drop('revenue',axis=1)
 X_test=X_test.fillna(0 )
abc.fit(train[feats], train['Target'] )<compute_test_metric>
ypred=model_XG.predict(X_test )
accuracy_score(train['Target'], abc.predict(train[feats]))<save_to_csv>
dtrain = xgb.DMatrix(x, y)
 dtest = xgb.DMatrix(X_test )
test[['Id','Target']].to_csv('submission.csv', index=False )<set_options>
xgb_params = {
'eta': 0.08,
'max_depth': 15,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=400, verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()
pd.set_option('max_columns', 140)
 pd.set_option('max_colwidth', 5000)
 pd.set_option('display.max_rows', 140)
 %matplotlib inline
plt.rcParams['figure.figsize'] =(12,8)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
y_predict1 = abs(model.predict(dtest))
train = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 codebook = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/codebook.csv')
 df = train.append(test)
 train.shape,test.shape, df.shape 
<filter>
train=x
y_train=np.log1p(y)
 test=X_test
y=y_train
print(codebook[codebook['Variable name']=='rez_esc'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='v18q1'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='v2a1'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='meaneduc'])
 print('----------------------------------')
 print(codebook[codebook['Variable name']=='SQBmeaned'] )<count_values>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values,y_train.values, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def eval_model(model, name):
start_time = time.time()
score = rmsle_cv(model)
 print("{} score: {:.4f}({:.4f}), execution time: {:.1f}".format(name, score.mean() , score.std() , time.time() -start_time))
df.v18q1.value_counts()<count_values>
mod_lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.005, random_state=1))
eval_model(mod_lasso, "lasso")
 mod_enet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
eval_model(mod_enet, "enet")
 mod_cat = CatBoostRegressor(iterations=10000, learning_rate=0.01,
depth=5, eval_metric='RMSE',
colsample_bylevel=0.7, random_seed = 17, silent=True,
bagging_temperature = 0.2, early_stopping_rounds=200)
 eval_model(mod_cat, "cat")
 mod_gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=10, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=5)
 eval_model(mod_gboost, "gboost")
 mod_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=10,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state=7, nthread=-1)
 eval_model(mod_xgb, "xgb")
 mod_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=1023,
learning_rate=0.05, n_estimators=650,
max_bin=58, bagging_fraction=0.80,max_depth=10,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=7, min_sum_hessian_in_leaf=11)
 eval_model(mod_lgb, "lgb" )
df.rez_esc.value_counts()<count_values>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 mod_stacked = StackingAveragedModels(base_models =(mod_cat, mod_xgb, mod_gboost, mod_lgb), meta_model = mod_lasso)
 eval_model(mod_stacked, "stacked" )
df.v2a1.value_counts()<count_values>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def predict(model):
model.fit(train.values,y_train.values)
 train_pred = model.predict(train.values)
 pred = np.expm1(model.predict(test.values))
print(rmsle(y_train, train_pred))
return(pred )
df.meaneduc.value_counts()<count_values>
prediction1=0
prediction = predict(mod_lasso)
 prediction = predict(mod_enet)
 prediction = predict(mod_xgb)
 prediction1+=prediction
prediction = predict(mod_gboost)
 prediction1+=prediction
prediction = predict(mod_lgb)
 prediction1+=prediction
prediction = predict(mod_stacked)
 prediction1+=prediction
prediction
df.SQBmeaned.value_counts()<drop_column>
new_test['id']=new_test.index
new_test['revenue']=prediction
new_test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 new_test[['id','revenue']].head()
<count_values><EOS> 
new_test['revenue']=prediction1/4
new_test[['id','revenue']].to_csv('submission_Dragon3.csv', index=False)
 new_test[['id','revenue']].head()