Kaggle competition: Two Sigma Connect: Rental Listing Inquiries
( X_train, y_train),(X_test, y_test)= mnist.load_data()<data_type_conversions>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import preprocessing
import datashader as ds
from datashader import transfer_functions as tf
num_pixels = X_train.shape[1] * X_train.shape[2]
X_train = X_train.reshape(( X_train.shape[0], num_pixels)).astype('float32')
 X_test = X_test.reshape(( X_test.shape[0], num_pixels)).astype('float32' )<feature_engineering>
train = pd.read_json(".. /input/two-sigma-connect-rental-listing-inquiries/train.json")
 train = train.reset_index()
train.pop("index")
 train.shape
X_train = X_train / 255
X_test = X_test / 255<categorify>
test = pd.read_json(".. /input/two-sigma-connect-rental-listing-inquiries/test.json")
 test = test.reset_index()
test.pop("index");
test.shape
y_train = np_utils.to_categorical(y_train)
 y_test = np_utils.to_categorical(y_test)
 num_classes = y_test.shape[1]<choose_model_class>
submission = pd.read_csv(".. /input/two-sigma-connect-rental-listing-inquiries/sample_submission.csv")
 submission.head()
def baseline_model() :
	
	model = Sequential()
	model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
	model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
	
	model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 	return model<train_model>
train["train_test"] = 'train'
test["train_test"] = 'test'
train = pd.concat([train, test],axis=0,sort=True)
 train.shape
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=200, verbose=2)
 scores = model.evaluate(X_test, y_test, verbose=0)
 print("Baseline Error: %.2f%%" %(100-scores[1]*100))<data_type_conversions>
train["price_type"] = np.log(train["price"])/(train["bedrooms"]+1)
 train["room_sum"] = train["bedrooms"]+train["bathrooms"]
train["num_features"] = train["features"].apply(len)
 train["num_description_words"] = train["description"].apply(lambda x: len(x.split(" ")) )
( X_train, y_train),(X_test, y_test)= mnist.load_data()
X_train = X_train.reshape(( X_train.shape[0], 28, 28, 1)).astype('float32')
 X_test = X_test.reshape(( X_test.shape[0], 28, 28, 1)).astype('float32')
 X_train = X_train / 255
X_test = X_test / 255
y_train = np_utils.to_categorical(y_train)
 y_test = np_utils.to_categorical(y_test)
 num_classes = y_test.shape[1]
def baseline_model() :
	
	model = Sequential()
	model.add(Conv2D(32,(5, 5), input_shape=(28, 28, 1), activation='relu'))
	model.add(MaxPooling2D())
 	model.add(Dropout(0.2))
	model.add(Flatten())
 	model.add(Dense(128, activation='relu'))
	model.add(Dense(num_classes, activation='softmax'))
	
	model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 	return model
model_simple = baseline_model()
model_simple.summary()<train_model>
ft = []
train['features'].apply(lambda x: ft.append(x))
ft=list(it.chain.from_iterable(ft))
print("Nombre d'éléments: " + str(len(ft)))
 uniq_ft = set(ft)
 print("Nombre d'éléments uniques: " + str(len(uniq_ft)) )
model_simple.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=200)
 scores = model_simple.evaluate(X_test, y_test, verbose=1)
 print("CNN Error: %.2f%%" %(100-scores[1]*100))<data_type_conversions>
image_timestamp = pd.read_csv('.. /input/hugoboum-renthop-image-timestamp/listing_image_time.csv')
 image_timestamp.columns = ['listing_id', 'timestamp']
image_timestamp.head()
( X_train, y_train),(X_test, y_test)= mnist.load_data()
X_train = X_train.reshape(( X_train.shape[0], 28, 28, 1)).astype('float32')
 X_test = X_test.reshape(( X_test.shape[0], 28, 28, 1)).astype('float32')
 X_train = X_train / 255
X_test = X_test / 255
y_train = np_utils.to_categorical(y_train)
 y_test = np_utils.to_categorical(y_test)
 num_classes = y_test.shape[1]
def large_model() :
	
	model = Sequential()
	model.add(Conv2D(30,(5, 5), input_shape=(28, 28, 1), activation='relu'))
	model.add(MaxPooling2D())
 	model.add(Conv2D(15,(3, 3), activation='relu'))
	model.add(MaxPooling2D())
 	model.add(Dropout(0.2))
	model.add(Flatten())
 	model.add(Dense(128, activation='relu'))
	model.add(Dense(50, activation='relu'))
	model.add(Dense(num_classes, activation='softmax'))
	
	model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 	return model
model_large = large_model()
model_large.summary()<train_model>
train = pd.merge(train,image_timestamp,on='listing_id',how='left' )
model_large.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=200)
 scores = model_large.evaluate(X_test, y_test, verbose=1)
 print("Large CNN Error: %.2f%%" %(100-scores[1]*100))<data_type_conversions>
latitude_median = np.median(train["latitude"])
 longitude_median = np.median(train["longitude"])
 print(latitude_median)
 print(longitude_median )
( X_train, y_train),(X_test, y_test)= mnist.load_data()
X_train = X_train.reshape(( X_train.shape[0], 28, 28, 1)).astype('float32')
 X_test = X_test.reshape(( X_test.shape[0], 28, 28, 1)).astype('float32')
 X_train = X_train / 255
X_test = X_test / 255
y_train = np_utils.to_categorical(y_train)
 y_test = np_utils.to_categorical(y_test)
 num_classes = y_test.shape[1]
def larger_model() :
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=3, padding='same', activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))
model.add(Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.1))
model.add(Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
model.add(Conv2D(filters=192, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.1))
model.add(Conv2D(filters=192, kernel_size=5, padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=2, padding='same'))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
model_larger = larger_model()
model_larger.summary()<train_model>
train['latitude'].values[train['latitude'] < 39] = latitude_median
train['latitude'].values[train['latitude'] > 41] = latitude_median
train['longitude'].values[train['longitude'] <-80] = longitude_median
train['longitude'].values[train['longitude'] > -70] = longitude_median
print('Valeurs abberantes ')
 print(train.query("latitude == 0")["latitude"].agg("count"))
print(train.query("longitude == 0")["longitude"].agg("count"))
model_larger.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=100)
 scores = model_larger.evaluate(X_test, y_test, verbose=1)
 print("Larger CNN Error: %.2f%%" %(100-scores[1]*100))<load_from_csv>
jb = jarque_bera(np.log(train["price"]))
print("p-value Jarque-Bera normality test: " + str(jb[1]))
X_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' ).values.astype('float32')
 X_test = X_test.reshape(-1, 28, 28, 1)
 X_test = X_test.astype('float32')/255
testY = model_larger.predict_classes(X_test, verbose=1 )<save_to_csv>
train.groupby("price_cat" ).agg("count" ).iloc[:,1]
sub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 sub['Label'] = testY
sub.to_csv('submission.csv',index=False )<set_options>
train["price_cat"] = train["price_cat"].astype("str" )
YouTubeVideo('3JQ3hYko51Y', width=800, height=450 )<import_modules>
le = preprocessing.LabelEncoder()
train["price_cat"] = le.fit_transform(train["price_cat"] )
import pandas as pd
import numpy as np
import time
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.models import Model
from keras.layers import Input, Conv2D, BatchNormalization, Activation
from keras.layers import Add, Flatten, AveragePooling2D, Dense, Dropout
from keras.callbacks import ModelCheckpoint, LearningRateScheduler
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import plot_model<load_from_csv>
streets = train.groupby(train["display_address"] ).agg(["count"] ).iloc[:,1]
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 X = train.drop(columns=['label'] ).values.reshape(-1, 28, 28, 1)/ 255
y = train['label'].values
test = test.values.reshape(-1, 28, 28, 1)/ 255
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1 )<prepare_x_and_y>
k=100
topstreets = streets.sort_values(ascending=False ).iloc[:k]
topstreets = pd.DataFrame(topstreets)
 topstreets["street"] = topstreets.index
topstreets = topstreets.unstack().iloc[:k]
topstreets = topstreets.reset_index().drop(["level_0","level_1"],axis=1)
 topstreets.columns = ["display_address","display_address_count"]
topstreets.head()
def residual_block(inputs, filters, strides=1):

 y = inputs
x = Conv2D(
 filters=filters,
kernel_size=3,
strides=strides,
padding='same',
)(inputs)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Conv2D(
 filters=filters,
kernel_size=3,
strides=1,
padding='same',
)(x)
 x = BatchNormalization()(x)
 if strides > 1:
y = Conv2D(
 filters=filters,
kernel_size=3,
strides=strides,
padding='same',
)(y)
 y = BatchNormalization()(y)
 x = Add()([x, y])
 x = Activation('relu' )(x)
 return x
def resnet(input_shape, num_classes, filters, stages):

 inputs = Input(shape=input_shape)
 x = Conv2D(
 filters=filters,
kernel_size=7,
strides=1,
padding='same',
)(inputs)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 for stage in stages:
x = residual_block(x, filters, strides=2)
 for i in range(stage-1):
x = residual_block(x, filters)
 filters *= 2
x = AveragePooling2D(4 )(x)
 x = Flatten()(x)
 x = Dropout(0.3 )(x)
 x = Dense(int(filters/4), activation='relu' )(x)
 outputs = Dense(num_classes, activation='softmax' )(x)
 model = Model(inputs=inputs, outputs=outputs)
 return model<choose_model_class>
train = pd.merge(train,topstreets,on='display_address',how='left' )
def train_model(epochs, filters, stages, batch_size, visualize=False):

 model = resnet(
 input_shape=X[0].shape,
num_classes=np.unique(y ).shape[-1],
filters=filters,
stages=stages 
)
 model.compile(
 loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'] 
)
 checkpoint = ModelCheckpoint(
 filepath=f'resnet-{int(time.time())}.dhf5',
monitor='loss',
save_best_only=True 
)
 annealer = LearningRateScheduler(lambda x: 1e-3 * 0.8**x)
 callbacks = [checkpoint, annealer]
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1 
)
 datagen.fit(X)
 history = model.fit_generator(
 datagen.flow(X_train, y_train, batch_size=batch_size),
validation_data=(X_test, y_test),
epochs=epochs,
verbose=2,
workers=12,
callbacks=callbacks 
)
 if visualize:
fig, axarr = plt.subplots(1, 2, figsize=(16, 8))
axarr[0].plot(history.history['accuracy'])
 axarr[0].plot(history.history['val_accuracy'])
 axarr[0].set_title('Model accuracy')
 axarr[0].set_ylabel('Accuracy')
 axarr[0].set_xlabel('Epoch')
 axarr[0].legend(['Train', 'Test'], loc='upper left')
 axarr[1].plot(history.history['loss'])
 axarr[1].plot(history.history['val_loss'])
 axarr[1].set_title('Model loss')
 axarr[1].set_ylabel('Loss')
 axarr[1].set_xlabel('Epoch')
 axarr[1].legend(['Train', 'Test'], loc='upper left')
 plt.show()
return model<train_model>
managers = train.groupby(train["manager_id"] ).agg(["count"] ).iloc[:,1]
managers = pd.DataFrame(managers)
 managers = managers.unstack()
managers["manager_id"] = managers.index
managers = managers.reset_index().drop(["level_0","level_1"],axis=1)
 managers.columns = ["manager_id","manager_id_count"]
managers = managers.iloc[:-1,:]
managers.head()
models = []
for i in range(1):
print('-------------------------')
 print('Model: ', i+1)
 print('-------------------------')
 model = train_model(
 epochs=10,
filters=64,
stages=[3, 3, 3],
batch_size=128,
visualize=True 
)
 models.append(model )<save_to_csv>
train = pd.merge(train,managers,on='manager_id',how='left' )
predictions = []
for model in models:
predictions.append(model.predict(test))
predictions = np.sum(predictions, axis=0)
 predictions = np.argmax(predictions, axis=1)
 submission = pd.DataFrame({'ImageId': np.arange(1, 28001, 1), 'Label': predictions})
 submission.to_csv('mnist_resnet_submission.csv', index=False )<import_modules>
train["created"] = train["created"].astype("datetime64" )
import plotly.graph_objects as go
import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from datetime import datetime
from pathlib import Path
from sklearn import preprocessing
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout
from keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler, MinMaxScaler<load_from_csv>
train["year"] = train["created"].dt.year
train["month"] = train["created"].dt.month
train["day"] = train["created"].dt.dayofweek
train["hour"] = train["created"].dt.hour
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 train.tail()<define_variables>
train_numeric = train._get_numeric_data()
train_numeric = pd.concat([train_numeric,train["interest_level"],train["train_test"]],axis=1)
 train_numeric.head()
mask = train['Date'].max()
world_cum_confirmed = sum(train[train['Date'] == mask].ConfirmedCases)
 world_cum_fatal = sum(train[train['Date'] == mask].Fatalities )<train_model>
train_numeric = train_numeric.drop(['year','train_test'],axis=1);
print('Number of Countires are: ', len(train['Country_Region'].unique()))
print('Training dataset ends at: ', mask)
 print('Number of cumulative confirmed cases worldwide are: ', world_cum_confirmed)
 print('Number of cumulative fatal cases worldwide are: ', world_cum_fatal )<sort_values>
le = preprocessing.LabelEncoder()
train_numeric["interest_level"] = le.fit_transform(train_numeric["interest_level"].fillna('0'))
np.unique(train_numeric["interest_level"] )
cum_per_country = train[train['Date'] == mask].groupby(['Date','Country_Region'] ).sum().sort_values(['ConfirmedCases'], ascending=False)
 cum_per_country[:10]<groupby>
train_numeric = train_numeric.query('interest_level>0' )
date = train['Date'].unique()
cc_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().ConfirmedCases
ft_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().Fatalities
cc_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().ConfirmedCases
ft_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().Fatalities
cc_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().ConfirmedCases
ft_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().Fatalities
cc_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().ConfirmedCases
ft_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().Fatalities
cc_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().ConfirmedCases
ft_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().Fatalities
fig = go.Figure()
fig.add_trace(go.Scatter(x=date, y=cc_us, name='US'))
fig.add_trace(go.Scatter(x=date, y=cc_ity, name='Italy'))
fig.add_trace(go.Scatter(x=date, y=cc_spn, name='Spain'))
fig.add_trace(go.Scatter(x=date, y=cc_gmn, name='Germany'))
fig.add_trace(go.Scatter(x=date, y=cc_frc, name='France'))
fig.update_layout(title="Plot of Cumulative Cases for Top 5 countires(except China)",
xaxis_title="Date",
yaxis_title="Cases")
 fig.update_xaxes(nticks=30)
 fig.show()<data_type_conversions>
vectorizer = TfidfVectorizer()
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train['Country_Region'] = train['Country_Region'].astype(str)
 test['Country_Region'] = test['Country_Region'].astype(str)
<feature_engineering>
tfidfeats = vectorizer.fit_transform(train["description"] )
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
train['Province_State'].fillna(EMPTY_VAL, inplace=True)
 train['Province_State'] = train.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test['Province_State'].fillna(EMPTY_VAL, inplace=True)
 test['Province_State'] = test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1 )<categorify>
from sklearn.decomposition import TruncatedSVD
le = preprocessing.LabelEncoder()
train['country_encoder'] = le.fit_transform(train['Country_Region'])
 train['date_int'] = train['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int)
 test['country_encoder'] = le.transform(test['Country_Region'])
 test['date_int'] = test['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int )<categorify>
svd = TruncatedSVD(n_components=100 )
le = preprocessing.LabelEncoder()
train['province_encoder'] = le.fit_transform(train['Province_State'])
 test['province_encoder'] = le.transform(test['Province_State'] )<count_duplicates>
lsa_feats = svd.fit_transform(tfidfeats )
start_time = time.time()
country = train['Country_Region'].drop_duplicates()
train_df = train.copy()
train_df.rename(columns={'Date': 'date', 'ConfirmedCases': 'cc_cases', 'Fatalities': 'ft_cases', 'Country_Region': 'country', 'Province_State': 'province'}, inplace=True)
 lags = np.arange(1,8,1)
 with tqdm(total = len(list(train_df['date'].unique())))as pbar:
for d in train_df['date'].drop_duplicates() :
for i in country:
province = train_df[train_df['country'] == i]['province'].drop_duplicates()
for j in province:
mask =(train_df['date'] == d)&(train_df['country'] == i)&(train_df['province'] == j)
 for lag in lags:
mask_org =(train_df['date'] ==(d - pd.Timedelta(days=lag)))&(train_df['country'] == i)&(train_df['province'] == j)
 try:
train_df.loc[mask, 'cc_cases_' + str(lag)] = train_df.loc[mask_org, 'cc_cases'].values
except:
train_df.loc[mask, 'cc_cases_' + str(lag)] = 0
try:
train_df.loc[mask, 'ft_cases_' + str(lag)] = train_df.loc[mask_org, 'ft_cases'].values
except:
train_df.loc[mask, 'ft_cases_' + str(lag)] = 0
pbar.update(1)
 print('Time spent for building features is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<save_to_csv>
lsa_feats = pd.DataFrame(lsa_feats)
 lsa_feats_colnames = ["lsa_" + str(i+1)for i in range(100)]
lsa_feats.columns = lsa_feats_colnames
train = pd.concat([train,lsa_feats],axis=1 )
train_df.to_csv(Path('/kaggle/working', 'train_df.csv'))
train_df = pd.read_csv(Path('/kaggle/working/', 'train_df.csv'), index_col = 0, parse_dates = ['date'])
 train_df[train_df['country'] == 'Italy'].tail()<split>
train["features"] = train["features"].astype("str" )
def split_train_val(df, val_ratio):
val_len = int(len(df)* val_ratio)
 train_set = df[:-val_len]
val_set = df[-val_len:]
return train_set, val_set<prepare_x_and_y>
tfidfeats2 = vectorizer.fit_transform(train["features"] )
test_fixed_cols = ['ForecastId', 'Province_State', 'Country_Region', 'Date']
fixed_cols = ['Id', 'province', 'country', 'date']
output_cols = ['cc_cases', 'ft_cases']
input_cols = list(set(train_df.columns.to_list())- set(fixed_cols)- set(output_cols))
print('output columns are ', output_cols)
 print('input columns are ', input_cols)
 X = train_df[input_cols]
y = train_df[output_cols]<split>
svd2 = TruncatedSVD(n_components=20 )
cc_input = ['cc_cases_1', 'cc_cases_2', 'cc_cases_3', 'cc_cases_4', 'cc_cases_5', 'cc_cases_6', 'cc_cases_7', 'country_encoder', 'province_encoder', 'date_int']
ft_input = ['ft_cases_1', 'ft_cases_2', 'ft_cases_3', 'ft_cases_4', 'ft_cases_5', 'ft_cases_6', 'ft_cases_7', 'country_encoder', 'province_encoder', 'date_int']
cc_output = ['cc_cases']
ft_output = ['ft_cases']
X_cc = X[cc_input]
X_ft = X[ft_input]
y_cc = y[cc_output]
y_ft = y[ft_output]
train_X_cc, val_X_cc = split_train_val(df = X_cc, val_ratio = 0.1)
 train_y_cc, val_y_cc = split_train_val(df = y_cc, val_ratio = 0.1)
 train_X_ft, val_X_ft = split_train_val(df = X_ft, val_ratio = 0.1)
 train_y_ft, val_y_ft = split_train_val(df = y_ft, val_ratio = 0.1 )<randomize_order>
lsa_feats2 = svd2.fit_transform(tfidfeats2 )
idx = np.random.RandomState(seed=42 ).permutation(train_X_cc.index)
 train_X_cc = train_X_cc.reindex(idx)
 train_y_cc = train_y_cc.reindex(idx)
 train_X_ft = train_X_ft.reindex(idx)
 train_y_ft = train_y_ft.reindex(idx)
<normalization>
lsa_feats2 = pd.DataFrame(lsa_feats2)
 lsa_feats2_colnames = ["lsa2_" + str(i+1)for i in range(20)]
lsa_feats2.columns = lsa_feats2_colnames
train = pd.concat([train,lsa_feats2],axis=1 )
<normalization>
from textblob import TextBlob
<data_type_conversions>
description_polarity, description_subjectivity = [],[]
for desc in train["description"]:
desc_blob= TextBlob(desc)
 description_polarity.append(desc_blob.sentiment.polarity)
 description_subjectivity.append(desc_blob.sentiment.subjectivity)
 train["description_polarity"] = description_polarity
train["description_subjectivity"] = description_subjectivity
X_train_cc = train_X_cc.to_numpy()
X_val_cc = val_X_cc.to_numpy()
X_train_ft = train_X_ft.to_numpy()
X_val_ft = val_X_ft.to_numpy()
y_train_cc = train_y_cc.to_numpy()
y_val_cc = val_y_cc.to_numpy()
y_train_ft = train_y_ft.to_numpy()
y_val_ft = val_y_ft.to_numpy()<compute_test_metric>
train["photos_count"] = train.photos.apply(len )
def root_mean_squared_log_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(K.log(y_pred + 1)- K.log(y_true + 1))))<choose_model_class>
import lightgbm as lgb
from sklearn.model_selection import GridSearchCV
def LSTM_model(n_1, input_dim, output_dim):
model = Sequential()
model.add(LSTM(n_1,input_shape=(1, input_dim), activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(output_dim, activation='relu'))
model.compile(loss=root_mean_squared_log_error, optimizer='adam')
 print(model.summary())
 return model<choose_model_class>
train = train.drop(['building_id','created','description','display_address',
'features','manager_id','photos','street_address'],axis=1)
 train['display_address_count'] = train['display_address_count'].astype("float")
 train['manager_id_count'] = train['manager_id_count'].astype("float")
model_cc = LSTM_model(4, X_train_cc.shape[-1], y_train_cc.shape[-1])
 model_ft = LSTM_model(4, X_train_ft.shape[-1], y_train_ft.shape[-1])
 early_stop = EarlyStopping(monitor='loss', patience=5, verbose=0, mode='min' )<train_model>
test = train[train.train_test == "test"]
train = train[train.train_test == "train"]
train.pop("train_test")
 train_target = train.pop("interest_level")
 test = test.drop(['interest_level','train_test'],axis=1)
 train.head()
print('Start model training')
 start_time = time.time()
history_cc = model_cc.fit(X_train_cc, y_train_cc, epochs = 100,validation_data =(X_val_cc, y_val_cc), verbose = 2, callbacks=[early_stop])
 model_cc.save("model_cc.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<train_model>
gradient_booster = lgb.LGBMClassifier()
print('Start model training')
 start_time = time.time()
history_ft = model_ft.fit(X_train_ft, y_train_ft, epochs = 100,validation_data =(X_val_ft, y_val_ft), verbose = 2, callbacks=[early_stop])
 model_ft.save("model_ft.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<predict_on_test>
grid_parameters = {'n_estimators': [100,500,1000],'num_leaves': [10,30,50],
'reg_alpha' : [0.2,0.4,0.6],'learning_rate' : [0.05,0.1]}
yhat_val_cc = model_cc.predict(X_val_cc)
 print(yhat_val_cc )<predict_on_test>
gs = GridSearchCV(estimator=gradient_booster, param_grid = grid_parameters,
cv= 4, scoring= 'neg_log_loss' )
yhat_val_ft = model_cc.predict(X_val_ft)
 print(yhat_val_ft )<data_type_conversions>
gs.fit(train,train_target )
start_time = time.time()
test['Country_Region'] = test['Country_Region'].astype(str)
 test['Province_State'] = test['Province_State'].astype(str)
 country = test['Country_Region'].drop_duplicates()
adj_input_cols = [e for e in input_cols if e not in('province_encoder', 'country_encoder', 'date_int')]
test_df = test.copy().join(pd.DataFrame(columns = adj_input_cols + output_cols))
test_df.rename(columns={'Date': 'date', 'Country_Region': 'country', 'Province_State': 'province'}, inplace=True)
 lags = np.arange(1,8,1)
 test_overlap_mask =(test_df['date'] <= train_df['date'].max())
 train_overlap_mask =(train_df['date'] >= test_df['date'].min())
 test_df.loc[test_overlap_mask, input_cols + output_cols] = train_df.loc[train_overlap_mask, input_cols + output_cols].values
pred_dt_range = pd.date_range(start = train_df['date'].max() + pd.Timedelta(days=1), end = test_df['date'].max() , freq = '1D')
 with tqdm(total = len(pred_dt_range)) as pbar:
for d in pred_dt_range:
for i in country:
province = test_df[test_df['country'] == i]['province'].drop_duplicates()
for j in province:
mask =(test_df['date'] == d)&(test_df['country'] == i)&(test_df['province'] == j)
 for lag in lags:
mask_org =(test_df['date'] ==(d - pd.Timedelta(days=lag)))&(test_df['country'] == i)&(test_df['province'] == j)
 try:
test_df.loc[mask, 'cc_cases_' + str(lag)] = test_df.loc[mask_org, 'cc_cases'].values
except:
test_df.loc[mask, 'cc_cases_' + str(lag)] = 0
try:
test_df.loc[mask, 'ft_cases_' + str(lag)] = test_df.loc[mask_org, 'ft_cases'].values
except:
test_df.loc[mask, 'ft_cases_' + str(lag)] = 0
test_X = test_df.loc[mask, input_cols]
test_X_cc = test_X[cc_input]
X_test_cc= test_X_cc
X_test_cc = X_test_cc.to_numpy().reshape(X_test_cc.shape[0], 1, X_test_cc.shape[1])
 next_cc = model_cc.predict(X_test_cc)
 next_cc_scaled = next_cc
test_X_ft = test_X[ft_input]
X_test_ft = test_X_ft
X_test_ft = X_test_ft.to_numpy().reshape(X_test_ft.shape[0], 1, X_test_ft.shape[1])
 next_ft = model_cc.predict(X_test_ft)
 next_ft_scaled = next_ft
test_df.loc[mask, 'cc_cases'] = next_cc_scaled
test_df.loc[mask, 'ft_cases'] = next_ft_scaled
pbar.update(1)
 print('Time spent for building features is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<create_dataframe>
best_log_loss = -1 * np.max(gs.cv_results_['mean_test_score'])
 print("La log-loss de validation croisée du meilleur parametrage est de " + str(best_log_loss))
submission = pd.DataFrame()
submission['ForecastId'] = test_df['ForecastId']
submission['ConfirmedCases'] = test_df['cc_cases']
submission['Fatalities'] = test_df['ft_cases']<save_to_csv>
gs.best_estimator_
submission.to_csv("submission.csv",index=False )<load_from_csv>
gs.best_estimator_.classes_
train = pd.read_csv(".. /input/train.csv")
 print(train.shape)
 train.head()<load_from_csv>
predictions = gs.predict_proba(test)
 predictions = pd.DataFrame(predictions)
 predictions.columns = ['high','low','medium']
test= pd.read_csv(".. /input/test.csv")
 print(test.shape)
 test.head()<split>
submission['high'] = predictions['high']
submission['medium'] = predictions['medium']
submission['low'] = predictions['low']
submission.head()
<data_type_conversions><EOS> 
submission.to_csv('projet ML.csv', index=False);