Kaggle competition: TalkingData Mobile User Demographics
<data_type_conversions><EOS> 
%matplotlib inline
<SOS> metric: MulticlassLoss Kaggle data source: talkingdata-mobile-user-demographics<compute_test_metric>
%matplotlib inline
def root_mean_squared_log_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(K.log(y_pred + 1)- K.log(y_true + 1))))<choose_model_class>
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
def LSTM_model(n_1, input_dim, output_dim):
model = Sequential()
model.add(LSTM(n_1,input_shape=(1, input_dim), activation='relu'))
model.add(Dense(output_dim, activation='relu'))
model.compile(loss=root_mean_squared_log_error, optimizer='adam')
 return model<choose_model_class>
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
K.clear_session()
model_cc = LSTM_model(4, X_train_cc.shape[-1], y_train_cc.shape[-1])
 model_ft = LSTM_model(4, X_train_ft.shape[-1], y_train_ft.shape[-1])
 early_stop_cc = EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='min')
 early_stop_ft = EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='min' )<train_model>
gatrain['trainrow'] = np.arange(gatrain.shape[0])
 gatest['testrow'] = np.arange(gatest.shape[0] )
print('Start model training')
 start_time = time.time()
history_cc = model_cc.fit(X_train_cc, y_train_cc, batch_size = 16, epochs = 100,validation_data =(X_val_cc, y_val_cc), verbose = 2, callbacks=[early_stop_cc])
 model_cc.save("model_cc.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<train_model>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
print('Start model training')
 start_time = time.time()
history_ft = model_ft.fit(X_train_ft, y_train_ft, batch_size = 16, epochs = 8,validation_data =(X_val_ft, y_val_ft), verbose = 2, callbacks=[early_stop_ft])
 model_ft.save("model_ft.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )<predict_on_test>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
yhat_val_cc = model_cc.predict(X_val_cc)
 print(yhat_val_cc[50:70] )<predict_on_test>
m = phone.phone_brand.str.cat(phone.device_model)
 modelencoder = LabelEncoder().fit(m)
 phone['model'] = modelencoder.transform(m)
 gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.model)))
 Xte_model = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.model)))
 print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))
yhat_val_ft = model_ft.predict(X_val_ft)
 print(yhat_val_ft[60:70] )<create_dataframe>
appencoder = LabelEncoder().fit(appevents.app_id)
 appevents['app'] = appencoder.transform(appevents.app_id)
 napps = len(appencoder.classes_)
 deviceapps =(appevents.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)
.groupby(['device_id','app'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 deviceapps.head()
submission = pd.DataFrame()
submission['ForecastId'] = test_df['ForecastId']
submission['ConfirmedCases'] = test_df['cc_cases']
submission['Fatalities'] = test_df['ft_cases']<save_to_csv>
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))
submission.to_csv("submission.csv",index=False )<import_modules>
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os<load_from_csv>
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )
data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv',index_col='Date',parse_dates=True )<categorify>
devicelabels =(deviceapps[['device_id','app']] 
.merge(applabels[['app','label']])
.groupby(['device_id','label'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 devicelabels.head()
data = data.fillna(value='empty' )<feature_engineering>
d = devicelabels.dropna(subset=['trainrow'])
 Xtr_label = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.label)) ,
shape=(gatrain.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
 Xte_label = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.label)) ,
shape=(gatest.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))
data['state_with_country'] = data['Province_State'] +'_'+ data['Country_Region']<drop_column>
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))
data= data.drop(labels=['Province_State','Country_Region','Id'],axis=1 )<define_variables>
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))
state_i_data =[]
for i in state_with_country_name:
state_i_data.append(i)
<drop_column>
targetencoder = LabelEncoder().fit(gatrain.group)
 y = targetencoder.transform(gatrain.group)
 nclasses = len(targetencoder.classes_ )
for i,j in enumerate(state_with_country_name):
state_i_data[i] = data[data['state_with_country'] == j ]
state_i_data[i] = state_i_data[i].drop(['state_with_country'],axis=1)
<load_from_csv>
def score(clf, random_state = 0):
kf = StratifiedKFold(y, n_folds=5, shuffle=True, random_state=random_state)
 pred = np.zeros(( y.shape[0],nclasses))
for itrain, itest in kf:
Xtr, Xte = Xtrain[itrain, :], Xtrain[itest, :]
ytr, yte = y[itrain], y[itest]
clf.fit(Xtr, ytr)
 pred[itest,:] = clf.predict_proba(Xte)
 return log_loss(yte, pred[itest, :])
 print("{:.5f}".format(log_loss(yte, pred[itest,:])) , end=' ')
 print('')
 return log_loss(y, pred )
prev_sub = pd.read_csv('/kaggle/input/covid19-forecasting-using-rnn/submission.csv')
 test_whole = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv',index_col='Date',parse_dates=True)
 test_whole = test_whole.fillna(value='empty')
<feature_engineering>
score(LogisticRegression(C=0.02))
test_whole['state_with_country'] = test_whole['Province_State'] +'_'+ test_whole['Country_Region']
test_whole.head()<feature_engineering>
score(LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs'))
prev_sub.index = test_whole.index<prepare_output>
clf = LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs')
 clf.fit(Xtrain, y)
 pred = pd.DataFrame(clf.predict_proba(Xtest), index = gatest.index, columns=targetencoder.classes_)
 pred.head()
<define_variables><EOS> 
pred.to_csv('logreg_subm.csv',index=True )