Kaggle competition: TalkingData Mobile User Demographics
<drop_column><EOS> 
%matplotlib inline
<SOS> metric: MulticlassLoss Kaggle data source: talkingdata-mobile-user-demographics<load_from_csv>
%matplotlib inline
test_whole = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv',index_col='Date',parse_dates=True)
 test_whole = test_whole.fillna(value='empty' )<feature_engineering>
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
test_whole['state_with_country'] = test_whole['Province_State'] +'_'+ test_whole['Country_Region']
test_whole.head()<define_variables>
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
state_i_data_for_test=[]
for i in state_with_country_name_for_test:
state_i_data_for_test.append(i)
<feature_engineering>
gatrain['trainrow'] = np.arange(gatrain.shape[0])
 gatest['testrow'] = np.arange(gatest.shape[0] )
for i,j in enumerate(state_with_country_name_for_test):
state_i_data_for_test[i] = test_whole[test_whole['state_with_country'] == j ]
state_i_data_for_test[i] = state_i_data_for_test[i].iloc[21:]
state_i_data_for_test[i] = state_i_data_for_test[i].drop(['ForecastId','state_with_country','Province_State','Country_Region'],axis=1)
 state_i_data_for_test.append(state_i_data_for_test[i] )<import_modules>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,LSTM
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator<normalization>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
result = []
full_scaler = MinMaxScaler()<train_on_grid>
m = phone.phone_brand.str.cat(phone.device_model)
 modelencoder = LabelEncoder().fit(m)
 phone['model'] = modelencoder.transform(m)
 gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.model)))
 Xte_model = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.model)))
 print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))
for i in range(len(state_with_country_name_for_test)) :
scaled_full_data = full_scaler.fit_transform(state_i_data[i])
 length = 1
batch_size = 1
generator = TimeseriesGenerator(scaled_full_data, scaled_full_data, length=length, batch_size=1)
 model = Sequential()
model.add(LSTM(96,input_shape=(length,scaled_full_data.shape[1])))
 model.add(Dense(scaled_full_data.shape[1]))
model.compile(optimizer='adam', loss='mse')
 model.fit_generator(generator,epochs=6)
 n_features = scaled_full_data.shape[1]
test_predictions = []
first_eval_batch = scaled_full_data[-length:]
current_batch = first_eval_batch.reshape(( 1, length, n_features))
for j in range(len(state_i_data_for_test[i])) :
current_pred = model.predict(current_batch)[0]
test_predictions.append(current_pred)
 current_batch = np.append(current_batch[:,1:,:],[[current_pred]],axis=1)
 true_predictions = full_scaler.inverse_transform(test_predictions)
 true_predictions = true_predictions.round()
true_predictions = pd.DataFrame(data=true_predictions,columns=state_i_data[1].columns)
 result.append(true_predictions)
 print('count:-',i)
<prepare_output>
appencoder = LabelEncoder().fit(appevents.app_id)
 appevents['app'] = appencoder.transform(appevents.app_id)
 napps = len(appencoder.classes_)
 deviceapps =(appevents.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)
.groupby(['device_id','app'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 deviceapps.head()
prediction = pd.DataFrame(data= state_i_data[0].iloc[-21:] ,columns=['ConfirmedCases','Fatalities'])
 prediction = prediction.append(result[0] )<prepare_output>
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))
for i in range(1,len(result)) :
prediction = prediction.append(state_i_data[i].iloc[-21:])
 prediction = prediction.append(result[i] )<define_variables>
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )
prediction.index = range(0,len(prediction))<load_from_csv>
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelencoder = LabelEncoder().fit(applabels.label_id)
 applabels['label'] = labelencoder.transform(applabels.label_id)
 nlabels = len(labelencoder.classes_ )
sub_format = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )<drop_column>
devicelabels =(deviceapps[['device_id','app']] 
.merge(applabels[['app','label']])
.groupby(['device_id','label'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 devicelabels.head()
sub_format = sub_format['ForecastId']<concatenate>
d = devicelabels.dropna(subset=['trainrow'])
 Xtr_label = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.label)) ,
shape=(gatrain.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
 Xte_label = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.label)) ,
shape=(gatest.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))
final = pd.concat([sub_format,prediction],axis=1 )<save_to_csv>
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))
final.to_csv('submission.csv',index=False )<load_from_csv>
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))
base_folder = '/kaggle/input/'
data_base = base_folder + 'covid19-global-forecasting-week-2/'
df = pd.read_csv(data_base + 'train.csv')
 df.rename(columns={'Province_State': 'Province/State', 'Country_Region': 'Country/Region'}, inplace=True)
 df['Province/State'].fillna('entire country', inplace=True)
 df<feature_engineering>
targetencoder = LabelEncoder().fit(gatrain.gender)
 y = targetencoder.transform(gatrain.gender)
 nclasses = len(targetencoder.classes_ )
def add_extra_features_from_previous_days(data_fr, tail_size=5):
cols_tmp = []
col_prefix = 'PreviousDay'
for i in range(0, tail_size):
col_cc = '{}-{}ConfirmedCases'.format(col_prefix, i)
 col_f = '{}-{}Fatalities'.format(col_prefix, i)
 data_fr[col_cc] = data_fr.groupby(['Country/Region', 'Province/State'])['ConfirmedCases'].shift(periods=i+1, fill_value=0)
 data_fr[col_f] = data_fr.groupby(['Country/Region', 'Province/State'])['Fatalities'].shift(periods=i+1, fill_value=0)
 data_fr[col_cc + 'Delta'] = data_fr.groupby(['Country/Region', 'Province/State'])[col_cc].diff().fillna(0)
 data_fr[col_f + 'Delta'] = data_fr.groupby(['Country/Region', 'Province/State'])[col_f].diff().fillna(0)
 cols_tmp += [col_cc, col_f, col_cc + 'Delta', col_f + 'Delta']
return cols_tmp
TAIL = 5
previous_days_cols = add_extra_features_from_previous_days(df, TAIL)
 df<data_type_conversions>
clf = LogisticRegression(C=0.08)
 clf.fit(Xtrain[70001:], y[70001:])
 pred = pd.DataFrame(clf.predict_proba(Xtrain[70001:]), index=gatrain.iloc[70001:].index, columns=targetencoder.classes_)
 pred.head()
<feature_engineering><EOS> 
pred.to_csv('test_gender.csv',index=True )