Kaggle competition: TalkingData Mobile User Demographics
<SOS> metric: MulticlassLoss Kaggle data source: talkingdata-mobile-user-demographics<prepare_output>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import os
from sklearn.preprocessing import LabelEncoder
from scipy.sparse import csr_matrix, hstack
import xgboost as xgb
from sklearn.cross_validation import StratifiedKFold
from sklearn.metrics import log_loss
for i in range(1,len(result)) :
prediction = prediction.append(state_i_data[i].iloc[-21:])
 prediction = prediction.append(result[i] )<define_variables>
def bintod(x):
if x < 3:
return 0
elif x < 8:
return 1
elif x < 20:
return 2
elif x < 23:
return 3
else:
return 0
def lngregion(x):
if x < 80:
return 1
elif x < 90:
return 2
elif x < 100:
return 3
elif x < 110:
return 4
elif x < 120:
return 5
elif x < 130:
return 6
elif x < 140:
return 7
else:
return 0
nlng = 8
nlat = 9
def latregion(x):
if x < 20:
return 1
elif x < 25:
return 2
elif x < 30:
return 3
elif x < 35:
return 4
elif x < 40:
return 5
elif x < 45:
return 6
elif x < 50:
return 7
elif x < 55:
return 8
else:
return 0
prediction.index = range(0,len(prediction)) 
<load_from_csv>
datadir = '.. /input'
gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),
index_col='device_id')
 gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),
index_col = 'device_id')
 phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))
phone = phone.drop_duplicates('device_id',keep='first' ).set_index('device_id')
 events = pd.read_csv(os.path.join(datadir,'events.csv'),
parse_dates=['timestamp'], index_col='event_id')
 appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
usecols=['event_id','app_id','is_active'],
dtype={'is_active':bool})
 applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
labelcat = pd.read_csv(os.path.join(datadir,'label_categories.csv'))
labelcat['category']=labelcat['category'].fillna('label-missing')
 labelcat.head()
sub_format = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )<drop_column>
gatrain['trainrow'] = np.arange(gatrain.shape[0])
 gatest['testrow'] = np.arange(gatest.shape[0] )
sub_format = sub_format['ForecastId']<concatenate>
events['longitude'] = events['longitude'].round(0)
 events['latitude'] = events['latitude'].round(0)
 events['longitude'] = events['longitude'].clip_lower(73.0 ).replace(73.0, np.NaN)
 events['longitude'] = events['longitude'].clip_upper(135.0 ).replace(135.0, np.NaN)
 events['latitude'] = events['latitude'].clip_lower(15.0 ).replace(15.0, np.NaN)
 events['latitude'] = events['latitude'].clip_upper(60.0 ).replace(60.0, np.NaN)
 events['latitude2'] =events.groupby(['device_id'])['latitude'].transform(lambda x: x.mode())
 events['longitude2'] =events.groupby(['device_id'])['longitude'].transform(lambda x: x.mode() )
final = pd.concat([sub_format,prediction],axis=1 )<save_to_csv>
events_latlng = events[['device_id', 'latitude2','longitude2']].drop_duplicates('device_id', keep='first')
 events_latlng = events_latlng.set_index('device_id')
 print('Number of devices with some lat long info',len(events_latlng['latitude2']))
print('out of that missing longitude: ', sum(events_latlng['longitude2'].isnull()))
print('out of that missing latitude: ', sum(events_latlng['latitude2'].isnull()))
events_latlng['lng_region'] = events_latlng['longitude2'].apply(lngregion)
 events_latlng['lat_region'] = events_latlng['latitude2'].apply(latregion)
 print("Frequencies longitude region:" '
', events_latlng['lng_region'].value_counts())
 print("Frequencies latitude region:" '
', events_latlng['lat_region'].value_counts() )
final.to_csv('submission.csv',index=False )<import_modules>
gatrain['lng_region'] = events_latlng['lng_region']
gatest['lng_region'] = events_latlng['lng_region']
gatrain['lng_region'] = gatrain.lng_region.fillna(value=0.0)
 gatest['lng_region'] = gatest.lng_region.fillna(value=0.0)
 Xtr_lng = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.lng_region)) ,
shape=(gatrain.shape[0],nlng))
Xte_lng = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.lng_region)) ,
shape=(gatest.shape[0],nlng))
print('Longitude features: train shape {}, test shape {}'.format(Xtr_lng.shape, Xte_lng.shape))
import plotly.graph_objects as go
import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from datetime import datetime
from pathlib import Path
from sklearn import preprocessing
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, LSTM, RNN, Dropout
from keras.callbacks import EarlyStopping
from keras import optimizers
from sklearn.preprocessing import StandardScaler, MinMaxScaler<load_from_csv>
gatrain['lat_region'] = events_latlng['lat_region']
gatest['lat_region'] = events_latlng['lat_region']
gatrain['lat_region'] = gatrain.lat_region.fillna(value=0.0)
 gatest['lat_region'] = gatest.lat_region.fillna(value=0.0)
 Xtr_lat = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.lat_region)) ,
shape=(gatrain.shape[0],nlat))
Xte_lat = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.lat_region)) ,
shape=(gatest.shape[0],nlat))
print('Latitude features: train shape {}, test shape {}'.format(Xtr_lat.shape, Xte_lat.shape))
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 train.tail()<define_variables>
events['timeadj'] =(events['longitude2']- 116.41)*4
events['timeadj'] = events['timeadj'].fillna(0.0 ).astype(int)
 events['timestamp2'] = events['timestamp'] + events['timeadj'].values.astype('timedelta64[m]')
 events['todh'] = events['timestamp2'].map(lambda x : x.hour)
 events['tod'] = events['todh'].apply(bintod)
 ntod = 4
eventod =(events.groupby(['device_id','tod'])['tod'].agg(['size'])
.merge(gatrain[['trainrow']], how='left',left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left',left_index=True, right_index=True)
.reset_index())
 eventod.head()
mask = train['Date'].max()
world_cum_confirmed = sum(train[train['Date'] == mask].ConfirmedCases)
 world_cum_fatal = sum(train[train['Date'] == mask].Fatalities )<train_model>
eventod['size'] = eventod['size'].map(lambda x: np.log(x+1))
t = eventod.dropna(subset=['trainrow'])
 Xtr_tod = csr_matrix(( t['size'].values,(t.trainrow, t.tod)) ,
shape=(gatrain.shape[0],ntod))
t = eventod.dropna(subset=['testrow'])
 Xte_tod = csr_matrix(( t['size'].values,(t.testrow, t.tod)) ,
shape=(gatest.shape[0],ntod))
print('TOD data: train shape {}, test shape {}'.format(Xtr_tod.shape, Xte_tod.shape))
print('Number of Countires are: ', len(train['Country_Region'].unique()))
print('Training dataset ends at: ', mask)
 print('Number of cumulative confirmed cases worldwide are: ', world_cum_confirmed)
 print('Number of cumulative fatal cases worldwide are: ', world_cum_fatal )<sort_values>
events["dow"] = events["timestamp"].map(lambda x : x.dayofweek)
 events['wkend'] = 0
events.ix[events.dow > 4,'wkend'] = 1
ndow = 2
evendow =(events.groupby(['device_id','wkend'])['wkend'].agg(['size'])
.merge(gatrain[['trainrow']], how='left',left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left',left_index=True, right_index=True)
.reset_index())
 evendow.head()
cum_per_country = train[train['Date'] == mask].groupby(['Date','Country_Region'] ).sum().sort_values(['ConfirmedCases'], ascending=False)
 cum_per_country[:10]<groupby>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
date = train['Date'].unique()
cc_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().ConfirmedCases
ft_us = train[train['Country_Region'] == 'US'].groupby(['Date'] ).sum().Fatalities
cc_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().ConfirmedCases
ft_ity = train[train['Country_Region'] == 'Italy'].groupby(['Date'] ).sum().Fatalities
cc_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().ConfirmedCases
ft_spn = train[train['Country_Region'] == 'Spain'].groupby(['Date'] ).sum().Fatalities
cc_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().ConfirmedCases
ft_gmn = train[train['Country_Region'] == 'Germany'].groupby(['Date'] ).sum().Fatalities
cc_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().ConfirmedCases
ft_frc = train[train['Country_Region'] == 'France'].groupby(['Date'] ).sum().Fatalities
fig = go.Figure()
fig.add_trace(go.Scatter(x=date, y=cc_us, name='US'))
fig.add_trace(go.Scatter(x=date, y=cc_ity, name='Italy'))
fig.add_trace(go.Scatter(x=date, y=cc_spn, name='Spain'))
fig.add_trace(go.Scatter(x=date, y=cc_gmn, name='Germany'))
fig.add_trace(go.Scatter(x=date, y=cc_frc, name='France'))
fig.update_layout(title="Plot of Cumulative Cases for Top 5 countires(except China)",
xaxis_title="Date",
yaxis_title="Cases")
 fig.update_xaxes(nticks=30)
 fig.show()<data_type_conversions>
brandencoder = LabelEncoder().fit(phone.phone_brand)
 phone['brand'] = brandencoder.transform(phone['phone_brand'])
 gatrain['brand'] = phone['brand']
gatest['brand'] = phone['brand']
Xtr_brand = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.brand)))
 Xte_brand = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.brand)))
 print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train['Country_Region'] = train['Country_Region'].astype(str)
 test['Country_Region'] = test['Country_Region'].astype(str)
<feature_engineering>
m = phone.phone_brand.str.cat(phone.device_model.str.split().str.get(0))
modelencoder = LabelEncoder().fit(m)
 phone['model'] = modelencoder.transform(m)
 gatrain['model'] = phone['model']
gatest['model'] = phone['model']
Xtr_model = csr_matrix(( np.ones(gatrain.shape[0]), 
(gatrain.trainrow, gatrain.model)))
 Xte_model = csr_matrix(( np.ones(gatest.shape[0]), 
(gatest.testrow, gatest.model)))
 print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
train['Province_State'].fillna(EMPTY_VAL, inplace=True)
 train['Province_State'] = train.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test['Province_State'].fillna(EMPTY_VAL, inplace=True)
 test['Province_State'] = test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1 )<categorify>
appencoder = LabelEncoder().fit(appevents.app_id)
 appevents['app'] = appencoder.transform(appevents.app_id)
 napps = len(appencoder.classes_)
 deviceapps =(appevents.merge(events[['device_id']], how='left',left_on='event_id',right_index=True)
.groupby(['device_id','app'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index() )
le = preprocessing.LabelEncoder()
train['country_encoder'] = le.fit_transform(train['Country_Region'])
 train['date_int'] = train['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int)
 test['country_encoder'] = le.transform(test['Country_Region'])
 test['date_int'] = test['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int )<categorify>
d = deviceapps.dropna(subset=['trainrow'])
 Xtr_app = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.app)) ,
shape=(gatrain.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
 Xte_app = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.app)) ,
shape=(gatest.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))
le = preprocessing.LabelEncoder()
train['province_encoder'] = le.fit_transform(train['Province_State'])
 test['province_encoder'] = le.transform(test['Province_State'] )<count_duplicates>
applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]
applabels['app'] = appencoder.transform(applabels.app_id)
 labelcat = labelcat.loc[labelcat.label_id.isin(applabels.label_id.unique())]
labelencoder = LabelEncoder().fit(labelcat.category)
 labelcat['label'] = labelencoder.transform(labelcat.category)
 nlabels = len(labelencoder.classes_)
 print('number of unique labels:',nlabels)
 print('recoded label categories', '/n',labelcat.head(n=20))
applabels=applabels.merge(labelcat[['label','label_id']],
how='left',left_on='label_id',right_on='label_id')
 devicelabels =(deviceapps[['device_id','app']] 
.merge(applabels[['app','label']])
.groupby(['device_id','label'])['app'].agg(['size'])
.merge(gatrain[['trainrow']], how='left', left_index=True, right_index=True)
.merge(gatest[['testrow']], how='left', left_index=True, right_index=True)
.reset_index())
 devicelabels.head()
<load_from_csv>
d = devicelabels.dropna(subset=['trainrow'])
 Xtr_label = csr_matrix(( np.ones(d.shape[0]),(d.trainrow, d.label)) ,
shape=(gatrain.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
 Xte_label = csr_matrix(( np.ones(d.shape[0]),(d.testrow, d.label)) ,
shape=(gatest.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))
train_df = pd.read_csv(Path('/kaggle/input/covid19w2', 'train_df.csv'), index_col = 0, parse_dates = ['date'])
 train_df = train_df[train_df['date_int']>=301]
train_df['weekday'] = train_df['date'].dt.weekday
train_df[train_df['country'] == 'Italy']<split>
Xtrain = hstack(( Xtr_brand, Xtr_model, Xtr_app, Xtr_label,
Xtr_tod, Xtr_dow, Xtr_lat, Xtr_lng), format='csr')
 Xtest = hstack(( Xte_brand, Xte_model, Xte_app, Xte_label,
Xte_tod, Xte_dow, Xte_lat, Xte_lng), format='csr')
 print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))
def split_train_val(df, val_ratio):
val_len = int(len(df)* val_ratio)
 train_set = df[:-val_len]
val_set = df[-val_len:]
return train_set, val_set<prepare_x_and_y>
targetencoder = LabelEncoder().fit(gatrain.group)
 y = targetencoder.transform(gatrain.group )
test_fixed_cols = ['ForecastId', 'Province_State', 'Country_Region', 'Date']
fixed_cols = ['Id', 'province', 'country', 'date']
output_cols = ['cc_cases', 'ft_cases']
input_cols = list(set(train_df.columns.to_list())- set(fixed_cols)- set(output_cols))
print('output columns are ', output_cols)
 print('input columns are ', input_cols)
 X = train_df[input_cols]
y = train_df[output_cols]<categorify>
params = {}
params['booster'] = 'gblinear'
params['objective'] = "multi:softprob"
params['eval_metric'] = 'mlogloss'
params['eta'] = 0.005
params['num_class'] = 12
params['lambda'] = 3
params['alpha'] = 2
cc_input = ['country_encoder', 'province_encoder', 'weekday', 'date_int','cc_cases_1', 'cc_cases_2', 'cc_cases_3', 'cc_cases_4', 'cc_cases_5', 'cc_cases_6', 'cc_cases_7']
ft_input = ['country_encoder', 'province_encoder', 'weekday' , 'date_int', 'ft_cases_1', 'ft_cases_2', 'ft_cases_3', 'ft_cases_4', 'ft_cases_5', 'ft_cases_6', 'ft_cases_7']
cc_output = ['cc_cases']
ft_output = ['ft_cases']
val_ratio = 0.05
X_cc = X[cc_input]
X_ft = X[ft_input]
y_cc = y[cc_output]
y_ft = y[ft_output]
train_X_cc, val_X_cc = split_train_val(df = X_cc, val_ratio = val_ratio)
 train_y_cc, val_y_cc = split_train_val(df = y_cc, val_ratio = val_ratio)
 train_X_ft, val_X_ft = split_train_val(df = X_ft, val_ratio = val_ratio)
 train_y_ft, val_y_ft = split_train_val(df = y_ft, val_ratio = val_ratio )<normalization>
kf = list(StratifiedKFold(y, n_folds=10, shuffle=True, random_state=4242)) [0]
Xtr, Xte = Xtrain[kf[0], :], Xtrain[kf[1], :]
ytr, yte = y[kf[0]], y[kf[1]]
print('Training set: ' + str(Xtr.shape))
print('Validation set: ' + str(Xte.shape))
d_train = xgb.DMatrix(Xtr, label=ytr)
 d_valid = xgb.DMatrix(Xte, label=yte)
 watchlist = [(d_train, 'train'),(d_valid, 'eval')]
<normalization><EOS> 
clf = xgb.train(params, d_train, 1000, watchlist, early_stopping_rounds=25)
 pred = clf.predict(xgb.DMatrix(Xtest))
pred = pd.DataFrame(pred, index = gatest.index, columns=targetencoder.classes_)
 pred.head()
pred.to_csv('sparse_xgb_v11.csv', index=True )