Kaggle competition: Google Cloud & NCAA® ML Competition 2018-Women's
<SOS> metric: LogLoss Kaggle data source: womens-machine-learning-competition-2018<data_type_conversions>
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.head(100 )<save_to_csv>
data_dir = '.. /input/'
tourresults = pd.read_csv(data_dir + 'WNCAATourneyCompactResults.csv')
 tourseeds = pd.read_csv(data_dir + 'WNCAATourneySeeds.csv')
 seasonresults = pd.read_csv(data_dir + 'WRegularSeasonCompactResults.csv' )
df_out.to_csv('submission.csv', index=False )<load_from_csv>
def GetWinsLosses(seasonresults, year, teamID):
Spec_season = seasonresults['Season']==year
teamIDW = seasonresults['WTeamID']==teamID
teamIDL = seasonresults['LTeamID']==teamID
wins = len(seasonresults[Spec_season & teamIDW])
 losses = len(seasonresults[Spec_season & teamIDL])
 return [wins, losses]
def seed_to_int(seed):
s_int = int(seed[1:3])
 return s_int
dirname = '/kaggle/input/covid19-global-forecasting-week-2/'
trainname = 'train.csv'
trainpath = dirname + trainname
traindata_raw = pd.read_csv(trainpath)
 traindata = collections.defaultdict(lambda :collections.defaultdict(list))
traindata_country = collections.defaultdict(lambda :collections.defaultdict(lambda :[0,0]))
traindata_state = collections.defaultdict(lambda :collections.defaultdict(lambda :[0,0]))
traindata_country_onlymainland = collections.defaultdict(lambda :collections.defaultdict(lambda :[0,0]))
for i in range(len(traindata_raw)) :
country = traindata_raw['Country_Region'][i]
state = traindata_raw['Province_State'][i]
date = traindata_raw['Date'][i]
cases = traindata_raw['ConfirmedCases'][i]
fata = traindata_raw['Fatalities'][i]
traindata[country][date].append(( state, cases, fata))
traindata_country[country][date][0] += cases
traindata_country[country][date][1] += fata
if state != state:
traindata_country_onlymainland[country][date][0] += cases
traindata_country_onlymainland[country][date][1] += fata
state_ = country + ':' + state if state == state else country + ':'
traindata_state[state_][date][0] += cases
traindata_state[state_][date][1] += fata 
<define_search_model>
year = 2018
df_seeds = tourseeds[tourseeds['Season']==year]
df_tour = tourresults[tourresults['Season']==year]
df_season = seasonresults[seasonresults['Season']==year]
df_seeds['seed_int'] = df_seeds.Seed.apply(seed_to_int)
 df_seeds.drop(labels=['Seed'], inplace=True, axis=1 )
polynomial = PolynomialFeatures(degree = 5)
 poly_model = LinearRegression()
predictdata_state = collections.defaultdict(lambda :[[],[]])
 for state in traindata_state:
state_history = [[],[]]
for date in traindata_state[state]:
state_history[0].append(traindata_state[state][date][0])
 state_history[1].append(traindata_state[state][date][1])
 i = 0
for perspective in state_history:
perspective_train = [i for i in perspective[:] if i!=0]
perspective = [i for i in perspective if i!=0]
x_all = np.array(range(1, len(perspective)+ 31))
x = x_all[:len(perspective_train)]
y = np.array(perspective_train)
 if len(perspective_train)< 2:
if not perspective_train:
predict_y = np.zeros(len(x_all[len(perspective_train):]))
else:
predict_y = np.array([perspective_train[-1]]*(len(x_all[len(perspective_train):])))
 predictdata_state[state][i] = predict_y
i =(i+1)% 2
continue
if 'China' in state and 'Macau' not in state and 'Hong Kong' not in state and 'Liaoning' not in state or 'Korea, South' in state:
def funexp(x, a, b, c):
return(x/a)**b + c
def funmi(x,a,b,c):
return a**(x + b)* c
def funsigmoid(x, a, b, c):
s = a /(1 + np.exp(-(x + b)/ c))
return s
p0 = [max(perspective_train), np.median(x), 5]
popt, pcov = curve_fit(funsigmoid, x, y, p0)
 predict_x = x_all[len(perspective_train):]
predict_y = [funsigmoid(i, popt[0], popt[1], popt[2])for i in predict_x]
if predict_y[0] < perspective_train[-1]:
predict_y = [i +(perspective_train[-1] - predict_y[0])for i in predict_y]
predict_y = [round(i)for i in predict_y]
predictdata_state[state][i] = predict_y
else:
x = x.reshape(-1,1)
 x_transformed = polynomial.fit_transform(x)
 poly_model.fit(x_transformed, y)
 predict_x = x_all[len(perspective_train):]
predict_y = poly_model.predict(polynomial.fit_transform(np.array(predict_x ).reshape(-1,1)))
 predict_y = [round(i)for i in predict_y]
predictdata_state[state][i] = predict_y
i =(i+1)% 2 
<load_from_csv>
seasonresultsNew = seasonresults.drop(['DayNum','WScore','LScore','WLoc','NumOT'],axis=1 )
testname = 'test.csv'
testpath = dirname + testname
testdata = pd.read_csv(testpath)
 submname = 'submission.csv'
submpath = dirname + submname
submdata = pd.read_csv(submpath)
 res = []
ind = 0
found = False
for i in range(len(testdata)) :
forecastid = testdata['ForecastId'][i]
state = testdata['Province_State'][i]
country = testdata['Country_Region'][i]
date = testdata['Date'][i]
state_ = country + ':' + state if state == state else country + ':'
if date == '2020-03-19':
found = True
if date == '2020-04-01':
found = False
ind = 0
if found:
res.append(traindata_state[state_][date])
 else:
res.append([int(predictdata_state[state_][0][ind]), int(predictdata_state[state_][1][ind])])
 ind += 1
subm_cases = [i[0] for i in res]
subm_fata = [i[1] for i in res]
submdata['ConfirmedCases'] = subm_cases
submdata['Fatalities'] = subm_fata
submdata.to_csv('submission.csv', index = False )<import_modules>
Teams = df_seeds.TeamID.sort_values().unique()
n_test_games = int(len(Teams)*(len(Teams)-1)/2)
 df_winseeds = df_seeds.rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_lossseeds = df_seeds.rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 CompleteDF = pd.DataFrame()
ii=0
IDlist = []
for t1 in Teams:
t1_seed = df_seeds[(df_seeds.TeamID == t1)&(df_seeds.Season == year)].seed_int.values[0]
CompleteDF.loc[ii,'seed'] = t1_seed
[winst1, lossest1] = GetWinsLosses(seasonresults, year, t1)
 CompleteDF.loc[ii,'Season'] = year
CompleteDF.loc[ii,'TeamID'] = t1
CompleteDF.loc[ii,'Wins'] = winst1
CompleteDF.loc[ii,'Losses'] = lossest1
ii+=1
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
BaseDF = seasonresults[seasonresults['Season']==year]
BaseDF.tail()
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<categorify>
WinerIDlessthanLoserID = BaseDF['WTeamID']<BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team1'] = BaseDF['WTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team1'] = BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team2'] = BaseDF['LTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team2'] = BaseDF['WTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Result'] = 1
BaseDF.loc[~WinerIDlessthanLoserID, 'Result'] = 0
BaseDF = BaseDF.drop(['DayNum','WScore','LScore','WLoc','NumOT','WTeamID','LTeamID'],axis=1)
train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax')
<feature_engineering>
FinalDF = pd.merge(left=BaseDF, right=CompleteDF, how='outer', left_on=['Season', 'Team1'], right_on=['Season', 'TeamID'])
 FinalDF.drop(['TeamID'],axis=1,inplace=True)
 FinalDF = FinalDF.rename(columns={'seed':'seedT1','Wins':'WinsT1','Losses':'LossesT1'})
 FinalDF = pd.merge(left=FinalDF, right=CompleteDF, how='right', left_on=['Season', 'Team2'], right_on=['Season', 'TeamID'])
 FinalDF.drop(['TeamID'],axis=1,inplace=True)
 FinalDF = FinalDF.rename(columns={'seed':'seedT2','Wins':'WinsT2','Losses':'LossesT2'})
 FinalDF = FinalDF.dropna()
<compute_test_metric>
FinalDF['Seed'] = FinalDF['seedT1']-FinalDF['seedT2']
FinalDF['Wins'] = FinalDF['WinsT1']-FinalDF['WinsT2']
FinalDF['Losses'] = FinalDF['LossesT1']-FinalDF['LossesT2']
FinalDF.drop(['seedT1','seedT2','WinsT1','WinsT2','LossesT1','LossesT2'],axis=1,inplace=True)
 FinalDF_full = FinalDF.copy()
<compute_test_metric>
X = FinalDF.drop(['Season','Result','Team1','Team2'],axis=1)
 y = FinalDF.Result.values
logreg = LogisticRegression(C=0.1)
 logreg.fit(X, y )
<compute_test_metric>
Teams = df_seeds.TeamID.sort_values().unique()
n_test_games = int(len(Teams)*(len(Teams)-1)/2)
 X_test = pd.DataFrame()
ii=0
IDlist = []
for t1 in Teams:
for t2 in Teams:
if t1 < t2:
t1_seed = df_seeds[(df_seeds.TeamID == t1)&(df_seeds.Season == year)].seed_int.values[0]
t2_seed = df_seeds[(df_seeds.TeamID == t2)&(df_seeds.Season == year)].seed_int.values[0]
X_test.loc[ii,'Seed'] = t1_seed - t2_seed
[winst1, lossest1] = GetWinsLosses(seasonresults, year, t1)
 [winst2, lossest2] = GetWinsLosses(seasonresults, year, t2)
 X_test.loc[ii,'Wins'] = winst1-winst2
X_test.loc[ii,'Losses'] = lossest1-lossest2
ii+=1
IDlist.append('{}_{}_{}'.format(year,t1,t2))
<feature_engineering>
preds = logreg.predict_proba(X_test)[:,1]
clipped_preds = np.clip(preds, 0.01, 0.99)
 df_final_season = pd.DataFrame(columns=['ID','Pred'])
 df_final_season.Pred = clipped_preds
df_final_season.ID = IDlist
<feature_engineering>
BaseDF = tourresults
WinerIDlessthanLoserID = BaseDF['WTeamID']<BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team1'] = BaseDF['WTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team1'] = BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team2'] = BaseDF['LTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team2'] = BaseDF['WTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Result'] = 1
BaseDF.loc[~WinerIDlessthanLoserID, 'Result'] = 0
BaseDF['Team1'] = BaseDF['Team1'].astype(int)
 BaseDF['Team2'] = BaseDF['Team2'].astype(int)
 BaseDF.loc[:,'ID']=BaseDF['Season'].astype(str)+'_'+BaseDF['Team1'].astype(str)+'_'+BaseDF['Team2'].astype(str)
 isSeason = BaseDF['Season'] == year
BaseDF = BaseDF[isSeason]
BaseDF = BaseDF.drop(['DayNum','WScore','LScore','WLoc','NumOT','WTeamID','LTeamID','Team1','Team2','Season'],axis=1 )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<save_to_csv>
BaseDF['ID']=IDlist
BaseDF['Result']=0
df_val = df_val_3
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission = submission.round({'ConfirmedCases': 0, 'Fatalities': 0})
 submission.to_csv('submission.csv', index=False)
 submission<define_variables>
PredsAndResults = pd.merge(left=BaseDF,right=df_final_season, on='ID' )
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
train_file = ".. /input/covid19-global-forecasting-week-2/train.csv"
test_file = ".. /input/covid19-global-forecasting-week-2/test.csv"
sub_file = ".. /input/covid19-global-forecasting-week-2/submission.csv"
Optimistic=True<load_from_csv>
PredsAndResults['Loss']=PredsAndResults['Result']*np.log(PredsAndResults['Pred'])+(1-PredsAndResults['Result'])*np.log(1-PredsAndResults['Pred'])
 -PredsAndResults.Loss.sum() /len(PredsAndResults )
df = pd.read_csv(train_file)
<data_type_conversions>
df_final_season.to_csv('Submission.csv', index=False )