Kaggle competition: COVID19 Global Forecasting (Week 2)
def apk(actual, predicted, k=10):

 if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i,p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i+1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=10):

 return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)])
 def extract_features(files, path):
features = {}
cnt = 0
for f in tqdm(files):
features[f] = {}
fs, data = scipy.io.wavfile.read(os.path.join(path, f))
abs_data = np.abs(data)
 diff_data = np.diff(data)
 def calc_part_features(data, n=2, prefix=''):
f_i = 1
for i in range(0, len(data), len(data)//n):
features[f]['{}mean_{}_{}'.format(prefix, f_i, n)] = np.mean(data[i:i + len(data)//n])
 features[f]['{}std_{}_{}'.format(prefix, f_i, n)] = np.std(data[i:i + len(data)//n])
 features[f]['{}min_{}_{}'.format(prefix, f_i, n)] = np.min(data[i:i + len(data)//n])
 features[f]['{}max_{}_{}'.format(prefix, f_i, n)] = np.max(data[i:i + len(data)//n])
 features[f]['len'] = len(data)
 if features[f]['len'] > 0:
n = 1
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 n = 2
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 n = 3
calc_part_features(data, n=n)
 calc_part_features(abs_data, n=n, prefix='abs_')
 calc_part_features(diff_data, n=n, prefix='diff_')
 cnt += 1
features = pd.DataFrame(features ).T.reset_index()
features.rename(columns={'index': 'fname'}, inplace=True)
 return features
path = os.path.join(data_path, 'audio_train')
 train_files = train.fname.values
train_features = extract_features(train_files, path)
 path = os.path.join(data_path, 'audio_test')
 test_files = ss.fname.values
test_features = extract_features(test_files, path )<merge>
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 print("*"*50)
 print(df_train.info())
 print("*"*50)
 print(df_test.info())
 print("*"*50 )
train_data = train_data.merge(train_features, on='fname', how='left')
 test_data = test_data.merge(test_features, on='fname', how='left')
 train_data.head()<prepare_x_and_y>
NULL_VAL = "NULL_VAL"
def fillState(state, country):
if state == NULL_VAL:
return country
return state
X_Train = df_train.loc[:, ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
X_Train['State'].fillna(NULL_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Test = df_test.loc[:, ['State', 'Country', 'Date', 'ForecastId']]
X_Test['State'].fillna(NULL_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 print("*"*50)
 print(X_Train.head())
 print("*"*50)
 print(X_Test.head())
 print("*"*50 )
X = train_data.drop(['label', 'fname'], axis=1)
 feature_names = list(X.columns)
 X = X.values
labels = np.sort(np.unique(train_data.label.values))
num_class = len(labels)
 c2i = {}
i2c = {}
for i, c in enumerate(labels):
c2i[c] = i
i2c[i] = c
y = np.array([c2i[x] for x in train_data.label.values] )<train_model>
le = LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 print("*"*50)
 print(X_Train.head())
 print("*"*50)
 print(X_Test.head())
 print("*"*50 )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=10, shuffle = True)
 clf = XGBClassifier(max_depth=5, learning_rate=0.05, n_estimators=3000,
n_jobs=-1, random_state=0, reg_alpha=0.2,
colsample_bylevel=0.9, colsample_bytree=0.9)
 clf.fit(X_train, y_train)
 print(accuracy_score(clf.predict(X_val), y_val))
def proba2labels(preds, i2c, k=3):
ans = []
ids = []
for p in preds:
idx = np.argsort(p)[::-1]
ids.append([i for i in idx[:k]])
 ans.append(' '.join([i2c[i] for i in idx[:k]]))
return ans, ids<save_to_csv>
le = LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
condition_train =(X_Train.Country == country)&(X_Train.State == state)
 X_Train_CS = X_Train.loc[condition_train, ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
condition_test =(X_Test.Country == country)&(X_Test.State == state)
 X_Test_CS = X_Test.loc[condition_test, ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
 df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
clf.fit(X, y)
 str_preds, _ = proba2labels(clf.predict_proba(test_data.drop(['label', 'fname'], axis = 1 ).values), i2c, k=3)
 subm = pd.DataFrame()
subm['fname'] = audio_test_files
subm['label'] = str_preds
subm.to_csv('submission.csv', index=False )<import_modules>
df_out.to_csv('submission.csv', index=False )