Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<categorify>
os.environ["WANDB_API_KEY"] = "0"
!pip uninstall -y transformers
!pip install transformers
!pip install nlp
strategy = None
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age<feature_engineering>
original_train = pd.read_csv(".. /input/contradictory-my-dear-watson/train.csv")
 original_train = sklearn.utils.shuffle(original_train)
 original_train = sklearn.utils.shuffle(original_train)
 validation_ratio = 0.2
nb_valid_examples = max(1, int(len(original_train)* validation_ratio))
original_valid = original_train[:nb_valid_examples]
original_train = original_train[nb_valid_examples:]
train_data['Age'] = train_data[['Age','Pclass']].apply(impute_age,axis=1 )<feature_engineering>
print(f"original - training: {len(original_train)} examples")
 original_train.head(10 )
test_data['Age'] = test_data[['Age','Pclass']].apply(impute_age,axis=1 )<save_to_csv>
print(f"original - validation: {len(original_valid)} examples")
 original_valid.head(10 )
y = train_data["Survived"]
features = ["Pclass","Sex","Age", "Family_tot", "Fare"]
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)
 model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
original_test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv")
 print(f"original - test: {len(original_test)} examples")
 original_test.head(10 )
if not sys.warnoptions:
warnings.simplefilter("ignore")
 warnings.filterwarnings("ignore")
<compute_test_metric>
mnli = nlp.load_dataset(path='glue', name='mnli' )
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
print(mnli, '
')
 print('The split names in MNLI dataset:')
 for k in mnli:
print(' ', k)
 print("
mnli['train'] is ", type(mnli['train']))
mnli['train']
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv")
 train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
print('The number of training examples in mnli dataset:', mnli['train'].num_rows)
 print('The number of validation examples in mnli dataset - part 1:', mnli['validation_matched'].num_rows)
 print('The number of validation examples in mnli dataset - part 2:', mnli['validation_mismatched'].num_rows, '
')
 print('The class names in mnli dataset:', mnli['train'].features['label'].names)
 print('The feature names in mnli dataset:', list(mnli['train'].features.keys()), '
')
 for elt in mnli['train']:
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', mnli['train'].features['label'].names[elt['label']])
 print('idx', elt['idx'])
 print('-' * 80)
 if elt['idx'] >= 10:
break
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()<feature_engineering>
mnli_train_df = pd.DataFrame(mnli['train'])
 mnli_valid_1_df = pd.DataFrame(mnli['validation_matched'])
 mnli_valid_2_df = pd.DataFrame(mnli['validation_mismatched'])
 mnli_train_df = mnli_train_df[['premise', 'hypothesis', 'label']]
mnli_valid_1_df = mnli_valid_1_df[['premise', 'hypothesis', 'label']]
mnli_valid_2_df = mnli_valid_2_df[['premise', 'hypothesis', 'label']]
mnli_train_df['lang_abv'] = 'en'
mnli_valid_1_df['lang_abv'] = 'en'
mnli_valid_2_df['lang_abv'] = 'en'
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()<compute_test_metric>
snli = nlp.load_dataset(path='snli')
 print('The number of training examples in snli dataset:', snli['train'].num_rows)
 print('The number of validation examples in snli dataset:', snli['validation'].num_rows, '
')
 print('The class names in snli dataset:', snli['train'].features['label'].names)
 print('The feature names in snli dataset:', list(snli['train'].features.keys()), '
')
 for idx, elt in enumerate(snli['train']):
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', snli['train'].features['label'].names[elt['label']])
 print('-' * 80)
 if idx >= 10:
break
method_list = ['Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2]
for i in range(0,2):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )<save_to_csv>
snli_train_df = pd.DataFrame(snli['train'])
 snli_valid_df = pd.DataFrame(snli['validation'])
 snli_train_df = snli_train_df[['premise', 'hypothesis', 'label']]
snli_valid_df = snli_valid_df[['premise', 'hypothesis', 'label']]
snli_train_df['lang_abv'] = 'en'
snli_valid_df['lang_abv'] = 'en'
df_val = df_val_2
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission<set_options>
xnli = nlp.load_dataset(path='xnli')
 print('The number of validation examples in xnli dataset:', xnli['validation'].num_rows, '
')
 print('The class names in xnli dataset:', xnli['validation'].features['label'].names)
 print('The feature names in xnli dataset:', list(xnli['validation'].features.keys()), '
')
 for idx, elt in enumerate(xnli['validation']):
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', xnli['validation'].features['label'].names[elt['label']])
 print('-' * 80)
 if idx >= 3:
break
<load_from_csv><EOS> 
buffer = {
'premise': [],
'hypothesis': [],
'label': [],
'lang_abv': []
}
for x in xnli['validation']:
label = x['label']
for idx, lang in enumerate(x['hypothesis']['language']):
hypothesis = x['hypothesis']['translation'][idx]
premise = x['premise'][lang]
buffer['premise'].append(premise)
 buffer['hypothesis'].append(hypothesis)
 buffer['label'].append(label)
 buffer['lang_abv'].append(lang)
 xnli_valid_df = pd.DataFrame(buffer)
 xnli_valid_df = xnli_valid_df[['premise', 'hypothesis', 'label', 'lang_abv']]