Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<data_type_conversions>
os.environ["WANDB_API_KEY"] = "0"
test_data['Age'].fillna(value = train_data['Age'].mean() , inplace = True)
 test_data['Fare'].fillna(value = train_data['Fare'].mean() , inplace = True )<define_variables>
df_train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 df_test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv' )
s =(test_data.dtypes == 'object')
 object_cols_test = list(s[s].index)
 print("Categorical variables:")
 print(object_cols_test )<categorify>
df_train.isnull().sum()
label_test_data = test_data.copy()
label_encoder = LabelEncoder()
for col in object_cols_test:
label_train_data[col] = label_encoder.fit(train_data_initial[col].astype(str))
label_test_data[col] = label_encoder.transform(test_data[col].astype(str))<train_model>
df_test.isnull().sum()
final_model = RandomForestClassifier(n_estimators=100, max_leaf_nodes=105, max_depth = 6, random_state= 0)
 final_model.fit(X, y)
 preds = final_model.predict(label_test_data )<save_to_csv>
def encode_sentence(s):
tokens = list(tokenizer.tokenize(s))
tokens.append('[SEP]')
 return tokenizer.convert_tokens_to_ids(tokens )
sub_df = pd.DataFrame(data={
'PassengerId': label_test_data.index,
'Survived': preds
})
 sub_df.to_csv('submission.csv', index=False)
 print('done' )<import_modules>
def bert_encode(premise, hypothesis, tokenizer):
sentence1 = tf.ragged.constant([encode_sentence(s)for s in np.array(premise)])
 sentence2 = tf.ragged.constant([encode_sentence(s)for s in np.array(hypothesis)])
 cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*(sentence1.shape[0])
 input_word_ids = tf.concat([cls, sentence1, sentence2], axis=-1)
 input_masks = tf.ones_like(input_word_ids ).to_tensor()
type_cls = tf.zeros_like(cls)
 type_s1 = tf.zeros_like(sentence1)
 type_s2 = tf.ones_like(sentence2)
 input_type_ids = tf.concat([type_cls, type_s1, type_s2], axis=-1 ).to_tensor()
inputs = {
'input_word_ids': input_word_ids.to_tensor() ,
'input_masks': input_masks,
'input_type_ids': input_type_ids}
return inputs
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from itertools import cycle, islice
import seaborn as sb
import matplotlib.dates as dates
import datetime as dt
from sklearn import preprocessing
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler
import plotly.graph_objects as go
import plotly_express as px
from sklearn.preprocessing import OrdinalEncoder<load_from_csv>
tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
 train_input = bert_encode(df_train.premise.values, df_train.hypothesis.values, tokenizer)
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv")
 train.head()<data_type_conversions>
K.clear_session()
train['Date'] = pd.to_datetime(train['Date'], format = '%Y-%m-%d')
 test['Date'] = pd.to_datetime(test['Date'], format = '%Y-%m-%d' )<count_values>
def build_bert_baseline(max_len=80):
bert_layer = TFBertModel.from_pretrained('bert-base-multilingual-cased')
 input_word_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_word_ids')
 input_masks = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_masks')
 input_type_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_type_ids')
 sequence_output = bert_layer([input_word_ids, input_masks, input_type_ids])[0]
output = tf.keras.layers.Dense(3, activation='softmax' )(sequence_output[:,0,:])
 model = tf.keras.Model(inputs=[input_word_ids,input_masks,input_type_ids],outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=3e-5),
loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 print(model.summary())
 return model
curr_date = train['Date'].max()
world_cum_confirmed = sum(train[train['Date'] == curr_date].ConfirmedCases)
 world_cum_fatal = sum(train[train['Date'] == curr_date].Fatalities)
 print('Number of Countires: ', len(train['Country_Region'].unique()))
print('End date in train dset: ', curr_date)
 print('Number of confirmed cases: ', world_cum_confirmed)
 print('Number of fatal cases: ', world_cum_fatal )<sort_values>
model = build_bert_baseline(max_len=80)
 filepath='best_weight.hdf5'
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
top_country_c = train[train['Date'] == curr_date].groupby(['Date','Country_Region'] ).sum().sort_values(['ConfirmedCases'], ascending=False)
 top_country_c.head(10 )<sort_values>
K.clear_session()
top_country_f = train[train['Date'] == curr_date].groupby(['Date','Country_Region'] ).sum().sort_values(['Fatalities'], ascending=False)
 top_country_f.head(10 )<count_values>
def build_bert_LSTM_head(max_len=80):
bert_layer = TFBertModel.from_pretrained('bert-base-multilingual-cased')
 input_word_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_word_ids')
 input_masks = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_masks')
 input_type_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_type_ids')
 sequence_output = bert_layer([input_word_ids, input_masks, input_type_ids])[0]
bi_lstm = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True))(sequence_output)
 avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)
 max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)
 concat = tf.keras.layers.concatenate([avg_pool, max_pool])
 dropout = tf.keras.layers.Dropout(0.3 )(concat)
 output = tf.keras.layers.Dense(3, activation="softmax" )(dropout)
 model = tf.keras.Model(inputs=[input_word_ids,input_masks,input_type_ids],outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=3e-5),
loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 print(model.summary())
 return model
pr_confirm = train['ConfirmedCases'].value_counts(normalize=True)
 pr_fatal = train['Fatalities'].value_counts(normalize=True)
 print(f'Percs of confirmed case = {pr_confirm[1:].sum() *100}%')
 print(f'Percs of fatality = {pr_fatal[1:].sum() *100}%' )<groupby>
bert_lstm_model = build_bert_LSTM_head(max_len=80)
 filepath='LSTM_best_weight.hdf5'
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
train_sum=train.groupby(['Date','Country_Region'] ).agg('sum' ).reset_index()
Country_cases(train_sum,'ConfirmedCases','US')
 Country_fatalities(train_sum,'Fatalities','US' )<feature_engineering>
K.clear_session()
train['MortalityRate'] = train['Fatalities'] / train['ConfirmedCases']
train['MortalityRate'] = train['MortalityRate'].fillna(0.0 )<feature_engineering>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
def add_daily_measures(df):
df.loc[0,'Daily Cases'] = df.loc[0,'ConfirmedCases']
df.loc[0,'Daily Deaths'] = df.loc[0,'Fatalities']
for i in range(1,len(df)) :
df.loc[i,'Daily Cases'] = df.loc[i,'ConfirmedCases'] - df.loc[i-1,'ConfirmedCases']
df.loc[i,'Daily Deaths'] = df.loc[i,'Fatalities'] - df.loc[i-1,'Fatalities']
df.loc[0,'Daily Cases'] = 0
df.loc[0,'Daily Deaths'] = 0
return df
df_world = train.copy()
df_world = df_world.groupby('Date',as_index=False)['ConfirmedCases','Fatalities'].sum()
df_world = add_daily_measures(df_world )<sort_values>
def roberta_encode(text,tokenizer,max_len=80):
encoded_text = tokenizer.batch_encode_plus(text,
pad_to_max_length=True,
max_length=max_len)
 return encoded_text
top_country_m = train[train['Date'] == curr_date].groupby(['Country_Region'] ).sum().sort_values(['MortalityRate'], ascending=False)
 top_country_m.head(10 )<feature_engineering>
tokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')
 train_text = df_train[['premise', 'hypothesis']].values.tolist()
encoded_train_text = roberta_encode(train_text,tokenizer,max_len=80 )
def create_features(df):
df['day'] = df['Date'].dt.day
df['month'] = df['Date'].dt.month
df['dayofweek'] = df['Date'].dt.dayofweek
df['dayofyear'] = df['Date'].dt.dayofyear
df['quarter'] = df['Date'].dt.quarter
df['weekofyear'] = df['Date'].dt.weekofyear
return df<split>
x_train, x_valid, y_train, y_valid = train_test_split(
 encoded_train_text['input_ids'],
df_train.label.values,
test_size=0.2, random_state=98 )
def train_dev_split(df, days):
date = df['Date'].max() - dt.timedelta(days=days)
 return df[df['Date'] <= date], df[df['Date'] > date]<categorify>
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE
def categoricalToInteger(df):
df.Province_State.fillna('NaN', inplace=True)
 oe = OrdinalEncoder()
df[['Province_State','Country_Region']] = oe.fit_transform(df.loc[:,['Province_State','Country_Region']])
 return df<split>
def create_tf_dataset(X, y,val,batch_size= BATCH_SIZE,auto=AUTO):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(98)
 if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(auto)
 else:
dataset = dataset.batch(batch_size ).prefetch(auto)
 return dataset
df_train, df_dev = train_dev_split(df_train,0)
 columns = ['day','month','dayofweek','dayofyear','quarter','weekofyear','Province_State', 'Country_Region','ConfirmedCases','Fatalities']
df_train = df_train[columns]
df_dev = df_dev[columns]<feature_engineering>
tf_train_dataset = create_tf_dataset(x_train,y_train,val=False)
 tf_val_dataset = create_tf_dataset(x_valid,y_valid,val=True )
df_test = categoricalToInteger(test)
 df_test = create_features(test)
 columns = ['day','month','dayofweek','dayofyear','quarter','weekofyear','Province_State', 'Country_Region']<data_type_conversions>
tf_train_dataset,tf_val_dataset
<feature_engineering>
def build_roberta(max_len=80):
roberta_layer = TFAutoModel.from_pretrained('jplu/tf-xlm-roberta-large')
 input_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_ids')
 sequence_output = roberta_layer(input_ids)[0]
output = tf.keras.layers.Dense(3, activation='softmax' )(sequence_output[:,0,:])
 model = tf.keras.Model(inputs=input_ids, outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 print(model.summary())
 return model
<choose_model_class>
with strategy.scope() :
roberta_model = build_roberta(max_len=80)
 filepath='roberta_best_weight.hdf5'
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
 n_steps = len(x_train)//BATCH_SIZE
roberta_model.fit(tf_train_dataset,steps_per_epoch=n_steps,
validation_data=tf_val_dataset,callbacks=[checkpoint],epochs=10 )
def RF() :
model = RandomForestRegressor(n_estimators = 100)
 return model
def XGB() :
model = XGBRegressor(n_estimators=1300)
 return model
def LGBM() :
model = LGBMRegressor(iterations=2)
 return model<prepare_x_and_y>
def model_cv_score(model):
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=98)
 E_cv = 0
for fold,(train_idx,val_idx)in enumerate(skf.split(df_train['premise'],df_train['label'])) :
print('
Fold %s:'%(fold))
K.clear_session()
train_premise = df_train['premise'].iloc[train_idx]
train_hypothesis = df_train['hypothesis'].iloc[train_idx]
train_label = df_train['label'].iloc[train_idx]
val_premise = df_train['premise'].iloc[val_idx]
val_hypothesis = df_train['hypothesis'].iloc[val_idx]
val_label = df_train['label'].iloc[val_idx]
cv_train_input = bert_encode(train_premise.values, train_hypothesis.values, tokenizer)
 cv_val_input = bert_encode(val_premise.values, val_hypothesis.values, tokenizer)
 filepath='best_weight_fold%s.h5'%(fold)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
 model.fit(cv_train_input, train_label.values, epochs =2,
batch_size = 16, callbacks=[checkpoint],
validation_data=(cv_val_input,val_label.values))
print('
Predicting OOF Error...... ')
 model.load_weights('best_weight_fold%s.h5'%(fold))
val_pred = [np.argmax(p)for p in model.predict(cv_val_input)]
oof_error = np.sum([pred!=true for pred,true in zip(val_pred,val_label)])/len(val_label)
 print('Fold_%s Error: %s' %(fold,oof_error))
E_cv += oof_error
print('
E_cv = %s'%(E_cv/5))
submission = []
for country in df_train.Country_Region.unique() :
df_train1 = df_train[df_train["Country_Region"]==country]
for state in df_train1.Province_State.unique() :
df_train2 = df_train1[df_train1["Province_State"]==state]
train = df_train2.values
X_train, y_train = train[:,:-2], train[:,-2:]
model1 = XGBRegressor(n_estimators=1100)
 model1.fit(X_train, y_train[:,0])
 model2 = XGBRegressor(n_estimators=1100)
 model2.fit(X_train, y_train[:,1])
 df_test1 = df_test[(df_test["Country_Region"]==country)&(df_test["Province_State"] == state)]
ForecastId = df_test1.ForecastId.values
df_test2 = df_test1[columns]
y_pred1 = np.round(model1.predict(df_test2.values),5)
 y_pred2 = np.round(model2.predict(df_test2.values),5)
 for i in range(len(y_pred1)) :
d = {'ForecastId':ForecastId[i], 'ConfirmedCases':y_pred1[i], 'Fatalities':y_pred2[i]}
submission.append(d )<save_to_csv>
test_text = df_test[['premise', 'hypothesis']].values.tolist()
encoded_test_text = roberta_encode(test_text,tokenizer,max_len=80)
 tf_test_dataset =(tf.data.Dataset 
.from_tensor_slices(encoded_test_text['input_ids'])
.batch(BATCH_SIZE))
roberta_model.load_weights('./roberta_best_weight.hdf5')
 pred = [np.argmax(res)for res in roberta_model.predict(tf_test_dataset)]
<load_from_csv><EOS> 
submission = pd.DataFrame({'id':df_test.id,'prediction':pred})
 submission.to_csv('submission.csv',index=False )