Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<drop_column>
os.environ['WANDB_API_KEY'] = '0'
warnings.filterwarnings("ignore" )
for i,j in enumerate(state_with_country_name):
state_i_data[i] = data[data['state_with_country'] == j ]
state_i_data[i] = state_i_data[i].drop(['state_with_country'],axis=1)
<load_from_csv>
DATA_PATH = '/kaggle/input/contradictory-my-dear-watson-eng/'
MODEL_PATH = '/kaggle/input/tf-roberta/'
prev_sub = pd.read_csv('/kaggle/input/covid19-forecasting-using-rnn/submission.csv')
 test_whole = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv',index_col='Date',parse_dates=True)
 test_whole = test_whole.fillna(value='empty')
<feature_engineering>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync )
test_whole['state_with_country'] = test_whole['Province_State'] +'_'+ test_whole['Country_Region']
test_whole.head()<feature_engineering>
train = pd.read_csv(DATA_PATH + '/trans_train.csv', index_col=False)
 test = pd.read_csv(DATA_PATH + '/trans_test.csv', index_col=False )
prev_sub.index = test_whole.index<prepare_output>
def basic_clean(row):
row['hypothesis'] = re.sub(' +', ' ', row['hypothesis'] ).strip().lower()
row['premise'] = re.sub(' +', ' ', row['premise'] ).strip().lower()
return row
train = train[train['hypothesis'].notna() ]
train = train[train['premise'].notna() ]
test = test[test['hypothesis'].notna() ]
test = test[test['premise'].notna() ]
train = train.apply(basic_clean, axis=1 ).reset_index()
test = test.apply(basic_clean, axis=1 ).reset_index()
prev_sub['state_with_country'] = test_whole['state_with_country']<define_variables>
MAX_LEN = 256
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=MODEL_PATH + 'vocab-roberta-base.json',
merges_file=MODEL_PATH + 'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True
)
forecast = []
for i in state_with_country_name:
forecast.append(i )<drop_column>
def encode_sentence(s):
return tokenizer.encode(s )
for i,j in enumerate(state_with_country_name):
forecast[i] = prev_sub[prev_sub['state_with_country'] == j ]
forecast[i] = forecast[i].iloc[13:21]
forecast[i] = forecast[i].drop(['ForecastId','state_with_country'],axis=1)
 state_i_data[i] = state_i_data[i].append(forecast[i] )<load_from_csv>
encode_sentence('I love machine learning' )
test_whole = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv',index_col='Date',parse_dates=True)
 test_whole = test_whole.fillna(value='empty' )<feature_engineering>
def roberta_encode(df, tokenizer):
ct = df.shape[0]
input_ids = np.ones(( ct, MAX_LEN), dtype='int32')
 attention_mask = np.zeros(( ct, MAX_LEN), dtype='int32')
 token_type_ids = np.zeros(( ct, MAX_LEN), dtype='int32')
 for k, row in df.iterrows() :
enc_hypothesis = tokenizer.encode(row['hypothesis'])
 enc_premise = tokenizer.encode(row['premise'])
 input_length = len(enc_hypothesis.ids)+ len(enc_premise.ids)+ 4
if input_length > MAX_LEN:
continue
input_ids[k,:input_length] = [0] + enc_hypothesis.ids + [2,2] + enc_premise.ids + [2]
attention_mask[k,:input_length] = 1
type_sep = np.zeros_like([0])
 type_s1 = np.zeros_like(enc_hypothesis.ids)
 type_s2 = np.ones_like(enc_premise.ids)
 z = [type_sep, type_s1, type_sep, type_sep, type_s2, type_sep]
token_type_ids[k,:input_length] = np.concatenate(z)
 return {
'input_word_ids': input_ids,
'input_mask': attention_mask,
'input_type_ids': token_type_ids
}
test_whole['state_with_country'] = test_whole['Province_State'] +'_'+ test_whole['Country_Region']
test_whole.head()<define_variables>
train_input = roberta_encode(train, tokenizer)
 test_input = roberta_encode(test, tokenizer )
state_i_data_for_test=[]
for i in state_with_country_name_for_test:
state_i_data_for_test.append(i)
<feature_engineering>
def build_model() :
input_word_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_word_ids')
 input_mask = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_mask')
 input_type_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_type_ids')
 config = RobertaConfig.from_pretrained(MODEL_PATH + 'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(MODEL_PATH + 'pretrained-roberta-base.h5', config=config)
 x = bert_model(input_word_ids, attention_mask=input_mask, token_type_ids=input_type_ids)
 x = x[0]
x = tf.keras.layers.Dropout(0.1 )(x)
 x = tf.keras.layers.Conv1D(1, 1 )(x)
 x = tf.keras.layers.Flatten()(x)
 x = tf.keras.layers.Dense(3, activation='softmax' )(x)
 model = tf.keras.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=x)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return model
for i,j in enumerate(state_with_country_name_for_test):
state_i_data_for_test[i] = test_whole[test_whole['state_with_country'] == j ]
state_i_data_for_test[i] = state_i_data_for_test[i].iloc[21:]
state_i_data_for_test[i] = state_i_data_for_test[i].drop(['ForecastId','state_with_country','Province_State','Country_Region'],axis=1)
 state_i_data_for_test.append(state_i_data_for_test[i] )<import_modules>
accuracy = []
history = []
VER = 'v0'
EPOCHS = 4
KFOLDS = 5
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
pred_test = np.zeros(( test.shape[0], 3))
skf = StratifiedKFold(n_splits=KFOLDS, shuffle=True, random_state=42)
 for fold,(idxT, idxV)in enumerate(skf.split(train_input['input_word_ids'], train.label.values)) :
print('
print('
print('
K.clear_session()
with strategy.scope() :
print('Building model...')
 model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5' %(VER, fold),
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
save_freq='epoch')
 kfold_train_input = {
'input_word_ids': train_input['input_word_ids'][idxT,],
'input_mask': train_input['input_mask'][idxT,],
'input_type_ids': train_input['input_type_ids'][idxT,]}
kfold_train_output = train.label.values[idxT,]
kfold_val_input = {
'input_word_ids': train_input['input_word_ids'][idxV,],
'input_mask': train_input['input_mask'][idxV,],
'input_type_ids': train_input['input_type_ids'][idxV,]}
kfold_val_output = train.label.values[idxV,]
print('Training...')
 kfold_history = model.fit(kfold_train_input,
kfold_train_output,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
verbose=1,
callbacks=[sv],
validation_data=(kfold_val_input, kfold_val_output))
history.append(kfold_history)
 print('Loading model...')
 model.load_weights('%s-roberta-%i.h5' %(VER, fold))
print('Predicting Test...')
 pred_test += model.predict(test_input)/ skf.n_splits
print('Predicting OOF...')
 oof = [np.argmax(i)for i in model.predict(kfold_val_input)]
kfold_accuracy = accuracy_score(oof, kfold_val_output)
 accuracy.append(kfold_accuracy)
 print('> FOLD %i - Accuracy: %.4f' %(fold + 1, kfold_accuracy))
print()
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,LSTM
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator<normalization>
print('> OVERALL KFold CV Accuracy: %.4f' % np.mean(accuracy))
result = []
full_scaler = MinMaxScaler()<train_on_grid>
submission = test.id.copy().to_frame()
submission['prediction'] = [np.argmax(i)for i in pred_test]
for i in range(len(state_with_country_name_for_test)) :
scaled_full_data = full_scaler.fit_transform(state_i_data[i])
 length = 1
batch_size = 1
generator = TimeseriesGenerator(scaled_full_data, scaled_full_data, length=length, batch_size=1)
 model = Sequential()
model.add(LSTM(96,input_shape=(length,scaled_full_data.shape[1])))
 model.add(Dense(scaled_full_data.shape[1]))
model.compile(optimizer='adam', loss='mse')
 model.fit_generator(generator,epochs=6)
 n_features = scaled_full_data.shape[1]
test_predictions = []
first_eval_batch = scaled_full_data[-length:]
current_batch = first_eval_batch.reshape(( 1, length, n_features))
for j in range(len(state_i_data_for_test[i])) :
current_pred = model.predict(current_batch)[0]
test_predictions.append(current_pred)
 current_batch = np.append(current_batch[:,1:,:],[[current_pred]],axis=1)
 true_predictions = full_scaler.inverse_transform(test_predictions)
 true_predictions = true_predictions.round()
true_predictions = pd.DataFrame(data=true_predictions,columns=state_i_data[1].columns)
 result.append(true_predictions)
 print('count:-',i)
<prepare_output>
submission.to_csv('submission.csv', index=False )
<prepare_output><EOS> 
submission.to_csv('submission.csv', index=False )