Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<feature_engineering>
!pip install transformers==3.0.2
!pip install nlp
test['Age'].replace(np.nan,np.median(test['Age'].dropna()),inplace=True)
 train['Age'].replace(np.nan,np.median(train['Age'].dropna()),inplace=True )<categorify>
np.random.seed(123 )
train['Sex'].replace({'male':1,'female':0},inplace=True)
 test['Sex'].replace({'male':1,'female':0},inplace=True )<categorify>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
embarked=pd.get_dummies(train['Embarked'])
 t_embarked=pd.get_dummies(test['Embarked'])
 train=train.join(embarked)
 test=test.join(t_embarked )<drop_column>
MODEL = 'joeddav/xlm-roberta-large-xnli'
EPOCHS = 4
MAXLEN = 120
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 auto = tf.data.experimental.AUTOTUNE
train.drop(['Name','SibSp','Parch','Ticket','Cabin','Embarked'],inplace=True,axis=1)
 test.drop(['Name','SibSp','Parch','Ticket','Cabin','Embarked'],inplace=True,axis=1 )<correct_missing_values>
def create_model(transformer_layer,learning_rate):
input_ids = Input(shape =(MAXLEN,), dtype = tf.int32)
 roberta = TFAutoModel.from_pretrained(transformer_layer)
 roberta = roberta([input_ids])[0]
out = GlobalAveragePooling1D()(roberta)
 out = Dense(3, activation = 'softmax' )(out)
 model = Model(inputs = input_ids, outputs = out)
 model.compile(
 optimizer = Adam(lr = learning_rate),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 return model
def tokeniZer(text,tokenizer):
encoded = tokenizer.batch_encode_plus(text, padding=True, max_length=MAXLEN, truncation=True)
 return np.array(encoded['input_ids'])
 def tokeniZer_old(dataset,tokenizer):
text = dataset[['premise', 'hypothesis']].values.tolist()
encoded = tokenizer.batch_encode_plus(text, padding=True, max_length=MAXLEN, truncation=True)
 x = encoded['input_ids'], encoded['attention_mask']
y = None
if 'label' in dataset.columns:
y = dataset.label.values
return encoded['input_ids'], encoded['attention_mask'], y
def load_mnli(use_validation=True):
result = []
dataset = load_dataset(path='glue', name='mnli')
 print(dataset['train'])
 keys = ['train', 'validation_matched','validation_mismatched'] if use_validation else ['train']
for k in keys:
for record in dataset[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result.append(( c1,c2,c3,'en'))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
def load_snli(use_validation=True):
result = []
dataset = load_dataset(path='snli')
 keys = ['train', 'validation'] if use_validation else ['train']
for k in keys:
for record in dataset[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result.append(( c1,c2,c3,'en'))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
def load_xnli() :
result = []
dataset = load_dataset(path='xnli')
 for k in dataset.keys() :
for record in dataset[k]:
hp, pr, lb = record['hypothesis'], record['premise'], record['label']
if hp and pr and lb in {0,1,2}:
for lang, translation in zip(hp['language'], hp['translation']):
pr_lang = pr.get(lang, None)
 if pr_lang is None:
continue
result.append(( pr_lang, translation, lb,lang))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
test['Fare'].replace({np.nan:np.median(test['Fare'].dropna())},inplace=True )<drop_column>
train = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/train.csv')
 test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv' )
test.set_index('PassengerId',inplace=True)
 train.set_index('PassengerId',inplace=True )<prepare_x_and_y>
mnli = load_mnli()
X_train=train.drop('Survived',axis=1)
 y_train=train['Survived']<predict_on_test>
total_train = train.drop(columns=['id'])
 total_train = pd.concat([total_train, mnli], axis=0)
 shuffled_data = shuffle(total_train ).reset_index(drop = True)
 X, y = shuffled_data[['premise', 'hypothesis']].values.tolist() , shuffled_data['label']
forest=RandomForestClassifier(max_depth=4,n_estimators=2000)
 forest.fit(X_train,y_train)
 y_pred=forest.predict_proba(test)
 y_pred=(y_pred[:,1]>0.45 ).astype(int)
 final_df=pd.DataFrame({'PassengerId':test.index,'Survived':y_pred})
 final_df.set_index('PassengerId',inplace=True )<save_to_csv>
x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25, random_state=2020 )
final_df.to_csv('submit.csv' )<prepare_output>
x_train = tokeniZer(x_train,tokenizer)
 x_valid = tokeniZer(x_valid,tokenizer)
 x_test = tokeniZer(test[['premise', 'hypothesis']].values.tolist() ,tokenizer)
 del mnli
gc.collect()
svc=SVC(C=1,gamma=10)
 svc.fit(X_train,y_train)
 y_pred2=svc.predict(test)
 final_df2=pd.DataFrame({'PassengerId':test.index,'Survived':y_pred2})
 final_df2.set_index('PassengerId',inplace=True )<save_to_csv>
def build_dataset(x, y, mode, batch_size):
if mode == "train":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 elif mode == "valid":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x, y)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(auto)
)
 elif mode == "test":
dataset =(
 tf.data.Dataset 
.from_tensor_slices(x)
.batch(BATCH_SIZE)
)
 else:
raise NotImplementedError
return dataset
final_df2.to_csv('submit.csv' )<compute_test_metric>
train_dataset = build_dataset(x_train, y_train, "train", BATCH_SIZE)
 valid_dataset = build_dataset(x_valid, y_valid, "valid", BATCH_SIZE)
 test_dataset = build_dataset(x_test, None, "test", BATCH_SIZE )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-3/"
model_directory="/kaggle/input/modelv3-dir/model"
geo_dir=None
extra_stable_columns=None
group_by_columns=None
if not group_by_columns is None and "continent" in group_by_columns:
assert not geo_dir is None
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 train=pd.merge(train,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 test=pd.merge(test,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 train.to_csv(directory + "train_plus_geo.csv", index=False)
 group_names=None
if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train=pd.merge(train,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]
train.to_csv(directory + "train_plus_groups.csv", index=False)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
print("max_train_date",(max_train_date))
print("max_test_date",(max_test_date))
target1="ConfirmedCases"
target2="Fatalities"
key="key" 
<categorify>
def create_xlm(transformer_layer,random_seed,learning_rate):
tf.keras.backend.clear_session()
tf.random.set_seed(random_seed)
 with strategy.scope() :
model = create_model(transformer_layer,learning_rate)
 model.summary()
return model
fix_target(train, key, target1, new_target_name=target1)
 fix_target(train, key, target2, new_target_name=target2)
 rate(train, key, target1, new_target_name="rate_" +target1)
 rate(train, key, target2, new_target_name="rate_" +target2)
 unique_keys=train[key].unique()
print(len(unique_keys))
train<categorify>
Xlm = create_xlm(MODEL,1124234,1e-5 )
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_data=None, groups_data=None, windows_group=[3,7], size_group=20,
days_back_confimed_group=[1,10,100]):
targets=[]
if not extra_data is None:
assert len(extra_stable_columns)==extra_data.shape[1]
if not groups_data is None:
assert len(group_names)==groups_data.shape[1]
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_data is None:
names+=[k for k in extra_stable_columns]
if not groups_data is None:
for gg in range(groups_data.shape[1]):
names+=["lag_rate_group_"+ str(gg+1)+ "_" + str(k+1)for k in range(size_group)]
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["std_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 row_features+=lag_confirmed_rate
for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
row_features+= std3_rate_confirmed
ewma3_rate_confirmed=generate_ewma_window(rate_confirmed, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 row_features+=lag_fatalities_rate
for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
row_features+= std3_rate_fatalities
ewma3_rate_fatalities=generate_ewma_window(rate_fatalities, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
if not extra_data is None:
row_features+=extra_data[i].tolist()
if not groups_data is None:
for gg in range(groups_data.shape[1]):
this_group=groups_data[:,gg].tolist()
lag_group_rate=get_lags(this_group, i, size=size_group)
 row_features+=lag_group_rate
for day in days_back_confimed_group:
days_ago_groupcount_2=days_ago_thresold_hit(this_group, i, day)
 row_features+=[days_ago_groupcount_2]
for window in windows_group:
ma3_rate_group,std3_rate_group= generate_ma_std_window(this_group, i, size=size_group, window=window)
 row_features+= ma3_rate_group
row_features+= std3_rate_group
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_stable_=None, group_nams=None,windows_group=[3,7],
size_group=20, days_back_confimed_group=[1,10,100]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities,
extra_data=mini_frame[extra_stable_].values if not extra_stable_ is None else None,
groups_data=mini_frame[group_nams].values if not group_nams is None else None,
windows_group=windows_group, size_group=size_group,
days_back_confimed_group=days_back_confimed_group)
 return mini_frame_with_features<predict_on_test>
callbacks = [tf.keras.callbacks.EarlyStopping(patience = 2,
monitor = 'val_loss',
restore_best_weights = True,
mode = 'min')]
steps_per_epoch = len(x_train)// BATCH_SIZE
history_xlm = Xlm.fit(train_dataset,
validation_data=valid_dataset,
steps_per_epoch=steps_per_epoch,
epochs = EPOCHS,
callbacks = callbacks)
<define_variables><EOS> 
predictions_xlm = Xlm.predict(test_dataset)
 predictions = predictions_xlm
final = np.argmax(predictions, axis = 1)
 submission = pd.DataFrame()
submission['id'] = test['id']
submission['prediction'] = final.astype(np.int32)
 submission.to_csv('submission.csv', index = False )