Kaggle competition: Jigsaw Multilingual Toxic Comment Classification
<SOS> metric: AUC Kaggle data source: jigsaw-multilingual-toxic-comment-classification<prepare_x_and_y>
%pip install transformers
features = ["bone_length", "rotting_flesh", "hair_length", "has_soul", "hair_soul", "hair_bone", "soul_bone"]
X_Train = pd.concat([train[features], pd.get_dummies(train["color"])], axis=1)
 y_Train = train["type"]
X_Test = pd.concat([test[features], pd.get_dummies(test["color"])], axis=1 )<categorify>
interativeEnvironment=False
localEnvironment=False
IN_COLAB=False
try:
except ImportError as e:
try:
IN_COLAB = True
except:
IN_COLAB = False
localEnvironment=True
os.environ['XLA_USE_BF16'] = "1"
localEnvironment
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 300)
 pd.set_option('display.width', None)
 %matplotlib inline
le_y = LabelEncoder()
y_Train = le_y.fit_transform(y_Train)
 print(X_Train.shape)
 print(y_Train.shape )<split>
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive() )
X_Train1, X_Test1, y_Train1, y_Test1 = train_test_split(X_Train, y_Train, test_size=0.20, random_state=36)
 print(X_Train1.shape)
 print(y_Train1.shape )<define_search_space>
forest = RandomForestClassifier(n_estimators = 20,
criterion = 'entropy',
max_features = 'auto')
 parameter_grid = {
'max_depth' : [None, 5, 20, 100],
'min_samples_split' : [2, 5, 7],
'min_weight_fraction_leaf' : [0.0, 0.1],
'max_leaf_nodes' : [20, 30],
}
grid_search_rf = GridSearchCV(forest, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_rf.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_rf.best_score_))
print('Best parameters: {}'.format(grid_search_rf.best_params_))<predict_on_test>
tf_env="tensorflow"
y_test_pred = grid_search_rf.predict(X_Test1)
 y_test_pred
accuracy_score(y_Test1, y_test_pred )<train_on_grid>
fit_verbocity=1
model=tf_env
nrows=None
interativeEnvironment=is_interactive()
interativeEnvironment=False
if localEnvironment:
warnings.warn("Nrows limited")
 pass
elif interativeEnvironment:
pass
else:
fit_verbocity=2
pooling_mode_cls_token="CLS_TOKEN"
pooling_mode_fc="FC"
pooling_mode=pooling_mode_fc
fc_dims=[]
trainable_transformer=False
use_dann=False
dann_lambda=-1
use_parcor=False
parcor_lambda=1
use_augmented_data=False
use_lowercase_data=False
use_finetuning=True
use_validation_during_pretraining=True
use_translated_data=True
use_pretraining=True
load_model=None
logreg = LogisticRegression()
parameter_grid = {'solver' : ['newton-cg', 'lbfgs'],
'multi_class' : ['multinomial'],
'C' : [0.005, 0.01, 1, 10],
'tol': [0.0001, 0.001, 0.005, 0.01]
}
grid_search_logit = GridSearchCV(logreg, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_logit.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_logit.best_score_))
print('Best parameters: {}'.format(grid_search_logit.best_params_))<predict_on_test>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
y_test_pred = grid_search_logit.predict(X_Test1)
 y_test_pred
accuracy_score(y_Test1, y_test_pred )<train_on_grid>
@tf.custom_gradient
def grad_reverse(x):
y = tf.identity(x)
 def custom_grad(dy):
return tf.math.scalar_mul(-1,dy)
 return y, custom_grad
class GradReverse(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
def call(self, x):
return grad_reverse(x )
svr = svm.SVC()
parameter_grid = {'kernel':('linear', 'rbf', 'poly'), 'C':[0.005, 0.01, 1, 10, 100, 1000], 'degree':[2, 3]}
grid_search_svc = GridSearchCV(svr, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_svc.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_svc.best_score_))
print('Best parameters: {}'.format(grid_search_svc.best_params_))<predict_on_test>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
y_test_pred = grid_search_svc.predict(X_Test1)
 y_test_pred
accuracy_score(y_Test1, y_test_pred )<train_on_grid>
AUTO = tf.data.experimental.AUTOTUNE
if not localEnvironment and not IN_COLAB:
GCS_DS_PATH = KaggleDatasets().get_gcs_path("jigsaw-multilingual-toxic-comment-classification")
 NEXAMPLESPEREPOCH=240000
EPOCHS = 10
if strategy.num_replicas_in_sync==1:
BATCH_SIZE = 4 * strategy.num_replicas_in_sync
else:
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
if localEnvironment:
MODEL='jplu/tf-xlm-roberta-base'
if IN_COLAB:
MODEL='jplu/tf-xlm-roberta-base'
print("BATCH_SIZE: ", BATCH_SIZE )
warnings.filterwarnings("ignore")
 nnet = neural_network.MLPClassifier()
parameter_grid = {'activation':['logistic','relu'], 'hidden_layer_sizes':[10,15,25], 'alpha':[1e-4, 1e-3, 1e-2, 1e-1], 'solver':['lbfgs','sgd'], 'learning_rate':['constant', 'adaptive']}
grid_search_nnet = GridSearchCV(nnet, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_nnet.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_nnet.best_score_))
print('Best parameters: {}'.format(grid_search_nnet.best_params_))<train_on_grid>
if IN_COLAB:
project_id="global-sun-279412"
sgd = SGDClassifier()
parameter_grid = {'average':[True],'loss':['log','perceptron','squared_loss'], 'penalty':[None,'l2','l1'], 'alpha':[0.001,0.01, 0.1, 1], 'l1_ratio':[0.15, 0.1, 0.3], 'learning_rate':['optimal'], 'random_state':[0]}
grid_search_sgd = GridSearchCV(sgd, param_grid=parameter_grid, cv=StratifiedKFold(3))
grid_search_sgd.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_sgd.best_score_))
print('Best parameters: {}'.format(grid_search_sgd.best_params_))<train_on_grid>
!gcloud config set project {project_id}
bag = ensemble.BaggingClassifier()
parameter_grid = {'base_estimator':[None,DummyClassifier() ,Perceptron() ,DecisionTreeClassifier() ,KNeighborsClassifier() ,SVC() ], "max_samples": [0.5, 1.0], "max_features": [1, 2, 4],"bootstrap": [True, False],"bootstrap_features": [True, False],'random_state':[0]}
grid_search_bag = GridSearchCV(bag, param_grid=parameter_grid, scoring='log_loss', refit='True', cv=StratifiedKFold(3))
grid_search_bag.fit(X_Train1, y_Train1)
 print('Best score: {}'.format(grid_search_bag.best_score_))
print('Best parameters: {}'.format(grid_search_bag.best_params_))<train_model>
bucket_name="jigsaw-tfrecords"
x=tf.contrib.learn.infer_real_valued_columns_from_input(X_Train1)
 tf_clf_dnn = learn.DNNClassifier(hidden_units=[16], n_classes=3, feature_columns=x, activation_fn=tf.sigmoid)
 tf_clf_dnn.fit(X_Train1, y_Train1, max_steps=5000)
 print(acc_s(y_Test1,tf_clf_dnn.predict(X_Test1)) )<train_on_grid>
if IN_COLAB:
auth.authenticate_user()
clf = tf.contrib.learn.LinearClassifier(
 feature_columns=tf.contrib.learn.infer_real_valued_columns_from_input(X_Train),
n_classes=3,
optimizer=tf.train.AdagradOptimizer(
 learning_rate=0.5, 
))
clf.fit(X_Train, y_Train, steps=500)
 y_pred = clf.predict(X_Test)
 pred = le_y.inverse_transform(y_pred )<create_dataframe>
if localEnvironment:
root =".. "
elif IN_COLAB:
root='gs://jigsaw-tfrecords/'
else:
root="/kaggle"
root
submission = pd.DataFrame({'id':test_id,
'type':pred} )<save_to_csv>
validation_fold_index=1
validation_fold_index
submission.to_csv('submission.csv', index=False )<save_to_csv>
def validationFoldRemoveFilter(validation_fold_index):
def _filter(features):
return tf.math.not_equal(features["ShardIndex"],validation_fold_index)
 return _filter
submission.to_csv('submission.csv', index=False )<import_modules>
def validationFoldOnlyFilter(validation_fold_index):
def _filter(features):
return tf.math.equal(features["ShardIndex"],validation_fold_index)
 return _filter
import numpy as np
import pandas as pd<load_from_csv>
if IN_COLAB:
TF_DS_PATH=root+"jigsaw-official"
elif not localEnvironment:
TF_DS_PATH = KaggleDatasets().get_gcs_path("jigsawaugmented")+"/tfrecords"
else:
TF_DS_PATH = ".. /tfrecords"
TF_DS_PATH
train = pd.read_csv(".. /input/train.csv", index_col=0)
 test = pd.read_csv(".. /input/test.csv", index_col=0 )<categorify>
file="en1_train_non_toxic_PropercaseSentences_0_4.tfrecord" 
("Sentences" not in file)or("LowercaseSentences" in file )
y_train = pd.get_dummies(train[["type"]], prefix="")
 train.drop("type", inplace=True, axis=1)
 train_test = pd.concat([train, test], axis=0)
 train_test.drop("color", inplace=True, axis=1)
 X_train = train_test.iloc[:len(y_train)]
X_test = train_test.iloc[len(y_train):]
del train_test
del train
del test<choose_model_class>
data_files=tf.io.gfile.glob(TF_DS_PATH + "/*.tfrecord")
 if use_augmented_data:
pass
elif use_lowercase_data:
data_files=list(filter(lambda file:("Sentences" not in file)or("LowercaseSentences" in file),data_files))
else:
data_files=list(filter(lambda file:"Sentences" not in file,data_files))
data_files
def sigmoid(X):
return 1.0 /(1.0 + np.exp(-X))
def softmax(X):
_sum = np.exp(X ).sum()
return np.exp(X)/ _sum
class neuralnet(object):
def __init__(self, num_input, num_hidden, num_output):
self._W1 =(np.random.random_sample(( num_input, num_hidden)) - 0.5 ).astype(np.float32)
 self._b1 = np.zeros(( 1, num_hidden)).astype(np.float32)
 self._W2 =(np.random.random_sample(( num_hidden, num_output)) - 0.5 ).astype(np.float32)
 self._b2 = np.zeros(( 1, num_output)).astype(np.float32)
 def forward(self,X):
net1 = np.matmul(X, self._W1)+ self._b1
y = sigmoid(net1)
 net2 = np.matmul(y, self._W2)+ self._b2
z = softmax(net2)
 return z,y
def backpropagation(self, X, target, eta):
z, y = self.forward(X)
 d2 =(z - target)
 d1 = y*(1.0-y)* np.matmul(d2, self._W2.T)
 self._W2 -= eta * np.matmul(y.T,d2)
 self._W1 -= eta * np.matmul(X.reshape(( -1,1)) ,d1)
 self._b2 -= eta * d2
self._b1 -= eta * d1<train_on_grid>
toxic_en_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non_toxic" not in file and "valid" not in file and "sharded" not in file, data_files))
toxic_en_files
num_hidden = 8
n_epochs = 1500
eta = 0.01
nn = neuralnet(X_train.shape[1], num_hidden, y_train.shape[1] )<train_model>
non_toxic_en_files=list(filter(lambda file:"nen" not in file and "non_toxic" in file and "valid" not in file and "sharded" not in file, data_files))
non_toxic_en_files
X = np.array(X_train, dtype=np.float32)
 Y = np.array(y_train, dtype=np.float32)
 for epoch in range(n_epochs):
for monster, target in zip(X,Y):
nn.backpropagation(monster, target, eta )<save_to_csv>
non_toxic_nen_files=list(filter(lambda file:"non_toxic" in file and "nen1" in file, data_files))
non_toxic_nen_files
with open('submission-{}-hidden.csv'.format(num_hidden), 'w')as f:
f.write("id,type
")
 for index, monster in X_test.iterrows() :
probs = nn.forward(np.array(monster, dtype=np.float32)) [0]
f.write("{},{}
".format(index, y_train.columns.values[np.argmax(probs)][1:]))<load_from_csv>
toxic_nen_files=list(filter(lambda file:"non_toxic" not in file and "nen1" in file, data_files))
toxic_nen_files
path = '.. /input/'
train_df = pd.read_csv(path+"/train.csv")
 test_df = pd.read_csv(path+"/test.csv" )<create_dataframe>
unknown_nen_files=list(filter(lambda file: "test" not in file and "nen2" in file, data_files))
unknown_nen_files
Targets_df=pd.DataFrame()
Targets_df["bandgap_energy_ev"]=train_df["bandgap_energy_ev"].copy()
Targets_df["formation_energy_ev_natom"]=train_df["formation_energy_ev_natom"].copy()
train_df=train_df.drop(["formation_energy_ev_natom","bandgap_energy_ev"],axis=1)
 train_id_df=pd.DataFrame()
train_id_df["id"]=train_df["id"].copy()
train_df=train_df.drop(["id"],axis=1)
 test_id_df=pd.DataFrame()
test_id_df["id"]=test_df["id"].copy()
test_df=test_df.drop(["id"],axis=1)
 combined_df = pd.concat([train_df, test_df], ignore_index=True)
 print("Total number of null values in the df","
")
 print(combined_df.isna().sum().sum() )<categorify>
valid_files=list(filter(lambda file:"test" not in file and "nen1" in file and "Sentences" not in file, data_files))
valid_files
numerical_df=pd.DataFrame.copy(combined_df[['number_of_total_atoms', 'percent_atom_al',
'percent_atom_ga', 'percent_atom_in', 'lattice_vector_1_ang',
'lattice_vector_2_ang', 'lattice_vector_3_ang',
'lattice_angle_alpha_degree', 'lattice_angle_beta_degree',
'lattice_angle_gamma_degree']])
 one_hot_df=pd.DataFrame.copy(combined_df[["spacegroup"]])
 one_hot_df=pd.get_dummies(one_hot_df,prefix=["spacegroup"],
columns=["spacegroup"])
 features_df=pd.concat([numerical_df,one_hot_df],axis=1)
 ")
 skewed_feats= numerical_df.skew()
skewed_feats = skewed_feats[skewed_feats > 0.1]
skewed_feats = skewed_feats.index
unskewed_feats= numerical_df.skew()
unskewed_feats = unskewed_feats[unskewed_feats < 0.1]
unskewed_feats = unskewed_feats.index
transform_df=pd.DataFrame()
transform_df[unskewed_feats]=(numerical_df[unskewed_feats]
- numerical_df[unskewed_feats].mean())/(numerical_df[unskewed_feats].max() - numerical_df[unskewed_feats].min())
 transform_df[skewed_feats] = np.log1p(numerical_df[skewed_feats])
 ")
 features_transform_df=pd.concat([transform_df,one_hot_df],axis=1)
 features_df=pd.concat([numerical_df,one_hot_df],axis=1 )<define_variables>
if use_translated_data:
if IN_COLAB:
TRANLSATED_DATA_PATH_ONE=root+"/translated-train-bias-all-langs"
TRANLSATED_DATA_PATH_TWO=root+"/jigsaw-train-multilingual-coments-google-api"
elif not localEnvironment:
TRANSLATED_DATA_PATH = KaggleDatasets().get_gcs_path("jigsawtranslated")+"/tfrecords-translated"
else:
pass
TRANSLATED_DATA_PATH
training_examples=features_df.iloc[0:2400].copy()
test_examples=features_df.iloc[2400:3000].copy()
training_examples_transform=features_transform_df.iloc[0:2400].copy()
test_examples_transform=features_transform_df.iloc[2400:3000].copy()<compute_train_metric>
translated_files=[]
if use_translated_data:
translated_files=tf.io.gfile.glob(TRANSLATED_DATA_PATH + "/*tfrecord")
 translated_files[:3],TRANSLATED_DATA_PATH
def rmsle_cv(model):
rmsle= np.sqrt(-cross_val_score(model, training_examples_transform, training_targets, scoring="neg_mean_squared_log_error", cv = 5))
return(rmsle )<train_on_grid>
toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" not in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(toxic_translated_files )
training_targets=Targets_df["bandgap_energy_ev"].copy()
model_linear=LinearRegression().fit(training_examples_transform, training_targets)
 linear_BG_pred = model_linear.predict(test_examples_transform)
 BG_rmsle=rmsle_cv(model_linear ).mean()
print("Band gap RMSLE:")
 print(BG_rmsle,"
")
 training_targets=Targets_df["formation_energy_ev_natom"].copy()
model_linear=LinearRegression().fit(training_examples_transform, training_targets)
 linear_EF_pred = model_linear.predict(test_examples_transform)
 EF_rmsle=rmsle_cv(model_linear ).mean()
print("Formation Energy RMSLE:")
 print(EF_rmsle,"
")
 print("Expected combined RMSLE")
 combined_rmsle=(EF_rmsle+BG_rmsle)/2
print(combined_rmsle)
 Predictions_df=pd.DataFrame()
Predictions_df["id"]=test_id_df["id"].copy()
Predictions_df["formation_energy_ev_natom"]=linear_EF_pred
Predictions_df["bandgap_energy_ev"]=linear_BG_pred
Predictions_df.to_csv("Linear_Nomad.csv",index=False )<train_model>
non_toxic_translated_files=list(filter(lambda file:"nen" not in file and "toxic" in file and "non" in file and "valid" not in file and "sharded" not in file, translated_files))
random.choice(non_toxic_translated_files )
training_targets=np.log1p(Targets_df["bandgap_energy_ev"].copy())
 dtrain = xgb.DMatrix(training_examples, label = training_targets)
 dtest = xgb.DMatrix(test_examples)
 params = {"max_depth":2,
"eta":0.1,
'gamma':0,
'subsample':0.8,
'colsample_bytree':1,
'min_child_weight':10,
'reg_alpha':7e-3,
'reg_lambda':1,
}
model_xgb = xgb.cv(params, dtrain, num_boost_round=500,early_stopping_rounds=100)
 model_xgb.loc[30:,["test-rmse-mean", "train-rmse-mean"]].plot()
last=len(model_xgb.loc[:])-1
print(model_xgb.loc[last:,["test-rmse-mean"]] )<train_model>
model_xgb = xgb.XGBRegressor(n_estimators=last,max_depth=2, learning_rate=0.1,
gamma=0,subsample=0.8,colsample_bytree=1,min_child_weight=10,
reg_alpha=7e-3,reg_lambda=1)
 model_xgb.fit(training_examples, training_targets)
 xgb.plot_importance(model_xgb)
 xgb_BG_preds = np.expm1(model_xgb.predict(test_examples))<prepare_x_and_y>
opus_langs=[['en', 'it'],
['en', 'ru'],
['en', 'tr'],
['en', 'pt_br'],
['en', 'fr'],
['en', 'pt'],
['en', 'es'],
['en', 'hi']]
if IN_COLAB:
OPUS_DS_PATH=root+"opus"
elif not localEnvironment:
OPUS_DS_PATH = KaggleDatasets().get_gcs_path("opus-for-jigsaw-sharded")
 else:
OPUS_DS_PATH = ".. /opus-processed"
OPUS_DS_PATH
training_targets=np.log1p(Targets_df["formation_energy_ev_natom"].copy())
 dtrain = xgb.DMatrix(training_examples, label = training_targets)
 dtest = xgb.DMatrix(test_examples)
 params = {"max_depth":4,
"eta":0.08,
'gamma':0,
'subsample':1,
'colsample_bytree':0.4,
'min_child_weight':3,
'reg_alpha':0,
'reg_lambda':0,
}
model_xgb = xgb.cv(params, dtrain, num_boost_round=500,early_stopping_rounds=100)
 model_xgb.loc[30:,["test-rmse-mean", "train-rmse-mean"]].plot()
last=len(model_xgb.loc[:])-1
print(model_xgb.loc[last:,["test-rmse-mean"]] )<train_model>
if IN_COLAB:
extension=".gz"
else:
extension=""
opus_files=tf.io.gfile.glob(OPUS_DS_PATH + "/*.tfrecord"+extension)
 if interativeEnvironment:
print(opus_files)
model_xgb = xgb.XGBRegressor(n_estimators=last,max_depth=4, learning_rate=0.08,
gamma= 0, colsample=1,colsample_bytree=0.4,min_child_weight=3,
reg_lambda=0)
 model_xgb.fit(training_examples, training_targets)
 xgb.plot_importance(model_xgb)
 xgb_EF_preds = np.expm1(model_xgb.predict(test_examples))<save_to_csv>
feature_description = {
'tokens': tf.io.FixedLenFeature([192], tf.int64),
'mask': tf.io.FixedLenFeature([192], tf.int64),
'toxic': tf.io.FixedLenFeature([], tf.float32),
's_toxic': tf.io.FixedLenFeature([], tf.float32),
'source':tf.io.FixedLenFeature([], tf.int64),
'toxicity_annotator_count':tf.io.FixedLenFeature([], tf.int64),
'lang':tf.io.FixedLenFeature([], tf.string),
'ShardIndex':tf.io.FixedLenFeature([], tf.int64),
}
def _parse_proto(example_proto):
example= tf.io.parse_single_example(example_proto, feature_description)
 return example
def _form_tuple(example):
y={"toxic":tf.cast(example['s_toxic'],tf.float32)}
if use_dann:
y["lang"]=tf.cast(tf.math.equal(example['lang'],'en'),tf.float32)
 if use_parcor:
y["is_same"]=example["is_same"]
return(example['tokens'],example['mask']),y
def smoother(smoothing=0.01):
def smooth(example):
label=tf.cast(example['s_toxic'],tf.float32)
 label=tf.cond(label>=1.0-smoothing,lambda :tf.ones_like(label)-smoothing,lambda :tf.cond(label<0.0+smoothing,lambda :tf.zeros_like(label)+smoothing,lambda :label))
example['s_toxic']=label
return example
return smooth
def isValidLabel(example):
example['isLabelValid']=tf.cond(example['toxic']==-1,lambda :tf.constant(0.0),lambda :tf.constant(1.0))
return example
Predictions_df=pd.DataFrame()
Predictions_df["id"]=test_id_df["id"].copy()
Predictions_df["formation_energy_ev_natom"]=xgb_EF_preds
Predictions_df["bandgap_energy_ev"]=xgb_BG_preds
Predictions_df.to_csv("XGB_Nomad.csv",index=False )<save_to_csv>
def perFileDataset(filename):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename ).map(_parse_proto)
 def perFileDatasetShuffled(filename):
return perFileDataset(filename ).shuffle(200000)
 def unsqueezeTargetDimensions(features,labels):
labels=tf.cast(labels,tf.float32)
 return features,tf.stack([tf.math.subtract(tf.constant([1.0]),labels),labels],axis=-1),
Predictions_df=pd.DataFrame()
Predictions_df["id"]=test_id_df["id"].copy()
stacked_EF_preds=0.9*xgb_EF_preds+0.1*linear_EF_pred
Predictions_df["formation_energy_ev_natom"]=stacked_EF_preds
stacked_BG_preds=0.9*xgb_BG_preds+0.1*linear_BG_pred
Predictions_df["bandgap_energy_ev"]=stacked_BG_preds
Predictions_df.to_csv("Stacked_Nomad.csv",index=False )<load_from_csv>
def toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))
warnings.filterwarnings('ignore')
 print(check_output(["ls", ".. /input"] ).decode("utf8"))
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<split>
def non_toxic_train_en_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_en_files ).shuffle(2048 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother())
 def non_toxic_train_translated_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(non_toxic_translated_files ).shuffle(2048*4 ).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12 ).map(smoother(0.1))
x_columns = [i for i in train.columns if i not in list(['id','formation_energy_ev_natom','bandgap_energy_ev'])]
label1 = 'formation_energy_ev_natom'
label2 = 'bandgap_energy_ev'
X = train[x_columns]
y = train[[label1,label2]]
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=2017)
 X_train = X_train.as_matrix()
X_valid = X_valid.as_matrix()
y_train_values1 = np.log1p(y_train['formation_energy_ev_natom'].values)
 y_train_values2 = np.log1p(y_train['bandgap_energy_ev'].values)
 y_valid_values1 = np.log1p(y_valid['formation_energy_ev_natom'].values)
 y_valid_values2 = np.log1p(y_valid['bandgap_energy_ev'].values )<compute_train_metric>
def toxic_train_nen_dataset(shardIndex=-1):
return tf.data.Dataset.from_tensor_slices(toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
clf1 = KernelRidge(kernel ='linear', alpha=1.0)
 clf2 = KernelRidge(kernel ='linear', alpha=1.0)
 clf1.fit(X_train,y_train_values1)
 clf2.fit(X_train,y_train_values2)
 preds1 = clf1.predict(X_valid)
 preds2 = clf2.predict(X_valid)
 y_pred1 = np.exp(preds1)-1
y_pred2 = np.exp(preds2)-1
rsme_valid1 = np.sqrt(mean_squared_error(y_valid_values1,preds1))
rsme_valid2 = np.sqrt(mean_squared_error(y_valid_values2,preds2))
rsme_total = np.sqrt(rsme_valid1*rsme_valid1+rsme_valid2*rsme_valid2)
 print('RSME for formation energy:')
 print(rsme_valid1)
 print('RSME for band gap:')
 print(rsme_valid2)
 print('RSME for total:')
 print(rsme_total )<compute_train_metric>
def non_toxic_train_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(non_toxic_nen_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother() ).shuffle(8000)
clf3 = KernelRidge(kernel ='polynomial', alpha=1.0)
 clf4 = KernelRidge(kernel ='polynomial', alpha=1.0)
 clf3.fit(X_train,y_train_values1)
 clf4.fit(X_train,y_train_values2)
 preds1 = clf3.predict(X_valid)
 preds2 = clf4.predict(X_valid)
 y_pred1 = np.exp(preds1)-1
y_pred2 = np.exp(preds2)-1
rsme_valid1 = np.sqrt(mean_squared_error(y_valid_values1,preds1))
rsme_valid2 = np.sqrt(mean_squared_error(y_valid_values2,preds2))
rsme_total = np.sqrt(rsme_valid1*rsme_valid1+rsme_valid2*rsme_valid2)
 print('RSME for formation energy:')
 print(rsme_valid1)
 print('RSME for band gap:')
 print(rsme_valid2)
 print('RSME for total:')
 print(rsme_total )<compute_train_metric>
def unknown_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(unknown_nen_files ).shuffle(2048 ).interleave(perFileDataset,num_parallel_calls=AUTO)
clf5 = KernelRidge(kernel ='rbf', alpha=1.0)
 clf6 = KernelRidge(kernel ='rbf', alpha=1.0)
 clf5.fit(X_train,y_train_values1)
 clf6.fit(X_train,y_train_values2)
 preds1 = clf5.predict(X_valid)
 preds2 = clf6.predict(X_valid)
 y_pred1 = np.exp(preds1)-1
y_pred2 = np.exp(preds2)-1
rsme_valid1 = np.sqrt(mean_squared_error(y_valid_values1,preds1))
rsme_valid2 = np.sqrt(mean_squared_error(y_valid_values2,preds2))
rsme_total = np.sqrt(rsme_valid1*rsme_valid1+rsme_valid2*rsme_valid2)
 print('RSME for formation energy:')
 print(rsme_valid1)
 print('RSME for band gap:')
 print(rsme_valid2)
 print('RSME for total:')
 print(rsme_total )<compute_train_metric>
def valid_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_files ).interleave(perFileDataset ).filter(validationFoldOnlyFilter(validation_fold_index))
clf7 = KernelRidge(kernel ='laplacian', alpha=1.0)
 clf8 = KernelRidge(kernel ='laplacian', alpha=1.0)
 clf7.fit(X_train,y_train_values1)
 clf8.fit(X_train,y_train_values2)
 preds1 = clf7.predict(X_valid)
 preds2 = clf8.predict(X_valid)
 y_pred1 = np.exp(preds1)-1
y_pred2 = np.exp(preds2)-1
rsme_valid1 = np.sqrt(mean_squared_error(y_valid_values1,preds1))
rsme_valid2 = np.sqrt(mean_squared_error(y_valid_values2,preds2))
rsme_total = np.sqrt(rsme_valid1*rsme_valid1+rsme_valid2*rsme_valid2)
 print('RSME for formation energy:')
 print(rsme_valid1)
 print('RSME for band gap:')
 print(rsme_valid2)
 print('RSME for total:')
 print(rsme_total )<save_to_csv>
def finetune_nen_dataset() :
return tf.data.Dataset.from_tensor_slices(valid_files ).shuffle(2048 ).interleave(perFileDataset ).filter(validationFoldRemoveFilter(validation_fold_index))
X_test = test[x_columns]
X_test = X_test.as_matrix()
preds1 = clf3.predict(X_test)
 preds2 = clf4.predict(X_test)
 y_pred1 = np.exp(preds1)-1
y_pred2 = np.exp(preds2)-1
krr = pd.DataFrame()
krr['id'] = test['id']
krr['formation_energy_ev_natom'] = y_pred1
krr['bandgap_energy_ev'] = y_pred2
krr.to_csv("krr_sub.csv", index=False )<load_from_csv>
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(non_toxic_train_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_train_non_toxic=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_train_non_toxic=5956
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(toxic_train_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_train_toxic=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_train_toxic=1061
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_unique_values>
%%time
if interativeEnvironment:
total_by_source=Counter()
total_by_toxic=Counter()
total=0
for i,l in tqdm(enumerate(unknown_nen_dataset())) :
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([l["s_toxic"].numpy() ])
 total_nen_unknown=total
total,total_by_toxic.most_common() ,total_by_source.most_common()
else:
total_nen_unknown=71078
spacegroups = train.spacegroup.unique()
spacegroups.sort()
print(spacegroups)
 n_spacegroups = len(spacegroups )<import_modules>
n_steps = NEXAMPLESPEREPOCH // BATCH_SIZE
num_nen=0
if use_validation_during_pretraining:
num_nen+=total_nen_train_toxic+total_nen_train_non_toxic
if use_dann:
num_nen += total_nen_unknown
n_steps,num_nen
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler, StandardScaler<define_variables>
num_dataset_section=2
if use_translated_data:
num_dataset_section+=2
def f0() :
return unknown_nen_dataset().repeat().take(total_nen_unknown)
 def f1() :
return toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f2() :
return non_toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f3() :
return toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f4() :
return non_toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f5() :
return toxic_train_nen_dataset()
def f6() :
return non_toxic_train_nen_dataset()
branches={0:f0,1:f1,2:f2,3:f3,4:f4,5:f5,6:f6}
mixedDatasetParams=[1,2]
if use_dann:
mixedDatasetParams.append(0)
 if use_validation_during_pretraining:
mixedDatasetParams.extend([5,6])
 if use_translated_data:
mixedDatasetParams.extend([3,4])
 def mapParamsToDataset(params):
return tf.switch_case(params,branches)
t1 = 'formation_energy_ev_natom'
t2 = 'bandgap_energy_ev'
transform_columns = ['number_of_total_atoms', 'percent_atom_al', 'percent_atom_ga', 'percent_atom_in', 'lattice_vector_1_ang', 'lattice_vector_2_ang', 'lattice_vector_3_ang', 'lattice_angle_alpha_degree', 'lattice_angle_beta_degree', 'lattice_angle_gamma_degree']
feature_columns = ['spacegroup'] + transform_columns 
<import_modules>
cycle_length=4
if use_dann:
cycle_length+=1
def non_parallel_dataset() :
return tf.data.Dataset.from_tensor_slices(mixedDatasetParams ).interleave(mapParamsToDataset, num_parallel_calls=AUTO,cycle_length=cycle_length ).take(NEXAMPLESPEREPOCH)
 if interativeEnvironment:
train_dict_dataset=non_parallel_dataset()
train_dict_dataset
import matplotlib.pyplot as plt
import matplotlib.cm as cm<normalization>
non_parallel_dataset()
all = pd.concat([train[feature_columns], test])
 scaler = MinMaxScaler()
scaler.fit(all[transform_columns])
 train[transform_columns] = scaler.transform(train[transform_columns])
 test[transform_columns] = scaler.transform(test[transform_columns] )<split>
%%time
if interativeEnvironment:
dlist=train_dict_dataset.repeat()
epoch_metrics={}
shard_metrics={}
for i,l in tqdm(enumerate(dlist)) :
epoch=str(i//NEXAMPLESPEREPOCH)
 shard=str(i//40000)
 if epoch=="10":
break
if epoch in epoch_metrics.keys() :
pass
else:
total_by_source=Counter()
total_by_toxic=Counter()
total_by_toxic_source=Counter()
total_by_s_toxic=Counter()
total=0
epoch_metrics[epoch]={"by_toxic":total_by_toxic,"by_source":total_by_source,"total":total,"by_both":total_by_toxic_source,"by_s_toxic":total_by_s_toxic}
if shard in shard_metrics.keys() :
pass
else:
stotal_by_source=Counter()
stotal_by_toxic=Counter()
stotal_by_toxic_source=Counter()
stotal_by_s_toxic=Counter()
stotal=0
shard_metrics[shard]={"by_toxic":stotal_by_toxic,"by_source":stotal_by_source,"total":stotal,"by_both":stotal_by_toxic_source,"by_s_toxic":stotal_by_s_toxic}
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 total_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 total_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 stotal+=1
stotal_by_source.update([l["source"].numpy() ])
 stotal_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 stotal_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 stotal_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 for epoch,epoch_metric in epoch_metrics.items() :
print("Epoch:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"By SToxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
" )
random_seed = 314
X_train, X_validation = train_test_split(train, test_size=0.2, random_state=random_seed)
 y_train = np.log1p(X_train[[t1, t2]])
 X_train = X_train.drop(['id', t1, t2], axis=1)
 y_validation = np.log1p(X_validation[[t1, t2]])
 X_validation = X_validation.drop(['id', t1, t2], axis=1)
 print(X_train.shape, y_train.shape)
 print(X_validation.shape, y_validation.shape )<import_modules>
if interativeEnvironment:
for epoch,epoch_metric in shard_metrics.items() :
print("Shard:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"Stoxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
" )
import tensorflow as tf<init_hyperparams>
def opus_files(lang):
return tf.io.gfile.glob(OPUS_DS_PATH + "/{}_*.tfrecord".format(lang)+extension)
 opus_files("es" )
tf.set_random_seed(1)
 np.random.seed(1)
 layers = [16] * 8
activation = tf.tanh
kernel_regularizer_l2 = None<categorify>
if use_parcor:
parallel_feature_description = {
'tokens_en': tf.io.FixedLenFeature([192], tf.int64),
'en_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'tokens_nen': tf.io.FixedLenFeature([192], tf.int64),
'nen_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'lang':tf.io.FixedLenFeature([], tf.string),
'source':tf.io.FixedLenFeature([], tf.int64,default_value=[5]),
's_toxic':tf.io.FixedLenFeature([2], tf.float32,default_value=[-1.0,-1.0]),
'is_same':tf.io.FixedLenFeature([], tf.float32,default_value=[1.0]),
}
def _parse_parallel_proto(example_proto):
example= tf.io.parse_single_example(example_proto, parallel_feature_description)
 return example
if IN_COLAB:
compression_type="GZIP"
else:
compression_type=""
def perFileParallelDataset(filename,feature_description=feature_description):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename,compression_type=compression_type ).map(_parse_parallel_proto)
 def make_parallel_pairs(example):
ret={}
ret["tokens"]=tf.stack([example["tokens_en"],example["tokens_nen"]])
 ret["mask"]=tf.stack([example["en_attn_mask"],example["nen_attn_mask"]])
 ret["s_toxic"]=example["s_toxic"]
ret["is_same"]=example["is_same"]
ret["lang"]=example["lang"]
return ret
def parallel_dataset(files):
return tf.data.Dataset.from_tensor_slices(files ).shuffle(2048 ).interleave(perFileParallelDataset ).map(make_parallel_pairs)
 def parallel_labelled_dataset(labelled_dataset):
return labelled_dataset.batch(2 ).map(make_parallel_labelled_pairs)
 def make_parallel_labelled_pairs(example):
example["is_same"]=tf.constant(0.0)
 return example
langs=["es","fr","pt","tr","ru","it"]
langFiles=list(map(opus_files,langs))
def all_parallel_dataset() :
return tf.data.Dataset.from_tensor_slices(langFiles ).interleave(parallel_dataset, num_parallel_calls=AUTO,cycle_length=len(langs))
def parallel_nonparallel_mixer(params):
return tf.cond(params=="parallel",lambda :all_parallel_dataset().take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple),lambda :parallel_labelled_dataset(non_parallel_dataset() ).take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple))
def parcor_dataset() :
return tf.data.Dataset.from_tensor_slices(["parallel","nonparallel"] ).interleave(parallel_nonparallel_mixer,num_parallel_calls=2 )
tf.reset_default_graph()
tf_is_training = tf.placeholder(tf.bool, None)
 tf_x = tf.placeholder(tf.float32,(None, X_train.shape[1]), name='tf_x')
 tf_y = tf.placeholder(tf.float32,(None, 2), name='tf_y' )<choose_model_class>
if interativeEnvironment:
ds=list(parallel_labelled_dataset(valid_nen_dataset() ).take(7))
print(list(map(lambda a:a["is_same"],ds)))
l = tf_x
for i, n in enumerate(layers):
l = tf.layers.dense(l, n, activation=activation, kernel_regularizer=kernel_regularizer_l2, name='layer%s' %(i + 1))
output = tf.layers.dense(l, 2, name='output' )<choose_model_class>
if use_parcor:
train_dataset=parcor_dataset().shuffle(NEXAMPLESPEREPOCH ).repeat().batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 valid_dataset=parallel_labelled_dataset(valid_nen_dataset() ).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 finetune_dataset=parallel_labelled_dataset(finetune_nen_dataset() ).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 else:
valid_dataset = valid_nen_dataset().map(_form_tuple ).batch(BATCH_SIZE ).prefetch(AUTO)
 train_dataset=non_parallel_dataset().shuffle(NEXAMPLESPEREPOCH ).map(_form_tuple ).repeat().batch(BATCH_SIZE ).prefetch(AUTO)
 finetune_dataset=finetune_nen_dataset().shuffle(NEXAMPLESPEREPOCH ).map(_form_tuple ).batch(BATCH_SIZE ).prefetch(AUTO)
 valid_dataset
loss_y = tf.sqrt(tf.reduce_mean(tf.squared_difference(tf_y, output), 0), name='tf_y_loss')
 loss_total = tf.reduce_mean(loss_y, name='tf_total_loss')
 rl = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
 if len(rl)> 0:
loss_total += tf.add_n(rl)
 optimizer = tf.train.AdamOptimizer(learning_rate=0.002)
 train_op = optimizer.minimize(loss_total )<set_options>
class WeightedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):
def __init__(self,
from_logits=False,
label_smoothing=0,
reduction=losses_utils.ReductionV2.AUTO,lossWeight=0.1,
name='binary_crossentropy'):

 super(WeightedBinaryCrossentropy, self ).__init__(
 name=name,
reduction=reduction,
from_logits=from_logits,
label_smoothing=label_smoothing)
 self.lossWeight=lossWeight
def call(self, y_true, y_pred):
l= tf.math.scalar_mul(self.lossWeight,super().call(y_true,y_pred))
return l
sess = tf.Session()
sess.run(tf.global_variables_initializer() )<train_model>
class MaskedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):
def __init__(self,
from_logits=False,
label_smoothing=0,
reduction=losses_utils.ReductionV2.AUTO,
name='binary_crossentropy'):

 super(MaskedBinaryCrossentropy, self ).__init__(
 name=name,
reduction=reduction,
from_logits=from_logits,
label_smoothing=label_smoothing)
 def call(self, y_true, y_pred):
if use_parcor:
y_true=tf.reshape(y_true,[-1,1])
 y_pred=tf.reshape(y_pred,[-1,1])
 y_pred=tf.squeeze(y_pred,axis =-1)
 y_true=tf.squeeze(y_true,axis =-1)
 mask=tf.math.greater_equal(y_true,0)
 mask=tf.cast(mask,y_true.dtype)
 masked_y_true=mask * y_true
l=super().call(masked_y_true,y_pred)
 l=mask*l
if use_parcor:
l=tf.reshape(l,[-1,2])
 l=l[:,0]+l[:,1]
return l
loss_data = []
for step in range(500):
_, lt, lt_y = sess.run([train_op, loss_total, loss_y], {tf_x: X_train, tf_y: y_train, tf_is_training: True})
 lv, lv_y = sess.run([loss_total, loss_y], {tf_x: X_validation, tf_y: y_validation, tf_is_training: False})
 loss_data.append([step, lt, lt_y[0], lt_y[1], lv, lv_y[0], lv_y[1]])
 loss_data = np.array(loss_data)
 print(loss_data[-1][1:] )<train_model>
class SquareLoss(tf.keras.losses.Loss):
def __init__(self,reduction=losses_utils.ReductionV2.AUTO, name="squareLoss",lossWeight=0):
super().__init__(reduction=reduction,name=name)
 self.lossWeight=lossWeight
def call(self, y_true, y_pred):
is_same_masked= y_pred*y_true
loss=tf.math.scalar_mul(self.lossWeight,is_same_masked*is_same_masked)
 return loss
loss, pred_y = sess.run([loss_total, output], {tf_x: X_validation, tf_y: y_validation, tf_is_training: False})
 print('loss:', loss )<prepare_output>
def build_model(transformer, max_len=512):
if use_parcor:
pair_input_word_ids = Input(shape=(2,max_len,), dtype=tf.int32, name="input_word_ids")
 pair_attention_masks = Input(shape=(2,max_len,), dtype=tf.int32, name="attention_mask")
 input_word_ids = tf.keras.backend.reshape(pair_input_word_ids,shape=(-1,max_len))
attention_masks= tf.keras.backend.reshape(pair_attention_masks,shape=(-1,max_len))
inputs=[pair_input_word_ids,pair_attention_masks]
else:
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 attention_masks = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 inputs=[input_word_ids,attention_masks]
transformer_output = transformer(( input_word_ids,attention_masks))
sequence_output = transformer_output[0]
transformer_output
if pooling_mode == pooling_mode_cls_token:
transformed_features = sequence_output[:, 0, :]
cls_token=transformed_features
else:
float_mask=tf.cast(tf.expand_dims(attention_masks,axis=-1),tf.float32)
 lengths=tf.reduce_sum(float_mask,axis=-2)
 masked_vals=tf.multiply(sequence_output,float_mask)
 sums=tf.reduce_sum(masked_vals,axis=-2)
 avgs=tf.math.divide(sums,lengths)
 maxes=tf.math.reduce_max(masked_vals,axis=-2)
 transformed_features=tf.concat([avgs,maxes],axis=-1)
 cls_token=transformed_features
for dims in fc_dims:
cls_token=Dense(dims, activation='relu' )(cls_token)
 if use_dann:
lang_token=GradReverse()(transformed_features)
 for dims in fc_dims:
lang_token=Dense(dims, activation='relu' )(lang_token)
 lang=Dense(1, activation="sigmoid",name="lang" )(lang_token)
 if use_parcor:
toxic = Dense(1, activation="sigmoid",name="unpair_toxic" )(cls_token)
 toxic=tf.keras.backend.reshape(toxic,shape=(-1,2,1))
toxic=tf.keras.layers.Layer(name="toxic" )(toxic)
 is_same=tf.keras.layers.Subtract(name="is_same" )([toxic[:,1,:],toxic[:,0,:]])
 else:
toxic = Dense(1, activation="sigmoid",name="toxic" )(cls_token)
 out=[toxic]
if use_parcor:
out.append(is_same)
 celoss=MaskedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='crossentropy_loss' 
)
 loss={"toxic":celoss}
if use_dann:
domainGap=WeightedBinaryCrossentropy(
 from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='domaingap_loss',lossWeight=dann_lambda 
)
 out.append(lang)
 loss["lang"]=domainGap
if use_parcor:
loss["is_same"]=SquareLoss(lossWeight=parcor_lambda)
 model = Model(inputs=inputs, outputs=out)
 model.compile(Adam(lr=5e-6), loss=loss, metrics={"toxic":['accuracy',tf.keras.metrics.AUC(name="auc"),tf.keras.metrics.AUC(name="pr-auc",curve="PR")]})
 return model
pd_y_pred = pd.DataFrame()
pd_y_pred[t1] = pred_y[:, 0]
pd_y_pred[t2] = pred_y[:, 1]
pd_y_pred['index'] = y_validation.index
pd_y_pred.set_index('index', inplace=True )<compute_test_metric>
def setTrainableLayers(transformer,embeddingsTrainable=False,poolerTrainable=False,hiddenTrainable=8):
transformer.layers[0].pooler.trainable=poolerTrainable
transformer.layers[0].embeddings.trainable=embeddingsTrainable
trainableHiddens=transformer.layers[0].encoder.layer[-1*hiddenTrainable:]
for layer in transformer.layers[0].encoder.layer:
if layer in trainableHiddens:
layer.trainable=True
else:
layer.trainable=False
for i, g in enumerate(spacegroups):
labels = y_validation.loc[X_validation[X_validation['spacegroup'] == g].index]
predictions = pd_y_pred.loc[X_validation[X_validation['spacegroup'] == g].index]
loss = np.sqrt(mean_squared_error(labels, predictions, multioutput='raw_values'))
print('spacegroup=%d' % g, *loss, np.mean(loss))<load_from_csv>
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 if load_model is not None:
model.load_weights(load_model)
 model.summary()
sample = pd.read_csv('.. /input/sample_submission.csv')
 sample.head()<train_model>
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
y_train = np.log1p(train[[t1, t2]])
 X_train = train.drop(['id', t1, t2], axis=1)
 sess = tf.Session()
sess.run(tf.global_variables_initializer())
 loss_data = []
for step in range(2000):
_, l, l_y = sess.run([train_op, loss_total, loss_y], {tf_x: X_train, tf_y: y_train, tf_is_training: True})
 loss_data.append([step, l, l_y[0], l_y[1]])
 print(loss_data[-1][1:])
 loss_data = np.array(loss_data )<save_to_csv>
if use_pretraining:
metric_prefix=""
if use_parcor:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)
X_test = test.drop(['id'], axis=1)
 pred_y = sess.run(output, {tf_x: X_test, tf_is_training: False})
 pred_y = np.expm1(pred_y)
 pred_y[pred_y[:, 0] < 0, 0] = 0
pred_y[pred_y[:, 1] < 0, 1] = 0
subm = pd.DataFrame()
subm['id'] = sample['id']
subm[t1] = pred_y[:, 0]
subm[t2] = pred_y[:, 1]
subm.to_csv("subm.csv", index=False)
 subm.head()<load_from_csv>
if use_pretraining:
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape)
 train.head(10 )<prepare_x_and_y>
%%time
sub = pd.read_csv(root+'/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv',nrows=nrows )
target_fe = np.log1p(train.formation_energy_ev_natom)
 target_be = np.log1p(train.bandgap_energy_ev)
 del train['formation_energy_ev_natom'], train['bandgap_energy_ev'], train['id'], test['id']<categorify>
test_files=list(filter(lambda file:"test" in file and "nen" in file, data_files))
test_files
train = pd.concat([train.drop(['spacegroup'], axis=1),
pd.get_dummies(train['spacegroup'], prefix='SG')], axis=1)
 test = pd.concat([test.drop(['spacegroup'], axis=1),
pd.get_dummies(test['spacegroup'], prefix='SG')], axis=1 )<train_on_grid>
if use_parcor:
test_dataset=parallel_labelled_dataset(tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1)).map(_form_tuple ).batch(int(BATCH_SIZE/2)).prefetch(AUTO)
 else:
test_dataset=tf.data.Dataset.from_tensor_slices(test_files ).interleave(perFileDataset,cycle_length=1 ).batch(BATCH_SIZE ).map(_form_tuple ).prefetch(AUTO)
def cv_train_model(X, y,
verbose_eval=None,
early_stopping_rounds=None,
params=None):
if type(y)is pd.core.frame.DataFrame:
y = y.values.ravel()
dstrain = lgb.Dataset(X, label=y)
 max_boost_round = 4000
if params is None:
lgb_params = {
'objective': 'regression_l2',
'learning_rate': 0.008,
'num_threads': 4,
'max_depth': 4,
'min_data_in_leaf': 23,
'feature_fraction': 0.93,
'bagging_fraction': 0.93,
'bagging_freq': 1,
'lambda_l2': 1e2,
'metric': ['mse']
}
print('lgb cv and training...')
 if verbose_eval is None:
verbose_eval = int(max_boost_round/30)
 if early_stopping_rounds is None:
early_stopping_rounds = int(max_boost_round/10)
 cv_lgb = lgb.cv(lgb_params, dstrain,
num_boost_round=max_boost_round,
nfold=10,
stratified=False,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stopping_rounds,
show_stdv=False)
 best_round = np.argmin(cv_lgb['l2-mean'])
 best_cv_mean = np.min(cv_lgb['l2-mean'])
 print('best round', best_round)
 print('best mse-mean', best_cv_mean)
 model_lgb = lgb.train(lgb_params, dstrain,
num_boost_round=best_round,
valid_sets=dstrain,
verbose_eval=verbose_eval)
 print('lgb cv and training finished...')
 return model_lgb, best_cv_mean
def get_feat_weight(model_lgb, feat_names, plot=True):
feat_weight = pd.DataFrame(model_lgb.feature_importance() ,
columns=['feature_importance'],
index=feat_names)
 if plot:
indices = np.argsort(feat_weight['feature_importance'])[::-1]
plt.figure(figsize=(12, 6))
plt.title('feature importance(lightgbm)')
 plt.bar(range(len(feat_weight)) , list(feat_weight.iloc[indices, 0]))
plt.xticks(range(len(feat_weight)) , feat_weight.iloc[indices].index,
rotation='vertical')
 plt.xlim([-1, len(feat_weight)])
 plt.show()
return feat_weight
def get_model_cv(df, y, plot=True, verbose_eval=False):
model_lgb, best_cv_mean = cv_train_model(df, y, verbose_eval=verbose_eval)
 feat_weight = get_feat_weight(model_lgb, feat_names=df.columns, plot=plot)
 print('best cv mean', best_cv_mean)
 return best_cv_mean, feat_weight, model_lgb<save_to_csv>
if use_pretraining:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
if use_finetuning:
write_file=write_root+'submission.unfinetuned.csv'
else:
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )
best_cv_mean_fe, feat_weight_fe, model_lgb_fe = get_model_cv(train, target_fe)
 pred_fe = np.expm1(model_lgb_fe.predict(test))
best_cv_mean_be, feat_weight_be, model_lgb_be = get_model_cv(train, target_be)
 pred_be = np.expm1(model_lgb_be.predict(test))
scr_total = np.mean([np.sqrt(best_cv_mean_fe), np.sqrt(best_cv_mean_be)])
 print(f'total cv score: {scr_total}')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['formation_energy_ev_natom'] = pred_fe
sub['bandgap_energy_ev'] = pred_be
sub.to_csv(f'sb_{scr_total}.csv', index=False )<compute_train_metric>
if use_finetuning:
now = datetime.now()
dt_string = now.strftime("%d_%m_%Y_%H_%M_%S_")
 out_path=dt_string+'zero_shot_pro.h5'
write_root=""
if IN_COLAB:
write_root=root+"outputs/"
out_path=write_root+out_path
print(out_path)
rg = RidgeCV(alphas=[0.003, 0.01, 0.3, 3, 10], cv=5)
 def kfold_cv(X, y, test, n_splits=10,
train_lgb=True,
lgb_ratio=0.8,
cv_pred_test=False):
kf = KFold(n_splits=n_splits, shuffle=True, random_state=233)
 cv_pred = np.zeros(( len(test)))
 scr_total = 0
for fold_id,(tr_idx, te_idx)in enumerate(kf.split(y)) :
print(f'Starting No.{fold_id} fold CV out of {n_splits}...')
 X_tr, y_tr = X.iloc[tr_idx], y.iloc[tr_idx]
X_te, y_te = X.iloc[te_idx], y.iloc[te_idx]
rg.fit(StandardScaler().fit_transform(X_tr), y_tr)
 pred_rg = rg.predict(StandardScaler().fit_transform(X_te))
mse = mean_squared_error(y_te, pred_rg)
 print('=======rg mse:', mse)
 if train_lgb==True:
_, _, model_lgb = get_model_cv(X_tr, y_tr, False, 0)
 print('=======lgb mse:',mean_squared_error(y_te, model_lgb.predict(X_te)))
 avg_mse = mean_squared_error(y_te,
pred_rg*(1-lgb_ratio)+\
model_lgb.predict(X_te)*lgb_ratio)
 print(f'=======avg mse: {avg_mse}')
 scr_total += avg_mse / n_splits
else:
scr_total += mse / n_splits
if cv_pred_test == True:
if train_lgb == True:
cv_pred +=(rg.predict(StandardScaler().fit_transform(
 test)) *(1-lgb_ratio)+ \
model_lgb.predict(test)*lgb_ratio)/n_splits
else:
cv_pred += rg.predict(StandardScaler().fit_transform(
 test)) /n_splits
print(f'score total: {scr_total}')
 if not cv_pred_test:
return scr_total
else:
return scr_total, np.expm1(cv_pred)
<set_options>
if use_finetuning:
metric_prefix=""
if use_parcor:
metric_prefix="_toxic"
mc = ModelCheckpoint(out_path, monitor="val"+metric_prefix+"_auc", mode='max',verbose=1,save_weights_only=True,save_best_only=True)
 es = EarlyStopping(monitor="val"+metric_prefix+"_auc", mode='max',patience=2,verbose=1,restore_best_weights=True)
 lrScheduler=ReduceLROnPlateau(
 monitor="val"+metric_prefix+"_loss",
factor=0.2,
patience=1,
verbose=1,
mode="min",
min_delta=0,
cooldown=0,
min_lr=1e-10,
)
warnings.filterwarnings('ignore')
 tf.random.set_seed(42)
<load_from_csv>
if use_finetuning:
train_history = model.fit(
 finetune_dataset,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose=1,
callbacks=[mc,es,lrScheduler]
)
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 features = [c for c in train.columns if 'feature' in c] 
<count_missing_values>
if use_finetuning:
l=model.predict(test_dataset, verbose=1)
 if use_dann:
l=l[0]
if use_parcor:
l=np.reshape(l,(-1,1))
sub['toxic'] = l[:,-1]
write_file=write_root+'submission.csv'
sub.to_csv(write_file, index=False )
<feature_engineering><EOS> 
def reset_keras() :
tf.keras.backend.clear_session()
print(gc.collect() )