Kaggle competition: Contradictory, My Dear Watson
( train.corr() **2)["SalePrice"].sort_values(ascending = False)[1:]<create_dataframe>
from transformers import BertTokenizer, TFBertModel
from transformers import RobertaTokenizer, TFRobertaModel
from transformers import XLMRobertaTokenizer, TFXLMRobertaModel
from transformers import AutoTokenizer
import matplotlib.pyplot as plt
import tensorflow as tf
from dask import bag, diagnostics
from sklearn.utils import shuffle
train = train[train.GrLivArea < 4500]
train.reset_index(drop = True, inplace = True)
 previous_train = train.copy()<prepare_x_and_y>
!pip install --quiet googletrans
train.drop(columns=['Id'],axis=1, inplace=True)
 test.drop(columns=['Id'],axis=1, inplace=True)
 y = train['SalePrice'].reset_index(drop=True)
 previous_train = train.copy()<drop_column>
submission = pd.read_csv("/kaggle/input/output/submission(2 ).csv")
 submission.head()
all_data = pd.concat(( train, test)).reset_index(drop = True)
 all_data.drop(['SalePrice'], axis = 1, inplace = True )<count_missing_values>
submission.to_csv("submission1.csv", index = False)
missing_percentage(all_data )<define_variables>
train = pd.read_csv("/kaggle/input/contradictory-my-dear-watson/train.csv")
 train.head()
missing_val_col = ["Alley",
"PoolQC",
"MiscFeature",
"Fence",
"FireplaceQu",
"GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
for i in missing_val_col:
all_data[i] = all_data[i].fillna('None' )<define_variables>
tokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large' )
missing_val_col2 = ['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for i in missing_val_col2:
all_data[i] = all_data[i].fillna(0)
 all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))<data_type_conversions>
def encode_sentence(s):
tokens = list(tokenizer.tokenize(s))
tokens.append('[SEP]')
 return tokenizer.convert_tokens_to_ids(tokens)
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['MSZoning'] = all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<data_type_conversions>
s = "I love machine learning"
encode_sentence(s )
all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['Utilities'] = all_data['Utilities'].fillna('AllPub')
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna("TA")
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['Electrical'] = all_data['Electrical'].fillna("SBrkr")
<count_missing_values>
def data_translate(source_data,dest_language):
translator = Translator()
if dest_language == 'zh':
dest_language = 'zh-cn'
dest_data = translator.translate(source_data, dest = dest_language ).text
return dest_data
missing_percentage(all_data )<sort_values>
def translation_augment(source_data, languages, fraction):
new_df = pd.DataFrame()
for lang in languages:
print(lang)
 sampled_rows = source_data.sample(frac=fraction, replace = False)
 prem_bag = bag.from_sequence(sampled_rows['premise'].tolist() ).map(data_translate, lang)
 hypothesis_bag = bag.from_sequence(sampled_rows['hypothesis'].tolist() ).map(data_translate, lang)
 with diagnostics.ProgressBar() :
prems = prem_bag.compute()
hyps = hypothesis_bag.compute()
aug_df = pd.DataFrame({'id': pd.Series([None]*len(sampled_rows)) ,
'premise': pd.Series(prems),
'hypothesis': pd.Series(hyps),
'lang_abv': pd.Series([lang]*len(sampled_rows)) ,
'language': pd.Series([None]*len(sampled_rows)) ,
'label': pd.Series(sampled_rows['label'].values)
 })
 new_df = new_df.append(aug_df)
 new_df = shuffle(new_df)
 return new_df
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 skewed_feats<sort_values>
def data_augment(train_df, fraction):
english_df = train.loc[train.lang_abv == 'en']
languages = list(set(train.lang_abv.values))
languages.remove('en')
 print(languages)
 translated_df = translation_augment(english_df,languages, fraction)
 train_df = train_df.append(translated_df)
 train_df = shuffle(train_df)
 return train_df
def fixing_skewness(df):

 numeric_feats = df.dtypes[df.dtypes != "object"].index
skewed_feats = df[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skewed_feats[abs(skewed_feats)> 0.5]
skewed_features = high_skew.index
for feat in skewed_features:
df[feat] = boxcox1p(df[feat], boxcox_normmax(df[feat] + 1))
fixing_skewness(all_data )<feature_engineering>
train = pd.read_csv('/kaggle/input/augment-data20/augmented_data_20percent.csv')
 train.head()
len(train )
all_data['TotalSF'] =(all_data['TotalBsmtSF']
+ all_data['1stFlrSF']
+ all_data['2ndFlrSF'])
 all_data['YrBltAndRemod'] = all_data['YearBuilt'] + all_data['YearRemodAdd']
all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1']
+ all_data['BsmtFinSF2']
+ all_data['1stFlrSF']
+ all_data['2ndFlrSF'] 
)
 all_data['Total_Bathrooms'] =(all_data['FullBath']
+(0.5 * all_data['HalfBath'])
 + all_data['BsmtFullBath']
+(0.5 * all_data['BsmtHalfBath'])
)
 all_data['Total_porch_sf'] =(all_data['OpenPorchSF']
+ all_data['3SsnPorch']
+ all_data['EnclosedPorch']
+ all_data['ScreenPorch']
+ all_data['WoodDeckSF'] 
)
<feature_engineering>
def bert_encode(premises, hypotheses, tokenizer):
num_examples = len(premises)
 sen1 = tf.ragged.constant([encode_sentence(s)for s in np.array(premises)])
 sen2 = tf.ragged.constant([encode_sentence(s)for s in np.array(hypotheses)])
 cls = [tokenizer.convert_tokens_to_ids(['CLS'])]*sen1.shape[0]
input_word_ids = tf.concat([cls, sen1, sen2], axis = -1)
 input_mask = tf.ones_like(input_word_ids ).to_tensor()
type_cls = tf.zeros_like(cls)
 type_sen1 = tf.zeros_like(sen1)
 type_sen2 = tf.ones_like(sen2)
 input_type_ids = tf.concat([type_cls, type_sen1, type_sen2], axis = -1 ).to_tensor()
inputs = {
'input_word_ids' : input_word_ids.to_tensor() ,
'input_mask': input_mask,
'input_type_ids': input_type_ids
}
return inputs
all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<drop_column>
train_input = bert_encode(train.premise.values, train.hypothesis.values, tokenizer )
all_data = all_data.drop(['Utilities', 'Street', 'PoolQC',], axis=1 )<categorify>
max_len = 80
def build_model() :
bert_encoder = TFRobertaModel.from_pretrained('jplu/tf-xlm-roberta-large')
 input_word_ids = tf.keras.Input(shape =(max_len,), dtype =tf.int32, name = "input_word_ids")
 input_mask = tf.keras.Input(shape =(max_len,), dtype= tf.int32, name = "input_mask")
 input_type_ids = tf.keras.Input(shape=(max_len,), dtype= tf.int32, name="input_type_ids")
 embedding = bert_encoder([input_word_ids, input_mask, input_type_ids])[0]
output = tf.keras.layers.Dense(3, activation = 'softmax' )(embedding[:,0,:])
 model = tf.keras.Model(inputs= [input_word_ids, input_mask, input_type_ids], outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss='sparse_categorical_crossentropy', metrics = 'accuracy')
 return model
final_features = pd.get_dummies(all_data ).reset_index(drop=True)
 final_features.shape<prepare_x_and_y>
model.fit(train_input, train.label.values, epochs=3, verbose=1, batch_size=16, validation_split=0.2 )
X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(y):, :]<drop_column>
model.save_weights('RoBertamodel_augmented_data_20_percent_adam_sparse_categorical_entropy.h5' )
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )<count_values>
test = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/test.csv')
 test.head()
counts = X.BsmtUnfSF.value_counts()<count_values>
test_input = bert_encode(test.premise.values, test.hypothesis.values, tokenizer )
counts.iloc[0]<count_values>
predictions = [np.argmax(i)for i in model.predict(test_input)]
for i in X.columns:
counts = X[i].value_counts()
print(counts )<train_model>
submission = test.id.copy().to_frame()
submission.head()
def overfit_reducer(df):

 overfit = []
for i in df.columns:
counts = df[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 return overfit
overfitted_features = overfit_reducer(X)
 X = X.drop(overfitted_features, axis=1)
 X_sub = X_sub.drop(overfitted_features, axis=1 )<split>
submission['prediction'] = predictions
<feature_engineering><EOS> 
submission.to_csv("submission.csv", index = False)