Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
class StackNet(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models,meta_final_model, meta_models1=None, meta_models2=None,add_prev_out=True, n_folds=10):
self.base_models = base_models
self.meta_models1 = meta_models1
self.meta_models2 = meta_models2
self.meta_final_model=meta_final_model
self.n_folds = n_folds
self.add_prev_out=add_prev_out
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
if self.meta_models1!=None:
self.meta_models1_ = [list() for x in self.meta_models1]
if self.meta_models2!=None:
self.meta_models2_ = [list() for x in self.meta_models2]
self.meta_final_model_=clone(self.meta_final_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions=np.hstack(( out_of_fold_predictions,X))
if self.meta_models1!=None:
out_of_fold_predictions2 = np.zeros(( X.shape[0],len(self.meta_models1)))
 if self.meta_models1!=None:
for i, model in enumerate(self.meta_models1):
for train_index, holdout_index in kfold.split(out_of_fold_predictions, y):
instance = clone(model)
 self.meta_models1_[i].append(instance)
 instance.fit(out_of_fold_predictions[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions[holdout_index])
 out_of_fold_predictions2[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions2=np.hstack(( out_of_fold_predictions2,X))
else:
out_of_fold_predictions2=out_of_fold_predictions
if self.meta_models2!=None:
out_of_fold_predictions3 = np.zeros(( X.shape[0],len(self.meta_models2)))
 if self.meta_models2!=None:
for i, model in enumerate(self.meta_models2):
for train_index, holdout_index in kfold.split(out_of_fold_predictions2, y):
instance = clone(model)
 self.meta_models2_[i].append(instance)
 instance.fit(out_of_fold_predictions2[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions2[holdout_index])
 out_of_fold_predictions3[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions3=np.hstack(( out_of_fold_predictions3,X))
else:
out_of_fold_predictions3=out_of_fold_predictions2
self.meta_final_model_.fit(out_of_fold_predictions3, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_])
 if self.add_prev_out:
meta_features=np.hstack(( meta_features,X))
if self.meta_models1!=None:
meta_features2 = np.column_stack([
np.column_stack([model.predict(meta_features)for model in meta_models1_] ).mean(axis=1)
 for meta_models1_ in self.meta_models1_ ])
 if self.add_prev_out:
meta_features2=np.hstack(( meta_features2,X))
else:
meta_features2=meta_features
if self.meta_models2!=None:
meta_features3 = np.column_stack([
np.column_stack([model.predict(meta_features2)for model in meta_models2_] ).mean(axis=1)
 for meta_models2_ in self.meta_models2_ ])
 if self.add_prev_out:
meta_features3=np.hstack(( meta_features3,X))
else:
meta_features3=meta_features2
return self.meta_final_model_.predict(meta_features3 )<choose_model_class>
from __future__ import absolute_import
lr=LogisticRegression(random_state=1)
 xgbm=xgb.XGBClassifier(objective='binary:hinge',random_state=1)
 lgbmm=lgb.LGBMClassifier(objective='huber',random_state=1)
 gbc=GradientBoostingClassifier(random_state=1)
 adc=AdaBoostClassifier(random_state=1)
 rf=RandomForestClassifier(random_state=1,n_jobs=-1,n_estimators=100)
 bc=BaggingClassifier(xgb.XGBClassifier(objective='binary:hinge',random_state=1),n_estimators =3,random_state=1)
 bc.fit(X_train,y_train)
 print(bc.__class__.__name__,accuracy_score(bc.predict(X_test),y_test))
sn=StackNet(( lr,gbc,adc,rf,bc,lgbmm,xgbm),xgbm)
 sn.fit(X_train,y_train)
 print(sn.__class__.__name__,accuracy_score(sn.predict(X_test),y_test))<save_to_csv>
package_dir = ".. /input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
sys.path.append(package_dir )
sn.fit(train.values,y.values)
 submission = pd.read_csv('.. /input/gender_submission.csv')
 submission['Survived'] = sn.predict(test.values)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
from __future__ import absolute_import
data=pd.read_csv('.. /input/train.csv')
 anwsering_this=pd.read_csv('.. /input/test.csv')
 submission_id=anwsering_this.Id.values<remove_duplicates>
from __future__ import division
if len(set(data.Id)) ==len(data.Id):
print("No duplicates for Id column")
 data.drop("Id", axis = 1, inplace = True)
<sort_values>
from __future__ import print_function
sep=data.shape[0]
data_fts=data.drop(['SalePrice'],axis=1)
 test_fts=anwsering_this.drop(['Id'],axis=1)
 all_instances=pd.concat([data_fts,test_fts] ).reset_index(drop=True)
 null_counts=all_instances.isnull().sum().sort_values(ascending=False)
 print(null_counts.head(38))<feature_engineering>
import numpy as np
import pandas as pd
from tqdm import tqdm
import os
import warnings
all_instances.loc[:,'PoolQC']=all_instances.loc[:,'PoolQC'].fillna('No')
 all_instances.loc[:,'MiscFeature']=all_instances.loc[:,'MiscFeature'].fillna('No')
 all_instances.loc[:,'Alley']=all_instances.loc[:,'Alley'].fillna('No')
 all_instances.loc[:,'Fence']=all_instances.loc[:,'Fence'].fillna('No')
 all_instances.loc[:,'FireplaceQu']=all_instances.loc[:,'FireplaceQu'].fillna('No')
 all_instances.loc[:,'LotFrontage']=all_instances.loc[:,'LotFrontage'].fillna(0)
 all_instances.loc[:,'GarageCond']=all_instances.loc[:,'GarageCond'].fillna('No')
 all_instances.loc[:,'GarageQual']=all_instances.loc[:,'GarageQual'].fillna('No')
 all_instances.loc[:,'GarageYrBlt']=all_instances.loc[:,'GarageYrBlt'].fillna(0)
 all_instances.loc[:,'GarageFinish']=all_instances.loc[:,'GarageFinish'].fillna('No')
 all_instances.loc[:,'GarageType']=all_instances.loc[:,'GarageType'].fillna('No')
 all_instances.loc[:,'BsmtCond']=all_instances.loc[:,'BsmtCond'].fillna('No')
 all_instances.loc[:,'BsmtExposure']=all_instances.loc[:,'BsmtExposure'].fillna('No_B')
 all_instances.loc[:,'BsmtQual']=all_instances.loc[:,'BsmtQual'].fillna('No')
 all_instances.loc[:,'BsmtFinType2']=all_instances.loc[:,'BsmtFinType2'].fillna('No')
 all_instances.loc[:,'BsmtFinType1']=all_instances.loc[:,'BsmtFinType1'].fillna('No')
 all_instances.loc[:,'MasVnrType']=all_instances.loc[:,'MasVnrType'].fillna('No')
 all_instances.loc[:,'MasVnrArea']=all_instances.loc[:,'MasVnrArea'].fillna(0)
 all_instances.loc[:,'MSZoning']=all_instances.loc[:,'MSZoning'].fillna("RL")
 all_instances.loc[:,'BsmtHalfBath']=all_instances.loc[:,'BsmtHalfBath'].fillna(0)
 all_instances.loc[:,'Utilities']=all_instances.loc[:,'Utilities'].fillna("AllPub")
 all_instances.loc[:,'Functional']=all_instances.loc[:,'Functional'].fillna("Typ")
 all_instances.loc[:,'BsmtFullBath']=all_instances.loc[:,'BsmtFullBath'].fillna(0)
 all_instances.loc[:,'BsmtFinSF2']=all_instances.loc[:,'BsmtFinSF2'].fillna(0)
 all_instances.loc[:,'BsmtFinSF1']=all_instances.loc[:,'BsmtFinSF1'].fillna(0)
 all_instances.loc[:,'Exterior2nd']=all_instances.loc[:,'Exterior2nd'].fillna("VinylSd")
 all_instances.loc[:,'BsmtUnfSF']=all_instances.loc[:,'BsmtUnfSF'].fillna(0)
 all_instances.loc[:,'TotalBsmtSF']=all_instances.loc[:,'TotalBsmtSF'].fillna(0)
 all_instances.loc[:,'Exterior1st']=all_instances.loc[:,'Exterior1st'].fillna("VinylSd")
 all_instances.loc[:,'SaleType']=all_instances.loc[:,'SaleType'].fillna("WD")
 all_instances.loc[:,'Electrical']=all_instances.loc[:,'Electrical'].fillna("SBrkr")
 all_instances.loc[:,'KitchenQual']=all_instances.loc[:,'KitchenQual'].fillna("TA")
 all_instances.loc[:,'GarageArea']=all_instances.loc[:,'GarageArea'].fillna(0)
 all_instances.loc[:,'GarageCars']=all_instances.loc[:,'GarageCars'].fillna(0)
 print("Remain "+str(all_instances.isnull().sum().sum())+" missing values.")
<data_type_conversions>
from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam
from pytorch_pretrained_bert import BertConfig
import gc
n2c=['MSSubClass','MoSold']
for i in n2c:
all_instances[i]=all_instances[i].astype(str)
 c2n_=['GarageCond','GarageQual','GarageFinish','FireplaceQu','KitchenQual','HeatingQC','BsmtFinType2','BsmtFinType1','BsmtExposure','BsmtCond','BsmtQual','ExterCond','ExterQual','Fence']
lv4_NA=['PoolQC']
lv5_full=['ExterQual','ExterCond','HeatingQC','KitchenQual']
lv5_NA=['BsmtQual','BsmtCond','FireplaceQu','GarageQual','GarageCond']
Bsmt_Type=['BsmtFinType2','BsmtFinType1']
all_instances['PoolQC']=all_instances['PoolQC'].replace({'No':0,'Fa':1,'TA':2,'Gd':3,'Ex':4})
 for i in lv5_full:
all_instances[i]=all_instances[i].replace({'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
 for i in lv5_NA:
all_instances[i]=all_instances[i].replace({'No':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
 for i in Bsmt_Type:
all_instances[i]=all_instances[i].replace({'No':0,'Unf':1,'LwQ':2,'Rec':3,'BLQ':4,'ALQ':5,'GLQ':6})
 all_instances['GarageFinish']=all_instances['GarageFinish'].replace({'No':0,'Unf':1,'RFn':2,'Fin':3})
 all_instances['BsmtExposure']=all_instances['BsmtExposure'].replace({'No_B':0,'No':1,'Mn':2,'Av':3,'Gd':4})
 all_instances['Fence']=all_instances['Fence'].replace({'No':0,'MnWw':1,'GdWo':2,'MnPrv':3,'GdPrv':4})
<feature_engineering>
import torch.utils.data
all_instances["OverallRate"] = all_instances["OverallQual"] * all_instances["OverallCond"]
all_instances["GarageRate"] = all_instances["GarageQual"] * all_instances["GarageCond"]
all_instances["ExterRate"] = all_instances["ExterQual"] * all_instances["ExterCond"]
all_instances["BsmtRate"]=all_instances["BsmtQual"] * all_instances["BsmtCond"]
all_instances["PoolRate"]=all_instances["PoolQC"] * all_instances["PoolArea"]
top_10_corr=list(corr['SalePrice'].sort_values(ascending=False ).index)[1:11]
for i in top_10_corr:
all_instances[i+'_s2']=all_instances[i] ** 2
all_instances[i+'_s3']=all_instances[i] ** 3
all_instances[i+'_sqrt']=np.sqrt(all_instances[i])
 print(all_instances.shape )<features_selection>
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )
categorical_features = all_instances.select_dtypes(include = ["object"] ).columns
numerical_features = all_instances.select_dtypes(exclude = ["object"] ).columns
print("Numerical features : " + str(len(numerical_features)))
 print("Categorical features : " + str(len(categorical_features)))
 all_instances_num = all_instances[numerical_features]
all_instances_cat = all_instances[categorical_features]
skewness = all_instances_num.apply(lambda x: skew(x))
skewness = skewness[abs(skewness)> 0.5]
print(str(skewness.shape[0])+ " skewed numerical features to log transform")
 skewed_features = skewness.index
all_instances_num[skewed_features] = np.log1p(all_instances_num[skewed_features])
 all_instances = pd.concat([all_instances_num, all_instances_cat], axis = 1)
 print(all_instances.shape)
<categorify>
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens)
 MAX_SEQUENCE_LENGTH = 280
SEED = 1234
BATCH_SIZE = 32
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
LARGE_BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-24_h-1024_a-16/uncased_L-24_H-1024_A-16/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
all_instances=pd.get_dummies(all_instances,drop_first=True)
 print(all_instances.shape )<prepare_x_and_y>
train_df =pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv' )
new_data=all_instances[:sep]
y=data['SalePrice']
new_test=all_instances[sep:] 
<normalization>
test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )
X_train,X_test,y_train,y_test=train_test_split(new_data,y,test_size=0.3,random_state=111)
 stdSc = StandardScaler()
X_train.loc[:, numerical_features] = stdSc.fit_transform(X_train.loc[:, numerical_features])
 X_test.loc[:, numerical_features] = stdSc.transform(X_test.loc[:, numerical_features])
<compute_train_metric>
BERT_SMALL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
BERT_LARGE_PATH = '.. /input/bert-pretrained-models/uncased_l-24_h-1024_a-16/uncased_L-24_H-1024_A-16/'
JIGSAW_BERT_SMALL_MODEL_PATH = '.. /input/bert-fast-all-v1/bert_pytorch.bin'
JIGSAW_BERT_LARGE_MODEL_PATH = '.. /input/pretrained-b-j/jigsaw-bert-large-uncased-len-220-fp16/epoch-1/pytorch_model.bin'
JIGSAW_BERT_SMALL_JSON_PATH = '.. /input/google-ais-bert-model-for-nlp/bert_config.json'
JIGSAW_BERT_LARGE_JSON_PATH = '.. /input/pretrained-b-j/jigsaw-bert-large-uncased-len-220-fp16/epoch-1/config.json'
NUM_BERT_MODELS = 2
INFER_BATCH_SIZE = 64
test_preds = np.zeros(( test_df.shape[0],NUM_BERT_MODELS))
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
print("Predicting BERT large model...... ")
 bert_config = BertConfig(JIGSAW_BERT_LARGE_JSON_PATH)
 tokenizer = BertTokenizer.from_pretrained(BERT_LARGE_PATH, cache_dir=None,do_lower_case=True)
 X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), MAX_SEQUENCE_LENGTH, tokenizer)
 test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
gc.collect()
model = BertForSequenceClassification(bert_config, num_labels=1)
 model.load_state_dict(torch.load(JIGSAW_BERT_LARGE_MODEL_PATH))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
model_preds = np.zeros(( len(X_test)))
 test_loader = torch.utils.data.DataLoader(test, batch_size=INFER_BATCH_SIZE, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 model_preds[i * INFER_BATCH_SIZE:(i + 1)* INFER_BATCH_SIZE] = pred[:, 0].detach().cpu().squeeze().numpy()
test_preds[:,0] = torch.sigmoid(torch.tensor(model_preds)).numpy().ravel()
del model
gc.collect()
print("Predicting BERT small model...... ")
 bert_config = BertConfig(JIGSAW_BERT_SMALL_JSON_PATH)
 tokenizer = BertTokenizer.from_pretrained(BERT_SMALL_PATH, cache_dir=None,do_lower_case=True)
 X_test = convert_lines(test_df["comment_text"].fillna("DUMMY_VALUE"), MAX_SEQUENCE_LENGTH, tokenizer)
 test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))
model = BertForSequenceClassification(bert_config, num_labels=1)
 model.load_state_dict(torch.load(JIGSAW_BERT_SMALL_MODEL_PATH))
model.to(device)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
model_preds = np.zeros(( len(X_test)))
 test_loader = torch.utils.data.DataLoader(test, batch_size=INFER_BATCH_SIZE, shuffle=False)
 tk0 = tqdm(test_loader)
 for i,(x_batch,)in enumerate(tk0):
pred = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 model_preds[i * INFER_BATCH_SIZE:(i + 1)* INFER_BATCH_SIZE] = pred[:, 0].detach().cpu().squeeze().numpy()
test_preds[:,1] = torch.sigmoid(torch.tensor(model_preds)).numpy().ravel()
del model
gc.collect()
bert_submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_preds.mean(axis=1)})
 bert_submission.to_csv('bert_submission.csv', index=False)
scorer = make_scorer(mean_squared_error, greater_is_better = False)
 def rmse_cv_train(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring = scorer, cv = 10))
return(rmse)
 def rmse_cv_test(model):
rmse= np.sqrt(-cross_val_score(model, X_test, y_test, scoring = scorer, cv = 10))
return(rmse)
<choose_model_class>
EMBEDDING_PATHS = ['.. /input/pickled-word-embedding/crawl-300d-2M.pkl',
'.. /input/pickled-word-embedding/glove.840B.300d.pkl']
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4*LSTM_UNITS
EPOCHS = 3
MAX_LEN = 220
IDENTITY_COLUMNS = [
'transgender', 'female', 'homosexual_gay_or_lesbian', 'muslim', 'hindu',
'white', 'black', 'psychiatric_or_mental_illness', 'jewish'
]
AUX_COLUMNS = ['target', 'severe_toxicity','obscene','identity_attack','insult','threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
lr = LinearRegression()
print("RMSE on Training set :", rmse_cv_train(lr ).mean())
 print("RMSE on Test set :", rmse_cv_test(lr ).mean())
<train_on_grid>
SYMBOLS_TO_ISOLATE = '.,?!-;*"…:—() %
SYMBOLS_TO_REMOVE = '
🍕\r🐵\xa0\ue014\t\uf818\uf04a\xad😢🐶️\uf0e0😜😎👊\u200b\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\u202a\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\x81エンジ故障\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\ufeff\u2028😉😤⛺🙂\u3000تحكسة👮💙فزط😏🍾🎉😞\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\u200a🌠🐟💫💰💎эпрд\x95🖐🙅⛲🍰🤐👆🙌\u2002💛🙁👀🙊🙉\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\x13🚬🤓\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\uf0b7\uf04c\x9f\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\u202d💤🍇\ue613小土豆🏡❔⁉\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\x9c\x9d🗑\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\u2000үսᴦᎥһͺ\u2007հ\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\uf203\uf09a\uf222\ue608\uf202\uf099\uf469\ue607\uf410\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\uf10aლڡ🐦\U0001f92f\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'
ISOLATE_DICT = {ord(c):f' {c} ' for c in SYMBOLS_TO_ISOLATE}
REMOVE_DICT = {ord(c):f'' for c in SYMBOLS_TO_REMOVE}
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
CONTRACTION_MAPPING = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def handle_punctuation(text):
text = text.translate(REMOVE_DICT)
 text = text.translate(ISOLATE_DICT)
 return text
def clean_contractions(text, mapping=CONTRACTION_MAPPING):

 specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def preprocess(x):
x = handle_punctuation(x)
 return x
def get_coefs(word, *arr):

 return word, np.asarray(arr,dtype='float32')
 def load_embeddings(path):
if path.split('.')[-1] in ['txt','vec']:
with open(path,'rb')as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 if path.split('.')[-1] =='pkl':
with open(path,'rb')as f:
return pickle.load(f)
 def build_matrix(word_index, path):

 embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix
ridge = RidgeCV(alphas = [0.01, 0.04, 0.08, 0.1, 0.4, 0.8, 1, 4, 8, 10, 40, 80])
 ridge.fit(X_train, y_train)
 alpha= ridge.alpha_
print("Best alpha :", alpha)
 print("Try again for more precision with alphas centered around " + str(alpha))
ridge = RidgeCV(alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8, alpha *.85,
alpha *.9, alpha *.95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
alpha * 1.25, alpha * 1.3, alpha * 1.35, alpha * 1.4],
cv = 10)
 ridge.fit(X_train, y_train)
 alpha_ridge = ridge.alpha_
print("Best alpha :", alpha_ridge)
 print("Ridge RMSE on Training set :", rmse_cv_train(ridge ).mean())
 print("Ridge RMSE on Test set :", rmse_cv_test(ridge ).mean())
 ridge_=Ridge(alpha=alpha_ridge)
 ridge_.fit(X_train,y_train)
 ridge_predict=ridge_.predict(X_test)
 ridge_RMSE=np.sqrt(mean_squared_error(ridge_predict,y_test))
print("Ridge regression RMSE :",ridge_RMSE)
<train_on_grid>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def build_model(embedding_matrix, num_aux_targets):

 words = Input(shape=(MAX_LEN,))
x = Embedding(*embedding_matrix.shape, weights = [embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x)
 ])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result =Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs =words, outputs =[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model
lasso = LassoCV(alphas = [0.0001, 0.0004, 0.0008, 0.001, 0.004, 0.008, 0.01, 0.04, 0.08, 0.1,
0.4, 0.8, 1],
max_iter = 50000, cv = 10)
 lasso.fit(X_train, y_train)
 alpha = lasso.alpha_
print("Best alpha :", alpha)
 print("Try again for more precision with alphas centered around " + str(alpha))
lasso = LassoCV(alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8,
alpha *.85, alpha *.9, alpha *.95, alpha, alpha * 1.05,
alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3, alpha * 1.35,
alpha * 1.4],
max_iter = 50000, cv = 10)
 lasso.fit(X_train, y_train)
 alpha_lasso = lasso.alpha_
print("Best alpha :", alpha_lasso)
 print("Lasso RMSE on Training set :", rmse_cv_train(lasso ).mean())
 print("Lasso RMSE on Test set :", rmse_cv_test(lasso ).mean())
 lasso_=Lasso(alpha=alpha_lasso)
 lasso_.fit(X_train,y_train)
 lasso_predict=lasso_.predict(X_test)
 lasso_RMSE=np.sqrt(mean_squared_error(lasso_predict,y_test))
print("Lasso regression RMSE :",lasso_RMSE )<train_on_grid>
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str)
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >=0.5, True, False )
elasticNet = ElasticNetCV(l1_ratio = [0.1, 0.3, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95, 1],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006,
0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
 elasticNet.fit(X_train, y_train)
 alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("Try again for more precision with l1_ratio centered around " + str(ratio))
elasticNet = ElasticNetCV(l1_ratio = [ratio *.85, ratio *.9, ratio *.95, ratio, ratio * 1.05, ratio * 1.1, ratio * 1.15],
alphas = [0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6],
max_iter = 50000, cv = 10)
 elasticNet.fit(X_train, y_train)
 if(elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha = elasticNet.alpha_
ratio = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio)
 print("Best alpha :", alpha)
 print("Now try again for more precision on alpha, with l1_ratio fixed at " + str(ratio)+
" and alpha centered around " + str(alpha))
elasticNet = ElasticNetCV(l1_ratio = ratio,
alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8, alpha *.85, alpha *.9,
alpha *.95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3,
alpha * 1.35, alpha * 1.4],
max_iter = 50000, cv = 10)
 elasticNet.fit(X_train, y_train)
 if(elasticNet.l1_ratio_ > 1):
elasticNet.l1_ratio_ = 1
alpha_eln = elasticNet.alpha_
ratio_eln = elasticNet.l1_ratio_
print("Best l1_ratio :", ratio_eln)
 print("Best alpha :", alpha_eln)
 print("ElasticNet RMSE on Training set :", rmse_cv_train(elasticNet ).mean())
 print("ElasticNet RMSE on Test set :", rmse_cv_test(elasticNet ).mean())
 elasticNet_=ElasticNet(l1_ratio=ratio_eln,alpha=alpha_eln)
 elasticNet_.fit(X_train,y_train)
 elasticNet_predict=elasticNet_.predict(X_test)
 elasticNet_RMSE=np.sqrt(mean_squared_error(elasticNet_predict,y_test))
print("elasticNet RMSE :",elasticNet_RMSE )<save_to_csv>
tokenizer = text.Tokenizer(filters = CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train,maxlen=MAX_LEN)
 x_test= sequence.pad_sequences(x_test, maxlen=MAX_LEN )
Final_model=Ridge(alpha=alpha_ridge)
 stdSc_ = StandardScaler()
new_data.loc[:, numerical_features] = stdSc_.fit_transform(new_data.loc[:, numerical_features])
 new_test.loc[:, numerical_features] = stdSc_.transform(new_test.loc[:, numerical_features])
 Final_model.fit(new_data,y)
 submission=Final_model.predict(new_test)
 submission=np.expm1(submission)
 df_submission=pd.DataFrame({'Id':submission_id,'SalePrice':submission})
 df_submission.to_csv('submission.csv',index=False )<load_from_csv>
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN]*(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])*train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights/=sample_weights.mean()
titanic = pd.read_csv(".. /input/train.csv")
 titanic_t = pd.read_csv(".. /input/test.csv" )<count_values>
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index,f)for f in EMBEDDING_PATHS], axis =-1)
 print("Embedding matrix shape:", embedding_matrix.shape)
 del train_df, tokenizer
gc.collect()
titanic['Sex'].value_counts()<drop_column>
checkpoint_predictions = []
weights = []
NUM_MODELS = 1
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=1,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks = [
LearningRateScheduler(lambda _: 1e-3*(0.55**global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 del model
gc.collect()
titanic = titanic.drop(['Cabin'], axis=1)
 titanic_t = titanic_t.drop(['Cabin'], axis=1 )<categorify>
predictions = np.average(checkpoint_predictions, weights=weights, axis=0)
 lstm_submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions
})
 lstm_submission.to_csv('submission.csv', index=False )
<data_type_conversions><EOS> 
submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': lstm_submission['prediction'].rank(pct=True)*0.3 + bert_submission['prediction'].rank(pct=True)*0.7})
 submission.to_csv('submission.csv', index=False)