Kaggle competition: House Prices - Advanced Regression Techniques
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x<choose_model_class>
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<load_from_csv>
df_train = df_train[df_train["GrLivArea"] < 4500]
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'] )<define_variables>
df_train['SalePrice'] = np.log1p(df_train['SalePrice'] )
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']<data_type_conversions>
n = df_train.shape[0]
for col in df_train.columns:
missing_pct = sum(df_train[col].isnull())*100.0/n
if missing_pct > 95.0:
print('{}: {:0.2f}%'.format(col, missing_pct))
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T
max_features = None
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))<string_transform>
df_train.drop(['PoolQC', 'MiscFeature'], axis=1, inplace=True)
 df_test.drop(['PoolQC', 'MiscFeature'], axis=1, inplace=True )
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<concatenate>
df_train[df_train['Electrical'].isna() ]
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()<prepare_x_and_y>
df_train = df_train[~df_train['Electrical'].isna() ]
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32)
 x_test_torch = torch.tensor(x_test, dtype=torch.long)
 batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )<choose_model_class>
cols = ["MSSubClass", "YrSold", 'MoSold']
df_train[cols] = df_train[cols].astype(str)
 df_test[cols] = df_test[cols].astype(str )
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
cname = 'GrLivArea'
df_train.corr() [cname][abs(df_train.corr() [cname])> 0.65]
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]})
 submission.to_csv('submission.csv', index=False )<import_modules>
cname = 'TotalBsmtSF'
df_train.corr() [cname][abs(df_train.corr() [cname])> 0.65]
import numpy as np
import pandas as pd
from keras.models import Model
from keras.layers import Input, Dense, LSTM, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, CuDNNGRU, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.optimizers import Adam, RMSprop, SGD
from keras.preprocessing import text, sequence
from keras.callbacks import LearningRateScheduler
import sys, os, re, csv, codecs
from keras.preprocessing.text import Tokenizer<define_variables>
col = ['GarageArea','1stFlrSF', '2ndFlrSF','TotRmsAbvGrd', 'GarageYrBlt']
df_train.drop(col, axis=1, inplace=True)
 df_test.drop(col, axis=1, inplace=True )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt']<define_variables>
def fill_missing_data(df):
df_data = df.copy()
categoricals = []
for cname,dtype in df_data.dtypes.items() :
if dtype == 'object':
categoricals.append(cname)
 df_data[categoricals] = df_data[categoricals].fillna('None')
 for cname in df_data.columns:
if cname not in categoricals:
df_data[cname] = df_data[cname].fillna(0)
 return df_data
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']<define_variables>
df_train = fill_missing_data(df_train)
 df_test = fill_missing_data(df_test )
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<compute_test_metric>
df_train['TotalPorchSF'] = df_train['OpenPorchSF'] + df_train['EnclosedPorch'] + df_train['3SsnPorch'] + df_train['ScreenPorch']
df_test['TotalPorchSF'] = df_test['OpenPorchSF'] + df_test['EnclosedPorch'] + df_test['3SsnPorch'] + df_test['ScreenPorch']
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )<load_pretrained>
df_train['TotalBaths'] = df_train['BsmtFullBath'] + df_train['FullBath'] + 0.5*(df_train['BsmtHalfBath'] + df_train['HalfBath'])
 df_test['TotalBaths'] = df_test['BsmtFullBath'] + df_test['FullBath'] + 0.5*(df_test['BsmtHalfBath'] + df_test['HalfBath'] )
def load_embedding(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f )<categorify>
df_train['TotalAreaSF'] = df_train['TotalBsmtSF'] + df_train['GrLivArea']
df_test['TotalAreaSF'] = df_test['TotalBsmtSF'] + df_test['GrLivArea']
def build_matrix(word_index, path):
embedding_index = load_embedding(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<choose_model_class>
df_train['Age'] = df_train['YrSold'].astype('int64')- df_train['YearBuilt']
df_test['Age'] = df_test['YrSold'].astype('int64')- df_test['YearBuilt']
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<load_from_csv>
def feature_engineering(df):
df_data = df.copy()
feature = {
'categorical':{
'MSSubClass': ['20', '30', '40', '45', '50', '60', '70', '75', '80', '85', '90', '120', '150', '160', '180', '190'],
'MSZoning': ['A', 'C', 'FV', 'I', 'RH', 'RL', 'RP', 'RM'],
'Alley': ['Grvl', 'Pave', 'None'],
'LandContour': ['Lvl', 'Bnk', 'HLS', 'Low'],
'LotConfig': ['Inside', 'Corner', 'CulDSac', 'FR2', 'FR3'],
'Neighborhood': ['Blmngtn', 'Blueste', 'BrDale', 'BrkSide', 'ClearCr', 'CollgCr', 'Crawfor', 'Edwards', 'Gilbert', 'IDOTRR', 'MeadowV', 'Mitchel',
'Names', 'NoRidge', 'NPkVill', 'NridgHt', 'NWAmes', 'OldTown', 'SWISU', 'Sawyer', 'SawyerW', 'Somerst', 'StoneBr', 'Timber', 'Veenker'],
'Condition1': ['Artery', 'Feedr', 'Norm', 'RRNn', 'RRAn', 'PosN', 'PosA', 'RRNe', 'RRAe'],
'Condition2': ['Artery', 'Feedr', 'Norm', 'RRNn', 'RRAn', 'PosN', 'PosA', 'RRNe', 'RRAe'],
'BldgType': ['1Fam', '2FmCon', 'Duplx', 'TwnhsE', 'TwnhsI'],
'HouseStyle': ['1Story', '1.5Fin', '1.5Unf', '2Story', '2.5Fin', '2.5Unf', 'SFoyer', 'SLvl'],
'RoofStyle': ['Flat', 'Gable', 'Gambrel', 'Hip', 'Mansard', 'Shed'],
'RoofMatl': ['ClyTile', 'CompShg', 'Membran', 'Metal', 'Roll', 'Tar&Grv', 'WdShake', 'WdShngl'],
'Exterior1st': ['AsbShng', 'AsphShn', 'BrkComm', 'BrkFace', 'CBlock', 'CemntBd', 'HdBoard', 'ImStucc', 'MetalSd', 'Other', 'Plywood', 'PreCast', 'Stone', 'Stucco',
'VinylSd', 'Wd Sdng', 'WdShing'],
'Exterior2nd': ['AsbShng', 'AsphShn', 'BrkComm', 'BrkFace', 'CBlock', 'CemntBd', 'HdBoard', 'ImStucc', 'MetalSd', 'Other', 'Plywood', 'PreCast', 'Stone', 'Stucco',
'VinylSd', 'Wd Sdng', 'WdShing'],
'MasVnrType': ['BrkCmn', 'BrkFace', 'CBlock', 'None', 'Stone'],
'Foundation': ['BrkTil', 'CBlock', 'PConc', 'Slab', 'Stone', 'Wood'],
'Heating': ['Floor', 'GasA', 'GasW', 'Grav', 'OthW', 'Wall'],
'Electrical': ['SBrkr', 'FuseA', 'FuseF', 'FuseP', 'Mix'],
'Functional': ['Typ', 'Min1', 'Min2', 'Mod', 'Maj1', 'Maj2', 'Sev', 'Sal'],
'GarageType': ['2Types', 'Attchd', 'Basment', 'BuiltIn', 'CarPort', 'Detchd', 'None'],
'GarageFinish': ['Fin', 'RFn', 'Unf', 'None'],
'PavedDrive': ['Y', 'P', 'N'],
'MiscFeature': ['Elev', 'Gar2', 'Othr', 'Shed', 'TenC', 'None'],
'MoSold': ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'],
'YrSold': ['2006', '2007', '2008', '2009', '2010'],
'SaleType': ['WD', 'CWD', 'VWD', 'New', 'COD', 'Con', 'ConLw', 'ConLI', 'ConLD', 'Oth'],
'SaleCondition': ['Normal', 'Abnorml', 'AdjLand', 'Alloca', 'Family', 'Partial']
},
'binary': {
'Street': ['Pave', 'Grvl'],
'CentralAir': ['Y', 'N']
},
'ordinal': {
'LotShape': ['None', 'IR3', 'IR2', 'IR1', 'Reg'],
'Utilities': ['None', 'NoSeWa', 'NoSewr', 'AllPub'],
'LandSlope': ['None', 'Sev', 'Mod', 'Gtl'],
'ExterQual': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'ExterCond': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'BsmtQual': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'BsmtCond': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'BsmtExposure': ['None', 'No', 'Mn', 'Av', 'Gd'],
'BsmtFinType1': ['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'],
'BsmtFinType2': ['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'],
'HeatingQC': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'KitchenQual': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'FireplaceQu': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'GarageQual': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'GarageCond': ['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'],
'Fence': ['None', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv'],
'PoolQC': ['None', 'Fa', 'Ta', 'Gd', 'Ex']
},
}
selected = []
for cname in df_data.columns:
if cname in feature['binary']:
default_value = feature['binary'][cname][0]
feature_name = cname + "_is_" + default_value
selected.append(feature_name)
 df_data[feature_name] = df_data[cname].apply(lambda x: int(x == default_value))
elif cname in feature['categorical']:
values = feature['categorical'][cname]
for val in values:
try:
new_name = "{}_{}".format(cname, val)
 selected.append(new_name)
 df_data[new_name] = df_data[cname].apply(lambda x: int(x == val))
except Exception as err:
print("One-hot encoding for {}_{}.Error: {}".format(cname, val, err))
elif cname in feature['ordinal']:
new_name = cname + "_ordinal"
selected.append(new_name)
 df_data[new_name] = df_data[cname].apply(lambda x: int(feature['ordinal'][cname].index(x)))
 else:
selected.append(cname)
 return df_data[selected]
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<prepare_x_and_y>
df_train = feature_engineering(df_train)
 df_test = feature_engineering(df_test)
 df_train
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str )<feature_engineering>
for col in df_train.columns:
if any(df_train[col])== False:
df_train.drop([col], axis=1, inplace=True)
 df_test.drop([col], axis=1, inplace=True )
for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False )<string_transform>
ids = df_test['Id']
y = df_train['SalePrice']
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))<string_transform>
df_train.drop(['Id', 'SalePrice'], axis=1, inplace=True)
 df_test.drop(['Id'], axis=1, inplace=True )
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen = MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen = MAX_LEN )<define_variables>
scaler = StandardScaler()
scaler.fit(df_train)
 train = scaler.transform(df_train)
 test = scaler.transform(df_test )
sample_weights = np.ones(len(x_train), dtype=np.float32)
 sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
 sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()<define_variables>
X_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.25, random_state=1)
 X_test, X_val, y_test, y_val = train_test_split(X_test, y_test, test_size=0.5, random_state=123 )
embedding_matrix = np.concatenate([build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 checkpoint_predictions = []
weights = []<train_model>
from sklearn.metrics import mean_squared_error
for model_idx in range(NUM_MODELS):
model = build_model(embedding_matrix, y_aux_train.shape[-1])
 for global_epoch in range(EPOCHS):
model.fit(
 x_train,
[y_train, y_aux_train],
batch_size=BATCH_SIZE,
epochs=1,
verbose=2,
sample_weight=[sample_weights.values, np.ones_like(sample_weights)],
callbacks=[
LearningRateScheduler(lambda _: 1e-3 *(0.55 ** global_epoch))
] 
)
 checkpoint_predictions.append(model.predict(x_test, batch_size=2048)[0].flatten())
 weights.append(2 ** global_epoch)
 predictions = np.average(checkpoint_predictions, weights=weights, axis=0 )<save_to_csv>
from sklearn.metrics import mean_squared_error
submission = pd.DataFrame.from_dict({
'id': test_df.id,
'prediction': predictions})
 submission.to_csv('submission.csv', index=False )<set_options>
param_init = {
"max_depth": 5,
"n_estimators": 3000,
"learning_rate": 0.01,
"subsample": 0.5,
"colsample_bytree": 0.7,
"min_child_weight": 1.5,
"reg_alpha": 0.75,
"reg_lambda": 0.4,
"seed": 42,
}
warnings.filterwarnings('ignore' )<feature_engineering>
xgb_model = xgboost.XGBRegressor(**param_init )
vocab.tqdm = tqdm.tqdm_notebook<define_variables>
param_fit = {
"eval_metric": "rmse",
"early_stopping_rounds": 500,
"verbose": 200,
"eval_set": [(X_val, y_val)]
}
path = ".. /input/jigsaw-unintended-bias-in-toxicity-classification/"
emb_path = ".. /input/embeddings-glove-crawl-torch-cached"
n_folds = 5
device = 'cuda'
gc.enable() ;<set_options>
xgb_model = xgb_model.fit(X_train, y_train, **param_fit )
seed = 7777
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True<categorify>
y_pred_xgb = xgb_model.predict(X_test )
class RegExCleaner() :
def __init__(self, expressions=[]):
r
self.expressions = expressions
@staticmethod
def _compile(expressions):
regexps = []
for pattern, repl in expressions.items() :
regexps.append(( re.compile(pattern), repl))
return regexps
@classmethod
def from_dict(cls, custom_dic):
r
return cls(cls._compile(custom_dic))
@classmethod
def from_vocab(cls, vocab):
r
pattern = re.compile("|".join(map(re.escape, vocab.keys())))
 repl = lambda match: vocab[match.group(0)]
return cls([(pattern, repl)])
 def __add__(self, b):
return RegExCleaner(self.expressions + b.expressions)
 def __call__(self, s):
for regex, repl in self.expressions:
s = regex.sub(repl, s)
 return s
tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)
 cleaner = RegExCleaner.from_dict({r'https?:/\/\S+':r' ',
r'[^A-Za-z0-9!.,?$'"]+':r' '})
 def preparation(s):
s = cleaner(s)
 return ' '.join(tknzr.tokenize(s))<data_type_conversions>
mean_squared_error(y_test, y_pred_xgb, squared=False )
class JigsawEvaluator:
def __init__(self, y_true, y_identity, power=-5, overall_model_weight=0.25):
self.y =(y_true >= 0.5 ).astype(int)
 self.y_i =(y_identity >= 0.5 ).astype(int)
 self.n_subgroups = self.y_i.shape[1]
self.power = power
self.overall_model_weight = overall_model_weight
@staticmethod
def _compute_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
def _compute_subgroup_auc(self, i, y_pred):
mask = self.y_i[:, i] == 1
return self._compute_auc(self.y[mask], y_pred[mask])
 def _compute_bpsn_auc(self, i, y_pred):
mask = self.y_i[:, i] + self.y == 1
return self._compute_auc(self.y[mask], y_pred[mask])
 def _compute_bnsp_auc(self, i, y_pred):
mask = self.y_i[:, i] + self.y != 1
return self._compute_auc(self.y[mask], y_pred[mask])
 def compute_bias_metrics_for_model(self, y_pred):
records = np.zeros(( 3, self.n_subgroups))
for i in range(self.n_subgroups):
records[0, i] = self._compute_subgroup_auc(i, y_pred)
 records[1, i] = self._compute_bpsn_auc(i, y_pred)
 records[2, i] = self._compute_bnsp_auc(i, y_pred)
 return records
def _calculate_overall_auc(self, y_pred):
return roc_auc_score(self.y, y_pred)
 def _power_mean(self, array):
total = sum(np.power(array, self.power))
return np.power(total / len(array), 1 / self.power)
 def get_final_metric(self, y_pred):
bias_metrics = self.compute_bias_metrics_for_model(y_pred)
 bias_score = np.average([
self._power_mean(bias_metrics[0]),
self._power_mean(bias_metrics[1]),
self._power_mean(bias_metrics[2])
 ])
 overall_score = self.overall_model_weight * self._calculate_overall_auc(y_pred)
 bias_score =(1 - self.overall_model_weight)* bias_score
return overall_score + bias_score
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
def as_torch(x):
return torch.FloatTensor([float(x)])
 class PartialSet() :
def __init__(self, ds, indices):
self.ds = ds
self.idx = indices
self.len = len(indices)
 self.fields = self.ds.fields
def update(self, idx):
self.idx = idx
self.len = len(idx)
 def __getitem__(self, idx):
return self.ds[self.idx[idx]]
def __len__(self):
return self.len
def sigmoid(x):
return 1/(1+np.exp(-x))
class DataFrameDataset(data.Dataset):
def __init__(self, df, fields, is_test=False, **kwargs):
keys = dict(fields)
 for n, f in list(keys.items()):
if isinstance(n, tuple):
keys.update(zip(n, f))
del keys[n]
keys = keys.keys()
examples = []
for i, row in tqdm.tqdm_notebook(df.iterrows() , total=len(df)) :
examples.append(data.Example.fromlist([row[k] for k in keys], fields))
super().__init__(examples, fields, **kwargs )<load_from_csv>
regressorTree = DecisionTreeRegressor(random_state = 0, min_samples_split=2, max_depth=6)
 regressorTree.fit(X_train, y_train )
cols = ['id', 'comment_text', 'target'] + identity_columns
dtypes = {'target': np.float16,'comment_text': object,'id': np.int32}
for c in identity_columns:
dtypes[c] = np.float16
df = pd.read_csv(os.path.join(path, 'train.csv'), usecols=cols, dtype=dtypes, index_col=[0])
 y = df.target.values > 0.5
skf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = seed)
 test_df = pd.read_csv(os.path.join(path, 'test.csv'),
dtype={'comment_text': object,'id': np.int32}, index_col=[0])
 test_df['prediction'] = 0<categorify>
y_pred_Tree = regressorTree.predict(X_test)
 mean_squared_error(y_test, y_pred_Tree, squared=False )
def sentence2numbers(s, fill_as=1):
seq = []
for w in s:
try:
seq.append(vocabulary.stoi[w])
 except KeyError:
seq.append(fill_as)
 return np.array(seq, dtype=np.int32 )<define_variables>
regressorGB = GradientBoostingRegressor(
 max_depth=5,
n_estimators=10000,
learning_rate=0.25 
)
 regressorGB.fit(X_train, y_train )
pad_token = 0
txt_field = data.Field(sequential=True, preprocessing=sentence2numbers,
pad_token=pad_token, use_vocab=False)
 num_field = data.Field(sequential=False, dtype=torch.float, use_vocab=False)
 idx_field = data.Field(sequential=False, dtype=torch.int64, use_vocab=False)
 train_fields = [ 
('id', idx_field), 
('target', num_field), 
('comment_text', txt_field),
]
test_fields = [ 
('id', idx_field), 
('comment_text', txt_field),
]<load_pretrained>
y_pred_GB = regressorGB.predict(X_test)
 mean_squared_error(y_test, y_pred_GB, squared=False )
vec = vocab.Vectors(os.path.join(emb_path, 'glove.840B.300d.txt'), cache=emb_path)
 vocabulary = vocab.Vocab(counter, max_size=500000, vectors=vec, specials=['<pad>', '<unk>'])
 torch.zero_(vocabulary.vectors[1]);
del vec
gc.collect() ;
print('Embedding vocab size: ', vocabulary.vectors.size(0))<create_dataframe>
regressorLasso = Lasso(alpha=0.0007)
 regressorLasso.fit(X_train, y_train )
ds = DataFrameDataset(df.reset_index() , train_fields)
 test_ds = DataFrameDataset(test_df.reset_index() , test_fields )<categorify>
y_pred_Lasso = regressorLasso.predict(X_test)
 mean_squared_error(y_test, y_pred_Lasso, squared=False )
class BatchWrapper:
def __init__(self, dl, mode='train'):
self.dl, self.mode = dl, mode
def __iter__(self):
if self.mode !='test':
for batch in self.dl:
yield(batch.comment_text, batch.target, batch.id)
 else:
for batch in self.dl:
yield(batch.comment_text, batch.id)
 def __len__(self):
return len(self.dl)
 def wrapper(ds, mode='train', **args):
dl = data.BucketIterator(ds, **args)
 return BatchWrapper(dl, mode )<categorify>
SalePrice_pred = xgb_model.predict(test)
 SalePrice_pred = np.exp(SalePrice_pred )
tloader = wrapper(test_ds, mode='test', batch_size=512, device='cuda',
sort_key=lambda x: len(x.comment_text),
sort_within_batch=True, shuffle=False, repeat=False )<predict_on_test>
submission = {'Id': ids, 'SalePrice': SalePrice_pred}
class BaseModule(nn.Module):
@torch.no_grad()
def prediction(self, x):
score = self.forward(x)
 return torch.sigmoid(score)
 @torch.no_grad()
def evaluate(self, x, y, func):
preds = self.forward(x)
 loss = func(preds.squeeze(-1), y)
 return preds, loss
class RecNN(BaseModule):
def __init__(self, embs_vocab, hidden_size, layers=1, dropout=0., bidirectional=False):
super().__init__()
self.hidden_size = hidden_size
self.bidirectional = bidirectional
self.num_layers = layers
self.emb = nn.Embedding.from_pretrained(embs_vocab)
 self.line = nn.Linear(embs_vocab.size(1), embs_vocab.size(1))
self.lstm = nn.LSTM(embs_vocab.size(1), self.hidden_size,
num_layers=layers, bidirectional=bidirectional, dropout=dropout)
 self.gru = nn.GRU(embs_vocab.size(1), self.hidden_size,
num_layers=layers, bidirectional=bidirectional, dropout=dropout)
 self.out = nn.Linear(self.hidden_size*(bidirectional + 1), 32)
 self.last = nn.Linear(32, 1)
 def forward(self, x):
embs = self.emb(x)
 lstm,(h, c)= self.lstm(embs)
 x = F.relu(self.line(embs), inplace=True)
 gru, h = self.gru(x, h)
 lstm = lstm + gru
lstm, _ = lstm.max(dim=0, keepdim=False)
 out = self.out(lstm)
 out = self.last(F.relu(out)).squeeze()
return out.squeeze(-1 )<categorify>
df_submission = pd.DataFrame(submission)
 df_submission
<save_to_csv><EOS> 
df_submission.to_csv('submission.csv', index=False )