Kaggle competition: Jigsaw Unintended Bias in Toxicity Classification
<SOS> metric: JigsawBiasAUC Kaggle data source: jigsaw-unintended-bias-in-toxicity-classification<predict_on_test>
tqdm.pandas()
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
TEXT_COL = 'comment_text'
VECS_PATH = Path('.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec')
 TRAIN_DATA = '.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv'
TEST_DATA = '.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv'
print('Average RMSLE score:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))<define_variables>
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15<save_to_csv>
misspell_dict = {"aren't": "are not", "can't": "cannot", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not",
"hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'll": "he will", "he's": "he is",
"i'd": "I had", "i'll": "I will", "i'm": "I am", "isn't": "is not",
"it's": "it is", "it'll": "it will", "i've": "I have", "let's": "let us",
"mightn't": "might not", "mustn't": "must not", "shan't": "shall not",
"she'd": "she would", "she'll": "she will", "she's": "she is",
"shouldn't": "should not", "that's": "that is", "there's": "there is",
"they'd": "they would", "they'll": "they will", "they're": "they are",
"they've": "they have", "we'd": "we would", "we're": "we are",
"weren't": "were not", "we've": "we have", "what'll": "what will",
"what're": "what are", "what's": "what is", "what've": "what have",
"where's": "where is", "who'd": "who would", "who'll": "who will",
"who're": "who are", "who's": "who is", "who've": "who have",
"won't": "will not", "wouldn't": "would not", "you'd": "you would",
"you'll": "you will", "you're": "you are", "you've": "you have",
"'re": " are", "wasn't": "was not", "we'll": " will", "tryin'": "trying"}
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<load_from_csv>
def _get_misspell(misspell_dict):
misspell_re = re.compile('(%s)' % '|'.join(misspell_dict.keys()))
return misspell_dict, misspell_re
train1 = pd.read_csv('.. /input/train.csv', header = 0, dtype={'Age': np.float64})
 test = pd.read_csv('.. /input/test.csv' , header = 0, dtype={'Age': np.float64})
 train=train1.drop(columns=['Survived'])
 allfeat = pd.concat([train, test],axis=0)
 print(train.info())
 print(train.shape)
 print(test.shape )<drop_column>
def replace_typical_misspell(text):
misspellings, misspellings_re = _get_misspell(misspell_dict)
 def replace(match):
return misspellings[match.group(0)]
return misspellings_re.sub(replace, text )
allfeat=allfeat.drop(columns=['PassengerId','Cabin','Ticket'] )<categorify>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']',
'>', '%', '=', '
'®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█',
'½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶',
'↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼',
'⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪',
'╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', "*"]
allfeat=pd.concat([allfeat,pd.get_dummies(allfeat['Pclass'])], axis=1)
 allfeat=allfeat.drop(columns=['Pclass'] )<categorify>
def clean_text(x):
x = str(x)
 for punct in puncts + list(string.punctuation):
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
allfeat=pd.concat([allfeat,pd.get_dummies(allfeat['Sex'])], axis=1)
 allfeat=allfeat.drop(columns=['Sex'] )<drop_column>
def clean_numbers(x):
return re.sub('\d+', ' ', x )
allfeat['FamilySize'] = allfeat['SibSp'] + allfeat['Parch'] + 1
allfeat=allfeat.drop(columns=['SibSp','Parch'] )<feature_engineering>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )
allfeat['IsAlone'] = 0
allfeat.loc[allfeat['FamilySize'] == 1, 'IsAlone'] = 1<categorify>
def load_fasttext(word_index):
embeddings_index = dict(get_coefs(*o.strip().split(' ')) for o in open(VECS_PATH))
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
allfeat['Embarked'] = allfeat['Embarked'].fillna('S')
 allfeat=pd.concat([allfeat,pd.get_dummies(allfeat['Embarked'])],axis=1)
 allfeat=allfeat.drop(columns='Embarked' )<categorify>
maxlen = 240
max_features = 100000
embed_size = 300
batch_size = 1024
train_epochs = 5
n_splits = 5
seed = 1000
allfeat['Fare'] = allfeat['Fare'].fillna(train['Fare'].median())
 allfeat['CategoricalFare'] = pd.qcut(allfeat['Fare'], 4)
 allfeat=pd.concat([allfeat,pd.get_dummies(allfeat['CategoricalFare'])],axis=1)
 allfeat=allfeat.drop(columns=['Fare','CategoricalFare'] )<categorify>
def load_and_prec() :
train = pd.read_csv(TRAIN_DATA, index_col='id')
 test = pd.read_csv(TEST_DATA, index_col='id')
 train['comment_text'] = train['comment_text'].str.lower()
test['comment_text'] = test['comment_text'].str.lower()
train['comment_text'] = train['comment_text'].apply(replace_typical_misspell)
 test['comment_text'] = test['comment_text'].apply(replace_typical_misspell)
 train['comment_text'] = train['comment_text'].apply(clean_text)
 test['comment_text'] = test['comment_text'].apply(clean_text)
 train['comment_text'] = train['comment_text'].apply(clean_numbers)
 test['comment_text'] = test['comment_text'].apply(clean_numbers)
 train_x = train['comment_text'].fillna('_
test_x = test['comment_text'].fillna('_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x))
train_x = tokenizer.texts_to_sequences(train_x)
 test_x = tokenizer.texts_to_sequences(test_x)
 train_x = pad_sequences(train_x, maxlen=maxlen)
 test_x = pad_sequences(test_x, maxlen=maxlen)
 train_y =(train['target'].values > 0.5 ).astype(int)
 np.random.seed(seed)
 train_idx = np.random.permutation(len(train_x))
train_x = train_x[train_idx]
train_y = train_y[train_idx]
return train_x, train_y, test_x, tokenizer.word_index
avg=allfeat['Age'].mean()
std=allfeat['Age'].std()
allfeat['Age']=allfeat['Age'].fillna(value=np.random.randint(avg-std,avg+std))
allfeat['Age'] = allfeat['Age'].astype(int)
 allfeat['CategoricalAge'] = pd.cut(allfeat['Age'], 5)
 allfeat=pd.concat([allfeat,pd.get_dummies(allfeat['CategoricalAge'])],axis=1)
 allfeat=allfeat.drop(columns=['Age','CategoricalAge'] )<categorify>
class Net(nn.Module):
def __init__(self, embedding_matrix):
super(Net, self ).__init__()
lstm_hidden_size = 120
gru_hidden_size = 60
self.gru_hidden_size = gru_hidden_size
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, lstm_hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(lstm_hidden_size*2, gru_hidden_size, bidirectional=True, batch_first=True)
 self.linear = nn.Linear(gru_hidden_size*6, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.unsqueeze(h_embedding.transpose(1, 2), 2)
 h_embedding = torch.squeeze(self.embedding_dropout(h_embedding)).transpose(1, 2)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, hh_gru = self.gru(h_lstm)
 hh_gru = hh_gru.view(-1, self.gru_hidden_size*2)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( hh_gru, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
allfeat['Title'] = allfeat['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 allfeat['Title'] = allfeat['Title'].replace('Mlle', 'Miss')
 allfeat['Title'] = allfeat['Title'].replace('Ms', 'Miss')
 allfeat['Title'] = allfeat['Title'].replace('Mme', 'Mrs')
 allfeat=pd.concat([allfeat,pd.get_dummies(allfeat['Title'])],axis=1)
 allfeat=allfeat.drop(columns='Title' )<prepare_x_and_y>
warnings.filterwarnings('ignore' )
allfeat.columns=['1', '2', '3', 'female', 'male', 'FamSize', 'IsAlone', 'C', 'Q', 'S', 'fare1', 'fare2', 'fare3', 'fare4', \
'age1', 'age2', 'age3', 'age4', 'age5', 'Master', 'Miss', 'Mr', 'Mrs', 'Rare']
print(list(allfeat))
X=allfeat[:][0:891]
testdf=allfeat[:][891:1309]
y=train1['Survived']<train_on_grid>
with timer('load data'):
train_x, train_y, test_x, word_index = load_and_prec()
embedding_matrix = load_fasttext(word_index )
param_grid = [{'min_child_weight': np.arange(0.1, 10.1, 0.1)}]
i=1
kf = StratifiedKFold(n_splits=10,random_state=1,shuffle=True)
 for train_index,test_index in kf.split(X,y):
print('
{} of kfold {}'.format(i,kf.n_splits))
xtr,xvl = X.loc[train_index],X.loc[test_index]
ytr,yvl = y[train_index],y[test_index]
model = GridSearchCV(XGBClassifier() , param_grid, cv=10, scoring= 'f1',iid=True)
 model.fit(xtr, ytr)
 print(model.best_params_)
 pred=model.predict(xvl)
 print('accuracy_score',accuracy_score(yvl,pred))
i+=1 
<save_to_csv>
import random
import os
op=pd.DataFrame(data={'PassengerId':test['PassengerId'],'Survived':model.predict(testdf)})
 op.to_csv('KFold_XGB_GridSearchCV_submission.csv',index=False )<define_variables>
_camel_re1 = re.compile(' (.)([A-Z][a-z]+)')
 _camel_re2 = re.compile('([a-z0-9] )([A-Z])')
 def camel2snake(name):
s1 = re.sub(_camel_re1, r'\1_\2', name)
 return re.sub(_camel_re2, r'\1_\2', s1 ).lower()
time1 = time.time()<set_options>
class TrainEvalCallback(Callback):
def begin_fit(self):
self.run.n_epochs = 0
self.run.n_iter = 0
def after_batch(self):
if not self.in_train: return
self.run.n_epochs+=1./self.iters
self.run.n_iter+=1
def begin_epoch(self):
self.run.n_epochs = self.epoch
self.model.train()
self.run.in_train = True
def begin_validate(self):
self.model.eval()
self.run.in_train = False
class CancelTrainException(Exception): pass
class CancelEpochException(Exception): pass
class CancelBatchException(Exception): pass
warnings.filterwarnings('ignore' )<load_from_csv>
from collections import Iterable
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<concatenate>
def listify(o):
if o is None: return []
if isinstance(o, list): return o
if isinstance(o, str): return [o]
if isinstance(o, Iterable): return list(o)
 return [o]
data = pd.concat(( train, test),ignore_index=True)
 data = data.drop("SalePrice",1 )<sort_values>
class Runner() :
def __init__(self, cbs=None, cb_funcs=None):
cbs = listify(cbs)
 for cbf in listify(cb_funcs):
cb = cbf()
setattr(self, cb.name, cb)
 cbs.append(cb)
 self.stop, self.cbs = False, [TrainEvalCallback() ] + cbs
@property
def opt(self): return self.learn.opt
@property
def model(self): return self.learn.model
@property
def loss_func(self): return self.learn.loss_func
@property
def data(self): return self.learn.data
def one_batch(self, xb, yb):
try:
self.xb, self.yb = xb, yb
self('begin_batch')
 self.pred = self.model(self.xb)
 self('after_pred')
 self.loss = self.loss_func(self.pred, self.yb)
 self('after_loss')
 if not self.in_train: return
self.loss.backward()
self('after_backward')
 self.opt.step()
self('after_step')
 self.opt.zero_grad()
except CancelBatchException: self('after_cancel_batch')
 finally: self('after_batch')
 def all_batches(self, dl):
self.iters = len(dl)
 try:
for xb, yb in progress_bar(dl, leave=False): self.one_batch(xb, yb)
 except CancelEpochException: self('after_cancel_epoch')
 def fit(self, epochs, learn):
self.epochs, self.learn, self.loss = epochs, learn, torch.tensor(0.)
try:
for cb in self.cbs: cb.set_runner(self)
 self('begin_fit')
 for epoch in range(epochs):
self.epoch = epoch
if not self('begin_epoch'): self.all_batches(self.data.train_dl)
 with torch.no_grad() :
if not self('begin_validate'): self.all_batches(self.data.valid_dl)
 self('after_epoch')
 except CancelTrainException: self('after_cancel_train')
 finally:
self('after_fit')
 self.learn = None
def __call__(self, cb_name):
res = False
for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name)and res
return res
s1 = data.isnull().sum() [data.isnull().sum() != 0]
s2 = s1/data.shape[0]
pd.concat({'Missing Values': s1,
'Missing Ratio': s2},axis=1 ).sort_values(by = 'Missing Ratio', ascending = False )<data_type_conversions>
class Learner() :
def __init__(self, model, opt, loss_func, data):
self.model, self.opt, self.loss_func, self.data = model, opt, loss_func, data
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'GarageCars',
'GarageArea', 'MasVnrArea', 'BsmtHalfBath'):
data[col] = data[col].fillna(0)
 data['LotFrontage'] = data['LotFrontage'].fillna(data['LotFrontage'].median())
 for col in('PoolQC', 'Fence', 'FireplaceQu','GarageType','GarageFinish', 'GarageQual',
'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'MSSubClass', 'MiscFeature'):
data[col] = data[col].fillna('None')
 for col in('Exterior1st', 'Exterior2nd', 'KitchenQual', 'Electrical', 'SaleType',
'MSZoning', 'GarageYrBlt'):
data[col] = data[col].fillna(data[col].mode() [0])
 data["Functional"] = data["Functional"].fillna("Typ")
 data = data.drop(['Id', 'Utilities'],1 )<prepare_output>
def get_model(data, lr=0.001):
model = Net(embedding_matrix ).to(device)
 return model, torch.optim.Adam(model.parameters() , lr )
TotalSF = data['1stFlrSF'] + data['2ndFlrSF'] + data['TotalBsmtSF']
data.insert(loc=0,column="TotalSF", value=TotalSF )<prepare_x_and_y>
class DataBunch() :
def __init__(self, train_dl, valid_dl):
self.train_dl, self.valid_dl = train_dl, valid_dl
@property
def train_ds(self): return self.train_dl.dataset
@property
def valid_ds(self): return self.valid_dl.dataset
y_log = np.log1p(Y )<sort_values>
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed ).split(train_x, train_y))
skew = abs(data.skew() ).sort_values(ascending = False)
 skew_features = skew[skew > 1].index
data[skew_features] = np.log1p(data[skew_features] )<categorify>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
data = pd.get_dummies(data )<prepare_x_and_y>
x_test_cuda = torch.tensor(test_x, dtype=torch.long ).to(device)
 test = torch.utils.data.TensorDataset(x_test_cuda)
 test_dl = DataLoader(test, batch_size=batch_size, shuffle=False )
X_train = data[:train.shape[0]]
X_test = data[train.shape[0]:]<import_modules>
def get_data(train_idx, valid_idx):
x_train_ds = torch.tensor(train_x[train_idx], dtype=torch.long ).to(device)
 y_train_ds = torch.tensor(train_y[train_idx, np.newaxis], dtype=torch.float32 ).to(device)
 x_val_ds = torch.tensor(train_x[valid_idx], dtype=torch.long ).to(device)
 y_val_ds = torch.tensor(train_y[valid_idx, np.newaxis], dtype=torch.float32 ).to(device)
 train_ds = torch.utils.data.TensorDataset(x_train_ds, y_train_ds)
 valid_ds = torch.utils.data.TensorDataset(x_val_ds, y_val_ds)
 train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
 valid_dl = DataLoader(valid_ds, batch_size=batch_size, shuffle=False)
 data = DataBunch(train_dl, valid_dl)
 return data
from sklearn.linear_model import Lasso, ElasticNet
from sklearn.kernel_ridge import KernelRidge
import xgboost as xgb
from sklearn.model_selection import KFold, cross_val_score, GridSearchCV<train_on_grid>
class FocalLoss(nn.Module):
def __init__(self, alpha=1, gamma=4, logits=False, reduce=True):
super(FocalLoss, self ).__init__()
self.alpha = alpha
self.gamma = gamma
self.logits = logits
self.reduce = reduce
def forward(self, inputs, targets):
if self.logits:
BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)
 else:
BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)
 pt = torch.exp(-BCE_loss)
 F_loss = self.alpha *(1-pt)**self.gamma * BCE_loss
if self.reduce:
return torch.mean(F_loss)
 else:
return F_loss
loss_fn = FocalLoss(logits=True )
model = Lasso()
grid = GridSearchCV(estimator=model, param_grid={'alpha': [100,10,1,0.1,0.01,0.001]})
 grid.fit(X_train, y_log)
 print('Lasso Regression parameters:')
 print('Alpha: ',grid.best_estimator_.alpha )<train_on_grid>
def get_learner(train_idx, valid_idx):
data = get_data(train_idx, valid_idx)
 learn = Learner(*get_model(data), loss_fn, data=data)
 return learn
model = ElasticNet()
grid = GridSearchCV(estimator=model, param_grid={'alpha': [100,10,1,0.1,0.01,0.001],
'l1_ratio': [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]})
 grid.fit(X_train, y_log)
 print('Elastic Net parameters:')
 print('Alpha: ',grid.best_estimator_.alpha)
 print('l1_ratio: ',grid.best_estimator_.l1_ratio )<train_on_grid>
class AvgStats() :
def __init__(self, metrics, in_train): self.metrics, self.in_train = listify(metrics), in_train
def reset(self):
self.tot_loss, self.count = 0., 0
self.tot_mets = [0.]*len(self.metrics)
 @property
def all_stats(self): return [self.tot_loss.item() ] + self.tot_mets
@property
def avg_stats(self): return [o/self.count for o in self.all_stats]
def __repr__(self):
if not self.count: return ''
return f"{'train' if self.in_train else 'valid'}: {self.avg_stats}"
def accumulate(self, run):
bn = run.xb.shape[0]
self.tot_loss+=run.loss*bn
self.count+=bn
for i, m in enumerate(self.metrics):
self.tot_mets[i]+=m(run.pred, run.yb)*bn
model = KernelRidge(kernel='polynomial')
 grid = GridSearchCV(estimator=model, param_grid={'alpha': [1000,10,1],
'coef0': [1000,10,1],
'degree': [2,3]})
 grid.fit(X_train, y_log)
 print('Kernel Ridge Regression parameters:')
 print('alpha: ',grid.best_estimator_.alpha)
 print('coef0: ',grid.best_estimator_.coef0)
 print('degree: ',grid.best_estimator_.degree )<train_on_grid>
class AvgStatsCallBack(Callback):
def __init__(self, metrics):
self.train_stats, self.valid_stats = AvgStats(metrics, True), AvgStats(metrics, False)
 def begin_epoch(self):
self.train_stats.reset()
self.valid_stats.reset()
def after_loss(self):
stats = self.train_stats if self.in_train else self.valid_stats
with torch.no_grad() : stats.accumulate(self.run)
 def after_epoch(self):
print(self.train_stats)
 print(self.valid_stats )
model = xgb.XGBRegressor()
grid = GridSearchCV(estimator=model, param_grid={'n_estimators': [300,400,500,600,700],
'max_depth': [2,3],
'learning_rate': [0.05,0.1]})
 grid.fit(X_train, y_log)
 print('XGB Regressor parameters:')
 print('alpha: ',grid.best_estimator_.n_estimators)
 print('max_depth: ',grid.best_estimator_.max_depth)
 print('degree: ',grid.best_estimator_.learning_rate )<compute_train_metric>
%matplotlib inline
def rmsle_cv(model):
kf = KFold(5, shuffle=True, random_state=42 ).get_n_splits(X_train.values)
 rmse= np.sqrt(-cross_val_score(model, X_train.values, y_log, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<compute_train_metric>
from sklearn.metrics import roc_auc_score
model_lasso = Lasso(alpha=0.001)
 score = rmsle_cv(model_lasso)
 print("Lasso Regression Score: {:.4f}".format(score.mean()))<compute_train_metric>
def roc(out, y):
score = roc_auc_score(y.cpu().detach().numpy() , out.cpu().detach().numpy())
 return score
model_enet = ElasticNet(alpha=0.001,l1_ratio=0.5)
 score = rmsle_cv(model_enet)
 print("ElasticNet Score: {:.4f}".format(score.mean()))<compute_train_metric>
class ParamScheduler(Callback):
_order = 1
def __init__(self, pname, sched_funcs): self.pname, self.sched_funcs = pname, sched_funcs
def begin_fit(self):
if not isinstance(self.sched_funcs,(list, tuple)) :
self.sched_funcs = [self.sched_funcs]*len(self.opt.param_groups)
 def set_param(self):
assert len(self.opt.param_groups)==len(self.sched_funcs)
 for pg, f in zip(self.opt.param_groups, self.sched_funcs):
pg[self.pname] = f(self.n_epochs/self.epochs)
 def begin_batch(self):
if self.in_train: self.set_param()
model_krr = KernelRidge(alpha=1000, coef0=1000, degree=2, kernel='polynomial')
 score = rmsle_cv(model_krr)
 print("Kernel Ridge Regressor Score: {:.4f}".format(score.mean()))<compute_train_metric>
class LR_Find(Callback):
_order = 1
def __init__(self, max_iter=100, min_lr=1e-6, max_lr=10):
self.max_iter, self.min_lr, self.max_lr = max_iter, min_lr, max_lr
self.best_loss = 1e9
def begin_batch(self):
if not self.in_train: return
pos = self.n_iter/self.max_iter
lr = self.min_lr*(self.max_lr/self.min_lr)**pos
for pg in self.opt.param_groups: pg['lr'] = lr
def after_step(self):
if self.n_iter>=self.max_iter or self.loss>self.best_loss*10:
raise CancelTrainException()
if self.loss<self.best_loss: self.best_loss = self.loss
model_xgb = xgb.XGBRegressor(n_estimators=700, max_depth=2, learning_rate=0.05)
 score = rmsle_cv(model_xgb)
 print("XGBoost Score: {:.4f}".format(score.mean()))<train_model>
run = Runner(cb_funcs=[LR_Find, Recorder] )
model_krr.fit(X_train,y_log)
 p_krr = np.expm1(model_krr.predict(X_test))
model_enet.fit(X_train,y_log)
 p_enet = np.expm1(model_enet.predict(X_test))
model_lasso.fit(X_train,y_log)
 p_lasso = np.expm1(model_lasso.predict(X_test))
model_xgb.fit(X_train,y_log)
 p_xgb = np.expm1(model_xgb.predict(X_test))<save_to_csv>
torch.cuda.empty_cache()
p_average =(p_lasso + p_xgb + p_krr + p_enet)/4
out = pd.Series(p_average,index=test.loc[:,'Id'])
 out.name = 'SalePrice'
out.to_csv('average.csv', header=True, index_label='Id' )<define_variables>
gc.collect()
Image(url= "https://i.ytimg.com/vi/1PhMWUoPDsk/maxresdefault.jpg" )<import_modules>
def annealer(f):
def _inner(start, end): return partial(f, start, end)
 return _inner
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
import math
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 survived = train_df['Survived']
passenger_id = test_df['PassengerId']<load_from_csv>
def pg_dicts(pgs): return [{'params': o} for o in pgs]
submission = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 submission.head()<count_missing_values>
def cos_1cycle_anneal(start, high, end):
return [sched_cos(start, high), sched_cos(high, end)]
print(train_df.isnull().sum())
 print(test_df.isnull().sum() )<feature_engineering>
from functools import partial
train_test_data = [train_df, test_df]
print(train_test_data)
 for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )<count_values>
phases = [0.2, 0.8]
train_df['Title'].value_counts() 
<count_values>
scheds = combine_scheds(phases, [sched_cos(1e-4, 5e-3), sched_cos(5e-3, 1e-3)] )
test_df['Title'].value_counts() 
<categorify>
from sklearn.metrics import roc_auc_score
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )<drop_column>
def accuracy(out, yb): return(torch.argmax(out, dim=1)==yb.long() ).float().mean()
train_df.drop('Name', axis=1, inplace=True)
 test_df.drop('Name', axis=1, inplace=True )<categorify>
run = Runner(cb_funcs=cbfs )
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )<categorify>
def train_and_eval() :
test_preds = np.zeros(len(test_x))
for fold,(train_idx, valid_idx)in enumerate(splits):
print('Fold:', fold)
 torch.cuda.empty_cache()
learn = get_learner(train_idx, valid_idx)
 gc.collect()
run = Runner(cb_funcs=cbfs)
 learn.model.train()
run.fit(4, learn)
 learn.model.eval()
test_preds_fold = np.zeros(len(test_dl.dataset))
for i,(x_batch,)in enumerate(test_dl):
with torch.no_grad() :
y_pred = learn.model(x_batch ).detach()
test_preds_fold[i*batch_size:(i+1)*batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds+=test_preds_fold/len(splits)
 del(learn)
 gc.collect()
print(f'Test {fold} added')
 print('Training Completed')
 return test_preds
train_df["Age"].fillna(train_df.groupby("Title")["Age"].transform("median"), inplace=True)
 test_df["Age"].fillna(test_df.groupby("Title")["Age"].transform("median"), inplace=True )<categorify>
def sigmoid(x): return 1/(1+np.exp(-x))
train_df.groupby("Title")["Age"].transform("median")
<feature_engineering>
preds = train_and_eval()
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 26), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 26)&(dataset['Age'] <= 36), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 36)&(dataset['Age'] <= 62), 'Age'] = 3,
dataset.loc[ dataset['Age'] > 62, 'Age'] = 4<count_values>
sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 sub['prediction'] = preds
sub.reset_index(drop=False, inplace=True)
 sub.head()
Pclass1 = train_df[train_df['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train_df[train_df['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train_df[train_df['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))<data_type_conversions>
sub.to_csv('submission.csv', index=False )
<categorify><EOS> 
sub.to_csv('submission.csv', index=False )