Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<data_type_conversions>
tqdm.pandas(desc='Progress')
for cols in('MSZoning','Electrical','Exterior2nd','Exterior1st','SaleType'):
all_data[cols]=all_data[cols].fillna(all_data[cols].mode() [0])
<categorify>
print(os.listdir(".. /input"))
all_data=all_data.drop(['Utilities'],axis=1)
 all_data['Functional']=all_data['Functional'].fillna('Typ')
 all_data['LotFrontage']=all_data.groupby(['Neighborhood'] ).transform(
 lambda x: x.fillna(x.median())
)
<create_dataframe>
embed_size = 300
max_features = 200000
maxlen = 40
batch_size=1024
missing=(all_data.isnull().sum().sort_values(ascending=False)/len(all_data)) [:35]
missing_data = pd.DataFrame({'Missing Ratio' :missing})
 missing_data.head(5 )<data_type_conversions>
def seed_everything(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
all_data['MSSubClass']=all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold']=all_data['YrSold'].apply(str)
 all_data['MoSold']=all_data['MoSold'].apply(str )<categorify>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
cols_for_lab_enc=('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols_for_lab_enc:
le=LabelEncoder()
le.fit(list(all_data[c].values))
all_data[c] = le.transform(list(all_data[c].values)) 
<feature_engineering>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
if bool(re.search(r'\d', x)) :
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] +
all_data['1stFlrSF'] + all_data['2ndFlrSF'])
 all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'] )<filter>
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
return train_X, test_X, train_y,tokenizer.word_index
num_feat=all_data.dtypes[all_data.dtypes!="object"].index
num_feat<sort_values>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words+1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words+1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words+1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
skews_col=all_data[num_feat].skew().sort_values(ascending=False)
 skews_col = skews_col[abs(skews_col)> 0.75]
skews_col<categorify>
class QuraData(data.Dataset):
def __init__(self,questions,labels,augument=False,training=True):
super(QuraData, self ).__init__()
self.augument=augument
self.questions=questions
self.labels= labels
self.len_ = len(self.questions)
 self.training=training
def shuffle(self,d):
return np.random.permutation(d.tolist())
 def dropout(self,d,p=0.5):
len_ = len(d)
 index = np.random.choice(len_,int(len_*p))
d[index]=0
return d
def __getitem__(self,index):
question,label = self.questions[index],self.labels[index,np.newaxis]
if self.training and self.augument :
question= self.dropout(question,p=0.05)
 question=torch.from_numpy(question ).long()
label=torch.LongTensor(label ).long()
return question,label
def __len__(self):
return self.len_
for c in skews_col.index:
all_data[c]=boxcox1p(all_data[c],0.1)
<sort_values>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
skews_col=all_data[num_feat].skew().sort_values(ascending=False)
 skews_col = skews_col[abs(skews_col)> 0.5]
skews_col<prepare_x_and_y>
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs
y=train['SalePrice']
all_data=all_data.drop(['SalePrice'],axis=1)
 all_data=pd.get_dummies(all_data)
 train=all_data[:len(train)]
test=all_data[len(train):] 
<import_modules>
def kmax_pooling(x, dim, k):
index = x.topk(k, dim = dim)[1].sort(dim = dim)[0]
return x.gather(dim, index )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC,ElasticNet,Lasso,Ridge
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split,GridSearchCV
from sklearn.metrics import mean_squared_error
from sklearn.neural_network import MLPRegressor
from sklearn.svm import SVR
import xgboost as xgb
import lightgbm as lgb
from bayes_opt import BayesianOptimization
from functools import partial
from sklearn.svm.libsvm import cross_validation
from sklearn.neighbors import KNeighborsRegressor
from mlxtend.regressor import StackingCVRegressor<compute_train_metric>
class Gate(nn.Module):
def __init__(self, input_size):
super(Gate, self ).__init__()
self.linear = nn.Linear(input_size, input_size)
 def forward(self, x):
x_proj = self.linear(x)
 gate = torch.sigmoid(x_proj)
 return x * gate
X_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.2)
 kfolds = KFold(n_splits=10, shuffle=True)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def rmsle_cv(model):
rmse= np.sqrt(-cross_val_score(model, train.values, y.values, scoring="neg_mean_squared_error", cv = kfolds.get_n_splits(train.values)))
 return rmse<train_model>
class SFU(nn.Module):

 def __init__(self, input_size, fusion_size):
super(SFU, self ).__init__()
self.linear_r = nn.Linear(input_size + fusion_size, input_size)
 self.linear_g = nn.Linear(input_size + fusion_size, input_size)
 def forward(self, x, fusions):
r_f = torch.cat([x, fusions], 2)
 r = F.tanh(self.linear_r(r_f))
g = F.sigmoid(self.linear_g(r_f))
o = g * r +(1-g)* x
return o
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<train_model>
kernel_sizes = [2,3,5]
class mergeNN(nn.Module):
def __init__(self,embedding_matrix):
super(mergeNN, self ).__init__()
self.model_name = 'LSTMText'
self.encoder = nn.Embedding.from_pretrained(torch.FloatTensor(embedding_matrix))
self.title_lstm = nn.LSTM(input_size = embed_size*2,\
hidden_size = 128,
num_layers =1,
bias = True,
batch_first = True,
bidirectional = True 
)
 self.title_gru = nn.GRU(input_size = 256,\
hidden_size = 128,
num_layers =1,
bias = True,
batch_first = True,
bidirectional = True 
)
 question_convs = [nn.Sequential(nn.Conv1d(in_channels = embed_size*2,out_channels = 48,kernel_size = kernel_size),
nn.BatchNorm1d(48),
nn.ReLU(inplace=True),
nn.MaxPool1d(kernel_size =(maxlen - kernel_size + 1)) 
)for kernel_size in kernel_sizes]
self.question_convs = nn.ModuleList(question_convs)
 self.lstm_attention = Attention(256, maxlen)
 self.fc = nn.Sequential(
 nn.Linear(912,256),
nn.BatchNorm1d(256),
nn.ReLU(inplace=True),
nn.Dropout(0.3),
nn.Linear(256,1)
)
 self.dropout=nn.Dropout2d(0.1)
 self.linear1=nn.Linear(600,256)
 for name, param in self.title_lstm.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 for name, param in self.title_gru.named_parameters() :
if 'bias' in name:
nn.init.constant_(param, 0.0)
 elif 'weight_ih' in name:
nn.init.kaiming_normal_(param)
 elif 'weight_hh' in name:
nn.init.orthogonal_(param)
 def forward(self, question):
question = self.encoder(question)
 question = torch.squeeze(self.dropout(torch.unsqueeze(question, 0)))
 question_lstm_out = self.title_lstm(question)[0]
question_res_out=self.linear1(question)+question_lstm_out
question_gru_out = self.title_gru(question_res_out)[0]
question_res_out2=question_res_out+question_gru_out
question_conv_out = kmax_pooling(( question_res_out2.permute(0,2,1)) ,2,2)
 question_conv_out = question_conv_out.view(question_conv_out.size(0), -1)
 question_attention_out=self.lstm_attention(question_res_out2)
 cnn_question_out = [question_conv(question.permute(0, 2, 1)) for question_conv in self.question_convs]
conv_out = torch.cat(cnn_question_out,dim=2)
 conv_out = conv_out.view(conv_out.size(0), -1)
 reshaped=torch.cat(( question_conv_out,question_attention_out,conv_out),1)
 logits = self.fc(( reshaped))
return logits
class AvarageModels(BaseEstimator, TransformerMixin, RegressorMixin):
def __init__(self,models):
self.models=models
def fit(self,X,y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self,X):
prediction= np.column_stack([m.predict(X)for m in self.models_])
 return np.mean(prediction,axis=1 )<train_model>
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=2018 ).split(train_X, train_y))
class StackNet(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models,meta_final_model, meta_models1=None, meta_models2=None,add_prev_out=True, n_folds=10):
self.base_models = base_models
self.meta_models1 = meta_models1
self.meta_models2 = meta_models2
self.meta_final_model=meta_final_model
self.n_folds = n_folds
self.add_prev_out=add_prev_out
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
if self.meta_models1!=None:
self.meta_models1_ = [list() for x in self.meta_models1]
if self.meta_models2!=None:
self.meta_models2_ = [list() for x in self.meta_models2]
self.meta_final_model_=clone(self.meta_final_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions=np.hstack(( out_of_fold_predictions,X))
out_of_fold_predictions2 = np.zeros(( X.shape[0],len(self.meta_models1)))
 if self.meta_models1!=None:
for i, model in enumerate(self.meta_models1):
for train_index, holdout_index in kfold.split(out_of_fold_predictions, y):
instance = clone(model)
 self.meta_models1_[i].append(instance)
 instance.fit(out_of_fold_predictions[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions[holdout_index])
 out_of_fold_predictions2[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions2=np.hstack(( out_of_fold_predictions2,X))
else:
out_of_fold_predictions2=out_of_fold_predictions
out_of_fold_predictions3 = np.zeros(( X.shape[0],len(self.meta_models2)))
 if self.meta_models2!=None:
for i, model in enumerate(self.meta_models2):
for train_index, holdout_index in kfold.split(out_of_fold_predictions2, y):
instance = clone(model)
 self.meta_models2_[i].append(instance)
 instance.fit(out_of_fold_predictions2[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions2[holdout_index])
 out_of_fold_predictions3[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions3=np.hstack(( out_of_fold_predictions3,X))
else:
out_of_fold_predictions3=out_of_fold_predictions2
self.meta_final_model_.fit(out_of_fold_predictions3, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_])
 if self.add_prev_out:
meta_features=np.hstack(( meta_features,X))
if self.meta_models1!=None:
meta_features2 = np.column_stack([
np.column_stack([model.predict(meta_features)for model in meta_models1_] ).mean(axis=1)
 for meta_models1_ in self.meta_models1_ ])
 if self.add_prev_out:
meta_features2=np.hstack(( meta_features2,X))
else:
meta_features2=meta_features
if self.meta_models2!=None:
meta_features3 = np.column_stack([
np.column_stack([model.predict(meta_features2)for model in meta_models2_] ).mean(axis=1)
 for meta_models2_ in self.meta_models2_ ])
 if self.add_prev_out:
meta_features3=np.hstack(( meta_features3,X))
else:
meta_features3=meta_features2
return self.meta_final_model_.predict(meta_features3 )<define_search_space>
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 embedding_matrix = np.concatenate(( glove_embeddings, paragram_embeddings), axis=1)
 np.shape(embedding_matrix )
max_iter=1e7
l1_ratio=[x/100 for x in range(65,100)]
alphas=[y*(10**-x)for x in range(2,6)for y in range(1,9)]
C=[x for x in range(1,30,3)]
epsilon=[y*(10**-x)for x in range(2,6)for y in range(1,9)]
gamma=[y*(10**-x)for x in range(2,6)for y in range(1,9)]
xgb_1=xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 lgb_1=lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 enet=make_pipeline(RobustScaler() ,ElasticNet(alpha=0.0005, l1_ratio=.9))
lasso=make_pipeline(RobustScaler() ,Lasso(alpha =0.0005))
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )<compute_train_metric>
def train(n_epochs):
train_preds = np.zeros(( len(train_X)))
 test_preds = np.zeros(( len(test_X)))
 for i,(train_idx, valid_idx)in enumerate(splits):
train_dataset= QuraData(train_X[train_idx.astype(int)],train_y[train_idx.astype(int)],augument=False,training=True)
 valid_dataset= QuraData(train_X[valid_idx.astype(int)],train_y[valid_idx.astype(int)],augument=False,training=False)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 model=mergeNN(embedding_matrix=embedding_matrix ).cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean')
 step_size = 500
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 valid_preds_fold= np.zeros(( len(valid_idx)))
 test_preds_fold= np.zeros(( len(test_X)))
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train(True)
 avg_loss = 0
for ii,(x_batch,y_batch)in enumerate(train_loader):
x_batch=Variable(x_batch ).cuda()
y_batch=Variable(y_batch ).cuda()
y_pred = model(x_batch)
 scheduler.batch_step()
loss = loss_fn(y_pred, y_batch.float())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.train(False)
 model.eval()
avg_val_loss = 0.
 with torch.no_grad() :
for i,(x_batch, y_batch)in enumerate(valid_loader):
x_batch=Variable(x_batch ).cuda()
y_batch=Variable(y_batch ).cuda()
y_pred = model(x_batch)
 avg_val_loss += loss_fn(y_pred, y_batch.float() ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(epoch + 1, n_epochs, avg_loss, avg_val_loss,elapsed_time))
model.eval()
with torch.no_grad() :

 for i,(x_batch,)in enumerate(test_loader):
x_batch=Variable(x_batch ).cuda()
y_pred = model(x_batch)
 test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
test_preds+=test_preds_fold/len(splits)
 return train_preds,test_preds
w=rmsle_cv(enet)
 print(w.mean() ,w.std())
 w=rmsle_cv(lasso)
 print(w.mean() ,w.std())
 w=rmsle_cv(GBoost)
 print(w.mean() ,w.std())
 w=rmsle_cv(KRR)
 print(w.mean() ,w.std())
 w=rmsle_cv(am)
 print(w.mean() ,w.std())
 w=rmsle_cv(sav)
 print(w.mean() ,w.std())
 xgb_1.fit(X_train.values,y_train.values)
 w=rmsle(xgb_1.predict(X_test.values),y_test.values)
 print(w)
 lgb_1.fit(X_train.values,y_train.values)
 w=rmsle(lgb_1.predict(X_test.values),y_test.values)
 print(w)
 sn.fit(X_train.values,y_train.values)
 w=rmsle(sn.predict(X_test.values),y_test.values)
 print(w )<save_to_csv>
train_preds,test_preds=train(n_epochs=3)
xgb_1.fit(train,y)
 lgb_1.fit(train,y)
 sn.fit(train.values,y.values)
 predicted_prices=np.expm1(sn.predict(test.values)) *0.7+np.expm1(xgb_1.predict(test)) *0.00+np.expm1(lgb_1.predict(test)) *0.3
my_submission = pd.DataFrame({'Id': test_Id, 'SalePrice': predicted_prices})
 my_submission.to_csv('submission.csv', index=False)
<set_options>
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(train_y,(train_preds > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
warnings.filterwarnings('ignore')
 % matplotlib inline<load_from_csv>
<train_model><EOS> 
pred_test_y =(test_preds > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False)