Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<save_to_csv>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 smp = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')
preds_df = pd.DataFrame({'first': pred_results[:,0], 'second': pred_results[:,1], 'third': pred_results[:,2]})
 preds_df = preds_df.replace(reverse_dict)
 preds_df['words'] = preds_df['first'] + " " + preds_df['second'] + " " + preds_df['third']
sub = pd.read_csv(INPUT_DIR + 'sample_submission.csv', index_col=['key_id'])
 sub['word'] = preds_df.words.values
sub.to_csv('1class_per_label_proto.csv')
 sub.head()<set_options>
train_na =(train.isnull().sum() / len(train)) * 100
train_na = train_na.drop(train_na[train_na==0].index ).sort_values(ascending=False)
 train_na_df = pd.DataFrame({'Missing Ratio' :train_na})
 train_na_df
warnings.filterwarnings('ignore')
 np.random.seed(60794)
 random.seed(60794)
 %matplotlib inline
%config InlineBackend.figure_format = 'retina' 
<set_options>
test_na =(test.isnull().sum() / len(test)) * 100
test_na = test_na.drop(test_na[test_na==0].index ).sort_values(ascending=False)
 test_na_df = pd.DataFrame({'Missing Ratio' :test_na})
 test_na_df
torch.manual_seed(60794)
 torch.backends.cudnn.deterministic = True 
<define_variables>
train['PoolQC'] = train['PoolQC'].fillna('None')
 test['PoolQC'] = test['PoolQC'].fillna('None' )
__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
'resnet152']
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
}
def swish(x):
return torch.sigmoid(x)* x
def conv3x3(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 def conv1x1(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
 class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = nn.BatchNorm2d(planes)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
self.conv1 = conv1x1(inplanes, planes)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = conv3x3(planes, planes, stride)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = conv1x1(planes, planes * self.expansion)
 self.bn3 = nn.BatchNorm2d(planes * self.expansion)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = swish(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = swish(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = swish(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, layers, num_classes=1000):
super(ResNet, self ).__init__()
self.inplanes = 64
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
bias=False)
 self.bn1 = nn.BatchNorm2d(64)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 conv1x1(self.inplanes, planes * block.expansion, stride),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = swish(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return x
def resnet18(pretrained=False, **kwargs):

 model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
 if pretrained:
model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
return model<load_pretrained>
train["MiscFeature"] = train["MiscFeature"].fillna("None")
 test["MiscFeature"] = test["MiscFeature"].fillna("None" )
def save_checkpoint(checkpoint_path, model, optimizer):
state = {'state_dict': model.state_dict() ,
'optimizer' : optimizer.state_dict() }
torch.save(state, checkpoint_path)
 print('model saved to %s' % checkpoint_path)
 def load_checkpoint(checkpoint_path, model, optimizer):
state = torch.load(checkpoint_path)
 model.load_state_dict(state['state_dict'])
 optimizer.load_state_dict(state['optimizer'])
 print('model loaded from %s' % checkpoint_path )<categorify>
train["Alley"] = train["Alley"].fillna("None")
 test["Alley"] = test["Alley"].fillna("None" )
en_dict = {}
path = '.. /input/quickdraw-doodle-recognition/train_simplified/'
filenames = glob.glob(os.path.join(path, '*.csv'))
filenames = sorted(filenames)
 def encode_files() :

 counter = 0
for fn in filenames:
en_dict[fn[:-4].split('/')[-1].replace(' ', '_')] = counter
counter += 1
encode_files()
dec_dict = {v: k for k, v in en_dict.items() }
def decode_labels(label):
return dec_dict[label]
def get_label(nfile):

 return en_dict[nfile.replace(' ', '_')[:-4]]<load_from_csv>
train["Fence"] = train["Fence"].fillna("None")
 test["Fence"] = test["Fence"].fillna("None" )
class DoodlesDataset(Dataset):

 def __init__(self, csv_file, root_dir, mode='train', nrows=1000, skiprows=None, size=256, transform=None):

 self.root_dir = root_dir
file = os.path.join(self.root_dir, csv_file)
 self.size = size
self.mode = mode
self.doodle = pd.read_csv(file, usecols=['drawing'], nrows=nrows, skiprows=skiprows)
 self.transform = transform
if self.mode == 'train':
self.label = get_label(csv_file)
 @staticmethod
def _draw(raw_strokes, size=256, lw=6, time_color=True):
BASE_SIZE = 256
img = np.zeros(( BASE_SIZE, BASE_SIZE), np.uint8)
 for t, stroke in enumerate(raw_strokes):
for i in range(len(stroke[0])- 1):
color = 255 - min(t, 10)* 13 if time_color else 255
_ = cv2.line(img,(stroke[0][i], stroke[1][i]), 
(stroke[0][i + 1], stroke[1][i + 1]), color, lw)
 if size != BASE_SIZE:
return cv2.resize(img,(size, size))
else:
return img
def __len__(self):
return len(self.doodle)
 def __getitem__(self, idx):
raw_strokes = ast.literal_eval(self.doodle.drawing[idx])
 sample = self._draw(raw_strokes, size=self.size, lw=6, time_color=True)
 if self.transform:
sample = self.transform(sample)
 if self.mode == 'train':
return(sample[None]/255 ).astype('float32'), self.label
else:
return(sample[None]/255 ).astype('float32' )<create_dataframe>
train["FireplaceQu"] = train["FireplaceQu"].fillna("None")
 test["FireplaceQu"] = test["FireplaceQu"].fillna("None" )
SIZE = 224
select_nrows = 8000
doodle_all = ConcatDataset([DoodlesDataset(fn.split('/')[-1], path,
skiprows=list(range(1,78000)) ,
nrows=select_nrows, size=SIZE)for fn in filenames] )<load_pretrained>
mapper = train.groupby("Neighborhood" ).median() ['LotFrontage'].to_dict()
for k,v in mapper.items() :
train.loc[(train['LotFrontage'].isnull() == True)&(train['Neighborhood'] == k), 'LotFrontage'] = v
test.loc[(test['LotFrontage'].isnull() == True)&(test['Neighborhood'] == k), 'LotFrontage'] = v
print('Train set:', len(doodle_all))
loader = DataLoader(doodle_all, batch_size=128, shuffle=True, num_workers=0)
<train_model>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None' )
def validation(lossf, scoref):
model.eval()
loss, score = 0, 0
vlen = len(valid_loader)
 for x, y in valid_loader:
x, y = x.to(device), y.to(device)
 output = model(x)
 loss += lossf(output, y ).item()
score += scoref(output, y)[0].item()
model.train()
return loss/vlen, score/vlen<compute_test_metric>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
train[col] = train[col].fillna(0)
 test[col] = test[col].fillna(0 )
def accuracy(output, target, topk=(3,)) :

 with torch.no_grad() :
maxk = max(topk)
 batch_size = target.size(0)
 _, pred = output.topk(maxk, 1, True, True)
 pred = pred.t()
correct = pred.eq(target.view(1, -1 ).expand_as(pred))
res = []
for k in topk:
correct_k = correct[:k].view(-1 ).float().sum(0, keepdim=True)
 res.append(correct_k.mul_(100.0 / batch_size))
return res<categorify>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
train[col] = train[col].fillna(0)
 test[col] = test[col].fillna(0 )
def mapk(output, target, k=3):

 with torch.no_grad() :
batch_size = target.size(0)
 _, pred = output.topk(k, 1, True, True)
 pred = pred.t()
correct = pred.eq(target.view(1, -1 ).expand_as(pred))
for i in range(k):
correct[i] = correct[i]*(k-i)
 score = correct[:k].view(-1 ).float().sum(0, keepdim=True)
 score.mul_(1.0 /(k * batch_size))
return score<choose_model_class>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
train[col] = train[col].fillna('None')
 test[col] = test[col].fillna('None' )
model = resnet18(pretrained=True)
<choose_model_class>
train["MasVnrType"] = train["MasVnrType"].fillna("None")
 test["MasVnrType"] = test["MasVnrType"].fillna("None")
 train["MasVnrArea"] = train["MasVnrArea"].fillna(0)
 test["MasVnrArea"] = test["MasVnrArea"].fillna(0 )
def squeeze_weights(m):
m.weight.data = m.weight.data.sum(dim=1)[:,None]
m.in_channels = 1
model.conv1.apply(squeeze_weights);
num_classes = 340
model.fc = nn.Linear(in_features=512, out_features=num_classes, bias=True )<define_search_space>
train['MSZoning'] = train['MSZoning'].fillna(train['MSZoning'].mode().item())
 test['MSZoning'] = test['MSZoning'].fillna(test['MSZoning'].mode().item() )
%%time
model(torch.randn(12,1,224,224)).size()<train_model>
train.drop('Utilities',axis=1,inplace=True)
 test.drop('Utilities',axis=1,inplace=True )
device = 'cuda'
model.to(device);<choose_model_class>
train["Functional"] = train["Functional"].fillna("Typ")
 test["Functional"] = test["Functional"].fillna("Typ" )
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=0.001, amsgrad=True )<load_pretrained>
train['Electrical'] = train['Electrical'].fillna(train['Electrical'].mode() [0])
 test['Electrical'] = test['Electrical'].fillna(test['Electrical'].mode() [0] )
load_checkpoint(".. /input/fold-two-doodler5/doodle-fold-two-2.pth", model, optimizer )<choose_model_class>
train['KitchenQual'] = train['KitchenQual'].fillna(train['KitchenQual'].mode() [0])
 test['KitchenQual'] = test['KitchenQual'].fillna(test['KitchenQual'].mode() [0] )
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5000,12000,18000], gamma=0.5 )<train_on_grid>
for col in ['Exterior1st','Exterior2nd']:
train[col] = train[col].fillna(train[col].mode() [0])
 test[col] = test[col].fillna(test[col].mode() [0] )
%%time
epochs = 1
lsize = len(loader)
 itr = 1
p_itr = 1000
model.train()
tloss, score = 0, 0
for epoch in range(epochs):
for x, y in tqdm.tqdm(loader):
x, y = x.to(device), y.to(device)
 optimizer.zero_grad()
output = model(x)
 loss = criterion(output, y)
 loss.backward()
optimizer.step()
tloss += loss.item()
score += mapk(output, y)[0].item()
scheduler.step()
if itr%p_itr==0:
print('Iteration {} -> Train Loss: {:.4f}, MAP@3: {:.3f}'.format(itr, tloss/p_itr, score/p_itr))
tloss, score = 0, 0
itr +=1
itr = itr%p_itr
print('Final -> Loss: {:.4f}, MAP@3: {:.3f}'.format(loss/itr, score/itr))
save_checkpoint('doodle-fold-two-%i.pth' % epochs, model, optimizer )<load_from_csv>
train['SaleType'] = train['SaleType'].fillna(train['SaleType'].mode() [0])
 test['SaleType'] = test['SaleType'].fillna(test['SaleType'].mode() [0] )
testset = DoodlesDataset('test_simplified.csv', '.. /input/quickdraw-doodle-recognition', mode='test', nrows=None, size=SIZE)
 testloader = DataLoader(testset, batch_size=128, shuffle=False, num_workers=0 )<find_best_params>
train['MSSubClass'] = train['MSSubClass'].fillna("None")
 test['MSSubClass'] = test['MSSubClass'].fillna("None" )
model.eval()
labels = np.empty(( 0,3))
for x in tqdm.tqdm(testloader):
x = x.to(device)
 output = model(x)
 _, pred = output.topk(3, 1, True, True)
 labels = np.concatenate([labels, pred], axis = 0 )<load_from_csv>
train_na =(train.isnull().sum() / len(train)) * 100
train_na = train_na.drop(train_na[train_na==0].index ).sort_values(ascending=False)
 train_na_df = pd.DataFrame({'Missing Ratio' :train_na})
 train_na_df
%%time
submission = pd.read_csv('.. /input/quickdraw-doodle-recognition/test_simplified.csv', index_col='key_id')
 submission.drop(['countrycode', 'drawing'], axis=1, inplace=True)
 submission['word'] = ''
for i, label in enumerate(labels):
submission.word.iloc[i] = " ".join([decode_labels(l)for l in label] )<save_to_csv>
test_na =(test.isnull().sum() / len(test)) * 100
test_na = test_na.drop(test_na[test_na==0].index ).sort_values(ascending=False)
 test_na_df = pd.DataFrame({'Missing Ratio' :test_na})
 test_na_df
submission.to_csv('preds_resnet18_foldone.csv' )<load_from_csv>
train['MSSubClass'] = train['MSSubClass'].apply(str)
 test['MSSubClass'] = test['MSSubClass'].apply(str)
 train['OverallCond'] = train['OverallCond'].astype(str)
 test['OverallCond'] = test['OverallCond'].astype(str)
 train['YrSold'] = train['YrSold'].astype(str)
 train['MoSold'] = train['MoSold'].astype(str)
 test['YrSold'] = test['YrSold'].astype(str)
 test['MoSold'] = test['MoSold'].astype(str )
df_train = pd.read_csv('.. /input/train.csv')
 df_train.head()<save_to_csv>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(set(train[c].unique().tolist() + test[c].unique().tolist())))
 train[c] = lbl.transform(list(train[c].values))
test[c] = lbl.transform(list(test[c].values))
p = df_train['is_duplicate'].mean()
print('Predicted score:', log_loss(df_train['is_duplicate'], np.zeros_like(df_train['is_duplicate'])+ p))
df_test = pd.read_csv('.. /input/test.csv')
 sub = pd.DataFrame({'test_id': df_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index=False)
 sub.head()<load_from_csv>
train['TotalSF'] = train['TotalBsmtSF'] + train['1stFlrSF'] + train['2ndFlrSF']
test['TotalSF'] = test['TotalBsmtSF'] + test['1stFlrSF'] + test['2ndFlrSF']
df_test = pd.read_csv('.. /input/test.csv')
 df_test.head()<define_variables>
y_train = train.SalePrice
train.drop(['SalePrice','Id'],axis=1,inplace=True)
 test_Ids = test['Id']
test.drop('Id',axis=1,inplace=True )
print('Total number of question pairs for testing: {}'.format(len(df_test)) )<feature_engineering>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features in train data to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
train[feat] = boxcox1p(train[feat], lam )
qmarks = np.mean(train_qs.apply(lambda x: '?' in x))
math = np.mean(train_qs.apply(lambda x: '[math]' in x))
fullstop = np.mean(train_qs.apply(lambda x: '.' in x))
capital_first = np.mean(train_qs.apply(lambda x: x[0].isupper()))
capitals = np.mean(train_qs.apply(lambda x: max([y.isupper() for y in x])))
 numbers = np.mean(train_qs.apply(lambda x: max([y.isdigit() for y in x])))
 print('Questions with question marks: {:.2f}%'.format(qmarks * 100))
print('Questions with [math] tags: {:.2f}%'.format(math * 100))
print('Questions with full stops: {:.2f}%'.format(fullstop * 100))
print('Questions with capitalised first letters: {:.2f}%'.format(capital_first * 100))
print('Questions with capital letters: {:.2f}%'.format(capitals * 100))
print('Questions with numbers: {:.2f}%'.format(numbers * 100))<string_transform>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features in test data to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
test[feat] = boxcox1p(test[feat], lam )
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
plt.figure(figsize=(15, 5))
train_word_match = df_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[df_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[df_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate')
 plt.legend()
plt.title('Label distribution over word_match_share', fontsize=15)
 plt.xlabel('word_match_share', fontsize=15 )<init_hyperparams>
train = pd.get_dummies(train)
 test = pd.get_dummies(test)
 missing_cols = set(train.columns)- set(test.columns)
 for c in missing_cols:
test[c] = 0
missing_cols = set(test.columns)- set(train.columns)
 for c in missing_cols:
train[c] = 0
test = test[train.columns.tolist() ]
train.shape,test.shape
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<sort_values>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
print('Most common words and weights:
')
 print(sorted(weights.items() , key=lambda x: x[1] if x[1] > 0 else 9999)[:10])
 print('
Least common words and weights: ')
(sorted(weights.items() , key=lambda x: x[1], reverse=True)[:10] )<feature_engineering>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<compute_test_metric>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
print('Original AUC:', roc_auc_score(df_train['is_duplicate'], train_word_match))
print(' TFIDF AUC:', roc_auc_score(df_train['is_duplicate'], tfidf_train_word_match.fillna(0)) )<prepare_x_and_y>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = df_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = df_test.apply(tfidf_word_match_share, axis=1, raw=True)
 y_train = df_train['is_duplicate'].values<concatenate>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
pos_train = x_train[y_train == 1]
neg_train = x_train[y_train == 0]
p = 0.165
scale =(( len(pos_train)/(len(pos_train)+ len(neg_train)))/ p)- 1
while scale > 1:
neg_train = pd.concat([neg_train, neg_train])
 scale -=1
neg_train = pd.concat([neg_train, neg_train[:int(scale * len(neg_train)) ]])
 print(len(pos_train)/(len(pos_train)+ len(neg_train)))
 x_train = pd.concat([pos_train, neg_train])
 y_train =(np.zeros(len(pos_train)) + 1 ).tolist() + np.zeros(len(neg_train)).tolist()
del pos_train, neg_train<split>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train, test_size=0.2, random_state=4242 )<train_model>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, verbosity=0,
random_state =7, nthread = -1 )
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'logloss'
params['eta'] = 0.02
params['max_depth'] = 4
d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
bst = xgb.train(params, d_train, 400, watchlist, early_stopping_rounds=50, verbose_eval=10 )<save_to_csv>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )<load_from_csv>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
tr = pd.read_csv('.. /input/train.csv')
 te = pd.read_csv('.. /input/test.csv')
 SCALE = 0.3627<feature_engineering>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def word_match_share(x):

 stops = set(stopwords.words('english'))
q1 = x.question1.fillna(' ' ).str.lower().str.split()
q2 = x.question2.fillna(' ' ).str.lower().str.split()
q1 = q1.map(lambda l : set(l)- stops)
 q2 = q2.map(lambda l : set(l)- stops)
 q = pd.DataFrame({'q1':q1, 'q2':q2})
 q['len_inter'] = q.apply(lambda row : len(row['q1'] & row['q2']), axis=1)
 q['len_tot'] = q.q1.map(len)+ q.q2.map(len)
 return(2 * q.len_inter / q.len_tot ).fillna(0 )<categorify>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def bin_model(tr, te, bins=100, vpos=1, vss=3):

 tr['word_share'] = word_match_share(tr)
 tr['binned_share'] =(bins * tr.word_share ).round()
pos = tr.groupby('binned_share' ).is_duplicate.sum()
cts = tr.binned_share.value_counts()
te['word_share'] = word_match_share(te)
 te['binned_share'] =(bins * te.word_share ).round()
te_pos = te.binned_share.map(pos, na_action='ignore' ).fillna(0)
 te_cts = te.binned_share.map(cts, na_action='ignore' ).fillna(0)
 prob =(te_pos + vpos)/(te_cts + vss)
 odds = prob /(1 - prob)
 scaled_odds = SCALE * odds
scaled_prob = scaled_odds /(1 + scaled_odds)
 sub = te[['test_id']].copy()
sub['is_duplicate'] = scaled_prob
return sub<save_to_csv>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
sub = bin_model(tr, te)
 sub.to_csv('no_ml_model.csv', index=False, float_format='%.6f')
 sub.head(10 )<load_from_csv>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print(check_output(["ls", ".. /input"] ).decode("utf8"))
print("This is just a test")
 quora_train = pd.read_csv(".. /input/train.csv")
 quora_test = pd.read_csv(".. /input/test.csv" )<count_unique_values>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Total number of question pairs for training: {}'.format(len(quora_train)))
 print('Duplicate pairs: {}%'.format(round(quora_train['is_duplicate'].mean() *100, 2)))
 qids = pd.Series(quora_train['qid1'].tolist() + quora_train['qid2'].tolist())
 print('Total number of questions in the training data: {}'.format(len(
 np.unique(qids))))
print('Number of questions that appear multiple times: {}'.format(np.sum(qids.value_counts() > 1)) )<save_to_csv>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
p = quora_train['is_duplicate'].mean()
print('Predicted score:', lgl(quora_train['is_duplicate'],
np.zeros_like(quora_train['is_duplicate'])+ p))
sub = pd.DataFrame({'test_id': quora_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index = False)
 print(sub.head() )<save_to_csv>
averaged_score = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_score)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
p = quora_train['is_duplicate'].mean()
print('Predicted score:', lgl(quora_train['is_duplicate'],
np.zeros_like(quora_train['is_duplicate'])+ p))
sub = pd.DataFrame({'test_id': quora_test['test_id'], 'is_duplicate': p})
 sub.to_csv('naive_submission.csv', index = False)
 print(sub.head() )<string_transform>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y.iloc[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
stops = set(stopwords.words("english"))
def word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_words_in_q1 = [w for w in q1words.keys() if w in q2words]
shared_words_in_q2 = [w for w in q2words.keys() if w in q1words]
R =(len(shared_words_in_q1)+ len(shared_words_in_q2)) /(len(q1words)+ len(q2words))
return R
print(stops)
 plt.figure(figsize=(15, 5))
train_word_match = quora_train.apply(word_match_share, axis=1, raw=True)
 plt.hist(train_word_match[quora_train['is_duplicate'] == 0], bins=20, normed=True, label='Not Duplicate')
 plt.hist(train_word_match[quora_train['is_duplicate'] == 1], bins=20, normed=True, alpha=0.7, label='Duplicate' )<data_type_conversions>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
train_qs = pd.Series(quora_train['question1'].tolist() + quora_train['question2'].tolist() ).astype(str)
 test_qs = pd.Series(quora_test['question1'].tolist() + quora_test['question2'].tolist() ).astype(str)
 dist_train = train_qs.apply(len)
 dist_test = test_qs.apply(len)
 print('ok' )<init_hyperparams>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def get_weight(count, eps=10000, min_count=2):
if count < min_count:
return 0
else:
return 1 /(count + eps)
 eps = 5000
train_qs.head(5)
 words =(" ".join(train_qs)).lower().split()
counts = Counter(words)
 weights = {word: get_weight(count)for word, count in counts.items() }<feature_engineering>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
def tfidf_word_match_share(row):
q1words = {}
q2words = {}
for word in str(row['question1'] ).lower().split() :
if word not in stops:
q1words[word] = 1
for word in str(row['question2'] ).lower().split() :
if word not in stops:
q2words[word] = 1
if len(q1words)== 0 or len(q2words)== 0:
return 0
shared_weights = [weights.get(w, 0)for w in q1words.keys() if w in q2words] + [weights.get(w, 0)for w in q2words.keys() if w in q1words]
total_weights = [weights.get(w, 0)for w in q1words] + [weights.get(w, 0)for w in q2words]
R = np.sum(shared_weights)/ np.sum(total_weights)
 return R<feature_engineering>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
x_train = pd.DataFrame()
x_test = pd.DataFrame()
x_train['word_match'] = train_word_match
x_train['tfidf_word_match'] = tfidf_train_word_match
x_test['word_match'] = quora_test.apply(word_match_share, axis=1, raw=True)
 x_test['tfidf_word_match'] = quora_test.apply(tfidf_word_match_share, axis=1, raw=True)
 x_train['gen2'] = np.sqrt(train_word_match)
 x_train['gen2_tfidf'] = np.sqrt(tfidf_train_word_match)
 x_test['gen2'] = np.sqrt(x_test['tfidf_word_match'])
 x_test['gen2_tfidf'] = np.sqrt(x_test['tfidf_word_match'])
 x_train['sin_w'] = np.sin(train_word_match)
 x_train['sin_tf'] = np.sin(tfidf_train_word_match)
 x_test['sin_w'] = np.sin(x_test['word_match'])
 x_test['sin_tf'] = np.sin(x_test['tfidf_word_match'])
 x_train['2w'] = train_word_match * train_word_match
x_train['2tf'] = tfidf_train_word_match * tfidf_train_word_match
x_test['2w'] = x_test['word_match'] * x_test['word_match']
x_test['2tf'] = x_test['tfidf_word_match'] * x_test['tfidf_word_match']
y_train = quora_train['is_duplicate'].values
print('OK' )<load_from_csv>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
train_orig = pd.read_csv('.. /input/train.csv', header=0)
 test_orig = pd.read_csv('.. /input/test.csv', header=0)
 tic0=timeit.default_timer()
df1 = train_orig[['question1']].copy()
df2 = train_orig[['question2']].copy()
df1_test = test_orig[['question1']].copy()
df2_test = test_orig[['question2']].copy()
df2.rename(columns = {'question2':'question1'},inplace=True)
 df2_test.rename(columns = {'question2':'question1'},inplace=True)
 train_questions = df1.append(df2)
 train_questions = train_questions.append(df1_test)
 train_questions = train_questions.append(df2_test)
 train_questions.info()
train_questions.drop_duplicates(subset = ['question1'],inplace=True)
 train_questions.reset_index(inplace=True,drop=True)
 questions_dict = pd.Series(train_questions.index.values,index=train_questions.question1.values ).to_dict()
train_cp = train_orig.copy()
test_cp = test_orig.copy()
train_cp.drop(['qid1','qid2'],axis=1,inplace=True)
 test_cp['is_duplicate'] = -1
test_cp.rename(columns={'test_id':'id'},inplace=True)
 comb = pd.concat([train_cp,test_cp])
 comb['q1_hash'] = comb['question1'].map(questions_dict)
 comb['q2_hash'] = comb['question2'].map(questions_dict)
 print('ok' )<count_values>
print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
q1_vc = comb.q1_hash.value_counts().to_dict()
q2_vc = comb.q2_hash.value_counts().to_dict()
def try_apply_dict(x,dict_to_apply):
try:
return dict_to_apply[x]
except KeyError:
return 0
comb['q1_freq'] = comb['q1_hash'].map(lambda x: try_apply_dict(x,q1_vc)+ try_apply_dict(x,q2_vc))
comb['q2_freq'] = comb['q2_hash'].map(lambda x: try_apply_dict(x,q1_vc)+ try_apply_dict(x,q2_vc))
print('test')
 train_comb = comb[comb['is_duplicate'] >= 0][['id','q1_hash','q2_hash','q1_freq','q2_freq','is_duplicate']]
test_comb = comb[comb['is_duplicate'] < 0][['id','q1_hash','q2_hash','q1_freq','q2_freq']]
train_comb.drop(['is_duplicate'],axis=1,inplace=True)
 print('ok' )<prepare_x_and_y>
Predictions = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
print('ok' )<split>
subm = pd.DataFrame()
subm['Id'] = test_Ids
subm['SalePrice'] = Predictions
subm.to_csv('submission.csv',index=False )