Kaggle competition: Titanic - Machine Learning from Disaster
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
warnings.filterwarnings('ignore')
 %matplotlib inline
%config InlineBackend.figure_format = 'retina'
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x
class EnsembleClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model1 = VisionTransformer.from_name('ViT-B_16', num_classes=5)
 self.model1.load_state_dict(torch.load('.. /input/vit-model-1/ViT-B_16.pt'))
self.model2 = CassvaImgClassifier(model_arch, n_class, pretrained)
 def forward(self, x):
x1 = self.model1(x)
 x2 = self.model2(x)
 return 0.6 * x1 + 0.4 * x2
def load(self, state_dict):
self.model2.load_state_dict(state_dict )<split>
gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train = pd.read_csv(".. /input/titanic/train.csv" )
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx,:].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = EnsembleClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 val_preds = []
tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load(torch.load('.. /input/fork-pytorch-efficientnet-baseline-train-amp-a/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
val_preds = np.mean(val_preds, axis=0)
 tst_preds = np.mean(tst_preds, axis=0)
 print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))
 print('fold {} validation accuracy = {:.5f}'.format(fold,(valid_.label.values==np.argmax(val_preds, axis=1)).mean()))
del model
torch.cuda.empty_cache()<feature_engineering>
passengerid = test.PassengerId
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()<save_to_csv>
train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test.to_csv('submission.csv', index=False )<import_modules>
train[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.models import Sequential
from keras.layers import Conv2D, Lambda, MaxPooling2D
from keras.layers import Dense, Dropout, Flatten
from keras.layers.normalization import BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.utils.np_utils import to_categorical 
<load_from_csv>
train[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
<train_model>
all_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
print(f"Training data size is {train.shape}
Testing data size is {test.shape}" )<prepare_x_and_y>
train[['Survived','Fare']].groupby(['Survived'], as_index=False ).mean().sort_values(by='Fare', ascending=False )
X = train.drop(['label'], 1 ).values
y = train['label'].values
test_x = test.values<feature_engineering>
train[['Survived','Age']].groupby(['Survived'], as_index=False ).mean().sort_values(by='Age', ascending=False )
X = X / 255.0
test_x = test_x / 255.0<split>
plt.imshow(train.corr() , cmap='hot', interpolation='nearest')
 plt.show()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0 )<normalization>
def missing_percentage(df):
total = df.isnull().sum().sort_values(ascending = False)
 percent = round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)
 return pd.concat([total, percent], axis=1, keys=['Total','Percent'] )
mean = np.mean(X_train)
 std = np.std(X_train)
 def standardize(x):
return(x-mean)/std<define_variables>
missing_percentage(train )
epochs = 50
batch_size = 64<choose_model_class>
missing_percentage(test )
model=Sequential()
model.add(Conv2D(filters=64, kernel_size =(3,3), activation="relu", input_shape=(28,28,1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=256, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(BatchNormalization())
 model.add(Dense(512,activation="relu"))
model.add(Dense(10,activation="softmax"))
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"] )<init_hyperparams>
def percent_value_counts(df, feature):
percent = pd.DataFrame(round(df.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2))
total = pd.DataFrame(df.loc[:,feature].value_counts(dropna=False))
total.columns = ['Total']
percent.columns = ['Percent']
return pd.concat([total, percent], axis=1 )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 train_gen = datagen.flow(X_train, y_train, batch_size=batch_size)
 test_gen = datagen.flow(X_test, y_test, batch_size=batch_size )<train_model>
percent_value_counts(train, 'Embarked' )
history = model.fit_generator(train_gen,
epochs = epochs,
steps_per_epoch = X_train.shape[0] // batch_size,
validation_data = test_gen,
validation_steps = X_test.shape[0] // batch_size )<predict_on_test>
train.loc[train['Embarked'].isnull() ,:]
pred = model.predict_classes(test_x, verbose=1 )<save_to_csv>
train["Embarked"].fillna('C',inplace=True )
sub['Label'] = pred
sub.to_csv("sub.csv", index=False)
 sub.head()<import_modules>
test.loc[test['Fare'].isnull() ,:]
import time
import torch
import matplotlib.pyplot as plt
from torch import nn, optim
from torchvision import transforms
from torch.utils.data import TensorDataset, DataLoader<init_hyperparams>
missing_value = test.loc[(test['Pclass']==3)&(test['Embarked']=='S')&(test['Sex']=="male"),'Fare'].mean()
test['Fare'].fillna(missing_value, inplace=True )
epochs = 50
batch_size = 32
lr = 1e-3
momentum = 0.9
weight_decay = 1e-4
train_ratio = 0.8
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<load_from_csv>
train = train.drop('Cabin',axis=1)
 test = test.drop("Cabin", axis=1 )
train_df = np.array(pd.read_csv('/kaggle/input/digit-recognizer/train.csv'))
imgs = train_df[:,1:].reshape(-1, 28, 28)
 labels = train_df[:,0]
imgs = np.expand_dims(imgs, axis=1)/ 255.0
num_train = int(imgs.shape[0] * train_ratio)
 train_imgs = imgs[:num_train]
train_labels = labels[:num_train]
val_imgs = imgs[num_train:]
val_labels = labels[num_train:]
train_dataset = TensorDataset(torch.tensor(train_imgs, dtype=torch.float), torch.tensor(train_labels))
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, pin_memory=True)
 val_dataset = TensorDataset(torch.tensor(val_imgs, dtype=torch.float), torch.tensor(val_labels))
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, pin_memory=True)
<define_search_model>
missing_percentage(train )
def conv_cell(cin, cout, k, s, p=None):
if not p:
p =(k-1)// 2
cell = nn.Sequential()
cell.add_module('conv', nn.Conv2d(in_channels=cin, out_channels=cout, kernel_size=k, stride=s, padding=p, bias=False))
cell.add_module('bn', nn.BatchNorm2d(cout))
cell.add_module('act', nn.ReLU())
 return cell
net = nn.Sequential(
 conv_cell(1, 32, 3, 2),
conv_cell(32, 32, 3, 1),
conv_cell(32, 32, 3, 1),
conv_cell(32, 64, 3, 2),
conv_cell(64, 64, 3, 1),
conv_cell(64, 64, 3, 1),
nn.AdaptiveAvgPool2d(1),
nn.Flatten() ,
nn.Linear(64, 10)
)
 print(net)
 net = net.to(device )<choose_model_class>
missing_percentage(test )
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters() , lr=lr, momentum=momentum, weight_decay=weight_decay )<categorify>
all_data['Title'] = all_data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(all_data['Title'], all_data['Sex'] )
def accuracy(dataloader, net):
total = 0
correct = 0
with torch.no_grad() :
for data in dataloader:
inputs, labels = data[0].to(device), data[1].to(device)
 outputs = net(inputs)
 _, predicted = torch.max(outputs.data, 1)
 total += labels.size(0)
 correct +=(predicted == labels ).sum().item()
return correct/total<find_best_params>
all_data['Title'] = all_data['Title'].replace(['Lady', 'Countess','Capt', 'Col',
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer','Dona'], 'Rare')
 all_data['Title'] = all_data['Title'].replace('Mlle', 'Miss')
 all_data['Title'] = all_data['Title'].replace('Ms', 'Miss')
 all_data['Title'] = all_data['Title'].replace('Mme', 'Mrs')
 all_data[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
best = 0
train_losses = []
train_accuracies = []
val_accuracies = []
for epoch in range(epochs):
tic = time.time()
epoch_loss = 0
for i, data in enumerate(train_loader):
inputs, labels = data[0].to(device), data[1].to(device)
 optimizer.zero_grad()
outputs = net(inputs)
 loss = loss_fn(outputs, labels)
 loss.backward()
optimizer.step()
epoch_loss += loss.item()
if i % 100 == 99:
print("[Epoch {}/{} Batch {}] lr: {}, loss: {:.4f}".format(
 epoch+1, epochs, i+1, optimizer.param_groups[0]['lr'], epoch_loss/(i+1)))
 toc = time.time()
train_acc = accuracy(train_loader, net)
 val_acc = accuracy(val_loader, net)
 print("Epoch {}, time: {:.2f}s, train accuracy: {:.4f}, val accuracy: {:.4f}".format(
 epoch+1, toc-tic, train_acc, val_acc))
print('-'*30)
 train_losses.append(epoch_loss/(i+1))
train_accuracies.append(train_acc)
 val_accuracies.append(val_acc)
 if val_acc > best:
best = val_acc
torch.save(net.state_dict() , 'mnist_best.pt')
 torch.save(net.state_dict() , 'mnist_last.pt')
 print("Training complete.Best validation accuracy:", best )<load_from_csv>
all_data = all_data.drop(['Name'], axis=1)
 all_data.head()
test_df = np.array(pd.read_csv('/kaggle/input/digit-recognizer/test.csv'))
test_imgs = test_df.reshape(-1, 28, 28)
 test_imgs = np.expand_dims(test_imgs, axis=1)/ 255.0
test_dataset = TensorDataset(torch.tensor(test_imgs, dtype=torch.float))
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 net.load_state_dict(torch.load('mnist_best.pt'))
net.eval()
test_results = []
for data in test_loader:
inputs = data[0].to(device)
 outputs = net(inputs)
 _, predicted = torch.max(outputs.data, axis=1)
 test_results.extend(predicted.tolist() )<save_to_csv>
all_data[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
num_test = len(test_results)
 test_df = pd.DataFrame(np.vstack(( np.arange(1, num_test+1), np.array(test_results)) ).T, columns=['ImageId', 'Label'])
 test_df.to_csv('submission.csv', index=False )<load_from_csv>
all_data['family_size'] = all_data.SibSp + all_data.Parch+1
def load_train_data(split=.0):
	with open('.. /input/digit-recognizer/train.csv')as file:
		data = np.asarray([x.replace('
', '' ).split(',')for x in file.readlines() [1:]])
 	train_data = data.astype('float32')[:, 1:].reshape(( len(data), 28, 28, 1))
	train_labels = to_categorical(data[:, 0])
 	
	indices = np.arange(len(data))
	np.random.shuffle(indices)
 	train_data = train_data[indices]
	train_labels = train_labels[indices]
	
	if split >.0:
		stop = int(len(data)* split)
 		val_data = train_data[:stop]
		val_labels = train_labels[:stop]
		train_data = train_data[stop:]
		train_labels = train_labels[stop:]
		return(train_data, train_labels),(val_data, val_labels)
 	else:
		return(train_data, train_labels)
 def load_test_data() :
	with open('.. /input/digit-recognizer/test.csv')as file:
		data = np.asarray([x.replace('
', '' ).split(',')for x in file.readlines() [1:]])
 	test_data = data.astype('float32' ).reshape(( data.shape[0], 28, 28, 1)) / 255
	return test_data
def get_model() :
	model = models.Sequential()
	model.add(layers.Conv2D(32,(3, 3), use_bias=False, padding='Same', input_shape=(28, 28, 1)))
 	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.Conv2D(32,(3, 3), use_bias=False, padding='Same'))
	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.MaxPooling2D(( 2, 2)))
 	model.add(layers.Dropout (.4))
	model.add(layers.Conv2D(64,(3, 3), use_bias=False, padding='Same'))
	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.Conv2D(64,(3, 3), use_bias=False, padding='Same'))
	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.MaxPooling2D(( 2, 2)))
 	model.add(layers.Dropout (.4))
	model.add(layers.Conv2D(128,(3, 3), use_bias=False, padding='Same'))
	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.Conv2D(128,(3, 3), use_bias=False, padding='Same'))
	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.Conv2D(128,(3, 3), use_bias=False, padding='Same'))
	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.MaxPooling2D(( 2, 2)))
 	model.add(layers.Dropout (.4))
	model.add(layers.Flatten())
 	model.add(layers.Dense(512, use_bias=False))
	model.add(layers.BatchNormalization())
 	model.add(layers.Activation('relu'))
	model.add(layers.Dropout (.4))
	model.add(layers.Dense(10, activation='softmax'))
	model.compile(optimizer='adam',
			 loss='categorical_crossentropy',
			 metrics=['accuracy'])
 	return model
def plot_losses(history):
	train_loss = history.history.get('loss')
 	val_loss = history.history.get('val_loss', None)
 	train_acc = history.history.get('accuracy')
 	val_acc = history.history.get('val_accuracy', None)
 	epochs = np.arange(len(train_loss)) + 1
	plt.figure(figsize=(10, 4))
	plt.subplot(121)
 	plt.plot(epochs, train_loss, color='blue', label='Training loss')
 	if val_loss:
		plt.plot(epochs, val_loss, color='red', label='Validation loss')
 	plt.xlabel('Epochs')
 	plt.ylabel('Loss')
 	plt.legend()
	plt.grid()
	plt.subplot(122)
 	plt.plot(epochs, train_acc, color='blue', label='Training acc')
 	if val_acc:
		plt.plot(epochs, val_acc, color='red', label='Validation acc')
 	plt.xlabel('Epochs')
 	plt.ylabel('Accuracy')
 	plt.legend()
	plt.grid()
	plt.show()
def confirm_submission(predict):
	data = [f'{n + 1},{x}
' for n, x in enumerate(predict)]
	with open('submission.csv', 'w')as file:
		file.write('ImageID,Label
')
 		for line in data:
			file.write(line)
<train_model>
all_data[['family_size', 'Survived']].groupby(['family_size'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
( train_data, train_labels),(val_data, val_labels)= load_train_data(split=.2)
 batch_size = 64
train_gen = ImageDataGenerator(
 	rescale=1./255,
	rotation_range=10,
	zoom_range=.1,
	width_shift_range=.1,
	height_shift_range=.1,
	fill_mode='nearest' ).flow(
 		train_data, train_labels,
		batch_size=batch_size)
 val_gen = ImageDataGenerator(
 	rescale=1./255 ).flow(
 		val_data, val_labels,
		batch_size=batch_size)
 model = get_model()
history = model.fit(train_gen,
	steps_per_epoch=len(train_data)// batch_size,
	validation_data=val_gen,
	validation_steps=len(val_data)// batch_size,
	epochs=100)
 plot_losses(history)
<train_model>
all_data['family_group'] = all_data['family_size'].map(family_group )
train_data, train_labels = load_train_data()
batch_size = 64
train_gen = ImageDataGenerator(
 	rescale=1./255,
	rotation_range=10,
	zoom_range=.1,
	width_shift_range=.1,
	height_shift_range=.1,
	fill_mode='nearest' ).flow(
 		train_data, train_labels,
		batch_size=batch_size)
 model = get_model()
history = model.fit(train_gen,
	steps_per_epoch=len(train_data)// batch_size,
	epochs=100)
 model.save('mnist.h5')
 plot_losses(history )<predict_on_test>
all_data = all_data.drop(['Parch', 'SibSp'], axis=1)
 all_data.head()
test_data = load_test_data()
model = models.load_model('mnist.h5')
 predict = np.argmax(model.predict(test_data), axis=1)
 confirm_submission(predict )<load_pretrained>
all_data['calculated_fare'] = all_data["Fare"]/all_data["family_size"]
all_data.head()
!apt-get install p7zip
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/train.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/test.tsv.7z
!p7zip -d -f -k /kaggle/input/mercari-price-suggestion-challenge/sample_submission.csv.7z<load_from_csv>
all_data = all_data.drop(['Fare', 'family_size'], axis=1 )
!unzip /kaggle/input/mercari-price-suggestion-challenge/sample_submission_stg2.csv.zip
!unzip /kaggle/input/mercari-price-suggestion-challenge/test_stg2.tsv.zip<set_options>
all_data.loc[all_data['calculated_fare'] <= 7.775, 'Fare_group'] = 1
all_data.loc[(all_data['calculated_fare'] > 7.775)&(all_data['calculated_fare'] <= 13), 'Fare_group'] =2
all_data.loc[ all_data['calculated_fare'] > 13, 'Fare_group'] = 3
all_data['Fare_group'] = all_data['Fare_group'].astype(int )
%matplotlib inline<load_from_csv>
all_data = all_data.drop(['FareBand', 'calculated_fare'], axis=1 )
train_df = pd.read_csv('train.tsv', sep='\t')
 train_df.shape<load_from_csv>
missing_percentage(all_data )
test_df = pd.read_csv('test_stg2.tsv', sep='\t')
 test_df.shape<feature_engineering>
def completing_age(df):
age_df = df.loc[:,:]
temp_train = age_df.loc[age_df.Age.notnull() ]
temp_test = age_df.loc[age_df.Age.isnull() ]
y = temp_train.Age.values
x = temp_train.loc[:,"family_group":].values
rfr = RandomForestRegressor(n_estimators=1500, n_jobs=-1)
 rfr.fit(x, y)
 predicted_age = rfr.predict(temp_test.loc[:,"family_group":])
 df.loc[df.Age.isnull() , "Age"] = predicted_age
return df
completing_age(all_data)
train_df['price'] = np.log1p(train_df['price'] )<count_values>
all_data['AgeBand'] = pd.cut(all_data['Age'], 5)
 all_data[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )
train_df['shipping'].value_counts()<count_values>
all_data.loc[ all_data['Age'] <= 16.136, 'Age_group'] = 1
all_data.loc[(all_data['Age'] > 16.136)&(all_data['Age'] <= 32.102), 'Age_group'] = 2
all_data.loc[(all_data['Age'] > 32.102)&(all_data['Age'] <= 48.068), 'Age_group'] = 3
all_data.loc[(all_data['Age'] > 48.068)&(all_data['Age'] <= 64.034), 'Age_group'] = 4
all_data.loc[ all_data['Age'] > 64.034, 'Age_group'] = 5
train_df['item_condition_id'].value_counts()<count_values>
all_data = all_data.drop(['Age', 'AgeBand'], axis=1)
 all_data.head()
train_df['item_description'].value_counts()<count_values>
all_data = pd.get_dummies(all_data, columns=['Title',"Pclass",'Embarked','Sex'], drop_first=False)
 all_data.head()
train_df[train_df['item_description'] == 'No description yet']['item_description'].count()<string_transform>
all_data['Age_group'] = all_data['Age_group'].astype(int)
 all_data.head()
def split_cat(category_name):
try:
return category_name.split('/')
 except:
return ['Other_Null', 'Other_Null', 'Other_Null']<feature_engineering>
X = train_df.drop(["Survived","PassengerId"], axis=1)
 Y = train_df["Survived"]
test = test_df.drop(["Survived","PassengerId"], axis=1 ).copy()
X.shape, Y.shape, test.shape
train_df['cat_dae'], train_df['cat_jung'], train_df['cat_so'] = zip(*train_df['category_name'].apply(lambda x: split_cat(x)))
 test_df['cat_dae'], test_df['cat_jung'], test_df['cat_so'] = zip(*test_df['category_name'].apply(lambda x: split_cat(x)) )<count_unique_values>
X_train, X_test, Y_train, Y_test = train_test_split(X, Y,test_size =.33, random_state=0 )
print(train_df['cat_dae'].nunique())
 print(train_df['cat_jung'].nunique())
 print(train_df['cat_so'].nunique() )<data_type_conversions>
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test)
 test= sc.transform(test )
train_df['brand_name'] = train_df['brand_name'].fillna(value='Other_Null')
 train_df['category_name'] = train_df['category_name'].fillna(value='Other_Null')
 train_df['item_description'] = train_df['item_description'].fillna(value='Other_Null')
 test_df['brand_name'] = test_df['brand_name'].fillna(value='Other_Null')
 test_df['category_name'] = test_df['category_name'].fillna(value='Other_Null')
 test_df['item_description'] = test_df['item_description'].fillna(value='Other_Null' )<count_missing_values>
X_train = pd.DataFrame(X_train, columns=headers)
 X_test = pd.DataFrame(X_test, columns=headers )
train_df.isnull().sum()<count_missing_values>
logreg = LogisticRegression(
 solver = 'liblinear',
penalty = 'l1' 
)
 logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_log
test_df.isnull().sum()<count_unique_values>
logreg = LogisticRegression()
parameters = {'penalty': ['l1','l2'],
'C': [0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,16.5,17,17.5,18]
}
grid_obj = GridSearchCV(logreg, parameters, scoring='accuracy')
 grid_obj = grid_obj.fit(X_train, Y_train)
 logreg = grid_obj.best_estimator_
logreg.fit(X_train, Y_train)
train_df['brand_name'].nunique()<count_values>
rfc_prediction = logreg.predict(X_test)
 rfc_score=accuracy_score(Y_test, rfc_prediction)
 print(rfc_score )
train_df['brand_name'].value_counts() [:5]<count_unique_values>
cv = StratifiedShuffleSplit(n_splits = 10, test_size =.25, random_state = 0)
 column_names = X.columns
X = sc.fit_transform(X)
 accuracies = cross_val_score(LogisticRegression(solver='liblinear'),X,Y, cv = cv)
 print("Cross-Validation accuracy scores:{}".format(accuracies))
print("Mean Cross-Validation accuracy score: {}".format(round(accuracies.mean() ,5)) )
train_df['name'].nunique()<count_values>
C_vals = [0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,16.5,17,17.5,18]
penalties = ['l1','l2']
cv = StratifiedShuffleSplit(n_splits = 10, test_size =.25)
 param = {'penalty': penalties, 'C': C_vals}
grid = GridSearchCV(estimator=LogisticRegression() ,
param_grid = param,
scoring = 'accuracy',
n_jobs =-1,
cv = cv 
)
 grid.fit(X, Y )
train_df['name'].value_counts() [:10]<feature_engineering>
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )
cnt_vec = CountVectorizer()
X_train_name = cnt_vec.fit_transform(train_df['name'])
 X_test_name = cnt_vec.transform(test_df['name'] )<prepare_x_and_y>
logreg_grid = grid.best_estimator_
logreg_grid.score(X,Y )
tfidf_descp = TfidfVectorizer(max_features=50000, ngram_range=(1, 3), stop_words='english')
 X_train_descp = tfidf_descp.fit_transform(train_df['item_description'])
 X_test_descp = tfidf_descp.transform(test_df['item_description'] )<categorify>
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_svc
lb_brand_name = LabelBinarizer(sparse_output=True)
 X_train_brand = lb_brand_name.fit_transform(train_df['brand_name'])
 X_test_brand = lb_brand_name.transform(test_df['brand_name'])
 lb_item_cond_id = LabelBinarizer(sparse_output=True)
 X_train_item_condition_id = lb_item_cond_id.fit_transform(train_df['item_condition_id'])
 X_test_item_condition_id = lb_item_cond_id.transform(test_df['item_condition_id'])
 lb_shipping = LabelBinarizer(sparse_output=True)
 X_train_shipping = lb_shipping.fit_transform(train_df['shipping'])
 X_test_shipping = lb_shipping.transform(test_df['shipping'])
 lb_cat_dae = LabelBinarizer(sparse_output=True)
 X_train_cat_dae = lb_cat_dae.fit_transform(train_df['cat_dae'])
 X_test_cat_dae = lb_cat_dae.transform(test_df['cat_dae'])
 lb_cat_jung = LabelBinarizer(sparse_output=True)
 X_train_cat_jung = lb_cat_jung.fit_transform(train_df['cat_jung'])
 X_test_cat_jung = lb_cat_jung.transform(test_df['cat_jung'])
 lb_cat_so = LabelBinarizer(sparse_output=True)
 X_train_cat_so = lb_cat_so.fit_transform(train_df['cat_so'])
 X_test_cat_so = lb_cat_so.transform(test_df['cat_so'] )<set_options>
Cs = [0.001, 0.01, 0.1, 1,1.5,2,2.5,3,4,5, 10]
gammas = [0.0001,0.001, 0.01, 0.1, 1]
param_grid = {'C': Cs, 'gamma' : gammas}
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 grid_search = GridSearchCV(SVC(kernel = 'rbf', probability=True), param_grid, cv=cv)
 grid_search.fit(X,Y )
sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 X_train = hstack(sparse_matrix_list ).tocsr()
print(type(X_train), X_train.shape)
 del X_train
gc.collect()<compute_test_metric>
print(grid_search.best_score_)
 print(grid_search.best_params_)
 print(grid_search.best_estimator_ )
def rmsle(y, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y)- np.log1p(y_pred), 2)))
 def evaluate_orig_price(y_test, preds):
preds_exmpm = np.expm1(preds)
 y_test_exmpm = np.expm1(y_test)
 return rmsle(y_test_exmpm, preds_exmpm )<find_best_model_class>
svc_grid = grid.best_estimator_
svc_grid.score(X,Y )
def model_train_predict(model, matrix_list):
X = hstack(matrix_list ).tocsr()
X_train, X_test, y_train, y_test = train_test_split(X, train_df['price'], test_size=0.2)
 model.fit(X_train, y_train)
 preds = model.predict(X_test)
 del X, X_train, X_test, y_train
gc.collect()
return preds, y_test<predict_on_test>
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_linear_svc
linear_model = Ridge(solver='lsqr', fit_intercept=False)
 sparse_matrix_list =(X_train_name, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 linear_preds, y_test = model_train_predict(model=linear_model, matrix_list=sparse_matrix_list)
 print('Item Description을 제외했을 때 rmsle 값:', evaluate_orig_price(y_test, linear_preds))
sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 linear_preds, y_test = model_train_predict(model=linear_model, matrix_list=sparse_matrix_list)
 print('Item Description을 포함한 rmsle 값:', evaluate_orig_price(y_test, linear_preds))<predict_on_test>
dectree = DecisionTreeClassifier()
dectree.fit(X_train, Y_train)
 Y_pred = dectree.predict(X_test)
 acc_dectree = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_dectree
lgbm_model = LGBMRegressor(n_estimators=200, learning_rate=0.5, num_leaves=125)
 sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 lgbm_preds, y_test = model_train_predict(model=lgbm_model, matrix_list=sparse_matrix_list)
 print('LightGBM rmsle 값:', evaluate_orig_price(y_test, lgbm_preds))<concatenate>
max_depth = range(1,20)
 criterion=["entropy", "gini"]
param = {'max_depth':max_depth,
'criterion': criterion}
grid = GridSearchCV(DecisionTreeClassifier() ,
param_grid = param,
verbose=False,
cv=StratifiedKFold(n_splits=20, random_state=15, shuffle=True),
n_jobs = -1)
 grid.fit(X,Y )
sparse_matrix_list =(X_train_name, X_train_descp, X_train_brand, X_train_item_condition_id, X_train_shipping, X_train_cat_dae, X_train_cat_jung, X_train_cat_so)
 X_train = hstack(sparse_matrix_list ).tocsr()
X_train<prepare_x_and_y>
print(grid.best_params_)
 print(grid.best_score_)
 print(grid.best_estimator_ )
y_train = train_df['price']
y_train<train_model>
dectree_grid = grid.best_estimator_
dectree_grid.score(X,Y )
lgbm_model = LGBMRegressor(n_estimators=200, learning_rate=0.5, num_leaves=125)
 lgbm_model.fit(X_train, y_train )<concatenate>
dot_data = StringIO()
export_graphviz(dectree_grid, out_file=dot_data,
feature_names=column_names, class_names =(["Survived" if int(i)is 1 else "Not_survived" for i in Y.unique() ]),
filled=True, rounded=True,
proportion=True,
special_characters=True)
(graph,)= pydot.graph_from_dot_data(dot_data.getvalue())
 Image(graph.create_png() )
sparse_matrix_list =(X_test_name, X_test_descp, X_test_brand, X_test_item_condition_id, X_test_shipping, X_test_cat_dae, X_test_cat_jung, X_test_cat_so)
 X_test = hstack(sparse_matrix_list ).tocsr()<predict_on_test>
rf = RandomForestClassifier(n_estimators=100)
 rf.fit(X_train, Y_train)
 Y_pred = rf.predict(X_test)
 acc_rf = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_rf
preds = lgbm_model.predict(X_test)
 preds<prepare_output>
n_estimators = [140,145,150,155,160];
max_depth = range(1,10);
criterions = ['gini', 'entropy'];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 parameters = {'n_estimators':n_estimators,
'max_depth':max_depth,
'criterion': criterions
}
grid = GridSearchCV(estimator=RandomForestClassifier(max_features='auto'),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 grid.fit(X,Y )
preds = np.expm1(preds)
 preds<load_from_csv>
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )
submission = pd.read_csv('sample_submission_stg2.csv')
 submission<prepare_output>
rf_grid = grid.best_estimator_
rf_grid.score(X,Y )
submission.loc[:, 'price'] = preds
submission<save_to_csv>
n_estimators = [10,30,50,70,80,150,160, 170,175,180,185];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 parameters = {'n_estimators':n_estimators,
}
grid = GridSearchCV(BaggingClassifier(base_estimator= None,
bootstrap_features=False),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 grid.fit(X,Y )
submission.to_csv('submission.csv', index=False )<save_to_csv>
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )
submission.to_csv('submission.csv', index=False )<load_from_csv>
bagging_grid = grid.best_estimator_
bagging_grid.score(X,Y )
data_train = pd.read_csv('.. /input/mercari-price-suggestion-challenge/train.tsv', delimiter='\t' )<split>
n_estimators = [100,140,145,150,160, 170,175,180,185];
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 learning_r = [0.1,1,0.01,0.5]
parameters = {'n_estimators':n_estimators,
'learning_rate':learning_r
}
grid = GridSearchCV(AdaBoostClassifier(base_estimator= None, 
),
param_grid=parameters,
cv=cv,
n_jobs = -1)
 grid.fit(X,Y )
y = np.log10(np.array(data_train['price'])+1)
 X = data_train.drop('price',axis=1)
 X_train,X_cv,Y_train,Y_cv = train_test_split(X, y, test_size=0.20, random_state=42)
 del(X, y ,data_train)
 gc.collect()<drop_column>
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )
X_train.drop('train_id', axis=1, inplace=True)
 X_cv.drop('train_id', axis=1, inplace=True )<compute_test_metric>
adaBoost_grid = grid.best_estimator_
adaBoost_grid.score(X,Y )
def rmsle(y, y0):
return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y0), 2)) )<define_variables>
gradient_boost = GradientBoostingClassifier()
gradient_boost.fit(X_train, Y_train)
 Y_pred = gradient_boost.predict(X_test)
 gradient_accy = round(accuracy_score(Y_pred, Y_test), 3)
 print(gradient_accy )
stopwords= ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've",\
"you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', \
'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their',\
'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', \
'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', \
'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', \
'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\
'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\
'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\
'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very', \
's', 't', 'can', 'will', 'just', 'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', \
've', 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn',\
"hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn',\
"mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't", \
'won', "won't", 'wouldn', "wouldn't"]<categorify>
XGBClassifier = XGBClassifier()
XGBClassifier.fit(X_train, Y_train)
 Y_pred = XGBClassifier.predict(X_test)
 XGBClassifier_accy = round(accuracy_score(Y_pred, Y_test), 3)
 print(XGBClassifier_accy )
stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 non_alphanumpunct = re.compile(u'[^A-Za-z0-9\.?!,; \(\)\[\]'"\$]+')
 RE_PUNCTUATION = '|'.join([re.escape(x)for x in string.punctuation])
 def concat_categories(x):
return set(x.values)
 def brandfinder(name, category):
for brand in brands_sorted_by_size:
if brand in name and category in brand_names_categories[brand]:
return brand
return 'Unknown'
def create_dictionary(col_name,data_frame= X_train):
dictionary = dict(zip(data_frame[col_name],data_frame.groupby(col_name)[col_name].transform('count')))
 return dictionary
def transform_col(data_frame, col_name):
dictionary = create_dictionary(col_name)
 transformed_column = []
for value in data_frame[col_name].values:
transformed_column.append(dictionary.get(value,1))
dictionary = None
del(dictionary)
 gc.collect()
return transformed_column
def clean_name(x):
if len(x):
x = non_alphanums.sub(' ', x ).split()
if len(x):
return x[0].lower()
return ''
def to_number(x):
try:
if not x.isdigit() :
return 0
x = int(x)
 if x > 100:
return 100
else:
return x
except:
return 0
def sum_numbers(desc):
if not isinstance(desc, str):
return 0
try:
return sum([to_number(s)for s in desc.split() ])
 except:
return 0
def normalize_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] \
if len(x)> 1 and x not in stopwords])
 def decontracted(phrase):
try:
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
except:
return 0
def cleaning_text(df):
preprocessed_item_description = []
for sentance in tqdm(df['item_description'].values):
sent = decontracted(str(sentance))
sent = sent.replace('\\r', ' ')
 sent = sent.replace('\"', ' ')
 sent = sent.replace('\
', ' ')
 sent = re.sub('[^A-Za-z0-9]+', ' ', sent)
 sent = ' '.join(e for e in sent.split() if e.lower() not in stopwords)
 preprocessed_item_description.append(sent.lower().strip())
 df = df.drop('item_description',axis=1)
 df['item_description'] = preprocessed_item_description
preprocessed_item_description = None
del(preprocessed_item_description)
 gc.collect()
return(df)
 def fill_brand(df):
brand_names_categories = dict(df[df['brand_name'] != 'missing'][['brand_name','category_name']].astype('str')\ 
.groupby('brand_name' ).agg(concat_categories ).reset_index().values.tolist())
 brands_sorted_by_size = list(sorted(filter(lambda y: len(y)>= 3, \
list(brand_names_categories.keys())) , \
key = lambda x: -len(x)))
 train_names_unknown_brands_train = df[df['brand_name'] == 'missing'][['name','category_name']].\
astype('str' ).values
train_estimated_brands_train = []
for name, category in tqdm(train_names_unknown_brands_train):
for brand in brands_sorted_by_size:
if brand in name and category in brand_names_categories[brand]:
brand_name = brand
else:
brand_name = 'missing'
train_estimated_brands_train.append(brand_name)
 df.loc[df['brand_name'] == 'missing', 'brand_name'] = train_estimated_brands_train
brand_names_categories = None
brands_sorted_by_size = None
train_names_unknown_brands_train = None
train_estimated_brands_train = None
brand_name= None
del(brand_names_categories,brands_sorted_by_size,train_names_unknown_brands_train,train_estimated_brands_train,brand_name)
 gc.collect()
return(df)
<data_type_conversions>
ExtraTreesClassifier = ExtraTreesClassifier()
ExtraTreesClassifier.fit(X_train, Y_train)
 Y_pred = ExtraTreesClassifier.predict(X_test)
 extraTree_accy = round(accuracy_score(Y_pred, Y_test), 3)
 print(extraTree_accy )
def preprocessing(df):
print("Text_cleaning Done")
 df['brand_name'] = df['brand_name'].fillna('missing')
 df['brand_name'] = df['brand_name'].astype('category')
 print("preprocessing for brand_name Done")
 df['category_name'] = df['category_name'].fillna('missing/missing/missing')
 df['category_name'] = df['category_name'].fillna('missing/missing/missing')
 print("preprocessing for category_name Done")
 df['item_description'] = df['item_description'].fillna('missing')
 print("preprocessing for item_description Done")
 df['category_name']= df['category_name'].str.split('/')
 df['main_category'] = df['category_name'].str.get(0 ).replace('', 'missing' ).astype('category')
 df['sub_category_1'] = df['category_name'].str.get(1 ).replace('', 'missing' ).astype('category')
 df['sub_category_2'] = df['category_name'].str.get(2 ).replace('', 'missing' ).astype('category')
 print("split main category in to 3 Done")
 df['item_condition_id'] = df['item_condition_id'].astype('category')
 df['item_condition_id'] = df['item_condition_id'].cat.add_categories(['missing'] ).fillna('missing')
 print("preprocessing for item_condition_id Done")
 df['shipping'] = df['shipping'].astype('category')
 df['shipping'] = df['shipping'].cat.add_categories(['missing'] ).fillna('missing')
 print("preprocessing for shipping Done")
 df['item_description'].fillna('missing', inplace=True)
 print("preprocessing for item_description Done")
 return(df)
 def adding_new_features(df):
df['name_first'] = df['name'].apply(clean_name)
 df['name_first_count'] = transform_col(data_frame = df, col_name = 'name_first')
 df['main_cat_count'] = transform_col(data_frame = df, col_name = 'main_category')
 df['sub_cat_1_count'] = transform_col(data_frame = df, col_name = 'sub_category_1')
 df['sub_cat_2_count'] = transform_col(data_frame = df, col_name = 'sub_category_2')
 df['brand_name_count'] = transform_col(data_frame = df, col_name = 'brand_name')
 df['DescriptionLower'] = df.item_description.str.count('[a-z]')
 df['NameLower'] = df.name.str.count('[a-z]')
 df['NameUpper'] = df.name.str.count('[A-Z]')
 df['DescriptionUpper'] = df.item_description.str.count('[A-Z]')
 df['name_len'] = df['name'].apply(lambda x: len(x))
df['des_len'] = df['item_description'].apply(lambda x: len(x))
df['name_desc_len_ratio'] = df['name_len']/df['des_len']
df['desc_word_count'] = df['item_description'].apply(lambda x: len(x.split()))
df['mean_des'] = df['item_description'].apply(lambda x: 0 if len(x)== 0 else float(len(x.split())) / len(x)) * 10
df['name_word_count'] = df['name'].apply(lambda x: len(x.split()))
df['mean_name'] = df['name'].apply(lambda x: 0 if len(x)== 0 else float(len(x.split())) / len(x)) * 10
df['desc_letters_per_word'] = df['des_len'] / df['desc_word_count']
df['name_letters_per_word'] = df['name_len'] / df['name_word_count']
df['NameLowerRatio'] = df['NameLower'] / df['name_len']
df['DescriptionLowerRatio'] = df['DescriptionLower'] / df['des_len']
df['NameUpperRatio'] = df['NameUpper'] / df['name_len']
df['DescriptionUpperRatio'] = df['DescriptionUpper'] / df['des_len']
df['NamePunctCount'] = df.name.str.count(RE_PUNCTUATION)
 df['DescriptionPunctCount'] = df.item_description.str.count(RE_PUNCTUATION)
 df['NamePunctCountRatio'] = df['NamePunctCount'] / df['name_word_count']
df['DescriptionPunctCountRatio'] = df['DescriptionPunctCount'] / df['desc_word_count']
df['NameDigitCount'] = df.name.str.count('[0-9]')
 df['DescriptionDigitCount'] = df.item_description.str.count('[0-9]')
 df['NameDigitCountRatio'] = df['NameDigitCount'] / df['name_word_count']
df['DescriptionDigitCountRatio'] = df['DescriptionDigitCount']/df['desc_word_count']
df['stopword_ratio_desc'] = df['item_description'].apply(lambda x: len([w for w in x.split() if w in stopwords])) / df['desc_word_count']
df['num_sum'] = df['item_description'].apply(sum_numbers)
 df['weird_characters_desc'] = df['item_description'].str.count(non_alphanumpunct)
 df['weird_characters_name'] = df['name'].str.count(non_alphanumpunct)
 df['prices_count'] = df['item_description'].str.count('[rm]')
 df['price_in_name'] = df['item_description'].str.contains('[rm]', regex=False ).astype('int')
 df.drop('category_name', axis=1, inplace=True)
 return(df )<feature_engineering>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn
preprocessing(X_train)
 adding_new_features(X_train )<count_missing_values>
knn_k = KNeighborsClassifier(metric='minkowski', p=2)
 cv = StratifiedShuffleSplit(n_splits=10, test_size=.25, random_state=2)
 accuracies = cross_val_score(knn_k, X,Y, cv = cv, scoring='accuracy')
 print("Cross-Validation accuracy scores:{}".format(accuracies))
print("Mean Cross-Validation accuracy score: {}".format(round(accuracies.mean() ,3)) )
X_train.columns[X_train.isna().any() ].tolist()<feature_engineering>
cv = StratifiedShuffleSplit(n_splits=10, test_size=.25, random_state=2)
 k_range = range(1,31)
 k_scores = []
for k in k_range:
knn_kk = KNeighborsClassifier(n_neighbors=k)
 scores = cross_val_score(knn_kk, X,Y, cv = cv, scoring = 'accuracy')
 k_scores.append(scores.mean())
 print("Accuracy scores are: {}
".format(k_scores))
print("Mean accuracy score: {}".format(np.mean(k_scores)) )
preprocessing(X_cv)
 adding_new_features(X_cv )<count_missing_values>
k_range = range(1,31)
 weights_options=['uniform','distance']
param = {'n_neighbors':k_range, 'weights':weights_options}
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30, random_state=15)
 grid = GridSearchCV(KNeighborsClassifier() , param,cv=cv,verbose = False, n_jobs=-1)
 grid.fit(X,Y )
X_cv.columns[X_cv.isna().any() ].tolist()<define_variables>
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )
total_cols = set(X_train.columns.values)
 basic_cols = {'name', 'item_condition_id', 'brand_name',
'shipping', 'item_description', 'main_category',
'sub_category_1', 'sub_category_2', 'name_first'}
numeric_cols = total_cols - basic_cols
cols_to_normalize = numeric_cols - {'price_in_name'}
text_cols = {'name', 'item_description'}
categorical_cols = {'item_condition_id','brand_name','shipping','main_category','sub_category_1','sub_category_2','name_first'}<train_model>
knn_grid= grid.best_estimator_
knn_grid.score(X,Y )
normalizer = Normalizer(copy=False)
 normalizer.fit(X_train[list(cols_to_normalize)] )<load_pretrained>
k_range = range(1,31)
 weights_options=['uniform','distance']
param = {'n_neighbors':k_range, 'weights':weights_options}
cv = StratifiedShuffleSplit(n_splits=10, test_size=.30)
 grid = RandomizedSearchCV(KNeighborsClassifier() , param,cv=cv,verbose = False, n_jobs=-1, n_iter=40)
 grid.fit(X,Y )
pickle.dump(normalizer, open("normalizer.pickle", "wb"),protocol=4)
 normalizer = None
del(normalizer)
 gc.collect()<categorify>
print(grid.best_score_)
 print(grid.best_params_)
 print(grid.best_estimator_ )
def normalize_dataframe(df):
with open('normalizer.pickle',mode='rb')as model_f:
normalizer_load = pickle.load(model_f)
 df[list(cols_to_normalize)]= normalizer_load.transform(df[list(cols_to_normalize)])
 df[list({'item_condition_id','shipping'})] = df[list({'item_condition_id','shipping'})].astype('category')
 normalizer_load = None
del(normalizer_load)
 gc.collect()
return df<create_dataframe>
knn_ran_grid = grid.best_estimator_
knn_ran_grid.score(X,Y )
normalize_dataframe(X_train)
 normalize_dataframe(X_cv )<drop_column>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_gaussian
wb_desc = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28,
"norm": "l2",
"tf": 1.0,
"idf": None}), procs=8)
 wb_desc.dictionary_freeze = True 
<categorify>
GaussianProcessClassifier = GaussianProcessClassifier()
GaussianProcessClassifier.fit(X_train, Y_train)
 Y_pred = GaussianProcessClassifier.predict(X_test)
 gau_pro_accy = round(accuracy_score(Y_pred, Y_test), 3)
 print(gau_pro_accy )
wb_desc.fit(X_train['item_description'])
 X_description_train_wb = wb_desc.transform(X_train['item_description'])
 X_description_cv_wb = wb_desc.fit_transform(X_cv['item_description'] )<define_variables>
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_perceptron
mask_desc = np.where(X_description_train_wb.getnnz(axis=0)> 3)[0]<find_best_params>
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(accuracy_score(Y_pred, Y_test)* 100, 2)
 acc_sgd
X_description_train_wb = X_description_train_wb[:, mask_desc]
X_description_cv_wb = X_description_cv_wb[:, mask_desc]<train_model>
all_models = [logreg,
logreg_grid,
svc,
svc_grid,
linear_svc,
dectree,
dectree_grid,
rf,
rf_grid,
bagging_grid,
adaBoost_grid,
gradient_boost,
XGBClassifier,
ExtraTreesClassifier,
knn,
knn_grid,
knn_ran_grid,
gaussian,
GaussianProcessClassifier,
perceptron,
sgd]
c = []
for i in all_models:
a = i.predict(X_test)
 b = accuracy_score(a, Y_test)
 c.append(b )
model_desc = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=5)
 model_desc.fit(X_description_train_wb, Y_train )<compute_train_metric>
models = pd.DataFrame({
'Model': ['logreg',
'logreg_grid',
'svc',
'svc_grid',
'linear_svc',
'dectree',
'dectree_grid',
'rf',
'rf_grid',
'bagging_grid',
'adaBoost_grid',
'gradient_boost',
'XGBClassifier',
'ExtraTreesClassifier',
'knn',
'knn_grid',
'knn_ran_grid',
'gaussian',
'GaussianProcessClassifier',
'perceptron',
'sgd'],
'Score': c })
 models.sort_values(by="Score",ascending=False )
pred_train_1 = model_desc.predict(X_description_train_wb)
 pred_cv_1 = model_desc.predict(X_description_cv_wb)
 print("Train rmsle: "+str(rmsle(10 ** Y_train-1, 10 ** pred_train_1-1)))
 print("CV rmsle: "+str(rmsle(10 ** Y_cv-1, 10 ** pred_cv_1-1)) )<load_pretrained>
final_pred_Y = bagging_grid.predict(test)
 final_pred_Y
pickle.dump(mask_desc, open("mask_desc.pickle", "wb"),protocol=4)
 pickle.dump(wb_desc, open("wb_desc.pickle", "wb"),protocol=4)
 pickle.dump(model_desc, open("model_desc.pickle", "wb"),protocol=4 )<drop_column>
submission = pd.DataFrame({
"PassengerId": passengerid,
"Survived": final_pred_Y
} )
mask_desc = None
wb_desc = None
model_desc = None
del(mask_desc,wb_desc,model_desc)
 gc.collect()<drop_column>
submission.PassengerId = submission.PassengerId.astype(int)
 submission.Survived = submission.Survived.astype(int )
wb_name = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29,
"norm": None,
"tf": 'binary',
"idf": None,
}), procs=8)
 wb_name.dictionary_freeze = True 
<categorify>
submission.to_csv("titanic_submission.csv", index=False )