Kaggle competition: Titanic - Machine Learning from Disaster
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<split>
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import accuracy_score
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )<choose_model_class>
train=pd.read_csv(".. /input/titanic/train.csv")
 tit1=train.select_dtypes(include=['float64','int64','object'])
 train.info()
test=pd.read_csv(".. /input/titanic/test.csv")
 tit2=test.select_dtypes(include=['float64','int64','object'])
 test.info()
<concatenate>
tit2['survived']=np.nan
tit2.head()
def conv_block(m, dim, acti, bn, res, do=0):
n = Conv2D(dim, 3, activation=acti, padding='same' )(m)
 n = BatchNormalization()(n)if bn else n
n = Dropout(do )(n)if do else n
n = Conv2D(dim, 3, activation=acti, padding='same' )(n)
 n = BatchNormalization()(n)if bn else n
return Concatenate()([m, n])if res else n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
if depth > 0:
n = conv_block(m, dim, acti, bn, res)
 m = MaxPooling2D()(n)if mp else Conv2D(dim, 3, strides=2, padding='same' )(n)
 m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res)
 if up:
m = UpSampling2D()(m)
 m = Conv2D(dim, 2, activation=acti, padding='same' )(m)
 else:
m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same' )(m)
 n = Concatenate()([n, m])
 m = conv_block(n, dim, acti, bn, res)
 else:
m = conv_block(m, dim, acti, bn, res, do)
 return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
i = Input(shape=img_shape)
 o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 o = Conv2D(out_ch, 1, activation='sigmoid' )(o)
 return Model(inputs=i, outputs=o )<train_model>
percent=round(np.mean(train['Survived']),3)*100
print("Percentage of Survivors:",percent )
model = UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True )<choose_model_class>
total=train['Survived'].sum()
total
men=train[train['Sex']=='male']
women=train[train['Sex']=='female']
m=men['Sex'].count()
w=women['Sex'].count()
print("male:",m)
 print("female:",w)
 print("percentage of women:",round(w/(m+w)*100))
print("percentage of men:",round(m/(m+w)*100))
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"] )<concatenate>
train.isnull().sum()
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0 )<train_model>
train.isnull().sum()
early_stopping = EarlyStopping(patience=10, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.1, patience=5, min_lr=0.000005, verbose=1)
 epochs = 200
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint, reduce_lr] )<predict_on_test>
train['Cabin'] = train['Cabin'].fillna('X')
 test['Cabin']=test['Cabin'].fillna('X' )
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 mask_valid = np.array([downsample(x)for x in y_valid] )<compute_train_metric>
train.loc[train.Age.isnull() ,'Age']=train.groupby("Pclass" ).Age.transform('median')
 test.loc[test.Age.isnull() ,'Age']=test.groupby("Pclass" ).Age.transform('median')
 print(train['Age'].isnull().sum() )
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(mask_valid, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )<find_best_params>
category = {'A':1, 'B':2, 'C':3, 'D':4, 'E':5, 'F':6, 'G':7, 'X':8, 'T':9}
train['Cabin'] = train['Cabin'].map(category)
 train['Cabin'].unique().tolist()
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]<categorify>
print(train.isnull().sum() )
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs<predict_on_test>
train["Embarked"] = train["Embarked"].fillna(mode(train["Embarked"]))
preds_test = model.predict(x_test )<compute_test_metric>
print(train.isnull().sum() )
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }<save_to_csv>
train["Sex"][train["Sex"] == "male"] = 0
train["Sex"][train["Sex"] == "female"] = 1
test["Sex"][test["Sex"] == "male"] = 0
test["Sex"][test["Sex"] == "female"] = 1
train["Embarked"][train["Embarked"] == "S"] = 0
train["Embarked"][train["Embarked"] == "C"] = 1
train["Embarked"][train["Embarked"] == "Q"] = 2
test["Embarked"][test["Embarked"] == "S"] = 0
test["Embarked"][test["Embarked"] == "C"] = 1
test["Embarked"][test["Embarked"] == "Q"] = 2
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv')
 timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M")
 print('Submission output to: sub-{}.csv'.format(timestamp))
sub.to_csv("sub-{}.csv".format(timestamp)) 
<set_options>
train['fam']=train['SibSp']+train['Parch']+1
test['fam']=test['SibSp']+test['Parch']+1
plt.style.use('seaborn-white')
 sns.set_style("white")
<train_model>
train['Age']=train['Age'].astype(str)
 test['Age']=test['Age'].astype(str )
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
<compute_test_metric>
cat={'2':1, '3':2 , '5':3, '1':4 ,'4':5,'8':6,'6':7,'7':8,'0':9,'9':10}
test['Age']=test['Age'].map(cat)
 test['Age'].unique().tolist()
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, K.sigmoid(y_pred)) + dice_loss(y_true, K.sigmoid(y_pred))
def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 logit_y_pred = K.log(y_pred /(1.- y_pred))
loss = weight *(logit_y_pred *(1.- y_true)+
K.log(1.+ K.exp(-K.abs(logit_y_pred)))+ K.maximum(-logit_y_pred, 0.))
 return K.sum(loss)/ K.sum(weight)
 def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
 w, m1, m2 = weight, y_true, y_pred
intersection =(m1 * m2)
 score =(2.* K.sum(w * intersection)+ smooth)/(K.sum(w * m1)+ K.sum(w * m2)+ smooth)
 loss = 1.- K.sum(score)
 return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
 y_pred = K.cast(y_pred, 'float32')
 averaged_mask = K.pool2d(
 y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
 weight = K.ones_like(averaged_mask)
 w0 = K.sum(weight)
 weight = 5.* K.exp(-5.* K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
 weight *=(w0 / w1)
 loss = weighted_bce_loss(y_true, y_pred, weight)+ dice_loss(y_true, y_pred)
 return loss

 def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_softmax(probas, labels, classes='all', per_image=False, ignore=None, order='BHWC'):

 if per_image:
def treat_image(prob_lab):
prob, lab = prob_lab
prob, lab = tf.expand_dims(prob, 0), tf.expand_dims(lab, 0)
 prob, lab = flatten_probas(prob, lab, ignore, order)
 return lovasz_softmax_flat(prob, lab, classes=classes)
 losses = tf.map_fn(treat_image,(probas, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_softmax_flat(*flatten_probas(probas, labels, ignore, order), classes=classes)
 return loss
def lovasz_softmax_flat(probas, labels, classes='all'):

 C = 1
losses = []
present = []
class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes
for c in class_to_sum:
fg = tf.cast(tf.equal(labels, c), probas.dtype)
 if classes == 'present':
present.append(tf.reduce_sum(fg)> 0)
 errors = tf.abs(fg - probas[:, c])
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort_{}".format(c))
fg_sorted = tf.gather(fg, perm)
 grad = lovasz_grad(fg_sorted)
 losses.append(
 tf.tensordot(errors_sorted, tf.stop_gradient(grad), 1, name="loss_class_{}".format(c)) 
)
 if len(class_to_sum)== 1:
return losses[0]
losses_tensor = tf.stack(losses)
 if classes == 'present':
present = tf.stack(present)
 losses_tensor = tf.boolean_mask(losses_tensor, present)
 loss = tf.reduce_mean(losses_tensor)
 return loss
def flatten_probas(probas, labels, ignore=None, order='BHWC'):

 if order == 'BCHW':
probas = tf.transpose(probas,(0, 2, 3, 1), name="BCHW_to_BHWC")
 order = 'BHWC'
if order != 'BHWC':
raise NotImplementedError('Order {} unknown'.format(order))
C = 1
probas = tf.reshape(probas,(-1, C))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return probas, labels
valid = tf.not_equal(labels, ignore)
 vprobas = tf.boolean_mask(probas, valid, name='valid_probas')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vprobas, vlabels
def keras_lovasz_softmax(labels,probas):
return lovasz_softmax(probas, labels)
 def keras_lovasz_hinge(labels,logits):
return lovasz_hinge(logits, labels, per_image=True, ignore=None)
 def mean_iou(y_true, y_pred):
prec = []
for t in np.arange(0.5, 1.0, 0.05):
y_pred_ = tf.to_int32(y_pred > t)
 score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)
 K.get_session().run(tf.local_variables_initializer())
 with tf.control_dependencies([up_opt]):
score = tf.identity(score)
 prec.append(score)
 return K.mean(K.stack(prec), axis=0)
 def focal_loss(y_true, y_pred):
gamma=0.75
alpha=0.25
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
pt_1 = K.clip(pt_1, 1e-3,.999)
 pt_0 = K.clip(pt_0, 1e-3,.999)
 return -K.sum(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) -K.sum(( 1-alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0)) 
<load_from_csv>
train['Title'] = train['Name'].map(lambda x: re.compile("([A-Za-z]+)\." ).search(x ).group())
 test['Title'] = test['Name'].map(lambda x: re.compile("([A-Za-z]+)\." ).search(x ).group())
 print(train['Title'].unique())
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]<feature_engineering>
train['Title'] = train['Title'].replace(['Lady.', 'Capt.', 'Col.',
'Don.', 'Dr.', 'Major.', 'Rev.', 'Jonkheer.', 'Dona.'], 'Rare.')
 train['Title'] = train['Title'].replace(['Countess.', 'Lady', 'Sir'], 'Royal')
 train['Title'] = train['Title'].replace('Mlle.', 'Miss.')
 train['Title'] = train['Title'].replace('Ms.', 'Miss.')
 train['Title'] = train['Title'].replace('Mme.', 'Mrs.')
 train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
test['Title'] = test['Title'].replace(['Lady.', 'Capt.', 'Col.',
'Don.', 'Dr.', 'Major.', 'Rev.', 'Jonkheer.', 'Dona.'], 'Rare.')
 test['Title'] = test['Title'].replace(['Countess.', 'Lady.', 'Sir.'], 'Royal.')
 test['Title'] = test['Title'].replace('Mlle.', 'Miss')
 test['Title'] = test['Title'].replace('Ms.', 'Miss.')
 test['Title'] = test['Title'].replace('Mme.', 'Mrs.')
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<feature_engineering>
title_mapping = {"Mr.": 1, "Miss.": 2, "Mrs.": 3, "Master.": 4, "Royal.": 5, "Rare.": 6}
train['Title'] = train['Title'].map(title_mapping)
 train['Title'] = train['Title'].fillna(0)
 train.head()
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )<categorify>
title_mapping = {"Mr.": 1, "Miss.": 2, "Mrs.": 3, "Master.": 4, "Royal.": 5, "Rare.": 6}
test['Title'] = test['Title'].map(title_mapping)
 test['Title'] = test['Title'].fillna(0)
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<split>
test = test.drop(['Ticket'], axis = 1)
 test = test.drop(['Name'], axis = 1)
 test = test.drop(['Parch'], axis = 1)
 test = test.drop(['Fare','SibSp'], axis = 1)
 train.drop(['Name', 'Ticket'], axis = 1, inplace = True)
 train = train.drop(['Parch'], axis = 1)
 train = train.drop(['Fare','SibSp'], axis = 1 )
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )<define_search_model>
X_train, X_test, y_train, y_test = train_test_split(train.drop(['Survived','PassengerId'], axis=1),
train['Survived'], test_size = 0.2,
random_state = 0 )
def conv_block(m, dim, acti, bn, res, do=0):
	n = Conv2D(dim, 3, activation=acti, padding='same' )(m)
 	n = BatchNormalization()(n)if bn else n
	n = Dropout(do )(n)if do else n
	n = Conv2D(dim, 3, activation=acti, padding='same' )(n)
 	n = BatchNormalization()(n)if bn else n
	return Concatenate()([m, n])if res else n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
	if depth > 0:
		n = conv_block(m, dim, acti, bn, res)
 		m = MaxPooling2D()(n)if mp else Conv2D(dim, 3, strides=2, padding='same' )(n)
 		m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res)
 		if up:
			m = UpSampling2D()(m)
 			m = Conv2D(dim, 2, activation=acti, padding='same' )(m)
 		else:
			m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same' )(m)
 		n = Concatenate()([n, m])
 		m = conv_block(n, dim, acti, bn, res)
 	else:
		m = conv_block(m, dim, acti, bn, res, do)
 	return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
		 dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
	i = Input(shape=img_shape)
 	o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 	o = Conv2D(out_ch, 1 )(o)
 	o = Activation("sigmoid" )(o)
 	return Model(inputs=i, outputs=o)
 def Logit_UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
		 dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
	i = Input(shape=img_shape)
 	o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 	o = Conv2D(out_ch, 1 )(o)
 	return Model(inputs=i, outputs=o )<choose_model_class>
logisticRegression = LogisticRegression(max_iter = 30000)
 logisticRegression.fit(X_train, y_train)
model = Logit_UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True)
<choose_model_class>
predictions = logisticRegression.predict(X_test)
 acc_LOG = round(accuracy_score(predictions, y_test)* 100, 2)
 print(acc_LOG)
 print(predictions)
model.compile(loss=bce_dice_loss, optimizer="adam", metrics=["accuracy",mean_iou])
<define_variables>
print(confusion_matrix(y_test, predictions))
train_gen = ImageDataGenerator(
 horizontal_flip=True,
shear_range=0)
 val_gen = ImageDataGenerator(
 horizontal_flip=True,
shear_range=0 )<train_model>
accuracy=(( 88+50)/(88+50+22+19))
print('accuracy is: ',(round(accuracy, 2)*100))
early_stopping = EarlyStopping(patience=10, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=5, min_lr=0.00001, verbose=1)
 epochs = 20
batch_size = 128
history1 = model.fit_generator(train_gen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train)/ 32,
epochs=epochs,
validation_data=val_gen.flow(x_valid, y_valid, batch_size=batch_size),
callbacks=[early_stopping, model_checkpoint, reduce_lr],shuffle=True)
 model.compile(loss=focal_loss, optimizer="sgd", metrics=["accuracy",mean_iou])
 epochs = 80
batch_size = 128
history2 = model.fit_generator(train_gen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train)/ 32,
epochs=epochs,
validation_data=val_gen.flow(x_valid, y_valid, batch_size=batch_size),
callbacks=[early_stopping, model_checkpoint, reduce_lr],shuffle=True )<load_pretrained>
randomforest = RandomForestClassifier(random_state = 5, criterion = 'gini', max_depth = 10, max_features = 'auto', n_estimators = 500)
 randomforest.fit(X_train, y_train)
 pred = randomforest.predict(X_test)
 acc_randomforest = round(accuracy_score(pred, y_test)* 100, 2)
 print(acc_randomforest)
model.load_weights("./keras.model" )<predict_on_test>
gbk = GradientBoostingClassifier()
gbk.fit(X_train, y_train)
 pred = gbk.predict(X_test)
 acc_gbk = round(accuracy_score(pred, y_test)* 100, 2)
 print(acc_gbk )
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )<compute_train_metric>
see={'TECHNIQUE':['RANDOM FOREST','LOGISTIC REGRESSION','GRADIENT BOOSTING'],'ACCURACY':[acc_randomforest,acc_LOG,acc_gbk]}
mod=pd.DataFrame(see)
 mod
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )<find_best_params>
ids = test['PassengerId']
predictions = randomforest.predict(test.drop('PassengerId', axis=1))
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]<categorify>
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import accuracy_score
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs<predict_on_test>
train=pd.read_csv(".. /input/titanic/train.csv")
 tit1=train.select_dtypes(include=['float64','int64','object'])
 train.info()
test=pd.read_csv(".. /input/titanic/test.csv")
 tit2=test.select_dtypes(include=['float64','int64','object'])
 test.info()
preds_test = model.predict(x_test )<compute_test_metric>
tit2['survived']=np.nan
tit2.head()
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }<save_to_csv>
percent=round(np.mean(train['Survived']),3)*100
print("Percentage of Survivors:",percent )
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )<set_options>
total=train['Survived'].sum()
total
men=train[train['Sex']=='male']
women=train[train['Sex']=='female']
m=men['Sex'].count()
w=women['Sex'].count()
print("male:",m)
 print("female:",w)
 print("percentage of women:",round(w/(m+w)*100))
print("percentage of men:",round(m/(m+w)*100))
pd.set_option('display.max_columns', None)
 %matplotlib inline
stop = stopwords.words('english')
 scaler = MinMaxScaler(feature_range=(0,1)) 
<load_from_csv>
train.isnull().sum()
train_df = pd.read_csv('.. /input/tmdb-clean-dataset/train.csv')
 test_df = pd.read_csv('.. /input/tmdb-clean-dataset/test.csv')
 merged_df = train_df.append(test_df,ignore_index=True )<count_values>
train.isnull().sum()
def get_count_houses(data_column,col_name):
ls = []
data = [extract_meta_2(str(row),col_name)for row in data_column]
data_2 = [house for production_houses in data for house in production_houses]
house_count = Counter(data_2)
 return house_count
def extract_single_meta(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace("'",'"')
 json_data = json_data.replace('None','"None"')
 json_data = json_data.replace('s n"','s n ')
 json_data = json_data.replace('ed"','ed ')
 json_data = json_data.replace('ld"','ld ')
 json_data = json_data.replace('n"s','ns')
 json_data = json_data.replace('o"s','os')
 json_data = json_data.replace('"s','s')
 data = json.loads(json_data)
 meta = [d[col_name] for d in data]
return meta[0]
def extract_meta(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace("'",'"')
 str_json_data = str(json_data)
 str_json_data = str_json_data.replace("l"A","l")
 data = None
data= json.loads(str_json_data)
 meta = [d[col_name] for d in data]
return meta
def extract_meta_2(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace(u'\xa0',u' ')
 json_data = json_data.replace("{'",'{"')
 json_data = json_data.replace("'}",'"}')
 json_data = json_data.replace("':",'":')
 json_data = json_data.replace(": '",': "')
 json_data = json_data.replace("',",'",')
 json_data = json_data.replace(", '",', "')
 json_data = json_data.replace("'",'')
 str_json_data = str(json_data)
 data= None
data= json.loads(str_json_data)
 meta = [d[col_name] for d in data]
return meta
def extract_metadata(data_column, col_name, col_type = "blank"):
all_meta = set()
counter = Counter()
if(col_type != 'production_companies' and col_type != 'keyword'):
distinct_meta = [extract_meta(str(row),col_name)for row in data_column]
else:
counter = get_count_houses(data_column,col_name)
 counter = {k:counter[k] for k in counter if counter[k] > 5}
distinct_meta = [extract_meta_2(str(row),col_name)for row in data_column]
flattened_meta = [data for row in distinct_meta for data in row]
distinct_meta = []
for k in counter:
if k in flattened_meta:
distinct_meta.append(k)
 for data in distinct_meta:
if(col_type != 'production_companies' and col_type != 'keyword'):
all_meta.update(set(data))
else:
all_meta.update(set([data]))
all_meta_dict = dict()
i = 0
for item in all_meta:
all_meta_dict[item] = i
i += 1
all_meta_list = list(all_meta)
 return all_meta, all_meta_dict, all_meta_list
def add_meta_to_dataframe(all_meta_dict,all_meta_list, _df, data_column, col_name, col_type='blank'):
if(col_type != 'production_companies' and col_type != 'keyword'):
distinct_meta = [extract_meta(str(row),col_name)for row in data_column]
else:
distinct_meta = [extract_meta_2(str(row),col_name)for row in data_column]
one_hot_np = np.empty([len(_df),len(all_meta_dict)])
 for i in range(0, len(_df)) :
one_hot_meta = np.zeros(len(all_meta_dict))
for j in range(0,len(distinct_meta[i])) :
if(distinct_meta[i][j] in all_meta_dict):
one_hot_meta[all_meta_dict[distinct_meta[i][j]]] = 1
one_hot_np[i] = one_hot_meta
one_hot_df = pd.DataFrame(one_hot_np)
 one_hot_df.columns = all_meta_list
if(col_name == 'iso_639_1'):
one_hot_df = one_hot_df['en']
else:
one_hot_df['sum' + col_name] = one_hot_df.sum(axis=1,skipna=False)
 _df_new = pd.concat([_df.reset_index(drop=True),one_hot_df],axis=1)
 return _df_new
genres = merged_df['genres']
genres.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(genres,"name")
 genres = train_df['genres']
genres.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df, genres,"name")
 genres_test = test_df['genres']
genres_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df,genres_test,"name")
 spoken_languages = merged_df['spoken_languages']
spoken_languages.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(spoken_languages,"iso_639_1")
 spoken_languages = train_df['spoken_languages']
spoken_languages.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df_new, spoken_languages,"iso_639_1")
 spoken_languages_test = test_df['spoken_languages']
spoken_languages_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df_new,spoken_languages_test,"iso_639_1")
 production_countries = merged_df['production_countries']
production_countries.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(production_countries,"iso_3166_1")
 production_countries = train_df['production_countries']
production_countries.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df_new, production_countries,"iso_3166_1")
 production_countries_test = test_df['production_countries']
production_countries_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df_new,production_countries_test,"iso_3166_1")

belongs_to_collection = train_df['belongs_to_collection']
belongs_to_collection.fillna('nan',inplace=True)
 result = [extract_single_meta(str(row),'name')for row in belongs_to_collection]
result = pd.DataFrame(result)
 result.columns = ['collection']
train_df_new = pd.concat([train_df_new.reset_index(drop=True),result],axis=1)
 belongs_to_collection_test = test_df['belongs_to_collection']
belongs_to_collection_test.fillna('nan',inplace=True)
 result = [extract_single_meta(str(row),'name')for row in belongs_to_collection_test]
result = pd.DataFrame(result)
 result.columns = ['collection']
test_df_new = pd.concat([test_df_new.reset_index(drop=True),result],axis=1)
 production_companies = merged_df['production_companies']
production_companies.fillna('nan',inplace=True)
 all_meta, all_meta_dict, all_meta_list = extract_metadata(production_companies,"name","production_companies")
 production_companies = train_df['production_companies']
production_companies.fillna('nan',inplace=True)
 train_df_new = add_meta_to_dataframe(all_meta_dict, all_meta_list, train_df_new, production_companies,"name","production_companies")
 production_companies_test = test_df['production_companies']
production_companies_test.fillna('nan',inplace=True)
 test_df_new = add_meta_to_dataframe(all_meta_dict,all_meta_list,test_df_new,production_companies_test,"name","production_companies")
<categorify>
train['Cabin'] = train['Cabin'].fillna('X')
 test['Cabin']=test['Cabin'].fillna('X' )
def extract_single_meta(json_data,col_name):
if(json_data == 'nan'):
return 'x'
json_data = json_data.replace("'",'"')
 json_data = json_data.replace('None','"None"')
 json_data = json_data.replace('s n"','s n ')
 json_data = json_data.replace('ed"','ed ')
 json_data = json_data.replace('ld"','ld ')
 json_data = json_data.replace('n"s','ns')
 json_data = json_data.replace('o"s','os')
 json_data = json_data.replace('"s','s')
 data = json.loads(json_data)
 meta = [d for d in data]
return meta[0]

 imdb_id = train_df['imdb_id']
cast = train_df['cast']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['-1','nan','nan','-1','-1','nan','10000','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 8 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
cast_df = DataFrame.from_records(main_ls)
 cast_df.columns = ['cast_id','character','credit_id','gender','id','name','order','profile_path','imdb_id']

 imdb_id = test_df['imdb_id']
cast = test_df['cast']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['-1','nan','nan','-1','-1','nan','10000','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 8 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
cast_df_test = DataFrame.from_records(main_ls)
 cast_df_test.columns = ['cast_id','character','credit_id','gender','id','name','order','profile_path','imdb_id'] 
<define_variables>
train.loc[train.Age.isnull() ,'Age']=train.groupby("Pclass" ).Age.transform('median')
 test.loc[test.Age.isnull() ,'Age']=test.groupby("Pclass" ).Age.transform('median')
 print(train['Age'].isnull().sum() )
imdb_id = train_df['imdb_id']
cast = train_df['crew']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['nan','nan','nan','nan','nan','nan','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 7 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
crew_df = DataFrame.from_records(main_ls)
 crew_df.columns = ['credit_id','department','gender','id','job','name','profile_path','imdb_id']

 imdb_id = test_df['imdb_id']
cast = test_df['crew']
main_ls = []
x = 0
i = 0
for item in cast:
i+= 1
ls = []
if str(item)== '[]':
main_ls.append(['nan','nan','nan','nan','nan','nan','nan'])
 x += 1
continue
item = str(item)
 item = item.replace("', '","'; '")
 item = item.replace(", '","; '")
 item= item.replace("{","")
 item= item.replace("},",";")
 item= item.replace("[","")
 item= item.replace("]","")
 item= item.replace("'","")
 item = item.replace('Jimmy;,','Jimmy')
 item = item.replace('Elektra Quartet;','Elektra Quartet')
 item_ls = item.split(';')
 i = 0
ls = []
for it in item_ls:
i += 1
it = it.replace('[','')
 it = it.replace(']','')
 it_ls = it.split(':')
 ls.append(str(it_ls[1] ).strip())
 if i % 7 == 0:
ls.append(imdb_id[x])
 main_ls.append(ls)
 ls = []
x += 1
crew_df_test = DataFrame.from_records(main_ls)
 crew_df_test.columns = ['credit_id','department','gender','id','job','name','profile_path','imdb_id']<merge>
category = {'A':1, 'B':2, 'C':3, 'D':4, 'E':5, 'F':6, 'G':7, 'X':8, 'T':9}
train['Cabin'] = train['Cabin'].map(category)
 train['Cabin'].unique().tolist()
cast_counts = cast_df[['cast_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['cast_counts']
cast_counts.reset_index(inplace=True)
 df_train= train_df_new.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = cast_df[['cast_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='cast_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_cast_count_0','gender_cast_count_1','gender_cast_count_2']
df_train = df_train.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_top_3 = cast_df[['order','gender','imdb_id']]
cast_gender_top_3['int_order'] = cast_gender_top_3.order.astype(int)
 cast_gender_top_3 = cast_gender_top_3.loc[cast_gender_top_3['int_order'] <= 2,]
cast_gender_top_3.drop('order',inplace=True,axis=1)
 cast_gender_top_3 = cast_gender_top_3.pivot_table(index='imdb_id',columns='int_order',values='gender',aggfunc='first')
 cast_gender_top_3.reset_index(inplace=True)
 cast_gender_top_3.columns=['imdb_id','cast_1_gender','cast_2_gender','cast_3_gender']
df_train = df_train.merge(cast_gender_top_3,how='left',left_on='imdb_id',right_on='imdb_id')
 top_actors = pd.read_csv('.. /input/top-1000-tmdb-actors/Top 1000 Actors and Actresses.csv',encoding='iso-8859-1')
 top_actors_name = top_actors[['Name']]
cast_df = cast_df.merge(top_actors_name,how='left',left_on='name',right_on='Name')
 print(cast_df.head(10))
cast_top_actors = cast_df[['imdb_id','Name']]
cast_top_actors = cast_top_actors.dropna()
cast_top_actors_2= cast_top_actors.groupby("imdb_id" ).count()
cast_top_actors_2.reset_index(inplace=True)
 cast_top_actors_2.columns=['imdb_id','count_top_actors']
df_train = df_train.merge(cast_top_actors_2,how='left',left_on='imdb_id',right_on='imdb_id')

cast_counts = cast_df_test[['cast_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['cast_counts']
cast_counts.reset_index(inplace=True)
 df_test= test_df_new.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = cast_df_test[['cast_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='cast_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_cast_count_0','gender_cast_count_1','gender_cast_count_2']
df_test = df_test.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_top_3 = cast_df_test[['order','gender','imdb_id']]
cast_gender_top_3['int_order'] = cast_gender_top_3.order.astype(int)
 cast_gender_top_3 = cast_gender_top_3.loc[cast_gender_top_3['int_order'] <= 2,]
cast_gender_top_3.drop('order',inplace=True,axis=1)
 cast_gender_top_3 = cast_gender_top_3.pivot_table(index='imdb_id',columns='int_order',values='gender',aggfunc='first')
 cast_gender_top_3.reset_index(inplace=True)
 cast_gender_top_3.columns=['imdb_id','cast_1_gender','cast_2_gender','cast_3_gender']
df_test = df_test.merge(cast_gender_top_3,how='left',left_on='imdb_id',right_on='imdb_id')
 top_actors = pd.read_csv('.. /input/top-1000-tmdb-actors/Top 1000 Actors and Actresses.csv',encoding='iso-8859-1')
 top_actors_name = top_actors[['Name']]
cast_df_test = cast_df_test.merge(top_actors_name,how='left',left_on='name',right_on='Name')
 cast_top_actors = cast_df_test[['imdb_id','Name']]
cast_top_actors = cast_top_actors.dropna()
cast_top_actors_2= cast_top_actors.groupby("imdb_id" ).count()
cast_top_actors_2.reset_index(inplace=True)
 cast_top_actors_2.columns=['imdb_id','count_top_actors']
df_test = df_test.merge(cast_top_actors_2,how='left',left_on='imdb_id',right_on='imdb_id' )<merge>
print(train.isnull().sum() )
cast_counts = crew_df[['credit_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['crew_counts']
cast_counts.reset_index(inplace=True)
 df_train= df_train.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df[['credit_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_crew_count_0','gender_crew_count_1','gender_crew_count_2']
df_train = df_train.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df[['credit_id', 'gender','imdb_id','department']].groupby(['imdb_id','gender','department'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot_table(index='imdb_id',columns=['gender','department'],values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 df_train = df_train.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_job_counts = crew_df[['credit_id', 'job','imdb_id']].groupby(['imdb_id','job'],as_index=False ).agg('count')
 crew_job_counts = crew_job_counts.pivot(index='imdb_id',columns='job',values='credit_id')
 crew_job_counts.reset_index(inplace=True)
 crew_job_counts.fillna(0,inplace=True)
 df_train = df_train.merge(crew_job_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_dept_counts = crew_df[['credit_id', 'department','imdb_id']].groupby(['imdb_id','department'],as_index=False ).agg('count')
 crew_dept_counts = crew_dept_counts.pivot(index='imdb_id',columns='department',values='credit_id')
 crew_dept_counts.reset_index(inplace=True)
 crew_dept_counts.fillna(0,inplace=True)
 df_train = df_train.merge(crew_dept_counts,how='left',left_on='imdb_id',right_on='imdb_id')

cast_counts = crew_df_test[['credit_id','imdb_id']].groupby('imdb_id' ).agg('count')
 cast_counts.columns = ['crew_counts']
cast_counts.reset_index(inplace=True)
 df_test= df_test.merge(cast_counts, how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df_test[['credit_id', 'gender','imdb_id']].groupby(['imdb_id','gender'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot(index='imdb_id',columns='gender',values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 cast_gender_counts.columns=['imdb_id','gender_crew_count_0','gender_crew_count_1','gender_crew_count_2']
df_test = df_test.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 cast_gender_counts = crew_df_test[['credit_id', 'gender','imdb_id','department']].groupby(['imdb_id','gender','department'],as_index=False ).agg('count')
 cast_gender_counts = cast_gender_counts.pivot_table(index='imdb_id',columns=['gender','department'],values='credit_id')
 cast_gender_counts.reset_index(inplace=True)
 cast_gender_counts.fillna(0,inplace=True)
 df_test = df_test.merge(cast_gender_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_job_counts = crew_df_test[['credit_id', 'job','imdb_id']].groupby(['imdb_id','job'],as_index=False ).agg('count')
 crew_job_counts = crew_job_counts.pivot(index='imdb_id',columns='job',values='credit_id')
 crew_job_counts.reset_index(inplace=True)
 crew_job_counts.fillna(0,inplace=True)
 df_test = df_test.merge(crew_job_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 crew_dept_counts = crew_df_test[['credit_id', 'department','imdb_id']].groupby(['imdb_id','department'],as_index=False ).agg('count')
 crew_dept_counts = crew_dept_counts.pivot(index='imdb_id',columns='department',values='credit_id')
 crew_dept_counts.reset_index(inplace=True)
 crew_dept_counts.fillna(0,inplace=True)
 df_test = df_test.merge(crew_dept_counts,how='left',left_on='imdb_id',right_on='imdb_id')
<groupby>
train["Embarked"] = train["Embarked"].fillna(mode(train["Embarked"]))
character = cast_df['character']
cast = cast_df['name']
character.fillna('x',inplace=True)
 cast.fillna('x',inplace=True)
 character_counts = Counter(character)
 cast_counts = Counter(cast)
 counter_char = {x: character_counts[x] for x in character_counts if character_counts[x] >= 40}
counter_cast = {x: cast_counts[x] for x in cast_counts if cast_counts[x] >= 40}
characters = [x for x in counter_char]
casts = [x for x in counter_cast]
cast_top = cast_df[cast_df['name'].isin(casts)]
cast_top_counts = cast_top[['imdb_id','name','cast_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_counts.reset_index(inplace=True)
 char_top = cast_df[cast_df['character'].isin(characters)]
char_top_counts = char_top[['imdb_id','character','cast_id']].groupby(['imdb_id','character'] ).agg('count')
 char_top_counts.reset_index(inplace=True)
 cast_top_test = cast_df_test[cast_df_test['name'].isin(casts)]
cast_top_test_counts = cast_top_test[['imdb_id','name','cast_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_test_counts.reset_index(inplace=True)
 char_top_test = cast_df_test[cast_df_test['character'].isin(characters)]
char_top_test_counts = char_top_test[['imdb_id','character','cast_id']].groupby(['imdb_id','character'] ).agg('count')
 char_top_test_counts.reset_index(inplace=True)
 cast_top_counts = cast_top_counts.pivot(index='imdb_id',columns='name',values='cast_id')
 cast_top_counts.fillna(0,inplace=True)
 cast_top_counts.reset_index(inplace=True)
 cast_top_test_counts = cast_top_test_counts.pivot(index='imdb_id',columns='name',values='cast_id')
 cast_top_test_counts.fillna(0,inplace=True)
 cast_top_test_counts.reset_index(inplace=True)
 char_top_counts = char_top_counts.pivot(index='imdb_id',columns='character',values='cast_id')
 char_top_counts.fillna(0,inplace=True)
 char_top_counts.reset_index(inplace=True)
 char_top_test_counts = char_top_test_counts.pivot(index='imdb_id',columns='character',values='cast_id')
 char_top_test_counts.fillna(0,inplace=True)
 char_top_test_counts.reset_index(inplace=True)
 df_train = df_train.merge(char_top_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_train = df_train.merge(cast_top_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_train.fillna(0,inplace=True)
 df_test = df_test.merge(char_top_test_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_test = df_test.merge(cast_top_test_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_test.fillna(0,inplace=True )<groupby>
print(train.isnull().sum() )
cast = crew_df['name']
cast.fillna('x',inplace=True)
 cast_counts = Counter(cast)
 counter_cast = {x: cast_counts[x] for x in cast_counts if cast_counts[x] >= 40}
casts = [x for x in counter_cast]
cast_top = crew_df[crew_df['name'].isin(casts)]
cast_top_counts = cast_top[['imdb_id','name','credit_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_counts.reset_index(inplace=True)
 cast_top_test = crew_df_test[crew_df_test['name'].isin(casts)]
cast_top_test_counts = cast_top_test[['imdb_id','name','credit_id']].groupby(['imdb_id','name'] ).agg('count')
 cast_top_test_counts.reset_index(inplace=True)
 cast_top_counts = cast_top_counts.pivot(index='imdb_id',columns='name',values='credit_id')
 cast_top_counts.fillna(0,inplace=True)
 cast_top_counts.reset_index(inplace=True)
 cast_top_test_counts = cast_top_test_counts.pivot(index='imdb_id',columns='name',values='credit_id')
 cast_top_test_counts.fillna(0,inplace=True)
 cast_top_test_counts.reset_index(inplace=True)
 df_train = df_train.merge(cast_top_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_train.fillna(0,inplace=True)
 df_test = df_test.merge(cast_top_test_counts,how='left',left_on='imdb_id',right_on='imdb_id')
 df_test.fillna(0,inplace=True )<feature_engineering>
train["Sex"][train["Sex"] == "male"] = 0
train["Sex"][train["Sex"] == "female"] = 1
test["Sex"][test["Sex"] == "male"] = 0
test["Sex"][test["Sex"] == "female"] = 1
train["Embarked"][train["Embarked"] == "S"] = 0
train["Embarked"][train["Embarked"] == "C"] = 1
train["Embarked"][train["Embarked"] == "Q"] = 2
test["Embarked"][test["Embarked"] == "S"] = 0
test["Embarked"][test["Embarked"] == "C"] = 1
test["Embarked"][test["Embarked"] == "Q"] = 2
df_train['overview'].fillna('',inplace=True)
 df_test['overview'].fillna('',inplace=True)
 df_train['len_original_title'] = df_train.original_title.apply(str ).apply(len)
 df_test['len_original_title'] = df_test.original_title.apply(len)
 df_train['len_overview'] = df_train.overview.apply(str ).apply(len)
 df_test['len_overview'] = df_test.overview.apply(str ).apply(len)
 df_train['len_tagline'] = df_train.tagline.apply(str ).apply(len)
 df_test['len_tagline'] = df_test.tagline.apply(str ).apply(len)
 df_train['len_title'] = df_train.title.apply(str ).apply(len)
 df_test['len_title'] = df_test.title.apply(str ).apply(len)
 df_train['num_words_title'] = df_train.title.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_title'] = df_test.title.apply(lambda x: len(str(x ).split(' ')))
 df_train['num_words_original_title'] = df_train.original_title.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_original_title'] = df_test.original_title.apply(lambda x: len(str(x ).split(' ')))
 df_train['num_words_tagline'] = df_train.tagline.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_tagline'] = df_test.tagline.apply(lambda x: len(str(x ).split(' ')))
 df_train['num_words_overview'] = df_train.overview.apply(lambda x: len(str(x ).split(' ')))
 df_test['num_words_overview'] = df_test.overview.apply(lambda x: len(str(x ).split(' ')) )<string_transform>
train['fam']=train['SibSp']+train['Parch']+1
test['fam']=test['SibSp']+test['Parch']+1
def senti(x):
return TextBlob(x ).sentiment
overview_train = df_train['overview']
overview_test = df_test['overview']
tagline_train = df_train['tagline']
tagline_test = df_test['tagline']
overview_train = overview_train.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
overview_train = overview_train.str.replace('[^\w\s]','')
 overview_test = overview_test.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
overview_test = overview_test.str.replace('[^\w\s]','')
 tagline_train = tagline_train.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
tagline_train = tagline_train.str.replace('[^\w\s]','')
 tagline_test = tagline_test.apply(lambda x: " ".join(str(x ).lower() for x in str(x ).split()))
tagline_test = tagline_test.str.replace('[^\w\s]','')
 overview_train = overview_train.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
overview_test = overview_test.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
tagline_train = tagline_train.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
tagline_test = tagline_test.apply(lambda x: " ".join(x for x in str(x ).split() if x not in stop))
st = PorterStemmer()
overview_train = overview_train.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
overview_test = overview_test.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
tagline_train = tagline_train.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
tagline_test = tagline_test.apply(lambda x: " ".join([st.stem(word)for word in str(x ).split() ]))
overview_train_senti = overview_train.apply(senti)
 overview_test_senti = overview_test.apply(senti)
 tagline_train_senti = tagline_train.apply(senti)
 tagline_test_senti = tagline_test.apply(senti)
 df_train['overview_senti_1'] = overview_train_senti.apply(lambda x: x[0])
 df_train['overview_senti_2'] = overview_train_senti.apply(lambda x: x[1])
 df_train['tagline_senti_1'] = tagline_train_senti.apply(lambda x: x[0])
 df_train['tagline_senti_2'] = tagline_train_senti.apply(lambda x: x[1])
 df_test['overview_senti_1'] = overview_test_senti.apply(lambda x: x[0])
 df_test['overview_senti_2'] = overview_test_senti.apply(lambda x: x[1])
 df_test['tagline_senti_1'] = tagline_test_senti.apply(lambda x: x[0])
 df_test['tagline_senti_2'] = tagline_test_senti.apply(lambda x: x[1] )<data_type_conversions>
train['Age']=train['Age'].astype(str)
 test['Age']=test['Age'].astype(str )
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date_2'].dt, part ).astype(int)
 return df
def fix_date(x):

 year = x.split('/')[2]
if len(year)== 2:
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
else:
return x
df_train['release_date_2'] = df_train['release_date'].apply(lambda x: fix_date(x))
df_test['release_date_2'] = df_test['release_date'].apply(lambda x: fix_date(x)if x != 0 else np.NaN)
 df_train['release_date_2'] = pd.to_datetime(df_train['release_date'])
 df_test['release_date_2'] = pd.to_datetime(df_test['release_date'])
 df_train = process_date(df_train)
 df_test = process_date(df_test)
 df_train.drop('release_date_2',axis=1,inplace=True)
 df_test.drop('release_date_2',axis=1,inplace=True )<merge>
cat={'2':1, '3':2 , '5':3, '1':4 ,'4':5,'8':6,'6':7,'7':8,'0':9,'9':10}
test['Age']=test['Age'].map(cat)
 test['Age'].unique().tolist()
train_sub = df_train[['id','collection','release_date_year']]
train_sub = train_sub[train_sub['collection'] != 'x']
train_sub['movie_rank'] = train_sub.groupby('collection')['release_date_year'].rank(ascending=True)
 train_sub.drop('collection',inplace=True,axis=1)
 train_sub.drop('release_date_year',inplace=True,axis=1)
 df_train = df_train.merge(train_sub,how='left',left_on='id',right_on='id')
 test_sub = df_test[['id','collection','release_date_year']]
test_sub = test_sub[test_sub['collection'] != 'x']
test_sub['movie_rank'] = test_sub.groupby('collection')['release_date_year'].rank(ascending=True)
 test_sub.drop('collection',inplace=True,axis=1)
 test_sub.drop('release_date_year',inplace=True,axis=1)
 df_test = df_test.merge(test_sub,how='left',left_on='id',right_on='id')
 df_train.loc[df_train.collection == 'x','movie_rank'] = np.NaN
df_test.loc[df_test.collection == 'x','movie_rank'] = np.NaN<categorify>
train['Title'] = train['Name'].map(lambda x: re.compile("([A-Za-z]+)\." ).search(x ).group())
 test['Title'] = test['Name'].map(lambda x: re.compile("([A-Za-z]+)\." ).search(x ).group())
 print(train['Title'].unique())
le = LabelEncoder()
le2 = LabelEncoder()
le3 = LabelEncoder()
merged_collection = pd.concat([df_train.collection,df_test.collection])
 merged_language = merged_df['original_language']
le.fit(merged_collection)
 le2.fit(merged_language)
 train_collection = df_train['collection']
test_collection = df_test['collection']
train_language = df_train['original_language']
test_language = df_test['original_language']
train_collection_labels = le.transform(train_collection)
 test_collection_labels = le.transform(test_collection)
 train_language_labels = le2.transform(train_language)
 test_language_labels = le2.transform(test_language)
 df_train['collection_encoded'] = train_collection_labels
df_test['collection_encoded'] = test_collection_labels
df_train['original_language_encoded'] = train_language_labels
df_test['original_language_encoded'] = test_language_labels<define_variables>
train['Title'] = train['Title'].replace(['Lady.', 'Capt.', 'Col.',
'Don.', 'Dr.', 'Major.', 'Rev.', 'Jonkheer.', 'Dona.'], 'Rare.')
 train['Title'] = train['Title'].replace(['Countess.', 'Lady', 'Sir'], 'Royal')
 train['Title'] = train['Title'].replace('Mlle.', 'Miss.')
 train['Title'] = train['Title'].replace('Ms.', 'Miss.')
 train['Title'] = train['Title'].replace('Mme.', 'Mrs.')
 train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
stop_english = stopwords.words('english')
 ls_all_words = []
ls_all_words_tagline = []
count_words_param = 30

 corpus_overview = df_train['overview']
corpus_overview_list = corpus_overview.tolist()
corpus_overview = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_list]
corpus_overview = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview]
corpus_overview = [str(txt ).lower() for txt in corpus_overview]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
for ele in sumbags_filter:
if(ele not in stop_english and ele not in ls_all_words):
ls_all_words.append(ele)
 list_index = list(range(len(df_train)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words):
df.set_value(i,ele,item[ele])
 i += 1
df_train = pd.concat([df_train.reset_index(drop=True),df],axis=1)
 corpus_overview = df_train['tagline']
corpus_overview_list = corpus_overview.tolist()
corpus_overview = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_list]
corpus_overview = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview]
corpus_overview = [str(txt ).lower() for txt in corpus_overview]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
for ele in sumbags_filter:
if(ele not in stop_english and ele not in ls_all_words_tagline):
ls_all_words_tagline.append(ele)
 list_index = list(range(len(df_train)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words_tagline)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words_tagline):
df.set_value(i,ele,item[ele])
 i += 1
df_train = pd.concat([df_train.reset_index(drop=True),df],axis=1)

corpus_overview_test = df_test['overview']
corpus_overview_test_list = corpus_overview_test.tolist()
corpus_overview_test = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_test_list]
corpus_overview_test = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview_test]
corpus_overview_test = [str(txt ).lower() for txt in corpus_overview_test]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview_test]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
list_index = list(range(len(df_test)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words):
df.set_value(i,ele,item[ele])
 i += 1
df_test = pd.concat([df_test.reset_index(drop=True),df],axis=1)
 corpus_overview_test = df_test['tagline']
corpus_overview_test_list = corpus_overview_test.tolist()
corpus_overview_test = [str(re.sub(' +',' ',str(txt)))for txt in corpus_overview_test_list]
corpus_overview_test = [str(re.sub('[^A-Za-z0-9' ]+','',str(txt)))for txt in corpus_overview_test]
corpus_overview_test = [str(txt ).lower() for txt in corpus_overview_test]
bagofwords = [collections.Counter(re.findall(r'\w+',txt)) for txt in corpus_overview_test]
sumbags = sum(bagofwords,collections.Counter())
 sumbags_filter = {x : sumbags[x] for x in sumbags if sumbags[x] >= count_words_param}
list_index = list(range(len(df_test)))
 df = pd.DataFrame(0, index=list_index, columns=ls_all_words_tagline)
 i = 0
for item in bagofwords:
for ele in item:
if(ele in ls_all_words_tagline):
df.set_value(i,ele,item[ele])
 i += 1
df_test = pd.concat([df_test.reset_index(drop=True),df],axis=1 )<feature_engineering>
test['Title'] = test['Title'].replace(['Lady.', 'Capt.', 'Col.',
'Don.', 'Dr.', 'Major.', 'Rev.', 'Jonkheer.', 'Dona.'], 'Rare.')
 test['Title'] = test['Title'].replace(['Countess.', 'Lady.', 'Sir.'], 'Royal.')
 test['Title'] = test['Title'].replace('Mlle.', 'Miss')
 test['Title'] = test['Title'].replace('Ms.', 'Miss.')
 test['Title'] = test['Title'].replace('Mme.', 'Mrs.')
df_test.loc[df_test['id'] == 3033,'budget'] = 250
df_test.loc[df_test['id'] == 3051,'budget'] = 50
df_test.loc[df_test['id'] == 3084,'budget'] = 337
df_test.loc[df_test['id'] == 3224,'budget'] = 4
df_test.loc[df_test['id'] == 3594,'budget'] = 25
df_test.loc[df_test['id'] == 3619,'budget'] = 500
df_test.loc[df_test['id'] == 3831,'budget'] = 3
df_test.loc[df_test['id'] == 3935,'budget'] = 500
df_test.loc[df_test['id'] == 4049,'budget'] = 995946
df_test.loc[df_test['id'] == 4424,'budget'] = 3
df_test.loc[df_test['id'] == 4460,'budget'] = 8
df_test.loc[df_test['id'] == 4555,'budget'] = 1200000
df_test.loc[df_test['id'] == 4624,'budget'] = 30
df_test.loc[df_test['id'] == 4645,'budget'] = 500
df_test.loc[df_test['id'] == 4709,'budget'] = 450
df_test.loc[df_test['id'] == 4839,'budget'] = 7
df_test.loc[df_test['id'] == 3125,'budget'] = 25
df_test.loc[df_test['id'] == 3142,'budget'] = 1
df_test.loc[df_test['id'] == 3201,'budget'] = 450
df_test.loc[df_test['id'] == 3222,'budget'] = 6
df_test.loc[df_test['id'] == 3545,'budget'] = 38
df_test.loc[df_test['id'] == 3670,'budget'] = 18
df_test.loc[df_test['id'] == 3792,'budget'] = 19
df_test.loc[df_test['id'] == 3881,'budget'] = 7
df_test.loc[df_test['id'] == 3969,'budget'] = 400
df_test.loc[df_test['id'] == 4196,'budget'] = 6
df_test.loc[df_test['id'] == 4221,'budget'] = 11
df_test.loc[df_test['id'] == 4222,'budget'] = 500
df_test.loc[df_test['id'] == 4285,'budget'] = 11
df_test.loc[df_test['id'] == 4319,'budget'] = 1
df_test.loc[df_test['id'] == 4639,'budget'] = 10
df_test.loc[df_test['id'] == 4719,'budget'] = 45
df_test.loc[df_test['id'] == 4822,'budget'] = 22
df_test.loc[df_test['id'] == 4829,'budget'] = 20
df_test.loc[df_test['id'] == 4969,'budget'] = 20
df_test.loc[df_test['id'] == 5021,'budget'] = 40
df_test.loc[df_test['id'] == 5035,'budget'] = 1
df_test.loc[df_test['id'] == 5063,'budget'] = 14
df_test.loc[df_test['id'] == 5119,'budget'] = 2
df_test.loc[df_test['id'] == 5214,'budget'] = 30
df_test.loc[df_test['id'] == 5221,'budget'] = 50
df_test.loc[df_test['id'] == 4903,'budget'] = 15
df_test.loc[df_test['id'] == 4983,'budget'] = 3
df_test.loc[df_test['id'] == 5102,'budget'] = 28
df_test.loc[df_test['id'] == 5217,'budget'] = 75
df_test.loc[df_test['id'] == 5224,'budget'] = 3
df_test.loc[df_test['id'] == 5469,'budget'] = 20
df_test.loc[df_test['id'] == 5840,'budget'] = 1
df_test.loc[df_test['id'] == 5960,'budget'] = 30
df_test.loc[df_test['id'] == 6506,'budget'] = 11
df_test.loc[df_test['id'] == 6553,'budget'] = 280
df_test.loc[df_test['id'] == 6561,'budget'] = 7
df_test.loc[df_test['id'] == 6582,'budget'] = 218
df_test.loc[df_test['id'] == 6638,'budget'] = 5
df_test.loc[df_test['id'] == 6749,'budget'] = 8
df_test.loc[df_test['id'] == 6759,'budget'] = 50
df_test.loc[df_test['id'] == 6856,'budget'] = 10
df_test.loc[df_test['id'] == 6858,'budget'] = 100
df_test.loc[df_test['id'] == 6876,'budget'] = 250
df_test.loc[df_test['id'] == 6972,'budget'] = 1
df_test.loc[df_test['id'] == 7079,'budget'] = 8000000
df_test.loc[df_test['id'] == 7150,'budget'] = 118
df_test.loc[df_test['id'] == 6506,'budget'] = 118
df_test.loc[df_test['id'] == 7225,'budget'] = 6
df_test.loc[df_test['id'] == 7231,'budget'] = 85
df_test.loc[df_test['id'] == 5222,'budget'] = 5
df_test.loc[df_test['id'] == 5322,'budget'] = 90
df_test.loc[df_test['id'] == 5350,'budget'] = 70
df_test.loc[df_test['id'] == 5378,'budget'] = 10
df_test.loc[df_test['id'] == 5545,'budget'] = 80
df_test.loc[df_test['id'] == 5810,'budget'] = 8
df_test.loc[df_test['id'] == 5926,'budget'] = 300
df_test.loc[df_test['id'] == 5927,'budget'] = 4
df_test.loc[df_test['id'] == 5986,'budget'] = 1
df_test.loc[df_test['id'] == 6053,'budget'] = 20
df_test.loc[df_test['id'] == 6104,'budget'] = 1
df_test.loc[df_test['id'] == 6130,'budget'] = 30
df_test.loc[df_test['id'] == 6301,'budget'] = 150
df_test.loc[df_test['id'] == 6276,'budget'] = 100
df_test.loc[df_test['id'] == 6473,'budget'] = 100
df_test.loc[df_test['id'] == 6842,'budget'] = 30
df_train.loc[df_train['id'] == 90,'budget'] = 30000000
df_train.loc[df_train['id'] == 118,'budget'] = 60000000
df_train.loc[df_train['id'] == 149,'budget'] = 18000000
df_train.loc[df_train['id'] == 464,'budget'] = 20000000
df_train.loc[df_train['id'] == 470,'budget'] = 13000000
df_train.loc[df_train['id'] == 513,'budget'] = 930000
df_train.loc[df_train['id'] == 797,'budget'] = 8000000
df_train.loc[df_train['id'] == 819,'budget'] = 90000000
df_train.loc[df_train['id'] == 850,'budget'] = 90000000
df_train.loc[df_train['id'] == 1007,'budget'] = 2
df_train.loc[df_train['id'] == 1112,'budget'] = 7500000
df_train.loc[df_train['id'] == 1131,'budget'] = 4300000
df_train.loc[df_train['id'] == 1359,'budget'] = 10000000
df_train.loc[df_train['id'] == 1542,'budget'] = 1
df_train.loc[df_train['id'] == 1570,'budget'] = 15800000
df_train.loc[df_train['id'] == 1571,'budget'] = 4000000
df_train.loc[df_train['id'] == 1714,'budget'] = 46000000
df_train.loc[df_train['id'] == 1721,'budget'] = 17500000
df_train.loc[df_train['id'] == 1885,'budget'] = 12
df_train.loc[df_train['id'] == 2091,'budget'] = 10
df_train.loc[df_train['id'] == 2268,'budget'] = 17500000
df_train.loc[df_train['id'] == 2491,'budget'] = 6
df_train.loc[df_train['id'] == 2602,'budget'] = 31000000
df_train.loc[df_train['id'] == 2612,'budget'] = 15000000
df_train.loc[df_train['id'] == 2696,'budget'] = 10000000
df_train.loc[df_train['id'] == 2801,'budget'] = 10000000
df_train.loc[df_train['id'] == 335,'budget'] = 2
df_train.loc[df_train['id'] == 348,'budget'] = 12
df_train.loc[df_train['id'] == 470,'budget'] = 13000000
df_train.loc[df_train['id'] == 513,'budget'] = 1100000
df_train.loc[df_train['id'] == 640,'budget'] = 6
df_train.loc[df_train['id'] == 696,'budget'] = 1
df_train.loc[df_train['id'] == 797,'budget'] = 8000000
df_train.loc[df_train['id'] == 850,'budget'] = 1500000
df_train.loc[df_train['id'] == 1199,'budget'] = 5
df_train.loc[df_train['id'] == 1282,'budget'] = 9
df_train.loc[df_train['id'] == 1347,'budget'] = 1
df_train.loc[df_train['id'] == 1755,'budget'] = 2
df_train.loc[df_train['id'] == 1801,'budget'] = 5
df_train.loc[df_train['id'] == 1918,'budget'] = 592
df_train.loc[df_train['id'] == 2033,'budget'] = 4
df_train.loc[df_train['id'] == 2118,'budget'] = 344
df_train.loc[df_train['id'] == 2252,'budget'] = 130
df_train.loc[df_train['id'] == 2256,'budget'] = 1
df_train.loc[df_train['id'] == 2696,'budget'] = 10000000<load_from_csv>
title_mapping = {"Mr.": 1, "Miss.": 2, "Mrs.": 3, "Master.": 4, "Royal.": 5, "Rare.": 6}
train['Title'] = train['Title'].map(title_mapping)
 train['Title'] = train['Title'].fillna(0)
 train.head()
train_add_features = pd.read_csv('.. /input/tmdb-additional-features/TrainAdditionalFeatures.csv')
 test_add_features = pd.read_csv('.. /input/tmdb-additional-features/TestAdditionalFeatures.csv')
 df_train = df_train.merge(train_add_features, how='left',left_on='imdb_id',right_on='imdb_id')
 df_test = df_test.merge(test_add_features, how='left',left_on='imdb_id',right_on='imdb_id')
<drop_column>
title_mapping = {"Mr.": 1, "Miss.": 2, "Mrs.": 3, "Master.": 4, "Royal.": 5, "Rare.": 6}
test['Title'] = test['Title'].map(title_mapping)
 test['Title'] = test['Title'].fillna(0)
train_data_cols = df_train.columns
test_data_cols = df_test.columns
for col_test in test_data_cols:
if str(col_test)not in train_data_cols and str(col_test)!= 'year_bucket' and not str(col_test ).__contains__('0')and not str(col_test ).__contains__('1')and not str(col_test ).__contains__('2'):
print('dropping ' + str(col_test))
df_test.drop(str(col_test),axis=1,inplace=True)
 for col_test in train_data_cols:
if str(col_test)not in test_data_cols and str(col_test)!= 'year_bucket' and not str(col_test ).__contains__('0')and not str(col_test ).__contains__('1')and not str(col_test ).__contains__('2'):
if(str(col_test)== 'revenue'):
continue
print('dropping ' + str(col_test))
df_train.drop(str(col_test),axis=1,inplace=True )<feature_engineering>
test = test.drop(['Ticket'], axis = 1)
 test = test.drop(['Name'], axis = 1)
 test = test.drop(['Parch'], axis = 1)
 test = test.drop(['Fare','SibSp'], axis = 1)
 train.drop(['Name', 'Ticket'], axis = 1, inplace = True)
 train = train.drop(['Parch'], axis = 1)
 train = train.drop(['Fare','SibSp'], axis = 1 )
def is_collection(row):
if(row == 0):
return 0
else:
return 1
def is_homepage(row):
if(row == 0):
return 0
else:
return 1
df_train['is_collection'] = df_train.belongs_to_collection.apply(is_collection)
 df_train['is_homepage'] = df_train.homepage.apply(is_homepage)
 df_test['is_collection'] = df_test.belongs_to_collection.apply(is_collection)
 df_test['is_homepage'] = df_test.homepage.apply(is_homepage)
 df_train['has_production_comp'] = df_train.production_companies.apply(is_collection)
 df_test['has_production_comp'] = df_test.production_companies.apply(is_collection)
 df_train['has_production_count'] = df_train.production_countries.apply(is_collection)
 df_test['has_production_count'] = df_test.production_countries.apply(is_collection)
 df_train['has_keywords'] = df_train.Keywords.apply(is_collection)
 df_test['has_keywords'] = df_test.Keywords.apply(is_collection)
 df_train['has_tagline'] = df_train.tagline.apply(is_collection)
 df_test['has_tagline'] = df_test.tagline.apply(is_collection )<drop_column>
X_train, X_test, y_train, y_test = train_test_split(train.drop(['Survived','PassengerId'], axis=1),
train['Survived'], test_size = 0.2,
random_state = 0 )
df_train.drop('belongs_to_collection',axis=1,inplace=True)
 df_train.drop('collection',axis=1,inplace=True)
 df_train.drop('genres',axis=1,inplace=True)
 df_train.drop('homepage',axis=1,inplace=True)
 df_train.drop('imdb_id',axis=1,inplace=True)
 df_train.drop('original_language',axis=1,inplace=True)
 df_train.drop('original_title',axis=1,inplace=True)
 df_train.drop('overview',axis=1,inplace=True)
 df_train.drop('poster_path',axis=1,inplace=True)
 df_train.drop('production_companies',axis=1,inplace=True)
 df_train.drop('production_countries',axis=1,inplace=True)
 df_train.drop('release_date',axis=1,inplace=True)
 df_train.drop('spoken_languages',axis=1,inplace=True)
 df_train.drop('status',axis=1,inplace=True)
 df_train.drop('tagline',axis=1,inplace=True)
 df_train.drop('title',axis=1,inplace=True)
 df_train.drop('Keywords',axis=1,inplace=True)
 df_train.drop('cast',axis=1,inplace=True)
 df_train.drop('crew',axis=1,inplace=True)
 df_test.drop('belongs_to_collection',axis=1,inplace=True)
 df_test.drop('collection',axis=1,inplace=True)
 df_test.drop('genres',axis=1,inplace=True)
 df_test.drop('homepage',axis=1,inplace=True)
 df_test.drop('imdb_id',axis=1,inplace=True)
 df_test.drop('original_language',axis=1,inplace=True)
 df_test.drop('original_title',axis=1,inplace=True)
 df_test.drop('overview',axis=1,inplace=True)
 df_test.drop('poster_path',axis=1,inplace=True)
 df_test.drop('production_companies',axis=1,inplace=True)
 df_test.drop('production_countries',axis=1,inplace=True)
 df_test.drop('release_date',axis=1,inplace=True)
 df_test.drop('spoken_languages',axis=1,inplace=True)
 df_test.drop('status',axis=1,inplace=True)
 df_test.drop('tagline',axis=1,inplace=True)
 df_test.drop('title',axis=1,inplace=True)
 df_test.drop('Keywords',axis=1,inplace=True)
 df_test.drop('cast',axis=1,inplace=True)
 df_test.drop('crew',axis=1,inplace=True )<feature_engineering>
logisticRegression = LogisticRegression(max_iter = 30000)
 logisticRegression.fit(X_train, y_train)
df_train['revenue'].fillna(0,inplace=True)
 df_train['log_revenue'] = np.log(df_train['revenue'])
 df_train['budget'].fillna(0,inplace=True)
 df_train['log_budget'] = np.log(df_train['budget'])
 df_train.drop('revenue',axis=1,inplace=True)
 df_train.drop('budget',axis=1,inplace=True)
 df_test['budget'].fillna(0,inplace=True)
 df_test['log_budget'] = np.log(df_test['budget'])
 df_test.drop('budget',axis=1,inplace=True)
 df_train['budget_year'] = df_train['log_budget'] / df_train['release_date_year']
df_test['budget_year'] = df_test['log_budget'] / df_test['release_date_year']
df_train['popularity_year'] = df_train['popularity'] / df_train['release_date_year']
df_test['popularity_year'] = df_test['popularity'] / df_test['release_date_year']
df_train.runtime.fillna(0,inplace=True)
 df_test.runtime.fillna(0,inplace=True)
 bins = [0, 40, 50, 60, 70, 80,90, 100,110,120,130,140,150,160,170,180,190,200,210,220,230,240,250,260,270,280,290,300,310,320,330,340,350]
labels = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32]
df_train['binned_runtime'] = pd.cut(df_train['runtime'], bins=bins, labels=labels)
 df_test['binned_runtime'] = pd.cut(df_test['runtime'], bins=bins, labels=labels)
 df_train['binned_runtime'] = pd.to_numeric(df_train['binned_runtime'])
 df_test['binned_runtime'] = pd.to_numeric(df_test['binned_runtime'] )<prepare_x_and_y>
predictions = logisticRegression.predict(X_test)
 acc_LOG = round(accuracy_score(predictions, y_test)* 100, 2)
 print(acc_LOG)
 print(predictions)
num_rounds = 10000
y_label = df_train['log_revenue']
X = df_train.drop('log_revenue',axis=1)
 X.drop('id',axis=1,inplace=True)
 X['cast_1_gender'] = X['cast_1_gender'].apply(int)
 X['cast_2_gender'] = X['cast_2_gender'].apply(int)
 X['cast_3_gender'] = X['cast_3_gender'].apply(int)
 X = X.replace([np.inf, -np.inf], 0 ).fillna(0)
 X.fillna(0,inplace=True)
 X_test = df_test.copy()
X_test.fillna(0,inplace=True)
 X_test_id = df_test['id']
X_test.drop('id',axis=1,inplace=True)
 X_test['cast_1_gender'] = X_test['cast_1_gender'].apply(int)
 X_test['cast_2_gender'] = X_test['cast_2_gender'].apply(int)
 X_test['cast_3_gender'] = X_test['cast_3_gender'].apply(int)
 X_test = X_test.replace([np.inf, -np.inf], 0 ).fillna(0)
 X_test.fillna(0,inplace=True)
 X = X.loc[:,~X.columns.duplicated() ]
X_test = X_test.loc[:,~X_test.columns.duplicated() ]
dtrain = xgb.DMatrix(data=X,label=y_label)
 dtest = xgb.DMatrix(data=X_test)
 watchlist = [(dtest, 'eval'),(dtrain, 'train')]
param = {'eta':0.01, 'max_depth': 2, 'booster': 'gbtree', 'colsample_bytree' : 0.3,'subsample' : 0.9}
res = xgb.cv(param,dtrain,num_rounds,10,metrics={'rmse'})
 print('cross validation')
 print(res)
 bst = xgb.train(param,dtrain,num_rounds)
 y_pred = bst.predict(dtrain)
 print('Train rmsle')
 print(mean_squared_error(y_label, y_pred)** 0.5)
 y_pred_reg = bst.predict(dtest)
 y_pred_reg_exp = np.exp(y_pred_reg)
 y_pred_reg_exp = pd.DataFrame(y_pred_reg_exp)
 test_preds = pd.concat([X_test_id,y_pred_reg_exp],axis=1)
 test_preds.columns = ['id','revenue']
test_preds.to_csv('submission_xgb.csv',index=False )<load_from_csv>
print(confusion_matrix(y_test, predictions))
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )<feature_engineering>
accuracy=(( 88+50)/(88+50+22+19))
print('accuracy is: ',(round(accuracy, 2)*100))
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000<feature_engineering>
randomforest = RandomForestClassifier(random_state = 5, criterion = 'gini', max_depth = 10, max_features = 'auto', n_estimators = 500)
 randomforest.fit(X_train, y_train)
 pred = randomforest.predict(X_test)
 acc_randomforest = round(accuracy_score(pred, y_test)* 100, 2)
 print(acc_randomforest)
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30<count_values>
gbk = GradientBoostingClassifier()
gbk.fit(X_train, y_train)
 pred = gbk.predict(X_test)
 acc_gbk = round(accuracy_score(pred, y_test)* 100, 2)
 print(acc_gbk )
pd.DataFrame(train.budget.value_counts() ).T<filter>
see={'TECHNIQUE':['RANDOM FOREST','LOGISTIC REGRESSION','GRADIENT BOOSTING'],'ACCURACY':[acc_randomforest,acc_LOG,acc_gbk]}
mod=pd.DataFrame(see)
 mod
<feature_engineering><EOS> 
ids = test['PassengerId']
predictions = randomforest.predict(test.drop('PassengerId', axis=1))
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )