Kaggle competition: Don't Get Kicked!
less_messy_data = raw_data.drop(messy_features, axis=1)
 n_features = len(list(less_messy_data))
print(f"We removed 5 features so our training data now has dimension:
1460x{n_features}" )<data_type_conversions>
train_data=pd.read_csv('/kaggle/input/DontGetKicked/training.csv')
 train_data.head()
<define_variables>
test_data=pd.read_csv('/kaggle/input/DontGetKicked/test.csv')
 test_data.head()
first_elements = [less_messy_data[i].iloc[0] for i in list(less_messy_data)]
categorical_features = []
numerical_features = []
for i in range(len(first_elements)) :
if type(first_elements[i])== str:
categorical_features.append(list(less_messy_data)[i])
 else:
numerical_features.append(list(less_messy_data)[i] )<count_values>
train_data.isnull().sum()
for feature in categorical_features:
most_occurring_category = less_messy_data[feature].value_counts().index[0]
less_messy_data[feature] = less_messy_data[feature].fillna(most_occurring_category)
 less_messy_data.head(20 )<feature_engineering>
test_data.isnull().sum()
for feature in numerical_features:
feature_mean = less_messy_data[feature].mean()
less_messy_data[feature] = less_messy_data[feature].fillna(np.floor(feature_mean))
less_messy_data.head(20 )<categorify>
train_data['IsBadBuy'].value_counts()
dummies = []
for cat in categorical_features:
dummies.append(pd.get_dummies(less_messy_data[cat]))
less_messy_data = less_messy_data.drop([cat], axis=1)
 n_dummies = len(dummies)
 dummies = [less_messy_data] + dummies
less_messy_data = pd.concat(dummies, axis=1)
 less_messy_data.head(5 )<feature_engineering>
train_data['Model'].value_counts()
columns = list(less_messy_data)
 add_num = 1
for _ in range(10):
for i in range(1, len(columns)) :
if columns[i] in columns[:i]:
columns[i] = columns[i] + str(add_num)
 add_num += 1
less_messy_data.columns = columns<normalization>
train_data.drop('Model',axis=1,inplace=True)
 test_data.drop("Model",axis=1,inplace=True )
remaining_numerical_features = list(less_messy_data)[:(n_features - n_dummies)]
scaler = MinMaxScaler()
scaled = scaler.fit_transform(less_messy_data[remaining_numerical_features])
 less_messy_data[remaining_numerical_features] = scaled
less_messy_data.head(5 )<split>
train_data['Trim'].value_counts()
test_data = less_messy_data.iloc[raw_len:, :]
less_messy_data = less_messy_data.iloc[:raw_len, :]<define_search_space>
train_data.drop('Trim',inplace=True,axis=1)
 test_data.drop('Trim',inplace=True,axis=1 )
test_parameters = {'n_estimators': [150, 300, 450, 600, 750, 1000],
'learning_rate': [0.01, 0.07, 0.1, 0.25, 0.4, 0.6, 1],
'subsample': [0.3, 0.5, 0.7, 0.9],
'max_depth': [3, 4, 5, 6, 7, 8, 9],
'colsample_bytree': [0.5, 0.7, 0.9],
'min_child_weight': [1, 2, 3, 4]
}
xgb_randsearch = RandomizedSearchCV(XGBRegressor() ,
param_distributions = test_parameters,
n_iter = 100,
scoring='neg_mean_squared_error',
verbose = 3,
n_jobs = -1)
 xgb_randsearch.fit(less_messy_data, target )<save_to_csv>
train_data['SubModel'].value_counts()
prediction = xgb_randsearch.predict(test_data)
 prediction = pd.DataFrame(prediction, columns=['SalePrice'])
 submission = np.zeros(( len(prediction), 2))
submission[:, 0] = test_Id.values.flatten()
submission[:, 1] = prediction.values.flatten()
submission = pd.DataFrame(submission, columns=['Id', 'SalePrice'])
 submission.Id = submission.Id.astype(int)
 submission.to_csv('.. /.. /.. /kaggle/working/base_line_submission.csv', index=False )<data_type_conversions>
train_data.drop('SubModel',inplace=True,axis=1)
 test_data.drop('SubModel',inplace=True,axis=1 )
labels = traindata.iloc[:,0].values.astype('int32')
 x_train = traindata.iloc[:,1:].values.astype('int32')
 x_test = testdata.values.astype('int32')
 x_train = x_train.reshape(-1,28,28,1)
 x_test = x_test.reshape(-1,28,28,1)
 y_train = tf.keras.utils.to_categorical(labels)
 print(x_train.shape)
 print(x_test.shape)
 print(y_train )<split>
train_data['Color'].value_counts()
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.9)
 print(x_train.shape)
 print(y_train.shape)
 print(x_val.shape)
 print(y_val.shape)
<choose_model_class>
test_data['Color'].value_counts()
image_generator = ImageDataGenerator(
 rescale = 1/255.0,
rotation_range = 20,
width_shift_range = 0.1,
height_shift_range = 0.1,
zoom_range = 0.1)
 validation_generator = ImageDataGenerator(
 rescale = 1/255.0 )<choose_model_class>
train_data['Color'].fillna(value='Color_Unknown',inplace=True)
 test_data['Color'].fillna(value='Color_Unknown',inplace=True )
model = models.Sequential()
model.add(layers.Conv2D(32,(3, 3), activation='relu', input_shape=(28, 28, 1)))
 model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(64,(3, 3), activation='relu'))
model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Flatten())
 model.add(layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.25))
model.add(layers.Dense(10, activation='softmax'))
model.summary()<train_model>
print("Number of null values in Color column "+str(train_data['Color'].isnull().sum()))
print("Number of null values in Color column "+str(test_data['Color'].isnull().sum()))
model.compile(optimizer='Adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
 history = model.fit_generator(image_generator.flow(x_train, y_train,batch_size = 512),
epochs =100,validation_data = validation_generator.flow(x_val, y_val, batch_size = 512), verbose=1 )<save_to_csv>
train_data['Transmission'].value_counts()
predict_generator = ImageDataGenerator(
 rescale = 1/255.0)
 predictions = model.predict_generator(predict_generator.flow(x_test, shuffle = False), verbose=0)
 predictions = np.argmax(predictions,axis=1)
 print(predictions)
 pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) , "Label": predictions} ).to_csv("preds.csv", index=False, header=True )<set_options>
test_data['Transmission'].value_counts()
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings("ignore",category=DeprecationWarning)
 print("<load_from_csv>
train_data[train_data['Transmission']=='Manual']
dir_contents = os.listdir('.. /input/')
 print("Dataset contents : {}".format(dir_contents))
print("Size of Training dataset : " + str(round(os.path.getsize('.. /input/train.csv')/(1024*2), 2)) +" KB")
 print("Size of Testing dataset : " + str(round(os.path.getsize('.. /input/test.csv')/(1024*2), 2)) +" KB" )<load_from_csv>
train_data['Transmission'].replace("Manual","MANUAL",inplace=True )
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 passengerIds = test_data['PassengerId']
print("Shape of Train dataset : {}".format(train_data.shape))
print("Shape of Test dataset : {}".format(test_data.shape))<count_values>
train_data['Transmission'].value_counts()
train_data['Survived'].value_counts(normalize=True)* 100<filter>
train_data['Transmission'].fillna(value="Transmission_unk",inplace=True)
 test_data['Transmission'].fillna(value="Transmission_unk",inplace=True )
train_data[(train_data['Age'] < 10)&(train_data['SibSp'] == 0)&(train_data['Parch'] == 0)]<count_values>
train_data['WheelTypeID'].value_counts()
print("
print(train_data['Sex'].value_counts())
 print("*************************")
 print("Percentage of Males who survived : {}%".format(round(train_data[(train_data['Sex']=='male')& 
(train_data['Survived']==1)].shape[0] * 100
/ train_data.shape[0], 2)))
 print("Percentage of Females who survived : {}%".format(round(train_data[(train_data['Sex']=='female')& 
(train_data['Survived']==1)].shape[0] * 100
/ train_data.shape[0], 2)) )<define_variables>
train_data.drop('WheelTypeID',inplace=True,axis=1)
 test_data.drop('WheelTypeID',inplace=True,axis=1 )
full_data = [train_data , test_data]
len(full_data )<feature_engineering>
train_data['WheelType'].value_counts()
def getTitleFromName(nameText):
title = str(nameText.split(', ')[1])
 title_search = re.search('([A-Za-z]+)\.', title ).group(1)
 return title_search
for data in full_data:
imp = SimpleImputer(missing_values=np.nan, strategy='median')
 data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
data['IsAlone'] = 0
data.loc[data['FamilySize']==1, 'IsAlone'] = 1
data['CabinAllotment'] = 0
data.loc[data['Cabin'].notnull() , 'CabinAllotment'] = 1
data['Embarked'] = data['Embarked'].fillna('S')
 data['Sex'] = data['Sex'].map({'male': 0, 'female': 1} ).astype(int)
 data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 data['Age'] = imp.fit_transform(data[['Age', 'Sex', 'FamilySize']])[:,0].astype(int)
 data['Fare'] = imp.fit_transform(data[['Fare', 'FamilySize', 'CabinAllotment']])
 data['PerTicket'] = data['Fare'] / data['FamilySize']
data.loc[ data['PerTicket'] <= 7.25, 'PerTicket'] = 0
data.loc[(data['PerTicket'] > 7.25)&(data['PerTicket'] <= 8.3), 'PerTicket'] = 1
data.loc[(data['PerTicket'] > 8.3)&(data['PerTicket'] <= 23.667), 'PerTicket'] = 2
data.loc[ data['PerTicket'] > 23.667, 'PerTicket'] = 3
data['PerTicket'] = data['PerTicket'].astype(int)
 data.loc[(data['Age'] >=0)&(data['Age'] <= 14), 'Age'] = 0
data.loc[(data['Age'] >=15)&(data['Age'] <= 24), 'Age'] = 1
data.loc[(data['Age'] >=25)&(data['Age'] <= 64), 'Age'] = 2
data.loc[data['Age'] >=65, 'Age'] = 3
data['Age'] = data['Age'].astype(int)
 data['Title'] = data['Name'].apply(getTitleFromName)
 data['Title'] = data['Title'].replace('Ms', 'Miss')
 data['Title'] = data['Title'].replace('Mlle', 'Miss')
 data['Title'] = data['Title'].replace('Mme', 'Mrs')
 data['Status'] = "General"
data.loc[data['Title'] == 'Capt','Status'] = 'Military'
data.loc[data['Title'] == 'Col','Status'] = 'Military'
data.loc[data['Title'] == 'Countess','Status'] = 'Political'
data.loc[data['Title'] == 'Don','Status'] = 'Military'
data.loc[data['Title'] == 'Dr','Status'] = 'General'
data.loc[data['Title'] == 'Jonkheer','Status'] = 'Political'
data.loc[data['Title'] == 'Lady','Status'] = 'Political'
data.loc[data['Title'] == 'Major','Status'] = 'Military'
data.loc[data['Title'] == 'Master','Status'] = 'General'
data.loc[data['Title'] == 'Rev','Status'] = 'Political'
data.loc[data['Title'] == 'Sir','Status'] = 'Military'
data['Rank'] = 0
data.loc[data['Title'] == 'Capt', 'Rank'] = 1
data.loc[data['Title'] == 'Col', 'Rank'] = 1
data.loc[data['Title'] == 'Major', 'Rank'] = 2
data.loc[data['Title'] == 'Don', 'Rank'] = 2
data.loc[data['Title'] == 'Sir', 'Rank'] = 0
data.loc[data['Title'] == 'Dr', 'Rank'] = 1
data.loc[data['Title'] == 'Master', 'Rank'] = 0
data.loc[data['Title'] == 'Miss', 'Rank'] = 0
data.loc[data['Title'] == 'Mr', 'Rank'] = 0
data.loc[data['Title'] == 'Mrs', 'Rank'] = 0
data.loc[data['Title'] == 'Countess', 'Rank'] = 2
data.loc[data['Title'] == 'Jonkheer', 'Rank'] = 0
data.loc[data['Title'] == 'Lady', 'Rank'] = 1
data.loc[data['Title'] == 'Rev', 'Rank'] = 1
data['Status'] = data['Status'].map({ 'General': 0, 'Military': 1, 'Political': 2})
 remove_features = ['PassengerId', 'Ticket', 'Cabin', 'SibSp', 'Parch', 'Name', 'Title', 'Fare']
train_data = train_data.drop(remove_features, axis=1)
 test_data = test_data.drop(remove_features, axis=1)
 train_data.head(5 )<prepare_x_and_y>
test_data['WheelType'].value_counts()
y_train = train_data['Survived'].values
X_train = train_data.drop('Survived', axis=1)
 X_test = test_data.values<categorify>
train_data['WheelType'].fillna(value='WheelType_unk',inplace=True)
 test_data['WheelType'].fillna(value='WheelType_unk',inplace=True )
encoding_clf = OneHotEncoder()
train_data_new = encoding_clf.fit_transform(X_train ).astype('int')
 test_data_new = encoding_clf.transform(X_test ).astype('int' )<concatenate>
train_data['WheelType'].value_counts()
y_train_new = tf.keras.utils.to_categorical(y_train, num_classes=2 )<choose_model_class>
train_data['Nationality'].value_counts()
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Dense(35, input_shape=(35,), activation=tf.nn.relu, kernel_initializer='he_uniform'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(100, activation=tf.nn.relu, kernel_initializer='he_uniform'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(40, activation=tf.nn.relu,kernel_initializer='he_uniform'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(10, activation=tf.nn.relu,kernel_initializer='he_uniform'))
model.add(tf.keras.layers.Dense(2, activation=tf.nn.softmax))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
 relu_model = model.fit(train_data_new, y_train_new, epochs=500, batch_size=7, verbose=1, validation_split=0.20 )<predict_on_test>
test_data['Nationality'].value_counts()
pred = model.predict_classes(test_data_new)
 pred<save_to_csv>
train_data['Nationality'].fillna(value='Nationality_unk',inplace=True)
 test_data['Nationality'].fillna(value='Nationality_unk',inplace=True )
DL_Submission = pd.DataFrame({ 'PassengerId': passengerIds,
'Survived': pred })
 DL_Submission.to_csv("DL_Submission.csv", index=False )<prepare_x_and_y>
train_data['Size'].value_counts()
<train_on_grid>
test_data['Size'].value_counts()
alpha_range = list([10** i for i in range(-10, 6, 1)] + [2**i for i in range(-5, -1, 1)])
 loss_range = list(['hinge', 'log', 'modified_huber', 'squared_hinge'])
 penalty_range = list(['l1', 'l2', 'elasticnet'])
 parameters = {'loss': loss_range, 'penalty': penalty_range, 'alpha': alpha_range}
model = SGDClassifier()
g_clf = GridSearchCV(model, parameters, cv=10, n_jobs=-1, scoring="accuracy")
 g_clf.fit(g_xtrain, g_ytrain)
 print("Model fitted perfectly.")
 print("Best Score(TRAIN): {}".format(g_clf.best_score_))
print("Best Params : {}".format(g_clf.best_params_))<create_dataframe>
train_data['Size'].fillna(value='Size_unk',inplace=True)
 test_data['Size'].fillna(value="Size_unk",inplace=True )
<choose_model_class>
train_data['TopThreeAmericanName'].value_counts()
<choose_model_class>
test_data['TopThreeAmericanName'].value_counts()
<load_from_csv>
train_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True)
 test_data['TopThreeAmericanName'].fillna(value='Top_unk',inplace=True )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 datasets = [train, test]
train.tail()<categorify>
train_data['PRIMEUNIT'].value_counts()
epic_titles = set(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'])
 female_titles = set(['Lady', 'Countess', 'Dona', 'Mlle', 'Ms', 'Mme'])
 deck = {"A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "U": 8}
def mapFare(fare):
if fare <= 7.91:
return 0
elif fare > 7.91 and fare <= 14.454:
return 1
elif fare > 14.454 and fare <= 31:
return 2
elif fare > 31:
return 3
def mapAge(age):
if age <= 16:
return 0
elif age > 16 and age <= 32:
return 1
elif age > 32 and age <= 48:
return 2
elif age > 48 and age <= 64:
return 3
elif age > 64:
return 4
for dataset in datasets:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
dataset.drop([ 'SibSp', 'Parch'], axis=1, inplace=True)
 dataset['Title'] = dataset['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 dataset['Title'] = dataset['Title'].apply(lambda x: x if x not in epic_titles else 'Epic')
 dataset['Sex'] = dataset['Sex'].fillna(dataset['Title'].apply(lambda x: 'female' if x in female_titles else 'male'))
dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Epic": 5}
dataset['Title'] = dataset['Title'].apply(lambda x: title_mapping[x] if x in title_mapping else 5)
 dataset['Sex'] = dataset['Sex'].map({"female": 0, "male": 1} ).astype(int)
 dataset['Age'] = dataset['Age'].fillna(dataset['Age'].mean())
 dataset['Age'] = dataset['Age'].apply(lambda x: mapAge(x))
dataset['Age'] = dataset['Age'].astype(int)
 dataset['Embarked'] = dataset['Embarked'].fillna('S')
 dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 dataset['Cabin'] = dataset['Cabin'].fillna("U0")
 dataset['Deck'] = dataset['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 dataset['Deck'] = dataset['Deck'].map(deck)
 dataset['Deck'] = dataset['Deck'].fillna(0)
 dataset['Deck'] = dataset['Deck'].astype(int)
 dataset.drop(['Cabin'], axis=1, inplace=True)
 familysizeDummies = pd.get_dummies(dataset['FamilySize'], prefix='familysize')
 dataset = pd.concat([dataset,familysizeDummies],axis=1)
 dataset = dataset.drop(columns=['FamilySize'])
 print(dataset.columns)
<groupby>
test_data['PRIMEUNIT'].value_counts()
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).agg(['mean', 'count', 'sum'] )<prepare_x_and_y>
train_data['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True)
 test_data['PRIMEUNIT'].fillna(value="Prime_unk",inplace=True )
y = train[['Survived']]
x = train.drop(columns=['Name', 'Ticket', 'Survived', 'PassengerId'])
 x.columns<train_model>
train_data['AUCGUART'].replace("AGREEN","GREEN",inplace=True)
 test_data['AUCGUART'].replace("ARED","RED",inplace=True )
train_x, test_x, train_y, test_y = train_test_split(x, np.ravel(y))<compute_train_metric>
train_data['AUCGUART'].fillna(value="AUC_unk",inplace=True)
 test_data['AUCGUART'].fillna(value="AUC_unk",inplace=True )
logreg_model = LogisticRegression()
logreg_model.fit(train_x, train_y)
 preds = logreg_model.predict(test_x)
 acc = accuracy_score(preds, test_y)
 print(f'Accuracy: {acc}')
<normalization>
train_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1)
 test_data.drop(['MMRAcquisitionAuctionAveragePrice','MMRAcquisitionAuctionCleanPrice',
'MMRAcquisitionRetailAveragePrice','MMRAcquisitonRetailCleanPrice',
'MMRCurrentAuctionAveragePrice','MMRCurrentAuctionCleanPrice',
'MMRCurrentRetailAveragePrice','MMRCurrentRetailCleanPrice'],
inplace=True,axis=1 )
scaler = MinMaxScaler(feature_range=[0, 1])
 train_x_scaled = scaler.fit_transform(train_x.iloc[1:, 0:8])
 test_x_scaled = scaler.fit_transform(test_x.iloc[1:, 0:len(train_x.columns)])
 train_x_centered = train_x - np.mean(train_x, axis = 0)
 test_x_centered = train_x - np.mean(test_x, axis = 0)
 train_x_centered.describe()<predict_on_test>
train_data.drop('PurchDate',axis=1,inplace=True)
 test_data.drop("PurchDate",axis=1,inplace=True )
for i in range(1, len(train_x.columns)) :
pca = PCA(n_components=i)
 pca.fit(train_x)
 train_x_pca = pca.transform(train_x)
 test_x_pca = pca.transform(test_x)
 logreg_model.fit(train_x_pca, train_y)
 preds_pca = logreg_model.predict(test_x_pca)
 acc_pca = accuracy_score(preds_pca, test_y)
 print(f'Accuracy with {i} principal components: {acc_pca}' )<init_hyperparams>
train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object'
warnings.simplefilter(action='ignore', category=FutureWarning)
 def plurality_value(parent_examples):
r = 0
return parent_examples['Survived'].value_counts().index[r]
def allValuesEqual(examples):
return len(examples['Survived'].value_counts())== 1
def partition(examples, column, value):

 true_rows, false_rows = examples.copy() , examples.copy()
for index, row in examples.iterrows() :
if row[column] == value:
false_rows = false_rows.drop([index])
 else:
true_rows = true_rows.drop([index])
 return true_rows, false_rows
def get_entropy(examples):
number_of_ones = 0
for index, row in examples.iterrows() :
if row['Survived'] == 1:
number_of_ones += 1
q = number_of_ones/len(examples)if len(examples)!= 0 else 0
return -(q*math.log2(q)+(1-q)*math.log2(( 1-q)))if(q != 0 and q!= 1)else 0
def get_remainder(one_rows, two_rows, entropy1, entropy2):
total = len(one_rows)+ len(two_rows)
 p, n = len(one_rows), len(two_rows)
 return(p/total)*entropy1 +(n/total)*entropy2
def find_best_split(examples, attributes, current_entropy):

 best_gain = 0
best_attribute = attributes[0]
info_gains = []
for attribute in attributes:
ones, twoes = partition(examples, attribute, 0)
 entropy1, entropy2 = get_entropy(ones), get_entropy(twoes)
 remainder = get_remainder(ones, twoes, entropy1, entropy2)
 info_gain = current_entropy - remainder
info_gains.append(info_gain)
 if info_gain >= best_gain:
best_gain, best_attribute = info_gain, attribute
return(best_attribute, best_gain)if best_gain > 0 else(random.choice(attributes), best_gain)
 def decision_tree_learning(examples, attributes, parent_examples, info_gain):

 if len(examples)== 0:
return plurality_value(parent_examples)
 elif allValuesEqual(examples):
return examples.iloc[0]['Survived']
elif len(attributes)== 0:
return plurality_value(examples)
 elif info_gain < 0.005:
return plurality_value(examples)
 current_entropy = get_entropy(examples)
 best_attribute, info_gain = find_best_split(examples, attributes, current_entropy)[0], find_best_split(examples, attributes, current_entropy)[1]
tree = Tree(best_attribute)
 attr_copy = attributes.copy()
attr_copy = attr_copy.drop(labels=best_attribute)
 for value in train[best_attribute].unique() :
next_examples = examples.loc[examples[best_attribute] == value]
subtree = decision_tree_learning(next_examples, attr_copy, examples, info_gain)
 tree.add_branch(value, subtree)
 return tree
class Tree:
def __init__(self, root, branches=None):
self.root = root
if branches == None:
branches = {}
self.branches = branches
def add_branch(self, label, branch):
self.branches[label] = branch
def predict(tree, example):
value = example[tree.root]
if isinstance(tree.branches[value], Tree):
return predict(tree.branches[value], example)
 else:
return tree.branches[value]
def print_tree(tree, value="", level=0):
print("\t" * level + str(value), end = " -> ")
 attribute = tree.root if isinstance(tree, Tree)else str(tree)
 print(attribute, end = "
")
 if isinstance(tree, Tree):
for label, subtree in tree.branches.items() :
print_tree(subtree, label, level + 1)
 def main() :
print("--------------------------------------------")
 train_new = train.copy()
train_new = train_new.drop(columns=['Name', 'Ticket', 'PassengerId'])
 test_new = test.copy()
y = train_new['Survived']
train_x, test_x, train_y, test_y = train_test_split(train_new, y, test_size=0.25)
 train_x_attributes = train_x.drop(columns=['Survived'] ).columns
tree = decision_tree_learning(train_x, train_x_attributes, train_x, 1)
 print_tree(tree)
 predictions = []
for index, example in test_x.iterrows() :
try:
prediction = predict(tree, example)
 except:
prediction = 0
print("her")
 predictions.append(prediction)
 acc = accuracy_score(test_y, predictions)
 print("Accuracy with my greedy Importance Decision Tree: ", acc)
 real_tree = decision_tree_learning(train, train_x_attributes, train, 1)
 predictions = []
for index, example in test.iterrows() :
try:
prediction = predict(real_tree, example)
 except:
prediction = 0
print("her")
 predictions.append(prediction)
 pred_df = test[['PassengerId']].copy()
pred_df['Survived'] = predictions
pred_df.to_csv('submission_DTC.csv', index=False)
<save_to_csv>
not_categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes!='object']
def predict_test(model, name):
test.fillna(0, inplace=True)
 model.fit(x, np.ravel(y))
test_data = test.drop(columns=['Name', 'Ticket', 'PassengerId'])
 preds = model.predict(test_data)
 pred_df = test[['PassengerId']].copy()
pred_df['Survived'] = preds
pred_df.to_csv(f'submission_{name}.csv', index=False )<choose_model_class>
for i in not_categorical:
maximum=np.max(train_data[i])
 train_data[i]=train_data[i]/maximum
maximum_test=np.max(test_data[i])
 test_data[i]=test_data[i]/maximum_test
names = ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Gaussian Process",
"Decision Tree", "Random Forest", "Neural Net", "AdaBoost",
"Naive Bayes", "QDA"]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025),
SVC(gamma=2, C=1),
GaussianProcessClassifier(1.0 * RBF(1.0)) ,
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5, n_estimators=800),
MLPClassifier(alpha=1, max_iter=500),
AdaBoostClassifier() ,
GaussianNB() ,
QuadraticDiscriminantAnalysis()
]
for i, classifier in enumerate(classifiers):
classifier.fit(train_x, train_y)
 preds = classifier.predict(test_x)
 print(f"Predicting with {names[i]}, accuracy: {accuracy_score(preds, test_y)}")
 predict_test(classifier, names[i] )<import_modules>
categorical=train_data.drop(['RefId','IsBadBuy'],axis=1 ).columns[train_data.drop(['RefId','IsBadBuy'],axis=1 ).dtypes=='object']
<choose_model_class>
train_data[categorical[0]]
<train_on_grid>
pd.get_dummies(train_data[categorical[0]] )
<train_model>
for i in categorical:
dummies=pd.get_dummies(train_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
train_data=pd.concat([train_data,dummies],axis=1)
 train_data.drop(i,inplace=True,axis=1)
 dummies=pd.get_dummies(test_data[i])
 dummies.columns=str(i)+'_'+dummies.columns
test_data=pd.concat([test_data,dummies],axis=1)
 test_data.drop(i,inplace=True,axis=1 )
<import_modules>
for i in train_data.drop('IsBadBuy',axis=1 ).columns:
if i not in test_data.columns:
test_data[i]=np.zeros(len(test_data))
sns.set_style("whitegrid")
 notebookstart = time.time()
pd.options.display.max_colwidth = 500
print("Keras Version: ",keras.__version__)
 print("Tensorflow Version: ", tensorflow.__version__)
 EMBEDDING_FILES = [
'.. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim',
'.. /input/gensim-embeddings-dataset/glove.840B.300d.gensim'
]
seed = 25
N_ROWS = None
BATCH_SIZE = 64
EPOCHS = 100
N_CLASSES = 1
f1_strategy = 'macro'
MAX_LEN = 34
AUX_COLUMNS = ['target']
TEXT_COLUMN = 'text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'<feature_engineering>
for i in test_data.columns:
if i not in train_data.columns:
train_data[i]=np.zeros(len(train_data))
def build_matrix(word_index, path):
embedding_index = KeyedVectors.load(path, mmap='r')
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
for candidate in [word, word.lower() ]:
if candidate in embedding_index:
embedding_matrix[i] = embedding_index[candidate]
break
return embedding_matrix
def text_processing(df):
df['keyword'] = df['keyword'].str.replace("%20", " ")
 df['hashtags'] = df['text'].apply(lambda x: " ".join(re.findall(r"
df['hash_loc_key'] = df[['hashtags', 'location','keyword']].astype(str ).apply(lambda x: " ".join(x), axis=1)
 df['hash_loc_key'] = df["hash_loc_key"].astype(str ).str.lower().str.strip().fillna('nan')
 textfeats = ['hash_loc_key', 'text']
for cols in textfeats:
df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split()))
df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100
if cols == "text":
df[cols+"_vader_Compound"]= df[cols].apply(lambda x:SIA.polarity_scores(x)['compound'])
 return df<load_from_csv>
test_data=test_data[train_data.drop("IsBadBuy",axis=1 ).columns]
print("Read Data")
 train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv', nrows = N_ROWS)
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv', nrows = N_ROWS)
 X = train_df[TEXT_COLUMN].astype(str)
 y = train_df[TARGET_COLUMN].values
test = test_df[TEXT_COLUMN].astype(str)
 print("Train Shape: {} Rows".format(X.shape[0]))
print("Test Shape: {} Rows".format(test.shape[0]))
print('Dependent Variable Factor Ratio: ',train_df[TARGET_COLUMN].value_counts(normalize=True ).to_dict())
 tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE, lower=True)
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 test = tokenizer.texts_to_sequences(test)
 length_info = [len(x)for x in X]
print("Train Sequence Length - Mean {:.1f} +/- {:.1f}, Max {:.1f}, Min {:.1f}".format(
 np.mean(length_info), np.std(length_info), np.max(length_info), np.min(length_info)))
 X = sequence.pad_sequences(X, maxlen=MAX_LEN)
 test = sequence.pad_sequences(test, maxlen=MAX_LEN)
 embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 print("Embeddings Matrix Shape:", embedding_matrix.shape)
 checkpoint_predictions = []
weights = []<feature_engineering>
X=train_data.drop(['RefId','IsBadBuy'],axis=1)
 y=train_data['IsBadBuy']
SIA = SentimentIntensityAnalyzer()
train_df = text_processing(train_df)
 test_df = text_processing(test_df)
 count_vectorizer = TfidfVectorizer(
 analyzer="word",
tokenizer=nltk.word_tokenize,
preprocessor=None,
stop_words='english',
sublinear_tf=True,
ngram_range=(1, 1),
max_features=None)
 hash_loc_tfidf = count_vectorizer.fit(train_df['hash_loc_key'])
 tfvocab = hash_loc_tfidf.get_feature_names()
print("Number of TF-IDF Features: {}".format(len(tfvocab)))
 train_tfidf = count_vectorizer.transform(train_df['hash_loc_key'])
 test_tfidf = count_vectorizer.transform(test_df['hash_loc_key'] )<normalization>
X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=42 )
dense_vars = [
'hash_loc_key_num_words',
'hash_loc_key_num_unique_words',
'hash_loc_key_words_vs_unique',
'text_num_words',
'text_num_unique_words',
'text_words_vs_unique',
'text_vader_Compound']
for d_i in dense_vars:
scaler = StandardScaler()
scaler.fit(train_df.loc[:,d_i].values.reshape(-1, 1))
train_df.loc[:,d_i] = scaler.transform(train_df.loc[:,d_i].values.reshape(-1, 1))
test_df.loc[:,d_i] = scaler.transform(test_df.loc[:,d_i].values.reshape(-1, 1))
train_num = hstack([csr_matrix(train_df.loc[:,dense_vars].values),train_tfidf] ).tocsr()
test_num = hstack([csr_matrix(test_df.loc[:,dense_vars].values),test_tfidf] ).tocsr()
num_cols = train_df[dense_vars].columns.tolist() + tfvocab<choose_model_class>
from sklearn.neighbors import KNeighborsClassifier
def build_model(embedding_matrix, n_classes, dropout=0.2):
words_inputs = Input(shape=(None,))
numeric_inputs = Input(shape=(len(num_cols),))
numeric_x = Dense(512, activation='relu' )(numeric_inputs)
 numeric_x = Dropout(dropout )(numeric_x)
 numeric_x = Dense(64, activation='relu' )(numeric_x)
 words_x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix],
trainable=False, input_length=MAX_LEN )(words_inputs)
 words_x = SpatialDropout1D(dropout )(words_x)
 words_x = Bidirectional(LSTM(128, return_sequences=True))(words_x)
 words_x = Bidirectional(LSTM(128, return_sequences=True))(words_x)
 emb_pool = concatenate([
GlobalMaxPooling1D()(words_x),
GlobalAveragePooling1D()(words_x)
 ])
 concat_x = concatenate([emb_pool, numeric_x])
 concat_x = Dropout(dropout )(concat_x)
 output = Dense(n_classes, activation='sigmoid' )(concat_x)
 model = Model(inputs=[words_inputs,numeric_inputs], outputs=output)
 opt = optimizers.Adam(learning_rate=0.00004, beta_1=0.9, beta_2=0.999, amsgrad=True)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['acc'])
 return model
model = build_model(embedding_matrix, N_CLASSES)
 model.summary()<choose_model_class>
from sklearn.neighbors import KNeighborsClassifier
oof_preds = np.zeros(X.shape[0])
 test_preds = np.zeros(test.shape[0])
 n_splits = 3
folds = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 plot_metrics = ['loss','acc']
fold_hist = {}
for i,(trn_idx, val_idx)in enumerate(folds.split(X)) :
modelstart = time.time()
model = build_model(embedding_matrix, N_CLASSES)
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=4, verbose=1,
mode='min', baseline=None, restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, min_lr=1e-7,
mode='min', verbose=1)
 history = model.fit(
 [X[trn_idx], train_num[trn_idx]],
y[trn_idx],
validation_data=([X[val_idx], train_num[val_idx]], y[val_idx]),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=0,
callbacks=[es, rlr] 
)
 best_index = np.argmin(history.history['val_loss'])
 fold_hist[i] = history
oof_preds[val_idx] = model.predict([X[val_idx], train_num[val_idx]] ).ravel()
test_preds += model.predict([test, test_num] ).ravel()
f1_sc = f1_score(y[val_idx],(oof_preds[val_idx] > 0.5 ).astype(int), average=f1_strategy)
 print("
FOLD {} COMPLETE in {:.1f} Minutes - Avg F1 {:.5f} - Best Epoch {}".format(i,(time.time() - modelstart)/60, f1_sc, best_index + 1))
best_metrics = {metric: scores[best_index] for metric, scores in history.history.items() }
pprint.pprint(best_metrics)
 f, ax = plt.subplots(1,len(plot_metrics),figsize = [12,4])
 for p_i,metric in enumerate(plot_metrics):
ax[p_i].plot(history.history[metric], label='Train ' + metric)
 ax[p_i].plot(history.history['val_' + metric], label='Val ' + metric)
 ax[p_i].set_title("{} Fold Loss Curve - {}
Best Epoch {}".format(i, metric, best_index))
ax[p_i].legend()
ax[p_i].axvline(x=best_index, c='black')
 plt.show()<compute_test_metric>
from sklearn.neighbors import KNeighborsClassifier
print("OOF Classification Report for Optimised Threshold: {:.3f}".format(best_cutoff))
print(classification_report(y,(oof_preds > best_cutoff ).astype(int), digits = 4))
print(f1_score(y,(oof_preds > cutoff ).astype(int), average=f1_strategy))
print("
OOF Non-Optimised Cutoff (.5)")
 print(classification_report(y,(oof_preds >.5 ).astype(int), digits = 4))
print(f1_score(y,(oof_preds >.5 ).astype(int), average=f1_strategy))
cnf_matrix = confusion_matrix(y,(oof_preds >.5 ).astype(int))
print("OOF Confusion Matrix")
 print(cnf_matrix)
 print("OOF Normalised Confusion Matrix")
 print(( cnf_matrix.astype('float')/ cnf_matrix.sum(axis=1)[:, np.newaxis] ).round(3))<sort_values>
KNN=KNeighborsClassifier(n_neighbors=11)
 KNN.fit(X_train,y_train )
show_cols = [
'id',
'keyword',
'location',
'text',
'target',
'oof_preds',
'error']
print("Look at False Negative")
 display(train_df[show_cols].sort_values(by = 'error', ascending=False ).iloc[:20])
 print("Look at False Positives")
 display(train_df[show_cols].sort_values(by = 'error', ascending=True ).iloc[:20] )<save_to_csv>
KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
metric_params=None, n_jobs=None, n_neighbors=11, p=2,
weights='uniform' )
submission = pd.DataFrame.from_dict({
'id': test_df.id,
TARGET_COLUMN:(( test_preds / n_splits)> best_cutoff ).astype(int)
 })
 submission.to_csv('submission_optimised_cutoff.csv', index=False)
 print(submission[TARGET_COLUMN].value_counts(normalize = True ).to_dict())
 submission.head()<save_to_csv>
KNN.score(X_test,y_test )
submission = pd.DataFrame.from_dict({
'id': test_df.id,
TARGET_COLUMN:(( test_preds / n_splits)>.5 ).astype(int)
 })
 submission.to_csv('submission_fixed_cutoff.csv', index=False)
 print(submission[TARGET_COLUMN].value_counts(normalize = True ).to_dict())
 submission.head()<save_to_csv>
predict=KNN.predict(test_data.drop('RefId',axis=1))
oof_pd = pd.DataFrame(oof_preds, columns = ['dense_oof'])
 oof_pd.to_csv("oof_dense_nn.csv" )<train_model>
Submission=pd.DataFrame(data=predict,columns=['IsBadBuy'])
 Submission.head()
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))<load_from_csv>
Submission['RefId']=test_data['RefId']
Submission.set_index('RefId',inplace=True )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 print(train.shape)
 print(test.shape)
<prepare_x_and_y>
Submission.head()
Submission.to_csv('Submission.csv' )