Kaggle competition: PadhAI: Text - Non Text Classification Level 4b
<SOS> metric: categorizationaccuracy Kaggle data source: padhai:-text-non-text-classification-level-4b<train_model>
np.random.seed(100)
 LEVEL = 'level_4a'
class TransformerMissingData(BaseEstimator, TransformerMixin):
def __init__(self, missing_age_masters_strategy = 'mean', missing_age_non_masters_strategy = 'mean'):
self.missing_age_masters_strategy = missing_age_masters_strategy
self.missing_age_non_masters_strategy = missing_age_non_masters_strategy
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
is_not_master =(X['Title_Master.'] == 0)
 is_master =(X['Title_Master.'] == 1)
 imp_non_master = SimpleImputer(missing_values = np.nan,
strategy = self.missing_age_non_masters_strategy)
 imp_master = SimpleImputer(missing_values = np.nan,
strategy = self.missing_age_masters_strategy)
 imp_fare = SimpleImputer(missing_values = np.nan,
strategy = 'mean')
 X.loc[is_not_master, 'Age'] = imp_non_master.fit_transform(X.loc[is_not_master, ['Age']])
 X.loc[is_master, 'Age'] = imp_master.fit_transform(X.loc[is_master, ['Age']])
 X.loc[:, 'Fare'] = imp_fare.fit_transform(X.loc[:, ['Fare']])
 return X<normalization>
class SigmoidNeuron:
def __init__(self):
self.w = None
self.b = None
def perceptron(self, x):
return np.dot(x, self.w.T)+ self.b
def sigmoid(self, x):
return 1.0/(1.0 + np.exp(-x))
def grad_w_mse(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
return(y_pred - y)* y_pred *(1 - y_pred)* x
def grad_b_mse(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
return(y_pred - y)* y_pred *(1 - y_pred)
 def grad_w_ce(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
if y == 0:
return y_pred * x
elif y == 1:
return -1 *(1 - y_pred)* x
else:
raise ValueError("y should be 0 or 1")
 def grad_b_ce(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
if y == 0:
return y_pred
elif y == 1:
return -1 *(1 - y_pred)
 else:
raise ValueError("y should be 0 or 1")
 def fit(self, X, Y, epochs=1, learning_rate=1, initialise=True, loss_fn="mse", display_loss=False):
if initialise:
self.w = np.random.randn(1, X.shape[1])
 self.b = 0
if display_loss:
loss = {}
for i in tqdm_notebook(range(epochs), total=epochs, unit="epoch"):
dw = 0
db = 0
for x, y in zip(X, Y):
if loss_fn == "mse":
dw += self.grad_w_mse(x, y)
 db += self.grad_b_mse(x, y)
 elif loss_fn == "ce":
dw += self.grad_w_ce(x, y)
 db += self.grad_b_ce(x, y)
 self.w -= learning_rate * dw
self.b -= learning_rate * db
if display_loss:
Y_pred = self.sigmoid(self.perceptron(X))
if loss_fn == "mse":
loss[i] = mean_squared_error(Y, Y_pred)
 elif loss_fn == "ce":
loss[i] = log_loss(Y, Y_pred)
 if display_loss:
plt.plot(loss.values())
 plt.xlabel('Epochs')
 if loss_fn == "mse":
plt.ylabel('Mean Squared Error')
 elif loss_fn == "ce":
plt.ylabel('Log Loss')
 plt.show()
def predict(self, X):
Y_pred = []
for x in X:
y_pred = self.sigmoid(self.perceptron(x))
Y_pred.append(y_pred)
 return np.array(Y_pred )
class TransformerNormalize(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
out = X
out.loc[:, ['Pclass']] = MinMaxScaler().fit_transform(out[['Pclass']])
 out.loc[:, ['Age', 'Fare']] = StandardScaler().fit_transform(out[['Age', 'Fare']])
 return out<load_from_csv>
languages = ['ta', 'hi', 'en']
images_train = read_all(".. /input/level_4b_train/level_4b/background/", key_prefix='bgr_')
 for language in languages:
images_train.update(read_all(".. /input/level_4b_train/level_4b/"+language, key_prefix=language+"_"))
print(len(images_train))
images_test = read_all(".. /input/level_4b_test/kaggle_level_4b", key_prefix='')
 print(len(images_test))
df_dummy = pd.concat([pd.read_csv('/kaggle/input/train.csv'), pd.read_csv('/kaggle/input/test.csv')], ignore_index = True)
 dummy_pipeline = Pipeline([ 
('prepare', TransformerSignificantData()), 
('dummify', TransformerDummify()), 
('normalize', TransformerNormalize())
 ])
 df_dummy = dummy_pipeline.transform(df_dummy)
 df_dummy.head()<define_variables>
scaler = MinMaxScaler()
X_scaled_train = scaler.fit_transform(X_train)
 X_scaled_test = scaler.transform(X_test )
related_to_age = ['Pclass', 'Title_Miss.', 'SibSp', 'Cabin_C', 'Title_Mr.', 'Fare', 'Parch']<count_missing_values>
sn_ce = SigmoidNeuron()
sn_ce.fit(X_scaled_train, Y_train, epochs=100, learning_rate=0.015, loss_fn="ce", display_loss=True )
df_pca = df_dummy.copy().loc[:, related_to_age]
df_pca.isnull().sum()<count_missing_values>
def print_accuracy(sn):
Y_pred_train = sn.predict(X_scaled_train)
 Y_pred_binarised_train =(Y_pred_train >= 0.5 ).astype("int" ).ravel()
accuracy_train = accuracy_score(Y_pred_binarised_train, Y_train)
 print("Train Accuracy : ", accuracy_train)
 print("-"*50 )
df_pca = df_pca.fillna(value = df_pca['Fare'].mean())
 df_pca.isnull().sum()<groupby>
print_accuracy(sn_ce )
np.cumsum(pca_age.explained_variance_ratio_ )<predict_on_test>
Y_pred_test = sn_ce.predict(X_scaled_test)
 Y_pred_binarised_test =(Y_pred_test >= 0.5 ).astype("int" ).ravel()
submission = {}
submission['ImageId'] = ID_test
submission['Class'] = Y_pred_binarised_test
submission = pd.DataFrame(submission)
 submission = submission[['ImageId', 'Class']]
submission = submission.sort_values(['ImageId'])
 submission.to_csv("submisision.csv", index=False )