Kaggle competition: PadhAI: Text - Non Text Classification Level 4b
<SOS> metric: categorizationaccuracy Kaggle data source: padhai:-text-non-text-classification-level-4b<choose_model_class>
warnings.filterwarnings("ignore")
 np.random.seed(100)
 LEVEL = 'level_4b'
model_1 = LGBMClassifier(learning_rate=0.01, n_estimators=1000, max_depth=None)
 model_2 = XGBClassifier(learning_rate=0.01, n_estimators=1000, max_depth=None)
 model_3 = AdaBoostClassifier(learning_rate=0.01, n_estimators=1000)
 model_4 = RandomForestClassifier(n_estimators=1000, random_state=42 )<find_best_params>
from padhai import MPNeuron,PerceptronWithSigmoid
estimators = []
estimators.append(( 'lgbm',model_1))
estimators.append(( 'xgb',model_2))
estimators.append(( 'adaboost',model_3))
estimators.append(( 'clf',model_4))<choose_model_class>
class SigmoidNeuron:
def __init__(self):
self.w = None
self.b = None
def perceptron(self, x):
return np.dot(x, self.w.T)+ self.b
def sigmoid(self, x):
return 1.0/(1.0 + np.exp(-x))
def grad_w_mse(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
return(y_pred - y)* y_pred *(1 - y_pred)* x
def grad_b_mse(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
return(y_pred - y)* y_pred *(1 - y_pred)
 def grad_w_ce(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
if y == 0:
return y_pred * x
elif y == 1:
return -1 *(1 - y_pred)* x
else:
raise ValueError("y should be 0 or 1")
 def grad_b_ce(self, x, y):
y_pred = self.sigmoid(self.perceptron(x))
if y == 0:
return y_pred
elif y == 1:
return -1 *(1 - y_pred)
 else:
raise ValueError("y should be 0 or 1")
 def fit_train_valid(self, X_train, Y_train,X_test,Y_test,epochs=1, learning_rate=1, initialise=True, loss_fn="mse", display_loss=False,lr_decay=False):
i = 0
if initialise:
np.random.seed(100)
 self.w = np.random.randn(1, X_train.shape[1])
 np.random.seed(100)
 self.b = np.random.randn()
if display_loss:
loss = {}
valid_loss = {}
while(i<epochs):
dw = 0
db = 0
for x, y in zip(X_train, Y_train):
if loss_fn == "mse":
dw += self.grad_w_mse(x, y)
 db += self.grad_b_mse(x, y)
 elif loss_fn == "ce":
dw += self.grad_w_ce(x, y)
 db += self.grad_b_ce(x, y)
 self.w -= learning_rate * dw
self.b -= learning_rate * db
if display_loss:
Y_pred = self.sigmoid(self.perceptron(X_train))
Y_test_pred = self.sigmoid(self.perceptron(X_test))
if loss_fn == "mse":
loss[i] = mean_squared_error(Y_train, Y_pred)
 valid_loss[i] = mean_squared_error(Y_test, Y_test_pred)
 elif loss_fn == "ce":
loss[i] = log_loss(Y_train, Y_pred)
 valid_loss[i] = log_loss(Y_test, Y_test_pred)
 if lr_decay:
if i>0 and valid_loss[i]-valid_loss[i-1]<0:
w = self.w
b = self.b
i+=1
elif i>0:
i = i-1
self.w = w
self.b = b
learning_rate/=2
else:
i+=1
else:
i+=1
if display_loss:
plt.plot(loss.values())
 plt.plot(valid_loss.values() ,color='orange')
 plt.xlabel('Epochs')
 if loss_fn == "mse":
plt.ylabel('Mean Squared Error')
 elif loss_fn == "ce":
plt.ylabel('Log Loss')
 plt.show()
def fit(self, X, Y, epochs=1, learning_rate=1, initialise=True, loss_fn="mse", display_loss=False):
if initialise:
np.random.seed(100)
 self.w = np.random.randn(1, X.shape[1])
 np.random.seed(100)
 self.b = np.random.randn()
if display_loss:
loss = {}
for i in range(epochs):
dw = 0
db = 0
for x, y in zip(X, Y):
if loss_fn == "mse":
dw += self.grad_w_mse(x, y)
 db += self.grad_b_mse(x, y)
 elif loss_fn == "ce":
dw += self.grad_w_ce(x, y)
 db += self.grad_b_ce(x, y)
 self.w -= learning_rate * dw
self.b -= learning_rate * db
if display_loss:
Y_pred = self.sigmoid(self.perceptron(X))
if loss_fn == "mse":
loss[i] = mean_squared_error(Y, Y_pred)
 elif loss_fn == "ce":
loss[i] = log_loss(Y, Y_pred)
 if display_loss:
plt.plot(loss.values())
 plt.xlabel('Epochs')
 if loss_fn == "mse":
plt.ylabel('Mean Squared Error')
 elif loss_fn == "ce":
plt.ylabel('Log Loss')
 plt.show()
def predict(self, X):
Y_pred = []
for x in X:
y_pred =1 if self.sigmoid(self.perceptron(x)) >=0.5 else 0
Y_pred.append(y_pred)
 return np.array(Y_pred )
hybrid_model = StackingClassifier(estimators )<train_model>
class Perceptron:
def __init__(self):
self.w = None
self.b = None
def model(self, x):
return 1 if(np.dot(self.w, x)>= self.b)else 0
def predict(self, X):
Y = []
for x in X:
result = self.model(x)
 Y.append(result)
 return np.array(Y)
 def fit(self, X, Y, epochs = 1, lr = 1,initialize=True,display_loss=False):
if initialize:
np.random.seed(100)
 self.w = np.random.randn(1,X.shape[1])
 np.random.seed(100)
 self.b = np.random.randn()
accuracy = {}
max_accuracy = 0
wt_matrix = []
for i in range(epochs):
for x, y in zip(X, Y):
y_pred = self.model(x)
 if y == 1 and y_pred == 0:
self.w = self.w + lr * x
self.b = self.b - lr * 1
elif y == 0 and y_pred == 1:
self.w = self.w - lr * x
self.b = self.b + lr * 1
if display_loss:
accuracy[i] = accuracy_score(self.predict(X), Y)
 if(accuracy[i] > max_accuracy):
max_accuracy = accuracy[i]
if display_loss:
print(max_accuracy)
 plt.title("Accuracy")
 plt.plot(accuracy.values())
 plt.xlabel("Epochs")
 plt.ylabel("Accuracy")
 plt.ylim([0, 1])
 plt.show()
hybrid_model.fit(X_train, y_train )<predict_on_test>
sys.path.insert(0,'.. /input/level_4b_train/' )
pred = hybrid_model.predict(X_test )<compute_test_metric>
languages = ['ta', 'hi', 'en']
images_train = read_all('.. /input/level_4b_train/'+LEVEL+"/"+"background", key_prefix='bgr_')
 for language in languages:
images_train.update(read_all('.. /input/level_4b_train/'+LEVEL+"/"+language, key_prefix=language+"_"))
print(len(images_train))
sys.path.insert(0,'.. /input/level_4b_test/')
 images_test = read_all('.. /input/level_4b_test/kaggle_'+LEVEL, key_prefix='')
 print(len(images_test))
accuracy_score(pred, y_test )<predict_on_test>
def pre_process_image(img):
img = img.reshape(64,64)
 ret,th1 = cv2.threshold(img,25,255,cv2.THRESH_BINARY)
 th1 = th1.ravel()
return th1
test['Survived'] = actual_pred = hybrid_model.predict(test )<save_to_csv>
X_train_processed = []
for i in range(X_train.shape[0]):
X_train_processed.append(pre_process_image(X_train[i]))
X_train = np.array(X_train_processed )
test[['PassengerId','Survived']].to_csv('submission.csv', index=False )<load_from_csv>
X_test_processed = []
for i in range(X_test.shape[0]):
X_test_processed.append(pre_process_image(X_test[i]))
X_test = np.array(X_test_processed )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv" ).sample(frac=1.)
test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<string_transform>
X_sub_train, X_sub_test, y_sub_train, y_sub_test = train_test_split(X_train, Y_train, test_size=0.1, random_state=34,stratify=Y_train )
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 X_train = np.array([tokenizer.encode(" ".join(simple_preprocess(text, min_len=2, max_len=15)) ,
add_special_tokens=True,
max_length=40,
pad_to_max_length=True)for text in train["text"]])
 X_test = np.array([tokenizer.encode(" ".join(simple_preprocess(text, min_len=2, max_len=15)) ,
add_special_tokens=True,
max_length=40,
pad_to_max_length=True)for text in test["text"]] )<choose_model_class>
sc = StandardScaler()
X_sc_sub_train = sc.fit_transform(X_sub_train)
 X_sc_sub_test = sc.transform(X_sub_test )
input_ids = Input(( 40,), dtype=int32)
 lm = TFRobertaModel.from_pretrained('roberta-base')
 sequence, cls = lm(input_ids)
 mean_pooling = GlobalAveragePooling1D()(sequence)
 max_pooling = GlobalMaxPooling1D()(sequence)
 pooling = concatenate([mean_pooling, max_pooling])
 dropout = Dropout(0.5, name="dropout" )(pooling)
 classification = Dense(1, activation="sigmoid", kernel_initializer=glorot_normal(seed=1), bias_initializer=glorot_normal(seed=1), name="classification" )(pooling)
 model = Model(input_ids, classification)
 plot_model(model )<train_model>
min_max = MinMaxScaler()
X_min_max_sub_train = min_max.fit_transform(X_sub_train)
 X_min_max_test = min_max.transform(X_sub_test )
def rate(epoch):
return 1.5e-5/(epoch + 1)
 scheduler = LearningRateScheduler(rate)
 model.compile(optimizer=Adam(beta_1=0.9, beta_2=0.999),
loss="binary_crossentropy",
metrics=["accuracy"])
 log = model.fit(X_train, train["target"].values,
callbacks=[scheduler],
batch_size=12,
epochs=3,
verbose=1 )<save_to_csv>
sn_mse = SigmoidNeuron()
test["proba"] = model.predict(X_test)
 test["target"] = test["proba"].apply(lambda p: int(p > 0.5))
test[["id", "target"]].to_csv("/kaggle/working/submission.csv", index=False )<set_options>
for learning_rate in [0.1,0.01,0.001,0.0001]:
sn_mse.fit_train_valid(X_sc_sub_train,y_sub_train,X_sc_sub_test,y_sub_test,display_loss=True,epochs=100,learning_rate=learning_rate,loss_fn="mse" )
%matplotlib inline 
<set_options>
for learning_rate in [0.0003,0.0006,0.0007,0.0008]:
sn_mse.fit_train_valid(X_sc_sub_train,y_sub_train,X_sc_sub_test,y_sub_test,display_loss=True,epochs=100,learning_rate=learning_rate,loss_fn="mse" )
COLOR = 'black'
mpl.rcParams['text.color'] = COLOR
mpl.rcParams['axes.labelcolor'] = COLOR
mpl.rcParams['xtick.color'] = COLOR
mpl.rcParams['ytick.color'] = COLOR
plt.rcParams.update({'font.size': 18})
 plt.subplots_adjust(wspace = 15, hspace = 15 )<load_from_csv>
sn_mse.fit_train_valid(X_sc_sub_train,y_sub_train,X_sc_sub_test,y_sub_test,display_loss=True,epochs=4500,learning_rate=0.0008,loss_fn="mse" )
original_training_df = pd.read_csv('/kaggle/input/train.csv')
 original_training_df.head()<count_missing_values>
accuracy_score(sn_mse.predict(X_sc_sub_train),y_sub_train )
original_training_df.isnull().sum()<count_missing_values>
accuracy_score(sn_mse.predict(X_sc_sub_test),y_sub_test )
original_training_df.isnull().sum().divide(len(original_training_df.index)).multiply(100 )<prepare_output>
X_sub_train, X_sub_test, y_sub_train, y_sub_test = train_test_split(X_train, Y_train, test_size=0.1, random_state=10,stratify=Y_train )
def get_title(dataframe_in):
dataframe_in['Title'] = dataframe_in['Name'].apply(lambda X: re.search('[A-Z]{1}[a-z]+\.', X ).group(0))
return dataframe_in
dataframe_transformations_test = original_training_df.copy()
dataframe_transformations_test = get_title(dataframe_transformations_test)
 dataframe_transformations_test.head()<drop_column>
print(np.mean(y_sub_train),np.mean(y_sub_test))
dataframe_transformations_test.drop('PassengerId', axis = 1, inplace = True )<data_type_conversions>
scaler = StandardScaler()
X_scaled_train = scaler.fit_transform(X_sub_train)
 X_scaled_test = scaler.transform(X_sub_test )
def convert_sex_to_number(dataframe_in):
dataframe_in['Sex'] = dataframe_in['Sex'].apply(lambda X: 0 if X == 'female' else 1)
 return dataframe_in
dataframe_transformations_test = convert_sex_to_number(dataframe_transformations_test )<feature_engineering>
sn_mse = sn_mse = SigmoidNeuron()
sn_mse.fit(X_scaled_train,y_sub_train,display_loss=True,epochs=4500,learning_rate=0.0008,loss_fn="mse" )
def get_cabin_letter(dataframe_in):
dataframe_in['Cabin'] = dataframe_in['Cabin'].apply(lambda X: re.search('[A-Za-z]{1}', X ).group(0 ).upper() if isinstance(X, str)else '?')
 return dataframe_in
def get_ticket_prefix(dataframe_in):
dataframe_in['Ticket'] = dataframe_in['Ticket'].apply(lambda X: X.split(' ')[0] if len(X.split(' ')) > 1 else '?')
 return dataframe_in
dataframe_transformations_test = get_cabin_letter(dataframe_transformations_test)
 dataframe_transformations_test = get_ticket_prefix(dataframe_transformations_test)
 dataframe_transformations_test.head()<feature_engineering>
def print_accuracy(sn):
Y_pred_train = sn.predict(X_scaled_train)
 Y_pred_binarised_train =(Y_pred_train >= 0.5 ).astype("int" ).ravel()
accuracy_train = accuracy_score(Y_pred_binarised_train, y_sub_train)
 print("Train Accuracy : ", accuracy_train)
 print("-"*50 )
def get_family_name(dataframe_in):
dataframe_in['FamilyName'] = dataframe_in['Name'].apply(lambda X: re.search('[A-Z]{1}[a-z ]+', X ).group(0))
return dataframe_in
dataframe_transformations_test = get_family_name(dataframe_transformations_test)
 dataframe_transformations_test.drop(['Name'], axis = 1, inplace = True)
 dict_cabins_per_family = dict()
for current_family_name in dataframe_transformations_test.FamilyName.unique().tolist() :
filter_family_name =(dataframe_transformations_test['FamilyName'] == current_family_name)
 filter_known_cabin =(dataframe_transformations_test['Cabin'] != '?')
 listCabinsFromFamily = dataframe_transformations_test.loc[(filter_family_name)&\ 
(filter_known_cabin)].Cabin.unique().tolist()
if len(listCabinsFromFamily)> 0:
max_v, mode = 0, None
for curr_cabin, v in Counter(listCabinsFromFamily ).items() :
if v > max_v:
max_v, mode = v, curr_cabin
dict_cabins_per_family[current_family_name] = mode
def get_family_cabin_per_row(df_row):
if df_row.FamilyName in dict_cabins_per_family and df_row.Cabin == '?':
out = dict_cabins_per_family[df_row.FamilyName]
else:
out = df_row.Cabin
return out
def get_family_cabin(df):
df['Cabin'] = df.apply(get_family_cabin_per_row, axis = 1)
 df['Cabin'] = df['Cabin'].fillna('?')
 return df
dataframe_transformations_test['Cabin'] = get_family_cabin(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()<count_values>
print_accuracy(sn_mse )
def embarked_fillna_median(df):
df.loc[:, ['Embarked']] = df['Embarked'].fillna(value = df['Embarked'].value_counts().idxmax())
 return df
dataframe_transformations_test = embarked_fillna_median(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()<categorify>
print(accuracy_score(sn_mse.predict(X_scaled_test),y_sub_test))
def fill_age_from_masters(df, strategy_in = 'median'):
is_master =(df['Title'] == 'Master.')
 imp = SimpleImputer(missing_values = np.nan, strategy = strategy_in)
 df.loc[is_master, 'Age'] = imp.fit_transform(df.loc[is_master][['Age']])
 return df
dataframe_transformations_test = fill_age_from_masters(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()<categorify>
mainScaler = StandardScaler()
X_train_scaled = mainScaler.fit_transform(X_train)
 X_test_scaled = mainScaler.transform(X_test )
def fill_age_from_non_masters(df, strategy_in = 'median'):
is_not_master =(df['Title'] != 'Master.')
 imp = SimpleImputer(missing_values = np.nan, strategy = strategy_in)
 df.loc[is_not_master, 'Age'] = imp.fit_transform(df.loc[is_not_master][['Age']])
 return df
dataframe_transformations_test = fill_age_from_non_masters(dataframe_transformations_test)
 dataframe_transformations_test.isnull().sum()<feature_engineering>
sn_mse = SigmoidNeuron()
sn_mse.fit(X_train_scaled,Y_train,display_loss=True,epochs=4500,learning_rate=0.0008,loss_fn="mse" )
def get_family_info(df_in):
df_in['FamilyMembers'] = df_in['Parch'] + df_in['SibSp'] + 1
df_in['Is_Mother'] = np.where(( df_in.Title=='Mrs.')&(df_in.Parch >0), 1, 0)
 return df_in<categorify>
print(accuracy_score(sn_mse.predict(X_train_scaled),Y_train))
<categorify><EOS> 
Y_pred_test = sn_mse.predict(X_test_scaled)
 Y_pred_binarised_test =(Y_pred_test >= 0.5 ).astype("int" ).ravel()
submission = {}
submission['ImageId'] = ID_test
submission['Class'] = Y_pred_binarised_test
submission = pd.DataFrame(submission)
 submission = submission[['ImageId', 'Class']]
submission = submission.sort_values(['ImageId'])
 submission.to_csv("submisision.csv", index=False )