Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<load_from_zip>
%matplotlib inline
!rm -rf test_images_224 test_images_224.zip<define_variables>
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )
submission = []<categorify>
train.isnull().sum().sort_values(ascending=False )
for i, pred in preds.items() :
y1 = np.argmax(pred[:168])
 y2 = np.argmax(pred[168:179])
 y3 = np.argmax(pred[179:])
 submission.append(( i, "Test_" + str(i)+ "_grapheme_root", y1))
submission.append(( i, "Test_" + str(i)+ "_vowel_diacritic", y2))
submission.append(( i, "Test_" + str(i)+ "_consonant_diacritic", y3))<save_to_csv>
dtype_df = train.dtypes.reset_index()
dtype_df.columns = ["Count", "Column Type"]
dtype_df.groupby("Column Type" ).aggregate('count' ).reset_index()
submission = pd.DataFrame(submission, columns=["id", "row_id", "target"] ).sort_values(by=["id", "row_id"] ).reset_index() [["row_id", "target"]]
submission.to_csv("submission.csv", index=False )<define_variables>
unique_df = train.nunique().reset_index()
unique_df.columns = ["col_name", "unique_count"]
inputs = ["test_image_data_0.parquet","test_image_data_1.parquet","test_image_data_2.parquet","test_image_data_3.parquet"] 
<define_variables>
train.drop(columns=constant_df['col_name'],inplace=True)
 test.drop(columns=constant_df['col_name'],inplace=True )
DIR = ".. /input/bengaliai-cv19/"<categorify>
train.drop("ID", axis = 1, inplace = True)
 y_train=train['target']
train.drop("target", axis = 1, inplace = True)
 test.drop("ID", axis = 1, inplace = True )
class EvalDataset(Dataset):
def __init__(self,df,transform=None,aug=None,norm=None):
self.norm = norm
self.df = df
self.data = 255 - self.df.iloc[:, 1:].values.reshape(-1, 137, 236 ).astype(np.uint8)
 self.transform = transform
self.aug = aug
def __getitem__(self, idx):
name = self.df.iloc[idx,0]
image_arr = cv2.resize(self.data[idx],(128,128))
image_arr = image_arr/255.0
if self.norm != None:
mean = self.norm['mean']
std = self.norm['std']
image_arr =(image_arr - mean)/std
image_arr = torch.from_numpy(image_arr.astype(np.float32))
return image_arr.unsqueeze(0 ).repeat(3,1,1),name
def __len__(self):
return len(self.data )<set_options>
y_train=np.log1p(y_train )
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
<choose_model_class>
X_train, X_valid, Y_train, Y_valid = train_test_split(train, y_train, test_size=0.05, random_state=0 )
class SEModule(nn.Module):
def __init__(self, channels=2048, reduction=16):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class MyModel(nn.Module):
def __init__(self,pretrained=True):
super().__init__()
if pretrained:
self.model = pretrainedmodels.__dict__["se_resnext50_32x4d"](pretrained="imagenet")
 else:
self.model = pretrainedmodels.__dict__["se_resnext50_32x4d"](pretrained=None)
 self.model = nn.Sequential(*list(self.model.children())[:-2])
 self.se_g = SEModule()
self.se_v = SEModule()
self.se_c = SEModule()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc_g = nn.Sequential(nn.Linear(2048,512), nn.ReLU() , nn.Dropout(p=0.2), nn.Linear(512,168))
self.fc_v = nn.Sequential(nn.Linear(2048,512), nn.ReLU() , nn.Dropout(p=0.2), nn.Linear(512,11))
self.fc_c = nn.Sequential(nn.Linear(2048,512), nn.ReLU() , nn.Dropout(p=0.2), nn.Linear(512,7))
def forward(self,x):
x = self.model(x)
 g = self.se_g(x)
 v = self.se_v(x)
 c = self.se_c(x)
 g = torch.flatten(self.avg_pool(g),1)
 v = torch.flatten(self.avg_pool(v),1)
 c = torch.flatten(self.avg_pool(c),1)
 g = self.fc_g(g)
 v = self.fc_v(v)
 c = self.fc_c(c)
 return g,v,c<create_dataframe>
RF_clf=RandomForestRegressor(random_state=42,n_jobs=-1)
results = []
mean = 13.4/255
std = 40.8/255
model = MyModel(pretrained=False)
 model.load_state_dict(torch.load('.. /input/seresnext-50-train/seresnext_50.pth'))
model.to(device)
 for inp in inputs:
df = pd.read_parquet(DIR+inp)
 kaggle_dataset = EvalDataset(df,norm={'mean':mean,'std':std})
 dl = DataLoader(kaggle_dataset,batch_size=128,num_workers=4)
 img,gra,vow,con = kaggle_out(model,dl,device)
 for i in range(len(img)) :
results.append(( f"{img[i]}_grapheme_root",gra[i]))
results.append(( f"{img[i]}_vowel_diacritic",vow[i]))
results.append(( f"{img[i]}_consonant_diacritic",con[i]))
df2 = pd.DataFrame(results, columns=["row_id", "target"] )<prepare_output>
RF_clf.fit(X_train,Y_train )
sub = df2<save_to_csv>
def evaluate(model, features, labels):
predictions = model.predict(features)
 errors = abs(predictions - labels)
 m = 100 * np.mean(errors / labels)
 accuracy = 100 - m
print('Model Performance')
 print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
 print('Accuracy = {:0.2f}%.'.format(accuracy))
return accuracy
sub.to_csv("submission.csv", index=False )<import_modules>
print('valid Accuracy')
 valid_accuracy = evaluate(RF_clf, X_valid, Y_valid )
IMG_SIZE=86
N_CHANNELS=1
def resize(df, size=86, need_progress_bar=True):
resized = {}
resize_size=86
if need_progress_bar:
for i in tqdm(range(df.shape[0])) :
image=df.loc[df.index[i]].values.reshape(137,236)
 _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
 contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]
idx = 0
ls_xmin = []
ls_ymin = []
ls_xmax = []
ls_ymax = []
for cnt in contours:
idx += 1
x,y,w,h = cv2.boundingRect(cnt)
 ls_xmin.append(x)
 ls_ymin.append(y)
 ls_xmax.append(x + w)
 ls_ymax.append(y + h)
 xmin = min(ls_xmin)
 ymin = min(ls_ymin)
 xmax = max(ls_xmax)
 ymax = max(ls_ymax)
 roi = image[ymin:ymax,xmin:xmax]
resized_roi = cv2.resize(roi,(resize_size, resize_size),interpolation=cv2.INTER_AREA)
 resized[df.index[i]] = resized_roi.reshape(-1)
 else:
for i in range(df.shape[0]):
image=df.loc[df.index[i]].values.reshape(137,236)
 _, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
 contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]
idx = 0
ls_xmin = []
ls_ymin = []
ls_xmax = []
ls_ymax = []
for cnt in contours:
idx += 1
x,y,w,h = cv2.boundingRect(cnt)
 ls_xmin.append(x)
 ls_ymin.append(y)
 ls_xmax.append(x + w)
 ls_ymax.append(y + h)
 xmin = min(ls_xmin)
 ymin = min(ls_ymin)
 xmax = max(ls_xmax)
 ymax = max(ls_ymax)
 roi = image[ymin:ymax,xmin:xmax]
resized_roi = cv2.resize(roi,(resize_size, resize_size),interpolation=cv2.INTER_AREA)
 resized[df.index[i]] = resized_roi.reshape(-1)
 resized = pd.DataFrame(resized ).T
return resized 
<load_from_csv>
RF_target=np.expm1(RF_clf.predict(test))
model = load_model('.. /input/bengali-alternative/densenet_169_rooty.h5')
 preds_dict = {
'grapheme_root': [],
'vowel_diacritic': [],
'consonant_diacritic': []
}
components = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']
target=[]
row_id=[]
for i in range(4):
df_test_img = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_{}.parquet'.format(i))
df_test_img.set_index('image_id', inplace=True)
 X_test = resize(df_test_img, need_progress_bar=False)/255
X_test = X_test.values.reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)
 preds = model.predict(X_test)
 for i, p in enumerate(preds_dict):
preds_dict[p] = np.argmax(preds[i], axis=1)
 for k,id in enumerate(df_test_img.index.values):
for i,comp in enumerate(components):
id_sample=id+'_'+comp
row_id.append(id_sample)
 target.append(preds_dict[comp][k])
 del df_test_img
del X_test
gc.collect()
df_sample = pd.DataFrame(
 {
'row_id': row_id,
'target':target
},
columns = ['row_id','target'] 
)
 df_sample.to_csv('submission.csv',index=False)
 df_sample.head()<import_modules>
NO_OF_FEATURES=1000
col = pd.DataFrame({'importance': RF_clf.feature_importances_, 'feature': train.columns} ).sort_values(
 by=['importance'], ascending=[False])[:NO_OF_FEATURES]['feature'].values
import numpy as np
import pandas as pd
import string
import re
from matplotlib import pyplot as plt
import nltk
from nltk.tokenize import TweetTokenizer
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords<load_pretrained>
train=train[col]
test=test[col]
nltk.download('stopwords' )<load_from_csv>
train["sum"] = train.sum(axis=1)
 test["sum"] = test.sum(axis=1)
 train["var"] = train.var(axis=1)
 test["var"] = test.var(axis=1)
 train["median"] = train.median(axis=1)
 test["median"] = test.median(axis=1)
 train["mean"] = train.mean(axis=1)
 test["mean"] = test.mean(axis=1)
 train["std"] = train.std(axis=1)
 test["std"] = test.std(axis=1)
 train["max"] = train.max(axis=1)
 test["max"] = test.max(axis=1)
 train["min"] =train.min(axis=1)
 test["min"] = test.min(axis=1)
 train["skew"] = train.skew(axis=1)
 test["skew"] = test.skew(axis=1 )
train_filepath = '/kaggle/input/nlp-getting-started/train.csv'
df_train = pd.read_csv(train_filepath, index_col='id')
 df_train.head()<count_missing_values>
X_train, X_valid, Y_train, Y_valid = train_test_split(train, y_train, test_size=0.1, random_state=0 )
df_train.isnull().sum() / len(df_train)* 100<choose_model_class>
def run_lgb(X_train, Y_train, X_valid, Y_valid, test):
params = {
"objective" : "regression",
"metric" : "rmse",
"task": "train",
"boosting type":'dart',
"num_leaves" :100,
"learning_rate" : 0.01,
"bagging_fraction" : 0.8,
"feature_fraction" : 0.8,
"bagging_frequency" : 6,
"bagging_seed" : 42,
"verbosity" : -1,
"seed": 42
}
lgtrain = lgb.Dataset(X_train, label=Y_train)
 lgval = lgb.Dataset(X_valid, label=Y_valid)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgtrain, lgval],
early_stopping_rounds=300,
verbose_eval=100,
evals_result=evals_result)
 lgb_prediction = np.expm1(model.predict(test, num_iteration=model.best_iteration))
return lgb_prediction, model, evals_result
tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)
 stopwords_english = stopwords.words('english')
 stemmer = PorterStemmer()<string_transform>
lgb_pred, model, evals_result = run_lgb(X_train, Y_train, X_valid, Y_valid, test)
 print("LightGBM Training Completed..." )
<feature_engineering><EOS> 
sub=pd.read_csv('.. /input/sample_submission.csv')
 sub_rf = pd.DataFrame()
sub_rf["target"] = RF_target
sub_rf["ID"] = sub["ID"]
sub_rf.to_csv("sub_rf.csv", index=False)
 sub_lgb = pd.DataFrame()
sub_lgb["target"] = lgb_pred
sub_lgb["ID"] = sub["ID"]
sub_lgb.to_csv("sub_lgb.csv", index=False)
 sub["target"] =(sub_lgb["target"] + sub_rf['target'])/2
print(sub.head())
 sub.to_csv('sub_lgb_xgb.csv', index=False)
 sub.head()