Kaggle competition: Give Me Some Credit
test_fea = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_fea = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_tar_nonsco = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_tar_sco = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<set_options>
warnings.filterwarnings("ignore")
 x_train = pd.read_csv('/kaggle/input/GiveMeSomeCredit/cs-training.csv')
 x_test = pd.read_csv('/kaggle/input/GiveMeSomeCredit/cs-test.csv')
 combine = [x_train, x_test]
def seed_everything(seed=1062):
np.random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=1062 )<normalization>
x_train.isnull().mean()
class Sparsemax(nn.Module):
def __init__(self, dim=None):
super(Sparsemax, self ).__init__()
self.dim = -1 if dim is None else dim
def forward(self, input):
input = input.transpose(0, self.dim)
 original_size = input.size()
input = input.reshape(input.size(0), -1)
 input = input.transpose(0, 1)
 dim = 1
number_of_logits = input.size(dim)
 input = input - torch.max(input, dim=dim, keepdim=True)[0].expand_as(input)
 zs = torch.sort(input=input, dim=dim, descending=True)[0]
range = torch.arange(start=1, end=number_of_logits + 1, device=device,step=1, dtype=input.dtype ).view(1, -1)
 range = range.expand_as(zs)
 bound = 1 + range * zs
cumulative_sum_zs = torch.cumsum(zs, dim)
 is_gt = torch.gt(bound, cumulative_sum_zs ).type(input.type())
 k = torch.max(is_gt * range, dim, keepdim=True)[0]
zs_sparse = is_gt * zs
taus =(torch.sum(zs_sparse, dim, keepdim=True)- 1)/ k
taus = taus.expand_as(input)
 self.output = torch.max(torch.zeros_like(input), input - taus)
 output = self.output
output = output.transpose(0, 1)
 output = output.reshape(original_size)
 output = output.transpose(0, self.dim)
 return output
def backward(self, grad_output):
dim = 1
nonzeros = torch.ne(self.output, 0)
 sum = torch.sum(grad_output * nonzeros, dim=dim)/ torch.sum(nonzeros, dim=dim)
 self.grad_input = nonzeros *(grad_output - sum.expand_as(grad_output))
return self.grad_input<init_hyperparams>
x_test.isnull().mean()
def initialize_non_glu(module,inp_dim,out_dim):
gain = np.sqrt(( inp_dim+out_dim)/np.sqrt(4*inp_dim))
torch.nn.init.xavier_normal_(module.weight, gain=gain)
 class GBN(nn.Module):
def __init__(self,inp,vbs=128,momentum=0.01):
super().__init__()
self.bn = nn.BatchNorm1d(inp,momentum=momentum)
 self.vbs = vbs
def forward(self,x):
chunk = torch.chunk(x,max(1,x.size(0)//self.vbs),0)
 res = [self.bn(y)for y in chunk ]
return torch.cat(res,0)
 class GLU(nn.Module):
def __init__(self,inp_dim,out_dim,fc=None,vbs=128):
super().__init__()
if fc:
self.fc = fc
else:
self.fc = nn.Linear(inp_dim,out_dim*2)
 self.bn = GBN(out_dim*2,vbs=vbs)
 self.od = out_dim
def forward(self,x):
x = self.bn(self.fc(x))
return x[:,:self.od]*torch.sigmoid(x[:,self.od:])
 class FeatureTransformer(nn.Module):
def __init__(self,inp_dim,out_dim,shared,n_ind,vbs=128):
super().__init__()
first = True
self.shared = nn.ModuleList()
if shared:
self.shared.append(GLU(inp_dim,out_dim,shared[0],vbs=vbs))
first= False
for fc in shared[1:]:
self.shared.append(GLU(out_dim,out_dim,fc,vbs=vbs))
else:
self.shared = None
self.independ = nn.ModuleList()
if first:
self.independ.append(GLU(inp,out_dim,vbs=vbs))
for x in range(first, n_ind):
self.independ.append(GLU(out_dim,out_dim,vbs=vbs))
self.scale = torch.sqrt(torch.tensor([.5],device=device))
def forward(self,x):
if self.shared:
x = self.shared[0](x)
 for glu in self.shared[1:]:
x = torch.add(x, glu(x))
x = x*self.scale
for glu in self.independ:
x = torch.add(x, glu(x))
x = x*self.scale
return x
class AttentionTransformer(nn.Module):
def __init__(self,inp_dim,out_dim,relax,vbs=128):
super().__init__()
self.fc = nn.Linear(inp_dim,out_dim)
 self.bn = GBN(out_dim,vbs=vbs)
 self.r = torch.tensor([relax],device=device)
 def forward(self,a,priors):
a = self.bn(self.fc(a))
mask = torch.sigmoid(a*priors)
 priors =priors*(self.r-mask)
 return mask
class DecisionStep(nn.Module):
def __init__(self,inp_dim,n_d,n_a,shared,n_ind,relax,vbs=128):
super().__init__()
self.fea_tran = FeatureTransformer(inp_dim,n_d+n_a,shared,n_ind,vbs)
 self.atten_tran = AttentionTransformer(n_a,inp_dim,relax,vbs)
 def forward(self,x,a,priors):
mask = self.atten_tran(a,priors)
 loss =(( -1)*mask*torch.log(mask+1e-10)).mean()
x = self.fea_tran(x*mask)
 return x,loss
class TabNet(nn.Module):
def __init__(self,inp_dim,final_out_dim,n_d=64,n_a=64,n_shared=2,n_ind=2,n_steps=5,relax=1.2,vbs=128):
super().__init__()
if n_shared>0:
self.shared = nn.ModuleList()
self.shared.append(nn.Linear(inp_dim,2*(n_d+n_a)))
 for x in range(n_shared-1):
self.shared.append(nn.Linear(n_d+n_a,2*(n_d+n_a)))
 else:
self.shared=None
self.first_step = FeatureTransformer(inp_dim,n_d+n_a,self.shared,n_ind)
 self.steps = nn.ModuleList()
for x in range(n_steps-1):
self.steps.append(DecisionStep(inp_dim,n_d,n_a,self.shared,n_ind,relax,vbs))
self.fc = nn.Linear(n_d,final_out_dim)
 self.bn = nn.BatchNorm1d(inp_dim)
 self.n_d = n_d
def forward(self,x):
x = self.bn(x)
 x_a = self.first_step(x)[:,self.n_d:]
loss = torch.zeros(1 ).to(x.device)
 out = torch.zeros(x.size(0),self.n_d ).to(x.device)
 priors = torch.ones(x.shape ).to(x.device)
 for step in self.steps:
x_te,l = step(x,x_a,priors)
 out += F.relu(x_te[:,:self.n_d])
 x_a = x_te[:,self.n_d:]
loss += l
return self.fc(out),loss<choose_model_class>
[x_train[x_train['NumberOfDependents'].isnull() ][x_train['MonthlyIncome'].notnull() ]['DebtRatio'].count() ,
x_test[x_test['NumberOfDependents'].isnull() ][x_test['MonthlyIncome'].notnull() ]['DebtRatio'].count() ]
class TabNetWithEmbed(nn.Module):
def __init__(self,inp_dim,final_out_dim,n_d=64,n_a=64,n_shared=2,n_ind=2,n_steps=5,relax=1.2,vbs=128):
super().__init__()
self.tabnet = TabNet(inp_dim,final_out_dim,n_d,n_a,n_shared,n_ind,n_steps,relax,vbs)
 self.cat_embed = []
self.emb1 = nn.Embedding(2,1)
 self.emb3 = nn.Embedding(3,1)
 self.cat_embed.append(self.emb1)
 self.cat_embed.append(self.emb3)
 def forward(self,catv,contv):
catv = catv.to(device)
 contv = contv.to(device)
 embeddings = [embed(catv[:,idx])for embed,idx in zip(self.cat_embed,range(catv.size(1)))]
catv = torch.cat(embeddings,1)
 x = torch.cat(( catv,contv),1 ).contiguous()
x,l = self.tabnet(x)
 return torch.sigmoid(x),l<train_model>
x_train[x_train['NumberOfDependents'].notnull() ][x_train['MonthlyIncome'].isnull() ][['NumberOfDependents']].median()
class DrugData(Dataset):
def __init__(self,df,out,train=True):
self.df = df
self.out = out
self.train=train
def __len__(self):
return len(self.df)
 def __getitem__(self,idx):
if self.train:
tar = np.where(self.out[idx].reshape(-1)==0,0.00001,0.99999)
 return torch.from_numpy(self.df[idx,:] ).float() ,torch.tensor(tar ).float()
else:
return torch.from_numpy(self.df[idx,:] ).float() ,torch.tensor(self.out[idx].reshape(-1)).float()<categorify>
import matplotlib.pyplot as plt
import seaborn as sns
train_fea['cp_dose'].replace({'D1':0,'D2':1},inplace=True)
 test_fea['cp_dose'].replace({'D1':0,'D2':1},inplace=True)
 train_fea['cp_time'].replace({24:0,72:1,48:2},inplace=True)
 test_fea['cp_time'].replace({24:0,72:1,48:2},inplace=True)
 train_fea['cp_type'].replace({'trt_cp':0,'ctl_vehicle':1},inplace=True)
 test_fea['cp_type'].replace({'trt_cp':0,'ctl_vehicle':1},inplace=True)
 train_fea = train_fea.drop(columns='sig_id')
 test_fea = test_fea.drop(columns='sig_id' )<define_variables>
[len(x_train[x_train['RevolvingUtilizationOfUnsecuredLines'] > 2])/ len(x_train),
len(x_test[x_test['RevolvingUtilizationOfUnsecuredLines'] > 2])/ len(x_train)]
cat_col = [0,2]
num_col = len(train_fea.columns )<drop_column>
[len(x_train[x_train['NumberOfTime30-59DaysPastDueNotWorse'] == 98]),
len(x_train[x_train['NumberOfTime60-89DaysPastDueNotWorse'] == 98]),
len(x_train[x_train['NumberOfTimes90DaysLate'] == 98]),
len(x_train[x_train['NumberOfTime30-59DaysPastDueNotWorse'] == 98][x_train['NumberOfTimes90DaysLate'] == 98][x_train['NumberOfTime60-89DaysPastDueNotWorse'] == 98])]
train_tar = train_tar_sco.drop(columns='sig_id' ).values<import_modules>
[len(x_test[x_test['NumberOfTime30-59DaysPastDueNotWorse'] == 98]),
len(x_test[x_test['NumberOfTime60-89DaysPastDueNotWorse'] == 98]),
len(x_test[x_test['NumberOfTimes90DaysLate'] == 98]),
len(x_test[x_test['NumberOfTime30-59DaysPastDueNotWorse'] == 98][x_test['NumberOfTimes90DaysLate'] == 98][x_test['NumberOfTime60-89DaysPastDueNotWorse'] == 98])]
from sklearn.feature_selection import VarianceThreshold
from sklearn.decomposition import PCA<concatenate>
for dataset in combine:
dataset.loc[dataset['MonthlyIncome'].isnull() ,'DebtRatio'] = dataset.loc[dataset['MonthlyIncome'].isnull() ,'DebtRatio'] / 3915;
dataset.loc[dataset['MonthlyIncome'] == 1,'DebtRatio'] = dataset.loc[dataset['MonthlyIncome'] == 1,'DebtRatio'] / 3915;
dataset.loc[dataset['MonthlyIncome'] == 0,'DebtRatio'] = dataset.loc[dataset['MonthlyIncome'] == 0,'DebtRatio'] / 3915;
data = pd.concat([train_fea,test_fea],ignore_index=True)
 g = [*(x for x in data.columns if 'g' in x)]
c = [*(x for x in data.columns if 'c-' in x)]<data_type_conversions>
for dataset in combine:
dataset.drop(columns = ['Unnamed: 0'], inplace = True)
 dataset['RevolvingUtilizationOfUnsecuredLines'].mask(dataset['RevolvingUtilizationOfUnsecuredLines'] > 2, 2, inplace=True)
 dataset['NumberOfTime60-89DaysPastDueNotWorse'].mask(dataset['NumberOfTime60-89DaysPastDueNotWorse'] > 95, 20, inplace=True)
 dataset['NumberOfTime30-59DaysPastDueNotWorse'].mask(dataset['NumberOfTime30-59DaysPastDueNotWorse'] > 95, 20, inplace=True)
 dataset['NumberOfTimes90DaysLate'].mask(dataset['NumberOfTimes90DaysLate'] > 95, 20, inplace=True)
 dataset['90_sum'] = dataset['NumberOfTime30-59DaysPastDueNotWorse'] + dataset['NumberOfTimes90DaysLate'] + dataset['NumberOfTime60-89DaysPastDueNotWorse']
dataset['DebtRatio'].mask(dataset['DebtRatio'] > 400, 400, inplace=True)
 dataset['MonthlyIncome'].replace(np.nan, 0, inplace=True)
 dataset['MonthlyIncome'].replace(1, 0, inplace=True)
 dataset['NumberOfDependents'].replace(np.nan, 0, inplace=True)
 x_train = x_train[x_train['age'] > 20]
x_train = x_train[x_train['MonthlyIncome'] <= 1e6]
x_train = x_train[x_train['NumberRealEstateLoansOrLines'] <= 50]
for col in g:
sel = QuantileTransformer(n_quantiles=1000,random_state=0,output_distribution='normal')
 sel.fit(data[col].to_numpy().reshape(-1,1))
data[col] = sel.transform(data[col].to_numpy().reshape(-1,1))
for col in c:
sel = QuantileTransformer(n_quantiles=1000,random_state=0,output_distribution='normal')
 sel.fit(data[col].to_numpy().reshape(-1,1))
data[col] = sel.transform(data[col].to_numpy().reshape(-1,1))<create_dataframe>
result = x_train['SeriousDlqin2yrs' ]
data = x_train.drop(['SeriousDlqin2yrs'], axis = 1)
 train_data, test_data, train_result, test_result = train_test_split(data, result, test_size = 0.2, stratify = result, random_state=33 )
pca_c = PCA(n_components=15)
 extra_c = pd.DataFrame(pca_c.fit_transform(data[c]))
pca_g = PCA(n_components=50)
 extra_g = pd.DataFrame(pca_g.fit_transform(data[g]))
data = pd.concat(( data,extra_c,extra_g),axis=1 )<import_modules>
models = []
scores = []
models.append("AdaBoost")
 print("AdaBoost:")
 model = AdaBoostClassifier()
result = cross_val_score(model,train_data,train_result,scoring='roc_auc',cv=StratifiedKFold(n_splits=10))
print('mean score:', result.mean())
 scores.append(result.mean())
 models.append("LGBM")
 print("LGBM:")
 model = LGBMClassifier()
result = cross_val_score(model,train_data,train_result,scoring='roc_auc',cv=StratifiedKFold(n_splits=10))
print('mean score:', result.mean())
 scores.append(result.mean())
 models.append("XGBoost")
 print("XGBoost:")
 model = XGBClassifier(eval_metric = 'auc')
 result = cross_val_score(model,train_data,train_result,scoring='roc_auc',cv=StratifiedKFold(n_splits=10))
print('mean score:', result.mean())
 scores.append(result.mean())
 models.append("GBDT")
 print("GBDT:")
 model = GradientBoostingClassifier()
result = cross_val_score(model,train_data,train_result,scoring='roc_auc',cv=StratifiedKFold(n_splits=10))
print('mean score:', result.mean())
 scores.append(result.mean())
 models.append("RandomForest")
 print("RandomForest:")
 model = RandomForestClassifier()
result = cross_val_score(model,train_data,train_result,scoring='roc_auc',cv=StratifiedKFold(n_splits=10))
print('mean score:', result.mean())
 scores.append(result.mean())
 models.append("SVM")
 print("SVM:")
 model = svm.SVC()
result = cross_val_score(model,train_data,train_result,scoring='roc_auc',cv=StratifiedKFold(n_splits=10))
print('mean score:', result.mean())
 scores.append(result.mean())
from umap import UMAP<create_dataframe>
model = LGBMClassifier()
model.fit(train_data, train_result)
 predict_proba = model.predict_proba(test_data)[:,1]
roc_auc_score(test_result, predict_proba )
umap_c = UMAP(random_state=256,n_components=15)
 extra_c = pd.DataFrame(umap_c.fit_transform(data[c]))
umap_g = UMAP(random_state=256,n_components=50)
 extra_g = pd.DataFrame(umap_g.fit_transform(data[g]))
data = pd.concat(( data,extra_c,extra_g),axis=1 )<split>
model = LGBMClassifier(n_estimators = 290, learning_rate = 0.025,max_depth = 6, num_leaves = 22)
 model.fit(train_data, train_result)
 predict_proba = model.predict_proba(test_data)[:,1]
roc_auc_score(test_result, predict_proba )
train_df = data.iloc[:len(train_fea),:].values
test_df= data.iloc[len(train_fea):,:].values<choose_model_class>
x_test = x_test.drop(['SeriousDlqin2yrs'],axis=1)
 y_test = model.predict_proba(x_test)[:,1]
ids = np.arange(1,101504)
 res = pd.DataFrame({'Id': ids, 'Probability': y_test})
 res.to_csv("submission.csv", index=False )