Kaggle competition: Influencers in Social Networks
<SOS> metric: AUC Kaggle data source: influencers-in-social-networks<import_modules>
warnings.simplefilter(action='ignore', category=FutureWarning)
 gc.enable()
from tqdm.notebook import tqdm<sort_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
imgs = [f for f in os.listdir(test_path)]
imgs = sorted(imgs, key=lambda s: int(s.split('.')[0]))<save_to_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def create_csv_submission(model, data_path, img_list, out_path):
submission_df = pd.DataFrame(columns=['img', 'pixels'])
 model.to(DEVICE)
 model.eval()
for i, img in enumerate(tqdm(img_list)) :
x = Image.open(os.path.join(data_path, img))
x = my_transforms(x)
 x = x.unsqueeze(0 ).to(DEVICE)
 pred_mask = model.predict(x)
 pred_mask = pred_mask.cpu()
pred_mask = transforms.Resize(size=(420, 580))(pred_mask)
 encoding = rle_encoding(pred_mask)
 pixels = ' '.join(map(str, encoding))
submission_df.loc[i] = [str(i+1), pixels]
submission_df.to_csv(out_path, index=False )<save_to_csv>
def pre_pro(df):
df = df.astype('float32')
 col = df.columns
for i in range(len(col)) :
m = df.loc[df[col[i]] != -np.inf, col[i]].min()
df[col[i]].replace(-np.inf,m,inplace=True)
 M = df.loc[df[col[i]] != np.inf, col[i]].max()
df[col[i]].replace(np.inf,M,inplace=True)
 df.fillna(0, inplace = True)
 return df
create_csv_submission(model=model,
data_path=test_path,
img_list=imgs,
out_path='submission.csv' )<load_from_csv>
def feat_eng(df):
df.replace(0, 0.001)
 df['follower_diff'] =(df['A_follower_count'] > df['B_follower_count'])
 df['following_diff'] =(df['A_following_count'] > df['B_following_count'])
 df['listed_diff'] =(df['A_listed_count'] > df['B_listed_count'])
 df['ment_rec_diff'] =(df['A_mentions_received'] > df['B_mentions_received'])
 df['rt_rec_diff'] =(df['A_retweets_received'] > df['B_retweets_received'])
 df['ment_sent_diff'] =(df['A_mentions_sent'] > df['B_mentions_sent'])
 df['rt_sent_diff'] =(df['A_retweets_sent'] > df['B_retweets_sent'])
 df['posts_diff'] =(df['A_posts'] > df['B_posts'])
 df['A_pop_ratio'] = df['A_mentions_sent']/df['A_listed_count']
df['A_foll_ratio'] = df['A_follower_count']/df['A_following_count']
df['A_ment_ratio'] = df['A_mentions_sent']/df['A_mentions_received']
df['A_rt_ratio'] = df['A_retweets_sent']/df['A_retweets_received']
df['B_pop_ratio'] = df['B_mentions_sent']/df['B_listed_count']
df['B_foll_ratio'] = df['B_follower_count']/df['B_following_count']
df['B_ment_ratio'] = df['B_mentions_sent']/df['B_mentions_received']
df['B_rt_ratio'] = df['B_retweets_sent']/df['B_retweets_received']
df['A/B_foll_ratio'] =(df['A_foll_ratio'] > df['B_foll_ratio'])
 df['A/B_ment_ratio'] =(df['A_ment_ratio'] > df['B_ment_ratio'])
 df['A/B_rt_ratio'] =(df['A_rt_ratio'] > df['B_rt_ratio'])
 df['nf1_diff'] =(df['A_network_feature_1'] > df['B_network_feature_1'])
 df['nf2_diff'] =(df['A_network_feature_2'] > df['B_network_feature_2'])
 df['nf3_diff'] =(df['A_network_feature_3'] > df['B_network_feature_3'])
 df['nf3_ratio'] = df['A_network_feature_3'] / df['B_network_feature_3']
df['nf2_ratio'] = df['A_network_feature_2'] / df['B_network_feature_2']
df['nf1_ratio'] = df['A_network_feature_1'] / df['B_network_feature_1']
return(pre_pro(df))
child=pd.read_csv('.. /input/child_wishlist_v2.csv',header=None,index_col=0)
 childArray=child.values 
<load_from_csv>
fe_train = feat_eng(train.copy())
 fe_test = feat_eng(test.copy() )
gift=pd.read_csv('.. /input/gift_goodkids_v2.csv',header=None,index_col=0)
 giftArray=gift.values 
<load_from_csv>
train_df = fe_train
test_df = fe_test
y_train = np.array(train_df['Choice'] )
submitReader=pd.read_csv('.. /input/sample_submission_random_v2.csv')
 submit=submitReader<compute_test_metric>
target = 'Choice'
predictors = train_df.columns.values.tolist() [1:]
def childListPosition(whichChild,whichGift):
temp=np.argwhere(childArray[whichChild]==whichGift)
 if temp.size>0:
return temp[0][0]+1
else:
return 0
def giftListPosition(whichGift,whichChild):
temp=np.argwhere(giftArray[whichGift]==whichChild)
 if temp.size>0:
return temp[0][0]+1
else:
return 0
def childScore(whichChild,whichGift):
position=childListPosition(whichChild,whichGift)
 if position>0:
return(100-position-1)*2
else:
return -1
def giftScore(whichGift,whichChild):
position=childListPosition(whichGift,whichChild)
 if position>0:
return(1000-position-1)*2
else:
return -1
def score(whichChild,whichGift):
tempa=childScore(whichChild,whichGift)
 tempb=giftScore(whichGift,whichChild)
 return tempa*1+tempb*1<normalization>
param_lgb = {
'feature_fraction': 0.4647875434283183,
'lambda_l1': 0.14487098904632512,
'lambda_l2': 0.9546002933329684,
'learning_rate': 0.050592093295320606,
'max_depth': int(round(7.696194993998026)) ,
'min_data_in_leaf': int(round(9.879507661608065)) ,
'min_gain_to_split': 0.7998292013880356,
'min_sum_hessian_in_leaf': 0.24962103361366683,
'num_leaves': int(round(2.854239951949671)) ,
'max_bin': 63,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'save_binary': True,
'seed': 1965,
'feature_fraction_seed': 1965,
'bagging_seed': 1965,
'drop_seed': 1965,
'data_random_seed': 1965,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False}
def changePosition(positionA,positionB):
tempa=submit.iloc[positionA][1]
tempb=submit.iloc[positionB][1]
submit.iloc[positionA][1]=tempb
submit.iloc[positionB][1]=tempa
return<compute_test_metric>
nfold = 20
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019)
 oof = np.zeros(len(fe_train))
predictions = np.zeros(( len(fe_test),nfold))
i = 1
for train_index, valid_index in skf.split(fe_train, fe_train.Choice.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(fe_train.iloc[train_index][predictors].values,
label=fe_train.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(fe_train.iloc[valid_index][predictors].values,
label=fe_train.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 clf = lgb.train(param_lgb, xg_train, 10000000, valid_sets = [xg_valid], verbose_eval=250, early_stopping_rounds = 100)
 oof[valid_index] = clf.predict(fe_train.iloc[valid_index][predictors].values, num_iteration=clf.best_iteration)
 predictions[:,i-1] += clf.predict(fe_test[predictors], num_iteration=clf.best_iteration)
 i = i + 1
print("
CV AUC: {:<0.8f}".format(metrics.roc_auc_score(fe_train.Choice.values, oof)) )
def currentScore(childA,childB):
giftA=submit.iloc[childA][1]
giftB=submit.iloc[childB][1]
scoreA=score(childA,giftA)
 scoreB=score(childB,giftB)
 return scoreA+scoreB
def predictScore(childA,childB):
giftA=submit.iloc[childA][1]
giftB=submit.iloc[childB][1]
scoreA=score(childA,giftB)
 scoreB=score(childB,giftA)
 return scoreA+scoreB<compute_test_metric>
lgb_bay = []
for i in range(len(predictions)) :
lgb_bay.append(predictions[i][-1] )
def agent(childA,childB):
tempA=currentScore(childA,childB)
 tempB=predictScore(childA,childB)
 if tempA>tempB:
return 0
elif tempA==tempB:
changePosition(childA,childB)
 elif tempA<tempB:
changePosition(childA,childB)
 print('childA:'+str(childA)
 +' childB:'+str(childB)
 +' currentScore:'+str(tempA)
 +' predictScore:'+str(tempB)
 +' improved:'+str(tempB-tempA)
)
 else:
return 1 
<save_to_csv>
submission = pd.read_csv('.. /input/sample_predictions.csv')
 submission['Choice'] = lgb_bay
submission.to_csv('sub.csv', index = False, header = True )