Kaggle competition: Costa Rican Household Poverty Level Prediction
ds['distance']=ds.apply(lambda x: haversine(( x['pickup_latitude'] ,x['pickup_longitude']),(x['dropoff_latitude'], x['dropoff_longitude'])) ,axis=1)
 test['distance']=test.apply(lambda x: haversine(( x['pickup_latitude'] ,x['pickup_longitude']),(x['dropoff_latitude'], x['dropoff_longitude'])) ,axis=1 )<feature_engineering>
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape
<feature_engineering>
df_all = df.append(test)
 df_all.shape
<feature_engineering>
df_all['dependency'].value_counts()
ds['pickup_weekday']=ds['pickup_datetime'].dt.weekday
ds['pickup_hour']=ds['pickup_datetime'].dt.hour
ds['pickup_month']=ds['pickup_datetime'].dt.month
ds['pickup_day']=ds['pickup_datetime'].dt.day<feature_engineering>
df_all['edjefa'].value_counts()
test['pickup_weekday']=test['pickup_datetime'].dt.weekday
test['pickup_hour']=test['pickup_datetime'].dt.hour
test['pickup_month']=test['pickup_datetime'].dt.month
test['pickup_day']=ds['pickup_datetime'].dt.day<categorify>
df_all['edjefe'].value_counts()
ds['isweekend']= ds.apply(lambda x :(x['pickup_weekday']==6 | x['pickup_weekday']==5),axis=1)
 ds['isweekend']=ds['isweekend'].map({True: 1, False:0})
 ds['store_and_fwd_flag']=ds['store_and_fwd_flag'].map({'N': 1, 'Y':0} )<categorify>
mapeamento = {'yes': 1, 'no': 0}
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )
test['isweekend']= test.apply(lambda x :(x['pickup_weekday']==6 | x['pickup_weekday']==5),axis=1)
 test['isweekend']=test['isweekend'].map({True: 1, False:0})
 test['store_and_fwd_flag']=test['store_and_fwd_flag'].map({'N': 1, 'Y':0} )<prepare_x_and_y>
df_all['dependency'].value_counts()
feature_cols=['vendor_id','passenger_count','pickup_id','dropoff_id','pickup_weekday','pickup_hour'
,'pickup_month','store_and_fwd_flag' ,'distance']
X=ds[feature_cols]
Y=ds['trip_duration_log']
test_features=test[feature_cols]<split>
df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )
X_train,X_test,Y_train,Y_test= b.train_test_split(X,Y,test_size=0.2, random_state=420)
 X_train,X_Val,Y_train,Y_Val= b.train_test_split(X_train,Y_train,test_size=0.1, random_state=420 )<train_model>
df_all.isnull().sum()
taxi_trip_model = XGBRegressor(n_estimators=200,learning_rate=0.05)
 taxi_trip_model.fit(X_train,Y_train,early_stopping_rounds=10,
eval_set=[(X_Val, Y_Val)])
 pred_test = taxi_trip_model.predict(X_test, ntree_limit=taxi_trip_model.best_ntree_limit )<compute_test_metric>
df_all[df_all['parentesco1'] == 1]['v2a1'].isnull().sum()
mae = mean_absolute_error(Y_test,pred_test)
 print(mae )<predict_on_test>
df_all['v18q'].value_counts()
predtest = taxi_trip_model.predict(test_features,ntree_limit=taxi_trip_model.best_ntree_limit )<prepare_x_and_y>
df_all.isnull().sum().sort_values()
y_test =np.exp(predtest)-1<save_to_csv>
df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )
output = pd.DataFrame({'id': test['id'],
'trip_duration': y_test})
 output.to_csv('submission_xgb_best.csv', index=False )<prepare_x_and_y>
df_all.Target.value_counts()
<set_options>
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]
pd.set_option("max_columns",300)
 pd.set_option("max_rows",1103)
 tqdm.pandas()<load_from_csv>
rf = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=701, min_impurity_decrease=0.0007, min_samples_leaf=2, min_samples_split=5, verbose=0, class_weight='balanced')
 rf.fit(train[feats], train['Target'])
 test['Target'] = rf.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )
df_train = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_path = '.. /input/imet-2019-fgvc6/train/'
label_df = pd.read_csv('.. /input/imet-2019-fgvc6/labels.csv' )<feature_engineering>
df_a = df_all[df_all['Target'] == 1]
df_b = df_all[df_all['Target'] == 2]
df_c = df_all[df_all['Target'] == 3]
df_d = df_all[df_all['Target'] == 4]
label_names = label_df['attribute_name'].values
num_labels = np.zeros(( df_train.shape[0],))
train_labels = np.zeros(( df_train.shape[0], len(label_names)))
 for row_index, row in enumerate(df_train['attribute_ids']):
num_labels[row_index] = len(row.split())
 for label in row.split() :
train_labels[row_index, int(label)] = 1<define_variables>
df_a_os = resample(df_a, replace=True, n_samples=len(df_d), random_state=42)
 df_b_os = resample(df_b, replace=True, n_samples=len(df_d), random_state=42)
 df_c_os = resample(df_c, replace=True, n_samples=len(df_d), random_state=42)
 df_os = pd.concat([df_a_os, df_b_os, df_c_os, df_d] )
culture, tag, unknown = 0, 0, 0
for l in label_names:
if l[:3] == 'cul':
culture += 1
elif l[:3] == 'tag':
tag += 1
else:
unknown += 1
print(f'Culture : {culture}')
 print(f'Tag : {tag}')
 print(f'Unknown : {unknown}')
 print(f'Total : {culture + tag + unknown}' )<define_variables>
df_os['Target'].value_counts()
label_sum = np.sum(train_labels, axis=0)
 culture_sequence = label_sum[:398].argsort() [::-1]
tag_sequence = label_sum[398:].argsort() [::-1]
culture_labels = [label_names[x][9:] for x in culture_sequence]
culture_counts = [label_sum[x] for x in culture_sequence]
tag_labels = [label_names[x + 398][5:] for x in tag_sequence]
tag_counts = [label_sum[x + 398] for x in tag_sequence]<save_to_csv>
train, test = train_test_split(df_os, test_size=0.2, random_state=42)
 rf.fit(train[feats], train['Target'])
 preds_test = rf.predict(test[feats])
 accuracy_score(test['Target'], preds_test )
df = pd.DataFrame({'Culture_label': culture_labels,'Culture_count': culture_counts})
 df.to_csv('cutr_labe.csv',index=True )<count_values>
df_b_us = resample(df_b, replace=False, n_samples=len(df_a),random_state=42)
 df_c_us = resample(df_c, replace=False, n_samples=len(df_a),random_state=42)
 df_d_us = resample(df_d, replace=False, n_samples=len(df_a),random_state=42)
 df_us = pd.concat([df_a, df_b_us, df_c_us, df_d_us] )
for i in range(len(tag_labels)) :
print(tag_labels[i],':',tag_counts[i] )<save_to_csv>
df_us['Target'].value_counts()
<feature_engineering><EOS> 
train, test = train_test_split(df_us, test_size=0.2, random_state=42)
 rf.fit(train[feats], train['Target'])
 preds_test = rf.predict(test[feats])
 accuracy_score(test['Target'], preds_test )