Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
plt.style.use('seaborn' )
POS_CASH_balance['NAME_CONTRACT_STATUS'] = pd.get_dummies(POS_CASH_balance['NAME_CONTRACT_STATUS'])
 POS_CASH_balance = POS_CASH_balance.fillna(0 )<groupby>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 train[:3]
pcb_count = POS_CASH_balance[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
POS_CASH_balance['SK_ID_PREV'] = POS_CASH_balance['SK_ID_CURR'].map(pcb_count['SK_ID_PREV'] )<groupby>
class UniversalSenteneceEncoder:
def __init__(self, encoder='universal-sentence-encoder', version='4'):
self.version = version
self.encoder = encoder
self.embd = hub.load(f"https://tfhub.dev/google/{encoder}/{version}",)
 def embed(self, sentences):
return self.embd(sentences)
 def squized(self, sentences):
return np.array(self.embd(tf.squeeze(tf.cast(sentences, tf.string))))
pos_cash_mean = POS_CASH_balance.groupby('SK_ID_CURR' ).mean()
pos_cash_mean.columns = ['pc_' + col for col in pos_cash_mean.columns]
pos_cash_mean = pos_cash_mean.reset_index()
pos_cash_mean.head()<merge>
%%time
use = UniversalSenteneceEncoder(encoder='universal-sentence-encoder-large',version='5' )
X = X.merge(right=pos_cash_mean, how='left', on='SK_ID_CURR')
 X.shape<set_options>
%%time
train = train[['text','target']].copy()
train['vects'] = use.squized(train.text ).tolist()
gc.enable()
del POS_CASH_balance
del pos_cash_mean
gc.collect()<load_from_csv>
%%time
test['vects'] = use.squized(test.text ).tolist()
bureau = pd.read_csv('.. /input/bureau.csv')
 bureau.head()<merge>
from sklearn import metrics
from sklearn.model_selection import train_test_split
import xgboost as xgb
bureau_mean = bureau.groupby('SK_ID_CURR' ).mean()
bureau_mean['buro_count'] = bureau[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
bureau_mean.columns = ['b_' + col for col in bureau_mean.columns]
X = X.merge(right=bureau_mean.reset_index() , how='left', on='SK_ID_CURR')
 X.shape<set_options>
train_,test_ = train_test_split(train,test_size=0.33,random_state=42,stratify=train.target )
gc.enable()
del bureau
del bureau_mean
gc.collect() 
<load_from_csv>
%%time
xgc = xgb.XGBClassifier(n_estimators=300)
 xgc.fit(pd.DataFrame(train_.vects.values.tolist()),train_['target'] )
previous_application = pd.read_csv('.. /input/previous_application.csv')
 previous_application.head()<groupby>
%%time
xgc_results = test_.copy()
xgc_results['y_pred'] = xgc.predict(pd.DataFrame(test_.vects.values.tolist()))
previous_app_mean = previous_application.groupby('SK_ID_CURR' ).mean()
previous_app_mean['SK_ID_PREV'] = previous_application[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count() ['SK_ID_PREV']
previous_app_mean.columns = ['pa_' + col for col in previous_app_mean.columns]
previous_app_mean.head()<merge>
print(metrics.classification_report(xgc_results.target,xgc_results.y_pred))
X = X.merge(right=previous_app_mean.reset_index() , how='left', on='SK_ID_CURR')
 X.shape<set_options>
from sklearn.model_selection import GridSearchCV
gc.enable()
del previous_app_mean
del previous_application
gc.collect()<load_from_csv>
from sklearn.model_selection import GridSearchCV
installments_payments = pd.read_csv('.. /input/installments_payments.csv')
 installments_payments.head()<merge>
%%time
xgc_full = xgb.XGBClassifier(n_estimators=300)
 xgc_full.fit(pd.DataFrame(train.vects.values.tolist()),train['target'] )
install_pay_mean = installments_payments.groupby('SK_ID_CURR' ).mean()
install_pay_mean['SK_ID_PREV'] = installments_payments[['SK_ID_CURR','SK_ID_PREV']].groupby('SK_ID_CURR' ).count() ['SK_ID_PREV']
install_pay_mean.columns = ['ip_' + col for col in install_pay_mean.columns]
X = X.merge(right=install_pay_mean.reset_index() , how='left', on='SK_ID_CURR' )<set_options>
%%time
xgc_results_full = test.copy()
xgc_results_full['y_pred'] = xgc_full.predict(pd.DataFrame(test.vects.values.tolist()))
<split><EOS> 
xgc_results_full[['id','y_pred']].rename(columns={'y_pred':'target'} ).to_csv('submissionv.csv',index=False )