Kaggle competition: Text classification
<SOS> metric: categorizationaccuracy Kaggle data source: text-classification<train_model>
sns.set()
%matplotlib inline
df_plt = df_params.dropna().reset_index()
s, x, y, z = zip(*df_plt[['Location', 'Alpha', 'Beta', 'Phi']].values )<filter>
DATA_DIR = '/kaggle/input/texts-classification-iad-hse-intro-2020/'
df_train = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))
df_test = pd.read_csv(os.path.join(DATA_DIR, 'test.csv'))
sub = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'))
df_plt.loc[(df_plt['Beta']==0)&(df_plt['Phi']==0)]<filter>
df_train.fillna('', inplace=True)
 df_test.fillna('', inplace=True )
df_plt.loc[(df_plt['Beta']==0)&(df_plt['Phi']==1)]<filter>
df_train['Category'].value_counts()
df_plt.loc[df_plt['Forecast per Million']>.01*1e6]<filter>
len(df_train['Category'].unique() )
df_plt.loc[[not(( a in [0,1])or 
(b in [0,1])or 
(abs(a-b)<1e-2)) for(a,b)in df_plt[['Alpha','Beta']].values]]<merge>
%%time
df_train['title&description'] = df_train['title'].str[:] + ' ' + df_train['description'].str[:]
df_train.head()
df_plt = pd.merge(df_plt, df_pop, how='left', on='Location')
 df_plt['Forecast'] = df_plt['Forecast per Million'] * df_plt['Population'] / 1000<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(df_train, df_train['Category'], random_state=13)
 X_train.shape, X_val.shape, y_train.shape, y_val.shape
df_r = pd.read_csv('.. /input/my-covid19-dataset/latlong.csv' ).set_index('Location', verify_integrity=True ).loc[:,['Lat', 'Long']]<load_from_csv>
%%time
tfidf = TfidfVectorizer(max_features=10000)
 X_train_tfidf = tfidf.fit_transform(X_train['title&description'] )
df_wdi =(pd.read_csv('.. /input/my-covid19-dataset/world-bank/world-development-indicators.csv')
.dropna(subset=['Country Code','Series Name'], how='all')
.set_index(['Country Code', 'Series Name'], verify_integrity=True)
.drop(columns=['Country Name', 'Series Code'])
.replace({'.. ': np.nan})
.dropna(how='all')
.astype(float))<categorify>
%%time
lr = LogisticRegression(random_state=13)
 lr.fit(X_train_tfidf, y_train )
wdi_default = df_wdi.fillna(df_wdi.median() ).loc['WLD',:]
df_wdi = df_wdi.fillna(wdi_default ).reset_index()<concatenate>
%%time
X_val_tfidf = tfidf.transform(X_val['title&description'] )
new_row = df_wdi.loc[df_wdi['Country Code']=='SGP'].copy(deep=True)
 new_row.loc[:,'Country Code'] = 'TWN'
df_wdi = df_wdi.append(new_row, ignore_index=True )<load_from_csv>
%%time
y_val_pred = lr.predict(X_val_tfidf)
 accuracy_score(y_val, y_val_pred )
df_ncd1 =(pd.read_csv('.. /input/my-covid19-dataset/ncd-risc/obesity/NCD_RisC_Lancet_2017_BMI_age_standardised_country.csv')
.loc[:,['ISO', 'Sex', 'Prevalence of BMI>=30 kg/m2(obesity)']] 
.rename(columns={'ISO': 'Country Code'})
.groupby('Country Code')
.mean() 
.reset_index() )<load_from_csv>
del X_train, y_train, X_val, y_val, X_train_tfidf, X_val_tfidf
df_ncd2 =(pd.read_csv('.. /input/my-covid19-dataset/ncd-risc/blood-pressure/NCD_RisC_Lancet_2016_BP_age_standardised_countries.csv')
.loc[:,['ISO', 'Sex', 'Prevalence of raised blood pressure']] 
.rename(columns={'ISO': 'Country Code'})
.groupby('Country Code')
.mean() 
.reset_index() )<load_from_csv>
df_test['title&description'] = df_test['title'].str[:] + ' ' + df_test['description'].str[:]
df_ihme = pd.read_csv('.. /input/my-covid19-dataset/ihme-gdb/IHME-GBD_2017_DATA-8e93cebf-1.csv')
 mask = [cause_name in ['Neoplasms','Cardiovascular diseases',
'Chronic respiratory diseases','Diabetes and kidney diseases']
for cause_name in df_ihme['cause_name']]
df_ihme =(df_ihme.rename(columns={'Neoplasms': 'Cancer prevalence'})
.loc[mask, ['location_name','cause_name','val']] 
.pivot(index='location_name', columns='cause_name', values='val')
.reset_index() 
.rename(columns={'location_name': 'Location'})
.merge(df_codes, how='left', left_on='Location', right_on='Country_Region')
.drop(columns=['Location','Country_Region']))<merge>
%%time
X_test_tfidf = tfidf.transform(df_test['title&description'])
 y_test_pred = lr.predict(X_test_tfidf )
lst_df = [df_wdi, df_ncd1, df_ncd2, df_ihme]
df_feat = reduce(lambda df_left, df_right: pd.merge(df_left, df_right, how='left', on='Country Code'), lst_df )<merge>
sub['Category'] = y_test_pred
sub.head()
df_feat =(df_cfr.reset_index() 
.merge(df_feat, how='left', on='Country Code' ).drop(columns=['Country Code']))<categorify>
sub.to_csv('submission.csv', index=False )
<prepare_x_and_y><EOS> 
sub.to_csv('submission.csv', index=False )