Kaggle competition: Text classification
<SOS> metric: categorizationaccuracy Kaggle data source: text-classification<feature_engineering>
import pandas as pd
import gc
df_pop['Population'] =(df_pop[['Population_P/S','Population_C/R']] 
.apply(lambda x: x[1] if np.isnan(x[0])else x[0], axis=1))<rename_columns>
Train = pd.read_csv('.. /input/texts-classification-iad-hse-intro-2020/train.csv' )
df_pop = df_pop.loc[:,['Location','Population']].set_index('Location', verify_integrity=True )<merge>
Test = pd.read_csv('.. /input/texts-classification-iad-hse-intro-2020/test.csv' )
df = pd.merge(df, df_pop, how='left', on='Location' )<feature_engineering>
Train.isnull().sum()
df['Fatalities per Million'] = df['Fatalities'] / df['Population'] * 1000
df['Confirmed Cases per Million'] = df['ConfirmedCases'] / df['Population'] * 1000<data_type_conversions>
Test.isnull().sum()
df['Day Count Confirmed'] =(df['ConfirmedCases']>0 ).groupby(df['Location'] ).cumsum().astype('int')
 df['Day Count Fatalities'] =(df['Fatalities']>0 ).groupby(df['Location'] ).cumsum().astype('int' )<data_type_conversions>
Train.fillna('', inplace=True )
df['Fatalities'] = df.loc[df['Day Count Fatalities']>0, 'Fatalities'].apply(lambda x: max(x,1))
df['Day Count Fatalities'] =(df['Fatalities']>0 ).groupby(df['Location'] ).cumsum().astype('int' )<feature_engineering>
Test.fillna('', inplace=True )
df['New Cases'] = df['ConfirmedCases'].groupby(df['Location'] ).diff() / df['ConfirmedCases']
df['New Fatalities'] = df['Fatalities'].groupby(df['Location'] ).diff() / df['Fatalities']
df['Case Fatality Rate'] = df['Fatalities'] / df['ConfirmedCases']<filter>
Train['title&description'] = Train['title'].str[:] + ' ' + Train['description'].str[:]
df_all = df
df = df[(df['Day Count Fatalities']>0)& 
(( df['Fatalities per Million']>1)| 
(df['New Fatalities']>1/7)) ] 
<load_from_csv>
Test['title&description'] = Test['title'].str[:] + ' ' + Test['description'].str[:]
df_testing =(pd.read_csv('.. /input/my-covid19-dataset/ourworldindata/tests/tests-vs-confirmed-cases-covid-19.csv')
.loc[:,['Entity', 'Total COVID-19 tests']] 
.rename(columns={'Entity':'Location', 'Total COVID-19 tests': 'Tests'}))<merge>
Train.drop(columns=['title', 'description'], inplace=True)
 Test.drop(columns=['title', 'description'], inplace=True )
df = pd.merge(df, df_testing, how='left', on='Location' )<feature_engineering>
gc.collect()
df['Tests per Million'] = df['Tests'] / df['Population'] * 1000
df['Tests per Million'].fillna(df['Confirmed Cases per Million'], inplace=True)
 df.drop(columns=['Province_State', 'Country_Region', 'Tests', 'Population'], inplace=True)
 df['Confirmed Rate'] = df['Confirmed Cases per Million'] / df['Tests per Million']<count_duplicates>
nltk.download('stopwords')
 ru_stopwords = list(stopwords.words("russian"))
mask =(~df.set_index(['Location','Date'], verify_integrity=True)
.index.get_level_values(0)
.duplicated(keep='last'))
df_plt =(df[mask].drop(columns=['Date','Country Code',
'Day Count Confirmed','ConfirmedCases','Confirmed Cases per Million',
'Confirmed Rate','Fatalities'])
.set_index(['Location'], verify_integrity=True))<set_options>
from sklearn.feature_extraction.text import TfidfVectorizer
df_plt.corr().style.background_gradient(cmap='Reds' ).set_precision(2 )<count_duplicates>
tf_idf = TfidfVectorizer(stop_words=ru_stopwords)
 tf_idf.fit(Train['title&description'] )
df = df.set_index(['Location','Day Count Fatalities'], verify_integrity=True ).sort_index()
mask = ~df.index.get_level_values(0 ).duplicated(keep='last' )<feature_engineering>
from sklearn.svm import LinearSVC
df_cfr = df.loc[mask,:].reset_index(level=1, drop=True ).loc[:,['Country Code', 'Case Fatality Rate']]
df_cfr[['Case Fatality Rate']] = df_cfr[['Case Fatality Rate']].apply(lambda x:(x-x.min())/(x.max() -x.min()))<define_variables>
Train_tf_idf = tf_idf.transform(Train['title&description'])
 Test_tf_idf = tf_idf.transform(Test['title&description'] )
state = 'US - New York'
fperiod = 90<filter>
clf = LinearSVC()
clf.fit(Train_tf_idf, Train['Category'] )
fatalities = df.loc[idx[state, :], 'Fatalities per Million'].reset_index(drop=True )<train_model>
del Train, Train_tf_idf
gc.collect()
init_alpha =.3
init_beta =.7
init_phi =.8
initial_level = fatalities[0]
initial_slope = fatalities[1] / fatalities[0]
start_params = [init_alpha, init_beta, initial_level, initial_slope, init_phi]
with warnings.catch_warnings() :
warnings.simplefilter('ignore')
 fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(start_params=start_params, remove_bias=True, use_basinhopping=True))
fcast = fmodel.forecast(fperiod )<create_dataframe>
Answer = pd.DataFrame(columns=['Id', 'Category'])
 Answer['Id'] = Test['itemid']
df_params = pd.DataFrame(index=df.index.unique(level='Location'),
columns=['Alpha', 'Beta', 'Phi' ,'Forecast per Million'])
 keys = ['smoothing_level', 'smoothing_slope', 'damping_slope']
for state in df.index.get_level_values(level=0 ).unique() :
fatalities = df.loc[idx[state, :], 'Fatalities per Million'].reset_index(drop=True)
 if len(fatalities.dropna().index)< 2: continue
with warnings.catch_warnings() :
warnings.simplefilter('ignore')
 fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(remove_bias=True, smoothing_seasonal=0))
_, beta, phi = list(map(fmodel.model.params.get, keys))
if not(( beta==0)and(phi==0)) :
init_alpha =.3
init_beta =.7
init_phi =.8
initial_level = fatalities[0]
initial_slope = fatalities[1] / fatalities[0]
start_params = [init_alpha, init_beta, initial_level, initial_slope, init_phi]
fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(start_params=start_params, remove_bias=True, use_basinhopping=True))
if fcast.iloc[-1]>.02*1e3:
fmodel =(ExponentialSmoothing(fatalities, trend='mul', damped=True, seasonal=None)
.fit(damping_slope=.8, remove_bias=True, use_basinhopping=True))
df_params.loc[state, ['Alpha', 'Beta', 'Phi']] = list(map(fmodel.model.params.get, keys))
df_params.loc[state, 'Forecast per Million'] = fmodel.forecast(fperiod ).iloc[-1]<data_type_conversions>
Answer['Category'] = clf.predict(Test_tf_idf )
<define_variables><EOS> 
Answer.to_csv('my_submission_tfidfvect_nltk_stopwords_lsvc.csv', index=None )