Kaggle competition: Sentiment Analysis in Russian
<SOS> metric: meanfscore Kaggle data source: sentiment-analysis-in-russian<prepare_x_and_y>
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords
from nltk import word_tokenize
import pandas as pd
import numpy as np
import re
from sklearn.model_selection import train_test_split
from collections import defaultdict
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score, classification_report
import json
df = df_sample.drop(columns = ['winPlacePerc','winPlacePerc_rank'])
 y = df_sample['winPlacePerc']
del(df_sample )<train_model>
with open('.. /input/train.json')as f:
raw_train = json.load(f)
 with open('.. /input/test.json')as f:
raw_test = json.load(f )
def split_vals(a, n : int):
return a[:n].copy() , a[n:].copy()
val_perc = 0.15
n_valid = int(val_perc * sample)
 n_trn = len(df)-n_valid
X_train, X_valid = split_vals(df, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 train_columns = list(X_train.columns)
 X_train = scaler.fit_transform(X_train)
 X_valid = scaler.transform(X_valid)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape)
<compute_train_metric>
def ru_token(string):

 return [i for i in word_tokenize(string)if re.match(r'[\u0400-\u04ffа́]+$', i)]
def print_score(m : RandomForestRegressor):
res = ['mae train: ', mae(m.predict(X_train), y_train),
'mae val: ', mae(m.predict(X_valid), y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )<train_model>
params = {}
params['tokenizer'] = ru_token
params['stop_words'] = stopwords.words('russian')
 params['ngram_range'] =(1, 3)
 params['min_df'] = 3
pi_model = RandomForestRegressor(n_estimators=10, n_jobs=-1 ).fit(X_train, y_train)
 perm = PermutationImportance(pi_model, random_state=1 ).fit(X_valid, y_valid)
<create_dataframe>
tfidf = TfidfVectorizer(**params )
fi_pi = pd.DataFrame({'importance':perm.feature_importances_, 'column':train_columns})
 to_keep = fi_pi.sort_values('importance', ascending=False)[0:80]['column']
to_keep = list(to_keep)
<train_model>
tfidf.fit([i['text'] for i in raw_train + raw_test] )
val_perc = 0.15
n_valid = int(val_perc * sample)
 n_trn = len(df)-n_valid
X_train, X_valid = split_vals(df, n_trn)
 y_train, y_valid = split_vals(y, n_trn)
 X_train = X_train[to_keep]
X_valid = X_valid[to_keep]
train_columns = list(X_train.columns)
 X_train = scaler.fit_transform(X_train)
 X_valid = scaler.transform(X_valid)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )<train_model>
train = {}
val = {}
tmp = defaultdict(list)
 for e in raw_train:
tmp[e['sentiment']].append(e['text'])
 for l in tmp:
train[l], val[l] = train_test_split(tmp[l], test_size=0.2, random_state=2018 )
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 params = {
'boosting_type': 'goss',
'objective': 'regression',
'metric': 'mae',
'learning_rate': 0.05,
'feature_fraction': 0.9,
'verbose': 0,
'n_estimators':10000,
}
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':15000,
'early_stopping_rounds':100, "num_leaves" : 40, "learning_rate" : 0.05,
"bagging_fraction" : 0.9, "bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.7
}
gbm = lgb.train(params,
lgb_train,
num_boost_round=1000,
valid_sets=lgb_eval,
verbose_eval=1000)
 display(mae(gbm.predict(X_valid, num_iteration=gbm.best_iteration),y_valid))<filter>
def upsampling_align(some_dict, random_state=2018):
rand = np.random.RandomState(random_state)
 upper = max([len(some_dict[l])for l in some_dict])
 print('upper bound: {}'.format(upper))
tmp = {}
for l in some_dict:
if len(some_dict[l])< upper:
repeat_time = int(upper/len(some_dict[l]))
remainder = upper % len(some_dict[l])
 _tmp = some_dict[l].copy()
rand.shuffle(_tmp)
 tmp[l] = some_dict[l] * repeat_time + _tmp[:remainder]
rand.shuffle(tmp[l])
 else:
tmp[l] = some_dict[l]
return tmp
( imp_df['importance_gain']<100 ).index<feature_engineering>
btrain = upsampling_align(train )
def sub(test):
test['headshot_rate'] = test['headshotKills'] / test['kills']
test['headshot_rate'] = test['headshot_rate'].fillna(0)
 test['totalDistance'] = test['rideDistance'] + test['walkDistance'] + test['swimDistance']
test['playersJoined'] = test.groupby('matchId')['matchId'].transform('count')
 test['killsNorm'] = test['kills']*(( 100-test['playersJoined'])/100 + 1)
 test['damageDealtNorm'] = test['damageDealt']*(( 100-test['playersJoined'])/100 + 1)
 test['maxPlaceNorm'] = test['maxPlace']*(( 100-test['playersJoined'])/100 + 1)
 test['matchDurationNorm'] = test['matchDuration']*(( 100-test['playersJoined'])/100 + 1)
 test['DBNOsNorm'] = test['DBNOs']*(( 100-test['playersJoined'])/100 + 1)
 test['healsandboosts'] = test['heals'] + test['boosts']
top = ['walkDistance','walkDistance','damageDealt']
bot = ['boosts','kills','kills']
for t,b in zip(top,bot):
test.loc[(test[b]!=0),t+'/'+b] = test[t]/test[b]
test.loc[(test[b]==0),t+'/'+b] = 0
features_to_agg = ['groupId','DBNOs','kills','totalDistance','headshot_rate',
'damageDealt','killPlace','walkDistance/boosts','walkDistance/kills','damageDealt/kills']
aggs = test[features_to_agg].groupby('groupId' ).agg([np.sum, np.mean, np.std])
 test_slice = test[['groupId']]
for col in list(aggs.columns.levels[0]):
for ag in list(aggs.columns.levels[1]):
test_slice = test_slice.join(aggs[col,ag], on='groupId')
 test_slice.columns = test_slice.columns.map(lambda x: x[0]+'_'+x[1] if type(x)==tuple else x)
 test_slice.fillna(0, inplace=True)
 del(aggs)
 test_slice.drop(columns='groupId', inplace=True)
 test = pd.concat([test, test_slice], axis=1)
 del(test_slice)
 features = list(test.columns)
 features.remove('Id')
 features.remove('matchType')
 features.remove('groupId')
 print(features)
 ranked = test.groupby('matchId')[features].rank(pct=True)
 ranked.columns = ranked.columns.map(lambda x: x + '_rank')
 test = pd.concat([test, ranked], axis=1)
 del(ranked)
 test.drop(columns=['matchType', 'groupId','matchId', 'Id'], inplace=True)
 test = test[train_columns]
test.fillna(0, inplace=True)
 test = scaler.fit_transform(test)
 return test
test = pd.read_csv(KAGGLE_DIR + 'test_V2.csv')
 id_ = test['Id']
test = sub(test )<prepare_output>
m_params = {}
m_params['solver'] = 'lbfgs'
m_params['multi_class'] = 'multinomial'
predictions = np.clip(a = gbm.predict(test, num_iteration=gbm.best_iteration), a_min=0.0, a_max=1.0)
 pred_df = pd.DataFrame({'Id' : id_, 'winPlacePerc' : predictions})
 pred_df 
<save_to_csv>
softmax = LogisticRegression(**m_params )
pred_df.to_csv("submission_2.csv", index=False )<set_options>
train_x = [j for i in sorted(btrain.keys())for j in btrain[i]]
train_y = [i for i in sorted(btrain.keys())for j in btrain[i]]
softmax.fit(tfidf.transform(train_x), train_y )
gc.enable()<load_from_csv>
test_x = [j for i in sorted(val.keys())for j in val[i]]
true = [i for i in sorted(val.keys())for j in val[i]]
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/pubg-finish-placement-prediction/test_V2.csv')
 test_idx = df.Id
print("remove some columns")
 target = 'winPlacePerc'
print("Adding Features")
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df["headshotKills"] + df["roadKills"]
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
print("Removing Na's From DF")
 df.fillna(0, inplace=True)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
pred = softmax.predict(tfidf.transform(test_x))
x_train, y_train, train_columns, _ = feature_engineering(True,False )<feature_engineering>
accuracy_score(true, pred )
x_test, _, _ , test_idx = feature_engineering(False,True )<drop_column>
lab = LabelEncoder()
c_true = lab.fit_transform(true)
 c_pred = lab.transform(pred)
 print(classification_report(c_true, c_pred, target_names=lab.classes_, digits=5))
x_train = reduce_mem_usage(x_train)
 x_test = reduce_mem_usage(x_test )<import_modules>
bval = upsampling_align(val )
import lightgbm as lgb<train_model>
b_test_x = [j for i in sorted(bval.keys())for j in bval[i]]
b_true = [i for i in sorted(bval.keys())for j in bval[i]]
b_pred = softmax.predict(tfidf.transform(b_test_x))
lab = LabelEncoder()
c_true = lab.fit_transform(b_true)
 c_pred = lab.transform(b_pred)
 print(classification_report(c_true, c_pred, target_names=lab.classes_, digits=5))
train_index = round(int(x_train.shape[0]*0.8))
dev_X = x_train[:train_index]
val_X = x_train[train_index:]
dev_y = y_train[:train_index]
val_y = y_train[train_index:]
gc.collect() ;
def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':20000, 'early_stopping_rounds':200,
"num_leaves" : 31, "learning_rate" : 0.05, "bagging_fraction" : 0.7,
"bagging_seed" : 0, "num_threads" : 4,"colsample_bytree" : 0.7
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y, model
pred_test, model = run_lgb(dev_X, dev_y, val_X, val_y, x_test )<load_from_csv>
sub_pred = softmax.predict(tfidf.transform([i['text'] for i in raw_test]))
sub_df = pd.DataFrame()
sub_df['id'] = [i['id'] for i in raw_test]
sub_df['sentiment'] = sub_pred
df_sub = pd.read_csv(".. /input/pubg-finish-placement-prediction/sample_submission_V2.csv")
 df_test = pd.read_csv(".. /input/pubg-finish-placement-prediction/test_V2.csv")
 df_sub['winPlacePerc'] = pred_test
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0
df_sub[["Id", "winPlacePerc"]].to_csv("submission.csv", index=False )<save_model>
sub_df.to_csv('softmax_reg.csv', index=False )