Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<find_best_params>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
Actual_1 = 'My life is totally awesome'
Predict_1 = 'awesome life'
Actual_2 = 'We are active kagglers'
Predict_2 = 'We are active kagglers'
print("Jaccard score for first set of scentences: {}".format(jaccard(Actual_1,Predict_1)))
 print("Jaccard score for second set of scentences: {}".format(jaccard(Actual_2,Predict_2)) )
grid_search_random_forest_classifier.best_params_<define_search_space>
stop=set(stopwords.words('english'))
warnings.filterwarnings("ignore")
 cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
xgb_param_grid={'colsample_bylevel':[0.1, 0.9, 1],
'colsample_bytree' :[0.2, 0.8, 1],
'gamma' :[0.99, 9, 99],
'max_depth' :[2, 4, 6, 8, 10],
'min_child_weight' :[1, 2, 4, 6, 8, 10],
'n_estimators' :[10, 20, 50, 70, 100, 200, 500, 1000]}
grid_search_xgboost_classifier = GridSearchCV(xgbClassifier, param_grid=xgb_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)
 grid_search_xgboost_classifier.fit(X_train, y_train)
 xgb_best = grid_search_xgboost_classifier.best_estimator_
grid_search_xgboost_classifier.best_score_<find_best_params>
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 train.sample(6 )
grid_search_xgboost_classifier.best_params_<split>
def miss_val(df):
total=df.isnull().sum()
return pd.concat([total],axis=1,keys=['Total'])
 print("Missing values for train dataset
")
 print(miss_val(train))
print("---------------------------------------------------------------------")
 print("Missing values for test dataset
")
 print(miss_val(test))
x_tr, x_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.3, random_state=random_state )<train_model>
train=train.dropna()
train.shape
random_forest_classifier = RandomForestClassifier(n_estimators=200, max_depth=3, max_features=8, min_samples_split=2, bootstrap=True, random_state=random_state)
 random_forest_classifier.fit(x_tr, y_tr)
 predicted_test = random_forest_classifier.predict(x_val)
 accuracy_random_forest = accuracy_score(y_val, predicted_test)
 f1_random_forest = f1_score(y_val, predicted_test)
 print("Random forest scores: ", accuracy_random_forest, f1_random_forest )<compute_train_metric>
train['target']=train['selected_text'].str.lower()
xgbClassifier = XGBClassifier(max_depth=4,n_estimators=20,gamma=9,colsample_bylevel=0.9,random_state=random_state)
 xgbClassifier.fit(x_tr, y_tr)
 predicted_test = xgbClassifier.predict(x_val)
 accuracy_xgb = accuracy_score(y_val, predicted_test)
 f1_xgb = f1_score(y_val, predicted_test)
 print("XGBooster scores: ", accuracy_xgb, f1_xgb )<save_to_csv>
def find_link(string):
url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', string)
 return "".join(url)
 train['target_url']=train['target'].apply(lambda x: find_link(x))
df=pd.DataFrame(train.loc[train['target_url']!=""]['sentiment'].value_counts() ).reset_index()
df.rename(columns={"index": "sentiment", "sentiment": "url_count"} )
xgbClassifier.fit(X_train, y_train)
 submission_prediction = xgbClassifier.predict(test_data)
 submission = pd.DataFrame({
'PassengerId': test_data['PassengerId'],
'Survived': submission_prediction
})
 submission_file_path = './titanic_submission.csv'
submission.to_csv(submission_file_path, index=False )<set_options>
def find_punct(text):
line = re.findall(r'[!"\$%&'() *+,\-.\/:;=
string="".join(line)
 return list(string )
pd.set_option('max_columns', None)
 plt.style.use('seaborn')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
<load_from_csv>
train['target_punct']=train['target'].apply(lambda x:find_punct(x))
train['target_punct_len']=train['target'].apply(lambda x:len(find_punct(x)) )
data_dict = {}
for i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/*'):
name = i.split('/')[-1].split('.')[0]
if name != 'MTeamSpellings':
data_dict[name] = pd.read_csv(i)
 else:
data_dict[name] = pd.read_csv(i, encoding='cp1252' )<merge>
punc_df=pd.DataFrame(train,columns=['target_punct','sentiment'])
 punc_df=punc_df[punc_df['target_punct'].map(lambda d: len(d)) > 0]
punc_df=punc_df.explode('target_punct')
 positive_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="positive"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'pos_punct'})
 negative_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="negative"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neg_punct'})
 neutral_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="neutral"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neut_punct'})
data_dict['MNCAATourneySeeds']['Seed'] = data_dict['MNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))
data_dict['MNCAATourneySeeds'] = data_dict['MNCAATourneySeeds'][['Season', 'TeamID', 'Seed']]
data_dict['MNCAATourneyCompactResults'] = data_dict['MNCAATourneyCompactResults'][['Season','WTeamID', 'LTeamID']]
df = pd.merge(data_dict['MNCAATourneyCompactResults'], data_dict['MNCAATourneySeeds'],
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 df = pd.merge(df, data_dict['MNCAATourneySeeds'], how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'])
 df = df.drop(['TeamID_x', 'TeamID_y'], axis=1)
 df.columns = ['Season', 'WTeamID', 'LTeamID', 'WSeed', 'LSeed']
df.head()<merge>
train['star']=train['target'].apply(lambda x:find_star(x))
train.loc[train['star']!=0]['sentiment'].value_counts().to_frame()
team_win_score = data_dict['MRegularSeasonCompactResults'].groupby(['Season', 'WTeamID'] ).agg({'WScore':['sum', 'count']} ).reset_index()
team_win_score.columns = ['Season', 'WTeamID', 'WScore_sum', 'WScore_count']
team_loss_score = data_dict['MRegularSeasonCompactResults'].groupby(['Season', 'LTeamID'] ).agg({'LScore':['sum', 'count']} ).reset_index()
team_loss_score.columns = ['Season', 'LTeamID', 'LScore_sum', 'LScore_count']
df = pd.merge(df, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 df = pd.merge(df, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 df = pd.merge(df, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 df = pd.merge(df, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 df.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 df['x_score'] = df['WScore_sum_x'] + df['LScore_sum_y']
df['y_score'] = df['WScore_sum_y'] + df['LScore_sum_x']
df['x_count'] = df['WScore_count_x'] + df['LScore_count_y']
df['y_count'] = df['WScore_count_y'] + df['WScore_count_x']
df = df.drop(['WScore_sum_x','WScore_count_x','LScore_sum_x','LScore_count_x',
'LScore_sum_y','LScore_count_y','WScore_sum_y','WScore_count_y'], axis =1)
 df.head()<feature_engineering>
def find_only_star(text):
if len(text.split())==1:
line=re.findall(r'[*]{2,5}',text)
 return len(line)
 else:
return 0
df_win = df.copy()
df_los = df.copy()
df_win = df_win[['WSeed', 'LSeed', 'x_score', 'y_score', 'x_count', 'y_count']]
df_los = df_los[['LSeed', 'WSeed', 'y_score', 'x_score', 'x_count', 'y_count']]
df_win.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']
df_los.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']
df_win['Seed_diff'] = df_win['Seed_1'] - df_win['Seed_2']
df_win['Score_diff'] = df_win['Score_1'] - df_win['Score_2']
df_los['Seed_diff'] = df_los['Seed_1'] - df_los['Seed_2']
df_los['Score_diff'] = df_los['Score_1'] - df_los['Score_2']<concatenate>
train['only_star']=train['target'].apply(lambda x:find_only_star(x))
train.loc[train['only_star']==1]['sentiment'].value_counts()
df_win['result'] = 1
df_los['result'] = 0
data = pd.concat(( df_win, df_los)).reset_index(drop=True )<merge>
train['target']= np.where(train['only_star']==1,"abusive",train['target'] )
test = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 test = test.drop(['Pred'], axis=1)
 test['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))
test['Team1'] = test['ID'].apply(lambda x: int(x.split('_')[1]))
test['Team2'] = test['ID'].apply(lambda x: int(x.split('_')[2]))
test = pd.merge(test, data_dict['MNCAATourneySeeds'], how='left', left_on=['Season', 'Team1'], right_on=['Season', 'TeamID'])
 test = pd.merge(test, data_dict['MNCAATourneySeeds'], how='left', left_on=['Season', 'Team2'], right_on=['Season', 'TeamID'])
 test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'Team1'], right_on=['Season', 'WTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'Team2'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'Team1'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'Team2'], right_on=['Season', 'WTeamID'])
 test.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 test.head()<feature_engineering>
def remove_link(string):
text = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +'," ",string)
 return " ".join(text.split() )
test['x_score'] = test['WScore_sum_x'] + test['LScore_sum_y']
test['y_score'] = test['WScore_sum_y'] + test['LScore_sum_x']
test['x_count'] = test['WScore_count_x'] + test['LScore_count_y']
test['y_count'] = test['WScore_count_y'] + test['WScore_count_x']
test = test[['Seed_x', 'Seed_y', 'x_score', 'y_score', 'x_count', 'y_count']]
test.columns = ['Seed_1', 'Seed_2', 'Score_1', 'Score_2', 'Count_1', 'Count_2']
test['Seed_diff'] = test['Seed_1'] - test['Seed_2']
test['Score_diff'] = test['Score_1'] - test['Score_2']
test_df = test<prepare_x_and_y>
def remove_punct(text):
line = re.sub(r'[!"\$%&'() *+,\-.\/:;=
return " ".join(line.split() )
params_lgb = {'num_leaves': 127,
'min_data_in_leaf': 10,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'logloss',
"verbosity": -1,
'random_state': 42,
}
X = data.drop('result', axis=1)
 y = data['result']<prepare_x_and_y>
train['target']=train['target'].apply(lambda x:remove_link(x))
train['target']=train['target'].apply(lambda x:remove_punct(x))
NFOLDS = 10
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds_lgb = np.zeros(test_df.shape[0])
 y_train_lgb = np.zeros(X.shape[0])
 y_oof = np.zeros(X.shape[0])
 feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params_lgb, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
y_train_lgb += clf.predict(X)/ NFOLDS
y_preds_lgb += clf.predict(test_df)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()<import_modules>
def create_corpus(data,feature,sentiment):
corpus=[]
for x in data[data['sentiment']==sentiment][feature].str.split() :
for i in x:
corpus.append(i)
 return corpus
<merge>
def corpus_sentiment(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word not in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y
df_compact = data_dict['MNCAATourneyCompactResults']
df_detailed = data_dict['MNCAATourneyDetailedResults']
results_merged = pd.merge(df_compact, df_detailed)
 results_merged.head()<feature_engineering>
def count_values(df,feature):
total=df.loc[:,feature].value_counts(dropna=False)
 percent=round(df.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2)
 return pd.concat([total,percent],axis=1,keys=['Total','Percent'])
results_merged['WeFG%'] =(results_merged['WFGM']+0.5*results_merged['WFGM3'])/ results_merged['WFGA']
results_merged['LeFG%'] =(results_merged['LFGM']+0.5*results_merged['LFGM3'])/ results_merged['LFGA']
results_merged['WTO%'] = results_merged['WTO'] /(results_merged['WFGA']+0.44*results_merged['WFTA'] +results_merged['WTO'])
 results_merged['LTO%'] = results_merged['LTO'] /(results_merged['LFGA']+0.44*results_merged['LFTA'] +results_merged['LTO'])
 results_merged['WFTR%'] = results_merged['WFTA'] / results_merged['WFGA']
results_merged['LFTR%'] = results_merged['LFTA'] / results_merged['LFGA']
results_merged['WORB%'] = results_merged['WOR'] / results_merged['LDR']
results_merged['LORB%'] = results_merged['LOR'] / results_merged['WDR']<load_from_csv>
count_values(train,'sentiment' )
ts = TrueSkill(draw_probability=0.01)
 beta = 25 / 6
def win_probability(p1, p2):
delta_mu = p1.mu - p2.mu
sum_sigma = p1.sigma * p1.sigma + p2.sigma * p2.sigma
denom = np.sqrt(2 *(beta * beta)+ sum_sigma)
 return ts.cdf(delta_mu / denom)
 submit = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 submit[['Season', 'Team1', 'Team2']] = submit.apply(lambda r:pd.Series([int(t)for t in r.ID.split('_')]), axis=1)
 df_tour = results_merged
teamIds = np.unique(np.concatenate([df_tour.WTeamID.values, df_tour.LTeamID.values]))
ratings = { tid:ts.Rating() for tid in teamIds }
def feed_season_results(season):
print("season = {}".format(season))
df1 = df_tour[df_tour.Season == season]
for r in df1.itertuples() :
ratings[r.WTeamID], ratings[r.LTeamID] = rate_1vs1(ratings[r.WTeamID], ratings[r.LTeamID])
 def update_pred(season):
beta = np.std([r.mu for r in ratings.values() ])
 print("beta = {}".format(beta))
submit.loc[submit.Season==season, 'Pred'] = submit[submit.Season==season].apply(lambda r:win_probability(ratings[r.Team1], ratings[r.Team2]), axis=1)
 for season in sorted(df_tour.Season.unique())[:-5]:
feed_season_results(season)
 update_pred(2015)
 feed_season_results(2015)
 update_pred(2016)
 feed_season_results(2016)
 update_pred(2017)
 feed_season_results(2017)
 update_pred(2018)
 feed_season_results(2018)
 update_pred(2019)
 submit.drop(['Season', 'Team1', 'Team2'], axis=1, inplace=True)
 y_preds_ts = submit['Pred']<load_from_csv>
count_values(train,'sentiment' )
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = 0.4*y_preds_lgb + 0.6*y_preds_ts<save_to_csv>
full_data=pd.concat([train,test])
 full_data['text']=full_data['text'].str.lower()
full_data.shape
submission_df.to_csv('submission.csv', index=False )<load_from_csv>
full_data['text_punct']=full_data['text'].apply(lambda x:find_punct(x))
full_data['text_punct_len']=full_data['text'].apply(lambda x:len(find_punct(x)) )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<load_from_csv>
punc_text_df=pd.DataFrame(full_data,columns=['text_punct','sentiment'])
 punc_text_df=punc_text_df[punc_text_df['text_punct'].map(lambda d: len(d)) > 0]
punc_text_df=punc_text_df.explode('text_punct')
 positive_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="positive"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'pos_punct'})
 negative_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="negative"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neg_punct'})
 neutral_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="neutral"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neut_punct'})
test = pd.read_csv("/kaggle/input/titanic/test.csv")
 test.head()<drop_column>
full_data['text']=full_data['text'].apply(lambda x:remove_link(x))
full_data['text']=full_data['text'].apply(lambda x:remove_punct(x))
train_data = train_data.drop(columns = ['Cabin','Name','Ticket','PassengerId'] )<data_type_conversions>
full_data.loc[full_data['text']=="",['text']]="nothing"
train_data['Age'].fillna(( train_data['Age'].mean()), inplace=True )<sort_values>
def corpus_sentiment_stop(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y
train_data[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 train_data[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<split>
train_word=full_data[:train.shape[0]]
test_word=full_data[train.shape[0]:]
yf = train_data.Survived
base_features = ['Parch','SibSp','Age', 'Fare','Pclass']
Xf = train_data[base_features]
train_X, val_X, train_y, val_y = train_test_split(Xf, yf, random_state=1)
 first_model = RandomForestRegressor(n_estimators=21, random_state=1 ).fit(train_X, train_y )<groupby>
pos_text_list=list(corpus_sentiment(train_word,'text','positive')[0])
 pos_target_list=list(corpus_sentiment(train_word,'target','positive')[0])
 neg_text_list=list(corpus_sentiment(train_word,'text','negative')[0])
 neg_target_list=list(corpus_sentiment(train_word,'target','negative')[0])
 neutral_text_list=list(corpus_sentiment(train_word,'text','neutral')[0])
 neutral_target_list=list(corpus_sentiment(train_word,'target','neutral')[0])
 pos_common_words_list=list(set(pos_text_list ).intersection(pos_target_list))
neg_common_words_list=list(set(neg_text_list ).intersection(neg_target_list))
neutral_common_words_list=list(set(neutral_text_list ).intersection(neutral_target_list))
train_data['FamilySize'] = train_data['SibSp'] + train_data['Parch']
train_data[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).agg('mean' )<feature_engineering>
results_jaccard=[]
for ind,row in train_word.iterrows() :
sentence1 = row.text
sentence2 = row.target
jaccard_score = jaccard(sentence1,sentence2)
 results_jaccard.append([sentence1,sentence2,jaccard_score] )
train_data['IsAlone'] = 0
train_data.loc[train_data['FamilySize'] == 0, 'IsAlone'] = 1
train_data[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()<sort_values>
jaccard = pd.DataFrame(results_jaccard,columns=["text","target","jaccard_score"])
 train_word = train_word.merge(jaccard,how='outer' )
train_data[["Fare", "Survived"]].groupby(['Survived'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 train_data.groupby(['Sex','Survived'])[['Fare']].agg(['min','mean','max'] )<train_model>
print("Jaccard score equals 1 : {}%".format(round(train_word.loc[train_word['jaccard_score']==1].shape[0]/train_word.shape[0]*100,2)))
 print("Jaccard score less than 0.3 : {}%".format(round(train_word.loc[train_word['jaccard_score']<0.3].shape[0]/train_word.shape[0]*100,2)) )
y2 = train_data.Survived
base_features2 = ['Parch','SibSp','Age', 'Fare','Pclass','Age*Class','FamilySize','IsAlone']
X2 = train_data[base_features2]
train_X2, val_X2, train_y2, val_y2 = train_test_split(X2, y2, random_state=1)
 second_model = RandomForestRegressor(n_estimators=21, random_state=1 ).fit(train_X2, train_y2)
 inter2 = pdp.pdp_interact(model=second_model, dataset=val_X2, model_features=base_features2, features=['Age', 'Pclass'])
 pdp.pdp_interact_plot(pdp_interact_out=inter2, feature_names=['Age', 'Pclass'], plot_type='contour')
 plt.show()<categorify>
train_word['difference']=abs(train_word['text_tweet_length']-train_word['target_tweet_length'] )
dummies_Sex = pd.get_dummies(train_data.Sex)
 dummies_Embarked = pd.get_dummies(train_data.Embarked)
 train_ready = pd.concat([train_data, dummies_Sex,dummies_Embarked], axis=1)
 train_ready.head()
train_ready = train_ready.drop(columns = ['Sex','Embarked'])
 train_ready.info()<drop_column>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train_ready = train_ready.drop(columns = ['Age*Class'])
 train_ready = train_ready.drop(columns = ['FamilySize'] )<compute_test_metric>
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 ct=train.shape[0]
input_ids=np.ones(( ct,MAX_LEN),dtype="int32")
 attention_mask=np.zeros(( ct,MAX_LEN),dtype="int32")
 token_type_ids=np.zeros(( ct,MAX_LEN),dtype="int32")
 start_tokens=np.zeros(( ct,MAX_LEN),dtype="int32")
 end_tokens=np.zeros(( ct,MAX_LEN),dtype="int32" )
for name in train_ready:
print(name, "column entropy :", round(stats.entropy(train_ready[name].value_counts(normalize=True), base=2),2))<drop_column>
for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ':
chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
test = test.drop(columns = ['Cabin','Name','Ticket','PassengerId'])
 test.head()<feature_engineering>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct_test = test.shape[0]
input_ids_t = np.ones(( ct_test,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
test['Age'].fillna(( test['Age'].mean()), inplace=True)
 test['Fare'].fillna(( test['Fare'].mean()), inplace=True)
 test.loc[ test['Fare'] <= 7.22, 'Fare'] = 0
test.loc[(test['Fare'] > 7.22)&(test['Fare'] <= 21.96), 'Fare'] = 1
test.loc[(test['Fare'] > 21.96)&(test['Fare'] <= 40.82), 'Fare'] = 2
test.loc[ test['Fare'] > 40.82, 'Fare'] = 3
test['FamilySize'] = test['SibSp'] + test['Parch'] + 1
test['IsAlone'] = 0
test.loc[test['FamilySize'] == 1, 'IsAlone'] = 1
test.loc[ test['Age'] <= 16, 'Age'] = 1
test.loc[(test['Age'] > 16)&(test['Age'] <= 32), 'Age'] = 2
test.loc[(test['Age'] > 32)&(test['Age'] <= 64), 'Age'] = 3
test.loc[ test['Age'] > 64, 'Age'] = 4
test['Age*Class'] = test.Age * test.Pclass
test.info()<categorify>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('relu' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
test_dummies_Sex = pd.get_dummies(test.Sex)
 test_dummies_Embarked = pd.get_dummies(test.Embarked)
 test_ready = pd.concat([test, test_dummies_Sex,test_dummies_Embarked], axis=1)
 test_ready.head()
test_ready = test_ready.drop(columns = ['Sex','Embarked'])
 test_ready = test_ready.drop(columns = ['Age*Class'])
 test_ready = test_ready.drop(columns = ['FamilySize'] )<split>
start_time=dt.now()
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
end_time=dt.now()
print(" ")
 print(" ")
 print("Time Taken to run above code :",(end_time-start_time ).total_seconds() /60," minutes" )
y = train_ready['Survived'].values
X = train_ready.drop('Survived',axis=1 ).values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=21, stratify=y)
 warnings.filterwarnings("ignore")
<define_search_model>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
clfs = []
seed = 3
clfs.append(( "LogReg", Pipeline([("Scaler", StandardScaler()),("LogReg", LogisticRegression())])))
 clfs.append(( "XGBClassifier",Pipeline([("Scaler", StandardScaler()),("XGB", XGBClassifier())])))
 clfs.append(( "KNN",Pipeline([("Scaler", StandardScaler()),("KNN", KNeighborsClassifier(n_neighbors=8)) ])))
 clfs.append(( "DecisionTreeClassifier",Pipeline([("Scaler", StandardScaler()),("DecisionTrees", DecisionTreeClassifier())])))
 clfs.append(( "RandomForestClassifier",Pipeline([("Scaler", StandardScaler()),("RandomForest", RandomForestClassifier())])))
 clfs.append(( "GradientBoostingClassifier",Pipeline([("Scaler", StandardScaler()),("GradientBoosting", GradientBoostingClassifier(n_estimators=100)) ])))
 clfs.append(( "RidgeClassifier",Pipeline([("Scaler", StandardScaler()),("RidgeClassifier", RidgeClassifier())])))
 clfs.append(( "BaggingRidgeClassifier", Pipeline([("Scaler", StandardScaler()),("BaggingClassifier", BaggingClassifier())])))
 clfs.append(( "ExtraTreesClassifier",Pipeline([("Scaler", StandardScaler()),("ExtraTrees", ExtraTreesClassifier())])))
 scoring = 'accuracy'
n_folds = 7
results, names = [], []
for name, model in clfs:
kfold = KFold(n_splits=n_folds, random_state=seed)
 cv_results = cross_val_score(model, X_train, y_train,cv= 5, scoring=scoring, n_jobs=-1)
 names.append(name)
 results.append(cv_results)
 msg = "%s: %f(+/- %f)" %(name, cv_results.mean() , cv_results.std())
 print(msg)
 fig = plt.figure(figsize=(15,6))
fig.suptitle('Classifier Algorithm Comparison', fontsize=22)
 ax = fig.add_subplot(111)
 sns.boxplot(x=names, y=results)
 ax.set_xticklabels(names)
 ax.set_xlabel("Algorithmn", fontsize=20)
 ax.set_ylabel("Accuracy of Models", fontsize=18)
 ax.set_xticklabels(ax.get_xticklabels() ,rotation=45)
 plt.show()<normalization>
test['selected_text'] = all
submission=test[['textID','selected_text']]
submission.to_csv('submission.csv',index=False)
 submission.head(5 )