Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<filter>
get_ipython().magic('matplotlib inline')
 plt.rcParams['figure.figsize'] =(8, 6)
 warnings.filterwarnings(action='ignore')
 warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 warnings.filterwarnings(action='ignore', category=FutureWarning)
 start = time.time()
train_good[train_good['diff_num']==0]<filter>
print('Version :', platform.python_version())
 print('Compiler :', platform.python_compiler())
 print('Build :', platform.python_build())
 print("
Current date and time using isoformat:")
 print(datetime.datetime.now().isoformat() )
train_outlier[train_outlier['share']==1]<filter>
save = True
n_row = None
debug = False
if debug is True: n_row = 80
n_splits = 2
n_iter = 70
scoring = 'accuracy'
rstate = 27
testset_size = 0.35
num_rounds = 800
n_tree_range = st.randint(100, num_rounds )
train_good[train_good['share']==1]<filter>
train_df = pd.read_csv(".. /input/train.csv", index_col='PassengerId', nrows=n_row)
 test_df = pd.read_csv(".. /input/test.csv", index_col='PassengerId', nrows=n_row)
 Survived = train_df['Survived'].copy()
train_df = train_df.drop('Survived', axis=1 ).copy()
df = pd.concat([test_df, train_df])
 traindex = train_df.index
testdex = test_df.index
del train_df
del test_df
df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['Name_length'] = df['Name'].apply(len)
 df['IsAlone'] = 0
df.loc[df['FamilySize'] == 1, 'IsAlone'] = 1
df['Title']=0
df['Title']=df.Name.str.extract('([A-Za-z]+)\.')
 df['Title'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don'],
['Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr'],inplace=True)
 df.loc[(df.Age.isnull())&(df.Title=='Mr'),'Age']= df.Age[df.Title=="Mr"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Mrs'),'Age']= df.Age[df.Title=="Mrs"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Master'),'Age']= df.Age[df.Title=="Master"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Miss'),'Age']= df.Age[df.Title=="Miss"].mean()
df.loc[(df.Age.isnull())&(df.Title=='Other'),'Age']= df.Age[df.Title=="Other"].mean()
df = df.drop('Name', axis=1)
 df['Embarked'] = df['Embarked'].fillna(df['Embarked'].mode().iloc[0])
 df['Fare'] = df['Fare'].fillna(df['Fare'].mean())
 df['Sex'] = df['Sex'].map({'female': 1,
'male': 0} ).astype(int)
 df['Title'] = df['Title'].map({'Mr': 0, 'Mrs': 1, 'Miss': 2, 'Master':3, 'Other':4})
 df['Title'] = df['Title'].fillna(df['Title'].mode().iloc[0])
 df['Title'] = df['Title'].astype(int)
 df['Embarked'] = df['Embarked'].map({'Q': 0, 'S': 1, 'C': 2} ).astype(int)
 df= df.drop(['Ticket', 'Cabin'], axis=1)
 categorical_features = ["Pclass","Sex","IsAlone","Title", "Embarked"]
len(train_outlier[train_outlier["text"].str.find(' ')== -1] )<filter>
continuous_features = ['Fare','Age','Name_length']
for col in continuous_features:
transf = df[col].values.reshape(-1,1)
 scaler = preprocessing.StandardScaler().fit(transf)
 df[col] = scaler.transform(transf)
 train_df = df.loc[traindex, :]
train_df['Survived'] = Survived
test_df = df.loc[testdex, :]
del df
len(train_good[train_good["text"].str.find(' ')== -1] )<feature_engineering>
X = train_df.drop(["Survived"] , axis=1)
 y = train_df["Survived"]
print("X, Y, Test Shape:",X.shape, y.shape, test_df.shape)
 results = pd.DataFrame(columns=['Model','Para','Test_Score','CV Mean','CV STDEV'])
 ensemble_models= {}
def subtext_analysis(col, subtext, df1, str1, df2, str2):
result = pd.DataFrame(columns = ['subtext', str1, str2, 'share,%'])
 if(len(df1)> 0)and(len(df2)> 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num1 = len(df1[df1[col].str.find(subtext[i])> -1])
 result.loc[i, str1] = num1
num2 = len(df2[df2[col].str.find(subtext[i])> -1])
 result.loc[i, str2] = num2
result.loc[i,'share,%'] = round(num1*100/num2,1)if num2 != 0 else 0
print('Number of all data is', len(df2))
display(result.sort_values(by=['share,%', str1], ascending=False))<feature_engineering>
print("Depedent Variable Distribution")
 print(y.value_counts(normalize=True)*100)
 print("0 = Died", "
1 = Survived" )
def subtext_analysis_one_df(col, subtext, df, str):
result = pd.DataFrame(columns = ['subtext', str, 'share of all,%'])
 num_all = len(df)
 if(num_all > 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num = len(df[df[col].str.find(subtext[i])> -1])
 result.loc[i, str] = num
result.loc[i,'share of all,%'] = round(num*100/num_all,1)
 print('Number of all data is', len(df))
display(result.sort_values(by='share of all,%', ascending=False))<define_variables>
imbalance_weight = y.value_counts(normalize=True)[0]/y.value_counts(normalize=True)[1]
print("Imbalance Weight: ",imbalance_weight )
subtext_test = ['SAD', 'bullying', 'Uh', 'oh', 'onna', 'fun', 'addicted', 'Power', 'well', 'unhappy', 'funny', 'Tears', 'Fears', 'sleeeeepy', ' ', ',', '?', '!' ,'!!', '!!!', ':/', '...', 'http', '****']<compute_test_metric>
levels = [2,4,6,8,10,12]
for x in levels:
pca = PCA(n_components=x)
 fit = pca.fit(train_df)
 print(( "{} Components
Explained Variance: {}
" ).format(x, fit.explained_variance_ratio_))
subtext_analysis("selected_text", subtext_test, train_outlier, 'train_outliers', train, 'train_all' )<compute_test_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testset_size, stratify=y,random_state=rstate)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape
cv = StratifiedShuffleSplit(n_splits=n_splits, test_size=0.2, random_state=rstate )
subtext_analysis("selected_text", subtext_test, train_good, 'train_good', train, 'train_all' )<compute_test_metric>
def save(model, modelname):

 global results
model.best_estimator_.fit(X_train, y_train)
 scores = cross_val_score(model.best_estimator_, X_train, y_train, cv=5,
scoring=scoring, verbose =0)
 CV_scores = scores.mean()
STDev = scores.std()
Test_scores = model.score(X_test, y_test)
 results = results.append({'Model': modelname,'Para': model.best_params_,'Test_Score': Test_scores,
'CV Mean':CV_scores, 'CV STDEV': STDev}, ignore_index=True)
 ensemble_models[modelname] = model.best_estimator_
print("
Evaluation Method: {}".format(scoring))
print("Optimal Model Parameters: {}".format(grid.best_params_))
print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(CV_scores, STDev, modelname))
print('Test_Score:', Test_scores)
 model.best_estimator_.fit(X_train, y_train)
 pred = model.predict(X_test)
 skplt.metrics.plot_confusion_matrix(y_test, pred, title="{} Confusion Matrix".format(modelname),
normalize=True,figsize=(6,6),text_fontsize='large')
 plt.show()
def norm_save(model,score, modelname):
global results
model.fit(X, y)
 submission = model.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,
'Survived':submission})
 CV_Score = score.mean()
Test_scores = model.score(X_test, y_test)
 STDev = score.std()
Test_Score = model.score(X_test, y_test)
 results = results.append({'Model': modelname,'Para': model,'Test_Score': Test_scores,
'CV Mean': CV_Score, 'CV STDEV': STDev}, ignore_index=True)
 ensemble_models[modelname] = model
print("
Evaluation Method: {}".format(scoring))
print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(CV_Score, STDev, modelname))
print('Test_Score:', Test_scores)
 model.fit(X_train, y_train)
 pred = model.predict(X_test)
 skplt.metrics.plot_confusion_matrix(y_test, pred, title="{} Confusion Matrix".format(modelname),
normalize=True,figsize=(6,6),text_fontsize='large')
 plt.show()
def eval_plot(model):
skplt.metrics.plot_roc_curve(y_test, model.predict_proba(X_test))
plt.show()
subtext_analysis_one_df("selected_text", subtext_test, train, 'test_all' )<compute_test_metric>
param_grid ={'n_neighbors': st.randint(1,40),
'weights':['uniform','distance']
}
grid = RandomizedSearchCV(KNeighborsClassifier() ,
param_grid,
cv=cv,
scoring=scoring,
verbose=1,
n_iter=n_iter,
random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "KNN" )
subtext_analysis_one_df("selected_text", subtext_test, test, 'test_all' )<feature_engineering>
param_grid ={'loss':["hinge","log","modified_huber","epsilon_insensitive","squared_epsilon_insensitive"]}
grid = GridSearchCV(SGDClassifier(max_iter=5, tol=None),
param_grid,cv=cv, scoring=scoring,
verbose=1)
 grid.fit(X_train, y_train)
 save(grid, "StochasticGradientDescent" )
test['text_chr3'] = test['text'].apply(rep_3chr)
 test.head(10 )<train_model>
DecisionTreeClassifier().get_params().keys()
train_outlier1 = train_outlier.nsmallest(1000, 'metric', keep='all')
 train_outlier2 = train_outlier.nsmallest(2000, 'metric', keep='all')
 train_outlier3 = train_outlier.nsmallest(3000, 'metric', keep='all')
 train_outlier5 = train_outlier.nsmallest(5000, 'metric', keep='all')
 train_outlier8 = train_outlier.nsmallest(8000, 'metric', keep='all' )<compute_test_metric>
tree = DecisionTreeClassifier()
print("Mean CV Accuracy:",cross_val_score(tree, X, y, cv=cv, scoring=scoring ).mean())
 feature_imp(tree )
subtext_analysis("selected_text", subtext_test, train_outlier1, 'in worst 1000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
param_grid ={'n_estimators': n_tree_range}
tree = DecisionTreeClassifier()
grid = RandomizedSearchCV(BaggingClassifier(tree),
param_grid, cv=cv, scoring=scoring,
verbose=1,n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "Bagger_ensemble" )
subtext_analysis("selected_text", subtext_test, train_outlier2, 'in worst 2000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
model = RandomForestClassifier()
print("Mean CV Accuracy:",cross_val_score(model, X, y, cv=cv, scoring=scoring ).mean())
 feature_imp(model )
subtext_analysis("selected_text", subtext_test, train_outlier3, 'in worst 3000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
param_grid ={
'max_depth': st.randint(3, 11),
'max_features':["sqrt", "log2"],
'max_leaf_nodes':st.randint(6, 10)
 }
model= RandomForestClassifier()
grid = RandomizedSearchCV(model,
param_grid, cv=cv,
scoring=scoring,
verbose=1,n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "Random_Forest" )
subtext_analysis("selected_text", subtext_test, train_outlier5, 'in worst 5000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
AdaBoostClassifier().get_params().keys()
subtext_analysis("selected_text", subtext_test, train_outlier8, 'in worst 8000 outliers', train_outlier, 'in all outliers' )<define_variables>
param_grid ={'n_estimators':n_tree_range,
'learning_rate':np.arange (.1, 4,.5)}
grid = RandomizedSearchCV(AdaBoostClassifier() ,
param_grid,cv=cv, scoring=scoring,
verbose=1, n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "AdaBoost_Ensemble" )
col_hist = ['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text',
'text_chr3', 'selected_text_chr3', 'selected_text_pred_chr3', 'metric']<drop_column>
GradientBoostingClassifier().get_params().keys()
data = train[['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text', 'diff_num', 'share', 'metric', 'res']].dropna()
data<import_modules>
param_grid ={'n_estimators': n_tree_range,
'loss': ['deviance', 'exponential'],
'learning_rate': [0.1, 0.01,0.05,0.001],
'max_depth': np.arange(2, 12, 1)}
grid = RandomizedSearchCV(GradientBoostingClassifier() ,
param_grid,cv=cv,
scoring=scoring,
verbose=1, n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "Gradient_Boosting" )
print('TF version',tf.__version__ )<define_variables>
params = {
"n_estimators": n_tree_range,
"max_depth": st.randint(3, 15),
"learning_rate": [0.05]
}
xgbreg = XGBClassifier(objective= 'binary:logistic', eval_metric="auc",
nthreads=2)
 grid = RandomizedSearchCV(xgbreg, params, n_jobs=1, verbose=1, n_iter=n_iter,
random_state=rstate, scoring=scoring)
 grid.fit(X_train,y_train, verbose=False)
 save(grid, "Sci_kit XGB" )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 1
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'neutral': 7974, 'negative': 2430}<load_from_csv>
model = XGBClassifier(n_estimators = num_rounds,
objective= 'binary:logistic',
learning_rate=0.01,
scale_pos_weight = imbalance_weight,
random_state=rstate,
scoring=scoring,
verbose_eval=10)
 model.fit(X_train,y_train, early_stopping_rounds=50, eval_set=[(X_test,
y_test)], verbose=False)
 score = cross_val_score(model, X_train,y_train, cv=cv)
 print("
xgBoost - CV Train : %.2f" % score.mean())
 print("xgBoost - Train : %.2f" % metrics.accuracy_score(model.predict(X_train), y_train))
print("xgBoost - Test : %.2f" % metrics.accuracy_score(model.predict(X_test), y_test))
norm_save(model,score, "XGBsklearn" )
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<string_transform>
xgtrain = xgb.DMatrix(X_train, label=y_train)
 xgtest = xgb.DMatrix(X_test, label=y_test)
 xgtesting = xgb.DMatrix(test_df)
 param = {'eta': 0.05,
'max_depth': 8,
'subsample': 0.8,
'colsample_bytree': 0.75,
'objective': 'binary:logistic',
'eval_metric': 'logloss',
'scale_pos_weight': imbalance_weight,
'seed': 23,
'lambda': 1.5,
'alpha':.6
}
clf_xgb_cv = xgb.cv(param, xgtrain, num_rounds,
stratified=True,
nfold=n_splits,
early_stopping_rounds=50,
verbose_eval=20)
 print("Optimal number of trees/estimators is %i" % clf_xgb_cv.shape[0])
 watchlist = [(xgtrain,'train'),(xgtest,'test')]
clf_xgb = xgb.train(param, xgtrain,clf_xgb_cv.shape[0], watchlist,verbose_eval=0)
 p = clf_xgb.predict(xgtest)
 fpr, tpr, thresholds = metrics.roc_curve(y_test,p)
 optimal_idx = np.argmax(tpr - fpr)
 optimal_threshold = thresholds[optimal_idx]
print("
Optimal Threshold: ",optimal_threshold)
 y_train_pred =(clf_xgb.predict(xgtrain, ntree_limit=clf_xgb.best_iteration)> optimal_threshold ).astype(int)
 y_test_pred =(clf_xgb.predict(xgtest, ntree_limit=clf_xgb.best_iteration)> optimal_threshold ).astype(int)
 score= metrics.accuracy_score(y_test_pred, y_test)
 print("
XGB - CV Train : %.2f" % score)
 train_score= metrics.accuracy_score(y_train_pred, y_train)
 print("XGB - Train : %.2f" % train_score)
 results = results.append({'Model': "BESTXGBOOST",'Para': param,'Test_Score': score,
'CV Mean':train_score, 'CV STDEV': None}, ignore_index=True)
 xgboostprobpred = clf_xgb.predict(xgtesting, ntree_limit=clf_xgb.best_iteration)
 xgboostpred =(xgboostprobpred > optimal_threshold ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':xgboostpred})
 submission.to_csv('BESTXGBOOST.csv',index=False )
stop = stopwords.words('english')
 stop += ["_________________________________", "u"]
punct = list(string.punctuation)
 punct.remove("-")
 punct.append(" " )<string_transform>
model = CatBoostClassifier(
 eval_metric='Logloss',
iterations = num_rounds,
use_best_model=True,
od_wait = 100,
random_seed=42,
logging_level = "Verbose",
metric_period = 100
)
def get_synonyms(word):

 if word.lower() in stop:
return [word], [1]
synonyms = set()
for syn in wordnet.synsets(word):
for l in syn.lemmas() :
synonym = l.name().replace("_", " " ).replace("-", " " ).lower()
synonym = "".join([char for char in synonym if char in ' qwertyuiopasdfghjklzxcvbnm'])
 synonyms.add(synonym)
 if word not in synonyms:
synonyms.add(word)
 n = len(synonyms)
 if n == 1:
word_ = "".join(list(filter(lambda x: x not in punct, word)))
 if word_.lower() in stop:
return [word, word_], [0.5, 0.5]
for syn in wordnet.synsets(word_):
for l in syn.lemmas() :
synonym = l.name().replace("_", " " ).replace("-", " " ).lower()
synonym = "".join([char for char in synonym if char in ' qwertyuiopasdfghjklzxcvbnm'])
 synonyms.add(synonym)
 if word_ not in synonyms:
synonyms.add(word_)
 n = len(synonyms)
 if n == 1:
probabilities = [1]
else:
probabilities = [0.5 if w==word else 0.5/(n-1)for w in synonyms]
return list(synonyms), probabilities<string_transform>
model = CatBoostClassifier(eval_metric='Accuracy',
iterations = 500,
scale_pos_weight= imbalance_weight,
random_seed=42,
logging_level = "Verbose",
metric_period = 100 
)
 model.fit(X,y,cat_features=categorical_features_pos )
def swap_words(words):
words = words.split()
if len(words)< 2:
return " ".join(words), False
random_idx = np.random.randint(0, len(words)-1)
 words[random_idx], words[random_idx+1] = words[random_idx+1], words[random_idx]
return " ".join(words), True<define_variables>
model.get_params()
for _ in range(5):
print(swap_words('The sun is shining today, this makes me feel so good !')[0] )<string_transform>
lgtrain = lgb.Dataset(X_train, y_train,
categorical_feature=categorical_features 
)
 lgvalid = lgb.Dataset(X_test, y_test,
categorical_feature=categorical_features 
)
 lgbm_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 12,
'learning_rate': 0.01,
'feature_fraction': 0.80,
'bagging_fraction': 0.80,
'bagging_freq': 5,
'max_bin':300,
'lambda_l2': 1.5,
'is_unbalance': True
}
modelstart = time.time()
lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=num_rounds,
valid_sets=[lgtrain,lgvalid],
valid_names=['train','valid'],
early_stopping_rounds=50,
verbose_eval=50 
)
 print("
Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
def new_row(row, n_samples=1):
text, selected_text, textID = row['text'], row['selected_text'], row['textID']
oth_text = text.replace(selected_text, " _________________________________ ")
 new_selected_text = [get_synonyms(word)for word in selected_text.split() ]
new_oth_text = [get_synonyms(word)for word in oth_text.split() ]
new_sentences = [row]
for i in range(n_samples):
oth_text_ = " ".join([np.random.choice(l_syn, p=p, replace=True)for l_syn, p in new_oth_text])
 selected_text_ = " ".join([np.random.choice(l_syn, p=p, replace=True)for l_syn, p in new_selected_text])
 text_ = oth_text_.replace("_________________________________", selected_text_)
 if not selected_text_ in text_:
print(f'Original : {text} with target {selected_text}, oth_text {oth_text}
Transformed : {text_} with target {selected_text_}, oth_text {oth_text_}')
 continue
row2 = dc(row)
 row2['text'] = text_
row2['selected_text'] = selected_text_
row2['textID'] = f'new_{textID}'
new_sentences.append(row2)
 for r in dc(new_sentences):
r_ = dc(r)
 if np.random.choice([True, False]):
selected_text, boo = swap_words(r_['selected_text'])
 if boo:
r_['text'] = r_['text'].replace(r_['selected_text'], selected_text)
 r_['selected_text'] = selected_text
else:
oth_text, _ = swap_words(r_['text'].replace(r_['selected_text'], " _________________________________ "))
r_['text'] = oth_text.replace("_________________________________",r_['selected_text'])
 else:
oth_text, _ = swap_words(r_['text'].replace(r_['selected_text'], " _________________________________ "))
r_['text'] = oth_text.replace("_________________________________", r_['selected_text'])
 r_['textID'] = f'new_{textID}'
new_sentences.append(r_)
 new_rows = pd.concat(new_sentences, axis=1 ).transpose().drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 new_rows = new_rows.loc[new_rows['text'].apply(len)<150]
counter = 0
for i, row in new_rows.iterrows() :
if row['textID'][:4] == 'new_':
row['textID'] = row['textID']+f'_{counter}'
counter += 1
return new_rows<count_duplicates>
p = lgb_clf.predict(X_test)
 fpr, tpr, thresholds = metrics.roc_curve(y_test,p)
 optimal_idx = np.argmax(tpr - fpr)
 optimal_threshold = thresholds[optimal_idx]
print("Optimal Threshold: ",optimal_threshold)
 y_train_pred =(lgb_clf.predict(X_train)> optimal_threshold ).astype(int)
 y_test_pred =(lgb_clf.predict(X_test)> optimal_threshold ).astype(int)
 score= metrics.accuracy_score(y_test_pred, y_test)
 print("
XGB - CV Train : %.2f" % score)
 train_score = metrics.accuracy_score(y_train_pred, y_train)
 print("XGB - Train : %.2f" % train_score)
 results = results.append({'Model': "LGBM",'Para': lgbm_params,'Test_Score': score,
'CV Mean':train_score, 'CV STDEV': None}, ignore_index=True)
 lgbmprobpred = lgb_clf.predict(test_df)
 lgbmpred =(lgbmprobpred > optimal_threshold ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':lgbmpred})
 submission.to_csv('LGBM.csv',index=False )
temp = [new_row(row, n_samples=2)for _, row in train.iterrows() ]
augmented_data = pd.concat(temp, axis=0)
 train['number'] = [t.shape[0] for t in temp]
train['number'] = train['number'].cumsum()
del temp
gc.collect()
augmented_data.drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 augmented_data.reset_index(drop=True, inplace=True)
 match_index = dc(train['number'])
 train.drop(columns='number', inplace=True)
 match_index = [0] + match_index.values.tolist()
match_borders = list(zip(match_index[:-1], match_index[1:]))
del match_index
gc.collect()
train['brackets'] = match_borders 
<save_to_csv>
modeblend = pd.DataFrame([xgboostpred,lgbmpred, lgbmpred] ).T
modeblend.head()
modeblend["Mode"] = modeblend.mode(axis=1)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':modeblend["Mode"]})
 submission.to_csv('Mode_GBM_Blend.csv',index=False)
 submission.head()
augmented_data.to_csv('extended_train.csv', index=False)
 train = augmented_data
del augmented_data<drop_column>
predblend = pd.DataFrame(np.array([xgboostprobpred,lgbmprobpred,lgbmpred])).T
predblend.columns = ["XG","LGBM","CAT"]
predblend["meanblend"] = predblend.mean(axis=1)
 predblend["blendout"] =(predblend["meanblend"] > 0.5 ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_df.index,'Survived':predblend["blendout"]})
 submission.to_csv('mean_GBM_blend.csv',index=False)
 submission.head()
train = train.loc[train.text_len<150]
train.drop(columns=["text_len"], inplace=True)
 train.reset_index(drop=True, inplace=True )<save_to_csv>
cv = StratifiedShuffleSplit(n_splits=n_splits, test_size=0.2, random_state=rstate )
train.to_csv('extended_train.csv', index=False )<define_variables>
model= LogisticRegression()
score = cross_val_score(model, X_train, y_train, cv=cv, scoring='accuracy')
 norm_save(LogisticRegression() ,score, "Logistic_Regression" )
ct = augmented_data.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k, row in augmented_data.iterrows() :
text1 = " "+" ".join(row['text'].split())
 text2 = " ".join(row['selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ':
chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[row['sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<define_variables>
logit = sm.OLS(y,sm.add_constant(X)).fit()
logit.summary()
ct_train = train.shape[0]
input_ids_train = np.ones(( ct_train,MAX_LEN),dtype='int32')
 attention_mask_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 token_type_ids_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 start_tokens_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 end_tokens_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 for k, row in train.iterrows() :
text1 = " "+" ".join(row['text'].split())
 text2 = " ".join(row['selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ':
chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[row['sentiment']]
input_ids_train[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_train[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens_train[k,toks[0]+2] = 1
end_tokens_train[k,toks[-1]+2] = 1<load_from_csv>
param_grid ={'max_iter': np.logspace(1, 5, 10 ).astype("int32"),
'hidden_layer_sizes': np.logspace(2, 3, 4 ).astype("int32"),
'activation':['identity', 'logistic', 'tanh', 'relu'],
'learning_rate': ['adaptive'],
'early_stopping': [True],
'alpha': np.logspace(2, 3, 4 ).astype("int32")
 }
model = MLPClassifier()
grid = RandomizedSearchCV(model,
param_grid, cv=cv, scoring=scoring,
verbose=1, n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "FFNeural_Net" )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('' )<feature_engineering>
model = LinearSVC()
scores= cross_val_score(model, X_train, y_train, cv=cv, scoring=scoring)
 norm_save(model, scores, "LinearSV" )
def test_new_row(row, n_samples=1):
text, textID = row['text'], row['textID']
new_text = [get_synonyms(word)for word in text.split() ]
new_sentences = [row]
for i in range(n_samples):
text_ = " ".join([np.random.choice(l_syn, p=p, replace=True)for l_syn, p in new_text])
 row2 = dc(row)
 row2['text'] = text_
row2['textID'] = f'new_{textID}'
new_sentences.append(row2)
 for r in dc(new_sentences):
r_ = dc(r)
 text, boo = swap_words(r_['text'])
 if boo:
r_['text'] = text
r_['textID'] = f'new_{textID}'
new_sentences.append(r_)
 new_rows = pd.concat(new_sentences, axis=1 ).transpose().drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 new_rows = new_rows.loc[new_rows['text'].apply(len)<150]
counter = 0
for i, row in new_rows.iterrows() :
if row['textID'][:4] == 'new_':
row['textID'] = row['textID']+f'_{counter}'
counter += 1
return new_rows<count_duplicates>
svc = SVC(kernel= 'rbf', probability=True)
 model = Pipeline(steps=[('svc', svc)])
 param_grid = {'svc__C': st.randint(1,10000),
'svc__gamma': np.logspace(1, -7, 10)}
grid = RandomizedSearchCV(model, param_grid,
cv=cv, verbose=1, scoring=scoring,
n_iter=n_iter, random_state=rstate)
 grid.fit(X_train, y_train)
 save(grid, "SVCrbf" )
temp = [test_new_row(row, n_samples=2)for _, row in test.iterrows() ]
test_augmented_data = pd.concat(temp, axis=0)
 test['number'] = [t.shape[0] for t in temp]
test['number'] = test['number'].cumsum()
del temp
gc.collect()
test_augmented_data.drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 test_augmented_data.reset_index(drop=True, inplace=True)
 test_match_index = dc(test['number'])
 test.drop(columns='number', inplace=True)
 test_match_index = [0] + test_match_index.values.tolist()
test_match_borders = list(zip(test_match_index[:-1], test_match_index[1:]))
del test_match_index
gc.collect()
test['brackets'] = test_match_borders<categorify>
pca = PCA()
svc = SVC(kernel= 'rbf',probability=True)
 model = Pipeline(steps=[('pca',pca), 
('svc', svc)])
 param_grid = {'svc__C': st.randint(1,10000),
'svc__gamma': np.logspace(1, -7, 10),
'pca__n_components': st.randint(1,len(X.columns)) }
grid = RandomizedSearchCV(model, param_grid,
cv=cv, verbose=1,
n_iter=n_iter, random_state=rstate, scoring=scoring)
 grid.fit(X_train, y_train)
 save(grid, "PCA_SVC" )
ct = test.shape[0]
test_input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 for k, row in test.iterrows() :
text1 = " "+" ".join(row['text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[row['sentiment']]
test_input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
ct = test_augmented_data.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k, row in test_augmented_data.iterrows() :
text1 = " "+" ".join(row['text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[row['sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<train_model>
model = GaussianNB()
score = cross_val_score(model, X_train, y_train, cv=cv, scoring=scoring)
 norm_save(model,score, "Gaussian" )
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
results.sort_values(by=["Test_Score"], ascending=False, inplace=True)
 results
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0):
return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
ignore = ["Catboost","BESTXGBOOST","LGBM", "Logistic GAM"]
not_proba_list = ['LinearSV','StochasticGradientDescent']
not_proba = results.query("Model in @not_proba_list")
 hard_models = results.query("Model not in @ignore")
 prob_models = results.query("Model not in @not_proba_list & Model not in @ignore")
 test_hard_pred_matrix = pd.DataFrame()
train_hard_pred_matrix = pd.DataFrame()
valid_hard_pred_matrix = pd.DataFrame()
test_soft_pred_matrix = pd.DataFrame()
train_soft_pred_matrix = pd.DataFrame()
valid_soft_pred_matrix = pd.DataFrame()
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=2,shuffle=True,random_state=SEED)
 for fold,(idx_T,idx_V)in enumerate(skf.split(input_ids_train,train.sentiment.values)) :
idxT = np.array([i for(a,b)in train.loc[idx_T, 'brackets'] for i in range(a, b)])
 idxV = np.array([i for(a,b)in train.loc[idx_V, 'brackets'] for i in range(a, b)])
 print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
alls = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = augmented_data.loc[k,'text']
else:
text1 = " "+" ".join(augmented_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 alls.append(jaccard(st,augmented_data.loc[k,'selected_text']))
jac.append(np.mean(alls))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(alls))
print()<feature_engineering>
models = list(zip([ensemble_models[x] for x in not_proba.Model], not_proba.Model))
clfs = []
print('5-fold cross validation:
')
 for clf, label in models:
scores = cross_val_score(clf, X_train, y_train, cv=cv, scoring=scoring, verbose=0)
 print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , label))
md = clf.fit(X_train, y_train)
 valid_hard_pred_matrix = pd.concat(( valid_hard_pred_matrix, pd.DataFrame({label: md.predict(X_test)})) , axis=1)
 clfs.append(md)
 print("Test Accuracy: %0.2f
" %(metrics.accuracy_score(clf.predict(X_test), y_test)))
 md = clf.fit(X,y)
 submission = md.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,
'Survived':submission})
 train_hard_pred_matrix = pd.concat(( train_hard_pred_matrix, pd.DataFrame({label: md.predict(X)})) , axis=1)
 test_hard_pred_matrix = pd.concat(( test_hard_pred_matrix, pd.DataFrame({label: submission})) , axis=1)
 df.to_csv("{}.csv".format(label),header=True,index=False)
 del clfs
print('>>>> OVERALL 3Fold CV Jaccard =',np.mean(jac))<feature_engineering>
warnings.filterwarnings(action='ignore', category=DeprecationWarning)
 models = list(zip([ensemble_models[x] for x in prob_models.Model], prob_models.Model))
plt.figure()
print('5-fold cross validation:
')
 clfs = []
for clf, model_label in models:
scores = cross_val_score(clf, X_train, y_train,cv=5, scoring=scoring, verbose=0)
 print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , model_label))
md = clf.fit(X_train, y_train)
 valid_soft_pred_matrix = pd.concat(( valid_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(X_test)[:,1]})) , axis=1)
 fpr, tpr, _ = roc_curve(y_test, md.predict_proba(X_test)[:,1])
 roc_auc = auc(fpr, tpr)
 print('ROC AUC: %0.2f' % roc_auc)
 plt.plot(fpr, tpr, label='{} ROC curve(area = {:.2})'.format(model_label, roc_auc))
clfs.append(md)
 print("Test Accuracy: %0.2f
" %(metrics.accuracy_score(clf.predict(X_test), y_test)))
 md = clf.fit(X,y)
 submission = md.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,
'Survived':submission})
 train_hard_pred_matrix = pd.concat(( train_hard_pred_matrix, pd.DataFrame({model_label: md.predict(X)})) , axis=1)
 test_hard_pred_matrix = pd.concat(( test_hard_pred_matrix, pd.DataFrame({model_label: submission})) , axis=1)
 train_soft_pred_matrix = pd.concat(( train_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(X)[:,1]})) , axis=1)
 test_soft_pred_matrix = pd.concat(( test_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(test_df)[:,1]})) , axis=1)
 df.to_csv("{}.csv".format(model_label),header=True,index=False)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('ROC Curve')
 plt.legend(loc="lower right")
 plt.show()
alls = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_augmented_data.loc[k,'text']
else:
text1 = " "+" ".join(test_augmented_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 alls.append(st)
 test_augmented_data['selected_text'] = alls<load_from_csv>
plt.figure()
voters = {}
for x in [2,3,5,7,10]:
ECH = EnsembleVoteClassifier([ensemble_models.get(key)for key in hard_models.Model[:x]], voting='hard')
 ECS = EnsembleVoteClassifier([ensemble_models.get(key)for key in prob_models.Model[:x]], voting='soft')
 print('
{}-Voting Models: 5-fold cross validation:
'.format(x))
for clf, model_label in zip([ECS, ECH],
['{}-VM-Ensemble Soft Voting'.format(x),
'{}-VM-Ensemble Hard Voting'.format(x)]):
scores = cross_val_score(clf, X_train, y_train, cv=5, scoring='accuracy')
 print("Train CV Accuracy: %0.2f(+/- %0.2f)[%s]" %(scores.mean() , scores.std() , model_label))
md = clf.fit(X_train, y_train)
 clfs.append(md)
 Test_Score = metrics.accuracy_score(clf.predict(X_test), y_test)
 print("Test Accuracy: %0.2f " % Test_Score)
 CV_Score = scores.mean()
STDev = scores.std()
global results
results = results.append({'Model': model_label,'Para': clf, 'CV Mean': CV_Score,
'Test_Score':Test_Score,'CV STDEV': STDev}, ignore_index=True)
 voters[model_label] = clf
md = clf.fit(X,y)
 submission = md.predict(test_df)
 df = pd.DataFrame({'PassengerId':test_df.index,'Survived':submission})
 df.to_csv("{}.csv".format(model_label),header=True,index=False)
 if clf is ECH:
train_hard_pred_matrix = pd.concat(( train_hard_pred_matrix, pd.DataFrame({model_label: md.predict(X)})) , axis=1)
 test_hard_pred_matrix = pd.concat(( test_hard_pred_matrix, pd.DataFrame({model_label: submission})) , axis=1)
 elif clf is ECS:
fpr, tpr, _ = roc_curve(y_test, md.predict_proba(X_test)[:,1])
 roc_auc = auc(fpr, tpr)
 print('ROC AUC: %0.2f' % roc_auc)
 plt.plot(fpr, tpr, label='{} ROC curve(area = {:.2})'.format(model_label, roc_auc))
train_soft_pred_matrix = pd.concat(( train_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(X)[:,1]})) , axis=1)
 test_soft_pred_matrix = pd.concat(( test_soft_pred_matrix, pd.DataFrame({model_label: md.predict_proba(test_df)[:,1]})) , axis=1)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('Soft Model ROC Curve')
 plt.legend(loc="lower right")
 plt.show()
test_augmented_data = pd.read_csv('.. /input/tweet-different-thresholds/test_augmented_data.csv' )<string_transform>
voters.get('10-VM-Ensemble Hard Voting' )
test_ = dc(test)
 def get_submission(threshold):
test1 = dc(test_)
 for ind,(a, b)in enumerate(test1.brackets.values.tolist() [0:5]):
df = test_augmented_data.loc[np.arange(a,b),['textID', 'text', 'selected_text']]
try:
selected_text = " ".join(df.loc[df.textID.str.contains('new_')==False,'selected_text'].values[0].split())
 text = " ".join(df.loc[df.textID.str.contains('new_')==False,'text'].values[0].split())
 except:
print(test1.loc[ind,:])
 print(df)
 break
print(f'Selected_text : {selected_text}, from text {text}')
 text = text.lower().split(selected_text.lower())
 print(f'When splitting :{text}')
 text_before, text_after = text[0].split() , text[-1].split()
rows = df.loc[df.textID.str.contains('new_')]
words_dic = []
i = 0
for word in text_before:
print(f'In text_before, processing word {i} : {word}')
 words_dic.append(word)
 l, p = get_synonyms(word)
 coeff = 0.5 if(word.lower() in stop or "".join(list(filter(lambda x: x not in punct, word)))in stop)else 1
rows[f'scoring_word_{i}'] = rows['selected_text'].apply(lambda x: coeff*np.average([w in x for w in l], weights=p))
i+=1
for word in selected_text.split() :
print(f'In selected_text, processing word {i} : {word}')
 words_dic.append(word)
 l, p = get_synonyms(word)
 coeff = 0.5 if(word.lower() in stop or "".join(list(filter(lambda x: x not in punct, word)))in stop)else 1
rows[f'scoring_word_{i}'] = rows['selected_text'].apply(lambda x: 2*coeff*np.average([w in x for w in l], weights=p))
i+=1
for word in text_after:
print(f'In text_after, processing word {i} : {word}')
 words_dic.append(word)
 l, p = get_synonyms(word)
 coeff = 0.5 if(word.lower() in stop or "".join(list(filter(lambda x: x not in punct, word)))in stop)else 1
rows[f'scoring_word_{i}'] = rows['selected_text'].apply(lambda x: coeff*np.average([w in x for w in l], weights=p))
i+=1
s = " ".join([words_dic[int(f.split('_')[-1])] for f in(rows[[f'scoring_word_{j}' for j in range(i)]].mean(axis=0)>=threshold ).index.tolist() ])
 test1.loc[ind, 'selected_text'] = " ".join([words_dic[int(f.split('_')[-1])] for f in(rows[[f'scoring_word_{j}' for j in range(i)]].mean(axis=0)>=threshold ).index.tolist() ])
 return test1
get_submission(threshold=0.5 )<prepare_x_and_y>
Xstack = X.copy()
ystack = y.copy()
X_trainstack = X_train.copy()
X_teststack = X_test.copy()
y_trainstack = y_train.copy()
y_teststack = y_test.copy()
test2 = get_submission(threshold=0.3)
 test3 = get_submission(threshold = 1 )<save_to_csv>
from sklearn.model_selection import cross_val_score
test1.to_csv('submission_05.csv', index=False)
 test2.to_csv('submission_03.csv', index=False)
 test3.to_csv('submission_1.csv', index=False )<save_to_csv>
num_trees = 10
verbose = True
clfs = [ensemble_models.get('KNN'),
ensemble_models.get('Sci_kit XGB')]
dataset_blend_train = np.zeros(( X_trainstack.shape[0], len(clfs)))
 dataset_blend_test = np.zeros(( X_teststack.shape[0], len(clfs)))
 dataset_blend_test_df = np.zeros(( test_df.shape[0], len(clfs)))
 print('5-fold cross validation:')
 for i, clf in enumerate(clfs):
scores = cross_val_score(clf, X_trainstack, y_trainstack, cv=4, scoring='accuracy')
 print("
print("Train CV Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std()))
clf.fit(X_trainstack, y_trainstack)
 print("Train Accuracy: %0.2f " %(metrics.accuracy_score(clf.predict(X_trainstack), y_trainstack)))
 dataset_blend_train[:,i] = clf.predict_proba(X_trainstack)[:, 1]
dataset_blend_test[:,i] = clf.predict_proba(X_teststack)[:, 1]
dataset_blend_test_df[:,i] = clf.predict_proba(test_df)[:, 1]
print("Test Accuracy: %0.2f
" %(metrics.accuracy_score(clf.predict(X_teststack), y_teststack)))
 print("
clf = LogisticRegression()
scores = cross_val_score(clf, dataset_blend_train, y_trainstack, cv=4, scoring=scoring)
 clf.fit(dataset_blend_train, y_trainstack)
 print("Train CV Accuracy: %0.2f(+/- %0.2f)" %(scores.mean() , scores.std()))
print("Train Accuracy: %0.2f " %(metrics.accuracy_score(clf.predict(dataset_blend_train), y_trainstack)))
 print("Test Accuracy: %0.2f " %(metrics.accuracy_score(clf.predict(dataset_blend_test), y_teststack)))
 if save == True:
pd.DataFrame({'PassengerId':test_df.index,
'Survived':clf.predict(dataset_blend_test_df)} ).to_csv(
 "{}.csv".format("Stacked"),header=True,index=False )
test_augmented_data.to_csv('test_augmented_data.csv', index=False )<save_to_csv>
score = cross_val_score(clf, X, y, cv=cv, scoring=scoring)
 norm_save(clf, score, "stacked")
 eval_plot(clf )
test[['textID','selected_text']].to_csv('submission.csv', index=False )<set_options>
train_hard_pred_matrix = train_hard_pred_matrix.set_index([traindex])
 train_hard_pred_matrix = pd.concat([train_hard_pred_matrix, Survived], axis=1)
 valid_hard_pred_matrix = valid_hard_pred_matrix.set_index([X_test.index])
 valid_hard_pred_matrix = pd.concat([valid_hard_pred_matrix, Survived[X_test.index]], axis=1)
 train_soft_pred_matrix = train_soft_pred_matrix.set_index([traindex])
 train_soft_pred_matrix = pd.concat([train_soft_pred_matrix, Survived], axis=1)
 valid_soft_pred_matrix = valid_soft_pred_matrix.set_index([X_test.index])
 valid_soft_pred_matrix = pd.concat([valid_soft_pred_matrix, Survived[X_test.index]], axis=1)
 test_hard_pred_matrix = test_hard_pred_matrix.set_index([testdex])
 valid_hard_pred_matrix = valid_hard_pred_matrix.set_index([X_test.index])
 test_soft_pred_matrix = test_soft_pred_matrix.set_index([testdex])
 valid_soft_pred_matrix = valid_soft_pred_matrix.set_index([X_test.index])
 test_hard_pred_matrix.to_csv("test_hard_pred_matrix.csv")
 test_soft_pred_matrix.to_csv("test_soft_pred_matrix.csv")
 valid_soft_pred_matrix.to_csv("valid_soft_pred_matrix.csv")
 train_hard_pred_matrix.to_csv("train_hard_pred_matrix.csv")
 train_soft_pred_matrix.to_csv("train_soft_pred_matrix.csv")
 valid_soft_pred_matrix.to_csv("valid_soft_pred_matrix.csv" )
pd.set_option('max_colwidth', 60)
 test.sample(25 )<set_options>
results.sort_values(by=["Test_Score"], ascending=False, inplace=True)
 results.to_csv("titanic_clf_results.csv", index=False)
 results
Image(".. /input/cycliclearningrate-1/keras_clr_triangular.png" )<import_modules>
top_models = [x for x in valid_soft_pred_matrix.columns
if x in list(results.sort_values(by="CV Mean",ascending = False ).Model[:15])]
print('TF version',tf.__version__ )<load_from_csv>
top_models = [x for x in valid_soft_pred_matrix.columns
if x in list(results.sort_values(by="CV Mean",ascending = False ).Model[:15])]
print("Models Chosen: ",top_models)
 valid_soft_pred_matrix = valid_soft_pred_matrix[top_models]
test_soft_pred_matrix = test_soft_pred_matrix[top_models]
pred_cols = list(valid_soft_pred_matrix.columns)
 def stack_features(tempdf):
tempdf['max'] = np.max(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['min'] = np.min(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['avg'] = np.mean(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['med'] = np.median(np.array([tempdf[col] for col in pred_cols]),axis=0)
 tempdf['std'] = np.std(np.array([tempdf[col] for col in pred_cols]),axis=0)
 for p1, p2 in itertools.combinations(pred_cols, 2):
tempdf['difference_%s__%s'%(p1,p2)] = tempdf[p2] - tempdf[p1]
tempdf['sums_%s__%s'%(p1,p2)] = tempdf[p2] + tempdf[p1]
return tempdf
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train_neg = train[train.sentiment == "negative"].reset_index() 
<define_variables>
all_soft_preds = pd.concat([valid_soft_pred_matrix, test_soft_pred_matrix],axis=0)
 all_soft_preds = stack_features(all_soft_preds)
 valid_soft_pred_matrix = all_soft_preds.loc[X_test.index, :]
test_soft_pred_matrix = all_soft_preds.loc[testdex, :]
X_stack = pd.concat([X_test, valid_soft_pred_matrix], axis=1)
 test_stack = pd.concat([test_df, test_soft_pred_matrix],axis=1)
 print("Train Shape: ", X_stack.shape)
 print("Test Shape: ", test_stack.shape)
 y_stack = y[X_test.index]
X_stack_train, X_stack_test, y_stack_train, y_stack_test = train_test_split(X_stack, y_stack, test_size=.30, stratify=y_stack,random_state=23)
 print(X_stack_train.shape, y_stack_train.shape, X_stack_test.shape, y_stack_test.shape)
 lgtrain = lgb.Dataset(data = X_stack_train,label = y_stack_train.values, categorical_feature = categorical_features)
 lgb_results = pd.DataFrame(columns = ["Rounds","Score","STDV", "LB", "Parameters"])
 fulllgtrain = lgb.Dataset(data = X_stack,label = y_stack.values, categorical_feature = categorical_features )
e = tokenizer.encode("going")
 print(e.ids )<define_variables>
model = LogisticRegression()
print("Mean CV Accuracy:",cross_val_score(model, X_stack, y_stack, cv=5, scoring="accuracy" ).mean())
 model.fit(X_stack, y_stack)
 logregpred = model.predict(test_stack)
 submission = pd.DataFrame({'PassengerId':test_stack.index,'Survived':logregpred})
 submission.to_csv('LogRegStack.csv',index=False )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(ct):
text1 = " "+" ".join(train.loc[k,'text'].split() ).strip()
text2 = " ".join(train.loc[k,'selected_text'].split() ).strip()
text1 = text1.replace("_","")
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 enc2 = tokenizer.encode(text2)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 print(offsets)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1 
<load_from_csv>
print("Light Gradient Boosting Classifier: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': ['binary_logloss'],
"learning_rate": 0.05,
"num_leaves": 20,
"max_depth": 5,
'is_unbalance': True
}
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 text1 = text1.replace("_","")
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<choose_model_class>
lgb_cv = lgb.cv(
 params = lgbm_params,
train_set = lgtrain,
num_boost_round=10000,
stratified=True,
nfold = 5,
verbose_eval=25,
seed = 23,
early_stopping_rounds=75,
categorical_feature= categorical_features)
 loss = lgbm_params["metric"][0]
optimal_rounds = np.argmin(lgb_cv[str(loss)+ '-mean'])
 best_cv_score = round(min(lgb_cv[str(loss)+ '-mean']), 7)
 best_std_score = round(lgb_cv[str(loss)+ '-stdv'][optimal_rounds],7)
 print("
Optimal Round: {}
Optimal Score: {} + {}".format(
 optimal_rounds,best_cv_score,best_std_score))
lgb_results = lgb_results.append({"Rounds": optimal_rounds,
"Score": best_cv_score,
"STDV": best_std_score,
"LB": None,
"Parameters": lgbm_params}, ignore_index=True )
def build_model() :

 ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x_bert = bert_model([ids,att,tok])
 logits = tf.keras.layers.Dense(2, activation="linear", kernel_initializer="he_normal" )(x_bert[0])
 print(x_bert[0].shape)
 start_logits, end_logits = tf.split(logits, 2, axis=-1)
 start_logits = tf.squeeze(start_logits, axis=-1)
 end_logits = tf.squeeze(end_logits, axis=-1)
 start_scores = tf.keras.layers.Activation('softmax' )(start_logits)
 end_scores = tf.keras.layers.Activation('softmax' )(end_logits)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[start_scores,end_scores])
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.00003)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer,metrics=['accuracy'])
 return model<set_options>
display(lgb_results.sort_values(by="Score",ascending = True))
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
self.scale_mode = 'cycle'
print(self.scale_mode)
 if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):
if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
print("Setting the lr to {}".format(self.base_lr))
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
<choose_model_class>
final_model_params = lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["Parameters"]
optimal_rounds = lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["Rounds"]
print("Parameters for Final Models:
",final_model_params)
 print("Score: {} +/- {}".format(lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["Score"],lgb_results.iloc[lgb_results["Score"].idxmin() ,:]["STDV"]))
print("Rounds: ", optimal_rounds )
clr = CyclicLR(
 	mode='triangular',
	base_lr= 1e-6,
	max_lr= 4e-5,
	step_size= 320,
scale_mode='cycle' )<init_hyperparams>
print("Find Optimal Cutoff")
 lgb_clf = lgb.train(
 final_model_params,
lgtrain,
num_boost_round = optimal_rounds + 1,
verbose_eval=100,
categorical_feature= categorical_features)
 p = lgb_clf.predict(X_stack_test)
 fpr, tpr, thresholds = metrics.roc_curve(y_stack_test,p)
 optimal_idx = np.argmax(tpr - fpr)
 optimal_threshold = thresholds[optimal_idx]
print("Optimal Threshold: ",optimal_threshold)
 y_stack_train_pred =(lgb_clf.predict(X_stack_train)> optimal_threshold ).astype(int)
 y_stack_test_pred =(lgb_clf.predict(X_stack_test)> optimal_threshold ).astype(int)
 score= metrics.accuracy_score(y_stack_test_pred, y_stack_test)
 print("
LGBM STACKER - CV Train : %.2f" % score )
jac = []; VER='v4'; DISPLAY=1
preds_start = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids.shape[0],MAX_LEN))
def scheduler(epoch):
if epoch <1:
return 0.00003
else:
return 0.00003 * tf.math.exp(-0.1 *(10 - epoch))
reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler,verbose=1)
 n_splits = 5
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=42)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print(idxT,idxV)
 print("Training for fold {}".format(fold))
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 hist = model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv,reduce_lr],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
model.load_weights('%s-roberta-%i.h5'%(VER,fold))
preds_start[idxV,],preds_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=1)
 all = []
for k in idxV:
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all)) 
<predict_on_test>
allmodelstart= time.time()
all_feature_importance_df = pd.DataFrame()
modelstart= time.time()
lgb_clf = lgb.train(
 final_model_params,
fulllgtrain,
num_boost_round = optimal_rounds + 1,
verbose_eval=100,
categorical_feature= categorical_features)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = X_stack.columns
fold_importance_df["importance"] = lgb_clf.feature_importance()
all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)
 lgbmprobpred = lgb_clf.predict(test_stack)
 lgbmpred =(lgbmprobpred > optimal_threshold ).astype(int)
 submission = pd.DataFrame({'PassengerId':test_stack.index,'Survived':lgbmpred})
 submission.to_csv('LGBM.csv',index=False)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
cols = all_feature_importance_df[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = all_feature_importance_df.loc[all_feature_importance_df.feature.isin(cols)]
plt.figure(figsize=(10,8))
sns.barplot(x="importance", y="feature",
data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features')
 plt.tight_layout()
plt.savefig('lgbm_importances.png' )
<feature_engineering><EOS> 
end = time.time()
print("Notebook took %0.2f minutes to Run"%(( end - start)/60))