Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<drop_column>
%matplotlib inline
pd.set_option('max_colwidth', 40 )
train_ID=train['Id']
test_ID=test['Id']
train.drop("Id",axis=1,inplace=True)
 test.drop("Id",axis=1,inplace=True )<filter>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
train=train[train["GrLivArea"]<4500]
train.reset_index(drop=True,inplace=True )<prepare_x_and_y>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
train['SalePrice']=np.log1p(train['SalePrice'])
 y=train['SalePrice']
train_features=train.drop('SalePrice',axis=1)
 test_features=test 
<concatenate>
Dropout_new = 0.15
n_split = 5
lr = 3e-5
features=pd.concat([train_features,test_features],axis=0)
 numeric_t = [f for f in features.columns if features.dtypes[f] != 'object']
char_t = [f for f in features.columns if features.dtypes[f] == 'object']<count_missing_values>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
for col in numeric_t:
if features[col].isnull().sum() >0:
print("{} is lack of {}".format(col,features[col].isnull().sum()))<count_missing_values>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
for col in char_t:
if features[col].isnull().sum() >0:
print("{} is lack of {}".format(col,features[col].isnull().sum()))<data_type_conversions>
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
features['MSSubClass'] = features['MSSubClass'].astype(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional']=features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st']=features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd']=features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))<data_type_conversions>
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
<feature_engineering>
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )
skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))<feature_engineering>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<categorify>
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )
final_features = pd.get_dummies(features ).reset_index(drop=True)
 print(final_features.shape)
 X = final_features.iloc[:len(y), :]
X_sub = final_features.iloc[len(X):, :]
print('X', X.shape, 'y', y.shape, 'X_sub', X_sub.shape)
 outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )<drop_column>
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_sub = X_sub.drop(overfit, axis=1 ).copy()<import_modules>
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )
import numpy as np
import pandas as pd
from datetime import datetime
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from mlxtend.regressor import StackingCVRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor<compute_test_metric>
train = train.replace({'sentiment': {'negative': -1, 'neutral': 0, 'positive': 1}})
 train['len_text'] = train['text'].str.len()
train['len_selected_text'] = train['selected_text'].str.len()
train['diff_num'] = train['end']-train['start']
train['share'] = train['len_selected_text']/train['len_text']
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse )<train_on_grid>
train['selected_text_pred'] = train['selected_text_pred'].map(lambda x: x.lstrip(' '))
train['len_selected_text_pred'] = train['selected_text_pred'].str.len()
train['diff_num_pred'] = train['end_pred']-train['start_pred']
train['share_pred'] = train['len_selected_text_pred']/train['len_text']
train['len_equal'] = 0
train.loc[(train['start'] == train['start_pred'])&(train['end'] == train['end_pred']), 'len_equal'] = 1
train['metric'] = metric_tse(train,'selected_text','selected_text_pred')
 train['res'] = 0
train.loc[train['metric'] == 1, 'res'] = 1
kfolds=KFold(n_splits=10,shuffle=True,random_state=42)
 scale=RobustScaler().fit(X)
 X1=scale.transform(X)
 model=Ridge()
rid_param_grid = {"alpha":[19.8]}
grid_search= GridSearchCV(model,param_grid=rid_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 rid_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )<train_on_grid>
def rep_3chr(text):
chr3 = 0
for word in text.split() :
for c in set(word):
if word.rfind(c+c+c)> -1:
chr3 = 1
return chr3
model=Lasso()
las_param_grid = {"alpha":[0.0005963623316594642]}
grid_search= GridSearchCV(model,param_grid=las_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 las_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )<train_on_grid>
train['text_chr3'] = train['text'].apply(rep_3chr)
 train['selected_text_chr3'] = train['selected_text'].apply(rep_3chr)
 train['selected_text_pred_chr3'] = train['selected_text_pred'].apply(rep_3chr )
model=ElasticNet()
ela_param_grid = {"alpha":[0.0006951927961775605],
"l1_ratio":[0.90]}
grid_search= GridSearchCV(model,param_grid=ela_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 ela_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )<train_on_grid>
train_outlier = train[train['res'] == 0].reset_index(drop=True)
 train_outlier
model=SVR()
svr_param_grid = {"C":[66],
"gamma":[6.105402296585326e-05]}
grid_search= GridSearchCV(model,param_grid=svr_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 svr_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )<find_best_params>
sh_out = str(round(len(train_outlier)*100/len(train),1))
print('Number of outliers is ' + sh_out + '% from training data' )
grid_search.best_params_<choose_model_class>
train_good = train[train['res'] == 1].reset_index(drop=True)
 train_good
model=GradientBoostingRegressor()
gbdt_param_grid = {"n_estimators":[2200],
"learning_rate":[0.05],
"max_depth":[3],
"max_features":["sqrt"],
"min_samples_leaf":[5],
"min_samples_split":[12],
"loss":["huber"]
}
grid_search= GridSearchCV(model,param_grid=gbdt_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 gbdt_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )<choose_model_class>
train_outlier[train_outlier['diff_num']==0]
model=LGBMRegressor()
lgbm_param_grid = {
'objective':['regression'],
'max_depth':[5],
'num_leaves':[12],
'learning_rate':[0.005],
'n_estimators':[5500],
'max_bin':[190],
'bagging_fraction':[0.2],
'feature_fraction':[0.2]
}
grid_search= GridSearchCV(model,param_grid=lgbm_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 lgbm_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )<normalization>
train_good[train_good['diff_num']==0]
kfolds=KFold(n_splits=5,shuffle=True,random_state=42)
 scale=RobustScaler().fit(X)
 X1=scale.transform(X)
 model=XGBRegressor()
xgb_param_grid = {"n_estimators":[3000],
"learning_rate":[0.01],
"max_depth":[3],
"subsample":[0.8],
"colsample_bytree":[0.8],
"gamma":[0],
"objective":['reg:linear'],
"min_child_weight":[2],
"reg_alpha":[0.1],
"reg_lambda":[0.5]
}
grid_search= GridSearchCV(model,param_grid=xgb_param_grid,cv=kfolds,scoring="neg_mean_squared_error",n_jobs= 10, verbose = 1)
 grid_search.fit(X1,y)
 xgb_best=grid_search.best_estimator_
np.sqrt(-grid_search.best_score_ )<train_model>
train_outlier[train_outlier['share']==1]
stack_gen = StackingCVRegressor(regressors=(rid_best, las_best, ela_best,
gbdt_best, xgb_best, lgbm_best),
meta_regressor=xgb_best,
use_features_in_secondary=True)
 stack_gen.fit(np.array(X1), np.array(y))<normalization>
train_good[train_good['share']==1]
X_sub=scale.transform(X_sub )<predict_on_test>
len(train_outlier[train_outlier["text"].str.find(' ')== -1] )
def blend_models_predict(X):
return(( 0.1 * ela_best.predict(X)) + \ 
(0.1 * las_best.predict(X)) + \ 
(0.1 * rid_best.predict(X)) + \ 
(0.1 * svr_best.predict(X)) + \ 
(0.1 * gbdt_best.predict(X)) + \ 
(0.1 * xgb_best.predict(X)) + \ 
(0.1 * lgbm_best.predict(X)) + \ 
(0.3 * stack_gen.predict(np.array(X))))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X1)))
 print('Predict submission', datetime.now() ,)
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)))
<load_from_csv>
len(train_good[train_good["text"].str.find(' ')== -1] )
sub_1 = pd.read_csv('.. /input/top-10-0-10943-stacking-mice-and-brutal-force/House_Prices_submit.csv')
 sub_2 = pd.read_csv('.. /input/hybrid-svm-benchmark-approach-0-11180-lb-top-2/hybrid_solution.csv')
 sub_3 = pd.read_csv('.. /input/lasso-model-for-regression-problem/lasso_sol22_Median.csv')
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(blend_models_predict(X_sub)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))
submission.to_csv("submission.csv", index=False )<import_modules>
def subtext_analysis(col, subtext, df1, str1, df2, str2):
result = pd.DataFrame(columns = ['subtext', str1, str2, 'share,%'])
 if(len(df1)> 0)and(len(df2)> 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num1 = len(df1[df1[col].str.find(subtext[i])> -1])
 result.loc[i, str1] = num1
num2 = len(df2[df2[col].str.find(subtext[i])> -1])
 result.loc[i, str2] = num2
result.loc[i,'share,%'] = round(num1*100/num2,1)if num2 != 0 else 0
print('Number of all data is', len(df2))
display(result.sort_values(by=['share,%', str1], ascending=False))
!pip install efficientnet 
<define_variables>
def subtext_analysis_one_df(col, subtext, df, str):
result = pd.DataFrame(columns = ['subtext', str, 'share of all,%'])
 num_all = len(df)
 if(num_all > 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num = len(df[df[col].str.find(subtext[i])> -1])
 result.loc[i, str] = num
result.loc[i,'share of all,%'] = round(num*100/num_all,1)
 print('Number of all data is', len(df))
display(result.sort_values(by='share of all,%', ascending=False))
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<define_variables>
subtext_test = ['SAD', 'bullying', 'Uh', 'oh', 'onna', 'fun', 'addicted', 'Power', 'well', 'unhappy', 'funny', 'Tears', 'Fears', 'sleeeeepy', ' ', ',', '?', '!' ,'!!', '!!!', ':/', '...', 'http', '****']
seed = 69
img_dir = ".. /input/plant-pathology-2020-fgvc7//images/"
path = ".. /input/plant-pathology-2020-fgvc7/"
input_shape =(512,512,3)
<load_from_csv>
subtext_analysis("selected_text", subtext_test, train_outlier, 'train_outliers', train, 'train_all' )
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sample_submission = pd.read_csv(path + 'sample_submission.csv')
<choose_model_class>
subtext_analysis("selected_text", subtext_test, train_good, 'train_good', train, 'train_all' )
gm_exp = tf.Variable(3.0, dtype = tf.float32)
 def generalized_mean_pool_2d(X):
pool =(tf.reduce_mean(tf.abs(X**(gm_exp)) ,
axis = [1, 2],
keepdims = False)+ 1.e-7)**(1./gm_exp)
 return pool
def create_model(input_shape):
input = Input(shape = input_shape)
 x_model = ResNet50(weights = 'imagenet', include_top = False, input_tensor = input, pooling = None,
classes = None)
 for layer in x_model.layers:
layer.trainable = True
lambda_layer = Lambda(generalized_mean_pool_2d)
 lambda_layer.trainable_weights.extend([gm_exp])
 x = lambda_layer(x_model.output)
 healthy = Dense(4, activation = 'softmax', name = 'plan_diseases' )(x)
 model = Model(inputs = x_model.input, outputs = healthy)
 return model 
<train_model>
subtext_analysis_one_df("selected_text", subtext_test, train, 'test_all' )
def _read(path):
img = cv2.imread(path)
 return img
class TrainDataGenerator(tf.keras.utils.Sequence):
def __init__(self , X_set, Y_set, img_dir, ids, batch_size = 2, img_size =(512,512,3), augmentation = False, GridMask = False):
self.X = X_set
self.Y = Y_set
self.batch_size = batch_size
self.ids = ids
self.img_size = img_size
self.img_dir = img_dir
self.augmentation = augmentation
self.GridMask = GridMask
self.on_epoch_end()
self.x_indexed = X_set[self.ids]
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X, Y = self.__generator__(indices)
 return X, Y
def on_epoch_end(self):
self.indices = np.arange(len(self.ids))
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
Y = np.empty(( self.batch_size, 4))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation == True:
aug = strong_aug(p=1.0)
 image = augment(aug, image)
 elif self.GridMask == True:
image = augment(transforms_train, image)
 X[i,] = image/255.
 Y[i,] = self.Y.loc[index].values
return X, Y
class TestDataGenerator(tf.keras.utils.Sequence):
def __init__(self, Test_set, img_dir, ids, batch_size = 3, img_size =(512,512,3), augmentation = None):
self.X = Test_set
self.img_dir = img_dir
self.ids = ids
self.batch_size = batch_size
self.img_size = img_size
self.augmentation = augmentation
def __len__(self):
return int(ceil(len(self.ids)/self.batch_size))
def __getitem__(self, index):
indices = self.ids[index*self.batch_size:(index+1)*self.batch_size]
X = self.__generator__(indices)
 return X
def __generator__(self, indices):
X = np.empty(( self.batch_size, *self.img_size))
for i, index in enumerate(indices):
ID = self.X[index]
image = _read(self.img_dir+ID+".jpg")
 image = cv2.resize(image,(512,512))
if self.augmentation is not None:
if self.augmentation == "HorizontalFlip":
image = augment(HorizontalFlip(p=1), image)
 elif self.augmentation == "VerticalFlip":
image = augment(VerticalFlip(p=1), image)
 elif self.augmentation == 'VH':
image = VH_augment(image)
 X[i,] = image/255.
 return X 
<split>
subtext_analysis_one_df("selected_text", subtext_test, test, 'test_all' )
tgt_cols = ['healthy', 'multiple_diseases', 'rust' , 'scab']
train_df = train[tgt_cols]
Y = pd.get_dummies(train_df)
 X = train['image_id']
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.15, random_state = seed)
 ids_train = np.array(X_train.index)
 ids_test = X_test.index
del train
del X
del Y<choose_model_class>
test['text_chr3'] = test['text'].apply(rep_3chr)
 test.head(10 )
model = create_model(input_shape)
 model.compile(optimizer = Adam(lr = 0.00016),
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy'] )<choose_model_class>
train_outlier1 = train_outlier.nsmallest(1000, 'metric', keep='all')
 train_outlier2 = train_outlier.nsmallest(2000, 'metric', keep='all')
 train_outlier3 = train_outlier.nsmallest(3000, 'metric', keep='all')
 train_outlier5 = train_outlier.nsmallest(5000, 'metric', keep='all')
 train_outlier8 = train_outlier.nsmallest(8000, 'metric', keep='all' )
best_w = ModelCheckpoint('plant_best.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
 last_w = ModelCheckpoint('plant_last.h5',
monitor='val_loss',
verbose=0,
save_best_only=False,
mode='auto',
period=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=2, min_lr=0.000001, verbose=1,cooldown=1)
 callbacks = [best_w, last_w, reduce_lr] 
<train_model>
subtext_analysis("selected_text", subtext_test, train_outlier1, 'in worst 1000 outliers', train_outlier, 'in all outliers' )
batch_size = 5
model.fit_generator(TrainDataGenerator(X_train,Y_train , img_dir, ids_train,
batch_size, augmentation = True, GridMask = True),
epochs=15,
verbose=1,
callbacks=callbacks,
validation_data=TrainDataGenerator(X_test, Y_test , img_dir, ids_test,
batch_size, augmentation = False, GridMask = False),
workers=1,
use_multiprocessing=False,
shuffle=True )<set_options>
subtext_analysis("selected_text", subtext_test, train_outlier2, 'in worst 2000 outliers', train_outlier, 'in all outliers' )
gc.collect()<load_pretrained>
subtext_analysis("selected_text", subtext_test, train_outlier3, 'in worst 3000 outliers', train_outlier, 'in all outliers' )
model.load_weights('plant_best.h5' )<predict_on_test>
subtext_analysis("selected_text", subtext_test, train_outlier5, 'in worst 5000 outliers', train_outlier, 'in all outliers' )
ids = test.index
Test_set = test['image_id']
data_generator_test = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = None)
 data_generator_test_Horizontal = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "HorizontalFlip")
 data_generator_test_Vertical = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VerticalFlip")
 data_generator_test_VH = TestDataGenerator(Test_set, img_dir, ids, batch_size, input_shape, augmentation = "VH")
 preds_1 = model.predict_generator(data_generator_test, verbose = 1)
 preds_2 = model.predict_generator(data_generator_test_Horizontal, verbose = 1)
 preds_3 = model.predict_generator(data_generator_test_Vertical, verbose = 1)
 preds_4 = model.predict_generator(data_generator_test_VH, verbose = 1)
 for index, pred in enumerate(zip(preds_1,preds_2, preds_3, preds_4)) :
sample_submission["healthy"][index] =(pred[0][0] + pred[1][0] + pred[2][0] + pred[3][0])/4
sample_submission["multiple_diseases"][index] =(pred[0][1] + pred[1][1] + pred[2][1] + pred[3][1])/4
sample_submission["rust"][index] =(pred[0][2] + pred[1][2] + pred[2][2] + pred[3][2])/4
sample_submission["scab"][index] =(pred[0][3] + pred[1][3] + pred[2][3] + pred[3][3])/4<save_to_csv>
subtext_analysis("selected_text", subtext_test, train_outlier8, 'in worst 8000 outliers', train_outlier, 'in all outliers' )
sample_submission.to_csv("submission.csv", index=False )<import_modules>
col_hist = ['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text',
'text_chr3', 'selected_text_chr3', 'selected_text_pred_chr3', 'metric']
<define_variables><EOS> 
data = train[['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text', 'diff_num', 'share', 'metric', 'res']].dropna()
data