Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<save_to_csv>
print('TF version',tf.__version__)
print("Gönderi hazırlanıyor")
 submision = pd.DataFrame()
submision['Id'] = test_ids
submision['SalePrice'] = xgb_pred
submision.to_csv('n_most_xgb_submission.csv',index=False)
 print("Gönderi kaydedildi")
 submision.head(20)
 print(xgb_train_pred[:20])
 print(y_train.values[:20])
 print(submision.values[:20] )<load_from_csv>
max_len = 128
train_mode = False
tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file = '.. /input/tf-roberta/vocab-roberta-base.json',
merges_file = '.. /input/tf-roberta/merges-roberta-base.txt',
add_prefix_space = True,
lowercase = True )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print(train.shape)
 display(train.head())
 train.fillna(value=-999.0,inplace=True)
 test.fillna(value=-999.0,inplace=True)
 print('Type numeric:')
 list_feature_Nan = []
for i in train.select_dtypes(exclude=['object'] ).columns:
if(train[i] == -999.0 ).astype(int ).sum() > 0:
print("Feature: ", i, "has", round(((train[i] == -999.0 ).astype(int ).sum() /1460)*100), "% of NaN")
 list_feature_Nan.append(i)
 print('Type object:')
 for i in train.select_dtypes(include=['object'] ).columns:
if(train[i] == -999.0 ).astype(int ).sum() > 0:
print("Feature: ", i, "has", round(((train[i] == -999.0 ).astype(int ).sum() /1460)*100), "% of NaN")
<prepare_x_and_y>
sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
data = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 data.head()
for i in list_feature_Nan:
train[i].replace(-999.0,train[i].mean() ,inplace=True)
 test[i].replace(-999.0,train[i].mean() ,inplace=True)
 y = np.array(pd.DataFrame(np.log(train.SalePrice)))
 y2 = np.array(pd.DataFrame(np.log(train.SalePrice)))
 mean_saleprice = pd.DataFrame(train.groupby(['GrLivArea'])['LotArea'].mean())
 mean_saleprice.columns = ['LotArea_bis']
train.drop('SalePrice',axis=1,inplace=True)
 all_data = all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'],test.loc[:,'MSSubClass':'SaleCondition']))
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
print(skewed_feats)
 train[skewed_feats] = np.log1p(train[skewed_feats])
 test[skewed_feats] = np.log1p(test[skewed_feats])
 train[skewed_feats] = train[skewed_feats].fillna(all_data[skewed_feats].mean())
 test[skewed_feats] = test[skewed_feats].fillna(all_data[skewed_feats].mean() )<categorify>
data['text'].replace('', np.nan, inplace=True)
 data.dropna(subset=['text'], inplace=True)
 data.reset_index(drop=True, inplace=True )
le = preprocessing.LabelEncoder()
train_str = train.select_dtypes(include=['object'])
 test_str = test.select_dtypes(include=['object'])
 display(train_str.head())
 print(train_str.columns.values)
 train.drop(train_str.columns.values,axis=1,inplace=True)
 test.drop(train_str.columns.values,axis=1,inplace=True )<merge>
x_train,x_test = train_test_split(data, test_size = 0.05, random_state=42)
 x_train,x_cv = train_test_split(x_train, test_size = 0.2, random_state = 42)
 print("x_train shape is", x_train.shape)
 print("x_cv shape is", x_cv.shape)
 print("x_test shape is", x_test.shape)
 x_train.reset_index(drop=True, inplace=True)
 x_cv.reset_index(drop=True, inplace=True)
 x_test.reset_index(drop=True, inplace=True )
train_str_dum = pd.get_dummies(train_str)
 test_str_dum = pd.get_dummies(test_str)
 columns_dum = list(set(train_str_dum)& set(test_str_dum))
train_str_dum = train_str_dum[columns_dum]
test_str_dum = test_str_dum[columns_dum]
train_flo = train.select_dtypes(exclude=['object'])
 test_flo = test.select_dtypes(exclude=['object'])
 new_train = pd.merge(train_flo,train_str_dum,left_index=True,right_index=True)
 new_test = pd.merge(test_flo,test_str_dum,left_index=True,right_index=True)
 display(new_train.head())
 print(new_train.columns.values )<drop_column>
tweet = 'WHY DO WE FALL? SO THAT WE CAN LEARN TO PICK OURSELVES BACK UP.'
selected_text = 'LEARN TO PICK OURSELVES BACK UP'
sentiment = 'positive'
print('text',tweet)
 print('select_text:',selected_text)
 print('sentiment:', sentiment)
 idx0 = None
idx1 = None
st_len = len(selected_text)
 tweet = " " + tweet
for i in range(len(tweet)) :
if(tweet[i:i+st_len]==selected_text):
idx0 = i
idx1 = i + st_len -1
break
char_targets = [0]*len(tweet)
 for i in range(len(tweet)) :
if idx0 != None and idx1!=None:
if i>=idx0 and i<=idx1:
char_targets[i] = 1
print('char_targets:',char_targets)
 tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
print('input_ids:',input_ids)
 print('offsets:',offsets)
 targets_index = []
for i,(off1,off2)in enumerate(offsets):
if sum(char_targets[off1:off2])>0:
targets_index.append(i)
 target_start = targets_index[0]
target_end = targets_index[-1]
print('target_start:',target_start)
 print('target_end:', target_end)
 ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 target_start+=1
target_end+=1
offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
train_clf = new_train.copy()
train_clf.drop('Id',axis=1,inplace=True)
 index = pd.DataFrame(test.Id,columns = ['Id'])
 test_clf = new_test.drop('Id',axis=1)
 train_clf2 = train_clf.drop(['LotFrontage','MasVnrArea','GarageYrBlt'],axis=1)
 train_clf2 = pd.merge(train_clf2,train[['LotFrontage','MasVnrArea','GarageYrBlt']],left_index=True,right_index=True)
 train_clf2['tot_sf'] = train_clf2['TotalBsmtSF'] + train_clf2['GrLivArea']
train_clf2['ratio_fl'] = train_clf2['2ndFlrSF'] / train_clf2['1stFlrSF']
train_clf2['garage_ex'] =(train_clf2['GarageQual_Gd'] + train_clf2['GarageQual_TA'] + train_clf2['GarageQual_Fa'] + train_clf2['GarageQual_Po'])*(train_clf2['GarageCond_Ex'])
 clus = KernelPCA(n_components = 25)
 train_clf2_pca = clus.fit_transform(train_clf2)
 train_clf3 = pd.merge(train_clf2,pd.DataFrame(train_clf2_pca),left_index=True,right_index=True)
 test_clf2 = test_clf.drop(['LotFrontage','MasVnrArea','GarageYrBlt'],axis=1)
 test_clf2 = pd.merge(test_clf2,test[['LotFrontage','MasVnrArea','GarageYrBlt']],left_index=True,right_index=True)
 test_clf2['tot_sf'] = test_clf2['TotalBsmtSF'] + test_clf2['GrLivArea']
test_clf2['ratio_fl'] = test_clf2['2ndFlrSF'] / test_clf2['1stFlrSF']
test_clf2['garage_ex'] =(test_clf2['GarageQual_Gd'] + test_clf2['GarageQual_TA'] + test_clf2['GarageQual_Fa'] + test_clf2['GarageQual_Po'])*(test_clf2['GarageCond_Ex'])
 test_clf3 = pd.merge(test_clf2,pd.DataFrame(clus.transform(test_clf2)) ,left_index=True,right_index=True)
 index = pd.DataFrame(test.Id,columns = ['Id'])
 def rmse(predictions, targets):
return np.sqrt(((predictions - targets)** 2 ).mean())
 def score(clf, train_np, random_state, folds, target, length):
kf = KFold(n = length , n_folds=folds, shuffle = True, random_state = random_state)
 for itrain, itest in kf:
Xtr, Xte = train_np[itrain], train_np[itest]
ytr, yte = target[itrain], target[itest]
clf.fit(Xtr, ytr.ravel())
 pred = pd.DataFrame(clf.predict(Xte))
return rmse(yte, pred)
 return rmse(y, pred )<prepare_x_and_y>
def text_process(tweet, sentiment, tokenizer, max_len, selected_text=None):

 if selected_text!=None:
tweet = " " + tweet
idx0 = None
idx1 = None
st_len = len(selected_text)
 for i in range(len(tweet)) :
if(tweet[i:i+st_len]==selected_text):
idx0 = i
idx1 = i + st_len -1
break
char_targets = [0]*len(tweet)
 for i in range(len(tweet)) :
if idx0 != None and idx1!=None:
if i>=idx0 and i<=idx1:
char_targets[i] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
targets_index = []
for i,(off1,off2)in enumerate(offsets):
if sum(char_targets[off1:off2])>0:
targets_index.append(i)
 target_start = targets_index[0]
target_end = targets_index[-1]
ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 target_start+=1
target_end+=1
offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 return {
'ids': ids,
'token_type_ids':token_type_ids,
'mask':mask,
'target_start':target_start,
'target_end':target_end,
'offsets':offsets
}
else:
tweet = " " + tweet
tok_tweet = tokenizer.encode(tweet)
 input_ids = tok_tweet.ids
offsets = tok_tweet.offsets
ids = [0] + input_ids + [2,2] + [sentiment_id[sentiment]] + [2]
token_type_ids = [0]*(len(ids))
mask = [1] * len(ids)
 offsets = [(0,0)]*1 + offsets + [(0,0)]*4
padding_length = max_len - len(ids)
 if padding_length > 0:
ids = ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 offsets = offsets +([(0, 0)] * padding_length)
 return {
'ids': ids,
'token_type_ids':token_type_ids,
'mask':mask,
'offsets':offsets
}
train_clf4 = train_clf3.copy()
test_clf4 = test_clf3.copy() 
<feature_engineering>
rows = x_train.shape[0]
train_ids = np.zeros(( rows,max_len), dtype = 'int32')
 train_mask = np.zeros(( rows,max_len), dtype = 'int32')
 train_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 train_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 train_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_train.shape[0]):
encoding = text_process(x_train.loc[i,'text'], x_train.loc[i,'sentiment'], tokenizer, max_len,x_train.loc[i,'selected_text'])
 train_ids[i] = encoding['ids']
train_start_idx[i,encoding['target_start']] = 1
train_end_idx[i, encoding['target_end']] = 1
train_type_ids[i] = encoding['token_type_ids']
train_mask[i] = encoding['mask']
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].min() > test_clf4[i].min() :
train_clf4[i][train_clf4[i]==train_clf4[i].min() ] = test_clf4[i].min()
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].min() < test_clf4[i].min() :
test_clf4[i][test_clf4[i]==test_clf4[i].min() ] = train_clf4[i].min()<feature_engineering>
i = 10
encoding = text_process(x_train.loc[i,'text'], x_train.loc[i,'sentiment'], tokenizer, max_len,x_train.loc[i,'selected_text'])
 train_ids[i] = encoding['ids']
train_start_idx[i,encoding['target_start']] = 1
train_end_idx[i, encoding['target_end']] = 1
train_type_ids[i] = encoding['token_type_ids']
train_mask[i] = encoding['mask']
tweet = x_train.loc[i,'text']
select_text = x_train.loc[i,'selected_text']
target_start = np.argmax(train_start_idx[i,])
 target_end = np.argmax(train_end_idx[i, ])
 offsets = encoding['offsets']
print('tweet:',tweet)
 print('selected_text:',select_text)
 print(tweet[offsets[target_start][0]:offsets[target_end][-1]] )
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].max() > test_clf4[i].max() :
test_clf4[i][test_clf4[i]==test_clf4[i].max() ] = train_clf4[i].max()
for i in train_clf4.select_dtypes(include=['float64'] ).columns:
if train_clf4[i].max() < test_clf4[i].max() :
train_clf4[i][train_clf4[i]==train_clf4[i].max() ] = test_clf4[i].max()<compute_test_metric>
rows = x_cv.shape[0]
cv_ids = np.zeros(( rows,max_len), dtype = 'int32')
 cv_mask = np.zeros(( rows,max_len), dtype = 'int32')
 cv_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 cv_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 cv_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_cv.shape[0]):
encoding = text_process(x_cv.loc[i,'text'], x_cv.loc[i,'sentiment'], tokenizer, max_len,x_cv.loc[i,'selected_text'])
 cv_ids[i] = encoding['ids']
cv_start_idx[i,encoding['target_start']] = 1
cv_end_idx[i, encoding['target_end']] = 1
cv_type_ids[i] = encoding['token_type_ids']
cv_mask[i] = encoding['mask']
def put_corr_feat(data,test,target):
data_new = data.copy()
test_new = test.copy()
for i in data.columns:
corr1=stats.pearsonr(np.array(pd.DataFrame(data[i])) ,target)[0]
corr2=stats.pearsonr(np.array(pd.DataFrame(data[i]**3)) ,target)[0]
if abs(corr2)>abs(corr1)*1.2:
print(i)
 data_new[i]=np.array(data_new[i])**3
test_new[i]=np.array(test_new[i])**3
return data_new, test_new
data1, test1 = put_corr_feat(train_clf3,test_clf3,y )<define_variables>
rows = x_test.shape[0]
test_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_mask = np.zeros(( rows,max_len), dtype = 'int32')
 test_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 test_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_test.shape[0]):
encoding = text_process(x_test.loc[i,'text'], x_test.loc[i,'sentiment'], tokenizer, max_len,x_test.loc[i,'selected_text'])
 test_ids[i] = encoding['ids']
test_start_idx[i,encoding['target_start']] = 1
test_end_idx[i, encoding['target_end']] = 1
test_type_ids[i] = encoding['token_type_ids']
test_mask[i] = encoding['mask']
def frange(start, stop, step):
i = start
a = []
while i < stop:
yield i
a.append(i)
 i += step
return a
Cs1 = [5000,6000,7000,8000,9000,10000]
res = []
res1 = []
res2 = []
res3 = []
res4 = []
Cs3=[]
train_np = np.array(train_clf3)
 robust_scaler = RobustScaler() 
<create_dataframe>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0):
return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
index = pd.DataFrame(test.Id,columns = ['Id'] )<concatenate>
config = RobertaConfig.from_pretrained('.. /input/tf-roberta/config-roberta-base.json')
 print(config )
np.random.seed(123)
 ntrain = train_clf4.shape[0]
sparse_data = []
tr_te = pd.concat(( train_clf4, test_clf4), axis = 0)
 tmp = csr_matrix(tr_te)
 sparse_data.append(tmp)
 xtr_te = hstack(sparse_data, format = 'csr')
 xtrain = xtr_te[:ntrain, :]
xtest = xtr_te[ntrain:, :]
def batch_generator(X, y, batch_size, shuffle):
number_of_batches = np.ceil(X.shape[0]/batch_size)
 counter = 0
sample_index = np.arange(X.shape[0])
 if shuffle:
np.random.shuffle(sample_index)
 while True:
batch_index = sample_index[batch_size*counter:batch_size*(counter+1)]
X_batch = X[batch_index,:].toarray()
y_batch = y[batch_index]
counter += 1
yield X_batch, y_batch
if(counter == number_of_batches):
if shuffle:
np.random.shuffle(sample_index)
 counter = 0
def batch_generatorp(X, batch_size, shuffle):
number_of_batches = X.shape[0] / np.ceil(X.shape[0]/batch_size)
 counter = 0
sample_index = np.arange(X.shape[0])
 while True:
batch_index = sample_index[batch_size * counter:batch_size *(counter + 1)]
X_batch = X[batch_index, :].toarray()
counter += 1
yield X_batch
if(counter == number_of_batches):
counter = 0
def nn_model() :
model = Sequential()
model.add(Dense(10, input_dim = xtrain.shape[1], init = 'he_normal'))
model.add(PReLU())
 model.add(Dropout(0.4))
model.add(Dense(5, init = 'he_normal'))
model.add(PReLU())
 model.add(Dropout(0.2))
model.add(Dense(1, init = 'he_normal'))
model.compile(loss = 'mse', optimizer = 'adadelta')
 return(model)
 nfolds = 2
folds = KFold(len(y), n_folds = nfolds, shuffle = True, random_state = 111)
 i = 0
nbags = 2
nepochs = 1
pred_oob = np.zeros(xtrain.shape[0])
 pred_test = np.zeros(xtest.shape[0])
 def model() :
for(inTr, inTe)in folds:
xtr = xtrain[inTr]
ytr = y[inTr]
xte = xtrain[inTe]
yte = y[inTe]
pred = np.zeros(xte.shape[0])
 for j in range(nbags):
model = nn_model()
fit = model.fit_generator(generator = batch_generator(xtr, ytr, 64, True),
nb_epoch = nepochs,
samples_per_epoch = xtr.shape[0],
verbose = 0)
 pred += model.predict_generator(generator = batch_generatorp(xte, 50, False), val_samples = xte.shape[0])[:,0]
score = rmse(np.exp(yte), np.exp(pred))
print(score)
 pred_test += model.predict_generator(generator = batch_generatorp(xtest, 25, False), val_samples = xtest.shape[0])[:,0]
pred /= nbags
pred_oob[inTe] = pred
i += 1 
<choose_model_class>
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 roberta = TFRobertaForQuestionAnswering.from_pretrained('.. /input/tf-roberta/pretrained-roberta-base.h5', config = config)
 x = roberta(ids, attention_mask = att, token_type_ids = tok)
 x1 = tf.keras.layers.Dropout(0.3 )(x[0])
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.3 )(x[1])
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs = [ids, att, tok], outputs=[x1, x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate = 3e-5)
 model.compile(loss= 'categorical_crossentropy', optimizer=optimizer)
 return model
clf = xgb.XGBRegressor(n_estimators = 6000, seed = 0, learning_rate = 0.01, max_depth = 3, subsample = 0.8, colsample_bytree = 0.8, colsample_bylevel = 0.8)
 clf2 = Ridge(alpha=21)
 clf3 = LassoCV(alphas = [1, 0.1, 0.001, 0.0005])
 Select = SelectKBest(f_regression, k = 268)
 train_fin = np.array(Select.fit_transform(train_clf4,y))
test_fin = np.array(Select.transform(test_clf4))
train_fin_ridge = np.array(robust_scaler.fit_transform(train_clf3))
test_fin_ridge = np.array(robust_scaler.transform(test_clf3))
kf1 = KFold(n = 1450 , n_folds=7, random_state=0, shuffle = True)
 i = 0
res1 = []
res4 = []
for itrain, itest in kf1:
i = i + 1
Xtr, Xte = train_fin[itrain], train_fin[itest]
ytr, yte = y[itrain], y[itest]
clf.fit(Xtr, ytr.ravel())
 if i == 1:
pred1 = pd.DataFrame(clf.predict(test_fin))
print("Fold 1 :", rmse(yte, pd.DataFrame(clf.predict(Xte))))
res1.append(rmse(yte, pd.DataFrame(clf.predict(Xte))))
if i > 1 :
pred1 = pred1 + pd.DataFrame(clf.predict(test_fin))
print("Fold ",i, " :", rmse(yte, pd.DataFrame(clf.predict(Xte))))
res1.append(rmse(yte, pd.DataFrame(clf.predict(Xte))))
clf2.fit(train_fin_ridge, y)
 pred2 = pd.DataFrame(clf2.predict(test_fin_ridge))
clf3.fit(train_fin_ridge, y)
 pred3 = pd.DataFrame(clf3.predict(test_fin_ridge))
pred_1 = pred1/7
for i in pred2[np.exp(pred2 ).values<1].index.values:
pred2[pred2.index == i] = pred_1[pred_1.index == i]
for i in pred2[pred2.values>12].index.values:
pred2[pred2.index == i] = pred_1[pred_1.index == i]
print(np.mean(res1))
pred_1 =(pred_1+2*pred2+pred3)/4
pred_2 =(pred_1+pred3)/2
pred_xg = pred_1
pred_1.columns = ['SalePrice']
pred_2.columns = ['SalePrice']
pred_xg.columns = ['SalePrice']
pred3.columns = ['SalePrice']
pred_final_1 = pd.DataFrame(np.exp(pred_1), index = new_test.index, columns = ['SalePrice'])
 pred_final_2 = pd.DataFrame(np.exp(pred_2), index = new_test.index, columns = ['SalePrice'])
 pred_Lasso = pd.DataFrame(np.exp(pred3), index = new_test.index, columns = ['SalePrice'])
 pred_xg = pd.DataFrame(np.exp(pred_xg), index = new_test.index, columns = ['SalePrice'])
 for i in pred_final_1[pred_final_1.values<100].index.values:
pred_final_1[pred_final_1.index == i] = 180921.1959
pred_final = pred_final_1
pred_submit = pd.merge(index,pred_final,left_index=True,right_index=True)
 pred_Lasso_submit = pd.merge(index,pred_Lasso,left_index=True,right_index=True)
 pred_xg_submit = pd.merge(index,pred_xg,left_index=True,right_index=True)
 pred_xg_lasso_submit = pd.merge(index,pred_final_2,left_index=True,right_index=True)
 pred_submit.to_csv('XG_Ridge_Lasso.csv',index=False)
 pred_Lasso_submit.to_csv('Lasso.csv',index=False)
 pred_xg_submit.to_csv('XGB.csv',index=False)
 pred_xg_lasso_submit.to_csv('XGB_Lasso.csv',index=False )<load_from_csv>
gc.collect()
train_sales = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')
 calendar_df = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 submission_file = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv' )<count_values>
if train_mode:
model.fit([train_ids, train_mask, train_type_ids], [train_start_idx, train_end_idx],
epochs = 3,
batch_size = 16,
verbose = True,
callbacks = [checkpoint],
validation_data =([cv_ids,cv_mask,cv_type_ids], [cv_start_idx, cv_end_idx]),
shuffle = True)
 else:
model.load_weights('.. /input/tse-best-roberta/best_model(6 ).h5' )
pd.Series(series_data.max(axis=1)).value_counts().tail(20 )<prepare_output>
rows = x_test.shape[0]
preds_start = np.zeros(( rows,max_len))
preds_end = np.zeros(( rows,max_len))
forecast = pd.DataFrame(series_data[:, -28:] ).mean(axis=1)
 forecast = pd.concat([forecast] * 28, axis=1)
 forecast.columns = [f'F{i}' for i in range(1, forecast.shape[1] + 1)]
forecast.head()<define_variables>
preds = model.predict([test_ids, test_mask, test_type_ids], verbose = True)
 preds_start += preds[0]
preds_end += preds[1]
validation_ids = train_sales['id'].values
evaluation_ids = [i.replace('validation', 'evaluation')for i in validation_ids]<concatenate>
score = 0
for k in range(x_test.shape[0]):
encoding = text_process(x_test.loc[k,'text'], x_test.loc[k,'sentiment'], tokenizer, max_len)
 offsets = encoding['offsets']
targets_start = np.argmax(preds_start[k,])
 targets_end = np.argmax(preds_end[k,])
 pred = x_test.loc[k,'text'][offsets[targets_start][0]:offsets[targets_end][-1]]
score+=jaccard(x_test.loc[k,'selected_text'], pred)
 score=score/x_test.shape[0]
print('score on local test_data',score )
ids = np.concatenate([validation_ids, evaluation_ids] )<create_dataframe>
for k in range(0,x_test.shape[0],100):
encoding = text_process(x_test.loc[k,'text'], x_test.loc[k,'sentiment'], tokenizer, max_len)
 offsets = encoding['offsets']
targets_start = np.argmax(preds_start[k,])
 targets_end = np.argmax(preds_end[k,])
 pred = x_test.loc[k,'text'][offsets[targets_start][0]:offsets[targets_end][-1]]
print('text:', x_test.text[k])
 print('selected text:', x_test.selected_text[k])
 print('sentiment:',x_test.sentiment[k])
 print('predicted:', pred)
 print('jaccard_score:', jaccard(pred, x_test.loc[k,'selected_text']))
print('
predictions = pd.DataFrame(ids, columns=['id'])
 forecast = pd.concat([forecast] * 2 ).reset_index(drop=True)
 predictions = pd.concat([predictions, forecast], axis=1 )<save_to_csv>
x_test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 rows = x_test.shape[0]
preds_start = np.zeros(( rows,max_len))
preds_end = np.zeros(( rows,max_len))
predictions.to_csv('submission.csv', index=False )<define_variables>
rows = x_test.shape[0]
test_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_mask = np.zeros(( rows,max_len), dtype = 'int32')
 test_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 test_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_test.shape[0]):
encoding = text_process(x_test.loc[i,'text'], x_test.loc[i,'sentiment'], tokenizer, max_len)
 test_ids[i] = encoding['ids']
test_type_ids[i] = encoding['token_type_ids']
test_mask[i] = encoding['mask']
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path<import_modules>
preds = model.predict([test_ids, test_mask, test_type_ids], verbose = True)
 preds_start += preds[0]
preds_end += preds[1]
import numpy as np
import pandas as pd
import torch.nn.functional as F
import os
import torch
import torch.nn as nn
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,models
from tqdm import tqdm_notebook as tqdm
import math
import torch.utils.model_zoo as model_zoo
import cv2
from sklearn.metrics import cohen_kappa_score
import openslide
import skimage.io
import random
from sklearn.metrics import cohen_kappa_score
import albumentations
from PIL import Image<set_options>
submission = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv' )
class config:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 IMG_WIDTH = 512
IMG_HEIGHT = 512
TEST_BATCH_SIZE = 16
CLASSES = 6
DEBUG = False<load_from_csv>
for k in range(x_test.shape[0]):
encoding = text_process(x_test.loc[k,'text'], x_test.loc[k,'sentiment'], tokenizer, max_len)
 offsets = encoding['offsets']
targets_start = np.argmax(preds_start[k,])
 targets_end = np.argmax(preds_end[k,])
 pred = x_test.loc[k,'text'][offsets[targets_start][0]:offsets[targets_end][-1]]
submission.loc[k,'selected_text'] = pred
BASE_PATH = '.. /input/prostate-cancer-grade-assessment'
data_dir = f'{BASE_PATH}/test_images'
test = pd.read_csv(f'{BASE_PATH}/test.csv')
 submission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')
 if config.DEBUG:
data_dir = f'{BASE_PATH}/train_images'
test = pd.read_csv(f'{BASE_PATH}/train.csv' ).head(200 )<set_options>
submission.to_csv('submission.csv', index = False )