Kaggle competition: Tweet Sentiment Extraction
def fixskew(data):
return boxcox1p(data,boxcox_normmax(data+1))<prepare_x_and_y>
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from nltk.stem.wordnet import WordNetLemmatizer
import re
import tensorflow as tf
import tensorflow.keras.backend as K
from sklearn.model_selection import StratifiedKFold
from transformers import *
import tokenizers
from keras.layers import Dense, Flatten, Conv1D, Dropout, Input
from keras.models import Model, load_model
from keras.callbacks import ModelCheckpoint, EarlyStopping
y = np.log(df_train['SalePrice'])
 df_test = pd.read_csv('.. /input/test.csv' ).drop('Id',axis=1)
 features = df_train.drop('SalePrice',axis=1 ).append(df_test)
 features = features.drop(['Utilities', 'Street', 'PoolQC'], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF1']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['TotalSF2'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['TotalBath'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['TotalPorchSF'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['HasPool'] =(features['PoolArea']!=0 ).astype('int')
 features['Has2ndFloor'] =(features['2ndFlrSF']!=0 ).astype('int')
 features['HasGarage'] =(features['GarageArea']!=0 ).astype('int')
 features['HasBsmt'] =(features['TotalBsmtSF']!=0 ).astype('int')
 features['HasFireplace'] =(features['Fireplaces']!=0 ).astype('int')
 numcols = features.columns[features.dtypes!='O']
for col in numcols:
features[col]=features[col].fillna(features[col].mean())
 catcols = features.columns[features.dtypes=='O']
for col in catcols:
target_map = df_train['SalePrice'].groupby(df_train[col].fillna('None')).mean()
features[col+'_target']=features[col].map(target_map ).fillna(df_train['SalePrice'].mean())
 features = pd.get_dummies(features.fillna('None'))
X = features[:len(y)]
X_test = features[len(y):]
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for col in X.columns:
counts = X[col].value_counts()
num_zeros = counts.iloc[0]
if num_zeros / len(X)* 100.0 > 99.0:
overfit.append(col)
 X = X.drop(overfit, axis=1)
 X_test = X_test.drop(overfit, axis=1 )<compute_train_metric>
train_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 submission_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv' )
kf = KFold(n_splits=5,shuffle=True)
 def scoreit(model,X=X):
return(np.sqrt(-cross_val_score(model,X,y,cv=kf,scoring='neg_mean_squared_error')) )<compute_test_metric>
train_df.isnull().sum()
lasso = make_pipeline(RobustScaler() , Lasso(alpha=3e-4))
scores = scoreit(lasso,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )<compute_test_metric>
train_df.dropna(inplace=True)
 train_df.reset_index(inplace=True )
randomforest = RandomForestRegressor(n_estimators=100)
 scores = scoreit(randomforest,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )<compute_test_metric>
train_df.isnull().sum()
xgb = make_pipeline(RobustScaler() , XGBRegressor(
 learning_rate=0.01, n_estimators=3400, max_depth=3, min_child_weight=0,
gamma=0, subsample=0.75, colsample_bytree=0.7, objective='reg:linear', nthread=-1,
scale_pos_weight=1, reg_alpha=0.00005))
scores = scoreit(xgb,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )<compute_test_metric>
def word(text):
review_word = BeautifulSoup(text ).get_text()
review_word = re.sub(r'[^a-zA-Z]',' ', review_word)
 words = review_word.lower().split()
stops = set(stopwords.words('english'))
words = [w for w in words if w not in stops]
return ' '.join(words)
 train_df['clean_text']=train_df['text'].apply(lambda x : word(x))
stack = StackingRegressor([lasso,xgb,randomforest],meta_regressor=xgb)
 scores = scoreit(stack,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )<save_to_csv>
def f_emoji(text):
sentences=[]
emoji_text = emoji.demojize(text)
 line = re.sub(":","",emoji_text)
 for sentence in line:
sett = re.sub("_"," ",sentence)
 sentences.append(' '.join(sett))
return ''.join(sentences)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_email(text):
line = re.compile(r'[\w\.-]+@[\w\.-]+')
 return line.sub(r'',text)
 def remove_hash(text):
line = re.sub(r'\
return ''.join(line)
 def remove_phone_num(text):
line = re.sub(r'\b\d{10}\b','', text)
 line = re.sub(r'(\d{3})\-(\d{3})\-(\d{4})','',line)
 return ''.join(line)
 def remove_year(text):
line=re.sub(r"\b(19[4-9][0-9]|20[0-1][0-9]|2020)\b",'',text)
 return ''.join(line)
 def remove_url(text):
url = re.sub(r'http[s]?[\:]?//\S+|www\.\S+\.\S+','',text)
 return ''.join(url)
 train_df['c_s_text']= train_df['selected_text'].copy()
train_df['c_s_text']= train_df['c_s_text'].apply(lambda x : remove_url(x))
train_df['c_s_text']= train_df['c_s_text'].apply(lambda x : remove_email(x))
train_df['c_s_text']= train_df['c_s_text'].apply(lambda x : remove_emoji(x))
train_df['c_s_text']= train_df['c_s_text'].apply(lambda x : f_emoji(x))
train_df['c_s_text']= train_df['c_s_text'].apply(lambda x : word(x))
lasso = make_pipeline(RobustScaler() , Lasso(alpha=3e-4))
lasso.fit(X,y)
 pred = pd.Series(np.exp(lasso.predict(X_test)))
 test_id = pd.read_csv('.. /input/test.csv')['Id']
output = pd.DataFrame({'Id':test_id, 'SalePrice':pred})
 output.to_csv('submission.csv', index=False )<import_modules>
sent_r=train_df.groupby('sentiment')['sentiment'].count()
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler<load_from_csv>
train_df.loc[train_df['sentiment']=='negative','c_s_text']
sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv" )<prepare_x_and_y>
neutral_df=train_df[train_df['sentiment']=='neutral']
negative_df = train_df[train_df['sentiment']=='negative']
positive_df = train_df[train_df['sentiment']=='positive']
print('neutral : text vs selected text equivalent rate : {:.2f}'.format(sum(neutral_df['clean_text']==neutral_df['c_s_text'])/len(neutral_df)*100))
print('negative : text vs selected text equivalent rate : {:.2f}'.format(sum(negative_df['clean_text']==negative_df['c_s_text'])/len(negative_df)*100))
print('neutral : text vs selected text equivalent rate : {:.2f}'.format(sum(positive_df['clean_text']==positive_df['c_s_text'])/len(positive_df)*100))
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_train = X_train / 255.0
X_test = test / 255.0
X_train = X_train.values.reshape(-1,28,28,1)
 X_test = X_test.values.reshape(-1,28,28,1)
 Y_train = to_categorical(Y_train, num_classes = 10 )<create_dataframe>
def get_top_ngram(corpus, n):
vec = CountVectorizer(ngram_range=(n,n), analyzer='word', max_features=5000)
 bag_of_words = vec.fit_transform(train_df[train_df['sentiment']==corpus]['c_s_text'])
 sum_words=bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0,idx])for word, idx in vec.vocabulary_.items() ]
words_freq = sorted(words_freq, key = lambda x:x[1], reverse=True)
 return words_freq[:20]
top_pos=get_top_ngram('positive',3)
 top_neu=get_top_ngram('neutral',3)
 top_neg=get_top_ngram('negative',3)
 print('Top Positive words: ',top_pos)
 print('Top Neutral words: ', top_neu)
 print('Top Negative words: ', top_neg )
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )<choose_model_class>
max_len = 150
vocab_path = '/kaggle/input/roberta-base/vocab.json'
merge_path = '/kaggle/input/roberta-base/merges.txt'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file = vocab_path,
merges_file = merge_path,
lowercase =True,
add_prefix_space=True 
)
 sentiment_id = {'positive':tokenizer.encode('positive' ).ids[0], 'negative':tokenizer.encode('negative' ).ids[0], 'neutral':tokenizer.encode('neutral' ).ids[0]}
train_df.reset_index(inplace=True )
nets = 15
model = [0] *nets
for j in range(nets):
model[j] = Sequential()
model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Flatten())
 model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )<train_model>
tot_tw = train_df.shape[0]
input_ids = np.ones(( tot_tw, max_len), dtype='int32')
 attention_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 token_type_ids = np.zeros(( tot_tw, max_len), dtype='int32')
 start_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 end_mask = np.zeros(( tot_tw, max_len), dtype='int32')
 for i in range(tot_tw):
set1 = " "+" ".join(train_df.loc[i,'text'].split())
 set2 = " ".join(train_df.loc[i,'selected_text'].split())
 idx = set1.find(set2)
 set2_loc = np.zeros(( len(set1)))
 set2_loc[idx:idx+len(set2)]=1
if set1[idx-1]==" ":
set2_loc[idx-1]=1
enc_set1 = tokenizer.encode(set1)
 selected_text_token_idx=[]
for k,(a,b)in enumerate(enc_set1.offsets):
sm = np.sum(set2_loc[a:b])
 if sm > 0:
selected_text_token_idx.append(k)
 senti_token = sentiment_id[train_df.loc[i,'sentiment']]
input_ids[i,:len(enc_set1.ids)+5] = [0]+enc_set1.ids+[2,2]+[senti_token]+[2]
attention_mask[i,:len(enc_set1.ids)+5]=1
if len(selected_text_token_idx)> 0:
start_mask[i,selected_text_token_idx[0]+1]=1
end_mask[i, selected_text_token_idx[-1]+1]=1
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * nets
epochs = 20
for j in range(nets):
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=66),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//66,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(j+1,epochs,max(history[j].history['accuracy']),max(history[j].history['val_accuracy'])) )<save_to_csv>
tot_test_tw = test_df.shape[0]
input_ids_t = np.ones(( tot_test_tw,max_len), dtype='int32')
 attention_mask_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 token_type_ids_t = np.zeros(( tot_test_tw,max_len), dtype='int32')
 for i in range(tot_test_tw):
set1 = " "+" ".join(test_df.loc[i,'text'].split())
 enc_set1 = tokenizer.encode(set1)
 s_token = sentiment_id[test_df.loc[i,'sentiment']]
input_ids_t[i,:len(enc_set1.ids)+5]=[0]+enc_set1.ids+[2,2]+[s_token]+[2]
attention_mask_t[i,:len(enc_set1.ids)+5]=1
results = np.zeros(( X_test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission_digit.csv",index=False )<set_options>
def build_model() :
ids = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 att = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( max_len,), dtype=tf.int32)
 config_path = RobertaConfig.from_pretrained('/kaggle/input/prerobertabase/config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained('/kaggle/input/prerobertabase/pretrained-roberta-base.h5', config=config_path)
 x = bert_model(ids, attention_mask = att, token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.LSTM(1024, return_sequences=True )(x1)
 x1 = tf.keras.layers.Conv1D(128,2, padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(16,2, padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.LSTM(1024, return_sequences=True )(x2)
 x2 = tf.keras.layers.Conv1D(128,2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(16,2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids,att,tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model
sns.set(style="ticks", context="talk")
<load_from_csv>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
<sort_values><EOS> 
jac =[]; DISPLAY=1
preds_start= np.zeros(( input_ids_t.shape[0],max_len))
preds_end= np.zeros(( input_ids_t.shape[0],max_len))
model = build_model()
print('Loading model.... ')
 model.load_weights('/kaggle/input/preroberta8/pre-roberta-v10.h5')
 print('-'*15)
 print('- RESULT -')
 print('-'*15)
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start = preds[0]
preds_end = preds[1]
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_df.loc[k,'text']
else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st)
 test_df['selected_text']=all
test_df[['textID','selected_text']].to_csv('submission.csv', index=False )