Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<set_options>
!pip install textstat
filterwarnings('ignore' )<categorify>
from sklearn.feature_extraction.text import CountVectorizer
from nltk.stem import WordNetLemmatizer,PorterStemmer
from collections import defaultdict,Counter
from wordcloud import WordCloud, STOPWORDS
from plotly.subplots import make_subplots
from nltk.tokenize import word_tokenize
import plotly.figure_factory as ff
from nltk.corpus import stopwords
from nltk.corpus import stopwords
import plotly.graph_objects as go
from textblob import TextBlob
import plotly.graph_objs as go
import matplotlib.pyplot as plt
from nltk.util import ngrams
import plotly.offline as py
import plotly.express as px
from statistics import *
from plotly import tools
import seaborn as sns
from tqdm import tqdm
import pandas as pd
import numpy as np
import textstat
import string
import json
import nltk
import gc
cLEncoder = LabelEncoder()
sLEncoder = LabelEncoder()
df.loc[:, 'Country'] = cLEncoder.fit_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.fit_transform(df.loc[:, 'State'] )<data_type_conversions>
py.init_notebook_mode(connected=True)
 nltk.download('stopwords')
 stop=set(stopwords.words('english'))
plt.style.use('seaborn')
X_Train = df[df.Date <= df_Train.Date.max() ]
X_Train.loc[:, 'Date_Int'] =(X_Train.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 yC_Train = X_Train.CC_1
yF_Train = X_Train.FC_1
X_Train = X_Train.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'])
 print(X_Train.shape, yC_Train.shape, yF_Train.shape )<choose_model_class>
train=pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 test=pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 target=train['sentiment']
cLGBMR = LGBMRegressor(num_leaves=85,learning_rate=10**-1.89,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.1),min_child_samples=2,subsample=0.97,subsample_freq=10,
colsample_bytree=0.68,reg_lambda=10**1.4,random_state=7,n_jobs=-1)
 fLGBMR = LGBMRegressor(num_leaves=26,learning_rate=10**-1.63,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.04),min_child_samples=14,subsample=0.66,subsample_freq=5,
colsample_bytree=0.8,reg_lambda=10**1.92,random_state=7,n_jobs=-1 )<train_model>
def basic_cleaning(text):
text=re.sub(r'https?://www\.\S+\.com','',text)
 text=re.sub(r'[^A-Za-z|\s]','',text)
 return text
def clean(df):
for col in ['text','selected_text']:
df[col]=df[col].astype(str ).apply(lambda x:basic_cleaning(x))
return df
colors=['blue','green','red']
sent=train.sentiment.unique()
cLGBMR.fit(X_Train, yC_Train, categorical_feature=['Country','State'])
 fLGBMR.fit(X_Train, yF_Train, categorical_feature=['Country','State'] )<data_type_conversions>
df=pd.concat([train,test])
 df['text']=df['text'].astype(str)
 df['seleceted_text']=df['selected_text'].astype(str )
X_Test = df[df.Date >= df_Test.Date.min() ]
X_Test.loc[:, 'Date_Int'] =(X_Test.loc[:, 'Date'].dt.strftime("%m%d")).astype('int16')
 X_Test.drop(columns=['Id', 'ForecastId', 'Date', 'ConfirmedCases', 'Fatalities', 'CC_1', 'FC_1'], axis=1, inplace=True )<predict_on_test>
def preprocess_news(df,stop=stop,n=1,col='text'):

 new_corpus=[]
stem=PorterStemmer()
lem=WordNetLemmatizer()
for text in df[col]:
words=[w for w in word_tokenize(text)if(w not in stop)]
words=[lem.lemmatize(w)for w in words if(len(w)>n)]
new_corpus.append(words)
 new_corpus=[word for l in new_corpus for word in l]
return new_corpus
nIds = range(X_Train.CSId.max() +1, X_Test.CSId.max() +1)
 yC_Pred = []
yF_Pred = []
for nId in nIds:
df = getDaysShift(df)
 df = getChangeGrowth(df)
 df = getMA(df)
 df = getCDSC(df)
 df = getFDSC(df)
 df.loc[df.CSId == nId, 'CC_1'] = cLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LConfirmedCases'] = df.loc[df.CSId == nId, 'CC_1'] + df.loc[df.CSId == nId, 'CD_1']
df.loc[df.CSId == nId,'ConfirmedCases'] = np.exp(df.loc[df.CSId == nId,'LConfirmedCases'])- 1
df.loc[df.CSId == nId, 'FC_1'] = fLGBMR.predict(X_Test[X_Test.CSId == nId])
 df.loc[df.CSId == nId,'LFatalities'] = df.loc[df.CSId == nId, 'FC_1'] + df.loc[df.CSId == nId, 'FD_1']
df.loc[df.CSId == nId,'Fatalities'] = np.exp(df.loc[df.CSId == nId,'LFatalities'])- 1<categorify>
df=clean(df )
df.loc[:, 'Country'] = cLEncoder.inverse_transform(df.loc[:, 'Country'])
 df.loc[:, 'State'] = sLEncoder.inverse_transform(df.loc[:, 'State'] )<filter>
def get_top_ngram(corpus, n=None):
vec = CountVectorizer(ngram_range=(n, n),stop_words=stop ).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:20]
df[df.Country =='Italy'][['ConfirmedCases', 'CC_1', 'Fatalities', 'FC_1', 'CSId']][75:99]<filter>
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
df_out = df.loc[df.ForecastId > 0, ['ForecastId', 'ConfirmedCases', 'Fatalities']]<data_type_conversions>
train['jaccard']=train.apply(lambda x : jaccard(x.text,x.selected_text),axis=1)
 positive=train[(train['sentiment']=='positive')&(train['jaccard']>0.9)]
df_out.ForecastId = df_out.ForecastId.astype('int' )<define_search_space>
from transformers import *
import tensorflow as tf
import tokenizers
df_out.iloc[np.r_[1, 42, 45, 97, 143, 175, 267, 327, 350, 420, 450, 540, 590, 680, 730, 2880, 2900, 2960, 3000, 3050, 7000, 7500, 8000, 8500, 9000, 9500, 10000, 10500, 11000, 11500, 12000, 12500, 13000, -1], :]<save_to_csv>
MAXLEN=128
PATH = '.. /input/tf-roberta/'
tokenizer=tokenizers.ByteLevelBPETokenizer(vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
df_out.to_csv('submission.csv', index=False )<import_modules>
ct=train.shape[0]
input_ids=np.ones(( ct,MAXLEN),dtype='int32')
 attention_mask=np.zeros(( ct,MAXLEN),dtype='int32')
 token_type_ids=np.zeros(( ct,MAXLEN),dtype='int32')
 start_tokens=np.zeros(( ct,MAXLEN),dtype='int32')
 end_tokens=np.zeros(( ct,MAXLEN),dtype='int32')
 for k in tqdm(range(ct)) :
text1=" "+" ".join(train.loc[k,"text"].split())
 text2=" ".join(train.loc[k,"selected_text"].split())
 idx=text1.find(text2)
 chars=np.zeros(len(text1))
chars[idx:idx+len(text2)]=1
if(text1[idx-1]==" "):
chars[idx-1]=1
enc=tokenizer.encode(text1)
 offsets=enc.offsets
toks=[]
for i,(a,b)in enumerate(offsets):
sm=np.sum(chars[a:b])
 if sm > 0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt<load_from_csv>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAXLEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAXLEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAXLEN),dtype='int32')
 for k in tqdm(range(test.shape[0])) :
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
df = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 df.shape<data_type_conversions>
def build_model() :
ids = tf.keras.layers.Input(( MAXLEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAXLEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAXLEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model
train_data = train_data.to_numpy()
train_labels = train_labels.to_numpy()<feature_engineering>
model=build_model()
train_data = train_data / 255<choose_model_class>
models=os.listdir(".. /input/tfroberta5fold6epochs")
 model_file_path=".. /input/tfroberta5fold6epochs/"
filters = 64
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=4*filters, kernel_size=(4,4), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
 ])
 model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'] )<train_model>
start_token=np.zeros(( ct,MAXLEN),dtype='float64')
 end_token=np.zeros(( ct,MAXLEN),dtype='float64')
 for model_path in models:
model.load_weights(model_file_path+model_path)
 pred=model.predict([input_ids_t,attention_mask_t,token_type_ids_t])
 start_token+=pred[0]/len(models)
 end_token+=pred[1]/len(models)
history = model.fit(train_data, train_labels, epochs=40, batch_size=32, verbose=0 )<load_from_csv>
start_token=np.argmax(start_token,axis=1)
 end_token=np.argmax(end_token,axis=1)
 selected_text=[]
for k in range(ct):
a=start_token[k]
b=end_token[k]
if a<b:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 selected_text.append(tokenizer.decode(enc.ids[a-1:b]))
else:
if test.loc[k,'sentiment']=="neutral":
selected_text.append(test.loc[k,'text'])
 else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 selected_text.append(tokenizer.decode(enc.ids[b-1:a]))
df_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 df_test.shape<data_type_conversions>
test['selected_text']=selected_text
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.head(5 )
test_data = df_test.to_numpy()
test_data = test_data / 255
test_data = test_data.reshape(( test_data.shape[0], 28, 28, 1))
test_data.shape<predict_on_test>
del model
gc.collect()
predictions = model.predict(test_data)
 predictions = np.asarray([np.argmax(prediction)for prediction in predictions])
 predictions.shape<prepare_output>
cols=['textID','text','sentiment','selected_text']
train_df=train[cols].copy()
del train
test_df=test.copy()
del test
gc.collect()
df_predictions = pd.DataFrame(predictions ).rename(columns={0: "Label"})
 df_predictions.index.names = ['ImageId']
df_predictions.index += 1
df_predictions.head()<save_to_csv>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation
from tensorflow.keras.layers import Bidirectional, GlobalMaxPool1D,Bidirectional
from tensorflow.keras.models import Model
from tensorflow.keras.layers import TimeDistributed
from tensorflow.keras.layers import concatenate
from tensorflow.compat.v1.keras.layers import CuDNNLSTM
import gc
import os
df_predictions.shape
df_predictions.to_csv("predictions.csv" )<import_modules>
train_df=pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 test_df=pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv" )
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt<load_from_csv>
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=128):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )
df = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 df.shape<data_type_conversions>
tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
 save_path = '/kaggle/working/distilbert_base_uncased/'
if not os.path.exists(save_path):
os.makedirs(save_path)
 tokenizer.save_pretrained(save_path)
 fast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=True)
 fast_tokenizer
train_data = train_data.to_numpy()
train_labels = train_labels.to_numpy()<feature_engineering>
x_train = fast_encode(train_df.text.astype(str), fast_tokenizer, maxlen=128)
 x_test = fast_encode(test_df.text.astype(str),fast_tokenizer,maxlen=128 )
train_data = train_data / 255<choose_model_class>
transformer_layer = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased' )
filters = 64
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(3,3), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=2*filters, kernel_size=(5,5), strides=2, padding='same', activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Conv2D(filters=4*filters, kernel_size=(4,4), activation=tf.nn.relu),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
 ])
 model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'] )<train_model>
def create_targets(df):
df['t_text'] = df['text'].apply(lambda x: tokenizer.tokenize(str(x)))
 df['t_selected_text'] = df['selected_text'].apply(lambda x: tokenizer.tokenize(str(x)))
 def func(row):
x,y = row['t_text'],row['t_selected_text'][:]
for offset in range(len(x)) :
d = dict(zip(x[offset:],y))
check = [k==v for k,v in d.items() ]
if all(check)== True:
break
return [0]*offset + [1]*len(y)+ [0]*(len(x)-offset-len(y))
df['targets'] = df.apply(func,axis=1)
 return df
train_df = create_targets(train_df)
 print('MAX_SEQ_LENGTH_TEXT', max(train_df['t_text'].apply(len)))
 print('MAX_TARGET_LENGTH',max(train_df['targets'].apply(len)))
 MAX_TARGET_LEN=108
history = model.fit(train_data, train_labels, epochs=40, batch_size=32, verbose=0 )<load_from_csv>
train_df['targets'] = train_df['targets'].apply(lambda x :x + [0] *(MAX_TARGET_LEN-len(x)))
 targets=np.asarray(train_df['targets'].values.tolist())
df_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 df_test.shape<data_type_conversions>
lb=LabelEncoder()
sent_train=lb.fit_transform(train_df['sentiment'])
 sent_test=lb.fit_transform(test_df['sentiment'] )
test_data = df_test.to_numpy()
test_data = test_data / 255
test_data = test_data.reshape(( test_data.shape[0], 28, 28, 1))
test_data.shape<predict_on_test>
def new_model(transformer_layer):
inp = Input(shape=(128,))
inp2= Input(shape=(1,))
embedding_matrix=transformer_layer.weights[0].numpy()
x = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],weights=[embedding_matrix],trainable=False )(inp)
 x = CuDNNLSTM(150, return_sequences=True,name='lstm_layer', )(x)
 x = CuDNNLSTM(100, return_sequences=False,name='lstm_layer-2', )(x)
 y =Dense(10,activation='relu' )(inp2)
 x= concatenate([x,y])
 x = Dense(MAX_TARGET_LEN,activation='sigmoid' )(x)
 model = Model(inputs=[inp,inp2], outputs=x)
 model.compile(loss='binary_crossentropy',
optimizer='adam')
 return model
predictions = model.predict(test_data)
 predictions = np.asarray([np.argmax(prediction)for prediction in predictions])
 predictions.shape<prepare_output>
model=new_model(transformer_layer)
 history=model.fit([x_train,sent_train],targets,epochs=3 )
df_predictions = pd.DataFrame(predictions ).rename(columns={0: "Label"})
 df_predictions.index.names = ['ImageId']
df_predictions.index += 1
df_predictions.head()<save_to_csv>
predictions=model.predict([x_test,sent_test] )
df_predictions.shape
df_predictions.to_csv("predictions.csv" )<load_from_csv>
def convert_output(sub,predictions):
preds=[]
for i,row in enumerate(sub['text']):
text,target=row.lower() ,predictions[i].tolist()
target=np.round(target ).tolist()
try:
start,end=target.index(1),target[::-1].index(1)
 text_list=tokenizer.tokenize(text)
 text_list=text_list+(( 108-len(text_list)) *['pad'])
 start_w,end_w=text_list[start],text_list[-end]
start=text.find(start_w.replace("
end=text.find(end_w.replace("
pred=text[start:end]
except:
pred=text
preds.append(pred)
 return preds
<load_from_csv><EOS> 
prediction_text=convert_output(test_df,predictions )