Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<load_from_csv>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
Actual_1 = 'My life is totally awesome'
Predict_1 = 'awesome life'
Actual_2 = 'We are active kagglers'
Predict_2 = 'We are active kagglers'
print("Jaccard score for first set of scentences: {}".format(jaccard(Actual_1,Predict_1)))
 print("Jaccard score for second set of scentences: {}".format(jaccard(Actual_2,Predict_2)) )
df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 df_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<prepare_x_and_y>
stop=set(stopwords.words('english'))
warnings.filterwarnings("ignore")
 cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='pearl')
x_train = np.array(df_train.iloc[:,1:])
 x_train = np.array([np.reshape(i,(28, 28, 1)) for i in x_train])
 y_train = np.array(df_train.iloc[:,0] )<normalization>
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 train.sample(6 )
x_train = x_train/255.0
y_train = keras.utils.to_categorical(y_train )<prepare_x_and_y>
def miss_val(df):
total=df.isnull().sum()
return pd.concat([total],axis=1,keys=['Total'])
 print("Missing values for train dataset
")
 print(miss_val(train))
print("---------------------------------------------------------------------")
 print("Missing values for test dataset
")
 print(miss_val(test))
x_test = np.array(df_test)
 x_test = np.array([np.reshape(i,(28, 28, 1)) for i in x_test])
 x_test = x_test/255.0<split>
train=train.dropna()
train.shape
X_train, X_test, Y_train, Y_test = train_test_split(x_train, y_train, test_size=0.2, stratify=y_train )<choose_model_class>
train['target']=train['selected_text'].str.lower()
model = keras.models.Sequential()
model.add(keras.layers.Conv2D(filters=32, kernel_size=(3,3), kernel_initializer='random_uniform', padding='same', activation='relu', input_shape=(X_train.shape[1:])))
 model.add(keras.layers.Conv2D(filters=32, kernel_size=(3,3), kernel_initializer='random_uniform', padding='same', activation='relu'))
model.add(keras.layers.MaxPool2D(pool_size=(2,2)))
 model.add(keras.layers.Conv2D(filters=64, kernel_size=(5,5), kernel_initializer='random_uniform', padding='same', activation='relu'))
model.add(keras.layers.Conv2D(filters=64, kernel_size=(5,5), kernel_initializer='random_uniform', padding='same', activation='relu'))
model.add(keras.layers.MaxPool2D(pool_size=(2,2)))
 model.add(keras.layers.Conv2D(filters=128, kernel_size=(7,7), kernel_initializer='random_uniform', padding='same', activation='relu'))
model.add(keras.layers.Conv2D(filters=128, kernel_size=(7,7), kernel_initializer='random_uniform', padding='same', activation='relu'))
model.add(keras.layers.MaxPool2D(pool_size=(3,3)))
 model.add(keras.layers.Conv2D(filters=256, kernel_size=(7,7), padding='same'))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Activation('relu'))
model.add(keras.layers.Dense(units=256, activation='relu'))
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Dense(units=100, activation='relu'))
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Flatten())
 model.add(keras.layers.Dense(units=y_train.shape[1], activation='softmax'))
print(model.summary())
 model.compile(loss='categorical_crossentropy',optimizer='Adam', metrics=['accuracy'] )<choose_model_class>
def find_link(string):
url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', string)
 return "".join(url)
 train['target_url']=train['target'].apply(lambda x: find_link(x))
df=pd.DataFrame(train.loc[train['target_url']!=""]['sentiment'].value_counts() ).reset_index()
df.rename(columns={"index": "sentiment", "sentiment": "url_count"} )
es = EarlyStopping(monitor='loss', mode='min', verbose=1, patience=5)
 filepath = "model.h5"
ckpt = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')
 rlp = ReduceLROnPlateau(monitor='loss', patience=2, factor=0.2 )<train_model>
def find_punct(text):
line = re.findall(r'[!"\$%&'() *+,\-.\/:;=
string="".join(line)
 return list(string )
history = model.fit(X_train, Y_train, batch_size=500, callbacks=[es, ckpt, rlp], epochs=100, validation_data=(X_test,Y_test))<predict_on_test>
train['target_punct']=train['target'].apply(lambda x:find_punct(x))
train['target_punct_len']=train['target'].apply(lambda x:len(find_punct(x)) )
id_img = []
label = []
for i in range(len(x_test)) :
id_img.append(i+1)
 label.append(np.argmax(model.predict(x_test[i:i+1])))
 img_id = np.array(id_img)
 label = np.array(label )<save_to_csv>
punc_df=pd.DataFrame(train,columns=['target_punct','sentiment'])
 punc_df=punc_df[punc_df['target_punct'].map(lambda d: len(d)) > 0]
punc_df=punc_df.explode('target_punct')
 positive_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="positive"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'pos_punct'})
 negative_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="negative"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neg_punct'})
 neutral_df=pd.DataFrame(punc_df.loc[punc_df['sentiment']=="neutral"]['target_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','target_punct':'neut_punct'})
op_df = pd.DataFrame()
op_df['ImageId'] = img_id
op_df['Label'] = label
op_df.to_csv("submission.csv", index=False )<set_options>
train['star']=train['target'].apply(lambda x:find_star(x))
train.loc[train['star']!=0]['sentiment'].value_counts().to_frame()
%matplotlib inline
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
warnings.filterwarnings("ignore")
 assert 'house-prices-advanced-regression-techniques' in os.listdir('.. /input/'), 'House Prices dataset not loaded!'<load_from_csv>
def find_only_star(text):
if len(text.split())==1:
line=re.findall(r'[*]{2,5}',text)
 return len(line)
 else:
return 0
HOME_FOLDER = '.. /input/house-prices-advanced-regression-techniques/'
df_train = pd.read_csv(HOME_FOLDER + 'train.csv', index_col='Id')
 df_test = pd.read_csv(HOME_FOLDER + 'test.csv', index_col='Id' )<prepare_x_and_y>
train['only_star']=train['target'].apply(lambda x:find_only_star(x))
train.loc[train['only_star']==1]['sentiment'].value_counts()
n_train, n_test = df_train.shape[0], df_test.shape[0]
df_full = pd.concat(( df_train, df_test), sort=False ).reset_index(drop=True)
 y_train = df_train['SalePrice']
df_full.drop(['SalePrice'], axis=1, inplace=True )<create_dataframe>
train['target']= np.where(train['only_star']==1,"abusive",train['target'] )
missing_ratio =(df_full.isnull().sum() / df_full.shape[0] * 100 ).sort_values(ascending=False)
 missing_ratio.drop(missing_ratio[missing_ratio == 0].index, inplace=True)
 missing_values = pd.DataFrame(missing_ratio, columns=['Missing Ratio'])
 print(missing_values )<data_type_conversions>
train['exclaim']=train['target'].apply(lambda x:find_exclam(x))
train.loc[train['exclaim']!=0]['sentiment'].value_counts().to_frame()
df_full['PoolQC'].fillna('None', inplace=True )<data_type_conversions>
def remove_link(string):
text = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +'," ",string)
 return " ".join(text.split() )
df_full['MiscFeature'].fillna('None', inplace=True )<data_type_conversions>
def remove_punct(text):
line = re.sub(r'[!"\$%&'() *+,\-.\/:;=
return " ".join(line.split() )
cols = ['Alley', 'Fence', 'FireplaceQu']
df_full[cols] = df_full[cols].fillna('None', inplace=False )<groupby>
train['target']=train['target'].apply(lambda x:remove_link(x))
train['target']=train['target'].apply(lambda x:remove_punct(x))
df_full['LotFrontage'] = df_full.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
def create_corpus(data,feature,sentiment):
corpus=[]
for x in data[data['sentiment']==sentiment][feature].str.split() :
for i in x:
corpus.append(i)
 return corpus
cols = ['GarageCond', 'GarageQual', 'GarageFinish', 'GarageType']
df_full[cols] = df_full[cols].fillna('None' )<data_type_conversions>
def corpus_sentiment(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word not in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y
cols = ['GarageYrBlt', 'GarageArea', 'GarageCars']
df_full[cols] = df_full[cols].fillna(0 )<data_type_conversions>
def count_values(df,feature):
total=df.loc[:,feature].value_counts(dropna=False)
 percent=round(df.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2)
 return pd.concat([total,percent],axis=1,keys=['Total','Percent'])
cols = ['BsmtCond', 'BsmtExposure', 'BsmtQual', 'BsmtFinType2', 'BsmtFinType1']
df_full[cols] = df_full[cols].fillna('None' )<data_type_conversions>
count_values(train,'sentiment' )
cols = ['BsmtHalfBath', 'BsmtFullBath', 'BsmtFinSF2', 'BsmtFinSF1', 'BsmtUnfSF', 'TotalBsmtSF']
df_full[cols] = df_full[cols].fillna(0 )<count_missing_values>
count_values(train,'sentiment' )
print(df_full[['MasVnrType', 'MasVnrArea']].loc[df_full['MasVnrType'].isnull() & df_full['MasVnrArea'].notnull() ])
 print('
Houses with price between 196.000 and 200.000:')
 print(df_full[['MasVnrType']].loc[(df_full['MasVnrArea'] <= 200.000)&(df_full['MasVnrArea'] >= 196.000)].squeeze().value_counts() )<feature_engineering>
full_data=pd.concat([train,test])
 full_data['text']=full_data['text'].str.lower()
full_data.shape
mask =(df_full['MasVnrType'].isnull())&(df_full['MasVnrArea'].notnull())
 df_full.loc[mask, 'MasVnrType'] = df_full.loc[mask, 'MasVnrType'].fillna('None')
 df_full['MasVnrType'].fillna('None', inplace=True)
 df_full['MasVnrArea'].fillna(0, inplace=True )<data_type_conversions>
full_data['text_punct']=full_data['text'].apply(lambda x:find_punct(x))
full_data['text_punct_len']=full_data['text'].apply(lambda x:len(find_punct(x)) )
df_full['MSZoning'].fillna(df_full['MSZoning'].mode() [0], inplace=True )<drop_column>
punc_text_df=pd.DataFrame(full_data,columns=['text_punct','sentiment'])
 punc_text_df=punc_text_df[punc_text_df['text_punct'].map(lambda d: len(d)) > 0]
punc_text_df=punc_text_df.explode('text_punct')
 positive_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="positive"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'pos_punct'})
 negative_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="negative"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neg_punct'})
 neutral_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="neutral"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neut_punct'})
df_full.drop(['Utilities'], axis=1, inplace=True )<rename_columns>
full_data['text']=full_data['text'].apply(lambda x:remove_link(x))
full_data['text']=full_data['text'].apply(lambda x:remove_punct(x))
df_full['Functional'].fillna('Typ', inplace=True )<sort_values>
full_data.loc[full_data['text']=="",['text']]="nothing"
assert df_full.isnull().sum().sort_values(ascending=False)[0] == 0, 'Mmm, there are still missing values! Check again.'<feature_engineering>
def corpus_sentiment_stop(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y
df_full['MSSubClass'] = df_full['MSSubClass'].apply(str)
 df_full['OverallCond'] = df_full['OverallCond'].apply(str)
 def mapper(month):
date = datetime.datetime(2000, month, 1)
 return date.strftime('%b')
 df_full['MoSold'] = df_full['MoSold'].apply(mapper )<categorify>
train_word=full_data[:train.shape[0]]
test_word=full_data[train.shape[0]:]
cols = ['MSSubClass', 'Street', 'Alley','LandSlope', 'OverallQual', 'OverallCond',
'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'HeatingQC', 'CentralAir', 'KitchenQual', 'Functional',
'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive',
'PoolQC', 'Fence', 'YrSold', 'MoSold']
for c in cols:
le = LabelEncoder()
le.fit(list(df_full[c].values))
df_full[c] = le.transform(list(df_full[c].values))<feature_engineering>
pos_text_list=list(corpus_sentiment(train_word,'text','positive')[0])
 pos_target_list=list(corpus_sentiment(train_word,'target','positive')[0])
 neg_text_list=list(corpus_sentiment(train_word,'text','negative')[0])
 neg_target_list=list(corpus_sentiment(train_word,'target','negative')[0])
 neutral_text_list=list(corpus_sentiment(train_word,'text','neutral')[0])
 neutral_target_list=list(corpus_sentiment(train_word,'target','neutral')[0])
 pos_common_words_list=list(set(pos_text_list ).intersection(pos_target_list))
neg_common_words_list=list(set(neg_text_list ).intersection(neg_target_list))
neutral_common_words_list=list(set(neutral_text_list ).intersection(neutral_target_list))
skewness = skewness[abs(skewness['Skew'])> 0.75]
print(f'There are {skewness.shape[0]} skewed numerical features to Box Cox transform.')
 lambda_param = 0.15
for feature in skewness.index:
df_full[feature] = boxcox1p(df_full[feature], lambda_param )<categorify>
results_jaccard=[]
for ind,row in train_word.iterrows() :
sentence1 = row.text
sentence2 = row.target
jaccard_score = jaccard(sentence1,sentence2)
 results_jaccard.append([sentence1,sentence2,jaccard_score] )
df_full = pd.get_dummies(df_full)
 print(f'Now there are {df_full.shape[1]} features.' )<prepare_x_and_y>
jaccard = pd.DataFrame(results_jaccard,columns=["text","target","jaccard_score"])
 train_word = train_word.merge(jaccard,how='outer' )
X_train = df_full[:n_train]
X_test = df_full[n_train:]<import_modules>
print("Jaccard score equals 1 : {}%".format(round(train_word.loc[train_word['jaccard_score']==1].shape[0]/train_word.shape[0]*100,2)))
 print("Jaccard score less than 0.3 : {}%".format(round(train_word.loc[train_word['jaccard_score']<0.3].shape[0]/train_word.shape[0]*100,2)) )
from sklearn.model_selection import KFold, cross_val_score
from xgboost import XGBRegressor<compute_train_metric>
train_word['difference']=abs(train_word['text_tweet_length']-train_word['target_tweet_length'] )
def model_validation(model, X, y, n_folds=5):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(X.values)
 score = -cross_val_score(model, X.values, y, scoring='neg_mean_squared_error', cv=kf)
 return np.sqrt(score )<choose_model_class>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
model_xgb = XGBRegressor(colsample_bytree=0.8, subsample=0.5,
learning_rate=0.05, max_depth=3,
min_child_weight=1.8, n_estimators=2000,
reg_alpha=0.1, reg_lambda=0.8, gamma=0.01,
silent=1, random_state=7, n_jobs=-1,
early_stopping_rounds=10)
 score = model_validation(model_xgb, X_train, y_train)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 ct=train.shape[0]
input_ids=np.ones(( ct,MAX_LEN),dtype="int32")
 attention_mask=np.zeros(( ct,MAX_LEN),dtype="int32")
 token_type_ids=np.zeros(( ct,MAX_LEN),dtype="int32")
 start_tokens=np.zeros(( ct,MAX_LEN),dtype="int32")
 end_tokens=np.zeros(( ct,MAX_LEN),dtype="int32" )
model_xgb.fit(X_train, y_train)
 predictions = np.expm1(model_xgb.predict(X_test))<save_to_csv>
for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
output = pd.DataFrame({'Id': df_test.index,
'SalePrice': predictions})
 output.to_csv('submission.csv', index=False )<load_from_csv>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct_test = test.shape[0]
input_ids_t = np.ones(( ct_test,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )<groupby>
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model
def missing_percentage(df):

 total = df.isnull().sum().sort_values(ascending = False)
 percent = round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)
 return pd.concat([total, percent], axis=1, keys=['Total','Percent'] )<count_missing_values>
%%time
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
%timeit -r2 -n10 missing_percentage(train )<count_missing_values>
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )
<count_missing_values><EOS> 
test['selected_text'] = all
submission=test[['textID','selected_text']]
submission.to_csv('submission.csv',index=False)
 submission.head(5 )