Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<categorify>
warnings.filterwarnings('ignore' )
df_all['Ticket_Frequency'] = df_all.groupby('Ticket')['Ticket'].transform('count' )<feature_engineering>
!cp -rp /kaggle/input/pyspellchecker-and-ekpharasis-libs-for-python /kaggle/working/
!cp -rp /kaggle/input/transformers280 /kaggle/working/
!pip install /kaggle/working/pyspellchecker-and-ekpharasis-libs-for-python/pyspellchecker-0.5.4-py2.py3-none-any.whl >> /dev/null
!pip install /kaggle/working/pyspellchecker-and-ekpharasis-libs-for-python/ekphrasis-0.5.1/ekphrasis-0.5.1/ >> /dev/null
!python /kaggle/working/pyspellchecker-and-ekpharasis-libs-for-python/ekphrasis-0.5.1/ekphrasis-0.5.1/setup.py install >> /dev/null
!mkdir -p /root/.ekphrasis/stats
!cp -r /kaggle/working/pyspellchecker-and-ekpharasis-libs-for-python/stats/* /root/.ekphrasis/stats >> /dev/null
!pip install /kaggle/working/transformers280/transformers-2.8.0 >> /dev/null
!rm -rf /kaggle/working/*
df_all['Title'] = df_all['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1<string_transform>
in_testing_mode = True
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
df_all['Family'] = extract_surname(df_all['Name'])
 df_train = df_all.loc[:890]
df_test = df_all.loc[891:]
dfs = [df_train, df_test]<define_variables>
import numpy as np
import pandas as pd
import random
import copy
import os
import re
from nltk.tokenize import SpaceTokenizer
from ekphrasis.classes.preprocessor import TextPreProcessor
from ekphrasis.classes.tokenizer import SocialTokenizer
from ekphrasis.dicts.emoticons import emoticons
from spellchecker import SpellChecker
import torch
import torch.nn as nn
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from transformers import RobertaConfig, RobertaModel, BertPreTrainedModel, RobertaTokenizer, AdamW, get_linear_schedule_with_warmup
import datetime, time
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA<feature_engineering>
input_dir = '/kaggle/input'
working_dir = '/kaggle/working'
data_dir = os.path.join(input_dir, "tweet-sentiment-extraction/")
 model_dir = os.path.join(input_dir, "robertabase/")
 train_file = os.path.join(data_dir, "train.csv")
 test_file = os.path.join(data_dir, "test.csv")
 sample_submission_file = os.path.join(data_dir, "sample_submission.csv")
 submission_file = os.path.join(working_dir, "submission.csv" )
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2<categorify>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )<categorify>
dfJaccards = copy.deepcopy(dfTrain)
 dfJaccards["jaccard"] = dfJaccards.apply(lambda df: jaccard(df.text, df.selected_text), axis=1)
 print("Ratios of selected text in text")
 dfJaccards = dfJaccards[['sentiment','jaccard']].groupby(['sentiment'] ).mean()
dfJaccards
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )<concatenate>
def stringtowords(text, selected_text):
text_words = str(text ).lower().split()
selected_text_words = str(selected_text ).lower().split()
selected_length = len(selected_text_words)
 start_char = "".join(text_words ).find("".join(selected_text_words))
if(start_char >= 0 and selected_length > 0):
remaining_char = start_char
for i,word in enumerate(text_words):
remaining_char-=len(word)
 if remaining_char < 0:
selected_text_words = text_words[i:i+selected_length]
break
text_word_numbers = [i for i in range(len(text_words)) ]
return text_words, text_word_numbers ,selected_text_words
df_all = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
df_all.drop(columns=drop_cols, inplace=True)
 df_all.head()<normalization>
test_string = "is back home now gonna miss every one"
test_selected_string = "onna mis"
print("Sample text:\t", test_string)
 print("Sample selected text:\t", test_selected_string)
 output_tweet_list, output_tweet_word_numbers, output_selected_list = stringtowords(test_string,test_selected_string)
 print("
stringtowords outputs")
 print("output tweet list: ",output_tweet_list)
 print("output tweet word numbers:",output_tweet_word_numbers)
 print("output selected text list: ",output_selected_list )
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))<choose_model_class>
dfTrain['text_w'], dfTrain['text_w_n'], dfTrain['selected_w'] = zip(*dfTrain.apply(lambda df: stringtowords(df.text,df.selected_text), axis=1))
dfTrain.head()
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
 leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1 )<train_model>
def fix_difference(old_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand_number):
new_word = [word for word in new_word if word != ""]
if len(new_word)== 0:
newtext_w.pop(word_ind+expand_number)
 if newtext_w_n is not None:
newtext_w_n.pop(word_ind+expand_number)
 expand_number -= 1
elif(len(new_word)> 1)|(new_word[0] != old_word):
for i, word in enumerate(new_word):
if i == 0:
newtext_w[word_ind+expand_number] = word
else:
expand_number += 1
newtext_w.insert(word_ind+expand_number,word)
 if newtext_w_n is not None:
newtext_w_n.insert(word_ind+expand_number,oldtext_w_n[word_ind])
 return newtext_w, newtext_w_n, expand_number
N = 5
oob = 0
probs = pd.DataFrame(np.zeros(( len(X_test), N * 2)) , columns=['Fold_{}_Prob_{}'.format(i, j)for i in range(1, N + 1)for j in range(2)])
 importances = pd.DataFrame(np.zeros(( X_train.shape[1], N)) , columns=['Fold_{}'.format(i)for i in range(1, N + 1)], index=df_all.columns)
 fprs, tprs, scores = [], [], []
skf = StratifiedKFold(n_splits=N, random_state=N, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(X_train, y_train), 1):
print('Fold {}
'.format(fold))
leaderboard_model.fit(X_train[trn_idx], y_train[trn_idx])
 trn_fpr, trn_tpr, trn_thresholds = roc_curve(y_train[trn_idx], leaderboard_model.predict_proba(X_train[trn_idx])[:, 1])
 trn_auc_score = auc(trn_fpr, trn_tpr)
 val_fpr, val_tpr, val_thresholds = roc_curve(y_train[val_idx], leaderboard_model.predict_proba(X_train[val_idx])[:, 1])
 val_auc_score = auc(val_fpr, val_tpr)
 scores.append(( trn_auc_score, val_auc_score))
fprs.append(val_fpr)
 tprs.append(val_tpr)
 probs.loc[:, 'Fold_{}_Prob_0'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 0]
probs.loc[:, 'Fold_{}_Prob_1'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 1]
importances.iloc[:, fold - 1] = leaderboard_model.feature_importances_
oob += leaderboard_model.oob_score_ / N
print('Fold {} OOB Score: {}
'.format(fold, leaderboard_model.oob_score_))
print('Average OOB Score: {}'.format(oob))<save_to_csv>
def cleanstring(oldtext_w, oldtext_w_n = None):
dictionary = {'w/':'with',
'dont':'do not',
'doesnt':'does not',
'didnt':'did not',
'cant':'can not',
'couldnt':'could not',
'wouldnt':'would not',
'shouldnt':'should not'
}
expand = 0
newtext_w = copy.deepcopy(oldtext_w)
 newtext_w_n = copy.deepcopy(oldtext_w_n)
 for word_ind ,current_word in enumerate(oldtext_w):
new_word = current_word
if current_word in dictionary:
new_word = dictionary[current_word]
new_word = new_word.split()
newtext_w, newtext_w_n, expand = fix_difference(current_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand)
 return newtext_w, newtext_w_n
class_survived = [col for col in probs.columns if col.endswith('Prob_1')]
probs['1'] = probs[class_survived].sum(axis=1)/ N
probs['0'] = probs.drop(columns=class_survived ).sum(axis=1)/ N
probs['pred'] = 0
pos = probs[probs['1'] >= 0.5].index
probs.loc[pos, 'pred'] = 1
y_pred = probs['pred'].astype(int)
 submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = y_pred.values
submission_df.to_csv('submissions.csv', header=True, index=False)
 submission_df.head(10 )<import_modules>
test_string = ["u","cant","seperate","it","w/","this"]
print("Before cleanstring")
 print(test_string)
 print("After cleanstring")
 print(cleanstring(test_string)[0] )
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import LabelEncoder<load_from_csv>
dfTrain['text_w'], dfTrain['text_w_n'] = zip(*dfTrain.apply(lambda df: cleanstring(df.text_w,df.text_w_n), axis=1))
dfTrain['selected_w'], _ = zip(*dfTrain.apply(lambda df: cleanstring(df.selected_w), axis=1))
print("===== E> Replaced unpunctuated texts with punctuated ones <3 =====" )
df_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 df_train.head(10 )<load_from_csv>
text_processor = TextPreProcessor(
 normalize=['url'],
annotate={},
fix_html=True,
segmenter="twitter",
corrector="twitter",
unpack_hashtags=False,
unpack_contractions=True,
spell_correct_elong=True,
tokenizer=SpaceTokenizer().tokenize,
dicts=[emoticons] 
)
 def tweetCleaner(oldtext_w, oldtext_w_n = None):
expand = 0
newtext_w = copy.deepcopy(oldtext_w)
 newtext_w_n = copy.deepcopy(oldtext_w_n)
 for word_ind ,current_word in enumerate(oldtext_w):
new_word = current_word
new_word = text_processor.pre_process_doc(current_word)
 for i,word in enumerate(new_word):
if "<" in word and ">" in word:
new_word[i] = re.sub("[<>]","",word)
 newtext_w, newtext_w_n, expand = fix_difference(current_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand)
 return newtext_w, newtext_w_n
df_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 df_test.head(10 )<count_values>
dfTrain['text_w'], dfTrain['text_w_n'] = zip(*dfTrain.apply(lambda df: tweetCleaner(df.text_w,df.text_w_n), axis=1))
dfTrain['selected_w'], _ = zip(*dfTrain.apply(lambda df: tweetCleaner(df.selected_w), axis=1))
print("===== E> Applied ekphrasis text preprocessor <3 =====" )
print('shape of the training data {}'.format(df_train.shape))
print('The number of passengers who survived {}'.format(np.array(df_train['Survived'].value_counts() [1])))
 print('The number of passengers who didn't survived {}'.format(np.array(df_train['Survived'].value_counts() [1])))
 print('There were {} men while {} women on Titanic'.format(df_train['Sex'].value_counts() [0], df_train['Sex'].value_counts() [1]))
print('Average age of passengers was {}'.format(df_train['Age'].mean()))
print('Average age of female passengers was {} while that of men was {}'.format(df_train.groupby('Sex')['Age'].mean() [0],\
df_train.groupby('Sex')['Age'].mean() [1]))
print()
print()
print("Checking for NULL values in each column")
 print(df_train.isnull().sum())
 df_train.groupby('Embarked')['Survived'].value_counts() 
<categorify>
def cleanstring2(oldtext_w, oldtext_w_n = None):
dictionary = {'ahah':'laugh',
'haha':'laugh',
'ahhah':'laugh',
'haaha':'laugh',
'hahha':'laugh',
'ahaah':'laugh',
'oxox':'kiss',
'xoxo':'kiss',
'xxox':'kiss',
'ooxo':'kiss'
}
expand = 0
newtext_w = copy.deepcopy(oldtext_w)
 newtext_w_n = copy.deepcopy(oldtext_w_n)
 for word_ind ,current_word in enumerate(oldtext_w):
new_word = current_word
for repeated in dictionary.keys() :
if repeated in current_word:
new_word = dictionary[repeated]
new_word = new_word.split()
newtext_w, newtext_w_n, expand = fix_difference(current_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand)
 return newtext_w, newtext_w_n
df_train['Age'].fillna(value= df_train['Age'].mean() , inplace=True)
 df_train['Embarked'].fillna(method='ffill', inplace=True)
 df_test['Age'].fillna(value= df_test['Age'].mean() , inplace=True)
 df_test['Fare'].fillna(value= df_test['Fare'].mean() , inplace=True)
 sex = LabelEncoder()
embark = LabelEncoder()
df_train['Sex'] = sex.fit_transform(df_train['Sex'])
 df_test['Sex'] = sex.fit_transform(df_test['Sex'])
 df_train['Embarked'] = embark.fit_transform(df_train['Embarked'])
 df_test['Embarked'] = embark.fit_transform(df_test['Embarked'])
 features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']
X_train = df_train[features]
y_train = df_train['Survived']
X_test = df_test[features]<save_to_csv>
test_string = ["ahhahahahaha","it","is","a","dummy","string","xoxoxooxoxoxo"]
print("Before cleanstring2")
 print(test_string)
 print("After cleanstring2")
 print(cleanstring2(test_string)[0] )
rfc = RandomForestClassifier(max_depth = 8 , n_estimators = 125, random_state= 4)
 rfc = rfc.fit(X_train, y_train)
 predict = rfc.predict(X_test)
 output = pd.DataFrame({'PassengerId': df_test.PassengerId, 'Survived': predict})
 output.to_csv('my_submission.csv', index=False )<import_modules>
dfTrain['text_w'], dfTrain['text_w_n'] = zip(*dfTrain.apply(lambda df: cleanstring2(df.text_w,df.text_w_n), axis=1))
dfTrain['selected_w'], _ = zip(*dfTrain.apply(lambda df: cleanstring2(df.selected_w), axis=1))
print("===== E> Some repeated structures replaced with their meanings <3 =====" )
from sklearn import preprocessing
from tensorflow.python import keras
from sklearn.impute import SimpleImputer
import tensorflow as tf
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Dropout<define_variables>
spell = SpellChecker(distance=1)
 def correctSpelling(oldtext_w, oldtext_w_n = None):
expand = 0
newtext_w = copy.deepcopy(oldtext_w)
 newtext_w_n = copy.deepcopy(oldtext_w_n)
 for word_ind ,current_word in enumerate(oldtext_w):
new_word = spell.correction(current_word)
 new_word = new_word.split()
newtext_w, newtext_w_n, expand = fix_difference(current_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand)
 return newtext_w, newtext_w_n
train_data = '/kaggle/input/titanic/train.csv'
test_data = "/kaggle/input/titanic/test.csv"<load_from_csv>
test_string = ["it","is","not","an","englidh","wotd"]
print("Before correctSpelling")
 print(test_string)
 print("After correctSpelling")
 print(correctSpelling(test_string)[0] )
train_panda = pd.read_csv(train_data, index_col = "PassengerId")
 train_panda.head()<load_from_csv>
dfTrain['text_w'], dfTrain['text_w_n'] = zip(*dfTrain.apply(lambda df: correctSpelling(df.text_w,df.text_w_n), axis=1))
dfTrain['selected_w'], _ = zip(*dfTrain.apply(lambda df: correctSpelling(df.selected_w), axis=1))
print("===== E> Fixed 1 char typos <3 =====" )
test_panda = pd.read_csv(test_data)
 test_panda.head()<categorify>
def addSpace(oldtext_w):
newtext_w = copy.deepcopy(oldtext_w)
 for word_ind ,current_word in enumerate(oldtext_w):
newtext_w[word_ind] = " "+current_word
return newtext_w
encoder = preprocessing.LabelEncoder()<define_variables>
dfTrain['text_w'] = dfTrain.apply(lambda df: addSpace(df.text_w), axis=1)
 dfTrain['selected_w'] = dfTrain.apply(lambda df: addSpace(df.selected_w), axis=1)
 print("===== E> Added spaces before every word <3 =====" )
cat_features = ["Sex", "Embarked"]<data_type_conversions>
dfTest['text_w'], dfTest['text_w_n'], _ = zip(*dfTest.apply(lambda df: stringtowords(df.text,""), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: cleanstring(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: tweetCleaner(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: cleanstring2(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: correctSpelling(df.text_w,df.text_w_n), axis=1))
dfTest['text_w'] = dfTest.apply(lambda df: addSpace(df.text_w), axis=1)
 dfTest.sample(n=5, random_state=50 )
train_panda["Embarked"] = train_panda["Embarked"].astype(str )<categorify>
print('Loading RoBERTa tokenizer...')
 tokenizer = RobertaTokenizer.from_pretrained(model_dir )
encoded_train = train_panda[cat_features].apply(encoder.fit_transform)
 encoded_test = test_panda[cat_features].apply(encoder.fit_transform )<define_variables>
def robertaTokenize(oldtext_w, oldtext_w_n = None):
expand = 0
newtext_w = copy.deepcopy(oldtext_w)
 newtext_w_n = copy.deepcopy(oldtext_w_n)
 for word_ind ,current_word in enumerate(oldtext_w):
new_word = tokenizer.tokenize(current_word)
 newtext_w, newtext_w_n, expand = fix_difference(current_word, new_word, oldtext_w_n, newtext_w, newtext_w_n, word_ind, expand)
 return newtext_w, newtext_w_n
num_features = ["Survived","Pclass","Age","SibSp","Parch","Fare"]
test_features= ["Pclass","Age","SibSp","Parch","Fare"]<merge>
test_string = [" hello"," worldasd"]
print("Before robertaTokenize")
 print(test_string)
 print("After robertaTokenize")
 print(robertaTokenize(test_string)[0] )
training_data = train_panda[num_features].join(encoded_train)
 test_data = test_panda[test_features].join(encoded_test )<count_missing_values>
dfTrain['text_w'], dfTrain['text_w_n'] = zip(*dfTrain.apply(lambda df: robertaTokenize(df.text_w,df.text_w_n), axis=1))
dfTrain['selected_w'], _ = zip(*dfTrain.apply(lambda df: robertaTokenize(df.selected_w), axis=1))
print("===== E> Tokenized every word using RoBERTa tokenization <3 =====" )
training_data.isnull().sum()<count_missing_values>
dfTest['text_w'], dfTest['text_w_n'] = zip(*dfTest.apply(lambda df: robertaTokenize(df.text_w,df.text_w_n), axis=1))
dfTest.sample(n=5, random_state=50 )
test_data.isnull().sum()<create_dataframe>
def find_sub_list(sub_list,super_list):
sub_list_length=len(sub_list)
 matching_list = []
found_start = 0
for start in range(0,sub_list_length):
for ind in(j for j,e in enumerate(super_list)if e==sub_list[start]):
found_start = ind-start
if sub_list_length < 2:
return [ind-start,ind-start+sub_list_length-1]
else:
for end in range(sub_list_length,start+1,-1):
if super_list[ind:ind+end-start]==sub_list[start:end]:
return [ind-start,ind-start+sub_list_length-1]
return [found_start, found_start+sub_list_length-1]
my_imputer = SimpleImputer()
imputed_train = pd.DataFrame(my_imputer.fit_transform(training_data))
imputed_test_data = pd.DataFrame(my_imputer.fit_transform(test_data))<rename_columns>
test_string = ["this","is","a","super","list"]
test_selected_string = ["r","list"]
print("super list")
 print(test_string)
 print("sub list")
 print(test_selected_string)
 print("location that find_sub_list found")
 print(find_sub_list(test_selected_string,test_string))
imputed_train.columns = training_data.columns
imputed_test_data.columns = test_data.columns<prepare_x_and_y>
dfTrain['selected_start_end'] = dfTrain.apply(lambda df: find_sub_list(df.selected_w,df.text_w), axis=1)
 dfTrain.head()
X = imputed_train.drop("Survived", axis = 1)
 y = imputed_train["Survived"]<define_variables>
if in_testing_mode:
dfTrain, dfLasttest = train_test_split(dfTrain, test_size=0.01, random_state=50)
 dfTrain.reset_index(drop=True, inplace=True)
 dfLasttest.reset_index(drop=True, inplace=True )
tf.random.set_seed(42 )<choose_model_class>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("the device we will run on is: ",device)
 if device.type == "cuda":
print("the GPU is: ", torch.cuda.get_device_name(0))
model = Sequential()<choose_model_class>
test_question = "positive"
test_string = [" hello"," world"]
test_context = robertaTokenize(test_string)[0]
print("question: ", test_question)
 print("context: ", test_context)
 test = tokenizer.encode_plus(
 [test_question],
test_context,
add_special_tokens = True,
max_length = 10,
pad_to_max_length = True,
return_token_type_ids = True,
return_attention_mask = True,
device=device,
return_tensors = 'pt', 
)
 print("input_ids: ",test["input_ids"])
 print("attention_mask: ",test["attention_mask"])
 print("token_type_ids: ",test["token_type_ids"])
 print("decoded: ",tokenizer.decode(test["input_ids"][0]))
model.add(Dense(7, activation = "relu", input_shape =(7,)))
 model.add(Dropout(0.2))<choose_model_class>
max_word_count = max(np.max(dfTrain["text_w"].apply(lambda x: len(x))), np.max(dfTest["text_w"].apply(lambda x: len(x))))
max_word_count+=5
print("Max clean word count is {}".format(max_word_count))
model.add(Dense(50, activation = "relu"))
model.add(Dropout(0.2))
model.add(Dense(50, activation = "relu"))<choose_model_class>
def convert_text2ids(sentiment, word_list, output, max_word_count, input_ids, attention_masks, token_type_ids, start_positions, end_positions):
encoded_dict = tokenizer.encode_plus(
 [sentiment],
word_list,
add_special_tokens = True,
max_length = max_word_count,
pad_to_max_length = True,
return_token_type_ids = True,
return_attention_mask = True,
device=device,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 token_type_ids.append(encoded_dict['token_type_ids'])
 if output is not None:
start_positions.append(output[0])
 end_positions.append(output[1] )
model.add(Dense(2, activation = "softmax"))<choose_model_class>
train_input_ids, train_attention_masks, train_token_type_ids, train_start_positions, train_end_positions = [], [], [], [], []
dfTrain.apply(lambda df: convert_text2ids(df.sentiment, df.text_w, df.selected_start_end, max_word_count, train_input_ids, train_attention_masks, train_token_type_ids, train_start_positions, train_end_positions), axis=1)
 train_input_ids = torch.cat(train_input_ids, dim=0)
 train_attention_masks = torch.cat(train_attention_masks, dim=0)
 train_token_type_ids = torch.cat(train_token_type_ids, dim=0)
 train_start_positions = torch.tensor(train_start_positions, dtype = torch.long)
 train_end_positions = torch.tensor(train_end_positions, dtype = torch.long )
model.compile(loss=keras.losses.sparse_categorical_crossentropy, optimizer = "adam", metrics = ['accuracy'] )<train_model>
batch_size = 64
epochs = 4
learning_rate = 5e-5
adam_epsilon = 1e-8
validation_size = 0.01
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )
model.fit(X, y,
batch_size=1,
epochs=700,
validation_split = 0.2,
verbose = 0 )<predict_on_test>
data = TensorDataset(train_input_ids,
train_attention_masks,
train_token_type_ids,
train_start_positions,
train_end_positions)
 train_data_size = int(( 1-validation_size)* len(data))
val_data_size = len(data)- train_data_size
train_dataset, val_dataset = torch.utils.data.random_split(data, [train_data_size, val_data_size])
 train_dataloader = DataLoader(train_dataset,
sampler=RandomSampler(train_dataset),
batch_size = batch_size)
 val_dataloader = DataLoader(val_dataset,
sampler=SequentialSampler(val_dataset),
batch_size = batch_size)
 print("train data length :", len(train_dataset))
print("validation data length :", len(val_dataset))
preds = model.predict(imputed_test_data )<predict_on_test>
random_step = random.randint(0,len(train_dataloader))
for step, batch in enumerate(train_dataloader):
if step == random_step:
random_data = random.randint(0,len(batch[0]))
sample_input_id = batch[0][random_data]
sample_attention_mask = batch[1][random_data]
sample_token_type_id = batch[2][random_data]
sample_start_position = batch[3][random_data]
sample_end_position = batch[4][random_data]
break
print("sample_input_id")
 print("\t",sample_input_id)
 print("sample_attention_mask")
 print("\t",sample_attention_mask)
 print("sample_token_type_id")
 print("\t",sample_token_type_id)
 print("sample_start_position")
 print("\t",sample_start_position)
 print("sample_end_position")
 print("\t",sample_end_position )
predictions = np.array(model.predict(imputed_test_data)).argmax(axis=1 )<prepare_output>
class tweetModel(BertPreTrainedModel):
def __init__(self, conf):
super(tweetModel, self ).__init__(conf)
 self.roberta = RobertaModel.from_pretrained(model_dir, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, dictionary):
outputs = self.roberta(
 dictionary["input_ids"],
attention_mask=dictionary["attention_mask"],
token_type_ids=dictionary["token_type_ids"] 
)
 out = torch.cat(( outputs[2][-1], outputs[2][-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 outputs =(start_logits, end_logits)
 if "start_positions" in dictionary and "end_positions" in dictionary:
if dictionary["start_positions"] is not None and dictionary["end_positions"] is not None:
if len(dictionary["start_positions"].size())> 1:
dictionary["start_positions"] = dictionary["start_positions"].squeeze(-1)
 if len(dictionary["end_positions"].size())> 1:
dictionary["end_positions"] = dictionary["end_positions"].squeeze(-1)
 ignored_index = start_logits.size(1)
 dictionary["start_positions"].clamp_(0, ignored_index)
 dictionary["end_positions"].clamp_(0, ignored_index)
 loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)
 start_loss = loss_fct(start_logits, dictionary["start_positions"])
 end_loss = loss_fct(end_logits, dictionary["end_positions"])
 total_loss =(start_loss + end_loss)/ 2
outputs =(total_loss,)+ outputs
return outputs
passenger_id=test_panda["PassengerId"]
results=passenger_id.to_frame()
results["Survived"]=predictions<save_to_csv>
rb_config = RobertaConfig.from_pretrained(model_dir , output_hidden_states = True)
 model = tweetModel(conf=rb_config)
 model.to(device )
results.to_csv("Titanic_ai_model.csv", index=False )<load_from_url>
param_optimizer = list(model.named_parameters())
 param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]
no_decay =['bias','LayerNorm.weight']
optimizer_grouped_parameters =[
{'params':[ p for n , p in param_optimizer
if not any(nd in n for nd in no_decay)],'weight_decay':0.01},
{'params':[p for n, p in param_optimizer
if any(nd in n for nd in no_decay)],'weigth_decay':0.0}
]
optimizer = AdamW(optimizer_grouped_parameters,
lr = learning_rate,
eps = adam_epsilon
)
!wget https://raw.githubusercontent.com/opencv/opencv/master/data/haarcascades/haarcascade_eye.xml
!wget https://raw.githubusercontent.com/opencv/opencv/master/data/haarcascades/haarcascade_frontalface_default.xml<load_from_csv>
total_steps = len(train_dataloader)* epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )
svc = pickle.load(open("/kaggle/input/fire-facenet/svc.pkl", "rb"))
enc = pickle.load(open("/kaggle/input/fire-facenet/encoder.pkl", "rb"))
mod = "/kaggle/input/fire-facenet/nn4.small2.v1.h5"
face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
 root = "/kaggle/input/asn10e-final-submission-coml-facial-recognition/"
infile = open(os.path.join(root,"person_id_mapping.csv"), "r")
 person_to_id = {line.split(",")[1][:-1]:int(line.split(",")[0])for line in infile if "person_id" not in line}
id_to_person = {person_to_id[key]:key for key in person_to_id}
sys.path.append("/kaggle/input/fire-facenet/models")
 nn4_small2_pretrained = create_model()
nn4_small2_pretrained.load_weights(mod)
 alignment = AlignDlib('/kaggle/input/fire-facenet/models/models/landmarks.dat' )<define_variables>
def train(model, optimizer, scheduler, epochs, train_dataloader, validation_dataloader, training_stats):
total_t0 = time.time()
for epoch_i in range(0, epochs):
print('
======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
batch_device = tuple(t.to(device)for t in batch)
 if step % 20 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
inputs = {
"input_ids": batch_device[0],
"attention_mask": batch_device[1],
"token_type_ids": batch_device[2],
"start_positions": batch_device[3],
"end_positions": batch_device[4],
}
model.zero_grad()
outputs = model(inputs)
 loss = outputs[0]
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print("
Average training loss: {0:.2f}".format(avg_train_loss))
print(" Training epoch took: {:}".format(training_time))
print("
Running Validation...")
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
nb_eval_steps = 0
for batch in validation_dataloader:
batch_device = tuple(t.to(device)for t in batch)
 inputs = {
"input_ids": batch_device[0],
"attention_mask": batch_device[1],
"token_type_ids": batch_device[2],
"start_positions": batch_device[3],
"end_positions": batch_device[4],
}
with torch.no_grad() :
outputs = model(inputs)
 loss = outputs[0]
total_eval_loss += loss.item()
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(" Validation Loss: {0:.2f}".format(avg_val_loss))
print(" Validation took: {:}".format(validation_time))
training_stats.append({
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Training Time': training_time,
'Validation Time': validation_time
})
 print("")
 print("Training complete!")
 print("Total training took {:}(h:mm:ss)".format(format_time(time.time() -total_t0)) )
ids = ["0_25", "0_34", "0_40", "0_37", "0_3", "0_2", "0_20", "0_27", "0_39", "0_33", "0_14", "0_17", "0_4", "0_46", "0_29", "0_35", "0_11", "0_41", "0_8", "0_42", "0_7", "0_1", "0_15", "0_0", "0_32", "0_16", "0_21", "0_13", "0_31", "0_30", "0_22", "0_19", "0_23", "0_24", "0_43", "0_36", "0_10", "0_44", "1_28", "1_26", "1_38", "1_45", "1_12", "1_9"]
ids0 = sorted([int(i.split("_")[1])for i in ids if i[0] == "0"])
 ids1 = sorted([int(i.split("_")[1])for i in ids if i[0] == "1"])
 pred = svc.decision_function(embedded)
 idx0 = np.asarray([i for i in range(len(X_paths)) if "0" in X_paths[i]])
 idx1 = np.asarray([i for i in range(len(X_paths)) if "1" in X_paths[i]] )<install_modules>
training_stats = []
train(model, optimizer, scheduler, epochs, train_dataloader, val_dataloader, training_stats )
!pip install munkres<categorify>
pd.set_option('precision', 2)
 df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('epoch')
 df_stats
m = Munkres()
def find_labels(pred, ids):
mat = [[-1*dist[i] for i in range(len(dist)) if i in ids] for dist in pred]
matches = {i[0]:i[1] for i in m.compute(mat)}
labels = [ids[matches[i]] if i in matches else None for i in range(len(pred)) ]
return labels
labels1 = find_labels(pred[idx1],ids1)
 labels0 = find_labels(pred[idx0],ids0)
 y2 = []
j = 0
k = 0
for i in range(len(X_paths)) :
if i in idx0:
y2.append(labels0[j])
 j += 1
else:
y2.append(labels1[k])
 k+=1 
<categorify>
random_step = random.randint(0,len(val_dataloader))
for step, batch in enumerate(val_dataloader):
if step == random_step:
random_data = random.randint(0,len(batch[0]))
sample_input_id = batch[0][random_data].to(device)
 sample_attention_mask = batch[1][random_data].to(device)
 sample_token_type_id = batch[2][random_data].to(device)
 sample_start_position = batch[3][random_data].to(device)
 sample_end_position = batch[4][random_data].to(device)
 break
print("sample_input_id")
 print("\t",sample_input_id)
 print("sample_attention_mask")
 print("\t",sample_attention_mask)
 print("sample_token_type_id")
 print("\t",sample_token_type_id)
 test_inputs = {
"input_ids": sample_input_id.view(1,-1 ).to(device),
"attention_mask": sample_attention_mask.view(1,-1 ).to(device),
"token_type_ids": sample_token_type_id.view(1,-1 ).to(device)
 }
sample_answer_start_scores, sample_answer_end_scores = model(test_inputs)
 answer_start = torch.argmax(sample_answer_start_scores)
 answer_end = torch.argmax(sample_answer_end_scores)
 print("
Real start: ", sample_start_position, 'Predicted start: ', answer_start)
 print("Real end: ", sample_end_position, 'Predicted end: ', answer_end )
def find_data(pnum, pred, idx):
ind = 0
mx = -100
for i, p in enumerate(pred):
if p[pnum] > mx:
ind = i
mx = p[pnum]
fname = X_paths[idx[ind]]
fx, fy, fw, fh = F[idx[ind]]
return {"filename": fname, "person_id":fname.replace(".jpg", "_%d"%pnum), 'xmin': fx, 'xmax': fx+fw, 'ymin': fy, 'ymax': fy+fh, 'img': idx[ind]}
def find_labels(pred, ids):
mat = [[-1*dist[i] for i in range(len(dist)) if i in ids] for dist in pred]
matches = {i[0]:i[1] for i in m.compute(mat)}
print(matches)
 labels = [ids[matches[i]] if i in matches else None for i in range(len(pred)) ]
return labels
preds1 = pred[idx1]
preds0 = pred[idx0]
y3 = {**{pnum:find_data(pnum, preds1, idx1)for pnum in id_to_person if pnum in ids1}, **{pnum:find_data(pnum, preds0, idx0)for pnum in id_to_person if pnum in ids0}}<set_options>
def getSelected(text, text_w_n, answer_start_scores, answer_end_scores):
start_sorted_indexes = np.argsort(-answer_start_scores, kind='quicksort', order=None)[0]
end_sorted_indexes = np.argsort(-answer_end_scores, kind='quicksort', order=None)[0]
found_start_end = False
for i in range(0,len(start_sorted_indexes)) :
current_start = start_sorted_indexes[i]
if current_start >= len(text_w_n):
continue
for j in range(0,i+2 if i+2<len(end_sorted_indexes)else len(end_sorted_indexes)) :
current_end = end_sorted_indexes[j]
if current_end >= len(text_w_n):
continue
if current_start <= current_end:
found_start_end = True
real_start = text_w_n[current_start]
real_end = text_w_n[current_end]
break
if found_start_end:
break
if not found_start_end:
return text
splitted_text = text.split()
selected_text = splitted_text[real_start:real_end+1]
selected_text = " ".join(selected_text)
 return selected_text
def predict_test(text, sentiment, text_w, text_w_n):
encoded_dict = tokenizer.encode_plus(
 [sentiment],
text_w,
add_special_tokens = True,
max_length = max_word_count,
pad_to_max_length = True,
return_token_type_ids = True,
return_attention_mask = True,
device=device,
return_tensors = 'pt', 
)
 encoded_dict["input_ids"] = encoded_dict["input_ids"].to(device)
 encoded_dict["token_type_ids"] = encoded_dict["token_type_ids"].to(device)
 encoded_dict["attention_mask"] = encoded_dict["attention_mask"].to(device)
 answer_start_scores, answer_end_scores = model(encoded_dict)
 answer_start_scores = answer_start_scores.cpu().detach().numpy()
answer_end_scores = answer_end_scores.cpu().detach().numpy()
selected_text = getSelected(text, text_w_n, answer_start_scores, answer_end_scores)
 return selected_text
warnings.simplefilter(action='ignore' )<load_from_csv>
x = random.randint(0,len(dfTest))
while dfTest.iloc[x].sentiment == "neutral":
x = random.randint(0,len(dfTest))
print(dfTest.iloc[x].text)
 print(dfTest.iloc[x].sentiment)
 predict_test(dfTest.iloc[x].text, dfTest.iloc[x].sentiment, dfTest.iloc[x].text_w, dfTest.iloc[x].text_w_n )
train_df = pd.read_csv('.. /input/titanic/train.csv')
 predict_df = pd.read_csv('.. /input/titanic/test.csv' )<concatenate>
dfTest["selected_text"] = dfTest.apply(lambda df: predict_test(df.text, df.sentiment, df.text_w, df.text_w_n), axis=1)
 dfTest['selected_text'] = dfTest['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 dfTest['selected_text'] = dfTest['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 dfTest['selected_text'] = dfTest['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 dfTest[["textID", "text", "sentiment", "selected_text"]].head(20 )
df=pd.concat([train_df,predict_df], ignore_index=True, sort =False)
 df.head()<sort_values>
dfContrib = pd.read_csv(sample_submission_file)
 dfContrib.head()
df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
dfContrib = pd.merge(dfContrib[["textID"]], dfTest[["textID","selected_text"]], on='textID')
 dfContrib.to_csv(submission_file,index=False)
 pd.read_csv(submission_file ).head()
df[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
df['Sex'] = df['Sex'].map({'female': 1, 'male': 0} ).astype(int )<feature_engineering>
dfNeutral = dfTrain[dfTrain['sentiment'] == 'neutral']
dfNeutral['jaccard'] = dfNeutral.apply(lambda df: jaccard(df.selected_text, df.text), axis=1)
 print("Average jaccard score of neutral data when text used as selected text: ",dfNeutral["jaccard"].mean())
 dfNeutral["predicted_selected_text"] = dfNeutral.apply(lambda df: predict_test(df.text, df.sentiment, df.text_w, df.text_w_n), axis=1)
 dfNeutral['predicted_jaccard'] = dfNeutral.apply(lambda df: jaccard(df.selected_text, df.predicted_selected_text), axis=1)
 print("Average jaccard score of neutral data when models prediction used as selected text: ",dfNeutral["predicted_jaccard"].mean())
 del(dfNeutral )
df['name_title'] = df['Name'].str.extract('([A-Za-z]+)\.', expand=False )<count_values>
if in_testing_mode:
dfLasttest["predicted_selected_text"] = dfLasttest.apply(lambda df: predict_test(df.text, df.sentiment, df.text_w, df.text_w_n), axis=1)
 dfLasttest["jaccard"] = dfLasttest.apply(lambda df: jaccard(df.selected_text, df.predicted_selected_text), axis=1)
 print("average jaccard score of test data is: ",dfLasttest["jaccard"].mean())
 print("average jaccard score of positive test data is: ",dfLasttest[dfLasttest['sentiment'] == 'positive']["jaccard"].mean())
 print("average jaccard score of negative test data is: ",dfLasttest[dfLasttest['sentiment'] == 'negative']["jaccard"].mean())
 print("average jaccard score of neutral test data is: ",dfLasttest[dfLasttest['sentiment'] == 'neutral']["jaccard"].mean() )
<count_values><EOS> 
if in_testing_mode:
dfTrain["predicted_selected_text"] = dfTrain.apply(lambda df: predict_test(df.text, df.sentiment, df.text_w, df.text_w_n), axis=1)
 dfTrain["jaccard"] = dfTrain.apply(lambda df: jaccard(df.selected_text, df.predicted_selected_text), axis=1)
 print("average jaccard score of train data is: ",dfTrain["jaccard"].mean())
 print("average jaccard score of positive train data is: ",dfTrain[dfTrain['sentiment'] == 'positive']["jaccard"].mean())
 print("average jaccard score of negative train data is: ",dfTrain[dfTrain['sentiment'] == 'negative']["jaccard"].mean())
 print("average jaccard score of neutral train data is: ",dfTrain[dfTrain['sentiment'] == 'neutral']["jaccard"].mean() )