Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<categorify>
%matplotlib inline
def clean_text(text):
url = re.compile(r'https?://\S+|www\.\S+')
 text = url.sub(r'', text)
 html = re.compile(r'<.*?>')
 text = html.sub(r'', text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags = re.UNICODE)
 text = emoji_pattern.sub(r'', text)
 table = str.maketrans('', '', string.punctuation)
 text = text.translate(table)
 text = text.lower()
text = re.sub(r"i'm", "i am", text)
 text = re.sub(r"you'll", "you will", text)
 text = re.sub(r"i'll", "i will", text)
 text = re.sub(r"she'll", "she will", text)
 text = re.sub(r"he'll", "he will", text)
 text = re.sub(r"he's", "he is", text)
 text = re.sub(r"she's", "she is", text)
 text = re.sub(r"that's", "that is", text)
 text = re.sub(r"what's", "what is", text)
 text = re.sub(r"where's", "where is", text)
 text = re.sub(r"there's", "there is", text)
 text = re.sub(r"here's", "here is", text)
 text = re.sub(r"who's", "who is", text)
 text = re.sub(r"how's", "how is", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"can't", "cannot", text)
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"don't", "do not", text)
 text = re.sub(r"shouldn't", "should not", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub(r" ", " ", text)
 text = "".join([c if c not in string.punctuation else "" for c in text])
 text = ''.join(c for c in text if not c.isdigit())
 text = re.sub(r"÷", "", text)
 text = re.sub(' +', ' ', text)
 return text
train_data['text'] = train_data['text'].apply(lambda x : clean_text(x))
test_data['text'] = test_data['text'].apply(lambda x : clean_text(x))<feature_engineering>
import scipy as sp
from functools import partial
from collections import Counter
def massage_text(text):
text = re.sub("[^a-zA-Z]", ' ', text)
 text = text.lower()
text = text.split()
lem = WordNetLemmatizer()
text = [lem.lemmatize(word)for word in text
if word not in set(stopwords.words('english')) ]
text = ' '.join(text)
 return text
print('--here goes nothing')
 print(text)
 train_data['text'] = train_data['text'].apply(lambda x : massage_text(x))
test_data['text'] = test_data['text'].apply(lambda x : massage_text(x))<feature_engineering>
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.model_selection import GroupShuffleSplit
from sklearn.metrics import accuracy_score
vectorizer = TfidfVectorizer()
Train = vectorizer.fit_transform(train_data['text'])
 test = vectorizer.transform(test_data['text'] )<train_on_grid>
warnings.filterwarnings('ignore' )
pbf = linear_model.RidgeClassifier(alpha=2.2)
 pbf.fit(Train, train_data["target"])
 scores = model_selection.cross_val_score(pbf, Train, train_data["target"], cv=3, scoring="f1")
 scores<compute_test_metric>
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb
from sklearn.svm import SVC
score = scores.mean() *100
print("Score: ", score )<train_model>
tr = pd.read_csv('.. /input/X_train.csv')
 te = pd.read_csv('.. /input/X_test.csv')
 target = pd.read_csv('.. /input/y_train.csv')
 ss = pd.read_csv('.. /input/sample_submission.csv' )
pbf.fit(Train, train_data["target"] )<predict_on_test>
tr0=tr
te0=te
pbf.predict(test )<load_from_csv>
df_target=pd.read_csv('.. /input/y_train.csv')
 df_target.head()
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = pbf.predict(test)
 sample_submission.head(20 )<save_to_csv>
le=LabelEncoder()
df_target['surface_id']=le.fit_transform(df_target['surface'])
 df_target.head()
sample_submission.to_csv("submission.csv", index=False )<import_modules>
df_target_s=df_target[['surface','surface_id']].groupby(by='surface' ).max().reset_index()
df_target_s
import numpy as np
import pandas as pd
from pathlib import Path
from fastai.text.all import *<define_variables>
df_target_cnt=target.groupby(by='surface' ).count() ['series_id']
df_target_cnt=df_target_cnt.reset_index()
datadir = Path.cwd().parent / 'input' / 'nlp-getting-started'<load_from_csv>
df_target_s=df_target_s.merge(df_target_cnt,on='surface')
 df_target_s
df_train = pd.read_csv(datadir/'train.csv')
 df_test = pd.read_csv(datadir/'test.csv')
 df_submission = pd.read_csv(datadir/'sample_submission.csv' )<load_pretrained>
df_target[df_target['group_id']==5]
dls_lm = TextDataLoaders.from_df(df_train, text_col='text', is_lm=True )<define_variables>
df_target_g=df_target[['group_id','surface_id']].groupby(by='group_id' ).count().reset_index()
df_target_g.head()
dls_lm.show_batch(max_n=2 )<load_pretrained>
print('Groups Nr Max Min',yg.nunique() ,yg.max() ,yg.min() )
learn_lm = language_model_learner(
 dls_lm, AWD_LSTM, drop_mult=0.3,
metrics=[accuracy, Perplexity() ] ).to_fp16()<train_model>
target['surface'] = le.fit_transform(target['surface'] )
learn_lm.fit_one_cycle(1, 2e-2 )<train_model>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
learn_lm.unfreeze()
learn_lm.fit_one_cycle(5,2e-3 )<categorify>
def fe_step0(actual):
actual['norm_quat'] =(actual['orientation_X']**2 + actual['orientation_Y']**2 + actual['orientation_Z']**2 + actual['orientation_W']**2)
 actual['mod_quat'] =(actual['norm_quat'])**0.5
actual['norm_X'] = actual['orientation_X'] / actual['mod_quat']
actual['norm_Y'] = actual['orientation_Y'] / actual['mod_quat']
actual['norm_Z'] = actual['orientation_Z'] / actual['mod_quat']
actual['norm_W'] = actual['orientation_W'] / actual['mod_quat']
return actual
learn_lm.save_encoder('finetuned' )<create_dataframe>
tr = fe_step0(tr)
 te = fe_step0(te )
dls = TextDataLoaders.from_df(df_train, text_col='text', label_col='target' )<define_variables>
def fe_step1(actual):

 x, y, z, w = actual['norm_X'].tolist() , actual['norm_Y'].tolist() , actual['norm_Z'].tolist() , actual['norm_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 actual['euler_x'] = nx
actual['euler_y'] = ny
actual['euler_z'] = nz
return actual
dls.show_batch(max_n=2 )<choose_model_class>
def feat_eng(data):
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['totl_angle'] = np.sqrt(data['euler_x'] ** 2 + data['euler_y'] ** 2 + data['euler_z'] ** 2)
 data['linear_acceleration_X_abs'] = data['linear_acceleration_X'].where(data['linear_acceleration_X']>=0, - data['linear_acceleration_X'])
 data['linear_acceleration_Y_abs'] = data['linear_acceleration_Y'].where(data['linear_acceleration_Y']>=0, - data['linear_acceleration_Y'])
 data['linear_acceleration_Z_abs'] = data['linear_acceleration_Z'].where(data['linear_acceleration_Z']>=0, - data['linear_acceleration_Z'])
 data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
data['angle_vs_acc'] = data['totl_angle'] / data['totl_linr_acc']
data['angle_vs_vel'] = data['totl_angle'] / data['totl_anglr_vel']
data['acc_int_vel'] = data['totl_linr_acc'] * data['totl_anglr_vel']
data['angle_int_acc'] = data['totl_angle'] * data['totl_linr_acc']
data['angle_int_vel'] = data['totl_angle'] * data['totl_anglr_vel']
data['angle_ints_vel_acc'] = data['totl_angle'] * data['totl_anglr_vel'] * data['totl_linr_acc']
def mean_change_of_abs_change(x):
return np.mean(np.diff(np.abs(np.diff(x))))
df = pd.DataFrame()
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_mean_change_of_abs_change'] = data.groupby('series_id')[col].apply(mean_change_of_abs_change)
 df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df
learn = text_classifier_learner(dls, AWD_LSTM, drop_mult=0.5, metrics=accuracy )<categorify>
tr = fe_step1(tr)
 te = fe_step1(te )
learn.load_encoder('finetuned' )<find_best_params>
%%time
tr = feat_eng(tr)
 te = feat_eng(te)
 tr.head()
lr_min,lr_steep = learn.lr_find()
print(f"Minimum/10: {lr_min:.2e}, steepest point: {lr_steep:.2e}" )<set_options>
def scale_df(df1, df2, feature_cols):
scaler = StandardScaler()
scaled_features = scaler.fit_transform(df1[feature_cols])
 df1[feature_cols] = scaled_features
scaled_test_features = scaler.transform(df2[feature_cols])
 df2[feature_cols] = scaled_test_features
return df1, df2
learn.fine_tune(5, 6e-3 )<define_variables>
tr,te=scale_df(tr,te,te.columns )
text_list = list(df_test['text'].values )<predict_on_test>
tr.fillna(0, inplace = True)
 te.fillna(0, inplace = True )
%%capture
prediction = [learn.predict(i)[0] for i in text_list]<data_type_conversions>
tr.replace(-np.inf, 0, inplace = True)
 tr.replace(np.inf, 0, inplace = True)
 te.replace(-np.inf, 0, inplace = True)
 te.replace(np.inf, 0, inplace = True )
df_submission['target'] = prediction
df_submission['target'] = df_submission['target'].astype(int )<save_to_csv>
svc_cols=[x for x in tr.columns if 'orientation_W' in x]
svc_cols=svc_cols+[x for x in tr.columns if 'orientation_Z' in x]
df_submission.to_csv('submission.csv', index=False )<import_modules>
clf_cols=[x for x in tr.columns if not(x in svc_cols)]+['group_id']
import pandas as pd<load_from_csv>
X=tr.join(df_target['group_id'],on='series_id' )
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 print(train.shape)
 train.head(5 )<load_from_csv>
y=X['group_id']
X=X[svc_cols]
T=te[svc_cols]
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 print(test.shape)
 test.head(5 )<import_modules>
CV_STEPS=10
folds = StratifiedKFold(n_splits=CV_STEPS, shuffle=True, random_state=876 )
import re<feature_engineering>
i=0
tst_preds_gr = np.zeros(( T.shape[0], CV_STEPS))
oof_preds_gr = np.zeros(( X.shape[0]))
splitter=folds.split(X,y)
 for(train_index, valid_index)in(splitter):
print('='*20, i, '='*20)
 X_train = X.iloc[train_index]
print('train size:',len(train_index))
X_valid = X.iloc[valid_index]
print('valid size:',len(valid_index))
y_train = y.iloc[train_index]
y_valid = y.iloc[valid_index]
clf = SVC(kernel='linear')
 clf.fit(X_train, y_train)
 oof_preds_gr[valid_index] = clf.predict(X_valid)
 tst_preds_gr[:,i] = clf.predict(T)
 print('score ', accuracy_score(oof_preds_gr[valid_index], y_valid.values))
i=i+1
def urlremove(text):
return re.sub(r"(?:\@|https?\://)\S+", "", text)
 train['text'] = train['text'].apply(urlremove)
 test['text'] = test['text'].apply(urlremove)
 train['text']<feature_engineering>
def extr_test_pred(tst_preds_cv):
gr_preds=np.zeros(len(tst_preds_cv),dtype=int)
 for r in range(0,len(tst_preds_cv)) :
gr_preds[r]=Counter(tst_preds_cv[r,:] ).most_common(1)[0][0]
return gr_preds
def remove_emoji(text):
emoji_pattern = re.compile(
 u"(\ud83d[\ude00-\ude4f])|"
u"(\ud83c[\udf00-\uffff])|"
u"(\ud83d[\u0000-\uddff])|"
u"(\ud83d[\ude80-\udeff])|"
u"(\ud83c[\udde0-\uddff])"
"+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train['text'] = train['text'].apply(remove_emoji)
 test['text'] = test['text'].apply(remove_emoji)
 train['text']<categorify>
tr['group_id']=y
def decontracted(text):
if '’' in text:
text = text.replace("’", "'")
 text = re.sub(r"won't", "will not", text)
 text = re.sub(r"can't", "can not", text)
 text = re.sub(r"n't", " not", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'s", " is", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'t", " not", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'m", " am", text)
 return text
train['text'] = train['text'].apply(decontracted)
 test['text'] = test['text'].apply(decontracted)
 train['text']<feature_engineering>
te['group_id']=extr_test_pred(tst_preds_gr )
def character_remove(text):
return re.sub('\W+|_', ' ', text)
 train.text = train.text.apply(character_remove)
 test.text = test.text.apply(character_remove)
 train.text<feature_engineering>
X=tr[clf_cols]
T=te[clf_cols]
y=pd.DataFrame(target['surface'],index=tr.index)
 LABELS=[0,1,2,3,4,5,6,7,8]
def remove_numbers(text):
pattern = r'[^a-zA-z.,!?/:;"'\s]'
return re.sub(pattern, '', text)
 train.text = train.text.apply(remove_numbers)
 train.text = train.text.str.lower()
train.text = train.text.str.strip()
test.text = test.text.apply(remove_numbers)
 test.text = test.text.str.lower()
test.text = test.text.str.strip()
train['text']<import_modules>
X.to_csv('X.csv', index=False)
 T.to_csv('T.csv', index=False)
 y.to_csv('y.csv', index=False )
from sklearn.feature_extraction.text import TfidfVectorizer<feature_engineering>
CV_STEPS=30
folds = StratifiedKFold(n_splits=CV_STEPS, shuffle=True, random_state=1984 )
tfidf = TfidfVectorizer(min_df=1, max_df=0.3, ngram_range=(1, 2))
print(tfidf)
 text_tfidf = tfidf.fit_transform(train['text'])
 text_tfidf2 = tfidf.transform(test['text'])
<compute_train_metric>
groups=yg.unique()
dummy=np.zeros(len(groups))
clf = LogisticRegression(C=4.5)
 scores = model_selection.cross_val_score(clf, text_tfidf, train["target"], cv=3, scoring="f1")
 scores<train_model>
def group_id_drop(W):
try:
W.drop('group_id',axis=1,inplace=True)
 except:
print('Info: group_id already dropped.')
 return(W )
clf.fit(text_tfidf, train["target"] )<save_to_csv>
tst_preds_rf = np.zeros(( T.shape[0], CV_STEPS))
oof_preds_rf = np.zeros(( X.shape[0]))
importances = np.zeros(( CV_STEPS,X.shape[1]-1))
i=0
splitter=folds.split(groups,dummy)
 for(tr_gr_idx, va_gr_idx)in(splitter):
print('-'*20, i, '-'*20)
 tr_groups=groups[tr_gr_idx]
va_groups=groups[va_gr_idx]
X_train = X[X['group_id'].isin(tr_groups)]
X_train.drop('group_id',axis=1,inplace=True)
 train_index = X_train.index
print('train size:',len(train_index))
X_valid = X[X['group_id'].isin(va_groups)]
X_valid.drop('group_id',axis=1,inplace=True)
 valid_index = X_valid.index
print('valid size:',len(valid_index))
y_train = y.iloc[train_index]
curr_labels=y_train['surface'].unique()
print('used labels:',len(curr_labels),' [',','.join([str(c)for c in curr_labels]),']')
 y_valid = y.iloc[valid_index]
clf = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(X_train, y_train)
 oof_preds_rf[valid_index] = clf.predict(X_valid)
 TWG=T.drop('group_id',axis=1)
 tst_preds_rf[:,i] = clf.predict(TWG)
 ass_labels=np.unique(tst_preds_rf[:,i])
 print('assigned labels:',len(ass_labels),' [',','.join([str(int(c)) for c in ass_labels]),']')
 print('score ', accuracy_score(oof_preds_rf[valid_index], y_valid.values))
importances[i,0:X.shape[1]] = clf.feature_importances_
i=i+1
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission["target"] = clf.predict(text_tfidf2)
 submission.to_csv("submission55.csv", index=False )<load_from_csv>
tst_preds_rf0 = tst_preds_rf
oof_preds_rf0 = oof_preds_rf
importances0 = importances
train_tweet = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_tweet = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 df_map = {"train": train_tweet, "test": test_tweet}<count_missing_values>
preds_rf0=extr_test_pred(tst_preds_rf0 )
check_null_values(df_map )<string_transform>
accuracy_score(target['surface'], oof_preds_rf0 )
def word_list(sentences):
result = [word.lower() for sentence in sentences
for word in word_tokenize(sentence)]
return result
def create_corpus(train_tweet):
df = train_tweet.copy()
class_0_corpus = word_list(df.query("target == 0")["text"])
 class_1_corpus = word_list(df.query("target == 1")["text"])
 return class_0_corpus, class_1_corpus<feature_engineering>
i=0
tst_preds_rf = np.zeros(( T.shape[0], CV_STEPS))
oof_preds_rf = np.zeros(( X.shape[0]))
importances = np.zeros(( CV_STEPS,X.shape[1]-1))
splitter=folds.split(X,y)
 for(train_index, valid_index)in(splitter):
print('
X_train = X.iloc[train_index]
print('train size:',len(train_index))
X_valid = X.iloc[valid_index]
print('valid size:',len(valid_index))
y_train = y.iloc[train_index]
curr_labels=y_train['surface'].unique()
X_train=group_id_drop(X_train)
 X_valid=group_id_drop(X_valid)
 print('used labels:',len(curr_labels),' [',','.join([str(c)for c in curr_labels]),']')
 y_valid = y.iloc[valid_index]
clf = RandomForestClassifier(n_estimators = 500, n_jobs = -1)
 clf.fit(X_train, y_train)
 oof_preds_rf[valid_index] = clf.predict(X_valid)
 TWG=T.drop('group_id',axis=1)
 tst_preds_rf[:,i] = clf.predict(TWG)
 ass_labels=np.unique(tst_preds_rf[:,i])
 print('assigned labels:',len(ass_labels),' [',','.join([str(int(c)) for c in ass_labels]),']')
 print('score ', accuracy_score(oof_preds_rf[valid_index], y_valid.values))
importances[i,0:X.shape[1]] = clf.feature_importances_
i=i+1
class_0_corpus, class_1_corpus = create_corpus(train_tweet)
 punc_list = [punct for punct in string.punctuation]
stop = list(stop )<count_values>
preds_rf=extr_test_pred(tst_preds_rf )
def sum_all() :
start = time.time()
result = 0
for num in range(100):
result += num
end = time.time()
print("Elapsed time: {:4f} sec".format(end - start))
print("The sum is {}".format(result))
sum_all()<normalization>
accuracy_score(target['surface'], oof_preds_rf )
def mul_all() :
start = time.time()
result = 0
for num in range(100):
result *= num
end = time.time()
print("Elapsed time: {:4f} sec".format(end - start))
print("The product is {}".format(result))
mul_all()<categorify>
CV_STEPS=10
folds = StratifiedKFold(n_splits=CV_STEPS, shuffle=True, random_state=1984 )
def simple_decorator(func):
def decorated() :
start = time.time()
result = func()
end = time.time()
print("{} function".format(func.__name__))
print("Elapsed time: {:4f} sec".format(end - start))
print("The result is {}".format(result))
print()
return result
return decorated 
<categorify>
NR_EPOCHS=1000
xgb_params = {
'booster' : 'gbtree',
'objective' : 'multi:softprob',
'num_class' : 9,
'seed': 1984,
'eta': 0.01,
'max_depth': 3,
'gamma': 0.1,
'lambda': 0.8,
'subsample': 0.8,
'colsample_bytree': 0.8,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
verbose_eval = 100
early_stop = 200
@simple_decorator
def sum_all() :
result = 0
for x in range(100):
result += x
return result
@simple_decorator
def mul_all() :
result = 0
for x in range(100):
result *= x
return result<string_transform>
def xgb_train_no_groups(params, X, y, T):
oof_train_preds = np.zeros(X.shape[0])
 tst_preds = np.zeros(( T.shape[0],CV_STEPS))
perf_dict_list=[]
i=0
splitter=folds.split(X,y)
 for(train_index, valid_index)in(splitter):
print('
X_train = X.iloc[train_index]
print('train size:',len(train_index))
X_valid = X.iloc[valid_index]
print('valid size:',len(valid_index))
y_train = y.iloc[train_index]
curr_labels=y_train['surface'].unique()
X_train=group_id_drop(X_train)
 X_valid=group_id_drop(X_valid)
 print('used labels:',len(curr_labels),' [',','.join([str(c)for c in curr_labels]),']')
 y_valid = y.iloc[valid_index]
d_train = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_train.columns)
 d_valid = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_valid.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
perf=dict()
model = xgb.train(dtrain=d_train, num_boost_round=NR_EPOCHS, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params,
evals_result=perf)
 valid_pred = model.predict(xgb.DMatrix(X_valid, feature_names=X_valid.columns), ntree_limit=model.best_ntree_limit)
 TWG=T.drop('group_id',axis=1)
 test_pred = model.predict(xgb.DMatrix(TWG, feature_names=TWG.columns), ntree_limit=model.best_ntree_limit)
 oof_train_preds[valid_index] = valid_pred.argmax(axis=1)
 tst_preds[:,i] = test_pred.argmax(axis=1)
 perf_dict_list = perf_dict_list + [perf]
print('Accuracy =',accuracy_score(target['surface'][valid_index], valid_pred.argmax(axis=1)))
 i += 1
return model, oof_train_preds, tst_preds, perf_dict_list
common_words(class_0_corpus, name="Class 0 Train")
 common_words(class_1_corpus, name="Class 1 Train" )<statistical_test>
def xgb_train_groups_based(params, X, y, T):
oof_train_preds = np.zeros(X.shape[0])
 tst_preds = np.zeros(( T.shape[0],CV_STEPS))
perf_dict_list=[]
i=0
splitter=folds.split(groups,dummy)
 for(tr_gr_idx, va_gr_idx)in(splitter):
print('-'*20, i, '-'*20)
 tr_groups=groups[tr_gr_idx]
va_groups=groups[va_gr_idx]
X_train = X[X['group_id'].isin(tr_groups)]
X_train.drop('group_id',axis=1,inplace=True)
 train_index = X_train.index
print('train size:',len(train_index))
X_valid = X[X['group_id'].isin(va_groups)]
X_valid.drop('group_id',axis=1,inplace=True)
 valid_index = X_valid.index
print('valid size:',len(valid_index))
y_train = y.iloc[train_index]
curr_labels=y_train['surface'].unique()
print('used labels:',len(curr_labels))
y_valid = y.iloc[valid_index]
d_train = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_train.columns)
 d_valid = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_valid.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
perf=dict()
model = xgb.train(dtrain=d_train, num_boost_round=NR_EPOCHS, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params,
evals_result=perf)
 valid_pred = model.predict(xgb.DMatrix(X_valid, feature_names=X_valid.columns), ntree_limit=model.best_ntree_limit)
 TWG=T.drop('group_id',axis=1)
 test_pred = model.predict(xgb.DMatrix(TWG, feature_names=TWG.columns), ntree_limit=model.best_ntree_limit)
 oof_train_preds[valid_index] = valid_pred.argmax(axis=1)
 tst_preds[:,i] = test_pred.argmax(axis=1)
 perf_dict_list = perf_dict_list + [perf]
print('Accuracy =',accuracy_score(target['surface'][valid_index], valid_pred.argmax(axis=1)))
 i += 1
return model, oof_train_preds, tst_preds, perf_dict_list
noise_patterns = {
"url" : r"https?(://\S+|\S+)|www\.\S+",
"html_tag" : r"<.*?>",
"non_ascii" : r"[^\x00-\x7f]+",
"RT word" : r"\b[Rr][Tt]\b",
"amp word" : r"\bamp\b",
}
def noise_check(df_map, patterns):
for key, df in df_map.items() :
print("{} DataFrame noises: ".format(key))
print("-" * 30)
 for indicator, pattern in patterns.items() :
count = df["text"].str.contains(pattern ).sum()
print("There were {:5} rows with {}".format(count, indicator))
print()
noise_check(df_map, noise_patterns )<string_transform>
%%time
model0, oof_preds_xgb0, tst_preds_xgb0, perf = xgb_train_groups_based(xgb_params, X,y,T )
def add_paren(text):
return "({})".format(text)
 noises_all = "|".join(add_paren(pattern)for pattern in noise_patterns.values())
 print(noises_all )<feature_engineering>
preds_xgb0=extr_test_pred(tst_preds_xgb0 )
def clean_noise(text):
return re.sub(noises_all, r" ", text)
 train_tweet["text"] = train_tweet["text"].apply(clean_noise)
 test_tweet["text"] = test_tweet["text"].apply(clean_noise)
 noise_check(df_map, noise_patterns )<randomize_order>
accuracy_score(target['surface'], oof_preds_xgb0 )
def random_sampling(df_map, n=5):
for key, df in df_map.items() :
print("{} sample from {} dataset".format(n, key))
print("-" * 30)
 print(df["text"].sample(n))
print()
random_sampling(df_map )<define_variables>
%%time
model, oof_preds_xgb, tst_preds_xgb, perf = xgb_train_no_groups(xgb_params, X,y,T )
word1 = "The"
word2 = "the"
print(word1.lower() == word2 )<categorify>
preds_xgb=extr_test_pred(tst_preds_xgb)
 preds_xgb
def decontracted(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
print(decontracted("I'm Lord of Darkness"))
print(decontracted("Who've made this burger?"))<string_transform>
accuracy_score(target['surface'], oof_preds_xgb )
def punct2espace(text):
return re.sub(",", "", text)
 def punct2wspace(text):
return re.sub(r"[{}]+".format(string.punctuation), " ", text)
 def residual_punc(text):
table = str.maketrans("", "", string.punctuation)
 return text.translate(table)
 def normalize_wspace(text):
return re.sub(r"\s+", " ", text)
 def replace_punctuations(text):
text = punct2espace(text)
 text = punct2wspace(text)
 text = residual_punc(text)
 text = normalize_wspace(text)
 return text.strip()
replace_punctuations(r"@@rakka@@ alhazimi@@hai typhoon--devastation \\\\\conclusively" )<install_modules>
preds=preds_rf0
!pip install pyspellchecker<string_transform>
ss['surface'] = le.inverse_transform(preds)
 ss['surface_id'] = preds
ss.head()
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 text = "speling correctin"
correct_spellings(text )<string_transform>
XC=X.copy()
TC=T.copy()
lemmatizer = WordNetLemmatizer()
wordnet_map = {"N":wordnet.NOUN, "V":wordnet.VERB, "J":wordnet.ADJ, "R":wordnet.ADV}
def lemmatize_words(text):
pos_tagged_text = nltk.pos_tag(text.split())
 return " ".join([lemmatizer.lemmatize(word, wordnet_map.get(pos[0], wordnet.NOUN)) for word, pos in pos_tagged_text])
 lemmatize_words("rakka alhazimi are cool" )<categorify>
from sklearn.decomposition import PCA
def normalize_text(text):
text = text.lower()
text = decontracted(text)
 text = replace_punctuations(text)
 text = lemmatize_words(text)
 return text
normalize_text("Hello andrew, I'm from kuvukiland nuce to meet you" )<install_modules>
pca = PCA(n_components=3, svd_solver='full',random_state=1984)
 pca.fit(XC)
 xpc= pca.transform(XC )
!pip install pandarallel<feature_engineering>
pca.explained_variance_ratio_
pandarallel.initialize()
start = time.time()
train_tweet["text"] = train_tweet["text"].parallel_apply(normalize_text)
 test_tweet["text"] = test_tweet["text"].parallel_apply(normalize_text)
 elapsed = time.time() - start
print("Elapsed time: {:.4f} min".format(elapsed / 60))<save_to_csv>
XPC = pd.DataFrame(data = xpc , columns = ['PC1', 'PC2','PC3'])
 XPC['surface_id'] = y
XPC.head()
train_tweet.to_csv("train_tweet.csv")
 test_tweet.to_csv("test_tweet.csv" )<string_transform>
selected_sufaces=[1,3,5,6,7,8]
english_vocab = nltk.corpus.words.words("en")
 english_vocab = set(english_vocab)
 corpus_0, corpus_1 = create_corpus(train_tweet)
 corpus = corpus_0 + corpus_1
tweet_words = set(corpus )<string_transform>
pcat=pca
tpc=pcat.transform(TC )
unknown_words = list(tweet_words.difference(english_vocab))
print("There were {} unknown words.".format(len(unknown_words)) )<string_transform>
TPC = pd.DataFrame(data = tpc , columns = ['PC1','PC2','PC3'])
 TPC['surface_id'] = preds
TPC.head()
MAX_LEN = 50
def tokenize(df_map):
sentence_list = []
for key, df in df_map.items() :
sentence = [" ".join(word for word in sen.split() if not word.isdigit())
 for sen in df["text"]]
length = df.shape[0]
sentence_list += sentence
tokenizer = preprocessing.text.Tokenizer()
tokenizer.fit_on_texts(sentence_list)
 tensor = tokenizer.texts_to_sequences(sentence_list)
 tensor = preprocessing.sequence.pad_sequences(tensor, padding="post", maxlen=MAX_LEN)
 input_tensor_train, input_tensor_test = tensor[:-length], tensor[-length:]
return input_tensor_train, input_tensor_test, tokenizer
X_train, X_test, tokenizer = tokenize(df_map)
 y_train = train_tweet["target"].values
y_train = y_train.reshape(-1, 1 )<load_from_csv>
coh = pd.DataFrame()
coh['prev_surface_id']=preds
coh['prev_group_id']=TC['group_id']
coh['prev_id']=coh.index
coh.head()
with open(".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec", "r")as vector:
fasttext = vector.readlines()
len(fasttext )<load_from_csv>
gs=df_target[['group_id','surface_id']].groupby(by=['group_id','surface_id'] ).first().reset_index()
gs[gs['surface_id']==1]
with open(".. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt", "r")as vector:
glove = vector.readlines()
len(glove )<categorify>
prev_merge=coh.merge(gs,left_on=['prev_surface_id','prev_group_id'],right_on=['surface_id','group_id'],how='left')
 prev_merge=prev_merge.groupby('prev_id' ).first()
prev_merge.fillna(-1, inplace=True)
 prev_merge.head()
def embed2dict(tokenizer, embedding):
word_index = tokenizer.word_index
embedding_dict = {line.split() [0]: line.split() [1:] for line in embedding[1:]
if word_index.get(line.split() [0])}
return embedding_dict
fasttext_dict = embed2dict(tokenizer, fasttext)
 glove_dict = embed2dict(tokenizer, glove )<split>
%%time
for i in range(0,len(prev_merge)) :
gr=prev_merge['group_id'].loc[i]
prev_gr=prev_merge['prev_group_id'].loc[i]
if(gr==-1):
prev_merge['surface_id'].loc[i]=gs['surface_id'][gs['group_id']==prev_gr].values[0]
fasttext_word = set(fasttext_dict.keys())
 glove_word = set(glove_dict.keys())
 intersection_word = fasttext_word.intersection(glove_word)
 print("Similar embedding words:", len(intersection_word))<categorify>
prev_merge=prev_merge.astype(int)
 prev_merge.head()
def update_embed_dict(embed_dict):
result = {word: vector for word, vector in embed_dict.items()
if word in intersection_word}
return result
fasttext_dict = update_embed_dict(fasttext_dict)
 glove_dict = update_embed_dict(glove_dict )<feature_engineering>
prev_merge['final_surface_id']=prev_merge['prev_surface_id']
prev_merge.head()
fasttext_dim = 300
glove_dim = 200
num_words = len(tokenizer.word_index)+ 1
def dict2vector(tokenizer, embed_dict, embed_dim):
vector = np.zeros(shape=(num_words, embed_dim))
word_index = tokenizer.word_index
for word, index in word_index.items() :
if index > num_words:
continue
if embed_dict.get(word):
vector[index] = embed_dict.get(word)
 return vector
fasttext_vector = dict2vector(tokenizer, fasttext_dict, fasttext_dim)
 glove_vector = dict2vector(tokenizer, glove_dict, glove_dim )<concatenate>
def counter_distance(cnt,ref):
vals=cnt.values()
tot=sum(vals)
 cnt=sorted(cnt.items())
 dist=0
for i in range(0,len(cnt)) :
dist+=np.abs(ref[i]-cnt[i][1]/tot)
 return dist
def concat_vector(vectors):
result = np.hstack(vectors)
 return result
embedding_vector = concat_vector([fasttext_vector, glove_vector] )<define_variables>
d_freq=[]
corrections=0
for i in range(0,len(prev_merge)) :
gr=prev_merge['group_id'].loc[i]
if(gr==-1):
hp1=prev_merge['final_surface_id']
hp2=hp1.copy()
hp2[i]=prev_merge['surface_id'].loc[i]
d_hp1=counter_distance(Counter(hp1),attended_prc)
 d_hp2=counter_distance(Counter(hp2),attended_prc)
 if(d_hp2<d_hp1):
d_freq=d_freq+[d_hp2]
corrections=corrections+1
prev_merge['final_surface_id'] = hp2
else:
d_freq=d_freq+[d_hp1]
print('Corrections made=',corrections )
BATCH_SIZE = 32
EPOCHS = 8
embedding_dim = embedding_vector.shape[1]<choose_model_class>
Counter(prev_merge['prev_surface_id'] )
def build_model(emb_init=None, emb_train=True):
embed_params = {"input_dim": num_words,
"output_dim": embedding_dim,
"embeddings_initializer": keras.initializers.Constant(
 embedding_vector if emb_init is None else emb_init),
"trainable": emb_train,
"mask_zero": True,
}
model = keras.Sequential([
keras.layers.Embedding(**embed_params),
keras.layers.Bidirectional(keras.layers.LSTM(128,
dropout=0.3,
recurrent_dropout=0.3,
return_sequences=True 
)) ,
keras.layers.Reshape(( 5, 10, 256)) ,
keras.layers.Conv2D(32, 2),
keras.layers.GlobalMaxPooling2D() ,
keras.layers.Dropout(0.3),
keras.layers.Dense(1, activation="sigmoid")])
 optimizer = keras.optimizers.Adam(1e-4, clipvalue=0.5)
 model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=["acc"])
 return model<train_model>
Counter(prev_merge['final_surface_id'] )
test_model.fit(X_train, y_train,
epochs=10,
batch_size=BATCH_SIZE,
validation_split=0.2, )<load_pretrained>
ss.drop('surface_id',axis=1,inplace=True)
 ss.to_csv('submission.csv', index=False )