Kaggle competition: Natural Language Processing with Disaster Tweets
wall_quality = ['epared1', 'epared2', 'epared3']
toCategorical(train, wall_quality, 'wall_quality')
 toCategorical(test, wall_quality, 'wall_quality' )<categorify>
df_train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 print(df_train.head() )
roof_quality = ['etecho1', 'etecho2', 'etecho3']
toCategorical(train, roof_quality, 'roof_quality')
 toCategorical(test, roof_quality, 'roof_quality' )<categorify>
df_train['keyword'].value_counts()
floor_quality = ['eviv1', 'eviv2', 'eviv3']
toCategorical(train, floor_quality, 'floor_quality')
 toCategorical(test, floor_quality, 'floor_quality' )<categorify>
df_train['location'].value_counts()
gender = ['male', 'female']
toCategorical(train, gender, 'gender')
 toCategorical(test, gender, 'gender' )<categorify>
percent_true = 0.42966
civil_status = ['estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7']
toCategorical(train, civil_status, 'civil_status')
 toCategorical(test, civil_status, 'civil_status' )<categorify>
df_test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 print(df_test.head() )
education = ['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']
toCategorical(train, education, 'education')
 toCategorical(test, education, 'education' )<categorify>
combined_text = ""
combined_true_text = ""
combined_false_text = ""
for index, row in df_train.iterrows() :
row_text = row['text'].lower() + ' '
combined_text += row_text
if row['target']:
combined_true_text += row_text
else:
combined_false_text += row_text
combined_text_words = nltk.word_tokenize(combined_text)
 true_text_words = nltk.word_tokenize(combined_true_text)
 false_text_words = nltk.word_tokenize(combined_false_text)
 print(combined_text_words[:50])
 print(true_text_words[:50])
 print(false_text_words[:50] )
house_ownership = ['tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']
toCategorical(train, house_ownership, 'house_ownership')
 toCategorical(test, house_ownership, 'house_ownership' )<categorify>
proba_true_list = [(key, freq_dist_true[key]*percent_true*len(combined_text_words)/(len(true_text_words)*freq_dist_combined[key])) for key in freq_dist_true.keys() if len(key)>3 and freq_dist_combined[key]>=25]
proba_true_list = sorted(proba_true_list, key=lambda x: np.absolute(percent_true - x[1]), reverse=True)
 print(proba_true_list )
region = ['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']
toCategorical(train, region, 'region')
 toCategorical(test, region, 'region' )<categorify>
important_word_list = [tpl[0] for tpl in proba_true_list if np.absolute(tpl[1]-percent_true)>0.1]
for w in important_word_list:
print(w )
area = ['area1', 'area2']
toCategorical(train, area, 'area')
 toCategorical(test, area, 'area' )<count_values>
sent_list = [nltk.word_tokenize(row['text'].lower())for(item, row)in df_train.iterrows() ] + [nltk.word_tokenize(row['text'].lower())for(item, row)in df_test.iterrows() ]
for c in [x for x in train.columns if x not in ignore]:
print(c, len(set(train[c])) )<define_variables>
word_embedding = Word2Vec(sent_list, size=75, min_count=5, workers=20, window=5,iter=1,sg=1 )
cat_vars = [
'hacdor',
'hacapo',
'v14a',
'refrig',
'v18q',
'cielorazo',
'dis',
'computer',
'television',
'mobilephone',
'wall_mat',
'floor_mat',
'roof_mat',
'water_prov',
'elec_prov',
'toilet',
'cooking',
'rubbish',
'wall_quality',
'roof_quality',
'floor_quality',
'gender',
'civil_status',
'education',
'house_ownership',
'region',
'area']<categorify>
word_embedding.train(alt_corpus, total_examples=word_embedding.corpus_count, epochs=25 )
train.replace({'dependency': {'no': 0, 'yes': 1}}, inplace=True)
 test.replace({'dependency': {'no': 0, 'yes': 1}}, inplace=True )<define_variables>
word_embedding.train(sent_list, total_examples=word_embedding.corpus_count, epochs=100 )
contin_vars = [x for x in train.columns if x not in ignore and x not in cat_vars]
for c in [x for x in contin_vars]:
print(c, len(set(train[c])) )<define_variables>
important_word_list_2 = [important_word_list[0]]
for w in important_word_list[1:]:
add_word = True
for w2 in important_word_list_2:
if np.absolute(word_embedding.wv.similarity(w,w2)) >0.65:
add_word = False
if add_word:
important_word_list_2.append(w)
 for w in important_word_list_2:
print(w )
dep = 'Target'<feature_engineering>
important_word_list = important_word_list_2
test.index=test['Id']
test[dep] = 0
len(test )<data_type_conversions>
def max_similarity(wd, text):
ret = 0
words = nltk.word_tokenize(text.lower())
 for w in words:
x = 0
if w in word_embedding.wv.vocab:
x = word_embedding.wv.similarity(w,wd)
 if x>ret:
ret=x
return ret
for v in cat_vars:
train[v] = train[v].astype('category' ).cat.as_ordered()
apply_cats(test, train)
 for v in contin_vars:
train[v] = train[v].fillna(0 ).astype('float32')
 test[v] = test[v].fillna(0 ).astype('float32')
 train.head(2 )<train_model>
def contains_one(s,lst):
if not isinstance(s,str):
s = s.decode('utf-8')
 for w in lst:
if re.search(re.escape(w),s,re.IGNORECASE):
return 1
return 0
pretrain = train[train.parentesco1 == 0].copy()
train = train[train.parentesco1 == 1].copy()
train.reset_index(inplace=True)
 pretrain.reset_index(inplace=True)
 n = len(train)
 print(f'Pretraining on {len(pretrain)}, final training on {n} points' )<prepare_x_and_y>
for w in important_word_list:
df_train[w] = df_train.apply(lambda x: max_similarity(w, x['text']), axis=1)
 df_test[w] = df_test.apply(lambda x: max_similarity(w, x['text']), axis=1 )
df, y, nas, mapper = proc_df(train[cat_vars+contin_vars+[dep]], 'Target', do_scale=True )<categorify>
df_targets = df_train['target']
df_prediction = pd.DataFrame()
df_prediction['id'] = df_test['id']
df_test, _, nas, mapper = proc_df(test[cat_vars+contin_vars+[dep]], 'Target', do_scale=True,
mapper=mapper, na_dict=nas )<categorify>
model = SVC()
df_pre, yp, nas, mapper = proc_df(pretrain[cat_vars+contin_vars+[dep]], 'Target', do_scale=True,
mapper=mapper, na_dict=nas )<define_variables>
model.fit(df_train, df_targets )
cat_sz = [(c, len(train[c].cat.categories)+1)for c in cat_vars]
emb_szs = [(c, min(50,(c+1)//2)) for _,c in cat_sz]<define_variables>
predictions = model.predict(df_test )
[x for x in df_test.columns if x not in df.columns] , [x for x in df.columns if x not in df_test.columns]<compute_test_metric>
df_prediction['target'] = predictions
def inv_y(a): return np.exp(a )<compute_test_metric>
df_prediction.to_csv("predictions.csv", index=False )