Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<choose_model_class>
!pip install pyspellchecker
net = make_model(len(FEATS))
ckpt = ModelCheckpoint("w.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.001)
 es = EarlyStopping(monitor='val_loss', patience=4)
 print(net.summary() )<train_model>
np.random.seed(31415)
 spell = spc.SpellChecker()
net.fit(xt, yt, batch_size=50_000, epochs=75, validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es] )<load_pretrained>
def add_location_to_text(row):

 if row['location'] is not np.nan:
return row['text']+" "+row['location']
return row['text']
def add_keywords_to_text(row):

 if row['keyword'] is not np.nan:
return row['text']+" "+row['keyword']
return row['text']
def add_sp1(text):

 def prepocess_text_for_spell(words):
words = list(filter(lambda x: len(x)>0, words))
return list(filter(lambda x:
not x.startswith("
x[0] != x[0].capitalize() , words))
words = prepocess_text_for_spell(text.split(" "))
return len(spell.unknown(words))
def add_wc(text):

 return len(text.split(' '))
def number_hash(text):

 words = list(filter(lambda x: len(x)>0, text.split(' ')))
 return len(list(filter(lambda x: x.startswith('
def number_of_chars(text):

 return len(text)
 def has_keyword(keyword):

 return int(not keyword is np.nan)
 def has_location(location):

 return int(not location is np.nan)
 def apply_all_feature(df):

 df.loc[:,'sp1'] = df['text'].apply(add_sp1 ).values
df.loc[:,'wc'] = df['text'].apply(add_wc ).values
df.loc[:,'hst'] = df['text'].apply(number_hash ).values
df.loc[:,'ch'] = df['text'].apply(number_of_chars ).values
df.loc[:,'loc'] = df['location'].apply(has_location ).values
df.loc[:,'text'] = df.apply(add_location_to_text,axis=1)
 df.loc[:,'text'] = df.apply(add_keywords_to_text,axis=1)
 df.loc[:,'hl'] = df.apply(has_location,axis=1)
 df.loc[:,'hk'] = df.apply(has_keyword,axis=1)
 def build_features(df_train,df_test,use_extra=True,use_nmf=False):

 Y_train = df_train['target'].values
n_train = len(Y_train)
 n_test = len(df_test)
 df_train = df_train.drop('target',axis=1)
 vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,stop_words='english',strip_accents='ascii',min_df=0.,ngram_range=(1,2))
df = df_train.append(df_test, ignore_index = True, sort=False)
 assert len(df)==n_train+n_test
df.loc[:,'text'] = df['text'].apply(lambda x: x.lower())
 X = vectorizer.fit_transform(df['text'])
 if use_extra:
apply_all_feature(df)
 X_custom = sklearn.preprocessing.scale(df[['sp1','wc','hst','ch','loc']].values,with_mean=False)
 X = scipy.sparse.hstack(( X,X_custom)).tocsr()
if use_nmf:
model = NMF(n_components=15, init='random', random_state=0)
 X = scipy.sparse.hstack(( X,model.fit_transform(X),model.fit_transform(X.log1p())) ).tocsr()
X_train = X[:n_train]
X_test = X[n_train:,:]
if 'target' in df_test.columns:
Y_test = df_test['target']
return X_train,X_test,Y_train,Y_test
else:
return X_train,X_test,Y_train,None
nett = make_model(len(FEATS))
nett.load_weights("w.h5" )<predict_on_test>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 df_train,df_test = sklearn.model_selection.train_test_split(train,test_size=0.33)
 X_train,X_test,Y_train,Y_test = build_features(df_train,df_test,use_extra=True,use_nmf=True )
pv = nett.predict(xv, batch_size=50_000, verbose=1)
 pe = nett.predict(xe, batch_size=50_000, verbose=1 )<compute_test_metric>
rreg = RidgeClassifier(alpha=.8,solver='sag')
 lass = Lasso(alpha=.01)
 rf = RandomForestClassifier(max_depth=20, random_state=0, n_estimators=100)
 svm = LinearSVC(penalty='l2',max_iter=10000)
 svm = SVC(kernel='linear',max_iter=10000 )
nett.evaluate(xv, yv, batch_size=50_000 )<define_variables>
rreg.fit(X_train, Y_train)
 Y_pred = rreg.predict(X_test)
 print(sklearn.metrics.classification_report(Y_test,Y_pred))
names = [f"F{i+1}" for i in range(28)]<create_dataframe>
lass.fit(X_train, Y_train)
 Y_pred = list(map(lambda x: x>=.5, lass.predict(X_test)))
 print(sklearn.metrics.classification_report(Y_test,Y_pred))
piv = pd.DataFrame(ids[:, 0], columns=["id"] )<feature_engineering>
rf.fit(X_train, Y_train)
 Y_pred = rf.predict(X_test)
 print(sklearn.metrics.classification_report(Y_test,Y_pred))
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
<create_dataframe>
svm.fit(X_train, Y_train)
 Y_pred = svm.predict(X_test)
 print(sklearn.metrics.classification_report(Y_test,Y_pred))
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2<save_to_csv>
x_train,x_test,y_train,y_test = build_features(train,
test,
use_extra = True,
use_nmf = True)
 kf = KFold(n_splits=5 )
sub.to_csv("submission.csv", index=False )<import_modules>
F = []
A = np.round(np.linspace(0,1,11),2)
 for a in tqdm(A):
rreg = RidgeClassifier(alpha=a,solver='sag')
 f1 = []
for i,(train_index, test_index)in enumerate(kf.split(x_train)) :
kf_x_train, kf_y_train = x_train[train_index], y_train[train_index]
kf_x_val, kf_y_val = x_train[test_index], y_train[test_index]
rreg.fit(kf_x_train, kf_y_train)
 y_pred = rreg.predict(kf_x_val)
 res = sklearn.metrics.classification_report(kf_y_val,y_pred, output_dict=True)
 f1.append(res['1']['f1-score'])
 F.append(f1 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc
import time<categorify>
X_train,X_test,Y_train,Y_test = build_features(train,test,use_extra=True,use_nmf=True )
<create_dataframe><EOS> 
rreg = RidgeClassifier(alpha=.3,solver='sag')
 rreg.fit(X_train, Y_train)
 sub = rreg.predict(X_test)
 pd.DataFrame(np.array([test['id'].values,sub] ).T,columns=['id','target'] ).to_csv('submission.csv',sep=',',
header=True,
index=False )