Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<compute_test_metric>
start_time = time.time()
metrics.classification_report(val_trues, val_preds )<predict_on_test>
%matplotlib inline
test_pred = np.array(test/255.)
test_pred = test_pred.reshape(( -1,28,28,1))
test_predictions = model.predict_classes(test_pred )<save_to_csv>
!pip install tensorflow-text==2.0.0 --user
sub_df = {'ImageId':list(range(1, len(test_predictions)+ 1)) ,'Label': test_predictions}
submission = pd.DataFrame(sub_df ).astype('int')
 submission.head()
submission.to_csv('submission.csv', index=False )<import_modules>
import tensorflow as tf
import tensorflow_hub as hub
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.callbacks import LearningRateScheduler<load_from_csv>
import tensorflow_text as text
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )<feature_engineering>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
X_train=X_train/255.0
test=test/255.0<train_model>
def clean(text):
text = re.sub(r"http\S+", " ", text)
 text = re.sub(r"RT ", " ", text)
 text = re.sub(r"[^a-zA-Z
text = re.sub(r"[0-9]", " ", text)
 text = re.sub(r"\s+", " ", text)
 text = text.strip()
return text
X_train = X_train.values.reshape(-1, 28, 28, 1)
 test = test.values.reshape(-1, 28, 28, 1 )<split>
train.text = train.text.apply(clean)
 test.text = test.text.apply(clean )
x_train,x_test,y_train,y_test = train_test_split(X_train,Y_train,test_size=0.2 )<choose_model_class>
use = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3" )
data_gen = ImageDataGenerator(
 rotation_range=12,
width_shift_range=0.12,
height_shift_range=0.12,
shear_range=0.12,
validation_split=0.2, )<train_model>
X_train = []
for r in tqdm(train.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_train.append(review_emb)
 X_train = np.array(X_train)
 y_train = train.target.values
X_test = []
for r in tqdm(test.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_test.append(review_emb)
 X_test = np.array(X_test )
data_gen.fit(x_train)
 data_gen.fit(x_test )<categorify>
train_arrays, test_arrays, train_labels, test_labels = train_test_split(X_train,
y_train,
random_state =42,
test_size=0.05 )
y_train = to_categorical(y_train, num_classes=10)
 y_test = to_categorical(y_test, num_classes=10 )<choose_model_class>
def svc_param_selection(X, y, nfolds):
Cs = [1.07]
gammas = [2.075]
param_grid = {'C': Cs, 'gamma' : gammas}
grid_search = GridSearchCV(SVC(kernel='rbf'), param_grid, cv=nfolds, n_jobs=8)
 grid_search.fit(X, y)
 grid_search.best_params_
return grid_search
model = svc_param_selection(train_arrays,train_labels, 5 )
model = Sequential()
model.add(Conv2D(filters=16, kernel_size=(3, 3), activation="relu", input_shape=(28,28,1)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu'))
model.add(Dropout(0.25))
model.add(Conv2D(filters=32, kernel_size=(3, 3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))<choose_model_class>
model.best_params_
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<train_model>
pred = model.predict(test_arrays )
batch_size = 64
epochs = 15
history = model.fit_generator(data_gen.flow(x_train, y_train, batch_size = batch_size), epochs = epochs,
validation_data =(x_test, y_test), verbose=1,
steps_per_epoch=x_train.shape[0] // batch_size, 
)
<predict_on_test>
cm = confusion_matrix(test_labels,pred)
 cm
pred = np.argmax(model.predict(test), axis=1)
 sub_df = {'ImageId':list(range(1, len(test)+ 1)) ,'Label':pred}
submission = pd.DataFrame(sub_df ).astype('int')
 submission.head()<save_to_csv>
accuracy = accuracy_score(test_labels,pred)
 accuracy
submission.to_csv('submission.csv', index=False )<save_to_csv>
test_pred = model.predict(X_test)
 submission['target'] = test_pred.round().astype(int)
submission.to_csv('submission.csv', index=False )<load_from_csv>
count = 2
prob_disaster = 0.9
keyword_list_disaster = list(ag[(ag['Count']>count)&(ag['Disaster Probability']>=prob_disaster)].index)
 keyword_list_disaster
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 train.head()<import_modules>
ids_disaster = test['id'][test.keyword.isin(keyword_list_disaster)].values
submission['target'][submission['id'].isin(ids_disaster)] = 1
<prepare_x_and_y><EOS> 
submission.to_csv("submission.csv", index=False)
 submission.head(10 )