Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<concatenate>
np.random.seed(2020)
all_data = train.append(test)
 train.head()<drop_column>
df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 df.head()
train = train.drop(['Soil_Type7','Soil_Type15'], axis=1 )<feature_engineering>
df['target'].value_counts()
a = all_data['Horizontal_Distance_To_Hydrology']
b = all_data['Vertical_Distance_To_Hydrology']
all_data['distance_to_hydrology'] = np.sqrt(np.power(a,2)+ np.power(b,2))
all_data['Horizontal_distance'] =(all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Roadways'] + all_data['Horizontal_Distance_To_Fire_Points'])/3
all_data['average_hillshade'] =(all_data['Hillshade_3pm'] + all_data['Hillshade_Noon'] + all_data['Hillshade_9am'])/3
all_data['Aspect_hillshade'] =(all_data['Aspect']*all_data['Hillshade_9am'])/255
all_data['slope_hillshade'] =(all_data['Slope']*all_data['Hillshade_Noon'])/255
all_data['Elevation'] = [math.floor(v/50.0)for v in all_data['Elevation']]<feature_engineering>
df[['keyword', 'location']] = df[['keyword', 'location']].astype('str')
 df.info()
all_data['EVDtH'] = all_data.Elevation-all_data.Vertical_Distance_To_Hydrology
all_data['EHDtH'] = all_data.Elevation-all_data.Horizontal_Distance_To_Hydrology*0.2
all_data['Distanse_to_Hydrolody'] =(all_data['Horizontal_Distance_To_Hydrology']**2+all_data['Vertical_Distance_To_Hydrology']**2)**0.5
all_data['Hydro_Fire_1'] = all_data['Horizontal_Distance_To_Hydrology']+all_data['Horizontal_Distance_To_Fire_Points']
all_data['Hydro_Fire_2'] = abs(all_data['Horizontal_Distance_To_Hydrology']-all_data['Horizontal_Distance_To_Fire_Points'])
 all_data['Hydro_Road_1'] = abs(all_data['Horizontal_Distance_To_Hydrology']+all_data['Horizontal_Distance_To_Roadways'])
 all_data['Hydro_Road_2'] = abs(all_data['Horizontal_Distance_To_Hydrology']-all_data['Horizontal_Distance_To_Roadways'])
 all_data['Fire_Road_1'] = abs(all_data['Horizontal_Distance_To_Fire_Points']+all_data['Horizontal_Distance_To_Roadways'])
 all_data['Fire_Road_2'] = abs(all_data['Horizontal_Distance_To_Fire_Points']-all_data['Horizontal_Distance_To_Roadways'])
<split>
nlp = spacy.load('en_core_web_lg' )
train_data = all_data.iloc[:train.shape[0],:]
test_data = all_data.iloc[train.shape[0]:,:]<feature_engineering>
with nlp.disable_pipes() :
text_vectors = np.array([nlp(text ).vector for text in df['text']] )
<normalization>
labels = df['target']
rs = RobustScaler()
rs.fit(train_data)
 train_data = rs.transform(train_data)
 test_data = rs.transform(test_data)
 train_data = pd.DataFrame(train_data, columns = all_data.columns)
 test_data = pd.DataFrame(test_data, columns = all_data.columns )<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(text_vectors, labels, test_size=0.2, random_state=3 )
<split>
mlp_model = Sequential([
Dense(64, input_dim=input_shape, activation='relu'),
Dropout(0.2),
Dense(32, activation='relu'),
Dense(1, activation='sigmoid'),
])
 mlp_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
x_train, x_test, y_train, y_test = train_test_split(train_data, y, test_size = 0.1, shuffle= True )<train_model>
mlp_model.fit(X_train, y_train,
batch_size=32,
epochs=30,
verbose=2,
callbacks=callbacks,
validation_data=(X_valid, y_valid))
def fit(model, X, y, X_test):
model.fit(X, y)
 pred = model.predict(X_test)
 return pred
def accuracy(y_actuals, y_predicted):
print(accuracy_score(y_actuals, y_predicted)) 
<compute_train_metric>
test_score = mlp_model.evaluate(X_valid, y_valid)
 print(test_score )
etc = ExtraTreesClassifier(n_estimators=400)
 print(cross_val_score(etc,x_train,y_train,cv = 5 ).mean() )<train_on_grid>
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 with nlp.disable_pipes() :
test_text_vectors = np.array([nlp(text ).vector for text in test['text']] )
<compute_train_metric><EOS> 
test_labels = mlp_model.predict_classes(test_text_vectors)
 submit_data = pd.DataFrame(test['id'])
 submit_data['target'] = test_labels
submit_data.to_csv('nlp_submit.csv', index=False )