Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_pretrained>
print(keras.__version__)
 print(tensorflow.__version__)
 EMBEDDING_FILES = [
'.. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim',
'.. /input/gensim-embeddings-dataset/glove.840B.300d.gensim'
]
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 20
MAX_LEN = 220
TEXT_COLUMN = 'text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
def prepare_data(df, config, data_dir):
X = np.empty(shape=(df.shape[0], config.dim[0], config.dim[1], 1))
input_length = config.audio_length
for i, fname in enumerate(df.index):
print(fname)
 file_path = data_dir + fname
data, _ = librosa.core.load(file_path, sr=config.sampling_rate, res_type="kaiser_fast")
 if len(data)> input_length:
max_offset = len(data)- input_length
offset = np.random.randint(max_offset)
 data = data[offset:(input_length+offset)]
else:
if input_length > len(data):
max_offset = input_length - len(data)
 offset = np.random.randint(max_offset)
 else:
offset = 0
data = np.pad(data,(offset, input_length - len(data)- offset), "constant")
 data = librosa.feature.mfcc(data, sr=config.sampling_rate, n_mfcc=config.n_mfcc)
 data = np.expand_dims(data, axis=-1)
 X[i,] = data
return X<save_to_csv>
print("Read Data")
 train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train_df.head(5 )
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-data-2d-conv-reduced-lr/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("2d_conv_ensembled_submission.csv", index=False )<load_from_csv>
def remove_urls(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 def process_tweets(tweets):
tweets = remove_urls(tweets)
 tweets = remove_html(tweets)
 tweets = remove_punct(tweets)
 return tweets
train_df["text"] = train_df["text"].map(process_tweets)
 test_df["text"] = test_df["text"].map(process_tweets )
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-data-2d-conv-reduced-lr/test_predictions_%d.npy"%i))
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-file/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("1d_2d_ensembled_submission.csv", index=False )<load_from_csv>
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
x_test = test_df[TEXT_COLUMN].astype(str)
 tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE, lower=False)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 checkpoint_predictions = []
weights = []
def build_matrix(word_index, path):
embedding_index = KeyedVectors.load(path, mmap='r')
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
for candidate in [word, word.lower() ]:
if candidate in embedding_index:
embedding_matrix[i] = embedding_index[candidate]
break
return embedding_matrix
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1)
 def build_model(embedding_matrix):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.5 )(x)
 x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True, dropout=0.3, recurrent_dropout=0.3))(x)
 x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True, dropout=0.3, recurrent_dropout=0.3))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x)
 ])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=result)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
loc_test = ".. /input/test.csv"
loc_train = ".. /input/train.csv"
loc_submission = "forest-cover-type-prediction.AspiringGuru.csv"
df_test = pd.read_csv(loc_test)
 df_train = pd.read_csv(loc_train)
 print("type(df_test)=", type(df_test), "df_test.shape=", df_test.shape)
 print("type(df_train)=", type(df_train), "df_train.shape=", df_train.shape)
 feature_cols = [col for col in df_train.columns if col not in ['Cover_Type', 'Id']]
X_train = df_train[feature_cols]
X_test = df_test[feature_cols]
train_y = df_train['Cover_Type']
test_ids = df_test['Id']
del df_train
del df_test
print("creating classifier")
 start_time = time.time()
clf = ensemble.RandomForestClassifier(n_estimators=200, n_jobs=-1, random_state=0)
 print("--- time to build ensemble.RandomForestClassifier %s seconds ---" %(time.time() - start_time))
print("fitting from train data")
 start_time = time.time()
clf.fit(X_train, train_y)
 print("--- time to clf.fit %s seconds ---" %(time.time() - start_time))
print("predicting from train data")
 start_time = time.time()
train_y_predicted = clf.predict(X_train)
 print("--- time to clf.predict %s seconds ---" %(time.time() - start_time))
print("type(train_y_predicted)=", type(train_y_predicted), "len(train_y_predicted)", len(train_y_predicted), "train_y_predicted.shape", train_y_predicted.shape)
 print("type(train_y)=", type(train_y), "train_y.shape=", train_y.shape)
 print("train_y = ", list(train_y[0:20, ]))
print("train_y_predicted = ", list(train_y_predicted[0:20, ]))
print("predicting from test data")
 start_time = time.time()
predicted = clf.predict(X_test)
 print("--- time to clf.predict %s seconds ---" %(time.time() - start_time))
print("type(predicted)=", type(predicted), "len(predicted)", len(predicted))
print("type(test_ids)=", type(test_ids), "len(test_ids)=", len(test_ids))
print("calculating accuracy_score on train data.")
 start_time = time.time()
score = accuracy_score(train_y, train_y_predicted)
 print("--- time to calcualte accuracy_score %s seconds ---" %(time.time() - start_time))
print("type(score)=", type(score), "score=", score)
 print("clf.predicting & writing to file")
 start_time = time.time()
with open(loc_submission, "w")as outfile:
outfile.write("Id,Cover_Type
")
 for e, val in enumerate(list(clf.predict(X_test))):
outfile.write("%s,%s
" %(test_ids[e], val))
print("--- time to clf.predict & write to file %s seconds ---" %(time.time() - start_time)) 
<load_from_csv>
print("Start Modeling")
 model = build_model(embedding_matrix)
 model.fit(
 x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_split=0.2,
verbose=1 
)
 print("Modeling Complete" )
<groupby><EOS> 
predictions = model.predict(x_test)
 submission = pd.DataFrame.from_dict({
'id': test_df.id,
TARGET_COLUMN: np.round(predictions ).astype(int ).reshape(x_test.shape[0])
 })
 submission.to_csv('lstm_submission.csv', index=False)
 submission.head()