Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<normalization>
random_state_split = 7
Dropout_num = 0
learning_rate = 6e-6
valid = 0.2
epochs_num = 3
batch_size_num = 16
target_corrected = False
target_big_corrected = False
def audio_norm(data):
max_data = np.max(data)
 min_data = np.min(data)
 data =(data-min_data)/(max_data-min_data+1e-6)
 return data-0.5<choose_model_class>
plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
def get_1d_dummy_model(config):
nclass = config.n_classes
input_length = config.audio_length
inp = Input(shape=(input_length,1))
x = GlobalMaxPool1D()(inp)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model
def get_1d_conv_model(config):
nclass = config.n_classes
input_length = config.audio_length
inp = Input(shape=(input_length,1))
x = Convolution1D(16, 9, activation=relu, padding="valid" )(inp)
 x = Convolution1D(16, 9, activation=relu, padding="valid" )(x)
 x = MaxPool1D(16 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = MaxPool1D(4 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = MaxPool1D(4 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(256, 3, activation=relu, padding="valid" )(x)
 x = Convolution1D(256, 3, activation=relu, padding="valid" )(x)
 x = GlobalMaxPool1D()(x)
 x = Dropout(rate=0.2 )(x)
 x = Dense(64, activation=relu )(x)
 x = Dense(1028, activation=relu )(x)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model<feature_engineering>
mpl.rc("savefig", dpi=1200 )
LABELS = list(train.label.unique())
 label_idx = {label: i for i, label in enumerate(LABELS)}
train.set_index("fname", inplace=True)
 test.set_index("fname", inplace=True)
 train["label_idx"] = train.label.apply(lambda x: label_idx[x])
 if not COMPLETE_RUN:
train = train[:2000]
test = test[:2000]<set_options>
tweet= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv')
 submission= pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv' )
config = Config(sampling_rate=16000, audio_duration=2, n_folds=10, learning_rate=0.001)
 if not COMPLETE_RUN:
config = Config(sampling_rate=100, audio_duration=1, n_folds=2, max_epochs=1 )<save_to_csv>
Real_len = tweet[tweet['target'] == 1].shape[0]
Not_len = tweet[tweet['target'] == 0].shape[0]
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-file/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("1d_conv_ensembled_submission.csv", index=False )<load_pretrained>
tweet['length'] = tweet['text'].apply(length )
SAMPLE_RATE = 44100
fname = '.. /input/freesound-audio-tagging/audio_train/' + '00044347.wav'
wav, _ = librosa.core.load(fname, sr=SAMPLE_RATE)
 wav = wav[:2*44100]<compute_test_metric>
import matplotlib.ticker as ticker
mfcc = librosa.feature.mfcc(wav, sr = SAMPLE_RATE, n_mfcc=40)
 mfcc.shape<import_modules>
def create_corpus(target):
corpus=[]
for x in tweet[tweet['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
GlobalMaxPool2D, MaxPool2D, concatenate, Activation)
<choose_model_class>
corpus=create_corpus(0)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)[:10]
def get_2d_dummy_model(config):
nclass = config.n_classes
inp = Input(shape=(config.dim[0],config.dim[1],1))
x = GlobalMaxPool2D()(inp)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model
def get_2d_conv_model(config):
nclass = config.n_classes
inp = Input(shape=(config.dim[0],config.dim[1],1))
x = Convolution2D(32,(4,10), padding="same" )(inp)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(32,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(32,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Convolution2D(32,(4,10), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = MaxPool2D()(x)
 x = Flatten()(x)
 x = Dense(64 )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model<set_options>
plt.figure(figsize=(16,5))
counter=Counter(corpus)
 most=counter.most_common()
x=[]
y=[]
for word,count in most[:40]:
if(word not in stop):
x.append(word)
 y.append(count )
config = Config(sampling_rate=44100, audio_duration=2, n_folds=10,
learning_rate=0.001, use_mfcc=True, n_mfcc=40)
 if not COMPLETE_RUN:
config = Config(sampling_rate=44100, audio_duration=2, n_folds=2,
max_epochs=1, use_mfcc=True, n_mfcc=40 )<load_pretrained>
def get_top_tweet_bigrams(corpus, n=None):
vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:n]
def prepare_data(df, config, data_dir):
X = np.empty(shape=(df.shape[0], config.dim[0], config.dim[1], 1))
input_length = config.audio_length
for i, fname in enumerate(df.index):
print(fname)
 file_path = data_dir + fname
data, _ = librosa.core.load(file_path, sr=config.sampling_rate, res_type="kaiser_fast")
 if len(data)> input_length:
max_offset = len(data)- input_length
offset = np.random.randint(max_offset)
 data = data[offset:(input_length+offset)]
else:
if input_length > len(data):
max_offset = input_length - len(data)
 offset = np.random.randint(max_offset)
 else:
offset = 0
data = np.pad(data,(offset, input_length - len(data)- offset), "constant")
 data = librosa.feature.mfcc(data, sr=config.sampling_rate, n_mfcc=config.n_mfcc)
 data = np.expand_dims(data, axis=-1)
 X[i,] = data
return X<save_to_csv>
df=pd.concat([tweet,test])
 df.shape
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-data-2d-conv-reduced-lr/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("2d_conv_ensembled_submission.csv", index=False )<load_from_csv>
example="New competition launched :https://www.kaggle.com/c/nlp-getting-started"
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-data-2d-conv-reduced-lr/test_predictions_%d.npy"%i))
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-file/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("1d_2d_ensembled_submission.csv", index=False )<set_options>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 remove_URL(example )
%matplotlib inline
tqdm_notebook().pandas(smoothing=0.7)
 mpl.rcParams['font.size'] = 14<define_variables>
df['text']=df['text'].apply(lambda x : remove_URL(x))
cache = False
run_full_notebook = False
train_root = '.. /input/audio_train/'
test_root = '.. /input/audio_test/'
if cache:
train_root_trimmed = '.. /input/audio_train_trimmed/'
test_root_trimmed = '.. /input/audio_test_trimmed/'
os.makedirs('.. /input/audio_train_trimmed', exist_ok=True)
 os.makedirs('.. /input/audio_test_trimmed', exist_ok=True)
 os.makedirs('.. /cache', exist_ok=True)
 os.makedirs('.. /output', exist_ok=True)
 else:
train_root_trimmed = train_root
test_root_trimmed = test_root<load_from_csv>
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 print(remove_html(example))
test_df = pd.read_csv(".. /input/sample_submission.csv")
 train_df = pd.read_csv(".. /input/train.csv")
 train_df.head()<count_unique_values>
df['text']=df['text'].apply(lambda x : remove_html(x))
n_test = test_df.shape[0]
n_training = train_df.shape[0]
n_categories = len(train_df.label.unique())
 print("Number of training examples: {}".format(n_training))
print("Number of testing examples: {}".format(n_test))
print("Number of unique categories: {}".format(n_categories))<normalization>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 remove_emoji("Omg another Earthquake 😔😔" )
random_int = random.randint(0, n_training)
 random_wavfile = train_df.fname.iloc[random_int]
print(train_df.label.iloc[random_int])
 play_audio(random_wavfile )<compute_test_metric>
df['text']=df['text'].apply(lambda x: remove_emoji(x))
def wavfile_stats(fname, root):
try:
data, fs = librosa.core.load(root + fname, sr=None)
 mean = np.mean(data)
 minimum = np.min(data)
 maximum = np.max(data)
 std = np.std(data)
 length = len(data)
 rms = np.sqrt(np.mean(data**2))
skewness = skew(data)
 kurt = kurtosis(data)
 return pd.Series([length, mean, minimum, maximum, std, rms, skewness, kurt])
 except ValueError:
print("Bad file at {}".format(fname))
return pd.Series([0, 0, 0, 0, 0, 0, 0, 0] )<load_from_csv>
def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 example="I am a
print(remove_punct(example))
if os.path.isfile('.. /cache/train_1.csv')and cache:
train_df = pd.read_csv('.. /cache/train_1.csv')
 test_df = pd.read_csv('.. /cache/test_1.csv')
 assert len(train_df.index)== n_training
assert len(test_df.index)== n_test
print("Files loaded from cache")
 else:
train_df[['length', 'data_mean', 'data_min', 'data_max', 'data_std', 'data_rms', 'skewness', 'kurtosis']] = \
train_df['fname'].progress_apply(wavfile_stats, root=train_root)
 test_df[['length', 'data_mean', 'data_min', 'data_max', 'data_std', 'data_rms', 'skewness', 'kurtosis']] = \
test_df['fname'].progress_apply(wavfile_stats, root=test_root)
 if cache:
train_df.to_csv('.. /cache/train_1.csv', index=False)
 test_df.to_csv('.. /cache/test_1.csv', index=False )<feature_engineering>
df['text']=df['text'].apply(lambda x : remove_punct(x))
train_df['rms_std'] = train_df['data_rms'] / train_df['data_std']
test_df['rms_std'] = test_df['data_rms'] / test_df['data_std']
train_df['max_min'] = train_df['data_max'] / train_df['data_min']
test_df['max_min'] = test_df['data_max'] / test_df['data_min']<correct_missing_values>
def cv(data):
count_vectorizer = CountVectorizer()
emb = count_vectorizer.fit_transform(data)
 return emb, count_vectorizer
list_corpus = df["text"].tolist()
list_labels = df["target"].tolist()
X_train, X_test, y_train, y_test = train_test_split(list_corpus, list_labels, test_size=0.2, random_state=random_state_split)
 X_train_counts, count_vectorizer = cv(X_train)
 X_test_counts = count_vectorizer.transform(X_test )
test_df.fillna(0, inplace=True )<split>
def tfidf(data):
tfidf_vectorizer = TfidfVectorizer()
train = tfidf_vectorizer.fit_transform(data)
 return train, tfidf_vectorizer
X_train_tfidf, tfidf_vectorizer = tfidf(X_train)
 X_test_tfidf = tfidf_vectorizer.transform(X_test )
def data_split(train_df, test_df, shuffle=True, test_size=0.25, random_state=0, verbose=True):
X_train = train_df.drop(['fname', 'label', 'manually_verified'], axis=1 ).values
X_test = test_df.drop(['fname', 'label'], axis=1 ).values
feature_names = list(test_df.drop(['fname', 'label'], axis=1 ).columns.values)
 y_train = pd.get_dummies(train_df.label)
 labels = y_train.columns.values
y_train = y_train.values
y_train = [np.argmax(row)for row in y_train]
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=test_size,
random_state=random_state, shuffle=shuffle)
 if verbose:
print("Train X shape = {}
Train y shape = {}".format(X_train.shape, len(y_train)))
 print("
Valid X shape = {}
Valid y shape = {}".format(X_valid.shape, len(y_valid)))
 assert X_train.shape[1] == X_valid.shape[1] == X_test.shape[1]
assert len(y_train)== X_train.shape[0]
assert len(y_valid)== X_valid.shape[0]
return X_train, X_valid, y_train, y_valid, X_test, feature_names, labels<split>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.metrics import classification_report, accuracy_score,make_scorer
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = data_split(train_df, test_df )<create_dataframe>
df_train = df[:7612]
df_test = df[7613:10875]
def lgb_dset(X_train, X_valid, y_train, y_valid, feature_names):
d_train = lgb.Dataset(X_train, label=y_train, feature_name=feature_names)
 d_valid = lgb.Dataset(X_valid, label=y_valid, feature_name=feature_names)
 params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': 5,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'num_threads': os.cpu_count() ,
'lambda_l2': 1.0,
'min_gain_to_split': 0,
'num_class': n_categories,
}
return d_train, d_valid, params<train_model>
X = df_train['text']
y = df_train['target']
d_train, d_valid, params = lgb_dset(X_train, X_valid, y_train, y_valid, feature_names)
 clf = lgb.train(params, d_train, num_boost_round=500, valid_sets=d_valid, verbose_eval=100, early_stopping_rounds=100 )<compute_test_metric>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1,random_state=42 )
def apk(actual, predicted, k=10):
if len(predicted)>k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i,p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i+1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=10):
return np.mean([apk(a,p,k)for a,p in zip(actual, predicted)] )<predict_on_test>
vectorizer=TfidfVectorizer(ngram_range=(1,3),min_df=3,strip_accents='unicode', use_idf=1,smooth_idf=1, sublinear_tf=1,max_features=None)
 vectorizer.fit(list(df['text']))
print('vocab length',len(vectorizer.vocabulary_))
p = clf.predict(X_valid, num_iteration=clf.best_iteration)
 predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
actual = [[i] for i in y_valid]
valid_score = mapk(actual, predictions, k=3)
 print("Score = {:.4f}".format(valid_score))<categorify>
X_train_onehot = vectorizer.transform(X_train ).todense()
X_val_onehot = vectorizer.transform(X_val ).todense()
def preds_to_labels(p, labels):
predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
prediction_labels = []
for pred in predictions:
label_list = []
for output in pred:
label_list.append(labels[output])
 prediction_labels.append(label_list)
 return prediction_labels
lab = preds_to_labels(p, labels)
 t = PrettyTable(['Truth', 'Prediction'])
 [t.add_row([labels[l[1][0]], l[0]])for l in zip(lab[:10], actual[:10])]
print(t )<predict_on_test>
lr_clf = LogisticRegression(max_iter=150,
penalty='l2',
solver='lbfgs',
random_state=0)
 lr_clf.fit(X_train_onehot, y_train)
 lr_pred = lr_clf.predict(X_val_onehot)
 print('accuracy score: ',accuracy_score(lr_pred,y_val))
print(classification_report(y_val, lr_pred))
preds = clf.predict(X_test, num_iteration=clf.best_iteration)
 lab = preds_to_labels(preds, labels )<normalization>
def create_corpus_new(df):
corpus=[]
for tweet in tqdm(df['text']):
words=[word.lower() for word in word_tokenize(tweet)]
corpus.append(words)
 return corpus
random_int = random.randint(0, n_test)
 random_wavfile = test_df.fname.iloc[random_int]
print(lab[random_int])
 play_audio(test_df.fname.iloc[random_int], dset='test' )<save_to_csv>
corpus=create_corpus_new(df )
def create_submission(predictions, name='submission.csv'):
predictions = ['{} {} {}'.format(x[0], x[1], x[2])for x in predictions]
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission.label = predictions
submission.to_csv('{}'.format(name), index=False)
 print("Submission saved to '{}'".format(name))
if run_full_notebook:
create_submission(lab, 'submission-{:.4f}.csv'.format(valid_score))<string_transform>
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word = values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()
def trim_silence(fname, root, window_length=0.5):
try:
trimmed_ends = 0
trimmed_int = 0
data, fs = librosa.core.load(root + fname, sr=None)
 length = len(data)
 data, _ = librosa.effects.trim(data, top_db=40)
 length_int = len(data)
 ratio_int = length_int/length
splits = librosa.effects.split(data, top_db=40)
 if len(splits)> 1:
data = np.concatenate([data[x[0]:x[1]] for x in splits])
 length_final = len(data)
 ratio_final = length_final/length_int
if cache:
librosa.output.write_wav('{}_trimmed/{}'.format(root[:-1], fname), data, fs)
 return pd.Series([length_int, length_final, ratio_int, ratio_final])
 except ValueError:
print("Bad file at {}".format(fname))
return pd.Series([0, 0, 0, 0] )<load_from_csv>
MAX_LEN=50
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )
if os.path.isfile('.. /cache/train_2.csv')and cache:
train_df = pd.read_csv('.. /cache/train_2.csv')
 test_df = pd.read_csv('.. /cache/test_2.csv')
 assert len(train_df.index)== n_training
assert len(test_df.index)== n_test
print("Files loaded from cache")
 else:
train_df[['length_int', 'length_final', 'ratio_int', 'ratio_final']] = \
train_df['fname'].progress_apply(trim_silence, root=train_root)
 test_df[['length_int', 'length_final', 'ratio_int', 'ratio_final']] = \
test_df['fname'].progress_apply(trim_silence, root=test_root)
 if cache:
train_df.to_csv('.. /cache/train_2.csv', index=False)
 test_df.to_csv('.. /cache/test_2.csv', index=False )<train_model>
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))
if run_full_notebook:
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = data_split(train_df, test_df)
 d_train, d_valid, params = lgb_dset(X_train, X_valid, y_train, y_valid, feature_names)
 clf = lgb.train(params, d_train, num_boost_round=1000, valid_sets=d_valid, verbose_eval=100, early_stopping_rounds=100 )<predict_on_test>
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i < num_words:
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
if run_full_notebook:
p = clf.predict(X_valid, num_iteration=clf.best_iteration)
 predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
actual = [[i] for i in y_valid]
valid_score = mapk(actual, predictions, k=3)
 print("Score = {:.4f}".format(valid_score))<save_to_csv>
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=3e-4)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'] )
if os.path.isfile('.. /cache/train_spectral.csv')and cache:
train_df = pd.read_csv('.. /cache/train_spectral.csv')
 test_df = pd.read_csv('.. /cache/test_spectral.csv')
 assert len(train_df.index)== n_training
assert len(test_df.index)== n_test
print("Files loaded from cache")
 else:
feature_names = spectral_features(return_fnames=True)
 train_df[feature_names] = train_df['fname'].progress_apply(spectral_features, root=train_root_trimmed)
 test_df[feature_names] = test_df['fname'].progress_apply(spectral_features, root=test_root_trimmed)
 if cache:
train_df.to_csv('.. /cache/train_spectral.csv', index=False)
 test_df.to_csv('.. /cache/test_spectral.csv', index=False )<predict_on_test>
train=tweet_pad[:tweet.shape[0]]
test=tweet_pad[tweet.shape[0]:]
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = data_split(train_df, test_df, verbose=False)
 d_train, d_valid, params = lgb_dset(X_train, X_valid, y_train, y_valid, feature_names)
 clf = lgb.train(params, d_train, num_boost_round=2000, valid_sets=d_valid, verbose_eval=200, early_stopping_rounds=100)
 p = clf.predict(X_valid, num_iteration=clf.best_iteration)
 predictions = [list(np.argsort(p[i])[::-1][:3])for i in range(len(p)) ]
actual = [[i] for i in y_valid]
valid_score = mapk(actual, predictions, k=3)
 print("
Score = {:.4f}".format(valid_score))<predict_on_test>
X_train,X_test,y_train,y_test=train_test_split(train,tweet['target'].values,test_size=0.2)
 print('Shape of train',X_train.shape)
 print("Shape of Validation ",X_test.shape )
p = clf.predict(X_test, num_iteration=clf.best_iteration)
 lab = preds_to_labels(p, labels)
 create_submission(lab, 'submission-{:.4f}.csv'.format(valid_score))<init_hyperparams>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
if run_full_notebook:
params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': 5,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'num_threads': os.cpu_count() ,
'lambda_l2': 1.0,
'min_gain_to_split': 0,
'num_class': n_categories,
}
X_train, X_valid, y_train, y_valid, X_test, feature_names, labels = \
data_split(train_df, test_df, test_size=0)
 d_train = lgb.Dataset(X_train, label=y_train, feature_name=feature_names )<find_best_model_class>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
if run_full_notebook:
print("Begin training...")
 clf = lgb.train(params, d_train, num_boost_round=1135)
 print("Begin test predictions...")
 p = clf.predict(X_test)
 lab = preds_to_labels(p, labels)
 create_submission(lab, 'submission-test.csv')
 print("Submission created.")
<define_variables>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
COMPLETE_RUN = False<set_options>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
if Dropout_num == 0:
out = Dense(1, activation='sigmoid' )(clf_output)
 else:
x = Dropout(Dropout_num )(clf_output)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
 return model
np.random.seed(1001)
 %matplotlib inline
matplotlib.style.use('ggplot' )<load_from_csv>
def clean_tweets(tweet):

 tweet = ''.join([x for x in tweet if x in string.printable])
 tweet = re.sub(r"http\S+", "", tweet)
 return tweet
train = pd.read_csv(".. /input/freesound-audio-tagging/train.csv")
 test = pd.read_csv(".. /input/freesound-audio-tagging/sample_submission.csv" )<count_unique_values>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )
print("Number of training examples=", train.shape[0], " Number of classes=", len(train.label.unique()))<count_values>
def remove_punctuations(text):
punctuations = '@
for p in punctuations:
text = text.replace(p, f' {p} ')
 text = text.replace('...', '...')
 if '...' not in text:
text = text.replace('.. ', '...')
 return text
print('Minimum samples per category = ', min(train.label.value_counts()))
print('Maximum samples per category = ', max(train.label.value_counts()))<load_pretrained>
abbreviations = {
"$" : " dollar ",
"€" : " euro ",
"4ao" : "for adults only",
"a.m" : "before midday",
"a3" : "anytime anywhere anyplace",
"aamof" : "as a matter of fact",
"acct" : "account",
"adih" : "another day in hell",
"afaic" : "as far as i am concerned",
"afaict" : "as far as i can tell",
"afaik" : "as far as i know",
"afair" : "as far as i remember",
"afk" : "away from keyboard",
"app" : "application",
"approx" : "approximately",
"apps" : "applications",
"asap" : "as soon as possible",
"asl" : "age, sex, location",
"atk" : "at the keyboard",
"ave." : "avenue",
"aymm" : "are you my mother",
"ayor" : "at your own risk",
"b&b" : "bed and breakfast",
"b+b" : "bed and breakfast",
"b.c" : "before christ",
"b2b" : "business to business",
"b2c" : "business to customer",
"b4" : "before",
"b4n" : "bye for now",
"b@u" : "back at you",
"bae" : "before anyone else",
"bak" : "back at keyboard",
"bbbg" : "bye bye be good",
"bbc" : "british broadcasting corporation",
"bbias" : "be back in a second",
"bbl" : "be back later",
"bbs" : "be back soon",
"be4" : "before",
"bfn" : "bye for now",
"blvd" : "boulevard",
"bout" : "about",
"brb" : "be right back",
"bros" : "brothers",
"brt" : "be right there",
"bsaaw" : "big smile and a wink",
"btw" : "by the way",
"bwl" : "bursting with laughter",
"c/o" : "care of",
"cet" : "central european time",
"cf" : "compare",
"cia" : "central intelligence agency",
"csl" : "can not stop laughing",
"cu" : "see you",
"cul8r" : "see you later",
"cv" : "curriculum vitae",
"cwot" : "complete waste of time",
"cya" : "see you",
"cyt" : "see you tomorrow",
"dae" : "does anyone else",
"dbmib" : "do not bother me i am busy",
"diy" : "do it yourself",
"dm" : "direct message",
"dwh" : "during work hours",
"e123" : "easy as one two three",
"eet" : "eastern european time",
"eg" : "example",
"embm" : "early morning business meeting",
"encl" : "enclosed",
"encl." : "enclosed",
"etc" : "and so on",
"faq" : "frequently asked questions",
"fawc" : "for anyone who cares",
"fb" : "facebook",
"fc" : "fingers crossed",
"fig" : "figure",
"fimh" : "forever in my heart",
"ft." : "feet",
"ft" : "featuring",
"ftl" : "for the loss",
"ftw" : "for the win",
"fwiw" : "for what it is worth",
"fyi" : "for your information",
"g9" : "genius",
"gahoy" : "get a hold of yourself",
"gal" : "get a life",
"gcse" : "general certificate of secondary education",
"gfn" : "gone for now",
"gg" : "good game",
"gl" : "good luck",
"glhf" : "good luck have fun",
"gmt" : "greenwich mean time",
"gmta" : "great minds think alike",
"gn" : "good night",
"g.o.a.t" : "greatest of all time",
"goat" : "greatest of all time",
"goi" : "get over it",
"gps" : "global positioning system",
"gr8" : "great",
"gratz" : "congratulations",
"gyal" : "girl",
"h&c" : "hot and cold",
"hp" : "horsepower",
"hr" : "hour",
"hrh" : "his royal highness",
"ht" : "height",
"ibrb" : "i will be right back",
"ic" : "i see",
"icq" : "i seek you",
"icymi" : "in case you missed it",
"idc" : "i do not care",
"idgadf" : "i do not give a damn fuck",
"idgaf" : "i do not give a fuck",
"idk" : "i do not know",
"ie" : "that is",
"i.e" : "that is",
"ifyp" : "i feel your pain",
"IG" : "instagram",
"iirc" : "if i remember correctly",
"ilu" : "i love you",
"ily" : "i love you",
"imho" : "in my humble opinion",
"imo" : "in my opinion",
"imu" : "i miss you",
"iow" : "in other words",
"irl" : "in real life",
"j4f" : "just for fun",
"jic" : "just in case",
"jk" : "just kidding",
"jsyk" : "just so you know",
"l8r" : "later",
"lb" : "pound",
"lbs" : "pounds",
"ldr" : "long distance relationship",
"lmao" : "laugh my ass off",
"lmfao" : "laugh my fucking ass off",
"lol" : "laughing out loud",
"ltd" : "limited",
"ltns" : "long time no see",
"m8" : "mate",
"mf" : "motherfucker",
"mfs" : "motherfuckers",
"mfw" : "my face when",
"mofo" : "motherfucker",
"mph" : "miles per hour",
"mr" : "mister",
"mrw" : "my reaction when",
"ms" : "miss",
"mte" : "my thoughts exactly",
"nagi" : "not a good idea",
"nbc" : "national broadcasting company",
"nbd" : "not big deal",
"nfs" : "not for sale",
"ngl" : "not going to lie",
"nhs" : "national health service",
"nrn" : "no reply necessary",
"nsfl" : "not safe for life",
"nsfw" : "not safe for work",
"nth" : "nice to have",
"nvr" : "never",
"nyc" : "new york city",
"oc" : "original content",
"og" : "original",
"ohp" : "overhead projector",
"oic" : "oh i see",
"omdb" : "over my dead body",
"omg" : "oh my god",
"omw" : "on my way",
"p.a" : "per annum",
"p.m" : "after midday",
"pm" : "prime minister",
"poc" : "people of color",
"pov" : "point of view",
"pp" : "pages",
"ppl" : "people",
"prw" : "parents are watching",
"ps" : "postscript",
"pt" : "point",
"ptb" : "please text back",
"pto" : "please turn over",
"qpsa" : "what happens",
"ratchet" : "rude",
"rbtl" : "read between the lines",
"rlrt" : "real life retweet",
"rofl" : "rolling on the floor laughing",
"roflol" : "rolling on the floor laughing out loud",
"rotflmao" : "rolling on the floor laughing my ass off",
"rt" : "retweet",
"ruok" : "are you ok",
"sfw" : "safe for work",
"sk8" : "skate",
"smh" : "shake my head",
"sq" : "square",
"srsly" : "seriously",
"ssdd" : "same stuff different day",
"tbh" : "to be honest",
"tbs" : "tablespooful",
"tbsp" : "tablespooful",
"tfw" : "that feeling when",
"thks" : "thank you",
"tho" : "though",
"thx" : "thank you",
"tia" : "thanks in advance",
"til" : "today i learned",
"tl;dr" : "too long i did not read",
"tldr" : "too long i did not read",
"tmb" : "tweet me back",
"tntl" : "trying not to laugh",
"ttyl" : "talk to you later",
"u" : "you",
"u2" : "you too",
"u4e" : "yours for ever",
"utc" : "coordinated universal time",
"w/" : "with",
"w/o" : "without",
"w8" : "wait",
"wassup" : "what is up",
"wb" : "welcome back",
"wtf" : "what the fuck",
"wtg" : "way to go",
"wtpa" : "where the party at",
"wuf" : "where are you from",
"wuzup" : "what is up",
"wywh" : "wish you were here",
"yd" : "yard",
"ygtr" : "you got that right",
"ynk" : "you never know",
"zzz" : "sleeping bored and tired"
}
fname = '.. /input/freesound-audio-tagging/audio_train/' + '00044347.wav'
ipd.Audio(fname )<normalization>
def convert_abbrev(word):
return abbreviations[word.lower() ] if word.lower() in abbreviations.keys() else word
wav = wave.open(fname)
 print("Sampling(frame)rate = ", wav.getframerate())
 print("Total samples(frames)= ", wav.getnframes())
 print("Duration = ", wav.getnframes() /wav.getframerate() )<train_model>
def convert_abbrev_in_text(text):
tokens = word_tokenize(text)
 tokens = [convert_abbrev(word)for word in tokens]
text = ' '.join(tokens)
 return text
rate, data = wavfile.read(fname)
 print("Sampling(frame)rate = ", rate)
 print("Total samples(frames)= ", data.shape)
 print(data )<import_modules>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
ModelCheckpoint, TensorBoard, ReduceLROnPlateau)
 GlobalMaxPool1D, Input, MaxPool1D, concatenate)
<set_options>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
class Config(object):
def __init__(self,
sampling_rate=16000, audio_duration=2, n_classes=41,
use_mfcc=False, n_folds=10, learning_rate=0.0001,
max_epochs=50, n_mfcc=20):
self.sampling_rate = sampling_rate
self.audio_duration = audio_duration
self.n_classes = n_classes
self.use_mfcc = use_mfcc
self.n_mfcc = n_mfcc
self.n_folds = n_folds
self.learning_rate = learning_rate
self.max_epochs = max_epochs
self.audio_length = self.sampling_rate * self.audio_duration
if self.use_mfcc:
self.dim =(self.n_mfcc, 1 + int(np.floor(self.audio_length/512)) , 1)
 else:
self.dim =(self.audio_length, 1 )<define_variables>
if target_big_corrected:
train["text"] = train["text"].apply(lambda x: clean_tweets(x))
test["text"] = test["text"].apply(lambda x: clean_tweets(x))
train["text"] = train["text"].apply(lambda x: remove_emoji(x))
test["text"] = test["text"].apply(lambda x: remove_emoji(x))
train["text"] = train["text"].apply(lambda x: remove_punctuations(x))
test["text"] = test["text"].apply(lambda x: remove_punctuations(x))
train["text"] = train["text"].apply(lambda x: convert_abbrev_in_text(x))
test["text"] = test["text"].apply(lambda x: convert_abbrev_in_text(x))
class DataGenerator(Sequence):
def __init__(self, config, data_dir, list_IDs, labels=None,
batch_size=64, preprocessing_fn=lambda x: x):
self.config = config
self.data_dir = data_dir
self.list_IDs = list_IDs
self.labels = labels
self.batch_size = batch_size
self.preprocessing_fn = preprocessing_fn
self.on_epoch_end()
self.dim = self.config.dim
def __len__(self):
return int(np.ceil(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_temp = [self.list_IDs[k] for k in indexes]
return self.__data_generation(list_IDs_temp)
 def on_epoch_end(self):
self.indexes = np.arange(len(self.list_IDs))
def __data_generation(self, list_IDs_temp):
cur_batch_size = len(list_IDs_temp)
 X = np.empty(( cur_batch_size, *self.dim))
input_length = self.config.audio_length
for i, ID in enumerate(list_IDs_temp):
file_path = self.data_dir + ID
data, _ = librosa.core.load(file_path, sr=self.config.sampling_rate,
res_type='kaiser_fast')
 if len(data)> input_length:
max_offset = len(data)- input_length
offset = np.random.randint(max_offset)
 data = data[offset:(input_length+offset)]
else:
if input_length > len(data):
max_offset = input_length - len(data)
 offset = np.random.randint(max_offset)
 else:
offset = 0
data = np.pad(data,(offset, input_length - len(data)- offset), "constant")
 if self.config.use_mfcc:
data = librosa.feature.mfcc(data, sr=self.config.sampling_rate,
n_mfcc=self.config.n_mfcc)
 data = np.expand_dims(data, axis=-1)
 else:
data = self.preprocessing_fn(data)[:, np.newaxis]
X[i,] = data
if self.labels is not None:
y = np.empty(cur_batch_size, dtype=int)
 for i, ID in enumerate(list_IDs_temp):
y[i] = self.labels[ID]
return X, to_categorical(y, num_classes=self.config.n_classes)
 else:
return X<normalization>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def audio_norm(data):
max_data = np.max(data)
 min_data = np.min(data)
 data =(data-min_data)/(max_data-min_data+1e-6)
 return data-0.5<choose_model_class>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
def get_1d_dummy_model(config):
nclass = config.n_classes
input_length = config.audio_length
inp = Input(shape=(input_length,1))
x = GlobalMaxPool1D()(inp)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model
def get_1d_conv_model(config):
nclass = config.n_classes
input_length = config.audio_length
inp = Input(shape=(input_length,1))
x = Convolution1D(16, 9, activation=relu, padding="valid" )(inp)
 x = Convolution1D(16, 9, activation=relu, padding="valid" )(x)
 x = MaxPool1D(16 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = MaxPool1D(4 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = Convolution1D(32, 3, activation=relu, padding="valid" )(x)
 x = MaxPool1D(4 )(x)
 x = Dropout(rate=0.1 )(x)
 x = Convolution1D(256, 3, activation=relu, padding="valid" )(x)
 x = Convolution1D(256, 3, activation=relu, padding="valid" )(x)
 x = GlobalMaxPool1D()(x)
 x = Dropout(rate=0.2 )(x)
 x = Dense(64, activation=relu )(x)
 x = Dense(1028, activation=relu )(x)
 out = Dense(nclass, activation=softmax )(x)
 model = models.Model(inputs=inp, outputs=out)
 opt = optimizers.Adam(config.learning_rate)
 model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 return model<feature_engineering>
random_state_split = 7
Dropout_num = 0
learning_rate = 3e-5
valid = 0.2
epochs_num = 3
batch_size_num = 16
target_corrected = True
target_big_corrected = True
LABELS = list(train.label.unique())
 label_idx = {label: i for i, label in enumerate(LABELS)}
train.set_index("fname", inplace=True)
 test.set_index("fname", inplace=True)
 train["label_idx"] = train.label.apply(lambda x: label_idx[x])
 if not COMPLETE_RUN:
train = train[:2000]
test = test[:2000]<set_options>
random_state_split = 7
Dropout_num = 0.3
learning_rate = 4e-5
validation_split = 0.3
epochs_num = 3
batch_size_num = 16
target_corrected = True
target_big_corrected = True
config = Config(sampling_rate=16000, audio_duration=2, n_folds=10, learning_rate=0.001)
 if not COMPLETE_RUN:
config = Config(sampling_rate=100, audio_duration=1, n_folds=2, max_epochs=1 )<save_to_csv>
checkpoint = ModelCheckpoint('model_BERT.h5', monitor='val_loss', save_best_only=False)
 train_history = model_BERT.fit(
 train_input, train_labels,
validation_split = valid,
epochs = epochs_num,
callbacks=[checkpoint],
batch_size = batch_size_num
)
pred_list = []
for i in range(10):
pred_list.append(np.load(".. /input/freesound-prediction-file/test_predictions_%d.npy"%i))
prediction = np.ones_like(pred_list[0])
 for pred in pred_list:
prediction = prediction*pred
prediction = prediction**(1./len(pred_list))
top_3 = np.array(LABELS)[np.argsort(-prediction, axis=1)[:, :3]]
predicted_labels = [' '.join(list(x)) for x in top_3]
test = pd.read_csv('.. /input/freesound-audio-tagging/sample_submission.csv')
 test['label'] = predicted_labels
test[['fname', 'label']].to_csv("1d_conv_ensembled_submission.csv", index=False )<load_pretrained>
model_BERT.load_weights('model_BERT.h5')
 test_pred_BERT = model_BERT.predict(test_input)
 test_pred_BERT_int = test_pred_BERT.round().astype('int' )
SAMPLE_RATE = 44100
fname = '.. /input/freesound-audio-tagging/audio_train/' + '00044347.wav'
wav, _ = librosa.core.load(fname, sr=SAMPLE_RATE)
 wav = wav[:2*44100]<compute_test_metric>
train_pred_BERT = model_BERT.predict(train_input)
 train_pred_BERT_int = train_pred_BERT.round().astype('int' )
mfcc = librosa.feature.mfcc(wav, sr = SAMPLE_RATE, n_mfcc=40)
 mfcc.shape<import_modules>
pred = pd.DataFrame(test_pred_BERT, columns=['preds'])
 pred.plot.hist()
GlobalMaxPool2D, MaxPool2D, concatenate, Activation)
<choose_model_class>
submission['target'] = test_pred_BERT_int
submission.head(10 )
<set_options><EOS> 
submission.to_csv("submission.csv", index=False, header=True )