Kaggle competition: Freesound Audio Tagging 2019
<SOS> metric: weightedlabelrankingaverageprecision Kaggle data source: freesound-audio-tagging-2019<categorify>
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import matplotlib.pyplot as plt
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
from imgaug import augmenters as iaa
import torch
import torch.nn as nn
import torch.nn.functional as F
from fastprogress import master_bar, progress_bar
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms
sentences = []
for s in all_data2["Address"]:
sentences.append(s.split(" "))
address_model = gensim.models.Word2Vec(sentences, min_count=1)
 encoded_address = np.zeros(( all_data2.shape[0], 100))
for i in range(len(sentences)) :
for j in range(len(sentences[i])) :
encoded_address[i] += address_model.wv[sentences[i][j]]
encoded_address[i] /= len(sentences[i])
 print('done' )<categorify>
NUM_CLASSES = 80
SIZE=128
checkpoint_file = ['model_best1.h5', 'model_best2.h5', 'model_best3.h5']
EPOCHS = [432, 0, 0]
TTA = [19, 0, 0]
BATCH_SIZE = 32
LR = 4e-4
PATIENCE = 10
LR_FACTOR = 0.8
CURATED_ONLY = True
TRAIN_AUGMENT = True
VALID_AUGMENT = False
MODEL = 'mobile'
SEED = 520
USE_MIXUP = True
MIXUP_PROB = 0.275
ACTIVATION = 'linear'
LOSS = 'BCEwithLogits'
all_data2.drop(['Address'],axis=1,inplace=True)
 categorical_features = ["DayOfWeek", "PdDistrict", "block", "special_time", "XYcluster",'ST']
ct = ColumnTransformer(transformers=[("categorical_features", OrdinalEncoder() , categorical_features)],
remainder="passthrough")
 all_data3 = ct.fit_transform(all_data2)
 print('done' )<categorify>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )
all_data4 = np.hstack(( all_data3,encoded_address))
print('done' )<split>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
train2 = all_data3[:num_train]
test2 = all_data3[num_train:]
print('done' )<data_type_conversions>
def tf_one_sample_positive_class_precisions(y_true, y_pred):
num_samples, num_classes = y_pred.shape
pos_class_indices = tf.where(y_true > 0)
 retrieved_classes = tf.nn.top_k(y_pred, k=num_classes ).indices
sample_range = tf.zeros(shape=tf.shape(tf.transpose(y_pred)) , dtype=tf.int32)
 sample_range = tf.add(sample_range, tf.range(tf.shape(y_pred)[0], delta=1))
sample_range = tf.transpose(sample_range)
 sample_range = tf.reshape(sample_range,(-1,num_classes*tf.shape(y_pred)[0]))
retrieved_classes = tf.reshape(retrieved_classes,(-1,num_classes*tf.shape(y_pred)[0]))
retrieved_class_map = tf.concat(( sample_range, retrieved_classes), axis=0)
 retrieved_class_map = tf.transpose(retrieved_class_map)
 retrieved_class_map = tf.reshape(retrieved_class_map,(tf.shape(y_pred)[0], num_classes, 2))
class_range = tf.zeros(shape=tf.shape(y_pred), dtype=tf.int32)
 class_range = tf.add(class_range, tf.range(num_classes, delta=1))
class_rankings = tf.scatter_nd(retrieved_class_map,
class_range,
tf.shape(y_pred))
num_correct_until_correct = tf.gather_nd(class_rankings, pos_class_indices)
 num_correct_until_correct_one = tf.add(num_correct_until_correct, 1)
 num_correct_until_correct_one = tf.cast(num_correct_until_correct_one, tf.float32)
 sample_label = pos_class_indices[:, 0]
sample_label = tf.reshape(sample_label,(-1, 1))
sample_label = tf.cast(sample_label, tf.int32)
 num_correct_until_correct = tf.reshape(num_correct_until_correct,(-1, 1))
retrieved_class_true_position = tf.concat(( sample_label,
num_correct_until_correct), axis=1)
 retrieved_pos = tf.ones(shape=tf.shape(retrieved_class_true_position)[0], dtype=tf.int32)
 retrieved_class_true = tf.scatter_nd(retrieved_class_true_position,
retrieved_pos,
tf.shape(y_pred))
retrieved_cumulative_hits = tf.cumsum(retrieved_class_true, axis=1)
 pos_ret_indices = tf.where(retrieved_class_true > 0)
 correct_rank = tf.gather_nd(retrieved_cumulative_hits, pos_ret_indices)
 correct_rank = tf.cast(correct_rank, tf.float32)
 precision_at_hits = tf.truediv(correct_rank, num_correct_until_correct_one)
 return pos_class_indices, precision_at_hits
def tf_lwlrap(y_true, y_pred):
num_samples, num_classes = y_pred.shape
pos_class_indices, precision_at_hits =(tf_one_sample_positive_class_precisions(y_true, y_pred))
pos_flgs = tf.cast(y_true > 0, tf.int32)
 labels_per_class = tf.reduce_sum(pos_flgs, axis=0)
 weight_per_class = tf.truediv(tf.cast(labels_per_class, tf.float32),
tf.cast(tf.reduce_sum(labels_per_class), tf.float32))
sum_precisions_by_classes = tf.zeros(shape=(num_classes), dtype=tf.float32)
 class_label = pos_class_indices[:,1]
sum_precisions_by_classes = tf.unsorted_segment_sum(precision_at_hits,
class_label,
num_classes)
 labels_per_class = tf.cast(labels_per_class, tf.float32)
 labels_per_class = tf.add(labels_per_class, 1e-7)
 per_class_lwlrap = tf.truediv(sum_precisions_by_classes,
tf.cast(labels_per_class, tf.float32))
out = tf.cast(tf.tensordot(per_class_lwlrap, weight_per_class, axes=1), dtype=tf.float32)
 return out
train_2 = sparse.hstack(( train2.astype(float),train_vect))
test_2 = sparse.hstack(( test2.astype(float),test_vect))
print('done' )<save_to_csv>
def BCEwithLogits(y_true, y_pred):
return K.mean(K.binary_crossentropy(y_true, y_pred, from_logits=True), axis=-1 )
model = LGBMClassifier(objective="multiclass", num_class=39, max_bin = 465, max_delta_step = 0.9,
learning_rate=0.4, num_leaves = 42, n_estimators=100, verbose=50)
 model.fit(train2, y_train)
 preds = model.predict_proba(test2)
 submission = pd.DataFrame(preds, columns=le.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission.to_csv('LGBM_final1.csv', index_label='Id')
 print('done' )<save_to_csv>
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 preprocessed_dir = Path('.. /input/fat2019_prep_mels1' )
model2 = LGBMClassifier(objective="multiclass", num_class=39, max_bin = 465, max_delta_step = 0.9,
learning_rate=0.4, num_leaves = 42, n_estimators=100, verbose=50)
 model2.fit(train_2, y_train)
 preds2 = model2.predict_proba(test_2)
 submission2 = pd.DataFrame(preds2, columns=le.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission2.to_csv('LGBM_final2.csv', index_label='Id')
 print('done' )<save_to_csv>
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}
sub_tot =(preds+preds2+prob)/3
submission4 = pd.DataFrame(sub_tot, columns=le.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission4.to_csv('final.csv', index_label='Id')
 print('done' )<import_modules>
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 if CURATED_ONLY:
train_df = train_curated
else:
train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)
 train_df.head()
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import lightgbm as lgb<load_from_csv>
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()
train_data = pd.read_csv('.. /input/sf-crime/train.csv.zip', parse_dates=['Dates'])
 test_data = pd.read_csv('.. /input/sf-crime/test.csv.zip', parse_dates=['Dates'] )<feature_engineering>
num_classes = len(labels)
 num_classes
all_features = pd.concat(( train_data.iloc[:, [0, 3, 4, 6, 7, 8]],
test_data.iloc[:, [1, 2, 3, 4, 5, 6]]),
sort=False)
 num_train = train_data.shape[0]
train_labels = pd.get_dummies(train_data['Category'] ).values
num_outputs = train_labels.shape[1]
train_labels = np.argmax(train_labels, axis=1)
 all_features['year'] = all_features.Dates.dt.year
all_features['month'] = all_features.Dates.dt.month
all_features['new_year'] = all_features['month'].apply(
 lambda x: 1 if x == 1 or x == 2 else 0)
 all_features['day'] = all_features.Dates.dt.day
all_features['hour'] = all_features.Dates.dt.hour
all_features['evening'] = all_features['hour'].apply(lambda x: 1
if x >= 18 else 0)
 wkm = {
'Monday': 0,
'Tuesday': 1,
'Wednesday': 2,
'Thursday': 3,
'Friday': 4,
'Saturday': 5,
'Sunday': 6
}
all_features['DayOfWeek'] = all_features['DayOfWeek'].apply(lambda x: wkm[x])
 all_features['weekend'] = all_features['DayOfWeek'].apply(
 lambda x: 1 if x == 4 or x == 5 else 0)
 OneHot_features = pd.get_dummies(all_features['PdDistrict'])
 all_features['block'] = all_features['Address'].apply(
 lambda x: 1 if 'block' in x.lower() else 0)
 PCA_features = all_features[['X', 'Y']].values
Standard_features = all_features[['DayOfWeek', 'year', 'month', 'day',
'hour']].values
OneHot_features = pd.concat([
OneHot_features, all_features[['new_year', 'evening', 'weekend', 'block']]
],
axis=1 ).values
scaler = StandardScaler()
scaler.fit(Standard_features)
 Standard_features = scaler.transform(Standard_features)
 pca = PCA(n_components=2)
 pca.fit(PCA_features)
 PCA_features = pca.transform(PCA_features)
 all_features = np.concatenate(
(PCA_features, Standard_features, OneHot_features), axis=1)
 train_features = all_features[:num_train]
num_inputs = train_features.shape[1]
test_features = all_features[num_train:]<create_dataframe>
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape
data_train = lgb.Dataset(train_features, label = train_labels )<init_hyperparams>
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 if CURATED_ONLY == False:
x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )
params = {
'boosting': 'gbdt',
'objective': 'multiclass',
'metrics' : 'multi_logloss',
'num_class': num_outputs,
'verbosity': 1,
'device_type':'gpu',
'gpu_platform_id':0,
'gpu_device_id':0,
'max_depth': 6,
'num_leaves': 50,
'min_data_in_leaf' : 20,
'feature_fraction': 0.8,
'learning_rate': 0.1,
}
gbm = lgb.train(params, data_train, num_boost_round = 214)
 gbm.save_model('.. /working/gbm(v1 ).txt')
 testResult = gbm.predict(test_features)
 sampleSubmission = pd.read_csv('.. /input/sf-crime/sampleSubmission.csv.zip')
 Result_pd = pd.DataFrame(testResult,
index=sampleSubmission.index,
columns=sampleSubmission.columns[1:])
 Result_pd.to_csv('.. /working/sampleSubmission(gbmv1 ).csv', index_label='Id' )<set_options>
def create_model_inception(n_out=NUM_CLASSES):
base_model =InceptionV3(weights=None, include_top=False)
 x0 = base_model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 return model
%matplotlib inline<load_from_csv>
def create_model_xception(n_out=NUM_CLASSES):
base_model = Xception(weights=None, include_top=False)
 x0 = base_model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 return model
train = pd.read_csv(".. /input/train.csv")
 train.head()<load_from_csv>
def create_model_mobile(n_out=NUM_CLASSES):
base_model =MobileNetV2(weights=None, include_top=False)
 x0 = base_model.output
x1 = GlobalAveragePooling2D()(x0)
 x2 = GlobalMaxPooling2D()(x0)
 x = Concatenate()([x1,x2])
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 return model
test = pd.read_csv('.. /input/test.csv')
 test.head()<count_unique_values>
def conv_simple_block(x, n_filters):
x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 x = AveragePooling2D()(x)
 return x
def create_model_simplecnn(n_out=NUM_CLASSES):
inp = Input(shape=(128,128,3))
x = conv_simple_block(inp,64)
 x = conv_simple_block(x,128)
 x = conv_simple_block(x,256)
 x = conv_simple_block(x,128)
 x = Flatten()(x)
 x = Dropout(0.2 )(x)
 x = Dense(128, activation='linear' )(x)
 x = PReLU()(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=inp, outputs=predictions)
 return model
print('Frequency of Categories: ', train.Category.nunique())
 print('Frequency of PdDistricts: ', train.PdDistrict.nunique())
 print('Frequency of DayOfWeeks: ', train.DayOfWeek.nunique() )<drop_column>
def output_of_lambda(input_shape):
return(input_shape[0], input_shape[2], input_shape[3])
 def my_max(x):
return K.max(x, axis=1, keepdims=False)
 def crnn_simple_block(x, n_filters):
x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = Activation("relu" )(x)
 x = Convolution2D(n_filters,(3,1), padding="same" )(x)
 x = Activation("relu" )(x)
 x = MaxPooling2D()(x)
 x = Dropout(0.2 )(x)
 return x
def create_model_crnn(n_out=NUM_CLASSES):
inp = Input(shape=(128,None,3))
x = crnn_simple_block(inp,64)
 x = crnn_simple_block(x,128)
 x = crnn_simple_block(x,256)
 x = Lambda(my_max, output_shape=output_of_lambda )(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = GlobalMaxPooling1D()(x)
 x = Dense(128, activation='linear' )(x)
 x = PReLU()(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=inp, outputs=predictions)
 return model
train = train.drop('Resolution', axis=1)
 train.sample(1 )<data_type_conversions>
def create_model_cnn8th(n_out=NUM_CLASSES):
regu=0
inp = Input(shape=(128,128,3))
x = Conv2D(48, 11, strides=(1,1),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(inp)
 x = BatchNormalization()(x)
 x = Conv2D(48, 11, strides=(2,3),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = MaxPooling2D(3, strides=(1,2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128, 5, strides=(1,1),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128, 5, strides=(2,3),kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = MaxPooling2D(3, strides=2 )(x)
 x = BatchNormalization()(x)
 x = Conv2D(192, 3, strides=1,kernel_initializer='he_uniform', activation='relu', padding='same' )(x)
 x = BatchNormalization()(x)
 x = Conv2D(192, 3, strides=1,kernel_initializer='he_uniform', activation='relu', padding='same' )(x)
 x = BatchNormalization()(x)
 x = Conv2D(128, 3, strides=1,kernel_initializer='he_uniform', activation='relu', padding='same',kernel_regularizer=regularizers.l2(regu))(x)
 x = MaxPooling2D(3, strides=(1,2))(x)
 x = BatchNormalization()(x)
 x = Flatten()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(n_out, activation=ACTIVATION )(x)
 model = Model(inputs=inp, outputs=predictions)
 return model
train['Date'] = pd.to_datetime(train.Dates)
 test['Date'] = pd.to_datetime(test.Dates)
 train = train.drop('Dates', axis=1)
 test = test.drop('Dates', axis=1)
 train.sample(1 )<feature_engineering>
K.clear_session()

 if MODEL == 'xception':
preprocess_input = preprocess_xception
model = create_model_xception(n_out=NUM_CLASSES)
 elif MODEL == 'inception':
preprocess_input = preprocess_inception
model = create_model_inception(n_out=NUM_CLASSES)
 elif MODEL == 'mobile':
preprocess_input = preprocess_mobile
model = create_model_mobile(n_out=NUM_CLASSES)
 elif MODEL == 'crnn':
preprocess_input = preprocess_mobile
model = create_model_crnn(n_out=NUM_CLASSES)
 elif MODEL == 'cnn8th':
preprocess_input = preprocess_mobile
model = create_model_cnn8th(n_out=NUM_CLASSES)
 else:
preprocess_input = preprocess_mobile
model = create_model_simplecnn(n_out=NUM_CLASSES)
 print(MODEL)
 model.summary()
train['IsDay'] = 0
train.loc[(train.Date.dt.hour > 6)&(train.Date.dt.hour < 20), 'IsDay' ] = 1
test['IsDay'] = 0
test.loc[(test.Date.dt.hour > 6)&(test.Date.dt.hour < 20), 'IsDay' ] = 1
train.sample(3 )<feature_engineering>
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_tf_lwlrap', factor=LR_FACTOR, patience=PATIENCE,
verbose=1, mode='max', min_delta=0.0001, cooldown=2, min_lr=1e-5)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 checkpoint = ModelCheckpoint(checkpoint_file[0], monitor='val_tf_lwlrap', verbose=1,
save_best_only=True, mode='max', save_weights_only = False)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
days_to_int_dic = {
'Monday': 1,
'Tuesday': 2,
'Wednesday': 3,
'Thursday': 4,
'Friday': 5,
'Saturday': 6,
'Sunday': 7,
}
train['DayOfWeek'] = train['DayOfWeek'].map(days_to_int_dic)
 test['DayOfWeek'] = test['DayOfWeek'].map(days_to_int_dic)
 train.DayOfWeek.unique()<feature_engineering>
x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)
 train_generator = FATTrainDataset.create_generator(
 x_trn, y_trn, BATCH_SIZE,(SIZE,SIZE,3), augument=TRAIN_AUGMENT, shuffling=True, mixup = USE_MIXUP, mixup_prob = MIXUP_PROB)
 validation_generator = FATTrainDataset.create_generator(
 x_val, y_val, BATCH_SIZE,(SIZE,SIZE,3), augument=VALID_AUGMENT, shuffling=False)
train['Hour'] = train.Date.dt.hour
train['Month'] = train.Date.dt.month
train['Year'] = train.Date.dt.year
train['Year'] = train['Year']
test['Hour'] = test.Date.dt.hour
test['Month'] = test.Date.dt.month
test['Year'] = test.Date.dt.year
test['Year'] = test['Year']
train.sample(1 )<feature_engineering>
train_steps = np.ceil(float(len(x_trn)) / float(BATCH_SIZE))
val_steps = np.ceil(float(len(x_val)) / float(BATCH_SIZE))
train_steps = train_steps.astype(int)
 val_steps = val_steps.astype(int)
 print(train_steps, val_steps)
 print(len(x_trn))
train['HourCos'] = np.cos(( train['Hour']*2*np.pi)/24)
 train['DayOfWeekCos'] = np.cos(( train['DayOfWeek']*2*np.pi)/7)
 train['MonthCos'] = np.cos(( train['Month']*2*np.pi)/12)
 test['HourCos'] = np.cos(( test['Hour']*2*np.pi)/24)
 test['DayOfWeekCos'] = np.cos(( test['DayOfWeek']*2*np.pi)/7)
 test['MonthCos'] = np.cos(( test['Month']*2*np.pi)/12)
 train.sample(1 )<categorify>
print(LOSS)
 if LOSS=='BCEwithLogits':
model.compile(loss=BCEwithLogits,
optimizer=Adam(lr=LR),
metrics=[tf_lwlrap,'categorical_accuracy'])
 else:
model.compile(loss=LOSS,
optimizer=Adam(lr=LR),
metrics=[tf_lwlrap,'categorical_accuracy'])
train = pd.get_dummies(train, columns=['PdDistrict'])
 test = pd.get_dummies(test, columns=['PdDistrict'])
 train.sample(2 )<categorify>
print(LR, PATIENCE, LR_FACTOR,BATCH_SIZE, TRAIN_AUGMENT, USE_MIXUP, MIXUP_PROB )
cat_le = LabelEncoder()
train['CategoryInt'] = pd.Series(cat_le.fit_transform(train.Category))
train.sample(5)
 cat_le.classes_<feature_engineering>
hist = model.fit_generator(
 train_generator,
steps_per_epoch=train_steps,
validation_data=validation_generator,
validation_steps=val_steps,
epochs=EPOCHS[0],
verbose=1,
callbacks=callbacks_list )
train['InIntersection'] = 1
train.loc[train.Address.str.contains('Block'), 'InIntersection'] = 0
test['InIntersection'] = 1
test.loc[test.Address.str.contains('Block'), 'InIntersection'] = 0<prepare_x_and_y>
print(K.eval(model.optimizer.lr))
feature_cols = ['X', 'Y', 'IsDay', 'DayOfWeek', 'Month', 'Hour', 'Year', 'InIntersection',
'PdDistrict_BAYVIEW', 'PdDistrict_CENTRAL', 'PdDistrict_INGLESIDE',
'PdDistrict_MISSION', 'PdDistrict_NORTHERN', 'PdDistrict_PARK',
'PdDistrict_RICHMOND', 'PdDistrict_SOUTHERN', 'PdDistrict_TARAVAL', 'PdDistrict_TENDERLOIN']
target_col = 'CategoryInt'
train_x = train[feature_cols]
train_y = train[target_col]
test_ids = test['Id']
test_x = test[feature_cols]<prepare_x_and_y>
if EPOCHS[1] > 0:
checkpoint = ModelCheckpoint(checkpoint_file[1], monitor='val_tf_lwlrap', verbose=1,
save_best_only=True, mode='max', save_weights_only = False)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
hist = model.fit_generator(
 train_generator,
steps_per_epoch=train_steps,
validation_data=validation_generator,
validation_steps=val_steps,
epochs=EPOCHS[1],
verbose=1,
callbacks=callbacks_list )
train_xgb = xgb.DMatrix(train_x, label=train_y)
 test_xgb = xgb.DMatrix(test_x )<init_hyperparams>
print(K.eval(model.optimizer.lr))
params = {
'max_depth': 5,
'eta': 0.4,
'silent': 1,
'objective': 'multi:softprob',
'num_class': 39,
}<compute_train_metric>
if EPOCHS[2] > 0:
checkpoint = ModelCheckpoint(checkpoint_file[2], monitor='val_tf_lwlrap', verbose=1,
save_best_only=True, mode='max', save_weights_only = False)
 callbacks_list = [checkpoint, csv_logger, reduceLROnPlat]
hist = model.fit_generator(
 train_generator,
steps_per_epoch=train_steps,
validation_data=validation_generator,
validation_steps=val_steps,
epochs=EPOCHS[2],
verbose=1,
callbacks=callbacks_list )
CROSS_VAL = False
if CROSS_VAL:
print('Doing Cross-validation...')
 cv = xgb.cv(params, train_xgb, nfold=3, early_stopping_rounds=10, metrics='mlogloss', verbose_eval=True)
 cv<save_to_csv>
print(K.eval(model.optimizer.lr))
SUBMIT = not CROSS_VAL
if SUBMIT:
print('Fitting Model...')
 m = xgb.train(params, train_xgb, 10)
 res = m.predict(test_xgb)
 cols = ['Id'] + cat_le.classes_
submission = pd.DataFrame(res, columns=cat_le.classes_)
 submission.insert(0, 'Id', test_ids)
 submission.to_csv('submission.csv', index=False)
 print('Done Outputing !')
 print(submission.sample(3))
else:
print('NOT SUBMITING' )<load_from_csv>
model.load_weights(checkpoint_file[0])
 validation_generator = FATTrainDataset.create_generator(
 x_val, y_val, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False)
 pred_val_y = model.predict_generator(validation_generator,steps=val_steps,verbose=1)
 for kk in range(len(TTA)) :
for ii in range(TTA[kk]):
validation_generator = FATTrainDataset.create_generator(
 x_val, y_val, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False)
 pred_val_y += model.predict_generator(validation_generator,steps=val_steps,verbose=1)
 if kk+1 < len(TTA)and TTA[kk+1] > 0:
model.load_weights(checkpoint_file[kk+1])
train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id' )<count_missing_values>
train_generator = FATTrainDataset.create_generator(
 x_trn, y_trn, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False)
 pred_train_y = model.predict_generator(train_generator,steps=train_steps,verbose=1 )
train.isnull().sum()<count_missing_values>
def calculate_overall_lwlrap_sklearn(truth, scores):

 sample_weight = np.sum(truth > 0, axis=1)
 nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)
 overall_lwlrap = sklearn.metrics.label_ranking_average_precision_score(
 truth[nonzero_weight_sample_indices, :] > 0,
scores[nonzero_weight_sample_indices, :],
sample_weight=sample_weight[nonzero_weight_sample_indices])
 return overall_lwlrap
train.isnull().sum()<feature_engineering>
print("lwlrap from sklearn.metrics for training data =", calculate_overall_lwlrap_sklearn(y_trn, pred_train_y))
print("val lwlrap from sklearn.metrics =", calculate_overall_lwlrap_sklearn(y_val, pred_val_y/10))
score, weight = calculate_per_class_lwlrap(y_val, pred_val_y)
 lwlrap =(score * weight ).sum()
print('direct calculation of val lwlrap : %.4f' %(lwlrap))
pd.options.display.max_columns=100
train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id')
 def feature_engineering(data):
data['Date'] = pd.to_datetime(data['Dates'].dt.date)
 data['Day'] = data['Dates'].dt.day
data['DayOfWeek'] = data['Dates'].dt.weekday
data['Month'] = data['Dates'].dt.month
data['Year'] = data['Dates'].dt.year
data['Hour'] = data['Dates'].dt.hour
data['Minute'] = data['Dates'].dt.minute
data['Block'] = data['Address'].str.contains('block', case=False ).apply(lambda x: 1 if x == True else 0)
 data.drop(columns=['Dates','Date','Address'], inplace=True)
 return data
train = feature_engineering(train)
 test = feature_engineering(test)
 train.drop(columns=['Descript','Resolution'], inplace=True )<categorify>
idx = np.sum(y_val,axis=1)> 1
print(y_val[idx, :].shape, y_val[idx==False, :].shape)
 print("val lwlrap for multi-labels =", calculate_overall_lwlrap_sklearn(y_val[idx], pred_val_y[idx]))
print("val lwlrap for single-label =", calculate_overall_lwlrap_sklearn(y_val[idx==False], pred_val_y[idx==False]))
le1 = LabelEncoder()
train['PdDistrict'] = le1.fit_transform(train['PdDistrict'])
 test['PdDistrict'] = le1.transform(test['PdDistrict'])
 le2 = LabelEncoder()
X = train.drop(columns=['Category'])
 y= le2.fit_transform(train['Category'] )<prepare_x_and_y>
test_steps = np.ceil(float(len(x_test)) / float(BATCH_SIZE)).astype(int)
train_xgb = xgb.DMatrix(X, label=y)
 test_xgb = xgb.DMatrix(test )<init_hyperparams>
model.load_weights(checkpoint_file[0])
 test_generator = FATTrainDataset.create_generator(
 x_test, x_test, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False, test_data=True)
 pred_test_y = model.predict_generator(test_generator,steps=test_steps,verbose=1)
 for kk in range(len(TTA)) :
for ii in range(TTA[kk]):
test_generator = FATTrainDataset.create_generator(
 x_test, x_test, BATCH_SIZE,(SIZE,SIZE,3), augument=False, shuffling=False, test_data=True)
 pred_test_y += model.predict_generator(test_generator,steps=test_steps,verbose=1)
 if kk+1 < len(TTA)and TTA[kk+1] > 0:
model.load_weights(checkpoint_file[kk+1] )
params = {
'max_depth': 6,
'eta': 0.3,
'num_boost_rounds' : 150 ,
'silent': 1,
'objective': 'multi:softprob',
'eval_metric' : 'mlogloss',
'learning_data' : 0.07,
'num_class': 39,
}<compute_train_metric>
sort_idx = np.argsort(labels ).astype(int)
CROSS_VAL = False
if CROSS_VAL:
print('Doing Cross-validation...')
 cv = xgb.cv(params, train_xgb, nfold=3, early_stopping_rounds=10, metrics='mlogloss', verbose_eval=True)
 cv<save_to_csv>
print(sort_idx )
SUBMIT = not CROSS_VAL
if SUBMIT:
print('Fitting Model...')
 m = xgb.train(params, train_xgb, 10)
 res = m.predict(test_xgb)
 submission = pd.DataFrame(res, columns=le2.inverse_transform 
(np.linspace(0, 38, 39, dtype='int16')) ,
index=test.index)
 submission.to_csv('submission.csv', index='Id')
 print(submission.sample(3))
else :
print('FAIL' )<import_modules>
sample_sub = pd.read_csv('.. /input/freesound-audio-tagging-2019/sample_submission.csv')
 test_Y_sort = pred_test_y[:, sort_idx]
sample_sub.iloc[:, 1:] = test_Y_sort
sample_sub.to_csv('submission.csv', index=False)
 sample_sub.head()