Kaggle competition: Titanic - Machine Learning from Disaster
counts = Counter(FP_text.split())
 print(counts )<data_type_conversions>
dataset = pd.read_csv(".. /input/train.csv")
 dataset.head()
df_test['prediction']=(y_pred_test_final >= model_cutoff ).astype(int )<drop_column>
dataset.Sex = dataset.Sex.replace("female", 0)
 dataset.Sex = dataset.Sex.replace("male", 1 )
df_test=df_test.drop(['question_text'], axis=1)
 df_test=df_test.drop(['length'], axis=1 )<save_to_csv>
y = dataset.Survived
features = ["Sex", "Age", "Parch", "SibSp"]
X = dataset[features]
df_test.to_csv(r'submission.csv', index = False )<feature_engineering>
X_tr, X_val, y_tr, y_val = train_test_split(X, y, random_state = 0 )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )<categorify>
my_model = XGBRegressor(n_estimators = 1000, learning_rate = 0.01)
 my_pipeline = Pipeline(steps=[ 
("model", my_model)
 ] )
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class<define_variables>
my_pipeline.fit(X_tr, y_tr, model__early_stopping_rounds = 10, model__eval_set = [(X_val, y_val)], model__verbose = False)
 preds = my_pipeline.predict(X_val )
TRAIN_MODE = False
CONTINUOUS_TRAIN = True
MIXMATCH_SSL = 0<define_variables>
mean_absolute_error(y_val, preds )
DATA = Path('.. /input/freesound-audio-tagging-2019')
 PREPROCESSED_N1K = Path('.. /input/fat2019_prep_mels1')
 PREPROCESSED_MP = Path('.. /input/fat2019-multipreprocessed-package')
 LAST_WEIGHTS = Path('.. /input/fat19-fastai-weights-of-mixup-mp')
 WORK = Path('work')
 Path(WORK ).mkdir(exist_ok=True, parents=True)
 CSV_TRAIN_CURATED = PREPROCESSED_MP/'train_curated_valid.csv'
CSV_TRAIN_NOISY = PREPROCESSED_MP/'train_noisy_valid.csv'
CSV_TRAIN_NOISY_BEST50S = PREPROCESSED_N1K/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
SAVED_WEIGHT = LAST_WEIGHTS/'last'
DEPLOYED_MODEL = LAST_WEIGHTS/'export.pkl'
def add_prefix_path(df, prefix):
fnames = df.fname.copy()
df.fname = prefix + os.sep
df.fname = df.fname.str.cat(fnames)
 return df
def filter_df(df):
few_labels = list(map(lambda p: len(p[1])<= 3, df.labels.str.split(',' ).iteritems()))
return df[few_labels]
df_curated = add_prefix_path(pd.read_csv(CSV_TRAIN_CURATED), 'train_curated')
 df_noisy = add_prefix_path(pd.read_csv(CSV_TRAIN_NOISY), 'train_noisy')
 df_noisy50s = add_prefix_path(pd.read_csv(CSV_TRAIN_NOISY_BEST50S), 'train_noisy')
 df_submission = pd.read_csv(CSV_SUBMISSION)
 df_test = add_prefix_path(df_submission.copy() , 'test')
 df_train = df_curated
print('Training dataset size:', len(df_train))
df_train.head()<categorify>
dataset2 = pd.read_csv(".. /input/test.csv")
 dataset2.Sex = dataset2.Sex.replace("female", 0)
 dataset2.Sex = dataset2.Sex.replace("male", 1)
 X = dataset2[features]
preds2 = my_pipeline.predict(X )
USE_MASK_FREQ = True
MASK_FREQ_RANGE = 8
MASK_FREQ_MAX_COUNT = 3
USE_MASK_TIME = True
MASK_TIME_RANGE = 8
MASK_TIME_MAX_COUNT = 3
def freq_mask(x, num=1, mask_size=10, mask_value=None, inplace=False):
cloned = x.clone() if not inplace else x
num_bins = cloned.shape[1]
mask_value = cloned.mean() if mask_value is None else mask_value
for i in range(num):
f = random.randrange(0, mask_size + 1)
 if f == 0 or f >= num_bins: continue
f_low = random.randrange(0, num_bins - f)
 f_high = f_low + f
cloned[:, f_low:f_high] = mask_value
return cloned
def time_mask(x, num=1, mask_size=10, mask_value=None, inplace=False):
cloned = x.clone() if not inplace else x
num_frames = cloned.shape[2]
mask_value = cloned.mean() if mask_value is None else mask_value
for i in range(num):
t = random.randrange(0, mask_size + 1)
 if t == 0 or t >= num_frames: continue
t_beg = random.randrange(0, num_frames - t)
 t_end = t_beg + t
cloned[:, :, t_beg:t_end] = mask_value
return cloned<load_pretrained>
submission.to_csv('submission.csv', index=False )
class AugmentationConfig:
padding_scale = 1.
 whitenoise = True
whitenoise_level = 1e-3
pitchshift = True
pitchshift_steps = 2.
 class PreproConfig:
sr = 44100
duration = 2.
 n_out = 128
n_mels = 128
n_fft = n_mels * 20
hop_len = int(sr * duration // n_out)
 sample_size = int(sr * duration)
 padding_size = int(sample_size * AugmentationConfig.padding_scale)
 def load_audio(file, eps=1e-6):
pcm, sr = librosa.load(file, sr=PreproConfig.sr)
 if len(pcm)<= 0:
pcm = np.zeros(PreproConfig.sample_size)+ 1e-8
sr = PreproConfig.sr
pcm_max = np.max(np.abs(pcm))
if pcm_max > eps:
pcm = pcm / pcm_max
min_size = PreproConfig.padding_size
if len(pcm)> min_size: pcm, _ = librosa.effects.trim(pcm)
 pcm = pad_zeros(pcm, min_size)
 return pcm, pcm_max
def pad_zeros(x, size):
if len(x)> size:
return x
else:
padding = size - len(x)
 offset = padding // 2
return np.pad(x,(offset, size - len(x)- offset), 'constant')
 def conv_pcm_to_magphs(x):
fft = librosa.core.stft(x, n_fft=PreproConfig.n_fft, hop_length=PreproConfig.hop_len)
 return np.abs(fft), np.angle(fft)
 def conv_magphs_to_mels(magphs):
mel_basis = librosa.filters.mel(PreproConfig.sr, PreproConfig.n_fft, n_mels=PreproConfig.n_mels)
 return np.dot(mel_basis, magphs)
 def conv_mag_to_melspec(mag):
mels = conv_magphs_to_mels(mag ** 2.)
mels = librosa.power_to_db(mels)
 m_max = mels.max()
m_min = mels.min()
return(mels - m_min)/(m_max - m_min)
 def normalize_melspec(melspec, eps=1e-6, dtype=np.uint8):
mean = melspec.mean()
std = melspec.std()
ms_std =(melspec - mean)/(std + eps)
 ms_min, ms_max = ms_std.min() , ms_std.max()
if(ms_max - ms_min)> eps:
ms_norm = 255.*(ms_std - ms_min)/(ms_max - ms_min)
 return ms_norm.astype(dtype)
 else:
return np.zeros_like(ms_std, dtype=dtype)
 def preprocess(pcm, debug=False, dtype=np.uint8):
melspec, melphss = conv_pcm_to_magphs(pcm)
 melspec = conv_mag_to_melspec(melspec)
 melspec = normalize_melspec(melspec, dtype=dtype)
 return melspec<load_pretrained>
submission.to_csv('submission.csv', index=False )