Kaggle competition: TrackML Particle Tracking Challenge
<SOS> metric: TrackML Kaggle data source: trackml-particle-identification<predict_on_test>
import numpy as np
import pandas as pd
import tensorflow as tf
from tqdm import tqdm
import glob
import os
from trackml.dataset import load_event, load_dataset
from trackml.score import score_event
from IPython.display import display
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization
from keras.layers.embeddings import Embedding
prediction = classifier.predict(test)
 predictions = np.argmax(prediction, axis=1 )<save_to_csv>
train = np.unique([p.split('-')[0] for p in sorted(glob.glob('.. /input/train_*/**')) ])
 test = np.unique([p.split('-')[0] for p in sorted(glob.glob('.. /input/test/**')) ])
 def read_files(path):
return [p.split('-')[0] for p in sorted(glob.glob(path)) ]
train_file_names = np.unique(read_files('.. /input/train_*/**.csv'))
test_file_names = np.unique(read_files('.. /input/test/**.csv'))
detector = pd.read_csv('.. /input/detectors.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )
submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("working/simple_cnn_kaggle.csv", index=False, header=True)
 submissions.shape<load_from_csv>
denoise_model = Sequential()
denoise_model.add(Dense(256, activation='relu', input_shape=[25, ]))
denoise_model.add(Dense(256, activation='relu'))
denoise_model.add(Dense(256, activation='relu'))
denoise_model.add(Dropout(0.5))
denoise_model.add(Dense(256, activation='relu'))
denoise_model.add(Dense(256, activation='relu'))
denoise_model.add(Dense(256, activation='relu'))
denoise_model.add(BatchNormalization())
 denoise_model.add(Dense(1, activation='sigmoid'))
denoise_model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'] )
train = pd.read_csv(".. /input/titanic/train.csv")
 y_train = train["Survived"]
train.drop("Survived",axis=1,inplace=True)
 test = pd.read_csv(".. /input/titanic/test.csv")
 sample_sub = pd.read_csv(".. /input/titanic/gender_submission.csv" )<feature_engineering>
def get_train_data_for_denoise(e):
hits, cells, truth = load_event(e, parts=['hits', 'cells', 'truth'])
 hits['event_id'] = int(e[-9:])
 cells = cells.groupby(by=['hit_id'])['ch0', 'ch1', 'value'].agg(
 ['mean', 'median', 'max', 'min', 'sum'] ).reset_index()
cells.columns = ['hit_id'] + ['-'.join([c2, c1])for c1 in [
'mean', 'median', 'max', 'min', 'sum']
for c2 in ['ch0', 'ch1', 'value']]
hits = pd.merge(hits, cells, how='left', on='hit_id')
 tcols = list(truth.columns)
 hits = pd.merge(hits, truth, how='left', on='hit_id')
 hits = norm_points(hits)
 cols = [c for c in hits.columns if c not in [
'event_id', 'hit_id', 'particle_id'] + tcols]
hits['target'] = hits['particle_id'].map(lambda x: 0 if x == 0 else 1)
 return hits[cols], hits['target'].values
def norm_points(df, g=0.7):
x = df.x.values
y = df.y.values
z = df.z.values
r = np.sqrt(x**2 + y**2 + z**2)
 df['x2'] = x / r
df['y2'] = y / r
r = np.sqrt(x**2 + y**2)
 df['z2'] =(z / r)* g
df['r'] = r
return df
def search_title(name):
match = re.search("([A-Za-z]+)\.",name)
 if match: return match.group(1)
 else: return ""
train.Name = train.Name.apply(search_title)
 test.Name = test.Name.apply(search_title )<feature_engineering>
train["Age"].fillna(train["Age"].median() , inplace=True)
 test ["Age"].fillna(train["Age"].median() , inplace=True)
 train["Embarked"].fillna(train["Embarked"].mode() [0], inplace=True)
 test ["Embarked"].fillna(train["Embarked"].mode() [0], inplace=True)
 train["Cabin"] = train["Cabin"].apply(( lambda x: 0 if type(x)== float else 1))
test ["Cabin"] = test ["Cabin"].apply(( lambda x: 0 if type(x)== float else 1))
train["TotalFamily"] = train["SibSp"] + train["Parch"]
test ["TotalFamily"] = test ["SibSp"] + test ["Parch"]
test ["Fare"].fillna(train["Fare"].median() , inplace=True )<categorify>
scl = preprocessing.StandardScaler()
dbscan = hdbscan.HDBSCAN(min_samples=3,
min_cluster_size=5,
cluster_selection_method='leaf',
prediction_data=False,
metric='braycurtis' )
oneHot = OneHotEncoder()
oneHotName = OneHotEncoder(handle_unknown="ignore")
 ordinal = OrdinalEncoder()
train = pd.concat([train,
pd.DataFrame(oneHot.fit_transform(train[["Embarked"]] ).toarray()),
pd.DataFrame(ordinal.fit_transform(train[["Sex"]])) ,
pd.DataFrame(oneHotName.fit_transform(train[["Name"]] ).toarray())], axis=1)
 train.drop(columns=["Sex","Embarked","Name","Ticket","PassengerId"],inplace=True)
 test = pd.concat([test,
pd.DataFrame(oneHot.transform(test[["Embarked"]] ).toarray()),
pd.DataFrame(ordinal.transform(test[["Sex"]])) ,
pd.DataFrame(oneHotName.transform(test[["Name"]] ).toarray())], axis=1)
 test.drop(columns=["Sex","Embarked","Name","Ticket","PassengerId"],inplace=True )<train_model>
def create_one_event_submission(event_id, hits, labels):
sub_data = np.column_stack(( [event_id]*len(hits), hits.hit_id.values, labels))
submission = pd.DataFrame(data=sub_data, columns=["event_id", "hit_id", "track_id"] ).astype(int)
 return submission
<normalization><EOS> 
submission = create_one_event_submission(0, hits, labels)
 score = score_event(truth, submission)
 print("Your score: ", score )