Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<feature_engineering>
le = preprocessing.LabelEncoder()
RAW_DATA_PATH = ".. /input/career-con-2019"
warnings.filterwarnings('ignore')
 %matplotlib inline
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100<feature_engineering>
data_target = pd.read_csv("{}/y_train.csv".format(RAW_DATA_PATH))
train_X = pd.read_csv('{}/X_train.csv'.format(RAW_DATA_PATH)).iloc[:,3:]
test_X = pd.read_csv('{}/X_test.csv'.format(RAW_DATA_PATH)).iloc[:,3:]
train_X = train_X.append(test_X)
 train_X = train_X.values.reshape(-1,128,10 )
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096<feature_engineering>
train_left_edges, train_left_linked = find_run_edges(train_X, edge='left')
 train_right_edges, train_right_linked = find_run_edges(train_X, edge='right')
 print('Found', len(train_left_edges), 'left edges and', len(train_right_edges), 'right edges.' )
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100<feature_engineering>
train_runs = find_runs(train_X, train_left_edges, train_right_edges )
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
lost_samples = np.array([ i for i in range(len(train_X)) if i not in np.concatenate(train_runs)])
 print(lost_samples)
 print(len(lost_samples))
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
first_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==821:
print(i)
 first_double_surface = i
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283<feature_engineering>
new_train_runs = [821, 974, 328, 1548, 172,
355, 957, 1481, 1046, 1650, 857, 724, 164, 1092, 1017, 1300,
1212, 536, 531, 1032, 994, 1501, 588, 579, 1177, 812, 1333,
1253]
train_runs[first_double_surface] = train_runs[first_double_surface][0:-len(new_train_runs)]
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087<feature_engineering>
train_runs.append(np.array(new_train_runs))
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090<feature_engineering>
second_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==3055:
print(i)
 second_double_surface = i
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
new_train_runs2 = [3055, 3360, 3662,
3780, 3663, 3091, 3769, 3175, 1957, 2712, 2063, 2708, 3139, 2722]
train_runs[second_double_surface] = train_runs[second_double_surface][0:-len(new_train_runs2)]
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
train_runs.append(np.array(new_train_runs2))
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402<feature_engineering>
third_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==2484:
print(i)
 third_double_surface = i
n=17
commits_df.loc[n,'commit_num'] = 26
commits_df.loc[n,'Dropout_model'] = 0.385
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
train_runs[third_double_surface]
n=18
commits_df.loc[n,'commit_num'] = 27
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.21
commits_df.loc[n,'LB_score'] = -6.8091<feature_engineering>
new_train_runs3 = [2484, 3062, 2290, 3517, 3293, 2651, 3767,
2029, 2558, 3580, 1874, 3373, 2514, 2308, 3160, 3161, 3613, 2511,
2469, 2990, 2780, 3756, 2376, 2616, 2540, 2039, 2219, 3743, 3198,
2584, 2752, 2304, 2887, 2841, 3480, 2517, 3020, 3424, 2027, 2652,
2648, 3433, 2359, 3392, 3164, 3798, 3642, 2713, 3405, 3673, 2369,
3411, 3595, 2242, 2307, 1897, 2834, 2350, 3795, 2948, 1856, 3486,
3353, 1966]
train_runs[third_double_surface] = train_runs[third_double_surface][0:-len(new_train_runs3)]
n=19
commits_df.loc[n,'commit_num'] = 28
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090<feature_engineering>
train_runs.append(np.array(new_train_runs3))
n=20
commits_df.loc[n,'commit_num'] = 29
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'GaussianNoise_stddev'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
fourth_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==3501:
print(i)
 fourth_double_surface = i
n=21
commits_df.loc[n,'commit_num'] = 31
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'GaussianNoise_stddev'] = 0.21
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
train_runs[fourth_double_surface]
n=22
commits_df.loc[n,'commit_num'] = 32
commits_df.loc[n,'Dropout_model'] = 0.24
commits_df.loc[n,'FVC_weight'] = 0.14
commits_df.loc[n,'GaussianNoise_stddev'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8106<feature_engineering>
new_train_runs4 = [3501, 2785]
train_runs[fourth_double_surface] = train_runs[fourth_double_surface][0:-len(new_train_runs4)]
train_runs.append(np.array(new_train_runs4))
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1<feature_engineering>
df_train_y = pd.DataFrame()
df_train_y['run_id'] = 0
df_train_y['run_pos'] = 0
for run_id in range(len(train_runs)) :
for run_pos in range(len(train_runs[run_id])) :
series_id = train_runs[run_id][run_pos]
df_train_y.at[ series_id, 'run_id' ] = run_id
df_train_y.at[ series_id, 'run_pos' ] = run_pos
df_train_y.tail()
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0<feature_engineering>
df_train_y['index'] = df_train_y.index
df_train_y = df_train_y.sort_values('index')
 df_train_y.rename(columns={'index':'series_id'}, inplace=True)
 df_train_y['run_id'] = df_train_y['run_id'].apply(lambda x: int(x))
df_train_y['run_pos'] = df_train_y['run_pos'].apply(lambda x: int(x))
df_train_y.tail()
commits_df['GaussianNoise_stddev'] = 0.2
commits_df.loc[commits_df['commit_num'] == 28, 'GaussianNoise_stddev'] = 0.25<sort_values>
run_id_train = df_train_y['run_id'][0:3810].values
run_id_test = df_train_y['run_id'][3810:7626].values
commits_df.sort_values(by=['LB_score'], ascending = False )<load_from_csv>
data_target['run_id'] = run_id_train
mapping_leak = {}
for i in range(0,3810):
cur_data = data_target.iloc[i]
mapping_leak.update({cur_data['run_id']: cur_data['surface']})
 unknown_run = []
ans_test = []
known_series = []
unknown_series = []
for i in range(0,3816):
if run_id_test[i] in mapping_leak:
ans_test.append(mapping_leak[run_id_test[i]])
 known_series.append(i)
 else:
ans_test.append('unknown')
 unknown_series.append(i)
 unknown_run.append(run_id_test[i] )
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )<concatenate>
sub = pd.read_csv("{}/sample_submission.csv".format(RAW_DATA_PATH))
sub['surface'] = ans_test
sub.head()
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )<feature_engineering>
best = pd.read_csv('.. /input/best-73/submission_24.csv')
 map_best_ans = {}
for i in range(0, best.shape[0]):
map_best_ans.update({best.iloc[i]['series_id'] : best.iloc[i]['surface'] })
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )<feature_engineering>
result = []
for i in range(0, sub.shape[0]):
if(sub.surface[i] == 'unknown'):
result.append(map_best_ans[i])
 else:
result.append(sub.surface[i] )
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
sub.surface = result
sub.to_csv('final_submission.csv', index=False )
<split><EOS> 
sub.surface.value_counts()