Kaggle competition: INGV - Volcanic Eruption Prediction
<SOS> metric: MAE Kaggle data source: ingv-volcanic-eruption-prediction<groupby>
pd.options.display.max_columns = None
%matplotlib inline
print(combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean())
 Pclass_1_mean_fare = combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().get([1] ).values[0]
Pclass_2_mean_fare = combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().get([2] ).values[0]
Pclass_3_mean_fare = combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().get([3] ).values[0]<categorify>
KAGGLE = True
if KAGGLE:
DIR = '.. /input/predict-volcanic-eruptions-ingv-oe'
else:
DIR = './predict-volcanic-eruptions-ingv-oe/'
combined_train_test['Pclass_Fare_Category'] = combined_train_test.apply(pclass_fare_category, args=(Pclass_1_mean_fare, Pclass_2_mean_fare, Pclass_3_mean_fare), axis = 1)
 print(combined_train_test['Pclass_Fare_Category'].groupby(by = combined_train_test['Pclass_Fare_Category'] ).count().sort_values(ascending = False))
le_fare = LabelEncoder()
le_fare.fit(np.array(['Pclass_1_Low_Fare', 'Pclass_1_High_Fare', 'Pclass_2_Low_Fare', 'Pclass_2_High_Fare', 'Pclass_3_Low_Fare', 'Pclass_3_High_Fare']))
combined_train_test['Pclass_Fare_Category'] = le_fare.transform(combined_train_test['Pclass_Fare_Category'] )<groupby>
train = pd.read_csv(os.path.join(DIR, 'train.csv'))
test = pd.read_csv(os.path.join(DIR, 'sample_submission.csv'))
train
print(combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().sort_values(ascending = True))
combined_train_test['Pclass'].replace([1, 2, 3],[Pclass_1_mean_fare, Pclass_2_mean_fare, Pclass_3_mean_fare], inplace = True )<categorify>
train['h:m:s'] =(train['time_to_eruption'] 
.apply(lambda x:datetime.timedelta(seconds = x/100)))
 train
combined_train_test['Family_Size'] = combined_train_test['Parch'] + combined_train_test['SibSp'] + 1
print(combined_train_test['Family_Size'].groupby(by = combined_train_test['Family_Size'] ).count().sort_values(ascending = False))
combined_train_test['Family_Size_Category'] = combined_train_test['Family_Size'].map(family_size_category)
 print(combined_train_test['Family_Size_Category'].groupby(by = combined_train_test['Family_Size_Category'] ).count().sort_values(ascending = False))
print(combined_train_test.groupby(['Survived', 'Family_Size_Category'])['Survived'].count())
 le_family = LabelEncoder()
le_family.fit(np.array(['Single', 'Small_Family', 'Large_Family']))
combined_train_test['Family_Size_Category'] = le_family.transform(combined_train_test['Family_Size_Category'])
 fam_size_cat_dummies_df = pd.get_dummies(combined_train_test['Family_Size_Category'],
prefix = combined_train_test[['Family_Size_Category']].columns[0])
 combined_train_test = pd.concat([combined_train_test, fam_size_cat_dummies_df], axis = 1 )<sort_values>
sample_df =(train.sort_values('time_to_eruption')
.reset_index() 
.rename(columns={'index': 'train_id'}))
sample_df = sample_df[sample_df.index %(len(train)// 5)== 5].reset_index(drop = True)
 sample_ids = sample_df['segment_id'].values
sample_df
print(combined_train_test['Age'].groupby(by = combined_train_test['Title'] ).mean().sort_values(ascending = True))<feature_engineering>
fs = 100
N = len(segment_df)
 n = 256
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2
combined_train_test['Age_Null'] = combined_train_test['Age'].apply(lambda x: 1 if(pd.notnull(x)) else 0 )<categorify>
def make_features(tgt):
tgt_df = train if tgt == 'train' else test
feature_set = []
for segment_id in tqdm(tgt_df['segment_id']):
segment_df = pd.read_csv(os.path.join(DIR,f'{tgt}/{segment_id}.csv'))
segment = [segment_id]
for sensor in segment_df.columns:
x = segment_df[sensor][:N]
if x.isna().sum() > 1000:
segment +=([np.NaN] * 10)
 continue
f, t, Z = scipy.signal.stft(x.fillna(0), fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
segment += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
feature_set.append(segment)
 cols = ['segment_id']
for i in range(10):
for j in ['A_pow', 'A_num','BH_pow', 'BH_num','BL_pow', 'BL_num','C_pow', 'C_num','D_pow', 'D_num']:
cols += [f's{i+1}_{j}']
feature_df = pd.DataFrame(feature_set, columns = cols)
 feature_df['segment_id'] = feature_df['segment_id'].astype('int')
 return feature_df
missing_age_df = pd.DataFrame(combined_train_test[['Age', 'Parch', 'Sex', 'SibSp', 'Family_Size', 'Family_Size_Category', 'Title', 'Fare']])
 missing_age_df = pd.get_dummies(missing_age_df, columns = ['Title', 'Family_Size_Category', 'Sex'])
 missing_age_df.shape
missing_age_df.info()
missing_age_train = missing_age_df[missing_age_df['Age'].notnull() ]
missing_age_test = missing_age_df[missing_age_df['Age'].isnull() ]<feature_engineering>
feature_df = make_features('train')
 train_set = pd.merge(train, feature_df, on = 'segment_id')
 train_set
combined_train_test.loc[(combined_train_test.Age.isnull()), 'Age'] = fill_missing_age(missing_age_train, missing_age_test )<feature_engineering>
df = train_set.drop(['segment_id', 'time_to_eruption','h:m:s'], axis=1)
 y = train_set['time_to_eruption']
X_train, X_val, y_train, y_val = train_test_split(df, y,
random_state = 42,
test_size = 0.2,
shuffle = True)
 features = X_train.columns.tolist()
cat_features = {}
if(sum(n < 0 for n in combined_train_test.Age.values.flatten())> 0):
combined_train_test.loc[combined_train_test.Age < 0, 'Age'] = np.nan
combined_train_test['Age'] = combined_train_test[['Age']].fillna(combined_train_test.groupby('Title' ).transform('mean'))<sort_values>
def do_lgb(X_train, y_train, X_val, y_val):
params = {'objective': 'rmse',
'metric': 'rmse',
'max_depth':14,
'min_data_in_leaf':5,
'num_leaves': 2**7 - 1,
'learning_rate': 0.05,
'feature_fraction': 0.7,
'bagging_fraction': 0.5,
'bagging_freq': 5,
'lambda_l1':80,
'num_iterations': 10000,
'seed': 42,
'verbose': 1
}
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 evals_result = {}
model = lgb.train(
 params,
lgb_train,
valid_sets =(lgb_train, lgb_eval),
feature_name = features,
categorical_feature = cat_features,
verbose_eval = 100,
evals_result = evals_result,
early_stopping_rounds = 200)
 return model
print(combined_train_test['Age'].groupby(by = combined_train_test['Title'] ).mean().sort_values(ascending = True))<categorify>
lgb_model = do_lgb(X_train, y_train, X_val, y_val )
combined_train_test['Age_Category'] = combined_train_test['Age'].map(age_group_cat)
 le_age = LabelEncoder()
le_age.fit(np.array(['Baby', 'Toddler', 'Child', 'Teenager', 'Adult', 'Middle_Aged', 'Senior_Citizen', 'Old']))
combined_train_test['Age_Category'] = le_age.transform(combined_train_test['Age_Category'])
 age_cat_dummies_df = pd.get_dummies(combined_train_test['Age_Category'],
prefix = combined_train_test[['Age_Category']].columns[0])
 combined_train_test = pd.concat([combined_train_test, age_cat_dummies_df], axis = 1 )<categorify>
feature_df = make_features('test')
 test_set = pd.merge(test, feature_df, on = 'segment_id')
 test_set
combined_train_test['Ticket_Letter'] = combined_train_test['Ticket'].str.split().str[0]
combined_train_test['Ticket_Letter'] = combined_train_test['Ticket_Letter'].apply(lambda x: np.NaN if x.isnumeric() else x)
 combined_train_test['Ticket_Number'] = combined_train_test['Ticket'].apply(lambda x: pd.to_numeric(x, errors='coerce'))
combined_train_test['Ticket_Number'].fillna(0, inplace = True)
 combined_train_test = pd.get_dummies(combined_train_test, columns = ['Ticket', 'Ticket_Letter'])
 combined_train_test.shape<categorify>
preds = lgb_model.predict(test_set.drop(['segment_id', 'time_to_eruption'], axis=1))
test['time_to_eruption'] = preds
test[['segment_id','time_to_eruption']]
combined_train_test['Cabin_Letter'] = combined_train_test['Cabin'].apply(lambda x: str(x)[0] if(pd.notnull(x)) else x)
 combined_train_test = pd.get_dummies(combined_train_test, columns = ['Cabin', 'Cabin_Letter'])
 combined_train_test.shape<normalization>
test[['segment_id','time_to_eruption']].to_csv('submission.csv', index=False )