Kaggle competition: LANL Earthquake Prediction
train = pd.read_json(".. /input/train.json")
 train['inc_angle'] = pd.to_numeric(train['inc_angle'],errors='coerce')
 test = pd.read_json(".. /input/test.json")
 test['inc_angle'] = pd.to_numeric(test['inc_angle'],errors='coerce')
 train = train[train['inc_angle'].isnull() == False].reset_index(drop = True)
 train['mean'] = np.NaN
train['count'] = np.NaN
test['mean'] = np.NaN
test['count'] = np.NaN
train['neighbor_iceberg'] = np.NaN
test['neighbor_iceberg'] = np.NaN<feature_engineering>
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler
from sklearn.svm import NuSVR
from sklearn.metrics import mean_absolute_error
for i in train.index:
mean = train[np.abs(train['inc_angle'] - train.loc[i, 'inc_angle'])< 0.00000001]
count = mean.shape[0] + test[ np.abs(test['inc_angle'] - \
train.loc[i, 'inc_angle'])< 0.00000001 ].shape[0] - 1
train.loc[i, 'count'] = count
try:
mean = mean.drop(i)['is_iceberg'].mean()
except:
mean = np.NaN
if mean == mean:
train.loc[i, 'mean'] = mean
else:
train.loc[i, 'mean'] = 0.5
temp = train[np.abs(train['inc_angle'] - train.loc[i, 'inc_angle'])< 0.025].drop(i)
 train.loc[i,'neighbor_iceberg'] = sum(temp['is_iceberg']==1)*1./(1+temp.shape[0])
 for i in test.index:
mean = train[ np.abs(train['inc_angle'] - test.loc[i, 'inc_angle'])< 0.00000001 ]
count = mean.shape[0] + test[ np.abs(test['inc_angle'] - \
test.loc[i, 'inc_angle'])< 0.00000001 ].shape[0] - 1
test.loc[i,'count'] = count
mean = mean['is_iceberg'].mean()
if mean == mean:
test.loc[i, 'mean'] = mean
else:
test.loc[i, 'mean'] = 0.5
temp = train[np.abs(train['inc_angle'] - test.loc[i, 'inc_angle'])< 0.025]
test.loc[i,'neighbor_iceberg'] = sum(temp['is_iceberg']==1)*1./(1+temp.shape[0] )<train_model>
train = pd.read_csv('.. /input/train.csv',nrows = 10**5, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})
clf = LogisticRegression()
clf.fit(train[['mean', 'count', 'neighbor_iceberg']], train['is_iceberg'] )<save_to_csv>
pd.options.display.precision = 15
predictions = pd.DataFrame([])
 predictions['id'] = test['id'].copy()
predictions['is_iceberg'] = clf.predict_proba(test[['mean', 'count', 'neighbor_iceberg']])[:,1]
predictions.to_csv("./submission.csv", index=False )<set_options>
train.rename({"acoustic_data": "signal", "time_to_failure": "time"}, axis="columns", inplace=True )
%matplotlib inline 
<load_from_disk>
train = train.fillna(train.mean() )
def load_train_data() :
data = pd.read_json('.. /input/train.json')
 return data
def load_test_data() :
data = pd.read_json('.. /input/test.json')
 return data
def denoise(x, weight, multichannel):
return np.asarray([denoise_tv_chambolle(item, weight=weight, multichannel=multichannel)for item in x])
 def gaussian_blur(x, sigma, multichannel):
return np.asarray([gaussian(item, sigma=sigma, multichannel=multichannel)for item in x])
 def color_composite(data):
rgb_arrays = []
for i, row in data.iterrows() :
band_1 = np.array(row['band_1'] ).reshape(75, 75)
 band_2 = np.array(row['band_2'] ).reshape(75, 75)
 band_3 = band_1 / band_2
r =(band_1 + abs(band_1.min())) / np.max(( band_1 + abs(band_1.min())))
 g =(band_2 + abs(band_2.min())) / np.max(( band_2 + abs(band_2.min())))
 b =(band_3 + abs(band_3.min())) / np.max(( band_3 + abs(band_3.min())))
 rgb = np.dstack(( r, g, b))
rgb_arrays.append(rgb)
 return np.array(rgb_arrays )<statistical_test>
def lee_filter(img, size):
img_mean = uniform_filter(img,(size, size))
img_sqr_mean = uniform_filter(img**2,(size, size))
img_variance = img_sqr_mean - img_mean**2
overall_variance = variance(img)
 img_weights = img_variance**2 /(img_variance**2 + overall_variance**2)
 img_output = img_mean + img_weights *(img - img_mean)
 return img_output<split>
x_train=x_train.fillna(x_train.mean() )
train_df = load_train_data()
test_df = load_test_data()<data_type_conversions>
label = np.array(train_df["is_iceberg"])
 train_angle = train_df['inc_angle']
train_angle = train_angle.replace('na',np.nan)
 train_angle = train_angle.astype(float ).fillna(np.mean(train_angle))
test_angle = test_df['inc_angle']
test_angle = test_angle.replace('na',np.nan)
 test_angle = test_angle.astype(float ).fillna(0)
 print('Done' )<categorify>
def create_dataset(data, labeled, weight_gray=0.05, weight_rgb=0.05):
band_1, band_2, images = data['band_1'].values, data['band_2'], color_composite(data)
 to_arr = lambda x: np.asarray([np.asarray(item)for item in x])
 band_1 = to_arr(band_1)
 band_2 = to_arr(band_2)
 band_3 =(band_1 / band_2)
 to_rgb = lambda x: np.asarray([(item + abs(item.min())) / np.max(( item + abs(item.min())))for item in x])
 band_1 = to_rgb(band_1)
 band_2 = to_rgb(band_2)
 band_3 = to_rgb(band_3)
 gray_reshape = lambda x: np.asarray([item.reshape(75, 75)for item in x])
 band_1 = gray_reshape(band_1)
 band_2 = gray_reshape(band_2)
 band_3 = gray_reshape(band_3)
 print('denoising')
 tf_reshape = lambda x: np.asarray([item.reshape(75, 75, 1)for item in x])
 band_1 = tf_reshape(band_1)
 band_2 = tf_reshape(band_2)
 band_3 = tf_reshape(band_3)
 img_reshape = lambda x: np.asarray([item.reshape(75,75,3)for item in x])
 images = img_reshape(images)
 band = np.concatenate([band_1, band_2, band_3], axis=3)
 x_angle = data.inc_angle
x_angle = x_angle.replace('na',np.nan)
 x_angle = x_angle.astype(float ).fillna(np.mean(x_angle))
x_angle = np.array(x_angle)
 if labeled:
y = np.array(data['is_iceberg'])
 else:
y = None
print('Done')
 return y, x_angle, band, images<prepare_x_and_y>
x_train = train[['signal']]
y_train = train[['time']]
y, x_angle, x_band, x_img = create_dataset(train_df, True )<concatenate>
scaler = StandardScaler()
scaler.fit(x_train)
 X_train_scaled = scaler.transform(x_train )
np.amax(x_band )<split>
svm = NuSVR()
svm.fit(X_train_scaled, y_train.values.flatten())
 y_pred = svm.predict(X_train_scaled )
x_train, x_val, \
x_angle_train, x_angle_val, \
y_train, y_val = train_test_split(x_img,x_angle, y, random_state=666, test_size=0.25)
 print('img', x_train.shape, y_train.shape)
 print('angle', x_angle_train.shape,y_val.shape)
<choose_model_class>
score = mean_absolute_error(y_train.values.flatten() , y_pred)
 print(f'Score: {score:0.3f}' )
adamax01 = Adamax(lr=0.0003 )<choose_model_class>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 submission.head()
def model2(kernel_size=3, filters_1=32, filters_2=32, filters_3=64, filters_4=128,
optimizers=adamax01,init='lecun_normal',relu_type='selu'):
band_input = Input(shape=(75, 75, 3))
cnn = BatchNormalization()(band_input)
 for i in range(1):
cnn = Conv2D(filters=filters_1, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 for i in range(1):
cnn = Conv2D(filters=filters_2, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 for i in range(1):
cnn = Conv2D(filters=filters_3, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 for i in range(1):
cnn = Conv2D(filters=filters_4, kernel_size=(kernel_size,kernel_size),
kernel_initializer=init )(cnn)
 cnn = Activation(relu_type )(cnn)
 cnn = MaxPooling2D(( 2,2), strides=(2,2))(cnn)
 cnn = Flatten()(cnn)
 cnn = Dense(512,activation= relu_type )(cnn)
 cnn = Dense(1, activation = 'sigmoid' )(cnn)
 simple_cnn = Model(inputs=[band_input],outputs=cnn)
 simple_cnn.compile(optimizer=optimizers, loss = 'binary_crossentropy', metrics = ['accuracy'])
 return simple_cnn
m = model2()
m.summary()<concatenate>
X_test = pd.DataFrame(columns=x_train.columns, dtype=np.float64, index=submission.index )
gen = ImageDataGenerator(horizontal_flip = True,
vertical_flip = True,
width_shift_range = 0,
height_shift_range = 0,
zoom_range = 0.2,
rotation_range = 20)
 def gen_flow_for_one_inputs(X1, y):
genX1 = gen.flow(X1,y, batch_size=64,seed=1)
 while True:
X1i = genX1.next()
yield X1i[0], X1i[1]
gen_flow = gen_flow_for_one_inputs(x_train, y_train)
 gen_val = gen_flow_for_one_inputs(x_val, y_val)
 model = model2()
ss = model.fit_generator(gen_flow, validation_data=(x_val,y_val),
steps_per_epoch=len(x_train)/ 64,
epochs=130,
callbacks = callbacks_list,
verbose = 1)
<train_model>
for seg_id in X_test.index:
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 x = seg['acoustic_data'].values
print(x)
 X_test.loc[seg_id,'signal'] = np.median(x )
model.load_weights(filepath=filepath)
 score = model.evaluate(x_train, y_train,verbose=1)
 print('Train loss:', score[0])
 print('Train accuracy:', score[1])
 test_score = model.evaluate(x_val, y_val,verbose=1)
 print('Test loss:', test_score[0])
 print('Test accuracy:', test_score[1])
<load_pretrained>
X_test.isnull().sum()
<create_dataframe><EOS> 
X_test_scaled = scaler.transform(X_test)
 submission['time_to_failure'] = svm.predict(X_test_scaled)
 submission.to_csv('submission.csv' )