Kaggle competition: Predictive Equipment Failures
<SOS> metric: meanfscore Kaggle data source: predictive-equipment-failures<set_options>
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.externals import joblib
%matplotlib inline 
<load_from_csv>
plt.rcParams['figure.figsize'] =(10, 8 )
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
train_og = pd.read_csv('/kaggle/input/equipfails/equip_failures_training_set.csv' )
train_images=train.iloc[:,1:].values
train_labels=train.iloc[:,0:1].values
test_X=test.iloc[:,:].values<data_type_conversions>
test_og = pd.read_csv('/kaggle/input/equipfails/equip_failures_test_set.csv' )
train_images = train_images.reshape(( -1, 28, 28, 1))
train_images = train_images.astype('float32')/ 255
test = test.values.reshape(-1,28,28,1)
 test = test / 255.0<categorify>
data = pd.concat([train_og, test_og], ignore_index=True, sort=False )
train_labels = to_categorical(train_labels )<define_variables>
data = data.replace('na', np.nan )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(train_images )<choose_model_class>
for c in data.columns:
data[c] = pd.to_numeric(data[c] )
model = models.Sequential()
model.add(layers.Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',activation ='relu', input_shape =(28,28,1)))
 model.add(layers.Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',activation ='relu'))
model.add(layers.MaxPool2D(pool_size=(2,2)))
 model.add(layers.Dropout(0.25))
model.add(layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(layers.MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(layers.Dropout(0.25))
model.add(layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(layers.MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(layers.Dropout(0.25))
model.add(layers.Flatten())
 model.add(layers.Dropout(0.25))
model.add(layers.Dense(256, activation='relu',kernel_regularizer=regularizers.l2(0.001)))
 model.add(layers.Dropout(0.35))
model.add(layers.Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] )<train_model>
data_nonna = data.iloc[:,2:].fillna(data.mean() )
history = model.fit_generator(datagen.flow(train_images,train_labels, batch_size=64),
epochs = 100,
verbose = 2, steps_per_epoch=train_images.shape[0] // 64
)<predict_on_test>
scaler = MinMaxScaler()
results = model.predict(test )<save_to_csv>
scaler.fit(data_nonna )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<save_model>
X = scaler.transform(data_nonna )
model.save('digit_clfr.h5' )<import_modules>
y = data.target.values
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler<load_from_csv>
m = train_og.shape[0]
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
X_train, X_valid, y_train, y_valid = train_test_split(X[:m], y[:m], test_size=0.3, random_state = 0 )
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_train = X_train / 255.0
X_test = test / 255.0
X_train = X_train.values.reshape(-1,28,28,1)
 X_test = X_test.values.reshape(-1,28,28,1)
 Y_train = to_categorical(Y_train, num_classes = 10 )<create_dataframe>
train_og.target.value_counts(normalize = True )
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )<choose_model_class>
def get_f1_score(clf, X_train, X_valid, y_train, y_valid):

 y_train_preds = clf.predict(X_train)
 train_f1_score = f1_score(y_pred=y_train_preds, y_true=y_train)
 y_valid_preds = clf.predict(X_valid)
 valid_f1_score = f1_score(y_pred=y_valid_preds, y_true=y_valid)
 return train_f1_score, valid_f1_score
nets = 15
model = [0] *nets
for j in range(nets):
model[j] = Sequential()
model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Flatten())
 model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )<train_model>
lrclassifier = LogisticRegression()
lrclassifier.fit(X_train, y_train )
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * nets
epochs = 45
for j in range(nets):
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )<save_to_csv>
get_f1_score(lrclassifier, X_train, X_valid, y_train, y_valid )
results = np.zeros(( X_test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST-CNN-ENSEMBLE.csv",index=False )<load_from_csv>
def select_sample(mult):
sample_size = mult*y_train[y_train==1].shape[0]
np.random.seed(2)
 sample_idx = np.concatenate([np.random.choice(np.where(y_train==0)[0], size = sample_size), np.where(y_train==1)[0]])
 X_train_sample = X_train[sample_idx]
y_train_sample = y_train[sample_idx]
return X_train_sample, y_train_sample
test_data_with_labels = pd.read_csv('.. /input/titanic-test-data/titanic.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv' )<set_options>
def get_f1_by_sample_lr(mult, is_sample=True):
if is_sample:
X_train_sample, y_train_sample = select_sample(mult)
 else:
X_train_sample, y_train_sample = X_train, y_train
lrclassifier = LogisticRegression()
lrclassifier.fit(X_train_sample, y_train_sample)
 train_f1_score, valid_f1_score = get_f1_score(lrclassifier, X_train_sample, X_valid, y_train_sample, y_valid)
 return {'sample_frac': X_train_sample.shape[0]/X_train.shape[0], 'train_f1_score': train_f1_score, 'valid_f1_score': valid_f1_score}
warnings.filterwarnings('ignore' )<feature_engineering>
num_mults = int(sum(y_train==0)/sum(y_train==1))
for i, name in enumerate(test_data_with_labels['name']):
if '"' in name:
test_data_with_labels['name'][i] = re.sub('"', '', name)
 for i, name in enumerate(test_data['Name']):
if '"' in name:
test_data['Name'][i] = re.sub('"', '', name )<filter>
f1_mult_lr = []
mult = 1
while mult < num_mults:
f1_mult_lr.append(get_f1_by_sample_lr(mult))
if mult==1:
mult+=9
else:
mult+=10
f1_mult_lr.append(get_f1_by_sample_lr(_, is_sample=False))
survived = []
for name in test_data['Name']:
survived.append(int(test_data_with_labels.loc[test_data_with_labels['name'] == name]['survived'].values[-1]))<save_to_csv>
f1_mult_lr_df = pd.io.json.json_normalize(f1_mult_lr )
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission['Survived'] = survived
submission.to_csv('submission.csv', index=False )<set_options>
def get_f1_by_sample_rf(mult, is_sample=True):
if is_sample:
X_train_sample, y_train_sample = select_sample(mult)
 else:
X_train_sample, y_train_sample = X_train, y_train
rfclassifier = RandomForestClassifier(n_estimators = 50, max_features = 10)
 rfclassifier.fit(X_train_sample, y_train_sample)
 train_f1_score, valid_f1_score = get_f1_score(rfclassifier, X_train_sample, X_valid, y_train_sample, y_valid)
 return {'sample_frac': X_train_sample.shape[0]/X_train.shape[0], 'train_f1_score': train_f1_score, 'valid_f1_score': valid_f1_score}
%matplotlib inline 
<import_modules>
f1_mult_rf = []
mult = 1
while mult < num_mults:
f1_mult_rf.append(get_f1_by_sample_rf(mult))
if mult==1:
mult+=9
else:
mult+=10
f1_mult_rf.append(get_f1_by_sample_rf(_, is_sample=False))
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler<define_variables>
f1_mult_rf_df = pd.io.json.json_normalize(f1_mult_rf )
train_file = ".. /input/train.csv"
test_file = ".. /input/test.csv"
output_file = "submission.csv"<split>
X_train_sample, y_train_sample = select_sample(mult=50)
 f1_est = []
num_estimators = 10
while num_estimators <= 100:
rfclassifier = RandomForestClassifier(n_estimators = num_estimators, max_features = 10)
 rfclassifier.fit(X_train_sample, y_train_sample)
 train_f1_score, valid_f1_score = get_f1_score(rfclassifier, X_train_sample, X_valid, y_train_sample, y_valid)
 f1_est.append({'num_estimators': num_estimators, 'train_f1_score': train_f1_score, 'valid_f1_score': valid_f1_score})
 num_estimators+=10
raw_data = np.loadtxt(train_file, skiprows=1, dtype='int', delimiter=',')
 x_train, x_val, y_train, y_val = train_test_split(
 raw_data[:,1:], raw_data[:,0], test_size=0.1 )<data_type_conversions>
f1_est_df = pd.io.json.json_normalize(f1_est )
x_train = x_train.astype("float32")/255.
 x_val = x_val.astype("float32")/255 .<categorify>
X_train_sample, y_train_sample = select_sample(mult=50)
 f1_feat = []
max_features = 2
while max_features <= 20:
rfclassifier = RandomForestClassifier(n_estimators = 60, max_features = max_features)
 rfclassifier.fit(X_train_sample, y_train_sample)
 train_f1_score, valid_f1_score = get_f1_score(rfclassifier, X_train_sample, X_valid, y_train_sample, y_valid)
 f1_feat.append({'max_features': max_features, 'train_f1_score': train_f1_score, 'valid_f1_score': valid_f1_score})
 max_features+=1
y_train = to_categorical(y_train)
 y_val = to_categorical(y_val)
 print(y_train[0] )<choose_model_class>
f1_feat_df = pd.io.json.json_normalize(f1_feat )
model = Sequential()
model.add(Conv2D(filters = 16, kernel_size =(3, 3), activation='relu',
input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 16, kernel_size =(3, 3), activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))<choose_model_class>
final_classifier = RandomForestClassifier(n_estimators = 60, max_features = 19)
 final_classifier.fit(X_train_sample, y_train_sample )
datagen = ImageDataGenerator(zoom_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1,
rotation_range = 10 )<choose_model_class>
get_f1_score(final_classifier, X_train_sample, X_valid, y_train_sample, y_valid )
model.compile(loss='categorical_crossentropy', optimizer = Adam(lr=1e-4), metrics=["accuracy"] )<choose_model_class>
X_test = X[m:]
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )<train_model>
y_test_preds = final_classifier.predict(X_test )
hist = model.fit_generator(datagen.flow(x_train, y_train, batch_size=16),
steps_per_epoch=500,
epochs=20,
verbose=2,
validation_data=(x_val[:400,:], y_val[:400,:]),
callbacks=[annealer] )<compute_test_metric>
submission = pd.DataFrame({'id': test_og.id, 'target': y_test_preds.astype(int)} )
final_loss, final_acc = model.evaluate(x_val, y_val, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))<compute_test_metric>
submission.to_csv('submission_dk_201910192153.csv',index=False )
y_hat = model.predict(x_val)
 y_pred = np.argmax(y_hat, axis=1)
 y_true = np.argmax(y_val, axis=1)
 cm = confusion_matrix(y_true, y_pred)
 print(cm )<data_type_conversions>
test_sample.replace('na', np.nan, inplace = True )
mnist_testset = np.loadtxt(test_file, skiprows=1, dtype='int', delimiter=',')
 x_test = mnist_testset.astype("float32")
 x_test = x_test.reshape(-1, 28, 28, 1)/255 .<predict_on_test>
test_sample_nonna = test_sample.fillna(data.mean() )
y_hat = model.predict(x_test, batch_size=64 )<prepare_output>
X_test_sample = scaler.transform(test_sample_nonna.iloc[:,1:] )
y_pred = np.argmax(y_hat,axis=1 )<save_to_csv>
y_test_sample_preds = final_classifier.predict(X_test_sample )
<set_options><EOS> 
data.mean().to_pickle('data_means.pkl' )