Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<set_options>
A = np.array([[1, 3, 5], [5, 4, 1], [3, 8, 6]])
 print(A)
 cov_matrix = np.cov(A, rowvar=False, bias=True)
 cov_matrix
warnings.filterwarnings("ignore" )<load_from_csv>
sns.set_style("dark")
train_data = pd.read_csv(path+'train.csv')
 samp_subm = pd.read_csv(path+'sample_submission.csv' )<define_variables>
train_df_org = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_df_org = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print("train_df_org shape ", train_df_org.shape)
 print("test_df_org shape ", test_df_org.shape)
 train_df_org.head()
print('Number train samples:', len(train_data.index))
print('Number test samples:', len(samp_subm.index))<load_from_csv>
standardized_data = StandardScaler().fit_transform(train_df_org)
 print(standardized_data.shape )
idnum = 2
image_id = train_data.loc[idnum, 'image_id']
data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')
 img = data_file.pixel_array<save_to_csv>
projected_vec = np.matmul(eigenvectors, sample_data.T)
 projected_vec.shape
samp_subm.to_csv('submission1.csv', index=False )<load_from_csv>
pca_dataframe = pd.DataFrame(data=projected_vec, columns=('1st_principal_comp', "2nd_principal_comp", 'labels'))
pca_dataframe.head()
pred_2class = pd.read_csv(".. /input/vinbigdata-2class-prediction/2-cls test pred.csv")
 low_threshold = 0.001
high_threshold = 0.87
pred_2class<load_from_csv>
pca = decomposition.PCA()
pca.n_components = 2
pca_data_with_scikit = pca.fit_transform(standardized_data)
 pca_data_with_scikit.shape
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(".. /input/vinbigdatastack/submission_postprocessed.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission2.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )<define_variables>
df_PCA_scikit = pd.DataFrame(data=pca_data_with_scikit, columns=('f1_PC', 'f2_PC', 'labels'))
df_PCA_scikit.head()
LABELS = ["isFraud"]<set_options>
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.models import Model
from tensorflow.keras import metrics
from tensorflow.keras import backend as K
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization
from tensorflow.keras.layers import Conv2D, MaxPool2D, AvgPool2D
from tensorflow.keras.optimizers import Adadelta
from keras.utils.np_utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.callbacks import LearningRateScheduler
%matplotlib inline
all_files = glob.glob(".. /input/lgmodels/*.csv")
 all_files 
<load_from_csv>
train_df_org = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 mnist_train_label = train_df_org.loc[:, "label"]
mnist_train_df = train_df_org.loc[:, "pixel0":]
predict_list = []
predict_list.append(pd.read_csv('.. /input/lgmodels/Submission-.9433.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9451.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9459.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9463.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-0.9467.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/Submission-.9440.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9454.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-0.9466.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-0.9475.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-0.9433.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-0.9468.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9452.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/Submission-.9429.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9449.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9457.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/Submission-.9438.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/Submission-.9442.csv')[LABELS].values)
 predict_list.append(pd.read_csv('.. /input/lgmodels/submission-.9469.csv')[LABELS].values)
<save_to_csv>
num_of_digit_classes = mnist_train_label_array.max() - mnist_train_label_array.min() + 1
mnist_train_label_array = to_categorical(mnist_train_label_array, num_classes=num_of_digit_classes)
 print('Shape of ytrain after encoding and converting to categorical values ', mnist_train_label_array.shape)
 print(mnist_train_label_array)
warnings.filterwarnings("ignore")
 print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions /= len(predict_list)
 submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('AggStacker.csv', index=False )<define_variables>
def run_model(input_shape=(28, 28, 1)) :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
return model
def run_model_compilation(model, optimizer='adam', loss='categorical_crossentropy'):
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
 def run_model_training(model, train, test, epochs, split):
history=model.fit(train, test, shuffle=True, epochs=epochs, validation_split=split)
 return history
sub_path = ".. /input/lgmodels/"
all_files = os.listdir(sub_path)
 all_files<feature_engineering>
cnn_model = run_model(( 28,28, 1))
run_model_compilation(cnn_model, 'adam', 'categorical_crossentropy')
 model_history = run_model_training(cnn_model, mnist_train_array, mnist_train_label_array, 100, 0.2 )
rank = np.tril(concat_sub.iloc[:,1:].corr().values,-1)
 m =(rank>0 ).sum()
m_gmean, s = 0, 0
for n in range(min(rank.shape[0],m)) :
mx = np.unravel_index(rank.argmin() , rank.shape)
 w =(m-n)/(m+n)
 print(w)
 m_gmean += w*(np.log(concat_sub.iloc[:,mx[0]+1])+np.log(concat_sub.iloc[:,mx[1]+1])) /2
s += w
rank[mx] = 1
m_gmean = np.exp(m_gmean/s )<save_to_csv>
final_predictions = cnn_model.predict(mnist_test_array)
 prediction_test_array = []
for i in final_predictions:
prediction_test_array.append(np.argmax(i))
<save_to_csv><EOS> 
submission = pd.DataFrame({
'ImageId': test_df_org.index+1,
'Label': prediction_test_array
})
 submission.to_csv('final_submission.csv', index=False )