Repository: lhnguyen102/cuTAGI (language: python)

Issue:
CNN Unit test
Added CNN unit test for GPU only. 


Discussion / hints:

Fix patch:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c1de5139d..b082ad692 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -160,6 +160,7 @@ set(GPU_SOURCES
   src/gpu_debug_utils.cpp
   test/test_gpu.cu
   test/cnn/test_cnn_gpu.cu
+  test/cnn_batch_norm/test_cnn_batch_norm_gpu.cu
   test/test_classification.cu
 )
 
diff --git a/main.cu b/main.cu
index 8598dc15f..df6df0d4b 100644
--- a/main.cu
+++ b/main.cu
@@ -50,9 +50,10 @@ int main(int argc, char* argv[]) {
         bool compute_gpu_tests = is_cuda_available();
         int num_tests_passed_cpu =
             test_cpu(user_input_options, compute_gpu_tests);
-        if (compute_gpu_tests)
+        // If cuda Available and output reinicialization not aborted
+        if (compute_gpu_tests && num_tests_passed_cpu >= 0)
             test_gpu(user_input_options, num_tests_passed_cpu);
-        else {
+        else if (!compute_gpu_tests) {
             std::cout << std::endl;
             std::cout
                 << "Unable to perform test on GPU: CUDA device unavailable."


Test patch:
diff --git a/test/cnn_batch_norm/test_cnn_batch_norm_gpu.cu b/test/cnn_batch_norm/test_cnn_batch_norm_gpu.cu
new file mode 100644
index 000000000..c7858ae31
--- /dev/null
+++ b/test/cnn_batch_norm/test_cnn_batch_norm_gpu.cu
@@ -0,0 +1,256 @@
+///////////////////////////////////////////////////////////////////////////////
+// File:         test_cnn_batch_norm_gpu.cu
+// Description:  Script to test the CNN batch norm GPU implementation of cuTAGI
+// Authors:      Miquel Florensa, Luong-Ha Nguyen & James-A. Goulet
+// Created:      April 5, 2023
+// Updated:      April 6, 2023
+// Contact:      miquelflorensa11@gmail.com & luongha.nguyen@gmail.com &
+//               james.goulet@polymtl.ca
+// Copyright (c) 2023 Miquel Florensa, Luong-Ha Nguyen & James-A. Goulet.
+// Some rights reserved.
+///////////////////////////////////////////////////////////////////////////////
+
+#include "test_cnn_batch_norm_gpu.cuh"
+
+// Specify network properties
+const std::vector<int> LAYERS = {2, 2, 6, 4, 2, 6, 4, 1, 1};
+const std::vector<int> NODES = {784, 0, 0, 0, 0, 0, 0, 150, 11};
+const std::vector<int> KERNELS = {4, 1, 3, 5, 1, 3, 1, 1, 1};
+const std::vector<int> STRIDES = {1, 0, 2, 1, 0, 2, 0, 0, 0};
+const std::vector<int> WIDTHS = {28, 0, 0, 0, 0, 0, 0, 0, 0};
+const std::vector<int> HEIGHTS = {28, 0, 0, 0, 0, 0, 0, 0, 0};
+const std::vector<int> FILTERS = {1, 4, 4, 4, 8, 8, 8, 1, 1};
+const std::vector<int> PADS = {1, 0, 0, 0, 0, 0, 0, 0, 0};
+const std::vector<int> PAD_TYPES = {1, 0, 0, 0, 0, 0, 0, 0, 0};
+const std::vector<int> ACTIVATIONS = {0, 4, 0, 0, 4, 0, 0, 4, 12};
+const int BATCH_SIZE = 2;
+const int SIGMA_V = 1;
+const int NUM_CLASSES = 10;
+const std::vector<float> MU = {0.1309};
+const std::vector<float> SIGMA = {1.0};
+
+bool test_cnn_batch_norm_gpu(bool recompute_outputs, std::string date,
+                             std::string arch, std::string data) {
+    // Create TAGI network
+    Network net;
+
+    net.layers = LAYERS;
+    net.nodes = NODES;
+    net.kernels = KERNELS;
+    net.strides = STRIDES;
+    net.widths = WIDTHS;
+    net.heights = HEIGHTS;
+    net.filters = FILTERS;
+    net.pads = PADS;
+    net.pad_types = PAD_TYPES;
+    net.activations = ACTIVATIONS;
+    net.batch_size = BATCH_SIZE;
+    net.sigma_v = SIGMA_V;
+
+    std::string device = "cuda";
+    net.device = device;
+    auto hrs = class_to_obs(NUM_CLASSES);
+    net.nye = hrs.n_obs;
+
+    if (net.activations.back() == net.act_names.hr_softmax) {
+        net.is_idx_ud = true;
+        auto hrs = class_to_obs(NUM_CLASSES);
+        net.nye = hrs.n_obs;
+    }
+
+    TagiNetwork tagi_net(net);
+
+    // Put it in the main test file
+    SavePath path;
+    path.curr_path = get_current_dir();
+    std::string data_path = path.curr_path + "/test/data/" + data;
+    std::string data_dir = path.curr_path + "/test/" + arch + "/data/";
+    std::string init_param_path_w =
+        data_dir + date + "_" + data + "_init_param_weights_w.csv";
+    std::string init_param_path_w_sc =
+        data_dir + date + "_" + data + "_init_param_weights_w_sc.csv";
+    std::string init_param_path_b =
+        data_dir + date + "_" + data + "_init_param_bias_b.csv";
+    std::string init_param_path_b_sc =
+        data_dir + date + "_" + data + "_init_param_bias_b_sc.csv";
+    std::string opt_param_path_w =
+        data_dir + date + "_" + data + "_opt_param_weights_w.csv";
+    std::string opt_param_path_w_sc =
+        data_dir + date + "_" + data + "_opt_param_weights_w_sc.csv";
+    std::string opt_param_path_b =
+        data_dir + date + "_" + data + "_opt_param_bias_b.csv";
+    std::string opt_param_path_b_sc =
+        data_dir + date + "_" + data + "_opt_param_bias_b_sc.csv";
+    std::string forward_states_path = data_dir + date +
+                                      "_forward_hidden_states_" + arch + "_" +
+                                      data + ".csv";
+    std::string backward_states_path = data_dir + date +
+                                       "_backward_hidden_states_" + arch + "_" +
+                                       data + ".csv";
+
+    // Train data
+    ImageData imdb = image_dataloader(data, data_path, MU, SIGMA, NUM_CLASSES,
+                                      tagi_net.prop);
+
+    std::vector<std::vector<float> *> weights;
+    weights.push_back(&tagi_net.theta.mw);
+    weights.push_back(&tagi_net.theta.Sw);
+    std::vector<std::vector<float> *> weights_sc;
+    weights_sc.push_back(&tagi_net.theta.mw_sc);
+    weights_sc.push_back(&tagi_net.theta.Sw_sc);
+
+    std::vector<std::vector<float> *> bias;
+    bias.push_back(&tagi_net.theta.mb);
+    bias.push_back(&tagi_net.theta.Sb);
+    std::vector<std::vector<float> *> bias_sc;
+    bias_sc.push_back(&tagi_net.theta.mb_sc);
+    bias_sc.push_back(&tagi_net.theta.Sb_sc);
+
+    // If we want to test but no data is available, we throw an error
+    if (!recompute_outputs && !directory_exists(data_dir)) {
+        throw std::runtime_error(
+            "Tested data are not available. Please regenerate or provide the "
+            "tested data");
+    } else if (recompute_outputs) {
+        // Cheks if the data directory exists and if not, creates it
+        if (!create_directory_if_not_exists(data_dir)) {
+            std::cout << "Error: could not create data directory" << std::endl;
+            return false;
+        }
+
+        write_vector_to_csv(init_param_path_w, "mw,Sw", weights);
+        write_vector_to_csv(init_param_path_w_sc, "mw_sc,Sw_sc", weights_sc);
+        write_vector_to_csv(init_param_path_b, "mb,Sb", bias);
+        write_vector_to_csv(init_param_path_b_sc, "mb_sc,Sb_sc", bias_sc);
+    }
+
+    // Read the initial parameters (see test_utils.cpp for more details)
+    read_vector_from_csv(init_param_path_w, weights);
+    read_vector_from_csv(init_param_path_w_sc, weights_sc);
+    read_vector_from_csv(init_param_path_b, bias);
+    read_vector_from_csv(init_param_path_b_sc, bias_sc);
+
+    tagi_net.theta_gpu.copy_host_to_device();
+
+    // Classify
+    train_classification(tagi_net, imdb, NUM_CLASSES);
+
+    tagi_net.theta_gpu.copy_device_to_host();
+    tagi_net.d_state_gpu.copy_device_to_host();
+
+    std::vector<std::vector<float> *> forward_states;
+    forward_states.push_back(&tagi_net.state.mz);
+    forward_states.push_back(&tagi_net.state.Sz);
+    forward_states.push_back(&tagi_net.state.ma);
+    forward_states.push_back(&tagi_net.state.Sa);
+    forward_states.push_back(&tagi_net.state.J);
+
+    std::vector<std::vector<float>> backward_states;
+    std::string backward_states_header = "";
+
+    for (int i = 0; i < net.layers.size() - 2; i++) {
+        backward_states_header +=
+            "mean_" + std::to_string(i) + ",sigma_" + std::to_string(i) + ",";
+        backward_states.push_back(
+            std::get<0>(tagi_net.get_inovation_mean_var(i)));
+        backward_states.push_back(
+            std::get<1>(tagi_net.get_inovation_mean_var(i)));
+    }
+
+    std::vector<std::vector<float> *> backward_states_ptr;
+    for (int i = 0; i < backward_states.size(); i++)
+        backward_states_ptr.push_back(&backward_states[i]);
+
+    if (recompute_outputs) {
+        // RESET OUPUTS
+
+        // Write the parameters and hidden states
+        write_vector_to_csv(opt_param_path_w, "mw,Sw", weights);
+        write_vector_to_csv(opt_param_path_w_sc, "mw_sc,Sw_sc", weights_sc);
+        write_vector_to_csv(opt_param_path_b, "mb,Sb", bias);
+        write_vector_to_csv(opt_param_path_b_sc, "mb_sc,Sb_sc", bias_sc);
+
+        // Write the forward hidden states
+        write_vector_to_csv(forward_states_path, "mz,Sz,ma,Sa,J",
+                            forward_states);
+
+        // Write the backward hidden states
+        write_vector_to_csv(backward_states_path, backward_states_header,
+                            backward_states_ptr);
+
+    } else {
+        // PERFORM TESTS
+
+        // Read the saved reference parameters
+        std::vector<std::vector<float> *> ref_weights;
+        std::vector<std::vector<float> *> ref_weights_sc;
+        std::vector<std::vector<float> *> ref_bias;
+        std::vector<std::vector<float> *> ref_bias_sc;
+
+        for (int i = 0; i < 2; i++) {
+            ref_weights.push_back(new std::vector<float>());
+            ref_weights_sc.push_back(new std::vector<float>());
+            ref_bias.push_back(new std::vector<float>());
+            ref_bias_sc.push_back(new std::vector<float>());
+        }
+
+        read_vector_from_csv(opt_param_path_w, ref_weights);
+        read_vector_from_csv(opt_param_path_w_sc, ref_weights_sc);
+        read_vector_from_csv(opt_param_path_b, ref_bias);
+        read_vector_from_csv(opt_param_path_b_sc, ref_bias_sc);
+
+        tagi_net.theta_gpu.copy_host_to_device();
+
+        // Compare optimal values with the ones we got
+        if (!compare_vectors(ref_weights, weights, data,
+                             "cnn batch norm. weights") ||
+            !compare_vectors(ref_weights_sc, weights_sc, data,
+                             "cnn batch norm. weights for residual network") ||
+            !compare_vectors(ref_bias, bias, data, "cnn batch norm. bias") ||
+            !compare_vectors(ref_bias_sc, bias_sc, data,
+                             "cnn batch norm. bias for residual network")) {
+            std::cout << "\033[1;31mTest for CNN BATCH NORMALIZATION PARAMS "
+                         "has FAILED in " +
+                             data + " data\033[0m\n"
+                      << std::endl;
+            return false;
+        }
+
+        // Read the saved forward hidden states reference
+        std::vector<std::vector<float> *> ref_forward_states;
+        for (int i = 0; i < 5; i++)
+            ref_forward_states.push_back(new std::vector<float>());
+
+        read_vector_from_csv(forward_states_path, ref_forward_states);
+
+        // Compare the saved forward hidden states with the ones we got
+        if (!compare_vectors(ref_forward_states, forward_states, data,
+                             "cnn batch norm. forward hidden states")) {
+            std::cout << "\033[1;31mTest for CNN BATCH NORMALIZATION FORWARD "
+                         "HIDDEN STATES has "
+                         "FAILED in " +
+                             data + " data\033[0m\n"
+                      << std::endl;
+            return false;
+        }
+
+        // Read the saved backward hidden states reference
+        std::vector<std::vector<float> *> ref_backward_states;
+        for (int i = 0; i < 2 * (net.layers.size() - 2); i++)
+            ref_backward_states.push_back(new std::vector<float>());
+
+        read_vector_from_csv(backward_states_path, ref_backward_states);
+
+        // Compare the saved backward hidden states with the ones we got
+        if (!compare_vectors(ref_backward_states, backward_states_ptr, data,
+                             "cnn batch norm. backward hidden states")) {
+            std::cout << "\033[1;31mTest for CNN BATCH NORMALIZATION BACKWARD "
+                         "HIDDEN STATES has "
+                         "FAILED in " +
+                             data + " data\033[0m\n"
+                      << std::endl;
+            return false;
+        }
+    }
+    return true;
+}
\ No newline at end of file
diff --git a/test/cnn_batch_norm/test_cnn_batch_norm_gpu.cuh b/test/cnn_batch_norm/test_cnn_batch_norm_gpu.cuh
new file mode 100644
index 000000000..c30d829ac
--- /dev/null
+++ b/test/cnn_batch_norm/test_cnn_batch_norm_gpu.cuh
@@ -0,0 +1,54 @@
+///////////////////////////////////////////////////////////////////////////////
+// File:         test_cnn_batch_norm_gpu.cuh
+// Description:  Header file for cnn batch norm test
+// Authors:      Miquel Florensa, Luong-Ha Nguyen & James-A. Goulet
+// Created:      April 5, 2023
+// Updated:      April 5, 2023
+// Contact:      miquelflorensa11@gmail.com, luongha.nguyen@gmail.com &
+//               james.goulet@polymtl.ca
+// Copyright (c) 2023 Miquel Florensa, Luong-Ha Nguyen & James-A. Goulet.
+// Some rights reserved.
+///////////////////////////////////////////////////////////////////////////////
+
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+#include <cstdlib>
+#include <ctime>
+#include <iomanip>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+#include "../../include/common.h"
+#include "../../include/cost.h"
+#include "../../include/dataloader.h"
+#include "../../include/derivative_calcul.cuh"
+#include "../../include/feed_forward.cuh"
+#include "../../include/global_param_update.cuh"
+#include "../../include/indices.h"
+#include "../../include/net_init.h"
+#include "../../include/net_prop.h"
+#include "../../include/param_feed_backward.cuh"
+#include "../../include/state_feed_backward.cuh"
+#include "../../include/struct_var.h"
+#include "../../include/tagi_network.cuh"
+#include "../../include/task.cuh"
+#include "../../include/utils.h"
+#include "../test_classification.cuh"
+#include "../test_dataloader.h"
+#include "../test_utils.h"
+
+/**
+ * @brief Test the CNN batch normalization network
+ *
+ * @param recompute_outputs indicates if the outputs should be recomputed
+ * @param date date of the test
+ * @param arch architecture of the network
+ * @param data dataset used
+ *
+ * @return true if the test was successful
+ */
+bool test_cnn_batch_norm_gpu(bool recompute_outputs, std::string date,
+                             std::string arch, std::string data);
\ No newline at end of file
diff --git a/test/data/last_dates.csv b/test/data/last_dates.csv
index b4952bab9..aa071ce03 100644
--- a/test/data/last_dates.csv
+++ b/test/data/last_dates.csv
@@ -1,2 +1,2 @@
 fnn,fnn_heteros,fnn_full_cov,fnn_derivatives,lstm,act_func,cnn,cnn_batch_norm,autoencoder
-2023_04_04,2023_04_04,2023_04_04,2023_04_04,2023_04_04,2023_04_04,2023_04_04,2023_02_21,2023_03_28
+2023_04_06,2023_04_06,2023_04_06,2023_04_06,2023_04_06,2023_04_06,2023_04_06,2023_04_06,2023_03_28
diff --git a/test/test_cpu.cpp b/test/test_cpu.cpp
index 746e72dda..9fa3e1580 100644
--- a/test/test_cpu.cpp
+++ b/test/test_cpu.cpp
@@ -403,8 +403,11 @@ int test_cpu(std::vector<std::string>& user_input_options,
                 write_dates(test_dates, test_num, date);
                 test_dates[test_num] = date;
             }
+            return 0;
+        } else {
+            // Reinizialization Aborted
+            return -2;
         }
-        return 0;
     }
 
     return -1;
diff --git a/test/test_gpu.cu b/test/test_gpu.cu
index ff36716e1..c1ba12692 100644
--- a/test/test_gpu.cu
+++ b/test/test_gpu.cu
@@ -3,7 +3,7 @@
 // Description:  Main script to test the GPU implementation of cuTAGI
 // Authors:      Florensa, Miquel, Luong-Ha Nguyen & James-A. Goulet
 // Created:      February 20, 2023
-// Updated:      April 4, 2023
+// Updated:      April 5, 2023
 // Contact:      miquelflorensa11@gmail.com, luongha.nguyen@gmail.com &
 //               james.goulet@polymtl.ca
 // Copyright (c) 2023 Miquel Florensa, Luong-Ha Nguyen & James-A. Goulet.
@@ -85,6 +85,20 @@ int test_gpu(std::vector<std::string>& user_input_options,
             if (test_result) num_test_passed++;
         }
 
+        // Perform test on GPU with batch normalization for classification task
+        if (test_architecture == "all" ||
+            test_architecture == "cnn_batch_norm") {
+            test_num = 7;  // CNN batch norm.
+
+            bool test_result = test_cnn_batch_norm_gpu(
+                false, test_dates[test_num], "cnn_batch_norm", "mnist");
+
+            print_test_results(single_test, test_result, NUM_TESTS, test_num,
+                               "CNN batch normalization");
+
+            if (test_result) num_test_passed++;
+        }
+
         // Number of tests passed
         if (test_architecture == "all") {
             std::cout << std::endl;
@@ -102,8 +116,8 @@ int test_gpu(std::vector<std::string>& user_input_options,
     ///////////////////////////////
 
     if (reinizialize_test_outputs.size() > 0) {
-        if (is_cuda_available() && (reinizialize_test_outputs == "all" ||
-                                    reinizialize_test_outputs == "cnn")) {
+        if (reinizialize_test_outputs == "all" ||
+            reinizialize_test_outputs == "cnn") {
             // Reinizialize test outputs for classification task
             std::cout << "Reinizializing CNN test outputs" << std::endl;
 
@@ -115,6 +129,21 @@ int test_gpu(std::vector<std::string>& user_input_options,
             write_dates(test_dates, test_num, date);
             test_dates[test_num] = date;
         }
+
+        if (reinizialize_test_outputs == "all" ||
+            reinizialize_test_outputs == "cnn_batch_norm") {
+            // Reinizialize test outputs for classification task
+            std::cout << "Reinizializing CNN batch norm. test outputs"
+                      << std::endl;
+
+            test_cnn_batch_norm_gpu(true, date, "cnn_batch_norm", "mnist");
+
+            test_num = 7;  // CNN
+
+            // Update de last date of the test
+            write_dates(test_dates, test_num, date);
+            test_dates[test_num] = date;
+        }
         return 0;
     }
 
diff --git a/test/test_gpu.cuh b/test/test_gpu.cuh
index 4f792c814..a9e9d1f54 100644
--- a/test/test_gpu.cuh
+++ b/test/test_gpu.cuh
@@ -20,6 +20,7 @@
 #include <string>
 
 #include "cnn/test_cnn_gpu.cuh"
+#include "cnn_batch_norm/test_cnn_batch_norm_gpu.cuh"
 #include "test_cpu.h"
 
 /**