// File: /workspace/github_cpp_makefile_star2_O2/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::InitWeightsAndBias()
void  ConstructLayer::InitWeightsAndBias()
{
    //std::cout<<"Layer id: "<<GetID()<<", W initialization."<<endl;
    // init weights and bias
    // no need to init active w&b, they will be filled in Batch starting phase
    // clear everything
    __weightMatrix.clear();
    __biasVector.clear();

    if(__trainingType == TrainingType::ResumeTraining)
    {
        LoadTrainedWeightsAndBias();
        return;
    }

    if(__p_data_interface == nullptr)
    {
        std::cout<<__func__<<" error: layer Need DataInterface already set."
            <<std::endl;
        exit(0);
    }
    //int batch_size = __p_data_interface->GetBatchSize();
    //int N = __p_data_interface->GetNumberOfBatches();

    // total_entries is used to randomly initialize weight matrix, 
    // using a Gaussian distribution, wieth mean=0, sigma = 1/sqrt(total_entries)
    //double total_entries = (double)batch_size * (double)N; // use this one, it is better (incorrect)
    //double total_entries = (double)batch_size;          // this one is not as good as the above one (incorrect)

    if(__type == LayerType::fullyConnected || __type == LayerType::output) // output layer is also a fully connected layer
    {
        int n_prev = 0;
        int n_curr = GetNumberOfNeurons();
        if(__prevLayer == nullptr) // input layer
        {
            std::cout<<"ERROR WARNING: layer"<<GetID()
                <<" has no prev layer, default 10 neruons for prev layer was used."
                <<std::endl;
            n_prev = 10;
        }
        else 
            n_prev = __prevLayer->GetNumberOfNeurons();

        // weight initialization
        Matrix w(n_curr, n_prev);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu actuation functions, use Xavier initialization
            // random with a normal distribution with sigma = 1/sqrt(number of fan-in neurons)
            w.RandomGaus(0., 1./sqrt((double)n_prev)); // (0, sqrt(n_neuron)) normal distribution

            //w.RandomGaus(0., 1./sqrt((double)total_entries)); // (0, sqrt(n_neuron)) normal distribution
        }
        else
        {
            // for ReLu actuation functions, use Kaiming He method: https://arxiv.org/pdf/1502.01852.pdf
            // step 1): initialize the matrix with a standard normal distribution
            w.RandomGaus(0., 1.);

            // step 2): then hadamard the weight matrix with a number sqrt(2/n)
            //          where n is the fan-in neurons (number of neurons in previous layer)
            w = w*sqrt(2./(double)n_prev);
        }
        __weightMatrix.push_back(w);

        // bias initialization
        Matrix b(n_curr, 1);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu neurons, use Xavier initialization, initialize it to 0
            b = b * 0.;
            //b.RandomGaus(0., 1.); // (0, 1) normal distribution
        }
        else if(__use_batch_normalization)
        {
            // if use batch normalization, b = 0
            // b is combined into \beta
            b = b*0.;
        }
        else
        {
            // for ReLu neurons, use Kaiming He method, initialize bias to 0
            b = b*0.;
        }
        __biasVector.push_back(b);
    }
    else if(__type == LayerType::cnn)
    {
        //auto norm_factor = __p_data_interface->GetBatchSize();
        for(size_t i=0;i<__n_kernels_cnn;i++)
        {
            Matrix w(__kernelDim);
            // weight matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w.RandomGaus(0., 1./sqrt((double)fan_in_neurons));
                //w.RandomGaus(0., 1./sqrt((double)total_entries));
            }
            else
            {
                // for ReLu neurons, use Kaiming He method
                w.RandomGaus(0., 1.);
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w = w * sqrt(2./ (double) fan_in_neurons);
            }
            __weightMatrix.push_back(w);

            Matrix b(1, 1);
            // bias matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                // Xavier also require b to be initialized with 0
                b = b * 0.;
                //b.RandomGaus(0., 1.); // (0, 1) normal distribution
            }
            else if(__use_batch_normalization)
            {
                // if use batch normalization, b = 0
                // b is combined into \beta
                b = b*0.;
            }
            else 
            {
                // for ReLu neurons, use Kaiming He method, set bias to 0.
                b = b * 0.;
            }
            __biasVector.push_back(b);
        }
    }
    else if(__type == LayerType::pooling)
    {
        for(size_t i=0;i<__n_kernels_cnn;i++) // pooling used cnn symbol
        {
            Matrix w(__kernelDim, 1); // pooling layer weight and bias matrix are not used, so set them to 1
            __weightMatrix.push_back(w);

            Matrix b(1, 1, 0);
            __biasVector.push_back(b);
        }
    }
    else {
        std::cout<<"Error: need layer type info before initing w&b."<<std::endl;
        exit(0);
    }
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %rdi, %r14
leaq 0x60(%rdi), %r15
movq %r15, %rdi
callq 0x11394
leaq 0x78(%r14), %rdi
movq %rdi, 0x30(%rsp)
callq 0x11394
cmpl $0x1, 0x40c(%r14)
jne 0x6a21
movq (%r14), %rax
movq %r14, %rdi
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
jmpq *0x128(%rax)
cmpq $0x0, 0x400(%r14)
je 0x6e4f
movl 0xc(%r14), %eax
cmpq $0x4, %rax
ja 0x6e80
leaq 0x1a678(%rip), %rcx     # 0x210bc
movslq (%rcx,%rax,4), %rax
addq %rcx, %rax
jmpq *%rax
movq (%r14), %rax
movq %r14, %rdi
callq *0x278(%rax)
movl %eax, %ebp
movq 0x3f0(%r14), %rdi
testq %rdi, %rdi
je 0x6c81
movq (%rdi), %rax
callq *0x278(%rax)
movl %eax, %r13d
jmp 0x6cc8
xorl %ebx, %ebx
leaq 0x38(%rsp), %r12
movq %rsp, %r13
leaq 0x18(%rsp), %rbp
cmpq 0x18(%r14), %rbx
jae 0x6e40
movq 0x20(%r14), %rsi
movq 0x28(%r14), %rdx
movq %r12, %rdi
callq 0x1776a
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x6afa
xorpd %xmm0, %xmm0
movq %r12, %rdi
movsd 0x1aa4b(%rip), %xmm1    # 0x21510
callq 0x187d8
movq 0x3f0(%r14), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
cvtsi2sd %eax, %xmm1
movsd 0x1aa32(%rip), %xmm0    # 0x21518
divsd %xmm1, %xmm0
ucomisd 0x1aa2e(%rip), %xmm0    # 0x21520
jb 0x6b3d
sqrtsd %xmm0, %xmm0
jmp 0x6b42
movq 0x3f0(%r14), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
cvtsi2sd %eax, %xmm0
ucomisd 0x1aa0a(%rip), %xmm0    # 0x21520
jb 0x6b1e
sqrtsd %xmm0, %xmm0
jmp 0x6b23
callq 0x43f8
movsd 0x1a9e5(%rip), %xmm1    # 0x21510
divsd %xmm0, %xmm1
xorpd %xmm0, %xmm0
movq %r12, %rdi
callq 0x187d8
jmp 0x6b60
callq 0x43f8
movq %r13, %rdi
movq %r12, %rsi
callq 0x182dc
movq %r12, %rdi
movq %r13, %rsi
callq 0x125b4
movq %r13, %rdi
callq 0x17a9a
movq %r15, %rdi
movq %r12, %rsi
callq 0x1139c
movq %r13, %rdi
pushq $0x1
popq %rdx
movq %rdx, %rsi
callq 0x176e6
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x6bb0
cmpb $0x1, 0x210(%r14)
jne 0x6bcc
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x182dc
movq %r13, %rdi
movq %rbp, %rsi
callq 0x125b4
jmp 0x6be6
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x182dc
movq %r13, %rdi
movq %rbp, %rsi
callq 0x125b4
jmp 0x6be6
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x182dc
movq %r13, %rdi
movq %rbp, %rsi
callq 0x125b4
movq %rbp, %rdi
callq 0x17a9a
movq 0x30(%rsp), %rdi
movq %r13, %rsi
callq 0x1139c
movq %r13, %rdi
callq 0x17a9a
movq %r12, %rdi
callq 0x17a9a
incq %rbx
jmp 0x6a8b
xorl %ebx, %ebx
leaq 0x38(%rsp), %r12
movq %rsp, %r13
pushq $0x1
popq %rbp
cmpq 0x18(%r14), %rbx
jae 0x6e40
movq 0x20(%r14), %rsi
movq 0x28(%r14), %rdx
movq %r12, %rdi
movsd 0x1a8d3(%rip), %xmm0    # 0x21510
callq 0x1787a
movq %r15, %rdi
movq %r12, %rsi
callq 0x1139c
xorpd %xmm0, %xmm0
movq %r13, %rdi
movq %rbp, %rsi
movq %rbp, %rdx
callq 0x177ee
movq 0x30(%rsp), %rdi
movq %r13, %rsi
callq 0x1139c
movq %r13, %rdi
callq 0x17a9a
movq %r12, %rdi
callq 0x17a9a
incq %rbx
jmp 0x6c20
movq 0x2c350(%rip), %rdi     # 0x32fd8
leaq 0x1ac83(%rip), %rsi     # 0x21912
callq 0x4240
movq %rax, %r12
movq (%r14), %rax
movq %r14, %rdi
callq *0x298(%rax)
movq %r12, %rdi
movl %eax, %esi
callq 0x43a0
leaq 0x1ac73(%rip), %rsi     # 0x21927
movq %rax, %rdi
callq 0x4240
movq %rax, %rdi
callq 0x4130
pushq $0xa
popq %r13
movslq %ebp, %r12
movslq %r13d, %rdx
leaq 0x38(%rsp), %rdi
movq %r12, %rsi
callq 0x176e6
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cvtsi2sd %r13d, %xmm0
cmpl $0x3, %eax
jne 0x6d2b
movsd %xmm0, 0x50(%rsp)
leaq 0x38(%rsp), %rdi
movsd 0x1a80c(%rip), %xmm1    # 0x21510
xorpd %xmm0, %xmm0
callq 0x187d8
movsd 0x1a803(%rip), %xmm0    # 0x21518
divsd 0x50(%rsp), %xmm0
ucomisd 0x1a7fd(%rip), %xmm0    # 0x21520
jb 0x6d5c
sqrtsd %xmm0, %xmm0
jmp 0x6d61
xorpd %xmm1, %xmm1
ucomisd %xmm1, %xmm0
jb 0x6d3b
sqrtsd %xmm0, %xmm0
jmp 0x6d40
callq 0x43f8
movsd 0x1a7c8(%rip), %xmm1    # 0x21510
divsd %xmm0, %xmm1
leaq 0x38(%rsp), %rdi
xorpd %xmm0, %xmm0
callq 0x187d8
jmp 0x6d83
callq 0x43f8
movq %rsp, %rdi
leaq 0x38(%rsp), %rsi
callq 0x182dc
leaq 0x38(%rsp), %rdi
movq %rsp, %rsi
callq 0x125b4
movq %rsp, %rdi
callq 0x17a9a
leaq 0x38(%rsp), %rsi
movq %r15, %rdi
callq 0x1139c
movq %rsp, %rdi
pushq $0x1
popq %rdx
movq %r12, %rsi
callq 0x176e6
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x6dd9
cmpb $0x1, 0x210(%r14)
jne 0x6df9
leaq 0x18(%rsp), %rdi
movq %rsp, %rsi
xorpd %xmm0, %xmm0
callq 0x182dc
movq %rsp, %rdi
leaq 0x18(%rsp), %rsi
callq 0x125b4
jmp 0x6e17
leaq 0x18(%rsp), %rdi
movq %rsp, %rsi
xorpd %xmm0, %xmm0
callq 0x182dc
movq %rsp, %rdi
leaq 0x18(%rsp), %rsi
callq 0x125b4
jmp 0x6e17
leaq 0x18(%rsp), %rdi
movq %rsp, %rsi
xorpd %xmm0, %xmm0
callq 0x182dc
movq %rsp, %rdi
leaq 0x18(%rsp), %rsi
callq 0x125b4
leaq 0x18(%rsp), %rdi
callq 0x17a9a
movq %rsp, %rsi
movq 0x30(%rsp), %rdi
callq 0x1139c
movq %rsp, %rdi
callq 0x17a9a
leaq 0x38(%rsp), %rdi
callq 0x17a9a
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movq 0x2c182(%rip), %rdi     # 0x32fd8
leaq 0x1aa74(%rip), %rsi     # 0x218d1
callq 0x4240
leaq 0x1aa7b(%rip), %rsi     # 0x218e4
movq %rax, %rdi
callq 0x4240
movq %rax, %rdi
callq 0x4130
xorl %edi, %edi
callq 0x4260
movq 0x2c151(%rip), %rdi     # 0x32fd8
leaq 0x1aad9(%rip), %rsi     # 0x21967
jmp 0x6e6c
jmp 0x6eb4
jmp 0x6eb4
jmp 0x6ecf
jmp 0x6ecf
jmp 0x6eb4
jmp 0x6ecf
jmp 0x6ecf
jmp 0x6edc
jmp 0x6edc
jmp 0x6ecf
jmp 0x6edc
jmp 0x6eb4
jmp 0x6ecf
jmp 0x6eb4
jmp 0x6ecf
jmp 0x6ecf
jmp 0x6ecf
jmp 0x6edc
movq %rax, %rbx
leaq 0x18(%rsp), %rdi
callq 0x17a9a
jmp 0x6ed2
jmp 0x6edc
jmp 0x6edc
jmp 0x6edc
jmp 0x6edc
jmp 0x6edc
jmp 0x6ecf
movq %rax, %rbx
movq %rsp, %rdi
callq 0x17a9a
jmp 0x6edf
movq %rax, %rbx
leaq 0x38(%rsp), %rdi
callq 0x17a9a
movq %rbx, %rdi
callq 0x43b0
nop