// File: /workspace/github_cpp_makefile_star2_O1/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::InitWeightsAndBias()
void  ConstructLayer::InitWeightsAndBias()
{
    //std::cout<<"Layer id: "<<GetID()<<", W initialization."<<endl;
    // init weights and bias
    // no need to init active w&b, they will be filled in Batch starting phase
    // clear everything
    __weightMatrix.clear();
    __biasVector.clear();

    if(__trainingType == TrainingType::ResumeTraining)
    {
        LoadTrainedWeightsAndBias();
        return;
    }

    if(__p_data_interface == nullptr)
    {
        std::cout<<__func__<<" error: layer Need DataInterface already set."
            <<std::endl;
        exit(0);
    }
    //int batch_size = __p_data_interface->GetBatchSize();
    //int N = __p_data_interface->GetNumberOfBatches();

    // total_entries is used to randomly initialize weight matrix, 
    // using a Gaussian distribution, wieth mean=0, sigma = 1/sqrt(total_entries)
    //double total_entries = (double)batch_size * (double)N; // use this one, it is better (incorrect)
    //double total_entries = (double)batch_size;          // this one is not as good as the above one (incorrect)

    if(__type == LayerType::fullyConnected || __type == LayerType::output) // output layer is also a fully connected layer
    {
        int n_prev = 0;
        int n_curr = GetNumberOfNeurons();
        if(__prevLayer == nullptr) // input layer
        {
            std::cout<<"ERROR WARNING: layer"<<GetID()
                <<" has no prev layer, default 10 neruons for prev layer was used."
                <<std::endl;
            n_prev = 10;
        }
        else 
            n_prev = __prevLayer->GetNumberOfNeurons();

        // weight initialization
        Matrix w(n_curr, n_prev);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu actuation functions, use Xavier initialization
            // random with a normal distribution with sigma = 1/sqrt(number of fan-in neurons)
            w.RandomGaus(0., 1./sqrt((double)n_prev)); // (0, sqrt(n_neuron)) normal distribution

            //w.RandomGaus(0., 1./sqrt((double)total_entries)); // (0, sqrt(n_neuron)) normal distribution
        }
        else
        {
            // for ReLu actuation functions, use Kaiming He method: https://arxiv.org/pdf/1502.01852.pdf
            // step 1): initialize the matrix with a standard normal distribution
            w.RandomGaus(0., 1.);

            // step 2): then hadamard the weight matrix with a number sqrt(2/n)
            //          where n is the fan-in neurons (number of neurons in previous layer)
            w = w*sqrt(2./(double)n_prev);
        }
        __weightMatrix.push_back(w);

        // bias initialization
        Matrix b(n_curr, 1);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu neurons, use Xavier initialization, initialize it to 0
            b = b * 0.;
            //b.RandomGaus(0., 1.); // (0, 1) normal distribution
        }
        else if(__use_batch_normalization)
        {
            // if use batch normalization, b = 0
            // b is combined into \beta
            b = b*0.;
        }
        else
        {
            // for ReLu neurons, use Kaiming He method, initialize bias to 0
            b = b*0.;
        }
        __biasVector.push_back(b);
    }
    else if(__type == LayerType::cnn)
    {
        //auto norm_factor = __p_data_interface->GetBatchSize();
        for(size_t i=0;i<__n_kernels_cnn;i++)
        {
            Matrix w(__kernelDim);
            // weight matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w.RandomGaus(0., 1./sqrt((double)fan_in_neurons));
                //w.RandomGaus(0., 1./sqrt((double)total_entries));
            }
            else
            {
                // for ReLu neurons, use Kaiming He method
                w.RandomGaus(0., 1.);
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w = w * sqrt(2./ (double) fan_in_neurons);
            }
            __weightMatrix.push_back(w);

            Matrix b(1, 1);
            // bias matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                // Xavier also require b to be initialized with 0
                b = b * 0.;
                //b.RandomGaus(0., 1.); // (0, 1) normal distribution
            }
            else if(__use_batch_normalization)
            {
                // if use batch normalization, b = 0
                // b is combined into \beta
                b = b*0.;
            }
            else 
            {
                // for ReLu neurons, use Kaiming He method, set bias to 0.
                b = b * 0.;
            }
            __biasVector.push_back(b);
        }
    }
    else if(__type == LayerType::pooling)
    {
        for(size_t i=0;i<__n_kernels_cnn;i++) // pooling used cnn symbol
        {
            Matrix w(__kernelDim, 1); // pooling layer weight and bias matrix are not used, so set them to 1
            __weightMatrix.push_back(w);

            Matrix b(1, 1, 0);
            __biasVector.push_back(b);
        }
    }
    else {
        std::cout<<"Error: need layer type info before initing w&b."<<std::endl;
        exit(0);
    }
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %rdi, %r14
movq 0x60(%rdi), %r15
movq 0x68(%rdi), %r12
cmpq %r15, %r12
je 0x6eee
movq %r15, %rbx
movq %rbx, %rdi
callq 0x1acb6
addq $0x18, %rbx
cmpq %r12, %rbx
jne 0x6ed9
movq %r15, 0x68(%r14)
movq 0x78(%r14), %r15
movq 0x80(%r14), %r12
cmpq %r15, %r12
je 0x6f19
movq %r15, %rbx
movq %rbx, %rdi
callq 0x1acb6
addq $0x18, %rbx
cmpq %r12, %rbx
jne 0x6f01
movq %r15, 0x80(%r14)
cmpl $0x1, 0x40c(%r14)
jne 0x6f3d
movq (%r14), %rax
movq %r14, %rdi
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
jmpq *0x128(%rax)
cmpq $0x0, 0x400(%r14)
je 0x73e5
movl 0xc(%r14), %eax
cmpq $0x4, %rax
ja 0x7416
leaq 0x60(%r14), %r15
leaq 0x78(%r14), %rcx
movq %rcx, 0x38(%rsp)
leaq 0x201bf(%rip), %rcx     # 0x2712c
movslq (%rcx,%rax,4), %rax
addq %rcx, %rax
jmpq *%rax
movq (%r14), %rax
movq %r14, %rdi
callq *0x278(%rax)
movl %eax, %ebp
movq 0x3f0(%r14), %rdi
testq %rdi, %rdi
je 0x71ce
movq (%rdi), %rax
callq *0x278(%rax)
movl %eax, %r13d
jmp 0x7244
cmpq $0x0, 0x18(%r14)
je 0x73d6
xorl %ebx, %ebx
leaq 0x40(%rsp), %r12
leaq 0x8(%rsp), %r13
leaq 0x20(%rsp), %rbp
movq 0x20(%r14), %rsi
movq 0x28(%r14), %rdx
movq %r12, %rdi
callq 0x1a8c0
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x7029
xorpd %xmm0, %xmm0
movq %r12, %rdi
movsd 0x2058f(%rip), %xmm1    # 0x27580
callq 0x1bee4
movq 0x3f0(%r14), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
xorps %xmm1, %xmm1
cvtsi2sd %eax, %xmm1
movsd 0x20573(%rip), %xmm0    # 0x27588
divsd %xmm1, %xmm0
ucomisd 0x2056f(%rip), %xmm0    # 0x27590
jb 0x706f
sqrtsd %xmm0, %xmm0
jmp 0x7074
movq 0x3f0(%r14), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
xorps %xmm0, %xmm0
cvtsi2sd %eax, %xmm0
ucomisd 0x20548(%rip), %xmm0    # 0x27590
jb 0x7050
sqrtsd %xmm0, %xmm0
jmp 0x7055
callq 0x4438
movsd 0x20523(%rip), %xmm1    # 0x27580
divsd %xmm0, %xmm1
xorpd %xmm0, %xmm0
movq %r12, %rdi
callq 0x1bee4
jmp 0x7092
callq 0x4438
movq %r13, %rdi
movq %r12, %rsi
callq 0x1b7d0
movq %r12, %rdi
movq %r13, %rsi
callq 0x14f80
movq %r13, %rdi
callq 0x1acb6
movq %r15, %rdi
movq %r12, %rsi
callq 0x13a14
movl $0x1, %esi
movl $0x1, %edx
movq %r13, %rdi
callq 0x1a82e
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x70e6
cmpb $0x1, 0x210(%r14)
jne 0x7102
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x1b7d0
movq %r13, %rdi
movq %rbp, %rsi
callq 0x14f80
jmp 0x711c
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x1b7d0
movq %r13, %rdi
movq %rbp, %rsi
callq 0x14f80
jmp 0x711c
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x1b7d0
movq %r13, %rdi
movq %rbp, %rsi
callq 0x14f80
movq %rbp, %rdi
callq 0x1acb6
movq 0x38(%rsp), %rdi
movq %r13, %rsi
callq 0x13a14
movq %r13, %rdi
callq 0x1acb6
movq %r12, %rdi
callq 0x1acb6
incq %rbx
cmpq 0x18(%r14), %rbx
jb 0x6fc1
jmp 0x73d6
cmpq $0x0, 0x18(%r14)
je 0x73d6
xorl %ebx, %ebx
leaq 0x40(%rsp), %r12
leaq 0x8(%rsp), %r13
movq 0x20(%r14), %rsi
movq 0x28(%r14), %rdx
movq %r12, %rdi
movsd 0x20403(%rip), %xmm0    # 0x27580
callq 0x1a9ec
movq %r15, %rdi
movq %r12, %rsi
callq 0x13a14
movl $0x1, %esi
movl $0x1, %edx
xorpd %xmm0, %xmm0
movq %r13, %rdi
callq 0x1a952
movq 0x38(%rsp), %rdi
movq %r13, %rsi
callq 0x13a14
movq %r13, %rdi
callq 0x1acb6
movq %r12, %rdi
callq 0x1acb6
incq %rbx
cmpq 0x18(%r14), %rbx
jb 0x716a
jmp 0x73d6
movq 0x2fe03(%rip), %r12     # 0x36fd8
leaq 0x207e5(%rip), %rsi     # 0x279c1
movl $0x14, %edx
movq %r12, %rdi
callq 0x4280
movq (%r14), %rax
movq %r14, %rdi
callq *0x298(%rax)
movq %r12, %rdi
movl %eax, %esi
callq 0x43c0
movq %rax, %r12
leaq 0x207cd(%rip), %rsi     # 0x279d6
movl $0x3f, %edx
movq %rax, %rdi
callq 0x4280
movq (%r12), %rax
movq -0x18(%rax), %rdi
addq %r12, %rdi
movl $0xa, %r13d
movl $0xa, %esi
callq 0x4250
movsbl %al, %esi
movq %r12, %rdi
callq 0x4030
movq %rax, %rdi
callq 0x4190
movslq %ebp, %r12
movslq %r13d, %rdx
leaq 0x40(%rsp), %rdi
movq %r12, %rsi
callq 0x1a82e
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x72a2
leaq 0x40(%rsp), %rdi
movsd 0x2030b(%rip), %xmm1    # 0x27580
xorpd %xmm0, %xmm0
callq 0x1bee4
xorps %xmm1, %xmm1
cvtsi2sd %r13d, %xmm1
movsd 0x202fa(%rip), %xmm0    # 0x27588
divsd %xmm1, %xmm0
ucomisd 0x202f6(%rip), %xmm0    # 0x27590
jb 0x72d8
sqrtsd %xmm0, %xmm0
jmp 0x72dd
cvtsi2sd %r13d, %xmm0
xorpd %xmm1, %xmm1
ucomisd %xmm1, %xmm0
jb 0x72b7
sqrtsd %xmm0, %xmm0
jmp 0x72bc
callq 0x4438
movsd 0x202bc(%rip), %xmm1    # 0x27580
divsd %xmm0, %xmm1
leaq 0x40(%rsp), %rdi
xorpd %xmm0, %xmm0
callq 0x1bee4
jmp 0x7305
callq 0x4438
leaq 0x8(%rsp), %rdi
leaq 0x40(%rsp), %rsi
callq 0x1b7d0
leaq 0x40(%rsp), %rdi
leaq 0x8(%rsp), %rsi
callq 0x14f80
leaq 0x8(%rsp), %rdi
callq 0x1acb6
leaq 0x40(%rsp), %rsi
movq %r15, %rdi
callq 0x13a14
leaq 0x8(%rsp), %rdi
movl $0x1, %edx
movq %r12, %rsi
callq 0x1a82e
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x7363
cmpb $0x1, 0x210(%r14)
jne 0x7387
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
xorpd %xmm0, %xmm0
callq 0x1b7d0
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x14f80
jmp 0x73a9
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
xorpd %xmm0, %xmm0
callq 0x1b7d0
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x14f80
jmp 0x73a9
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
xorpd %xmm0, %xmm0
callq 0x1b7d0
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x14f80
leaq 0x20(%rsp), %rdi
callq 0x1acb6
leaq 0x8(%rsp), %rsi
movq 0x38(%rsp), %rdi
callq 0x13a14
leaq 0x8(%rsp), %rdi
callq 0x1acb6
leaq 0x40(%rsp), %rdi
callq 0x1acb6
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movq 0x2fbec(%rip), %rdi     # 0x36fd8
leaq 0x2058d(%rip), %rsi     # 0x27980
callq 0x4260
leaq 0x20594(%rip), %rsi     # 0x27993
movq %rax, %rdi
callq 0x4260
movq %rax, %rdi
callq 0x4100
xorl %edi, %edi
callq 0x4290
movq 0x2fbbb(%rip), %rdi     # 0x36fd8
leaq 0x205f2(%rip), %rsi     # 0x27a16
jmp 0x7402
jmp 0x744c
jmp 0x744c
jmp 0x7465
jmp 0x7465
jmp 0x744c
jmp 0x7465
jmp 0x7465
jmp 0x7474
jmp 0x7474
jmp 0x7465
jmp 0x744c
jmp 0x7465
jmp 0x744c
jmp 0x7465
jmp 0x7474
jmp 0x7474
jmp 0x7465
jmp 0x7465
jmp 0x7474
movq %rax, %rbx
leaq 0x20(%rsp), %rdi
callq 0x1acb6
jmp 0x7468
jmp 0x7474
jmp 0x7474
jmp 0x7474
jmp 0x7474
jmp 0x7465
movq %rax, %rbx
leaq 0x8(%rsp), %rdi
callq 0x1acb6
jmp 0x7477
movq %rax, %rbx
leaq 0x40(%rsp), %rdi
callq 0x1acb6
movq %rbx, %rdi
callq 0x43e0
nop