// File: /workspace/github_cpp_makefile_star2_O3/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::InitWeightsAndBias()
void  ConstructLayer::InitWeightsAndBias()
{
    //std::cout<<"Layer id: "<<GetID()<<", W initialization."<<endl;
    // init weights and bias
    // no need to init active w&b, they will be filled in Batch starting phase
    // clear everything
    __weightMatrix.clear();
    __biasVector.clear();

    if(__trainingType == TrainingType::ResumeTraining)
    {
        LoadTrainedWeightsAndBias();
        return;
    }

    if(__p_data_interface == nullptr)
    {
        std::cout<<__func__<<" error: layer Need DataInterface already set."
            <<std::endl;
        exit(0);
    }
    //int batch_size = __p_data_interface->GetBatchSize();
    //int N = __p_data_interface->GetNumberOfBatches();

    // total_entries is used to randomly initialize weight matrix, 
    // using a Gaussian distribution, wieth mean=0, sigma = 1/sqrt(total_entries)
    //double total_entries = (double)batch_size * (double)N; // use this one, it is better (incorrect)
    //double total_entries = (double)batch_size;          // this one is not as good as the above one (incorrect)

    if(__type == LayerType::fullyConnected || __type == LayerType::output) // output layer is also a fully connected layer
    {
        int n_prev = 0;
        int n_curr = GetNumberOfNeurons();
        if(__prevLayer == nullptr) // input layer
        {
            std::cout<<"ERROR WARNING: layer"<<GetID()
                <<" has no prev layer, default 10 neruons for prev layer was used."
                <<std::endl;
            n_prev = 10;
        }
        else 
            n_prev = __prevLayer->GetNumberOfNeurons();

        // weight initialization
        Matrix w(n_curr, n_prev);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu actuation functions, use Xavier initialization
            // random with a normal distribution with sigma = 1/sqrt(number of fan-in neurons)
            w.RandomGaus(0., 1./sqrt((double)n_prev)); // (0, sqrt(n_neuron)) normal distribution

            //w.RandomGaus(0., 1./sqrt((double)total_entries)); // (0, sqrt(n_neuron)) normal distribution
        }
        else
        {
            // for ReLu actuation functions, use Kaiming He method: https://arxiv.org/pdf/1502.01852.pdf
            // step 1): initialize the matrix with a standard normal distribution
            w.RandomGaus(0., 1.);

            // step 2): then hadamard the weight matrix with a number sqrt(2/n)
            //          where n is the fan-in neurons (number of neurons in previous layer)
            w = w*sqrt(2./(double)n_prev);
        }
        __weightMatrix.push_back(w);

        // bias initialization
        Matrix b(n_curr, 1);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu neurons, use Xavier initialization, initialize it to 0
            b = b * 0.;
            //b.RandomGaus(0., 1.); // (0, 1) normal distribution
        }
        else if(__use_batch_normalization)
        {
            // if use batch normalization, b = 0
            // b is combined into \beta
            b = b*0.;
        }
        else
        {
            // for ReLu neurons, use Kaiming He method, initialize bias to 0
            b = b*0.;
        }
        __biasVector.push_back(b);
    }
    else if(__type == LayerType::cnn)
    {
        //auto norm_factor = __p_data_interface->GetBatchSize();
        for(size_t i=0;i<__n_kernels_cnn;i++)
        {
            Matrix w(__kernelDim);
            // weight matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w.RandomGaus(0., 1./sqrt((double)fan_in_neurons));
                //w.RandomGaus(0., 1./sqrt((double)total_entries));
            }
            else
            {
                // for ReLu neurons, use Kaiming He method
                w.RandomGaus(0., 1.);
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w = w * sqrt(2./ (double) fan_in_neurons);
            }
            __weightMatrix.push_back(w);

            Matrix b(1, 1);
            // bias matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                // Xavier also require b to be initialized with 0
                b = b * 0.;
                //b.RandomGaus(0., 1.); // (0, 1) normal distribution
            }
            else if(__use_batch_normalization)
            {
                // if use batch normalization, b = 0
                // b is combined into \beta
                b = b*0.;
            }
            else 
            {
                // for ReLu neurons, use Kaiming He method, set bias to 0.
                b = b * 0.;
            }
            __biasVector.push_back(b);
        }
    }
    else if(__type == LayerType::pooling)
    {
        for(size_t i=0;i<__n_kernels_cnn;i++) // pooling used cnn symbol
        {
            Matrix w(__kernelDim, 1); // pooling layer weight and bias matrix are not used, so set them to 1
            __weightMatrix.push_back(w);

            Matrix b(1, 1, 0);
            __biasVector.push_back(b);
        }
    }
    else {
        std::cout<<"Error: need layer type info before initing w&b."<<std::endl;
        exit(0);
    }
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %rdi, %r14
movq 0x60(%rdi), %r15
movq 0x68(%rdi), %r12
cmpq %r15, %r12
je 0x6e02
movq %r15, %rbx
movq %rbx, %rdi
callq 0x1abf2
addq $0x18, %rbx
cmpq %r12, %rbx
jne 0x6ded
movq %r15, 0x68(%r14)
movq 0x78(%r14), %r15
movq 0x80(%r14), %r12
cmpq %r15, %r12
je 0x6e2d
movq %r15, %rbx
movq %rbx, %rdi
callq 0x1abf2
addq $0x18, %rbx
cmpq %r12, %rbx
jne 0x6e15
movq %r15, 0x80(%r14)
cmpl $0x1, 0x40c(%r14)
jne 0x6e51
movq (%r14), %rax
movq %r14, %rdi
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
jmpq *0x128(%rax)
cmpq $0x0, 0x400(%r14)
je 0x72f9
movl 0xc(%r14), %eax
cmpq $0x4, %rax
ja 0x732a
leaq 0x60(%r14), %r15
leaq 0x78(%r14), %rcx
movq %rcx, 0x38(%rsp)
leaq 0x202ab(%rip), %rcx     # 0x2712c
movslq (%rcx,%rax,4), %rax
addq %rcx, %rax
jmpq *%rax
movq (%r14), %rax
movq %r14, %rdi
callq *0x278(%rax)
movl %eax, %ebp
movq 0x3f0(%r14), %rdi
testq %rdi, %rdi
je 0x70e2
movq (%rdi), %rax
callq *0x278(%rax)
movl %eax, %r13d
jmp 0x7158
cmpq $0x0, 0x18(%r14)
je 0x72ea
xorl %ebx, %ebx
leaq 0x40(%rsp), %r12
leaq 0x8(%rsp), %r13
leaq 0x20(%rsp), %rbp
movq 0x20(%r14), %rsi
movq 0x28(%r14), %rdx
movq %r12, %rdi
callq 0x1a800
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x6f3d
xorpd %xmm0, %xmm0
movq %r12, %rdi
movsd 0x2067b(%rip), %xmm1    # 0x27580
callq 0x1be10
movq 0x3f0(%r14), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
xorps %xmm1, %xmm1
cvtsi2sd %eax, %xmm1
movsd 0x2065f(%rip), %xmm0    # 0x27588
divsd %xmm1, %xmm0
ucomisd 0x2065b(%rip), %xmm0    # 0x27590
jb 0x6f83
sqrtsd %xmm0, %xmm0
jmp 0x6f88
movq 0x3f0(%r14), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
xorps %xmm0, %xmm0
cvtsi2sd %eax, %xmm0
ucomisd 0x20634(%rip), %xmm0    # 0x27590
jb 0x6f64
sqrtsd %xmm0, %xmm0
jmp 0x6f69
callq 0x4438
movsd 0x2060f(%rip), %xmm1    # 0x27580
divsd %xmm0, %xmm1
xorpd %xmm0, %xmm0
movq %r12, %rdi
callq 0x1be10
jmp 0x6fa6
callq 0x4438
movq %r13, %rdi
movq %r12, %rsi
callq 0x1b70c
movq %r12, %rdi
movq %r13, %rsi
callq 0x14e2c
movq %r13, %rdi
callq 0x1abf2
movq %r15, %rdi
movq %r12, %rsi
callq 0x138a4
movl $0x1, %esi
movl $0x1, %edx
movq %r13, %rdi
callq 0x1a76e
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x6ffa
cmpb $0x1, 0x210(%r14)
jne 0x7016
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x1b70c
movq %r13, %rdi
movq %rbp, %rsi
callq 0x14e2c
jmp 0x7030
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x1b70c
movq %r13, %rdi
movq %rbp, %rsi
callq 0x14e2c
jmp 0x7030
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %r13, %rsi
callq 0x1b70c
movq %r13, %rdi
movq %rbp, %rsi
callq 0x14e2c
movq %rbp, %rdi
callq 0x1abf2
movq 0x38(%rsp), %rdi
movq %r13, %rsi
callq 0x138a4
movq %r13, %rdi
callq 0x1abf2
movq %r12, %rdi
callq 0x1abf2
incq %rbx
cmpq 0x18(%r14), %rbx
jb 0x6ed5
jmp 0x72ea
cmpq $0x0, 0x18(%r14)
je 0x72ea
xorl %ebx, %ebx
leaq 0x40(%rsp), %r12
leaq 0x8(%rsp), %r13
movq 0x20(%r14), %rsi
movq 0x28(%r14), %rdx
movq %r12, %rdi
movsd 0x204ef(%rip), %xmm0    # 0x27580
callq 0x1a92c
movq %r15, %rdi
movq %r12, %rsi
callq 0x138a4
movl $0x1, %esi
movl $0x1, %edx
xorpd %xmm0, %xmm0
movq %r13, %rdi
callq 0x1a892
movq 0x38(%rsp), %rdi
movq %r13, %rsi
callq 0x138a4
movq %r13, %rdi
callq 0x1abf2
movq %r12, %rdi
callq 0x1abf2
incq %rbx
cmpq 0x18(%r14), %rbx
jb 0x707e
jmp 0x72ea
movq 0x2feef(%rip), %r12     # 0x36fd8
leaq 0x20892(%rip), %rsi     # 0x27982
movl $0x14, %edx
movq %r12, %rdi
callq 0x4280
movq (%r14), %rax
movq %r14, %rdi
callq *0x298(%rax)
movq %r12, %rdi
movl %eax, %esi
callq 0x43c0
movq %rax, %r12
leaq 0x2087a(%rip), %rsi     # 0x27997
movl $0x3f, %edx
movq %rax, %rdi
callq 0x4280
movq (%r12), %rax
movq -0x18(%rax), %rdi
addq %r12, %rdi
movl $0xa, %r13d
movl $0xa, %esi
callq 0x4250
movsbl %al, %esi
movq %r12, %rdi
callq 0x4030
movq %rax, %rdi
callq 0x4190
movslq %ebp, %r12
movslq %r13d, %rdx
leaq 0x40(%rsp), %rdi
movq %r12, %rsi
callq 0x1a76e
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x71b6
leaq 0x40(%rsp), %rdi
movsd 0x203f7(%rip), %xmm1    # 0x27580
xorpd %xmm0, %xmm0
callq 0x1be10
xorps %xmm1, %xmm1
cvtsi2sd %r13d, %xmm1
movsd 0x203e6(%rip), %xmm0    # 0x27588
divsd %xmm1, %xmm0
ucomisd 0x203e2(%rip), %xmm0    # 0x27590
jb 0x71ec
sqrtsd %xmm0, %xmm0
jmp 0x71f1
cvtsi2sd %r13d, %xmm0
xorpd %xmm1, %xmm1
ucomisd %xmm1, %xmm0
jb 0x71cb
sqrtsd %xmm0, %xmm0
jmp 0x71d0
callq 0x4438
movsd 0x203a8(%rip), %xmm1    # 0x27580
divsd %xmm0, %xmm1
leaq 0x40(%rsp), %rdi
xorpd %xmm0, %xmm0
callq 0x1be10
jmp 0x7219
callq 0x4438
leaq 0x8(%rsp), %rdi
leaq 0x40(%rsp), %rsi
callq 0x1b70c
leaq 0x40(%rsp), %rdi
leaq 0x8(%rsp), %rsi
callq 0x14e2c
leaq 0x8(%rsp), %rdi
callq 0x1abf2
leaq 0x40(%rsp), %rsi
movq %r15, %rdi
callq 0x138a4
leaq 0x8(%rsp), %rdi
movl $0x1, %edx
movq %r12, %rsi
callq 0x1a76e
movq (%r14), %rax
movq %r14, %rdi
callq *0x2b0(%rax)
cmpl $0x3, %eax
jne 0x7277
cmpb $0x1, 0x210(%r14)
jne 0x729b
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
xorpd %xmm0, %xmm0
callq 0x1b70c
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x14e2c
jmp 0x72bd
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
xorpd %xmm0, %xmm0
callq 0x1b70c
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x14e2c
jmp 0x72bd
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
xorpd %xmm0, %xmm0
callq 0x1b70c
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x14e2c
leaq 0x20(%rsp), %rdi
callq 0x1abf2
leaq 0x8(%rsp), %rsi
movq 0x38(%rsp), %rdi
callq 0x138a4
leaq 0x8(%rsp), %rdi
callq 0x1abf2
leaq 0x40(%rsp), %rdi
callq 0x1abf2
addq $0x58, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movq 0x2fcd8(%rip), %rdi     # 0x36fd8
leaq 0x2063a(%rip), %rsi     # 0x27941
callq 0x4260
leaq 0x20641(%rip), %rsi     # 0x27954
movq %rax, %rdi
callq 0x4260
movq %rax, %rdi
callq 0x4100
xorl %edi, %edi
callq 0x4290
movq 0x2fca7(%rip), %rdi     # 0x36fd8
leaq 0x2069f(%rip), %rsi     # 0x279d7
jmp 0x7316
jmp 0x7360
jmp 0x7360
jmp 0x7379
jmp 0x7379
jmp 0x7360
jmp 0x7379
jmp 0x7379
jmp 0x7388
jmp 0x7388
jmp 0x7379
jmp 0x7360
jmp 0x7379
jmp 0x7360
jmp 0x7379
jmp 0x7388
jmp 0x7388
jmp 0x7379
jmp 0x7379
jmp 0x7388
movq %rax, %rbx
leaq 0x20(%rsp), %rdi
callq 0x1abf2
jmp 0x737c
jmp 0x7388
jmp 0x7388
jmp 0x7388
jmp 0x7388
jmp 0x7379
movq %rax, %rbx
leaq 0x8(%rsp), %rdi
callq 0x1abf2
jmp 0x738b
movq %rax, %rbx
leaq 0x40(%rsp), %rdi
callq 0x1abf2
movq %rbx, %rdi
callq 0x43e0
nop