// File: xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::InitWeightsAndBias()
void  ConstructLayer::InitWeightsAndBias()
{
    //std::cout<<"Layer id: "<<GetID()<<", W initialization."<<endl;
    // init weights and bias
    // no need to init active w&b, they will be filled in Batch starting phase
    // clear everything
    __weightMatrix.clear();
    __biasVector.clear();

    if(__trainingType == TrainingType::ResumeTraining)
    {
        LoadTrainedWeightsAndBias();
        return;
    }

    if(__p_data_interface == nullptr)
    {
        std::cout<<__func__<<" error: layer Need DataInterface already set."
            <<std::endl;
        exit(0);
    }
    //int batch_size = __p_data_interface->GetBatchSize();
    //int N = __p_data_interface->GetNumberOfBatches();

    // total_entries is used to randomly initialize weight matrix, 
    // using a Gaussian distribution, wieth mean=0, sigma = 1/sqrt(total_entries)
    //double total_entries = (double)batch_size * (double)N; // use this one, it is better (incorrect)
    //double total_entries = (double)batch_size;          // this one is not as good as the above one (incorrect)

    if(__type == LayerType::fullyConnected || __type == LayerType::output) // output layer is also a fully connected layer
    {
        int n_prev = 0;
        int n_curr = GetNumberOfNeurons();
        if(__prevLayer == nullptr) // input layer
        {
            std::cout<<"ERROR WARNING: layer"<<GetID()
                <<" has no prev layer, default 10 neruons for prev layer was used."
                <<std::endl;
            n_prev = 10;
        }
        else 
            n_prev = __prevLayer->GetNumberOfNeurons();

        // weight initialization
        Matrix w(n_curr, n_prev);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu actuation functions, use Xavier initialization
            // random with a normal distribution with sigma = 1/sqrt(number of fan-in neurons)
            w.RandomGaus(0., 1./sqrt((double)n_prev)); // (0, sqrt(n_neuron)) normal distribution

            //w.RandomGaus(0., 1./sqrt((double)total_entries)); // (0, sqrt(n_neuron)) normal distribution
        }
        else
        {
            // for ReLu actuation functions, use Kaiming He method: https://arxiv.org/pdf/1502.01852.pdf
            // step 1): initialize the matrix with a standard normal distribution
            w.RandomGaus(0., 1.);

            // step 2): then hadamard the weight matrix with a number sqrt(2/n)
            //          where n is the fan-in neurons (number of neurons in previous layer)
            w = w*sqrt(2./(double)n_prev);
        }
        __weightMatrix.push_back(w);

        // bias initialization
        Matrix b(n_curr, 1);
        if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
        {
            // for non-ReLu neurons, use Xavier initialization, initialize it to 0
            b = b * 0.;
            //b.RandomGaus(0., 1.); // (0, 1) normal distribution
        }
        else if(__use_batch_normalization)
        {
            // if use batch normalization, b = 0
            // b is combined into \beta
            b = b*0.;
        }
        else
        {
            // for ReLu neurons, use Kaiming He method, initialize bias to 0
            b = b*0.;
        }
        __biasVector.push_back(b);
    }
    else if(__type == LayerType::cnn)
    {
        //auto norm_factor = __p_data_interface->GetBatchSize();
        for(size_t i=0;i<__n_kernels_cnn;i++)
        {
            Matrix w(__kernelDim);
            // weight matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w.RandomGaus(0., 1./sqrt((double)fan_in_neurons));
                //w.RandomGaus(0., 1./sqrt((double)total_entries));
            }
            else
            {
                // for ReLu neurons, use Kaiming He method
                w.RandomGaus(0., 1.);
                int fan_in_neurons = __prevLayer->GetNumberOfNeurons();
                w = w * sqrt(2./ (double) fan_in_neurons);
            }
            __weightMatrix.push_back(w);

            Matrix b(1, 1);
            // bias matrix initialization
            if(GetNeuronActuationFuncType() != ActuationFuncType::Relu)
            {
                // for non-ReLu neurons, use Xavier initialization
                // Xavier also require b to be initialized with 0
                b = b * 0.;
                //b.RandomGaus(0., 1.); // (0, 1) normal distribution
            }
            else if(__use_batch_normalization)
            {
                // if use batch normalization, b = 0
                // b is combined into \beta
                b = b*0.;
            }
            else 
            {
                // for ReLu neurons, use Kaiming He method, set bias to 0.
                b = b * 0.;
            }
            __biasVector.push_back(b);
        }
    }
    else if(__type == LayerType::pooling)
    {
        for(size_t i=0;i<__n_kernels_cnn;i++) // pooling used cnn symbol
        {
            Matrix w(__kernelDim, 1); // pooling layer weight and bias matrix are not used, so set them to 1
            __weightMatrix.push_back(w);

            Matrix b(1, 1, 0);
            __biasVector.push_back(b);
        }
    }
    else {
        std::cout<<"Error: need layer type info before initing w&b."<<std::endl;
        exit(0);
    }
}

// Assembly:
subq $0x1e8, %rsp            # imm = 0x1E8
movq %rdi, 0x1e0(%rsp)
movq 0x1e0(%rsp), %rdi
movq %rdi, 0x30(%rsp)
addq $0x60, %rdi
callq 0x1fa00
movq 0x30(%rsp), %rdi
addq $0x78, %rdi
callq 0x1fa00
movq 0x30(%rsp), %rax
cmpl $0x1, 0x40c(%rax)
jne 0x90a4
movq 0x30(%rsp), %rdi
movq (%rdi), %rax
callq *0x128(%rax)
jmp 0x99de
movq 0x30(%rsp), %rax
cmpq $0x0, 0x400(%rax)
jne 0x90eb
movq 0x61f1e(%rip), %rdi     # 0x6afd8
leaq 0x49347(%rip), %rsi     # 0x52408
callq 0x4230
movq %rax, %rdi
leaq 0x4934b(%rip), %rsi     # 0x5241b
callq 0x4230
movq %rax, %rdi
movq 0x61ed1(%rip), %rsi     # 0x6afb0
callq 0x4260
xorl %edi, %edi
callq 0x4290
movq 0x30(%rsp), %rax
cmpl $0x0, 0xc(%rax)
je 0x9105
movq 0x30(%rsp), %rax
cmpl $0x4, 0xc(%rax)
jne 0x94d9
movq 0x30(%rsp), %rdi
movl $0x0, 0x1dc(%rsp)
movq (%rdi), %rax
callq *0x278(%rax)
movl %eax, %ecx
movq 0x30(%rsp), %rax
movl %ecx, 0x1d8(%rsp)
cmpq $0x0, 0x3f0(%rax)
jne 0x9193
movq 0x61e9b(%rip), %rdi     # 0x6afd8
leaq 0x49305(%rip), %rsi     # 0x52449
callq 0x4230
movq 0x30(%rsp), %rdi
movq %rax, 0x28(%rsp)
movq (%rdi), %rax
callq *0x298(%rax)
movq 0x28(%rsp), %rdi
movl %eax, %esi
callq 0x43e0
movq %rax, %rdi
leaq 0x492ec(%rip), %rsi     # 0x5245e
callq 0x4230
movq %rax, %rdi
movq 0x61e2f(%rip), %rsi     # 0x6afb0
callq 0x4260
movl $0xa, 0x1dc(%rsp)
jmp 0x91af
movq 0x30(%rsp), %rax
movq 0x3f0(%rax), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
movl %eax, 0x1dc(%rsp)
movslq 0x1d8(%rsp), %rsi
movslq 0x1dc(%rsp), %rdx
leaq 0x1c0(%rsp), %rdi
callq 0x3b3b0
movq 0x30(%rsp), %rdi
movq (%rdi), %rax
movq 0x2b0(%rax), %rax
callq *%rax
movl %eax, 0x24(%rsp)
jmp 0x91e3
movl 0x24(%rsp), %eax
cmpl $0x3, %eax
je 0x9236
cvtsi2sdl 0x1dc(%rsp), %xmm0
callq 0x4468
movsd 0x48e06(%rip), %xmm1    # 0x52008
divsd %xmm0, %xmm1
leaq 0x1c0(%rsp), %rdi
xorps %xmm0, %xmm0
callq 0x3dac0
jmp 0x9218
jmp 0x92cd
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
jmp 0x94c7
leaq 0x1c0(%rsp), %rdi
xorps %xmm0, %xmm0
movsd 0x48dbf(%rip), %xmm1    # 0x52008
callq 0x3dac0
jmp 0x9250
cvtsi2sdl 0x1dc(%rsp), %xmm1
movsd 0x48db7(%rip), %xmm0    # 0x52018
divsd %xmm1, %xmm0
callq 0x4468
leaq 0x198(%rsp), %rdi
leaq 0x1c0(%rsp), %rsi
callq 0x3cfd0
jmp 0x9281
leaq 0x1c0(%rsp), %rdi
leaq 0x198(%rsp), %rsi
callq 0x1fa20
jmp 0x9298
leaq 0x198(%rsp), %rdi
callq 0x3bb80
jmp 0x92cd
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0x198(%rsp), %rdi
callq 0x3bb80
jmp 0x94c7
movq 0x30(%rsp), %rdi
addq $0x60, %rdi
leaq 0x1c0(%rsp), %rsi
callq 0x1fa50
jmp 0x92e5
movslq 0x1d8(%rsp), %rsi
leaq 0x180(%rsp), %rdi
movl $0x1, %edx
callq 0x3b3b0
jmp 0x9301
movq 0x30(%rsp), %rdi
movq (%rdi), %rax
movq 0x2b0(%rax), %rax
callq *%rax
movl %eax, 0x20(%rsp)
jmp 0x9318
movl 0x20(%rsp), %eax
cmpl $0x3, %eax
je 0x93a7
leaq 0x168(%rsp), %rdi
leaq 0x180(%rsp), %rsi
xorps %xmm0, %xmm0
callq 0x3cfd0
jmp 0x933f
leaq 0x180(%rsp), %rdi
leaq 0x168(%rsp), %rsi
callq 0x1fa20
jmp 0x9356
leaq 0x168(%rsp), %rdi
callq 0x3bb80
jmp 0x9483
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
jmp 0x94ba
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0x168(%rsp), %rdi
callq 0x3bb80
jmp 0x94ba
movq 0x30(%rsp), %rax
testb $0x1, 0x210(%rax)
je 0x941e
leaq 0x150(%rsp), %rdi
leaq 0x180(%rsp), %rsi
xorps %xmm0, %xmm0
callq 0x3cfd0
jmp 0x93cf
leaq 0x180(%rsp), %rdi
leaq 0x150(%rsp), %rsi
callq 0x1fa20
jmp 0x93e6
leaq 0x150(%rsp), %rdi
callq 0x3bb80
jmp 0x9481
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0x150(%rsp), %rdi
callq 0x3bb80
jmp 0x94ba
leaq 0x138(%rsp), %rdi
leaq 0x180(%rsp), %rsi
xorps %xmm0, %xmm0
callq 0x3cfd0
jmp 0x9438
leaq 0x180(%rsp), %rdi
leaq 0x138(%rsp), %rsi
callq 0x1fa20
jmp 0x944f
leaq 0x138(%rsp), %rdi
callq 0x3bb80
jmp 0x9481
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0x138(%rsp), %rdi
callq 0x3bb80
jmp 0x94ba
jmp 0x9483
movq 0x30(%rsp), %rdi
addq $0x78, %rdi
leaq 0x180(%rsp), %rsi
callq 0x1fa50
jmp 0x949b
leaq 0x180(%rsp), %rdi
callq 0x3bb80
leaq 0x1c0(%rsp), %rdi
callq 0x3bb80
jmp 0x99de
leaq 0x180(%rsp), %rdi
callq 0x3bb80
leaq 0x1c0(%rsp), %rdi
callq 0x3bb80
jmp 0x99e6
movq 0x30(%rsp), %rax
cmpl $0x1, 0xc(%rax)
jne 0x98ac
movq $0x0, 0x130(%rsp)
movq 0x30(%rsp), %rcx
movq 0x130(%rsp), %rax
cmpq 0x18(%rcx), %rax
jae 0x98a7
movq 0x30(%rsp), %rax
movups 0x20(%rax), %xmm0
movaps %xmm0, 0x100(%rsp)
movq 0x100(%rsp), %rsi
movq 0x108(%rsp), %rdx
leaq 0x118(%rsp), %rdi
callq 0x3b4a0
movq 0x30(%rsp), %rdi
movq (%rdi), %rax
movq 0x2b0(%rax), %rax
callq *%rax
movl %eax, 0x1c(%rsp)
jmp 0x9550
movl 0x1c(%rsp), %eax
cmpl $0x3, %eax
je 0x95cc
movq 0x30(%rsp), %rax
movq 0x3f0(%rax), %rdi
movq (%rdi), %rax
movq 0x278(%rax), %rax
callq *%rax
movl %eax, 0x18(%rsp)
jmp 0x9577
movl 0x18(%rsp), %eax
movl %eax, 0xfc(%rsp)
cvtsi2sdl 0xfc(%rsp), %xmm0
callq 0x4468
movsd 0x48a70(%rip), %xmm1    # 0x52008
divsd %xmm0, %xmm1
leaq 0x118(%rsp), %rdi
xorps %xmm0, %xmm0
callq 0x3dac0
jmp 0x95ae
jmp 0x968c
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
jmp 0x9895
leaq 0x118(%rsp), %rdi
xorps %xmm0, %xmm0
movsd 0x48a29(%rip), %xmm1    # 0x52008
callq 0x3dac0
jmp 0x95e6
movq 0x30(%rsp), %rax
movq 0x3f0(%rax), %rdi
movq (%rdi), %rax
movq 0x278(%rax), %rax
callq *%rax
movl %eax, 0x14(%rsp)
jmp 0x9604
movl 0x14(%rsp), %eax
movl %eax, 0xf8(%rsp)
cvtsi2sdl 0xf8(%rsp), %xmm1
movsd 0x489f8(%rip), %xmm0    # 0x52018
divsd %xmm1, %xmm0
callq 0x4468
leaq 0xe0(%rsp), %rdi
leaq 0x118(%rsp), %rsi
callq 0x3cfd0
jmp 0x9640
leaq 0x118(%rsp), %rdi
leaq 0xe0(%rsp), %rsi
callq 0x1fa20
jmp 0x9657
leaq 0xe0(%rsp), %rdi
callq 0x3bb80
jmp 0x968c
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0xe0(%rsp), %rdi
callq 0x3bb80
jmp 0x9895
movq 0x30(%rsp), %rdi
addq $0x60, %rdi
leaq 0x118(%rsp), %rsi
callq 0x1fa50
jmp 0x96a4
leaq 0xc8(%rsp), %rdi
movl $0x1, %edx
movq %rdx, %rsi
callq 0x3b3b0
jmp 0x96bb
movq 0x30(%rsp), %rdi
movq (%rdi), %rax
movq 0x2b0(%rax), %rax
callq *%rax
movl %eax, 0x10(%rsp)
jmp 0x96d2
movl 0x10(%rsp), %eax
cmpl $0x3, %eax
je 0x9761
leaq 0xb0(%rsp), %rdi
leaq 0xc8(%rsp), %rsi
xorps %xmm0, %xmm0
callq 0x3cfd0
jmp 0x96f9
leaq 0xc8(%rsp), %rdi
leaq 0xb0(%rsp), %rsi
callq 0x1fa20
jmp 0x9710
leaq 0xb0(%rsp), %rdi
callq 0x3bb80
jmp 0x983d
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
jmp 0x9888
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0xb0(%rsp), %rdi
callq 0x3bb80
jmp 0x9888
movq 0x30(%rsp), %rax
testb $0x1, 0x210(%rax)
je 0x97d8
leaq 0x98(%rsp), %rdi
leaq 0xc8(%rsp), %rsi
xorps %xmm0, %xmm0
callq 0x3cfd0
jmp 0x9789
leaq 0xc8(%rsp), %rdi
leaq 0x98(%rsp), %rsi
callq 0x1fa20
jmp 0x97a0
leaq 0x98(%rsp), %rdi
callq 0x3bb80
jmp 0x983b
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0x98(%rsp), %rdi
callq 0x3bb80
jmp 0x9888
leaq 0x80(%rsp), %rdi
leaq 0xc8(%rsp), %rsi
xorps %xmm0, %xmm0
callq 0x3cfd0
jmp 0x97f2
leaq 0xc8(%rsp), %rdi
leaq 0x80(%rsp), %rsi
callq 0x1fa20
jmp 0x9809
leaq 0x80(%rsp), %rdi
callq 0x3bb80
jmp 0x983b
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0x80(%rsp), %rdi
callq 0x3bb80
jmp 0x9888
jmp 0x983d
movq 0x30(%rsp), %rdi
addq $0x78, %rdi
leaq 0xc8(%rsp), %rsi
callq 0x1fa50
jmp 0x9855
leaq 0xc8(%rsp), %rdi
callq 0x3bb80
leaq 0x118(%rsp), %rdi
callq 0x3bb80
movq 0x130(%rsp), %rax
addq $0x1, %rax
movq %rax, 0x130(%rsp)
jmp 0x94f4
leaq 0xc8(%rsp), %rdi
callq 0x3bb80
leaq 0x118(%rsp), %rdi
callq 0x3bb80
jmp 0x99e6
jmp 0x99dc
movq 0x30(%rsp), %rax
cmpl $0x2, 0xc(%rax)
jne 0x99b1
movq $0x0, 0x78(%rsp)
movq 0x30(%rsp), %rcx
movq 0x78(%rsp), %rax
cmpq 0x18(%rcx), %rax
jae 0x99af
movq 0x30(%rsp), %rax
movups 0x20(%rax), %xmm0
movaps %xmm0, 0x50(%rsp)
movq 0x50(%rsp), %rsi
movq 0x58(%rsp), %rdx
leaq 0x60(%rsp), %rdi
movq %rdi, 0x8(%rsp)
movsd 0x48706(%rip), %xmm0    # 0x52008
callq 0x3b680
movq 0x30(%rsp), %rdi
movq 0x8(%rsp), %rsi
addq $0x60, %rdi
callq 0x1fa50
jmp 0x991c
leaq 0x38(%rsp), %rdi
movl $0x1, %edx
xorps %xmm0, %xmm0
movq %rdx, %rsi
callq 0x3b590
jmp 0x9933
movq 0x30(%rsp), %rdi
addq $0x78, %rdi
leaq 0x38(%rsp), %rsi
callq 0x1fa50
jmp 0x9948
leaq 0x38(%rsp), %rdi
callq 0x3bb80
leaq 0x60(%rsp), %rdi
callq 0x3bb80
movq 0x78(%rsp), %rax
addq $0x1, %rax
movq %rax, 0x78(%rsp)
jmp 0x98c4
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
jmp 0x99a3
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x1b8(%rsp)
movl %eax, 0x1b4(%rsp)
leaq 0x38(%rsp), %rdi
callq 0x3bb80
leaq 0x60(%rsp), %rdi
callq 0x3bb80
jmp 0x99e6
jmp 0x99da
movq 0x61620(%rip), %rdi     # 0x6afd8
leaq 0x48adf(%rip), %rsi     # 0x5249e
callq 0x4230
movq %rax, %rdi
movq 0x615e2(%rip), %rsi     # 0x6afb0
callq 0x4260
xorl %edi, %edi
callq 0x4290
jmp 0x99dc
jmp 0x99de
addq $0x1e8, %rsp            # imm = 0x1E8
retq
movq 0x1b8(%rsp), %rdi
callq 0x43f0
nopw %cs:(%rax,%rax)