// File: /workspace/github_cpp_makefile_star2_O2/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::UpdateWeightsAndBiasFC()
void ConstructLayer::UpdateWeightsAndBiasFC()
{
    //cout<<__func__<<" started..."<<endl;
    // after finishing one batch, update weights and bias, for FC layer
    size_t M = __imageDeltaFull.size(); // batch size
    if( M != __wGradient.size() ) {
        std::cout<<__func__<<" Error: update FC weights, batch size not match."<<std::endl;
        std::cout<<"           weight gradient batch size: "<<__wGradient.size()<<std::endl;
        std::cout<<"           delta image batch size: "<<__imageDeltaFull.size()<<std::endl;
        exit(0);
    }
    if( __wGradient[0].OutputImageFromKernel.size() != 1 || __bGradient[0].OutputImageFromKernel.size()!=1) 
    {
        std::cout<<__func__<<" Error: update FC weiths, more than 1 w gradient matrix found."
            <<std::endl;
        exit(0);
    }

    // gradient descent
    if(__weightMatrix.size() != __neuronDim.k) {
        std::cout<<__func__<<" Error: update FC layer weights, more than 1 weight matrix found."<<std::endl;
        std::cout<<"__weightMatrix size: "<<__weightMatrix.size()<<std::endl;
        std::cout<<"neuron dimension: "<<__neuronDim<<std::endl;
        exit(0);
    }
    Matrix dw((__weightMatrix[0]).Dimension(), 0); 
    for(size_t i=0;i<M;i++){ // sum x (batches)
        dw  = dw + __wGradient[i].OutputImageFromKernel[0];
        //cout<<__wGradient[i].OutputImageFromKernel[0]<<endl;
        //cout<<i<<" before sum"<<endl;
        //getchar();
    }

    if(__weights_optimizer == WeightsOptimizer::SGD)
    {
        // standard SGD 
        dw = dw * double(__learningRate/(double)M); // over batch 
    }
    else if(__weights_optimizer == WeightsOptimizer::Adam)
    {
        // Adam optimizer
        dw = dw * (1./ (double)M );

        //Matrix tmp = dw * __learningRate;
        //cout<<"Before adam: "<<endl<<dw<<endl;
        dw = AdamOptimizer(dw, 0);
        //cout<<"after adam: "<<endl<<dw<<endl;
        //getchar();
    }
    else
    {
        std::cout<<__func__<<" Error: unsupported optimization method."<<std::endl;
        exit(0);
    }
    //cout<<"learning rate: "<<double(__learningRate/(double)M)<<endl;

    // Get filter Matrix for masking Regularization item
    assert(__activeFlag.size() == 1);
    auto convertFilterToMatrix = [&](Filter2D & F) -> Matrix
    {
        auto dim = F.Dimension();
        Matrix M(dim, 0);
        for(size_t i=0;i<dim.first;i++)
        {
            for(size_t j=0;j<dim.second;j++)
            {
                if(F[i][j]) M[i][j] = 1;
            }
        }
        return M;
    };
    Matrix currentLayerFilter = convertFilterToMatrix(__activeFlag[0]);
    //cout<<currentLayerFilter.Dimension()<<endl;
    Matrix prevLayerFilter(1, dw.Dimension().second, 1);
    //cout<<"before: "<<endl<<prevLayerFilter<<endl;

    if(__prevLayer->GetType() == LayerType::input) // only fc layer need this, other layers won't affect
    {
        auto & prevLayerFilters = __prevLayer->GetActiveFlag();
        assert(prevLayerFilters.size() == 1);
        assert(prevLayerFilters[0].Dimension().first == (size_t)__prevLayer->GetNumberOfNeurons());

        Matrix filter = convertFilterToMatrix(prevLayerFilters[0]);
        prevLayerFilter = filter.Transpose();
        //cout<<"after: "<<endl<<prevLayerFilter<<endl;
    }

    // following is two flag matrix (dim[n,1] x dim[1, m], with bool elements) product
    // all the elements of the product will be either 0 or 1
    // this product can be used to mask out disabled elements in weight matrix regularization
    Matrix F = currentLayerFilter * prevLayerFilter;
    //cout<<F<<endl;
    //cout<<prevLayerFilter<<endl;
    //cout<<dw<<endl;
    //getchar();

    assert(F.Dimension() == (__weightMatrix[0]).Dimension());
    // Regularization
    //double f_regularization = 0.;
    if(__regularizationMethod == Regularization::L2) 
    {
        // obsolete
        //f_regularization = 1 - __learningRate * __regularizationParameter / M;
        //(__weightMatrix[0]) = (__weightMatrix[0]) * f_regularization - dw;

        // new
        // hadamard multiply with F to maks out all inactive elements
        Matrix regularization_M = (__weightMatrix[0]^F) *(__learningRate * __regularizationParameter/((double)M)); 
        Matrix total_correction_M = regularization_M + dw; // dw already include learning rate during generation
        __weightMatrix[0] = __weightMatrix[0] - total_correction_M;
    } 
    else if(__regularizationMethod == Regularization::L1) 
    {
        Matrix _t = (__weightMatrix[0]); // make a copy of weight matrix
        _t(&SGN); // get the sign for each element in weight matrix
        _t = _t^F; // hadamard operation to mask out all inactive elements
        _t = _t * (__learningRate*__regularizationParameter/(double)M); // L1 regularization part
        (__weightMatrix[0]) = (__weightMatrix[0]) -  _t; // apply L1 regularization to weight matrix
        (__weightMatrix[0]) = (__weightMatrix[0]) - dw; // apply gradient decsent part
    } 
    else 
    {
        std::cout<<__func__<<" Error: update FC weights, unsupported regularization."<<std::endl;
        exit(0);
    }

    //cout<<"reached hrere..."<<endl;

    if(!__use_batch_normalization || 
            (__type != LayerType::cnn && __type != LayerType::fullyConnected && __type != LayerType::output))
    {
        // for cnn, fc, output layers, if BN is not used, one need to update b paramter.
        // if BN is used, b parameter is discarded (combined into \beta parameter)
        // bias
        Matrix db(__biasVector[0].Dimension());
        for(size_t i=0;i<M;i++){
            db = db + __bGradient[i].OutputImageFromKernel[0];
        }
        db = db / (double)M;
        db = db * __learningRate;
        __biasVector[0] = __biasVector[0] - db;
    }
    //cout<<__func__<<" ended.."<<endl;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0xc8, %rsp
movq %rdi, %rbx
movq 0x1e8(%rdi), %rcx
subq 0x1e0(%rdi), %rcx
pushq $0x18
popq %r12
movq %rcx, %rax
cqto
idivq %r12
movq %rax, %r14
movq 0x90(%rdi), %rax
movq 0x98(%rdi), %rdx
subq %rax, %rdx
cmpq %rdx, %rcx
jne 0xdb53
movq 0x8(%rax), %rcx
subq (%rax), %rcx
cmpq $0x18, %rcx
jne 0xdb2c
movq 0xa8(%rbx), %rax
movq 0x8(%rax), %rcx
subq (%rax), %rcx
cmpq $0x18, %rcx
jne 0xdb2c
movq 0x60(%rbx), %rdi
movq 0x68(%rbx), %rax
subq %rdi, %rax
cqto
idivq %r12
cmpq 0x3a8(%rbx), %rax
jne 0xdbe4
callq 0x17a6a
leaq 0x50(%rsp), %r15
xorpd %xmm0, %xmm0
movq %r15, %rdi
movq %rax, %rsi
callq 0x1787a
xorl %r13d, %r13d
leaq 0x38(%rsp), %r12
movq %r14, %rbp
subq $0x1, %rbp
jb 0xd5ce
movq 0x90(%rbx), %rax
movq (%rax,%r13), %rdx
movq %r12, %rdi
movq %r15, %rsi
callq 0x17cc4
movq %r15, %rdi
movq %r12, %rsi
callq 0x125b4
movq %r12, %rdi
callq 0x17a9a
addq $0x18, %r13
jmp 0xd599
movl 0x440(%rbx), %eax
movq %r14, %xmm1
testl %eax, %eax
movdqa %xmm1, 0x80(%rsp)
je 0xd670
cmpl $0x1, %eax
jne 0xdcae
movapd 0x80(%rsp), %xmm0
unpcklps 0x13a1d(%rip), %xmm0    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
subpd 0x13a25(%rip), %xmm0    # 0x21030
movapd %xmm0, %xmm1
unpckhpd %xmm0, %xmm1            # xmm1 = xmm1[1],xmm0[1]
addsd %xmm0, %xmm1
movsd 0x13ef1(%rip), %xmm0    # 0x21510
divsd %xmm1, %xmm0
leaq 0x38(%rsp), %rdi
leaq 0x50(%rsp), %rsi
callq 0x182dc
leaq 0x50(%rsp), %rdi
leaq 0x38(%rsp), %rsi
callq 0x125b4
leaq 0x38(%rsp), %rdi
callq 0x17a9a
leaq 0x38(%rsp), %rdi
leaq 0x50(%rsp), %rdx
movq %rbx, %rsi
xorl %ecx, %ecx
callq 0xeeb2
leaq 0x50(%rsp), %rdi
leaq 0x38(%rsp), %rsi
callq 0x125b4
jmp 0xd6b6
movsd 0xc0(%rbx), %xmm0
punpckldq 0x139a0(%rip), %xmm1 # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
subpd 0x139a8(%rip), %xmm1    # 0x21030
movapd %xmm1, %xmm2
unpckhpd %xmm1, %xmm2            # xmm2 = xmm2[1],xmm1[1]
addsd %xmm1, %xmm2
divsd %xmm2, %xmm0
leaq 0x38(%rsp), %rdi
leaq 0x50(%rsp), %rsi
callq 0x182dc
leaq 0x50(%rsp), %rdi
leaq 0x38(%rsp), %rsi
callq 0x125b4
leaq 0x38(%rsp), %rdi
callq 0x17a9a
movq 0x108(%rbx), %rsi
movq 0x110(%rbx), %rax
subq %rsi, %rax
cmpq $0x18, %rax
jne 0xdc70
leaq 0x38(%rsp), %rdi
callq 0xf3f6
leaq 0x50(%rsp), %rdi
callq 0x17a6a
leaq 0x98(%rsp), %rdi
pushq $0x1
popq %rsi
movsd 0x13e0e(%rip), %xmm0    # 0x21510
callq 0x177ee
movq 0x3f0(%rbx), %rdi
movq (%rdi), %rax
callq *0x250(%rax)
cmpl $0x3, %eax
jne 0xd7ac
movq 0x3f0(%rbx), %rdi
movq (%rdi), %rax
callq *0x268(%rax)
movq %rax, %r15
movq (%rax), %rdi
movq 0x8(%rax), %rax
subq %rdi, %rax
cmpq $0x18, %rax
jne 0xdcda
callq 0x1145e
movq %rax, %r12
movq 0x3f0(%rbx), %rdi
movq (%rdi), %rax
callq *0x278(%rax)
cltq
cmpq %rax, %r12
jne 0xdcf9
movq (%r15), %rsi
leaq 0x68(%rsp), %rdi
callq 0xf3f6
leaq 0x20(%rsp), %rdi
leaq 0x68(%rsp), %rsi
callq 0x18e22
leaq 0x98(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x125b4
leaq 0x20(%rsp), %rdi
callq 0x17a9a
leaq 0x68(%rsp), %rdi
callq 0x17a9a
leaq 0x68(%rsp), %rdi
leaq 0x38(%rsp), %rsi
leaq 0x98(%rsp), %rdx
callq 0x17aac
leaq 0x68(%rsp), %rdi
callq 0x17a6a
movq %rax, %r15
movq %rdx, %r12
movq 0x60(%rbx), %rdi
callq 0x17a6a
movq %rdx, %rcx
movq %r15, %rdi
movq %r12, %rsi
movq %rax, %rdx
callq 0x175e8
testb %al, %al
je 0xdc8f
movl 0xc8(%rbx), %eax
cmpl $0x1, %eax
je 0xd8bd
testl %eax, %eax
jne 0xdd18
movq 0x60(%rbx), %rsi
leaq 0x8(%rsp), %rdi
leaq 0x68(%rsp), %rdx
callq 0x17f64
movsd 0xc0(%rbx), %xmm0
mulsd 0xd0(%rbx), %xmm0
movapd 0x80(%rsp), %xmm1
unpcklps 0x137e1(%rip), %xmm1    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
subpd 0x137e9(%rip), %xmm1    # 0x21030
movapd %xmm1, %xmm2
unpckhpd %xmm1, %xmm2            # xmm2 = xmm2[1],xmm1[1]
addsd %xmm1, %xmm2
divsd %xmm2, %xmm0
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
callq 0x182dc
leaq 0x8(%rsp), %rdi
callq 0x17a9a
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
leaq 0x50(%rsp), %rdx
callq 0x17cc4
movq 0x60(%rbx), %rsi
leaq 0xb0(%rsp), %rdi
leaq 0x8(%rsp), %rdx
callq 0x17eb0
movq 0x60(%rbx), %rdi
leaq 0xb0(%rsp), %rsi
callq 0x125b4
leaq 0xb0(%rsp), %rdi
callq 0x17a9a
jmp 0xd9b5
movq 0x60(%rbx), %rsi
leaq 0x20(%rsp), %rdi
callq 0x1796e
leaq 0x1bc6(%rip), %rsi      # 0xf498
leaq 0x20(%rsp), %rdi
callq 0x188a2
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
leaq 0x68(%rsp), %rdx
callq 0x17f64
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
callq 0x125b4
leaq 0x8(%rsp), %rdi
callq 0x17a9a
movsd 0xc0(%rbx), %xmm0
mulsd 0xd0(%rbx), %xmm0
movapd 0x80(%rsp), %xmm1
unpcklps 0x136f7(%rip), %xmm1    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
subpd 0x136ff(%rip), %xmm1    # 0x21030
movapd %xmm1, %xmm2
unpckhpd %xmm1, %xmm2            # xmm2 = xmm2[1],xmm1[1]
addsd %xmm1, %xmm2
divsd %xmm2, %xmm0
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x182dc
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
callq 0x125b4
leaq 0x8(%rsp), %rdi
callq 0x17a9a
movq 0x60(%rbx), %rsi
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rdx
callq 0x17eb0
movq 0x60(%rbx), %rdi
leaq 0x8(%rsp), %rsi
callq 0x125b4
leaq 0x8(%rsp), %rdi
callq 0x17a9a
movq 0x60(%rbx), %rsi
leaq 0x8(%rsp), %rdi
leaq 0x50(%rsp), %rdx
callq 0x17eb0
movq 0x60(%rbx), %rdi
leaq 0x8(%rsp), %rsi
callq 0x125b4
leaq 0x8(%rsp), %rdi
callq 0x17a9a
leaq 0x20(%rsp), %rdi
callq 0x17a9a
cmpb $0x1, 0x210(%rbx)
jne 0xd9e6
movl 0xc(%rbx), %eax
cmpl $0x4, %eax
ja 0xd9e6
pushq $0x13
popq %rcx
btl %eax, %ecx
jb 0xdaef
movq 0x78(%rbx), %rdi
callq 0x17a6a
leaq 0x20(%rsp), %rdi
movq %rax, %rsi
callq 0x1776a
xorl %r13d, %r13d
leaq 0x8(%rsp), %r15
leaq 0x20(%rsp), %r12
subq $0x1, %r14
jb 0xda3e
movq 0xa8(%rbx), %rax
movq (%rax,%r13), %rdx
movq %r15, %rdi
movq %r12, %rsi
callq 0x17cc4
movq %r12, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
addq $0x18, %r13
jmp 0xda09
movapd 0x80(%rsp), %xmm1
unpcklps 0x135d2(%rip), %xmm1    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
subpd 0x135da(%rip), %xmm1    # 0x21030
movapd %xmm1, %xmm0
unpckhpd %xmm1, %xmm0            # xmm0 = xmm0[1],xmm1[1]
addsd %xmm1, %xmm0
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x18348
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
callq 0x125b4
leaq 0x8(%rsp), %rdi
callq 0x17a9a
movsd 0xc0(%rbx), %xmm0
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rsi
callq 0x182dc
leaq 0x20(%rsp), %rdi
leaq 0x8(%rsp), %rsi
callq 0x125b4
leaq 0x8(%rsp), %rdi
callq 0x17a9a
movq 0x78(%rbx), %rsi
leaq 0x8(%rsp), %rdi
leaq 0x20(%rsp), %rdx
callq 0x17eb0
movq 0x78(%rbx), %rdi
leaq 0x8(%rsp), %rsi
callq 0x125b4
leaq 0x8(%rsp), %rdi
callq 0x17a9a
leaq 0x20(%rsp), %rdi
callq 0x17a9a
leaq 0x68(%rsp), %rdi
callq 0x17a9a
leaq 0x98(%rsp), %rdi
callq 0x17a9a
leaq 0x38(%rsp), %rdi
callq 0x17a9a
leaq 0x50(%rsp), %rdi
callq 0x17a9a
addq $0xc8, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movq 0x254a5(%rip), %rdi     # 0x32fd8
leaq 0x14f1b(%rip), %rsi     # 0x22a55
callq 0x4240
leaq 0x14fa3(%rip), %rsi     # 0x22ae9
movq %rax, %rdi
callq 0x4240
jmp 0xdc61
movq 0x2547e(%rip), %r14     # 0x32fd8
leaq 0x14ef4(%rip), %rsi     # 0x22a55
movq %r14, %rdi
callq 0x4240
leaq 0x14efc(%rip), %rsi     # 0x22a6c
movq %rax, %rdi
callq 0x4240
movq %rax, %rdi
callq 0x4130
leaq 0x14f16(%rip), %rsi     # 0x22a9d
movq %r14, %rdi
callq 0x4240
movq %rax, %rdi
movq 0x98(%rbx), %rax
subq 0x90(%rbx), %rax
cqto
idivq %r12
movq %rax, %rsi
callq 0x4090
movq %rax, %rdi
callq 0x4130
leaq 0x14f09(%rip), %rsi     # 0x22ac5
movq %r14, %rdi
callq 0x4240
movq %rax, %rdi
movq 0x1e8(%rbx), %rax
subq 0x1e0(%rbx), %rax
cqto
idivq %r12
movq %rax, %rsi
callq 0x4090
jmp 0xdc61
leaq 0x398(%rbx), %r14
movq 0x253e6(%rip), %r15     # 0x32fd8
leaq 0x14e5c(%rip), %rsi     # 0x22a55
movq %r15, %rdi
callq 0x4240
leaq 0x14f20(%rip), %rsi     # 0x22b28
movq %rax, %rdi
callq 0x4240
movq %rax, %rdi
callq 0x4130
leaq 0x14f4b(%rip), %rsi     # 0x22b6a
movq %r15, %rdi
callq 0x4240
movq %rax, %rdi
movq 0x68(%rbx), %rax
subq 0x60(%rbx), %rax
cqto
idivq %r12
movq %rax, %rsi
callq 0x4090
movq %rax, %rdi
callq 0x4130
leaq 0x14f32(%rip), %rsi     # 0x22b80
movq %r15, %rdi
callq 0x4240
movq %rax, %rdi
movq %r14, %rsi
callq 0x1740e
movq %rax, %rdi
callq 0x4130
xorl %edi, %edi
callq 0x4260
leaq 0x14640(%rip), %rdi     # 0x222b7
leaq 0x13a5c(%rip), %rsi     # 0x216da
leaq 0x14f37(%rip), %rcx     # 0x22bbc
movl $0xd4a, %edx            # imm = 0xD4A
callq 0x4180
leaq 0x14fc4(%rip), %rdi     # 0x22c5a
leaq 0x13a3d(%rip), %rsi     # 0x216da
leaq 0x14f18(%rip), %rcx     # 0x22bbc
movl $0xd71, %edx            # imm = 0xD71
callq 0x4180
movq 0x25323(%rip), %rdi     # 0x32fd8
leaq 0x14d99(%rip), %rsi     # 0x22a55
callq 0x4240
leaq 0x14ecb(%rip), %rsi     # 0x22b93
movq %rax, %rdi
callq 0x4240
movq %rax, %rdi
callq 0x4130
jmp 0xdc69
leaq 0x14f09(%rip), %rdi     # 0x22bea
leaq 0x139f2(%rip), %rsi     # 0x216da
leaq 0x14ecd(%rip), %rcx     # 0x22bbc
movl $0xd60, %edx            # imm = 0xD60
callq 0x4180
leaq 0x14f07(%rip), %rdi     # 0x22c07
leaq 0x139d3(%rip), %rsi     # 0x216da
leaq 0x14eae(%rip), %rcx     # 0x22bbc
movl $0xd61, %edx            # imm = 0xD61
callq 0x4180
movq 0x252b9(%rip), %rdi     # 0x32fd8
leaq 0x14d2f(%rip), %rsi     # 0x22a55
callq 0x4240
leaq 0x14f59(%rip), %rsi     # 0x22c8b
movq %rax, %rdi
callq 0x4240
movq %rax, %rdi
callq 0x4130
jmp 0xdc69
jmp 0xddeb
jmp 0xddbb
jmp 0xddb6
jmp 0xddbb
jmp 0xddb6
jmp 0xddbb
jmp 0xddb6
movq %rax, %rbx
leaq 0xb0(%rsp), %rdi
callq 0x17a9a
jmp 0xddbe
jmp 0xddbb
jmp 0xddbb
jmp 0xddb6
jmp 0xddb6
jmp 0xddb6
movq %rax, %rbx
leaq 0x8(%rsp), %rdi
jmp 0xddcd
jmp 0xddb1
jmp 0xddb1
jmp 0xddb6
jmp 0xddb1
jmp 0xdda6
jmp 0xdda6
jmp 0xddf0
jmp 0xddeb
jmp 0xddf0
jmp 0xddf0
jmp 0xddeb
jmp 0xddeb
jmp 0xddbb
jmp 0xddb6
jmp 0xddbb
jmp 0xddb6
jmp 0xddbb
jmp 0xddb6
jmp 0xdda6
jmp 0xdda6
movq %rax, %rbx
jmp 0xdddc
jmp 0xddeb
jmp 0xddb1
jmp 0xddf0
movq %rax, %rbx
jmp 0xddd2
movq %rax, %rbx
jmp 0xddc8
movq %rax, %rbx
leaq 0x8(%rsp), %rdi
callq 0x17a9a
leaq 0x20(%rsp), %rdi
callq 0x17a9a
leaq 0x68(%rsp), %rdi
callq 0x17a9a
leaq 0x98(%rsp), %rdi
callq 0x17a9a
jmp 0xddf3
movq %rax, %rbx
jmp 0xddfd
movq %rax, %rbx
leaq 0x38(%rsp), %rdi
callq 0x17a9a
leaq 0x50(%rsp), %rdi
callq 0x17a9a
movq %rbx, %rdi
callq 0x43b0
nop