// File: /workspace/github_cpp_makefile_star2_O2/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::UpdateWeightsAndBiasCNN()
void ConstructLayer::UpdateWeightsAndBiasCNN()
{
    // after finishing one training batch, update weights and bias gradient, (CNN layer)
    // cnn layer is different with FC layer. For FC layer, different 
    // neurons have different weights, however, for CNN layer, different
    // neurons in one image share the same weights and bias.
    // so when you are
    // using this function in layer class, you need to loop over every 
    // neuron for FC layer, but you should not loop over neruons in 
    // the same image for CNN layer, you should just use only one neuron,
    // any neuron would be fine, instead you need to loop over images. --- This is old design

    // after finishing one batch, update weights and bias, CNN layer
    size_t M = __imageDeltaFull.size(); // batch size
    if( M != __wGradient.size() ) {
        std::cout<<"Error: update FC weights, batch size not match."<<std::endl;
        exit(0);
    }

    // in case use drop out, need to filter out all inactive elements
    // Get filter Matrix from Filter2D structure
    assert(__activeFlag.size() == __weightMatrix.size());
    auto convertFilterToMatrix = [&](Filter2D & F) -> Matrix
    {
        auto dim = F.Dimension();
        Matrix _M(dim, 0);
        for(size_t i=0;i<dim.first;i++)
        {
            for(size_t j=0;j<dim.second;j++)
            {
                if(F[i][j]) _M[i][j] = 1;
            }
        }
        return _M;
    };

    // loop for kernel
    size_t nKernel = __weightMatrix.size();
    for(size_t k=0;k<nKernel;k++)
    {
        // 1) get gradient
        // gradient descent
        Matrix dw(__weightMatrix[k].Dimension());
        // loop for batch
        for(size_t i=0;i<M;i++){ 
            dw  = dw + __wGradient[i].OutputImageFromKernel[k];
        }
        if(__weights_optimizer == WeightsOptimizer::SGD)
        {
            // stochastic gradient descent
            dw = dw * double(__learningRate/(double)M); // gradients average over batch size
        }
        else if(__weights_optimizer == WeightsOptimizer::Adam)
        {
            // adam optimize
            dw = dw * (1./(double)M);
            dw = AdamOptimizer(dw, k);
        }
        else
        {
            std::cout<<__func__<<" Error: unsupported weights optimizer."<<std::endl;
            exit(0);
        }

        // Hadamard F to mask out all inactive elements
        Matrix F = convertFilterToMatrix(__activeFlag[k]);
        assert(F.Dimension() == (__weightMatrix[k]).Dimension());
        dw = dw^F; // for safe reason, mask out all inactive elements

        // regularization part
        //double f_regularization = 0;
        if(__regularizationMethod == Regularization::L2)
        {
            // obsolete
            //f_regularization = 1 - __learningRate * __regularizationParameter / (double)M;
            //(__weightMatrix[k]) = (__weightMatrix[k])*f_regularization;
            //(__weightMatrix[k]) = (__weightMatrix[k]) - dw;

            Matrix regularization_M = (__weightMatrix[k]^F) * (__learningRate * __regularizationParameter/((double)M));
            Matrix total_correction_M = regularization_M + dw; // dw already have learing rate multiplied
            __weightMatrix[k] = __weightMatrix[k] - total_correction_M;
        }
        else if(__regularizationMethod == Regularization::L1)
        {
            Matrix tmp = (__weightMatrixActive[k]);
            tmp(&SGN);
            tmp = tmp^F; // hadamard operation to mask out all inactive elements
            tmp = tmp * (__learningRate*__regularizationParameter/(double)M);
            //(*__w) = (*__w) - tmp - dw;
            (__weightMatrix[k]) = (__weightMatrix[k]) - dw;
            (__weightMatrix[k]) = (__weightMatrix[k]) - tmp;
        }
        else {
            std::cout<<"Error: update CNN weights, unsupported regularizaton method."<<std::endl;
            exit(0);
        }

        if(!__use_batch_normalization || 
                (__type != LayerType::cnn && __type != LayerType::fullyConnected && __type != LayerType::output))
        {
            // for cnn, output fc layers, if BN is not used, one need to update b paramter
            // if BN is used, then b prameter is discarded (combined into \beta paramter), no need to update b parmater
            // update bias
            Matrix db(1, 1);
            // loop for batch
            for(size_t i=0;i<M;i++){
                db = db + __bGradient[i].OutputImageFromKernel[k];
            }
            db = db * double(__learningRate / (double)M);
            __biasVector[k] = __biasVector[k] - db;
        }
    }
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0xc8, %rsp
movq %rdi, %r14
movq 0x98(%rdi), %rdi
movq 0x1e8(%r14), %rcx
subq 0x1e0(%r14), %rcx
pushq $0x18
popq %rsi
movq %rcx, %rax
cqto
idivq %rsi
movq %rax, 0x30(%rsp)
subq 0x90(%r14), %rdi
cmpq %rdi, %rcx
jne 0xe3f9
movq 0x110(%r14), %rdi
subq 0x108(%r14), %rdi
movq 0x68(%r14), %rcx
subq 0x60(%r14), %rcx
movq %rcx, %rax
cqto
idivq %rsi
movq %rax, 0xa8(%rsp)
cmpq %rcx, %rdi
jne 0xe41b
movsd 0x30(%rsp), %xmm0
unpcklps 0x1318b(%rip), %xmm0    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
subpd 0x13193(%rip), %xmm0    # 0x21030
movapd %xmm0, %xmm1
unpckhpd %xmm0, %xmm1            # xmm1 = xmm1[1],xmm0[1]
addsd %xmm0, %xmm1
movsd 0x1365f(%rip), %xmm0    # 0x21510
movapd %xmm1, 0x70(%rsp)
divsd %xmm1, %xmm0
movsd %xmm0, 0xa0(%rsp)
xorl %eax, %eax
leaq 0x88(%rsp), %rbp
movq %r14, 0x10(%rsp)
cmpq 0xa8(%rsp), %rax
je 0xe37f
movq %rax, 0x50(%rsp)
imulq $0x18, %rax, %r15
movq 0x60(%r14), %rdi
addq %r15, %rdi
callq 0x17a6a
leaq 0x58(%rsp), %rdi
movq %rax, %rsi
callq 0x1776a
xorl %ebx, %ebx
movq 0x30(%rsp), %r14
subq $0x1, %r14
jb 0xdf4c
movq 0x10(%rsp), %rax
movq 0x90(%rax), %rax
movq (%rax,%rbx), %rdx
addq %r15, %rdx
movq %rbp, %rdi
leaq 0x58(%rsp), %r12
movq %r12, %rsi
callq 0x17cc4
movq %r12, %rdi
movq %rbp, %rsi
callq 0x125b4
movq %rbp, %rdi
callq 0x17a9a
addq $0x18, %rbx
jmp 0xdf0a
movq 0x10(%rsp), %rbx
movl 0x440(%rbx), %eax
testl %eax, %eax
movq %r15, 0x8(%rsp)
je 0xdfb5
cmpl $0x1, %eax
leaq 0x58(%rsp), %r14
jne 0xe3b0
movq %rbp, %rdi
movq %r14, %rsi
movsd 0xa0(%rsp), %xmm0
callq 0x182dc
movq %r14, %rdi
movq %rbp, %rsi
callq 0x125b4
movq %rbp, %rdi
callq 0x17a9a
movq %rbp, %rdi
movq %rbx, %rsi
movq %r14, %rdx
movq 0x50(%rsp), %rcx
callq 0xeeb2
movq %r14, %rdi
movq %rbp, %rsi
callq 0x125b4
jmp 0xdfde
movsd 0xc0(%rbx), %xmm0
divsd 0x70(%rsp), %xmm0
movq %rbp, %rdi
leaq 0x58(%rsp), %r14
movq %r14, %rsi
callq 0x182dc
movq %r14, %rdi
movq %rbp, %rsi
callq 0x125b4
movq %rbp, %rdi
callq 0x17a9a
movq 0x108(%rbx), %r15
addq 0x8(%rsp), %r15
movq %r15, %rdi
callq 0x1145e
movq %rax, %r13
movq %rdx, %r14
xorpd %xmm0, %xmm0
movq %rbp, %rdi
movq %rax, %rsi
callq 0x1787a
xorl %ebx, %ebx
cmpq %r13, %rbx
je 0xe058
xorl %r12d, %r12d
cmpq %r12, %r14
je 0xe053
imulq $0x28, %rbx, %rdi
addq (%r15), %rdi
movq %r12, %rsi
callq 0x11440
testq %rdx, (%rax)
je 0xe04e
movq %rbp, %rdi
movq %rbx, %rsi
callq 0x17c46
movq (%rax), %rax
movabsq $0x3ff0000000000000, %rcx # imm = 0x3FF0000000000000
movq %rcx, (%rax,%r12,8)
incq %r12
jmp 0xe019
incq %rbx
jmp 0xe011
movq %rbp, %rdi
callq 0x17a6a
movq %rax, %rbx
movq %rdx, %r14
movq 0x10(%rsp), %rax
movq 0x60(%rax), %rdi
addq 0x8(%rsp), %rdi
callq 0x17a6a
movq %rdx, %rcx
movq %rbx, %rdi
movq %r14, %rsi
movq %rax, %rdx
callq 0x175e8
testb %al, %al
leaq 0x58(%rsp), %r12
leaq 0x38(%rsp), %r15
je 0xe391
leaq 0x18(%rsp), %rbx
movq %rbx, %rdi
movq %r12, %rsi
movq %rbp, %rdx
callq 0x17f64
movq 0x10(%rsp), %r14
movq %r12, %rdi
movq %rbx, %rsi
callq 0x125b4
movq %rbx, %rdi
callq 0x17a9a
movl 0xc8(%r14), %eax
cmpl $0x1, %eax
je 0xe189
testl %eax, %eax
jne 0xe3dc
movq 0x60(%r14), %rsi
addq 0x8(%rsp), %rsi
movq %r15, %rdi
movq %rbp, %rdx
callq 0x17f64
movsd 0xc0(%r14), %xmm0
mulsd 0xd0(%r14), %xmm0
divsd 0x70(%rsp), %xmm0
leaq 0x18(%rsp), %rbx
movq %rbx, %rdi
movq %r15, %rsi
callq 0x182dc
movq %r15, %rdi
callq 0x17a9a
movq %r15, %rdi
movq %rbx, %rsi
leaq 0x58(%rsp), %rdx
callq 0x17cc4
movq 0x60(%r14), %rsi
movq %r14, %r12
movq 0x8(%rsp), %r14
addq %r14, %rsi
leaq 0xb0(%rsp), %rbx
movq %rbx, %rdi
movq %r15, %rdx
callq 0x17eb0
movq 0x60(%r12), %rdi
addq %r14, %rdi
movq %rbx, %rsi
callq 0x125b4
movq %rbx, %rdi
callq 0x17a9a
movq %r15, %rdi
callq 0x17a9a
leaq 0x18(%rsp), %rdi
callq 0x17a9a
movq 0x10(%rsp), %r14
jmp 0xe268
movq 0xd8(%r14), %rsi
addq 0x8(%rsp), %rsi
movq %rbx, %rdi
callq 0x1796e
movq %rbx, %rdi
leaq 0x12f1(%rip), %rsi      # 0xf498
callq 0x188a2
movq %r15, %rdi
movq %rbx, %rsi
movq %rbp, %rdx
callq 0x17f64
movq %rbx, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
movsd 0xc0(%r14), %xmm0
mulsd 0xd0(%r14), %xmm0
divsd 0x70(%rsp), %xmm0
movq %r15, %rdi
movq %rbx, %rsi
callq 0x182dc
movq %rbx, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
movq 0x60(%r14), %rsi
movq 0x8(%rsp), %rbx
addq %rbx, %rsi
movq %r15, %rdi
leaq 0x58(%rsp), %rdx
callq 0x17eb0
movq 0x60(%r14), %rdi
addq %rbx, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
movq 0x60(%r14), %rsi
addq %rbx, %rsi
movq %r15, %rdi
leaq 0x18(%rsp), %rdx
callq 0x17eb0
movq 0x60(%r14), %rdi
addq %rbx, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
leaq 0x18(%rsp), %rdi
callq 0x17a9a
cmpb $0x1, 0x210(%r14)
jne 0xe287
movl 0xc(%r14), %eax
cmpl $0x4, %eax
ja 0xe287
pushq $0x13
popq %rcx
btl %eax, %ecx
jb 0xe360
leaq 0x18(%rsp), %rdi
pushq $0x1
popq %rdx
movq %rdx, %rsi
callq 0x176e6
xorl %ebx, %ebx
movq 0x30(%rsp), %r14
subq $0x1, %r14
jb 0xe2e8
movq 0x10(%rsp), %rax
movq 0xa8(%rax), %rax
movq (%rax,%rbx), %rdx
addq 0x8(%rsp), %rdx
movq %r15, %rdi
movq %r15, %r12
leaq 0x18(%rsp), %r15
movq %r15, %rsi
callq 0x17cc4
movq %r15, %rdi
movq %r12, %rsi
callq 0x125b4
movq %r12, %r15
movq %r12, %rdi
callq 0x17a9a
addq $0x18, %rbx
jmp 0xe29e
movq 0x10(%rsp), %r14
movsd 0xc0(%r14), %xmm0
divsd 0x70(%rsp), %xmm0
movq %r15, %rdi
leaq 0x18(%rsp), %rbx
movq %rbx, %rsi
callq 0x182dc
movq %rbx, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
movq 0x78(%r14), %rsi
movq %r14, %r12
movq 0x8(%rsp), %r14
addq %r14, %rsi
movq %r15, %rdi
movq %rbx, %rdx
callq 0x17eb0
addq 0x78(%r12), %r14
movq %r14, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
leaq 0x18(%rsp), %rdi
callq 0x17a9a
movq 0x10(%rsp), %r14
movq %rbp, %rdi
callq 0x17a9a
leaq 0x58(%rsp), %rdi
callq 0x17a9a
movq 0x50(%rsp), %rax
incq %rax
jmp 0xded3
addq $0xc8, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0x14a04(%rip), %rdi     # 0x22d9c
leaq 0x1333b(%rip), %rsi     # 0x216da
leaq 0x14988(%rip), %rcx     # 0x22d2e
movl $0xe31, %edx            # imm = 0xE31
callq 0x4180
movq 0x24c21(%rip), %rdi     # 0x32fd8
leaq 0x1499f(%rip), %rsi     # 0x22d5d
callq 0x4240
leaq 0x149ab(%rip), %rsi     # 0x22d75
movq %rax, %rdi
callq 0x4240
movq %rax, %rdi
callq 0x4130
jmp 0xe414
movq 0x24bf5(%rip), %rdi     # 0x32fd8
leaq 0x149e3(%rip), %rsi     # 0x22dcd
callq 0x4240
movq %rax, %rdi
callq 0x4130
jmp 0xe414
movq 0x24bd8(%rip), %rdi     # 0x32fd8
leaq 0x14666(%rip), %rsi     # 0x22a6d
callq 0x4240
movq %rax, %rdi
callq 0x4130
xorl %edi, %edi
callq 0x4260
leaq 0x148df(%rip), %rdi     # 0x22d01
leaq 0x132b1(%rip), %rsi     # 0x216da
leaq 0x148fe(%rip), %rcx     # 0x22d2e
movl $0xe04, %edx            # imm = 0xE04
callq 0x4180
jmp 0xe4b9
jmp 0xe4b0
jmp 0xe497
jmp 0xe497
movq %rax, %rbx
leaq 0x38(%rsp), %rdi
jmp 0xe4a9
jmp 0xe497
jmp 0xe4b9
jmp 0xe492
jmp 0xe492
jmp 0xe492
jmp 0xe492
jmp 0xe4b9
jmp 0xe492
movq %rax, %rbx
leaq 0xb0(%rsp), %rdi
callq 0x17a9a
jmp 0xe49a
jmp 0xe492
jmp 0xe497
jmp 0xe497
jmp 0xe4b9
jmp 0xe4b0
jmp 0xe4b0
jmp 0xe4b0
jmp 0xe4b9
jmp 0xe4b9
jmp 0xe497
jmp 0xe492
jmp 0xe497
jmp 0xe492
jmp 0xe4b9
jmp 0xe4b9
jmp 0xe492
jmp 0xe4b0
jmp 0xe4b9
movq %rax, %rbx
jmp 0xe4a4
movq %rax, %rbx
leaq 0x38(%rsp), %rdi
callq 0x17a9a
leaq 0x18(%rsp), %rdi
callq 0x17a9a
jmp 0xe4bc
movq %rax, %rbx
jmp 0xe4c9
jmp 0xe4b9
jmp 0xe4b9
movq %rax, %rbx
leaq 0x88(%rsp), %rdi
callq 0x17a9a
leaq 0x58(%rsp), %rdi
callq 0x17a9a
movq %rbx, %rdi
callq 0x43b0
nop