// File: /workspace/github_cpp_makefile_star2_O2/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::BatchNormalization_GetGradientOnParameters_2DLayer()
void ConstructLayer::BatchNormalization_GetGradientOnParameters_2DLayer() // TO-DO: need to parallelize
{
    // gradients of cost function on \sigma_B^2, \mu_B, \gamma, \beta
    size_t nK = __n_kernels_cnn; // cnn layer = nKernels
    double epsilon = 1e-8;
    size_t batch_size = __p_data_interface->GetBatchSize();

    // for cnn layer, each kernel has one \mu_B, \sigma_B^2, \gamma, \beta, all of these are one number
    // so the gradients on these parameters are all numbers, not matrices

    for(size_t k=0;k<nK;k++)
    {
        // mu_B
        // both cnn and fc have one average and variance
        Matrix & mu_B = __imageAverage_BN[0].OutputImageFromKernel[k];
        // sigma_B^2
        Matrix & sigma_B = __imageVariance_BN[0].OutputImageFromKernel[k];

        Matrix Var_inverse_half_cubic = sigma_B + epsilon;
        Var_inverse_half_cubic(inverse_half_cubic);
        Var_inverse_half_cubic = Var_inverse_half_cubic * (-1./2.);

        Matrix Var_inverse_square_root = sigma_B + epsilon;
        Var_inverse_square_root(inverse_square_root);
        Var_inverse_square_root = Var_inverse_square_root * (-1.);

        // 1) \partial{C}/\partial{\sigma_B^2}
        Matrix partial_C_over_partial_sigma_B2(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        for(size_t i=0;i<batch_size;i++)
        {
            Matrix partial_C_over_partial_hat_x = __imageDelta[i].OutputImageFromKernel[k] * (__gamma_BN[k])[0][0];

            Matrix tmp = __imageZ[i].OutputImageFromKernel[k] - mu_B[0][0];
            tmp = tmp ^ partial_C_over_partial_hat_x;

            partial_C_over_partial_sigma_B2 = partial_C_over_partial_sigma_B2 + tmp;
        }
        auto dim = partial_C_over_partial_sigma_B2.Dimension();
        double sum = partial_C_over_partial_sigma_B2.SumInSection(0, dim.first, 0, dim.second);
        sum *= Var_inverse_half_cubic[0][0]; 
        Matrix res(1, 1, sum); // for cnn layer, result should be in DIM(1,1) matrix form

        // \partial{C}/\partial{\sigma_B^2}  result
        __v_partial_C_over_partial_sigma_square.push_back(res);

        // 2) \partial{C}/\partial{mu_B}
        Matrix part1(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        Matrix part2(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        for(size_t i=0;i<batch_size;i++)
        {
            Matrix partial_C_over_partial_hat_x = __imageDelta[i].OutputImageFromKernel[k] * (__gamma_BN[k])[0][0];
            part1 = part1 + partial_C_over_partial_hat_x;

            part2 = part2 + __imageZ[i].OutputImageFromKernel[k];
            part2 = part2 - mu_B[0][0];
        }
        double sum1 = part1.SumInSection(0, dim.first, 0, dim.second);
        sum1 = sum1 * Var_inverse_square_root[0][0];

        double sum2 = part2.SumInSection(0, dim.first, 0, dim.second);
        sum2 = sum2 * (-2./(double)batch_size/(double)dim.first/(double)dim.second);
        sum2 = sum2 * res[0][0];

        double tmp = sum1 + sum2;
        res[0][0] = tmp;
        // \partial{C}/\partial{mu_B}   result
        __v_partial_C_over_partial_mu.push_back(res);

        // 3) \partial{C}/\partial{\gamma} and \partial{C}/\partial{beta}
        Matrix partial_C_over_partial_gamma(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        Matrix partial_C_over_partial_beta(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        for(size_t i=0;i<batch_size;i++)
        {
            Matrix tmp = __imageDelta[i].OutputImageFromKernel[k] ^ __imageZ_BN[i].OutputImageFromKernel[k];
            partial_C_over_partial_gamma = partial_C_over_partial_gamma +  tmp;

            partial_C_over_partial_beta = partial_C_over_partial_beta + __imageDelta[i].OutputImageFromKernel[k];
        }

        sum1 = partial_C_over_partial_gamma.SumInSection(0, dim.first, 0, dim.second);
        sum2 = partial_C_over_partial_beta.SumInSection(0, dim.first, 0, dim.second);
        res[0][0] = sum1;
        __v_partial_C_over_partial_gamma.push_back(res);
        res[0][0] = sum2;
        __v_partial_C_over_partial_beta.push_back(res);
    }
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x168, %rsp            # imm = 0x168
movq 0x18(%rdi), %rax
movq %rax, 0xc0(%rsp)
movq 0x400(%rdi), %rax
movslq (%rax), %rax
movq %rax, 0x30(%rsp)
movq %rax, %xmm0
punpckldq 0x181df(%rip), %xmm0 # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
subpd 0x181e7(%rip), %xmm0    # 0x21030
leaq 0x338(%rdi), %rax
movq %rax, 0xb8(%rsp)
movapd %xmm0, %xmm1
unpckhpd %xmm0, %xmm1            # xmm1 = xmm1[1],xmm0[1]
addsd %xmm0, %xmm1
movsd 0x186ac(%rip), %xmm0    # 0x21518
divsd %xmm1, %xmm0
movsd %xmm0, 0xb0(%rsp)
leaq 0x350(%rdi), %rax
movq %rax, 0xa8(%rsp)
leaq 0x368(%rdi), %rax
movq %rax, 0xa0(%rsp)
movq %rdi, 0x8(%rsp)
leaq 0x380(%rdi), %rax
movq %rax, 0x98(%rsp)
xorl %r14d, %r14d
leaq 0xd8(%rsp), %r15
leaq 0x120(%rsp), %r13
cmpq 0xc0(%rsp), %r14
je 0x95a6
imulq $0x18, %r14, %rdx
movq 0x8(%rsp), %rcx
movq 0x278(%rcx), %rax
movq 0x2a8(%rcx), %rcx
movq (%rax), %rax
addq %rdx, %rax
movq %rax, 0x10(%rsp)
movq (%rcx), %r12
movq %rdx, (%rsp)
addq %rdx, %r12
leaq 0x138(%rsp), %rbx
movq %rbx, %rdi
movq %r12, %rsi
movsd 0x1861a(%rip), %xmm0    # 0x21528
callq 0x180cc
movq %rbx, %rdi
leaq 0x79c(%rip), %rsi       # 0x96b9
callq 0x188a2
leaq 0x108(%rsp), %rbp
movq %rbp, %rdi
movq %rbx, %rsi
movsd 0x185f8(%rip), %xmm0    # 0x21530
callq 0x182dc
movq %rbx, %rdi
movq %rbp, %rsi
callq 0x125b4
movq %rbp, %rdi
callq 0x17a9a
movq %rbp, %rdi
movq %r12, %rsi
movsd 0x185ca(%rip), %xmm0    # 0x21528
callq 0x180cc
movq %rbp, %rdi
leaq 0x79d(%rip), %rsi       # 0x970a
callq 0x188a2
leaq 0xf0(%rsp), %r12
movq %r12, %rdi
movq %rbp, %rsi
movsd 0x185b0(%rip), %xmm0    # 0x21538
callq 0x182dc
movq %rbp, %rdi
movq %r12, %rsi
callq 0x125b4
movq %r14, 0xd0(%rsp)
movq %r12, %rdi
callq 0x17a9a
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq (%rsp), %rdi
callq 0x17a6a
xorpd %xmm0, %xmm0
movq %r12, %rdi
movq %rax, %rsi
callq 0x1787a
xorl %edx, %edx
movq 0x30(%rsp), %rbp
subq $0x1, %rbp
jb 0x90d5
movq 0x8(%rsp), %rcx
movq 0x180(%rcx), %rax
movq %rdx, %r14
movq (%rax,%rdx), %r12
movq 0x308(%rcx), %rdi
movq (%rsp), %rbx
addq %rbx, %rdi
xorl %esi, %esi
callq 0x17c46
addq %rbx, %r12
movq (%rax), %rax
movsd (%rax), %xmm0
leaq 0x68(%rsp), %rdi
movq %r12, %rsi
callq 0x182dc
movq %r15, %r13
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax,%r14), %r12
movq 0x10(%rsp), %rdi
xorl %esi, %esi
callq 0x17c46
leaq 0x50(%rsp), %r15
addq (%rsp), %r12
movq (%rax), %rax
movsd (%rax), %xmm0
movq %r15, %rdi
movq %r12, %rsi
callq 0x18270
leaq 0x38(%rsp), %rbx
movq %rbx, %rdi
movq %r15, %rsi
leaq 0x68(%rsp), %rdx
callq 0x17f64
movq %r15, %rdi
movq %rbx, %rsi
callq 0x125b4
movq %rbx, %rdi
callq 0x17a9a
movq %rbx, %rdi
leaq 0xf0(%rsp), %r12
movq %r12, %rsi
movq %r15, %rdx
callq 0x17cc4
movq %r12, %rdi
movq %rbx, %rsi
callq 0x125b4
movq %rbx, %rdi
callq 0x17a9a
leaq 0x50(%rsp), %rdi
callq 0x17a9a
leaq 0x68(%rsp), %rdi
callq 0x17a9a
movq %r14, %rdx
addq $0x18, %rdx
movq %r13, %r15
leaq 0x120(%rsp), %r13
jmp 0x8fd6
movq %r12, %rdi
callq 0x17a6a
movq %rdx, %r8
movq %r12, %rdi
xorl %esi, %esi
movq %rax, 0x18(%rsp)
movq %rax, %rdx
xorl %ecx, %ecx
movq %r8, 0x20(%rsp)
callq 0x19b90
movsd %xmm0, 0x28(%rsp)
leaq 0x138(%rsp), %rdi
leaq 0x68(%rsp), %r14
xorl %esi, %esi
callq 0x17c46
movq (%rax), %rax
movsd 0x28(%rsp), %xmm0
mulsd (%rax), %xmm0
movq %r14, %rdi
pushq $0x1
popq %rdx
movq %rdx, %rsi
callq 0x177ee
movq 0xb8(%rsp), %rdi
movq %r14, %rsi
callq 0x1139c
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq (%rsp), %rdi
callq 0x17a6a
xorpd %xmm0, %xmm0
leaq 0x50(%rsp), %rdi
movq %rax, %rsi
callq 0x1787a
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq (%rsp), %rdi
callq 0x17a6a
xorpd %xmm0, %xmm0
leaq 0x38(%rsp), %rdi
movq %rax, %rsi
callq 0x1787a
xorl %edx, %edx
movq 0x30(%rsp), %rbp
subq $0x1, %rbp
jb 0x92a0
movq 0x8(%rsp), %rcx
movq 0x180(%rcx), %rax
movq %rdx, %r14
movq (%rax,%rdx), %r12
movq 0x308(%rcx), %rdi
movq (%rsp), %r13
addq %r13, %rdi
xorl %esi, %esi
callq 0x17c46
leaq 0x80(%rsp), %rbx
addq %r13, %r12
movq (%rax), %rax
movsd (%rax), %xmm0
movq %rbx, %rdi
movq %r12, %rsi
callq 0x182dc
movq %r15, %rdi
movq %r15, %r12
leaq 0x50(%rsp), %r15
movq %r15, %rsi
movq %rbx, %rdx
callq 0x17cc4
movq %r15, %rdi
movq %r12, %rsi
callq 0x125b4
movq %r12, %r15
movq %r12, %rdi
callq 0x17a9a
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax,%r14), %rdx
addq (%rsp), %rdx
movq %r12, %rdi
leaq 0x38(%rsp), %rbx
movq %rbx, %rsi
callq 0x17cc4
movq %rbx, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
movq 0x10(%rsp), %rdi
xorl %esi, %esi
callq 0x17c46
leaq 0x38(%rsp), %rbx
movq (%rax), %rax
movsd (%rax), %xmm0
movq %r15, %rdi
movq %rbx, %rsi
callq 0x18270
movq %rbx, %rdi
movq %r15, %rsi
callq 0x125b4
movq %r15, %rdi
callq 0x17a9a
leaq 0x80(%rsp), %rdi
callq 0x17a9a
movq %r14, %rdx
addq $0x18, %rdx
leaq 0x120(%rsp), %r13
jmp 0x9197
leaq 0x50(%rsp), %rdi
xorl %esi, %esi
movq 0x18(%rsp), %rdx
xorl %ecx, %ecx
movq 0x20(%rsp), %r8
callq 0x19b90
movsd %xmm0, 0x10(%rsp)
leaq 0x108(%rsp), %rdi
xorl %esi, %esi
callq 0x17c46
movq (%rax), %rax
movsd 0x10(%rsp), %xmm0
mulsd (%rax), %xmm0
movsd %xmm0, 0x10(%rsp)
leaq 0x38(%rsp), %rdi
xorl %esi, %esi
movq 0x18(%rsp), %rdx
xorl %ecx, %ecx
movq 0x20(%rsp), %r8
callq 0x19b90
movsd %xmm0, 0x28(%rsp)
leaq 0x68(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x17c46
movq (%rax), %rax
movsd (%rax), %xmm0
movsd %xmm0, 0xc8(%rsp)
movq %r14, %rdi
xorl %esi, %esi
callq 0x17c46
movsd 0x18(%rsp), %xmm0
movsd 0x17ceb(%rip), %xmm2    # 0x21020
unpcklps %xmm2, %xmm0            # xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
movapd 0x17cf0(%rip), %xmm3    # 0x21030
subpd %xmm3, %xmm0
movapd %xmm0, %xmm1
unpckhpd %xmm0, %xmm1            # xmm1 = xmm1[1],xmm0[1]
addsd %xmm0, %xmm1
movsd 0xb0(%rsp), %xmm0
divsd %xmm1, %xmm0
movsd 0x20(%rsp), %xmm1
unpcklps %xmm2, %xmm1            # xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
subpd %xmm3, %xmm1
movapd %xmm1, %xmm2
unpckhpd %xmm1, %xmm2            # xmm2 = xmm2[1],xmm1[1]
addsd %xmm1, %xmm2
divsd %xmm2, %xmm0
mulsd 0x28(%rsp), %xmm0
mulsd 0xc8(%rsp), %xmm0
movsd 0x10(%rsp), %xmm1
subsd %xmm0, %xmm1
movq (%rax), %rax
movsd %xmm1, (%rax)
movq 0xa8(%rsp), %rdi
movq %r14, %rsi
callq 0x1139c
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq (%rsp), %rdi
callq 0x17a6a
xorpd %xmm0, %xmm0
leaq 0x80(%rsp), %rdi
movq %rax, %rsi
callq 0x1787a
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq (%rsp), %rdi
callq 0x17a6a
xorpd %xmm0, %xmm0
movq %r15, %rdi
movq %rax, %rsi
callq 0x1787a
xorl %r14d, %r14d
movq 0x30(%rsp), %r12
movq (%rsp), %rdi
subq $0x1, %r12
jb 0x94b0
movq 0x8(%rsp), %rcx
movq 0x180(%rcx), %rax
movq 0x218(%rcx), %rcx
movq (%rax,%r14), %rsi
addq %rdi, %rsi
movq (%rcx,%r14), %rdx
addq %rdi, %rdx
movq %r13, %rdi
callq 0x17f64
leaq 0x150(%rsp), %rbp
movq %rbp, %rdi
leaq 0x80(%rsp), %rbx
movq %rbx, %rsi
movq %r13, %rdx
callq 0x17cc4
movq %rbx, %rdi
movq %rbp, %rsi
callq 0x125b4
movq %rbp, %rdi
callq 0x17a9a
movq 0x8(%rsp), %rax
movq 0x180(%rax), %rax
movq (%rax,%r14), %rdx
addq (%rsp), %rdx
movq %rbp, %rdi
movq %r15, %rsi
callq 0x17cc4
movq %r15, %rdi
movq %rbp, %rsi
callq 0x125b4
movq %rbp, %rdi
callq 0x17a9a
movq %r13, %rdi
callq 0x17a9a
addq $0x18, %r14
jmp 0x9405
leaq 0x80(%rsp), %rbx
movq %rbx, %rdi
xorl %esi, %esi
movq 0x18(%rsp), %rbp
movq %rbp, %rdx
xorl %ecx, %ecx
movq 0x20(%rsp), %r12
movq %r12, %r8
callq 0x19b90
movsd %xmm0, (%rsp)
leaq 0x68(%rsp), %r14
movq %r15, %rdi
xorl %esi, %esi
movq %rbp, %rdx
xorl %ecx, %ecx
movq %r12, %r8
callq 0x19b90
movsd %xmm0, 0x10(%rsp)
movq %r14, %rdi
xorl %esi, %esi
callq 0x17c46
movq (%rax), %rax
movsd (%rsp), %xmm0
movsd %xmm0, (%rax)
movq 0xa0(%rsp), %rdi
movq %r14, %rsi
callq 0x1139c
movq %r14, %rdi
xorl %esi, %esi
callq 0x17c46
movq (%rax), %rax
movsd 0x10(%rsp), %xmm0
movsd %xmm0, (%rax)
movq 0x98(%rsp), %rdi
movq %r14, %rsi
callq 0x1139c
movq %r15, %rdi
callq 0x17a9a
movq %rbx, %rdi
callq 0x17a9a
leaq 0x38(%rsp), %rdi
callq 0x17a9a
leaq 0x50(%rsp), %rdi
callq 0x17a9a
movq %r14, %rdi
callq 0x17a9a
leaq 0xf0(%rsp), %rdi
callq 0x17a9a
leaq 0x108(%rsp), %rdi
callq 0x17a9a
leaq 0x138(%rsp), %rdi
callq 0x17a9a
movq 0xd0(%rsp), %r14
incq %r14
jmp 0x8ebe
addq $0x168, %rsp            # imm = 0x168
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
jmp 0x96b4
jmp 0x96af
jmp 0x96b4
jmp 0x96b4
jmp 0x95d6
jmp 0x95f4
jmp 0x95d6
jmp 0x95f4
jmp 0x95f4
movq %rax, %rbx
jmp 0x969a
jmp 0x964d
jmp 0x964d
jmp 0x964d
jmp 0x964d
jmp 0x9652
jmp 0x96b4
jmp 0x9646
jmp 0x96af
movq %rax, %rbx
jmp 0x968d
jmp 0x962e
jmp 0x9602
jmp 0x9607
movq %rax, %rbx
jmp 0x9617
movq %rax, %rbx
leaq 0x150(%rsp), %rdi
callq 0x17a9a
leaq 0x120(%rsp), %rdi
callq 0x17a9a
jmp 0x9631
jmp 0x962e
jmp 0x962e
jmp 0x962e
jmp 0x9652
movq %rax, %rbx
leaq 0xd8(%rsp), %rdi
callq 0x17a9a
jmp 0x9655
jmp 0x9652
jmp 0x9646
jmp 0x964d
movq %rax, %rbx
jmp 0x966c
jmp 0x964d
movq %rax, %rbx
jmp 0x9662
movq %rax, %rbx
leaq 0x80(%rsp), %rdi
callq 0x17a9a
leaq 0x38(%rsp), %rdi
callq 0x17a9a
leaq 0x50(%rsp), %rdi
callq 0x17a9a
leaq 0x68(%rsp), %rdi
callq 0x17a9a
leaq 0xf0(%rsp), %rdi
callq 0x17a9a
leaq 0x108(%rsp), %rdi
callq 0x17a9a
leaq 0x138(%rsp), %rdi
callq 0x17a9a
movq %rbx, %rdi
callq 0x43b0
movq %rax, %rbx
jmp 0x9676
movq %rax, %rbx
jmp 0x9680