// File: /workspace/github_cpp_makefile_star2_O1/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::BatchNormalization_GetGradientOnParameters_2DLayer()
void ConstructLayer::BatchNormalization_GetGradientOnParameters_2DLayer() // TO-DO: need to parallelize
{
    // gradients of cost function on \sigma_B^2, \mu_B, \gamma, \beta
    size_t nK = __n_kernels_cnn; // cnn layer = nKernels
    double epsilon = 1e-8;
    size_t batch_size = __p_data_interface->GetBatchSize();

    // for cnn layer, each kernel has one \mu_B, \sigma_B^2, \gamma, \beta, all of these are one number
    // so the gradients on these parameters are all numbers, not matrices

    for(size_t k=0;k<nK;k++)
    {
        // mu_B
        // both cnn and fc have one average and variance
        Matrix & mu_B = __imageAverage_BN[0].OutputImageFromKernel[k];
        // sigma_B^2
        Matrix & sigma_B = __imageVariance_BN[0].OutputImageFromKernel[k];

        Matrix Var_inverse_half_cubic = sigma_B + epsilon;
        Var_inverse_half_cubic(inverse_half_cubic);
        Var_inverse_half_cubic = Var_inverse_half_cubic * (-1./2.);

        Matrix Var_inverse_square_root = sigma_B + epsilon;
        Var_inverse_square_root(inverse_square_root);
        Var_inverse_square_root = Var_inverse_square_root * (-1.);

        // 1) \partial{C}/\partial{\sigma_B^2}
        Matrix partial_C_over_partial_sigma_B2(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        for(size_t i=0;i<batch_size;i++)
        {
            Matrix partial_C_over_partial_hat_x = __imageDelta[i].OutputImageFromKernel[k] * (__gamma_BN[k])[0][0];

            Matrix tmp = __imageZ[i].OutputImageFromKernel[k] - mu_B[0][0];
            tmp = tmp ^ partial_C_over_partial_hat_x;

            partial_C_over_partial_sigma_B2 = partial_C_over_partial_sigma_B2 + tmp;
        }
        auto dim = partial_C_over_partial_sigma_B2.Dimension();
        double sum = partial_C_over_partial_sigma_B2.SumInSection(0, dim.first, 0, dim.second);
        sum *= Var_inverse_half_cubic[0][0]; 
        Matrix res(1, 1, sum); // for cnn layer, result should be in DIM(1,1) matrix form

        // \partial{C}/\partial{\sigma_B^2}  result
        __v_partial_C_over_partial_sigma_square.push_back(res);

        // 2) \partial{C}/\partial{mu_B}
        Matrix part1(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        Matrix part2(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        for(size_t i=0;i<batch_size;i++)
        {
            Matrix partial_C_over_partial_hat_x = __imageDelta[i].OutputImageFromKernel[k] * (__gamma_BN[k])[0][0];
            part1 = part1 + partial_C_over_partial_hat_x;

            part2 = part2 + __imageZ[i].OutputImageFromKernel[k];
            part2 = part2 - mu_B[0][0];
        }
        double sum1 = part1.SumInSection(0, dim.first, 0, dim.second);
        sum1 = sum1 * Var_inverse_square_root[0][0];

        double sum2 = part2.SumInSection(0, dim.first, 0, dim.second);
        sum2 = sum2 * (-2./(double)batch_size/(double)dim.first/(double)dim.second);
        sum2 = sum2 * res[0][0];

        double tmp = sum1 + sum2;
        res[0][0] = tmp;
        // \partial{C}/\partial{mu_B}   result
        __v_partial_C_over_partial_mu.push_back(res);

        // 3) \partial{C}/\partial{\gamma} and \partial{C}/\partial{beta}
        Matrix partial_C_over_partial_gamma(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        Matrix partial_C_over_partial_beta(__imageZ[0].OutputImageFromKernel[k].Dimension(), 0.);
        for(size_t i=0;i<batch_size;i++)
        {
            Matrix tmp = __imageDelta[i].OutputImageFromKernel[k] ^ __imageZ_BN[i].OutputImageFromKernel[k];
            partial_C_over_partial_gamma = partial_C_over_partial_gamma +  tmp;

            partial_C_over_partial_beta = partial_C_over_partial_beta + __imageDelta[i].OutputImageFromKernel[k];
        }

        sum1 = partial_C_over_partial_gamma.SumInSection(0, dim.first, 0, dim.second);
        sum2 = partial_C_over_partial_beta.SumInSection(0, dim.first, 0, dim.second);
        res[0][0] = sum1;
        __v_partial_C_over_partial_gamma.push_back(res);
        res[0][0] = sum2;
        __v_partial_C_over_partial_beta.push_back(res);
    }
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x178, %rsp            # imm = 0x178
movq %rdi, 0x8(%rsp)
movq 0x18(%rdi), %rax
movq %rax, 0xb8(%rsp)
testq %rax, %rax
je 0xa1d7
movq 0x8(%rsp), %rdx
movq 0x400(%rdx), %rax
movslq (%rax), %rcx
leaq 0x338(%rdx), %rax
movq %rax, 0xb0(%rsp)
movq %rcx, 0x20(%rsp)
movq %rcx, %xmm0
punpckldq 0x1d5e9(%rip), %xmm0 # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
subpd 0x1d5f1(%rip), %xmm0    # 0x27050
movapd %xmm0, %xmm1
unpckhpd %xmm0, %xmm1            # xmm1 = xmm1[1],xmm0[1]
addsd %xmm0, %xmm1
movsd 0x1db15(%rip), %xmm0    # 0x27588
divsd %xmm1, %xmm0
movsd %xmm0, 0xa8(%rsp)
leaq 0x350(%rdx), %rax
movq %rax, 0xa0(%rsp)
leaq 0x368(%rdx), %rax
movq %rax, 0x98(%rsp)
leaq 0x380(%rdx), %rax
movq %rax, 0x90(%rsp)
xorl %r14d, %r14d
leaq 0x130(%rsp), %r15
leaq 0x118(%rsp), %rbx
leaq 0xe8(%rsp), %r13
leaq (,%r14,8), %rax
leaq (%rax,%rax,2), %rdx
movq 0x8(%rsp), %rcx
movq 0x278(%rcx), %rax
movq 0x2a8(%rcx), %rcx
movq (%rax), %rax
addq %rdx, %rax
movq %rax, 0x28(%rsp)
movq (%rcx), %r12
movq %rdx, 0xc8(%rsp)
addq %rdx, %r12
movq %r15, %rdi
movq %r12, %rsi
movsd 0x1da8a(%rip), %xmm0    # 0x27598
callq 0x1b4b6
movq %r15, %rdi
leaq 0x7cd(%rip), %rsi       # 0xa2ea
callq 0x1bfe6
movq %rbx, %rdi
movq %r15, %rsi
movsd 0x1da70(%rip), %xmm0    # 0x275a0
callq 0x1b7d0
movq %r15, %rdi
movq %rbx, %rsi
callq 0x14f80
movq %rbx, %rdi
callq 0x1acb6
movq %rbx, %rdi
movq %r12, %rsi
movsd 0x1da42(%rip), %xmm0    # 0x27598
callq 0x1b4b6
movq %rbx, %rdi
leaq 0x7d6(%rip), %rsi       # 0xa33b
callq 0x1bfe6
leaq 0x100(%rsp), %r12
movq %r13, %rdi
movq %rbx, %rsi
movsd 0x1da28(%rip), %xmm0    # 0x275a8
callq 0x1b7d0
movq %rbx, %rdi
movq %r13, %rsi
callq 0x14f80
movq %r13, %rdi
callq 0x1acb6
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq %r14, 0xc0(%rsp)
leaq (%r14,%r14,2), %rcx
movq %rcx, 0x18(%rsp)
leaq (,%rcx,8), %rcx
movq (%rax), %rdi
movq %rcx, 0x10(%rsp)
addq %rcx, %rdi
callq 0x1ac7c
xorpd %xmm0, %xmm0
movq %r13, %rdi
movq %rax, %rsi
callq 0x1a9ec
cmpl $0x0, 0x20(%rsp)
leaq 0xd0(%rsp), %r13
je 0x9cea
xorl %r14d, %r14d
movq 0x20(%rsp), %rbp
movq 0x8(%rsp), %rcx
movq 0x180(%rcx), %rax
movq (%rax,%r14), %r15
movq 0x308(%rcx), %rdi
addq 0x10(%rsp), %rdi
xorl %esi, %esi
callq 0x1ae76
movq 0x18(%rsp), %rcx
leaq (%r15,%rcx,8), %rsi
movq (%rax), %rax
movsd (%rax), %xmm0
leaq 0x78(%rsp), %rdi
callq 0x1b7d0
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax,%r14), %r15
movq 0x28(%rsp), %rdi
xorl %esi, %esi
callq 0x1ae76
leaq 0x60(%rsp), %r13
movq 0x18(%rsp), %rcx
leaq (%r15,%rcx,8), %rsi
movq (%rax), %rax
movsd (%rax), %xmm0
movq %r13, %rdi
callq 0x1b714
leaq 0x100(%rsp), %r12
movq %r12, %rdi
movq %r13, %rsi
leaq 0x78(%rsp), %rdx
callq 0x1b2b2
movq %r13, %rdi
movq %r12, %rsi
callq 0x14f80
movq %r12, %rdi
callq 0x1acb6
movq %r12, %rdi
leaq 0xe8(%rsp), %r15
movq %r15, %rsi
movq %r13, %rdx
callq 0x1aef8
movq %r15, %rdi
movq %r12, %rsi
callq 0x14f80
movq %r12, %rdi
callq 0x1acb6
leaq 0x60(%rsp), %rdi
callq 0x1acb6
leaq 0x78(%rsp), %rdi
callq 0x1acb6
addq $0x18, %r14
decq %rbp
leaq 0xd0(%rsp), %r13
jne 0x9bf7
leaq 0xe8(%rsp), %r14
movq %r14, %rdi
callq 0x1ac7c
movq %rdx, %r8
movq %r14, %rdi
xorl %esi, %esi
movq %rax, 0x38(%rsp)
movq %rax, %rdx
xorl %ecx, %ecx
movq %r8, 0x40(%rsp)
callq 0x1d87e
movsd %xmm0, 0x30(%rsp)
leaq 0x130(%rsp), %rdi
xorl %esi, %esi
callq 0x1ae76
movq (%rax), %rax
movsd 0x30(%rsp), %xmm0
mulsd (%rax), %xmm0
movl $0x1, %esi
movl $0x1, %edx
leaq 0x78(%rsp), %r14
movq %r14, %rdi
callq 0x1a952
movq 0xb0(%rsp), %rdi
movq %r14, %rsi
callq 0x13a14
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq 0x10(%rsp), %rdi
callq 0x1ac7c
xorpd %xmm0, %xmm0
leaq 0x60(%rsp), %rdi
movq %rax, %rsi
callq 0x1a9ec
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq 0x10(%rsp), %rdi
callq 0x1ac7c
xorpd %xmm0, %xmm0
movq %r12, %rdi
movq %rax, %rsi
callq 0x1a9ec
cmpl $0x0, 0x20(%rsp)
je 0x9ea3
xorl %r14d, %r14d
movq 0x20(%rsp), %rbp
movq 0x8(%rsp), %rcx
movq 0x180(%rcx), %rax
movq (%rax,%r14), %r15
movq 0x308(%rcx), %rdi
addq 0x10(%rsp), %rdi
xorl %esi, %esi
callq 0x1ae76
movq 0x18(%rsp), %rcx
leaq (%r15,%rcx,8), %rsi
movq (%rax), %rax
movsd (%rax), %xmm0
movq %r13, %rdi
callq 0x1b7d0
leaq 0x48(%rsp), %rbx
movq %rbx, %rdi
leaq 0x60(%rsp), %r15
movq %r15, %rsi
movq %r13, %rdx
callq 0x1aef8
movq %r15, %rdi
movq %rbx, %rsi
callq 0x14f80
movq %rbx, %rdi
callq 0x1acb6
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax,%r14), %rdx
addq 0x10(%rsp), %rdx
movq %rbx, %rdi
movq %r12, %rsi
callq 0x1aef8
movq %r12, %rdi
movq %rbx, %rsi
callq 0x14f80
movq %rbx, %rdi
callq 0x1acb6
movq 0x28(%rsp), %rdi
xorl %esi, %esi
callq 0x1ae76
movq (%rax), %rax
movsd (%rax), %xmm0
movq %rbx, %rdi
movq %r12, %rsi
callq 0x1b714
movq %r12, %rdi
movq %rbx, %rsi
callq 0x14f80
movq %rbx, %rdi
callq 0x1acb6
movq %r13, %rdi
callq 0x1acb6
addq $0x18, %r14
decq %rbp
jne 0x9dc4
leaq 0x60(%rsp), %rdi
xorl %esi, %esi
movq 0x38(%rsp), %r14
movq %r14, %rdx
xorl %ecx, %ecx
movq 0x40(%rsp), %rbx
movq %rbx, %r8
callq 0x1d87e
movsd %xmm0, 0x18(%rsp)
leaq 0x118(%rsp), %rdi
xorl %esi, %esi
callq 0x1ae76
movq (%rax), %rax
movsd 0x18(%rsp), %xmm0
mulsd (%rax), %xmm0
movsd %xmm0, 0x18(%rsp)
movq %r12, %rdi
xorl %esi, %esi
movq %r14, %rdx
xorl %ecx, %ecx
movq %rbx, %r8
callq 0x1d87e
movsd %xmm0, 0x28(%rsp)
leaq 0x78(%rsp), %r14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1ae76
movq (%rax), %rax
movsd (%rax), %xmm0
movsd %xmm0, 0x30(%rsp)
movq %r14, %rdi
xorl %esi, %esi
callq 0x1ae76
movsd 0x38(%rsp), %xmm0
movsd 0x1d10b(%rip), %xmm2    # 0x27040
unpcklps %xmm2, %xmm0            # xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
movapd 0x1d110(%rip), %xmm3    # 0x27050
subpd %xmm3, %xmm0
movapd %xmm0, %xmm1
unpckhpd %xmm0, %xmm1            # xmm1 = xmm1[1],xmm0[1]
addsd %xmm0, %xmm1
movsd 0xa8(%rsp), %xmm0
divsd %xmm1, %xmm0
movsd 0x40(%rsp), %xmm1
unpcklps %xmm2, %xmm1            # xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
subpd %xmm3, %xmm1
movapd %xmm1, %xmm2
unpckhpd %xmm1, %xmm2            # xmm2 = xmm2[1],xmm1[1]
addsd %xmm1, %xmm2
divsd %xmm2, %xmm0
mulsd 0x28(%rsp), %xmm0
mulsd 0x30(%rsp), %xmm0
movsd 0x18(%rsp), %xmm1
subsd %xmm0, %xmm1
movq (%rax), %rax
movsd %xmm1, (%rax)
movq 0xa0(%rsp), %rdi
movq %r14, %rsi
callq 0x13a14
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq 0x10(%rsp), %rdi
callq 0x1ac7c
xorpd %xmm0, %xmm0
movq %r13, %rdi
movq %rax, %rsi
callq 0x1a9ec
movq 0x8(%rsp), %rax
movq 0x150(%rax), %rax
movq (%rax), %rdi
addq 0x10(%rsp), %rdi
callq 0x1ac7c
xorpd %xmm0, %xmm0
leaq 0x48(%rsp), %rdi
movq %rax, %rsi
callq 0x1a9ec
cmpl $0x0, 0x20(%rsp)
leaq 0x160(%rsp), %rbx
je 0xa0cb
xorl %r14d, %r14d
movq 0x20(%rsp), %r13
movq 0x8(%rsp), %rcx
movq 0x180(%rcx), %rax
movq 0x218(%rcx), %rcx
movq (%rax,%r14), %rsi
movq 0xc8(%rsp), %rax
addq %rax, %rsi
movq (%rcx,%r14), %rdx
addq %rax, %rdx
movq %rbx, %rdi
callq 0x1b2b2
leaq 0x148(%rsp), %rbp
movq %rbp, %rdi
leaq 0xd0(%rsp), %r15
movq %r15, %rsi
movq %rbx, %rdx
callq 0x1aef8
movq %r15, %rdi
movq %rbp, %rsi
callq 0x14f80
movq %rbp, %rdi
callq 0x1acb6
movq 0x8(%rsp), %rax
movq 0x180(%rax), %rax
movq (%rax,%r14), %rdx
addq 0x10(%rsp), %rdx
movq %rbp, %rdi
leaq 0x48(%rsp), %rbx
movq %rbx, %rsi
callq 0x1aef8
movq %rbx, %rdi
movq %rbp, %rsi
callq 0x14f80
movq %rbp, %rdi
callq 0x1acb6
leaq 0x160(%rsp), %rbx
movq %rbx, %rdi
callq 0x1acb6
addq $0x18, %r14
decq %r13
jne 0xa014
leaq 0xd0(%rsp), %r15
movq %r15, %rdi
xorl %esi, %esi
movq 0x38(%rsp), %r13
movq %r13, %rdx
xorl %ecx, %ecx
movq 0x40(%rsp), %rbx
movq %rbx, %r8
callq 0x1d87e
movsd %xmm0, 0x10(%rsp)
leaq 0x78(%rsp), %r14
leaq 0x48(%rsp), %rdi
xorl %esi, %esi
movq %r13, %rdx
xorl %ecx, %ecx
movq %rbx, %r8
callq 0x1d87e
movsd %xmm0, 0x18(%rsp)
movq %r14, %rdi
xorl %esi, %esi
callq 0x1ae76
movq (%rax), %rax
movsd 0x10(%rsp), %xmm0
movsd %xmm0, (%rax)
movq 0x98(%rsp), %rdi
movq %r14, %rsi
callq 0x13a14
movq %r14, %rdi
xorl %esi, %esi
callq 0x1ae76
movq (%rax), %rax
movsd 0x18(%rsp), %xmm0
movsd %xmm0, (%rax)
movq 0x90(%rsp), %rdi
movq %r14, %rsi
callq 0x13a14
leaq 0x48(%rsp), %rdi
callq 0x1acb6
movq %r15, %rdi
callq 0x1acb6
movq %r12, %rdi
callq 0x1acb6
leaq 0x60(%rsp), %rdi
callq 0x1acb6
movq %r14, %rdi
callq 0x1acb6
leaq 0xe8(%rsp), %r13
movq %r13, %rdi
callq 0x1acb6
leaq 0x118(%rsp), %rbx
movq %rbx, %rdi
callq 0x1acb6
leaq 0x130(%rsp), %r15
movq %r15, %rdi
callq 0x1acb6
movq 0xc0(%rsp), %r14
incq %r14
cmpq 0xb8(%rsp), %r14
jne 0x9ac8
addq $0x178, %rsp            # imm = 0x178
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
jmp 0xa2e5
jmp 0xa2e5
jmp 0xa2e0
jmp 0xa205
jmp 0xa225
jmp 0xa225
jmp 0xa2e5
jmp 0xa205
movq %rax, %rbx
jmp 0xa2cb
jmp 0xa225
jmp 0xa27b
jmp 0xa27b
jmp 0xa27b
jmp 0xa280
jmp 0xa27b
jmp 0xa2e5
jmp 0xa2e0
jmp 0xa274
movq %rax, %rbx
jmp 0xa2be
jmp 0xa261
jmp 0xa261
jmp 0xa235
jmp 0xa247
movq %rax, %rbx
leaq 0x148(%rsp), %rdi
callq 0x1acb6
jmp 0xa24a
movq %rax, %rbx
leaq 0x160(%rsp), %rdi
callq 0x1acb6
jmp 0xa264
jmp 0xa280
jmp 0xa261
jmp 0xa261
jmp 0xa280
movq %rax, %rbx
leaq 0x48(%rsp), %rdi
callq 0x1acb6
jmp 0xa283
jmp 0xa274
jmp 0xa27b
movq %rax, %rbx
jmp 0xa29d
jmp 0xa27b
movq %rax, %rbx
jmp 0xa290
movq %rax, %rbx
leaq 0xd0(%rsp), %rdi
callq 0x1acb6
leaq 0x100(%rsp), %rdi
callq 0x1acb6
leaq 0x60(%rsp), %rdi
callq 0x1acb6
leaq 0x78(%rsp), %rdi
callq 0x1acb6
leaq 0xe8(%rsp), %rdi
callq 0x1acb6
leaq 0x118(%rsp), %rdi
callq 0x1acb6
leaq 0x130(%rsp), %rdi
callq 0x1acb6
movq %rbx, %rdi
callq 0x43e0
movq %rax, %rbx
jmp 0xa2a7
movq %rax, %rbx
jmp 0xa2b1