// File: /workspace/github_cpp_makefile_star2_O1/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::AdamOptimizer(Matrix const&, int)
Matrix ConstructLayer::AdamOptimizer(const Matrix &dw, int kernel_index)
{
    //cout<<"----------------------------------------------------------"<<endl;
    // Adam optimizer, refer to: https://arxiv.org/abs/1412.6980
    // sanity check
    assert(Momentum_1st_order.size() == Momentum_2nd_order.size());
    if(Momentum_1st_order.size() == 0)
    {
        // initiaize with 0's
        if(__type == LayerType::fullyConnected || __type == LayerType::output || __type == LayerType::input)
        {
            assert(kernel_index == 0);
            Momentum_1st_order.resize(1, Matrix(dw.Dimension(), 0));
            Momentum_2nd_order.resize(1, Matrix(dw.Dimension(), 0));
        }
        else if(__type == LayerType::cnn || __type == LayerType::pooling)
        {
            assert(__n_kernels_cnn > 0);
            Momentum_1st_order.resize(__n_kernels_cnn, Matrix(dw.Dimension(), 0));
            Momentum_2nd_order.resize(__n_kernels_cnn, Matrix(dw.Dimension(), 0));
        }
    }

    // g_t
    Matrix g_t = dw;
    //cout<<"g_t: "<<endl<<g_t<<endl;

    // these parameters are from paper: https://arxiv.org/abs/1412.6980
    //     according to the ref: learning rate is best to be 0.001
    double beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8;
    // get m_t
    Matrix & m_t_1 = Momentum_1st_order[kernel_index];
    //cout<<"m_t_1: "<<endl<<m_t_1<<endl;
    //cout<<" 1 - beta1: "<<1.-beta1<<endl;
    Matrix m_tmp = g_t * (1. - beta1);
    //cout<<"m_tmp: "<<endl<<m_tmp<<endl;
    Matrix m_t = (m_t_1 * beta1) + m_tmp;
    //Matrix debug_tmp = m_t_1 * beta1;
    //cout<<"m_t_1*beta1: "<<endl<<(debug_tmp)<<endl;
    //cout<<"m_t: = (m_t_1 * beta1) + g_t * (1 - beta1): "<<endl<<m_t<<endl;
    // save m_t
    Momentum_1st_order[kernel_index] = m_t; // update 1st order momentum
    // get v_t
    Matrix & v_t_1 = Momentum_2nd_order[kernel_index];
    //cout<<"v_t_1: "<<endl<<v_t_1<<endl;
    m_tmp = (g_t^g_t) * (1. - beta2);
    //cout<<"g_t^2 * (1. - beta2(0.999))"<<endl<<m_tmp<<endl;
    Matrix v_t = (v_t_1 *beta2) + m_tmp;
    //debug_tmp = v_t_1 *beta2;
    //cout<<"v_t_1 * beta2:"<<endl<<debug_tmp<<endl;
    //cout<<"vt = v_t_1*beta2 + g_t^2*(1-beta2): "<<endl<<v_t<<endl;
    // save v_t
    Momentum_2nd_order[kernel_index] = v_t; // update 2nd order momentum

    // get bias corrected m_t and v-t
    __beta1_to_power_t *= beta1;
    __beta2_to_power_t *= beta2;
    Matrix bias_corrected_1st_order_moment = m_t / (1. - __beta1_to_power_t);
    //cout<<"bias corrected m_t: /"<<(1. - __beta1_to_power_t)<<endl<<bias_corrected_1st_order_moment<<endl;
    Matrix bias_corrected_2nd_order_moment = v_t / (1. - __beta2_to_power_t);
    //cout<<"bias corrected v_t: /"<<(1. - __beta2_to_power_t)<<endl<<bias_corrected_2nd_order_moment<<endl;

    // get d_theta_t 
    Matrix m_epsilon(g_t.Dimension(), epsilon);
    bias_corrected_2nd_order_moment(__AdamSquareRoot);
    //cout<<"sqrt(^v_t): "<<endl<<bias_corrected_2nd_order_moment<<endl;
    Matrix square_root_of_vt = bias_corrected_2nd_order_moment;
    Matrix denominator = square_root_of_vt + m_epsilon;
    //cout<<"m_epsilon: "<<endl<<m_epsilon<<endl;
    //cout<<"sqrt(^v_t): + epsilon"<<endl<<denominator<<endl;
    Matrix ratio = bias_corrected_1st_order_moment / denominator;
    //cout<<"^m_t/(sqrt(^v_t) + epsilon):"<<endl<<ratio<<endl;

    // according to the reference, alpha best to be 0.001, however, 
    // based on my test, it is too slow, alpha = 0.06 works best
    double alpha = __learningRate;
    Matrix dw_Adam = ratio * alpha;
    //Matrix dw_Adam = ratio * __learningRate;
    //cout<<"learning rate: "<<__learningRate<<endl;
    //cout<<"dw_Adam: "<<endl<<dw_Adam<<endl;
    //getchar();

    return dw_Adam;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x108, %rsp            # imm = 0x108
movl %ecx, %ebp
movq %rdx, %r13
movq %rsi, %r14
movq %rdi, 0x58(%rsp)
movq 0x448(%rsi), %rax
movq 0x450(%rsi), %rcx
movq %rcx, %rdx
subq %rax, %rdx
movq 0x468(%rsi), %rsi
subq 0x460(%r14), %rsi
cmpq %rsi, %rdx
jne 0x10e6b
leaq 0x448(%r14), %rdx
movq %rdx, 0x20(%rsp)
leaq 0x460(%r14), %r15
cmpq %rax, %rcx
jne 0x10bc5
movl 0xc(%r14), %eax
cmpl $0x4, %eax
ja 0x10bc5
movl $0x19, %ecx
btl %eax, %ecx
jae 0x10b4a
testl %ebp, %ebp
jne 0x10e8a
movq %r13, %rdi
callq 0x1ac7c
leaq 0x8(%rsp), %rbx
xorpd %xmm0, %xmm0
movq %rbx, %rdi
movq %rax, %rsi
callq 0x1a9ec
movl $0x1, %esi
movq 0x20(%rsp), %rdi
movq %rbx, %rdx
callq 0x1460e
leaq 0x8(%rsp), %rdi
callq 0x1acb6
movq %r13, %rdi
callq 0x1ac7c
leaq 0x8(%rsp), %rbx
xorpd %xmm0, %xmm0
movq %rbx, %rdi
movq %rax, %rsi
callq 0x1a9ec
movl $0x1, %esi
movq %r15, %rdi
movq %rbx, %rdx
callq 0x1460e
jmp 0x10bbb
movq 0x18(%r14), %rbx
testq %rbx, %rbx
je 0x10ea9
movq %r13, %rdi
callq 0x1ac7c
leaq 0x8(%rsp), %r12
xorpd %xmm0, %xmm0
movq %r12, %rdi
movq %rax, %rsi
callq 0x1a9ec
movq 0x20(%rsp), %rdi
movq %rbx, %rsi
movq %r12, %rdx
callq 0x1460e
leaq 0x8(%rsp), %rdi
callq 0x1acb6
movq 0x18(%r14), %rbx
movq %r13, %rdi
callq 0x1ac7c
leaq 0x8(%rsp), %r12
xorpd %xmm0, %xmm0
movq %r12, %rdi
movq %rax, %rsi
callq 0x1a9ec
movq %r15, %rdi
movq %rbx, %rsi
movq %r12, %rdx
callq 0x1460e
leaq 0x8(%rsp), %rdi
callq 0x1acb6
movq %r15, %r12
movl %ebp, %r15d
leaq 0x8(%rsp), %rbx
movq %rbx, %rdi
movq %r13, %rsi
callq 0x1aafe
movq 0x20(%rsp), %rax
movq (%rax), %rbp
leaq 0x60(%rsp), %rdi
movsd 0x169e0(%rip), %xmm0    # 0x275d0
movq %rbx, %rsi
callq 0x1b7d0
movslq %r15d, %rax
leaq (%rax,%rax,2), %r13
leaq (,%r13,8), %rsi
addq %rbp, %rsi
leaq 0x28(%rsp), %rdi
movsd 0x169c1(%rip), %xmm0    # 0x275d8
callq 0x1b7d0
leaq 0x90(%rsp), %rdi
leaq 0x28(%rsp), %rsi
leaq 0x60(%rsp), %rdx
callq 0x1aef8
movq %r12, %rbx
leaq 0x28(%rsp), %rdi
callq 0x1acb6
shlq $0x3, %r13
movq 0x20(%rsp), %rax
movq (%rax), %rdi
addq %r13, %rdi
leaq 0x90(%rsp), %rsi
callq 0x14f80
movq (%rbx), %r12
addq %r13, %r12
leaq 0x40(%rsp), %rdi
leaq 0x8(%rsp), %rdx
movq %rdx, %rsi
callq 0x1b2b2
leaq 0x28(%rsp), %rdi
leaq 0x40(%rsp), %rsi
movsd 0x1695a(%rip), %xmm0    # 0x275e0
callq 0x1b7d0
leaq 0x60(%rsp), %rdi
leaq 0x28(%rsp), %rsi
callq 0x14f80
leaq 0x28(%rsp), %rdi
callq 0x1acb6
leaq 0x40(%rsp), %rdi
callq 0x1acb6
leaq 0x40(%rsp), %rdi
movsd 0x1692d(%rip), %xmm0    # 0x275e8
movq %r12, %rsi
callq 0x1b7d0
leaq 0x28(%rsp), %rdi
leaq 0x40(%rsp), %rsi
leaq 0x60(%rsp), %rdx
callq 0x1aef8
leaq 0x40(%rsp), %rdi
callq 0x1acb6
addq (%rbx), %r13
leaq 0x28(%rsp), %rsi
movq %r13, %rdi
callq 0x14f80
movupd 0x478(%r14), %xmm1
mulpd 0x1636e(%rip), %xmm1    # 0x27070
movupd %xmm1, 0x478(%r14)
movsd 0x1686d(%rip), %xmm0    # 0x27580
subsd %xmm1, %xmm0
leaq 0x40(%rsp), %rdi
leaq 0x90(%rsp), %rsi
callq 0x1b88c
movsd 0x1684f(%rip), %xmm0    # 0x27580
subsd 0x480(%r14), %xmm0
leaq 0x78(%rsp), %rdi
leaq 0x28(%rsp), %rsi
callq 0x1b88c
leaq 0x8(%rsp), %rdi
callq 0x1ac7c
leaq 0xf0(%rsp), %rdi
movsd 0x16835(%rip), %xmm0    # 0x27598
movq %rax, %rsi
callq 0x1a9ec
leaq 0x265(%rip), %rsi       # 0x10fd7
leaq 0x78(%rsp), %rdi
callq 0x1bfe6
leaq 0xd8(%rsp), %rdi
leaq 0x78(%rsp), %rsi
callq 0x1aafe
leaq 0xc0(%rsp), %rdi
leaq 0xd8(%rsp), %rsi
leaq 0xf0(%rsp), %rdx
callq 0x1aef8
leaq 0xa8(%rsp), %rdi
leaq 0x40(%rsp), %rsi
leaq 0xc0(%rsp), %rdx
callq 0x1b3b4
movsd 0xc0(%r14), %xmm0
leaq 0xa8(%rsp), %rsi
movq 0x58(%rsp), %rbx
movq %rbx, %rdi
callq 0x1b7d0
leaq 0xa8(%rsp), %rdi
callq 0x1acb6
leaq 0xc0(%rsp), %rdi
callq 0x1acb6
leaq 0xd8(%rsp), %rdi
callq 0x1acb6
leaq 0xf0(%rsp), %rdi
callq 0x1acb6
leaq 0x78(%rsp), %rdi
callq 0x1acb6
leaq 0x40(%rsp), %rdi
callq 0x1acb6
leaq 0x28(%rsp), %rdi
callq 0x1acb6
leaq 0x90(%rsp), %rdi
callq 0x1acb6
leaq 0x60(%rsp), %rdi
callq 0x1acb6
leaq 0x8(%rsp), %rdi
callq 0x1acb6
movq %rbx, %rax
addq $0x108, %rsp            # imm = 0x108
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0x18111(%rip), %rdi     # 0x28f83
leaq 0x168d1(%rip), %rsi     # 0x2774a
leaq 0x1813a(%rip), %rcx     # 0x28fba
movl $0xe99, %edx            # imm = 0xE99
callq 0x4160
leaq 0x18163(%rip), %rdi     # 0x28ff4
leaq 0x168b2(%rip), %rsi     # 0x2774a
leaq 0x1811b(%rip), %rcx     # 0x28fba
movl $0xe9f, %edx            # imm = 0xE9F
callq 0x4160
leaq 0x18156(%rip), %rdi     # 0x29006
leaq 0x16893(%rip), %rsi     # 0x2774a
leaq 0x180fc(%rip), %rcx     # 0x28fba
movl $0xea5, %edx            # imm = 0xEA5
callq 0x4160
jmp 0x10f65
jmp 0x10f65
jmp 0x10f65
jmp 0x10f65
movq %rax, %rbx
leaq 0xa8(%rsp), %rdi
callq 0x1acb6
jmp 0x10ef1
movq %rax, %rbx
leaq 0xc0(%rsp), %rdi
callq 0x1acb6
jmp 0x10f03
movq %rax, %rbx
leaq 0xd8(%rsp), %rdi
callq 0x1acb6
jmp 0x10f17
jmp 0x10f14
movq %rax, %rbx
leaq 0xf0(%rsp), %rdi
callq 0x1acb6
jmp 0x10f6d
movq %rax, %rbx
jmp 0x10f77
jmp 0x10f2d
movq %rax, %rbx
jmp 0x10f81
jmp 0x10f45
jmp 0x10f51
movq %rax, %rbx
leaq 0x28(%rsp), %rdi
callq 0x1acb6
jmp 0x10f48
movq %rax, %rbx
leaq 0x40(%rsp), %rdi
jmp 0x10f86
jmp 0x10f51
movq %rax, %rbx
jmp 0x10f8b
movq %rax, %rbx
leaq 0x28(%rsp), %rdi
jmp 0x10f93
movq %rax, %rbx
jmp 0x10f98
movq %rax, %rbx
jmp 0x10fa2
movq %rax, %rbx
leaq 0x78(%rsp), %rdi
callq 0x1acb6
leaq 0x40(%rsp), %rdi
callq 0x1acb6
leaq 0x28(%rsp), %rdi
callq 0x1acb6
leaq 0x90(%rsp), %rdi
callq 0x1acb6
leaq 0x60(%rsp), %rdi
callq 0x1acb6
leaq 0x8(%rsp), %rdi
callq 0x1acb6
movq %rbx, %rdi
callq 0x43e0