// File: /workspace/github_cpp_makefile_star2_O3/xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::AdamOptimizer(Matrix const&, int)
Matrix ConstructLayer::AdamOptimizer(const Matrix &dw, int kernel_index)
{
    //cout<<"----------------------------------------------------------"<<endl;
    // Adam optimizer, refer to: https://arxiv.org/abs/1412.6980
    // sanity check
    assert(Momentum_1st_order.size() == Momentum_2nd_order.size());
    if(Momentum_1st_order.size() == 0)
    {
        // initiaize with 0's
        if(__type == LayerType::fullyConnected || __type == LayerType::output || __type == LayerType::input)
        {
            assert(kernel_index == 0);
            Momentum_1st_order.resize(1, Matrix(dw.Dimension(), 0));
            Momentum_2nd_order.resize(1, Matrix(dw.Dimension(), 0));
        }
        else if(__type == LayerType::cnn || __type == LayerType::pooling)
        {
            assert(__n_kernels_cnn > 0);
            Momentum_1st_order.resize(__n_kernels_cnn, Matrix(dw.Dimension(), 0));
            Momentum_2nd_order.resize(__n_kernels_cnn, Matrix(dw.Dimension(), 0));
        }
    }

    // g_t
    Matrix g_t = dw;
    //cout<<"g_t: "<<endl<<g_t<<endl;

    // these parameters are from paper: https://arxiv.org/abs/1412.6980
    //     according to the ref: learning rate is best to be 0.001
    double beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8;
    // get m_t
    Matrix & m_t_1 = Momentum_1st_order[kernel_index];
    //cout<<"m_t_1: "<<endl<<m_t_1<<endl;
    //cout<<" 1 - beta1: "<<1.-beta1<<endl;
    Matrix m_tmp = g_t * (1. - beta1);
    //cout<<"m_tmp: "<<endl<<m_tmp<<endl;
    Matrix m_t = (m_t_1 * beta1) + m_tmp;
    //Matrix debug_tmp = m_t_1 * beta1;
    //cout<<"m_t_1*beta1: "<<endl<<(debug_tmp)<<endl;
    //cout<<"m_t: = (m_t_1 * beta1) + g_t * (1 - beta1): "<<endl<<m_t<<endl;
    // save m_t
    Momentum_1st_order[kernel_index] = m_t; // update 1st order momentum
    // get v_t
    Matrix & v_t_1 = Momentum_2nd_order[kernel_index];
    //cout<<"v_t_1: "<<endl<<v_t_1<<endl;
    m_tmp = (g_t^g_t) * (1. - beta2);
    //cout<<"g_t^2 * (1. - beta2(0.999))"<<endl<<m_tmp<<endl;
    Matrix v_t = (v_t_1 *beta2) + m_tmp;
    //debug_tmp = v_t_1 *beta2;
    //cout<<"v_t_1 * beta2:"<<endl<<debug_tmp<<endl;
    //cout<<"vt = v_t_1*beta2 + g_t^2*(1-beta2): "<<endl<<v_t<<endl;
    // save v_t
    Momentum_2nd_order[kernel_index] = v_t; // update 2nd order momentum

    // get bias corrected m_t and v-t
    __beta1_to_power_t *= beta1;
    __beta2_to_power_t *= beta2;
    Matrix bias_corrected_1st_order_moment = m_t / (1. - __beta1_to_power_t);
    //cout<<"bias corrected m_t: /"<<(1. - __beta1_to_power_t)<<endl<<bias_corrected_1st_order_moment<<endl;
    Matrix bias_corrected_2nd_order_moment = v_t / (1. - __beta2_to_power_t);
    //cout<<"bias corrected v_t: /"<<(1. - __beta2_to_power_t)<<endl<<bias_corrected_2nd_order_moment<<endl;

    // get d_theta_t 
    Matrix m_epsilon(g_t.Dimension(), epsilon);
    bias_corrected_2nd_order_moment(__AdamSquareRoot);
    //cout<<"sqrt(^v_t): "<<endl<<bias_corrected_2nd_order_moment<<endl;
    Matrix square_root_of_vt = bias_corrected_2nd_order_moment;
    Matrix denominator = square_root_of_vt + m_epsilon;
    //cout<<"m_epsilon: "<<endl<<m_epsilon<<endl;
    //cout<<"sqrt(^v_t): + epsilon"<<endl<<denominator<<endl;
    Matrix ratio = bias_corrected_1st_order_moment / denominator;
    //cout<<"^m_t/(sqrt(^v_t) + epsilon):"<<endl<<ratio<<endl;

    // according to the reference, alpha best to be 0.001, however, 
    // based on my test, it is too slow, alpha = 0.06 works best
    double alpha = __learningRate;
    Matrix dw_Adam = ratio * alpha;
    //Matrix dw_Adam = ratio * __learningRate;
    //cout<<"learning rate: "<<__learningRate<<endl;
    //cout<<"dw_Adam: "<<endl<<dw_Adam<<endl;
    //getchar();

    return dw_Adam;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x108, %rsp            # imm = 0x108
movl %ecx, %ebp
movq %rdx, %r13
movq %rsi, %r14
movq %rdi, 0x58(%rsp)
movq 0x448(%rsi), %rax
movq 0x450(%rsi), %rcx
movq %rcx, %rdx
subq %rax, %rdx
movq 0x468(%rsi), %rsi
subq 0x460(%r14), %rsi
cmpq %rsi, %rdx
jne 0x10bdd
leaq 0x448(%r14), %rdx
movq %rdx, 0x20(%rsp)
leaq 0x460(%r14), %r15
cmpq %rax, %rcx
jne 0x10937
movl 0xc(%r14), %eax
cmpl $0x4, %eax
ja 0x10937
movl $0x19, %ecx
btl %eax, %ecx
jae 0x108bc
testl %ebp, %ebp
jne 0x10bfc
movq %r13, %rdi
callq 0x1abb8
leaq 0x8(%rsp), %rbx
xorpd %xmm0, %xmm0
movq %rbx, %rdi
movq %rax, %rsi
callq 0x1a92c
movl $0x1, %esi
movq 0x20(%rsp), %rdi
movq %rbx, %rdx
callq 0x144ce
leaq 0x8(%rsp), %rdi
callq 0x1abf2
movq %r13, %rdi
callq 0x1abb8
leaq 0x8(%rsp), %rbx
xorpd %xmm0, %xmm0
movq %rbx, %rdi
movq %rax, %rsi
callq 0x1a92c
movl $0x1, %esi
movq %r15, %rdi
movq %rbx, %rdx
callq 0x144ce
jmp 0x1092d
movq 0x18(%r14), %rbx
testq %rbx, %rbx
je 0x10c1b
movq %r13, %rdi
callq 0x1abb8
leaq 0x8(%rsp), %r12
xorpd %xmm0, %xmm0
movq %r12, %rdi
movq %rax, %rsi
callq 0x1a92c
movq 0x20(%rsp), %rdi
movq %rbx, %rsi
movq %r12, %rdx
callq 0x144ce
leaq 0x8(%rsp), %rdi
callq 0x1abf2
movq 0x18(%r14), %rbx
movq %r13, %rdi
callq 0x1abb8
leaq 0x8(%rsp), %r12
xorpd %xmm0, %xmm0
movq %r12, %rdi
movq %rax, %rsi
callq 0x1a92c
movq %r15, %rdi
movq %rbx, %rsi
movq %r12, %rdx
callq 0x144ce
leaq 0x8(%rsp), %rdi
callq 0x1abf2
movq %r15, %r12
movl %ebp, %r15d
leaq 0x8(%rsp), %rbx
movq %rbx, %rdi
movq %r13, %rsi
callq 0x1aa3e
movq 0x20(%rsp), %rax
movq (%rax), %rbp
leaq 0x60(%rsp), %rdi
movsd 0x16c6e(%rip), %xmm0    # 0x275d0
movq %rbx, %rsi
callq 0x1b70c
movslq %r15d, %rax
leaq (%rax,%rax,2), %r13
leaq (,%r13,8), %rsi
addq %rbp, %rsi
leaq 0x28(%rsp), %rdi
movsd 0x16c4f(%rip), %xmm0    # 0x275d8
callq 0x1b70c
leaq 0x90(%rsp), %rdi
leaq 0x28(%rsp), %rsi
leaq 0x60(%rsp), %rdx
callq 0x1ae34
movq %r12, %rbx
leaq 0x28(%rsp), %rdi
callq 0x1abf2
shlq $0x3, %r13
movq 0x20(%rsp), %rax
movq (%rax), %rdi
addq %r13, %rdi
leaq 0x90(%rsp), %rsi
callq 0x14e2c
movq (%rbx), %r12
addq %r13, %r12
leaq 0x40(%rsp), %rdi
leaq 0x8(%rsp), %rdx
movq %rdx, %rsi
callq 0x1b1ee
leaq 0x28(%rsp), %rdi
leaq 0x40(%rsp), %rsi
movsd 0x16be8(%rip), %xmm0    # 0x275e0
callq 0x1b70c
leaq 0x60(%rsp), %rdi
leaq 0x28(%rsp), %rsi
callq 0x14e2c
leaq 0x28(%rsp), %rdi
callq 0x1abf2
leaq 0x40(%rsp), %rdi
callq 0x1abf2
leaq 0x40(%rsp), %rdi
movsd 0x16bbb(%rip), %xmm0    # 0x275e8
movq %r12, %rsi
callq 0x1b70c
leaq 0x28(%rsp), %rdi
leaq 0x40(%rsp), %rsi
leaq 0x60(%rsp), %rdx
callq 0x1ae34
leaq 0x40(%rsp), %rdi
callq 0x1abf2
addq (%rbx), %r13
leaq 0x28(%rsp), %rsi
movq %r13, %rdi
callq 0x14e2c
movupd 0x478(%r14), %xmm1
mulpd 0x165fc(%rip), %xmm1    # 0x27070
movupd %xmm1, 0x478(%r14)
movsd 0x16afb(%rip), %xmm0    # 0x27580
subsd %xmm1, %xmm0
leaq 0x40(%rsp), %rdi
leaq 0x90(%rsp), %rsi
callq 0x1b7c8
movsd 0x16add(%rip), %xmm0    # 0x27580
subsd 0x480(%r14), %xmm0
leaq 0x78(%rsp), %rdi
leaq 0x28(%rsp), %rsi
callq 0x1b7c8
leaq 0x8(%rsp), %rdi
callq 0x1abb8
leaq 0xf0(%rsp), %rdi
movsd 0x16ac3(%rip), %xmm0    # 0x27598
movq %rax, %rsi
callq 0x1a92c
leaq 0x3a4(%rip), %rsi       # 0x10e88
leaq 0x78(%rsp), %rdi
callq 0x1bf18
leaq 0xd8(%rsp), %rdi
leaq 0x78(%rsp), %rsi
callq 0x1aa3e
leaq 0xc0(%rsp), %rdi
leaq 0xd8(%rsp), %rsi
leaq 0xf0(%rsp), %rdx
callq 0x1ae34
leaq 0xa8(%rsp), %rdi
leaq 0x40(%rsp), %rsi
leaq 0xc0(%rsp), %rdx
callq 0x1b2f0
movsd 0xc0(%r14), %xmm0
leaq 0xa8(%rsp), %rsi
movq 0x58(%rsp), %rbx
movq %rbx, %rdi
callq 0x1b70c
leaq 0xa8(%rsp), %rdi
callq 0x1abf2
leaq 0xc0(%rsp), %rdi
callq 0x1abf2
leaq 0xd8(%rsp), %rdi
callq 0x1abf2
leaq 0xf0(%rsp), %rdi
callq 0x1abf2
leaq 0x78(%rsp), %rdi
callq 0x1abf2
leaq 0x40(%rsp), %rdi
callq 0x1abf2
leaq 0x28(%rsp), %rdi
callq 0x1abf2
leaq 0x90(%rsp), %rdi
callq 0x1abf2
leaq 0x60(%rsp), %rdi
callq 0x1abf2
leaq 0x8(%rsp), %rdi
callq 0x1abf2
movq %rbx, %rax
addq $0x108, %rsp            # imm = 0x108
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0x18360(%rip), %rdi     # 0x28f44
leaq 0x16b5f(%rip), %rsi     # 0x2774a
leaq 0x18389(%rip), %rcx     # 0x28f7b
movl $0xe99, %edx            # imm = 0xE99
callq 0x4160
leaq 0x183b2(%rip), %rdi     # 0x28fb5
leaq 0x16b40(%rip), %rsi     # 0x2774a
leaq 0x1836a(%rip), %rcx     # 0x28f7b
movl $0xe9f, %edx            # imm = 0xE9F
callq 0x4160
leaq 0x183a5(%rip), %rdi     # 0x28fc7
leaq 0x16b21(%rip), %rsi     # 0x2774a
leaq 0x1834b(%rip), %rcx     # 0x28f7b
movl $0xea5, %edx            # imm = 0xEA5
callq 0x4160
jmp 0x10cd7
jmp 0x10cd7
jmp 0x10cd7
jmp 0x10cd7
movq %rax, %rbx
leaq 0xa8(%rsp), %rdi
callq 0x1abf2
jmp 0x10c63
movq %rax, %rbx
leaq 0xc0(%rsp), %rdi
callq 0x1abf2
jmp 0x10c75
movq %rax, %rbx
leaq 0xd8(%rsp), %rdi
callq 0x1abf2
jmp 0x10c89
jmp 0x10c86
movq %rax, %rbx
leaq 0xf0(%rsp), %rdi
callq 0x1abf2
jmp 0x10cdf
movq %rax, %rbx
jmp 0x10ce9
jmp 0x10c9f
movq %rax, %rbx
jmp 0x10cf3
jmp 0x10cb7
jmp 0x10cc3
movq %rax, %rbx
leaq 0x28(%rsp), %rdi
callq 0x1abf2
jmp 0x10cba
movq %rax, %rbx
leaq 0x40(%rsp), %rdi
jmp 0x10cf8
jmp 0x10cc3
movq %rax, %rbx
jmp 0x10cfd
movq %rax, %rbx
leaq 0x28(%rsp), %rdi
jmp 0x10d05
movq %rax, %rbx
jmp 0x10d0a
movq %rax, %rbx
jmp 0x10d14
movq %rax, %rbx
leaq 0x78(%rsp), %rdi
callq 0x1abf2
leaq 0x40(%rsp), %rdi
callq 0x1abf2
leaq 0x28(%rsp), %rdi
callq 0x1abf2
leaq 0x90(%rsp), %rdi
callq 0x1abf2
leaq 0x60(%rsp), %rdi
callq 0x1abf2
leaq 0x8(%rsp), %rdi
callq 0x1abf2
movq %rbx, %rdi
callq 0x43e0