// File: xbai0624[P]dnn/src/ConstructLayer.cpp
// Function: ConstructLayer::AdamOptimizer(Matrix const&, int)
Matrix ConstructLayer::AdamOptimizer(const Matrix &dw, int kernel_index)
{
    //cout<<"----------------------------------------------------------"<<endl;
    // Adam optimizer, refer to: https://arxiv.org/abs/1412.6980
    // sanity check
    assert(Momentum_1st_order.size() == Momentum_2nd_order.size());
    if(Momentum_1st_order.size() == 0)
    {
        // initiaize with 0's
        if(__type == LayerType::fullyConnected || __type == LayerType::output || __type == LayerType::input)
        {
            assert(kernel_index == 0);
            Momentum_1st_order.resize(1, Matrix(dw.Dimension(), 0));
            Momentum_2nd_order.resize(1, Matrix(dw.Dimension(), 0));
        }
        else if(__type == LayerType::cnn || __type == LayerType::pooling)
        {
            assert(__n_kernels_cnn > 0);
            Momentum_1st_order.resize(__n_kernels_cnn, Matrix(dw.Dimension(), 0));
            Momentum_2nd_order.resize(__n_kernels_cnn, Matrix(dw.Dimension(), 0));
        }
    }

    // g_t
    Matrix g_t = dw;
    //cout<<"g_t: "<<endl<<g_t<<endl;

    // these parameters are from paper: https://arxiv.org/abs/1412.6980
    //     according to the ref: learning rate is best to be 0.001
    double beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8;
    // get m_t
    Matrix & m_t_1 = Momentum_1st_order[kernel_index];
    //cout<<"m_t_1: "<<endl<<m_t_1<<endl;
    //cout<<" 1 - beta1: "<<1.-beta1<<endl;
    Matrix m_tmp = g_t * (1. - beta1);
    //cout<<"m_tmp: "<<endl<<m_tmp<<endl;
    Matrix m_t = (m_t_1 * beta1) + m_tmp;
    //Matrix debug_tmp = m_t_1 * beta1;
    //cout<<"m_t_1*beta1: "<<endl<<(debug_tmp)<<endl;
    //cout<<"m_t: = (m_t_1 * beta1) + g_t * (1 - beta1): "<<endl<<m_t<<endl;
    // save m_t
    Momentum_1st_order[kernel_index] = m_t; // update 1st order momentum
    // get v_t
    Matrix & v_t_1 = Momentum_2nd_order[kernel_index];
    //cout<<"v_t_1: "<<endl<<v_t_1<<endl;
    m_tmp = (g_t^g_t) * (1. - beta2);
    //cout<<"g_t^2 * (1. - beta2(0.999))"<<endl<<m_tmp<<endl;
    Matrix v_t = (v_t_1 *beta2) + m_tmp;
    //debug_tmp = v_t_1 *beta2;
    //cout<<"v_t_1 * beta2:"<<endl<<debug_tmp<<endl;
    //cout<<"vt = v_t_1*beta2 + g_t^2*(1-beta2): "<<endl<<v_t<<endl;
    // save v_t
    Momentum_2nd_order[kernel_index] = v_t; // update 2nd order momentum

    // get bias corrected m_t and v-t
    __beta1_to_power_t *= beta1;
    __beta2_to_power_t *= beta2;
    Matrix bias_corrected_1st_order_moment = m_t / (1. - __beta1_to_power_t);
    //cout<<"bias corrected m_t: /"<<(1. - __beta1_to_power_t)<<endl<<bias_corrected_1st_order_moment<<endl;
    Matrix bias_corrected_2nd_order_moment = v_t / (1. - __beta2_to_power_t);
    //cout<<"bias corrected v_t: /"<<(1. - __beta2_to_power_t)<<endl<<bias_corrected_2nd_order_moment<<endl;

    // get d_theta_t 
    Matrix m_epsilon(g_t.Dimension(), epsilon);
    bias_corrected_2nd_order_moment(__AdamSquareRoot);
    //cout<<"sqrt(^v_t): "<<endl<<bias_corrected_2nd_order_moment<<endl;
    Matrix square_root_of_vt = bias_corrected_2nd_order_moment;
    Matrix denominator = square_root_of_vt + m_epsilon;
    //cout<<"m_epsilon: "<<endl<<m_epsilon<<endl;
    //cout<<"sqrt(^v_t): + epsilon"<<endl<<denominator<<endl;
    Matrix ratio = bias_corrected_1st_order_moment / denominator;
    //cout<<"^m_t/(sqrt(^v_t) + epsilon):"<<endl<<ratio<<endl;

    // according to the reference, alpha best to be 0.001, however, 
    // based on my test, it is too slow, alpha = 0.06 works best
    double alpha = __learningRate;
    Matrix dw_Adam = ratio * alpha;
    //Matrix dw_Adam = ratio * __learningRate;
    //cout<<"learning rate: "<<__learningRate<<endl;
    //cout<<"dw_Adam: "<<endl<<dw_Adam<<endl;
    //getchar();

    return dw_Adam;
}

// Assembly:
subq $0x2f8, %rsp            # imm = 0x2F8
movq %rdi, 0x70(%rsp)
movq %rdi, %rax
movq %rax, 0x78(%rsp)
movq %rdi, 0x2f0(%rsp)
movq %rsi, 0x2e8(%rsp)
movq %rdx, 0x2e0(%rsp)
movl %ecx, 0x2dc(%rsp)
movq 0x2e8(%rsp), %rdi
movq %rdi, 0x80(%rsp)
addq $0x448, %rdi            # imm = 0x448
callq 0x1f9c0
movq 0x80(%rsp), %rdi
movq %rax, 0x88(%rsp)
addq $0x460, %rdi            # imm = 0x460
callq 0x1f9c0
movq %rax, %rcx
movq 0x88(%rsp), %rax
cmpq %rcx, %rax
jne 0x193bd
jmp 0x193dc
leaq 0x3a647(%rip), %rdi     # 0x53a0b
leaq 0x38e07(%rip), %rsi     # 0x521d2
movl $0xe99, %edx            # imm = 0xE99
leaq 0x3a66b(%rip), %rcx     # 0x53a42
callq 0x4170
movq 0x80(%rsp), %rdi
addq $0x448, %rdi            # imm = 0x448
callq 0x1f9c0
cmpq $0x0, %rax
jne 0x19740
movq 0x80(%rsp), %rax
cmpl $0x0, 0xc(%rax)
je 0x19428
movq 0x80(%rsp), %rax
cmpl $0x4, 0xc(%rax)
je 0x19428
movq 0x80(%rsp), %rax
cmpl $0x3, 0xc(%rax)
jne 0x19594
cmpl $0x0, 0x2dc(%rsp)
jne 0x19434
jmp 0x19453
leaq 0x3a641(%rip), %rdi     # 0x53a7c
leaq 0x38d90(%rip), %rsi     # 0x521d2
movl $0xe9f, %edx            # imm = 0xE9F
leaq 0x3a5f4(%rip), %rcx     # 0x53a42
callq 0x4170
movq 0x80(%rsp), %rax
addq $0x448, %rax            # imm = 0x448
movq %rax, 0x60(%rsp)
movq 0x2e0(%rsp), %rdi
callq 0x3bae0
movq %rax, 0x2b0(%rsp)
movq %rdx, 0x2b8(%rsp)
movq 0x2b0(%rsp), %rsi
movq 0x2b8(%rsp), %rdx
leaq 0x2c0(%rsp), %rdi
movq %rdi, 0x68(%rsp)
xorps %xmm0, %xmm0
callq 0x3b680
movq 0x60(%rsp), %rdi
movq 0x68(%rsp), %rdx
movl $0x1, %esi
callq 0x21bf0
jmp 0x194be
leaq 0x2c0(%rsp), %rdi
callq 0x3bb80
movq 0x80(%rsp), %rax
addq $0x460, %rax            # imm = 0x460
movq %rax, 0x50(%rsp)
movq 0x2e0(%rsp), %rdi
callq 0x3bae0
movq %rax, 0x278(%rsp)
movq %rdx, 0x280(%rsp)
movq 0x278(%rsp), %rsi
movq 0x280(%rsp), %rdx
leaq 0x288(%rsp), %rdi
movq %rdi, 0x58(%rsp)
xorps %xmm0, %xmm0
callq 0x3b680
movq 0x50(%rsp), %rdi
movq 0x58(%rsp), %rdx
movl $0x1, %esi
callq 0x21bf0
jmp 0x19536
leaq 0x288(%rsp), %rdi
callq 0x3bb80
jmp 0x1973e
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0x2c0(%rsp), %rdi
callq 0x3bb80
jmp 0x19dd5
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0x288(%rsp), %rdi
callq 0x3bb80
jmp 0x19dd5
movq 0x80(%rsp), %rax
cmpl $0x1, 0xc(%rax)
je 0x195b4
movq 0x80(%rsp), %rax
cmpl $0x2, 0xc(%rax)
jne 0x1973c
movq 0x80(%rsp), %rax
cmpq $0x0, 0x18(%rax)
jbe 0x195c5
jmp 0x195e4
leaq 0x3a4c2(%rip), %rdi     # 0x53a8e
leaq 0x38bff(%rip), %rsi     # 0x521d2
movl $0xea5, %edx            # imm = 0xEA5
leaq 0x3a463(%rip), %rcx     # 0x53a42
callq 0x4170
movq 0x80(%rsp), %rax
movq %rax, %rcx
addq $0x448, %rcx            # imm = 0x448
movq %rcx, 0x38(%rsp)
movq 0x18(%rax), %rax
movq %rax, 0x40(%rsp)
movq 0x2e0(%rsp), %rdi
callq 0x3bae0
movq %rax, 0x250(%rsp)
movq %rdx, 0x258(%rsp)
movq 0x250(%rsp), %rsi
movq 0x258(%rsp), %rdx
leaq 0x260(%rsp), %rdi
movq %rdi, 0x48(%rsp)
xorps %xmm0, %xmm0
callq 0x3b680
movq 0x38(%rsp), %rdi
movq 0x40(%rsp), %rsi
movq 0x48(%rsp), %rdx
callq 0x21bf0
jmp 0x1965c
leaq 0x260(%rsp), %rdi
callq 0x3bb80
movq 0x80(%rsp), %rax
movq %rax, %rcx
addq $0x460, %rcx            # imm = 0x460
movq %rcx, 0x20(%rsp)
movq 0x18(%rax), %rax
movq %rax, 0x28(%rsp)
movq 0x2e0(%rsp), %rdi
callq 0x3bae0
movq %rax, 0x228(%rsp)
movq %rdx, 0x230(%rsp)
movq 0x228(%rsp), %rsi
movq 0x230(%rsp), %rdx
leaq 0x238(%rsp), %rdi
movq %rdi, 0x30(%rsp)
xorps %xmm0, %xmm0
callq 0x3b680
movq 0x20(%rsp), %rdi
movq 0x28(%rsp), %rsi
movq 0x30(%rsp), %rdx
callq 0x21bf0
jmp 0x196e1
leaq 0x238(%rsp), %rdi
callq 0x3bb80
jmp 0x1973c
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0x260(%rsp), %rdi
callq 0x3bb80
jmp 0x19dd5
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0x238(%rsp), %rdi
callq 0x3bb80
jmp 0x19dd5
jmp 0x1973e
jmp 0x19740
movq 0x2e0(%rsp), %rsi
leaq 0x210(%rsp), %rdi
movq %rdi, 0x18(%rsp)
callq 0x3b810
movq 0x80(%rsp), %rdi
movabsq $0x3feccccccccccccd, %rax # imm = 0x3FECCCCCCCCCCCCD
movq %rax, 0x208(%rsp)
movabsq $0x3feff7ced916872b, %rax # imm = 0x3FEFF7CED916872B
movq %rax, 0x200(%rsp)
movabsq $0x3e45798ee2308c3a, %rax # imm = 0x3E45798EE2308C3A
movq %rax, 0x1f8(%rsp)
addq $0x448, %rdi            # imm = 0x448
movslq 0x2dc(%rsp), %rsi
callq 0x1f9e0
movq 0x18(%rsp), %rsi
movq %rax, 0x1f0(%rsp)
movsd 0x208(%rsp), %xmm1
movsd 0x3883e(%rip), %xmm0    # 0x52008
subsd %xmm1, %xmm0
leaq 0x1d8(%rsp), %rdi
callq 0x3cfd0
jmp 0x197dd
movq 0x1f0(%rsp), %rsi
movsd 0x208(%rsp), %xmm0
leaq 0x1a8(%rsp), %rdi
callq 0x3cfd0
jmp 0x197fd
leaq 0x1c0(%rsp), %rdi
leaq 0x1a8(%rsp), %rsi
leaq 0x1d8(%rsp), %rdx
callq 0x3bf90
jmp 0x1981c
leaq 0x1a8(%rsp), %rdi
callq 0x3bb80
movq 0x80(%rsp), %rdi
addq $0x448, %rdi            # imm = 0x448
movslq 0x2dc(%rsp), %rsi
callq 0x1f9e0
movq %rax, %rdi
leaq 0x1c0(%rsp), %rsi
callq 0x1fa20
jmp 0x19857
movq 0x80(%rsp), %rdi
addq $0x460, %rdi            # imm = 0x460
movslq 0x2dc(%rsp), %rsi
callq 0x1f9e0
movq %rax, 0x1a0(%rsp)
leaq 0x170(%rsp), %rdi
leaq 0x210(%rsp), %rdx
movq %rdx, %rsi
callq 0x3c680
jmp 0x19895
movsd 0x200(%rsp), %xmm1
movsd 0x38762(%rip), %xmm0    # 0x52008
subsd %xmm1, %xmm0
leaq 0x188(%rsp), %rdi
leaq 0x170(%rsp), %rsi
callq 0x3cfd0
jmp 0x198c1
leaq 0x1d8(%rsp), %rdi
leaq 0x188(%rsp), %rsi
callq 0x1fa20
jmp 0x198d8
leaq 0x188(%rsp), %rdi
callq 0x3bb80
leaq 0x170(%rsp), %rdi
callq 0x3bb80
movq 0x1a0(%rsp), %rsi
movsd 0x200(%rsp), %xmm0
leaq 0x140(%rsp), %rdi
callq 0x3cfd0
jmp 0x19912
leaq 0x158(%rsp), %rdi
leaq 0x140(%rsp), %rsi
leaq 0x1d8(%rsp), %rdx
callq 0x3bf90
jmp 0x19931
leaq 0x140(%rsp), %rdi
callq 0x3bb80
movq 0x80(%rsp), %rdi
addq $0x460, %rdi            # imm = 0x460
movslq 0x2dc(%rsp), %rsi
callq 0x1f9e0
movq %rax, %rdi
leaq 0x158(%rsp), %rsi
callq 0x1fa20
jmp 0x1996c
movq 0x80(%rsp), %rax
movsd 0x208(%rsp), %xmm1
movsd 0x478(%rax), %xmm0
mulsd %xmm1, %xmm0
movsd %xmm0, 0x478(%rax)
movsd 0x200(%rsp), %xmm1
movsd 0x480(%rax), %xmm0
mulsd %xmm1, %xmm0
movsd %xmm0, 0x480(%rax)
movsd 0x478(%rax), %xmm1
movsd 0x3864a(%rip), %xmm0    # 0x52008
subsd %xmm1, %xmm0
leaq 0x128(%rsp), %rdi
leaq 0x1c0(%rsp), %rsi
callq 0x3d140
jmp 0x199d9
movq 0x80(%rsp), %rax
movsd 0x480(%rax), %xmm1
movsd 0x38617(%rip), %xmm0    # 0x52008
subsd %xmm1, %xmm0
leaq 0x110(%rsp), %rdi
leaq 0x158(%rsp), %rsi
callq 0x3d140
jmp 0x19a0c
leaq 0x210(%rsp), %rdi
callq 0x3bae0
movq %rdx, 0x8(%rsp)
movq %rax, 0x10(%rsp)
jmp 0x19a25
movq 0x8(%rsp), %rax
movq 0x10(%rsp), %rcx
movq %rcx, 0xe8(%rsp)
movq %rax, 0xf0(%rsp)
movsd 0x1f8(%rsp), %xmm0
movq 0xe8(%rsp), %rsi
movq 0xf0(%rsp), %rdx
leaq 0xf8(%rsp), %rdi
callq 0x3b680
jmp 0x19a67
leaq 0x712(%rip), %rsi       # 0x1a180
leaq 0x110(%rsp), %rdi
callq 0x3dc20
jmp 0x19a7d
leaq 0xd0(%rsp), %rdi
leaq 0x110(%rsp), %rsi
callq 0x3b810
jmp 0x19a94
leaq 0xb8(%rsp), %rdi
leaq 0xd0(%rsp), %rsi
leaq 0xf8(%rsp), %rdx
callq 0x3bf90
jmp 0x19ab3
leaq 0xa0(%rsp), %rdi
leaq 0x128(%rsp), %rsi
leaq 0xb8(%rsp), %rdx
callq 0x3c880
jmp 0x19ad2
movq 0x70(%rsp), %rdi
movq 0x80(%rsp), %rax
movsd 0xc0(%rax), %xmm0
movsd %xmm0, 0x98(%rsp)
movb $0x0, 0x97(%rsp)
movsd 0x98(%rsp), %xmm0
leaq 0xa0(%rsp), %rsi
callq 0x3cfd0
jmp 0x19b10
movb $0x1, 0x97(%rsp)
testb $0x1, 0x97(%rsp)
jne 0x19cd1
jmp 0x19cc7
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19dc8
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19dbb
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0x1a8(%rsp), %rdi
callq 0x3bb80
jmp 0x19dbb
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19dae
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19bd3
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0x188(%rsp), %rdi
callq 0x3bb80
leaq 0x170(%rsp), %rdi
callq 0x3bb80
jmp 0x19dae
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0x140(%rsp), %rdi
callq 0x3bb80
jmp 0x19dae
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19da1
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19d94
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19d87
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19d7a
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19d6d
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
jmp 0x19d60
movq %rax, %rcx
movl %edx, %eax
movq %rcx, 0x2a8(%rsp)
movl %eax, 0x2a4(%rsp)
leaq 0xa0(%rsp), %rdi
callq 0x3bb80
jmp 0x19d60
movq 0x70(%rsp), %rdi
callq 0x3bb80
leaq 0xa0(%rsp), %rdi
callq 0x3bb80
leaq 0xb8(%rsp), %rdi
callq 0x3bb80
leaq 0xd0(%rsp), %rdi
callq 0x3bb80
leaq 0xf8(%rsp), %rdi
callq 0x3bb80
leaq 0x110(%rsp), %rdi
callq 0x3bb80
leaq 0x128(%rsp), %rdi
callq 0x3bb80
leaq 0x158(%rsp), %rdi
callq 0x3bb80
leaq 0x1c0(%rsp), %rdi
callq 0x3bb80
leaq 0x1d8(%rsp), %rdi
callq 0x3bb80
leaq 0x210(%rsp), %rdi
callq 0x3bb80
movq 0x78(%rsp), %rax
addq $0x2f8, %rsp            # imm = 0x2F8
retq
leaq 0xb8(%rsp), %rdi
callq 0x3bb80
leaq 0xd0(%rsp), %rdi
callq 0x3bb80
leaq 0xf8(%rsp), %rdi
callq 0x3bb80
leaq 0x110(%rsp), %rdi
callq 0x3bb80
leaq 0x128(%rsp), %rdi
callq 0x3bb80
leaq 0x158(%rsp), %rdi
callq 0x3bb80
leaq 0x1c0(%rsp), %rdi
callq 0x3bb80
leaq 0x1d8(%rsp), %rdi
callq 0x3bb80
leaq 0x210(%rsp), %rdi
callq 0x3bb80
movq 0x2a8(%rsp), %rdi
callq 0x43f0
nopw %cs:(%rax,%rax)