// File: /workspace/github_cpp_makefile_star2_O2/Kingsford-Group[P]ghost2/include/Eigen/src/Core/products/SelfadjointMatrixVector.h
// Function: Eigen::internal::selfadjoint_matrix_vector_product<double, long, 0, 1, false, false, 0>::run(long, double const*, long, double const*, long, double*, double)
EIGEN_DONT_INLINE void selfadjoint_matrix_vector_product<Scalar,Index,StorageOrder,UpLo,ConjugateLhs,ConjugateRhs,Version>::run(
  Index size,
  const Scalar*  lhs, Index lhsStride,
  const Scalar* _rhs, Index rhsIncr,
  Scalar* res,
  Scalar alpha)
{
  typedef typename packet_traits<Scalar>::type Packet;
  const Index PacketSize = sizeof(Packet)/sizeof(Scalar);

  enum {
    IsRowMajor = StorageOrder==RowMajor ? 1 : 0,
    IsLower = UpLo == Lower ? 1 : 0,
    FirstTriangular = IsRowMajor == IsLower
  };

  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs,  IsRowMajor), ConjugateRhs> cj0;
  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1;
  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex, ConjugateRhs> cjd;

  conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs,  IsRowMajor), ConjugateRhs> pcj0;
  conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> pcj1;

  Scalar cjAlpha = ConjugateRhs ? numext::conj(alpha) : alpha;

  // FIXME this copy is now handled outside product_selfadjoint_vector, so it could probably be removed.
  // if the rhs is not sequentially stored in memory we copy it to a temporary buffer,
  // this is because we need to extract packets
  ei_declare_aligned_stack_constructed_variable(Scalar,rhs,size,rhsIncr==1 ? const_cast<Scalar*>(_rhs) : 0);  
  if (rhsIncr!=1)
  {
    const Scalar* it = _rhs;
    for (Index i=0; i<size; ++i, it+=rhsIncr)
      rhs[i] = *it;
  }

  Index bound = (std::max)(Index(0),size-8) & 0xfffffffe;
  if (FirstTriangular)
    bound = size - bound;

  for (Index j=FirstTriangular ? bound : 0;
       j<(FirstTriangular ? size : bound);j+=2)
  {
    const Scalar* EIGEN_RESTRICT A0 = lhs + j*lhsStride;
    const Scalar* EIGEN_RESTRICT A1 = lhs + (j+1)*lhsStride;

    Scalar t0 = cjAlpha * rhs[j];
    Packet ptmp0 = pset1<Packet>(t0);
    Scalar t1 = cjAlpha * rhs[j+1];
    Packet ptmp1 = pset1<Packet>(t1);

    Scalar t2(0);
    Packet ptmp2 = pset1<Packet>(t2);
    Scalar t3(0);
    Packet ptmp3 = pset1<Packet>(t3);

    size_t starti = FirstTriangular ? 0 : j+2;
    size_t endi   = FirstTriangular ? j : size;
    size_t alignedStart = (starti) + internal::first_aligned(&res[starti], endi-starti);
    size_t alignedEnd = alignedStart + ((endi-alignedStart)/(PacketSize))*(PacketSize);

    // TODO make sure this product is a real * complex and that the rhs is properly conjugated if needed
    res[j]   += cjd.pmul(numext::real(A0[j]), t0);
    res[j+1] += cjd.pmul(numext::real(A1[j+1]), t1);
    if(FirstTriangular)
    {
      res[j]   += cj0.pmul(A1[j],   t1);
      t3       += cj1.pmul(A1[j],   rhs[j]);
    }
    else
    {
      res[j+1] += cj0.pmul(A0[j+1],t0);
      t2 += cj1.pmul(A0[j+1], rhs[j+1]);
    }

    for (size_t i=starti; i<alignedStart; ++i)
    {
      res[i] += t0 * A0[i] + t1 * A1[i];
      t2 += numext::conj(A0[i]) * rhs[i];
      t3 += numext::conj(A1[i]) * rhs[i];
    }
    // Yes this an optimization for gcc 4.3 and 4.4 (=> huge speed up)
    // gcc 4.2 does this optimization automatically.
    const Scalar* EIGEN_RESTRICT a0It  = A0  + alignedStart;
    const Scalar* EIGEN_RESTRICT a1It  = A1  + alignedStart;
    const Scalar* EIGEN_RESTRICT rhsIt = rhs + alignedStart;
          Scalar* EIGEN_RESTRICT resIt = res + alignedStart;
    for (size_t i=alignedStart; i<alignedEnd; i+=PacketSize)
    {
      Packet A0i = ploadu<Packet>(a0It);  a0It  += PacketSize;
      Packet A1i = ploadu<Packet>(a1It);  a1It  += PacketSize;
      Packet Bi  = ploadu<Packet>(rhsIt); rhsIt += PacketSize; // FIXME should be aligned in most cases
      Packet Xi  = pload <Packet>(resIt);

      Xi    = pcj0.pmadd(A0i,ptmp0, pcj0.pmadd(A1i,ptmp1,Xi));
      ptmp2 = pcj1.pmadd(A0i,  Bi, ptmp2);
      ptmp3 = pcj1.pmadd(A1i,  Bi, ptmp3);
      pstore(resIt,Xi); resIt += PacketSize;
    }
    for (size_t i=alignedEnd; i<endi; i++)
    {
      res[i] += cj0.pmul(A0[i], t0) + cj0.pmul(A1[i],t1);
      t2 += cj1.pmul(A0[i], rhs[i]);
      t3 += cj1.pmul(A1[i], rhs[i]);
    }

    res[j]   += alpha * (t2 + predux(ptmp2));
    res[j+1] += alpha * (t3 + predux(ptmp3));
  }
  for (Index j=FirstTriangular ? 0 : bound;j<(FirstTriangular ? bound : size);j++)
  {
    const Scalar* EIGEN_RESTRICT A0 = lhs + j*lhsStride;

    Scalar t1 = cjAlpha * rhs[j];
    Scalar t2(0);
    // TODO make sure this product is a real * complex and that the rhs is properly conjugated if needed
    res[j] += cjd.pmul(numext::real(A0[j]), t1);
    for (Index i=FirstTriangular ? 0 : j+1; i<(FirstTriangular ? j : size); i++)
    {
      res[i] += cj0.pmul(A0[i], t1);
      t2 += cj1.pmul(A0[i], rhs[i]);
    }
    res[j] += alpha * t2;
  }
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x88, %rsp
movq %r9, %rbx
movq %rdx, %r14
movq %rsi, %r15
movq %rdi, %r12
movq %rdi, %rax
shrq $0x3d, %rax
je 0x2b233
movaps %xmm0, -0x50(%rbp)
movq %rcx, %r13
movq %r8, -0x30(%rbp)
callq 0x23674
movq -0x30(%rbp), %r8
movq %r13, %rcx
movapd -0x50(%rbp), %xmm0
cmpq $0x1, %r8
setne %al
testq %rcx, %rcx
sete %dl
orb %al, %dl
leaq (,%r12,8), %rdi
testb %dl, %dl
je 0x2b26c
cmpq $0x4e20, %rdi           # imm = 0x4E20
ja 0x2b282
movq %rsp, %rax
addq $0xf, %rdi
andq $-0x10, %rdi
subq %rdi, %rax
movq %rax, %rsp
xorl %edx, %edx
jmp 0x2b2a1
andq $0x0, -0x70(%rbp)
cmpq $0x4e21, %rdi           # imm = 0x4E21
movq %r12, -0x68(%rbp)
setae -0x60(%rbp)
jmp 0x2b2dd
movapd %xmm0, -0x50(%rbp)
movq %rcx, %r13
movq %r8, -0x30(%rbp)
callq 0x236a1
movq -0x30(%rbp), %r8
movq %r13, %rcx
movapd -0x50(%rbp), %xmm0
movb $0x1, %dl
movq %rax, -0x70(%rbp)
movq %r12, -0x68(%rbp)
movb %dl, -0x60(%rbp)
cmpq $0x1, %r8
je 0x2b2da
xorl %esi, %esi
testq %r12, %r12
movl $0x0, %edx
cmovgq %r12, %rdx
shlq $0x3, %r8
cmpq %rsi, %rdx
je 0x2b2da
movsd (%rcx), %xmm1
movsd %xmm1, (%rax,%rsi,8)
incq %rsi
addq %r8, %rcx
jmp 0x2b2c4
movq %rax, %rcx
cmpq $0x9, %r12
pushq $0x8
popq %rax
cmovgeq %r12, %rax
movl $0xfffffff8, %edx       # imm = 0xFFFFFFF8
addq %rax, %rdx
movq %rdx, -0x78(%rbp)
movl %edx, %r13d
andl $-0x2, %r13d
movapd %xmm0, %xmm13
unpcklpd %xmm0, %xmm13           # xmm13 = xmm13[0],xmm0[0]
leaq 0x10(%r15), %rsi
movq %r14, %rax
shlq $0x4, %rax
movq %rax, -0x80(%rbp)
leaq 0x10(%rax), %rdx
leaq (%r15,%r14,8), %r8
addq $0x10, %r8
leaq 0x10(%rbx), %r9
leaq 0x10(%rcx), %rax
movq %rax, -0x58(%rbp)
leaq (%r15,%r14,8), %r11
pushq $0x2
popq %rdi
xorl %r10d, %r10d
xorpd %xmm1, %xmm1
movq %r15, -0x30(%rbp)
movq %rcx, -0x50(%rbp)
movq %r14, -0x90(%rbp)
movq %r13, -0x88(%rbp)
cmpq %r13, %r10
jae 0x2b5d5
movq %r10, %rax
imulq %r14, %rax
movq %rax, -0x38(%rbp)
movq %r10, %rax
orq $0x1, %rax
imulq %r14, %rax
leaq 0x2(%r10), %rcx
leal (%rbx,%rcx,8), %r13d
movq %r12, %r14
movq %rcx, -0xa8(%rbp)
subq %rcx, %r14
shrl $0x3, %r13d
andl $0x1, %r13d
cmpq %r14, %r13
cmovaeq %r14, %r13
testb $0x7, %bl
movq -0x38(%rbp), %rcx
movq %rdi, -0xa0(%rbp)
movq -0x30(%rbp), %rdi
leaq (%rdi,%rcx,8), %rcx
movq %rcx, -0x98(%rbp)
movq -0xa0(%rbp), %rdi
movq -0x30(%rbp), %rcx
leaq (%rcx,%rax,8), %rax
movq %rax, -0x38(%rbp)
movq -0x50(%rbp), %rax
movupd (%rax,%r10,8), %xmm2
mulpd %xmm13, %xmm2
cmovneq %r14, %r13
leaq 0x2(%r13,%r10), %rax
movq %r12, %r13
movq -0x98(%rbp), %r14
movsd (%r14,%r10,8), %xmm3
mulsd %xmm2, %xmm3
addsd (%rbx,%r10,8), %xmm3
subq %rax, %r13
movsd %xmm3, (%rbx,%r10,8)
movapd %xmm2, %xmm4
unpckhpd %xmm2, %xmm4            # xmm4 = xmm4[1],xmm2[1]
movq -0x38(%rbp), %rcx
movsd 0x8(%rcx,%r10,8), %xmm3
mulsd %xmm4, %xmm3
addsd 0x8(%rbx,%r10,8), %xmm3
movsd %xmm3, 0x8(%rbx,%r10,8)
movsd 0x8(%r14,%r10,8), %xmm5
mulsd %xmm2, %xmm5
addsd %xmm3, %xmm5
movsd %xmm5, 0x8(%rbx,%r10,8)
movsd 0x8(%r14,%r10,8), %xmm3
movq %r10, -0x38(%rbp)
movq -0x50(%rbp), %rcx
mulsd 0x8(%rcx,%r10,8), %xmm3
andq $-0x2, %r13
addsd %xmm1, %xmm3
movq %xmm3, %xmm3            # xmm3 = xmm3[0],zero
xorl %r10d, %r10d
movq -0x58(%rbp), %rcx
leaq (%rdi,%r10), %r14
cmpq %rax, %r14
jae 0x2b4b2
movsd (%r8,%r10,8), %xmm5
mulsd %xmm4, %xmm5
movsd (%rsi,%r10,8), %xmm6
mulsd %xmm2, %xmm6
addsd %xmm5, %xmm6
addsd (%r9,%r10,8), %xmm6
movsd %xmm6, (%r9,%r10,8)
movsd (%rsi,%r10,8), %xmm5
movhpd (%r8,%r10,8), %xmm5     # xmm5 = xmm5[0],mem[0]
movsd (%rcx,%r10,8), %xmm6
unpcklpd %xmm6, %xmm6            # xmm6 = xmm6[0,0]
mulpd %xmm5, %xmm6
addpd %xmm6, %xmm3
incq %r10
jmp 0x2b462
movapd %xmm2, %xmm7
unpcklpd %xmm2, %xmm7            # xmm7 = xmm7[0],xmm2[0]
addq %rax, %r13
xorpd %xmm6, %xmm6
xorpd %xmm5, %xmm5
movq -0x90(%rbp), %r14
movq -0x50(%rbp), %rcx
movq -0x38(%rbp), %r10
cmpq %r13, %rax
jae 0x2b577
movupd (%r15,%rax,8), %xmm8
movupd (%r11,%rax,8), %xmm9
movupd (%rcx,%rax,8), %xmm10
movapd %xmm4, %xmm11
mulpd %xmm9, %xmm11
addpd (%rbx,%rax,8), %xmm11
movapd %xmm7, %xmm12
mulpd %xmm8, %xmm12
addpd %xmm11, %xmm12
mulpd %xmm10, %xmm8
addpd %xmm8, %xmm5
mulpd %xmm10, %xmm9
addpd %xmm9, %xmm6
movapd %xmm12, (%rbx,%rax,8)
addq $0x2, %rax
jmp 0x2b4d4
movsd (%r15,%r13,8), %xmm4
movhpd (%r11,%r13,8), %xmm4    # xmm4 = xmm4[0],mem[0]
mulpd %xmm2, %xmm4
movapd %xmm4, %xmm7
unpckhpd %xmm4, %xmm7            # xmm7 = xmm7[1],xmm4[1]
addsd %xmm4, %xmm7
addsd (%rbx,%r13,8), %xmm7
movsd %xmm7, (%rbx,%r13,8)
movsd (%rcx,%r13,8), %xmm4
movsd (%r15,%r13,8), %xmm7
movhpd (%r11,%r13,8), %xmm7    # xmm7 = xmm7[0],mem[0]
unpcklpd %xmm4, %xmm4            # xmm4 = xmm4[0,0]
mulpd %xmm4, %xmm7
addpd %xmm7, %xmm3
incq %r13
cmpq %r12, %r13
jb 0x2b52e
movapd %xmm5, %xmm2
unpcklpd %xmm6, %xmm2            # xmm2 = xmm2[0],xmm6[0]
unpckhpd %xmm6, %xmm5            # xmm5 = xmm5[1],xmm6[1]
addpd %xmm2, %xmm5
addpd %xmm3, %xmm5
movupd (%rbx,%r10,8), %xmm2
mulpd %xmm13, %xmm5
addpd %xmm2, %xmm5
movupd %xmm5, (%rbx,%r10,8)
addq %rdx, %rsi
addq %rdx, %r8
addq $0x10, %r9
addq $0x10, -0x58(%rbp)
addq $0x2, %rdi
movq -0x80(%rbp), %rax
addq %rax, %r15
addq %rax, %r11
movq -0xa8(%rbp), %r10
movq -0x88(%rbp), %r13
jmp 0x2b350
movq -0x78(%rbp), %r15
shrl %r15d
imulq %r15, %rdx
movq -0x30(%rbp), %r11
addq %r11, %rdx
leaq 0x8(,%r14,8), %rax
shlq $0x4, %r15
leaq (%rbx,%r15), %rsi
addq %rcx, %r15
pushq $0x1
popq %rdi
cmpq %r12, %r13
jge 0x2b697
movq %r13, %r8
imulq %r14, %r8
leaq (%r11,%r8,8), %r8
movsd (%rcx,%r13,8), %xmm3
mulsd %xmm0, %xmm3
movsd (%r8,%r13,8), %xmm1
mulsd %xmm3, %xmm1
addsd (%rbx,%r13,8), %xmm1
movsd %xmm1, (%rbx,%r13,8)
leaq 0x1(%r13), %r8
xorpd %xmm1, %xmm1
movq %rdi, %r9
leaq (%r9,%r13), %r10
cmpq %r12, %r10
jge 0x2b670
movsd (%rdx,%r9,8), %xmm2
mulsd %xmm3, %xmm2
addsd (%rsi,%r9,8), %xmm2
movsd %xmm2, (%rsi,%r9,8)
movsd (%rdx,%r9,8), %xmm2
mulsd (%r15,%r9,8), %xmm2
addsd %xmm2, %xmm1
incq %r9
jmp 0x2b63c
mulsd %xmm0, %xmm1
addsd (%rbx,%r13,8), %xmm1
movsd %xmm1, (%rbx,%r13,8)
addq %rax, %rdx
addq $0x8, %rsi
addq $0x8, %r15
movq %r8, %r13
movq -0x50(%rbp), %rcx
jmp 0x2b5fd
leaq -0x70(%rbp), %rdi
callq 0x25cb2
leaq -0x28(%rbp), %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
nop