// File: /workspace/github_cpp_makefile_star2_O1/Kingsford-Group[P]ghost2/include/Eigen/src/Core/products/SelfadjointMatrixVector.h
// Function: Eigen::internal::selfadjoint_matrix_vector_product<double, long, 0, 1, false, false, 0>::run(long, double const*, long, double const*, long, double*, double)
EIGEN_DONT_INLINE void selfadjoint_matrix_vector_product<Scalar,Index,StorageOrder,UpLo,ConjugateLhs,ConjugateRhs,Version>::run(
  Index size,
  const Scalar*  lhs, Index lhsStride,
  const Scalar* _rhs, Index rhsIncr,
  Scalar* res,
  Scalar alpha)
{
  typedef typename packet_traits<Scalar>::type Packet;
  const Index PacketSize = sizeof(Packet)/sizeof(Scalar);

  enum {
    IsRowMajor = StorageOrder==RowMajor ? 1 : 0,
    IsLower = UpLo == Lower ? 1 : 0,
    FirstTriangular = IsRowMajor == IsLower
  };

  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs,  IsRowMajor), ConjugateRhs> cj0;
  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1;
  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex, ConjugateRhs> cjd;

  conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs,  IsRowMajor), ConjugateRhs> pcj0;
  conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> pcj1;

  Scalar cjAlpha = ConjugateRhs ? numext::conj(alpha) : alpha;

  // FIXME this copy is now handled outside product_selfadjoint_vector, so it could probably be removed.
  // if the rhs is not sequentially stored in memory we copy it to a temporary buffer,
  // this is because we need to extract packets
  ei_declare_aligned_stack_constructed_variable(Scalar,rhs,size,rhsIncr==1 ? const_cast<Scalar*>(_rhs) : 0);  
  if (rhsIncr!=1)
  {
    const Scalar* it = _rhs;
    for (Index i=0; i<size; ++i, it+=rhsIncr)
      rhs[i] = *it;
  }

  Index bound = (std::max)(Index(0),size-8) & 0xfffffffe;
  if (FirstTriangular)
    bound = size - bound;

  for (Index j=FirstTriangular ? bound : 0;
       j<(FirstTriangular ? size : bound);j+=2)
  {
    const Scalar* EIGEN_RESTRICT A0 = lhs + j*lhsStride;
    const Scalar* EIGEN_RESTRICT A1 = lhs + (j+1)*lhsStride;

    Scalar t0 = cjAlpha * rhs[j];
    Packet ptmp0 = pset1<Packet>(t0);
    Scalar t1 = cjAlpha * rhs[j+1];
    Packet ptmp1 = pset1<Packet>(t1);

    Scalar t2(0);
    Packet ptmp2 = pset1<Packet>(t2);
    Scalar t3(0);
    Packet ptmp3 = pset1<Packet>(t3);

    size_t starti = FirstTriangular ? 0 : j+2;
    size_t endi   = FirstTriangular ? j : size;
    size_t alignedStart = (starti) + internal::first_aligned(&res[starti], endi-starti);
    size_t alignedEnd = alignedStart + ((endi-alignedStart)/(PacketSize))*(PacketSize);

    // TODO make sure this product is a real * complex and that the rhs is properly conjugated if needed
    res[j]   += cjd.pmul(numext::real(A0[j]), t0);
    res[j+1] += cjd.pmul(numext::real(A1[j+1]), t1);
    if(FirstTriangular)
    {
      res[j]   += cj0.pmul(A1[j],   t1);
      t3       += cj1.pmul(A1[j],   rhs[j]);
    }
    else
    {
      res[j+1] += cj0.pmul(A0[j+1],t0);
      t2 += cj1.pmul(A0[j+1], rhs[j+1]);
    }

    for (size_t i=starti; i<alignedStart; ++i)
    {
      res[i] += t0 * A0[i] + t1 * A1[i];
      t2 += numext::conj(A0[i]) * rhs[i];
      t3 += numext::conj(A1[i]) * rhs[i];
    }
    // Yes this an optimization for gcc 4.3 and 4.4 (=> huge speed up)
    // gcc 4.2 does this optimization automatically.
    const Scalar* EIGEN_RESTRICT a0It  = A0  + alignedStart;
    const Scalar* EIGEN_RESTRICT a1It  = A1  + alignedStart;
    const Scalar* EIGEN_RESTRICT rhsIt = rhs + alignedStart;
          Scalar* EIGEN_RESTRICT resIt = res + alignedStart;
    for (size_t i=alignedStart; i<alignedEnd; i+=PacketSize)
    {
      Packet A0i = ploadu<Packet>(a0It);  a0It  += PacketSize;
      Packet A1i = ploadu<Packet>(a1It);  a1It  += PacketSize;
      Packet Bi  = ploadu<Packet>(rhsIt); rhsIt += PacketSize; // FIXME should be aligned in most cases
      Packet Xi  = pload <Packet>(resIt);

      Xi    = pcj0.pmadd(A0i,ptmp0, pcj0.pmadd(A1i,ptmp1,Xi));
      ptmp2 = pcj1.pmadd(A0i,  Bi, ptmp2);
      ptmp3 = pcj1.pmadd(A1i,  Bi, ptmp3);
      pstore(resIt,Xi); resIt += PacketSize;
    }
    for (size_t i=alignedEnd; i<endi; i++)
    {
      res[i] += cj0.pmul(A0[i], t0) + cj0.pmul(A1[i],t1);
      t2 += cj1.pmul(A0[i], rhs[i]);
      t3 += cj1.pmul(A1[i], rhs[i]);
    }

    res[j]   += alpha * (t2 + predux(ptmp2));
    res[j+1] += alpha * (t3 + predux(ptmp3));
  }
  for (Index j=FirstTriangular ? 0 : bound;j<(FirstTriangular ? bound : size);j++)
  {
    const Scalar* EIGEN_RESTRICT A0 = lhs + j*lhsStride;

    Scalar t1 = cjAlpha * rhs[j];
    Scalar t2(0);
    // TODO make sure this product is a real * complex and that the rhs is properly conjugated if needed
    res[j] += cjd.pmul(numext::real(A0[j]), t1);
    for (Index i=FirstTriangular ? 0 : j+1; i<(FirstTriangular ? j : size); i++)
    {
      res[i] += cj0.pmul(A0[i], t1);
      t2 += cj1.pmul(A0[i], rhs[i]);
    }
    res[j] += alpha * t2;
  }
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %rdx, -0x38(%rbp)
movq %rsi, -0x30(%rbp)
movq %rdi, %rax
shrq $0x3d, %rax
jne 0x319e2
movq %r9, %rbx
movq %r8, %r13
movq %rcx, %r14
xorl %eax, %eax
cmpq $0x1, %r8
cmovneq %rax, %rcx
movq %rcx, %rsi
testq %rcx, %rcx
movq %rdi, -0x48(%rbp)
jne 0x315d8
leaq (,%rdi,8), %rcx
cmpq $0x9c4, %rdi            # imm = 0x9C4
ja 0x315b6
movq %rsp, %rax
addq $0xf, %rcx
andq $-0x10, %rcx
subq %rcx, %rax
movq %rax, %rsp
movq %rax, %rsi
jmp 0x315d8
movq %rcx, %rdi
movsd %xmm0, -0x40(%rbp)
callq 0xd750
movq -0x48(%rbp), %rdi
movsd -0x40(%rbp), %xmm0
movq %rax, %rsi
testq %rax, %rax
je 0x319e2
movq %rax, -0x58(%rbp)
cmpq $0x1, %r13
sete %cl
testq %rdi, %rdi
sete %dl
orb %cl, %dl
movq %rsi, %rax
jne 0x31618
cmpq $0x2, %rdi
movl $0x1, %ecx
cmovgeq %rdi, %rcx
shlq $0x3, %r13
xorl %edx, %edx
movsd (%r14), %xmm1
movsd %xmm1, (%rax,%rdx,8)
incq %rdx
addq %r13, %r14
cmpq %rdx, %rcx
jne 0x31603
cmpq $0x9, %rdi
movl $0x8, %ecx
cmovgeq %rdi, %rcx
movl $0xfffffff8, %edx       # imm = 0xFFFFFFF8
addq %rcx, %rdx
movl $0xfffffffe, %r15d      # imm = 0xFFFFFFFE
movq %rdx, -0x50(%rbp)
andq %rdx, %r15
je 0x318e0
leaq 0x10(%rax), %rsi
leaq 0x10(%rbx), %r8
movq -0x38(%rbp), %rdx
movq -0x30(%rbp), %r12
leaq (%r12,%rdx,8), %r9
addq $0x10, %r9
movq %rdx, %rcx
shlq $0x4, %rcx
movq %rcx, -0x70(%rbp)
addq $0x10, %rcx
movq %rcx, -0x68(%rbp)
leaq 0x10(%r12), %r14
leaq (%r12,%rdx,8), %r13
xorl %ecx, %ecx
movq %r15, -0x78(%rbp)
movq %rax, -0x60(%rbp)
movq %rsi, -0x40(%rbp)
movq %rcx, %rax
movq -0x38(%rbp), %r11
imulq %r11, %rax
movq %rcx, %r10
orq $0x1, %r10
movq %r10, -0x80(%rbp)
imulq %r11, %r10
leaq 0x2(%rcx), %rdx
leal (%rbx,%rdx,8), %r15d
movq %rdi, %r11
subq %rdx, %r11
shrl $0x3, %r15d
andl $0x1, %r15d
cmpq %r11, %r15
cmovaeq %r11, %r15
testb $0x7, %bl
movq -0x30(%rbp), %rsi
leaq (%rsi,%rax,8), %rdi
movq -0x60(%rbp), %rax
leaq (%rsi,%r10,8), %r10
movsd (%rax,%rcx,8), %xmm13
mulsd %xmm0, %xmm13
movsd 0x8(%rax,%rcx,8), %xmm1
mulsd %xmm0, %xmm1
cmovneq %r11, %r15
movsd (%rdi,%rcx,8), %xmm2
mulsd %xmm13, %xmm2
addsd (%rbx,%rcx,8), %xmm2
movsd %xmm2, (%rbx,%rcx,8)
movsd 0x8(%r10,%rcx,8), %xmm2
mulsd %xmm1, %xmm2
addsd 0x8(%rbx,%rcx,8), %xmm2
movsd %xmm2, 0x8(%rbx,%rcx,8)
movsd 0x8(%rdi,%rcx,8), %xmm3
mulsd %xmm13, %xmm3
addsd %xmm2, %xmm3
movsd %xmm3, 0x8(%rbx,%rcx,8)
movsd 0x8(%rdi,%rcx,8), %xmm3
mulsd 0x8(%rax,%rcx,8), %xmm3
leaq (%r15,%rcx), %r10
addq $0x2, %r10
xorpd %xmm2, %xmm2
addsd %xmm2, %xmm3
cmpq %r10, %rdx
jae 0x31798
xorl %edi, %edi
movq -0x40(%rbp), %rsi
movsd (%r9,%rdi,8), %xmm4
mulsd %xmm1, %xmm4
movsd (%r14,%rdi,8), %xmm5
mulsd %xmm13, %xmm5
addsd %xmm4, %xmm5
addsd (%r8,%rdi,8), %xmm5
movsd %xmm5, (%r8,%rdi,8)
movsd (%rsi,%rdi,8), %xmm4
movsd (%r14,%rdi,8), %xmm5
mulsd %xmm4, %xmm5
mulsd (%r9,%rdi,8), %xmm4
addsd %xmm5, %xmm3
addsd %xmm4, %xmm2
incq %rdi
cmpq %rdi, %r15
jne 0x3174c
jmp 0x3179c
movq -0x40(%rbp), %rsi
movq -0x48(%rbp), %rdi
movq %rdi, %r15
subq %r10, %r15
andq $-0x2, %r15
addq %r10, %r15
xorpd %xmm4, %xmm4
xorpd %xmm5, %xmm5
cmpq %r15, %r10
jae 0x3186f
movapd %xmm13, %xmm6
unpcklpd %xmm13, %xmm6           # xmm6 = xmm6[0],xmm13[0]
movapd %xmm1, %xmm7
unpcklpd %xmm1, %xmm7            # xmm7 = xmm7[0],xmm1[0]
movupd (%r12,%r10,8), %xmm8
movupd (%r13,%r10,8), %xmm9
movupd (%rax,%r10,8), %xmm10
movapd %xmm7, %xmm11
mulpd %xmm9, %xmm11
addpd (%rbx,%r10,8), %xmm11
movapd %xmm6, %xmm12
mulpd %xmm8, %xmm12
addpd %xmm11, %xmm12
mulpd %xmm10, %xmm8
addpd %xmm8, %xmm5
mulpd %xmm10, %xmm9
addpd %xmm9, %xmm4
movapd %xmm12, (%rbx,%r10,8)
addq $0x2, %r10
cmpq %r15, %r10
jb 0x317d0
jmp 0x3186f
movsd (%r12,%r15,8), %xmm6
mulsd %xmm13, %xmm6
movsd (%r13,%r15,8), %xmm7
mulsd %xmm1, %xmm7
addsd %xmm6, %xmm7
addsd (%rbx,%r15,8), %xmm7
movsd %xmm7, (%rbx,%r15,8)
movsd (%rax,%r15,8), %xmm6
movsd (%r12,%r15,8), %xmm7
mulsd %xmm6, %xmm7
mulsd (%r13,%r15,8), %xmm6
addsd %xmm7, %xmm3
addsd %xmm6, %xmm2
incq %r15
cmpq %rdi, %r15
jb 0x31827
movapd %xmm5, %xmm1
unpckhpd %xmm5, %xmm1            # xmm1 = xmm1[1],xmm5[1]
addsd %xmm5, %xmm1
addsd %xmm3, %xmm1
mulsd %xmm0, %xmm1
addsd (%rbx,%rcx,8), %xmm1
movsd %xmm1, (%rbx,%rcx,8)
movapd %xmm4, %xmm1
unpckhpd %xmm4, %xmm1            # xmm1 = xmm1[1],xmm4[1]
addsd %xmm4, %xmm1
addsd %xmm2, %xmm1
mulsd %xmm0, %xmm1
movq -0x80(%rbp), %rcx
addsd (%rbx,%rcx,8), %xmm1
movsd %xmm1, (%rbx,%rcx,8)
addq $0x10, %rsi
addq $0x10, %r8
movq -0x68(%rbp), %rcx
addq %rcx, %r9
addq %rcx, %r14
movq -0x70(%rbp), %rcx
addq %rcx, %r13
addq %rcx, %r12
movq %rdx, %rcx
movq -0x78(%rbp), %r15
cmpq %r15, %rdx
jb 0x3167e
cmpq %rdi, %r15
jge 0x319c1
movq -0x50(%rbp), %r9
shrl %r9d
movq %r9, %rdx
shlq $0x4, %rdx
leaq (%rax,%rdx), %rsi
addq %rbx, %rdx
movq -0x38(%rbp), %r8
movq %r8, %rcx
shlq $0x4, %rcx
addq $0x10, %rcx
imulq %r9, %rcx
addq -0x30(%rbp), %rcx
leaq 0x8(,%r8,8), %r8
movq %r15, %r9
imulq -0x38(%rbp), %r9
movq -0x30(%rbp), %r10
leaq (%r10,%r9,8), %r9
movsd (%rax,%r15,8), %xmm3
mulsd %xmm0, %xmm3
movsd (%r9,%r15,8), %xmm1
mulsd %xmm3, %xmm1
addsd (%rbx,%r15,8), %xmm1
movsd %xmm1, (%rbx,%r15,8)
leaq 0x1(%r15), %r9
xorpd %xmm1, %xmm1
cmpq %rdi, %r9
jge 0x3199a
movl $0x1, %r10d
movsd (%rcx,%r10,8), %xmm2
mulsd %xmm3, %xmm2
addsd (%rdx,%r10,8), %xmm2
movsd %xmm2, (%rdx,%r10,8)
movsd (%rcx,%r10,8), %xmm2
mulsd (%rsi,%r10,8), %xmm2
addsd %xmm2, %xmm1
leaq (%r15,%r10), %r11
incq %r11
incq %r10
cmpq -0x48(%rbp), %r11
movq -0x48(%rbp), %rdi
jl 0x31960
mulsd %xmm0, %xmm1
addsd (%rbx,%r15,8), %xmm1
movsd %xmm1, (%rbx,%r15,8)
addq $0x8, %rsi
addq $0x8, %rdx
addq %r8, %rcx
movq %r9, %r15
cmpq %rdi, %r9
jl 0x3191d
cmpq $0x9c5, %rdi            # imm = 0x9C5
jb 0x319d3
movq -0x58(%rbp), %rdi
callq 0xd660
leaq -0x28(%rbp), %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movl $0x8, %edi
callq 0xd260
movq 0x4757d(%rip), %rcx     # 0x78f70
addq $0x10, %rcx
movq %rcx, (%rax)
movq 0x4759f(%rip), %rsi     # 0x78fa0
movq 0x47538(%rip), %rdx     # 0x78f40
movq %rax, %rdi
callq 0xd990