// File: /workspace/github_cpp_makefile_star2_O3/Kingsford-Group[P]ghost2/include/Eigen/src/Core/products/SelfadjointMatrixVector.h
// Function: Eigen::internal::selfadjoint_matrix_vector_product<double, long, 0, 1, false, false, 0>::run(long, double const*, long, double const*, long, double*, double)
EIGEN_DONT_INLINE void selfadjoint_matrix_vector_product<Scalar,Index,StorageOrder,UpLo,ConjugateLhs,ConjugateRhs,Version>::run(
  Index size,
  const Scalar*  lhs, Index lhsStride,
  const Scalar* _rhs, Index rhsIncr,
  Scalar* res,
  Scalar alpha)
{
  typedef typename packet_traits<Scalar>::type Packet;
  const Index PacketSize = sizeof(Packet)/sizeof(Scalar);

  enum {
    IsRowMajor = StorageOrder==RowMajor ? 1 : 0,
    IsLower = UpLo == Lower ? 1 : 0,
    FirstTriangular = IsRowMajor == IsLower
  };

  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs,  IsRowMajor), ConjugateRhs> cj0;
  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1;
  conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex, ConjugateRhs> cjd;

  conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs,  IsRowMajor), ConjugateRhs> pcj0;
  conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> pcj1;

  Scalar cjAlpha = ConjugateRhs ? numext::conj(alpha) : alpha;

  // FIXME this copy is now handled outside product_selfadjoint_vector, so it could probably be removed.
  // if the rhs is not sequentially stored in memory we copy it to a temporary buffer,
  // this is because we need to extract packets
  ei_declare_aligned_stack_constructed_variable(Scalar,rhs,size,rhsIncr==1 ? const_cast<Scalar*>(_rhs) : 0);  
  if (rhsIncr!=1)
  {
    const Scalar* it = _rhs;
    for (Index i=0; i<size; ++i, it+=rhsIncr)
      rhs[i] = *it;
  }

  Index bound = (std::max)(Index(0),size-8) & 0xfffffffe;
  if (FirstTriangular)
    bound = size - bound;

  for (Index j=FirstTriangular ? bound : 0;
       j<(FirstTriangular ? size : bound);j+=2)
  {
    const Scalar* EIGEN_RESTRICT A0 = lhs + j*lhsStride;
    const Scalar* EIGEN_RESTRICT A1 = lhs + (j+1)*lhsStride;

    Scalar t0 = cjAlpha * rhs[j];
    Packet ptmp0 = pset1<Packet>(t0);
    Scalar t1 = cjAlpha * rhs[j+1];
    Packet ptmp1 = pset1<Packet>(t1);

    Scalar t2(0);
    Packet ptmp2 = pset1<Packet>(t2);
    Scalar t3(0);
    Packet ptmp3 = pset1<Packet>(t3);

    size_t starti = FirstTriangular ? 0 : j+2;
    size_t endi   = FirstTriangular ? j : size;
    size_t alignedStart = (starti) + internal::first_aligned(&res[starti], endi-starti);
    size_t alignedEnd = alignedStart + ((endi-alignedStart)/(PacketSize))*(PacketSize);

    // TODO make sure this product is a real * complex and that the rhs is properly conjugated if needed
    res[j]   += cjd.pmul(numext::real(A0[j]), t0);
    res[j+1] += cjd.pmul(numext::real(A1[j+1]), t1);
    if(FirstTriangular)
    {
      res[j]   += cj0.pmul(A1[j],   t1);
      t3       += cj1.pmul(A1[j],   rhs[j]);
    }
    else
    {
      res[j+1] += cj0.pmul(A0[j+1],t0);
      t2 += cj1.pmul(A0[j+1], rhs[j+1]);
    }

    for (size_t i=starti; i<alignedStart; ++i)
    {
      res[i] += t0 * A0[i] + t1 * A1[i];
      t2 += numext::conj(A0[i]) * rhs[i];
      t3 += numext::conj(A1[i]) * rhs[i];
    }
    // Yes this an optimization for gcc 4.3 and 4.4 (=> huge speed up)
    // gcc 4.2 does this optimization automatically.
    const Scalar* EIGEN_RESTRICT a0It  = A0  + alignedStart;
    const Scalar* EIGEN_RESTRICT a1It  = A1  + alignedStart;
    const Scalar* EIGEN_RESTRICT rhsIt = rhs + alignedStart;
          Scalar* EIGEN_RESTRICT resIt = res + alignedStart;
    for (size_t i=alignedStart; i<alignedEnd; i+=PacketSize)
    {
      Packet A0i = ploadu<Packet>(a0It);  a0It  += PacketSize;
      Packet A1i = ploadu<Packet>(a1It);  a1It  += PacketSize;
      Packet Bi  = ploadu<Packet>(rhsIt); rhsIt += PacketSize; // FIXME should be aligned in most cases
      Packet Xi  = pload <Packet>(resIt);

      Xi    = pcj0.pmadd(A0i,ptmp0, pcj0.pmadd(A1i,ptmp1,Xi));
      ptmp2 = pcj1.pmadd(A0i,  Bi, ptmp2);
      ptmp3 = pcj1.pmadd(A1i,  Bi, ptmp3);
      pstore(resIt,Xi); resIt += PacketSize;
    }
    for (size_t i=alignedEnd; i<endi; i++)
    {
      res[i] += cj0.pmul(A0[i], t0) + cj0.pmul(A1[i],t1);
      t2 += cj1.pmul(A0[i], rhs[i]);
      t3 += cj1.pmul(A1[i], rhs[i]);
    }

    res[j]   += alpha * (t2 + predux(ptmp2));
    res[j+1] += alpha * (t3 + predux(ptmp3));
  }
  for (Index j=FirstTriangular ? 0 : bound;j<(FirstTriangular ? bound : size);j++)
  {
    const Scalar* EIGEN_RESTRICT A0 = lhs + j*lhsStride;

    Scalar t1 = cjAlpha * rhs[j];
    Scalar t2(0);
    // TODO make sure this product is a real * complex and that the rhs is properly conjugated if needed
    res[j] += cjd.pmul(numext::real(A0[j]), t1);
    for (Index i=FirstTriangular ? 0 : j+1; i<(FirstTriangular ? j : size); i++)
    {
      res[i] += cj0.pmul(A0[i], t1);
      t2 += cj1.pmul(A0[i], rhs[i]);
    }
    res[j] += alpha * t2;
  }
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x58, %rsp
movq %rdx, -0x38(%rbp)
movq %rsi, -0x30(%rbp)
movq %rdi, %rax
shrq $0x3d, %rax
jne 0x306f0
movq %r9, %rbx
movq %r8, %r15
movq %rcx, %r13
cmpq $0x1, %r8
sete %al
testq %rcx, %rcx
setne %cl
testb %al, %cl
movq %rdi, -0x40(%rbp)
jne 0x302d2
leaq (,%rdi,8), %rcx
cmpq $0x9c4, %rdi            # imm = 0x9C4
ja 0x302dc
movq %rsp, %rax
addq $0xf, %rcx
andq $-0x10, %rcx
subq %rcx, %rax
movq %rax, %rsp
jmp 0x302fa
movq $0x0, -0x48(%rbp)
jmp 0x3032e
movq %rcx, %rdi
movaps %xmm0, -0x60(%rbp)
callq 0xc750
movq -0x40(%rbp), %rdi
movapd -0x60(%rbp), %xmm0
testq %rax, %rax
je 0x306f0
cmpq $0x1, %r15
sete %dl
testq %rdi, %rdi
sete %cl
orb %dl, %cl
movq %rax, -0x48(%rbp)
jne 0x3032b
shlq $0x3, %r15
xorl %ecx, %ecx
movsd (%r13), %xmm1
movsd %xmm1, (%rax,%rcx,8)
incq %rcx
addq %r15, %r13
cmpq %rcx, %rdi
jne 0x30315
movq %rax, %r13
cmpq $0x9, %rdi
movl $0x8, %eax
cmovgeq %rdi, %rax
movl $0xfffffff8, %ecx       # imm = 0xFFFFFFF8
addq %rax, %rcx
movl $0xfffffffe, %r14d      # imm = 0xFFFFFFFE
movq %rcx, -0x68(%rbp)
andq %rcx, %r14
je 0x305d9
movapd %xmm0, %xmm13
unpcklpd %xmm0, %xmm13           # xmm13 = xmm13[0],xmm0[0]
leaq 0x10(%r13), %rdx
leaq 0x10(%rbx), %rsi
movq -0x38(%rbp), %rax
movq -0x30(%rbp), %r12
leaq (%r12,%rax,8), %r8
addq $0x10, %r8
movq %rax, %rcx
shlq $0x4, %rcx
movq %rcx, -0x78(%rbp)
addq $0x10, %rcx
movq %rcx, -0x70(%rbp)
leaq 0x10(%r12), %r11
leaq (%r12,%rax,8), %r15
xorl %eax, %eax
xorpd %xmm1, %xmm1
movq %r14, -0x80(%rbp)
movq %rdx, -0x60(%rbp)
movq %rax, %rcx
movq -0x38(%rbp), %r10
imulq %r10, %rcx
movq %rax, %r9
orq $0x1, %r9
imulq %r10, %r9
leaq 0x2(%rax), %rdx
leal (%rbx,%rdx,8), %r14d
movq %rdi, %r10
subq %rdx, %r10
shrl $0x3, %r14d
andl $0x1, %r14d
cmpq %r10, %r14
cmovaeq %r10, %r14
testb $0x7, %bl
movq -0x30(%rbp), %rdi
leaq (%rdi,%rcx,8), %rcx
leaq (%rdi,%r9,8), %r9
movupd (%r13,%rax,8), %xmm2
mulpd %xmm13, %xmm2
movapd %xmm2, %xmm4
unpckhpd %xmm2, %xmm4            # xmm4 = xmm4[1],xmm2[1]
cmovneq %r10, %r14
movq %rdx, %r10
movsd (%rcx,%rax,8), %xmm3
mulsd %xmm2, %xmm3
addsd (%rbx,%rax,8), %xmm3
movsd %xmm3, (%rbx,%rax,8)
movsd 0x8(%r9,%rax,8), %xmm3
mulsd %xmm4, %xmm3
addsd 0x8(%rbx,%rax,8), %xmm3
movsd %xmm3, 0x8(%rbx,%rax,8)
movsd 0x8(%rcx,%rax,8), %xmm5
mulsd %xmm2, %xmm5
addsd %xmm3, %xmm5
movsd %xmm5, 0x8(%rbx,%rax,8)
movsd 0x8(%rcx,%rax,8), %xmm3
mulsd 0x8(%r13,%rax,8), %xmm3
leaq (%r14,%rax), %rcx
addq $0x2, %rcx
addsd %xmm1, %xmm3
movq %xmm3, %xmm3            # xmm3 = xmm3[0],zero
cmpq %rcx, %rdx
jae 0x304b2
xorl %r9d, %r9d
movq -0x60(%rbp), %rdx
movsd (%r8,%r9,8), %xmm5
mulsd %xmm4, %xmm5
movsd (%r11,%r9,8), %xmm6
mulsd %xmm2, %xmm6
addsd %xmm5, %xmm6
addsd (%rsi,%r9,8), %xmm6
movsd %xmm6, (%rsi,%r9,8)
movsd (%r11,%r9,8), %xmm5
movhpd (%r8,%r9,8), %xmm5      # xmm5 = xmm5[0],mem[0]
movsd (%rdx,%r9,8), %xmm6
unpcklpd %xmm6, %xmm6            # xmm6 = xmm6[0,0]
mulpd %xmm5, %xmm6
addpd %xmm6, %xmm3
incq %r9
cmpq %r9, %r14
jne 0x30466
jmp 0x304b6
movq -0x60(%rbp), %rdx
movq -0x40(%rbp), %rdi
movq %rdi, %r14
subq %rcx, %r14
andq $-0x2, %r14
addq %rcx, %r14
xorpd %xmm6, %xmm6
xorpd %xmm5, %xmm5
cmpq %r14, %rcx
jae 0x30581
movapd %xmm2, %xmm7
unpcklpd %xmm2, %xmm7            # xmm7 = xmm7[0],xmm2[0]
movupd (%r12,%rcx,8), %xmm8
movupd (%r15,%rcx,8), %xmm9
movupd (%r13,%rcx,8), %xmm10
movapd %xmm4, %xmm11
mulpd %xmm9, %xmm11
addpd (%rbx,%rcx,8), %xmm11
movapd %xmm7, %xmm12
mulpd %xmm8, %xmm12
addpd %xmm11, %xmm12
mulpd %xmm10, %xmm8
addpd %xmm8, %xmm5
mulpd %xmm10, %xmm9
addpd %xmm9, %xmm6
movapd %xmm12, (%rbx,%rcx,8)
addq $0x2, %rcx
cmpq %r14, %rcx
jb 0x304e0
jmp 0x30581
movsd (%r12,%r14,8), %xmm4
movhpd (%r15,%r14,8), %xmm4    # xmm4 = xmm4[0],mem[0]
mulpd %xmm2, %xmm4
movapd %xmm4, %xmm7
unpckhpd %xmm4, %xmm7            # xmm7 = xmm7[1],xmm4[1]
addsd %xmm4, %xmm7
addsd (%rbx,%r14,8), %xmm7
movsd %xmm7, (%rbx,%r14,8)
movsd (%r13,%r14,8), %xmm4
unpcklpd %xmm4, %xmm4            # xmm4 = xmm4[0,0]
movsd (%r12,%r14,8), %xmm7
movhpd (%r15,%r14,8), %xmm7    # xmm7 = xmm7[0],mem[0]
mulpd %xmm4, %xmm7
addpd %xmm7, %xmm3
incq %r14
cmpq %rdi, %r14
jb 0x30537
movapd %xmm5, %xmm2
unpcklpd %xmm6, %xmm2            # xmm2 = xmm2[0],xmm6[0]
unpckhpd %xmm6, %xmm5            # xmm5 = xmm5[1],xmm6[1]
addpd %xmm2, %xmm5
addpd %xmm3, %xmm5
movupd (%rbx,%rax,8), %xmm2
mulpd %xmm13, %xmm5
addpd %xmm2, %xmm5
movupd %xmm5, (%rbx,%rax,8)
addq $0x10, %rdx
addq $0x10, %rsi
movq -0x70(%rbp), %rax
addq %rax, %r8
addq %rax, %r11
movq -0x78(%rbp), %rax
addq %rax, %r15
addq %rax, %r12
movq %r10, %rax
movq -0x80(%rbp), %r14
cmpq %r14, %r10
jb 0x3039e
cmpq %rdi, %r14
jge 0x306cf
movq -0x68(%rbp), %r8
shrl %r8d
movq %r8, %rcx
shlq $0x4, %rcx
leaq (%rcx,%r13), %rdx
addq %rbx, %rcx
movq -0x38(%rbp), %rax
movq %rax, %rsi
shlq $0x4, %rsi
addq $0x10, %rsi
imulq %r8, %rsi
addq -0x30(%rbp), %rsi
leaq 0x8(,%rax,8), %rax
movq %r14, %r8
imulq -0x38(%rbp), %r8
movq -0x30(%rbp), %r9
leaq (%r9,%r8,8), %r8
movsd (%r13,%r14,8), %xmm3
mulsd %xmm0, %xmm3
movsd (%r8,%r14,8), %xmm1
mulsd %xmm3, %xmm1
addsd (%rbx,%r14,8), %xmm1
movsd %xmm1, (%rbx,%r14,8)
leaq 0x1(%r14), %r8
cmpq %rdi, %r8
jge 0x306bd
xorpd %xmm1, %xmm1
movl $0x1, %r9d
movsd (%rsi,%r9,8), %xmm2
mulsd %xmm3, %xmm2
addsd (%rcx,%r9,8), %xmm2
movsd %xmm2, (%rcx,%r9,8)
movsd (%rsi,%r9,8), %xmm2
mulsd (%rdx,%r9,8), %xmm2
addsd %xmm2, %xmm1
leaq (%r14,%r9), %r10
incq %r10
incq %r9
cmpq -0x40(%rbp), %r10
movq -0x40(%rbp), %rdi
jl 0x3065a
mulsd %xmm0, %xmm1
addsd (%rbx,%r14,8), %xmm1
movsd %xmm1, (%rbx,%r14,8)
addq $0x8, %rdx
addq $0x8, %rcx
addq %rax, %rsi
movq %r8, %r14
cmpq %rdi, %r8
jl 0x30616
jmp 0x306cf
xorpd %xmm2, %xmm2
mulsd %xmm2, %xmm0
addsd %xmm1, %xmm0
movsd %xmm0, (%rbx,%r14,8)
cmpq $0x9c5, %rdi            # imm = 0x9C5
jb 0x306e1
movq -0x48(%rbp), %rdi
callq 0xc660
leaq -0x28(%rbp), %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movl $0x8, %edi
callq 0xc260
movq 0x4786f(%rip), %rcx     # 0x77f70
addq $0x10, %rcx
movq %rcx, (%rax)
movq 0x47891(%rip), %rsi     # 0x77fa0
movq 0x4782a(%rip), %rdx     # 0x77f40
movq %rax, %rdi
callq 0xc990