// File: /workspace/github_cpp_makefile_star2_O2/j-faria[P]kima/eigen/Eigen/src/Core/products/GeneralMatrixVector.h
// Function: Eigen::internal::general_matrix_vector_product<long, double, Eigen::internal::const_blas_data_mapper<double, long, 0>, 0, false, double, Eigen::internal::const_blas_data_mapper<double, long, 0>, false, 0>::run(long, long, Eigen::internal::const_blas_data_mapper<double, long, 0> const&, Eigen::internal::const_blas_data_mapper<double, long, 0> const&, double*, long, double)
EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product<Index,LhsScalar,LhsMapper,ColMajor,ConjugateLhs,RhsScalar,RhsMapper,ConjugateRhs,Version>::run(
  Index rows, Index cols,
  const LhsMapper& alhs,
  const RhsMapper& rhs,
        ResScalar* res, Index resIncr,
  RhsScalar alpha)
{
  EIGEN_UNUSED_VARIABLE(resIncr);
  eigen_internal_assert(resIncr==1);

  // The following copy tells the compiler that lhs's attributes are not modified outside this function
  // This helps GCC to generate propoer code.
  LhsMapper lhs(alhs);

  conj_helper<LhsScalar,RhsScalar,ConjugateLhs,ConjugateRhs> cj;
  conj_helper<LhsPacket,RhsPacket,ConjugateLhs,ConjugateRhs> pcj;
  conj_helper<LhsPacketHalf,RhsPacketHalf,ConjugateLhs,ConjugateRhs> pcj_half;
  conj_helper<LhsPacketQuarter,RhsPacketQuarter,ConjugateLhs,ConjugateRhs> pcj_quarter;

  const Index lhsStride = lhs.stride();
  // TODO: for padded aligned inputs, we could enable aligned reads
  enum { LhsAlignment = Unaligned,
         ResPacketSize = Traits::ResPacketSize,
         ResPacketSizeHalf = HalfTraits::ResPacketSize,
         ResPacketSizeQuarter = QuarterTraits::ResPacketSize,
         LhsPacketSize = Traits::LhsPacketSize,
         HasHalf = (int)ResPacketSizeHalf < (int)ResPacketSize,
         HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf
  };

  const Index n8 = rows-8*ResPacketSize+1;
  const Index n4 = rows-4*ResPacketSize+1;
  const Index n3 = rows-3*ResPacketSize+1;
  const Index n2 = rows-2*ResPacketSize+1;
  const Index n1 = rows-1*ResPacketSize+1;
  const Index n_half = rows-1*ResPacketSizeHalf+1;
  const Index n_quarter = rows-1*ResPacketSizeQuarter+1;

  // TODO: improve the following heuristic:
  const Index block_cols = cols<128 ? cols : (lhsStride*sizeof(LhsScalar)<32000?16:4);
  ResPacket palpha = pset1<ResPacket>(alpha);
  ResPacketHalf palpha_half = pset1<ResPacketHalf>(alpha);
  ResPacketQuarter palpha_quarter = pset1<ResPacketQuarter>(alpha);

  for(Index j2=0; j2<cols; j2+=block_cols)
  {
    Index jend = numext::mini(j2+block_cols,cols);
    Index i=0;
    for(; i<n8; i+=ResPacketSize*8)
    {
      ResPacket c0 = pset1<ResPacket>(ResScalar(0)),
                c1 = pset1<ResPacket>(ResScalar(0)),
                c2 = pset1<ResPacket>(ResScalar(0)),
                c3 = pset1<ResPacket>(ResScalar(0)),
                c4 = pset1<ResPacket>(ResScalar(0)),
                c5 = pset1<ResPacket>(ResScalar(0)),
                c6 = pset1<ResPacket>(ResScalar(0)),
                c7 = pset1<ResPacket>(ResScalar(0));

      for(Index j=j2; j<jend; j+=1)
      {
        RhsPacket b0 = pset1<RhsPacket>(rhs(j,0));
        c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0);
        c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1);
        c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*2,j),b0,c2);
        c3 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*3,j),b0,c3);
        c4 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*4,j),b0,c4);
        c5 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*5,j),b0,c5);
        c6 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*6,j),b0,c6);
        c7 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*7,j),b0,c7);
      }
      pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0)));
      pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1)));
      pstoreu(res+i+ResPacketSize*2, pmadd(c2,palpha,ploadu<ResPacket>(res+i+ResPacketSize*2)));
      pstoreu(res+i+ResPacketSize*3, pmadd(c3,palpha,ploadu<ResPacket>(res+i+ResPacketSize*3)));
      pstoreu(res+i+ResPacketSize*4, pmadd(c4,palpha,ploadu<ResPacket>(res+i+ResPacketSize*4)));
      pstoreu(res+i+ResPacketSize*5, pmadd(c5,palpha,ploadu<ResPacket>(res+i+ResPacketSize*5)));
      pstoreu(res+i+ResPacketSize*6, pmadd(c6,palpha,ploadu<ResPacket>(res+i+ResPacketSize*6)));
      pstoreu(res+i+ResPacketSize*7, pmadd(c7,palpha,ploadu<ResPacket>(res+i+ResPacketSize*7)));
    }
    if(i<n4)
    {
      ResPacket c0 = pset1<ResPacket>(ResScalar(0)),
                c1 = pset1<ResPacket>(ResScalar(0)),
                c2 = pset1<ResPacket>(ResScalar(0)),
                c3 = pset1<ResPacket>(ResScalar(0));

      for(Index j=j2; j<jend; j+=1)
      {
        RhsPacket b0 = pset1<RhsPacket>(rhs(j,0));
        c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0);
        c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1);
        c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*2,j),b0,c2);
        c3 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*3,j),b0,c3);
      }
      pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0)));
      pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1)));
      pstoreu(res+i+ResPacketSize*2, pmadd(c2,palpha,ploadu<ResPacket>(res+i+ResPacketSize*2)));
      pstoreu(res+i+ResPacketSize*3, pmadd(c3,palpha,ploadu<ResPacket>(res+i+ResPacketSize*3)));

      i+=ResPacketSize*4;
    }
    if(i<n3)
    {
      ResPacket c0 = pset1<ResPacket>(ResScalar(0)),
                c1 = pset1<ResPacket>(ResScalar(0)),
                c2 = pset1<ResPacket>(ResScalar(0));

      for(Index j=j2; j<jend; j+=1)
      {
        RhsPacket b0 = pset1<RhsPacket>(rhs(j,0));
        c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0);
        c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1);
        c2 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*2,j),b0,c2);
      }
      pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0)));
      pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1)));
      pstoreu(res+i+ResPacketSize*2, pmadd(c2,palpha,ploadu<ResPacket>(res+i+ResPacketSize*2)));

      i+=ResPacketSize*3;
    }
    if(i<n2)
    {
      ResPacket c0 = pset1<ResPacket>(ResScalar(0)),
                c1 = pset1<ResPacket>(ResScalar(0));

      for(Index j=j2; j<jend; j+=1)
      {
        RhsPacket b0 = pset1<RhsPacket>(rhs(j,0));
        c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*0,j),b0,c0);
        c1 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+LhsPacketSize*1,j),b0,c1);
      }
      pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0)));
      pstoreu(res+i+ResPacketSize*1, pmadd(c1,palpha,ploadu<ResPacket>(res+i+ResPacketSize*1)));
      i+=ResPacketSize*2;
    }
    if(i<n1)
    {
      ResPacket c0 = pset1<ResPacket>(ResScalar(0));
      for(Index j=j2; j<jend; j+=1)
      {
        RhsPacket b0 = pset1<RhsPacket>(rhs(j,0));
        c0 = pcj.pmadd(lhs.template load<LhsPacket,LhsAlignment>(i+0,j),b0,c0);
      }
      pstoreu(res+i+ResPacketSize*0, pmadd(c0,palpha,ploadu<ResPacket>(res+i+ResPacketSize*0)));
      i+=ResPacketSize;
    }
    if(HasHalf && i<n_half)
    {
      ResPacketHalf c0 = pset1<ResPacketHalf>(ResScalar(0));
      for(Index j=j2; j<jend; j+=1)
      {
        RhsPacketHalf b0 = pset1<RhsPacketHalf>(rhs(j,0));
        c0 = pcj_half.pmadd(lhs.template load<LhsPacketHalf,LhsAlignment>(i+0,j),b0,c0);
      }
      pstoreu(res+i+ResPacketSizeHalf*0, pmadd(c0,palpha_half,ploadu<ResPacketHalf>(res+i+ResPacketSizeHalf*0)));
      i+=ResPacketSizeHalf;
    }
    if(HasQuarter && i<n_quarter)
    {
      ResPacketQuarter c0 = pset1<ResPacketQuarter>(ResScalar(0));
      for(Index j=j2; j<jend; j+=1)
      {
        RhsPacketQuarter b0 = pset1<RhsPacketQuarter>(rhs(j,0));
        c0 = pcj_quarter.pmadd(lhs.template load<LhsPacketQuarter,LhsAlignment>(i+0,j),b0,c0);
      }
      pstoreu(res+i+ResPacketSizeQuarter*0, pmadd(c0,palpha_quarter,ploadu<ResPacketQuarter>(res+i+ResPacketSizeQuarter*0)));
      i+=ResPacketSizeQuarter;
    }
    for(;i<rows;++i)
    {
      ResScalar c0(0);
      for(Index j=j2; j<jend; j+=1)
        c0 += cj.pmul(lhs(i,j), rhs(j,0));
      res[i] += alpha*c0;
    }
  }
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
movq (%rdx), %r10
movq 0x8(%rdx), %rdx
leaq -0xf(%rdi), %r9
leaq -0x7(%rdi), %rax
movq %rax, -0x20(%rsp)
leaq -0x5(%rdi), %rax
movq %rax, -0x28(%rsp)
leaq -0x3(%rdi), %rax
movq %rax, -0x30(%rsp)
leaq (,%rdx,8), %r14
cmpq $0x7d00, %r14           # imm = 0x7D00
movabsq $0x10, %rax
movabsq $0x4, %r11
cmovbq %rax, %r11
leaq -0x1(%rdi), %rax
movq %rax, -0x40(%rsp)
cmpq $0x80, %rsi
cmovlq %rsi, %r11
movapd %xmm0, %xmm1
unpcklpd %xmm0, %xmm1            # xmm1 = xmm1[0],xmm0[0]
leaq 0x70(%r10), %rbx
movq %r11, -0x38(%rsp)
imulq %rdx, %r11
movq %r11, -0x48(%rsp)
leaq (,%r11,8), %rax
movq %rax, -0x50(%rsp)
leaq 0x30(%r10), %rax
movq %rax, -0x60(%rsp)
leaq 0x20(%r10), %rax
movq %rax, -0x68(%rsp)
movabsq $0x2, %rax
movq %rax, -0x70(%rsp)
xorl %r11d, %r11d
movl $0x80, %r12d
movq %r10, -0x58(%rsp)
movq %r10, -0x78(%rsp)
movq %rsi, -0x18(%rsp)
cmpq %rsi, %r11
jge 0x1cfcb
movq -0x38(%rsp), %rax
addq %r11, %rax
cmpq %rsi, %rax
movq %rax, -0x10(%rsp)
cmovlq %rax, %rsi
movq -0x60(%rsp), %r10
movq %rbx, -0x8(%rsp)
movq %rbx, %rax
xorl %ebx, %ebx
cmpq %r9, %rbx
jge 0x1ccea
movq (%rcx), %rbp
xorpd %xmm2, %xmm2
movq %rax, %r13
xorpd %xmm3, %xmm3
xorpd %xmm4, %xmm4
xorpd %xmm5, %xmm5
xorpd %xmm6, %xmm6
xorpd %xmm7, %xmm7
xorpd %xmm8, %xmm8
xorpd %xmm9, %xmm9
movq %r11, %r15
cmpq %rsi, %r15
jge 0x1cc27
movsd (%rbp,%r15,8), %xmm10
unpcklpd %xmm10, %xmm10          # xmm10 = xmm10[0,0]
movupd -0x70(%r13), %xmm11
movupd -0x60(%r13), %xmm12
movupd -0x50(%r13), %xmm13
movupd -0x40(%r13), %xmm14
mulpd %xmm10, %xmm11
addpd %xmm11, %xmm9
mulpd %xmm10, %xmm12
addpd %xmm12, %xmm8
mulpd %xmm10, %xmm13
addpd %xmm13, %xmm7
mulpd %xmm10, %xmm14
addpd %xmm14, %xmm6
movupd -0x30(%r13), %xmm11
mulpd %xmm10, %xmm11
addpd %xmm11, %xmm5
movupd -0x20(%r13), %xmm11
mulpd %xmm10, %xmm11
addpd %xmm11, %xmm4
movupd -0x10(%r13), %xmm11
mulpd %xmm10, %xmm11
addpd %xmm11, %xmm3
movupd (%r13), %xmm11
mulpd %xmm10, %xmm11
addpd %xmm11, %xmm2
incq %r15
addq %r14, %r13
jmp 0x1cb87
mulpd %xmm1, %xmm9
movupd (%r8,%rbx,8), %xmm10
addpd %xmm9, %xmm10
movupd 0x10(%r8,%rbx,8), %xmm9
movupd 0x20(%r8,%rbx,8), %xmm11
movupd 0x30(%r8,%rbx,8), %xmm12
movupd %xmm10, (%r8,%rbx,8)
mulpd %xmm1, %xmm8
addpd %xmm9, %xmm8
movupd %xmm8, 0x10(%r8,%rbx,8)
mulpd %xmm1, %xmm7
addpd %xmm11, %xmm7
movupd %xmm7, 0x20(%r8,%rbx,8)
mulpd %xmm1, %xmm6
addpd %xmm12, %xmm6
movupd %xmm6, 0x30(%r8,%rbx,8)
movupd 0x40(%r8,%rbx,8), %xmm6
mulpd %xmm1, %xmm5
addpd %xmm6, %xmm5
movupd %xmm5, 0x40(%r8,%rbx,8)
movupd 0x50(%r8,%rbx,8), %xmm5
mulpd %xmm1, %xmm4
addpd %xmm5, %xmm4
movupd %xmm4, 0x50(%r8,%rbx,8)
movupd 0x60(%r8,%rbx,8), %xmm4
mulpd %xmm1, %xmm3
addpd %xmm4, %xmm3
movupd %xmm3, 0x60(%r8,%rbx,8)
movupd 0x70(%r8,%rbx,8), %xmm3
mulpd %xmm1, %xmm2
addpd %xmm3, %xmm2
movupd %xmm2, 0x70(%r8,%rbx,8)
addq $0x10, %rbx
addq %r12, %rax
addq %r12, %r10
jmp 0x1cb53
cmpq -0x20(%rsp), %rbx
jge 0x1cdba
movq (%rcx), %rax
xorpd %xmm2, %xmm2
xorpd %xmm3, %xmm3
xorpd %xmm4, %xmm4
xorpd %xmm5, %xmm5
movq %r11, %r15
cmpq %rsi, %r15
jge 0x1cd5f
movsd (%rax,%r15,8), %xmm6
unpcklpd %xmm6, %xmm6            # xmm6 = xmm6[0,0]
movupd -0x30(%r10), %xmm7
movupd -0x20(%r10), %xmm8
movupd -0x10(%r10), %xmm9
movupd (%r10), %xmm10
mulpd %xmm6, %xmm7
addpd %xmm7, %xmm5
mulpd %xmm6, %xmm8
addpd %xmm8, %xmm4
mulpd %xmm6, %xmm9
addpd %xmm9, %xmm3
mulpd %xmm6, %xmm10
addpd %xmm10, %xmm2
incq %r15
addq %r14, %r10
jmp 0x1cd0b
mulpd %xmm1, %xmm5
movupd (%r8,%rbx,8), %xmm6
addpd %xmm5, %xmm6
movupd 0x10(%r8,%rbx,8), %xmm5
movupd 0x20(%r8,%rbx,8), %xmm7
movupd 0x30(%r8,%rbx,8), %xmm8
movupd %xmm6, (%r8,%rbx,8)
mulpd %xmm1, %xmm4
addpd %xmm5, %xmm4
movupd %xmm4, 0x10(%r8,%rbx,8)
mulpd %xmm1, %xmm3
addpd %xmm7, %xmm3
movupd %xmm3, 0x20(%r8,%rbx,8)
mulpd %xmm1, %xmm2
addpd %xmm8, %xmm2
movupd %xmm2, 0x30(%r8,%rbx,8)
orq $0x8, %rbx
cmpq -0x28(%rsp), %rbx
jge 0x1ce66
movq (%rcx), %rax
movq -0x68(%rsp), %r10
leaq (%r10,%rbx,8), %r10
xorpd %xmm2, %xmm2
xorpd %xmm3, %xmm3
xorpd %xmm4, %xmm4
movq %r11, %r15
cmpq %rsi, %r15
jge 0x1ce22
movsd (%rax,%r15,8), %xmm5
unpcklpd %xmm5, %xmm5            # xmm5 = xmm5[0,0]
movupd -0x20(%r10), %xmm6
movupd -0x10(%r10), %xmm7
movupd (%r10), %xmm8
mulpd %xmm5, %xmm6
addpd %xmm6, %xmm4
mulpd %xmm5, %xmm7
addpd %xmm7, %xmm3
mulpd %xmm5, %xmm8
addpd %xmm8, %xmm2
incq %r15
addq %r14, %r10
jmp 0x1cde0
mulpd %xmm1, %xmm4
movupd (%r8,%rbx,8), %xmm5
addpd %xmm4, %xmm5
movupd 0x10(%r8,%rbx,8), %xmm4
movupd 0x20(%r8,%rbx,8), %xmm6
movupd %xmm5, (%r8,%rbx,8)
mulpd %xmm1, %xmm3
addpd %xmm4, %xmm3
movupd %xmm3, 0x10(%r8,%rbx,8)
mulpd %xmm1, %xmm2
addpd %xmm6, %xmm2
movupd %xmm2, 0x20(%r8,%rbx,8)
addq $0x6, %rbx
cmpq -0x30(%rsp), %rbx
jge 0x1ceeb
movq (%rcx), %rax
movq -0x58(%rsp), %r10
leaq (%r10,%rbx,8), %r10
xorpd %xmm3, %xmm3
movq -0x70(%rsp), %r15
xorpd %xmm2, %xmm2
movq %r11, %r13
cmpq %rsi, %r13
jge 0x1cebd
movsd (%rax,%r13,8), %xmm4
unpcklpd %xmm4, %xmm4            # xmm4 = xmm4[0,0]
movupd -0x10(%r10,%r15,8), %xmm5
movupd (%r10,%r15,8), %xmm6
mulpd %xmm4, %xmm5
addpd %xmm5, %xmm3
mulpd %xmm4, %xmm6
addpd %xmm6, %xmm2
incq %r13
addq %rdx, %r15
jmp 0x1ce89
mulpd %xmm1, %xmm3
movupd (%r8,%rbx,8), %xmm4
addpd %xmm3, %xmm4
movupd 0x10(%r8,%rbx,8), %xmm3
movupd %xmm4, (%r8,%rbx,8)
mulpd %xmm1, %xmm2
addpd %xmm3, %xmm2
movupd %xmm2, 0x10(%r8,%rbx,8)
addq $0x4, %rbx
cmpq -0x40(%rsp), %rbx
jge 0x1cf41
movq (%rcx), %rax
movq -0x78(%rsp), %r10
leaq (%r10,%rbx,8), %r10
xorpd %xmm2, %xmm2
movq %r11, %r15
cmpq %rsi, %r15
jge 0x1cf29
movsd (%rax,%r15,8), %xmm3
unpcklpd %xmm3, %xmm3            # xmm3 = xmm3[0,0]
movupd (%r10), %xmm4
mulpd %xmm3, %xmm4
addpd %xmm4, %xmm2
incq %r15
addq %r14, %r10
jmp 0x1cf05
movupd (%r8,%rbx,8), %xmm3
mulpd %xmm1, %xmm2
addpd %xmm3, %xmm2
movupd %xmm2, (%r8,%rbx,8)
addq $0x2, %rbx
movq -0x78(%rsp), %rax
leaq (%rax,%rbx,8), %rax
cmpq %rdi, %rbx
jge 0x1cf91
movq (%rcx), %r10
xorpd %xmm2, %xmm2
movq %rax, %r15
movq %r11, %r13
cmpq %rsi, %r13
jge 0x1cf78
movsd (%r15), %xmm3
mulsd (%r10,%r13,8), %xmm3
addsd %xmm3, %xmm2
incq %r13
addq %r14, %r15
jmp 0x1cf5c
mulsd %xmm0, %xmm2
addsd (%r8,%rbx,8), %xmm2
movsd %xmm2, (%r8,%rbx,8)
incq %rbx
addq $0x8, %rax
jmp 0x1cf4a
movq -0x8(%rsp), %rbx
movq -0x50(%rsp), %rax
addq %rax, %rbx
addq %rax, -0x60(%rsp)
addq %rax, -0x68(%rsp)
movq -0x70(%rsp), %rsi
addq -0x48(%rsp), %rsi
movq %rsi, -0x70(%rsp)
addq %rax, -0x78(%rsp)
movq -0x10(%rsp), %r11
movq -0x18(%rsp), %rsi
jmp 0x1cb27
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq