// File: Kingsford-Group[P]ghost2/include/Eigen/src/Core/products/GeneralMatrixMatrix.h
// Function: Eigen::internal::general_matrix_matrix_product<long, double, 0, false, double, 0, false, 0>::run(long, long, long, double const*, long, double const*, long, double*, long, double, Eigen::internal::level3_blocking<double, double>&, Eigen::internal::GemmParallelInfo<long>*)
static void run(Index rows, Index cols, Index depth,
  const LhsScalar* _lhs, Index lhsStride,
  const RhsScalar* _rhs, Index rhsStride,
  ResScalar* res, Index resStride,
  ResScalar alpha,
  level3_blocking<LhsScalar,RhsScalar>& blocking,
  GemmParallelInfo<Index>* info = 0)
{
  const_blas_data_mapper<LhsScalar, Index, LhsStorageOrder> lhs(_lhs,lhsStride);
  const_blas_data_mapper<RhsScalar, Index, RhsStorageOrder> rhs(_rhs,rhsStride);

  typedef gebp_traits<LhsScalar,RhsScalar> Traits;

  Index kc = blocking.kc();                   // cache block size along the K direction
  Index mc = (std::min)(rows,blocking.mc());  // cache block size along the M direction
  //Index nc = blocking.nc(); // cache block size along the N direction

  gemm_pack_lhs<LhsScalar, Index, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs;
  gemm_pack_rhs<RhsScalar, Index, Traits::nr, RhsStorageOrder> pack_rhs;
  gebp_kernel<LhsScalar, RhsScalar, Index, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp;

#ifdef EIGEN_HAS_OPENMP
  if(info)
  {
    // this is the parallel version!
    Index tid = omp_get_thread_num();
    Index threads = omp_get_num_threads();
    
    std::size_t sizeA = kc*mc;
    std::size_t sizeW = kc*Traits::WorkSpaceFactor;
    ei_declare_aligned_stack_constructed_variable(LhsScalar, blockA, sizeA, 0);
    ei_declare_aligned_stack_constructed_variable(RhsScalar, w, sizeW, 0);
    
    RhsScalar* blockB = blocking.blockB();
    eigen_internal_assert(blockB!=0);

    // For each horizontal panel of the rhs, and corresponding vertical panel of the lhs...
    for(Index k=0; k<depth; k+=kc)
    {
      const Index actual_kc = (std::min)(k+kc,depth)-k; // => rows of B', and cols of the A'

      // In order to reduce the chance that a thread has to wait for the other,
      // let's start by packing A'.
      pack_lhs(blockA, &lhs(0,k), lhsStride, actual_kc, mc);

      // Pack B_k to B' in a parallel fashion:
      // each thread packs the sub block B_k,j to B'_j where j is the thread id.

      // However, before copying to B'_j, we have to make sure that no other thread is still using it,
      // i.e., we test that info[tid].users equals 0.
      // Then, we set info[tid].users to the number of threads to mark that all other threads are going to use it.
      while(info[tid].users!=0) {}
      info[tid].users += threads;

      pack_rhs(blockB+info[tid].rhs_start*actual_kc, &rhs(k,info[tid].rhs_start), rhsStride, actual_kc, info[tid].rhs_length);

      // Notify the other threads that the part B'_j is ready to go.
      info[tid].sync = k;

      // Computes C_i += A' * B' per B'_j
      for(Index shift=0; shift<threads; ++shift)
      {
        Index j = (tid+shift)%threads;

        // At this point we have to make sure that B'_j has been updated by the thread j,
        // we use testAndSetOrdered to mimic a volatile access.
        // However, no need to wait for the B' part which has been updated by the current thread!
        if(shift>0)
          while(info[j].sync!=k) {}

        gebp(res+info[j].rhs_start*resStride, resStride, blockA, blockB+info[j].rhs_start*actual_kc, mc, actual_kc, info[j].rhs_length, alpha, -1,-1,0,0, w);
      }

      // Then keep going as usual with the remaining A'
      for(Index i=mc; i<rows; i+=mc)
      {
        const Index actual_mc = (std::min)(i+mc,rows)-i;

        // pack A_i,k to A'
        pack_lhs(blockA, &lhs(i,k), lhsStride, actual_kc, actual_mc);

        // C_i += A' * B'
        gebp(res+i, resStride, blockA, blockB, actual_mc, actual_kc, cols, alpha, -1,-1,0,0, w);
      }

      // Release all the sub blocks B'_j of B' for the current thread,
      // i.e., we simply decrement the number of users by 1
      for(Index j=0; j<threads; ++j)
        #pragma omp atomic
        --(info[j].users);
    }
  }
  else
#endif // EIGEN_HAS_OPENMP
  {
    EIGEN_UNUSED_VARIABLE(info);

    // this is the sequential version!
    std::size_t sizeA = kc*mc;
    std::size_t sizeB = kc*cols;
    std::size_t sizeW = kc*Traits::WorkSpaceFactor;

    ei_declare_aligned_stack_constructed_variable(LhsScalar, blockA, sizeA, blocking.blockA());
    ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, blocking.blockB());
    ei_declare_aligned_stack_constructed_variable(RhsScalar, blockW, sizeW, blocking.blockW());

    // For each horizontal panel of the rhs, and corresponding panel of the lhs...
    // (==GEMM_VAR1)
    for(Index k2=0; k2<depth; k2+=kc)
    {
      const Index actual_kc = (std::min)(k2+kc,depth)-k2;

      // OK, here we have selected one horizontal panel of rhs and one vertical panel of lhs.
      // => Pack rhs's panel into a sequential chunk of memory (L2 caching)
      // Note that this panel will be read as many times as the number of blocks in the lhs's
      // vertical panel which is, in practice, a very low number.
      pack_rhs(blockB, &rhs(k2,0), rhsStride, actual_kc, cols);

      // For each mc x kc block of the lhs's vertical panel...
      // (==GEPP_VAR1)
      for(Index i2=0; i2<rows; i2+=mc)
      {
        const Index actual_mc = (std::min)(i2+mc,rows)-i2;

        // We pack the lhs's block into a sequential chunk of memory (L1 caching)
        // Note that this block will be read a very high number of times, which is equal to the number of
        // micro vertical panel of the large rhs's panel (e.g., cols/4 times).
        pack_lhs(blockA, &lhs(i2,k2), lhsStride, actual_kc, actual_mc);

        // Everything is packed, we can now call the block * panel kernel:
        gebp(res+i2, resStride, blockA, blockB, actual_mc, actual_kc, cols, alpha, -1, -1, 0, 0, blockW);
      }
    }
  }
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
subq $0x1f0, %rsp            # imm = 0x1F0
movq 0x30(%rbp), %rax
movq 0x28(%rbp), %rax
movq 0x20(%rbp), %rax
movq 0x18(%rbp), %rax
movq 0x10(%rbp), %rax
movq %rdi, -0x20(%rbp)
movq %rsi, -0x28(%rbp)
movq %rdx, -0x30(%rbp)
movq %rcx, -0x38(%rbp)
movq %r8, -0x40(%rbp)
movq %r9, -0x48(%rbp)
movsd %xmm0, -0x50(%rbp)
movq -0x38(%rbp), %rsi
movq -0x40(%rbp), %rdx
leaq -0x60(%rbp), %rdi
callq 0xa8ab0
movq -0x48(%rbp), %rsi
movq 0x10(%rbp), %rdx
leaq -0x70(%rbp), %rdi
callq 0xa8ab0
movq 0x28(%rbp), %rdi
callq 0xa8ae0
movq %rax, -0x78(%rbp)
movq 0x28(%rbp), %rdi
callq 0xa8b00
movq %rax, -0x88(%rbp)
leaq -0x20(%rbp), %rdi
leaq -0x88(%rbp), %rsi
callq 0x37360
movq (%rax), %rax
movq %rax, -0x80(%rbp)
leaq 0x30(%rbp), %rdi
callq 0xa8b20
movq -0x78(%rbp), %rax
imulq -0x80(%rbp), %rax
movq %rax, -0x98(%rbp)
movq -0x78(%rbp), %rax
imulq -0x28(%rbp), %rax
movq %rax, -0xa0(%rbp)
movq -0x78(%rbp), %rax
shlq $0x3, %rax
movq %rax, -0xa8(%rbp)
movq -0x98(%rbp), %rax
movq %rax, -0x8(%rbp)
movabsq $0x1fffffffffffffff, %rax # imm = 0x1FFFFFFFFFFFFFFF
cmpq %rax, -0x8(%rbp)
jbe 0xa844a
callq 0x4a840
movq 0x28(%rbp), %rdi
callq 0xa8b30
cmpq $0x0, %rax
je 0xa846b
movq 0x28(%rbp), %rdi
callq 0xa8b30
movq %rax, -0x150(%rbp)
jmp 0xa84c8
movq -0x98(%rbp), %rax
shlq $0x3, %rax
cmpq $0x4e20, %rax           # imm = 0x4E20
ja 0xa84a3
movq -0x98(%rbp), %rax
leaq 0xf(,%rax,8), %rcx
andq $-0x10, %rcx
movq %rsp, %rax
subq %rcx, %rax
movq %rax, %rsp
movq %rax, -0x158(%rbp)
jmp 0xa84ba
movq -0x98(%rbp), %rdi
shlq $0x3, %rdi
callq 0x4a8c0
movq %rax, -0x158(%rbp)
movq -0x158(%rbp), %rax
movq %rax, -0x150(%rbp)
movq -0x150(%rbp), %rax
movq %rax, -0xb0(%rbp)
movq 0x28(%rbp), %rdi
callq 0xa8b30
cmpq $0x0, %rax
jne 0xa84f5
movq -0xb0(%rbp), %rax
movq %rax, -0x160(%rbp)
jmp 0xa8500
xorl %eax, %eax
movq %rax, -0x160(%rbp)
jmp 0xa8500
movq -0x160(%rbp), %rsi
movq -0x98(%rbp), %rdx
movq -0x98(%rbp), %rax
shlq $0x3, %rax
cmpq $0x4e20, %rax           # imm = 0x4E20
seta %al
movzbl %al, %ecx
andl $0x1, %ecx
leaq -0xc8(%rbp), %rdi
callq 0x541f0
movq -0xa0(%rbp), %rax
movq %rax, -0x10(%rbp)
movabsq $0x1fffffffffffffff, %rax # imm = 0x1FFFFFFFFFFFFFFF
cmpq %rax, -0x10(%rbp)
jbe 0xa8558
callq 0x4a840
jmp 0xa8556
jmp 0xa8558
jmp 0xa855a
movq 0x28(%rbp), %rdi
callq 0xa8b50
movq %rax, -0x168(%rbp)
jmp 0xa856c
movq -0x168(%rbp), %rax
cmpq $0x0, %rax
je 0xa859b
movq 0x28(%rbp), %rdi
callq 0xa8b50
movq %rax, -0x170(%rbp)
jmp 0xa858b
movq -0x170(%rbp), %rax
movq %rax, -0x178(%rbp)
jmp 0xa860a
movq -0xa0(%rbp), %rax
shlq $0x3, %rax
cmpq $0x4e20, %rax           # imm = 0x4E20
ja 0xa85d3
movq -0xa0(%rbp), %rax
leaq 0xf(,%rax,8), %rcx
andq $-0x10, %rcx
movq %rsp, %rax
subq %rcx, %rax
movq %rax, %rsp
movq %rax, -0x180(%rbp)
jmp 0xa85fc
movq -0xa0(%rbp), %rdi
shlq $0x3, %rdi
callq 0x4a8c0
movq %rax, -0x188(%rbp)
jmp 0xa85ec
movq -0x188(%rbp), %rax
movq %rax, -0x180(%rbp)
jmp 0xa85fc
movq -0x180(%rbp), %rax
movq %rax, -0x178(%rbp)
movq -0x178(%rbp), %rax
movq %rax, -0xe0(%rbp)
movq 0x28(%rbp), %rdi
callq 0xa8b50
movq %rax, -0x190(%rbp)
jmp 0xa862a
movq -0x190(%rbp), %rax
cmpq $0x0, %rax
jne 0xa8647
movq -0xe0(%rbp), %rax
movq %rax, -0x198(%rbp)
jmp 0xa8652
xorl %eax, %eax
movq %rax, -0x198(%rbp)
jmp 0xa8652
movq -0x198(%rbp), %rsi
movq -0xa0(%rbp), %rdx
movq %rdx, %rax
shlq $0x3, %rax
subq $0x4e21, %rax           # imm = 0x4E21
setae %al
movzbl %al, %ecx
leaq -0xf8(%rbp), %rdi
callq 0x541f0
jmp 0xa8681
movq -0xa8(%rbp), %rax
movq %rax, -0x18(%rbp)
movabsq $0x1fffffffffffffff, %rax # imm = 0x1FFFFFFFFFFFFFFF
cmpq %rax, -0x18(%rbp)
jbe 0xa86a5
callq 0x4a840
jmp 0xa86a3
jmp 0xa86a5
jmp 0xa86a7
movq 0x28(%rbp), %rdi
callq 0xa8b70
movq %rax, -0x1a0(%rbp)
jmp 0xa86b9
movq -0x1a0(%rbp), %rax
cmpq $0x0, %rax
je 0xa86e8
movq 0x28(%rbp), %rdi
callq 0xa8b70
movq %rax, -0x1a8(%rbp)
jmp 0xa86d8
movq -0x1a8(%rbp), %rax
movq %rax, -0x1b0(%rbp)
jmp 0xa8757
movq -0xa8(%rbp), %rax
shlq $0x3, %rax
cmpq $0x4e20, %rax           # imm = 0x4E20
ja 0xa8720
movq -0xa8(%rbp), %rax
leaq 0xf(,%rax,8), %rcx
andq $-0x10, %rcx
movq %rsp, %rax
subq %rcx, %rax
movq %rax, %rsp
movq %rax, -0x1b8(%rbp)
jmp 0xa8749
movq -0xa8(%rbp), %rdi
shlq $0x3, %rdi
callq 0x4a8c0
movq %rax, -0x1c0(%rbp)
jmp 0xa8739
movq -0x1c0(%rbp), %rax
movq %rax, -0x1b8(%rbp)
jmp 0xa8749
movq -0x1b8(%rbp), %rax
movq %rax, -0x1b0(%rbp)
movq -0x1b0(%rbp), %rax
movq %rax, -0x100(%rbp)
movq 0x28(%rbp), %rdi
callq 0xa8b70
movq %rax, -0x1c8(%rbp)
jmp 0xa8777
movq -0x1c8(%rbp), %rax
cmpq $0x0, %rax
jne 0xa8794
movq -0x100(%rbp), %rax
movq %rax, -0x1d0(%rbp)
jmp 0xa879f
xorl %eax, %eax
movq %rax, -0x1d0(%rbp)
jmp 0xa879f
movq -0x1d0(%rbp), %rsi
movq -0xa8(%rbp), %rdx
movq %rdx, %rax
shlq $0x3, %rax
subq $0x4e21, %rax           # imm = 0x4E21
setae %al
movzbl %al, %ecx
leaq -0x118(%rbp), %rdi
callq 0x541f0
jmp 0xa87ce
movq $0x0, -0x120(%rbp)
movq -0x120(%rbp), %rax
cmpq -0x30(%rbp), %rax
jge 0xa8a5e
movq -0x120(%rbp), %rax
movq -0x78(%rbp), %rcx
addq %rcx, %rax
movq %rax, -0x130(%rbp)
leaq -0x130(%rbp), %rdi
leaq -0x30(%rbp), %rsi
callq 0x37360
movq (%rax), %rax
movq -0x120(%rbp), %rcx
subq %rcx, %rax
movq %rax, -0x128(%rbp)
movq -0xe0(%rbp), %rax
movq %rax, -0x1e0(%rbp)
movq -0x120(%rbp), %rsi
xorl %eax, %eax
movl %eax, %edx
leaq -0x70(%rbp), %rdi
callq 0xa8e70
movq %rax, -0x1d8(%rbp)
jmp 0xa884e
movq -0x1d8(%rbp), %rdx
movq -0x1e0(%rbp), %rsi
movq 0x10(%rbp), %rcx
movq -0x128(%rbp), %r8
movq -0x28(%rbp), %r9
subq $0x10, %rsp
movq %rsp, %rax
movq $0x0, 0x8(%rax)
movq $0x0, (%rax)
leaq -0x8a(%rbp), %rdi
callq 0xa8b90
addq $0x10, %rsp
jmp 0xa8893
movq $0x0, -0x138(%rbp)
movq -0x138(%rbp), %rax
cmpq -0x20(%rbp), %rax
jge 0xa8a45
movq -0x138(%rbp), %rax
movq -0x80(%rbp), %rcx
addq %rcx, %rax
movq %rax, -0x148(%rbp)
leaq -0x148(%rbp), %rdi
leaq -0x20(%rbp), %rsi
callq 0x37360
movq (%rax), %rax
movq -0x138(%rbp), %rcx
subq %rcx, %rax
movq %rax, -0x140(%rbp)
movq -0xb0(%rbp), %rax
movq %rax, -0x1f0(%rbp)
movq -0x138(%rbp), %rsi
movq -0x120(%rbp), %rdx
leaq -0x60(%rbp), %rdi
callq 0xa8e70
movq %rax, -0x1e8(%rbp)
jmp 0xa8916
movq -0x1e8(%rbp), %rdx
movq -0x1f0(%rbp), %rsi
movq -0x40(%rbp), %rcx
movq -0x128(%rbp), %r8
movq -0x140(%rbp), %r9
subq $0x10, %rsp
movq %rsp, %rax
movq $0x0, 0x8(%rax)
movq $0x0, (%rax)
leaq -0x89(%rbp), %rdi
callq 0xa8ea0
addq $0x10, %rsp
jmp 0xa895e
movq 0x18(%rbp), %rax
movq -0x138(%rbp), %rcx
leaq (%rax,%rcx,8), %rsi
movq 0x20(%rbp), %rdx
movq -0xb0(%rbp), %rcx
movq -0xe0(%rbp), %r8
movq -0x140(%rbp), %r9
movq -0x128(%rbp), %rdi
movq -0x28(%rbp), %r10
movsd -0x50(%rbp), %xmm0
movq -0x100(%rbp), %r11
subq $0x40, %rsp
movq %rsp, %rax
movq %r11, 0x30(%rax)
movq %r10, 0x8(%rax)
movq %rdi, (%rax)
movq $0x0, 0x28(%rax)
movq $0x0, 0x20(%rax)
movq $-0x1, 0x18(%rax)
movq $-0x1, 0x10(%rax)
leaq -0x8b(%rbp), %rdi
callq 0xa91e0
addq $0x40, %rsp
jmp 0xa89e1
jmp 0xa89e3
movq -0x80(%rbp), %rax
addq -0x138(%rbp), %rax
movq %rax, -0x138(%rbp)
jmp 0xa889e
movq %rax, %rcx
movl %edx, %eax
movq %rcx, -0xd0(%rbp)
movl %eax, -0xd4(%rbp)
jmp 0xa8a93
movq %rax, %rcx
movl %edx, %eax
movq %rcx, -0xd0(%rbp)
movl %eax, -0xd4(%rbp)
jmp 0xa8a87
movq %rax, %rcx
movl %edx, %eax
movq %rcx, -0xd0(%rbp)
movl %eax, -0xd4(%rbp)
leaq -0x118(%rbp), %rdi
callq 0x55c10
jmp 0xa8a87
jmp 0xa8a47
movq -0x78(%rbp), %rax
addq -0x120(%rbp), %rax
movq %rax, -0x120(%rbp)
jmp 0xa87d9
leaq -0x118(%rbp), %rdi
callq 0x55c10
leaq -0xf8(%rbp), %rdi
callq 0x55c10
leaq -0xc8(%rbp), %rdi
callq 0x55c10
movq %rbp, %rsp
popq %rbp
retq
leaq -0xf8(%rbp), %rdi
callq 0x55c10
leaq -0xc8(%rbp), %rdi
callq 0x55c10
movq -0xd0(%rbp), %rdi
callq 0x10cc0
nopl (%rax,%rax)