// File: /workspace/github_cpp_makefile_star2_O3/Pyboon[P]sage/./Eigen/src/Core/products/GeneralMatrixMatrix.h
// Function: Eigen::internal::general_matrix_matrix_product<long, std::complex<double>, 1, true, std::complex<double>, 0, false, 0>::run(long, long, long, std::complex<double> const*, long, std::complex<double> const*, long, std::complex<double>*, long, std::complex<double>, Eigen::internal::level3_blocking<std::complex<double>, std::complex<double>>&, Eigen::internal::GemmParallelInfo<long>*)
static void run(Index rows, Index cols, Index depth,
  const LhsScalar* _lhs, Index lhsStride,
  const RhsScalar* _rhs, Index rhsStride,
  ResScalar* _res, Index resStride,
  ResScalar alpha,
  level3_blocking<LhsScalar,RhsScalar>& blocking,
  GemmParallelInfo<Index>* info = 0)
{
  typedef const_blas_data_mapper<LhsScalar, Index, LhsStorageOrder> LhsMapper;
  typedef const_blas_data_mapper<RhsScalar, Index, RhsStorageOrder> RhsMapper;
  typedef blas_data_mapper<typename Traits::ResScalar, Index, ColMajor> ResMapper;
  LhsMapper lhs(_lhs,lhsStride);
  RhsMapper rhs(_rhs,rhsStride);
  ResMapper res(_res, resStride);

  Index kc = blocking.kc();                   // cache block size along the K direction
  Index mc = (std::min)(rows,blocking.mc());  // cache block size along the M direction
  Index nc = (std::min)(cols,blocking.nc());  // cache block size along the N direction

  gemm_pack_lhs<LhsScalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, LhsStorageOrder> pack_lhs;
  gemm_pack_rhs<RhsScalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
  gebp_kernel<LhsScalar, RhsScalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp;

#ifdef EIGEN_HAS_OPENMP
  if(info)
  {
    // this is the parallel version!
    int tid = omp_get_thread_num();
    int threads = omp_get_num_threads();

    LhsScalar* blockA = blocking.blockA();
    eigen_internal_assert(blockA!=0);

    std::size_t sizeB = kc*nc;
    ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, 0);

    // For each horizontal panel of the rhs, and corresponding vertical panel of the lhs...
    for(Index k=0; k<depth; k+=kc)
    {
      const Index actual_kc = (std::min)(k+kc,depth)-k; // => rows of B', and cols of the A'

      // In order to reduce the chance that a thread has to wait for the other,
      // let's start by packing B'.
      pack_rhs(blockB, rhs.getSubMapper(k,0), actual_kc, nc);

      // Pack A_k to A' in a parallel fashion:
      // each thread packs the sub block A_k,i to A'_i where i is the thread id.

      // However, before copying to A'_i, we have to make sure that no other thread is still using it,
      // i.e., we test that info[tid].users equals 0.
      // Then, we set info[tid].users to the number of threads to mark that all other threads are going to use it.
      while(info[tid].users!=0) {}
      info[tid].users += threads;

      pack_lhs(blockA+info[tid].lhs_start*actual_kc, lhs.getSubMapper(info[tid].lhs_start,k), actual_kc, info[tid].lhs_length);

      // Notify the other threads that the part A'_i is ready to go.
      info[tid].sync = k;

      // Computes C_i += A' * B' per A'_i
      for(int shift=0; shift<threads; ++shift)
      {
        int i = (tid+shift)%threads;

        // At this point we have to make sure that A'_i has been updated by the thread i,
        // we use testAndSetOrdered to mimic a volatile access.
        // However, no need to wait for the B' part which has been updated by the current thread!
        if (shift>0) {
          while(info[i].sync!=k) {
          }
        }

        gebp(res.getSubMapper(info[i].lhs_start, 0), blockA+info[i].lhs_start*actual_kc, blockB, info[i].lhs_length, actual_kc, nc, alpha);
      }

      // Then keep going as usual with the remaining B'
      for(Index j=nc; j<cols; j+=nc)
      {
        const Index actual_nc = (std::min)(j+nc,cols)-j;

        // pack B_k,j to B'
        pack_rhs(blockB, rhs.getSubMapper(k,j), actual_kc, actual_nc);

        // C_j += A' * B'
        gebp(res.getSubMapper(0, j), blockA, blockB, rows, actual_kc, actual_nc, alpha);
      }

      // Release all the sub blocks A'_i of A' for the current thread,
      // i.e., we simply decrement the number of users by 1
      for(Index i=0; i<threads; ++i)
        #pragma omp atomic
        info[i].users -= 1;
    }
  }
  else
#endif // EIGEN_HAS_OPENMP
  {
    EIGEN_UNUSED_VARIABLE(info);

    // this is the sequential version!
    std::size_t sizeA = kc*mc;
    std::size_t sizeB = kc*nc;

    ei_declare_aligned_stack_constructed_variable(LhsScalar, blockA, sizeA, blocking.blockA());
    ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, blocking.blockB());

    const bool pack_rhs_once = mc!=rows && kc==depth && nc==cols;

    // For each horizontal panel of the rhs, and corresponding panel of the lhs...
    for(Index i2=0; i2<rows; i2+=mc)
    {
      const Index actual_mc = (std::min)(i2+mc,rows)-i2;

      for(Index k2=0; k2<depth; k2+=kc)
      {
        const Index actual_kc = (std::min)(k2+kc,depth)-k2;

        // OK, here we have selected one horizontal panel of rhs and one vertical panel of lhs.
        // => Pack lhs's panel into a sequential chunk of memory (L2/L3 caching)
        // Note that this panel will be read as many times as the number of blocks in the rhs's
        // horizontal panel which is, in practice, a very low number.
        pack_lhs(blockA, lhs.getSubMapper(i2,k2), actual_kc, actual_mc);

        // For each kc x nc block of the rhs's horizontal panel...
        for(Index j2=0; j2<cols; j2+=nc)
        {
          const Index actual_nc = (std::min)(j2+nc,cols)-j2;

          // We pack the rhs's block into a sequential chunk of memory (L2 caching)
          // Note that this block will be read a very high number of times, which is equal to the number of
          // micro horizontal panel of the large rhs's panel (e.g., rows/12 times).
          if((!pack_rhs_once) || i2==0)
            pack_rhs(blockB, rhs.getSubMapper(k2,j2), actual_kc, actual_nc);

          // Everything is packed, we can now call the panel * block kernel:
          gebp(res.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, alpha);
        }
      }
    }
  }
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x108, %rsp            # imm = 0x108
movsd %xmm1, -0x128(%rbp)
movsd %xmm0, -0x120(%rbp)
movq %r9, -0xd8(%rbp)
movq %r8, -0xc0(%rbp)
movq %rcx, -0xd0(%rbp)
movq %rdx, %r13
movq 0x28(%rbp), %rbx
movq 0x20(%rbx), %rdx
movq 0x10(%rbx), %r15
movq 0x18(%rbx), %r12
cmpq %rdi, %r15
movq %rdi, %rcx
cmovlq %r15, %rcx
cmpq %rsi, %r12
movq %rsi, %rax
cmovlq %r12, %rax
movq %rax, -0x58(%rbp)
movq %rcx, -0xb0(%rbp)
movq %rdx, -0x50(%rbp)
imulq %rdx, %rcx
movq %rcx, %rax
shrq $0x3c, %rax
jne 0x1294e
movq %rdi, %r8
movq (%rbx), %rax
movq %rax, -0x68(%rbp)
testq %rax, %rax
movq %rsi, -0x60(%rbp)
je 0x12571
xorl %eax, %eax
jmp 0x125ca
movq %rcx, %rdi
shlq $0x4, %rdi
cmpq $0x2000, %rcx           # imm = 0x2000
ja 0x12598
movq %rsp, %rax
addq $0x1e, %rdi
andq $-0x10, %rdi
subq %rdi, %rax
movq %rax, %rsp
movq %rax, -0x68(%rbp)
jmp 0x125ca
movq %r8, -0x88(%rbp)
movq %rcx, %r14
callq 0x3360
testb $0xf, %al
jne 0x129b0
movq %rax, -0x68(%rbp)
testq %rax, %rax
movq %r14, %rcx
movq -0x60(%rbp), %rsi
movq -0x88(%rbp), %r8
je 0x129b2
movabsq $0xfffffffffffffff, %rdx # imm = 0xFFFFFFFFFFFFFFF
movq -0x58(%rbp), %r9
imulq -0x50(%rbp), %r9
cmpq %rdx, %r9
ja 0x1297c
movq 0x8(%rbx), %rdx
movq %rdx, -0x70(%rbp)
testq %rdx, %rdx
movq %rcx, -0x48(%rbp)
movq %rax, -0x40(%rbp)
movq %r9, -0xa8(%rbp)
je 0x1260c
movq $0x0, -0x38(%rbp)
jmp 0x1265f
movq %r9, %rdi
shlq $0x4, %rdi
cmpq $0x2000, %r9            # imm = 0x2000
ja 0x12637
movq %rsp, %rcx
addq $0x1e, %rdi
andq $-0x10, %rdi
subq %rdi, %rcx
movq %rcx, %rsp
movq %rcx, -0x38(%rbp)
movq %rcx, -0x70(%rbp)
jmp 0x1265f
movq %r8, %rbx
callq 0x3360
testb $0xf, %al
jne 0x129b4
movq %rax, -0x70(%rbp)
movq %rax, -0x38(%rbp)
testq %rax, %rax
movq -0x60(%rbp), %rsi
movq %rbx, %r8
je 0x129d3
cmpq %rsi, %r12
setl %al
cmpq %r8, %r15
setge %cl
cmpq %r13, -0x50(%rbp)
setne %dl
testq %r8, %r8
jle 0x12916
movq 0x20(%rbp), %rsi
movq 0x18(%rbp), %rdi
movq %rdi, -0xb8(%rbp)
movq 0x10(%rbp), %rdi
orb %cl, %dl
orb %al, %dl
movb %dl, -0x29(%rbp)
movq -0x50(%rbp), %rax
shlq $0x4, %rax
movq %rax, -0xf0(%rbp)
movq -0x58(%rbp), %rcx
imulq %rcx, %rdi
shlq $0x4, %rdi
movq %rdi, -0x118(%rbp)
movq -0xb0(%rbp), %rax
shlq $0x4, %rax
movq %rax, -0xe0(%rbp)
imulq %rcx, %rsi
shlq $0x4, %rsi
movq %rsi, -0x110(%rbp)
xorl %eax, %eax
movq %r13, -0xf8(%rbp)
movq %r8, -0x88(%rbp)
movq -0xb0(%rbp), %rcx
addq %rax, %rcx
cmpq %r8, %rcx
movq %rcx, -0xe8(%rbp)
cmovlq %rcx, %r8
movq %r8, -0x98(%rbp)
testq %r13, %r13
jle 0x128f1
subq %rax, -0x98(%rbp)
movq %rax, %rcx
imulq -0xc0(%rbp), %rcx
shlq $0x4, %rcx
addq -0xd0(%rbp), %rcx
movq %rcx, -0x100(%rbp)
testq %rax, %rax
sete %al
orb -0x29(%rbp), %al
movb %al, -0x2a(%rbp)
movq -0xd8(%rbp), %rax
movq %rax, -0x90(%rbp)
xorl %eax, %eax
movq -0x50(%rbp), %rcx
leaq (%rax,%rcx), %rbx
cmpq %r13, %rbx
movq %r13, %rcx
cmovlq %rbx, %rcx
subq %rax, %rcx
shlq $0x4, %rax
addq -0x100(%rbp), %rax
movq %rax, -0x80(%rbp)
movq -0xc0(%rbp), %rax
movq %rax, -0x78(%rbp)
subq $0x8, %rsp
leaq -0x2d(%rbp), %rdi
movq -0x68(%rbp), %rsi
leaq -0x80(%rbp), %rdx
movq %rcx, -0xc8(%rbp)
movq -0x98(%rbp), %r8
xorl %r9d, %r9d
pushq $0x0
callq 0x12a4a
addq $0x10, %rsp
movq %rbx, -0x108(%rbp)
movq -0x60(%rbp), %rax
testq %rax, %rax
jle 0x128c5
movq -0xb8(%rbp), %r13
movq -0x90(%rbp), %rcx
movq %rcx, -0xa0(%rbp)
xorl %ebx, %ebx
movq -0x58(%rbp), %r12
xorl %r14d, %r14d
cmpq %r12, %rax
movq %r12, %r15
cmovlq %rax, %r15
addq %rbx, %r15
cmpb $0x0, -0x2a(%rbp)
je 0x1282b
movq -0xa0(%rbp), %rax
movq %rax, -0x80(%rbp)
movq 0x10(%rbp), %rax
movq %rax, -0x78(%rbp)
subq $0x8, %rsp
leaq -0x2c(%rbp), %rdi
movq -0x70(%rbp), %rsi
leaq -0x80(%rbp), %rdx
movq -0xc8(%rbp), %rcx
movq %r15, %r8
xorl %r9d, %r9d
pushq $0x0
callq 0x12ad2
addq $0x10, %rsp
movq %r13, -0x80(%rbp)
movq 0x20(%rbp), %rax
movq %rax, -0x78(%rbp)
subq $0x30, %rsp
xorps %xmm0, %xmm0
movups %xmm0, 0x18(%rsp)
movq $-0x1, %rax
movq %rax, 0x10(%rsp)
movq %rax, 0x8(%rsp)
movq %r15, (%rsp)
leaq -0x2b(%rbp), %rdi
leaq -0x80(%rbp), %rsi
movq -0x68(%rbp), %rdx
movq -0x70(%rbp), %rcx
movq -0x98(%rbp), %r8
movq -0xc8(%rbp), %r9
movsd -0x120(%rbp), %xmm0
movsd -0x128(%rbp), %xmm1
callq 0x12c8a
addq $0x30, %rsp
movq -0x58(%rbp), %rax
addq %rax, %r14
addq %rax, %r12
subq %rax, %rbx
movq -0xa0(%rbp), %rax
addq -0x118(%rbp), %rax
movq %rax, -0xa0(%rbp)
addq -0x110(%rbp), %r13
movq -0x60(%rbp), %rax
cmpq %rax, %r14
jl 0x127dd
movq -0x90(%rbp), %rax
addq -0xf0(%rbp), %rax
movq %rax, -0x90(%rbp)
movq -0xf8(%rbp), %r13
movq -0x108(%rbp), %rax
cmpq %r13, %rax
jl 0x12750
movq -0xe0(%rbp), %rax
addq %rax, -0xb8(%rbp)
movq -0x88(%rbp), %r8
movq -0xe8(%rbp), %rax
cmpq %r8, %rax
jl 0x126e8
cmpq $0x2001, -0xa8(%rbp) # imm = 0x2001
jb 0x1292c
movq -0x38(%rbp), %rdi
callq 0x3310
cmpq $0x2001, -0x48(%rbp)    # imm = 0x2001
jb 0x1293f
movq -0x40(%rbp), %rdi
callq 0x3310
leaq -0x28(%rbp), %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
movl $0x8, %edi
callq 0x3130
movq 0x1c661(%rip), %rcx     # 0x2efc0
addq $0x10, %rcx
movq %rcx, (%rax)
movq 0x1c663(%rip), %rsi     # 0x2efd0
movq 0x1c634(%rip), %rdx     # 0x2efa8
movq %rax, %rdi
callq 0x3430
movq %rax, %r15
movq %rcx, %r14
movl $0x8, %edi
callq 0x3130
movq 0x1c62d(%rip), %rcx     # 0x2efc0
addq $0x10, %rcx
movq %rcx, (%rax)
movq 0x1c62f(%rip), %rsi     # 0x2efd0
movq 0x1c600(%rip), %rdx     # 0x2efa8
movq %rax, %rdi
callq 0x3430
jmp 0x129b4
jmp 0x1294e
leaq 0x9f40(%rip), %rdi      # 0x1c8fb
leaq 0x9ff3(%rip), %rsi      # 0x1c9b5
leaq 0xa00b(%rip), %rcx      # 0x1c9d4
movl $0xa1, %edx
callq 0x31b0
movl $0x8, %edi
callq 0x3130
movq 0x1c5dc(%rip), %rcx     # 0x2efc0
addq $0x10, %rcx
movq %rcx, (%rax)
movq 0x1c5de(%rip), %rsi     # 0x2efd0
movq 0x1c5af(%rip), %rdx     # 0x2efa8
movq %rax, %rdi
callq 0x3430
movq %rax, %rbx
jmp 0x12a2b
movq %rax, %rbx
movq %r15, %rdi
jmp 0x12a33
jmp 0x12a12
jmp 0x12a12
movq %rax, %rbx
cmpq $0x2001, -0xa8(%rbp) # imm = 0x2001
movq -0x38(%rbp), %rdi
jb 0x12a2b
callq 0x3310
movq -0x48(%rbp), %r14
movq -0x40(%rbp), %rdi
cmpq $0x2001, %r14           # imm = 0x2001
jb 0x12a41
callq 0x3310
movq %rbx, %rdi
callq 0x3450
nop