// File: /workspace/github_c_makefile_star2_O1/LPD-EPFL[P]mctop/src/mctop.c
// Function: mem_bandwidth
void*
mem_bandwidth(void* param)
{
  tld_t* tld = (tld_t*) param;
  const int tid = tld->id;
  const uint n_threads = tld->n_threads;
  pthread_barrier_t* barrier = tld->barrier;
  mctop_t* topo = tld->topo;
  const uint n_u64 = test_mem_bw_size / sizeof(uint64_t);

  volatile cache_line_t** mem_bw = malloc_assert(mctop_get_num_nodes(topo) * sizeof(cache_line_t*));
  for (int n = 0; n < mctop_get_num_nodes(topo); n++)
    {
      mem_bw[n] = numa_alloc_onnode(test_mem_bw_size, n);
      bzero((void*) mem_bw[n], test_mem_bw_size);
    }

  ID0_DO(mem_bw_gbps_r = (double*) malloc_assert(n_threads * sizeof(double));
	 mem_bw_gbps_w = (double*) malloc_assert(n_threads * sizeof(double)));

  double progress_step = 100.0 / (topo->n_sockets * topo->n_sockets);
  uint progress = 0;
  ID0_DO(NOT_VERBOSE(printf("# Progress : %6.1f%%", 0.0)); fflush(stdout));

  for (int n = 0; n < mctop_get_num_nodes(topo); n++)
    {
      mctop_run_on_socket_nm(topo, n);
      VERBOSE(ID0_DO(printf(" ######## Run Socket %d\n", n);););
      for (int mem_on = 0; mem_on < mctop_get_num_nodes(topo); mem_on++)
	{
	  VERBOSE(ID0_DO(printf(" #### Mem Node %d\n", mem_on)););

	  pthread_barrier_wait(barrier);
	  if (tid == 0)
	    {
	      dvfs_scale_up(test_num_dvfs_reps, test_dvfs_ratio, NULL);
	      double bw_gbps_r = mem_bw_estimate(mem_bw[mem_on], BW_READ, n_u64, test_mem_bw_num_reps);
	      mem_bw_table1_r[n][mem_on] = bw_gbps_r;
	      double bw_gbps_w = mem_bw_estimate(mem_bw[mem_on], BW_WRITE, n_u64, test_mem_bw_num_reps);
	      mem_bw_table1_w[n][mem_on] = bw_gbps_w;
	    }
	  pthread_barrier_wait(barrier);

	  dvfs_scale_up(test_num_dvfs_reps, test_dvfs_ratio, NULL);
	  mini_barrier(&mem_bw_barrier, n_threads);

	  double bw_gbps_r = mem_bw_estimate(mem_bw[mem_on], BW_READ, n_u64, test_mem_bw_num_reps);
	  mem_bw_gbps_r[tid] = bw_gbps_r;
	  double bw_gbps_w = mem_bw_estimate(mem_bw[mem_on], BW_WRITE, n_u64, test_mem_bw_num_reps);
	  mem_bw_gbps_w[tid] = bw_gbps_w;

	  pthread_barrier_wait(barrier);

	  if (tid == 0)
	    {
	      mem_bw_barrier = 0;
	      double tot_bw_r = 0, tot_bw_w = 0;
	      VERBOSE(printf("   READ  BW ("););
	      for (int i = 0; i < n_threads; i++)
		{
		  VERBOSE(printf("+%2.2f", mem_bw_gbps_r[i]););
		  tot_bw_r += mem_bw_gbps_r[i];
		}
	      VERBOSE(printf(") = %f GB/s\n", tot_bw_r););
	      mem_bw_table_r[n][mem_on] = tot_bw_r;

	      VERBOSE(printf("   WRITE BW ("););
	      for (int i = 0; i < n_threads; i++)
		{
		  VERBOSE(printf("+%2.2f", mem_bw_gbps_w[i]););
		  tot_bw_w += mem_bw_gbps_w[i];
		}
	      VERBOSE(printf(") = %f GB/s\n", tot_bw_w););
	      mem_bw_table_w[n][mem_on] = tot_bw_w;

	      NOT_VERBOSE(printf("\r# Progress : %6.1f%%", ++progress * progress_step); fflush(stdout););
	    }
	}
    }

  for (int n = 0; n < mctop_get_num_nodes(topo); n++)
    {
      numa_free((void*) mem_bw[n], test_mem_bw_size);
    }
  free(mem_bw);


  if (tid == 0)
    {
      NOT_VERBOSE(printf("\n"););
      free((void*) mem_bw_gbps_r);
      free((void*) mem_bw_gbps_w);
    }
  return NULL;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x38, %rsp
movslq (%rdi), %r14
movl 0x4(%rdi), %ebp
movq 0x18(%rdi), %rax
movq %rax, 0x18(%rsp)
movq 0x20(%rdi), %r13
movq 0xd935(%rip), %r15      # 0x11220
movq %r13, %rdi
callq 0x8c6f
leaq (,%rax,8), %rdi
callq 0x21d0
movq %rax, 0x10(%rsp)
testq %rax, %rax
je 0x3dc4
movq %r13, %rdi
callq 0x8c6f
testq %rax, %rax
movq 0x10(%rsp), %rbx
je 0x3957
xorl %r12d, %r12d
movq 0xd8f6(%rip), %rdi      # 0x11220
movl %r12d, %esi
callq 0x21f0
movq %rax, (%rbx,%r12,8)
movq 0xd8e3(%rip), %rdx      # 0x11220
movq %rax, %rdi
xorl %esi, %esi
callq 0x2120
incq %r12
movq %r13, %rdi
callq 0x8c6f
cmpq %r12, %rax
ja 0x3923
testl %r14d, %r14d
jne 0x3994
leaq (,%rbp,8), %r12
movq %r12, %rdi
callq 0x21d0
testq %rax, %rax
je 0x3dc4
movq %rax, 0xd95c(%rip)      # 0x112d8
movq %r12, %rdi
callq 0x21d0
testq %rax, %rax
je 0x3dc4
movq %rax, 0xd94c(%rip)      # 0x112e0
movl 0x18(%r13), %r12d
testl %r14d, %r14d
je 0x3db3
movq %r13, %rdi
callq 0x8c6f
testq %rax, %rax
je 0x3d61
imull %r12d, %r12d
cvtsi2sd %r12, %xmm0
movsd 0x7645(%rip), %xmm1     # 0xb008
divsd %xmm0, %xmm1
movsd %xmm1, 0x30(%rsp)
shrq $0x3, %r15
movl %r15d, %r12d
xorl %ebx, %ebx
movq $0x0, 0x20(%rsp)
movq %r13, 0x28(%rsp)
movq %r13, %rdi
movl %ebx, %esi
callq 0x9038
cmpl $0x0, 0xd893(%rip)      # 0x11288
jne 0x3d43
movq %r13, %rdi
callq 0x8c6f
testq %rax, %rax
je 0x3d28
xorl %r13d, %r13d
movq %rbx, %r15
cmpl $0x0, 0xd86f(%rip)      # 0x11288
movq 0x18(%rsp), %rbx
jne 0x3c90
movq %rbx, %rdi
callq 0x2090
testl %r14d, %r14d
jne 0x3a9c
movl $0x6acfc0, %edi         # imm = 0x6ACFC0
movsd 0x75d2(%rip), %xmm0     # 0xb010
xorl %esi, %esi
callq 0x5e1d
movq 0x10(%rsp), %rax
movq (%rax,%r13,8), %rbx
movl $0x4, %ecx
movq %rbx, %rdi
xorl %esi, %esi
movq %r12, %rdx
callq 0x375f
movq 0xd881(%rip), %rax      # 0x112e8
movq (%rax,%r15,8), %rax
movsd %xmm0, (%rax,%r13,8)
movl $0x4, %ecx
movq %rbx, %rdi
movl $0x1, %esi
movq %r12, %rdx
callq 0x375f
movq 0xd863(%rip), %rax      # 0x112f0
movq (%rax,%r15,8), %rax
movq 0x18(%rsp), %rbx
movsd %xmm0, (%rax,%r13,8)
movq %rbx, %rdi
callq 0x2090
movl $0x6acfc0, %edi         # imm = 0x6ACFC0
movsd 0x755f(%rip), %xmm0     # 0xb010
xorl %esi, %esi
callq 0x5e1d
lock
incl 0xd7fd(%rip)            # 0x112bc
movl 0xd7f7(%rip), %eax      # 0x112bc
cmpl %ebp, %eax
je 0x3acd
pause
jmp 0x3abf
movq 0x10(%rsp), %rax
movq (%rax,%r13,8), %rbx
movl $0x4, %ecx
movq %rbx, %rdi
xorl %esi, %esi
movq %r12, %rdx
callq 0x375f
movq 0xd7e9(%rip), %rax      # 0x112d8
movsd %xmm0, (%rax,%r14,8)
movl $0x4, %ecx
movq %rbx, %rdi
movl $0x1, %esi
movq %r12, %rdx
callq 0x375f
movq 0xd7cf(%rip), %rax      # 0x112e0
movsd %xmm0, (%rax,%r14,8)
movq 0x18(%rsp), %rdi
callq 0x2090
testl %r14d, %r14d
je 0x3b2e
movq %r15, %rbx
jmp 0x3c75
movl $0x0, 0xd784(%rip)      # 0x112bc
cmpl $0x0, 0xd749(%rip)      # 0x11288
jne 0x3ca2
xorpd %xmm0, %xmm0
testq %rbp, %rbp
je 0x3b97
xorl %ebx, %ebx
cmpl $0x0, 0xd731(%rip)      # 0x11288
jne 0x3b6f
movq 0xd778(%rip), %rax      # 0x112d8
addsd (%rax,%rbx,8), %xmm0
incq %rbx
cmpq %rbx, %rbp
jne 0x3b50
jmp 0x3b97
movq 0xd762(%rip), %rax      # 0x112d8
movsd %xmm0, 0x8(%rsp)
movsd (%rax,%rbx,8), %xmm0
leaq 0x77a1(%rip), %rdi      # 0xb329
movb $0x1, %al
callq 0x2100
movsd 0x8(%rsp), %xmm0
jmp 0x3b59
cmpl $0x0, 0xd6ea(%rip)      # 0x11288
jne 0x3cb5
movq 0xd74d(%rip), %rax      # 0x112f8
movq (%rax,%r15,8), %rax
movsd %xmm0, (%rax,%r13,8)
cmpl $0x0, 0xd6cc(%rip)      # 0x11288
jne 0x3cda
xorpd %xmm0, %xmm0
testq %rbp, %rbp
je 0x3c14
xorl %ebx, %ebx
cmpl $0x0, 0xd6b4(%rip)      # 0x11288
jne 0x3bec
movq 0xd703(%rip), %rax      # 0x112e0
addsd (%rax,%rbx,8), %xmm0
incq %rbx
cmpq %rbx, %rbp
jne 0x3bcd
jmp 0x3c14
movq 0xd6ed(%rip), %rax      # 0x112e0
movsd %xmm0, 0x8(%rsp)
movsd (%rax,%rbx,8), %xmm0
leaq 0x7724(%rip), %rdi      # 0xb329
movb $0x1, %al
callq 0x2100
movsd 0x8(%rsp), %xmm0
jmp 0x3bd6
cmpl $0x0, 0xd66d(%rip)      # 0x11288
jne 0x3ced
movq 0xd6d8(%rip), %rax      # 0x11300
movq %r15, %rbx
movq (%rax,%r15,8), %rax
movsd %xmm0, (%rax,%r13,8)
cmpl $0x0, 0xd64c(%rip)      # 0x11288
jne 0x3c75
movq 0x20(%rsp), %rax
incl %eax
movq %rax, 0x20(%rsp)
xorps %xmm0, %xmm0
cvtsi2sd %rax, %xmm0
mulsd 0x30(%rsp), %xmm0
leaq 0x76ec(%rip), %rdi      # 0xb34b
movb $0x1, %al
callq 0x2100
movq 0xd35b(%rip), %rax      # 0x10fc8
movq (%rax), %rdi
callq 0x21e0
incq %r13
movq 0x28(%rsp), %rdi
callq 0x8c6f
cmpq %r13, %rax
ja 0x3a12
jmp 0x3d28
testl %r14d, %r14d
je 0x3d12
movq %rbx, %rdi
callq 0x2090
jmp 0x3a9c
leaq 0x7672(%rip), %rdi      # 0xb31b
xorl %eax, %eax
callq 0x2100
jmp 0x3b45
leaq 0x7674(%rip), %rdi      # 0xb330
movsd %xmm0, 0x8(%rsp)
movsd 0x8(%rsp), %xmm0
movb $0x1, %al
callq 0x2100
movsd 0x8(%rsp), %xmm0
jmp 0x3ba4
leaq 0x765c(%rip), %rdi      # 0xb33d
xorl %eax, %eax
callq 0x2100
jmp 0x3bc2
leaq 0x763c(%rip), %rdi      # 0xb330
movsd %xmm0, 0x8(%rsp)
movsd 0x8(%rsp), %xmm0
movb $0x1, %al
callq 0x2100
movsd 0x8(%rsp), %xmm0
jmp 0x3c21
leaq 0x75ef(%rip), %rdi      # 0xb308
movl %r13d, %esi
xorl %eax, %eax
callq 0x2100
jmp 0x3a24
incq %rbx
movq 0x28(%rsp), %r13
movq %r13, %rdi
callq 0x8c6f
cmpq %rbx, %rax
ja 0x39e4
jmp 0x3d61
testl %r14d, %r14d
jne 0x39fb
leaq 0x759c(%rip), %rdi      # 0xb2ef
movl %ebx, %esi
xorl %eax, %eax
callq 0x2100
jmp 0x39fb
movq %r13, %rdi
callq 0x8c6f
testq %rax, %rax
movq 0x10(%rsp), %r15
je 0x3d95
xorl %ebx, %ebx
movq (%r15,%rbx,8), %rdi
movq 0xd4a0(%rip), %rsi      # 0x11220
callq 0x20c0
incq %rbx
movq %r13, %rdi
callq 0x8c6f
cmpq %rbx, %rax
ja 0x3d75
movq %r15, %rdi
callq 0x2030
testl %r14d, %r14d
je 0x3dbd
xorl %eax, %eax
addq $0x38, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
callq 0x26fa
jmp 0x39a1
callq 0x2725
jmp 0x3da2
leaq 0x957d(%rip), %rdi      # 0xd348
leaq 0x7d2d(%rip), %rsi      # 0xbaff
leaq 0x7d3f(%rip), %rcx      # 0xbb18
movl $0x59, %edx
callq 0x2110