// File: /workspace/github_c_makefile_star2_O2/google[P]multichase/pingpong.c
// Function: main
int main(int argc, char **argv) {
  void *(*thread_fn)(void *data) = NULL;
  int c;
  char *p;

  while ((c = getopt(argc, argv, "c:lur:xs:")) != -1) {
    switch (c) {
      case 'l':
        if (thread_fn) goto thread_fn_error;
        thread_fn = locked_loop;
        break;
      case 'u':
        if (thread_fn) goto thread_fn_error;
        thread_fn = unlocked_loop;
        break;
      case 'x':
        if (thread_fn) goto thread_fn_error;
        thread_fn = xadd_loop;
        break;
      case 'r':
        nr_relax = strtoul(optarg, &p, 0);
        if (*p) {
          fprintf(stderr, "-r requires a numeric argument\n");
          exit(1);
        }
        break;
      case 'c':
        nr_tested_cores = strtoul(optarg, &p, 0);
        if (*p) {
          fprintf(stderr, "-c requires a numeric argument\n");
          exit(1);
        }
        break;
      case 's':
        nr_array_elts = strtoul(optarg, &p, 0);
        if (*p) {
          fprintf(stderr, "-s requires a numeric argument\n");
          exit(1);
        }
        if (posix_memalign((void **)&communication_array, 1ull << 21,
                           nr_array_elts * sizeof(*communication_array))) {
          fprintf(stderr, "posix_memalign failed\n");
          exit(1);
        }
        break;
      default:
        fprintf(stderr,
                "usage: %s [-l | -u | -x] [-r nr_relax] [-s "
                "nr_array_elts_to_dirty] [-c nr_tested_cores]\n",
                argv[0]);
        exit(1);
    }
  }
  if (thread_fn == NULL) {
  thread_fn_error:
    fprintf(stderr, "must specify exactly one of -u, -l or -x\n");
    exit(1);
  }

  setvbuf(stdout, NULL, _IONBF, BUFSIZ);

  // find the active cpus
  cpu_set_t cpus;
  if (sched_getaffinity(getpid(), sizeof(cpus), &cpus)) {
    perror("sched_getaffinity");
    exit(1);
  }

  printf(
      "avg latency to communicate a modified line from one core to another\n");
  printf("times are in ns\n\n");

  // print top row header
  const int col_width = 8;
  size_t first_cpu = ~0;
  size_t last_cpu = 0;
  printf("   ");
  for (size_t j = 0; j < CPU_SETSIZE; ++j) {
    if (CPU_ISSET(j, &cpus)) {
      if (first_cpu > j) {
        first_cpu = j;
      } else {
        printf("%*zu", col_width, j);
      }
      if (last_cpu < j) {
        last_cpu = j;
      }
    }
  }
  printf("\n");

  for (size_t i = 0, core = 0; i < last_cpu && core < nr_tested_cores; ++i) {
    if (!CPU_ISSET(i, &cpus)) {
      continue;
    }
    ++core;
    thread_args_t even;
    CPU_ZERO(&even.cpus);
    CPU_SET(i, &even.cpus);
    even.me = 0;
    even.buddy = 1;
    printf("%2zu:", i);
    for (size_t j = first_cpu + 1; j <= i; ++j) {
      if (CPU_ISSET(j, &cpus)) {
        printf("%*s", col_width, "");
      }
    }
    for (size_t j = i + 1; j <= last_cpu; ++j) {
      if (!CPU_ISSET(j, &cpus)) {
        continue;
      }

      thread_args_t odd;
      CPU_ZERO(&odd.cpus);
      CPU_SET(j, &odd.cpus);
      odd.me = 1;
      odd.buddy = 0;
      __sync_lock_test_and_set(&nr_pingpongs.x, 0);
      pthread_t odd_thread;
      if (pthread_create(&odd_thread, NULL, thread_fn, &odd)) {
        perror("pthread_create odd");
        exit(1);
      }
      pthread_t even_thread;
      if (pthread_create(&even_thread, NULL, thread_fn, &even)) {
        perror("pthread_create even");
        exit(1);
      }

      uint64_t last_stamp = now_nsec();
      double best_sample = 1. / 0.;  // infinity
      for (size_t sample_no = 0; sample_no < NR_SAMPLES; ++sample_no) {
        usleep(SAMPLE_US);
        atomic_t s = __sync_lock_test_and_set(&nr_pingpongs.x, 0);
        uint64_t time_stamp = now_nsec();
        double sample = (time_stamp - last_stamp) / (double)s;
        last_stamp = time_stamp;
        if (sample < best_sample) {
          best_sample = sample;
        }
      }
      printf("%*.1f", col_width, best_sample);

      stop_loops = 1;
      if (pthread_join(odd_thread, NULL)) {
        perror("pthread_join odd_thread");
        exit(1);
      }
      if (pthread_join(even_thread, NULL)) {
        perror("pthread_join even_thread");
        exit(1);
      }
      stop_loops = 0;

      if (munmap(pingpong_mutex, getpagesize())) {
        perror("munmap");
        exit(1);
      }
      pingpong_mutex = NULL;
    }
    printf("\n");
  }
  printf("\n");

  return 0;
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x1d8, %rsp            # imm = 0x1D8
movq %rsi, %r14
movl %edi, %ebp
movq $0x0, (%rsp)
leaq 0xb289f(%rip), %r15     # 0x4b404c
movq $0x4ef060, %rbx         # imm = 0x4EF060
leaq 0x10(%rsp), %r12
leaq 0xb2870(%rip), %r13     # 0x4b4030
movl %ebp, %edi
movq %r14, %rsi
movq %r15, %rdx
callq 0x456b20
leal -0x72(%rax), %ecx
cmpl $0x6, %ecx
ja 0x401802
movslq (%r13,%rcx,4), %rax
addq %r13, %rax
jmpq *%rax
movq (%rbx), %rdi
movq %r12, %rsi
xorl %edx, %edx
callq 0x40b600
movq %rax, 0xe591d(%rip)     # 0x4e7110
movq 0x10(%rsp), %rax
cmpb $0x0, (%rax)
je 0x4017c0
jmp 0x401cda
cmpl $0x63, %eax
jne 0x401871
movq (%rbx), %rdi
movq %r12, %rsi
xorl %edx, %edx
callq 0x40b600
movq %rax, 0xe58fd(%rip)     # 0x4e7118
movq 0x10(%rsp), %rax
cmpb $0x0, (%rax)
je 0x4017c0
jmp 0x401d15
movq (%rbx), %rdi
movq %r12, %rsi
xorl %edx, %edx
callq 0x40b600
movq %rax, 0xe7fc2(%rip)     # 0x4e9800
movq 0x10(%rsp), %rcx
cmpb $0x0, (%rcx)
jne 0x401d02
shlq $0x3, %rax
movl $0x200000, %esi         # imm = 0x200000
leaq 0xe7fac(%rip), %rdi     # 0x4e9808
movq %rax, %rdx
callq 0x4283b0
testl %eax, %eax
je 0x4017c0
jmp 0x401ced
cmpl $-0x1, %eax
je 0x4018b6
cmpl $0x6c, %eax
jne 0x401cbd
cmpq $0x0, (%rsp)
leaq 0x4c9(%rip), %rax       # 0x401d54
jmp 0x4018a7
cmpq $0x0, (%rsp)
leaq 0x539(%rip), %rax       # 0x401dd2
jmp 0x4018a7
cmpq $0x0, (%rsp)
leaq 0x5aa(%rip), %rax       # 0x401e51
movq %rax, (%rsp)
je 0x4017c0
jmp 0x401d33
cmpq $0x0, (%rsp)
je 0x401d33
movq $0x4e7730, %rax         # imm = 0x4E7730
movq (%rax), %rdi
pushq $0x2
popq %rdx
movl $0x2000, %ecx           # imm = 0x2000
xorl %esi, %esi
callq 0x413230
callq 0x455450
leaq 0x158(%rsp), %rdx
movl $0x80, %esi
movl %eax, %edi
callq 0x456d20
testl %eax, %eax
jne 0x401d48
leaq 0xb28ef(%rip), %rdi     # 0x4b41f1
callq 0x413090
leaq 0xb2927(%rip), %rdi     # 0x4b4235
callq 0x413090
leaq 0xb2848(%rip), %rdi     # 0x4b4162
xorl %eax, %eax
callq 0x40bee0
pushq $-0x1
popq %rbx
leaq 0xb283b(%rip), %r14     # 0x4b4166
pushq $0x8
popq %rbp
xorl %r15d, %r15d
movq $0x0, 0x8(%rsp)
cmpq $0x400, %r15            # imm = 0x400
je 0x40198b
movq %r15, %rax
shrq $0x6, %rax
movq 0x158(%rsp,%rax,8), %rax
btq %r15, %rax
jae 0x401986
movq %r15, %rax
cmpq %r15, %rbx
ja 0x401972
movq %r14, %rdi
movl %ebp, %esi
movq %r15, %rdx
xorl %eax, %eax
callq 0x40bee0
movq %rbx, %rax
movq 0x8(%rsp), %rcx
cmpq %r15, %rcx
cmovbeq %r15, %rcx
movq %rcx, 0x8(%rsp)
movq %rax, %rbx
incq %r15
jmp 0x40193a
pushq $0xa
popq %rdi
callq 0x413420
movq 0xe577e(%rip), %rax     # 0x4e7118
incq %rbx
movq %rbx, 0x20(%rsp)
pushq $0x8
popq %r13
movq $0x0, 0x18(%rsp)
xorl %ebp, %ebp
movq (%rsp), %rbx
cmpq 0x8(%rsp), %rbp
jae 0x401c69
cmpq %rax, 0x18(%rsp)
jae 0x401c69
movq %rbp, %rdx
shrq $0x6, %rdx
movq 0x158(%rsp,%rdx,8), %rdi
pushq $0x1
popq %rsi
movl %ebp, %ecx
shlq %cl, %rsi
btq %rbp, %rdi
jae 0x401c61
xorpd %xmm0, %xmm0
movapd %xmm0, 0x140(%rsp)
movapd %xmm0, 0x130(%rsp)
movapd %xmm0, 0x120(%rsp)
movapd %xmm0, 0x110(%rsp)
movapd %xmm0, 0x100(%rsp)
movapd %xmm0, 0xf0(%rsp)
movapd %xmm0, 0xe0(%rsp)
movapd %xmm0, 0xd0(%rsp)
orq %rsi, 0xd0(%rsp,%rdx,8)
movabsq $0x100000000, %rax      # imm = 0x100000000
movq %rax, 0x150(%rsp)
leaq 0xb2712(%rip), %rdi     # 0x4b416b
movq %rbp, %rsi
xorl %eax, %eax
callq 0x40bee0
movq 0x20(%rsp), %r14
cmpq %rbp, %r14
ja 0x401a9f
movq %r14, %rax
shrq $0x6, %rax
movq 0x158(%rsp,%rax,8), %rax
btq %r14, %rax
jae 0x401a9a
leaq 0xb26e8(%rip), %rdi     # 0x4b4171
movl %r13d, %esi
leaq 0xca842(%rip), %rdx     # 0x4cc2d5
xorl %eax, %eax
callq 0x40bee0
incq %r14
jmp 0x401a68
incq 0x18(%rsp)
movq %rbp, %r14
cmpq 0x8(%rsp), %r14
je 0x401c52
incq %r14
movq %r14, %rax
shrq $0x6, %rax
movq 0x158(%rsp,%rax,8), %rsi
pushq $0x1
popq %rdx
movl %r14d, %ecx
shlq %cl, %rdx
btq %r14, %rsi
jae 0x401aa7
xorpd %xmm0, %xmm0
movapd %xmm0, 0xb0(%rsp)
movapd %xmm0, 0xa0(%rsp)
movapd %xmm0, 0x90(%rsp)
movapd %xmm0, 0x80(%rsp)
movapd %xmm0, 0x70(%rsp)
movapd %xmm0, 0x60(%rsp)
movapd %xmm0, 0x50(%rsp)
movapd %xmm0, 0x40(%rsp)
orq %rdx, 0x40(%rsp,%rax,8)
movq $0x1, 0xc0(%rsp)
xorl %eax, %eax
xchgl %eax, 0xe80d4(%rip)     # 0x4e9c00
leaq 0x38(%rsp), %rdi
xorl %esi, %esi
movq %rbx, %rdx
leaq 0x40(%rsp), %rcx
callq 0x41d480
testl %eax, %eax
jne 0x401ca9
leaq 0x30(%rsp), %rdi
xorl %esi, %esi
movq %rbx, %rdx
leaq 0xd0(%rsp), %rcx
callq 0x41d480
testl %eax, %eax
jne 0x401c97
callq 0x401ed5
movq %rax, %r12
pushq $0x5
popq %r15
movsd 0xb248d(%rip), %xmm0    # 0x4b4008
subq $0x1, %r15
jb 0x401bdd
movl $0x3d090, %edi          # imm = 0x3D090
movsd %xmm0, 0x28(%rsp)
callq 0x457e60
xorl %ebx, %ebx
xchgl %ebx, 0xe8067(%rip)     # 0x4e9c00
callq 0x401ed5
movq %rax, %rcx
subq %r12, %rcx
movq %rcx, %xmm0
punpckldq 0xb245f(%rip), %xmm0 # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
subpd 0xb2467(%rip), %xmm0    # 0x4b4020
movapd %xmm0, %xmm1
unpckhpd %xmm0, %xmm1            # xmm1 = xmm1[1],xmm0[1]
addsd %xmm0, %xmm1
cvtsi2sd %rbx, %xmm0
divsd %xmm0, %xmm1
minsd 0x28(%rsp), %xmm1
movq %rax, %r12
movapd %xmm1, %xmm0
jmp 0x401b7b
leaq 0xb25b8(%rip), %rdi     # 0x4b419c
movl %r13d, %esi
movb $0x1, %al
callq 0x40bee0
movl $0x1, 0xe8408(%rip)     # 0x4ea000
movq 0x38(%rsp), %rdi
xorl %esi, %esi
callq 0x41e520
testl %eax, %eax
jne 0x401c8e
movq 0x30(%rsp), %rdi
xorl %esi, %esi
callq 0x41e520
testl %eax, %eax
jne 0x401c85
movl $0x0, 0xe83da(%rip)     # 0x4ea000
movq 0xe83db(%rip), %r12     # 0x4ea008
callq 0x457dd0
movslq %eax, %rsi
movq %r12, %rdi
callq 0x457f30
testl %eax, %eax
jne 0x401ca0
andq $0x0, 0xe83bf(%rip)     # 0x4ea008
movq (%rsp), %rbx
jmp 0x401aa7
pushq $0xa
popq %rdi
callq 0x413420
movq 0xe54b7(%rip), %rax     # 0x4e7118
incq %rbp
jmp 0x4019b5
pushq $0xa
popq %rdi
callq 0x413420
xorl %eax, %eax
addq $0x1d8, %rsp            # imm = 0x1D8
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
leaq 0xb252e(%rip), %rdi     # 0x4b41ba
jmp 0x401cb0
leaq 0xb250d(%rip), %rdi     # 0x4b41a2
jmp 0x401cb0
leaq 0xb24ea(%rip), %rdi     # 0x4b4188
jmp 0x401cb0
leaq 0xb252c(%rip), %rdi     # 0x4b41d3
jmp 0x401cb0
leaq 0xb24c5(%rip), %rdi     # 0x4b4175
callq 0x40c110
pushq $0x1
popq %rdi
callq 0x40b2f0
movq $0x4e7728, %rax         # imm = 0x4E7728
movq (%rax), %rdi
movq (%r14), %rdx
leaq 0xb23fc(%rip), %rsi     # 0x4b40cd
xorl %eax, %eax
callq 0x40be20
jmp 0x401cb5
movq $0x4e7728, %rax         # imm = 0x4E7728
movq (%rax), %rcx
leaq 0xb236b(%rip), %rdi     # 0x4b4056
jmp 0x401d26
movq $0x4e7728, %rax         # imm = 0x4E7728
movq (%rax), %rcx
leaq 0xb23b8(%rip), %rdi     # 0x4b40b6
pushq $0x16
jmp 0x401d28
movq $0x4e7728, %rax         # imm = 0x4E7728
movq (%rax), %rcx
leaq 0xb2383(%rip), %rdi     # 0x4b4096
jmp 0x401d26
movq $0x4e7728, %rax         # imm = 0x4E7728
movq (%rax), %rcx
leaq 0xb2350(%rip), %rdi     # 0x4b4076
pushq $0x1f
popq %rsi
pushq $0x1
popq %rdx
callq 0x412dc0
jmp 0x401cb5
movq $0x4e7728, %rax         # imm = 0x4E7728
movq (%rax), %rcx
leaq 0xb23e2(%rip), %rdi     # 0x4b4126
pushq $0x29
jmp 0x401d28
leaq 0xb2401(%rip), %rdi     # 0x4b4150
jmp 0x401cb0