// File: /workspace/github_c_makefile_star2_O2/USArmyResearchLab[P]ARL_OpenSHMEM/src/shmem_init.c
// Function: shmem_init
SHMEM_SCOPE
void shmem_init(void)
{
	int run_id;
	int my_pe;
	int n_pes;
	int n_pes_log2;
	size_t heap_size;

	debug("shmem_init begin");

	bzero(&__shmem,sizeof(shmem_internals_t));

	const char* env_var_run_id = getenv("SHMEM_RUN_ID");
	const char* env_var_my_pe = getenv("SHMEM_MY_PE");
	const char* env_var_n_pes = getenv("SHMEM_N_PES");
	const char* env_var_heap_size = getenv("SHMEM_SYMMETRIC_HEAP_SIZE");

	if ( env_var_run_id && env_var_my_pe && env_var_n_pes ) {
		run_id = atoi(env_var_run_id);
		my_pe = atoi(env_var_my_pe);
		n_pes = atoi(env_var_n_pes);
	} else {
		run_id = getpid();
		my_pe = 0;
		n_pes = 1;
		fprintf(stderr,"shmem warning: SHMEM_RUN_ID, SHMEM_MY_PE, or"
			" SHMEM_N_PES not set. Defaulting to serial execution\n");
	}

	if (env_var_heap_size) {
		size_t len = strnlen(env_var_heap_size,1024);
		unsigned int x = 1;
		if (env_var_heap_size[len-1]=='K') {
			x = 1024;
		} else if (env_var_heap_size[len-1]=='M') {
			x = 1024*1024;
		}
		heap_size = x * atoi(env_var_heap_size);
	} else {
		heap_size = 128*1024*1024;
		if (my_pe == 0) {
			fprintf(stderr,"shmem warning: SHMEM_SYMMETRIC_HEAP_SIZE not set."
				" Defaulting to %ld bytes\n", heap_size);
		}
	}

	debug("run_id=%d\nmy_pe=%d\nn_pes=%d\nheap_size=%ld\n",
		run_id, my_pe, n_pes, heap_size);

	if ( n_pes < 0 || n_pes > SHMEM_MAX_PES ) {
		fprintf(stderr,"shmem error: n_pes out of range: %d\n",n_pes);
		exit(-1);
	}

	if ( my_pe < 0 || my_pe >= n_pes ) {
		fprintf(stderr,"shmem error: my_pe out of range: %d\n",my_pe);
		exit(-1);
	}


	// log2_ceil of n_pes precalculated once
	n_pes_log2 = 0;
	unsigned int x = n_pes - 1;
	while (x > 0) {
		n_pes_log2++;
		x >>= 1;
	}

	__shmem.run_id = run_id;
	__shmem_my_pe = __shmem.my_pe = my_pe;
	__shmem_n_pes = __shmem.n_pes = n_pes;
	__shmem.n_pes_log2 = n_pes_log2;
	__shmem.heap_size = heap_size;


	__shmem.giant_barrier_count = 0;
	sem_init(&__shmem.giant_mutex,1,0);
	sem_init(&__shmem.giant_barrier_gate1,1,0);
	sem_init(&__shmem.giant_barrier_gate2,1,0);

	sem_init(&__shmem.sync_mutex,1,0);


	/* First we need to remap .data and .bss into a shm allocation. */

	int data_fd;
	void* data_start;
	size_t data_size;

	find_data_pages(&data_start,&data_size);

	intptr_t data_shm_base 
		= (intptr_t)shm_alloc(n_pes*data_size,run_id,0,&data_fd);
	
	munmap((void*)(data_shm_base+my_pe*data_size), data_size);

	remap_pages(data_fd, my_pe*data_size, data_start, data_size);

	__shmem.data_fd = data_fd;
	__shmem.data_base = (intptr_t)data_start;
	__shmem.data_size = data_size;
	__shmem.data_shm_base = data_shm_base;

	debug("symmetric data:\n\tdata_fd=%d\n\tdata_base=%p\n\tdata_size=%ld\n"
		"\tdata_shm_base=%p\n",data_fd,data_start,data_size,(void*)data_shm_base);


	/* we now have enough information to setup the symmetric heap -DAR */

	debug("shmem_init setup shared memory region");

	intptr_t heap_shm_base = (intptr_t)shm_alloc(n_pes*heap_size,run_id,1,0);

	debug("shmem_init shm_alloc returned %p",(void*)heap_shm_base);

	__shmem.heap_shm_base = heap_shm_base;

	debug("__shmem.heap_shm_base %p\n",(void*)__shmem.heap_shm_base);


	int c, r;
	for (c = 0, r = 1; r < __shmem.n_pes; c++, r <<= 1)
	{
		int to = __shmem.my_pe + r;
		if (to >= __shmem.n_pes) to -= __shmem.n_pes;
		long* lock = (long*)(__shmem.barrier_sync + c);
		long* remote_lock = (long*)shmem_ptr((void*)lock, to);
		__shmem.barrier_psync[c] = remote_lock;
	}
	debug("pe %d barrier_psync[0] %p",__shmem.my_pe,__shmem.barrier_psync[0]);


	__shmem.local_mem_base 
		= __shmem.heap_shm_base + my_pe * heap_size + getpagesize();

	__shmem.free_mem = __shmem.local_mem_base;

	__shmem.xmem0 = xmalloc_init( 
		(void*)__shmem.local_mem_base, heap_size - getpagesize(), 0 );

	debug("[%d] before barrier\n",my_pe);

	__shmem.p_giant_mutex = (sem_t*)shmem_ptr((void*)&__shmem.giant_mutex,0);

	if (my_pe == 0) sem_post(&__shmem.giant_mutex);
	sem_post(&__shmem.sync_mutex);


	shmem_sync_all();


	debug("[%d] after barrier\n",my_pe);


	if (__shmem.n_pes == 1) return;

	if (!__shmem.my_pe) {
		debug("master set psync to 1 @ %p",__shmem.barrier_psync[0]);
		*__shmem.barrier_psync[0] = 1;
	}

	debug("spin on sync at %p",__shmem.barrier_sync);
	while(*__shmem.barrier_sync == SHMEM_SYNC_VALUE);

	if (__shmem.my_pe) {
		debug("slave set psync to 1 @ %p",__shmem.barrier_psync[0]);
		*__shmem.barrier_psync[0] = 1;
	}

	debug("set sync to SHMEM_SYNC_VALUE @ %p",__shmem.barrier_sync);
	*__shmem.barrier_sync = SHMEM_SYNC_VALUE;

	debug("shmem_init done");

	
}

// Assembly:
pushq %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0x38, %rsp
leaq 0x4b83(%rip), %rdi      # 0x61f0
xorl %r13d, %r13d
movl $0x298, %edx            # imm = 0x298
xorl %esi, %esi
callq 0x1130
leaq 0x2a05(%rip), %rdi      # 0x4088
callq 0x1030
movq %rax, %r14
leaq 0x2a03(%rip), %rdi      # 0x4095
callq 0x1030
movq %rax, %r15
leaq 0x2a00(%rip), %rdi      # 0x40a1
callq 0x1030
movq %rax, %r12
leaq 0x29fd(%rip), %rdi      # 0x40ad
callq 0x1030
movq %rax, %rbx
testq %r14, %r14
je 0x1a25
testq %r15, %r15
je 0x1a25
testq %r12, %r12
je 0x1a25
movq %r14, %rdi
callq 0x1210
movl %eax, (%rsp)
movq %r15, %rdi
callq 0x1210
movl %eax, %r13d
movq %r12, %rdi
callq 0x1210
movl %eax, %r14d
testq %rbx, %rbx
je 0x172f
movl $0x400, %esi            # imm = 0x400
movq %rbx, %rdi
callq 0x1140
movb -0x1(%rbx,%rax), %bpl
movq %rbx, %rdi
callq 0x1210
movl %eax, %ebx
xorl %eax, %eax
cmpb $0x4d, %bpl
pushq $0x14
popq %rdx
cmovnel %eax, %edx
cmpb $0x4b, %bpl
pushq $0xa
popq %rcx
cmovnel %edx, %ecx
shll %cl, %ebx
jmp 0x173d
movl $0x8000000, %ebx        # imm = 0x8000000
testl %r13d, %r13d
je 0x1a56
cmpl $0x10001, %r14d         # imm = 0x10001
jae 0x1a80
cmpl %r14d, %r13d
jae 0x1a96
leal -0x1(%r14), %ecx
xorl %eax, %eax
testl %ecx, %ecx
je 0x1763
incl %eax
shrl %ecx
jmp 0x1759
movl (%rsp), %ecx
movl %ecx, 0x4a84(%rip)      # 0x61f0
movl %r13d, 0x4a81(%rip)     # 0x61f4
movl %r13d, 0x49c6(%rip)     # 0x6140
movl %r14d, 0x4a77(%rip)     # 0x61f8
movl %r14d, 0x49bc(%rip)     # 0x6144
movl %eax, 0x4a6e(%rip)      # 0x61fc
movq %rbx, 0x30(%rsp)
movq %rbx, 0x4ba6(%rip)      # 0x6340
andl $0x0, 0x4b17(%rip)      # 0x62b8
leaq 0x4ae8(%rip), %rdi      # 0x6290
pushq $0x1
popq %r12
xorl %ebp, %ebp
movl %r12d, %esi
xorl %edx, %edx
callq 0x11d0
leaq 0x4b01(%rip), %rdi      # 0x62c0
movl %r12d, %esi
xorl %edx, %edx
callq 0x11d0
leaq 0x4b10(%rip), %rdi      # 0x62e0
movl %r12d, %esi
xorl %edx, %edx
callq 0x11d0
leaq 0x4b1f(%rip), %rdi      # 0x6300
movl %r12d, %esi
xorl %edx, %edx
callq 0x11d0
callq 0x1220
movslq %eax, %rdx
movq %rdx, %rax
negq %rax
movl %r13d, %ebx
leaq 0x492d(%rip), %r13      # 0x6130
andq %rax, %r13
leaq 0x4cab(%rip), %rcx      # 0x64b8
subq %r13, %rcx
andq %rcx, %rax
movq %rdx, 0x28(%rsp)
leaq (%rax,%rdx), %r15
cmpq %rcx, %rax
cmoveq %rcx, %r15
movl %r14d, %eax
movq %r15, %rdi
movq %rax, 0x18(%rsp)
imulq %rax, %rdi
leaq 0xc(%rsp), %r14
movl (%rsp), %esi
xorl %edx, %edx
movq %r14, %rcx
callq 0x1ab9
movl %ebx, 0x8(%rsp)
movl %ebx, %ecx
movq %r15, %rbx
movq %rcx, 0x20(%rsp)
imulq %rcx, %rbx
movq %rax, 0x10(%rsp)
leaq (%rax,%rbx), %rdi
movq %r15, %rsi
callq 0x11c0
movl (%r14), %eax
movl %eax, 0x4(%rsp)
pushq $0x3
popq %rdx
pushq $0x22
popq %rcx
pushq $-0x1
popq %r8
xorl %edi, %edi
movq %r15, %rsi
xorl %r9d, %r9d
callq 0x10e0
testq %rax, %rax
je 0x18db
movq %rax, %r14
movq %rbx, %r9
movq %r15, %rbx
shrq $0x3, %rbx
xorl %eax, %eax
cmpq %rax, %rbx
je 0x18ac
movq (%r13,%rax,8), %rcx
movq %rcx, (%r14,%rax,8)
incq %rax
jmp 0x1899
pushq $0x3
popq %rdx
pushq $0x11
popq %rcx
movq %r13, %rdi
movq %r15, %rsi
movl 0x4(%rsp), %r8d
callq 0x10e0
testq %rax, %rax
je 0x18db
xorl %ecx, %ecx
cmpq %rcx, %rbx
je 0x18db
movq (%r14,%rcx,8), %rdx
movq %rdx, (%rax,%rcx,8)
incq %rcx
jmp 0x18c9
movl 0xc(%rsp), %eax
movl %eax, 0x4a3b(%rip)      # 0x6320
movq %r13, 0x4a44(%rip)      # 0x6330
movq %r15, 0x4a35(%rip)      # 0x6328
movq 0x10(%rsp), %rax
movq %rax, 0x4a39(%rip)      # 0x6338
movq 0x30(%rsp), %r15
movq 0x18(%rsp), %rdi
imulq %r15, %rdi
movl (%rsp), %esi
movl %r12d, %edx
xorl %ecx, %ecx
callq 0x1ab9
movq %rax, 0x4a27(%rip)      # 0x6348
leaq 0x4a38(%rip), %rbx      # 0x6360
movl 0x8(%rsp), %r14d
movl 0x48c5(%rip), %eax      # 0x61f8
cmpl %eax, %r12d
jge 0x195d
movl 0x48b6(%rip), %esi      # 0x61f4
addl %r12d, %esi
cmpl %eax, %esi
cmovll %ebp, %eax
subl %eax, %esi
movq %rbx, %rdi
callq 0x1c6c
movq %rax, 0x40(%rbx)
addl %r12d, %r12d
addq $0x8, %rbx
jmp 0x192d
movq 0x20(%rsp), %rdi
imulq %r15, %rdi
movq 0x28(%rsp), %rax
addq %rax, %rdi
addq 0x49d3(%rip), %rdi      # 0x6348
movq %rdi, 0x49d4(%rip)      # 0x6350
movq %rdi, 0x49d5(%rip)      # 0x6358
subq %rax, %r15
movq %r15, %rsi
xorl %edx, %edx
callq 0x1df8
movq %rax, 0x4ae9(%rip)      # 0x6480
leaq 0x48f2(%rip), %rdi      # 0x6290
xorl %esi, %esi
callq 0x1c6c
movq %rax, 0x4904(%rip)      # 0x62b0
testl %r14d, %r14d
jne 0x19bd
leaq 0x48d8(%rip), %rdi      # 0x6290
callq 0x1160
leaq 0x493c(%rip), %rdi      # 0x6300
callq 0x1160
callq 0x1cb4
cmpl $0x1, 0x4823(%rip)      # 0x61f8
je 0x1a16
movl 0x4817(%rip), %eax      # 0x61f4
testl %eax, %eax
jne 0x19ef
movq 0x49b8(%rip), %rcx      # 0x63a0
movq $0x1, (%rcx)
cmpq $0x0, 0x4969(%rip)      # 0x6360
je 0x19ef
testl %eax, %eax
je 0x1a0b
movq 0x499c(%rip), %rax      # 0x63a0
movq $0x1, (%rax)
movq $0x0, 0x494a(%rip) # 0x6360
addq $0x38, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
pushq $0x1
popq %r14
callq 0x10a0
movl %eax, (%rsp)
movq 0x45c0(%rip), %rax      # 0x5ff8
movq (%rax), %rcx
leaq 0x2685(%rip), %rdi      # 0x40c7
pushq $0x61
popq %rsi
pushq $0x1
popq %rdx
callq 0x1240
testq %rbx, %rbx
jne 0x16f9
movq 0x459b(%rip), %rax      # 0x5ff8
movq (%rax), %rdi
leaq 0x26c2(%rip), %rsi      # 0x4129
xorl %r13d, %r13d
movl $0x8000000, %ebx        # imm = 0x8000000
movl $0x8000000, %edx        # imm = 0x8000000
xorl %eax, %eax
callq 0x1180
jmp 0x173d
movq 0x4571(%rip), %rax      # 0x5ff8
movq (%rax), %rdi
leaq 0x26e3(%rip), %rsi      # 0x4174
movl %r14d, %edx
jmp 0x1aaa
movq 0x455b(%rip), %rax      # 0x5ff8
movq (%rax), %rdi
leaq 0x26f2(%rip), %rsi      # 0x4199
movl %r13d, %edx
xorl %eax, %eax
callq 0x1180
pushq $-0x1
popq %rdi
callq 0x1230