Package: shmutils
I’ve wondered why isn’t it easier to have multiple processes be able to have a shared memory space.I’ve also wondered about how to pass shared memory definitions between processes.Examplesimportmultiprocessingfromcontextlibimportsuppressfromconcurrent.futuresimportProcessPoolExecutor,as_completedfromshmutilsimportMappedMemory,MapFlags,MapProtectionsfromshmutils.utilsimportcffiwrapperfromshmutils.lockimportLockfromshmutils.shmimportshm_open,shm_unlinkdef_increment_to(m,lock:Lock,value:cffiwrapper,limit:int):"""
    Returns the number of times we increments the number
    """count=value[0]num_incr=0whilecount<limit:withlock:count=value[0]ifcount>=limit:breaknum_incr+=1value[0]=count+1returnnum_incrif__name__=="__main__":withsuppress(FileNotFoundError):shm_unlink("test-lock")fd=shm_open("test-lock","w+")size=4096*100fd.truncate(size)assertfd.size()==sizewithMappedMemory(0,size,MapProtections.READ_WRITE,MapFlags.SHARED,fd)asm:withProcessPoolExecutor(mp_context=multiprocessing.get_context("fork"))asexe:lock=Lock(m)counter=m.new("size_t *",0)withlock:result1=exe.submit(_increment_to,m,lock,cffiwrapper(counter,m),2_0000)result2=exe.submit(_increment_to,m,lock,cffiwrapper(counter,m),2_0000)tuple(as_completed((result1,result2)))r1,r2=result1.result(),result2.result()assert(result1.exception(),result2.exception())==(None,None)assertcounter[0]==2_0000assertr1+r2==2_0000A MappedMemory is helpful for straddling the relative/absolute address world.Roadmap✅ Virtual address stabilityMappedMemory.at(relative address to bytes)MappedMemory.absolute_at(absolute address to bytes)MappedMemory.abs_address_at(relative address -> absolute address)MappedMemory.relative_address_at(absolute address -> relative address)🚧 Support Pythonmultiprocessing.get_context("spawn")☐ figure out pickling ofcffi.CData☐ switch toinstructfor internal classesimplement something like:frominstructimportCBase,class_metadata,astuplefromshmutilsimportMemoryMap,MemoryFlags,ffiasshmffi...fd=shm_open(...).truncate(PAGE_SIZE)page=MemoryMap(None,PAGE_SIZE,flags=MemoryFlags.SHARED,fd=fd)...ffi=cffi.FFI()# You can include other ffi's to reuse c type declarationsffi.include(shmffi)ffi.cdef('''
    typedef enum {INACTIVE, ACTIVE, DELETED} OrgUserStatus_t;
''')# pass into the instruct.CBase class an ffi instead of ``instruct.default_ffi``classUser(CBase,ffi=ffi):__slots__='''
    struct org_user_t {
        uint64_t id;
        char     *fullname;
        uint8_t  fullname_len;
        OrgUserStatus_t status;
    };
    '''assertUser.__slots__==()assertffi.typeof(class_metadata(User,"cdecl"))isffi.typeof('struct org_user_t)assertffi.sizeof('struct org_user_t')==class_metadata(User,"csizeof")assertffiisclass_metadata(User,"default_ffi")lib=ffi.dlopen(None)# Allocate using ``ffi.new``u=User.new(12345,b"Autumn",6,lib.ACTIVE)assertUser.typeof(u)=='struct org_user_t*'assertffi.typeof(u.id)isffi.typeof('uint64_t')assertffi.typeof(u.fullname)isffi.typeof('char*')assertffi.typeof(u.fullname_len)isffi.typeof('uint8_t')assertlen(memoryview(User.getbuffer(u)))==User.sizeof()assertlen(memoryview(User.getbuffer(u)))==User.sizeof(u)assertnothasattr(u,'sizeof')assertu.__internals__["heap"]isNone# Allocate using an alternate function# in this case, use the ``.new`` malloc for the# shared pageSharedUser:Type[User]=User.with_heap(page)u2=SharedUser.new()assertu2.__internals__["heap"]ispageassertu2.id==0assertu2.fullname==ffi.NULLassertu2.fullname_len==0assertu2.status==0# as far as the cdata is concerned, it points into the ``page``'s heap# User the ``CBase``'s ``.addressof`` call to get a pointer to the entityabs_ptr=ffi.cast('uintptr_t',User.addressof(u2))assertpage.address.begin<=int(abs_ptr)<page.address.end# page contents and buffer match each otherassertpage.absolute_at[abs_ptr:abs_ptr+User.sizeof()]==User.getbuffer()[0:User.sizeof()]# demo assign# allocate space for a nameraw_u2_fullname=page.new('char*',b'Autumn Jolitz')u2.id=4123u2.fullname=u2_fullname# assign the pointeru2.fullname_len=13u2.status=lib.ACTIVEassertastuple(u2)==(u2.id,raw_u2_fullname,u2.fullname_len,lib.ACTIVE)u2_copy=pickle.loads(pickle.dumps(u2))assertu2_copy.__internals__["heap"]ispageassertastuple(u2)==astuple(u2_copy)☐ split Locks intoRawLock|RawRLock(consumes amemoryview|bytebuffer|ffi.buffer, allocates from 0 to length of lock size)☐ split Locks intoLock|RLock☐ reimplement locking in terms of a condition variable☐ use liblfs for a freelist☐ make a shared heap process-safeLimitationsToy allocator, does not synchronize writes between processes (only parent is expected to have allocated ahead of time).spawn vs forkUse “fork” multiprocessing method instead.The “spawn” multiprocessing method is subject to ASLR andsometimesthe kernel locates a child process starting much higher than our process. This has the effect of breaking the requirement for absolute pointers working.However, it is observed that if a parent process manages to get a high enough memory page, the probability of the child process being able tommap(2)the same address increases significantly.importmultiprocessingfromcontextlibimportsuppressfromconcurrent.futuresimportProcessPoolExecutor,waitfrommmapimportPAGESIZEfromshmutilsimportMappedMemory,MapFlagsfromshmutils.mmapimportmunmap,round_to_page_sizefromshmutils.shmimportshm_open,ffi,shm_unlinkfromshmutils.utilsimportcffiwrapper# cffiwrapper - use to pickle/unpickle cffi objects between processesdef_set_data_to(value:cffiwrapper,to:int)->int:was=value[0]foriinrange(was,to):value[0]=ivalue[0]=toreturnwasif__name__=="__main__":withsuppress(FileNotFoundError):shm_unlink("test-mmap-spawn")withshm_open("test-mmap-spawn","x+")asfd:shared_size=round_to_page_size(1024*1024*1024)fd.truncate(shared_size)# Allocate a dummy 512 MiB blockrangeunused_space=MappedMemory(None,512*1024*1024)# write to the pages to ensure we're not being fooledunused_space[len(unused_space)-PAGESIZE:len(unused_space)-PAGESIZE+4]=b"sink"# Calculate the last page in the unused space rangestart_address:int=unused_space.abs_address_at[len(unused_space)-PAGESIZE]# detach the unused space guts so we can free all bu the last pageraw_address,size=unused_space.detach()# free all BUT the last pagemunmap(raw_address,size-PAGESIZE)delunused_space# Prove our start address is the last page of the mostly freed range# (our last page is still mapped.)assertint(ffi.cast("uintptr_t",raw_address))+size-PAGESIZE==start_addresswithMappedMemory(start_address,shared_size,flags=MapFlags.SHARED|MapFlags.FIXED,fd=fd)asm:withProcessPoolExecutor(1,mp_context=multiprocessing.get_context("spawn"))asexe:value=m.new("int64_t*",1923)assertvalue[0]==1923# The child process will now be able to mess with this counterfuture=exe.submit(_set_data_to,cffiwrapper(value,m),8900)wait([future])# And we can see the results both on the value in memory and from the# returnassertfuture.done()andnotfuture.exception()assert(future.result(),value[0])==(1923,8900)