Package: pythonic-toolbox
Pythonic toolboxTable of ContentsIntroductionInstallationUsagecontextSkipContextdecoratorsignore_unexpected_kwargsretrydeque_utilsdeque_pop_anydeque_splitdict_utilsDictObjFinalDictObjRangeKeyDictStrKeyIdDictcollect_leavesdict_untilselect_list_of_dictsunique_list_of_dictswalk_leavesfunctional_utilsfilter_multilist_utilsfilter_allowablesort_with_custom_ordersunpack_listuntilstring_utilssubstitute_string_template_dictREADME.md is auto generated by the scripttests/generate_readme_markdown.pyfrom testing files,DO NOT EDIT DIRECTLY!;)python3tests/generate_readme_markdown.pyIntroductionA python3.6+ toolbox with multi useful utils, functions, decorators in pythonic way, and is fully tested from python3.6 to python3.11 .Installationpip3installpythonic-toolbox--upgradeUsagecontextSkipContextimportitertoolsimportpytestfrompythonic_toolbox.utils.context_utilsimportSkipContext# Usage: define a class that inherits the SkipContext,# and takes control of the skip or not logicclassMyWorkStation(SkipContext):def__init__(self,week_day:str):working_days={'monday','tuesday','wednesday','thursday','friday'}weekends={'saturday','sunday'}ifweek_day.lower()notinworking_days.union(weekends):raiseValueError(f'Invalid weekday{week_day}')skip=Trueifweek_day.lower()inweekendselseFalsesuper(MyWorkStation,self).__init__(skip=skip)seven_week_days=['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']logged_opening_days=[]total_working_hours=0forcur_week_dayinseven_week_days:# MyWorkStation will skip the code block when encountering weekendswithMyWorkStation(week_day=cur_week_day):# log this working daylogged_opening_days.append(cur_week_day)# accumulate working hours, 8 hours on each working daytotal_working_hours+=8# only working days are loggedassertlogged_opening_days==['Monday','Tuesday','Wednesday','Thursday','Friday']asserttotal_working_hours==8*5# test basic SkipContextcount_iterator=itertools.count(start=0,step=1)flg_skip=TruewithSkipContext(skip=flg_skip):# if skip = True, all codes inside the context will be skipped(not executed)next(count_iterator)# this will not be executedassertsum([1,1])==3raiseException('Codes will not be executed')assertnext(count_iterator)==0# check previous context is skippedflg_skip=FalsewithSkipContext(skip=flg_skip):# codes will be executed as normal, if skip = Falsenext(count_iterator)# generate value 1assertsum([1,1])==2assertnext(count_iterator)==2# check previous context is executedwithpytest.raises(Exception)asexec_info:withSkipContext(skip=False):# if skip = False, this SkipContextManager is transparent,# internal exception will be detected as normalraiseException('MyError')assertexec_info.value.args[0]=='MyError'# another example: ensure there will be only one job, who acquire the lock, run the increase +1frommultiprocessingimportManager,Poolimporttimefrompythonic_toolbox.utils.context_utilsimportSkipContextdefplain_cronjob_increase(ns,lock):start=time.time()withlock:now=time.time()ifnow-start>=0.5:passelse:ns.cnt+=1time.sleep(1)returnns.cntclassPreemptiveLockContext(SkipContext):def__init__(self,lock):self.start_time=time.perf_counter()self.lock=lockself.acquired=self.lock.acquire(timeout=0.5)skip=notself.acquiredsuper(PreemptiveLockContext,self).__init__(skip=skip)def__exit__(self,type,value,traceback):ifself.acquired:time.sleep(1)self.lock.release()iftypeisNone:return# No exceptionelse:ifissubclass(type,self.SkipContentException):returnTrue# Suppress special SkipWithBlockExceptionreturnFalsedefcronjob_increase(ns,lock):# for those who cannot acquire the lock within some time# this context block will be skipped, quite simplewithPreemptiveLockContext(lock):ns.cnt+=1returnns.cntmanager=Manager()lock=manager.Lock()ns=manager.Namespace()pool=Pool(2)ns.cnt=0processes=[pool.apply_async(plain_cronjob_increase,args=(ns,lock))for__inrange(0,2)]result=[p.get()forpinprocesses]assertresult==[1,1]assertns.cnt==1# reset global cnt=0ns.cnt=0processes=[pool.apply_async(cronjob_increase,args=(ns,lock))for__inrange(0,2)]result=[p.get()forpinprocesses]assertresult==[1,1]assertns.cnt==1decoratorsignore_unexpected_kwargsimportpytestfrompythonic_toolbox.decorators.commonimportignore_unexpected_kwargs# Following functions are named under Metasyntactic Variables, like:# foobar, foo, bar, baz, qux, quux, quuz, corge,# grault, garply, waldo, fred, plugh, xyzzy, thuddeffoo(a,b=0,c=3):returna,b,cdct={'a':1,'b':2,'d':4}withpytest.raises(TypeError)as__:assertfoo(**dct)==(1,2,3)wrapped_foo=ignore_unexpected_kwargs(foo)assertwrapped_foo(**dct)==(1,2,3)assertwrapped_foo(0,0,0)==(0,0,0)assertwrapped_foo(a=1,b=2,c=3)==(1,2,3)@ignore_unexpected_kwargsdefbar(*args:int):returnsum(args)# should not change original behaviorassertbar(1,2,3)==6assertbar(1,2,3,unexpected='Gotcha')==6nums=[1,2,3]assertbar(*nums,unexpected='Gotcha')==6@ignore_unexpected_kwargsdefqux(a,b,**kwargs):# function with Parameter.VAR_KEYWORD Aka **kwargsreturna,b,kwargs.get('c',3),kwargs.get('d',4)assertqux(**{'a':1,'b':2,'d':4,'e':5})==(1,2,3,4)classPerson:@ignore_unexpected_kwargsdef__init__(self,name,age,sex):self.name=nameself.age=ageself.sex=sex@classmethod@ignore_unexpected_kwargsdefcreate(cls,name,age,sex):returncls(name,age,sex)@staticmethod@ignore_unexpected_kwargsdefgreetings(name):returnf'Hello, I am{name}'params={'name':'albert','age':34,'sex':'male','height':'170cm',}__=Person(**params)__=Person('albert',35,'male',height='170cm')# test cases for classmethod, staticmethod__=Person.create(**params)assertPerson.greetings(**params)retryimportpytestfrompythonic_toolbox.decorators.commonimportretry# use decorator without any arguments, using retry default params@retrydeffunc_fail_first_time():"""func_fail_first_time"""self=func_fail_first_timeifnothasattr(self,'call_times'):# set attribute call_times for function, to count call timesself.call_times=0self.call_times+=1ifself.call_times==1:raiseException('Fail when first called')return'ok'assertfunc_fail_first_time()=='ok'assertfunc_fail_first_time.call_times==2assertfunc_fail_first_time.__doc__=='func_fail_first_time'@retry(tries=2,delay=0.1)# use decorator with customized paramsdeffunc_fail_twice():"""func_fail_twice"""self=func_fail_twiceifnothasattr(self,'call_times'):self.call_times=0self.call_times+=1ifself.call_times<=2:raiseException('Fail when called first, second time')return'ok'assertfunc_fail_twice()=='ok'assertfunc_fail_twice.call_times==3assertfunc_fail_twice.__doc__=='func_fail_twice'@retry(tries=2,delay=0.1)deffunc_fail_three_times():"""func_fail_three_times"""self=func_fail_three_timesifnothasattr(self,'call_times'):self.call_times=0self.call_times+=1ifself.call_times<=3:# 1, 2, 3raiseException('Fail when called first, second, third time')return'ok'withpytest.raises(Exception)asexec_info:func_fail_three_times()assertfunc_fail_three_times.call_times==3assertexec_info.value.args[0]=='Fail when called first, second, third time'importasyncio@retryasyncdefasync_func_fail_first_time():"""async_func_fail_first_time"""self=async_func_fail_first_timeifnothasattr(self,'call_times'):self.call_times=0self.call_times+=1ifself.call_times==1:raiseException('Fail when first called')return'ok'@retry(delay=0.1)asyncdefasync_func_fail_first_time2():"""async_func_fail_first_time2"""self=async_func_fail_first_time2ifnothasattr(self,'call_times'):self.call_times=0self.call_times+=1ifself.call_times==1:raiseException('Fail when first called')return'ok'asyncdefasync_main():assertawaitasync_func_fail_first_time()=='ok'assertasync_func_fail_first_time.__doc__=='async_func_fail_first_time'assertasync_func_fail_first_time.call_times==2assertawaitasync_func_fail_first_time2()=='ok'assertasync_func_fail_first_time2.call_times==2assertasync_func_fail_first_time2.__doc__=='async_func_fail_first_time2'loop=asyncio.get_event_loop()ifloop.is_closed():loop=asyncio.new_event_loop()try:loop.run_until_complete(async_main())finally:loop.close()importrandomfail_count=0@retry(delay=0.1)asyncdefalways_fail_func():nonlocalfail_countfail_count+=1awaitasyncio.sleep(random.random())raiseValueError()asyncdefasync_main_for_always_fail():nonlocalfail_counttasks=[always_fail_func()foriinrange(0,3)]results=awaitasyncio.gather(*tasks,return_exceptions=True)assertall(map(lambdae:isinstance(e,ValueError),results))assertfail_count==2*3# each func run twice, three func callsloop=asyncio.get_event_loop()ifloop.is_closed():loop=asyncio.new_event_loop()try:loop.run_until_complete(async_main_for_always_fail())finally:loop.close()deque_utilsdeque_pop_anyfromcollectionsimportdequeimportpytestfrompythonic_toolbox.utils.deque_utilsimportdeque_pop_anyqueue=deque([1,2,3,4,5])assertdeque_pop_any(queue,idx=1)==2assertqueue==deque([1,3,4,5])# edge case: same as deque.popleft()queue=deque([1,2,3,4,5])assertdeque_pop_any(queue,idx=0)==1assertqueue==deque([2,3,4,5])# edge case: same as deque.popright()queue=deque([1,2,3,4,5])assertdeque_pop_any(queue,idx=len(queue)-1)==5assertqueue==deque([1,2,3,4])queue=deque([1,2,3,4,5])withpytest.raises(IndexError)asexec_info:deque_pop_any(queue,idx=102)# edge case: pop from empty dequequeue=deque()withpytest.raises(IndexError)asexec_info:deque_pop_any(queue,idx=0)assertexec_info.value.args[0]=='pop from empty deque'deque_splitimportpytestfromcollectionsimportdequefrompythonic_toolbox.utils.deque_utilsimportdeque_splitqueue1,queue2=deque_split(deque([1,2,3,4,5]),num=3)assertqueue1==deque([1,2,3])assertqueue2==deque([4,5])queue1,queue2=deque_split(deque([1,2,3,4,5]),num=0)assertqueue1==deque([])assertqueue2==deque([1,2,3,4,5])queue1,queue2=deque_split(deque([1,2,3,4,5]),num=100)assertqueue1==deque([1,2,3,4,5])assertqueue2==deque([])withpytest.raises(ValueError)asexec_info:deque_split(deque([1,2,3,4,5]),-1)assertexec_info.value.args[0]=='num must be integer: 0 <= num <= sys.maxsize'dict_utilsDictObjfromcopyimportdeepcopyimportpytestfrompythonic_toolbox.utils.dict_utilsimportDictObjnaive_dct={'key1':'val1','key2':'val2',}obj=DictObj(naive_dct)# test basic functional methods like dictassertlen(obj)==2assertbool(obj)isTrue# same behavior like ordinary dict according to the python version (FILO for popitem for 3.6+)assertobj.popitem()==('key2','val2')assertobj.popitem()==('key1','val1')withpytest.raises(KeyError)as__:obj.popitem()# a key can be treated like an attribute# an attribute can be treated like a keyobj.key3='val3'assertobj.pop('key3')=='val3'withpytest.raises(KeyError)as__:obj.pop('key4')obj.key5='val5'delobj.key5withpytest.raises(KeyError)as__:obj.pop('key5')withpytest.raises(AttributeError)as__:delobj.key5# test deepcopyobj=DictObj({'languages':['Chinese','English']})copied_obj=deepcopy(obj)assertcopied_obj==objcopied_obj.languages=obj.languages+['Japanese']assertobj.languages==['Chinese','English']assertcopied_obj.languages==['Chinese','English','Japanese']assertcopied_obj!=objperson_dct={'name':'Albert','age':'34','sex':'Male','languages':['Chinese','English']}person=DictObj(person_dct)assertDictObj(person_dct)==DictObj(person_dct)assertperson.to_dict()==person_dctassertset(person.keys())=={'name','age','sex','languages'}asserthasattr(person,'name')isTrueassertperson.name=='Albert'assertperson['name']=='Albert'person.languages.append('Japanese')assertperson.languages==['Chinese','English','Japanese']person.height='170'assertperson['height']=='170'assert'height'inpersonassert'height'inperson.keys()asserthasattr(person,'height')isTruedelperson['height']assert'height'notinpersonassert'height'notinperson.keys()person['height']='170cm'person.update({'weight':'50'})weight_val=person.pop('weight')assertweight_val=='50'person.update(DictObj({'weight':'50kg'}))assertperson.weight=='50kg'expected={'name':'Albert','age':'34','sex':'Male','languages':['Chinese','English','Japanese'],# appended new language'height':'170cm',# new added attribute'weight':'50kg',# new added attribute}assertperson.to_dict()==expectedrepr_expected:str=("{'name': 'Albert', 'age': '34', 'sex': 'Male', ""'languages': ['Chinese', 'English', 'Japanese'],"" 'height': '170cm', 'weight': '50kg'}")assertrepr(person)==repr_expected# nested structure will be detected, and changed to DictObjchessboard_data={'position':[[{'name':'knight'},{'name':'pawn'}],[{'name':'pawn'},{'name':'queen'}],]}chessboard_obj=DictObj(chessboard_data)# test comparing instances of DictObjassertDictObj(chessboard_data)==DictObj(chessboard_data)assertisinstance(chessboard_obj.position,list)assertlen(chessboard_obj.position)==2assertisinstance(chessboard_obj.position[0][0],DictObj)assertchessboard_obj.position[0][0].name=='knight'assertchessboard_obj.position[1][1].name=='queen'# edge case empty DictObjempty_dict_obj=DictObj({})assertlen(empty_dict_obj)==0assertbool(empty_dict_obj)isFalseobj_dict=DictObj({'data':'oops'})assertobj_dict.data=='oops'# params validationinvalid_key_dct={1:'1',}# test when dict's key is not strwithpytest.raises(ValueError)as__:__=DictObj(invalid_key_dct)complicated_key_dct={'1abc':'Gotcha',# '1abc' is not valid identifier for Python, so obj.1abc will cause SyntaxError'class':'MyClass',# 'class' is keyword in Python, so obj.class will cause SyntaxError}obj_dict=DictObj(complicated_key_dct)assertobj_dict['1abc']=='Gotcha'assertgetattr(obj_dict,'1abc')=='Gotcha'# you can access '1abc' as attribute by adding prefix '_'assertobj_dict._1abc=='Gotcha'delobj_dict._1abcassertobj_dict['class']=='MyClass'assertgetattr(obj_dict,'class')=='MyClass'# you can access 'class' as attribute by adding prefix '_'assertobj_dict._class=='MyClass'# test re-assign new value for 'class'obj_dict._class='MyClass2'assertobj_dict._class=='MyClass2'assertobj_dict['class']=='MyClass2'assertgetattr(obj_dict,'class')=='MyClass2'delobj_dict._class# if assign new attributes (_2, _try), ObjDict will treat it like what the originally are# this is fully considered by design, you're not encouraged to mess up keysobj_dict._2x='NewAttr'assertobj_dict._2x=='NewAttr'assertobj_dict['_2x']=='NewAttr'withpytest.raises(KeyError):__=obj_dict['2x']withpytest.raises(AttributeError):__=getattr(obj_dict,'2x')obj_dict._try='NewAttr'assertobj_dict._try=='NewAttr'assertobj_dict['_try']=='NewAttr'withpytest.raises(KeyError):__=obj_dict['NewAttr']withpytest.raises(AttributeError):__=getattr(obj_dict,'NewAttr')# Demo for messing up key 'class'# delete and re-assign _classcomplicated_key_dct={'class':'MyClass',# 'class' is keyword in Python, so obj.class will cause SyntaxError}obj_dict=DictObj(complicated_key_dct)assertobj_dict['class']=='MyClass'obj_dict._class='MyClass2'assertobj_dict['class']=='MyClass2'delobj_dict._class# obj_dict has no knowledge about 'class' or '_class'# so '_class' is a brand-new attribute, and will be stored as '_class'obj_dict._class='MyClass3'withpytest.raises(KeyError):# Oops!!! by-design# 'class' cannot be accessed as key anymore,# because we store '_class' as key as other valid keys behaveassertobj_dict['class']=='MyClass3'assertobj_dict['_class']=='MyClass3'# thread safe testingimportsysfromthreadingimportThreadfrompythonic_toolbox.decorators.decorator_utilsimportmethod_synchronizedclassMyObjDict(DictObj):# implement a thread-safe method to increase the value of cnt@method_synchronizeddefincrease_cnt_by_n(self,n):self.cnt+=ndefincrease_cnt_by_100(dict_obj):foriinrange(100):dict_obj.increase_cnt_by_n(1)sw_interval=sys.getswitchinterval()try:sys.setswitchinterval(0.0001)my_dict_obj=MyObjDict({'cnt':0})threads=[Thread(target=increase_cnt_by_100,args=(my_dict_obj,))for_inrange(100)][t.start()fortinthreads][t.join()fortinthreads]assertmy_dict_obj.cnt==10000finally:sys.setswitchinterval(sw_interval)# test copy/deepcopy of DictObjimportcopyperson=DictObj({'name':'albert','age':33})team=DictObj({'leader':person})shallow_copy_of_team=copy.copy(team)assertteam.leaderisshallow_copy_of_team.leaderdeep_copy_of_team=copy.deepcopy(team)assertteam.leaderisnotdeep_copy_of_team.leaderassertteam.leader==deep_copy_of_team.leaderFinalDictObjfromtypingimportcastimportpytestfrompythonic_toolbox.utils.dict_utilsimportFinalDictObjperson_dct={'name':'Albert','age':'34','sex':'Male','languages':['Chinese','English']}fixed_person=FinalDictObj(person_dct)assertfixed_person.name=='Albert'# FINAL means once initialized, you cannot change the key/attribute anymorewithpytest.raises(RuntimeError)asexec_info:fixed_person.name='Steve'expected_error_str='Cannot modify attribute/item in an already initialized FinalDictObj'assertexec_info.value.args[0]==expected_error_strwithpytest.raises(RuntimeError)as__:fixed_person.popitem()withpytest.raises(RuntimeError)as__:fixed_person.pop('name')assertisinstance(fixed_person.languages,tuple)withpytest.raises(AttributeError)asexec_info:# list values are changed into tuple to avoid being modifiedcast(list,fixed_person.languages).append('Japanese')expected_error_str="'tuple' object has no attribute 'append'"assertexec_info.value.args[0]==expected_error_strassertfixed_person.to_dict()==person_dct# nested structure will be detected, and changed to FinalDictObjchessboard_data={'position':[[{'name':'knight'},{'name':'pawn'}],[{'name':'pawn'},{'name':'queen'}],]}chessboard_obj=FinalDictObj(chessboard_data)# test comparing instances of FinalDictObjassertFinalDictObj(chessboard_data)==FinalDictObj(chessboard_data)assertisinstance(chessboard_obj.position,tuple)assertisinstance(chessboard_obj.position[0][0],FinalDictObj)assertchessboard_obj.position[1][1].name=='queen'withpytest.raises(RuntimeError)as__:chessboard_obj.position[1][1].name='knight'# test for keyword/non-identifier key as attributefinal_obj_dict=FinalDictObj({'class':'MyClass',# 'class' is keyword in Python, so obj.class will cause SyntaxError})assertfinal_obj_dict['class']=='MyClass'assertgetattr(final_obj_dict,'class')=='MyClass'assertfinal_obj_dict._class=='MyClass'# test copy/deepcopy of FileDictObjimportcopyperson=FinalDictObj({'name':'albert','age':33})team=FinalDictObj({'leader':person})shallow_copy_of_team=copy.copy(team)assertteam.leaderisshallow_copy_of_team.leaderassertteam.leader==shallow_copy_of_team.leaderdeep_copy_of_team=copy.deepcopy(team)assertteam.leaderisnotdeep_copy_of_team.leaderassertteam.leader==deep_copy_of_team.leaderRangeKeyDictimportpytestfrompythonic_toolbox.utils.dict_utilsimportRangeKeyDict# test normal caserange_key_dict:RangeKeyDict[float,str]=RangeKeyDict({(float('-inf'),0):'Negative',(0,60):'F',# 0 <= val < 60(60,70):'D',# 60 <= val < 70(70,80):'C',# 70 <= val < 80(80,90):'B',# 80 <= val < 90(90,100):'A',# 90 <= val < 100100:'A+',# val == 100})# Big O of querying is O(log n), n is the number of ranges, due to using bisect insideassertrange_key_dict[-1]=='Negative'assertrange_key_dict[0]=='F'assertrange_key_dict[55]=='F'assertrange_key_dict[60]=='D'assertrange_key_dict[75]=='C'assertrange_key_dict[85]=='B'assertrange_key_dict[95]=='A'assertrange_key_dict[100]=='A+'withpytest.raises(KeyError)asexec_info:_=range_key_dict['95']# when key is not comparable with other integer keysassertexec_info.value.args[0]=="KeyError: '95' is not comparable with other keys"withpytest.raises(KeyError)asexec_info:_=range_key_dict[150]assertexec_info.value.args[0]=='KeyError: 150'assertrange_key_dict.get(150,'N/A')=='N/A'# test comparison with other RangeKeyDictassertRangeKeyDict({(0,10):'1'})==RangeKeyDict({(0,10):'1'})assertRangeKeyDict({(0,10):'1'})!=RangeKeyDict({(0,10):'2'})assertRangeKeyDict({(0,10):'1'})!=RangeKeyDict({(0,1000):'1'})withpytest.raises(ValueError):# [1, 1) is not a valid range# there's no value x satisfy 1 <= x < 1RangeKeyDict({(1,1):'1'})withpytest.raises(ValueError):# [1, -1) is not a valid rangeRangeKeyDict({(1,-1):'1'})# validate input keys types and detect range overlaps(segment intersect)withpytest.raises(ValueError)asexec_info:RangeKeyDict({(0,10):'val-between-0-and-10',(0,5):'val-between-0-and-5'})expected_error_msg=("Duplicated left boundary key 0 detected: ""(0, 10): 'val-between-0-and-10', (0, 5): 'val-between-0-and-5'")assertexec_info.value.args[0]==expected_error_msgwithpytest.raises(ValueError)asexec_info:RangeKeyDict({(0,10):'val-between-0-and-10',(5,15):'val-between-5-and-15'})expected_error_msg=("Overlap detected: ""(0, 10): 'val-between-0-and-10', (5, 15): 'val-between-5-and-15'")assertexec_info.value.args[0]==expected_error_msg# test RangeKeyDict with no continuous rangesrange_key_dict:RangeKeyDict[float,str]=RangeKeyDict({(0,60):'F',# 0 <= val < 60(70,80):'C',# 70 <= val < 80})assertrange_key_dict[10]=='F'withpytest.raises(KeyError)asexec_info:_=range_key_dict[-100]assertexec_info.value.args[0]=='KeyError: -100'withpytest.raises(KeyError)asexec_info:_=range_key_dict[65]assertexec_info.value.args[0]=='KeyError: 65'withpytest.raises(KeyError)asexec_info:_=range_key_dict[100]assertexec_info.value.args[0]=='KeyError: 100'fromfunctoolsimporttotal_ordering@total_orderingclassAge:def__init__(self,val:float):ifnotisinstance(val,(int,float)):raiseValueError('Invalid age value')self.val=valdef__le__(self,other):returnself.val<=other.valdef__repr__(self):returnf'Age({repr(self.val)})'def__hash__(self):returnhash(self.val)age_categories_map:RangeKeyDict[Age,str]=RangeKeyDict({(Age(0),Age(2)):'Baby',(Age(2),Age(15)):'Children',(Age(15),Age(25)):'Youth',(Age(25),Age(65)):'Adults',(Age(65),Age(123)):'Seniors',})assertage_categories_map[Age(0.5)]=='Baby'assertage_categories_map[Age(12)]=='Children'assertage_categories_map[Age(20)]=='Youth'assertage_categories_map[Age(35)]=='Adults'assertage_categories_map[Age(70)]=='Seniors'StrKeyIdDictimportpytestfrompythonic_toolbox.utils.dict_utilsimportStrKeyIdDictdata={1:'a',2:'b','3':'c'}my_dict=StrKeyIdDict(data)# usage: value can be accessed by id (str: int-like/uuid-like/whatever) or id (int)assertmy_dict['1']==my_dict[1]=='a'assertmy_dict.keys()=={'1','2','3'}# all keys are str typemy_dict['4']='d'assertmy_dict['4']=='d'my_dict[4]='d'assertmy_dict['4']=='d'my_dict.update({4:'d'})assertmy_dict['4']=='d'# test comparing instances of the classassertStrKeyIdDict(data)==StrKeyIdDict(data)assertStrKeyIdDict(data)!=StrKeyIdDict(dict(data,**{'4':'d'}))assertStrKeyIdDict(data)=={'1':'a','2':'b','3':'c'}assertStrKeyIdDict(data)!={'1':'a','2':'b','3':'d'}assertStrKeyIdDict(data)!={1:'a',2:'b',3:'c'}# StrKeyIdDict assumes all keys are strings# test delete keydelmy_dict[4]assertmy_dict.keys()=={'1','2','3'}# '4' is not in the dict anymore# assign value to an arbitrary string key that is not in the dictmy_dict.update({'some-uuid':'something'})assertmy_dict['some-uuid']=='something'withpytest.raises(TypeError):# key '1', 1 both stands for key '1',# so we get duplicated keys when initializing instance, oops!my_dict=StrKeyIdDict({'1':'a',1:'A'})assertmy_dict.get(1)=='a'assertmy_dict.get('NotExistKey')isNoneassertmy_dict.get('NotExistKey','NotExistValue')=='NotExistValue'# test edge casesassertStrKeyIdDict()=={}# test shallow copymy_dict[5]=['e1','e2','e3']copy_dict=my_dict.copy()copy_dict[1]='A'assertmy_dict[1]=='a'my_dict['5'].append('e4')assertcopy_dict['5']==['e1','e2','e3','e4']# test deep copyfromcopyimportdeepcopycopy_dict=deepcopy(my_dict)my_dict[5].append('e5')assertmy_dict['5']==['e1','e2','e3','e4','e5']assertcopy_dict[5]==['e1','e2','e3','e4']# test constructormy_dict=StrKeyIdDict(uuid1='a',uuid2='b')assertmy_dict['uuid1']=='a'# test constructor (from keys)my_dict=StrKeyIdDict.fromkeys([1,2,3],None)assertmy_dict=={'1':None,'2':None,'3':None}# test update and overwritemy_dict.update(StrKeyIdDict({1:'a',2:'b',3:'c',4:'d'}))assertmy_dict=={'1':'a','2':'b','3':'c','4':'d'}my_dict=StrKeyIdDict([(1,'a'),(2,'b'),(3,'c'),(4,'d')])assertmy_dict['1']==my_dict[1]=='a'# reassign StrKeyIdDict instance to another StrKeyIdDict instancemy_dict=StrKeyIdDict(my_dict)assertmy_dict=={'1':'a','2':'b','3':'c','4':'d'}assertdict(my_dict)=={'1':'a','2':'b','3':'c','4':'d'}# test case when "key" is "data", which is a reserved keyword inside StrKeyIdDictmy_dict=StrKeyIdDict({'data':'data_value','1':'a'})assertmy_dict['data']=='data_value'assertmy_dict['1']=='a'# delete key 'data', should not affect other keysdelmy_dict['data']assertmy_dict['1']=='a'collect_leavesfrompythonic_toolbox.utils.dict_utilsimportcollect_leaves# a nested dict-like structmy_dict={'node_1':{'node_1_1':{'node_1_1_1':'A',},'node_1_2':{'node_1_2_1':'B','node_1_2_2':'C','node_1_2_3':None,},'node_1_3':[# dict list{'node_1_3_1_1':'D','node_1_3_1_2':'E',},{'node_1_3_2_1':'FF','node_1_3_2_2':'GG',}]}}expected=['A','B','C',None,'D','E','FF','GG']assertcollect_leaves(my_dict)==expectedexpected=['A','B','C','D','E','FF','GG']assertcollect_leaves(my_dict,leaf_pred=lambdalf:lf)==expectedassertcollect_leaves(my_dict,keypath_pred=lambdakp:len(kp)==1)==[]expected=['B','C']assertcollect_leaves(my_dict,keypath_pred=lambdakp:kp[-1]in{'node_1_2_1','node_1_2_2'})==expectedexpected=['C']assertcollect_leaves(my_dict,leaf_pred=lambdalf:lf=='C')==expectedassertcollect_leaves(my_dict,keypath_pred=lambdakp:kp[-1]=='node_1_2_2',leaf_pred=lambdalf:lf=='C')==expectedassertcollect_leaves(my_dict,keypath_pred=lambdakp:kp[-1]=='node_1_1_1',leaf_pred=lambdalf:lf=='C')==[]expected=['D','E','FF','GG']assertcollect_leaves(my_dict,keypath_pred=lambdakp:len(kp)>=2andkp[-2]=='node_1_3')==expectedexpected=['FF','GG']assertcollect_leaves(my_dict,keypath_pred=lambdakp:len(kp)>=2andkp[-2]=='node_1_3',leaf_pred=lambdalf:isinstance(lf,str)andlen(lf)==2)==expected# edge casesassertcollect_leaves([])==[]assertcollect_leaves({})==[]assertcollect_leaves(None)==[]dict_untilfrompythonic_toolbox.utils.dict_utilsimportdict_untildata={'full_name':'Albert Lee','pen_name':None}assertdict_until(data,keys=['name','full_name'])=='Albert Lee'assertdict_until(data,keys=['full_name','name'])=='Albert Lee'assertdict_until(data,keys=['name','english_name'])isNoneassertdict_until(data,keys=['name','english_name'],default='anonymous')=='anonymous'# test when pen_name is set None on purposeassertdict_until(data,keys=['pen_name'],default='anonymous')isNone# test when value with None value is not acceptableassertdict_until(data,keys=['pen_name'],terminate=lambdax:xisnotNone,default='anonymous')=='anonymous'select_list_of_dictsfrompythonic_toolbox.utils.dict_utilsimportselect_list_of_dictsdict_lst=[{'name':'Tony Stark','sex':'male','age':49,'alias':'Iron Man'},{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'},# another Peter Parker from multiverse{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'},# age unknown for Carol Danvers, no age field{'name':'Carol Danvers','sex':'female','alias':'Captain Marvel'},{'name':'Natasha Romanoff','sex':'female','age':35,'alias':'Black Widow'},]assertselect_list_of_dicts(dict_lst,look_like={'name':'Peter Parker'})==[{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'},{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'}]assertselect_list_of_dicts(dict_lst,look_like={'sex':'female'})==[{'name':'Carol Danvers','sex':'female','alias':'Captain Marvel'},{'name':'Natasha Romanoff','sex':'female','age':35,'alias':'Black Widow'}]assertselect_list_of_dicts(dict_lst,look_like={'sex':'female'},keys=['name'])==[{'name':'Carol Danvers'},{'name':'Natasha Romanoff'}]# unique is supported for return listassertselect_list_of_dicts(dict_lst,look_like={'sex':'male'},keys=['name','age'])==[{'name':'Tony Stark','age':49},{'name':'Peter Parker','age':16},{'name':'Peter Parker','age':16},]assertselect_list_of_dicts(dict_lst,look_like={'sex':'male'},keys=['name','age'],unique=True)==[{'name':'Tony Stark','age':49},{'name':'Peter Parker','age':16}]# dict keys are ordered as the keys passed-inassertlist(select_list_of_dicts(dict_lst,keys=['name','age'],unique=True)[0].keys())==['name','age']assertlist(select_list_of_dicts(dict_lst,keys=['age','name'],unique=True)[0].keys())==['age','name']# locate Captain Marvel, with default val for missing keyassertselect_list_of_dicts(dict_lst,look_like={'alias':'Captain Marvel'},keys=['name','sex','age','alias'],val_for_missing_key='Unknown')[0]['age']=='Unknown'# edge cases, get the original dictassertselect_list_of_dicts([])==[]assertselect_list_of_dicts(dict_lst)==dict_lst# new list of dicts is returned, leaving the original list of dicts untouchedblack_widow=select_list_of_dicts(dict_lst,look_like={'name':'Natasha Romanoff'})[0]black_widow['age']+=1assertblack_widow['age']==36# we don't modify the original dict data, Natasha is always 35 years oldassertselect_list_of_dicts(dict_lst,look_like={'name':'Natasha Romanoff'})[0]['age']==35# preds provide more flexibility, filter the ones with age infoassertlen(select_list_of_dicts(dict_lst,preds=[lambdad:'age'ind,lambdad:d['age']>=0]))==4assertlen(select_list_of_dicts(dict_lst,preds=[lambdad:'age'ind,lambdad:d['age']>=0],unique=True))==3# combine look_like and preds parametersexpected=[{'name':'Tony Stark','sex':'male','age':49,'alias':'Iron Man'}]assertselect_list_of_dicts(dict_lst,look_like={'sex':'male'},preds=[lambdad:'age'ind,lambdad:d['age']>20])==expected# empty list is returned if no dict matches the criteriaassertselect_list_of_dicts(dict_lst,look_like={'sex':'male'},preds=[lambdad:'sex'indandd['sex']=='female'])==[]unique_list_of_dictsfrompythonic_toolbox.utils.dict_utilsimportunique_list_of_dictsdict_lst=[{'name':'Tony Stark','sex':'male','age':49,'alias':'Iron Man'},{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'},# Peter Parkers from multiverse in same age.{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'},{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'},]# Only one Peter Parker will be kept, for all data are exactly same.assertunique_list_of_dicts(dict_lst)==[{'name':'Tony Stark','sex':'male','age':49,'alias':'Iron Man'},{'name':'Peter Parker','sex':'male','age':16,'alias':'Spider Man'},]# edge casesassertunique_list_of_dicts([])==[]walk_leavesfrompythonic_toolbox.utils.dict_utilsimportwalk_leavesdata={'k1':{'k1_1':1,'k1_2':2,},'k2':'N/A',# stands for not available}expected={'k1':{'k1_1':2,'k1_2':4,},'k2':'N/A',# stands for not available}assertwalk_leaves(data)==data# no transform function provided, just a deepcopyassertwalk_leaves(data,trans_fun=lambdax:x*2ifisinstance(x,int)elsex)==expected# if inplace is set True, will change data inplace, return nothingassertwalk_leaves(data,trans_fun=lambdax:x*2ifisinstance(x,int)elsex,inplace=True)isNoneassertdata==expecteddata=[{'name':'lml','age':33},{'name':'albert','age':18}]expected=[{'name':'lml','age':66},{'name':'albert','age':36}]assertwalk_leaves(data,trans_fun=lambdax:x*2ifisinstance(x,int)elsex)==expectedassertwalk_leaves(data,trans_fun=lambdax:x*2ifisinstance(x,int)elsex,inplace=True)isNoneassertdata==expected# edge casesassertwalk_leaves(None)isNoneassertwalk_leaves([])==[]assertwalk_leaves({})=={}assertwalk_leaves(None,inplace=True)isNoneassertwalk_leaves([],inplace=True)isNoneassertwalk_leaves({},inplace=True)isNonefunctional_utilsfilter_multifrompythonic_toolbox.utils.functional_utilsimportlfilter_multi,filter_multifromcollections.abcimportIterabledefis_even(x):returnx%2==0defis_divisible_by_5(x):returnx%5==0# select numbers which are divisible by 2 and 5assertlfilter_multi([is_even,is_divisible_by_5],range(1,30))==[10,20]assertlfilter_multi([is_even,is_divisible_by_5],[5,10,15,20])==[10,20]fromitertoolsimportcount,takewhile# if you want to pass an iterator, make sure the iterator will end/break,# Note: a bare count(start=0, step=2) will generate number like 0, 2, 4, 6, .... (never ends)even_numbers_less_equal_than_50=takewhile(lambdax:x<=50,count(start=0,step=2))expected=[0,10,20,30,40,50]assertlfilter_multi([is_even,is_divisible_by_5],even_numbers_less_equal_than_50)==expected# testing for filter_multi, not converted to list directlynum_iterator=filter_multi([is_even,is_divisible_by_5],[5,10,15,20])asserttype(num_iterator)isfilterassertisinstance(num_iterator,Iterable)expected=[10,20]foridx,valueinenumerate(num_iterator):assertvalue==expected[idx]# when items are infinite, choose filter_multi instead of lfilter_multiexpected=[0,10,20,30,40,50]foridx,valueinenumerate(filter_multi([is_even,is_divisible_by_5],count(start=0,step=1))):ifvalue>50:breakelse:assertvalue==expected[idx]list_utilsfilter_allowablefrompythonic_toolbox.utils.list_utilsimportfilter_allowablefruits=['apple','banana','orange']vegetables=['carrot','potato','tomato']meats=['beef','chicken','fish']foods=fruits+vegetables+meatsassertlist(filter_allowable(foods))==foodsassertlist(filter_allowable(foods,allow_list=[],block_list=[]))==foodsassertlist(filter_allowable(foods,allow_list=['apple','banana','blueberry']))==['apple','banana']assertlist(filter_allowable(foods,allow_list=[],block_list=foods))==[]assertlist(filter_allowable(foods,block_list=meats))==fruits+vegetablesassertlist(filter_allowable(foods,allow_list=['apple'],block_list=[]))==['apple']assertlist(filter_allowable(foods,allow_list=['apple'],block_list=['apple']))==[]assertlist(filter_allowable(foods+['blueberry'],allow_list=[],block_list=foods))==['blueberry']assertlist(filter_allowable(['blueberry'],allow_list=[],block_list=[]))==['blueberry']assertlist(filter_allowable(['blueberry'],allow_list=[],block_list=['apple','banana']))==['blueberry']assertlist(filter_allowable(['blueberry'],allow_list=['orange'],block_list=['apple','banana']))==[]# test cases with parameter keyassertlist(filter_allowable(foods,allow_list=['a','b'],key=lambdax:x[0]))==['apple','banana','beef']# test some basic casesassertlist(filter_allowable())==[]assertlist(filter_allowable(candidates=None))==[]assertlist(filter_allowable(candidates=[]))==[]assertlist(filter_allowable(candidates=[],allow_list=[],block_list=[]))==[]sort_with_custom_ordersfromoperatorimportitemgetterfromtypingimportListimportpytestfrompythonic_toolbox.utils.list_utilsimportsort_with_custom_orders# basic usagevalues=['branch2','branch1','branch3','master','release']expected=['master','release','branch1','branch2','branch3']assertsort_with_custom_orders(values,prefix_orders=['master','release'])==expectedassertsort_with_custom_orders(values,prefix_orders=['master','release'],reverse=True)==expected[::-1]values=[1,2,3,9,9]expected=[9,9,1,2,3]assertsort_with_custom_orders(values,prefix_orders=[9,8,7])==expectedvalues=[1,2,3,9]expected=[9,2,3,1]assertsort_with_custom_orders(values,prefix_orders=[9],suffix_orders=[1])==expectedassertsort_with_custom_orders([])==[]assertsort_with_custom_orders([],prefix_orders=[],suffix_orders=[])==[]assertsort_with_custom_orders([],prefix_orders=['master'])==[]# tests for unhashable valuesvalues=[[2,2],[1,1],[3,3],[6,0]]assertsort_with_custom_orders(values,prefix_orders=[[3,3]])==[[3,3],[1,1],[2,2],[6,0]]# if "key" is provided, items are sorted in order of key(item)# items in prefix_orders/suffix_orders don't need to be one-one correspondence with items to sort# sum([6]) == sum([3, 3]) == sum([6, 0])assertsort_with_custom_orders(values,prefix_orders=[[6]],key=sum)==[[3,3],[6,0],[1,1],[2,2]]# tests for list of dictsvalues=[{2:2},{1:1},{1:2}]assertsort_with_custom_orders(values,prefix_orders=[{2:2}],key=lambdadata:sum(data.values()))==[{2:2},{1:2},{1:1}]branch_info:List[dict]=[{'branch':'master','commit_id':'v1.2'},{'branch':'release','commit_id':'v1.1'}]# Assume that we prefer choosing branch in order: release > master > others (develop, hotfix etc.)res=sort_with_custom_orders(branch_info,prefix_orders=[{'branch':'release'},{'branch':'master'}],key=itemgetter('branch'))expected=[{'branch':'release','commit_id':'v1.1'},{'branch':'master','commit_id':'v1.2'}]assertres==expectedbranch_info=[{'branch':'develop','commit_id':'v1.3'},{'branch':'master','commit_id':'v1.2'}]res=sort_with_custom_orders(branch_info,prefix_orders=[{'branch':'release'},{'branch':'master'}],key=itemgetter('branch'))expected=[{'branch':'master','commit_id':'v1.2'},{'branch':'develop','commit_id':'v1.3'}]assertres==expected# tests for exceptionswithpytest.raises(ValueError)asexec_info:sort_with_custom_orders([1,2,3],prefix_orders=[3],suffix_orders=[3])assertexec_info.value.args[0]=='prefix and suffix contains same value'withpytest.raises(ValueError)asexec_info:sort_with_custom_orders([1,2,3],prefix_orders=[1,1])assertexec_info.value.args[0]=='prefix_orders contains duplicated values'# tests for classclassPerson:def__init__(self,id,name,age):self.id=idself.name=nameself.age=agedef__lt__(self,other:'Person'):returnself.age<other.agedef__eq__(self,other:'Person'):returnself.age==other.agedef__hash__(self):returnself.iddef__str__(self):returnf'Person({self.id},{self.name},{self.age})'def__repr__(self):returnstr(self)Albert=Person(1,'Albert',28)Alice=Person(2,'Alice',26)Menglong=Person(3,'Menglong',33)persons=[Albert,Alice,Menglong]expected=[Alice,Albert,Menglong]assertsort_with_custom_orders(persons)==expectedexpected=[Menglong,Alice,Albert]assertsort_with_custom_orders(persons,prefix_orders=[Menglong,Person(4,'Anyone',40)])==expectedunpack_listimportpytestfrompythonic_toolbox.utils.list_utilsimportunpack_listfirst,second,third=unpack_list(['a','b','c','d'],target_num=3)assertfirst=='a'andsecond=='b'andthird=='c'first,second,third=unpack_list(['a','b'],target_num=3,default=None)assertfirst=='a'andsecond=='b'andthirdisNonefirst,second,third=unpack_list(range(1,3),target_num=3,default=None)assertfirst==1andsecond==2andthirdisNonefirst,second,third=unpack_list([],target_num=3,default=0)assertfirst==second==third==0first,second,*rest=unpack_list(['a','b','c'],target_num=4,default='x')assertfirst=='a'andsecond=='b'andrest==['c','x']# test case for type rangefirst,second,third=unpack_list(range(1,3),target_num=3,default=None)assertfirst==1andsecond==2andthirdisNonedeffib():a,b=0,1while1:yieldaa,b=b,a+b# test case for type generatorfib_generator=fib()# generates data like [0, 1, 1, 2, 3, 5, 8, 13, 21 ...]first,second,third,*rest=unpack_list(fib_generator,target_num=6)assertfirst==0andsecond==1andthird==1assertrest==[2,3,5]seventh,eighth=unpack_list(fib_generator,target_num=2)assertseventh==8andeighth==13# test edge case, nothing to unpackempty=unpack_list([],target_num=0,default=None)assertempty==[]res=unpack_list([],target_num=2,default=None)assertres==[None,None]empty=unpack_list(['a','b'],target_num=0,default=None)assertempty==[]empty=unpack_list(range(0,0),target_num=0)assertempty==[]empty=unpack_list(iter([]),target_num=0,default=None)assertempty==[]withpytest.raises(ValueError):# ValueError: not enough values to unpack (expected 3, got 2)first,second,third=unpack_list([1,2],target_num=2)untilfromitertoolsimportcountfrompythonic_toolbox.utils.list_utilsimportuntil# basic usagecounter=count(1,2)# generator of odd numbers: 1, 3, 5, 7 ...assertuntil(counter,lambdax:x>10)==11assertuntil([1,2,3],lambdax:x>10,default=11)==11# test case for when there's no default value and no item in the iterable satisfies the conditionassertuntil([1,2,3],lambdax:x>10)isNone# edge casesassertuntil([],default=3)==3# nothing provided, return defaultassertuntil(None,lambdax:x>10,default=11)==11# test case for when there's no item in the counter satisfies the condition# the following codes will run forever, so comment them out# counter = count(1, 2)  # generator of odd numbers: 1, 3, 5, 7 ...# assert until(counter, lambda x: x % 2 == 0) is None# test case for when max_iter_num is provided, only iterate the counter for max_iter_num timescounter=count(1,2)# generator of odd numbers: 1, 3, 5, 7 ...assertuntil(counter,lambdax:x%2==0,default=None,max_iter_num=100)isNonenumbers=[1,2,3,4,5,6]assertuntil(numbers,lambdax:x>=5,default=None,max_iter_num=1)isNoneassertuntil(numbers,lambdax:x>=5,default=None,max_iter_num=4)isNoneassertuntil(numbers,lambdax:x>=5,default=None,max_iter_num=5)==5assertuntil(numbers,lambdax:x>=5,default=None,max_iter_num=100)==5string_utilssubstitute_string_template_dictfromunittest.mockimportpatch,PropertyMockimportpytestfrompythonic_toolbox.utils.string_utilsimportsubstitute_string_template_dict,CycleError# simple usage# both $variable ${variable} declarations are supported in string template formatstr_template_dict={'greeting':'Good Morning, Everyone!','first_name':'Albert','last_name':'Lee','full_name':'$first_name $last_name','age':34,'speech':'$greeting, I am $full_name, a ${age}-year-old programmer, very glad to meet you!'}output_dict=substitute_string_template_dict(str_template_dict)assertoutput_dict['full_name']=='Albert Lee'expected_speech='Good Morning, Everyone!, I am Albert Lee, a 34-year-old programmer, very glad to meet you!'assertoutput_dict['speech']==expected_speech# complex usage, with dynamic values, and multi value-providing holdersstr_template_dict={'first_name':'Daenerys','last_name':'Targaryen','nick_name':'Dany','full_name':'$first_name $last_name','speech':"$nick_name: I'm $full_name ($title1, $title2, $title3), it's $current_time_str, $greeting!",}variables_dict={'title1':'Queen of Meereen','title2':'Mother of Dragons'}classDynamicVariables:@propertydefcurrent_time_str(self):importdatetimereturndatetime.datetime.now().strftime("%H:%M:%S")classDefaultUnknownTitle:"""A class will always return UnknownTitle, when try to access attribute liketitle1, title2, ..., titleX"""def__getattribute__(self,item):ifisinstance(item,str)anditem.startswith('title')anditem[len(item)-1:].isdigit():return'UnknownTitle'returnsuper(DefaultUnknownTitle,self).__getattribute__(item)expected_speech=("Dany: I'm Daenerys Targaryen (Queen of Meereen, Mother of Dragons, UnknownTitle), ""it's 08:00:00, good morning everyone!")# using mock to make DynamicVariables().current_time_str always return 08:00:00withpatch.object(DynamicVariables,'current_time_str',return_value='08:00:00',new_callable=PropertyMock):output_dict=substitute_string_template_dict(str_template_dict,variables_dict,DynamicVariables(),DefaultUnknownTitle(),greeting='good morning everyone')assertoutput_dict['speech']==expected_speech# edge casesassertsubstitute_string_template_dict({})=={}# cycle detectionstr_template_dict={'variable_a':'Hello $variable_b',# variable_a depends on variable_b'variable_b':'Hello $variable_a',# variable_b depends on variable_a, it's a cycle!}withpytest.raises(CycleError)asexec_info:substitute_string_template_dict(str_template_dict)