Package: sdk-42946c74-f393-4856-9a84-41c4a80cba69
pyraisdkDynamic BatchingDescriptionWhen we deploy a model in AML with GPU instance to provide inference service, if it occupies GPU for inferencing in each request separately, it will be quite inefficient. This is a shared module helping to collect data items from different requests, and inferencing batchly in a backend thread. This will considerably improve usage efficiency of GPU.Usage ExamplesBuildYourModelclass inherited frompyraisdk.dynbatch.BaseModel.fromtypingimportListfrompyraisdk.dynbatchimportBaseModelclassSimpleModel(BaseModel):defpredict(self,items:List[str])->List[int]:rs=[]foriteminitems:rs.append(len(item))returnrsdefpreprocess(self,items:List[str])->List[str]:rs=[]foriteminitems:rs.append(f'[{item}]')returnrsInitialize apyraisdk.dynbatch.DynamicBatchModelwithYourModelinstance, and callpredict / predict_onefor inferencing.frompyraisdk.dynbatchimportDynamicBatchModel# prepare modelsimple_model=SimpleModel()batch_model=DynamicBatchModel(simple_model)# predictitems=['abc','123456','xyzcccffaffaaa']predictions=batch_model.predict(items)assertpredictions==[5,8,16]# predict_oneitem='abc'prediction=batch_model.predict_one(item)assertprediction==5Concurrent requests topredict / predict_one, in different threads.fromthreadingimportThreadfrompyraisdk.dynbatchimportDynamicBatchModel# prepare modelsimple_model=SimpleModel()batch_model=DynamicBatchModel(simple_model)# thread run functiondefrun(name,num):forstepinrange(num):item=f'{name}-{step}'prediction=batch_model.predict_one(item)assertprediction==len(item)+2# start concurrent inferencethreads=[Thread(target=run,args=(f'{tid}',100))fortidinrange(20)]fortinthreads:t.start()fortinthreads:t.join()Loging & EventsDescriptionThis module is for logging and event tracing.interfacedefinitialize(eh_hostname:Optional[str]=None,client_id:Optional[str]=None,eh_conn_str:Optional[str]=None,eh_structured:Optional[str]=None,eh_unstructured:Optional[str]=None,role:Optional[str]=None,instance:Optional[str]=None,)Parameter description forinitialize:eh_hostname: Fully Qualified Namespace aka EH Endpoint URL (*.servicebus.windows.net). Default, read $EVENTHUB_NAMESPACEclient_id: client_id of service principal. Default, read $UAI_CLIENT_IDeh_conn_str: connection string of eventhub namespace. Default, read $EVENTHUB_CONN_STRINGeh_structured: structured eventhub name. Default, read $EVENTHUB_AUX_STRUCTUREDeh_unstructured: unstructured eventhub name. Default, read $EVENTHUB_AUX_UNSTRUCTUREDrole: role, Default: RemoteModel_${ENDPOINT_NAME}instance: instance, Default: "${ENDPOINT_VERSION}|{os.uname()[1]}" or "${ENDPOINT_VERSION}|{_probably_unique_id()}"defevent(self,key:str,code:str,numeric:float,detail:str='',corr_id:str='',elem:int=-1)definfof(self,format:str,*args:Any)definfocf(self,corr_id:str,elem:int,format:str,*args:Any)defwarnf(self,format:str,*args:Any)defwarncf(self,corr_id:str,elem:int,format:str,*args:Any)deferrorf(self,format:str,*args:Any)deferrorcf(self,corr_id:str,elem:int,ex:Optional[Exception],format:str,*args:Any)deffatalf(self,format:str,*args:Any)deffatalcf(self,corr_id:str,elem:int,ex:Optional[Exception],format:str,*args:Any)examples# export EVENTHUB_AUX_UNSTRUCTURED='ehunstruct'# export EVENTHUB_AUX_STRUCTURED='ehstruct'# export UAI_CLIENT_ID='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'# export EVENTHUB_NAMESPACE='xxx.servicebus.windows.net'frompyraisdkimportrlogrlog.initialize()rlog.infof('this is a info message%s',123)rlog.event('LifetimeEvent','STOP_GRACEFUL_SIGNAL',0,'detail info')# export EVENTHUB_AUX_UNSTRUCTURED='ehunstruct'# export EVENTHUB_AUX_STRUCTURED='ehstruct'# export EVENTHUB_CONN_STRING='<connection string>'frompyraisdkimportrlogrlog.initialize()rlog.infocf('corrid',-1,'this is a info message:%s',123)rlog.event('RequestDuration','200',0.01,'this is duration in seconds')frompyraisdkimportrlogrlog.initialize(eh_structured='ehstruct',eh_unstructured='ehunstruct',eh_conn_str='<eventhub-conn-str>')rlog.errorcf('corrid',-1,Exception('error msg'),'error message:%s%s',1,2)rlog.event('CpuUsage','',0.314,detail='cpu usage',corr_id='corrid',elem=-1)