Package: fabrique-atelier
Library for fabrique ml pipelines development.You can copy pipeline project examples from hereimportfabrique_atelierprint(fabrique_atelier.__path__[0]+'/examples')Model decomposition stepsYou can find this decomposed example in /examples/fake_lightgbm1. We have samples and model inference code# load samplessamp_dir='./samples'filenames=[fforfinos.listdir(samp_dir)ifos.path.isfile(f'{samp_dir}/{f}')]samples=[]forfilenameinfilenames:withopen(f'{samp_dir}/{filename}')asfp:samples.append(fp.read())sample is json string{"ts":1622637967.6218433,"uid":147757848,"number":"00000125","type":"out","traffic":{"about":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"services":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"contacts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"roaming":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"tariffs":[14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"simcards":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"balance":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"internet":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"messaging":[0,0,0,1,1,1,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0],"support":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}}# load modelimportlightgbmaslgbbst=lgb.Booster(model_file='../model.txt')# init modelclasses=["about","services","contacts","roaming","tariffs","simcards","balance","internet","messaging","support"]# simple inference codeimportnumpyasnpreference_results=[]# we will use this results for testsforsampleinsamples:## 1. extract features from json message# 1.1. parse message and get featuresmes=json.loads(sample)features=np.array([mes['traffic'][cls]forclsinclasses]).flatten()## 2. make prediction result# 2.1 get predictionpred_vals=bst.predict([features,])[0]# 2.2 normalize scores, get classscores=pred_vals/pred_vals.sum()reason=classes[scores.argmax()]# 2.3 make and serialize messagescores_dict={cls:round(scores[i],2)fori,clsinenumerate(classes)}res=dict(ts=mes['ts'],uid=mes['uid'],number=mes['number'],classes=scores_dict,reason=reason)reference_results.append(json.dumps(res,indent=2))2. Split inference into feature extraction and lightgbm prediction# splitted inference codesplitted_results=[]feature_msgs=[]forsampleinsamples:## 1. extract features from json message# 1.1. parse message and get featuresmes=json.loads(sample)features=np.array([mes['traffic'][cls]forclsinclasses]).flatten()# 1.2. create intermediate messagedata=dict(ts=mes['ts'],uid=mes['uid'],number=mes['number'],features=features.tolist())feature_msgs.append(data)## 2. make prediction resultfeatures_batch=np.array([data['features']fordatainfeature_msgs])# 2.1 get predictionpred_batch=bst.predict(features_batch)# microbatch predictionfori,mesinenumerate(feature_msgs):scores=pred_batch[i]# 2.2 normalize scores, get classtry:scores=scores/scores.sum()# try to normalizeexcept:passreason_num=scores.argmax()reason=classes[reason_num]# 2.3 make and serialize messagescores_dict={cls:round(scores[i],2)fori,clsinenumerate(classes)}res=dict(ts=mes['ts'],uid=mes['uid'],number=mes['number'],classes=scores_dict,reason=reason)splitted_results.append(json.dumps(res,indent=2))check if results are equalassertset(splitted_results)==set(reference_results)3. Make pipeline.pyfromfabrique_atelier.actorsimportPipeline,ProcessorimportjsonimportnumpyasnpimportlightgbmaslgbclassExtractFeatures(Processor):def__init__(self):self.classes=["about","services","contacts","roaming","tariffs","simcards","balance","internet","messaging","support"]defget_result(self,body):## 1. extract features from json message# 1.1. parse message and get featuresmes=json.loads(body['data'])features=np.array([mes['traffic'][cls]forclsinself.classes]).flatten()# 1.2. create intermediate messagedata=dict(ts=mes['ts'],uid=mes['uid'],number=mes['number'],features=features.tolist())return{'data':data}classScoringModel(Processor):def__init__(self):self.classes=["about","services","contacts","roaming","tariffs","simcards","balance","internet","messaging","support"]self.bst=lgb.Booster(model_file='./model.txt')# init modeldefget_batch_result(self,batch):## 2. make prediction resultfeatures_batch=np.array([body['data']['features']forbodyinbatch])# 2.1 get predictionpred_batch=self.bst.predict(features_batch)# microbatch predictionout_batch=[]fori,bodyinenumerate(batch):in_data=body['data']scores=pred_batch[i]# 2.2 normalize scores, get classtry:scores=scores/scores.sum()# try to normalizeexcept:passreason_num=scores.argmax()reason=self.classes[reason_num]# 2.3 make and serialize messagescores_dict={cls:round(scores[i],2)fori,clsinenumerate(self.classes)}out_data=dict(ts=in_data['ts'],uid=in_data['uid'],number=in_data['number'],classes=scores_dict,reason=reason)out_body=dict(data=json.dumps(out_data).encode(),metrics={"reason_num":int(reason_num)})out_batch.append(out_body)returnout_batch# topologypipeline=Pipeline(['extractor','model'])ids=pipeline.idsnodes=pipeline.nodesnodes.extractor=ExtractFeatures.to(ids.model)nodes.model=ScoringModel.to(ids.end)4. Run emulation and check resultsrun ./dev/start_pipeline.py with this codefrompipelineimportpipeline# load samples## code from first decomposition step above ### start emulationpipeline.start_emulation(samples)# simple inference code## code from first decomposition step above ### load results of emulationresult_dir='./out/data'filenames=[fforfinos.listdir(result_dir)ifos.path.isfile(f'{result_dir}/{f}')]results=[]forfilenameinfilenames:withopen(f'{result_dir}/{filename}')asfp:results.append(fp.read())#check if results are equalassertset(results)==set(reference_results)