Package: kpick
AI Picking packageSystem Requirements:-Ubuntu16.04or18.04
-CUDA>=10.0,CUDNN>=7-Pytorch>=1.4.0Install NVIDIA drivercheckGPUinfo:sudolshw-Cdisplayorhwinfo--gfxcard--short
Install:
sudoadd-apt-repositoryppa:graphics-drivers/ppa
sudoaptupdate
reboot
Open'Software and update/ Addtional Drivers'andselectproperdriver
rebootInstall CUDA and CUDNN-Download*.runfilefromhttps://developer.nvidia.com/cuda-toolkit-archive
sudoshcuda_XXX.run
-Followthecommandlinepromts:
***Note:Answer'NO'forquestion"Install NVIDIA Accelerated Graphics Driver for Linux-XXX?"-DownloadCUDNNfromhttps://developer.nvidia.com/rdp/cudnn-archive
-Extracttarfile
sudocp/cuda/include/*/usr/loca/cuda-XX/include
sudocp/cuda/lib64/*/usr/local/cuda-XX/lib64
-SetupCUDApath
sudogedit~/.bashrc
Add2linestothefile:PATH=/usr/local/cuda/bin:$PATHLD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATHsource~/.bashrc

sudogedit/etc/ld.so.conf.d/cuda.conf
Add:/usr/local/cuda/lib64
sudoldconfig
rebootInstallsudoaptinstallpython3.6-dev
sudoaptinstallpython3.6-tkcd$ROOTpipinstall-e.Download pretrained network and cofigurations filehttps://drive.google.com/drive/folders/1hdeJUe0EzIO6be5i-m8ygYpbnDGpHrKy?usp=sharing** Place *.cfg files into $ROOT/configs, and trained *.pth to $ROOT/data/model/{grip or suction}_evaluatorDemosSuction Detection with RGBD from realsensefromttcv.sensor.realsense_sensorimportRSSensor
importcv2# get data from realsenesensor=RSSensor()sensor.start()foriinrange(5):rgb,depth=sensor.get_data()# define detectordetector=SuctionDetector(cfg_path='configs/suction_net.cfg')# set crop areabbox=(630,330,1000,670)# left, top, right, bottom# find suction  poseSuctions=detector.find_suction_pose_rgb_depth(rgb=rgb,depth=depth,bbox=bbox)out=detector.show_suctions_rgb_depth(rgb,depth,Suctions,bbox=bbox)# showcv2.imshow('rgb',rgb[:,:,::-1])cv2.imshow('depth',depth)cv2.imshow('out',out[:,:,::-1])# release realsensecv2.waitKey()sensor.stop()Suction Detection with GUIfromttcv.basic.basic_objectsimportDetGui
fromkpick.processing.suction_detection_v2importSuctionDetector
classSuctionGuiDetector(SuctionDetector,DetGui):defgui_process_single(self,rgbd,method_ind=0,filename='unnamed',disp_mode='rgb'):ifmethod_ind==0:ret=self.show_suctions(rgbd=rgbd,Suctions=self.find_suction_pose_multiscale(rgbd=rgbd),disp_mode=disp_mode)returnret

fromttcv.sensor.realsense_sensorimportget_realsense_modules
fromttcv.basic.basic_guiimportBasGUI,GuiModulecfg_path='configs/suction_net.cfg'suction_detect_module=GuiModule(SuctionGuiDetector,type='suction_detector',name='Suction Detector',category='detector',cfg_path=cfg_path,num_mod_method=1)BasGUI(title='Stefan Pose Maker',modules=[suction_detect_module]+get_realsense_modules(),)Suction Detection With TCP/IP connectServerfromttcv.basic.basic_tcp_connectimportServerThread
fromkpick.processing.suction_detection_v2importSuctionDetector


classSuctionSeverDetector(SuctionDetector,ServerThread):def__init__(self,cfg_path,host='localhost',port=8888):SuctionDetector.__init__(self,cfg_path=cfg_path)print('{} Detector initialized '.format('+'*10))ServerThread.__init__(self,host=host,port=port)print('{} Server initialized '.format('+'*10))defprocess_received_data(self):Suctions=self.find_suction_pose_rgb_depth(rgb=self.data['rgb'],depth=self.data['depth'],bbox=self.data['bbox'],ws_pts=self.data['ws_pts'])det={'scores':Suctions.scores.tolist(),'locs':Suctions.get_suction_locs(),'disp_color':Suctions.disp_colors,'norms':Suctions.norm_vectors(),'best_ind':Suctions.best_ind}out=self.show_suctions_rgb_depth(rgb=self.data['rgb'],depth=self.data['depth'],Suctions=Suctions,bbox=self.data['bbox'],ws_pts=self.data['ws_pts'],disp_mode=self.data['disp_mode'])return{'im':out,'det':det}if__name__=='__main__':SuctionSeverDetector(cfg_path='configs/suction_net.cfg').listen()Client GUIfromttcv.basic.basic_objectsimportDetGuiObj
fromttcv.basic.basic_tcp_connectimportClientThread
fromttcv.utils.proc_utilsimportCFG
fromttcv.basic.basic_guiimportBasGUI,GuiModule
fromttcv.sensor.realsense_sensorimportget_realsense_modules


defdemo_client_gui(host='localhost',port=8888):classClientGui(DetGuiObj,ClientThread):def__init__(self,args=None,cfg_path=None):DetGuiObj.__init__(self,args=args,cfg_path=cfg_path)ClientThread.__init__(self,host=self.args.host,port=self.args.port)defgui_process_single(self,rgbd,method_ind=0,filename='unnamed',disp_mode='rgb'):rets=self.send_and_get_return({'rgb':rgbd.rgb,'depth':rgbd.depth,'bbox':rgbd.workspace.bbox,'ws_pts':rgbd.workspace.pts,'disp_mode':disp_mode})print('Return received ...')ifretsisNone:print('None return ...')returnrgbd.disp(mode=disp_mode)#++++++++++++++++++++++++++++ Robot actiont heredet=rets['det']returnrets['im']args=CFG()args.host,args.port=host,portclient_module=GuiModule(ClientGui,type='client_gui',name='Client GUI',category='detector',run_thread=True,args=args)BasGUI(title='Client Gui',modules=[client_module,]+get_realsense_modules())if__name__=='__main__':demo_client_gui()Client with realsensefromttcv.sensor.realsense_sensorimportRSSensor
importcv2
fromttcv.basic.basic_tcp_connectimportClientThread# get data from realsenesensor=RSSensor()sensor.start()# for i in range(5): rgb, depth = sensor.get_data()# define detector# detector = SuctionDetector(cfg_path='configs/suction_net.cfg')# set crop areabbox=None# (630, 330, 1000, 670)    # left, top, right, bottomws_pts=[(177,48),(1082,40),(1104,650),(855,655),(760,512),(692,517),(762,663),(136,646)]# init clientclient=ClientThread()whileTrue:rgb,depth=sensor.get_data()cv2.imshow('im',rgb[:,:,::-1])# find suction  pose# Suctions = detector.find_suction_pose_rgb_depth(rgb=rgb, depth=depth, bbox=bbox)# out = detector.show_suctions_rgb_depth(rgb, depth, Suctions,  bbox=bbox)sent_dict={'rgb':rgb,'depth':depth,'bbox':None,'ws_pts':ws_pts,'disp_mode':'rgb'}rets=client.send_and_get_return(sent_dict)out,det=rets['im'],rets['det']cv2.imshow('out',out[:,:,::-1])ifcv2.waitKey(10)==27:break# release realsensesensor.stop()Grasp Detection with Realsensefromttcv.sensor.realsense_sensorimportRSSensor
fromkpick.processing.grip_detection_v6importGripDetector
importcv2# get data from realsenesensor=RSSensor()sensor.start()foriinrange(5):rgb,depth=sensor.get_data()# define detectordetector=GripDetector(cfg_path='configs/grip_net.cfg')# set crop areabbox=(630,330,1000,670)# left, top, right, bottom# find suction  poseGrips=detector.find_grip_pose_rgb_depth(rgb=rgb,depth=depth,bbox=bbox)out=detector.show_grips_rgb_depth(rgb,depth,Grips,bbox=bbox)# showcv2.imshow('rgb',rgb[:,:,::-1])cv2.imshow('depth',depth)cv2.imshow('out',out[:,:,::-1])# release realsensecv2.waitKey()sensor.stop()Grasp Detection with GUIclassGripGuiDetector(GripDetector,DetGui):defgui_process_single(self,rgbd,method_ind=0,filename='unnamed',disp_mode='rgb'):ifmethod_ind==0:ret=self.show_grips(rgbd=rgbd,Grips=self.find_grip_pose_from_edges_v3(rgbd=rgbd),disp_mode=disp_mode)returnret

fromttcv.sensor.realsense_sensorimportget_realsense_modules
fromttcv.basic.basic_guiimportBasGUI,GuiModulecfg_path='configs/grip_net.cfg'detect_module=GuiModule(GripGuiDetector,type='grip_detector',name='Grip Detector',category='detector',cfg_path=cfg_path,num_mod_method=1)BasGUI(title='Grip Detection GUI',modules=[detect_module]+get_realsense_modules(),)Grasp Detection Serverfromttcv.basic.basic_tcp_connectimportServerThread
fromkpick.processing.grip_detection_v6importGripDetector


classGripSeverDetector(GripDetector,ServerThread):def__init__(self,cfg_path,host='localhost',port=8888):GripDetector.__init__(self,cfg_path=cfg_path)print('{} Detector initialized '.format('+'*10))ServerThread.__init__(self,host=host,port=port)print('{} Server initialized '.format('+'*10))defprocess_received_data(self):Grips=self.find_grip_pose_rgb_depth(rgb=self.data['rgb'],depth=self.data['depth'],bbox=self.data['bbox'],ws_pts=self.data['ws_pts'])det={'scores':Grips.scores,'locs':Grips.get_grip_centers(),'disp_color':Grips.disp_colors,'best_ind':Grips.best_ind}out=self.show_grips_rgb_depth(rgb=self.data['rgb'],depth=self.data['depth'],Grips=Grips,bbox=self.data['bbox'],ws_pts=self.data['ws_pts'],disp_mode=self.data['disp_mode'])return{'im':out,'det':det}if__name__=='__main__':GripSeverDetector(cfg_path='configs/grip_net.cfg').listen()