Package: bender-client
Bender Client for Python:warning: The fullDOCUMENTATIONon bender-python-client can be foundHERE.SetupCreate an account for free atbender.dreem.comInstall bender in your Python environment withpip install bender-clientUsage ExampleLet's use the famous MNIST example where we try to recognize handwritten digits in images.The code of the algorithm usingPyTorchis the following :To use this example, do not forget topip install numpy torch torchvision.from__future__importprint_functionimporttorchimporttorch.nnasnnimporttorch.nn.functionalasFimporttorch.optimasoptimfromtorchvisionimportdatasets,transformsclassNet(nn.Module):def__init__(self,dropout=True,activation="relu",kernel_size=5,conv_depth=10,linear_depth=50):super(Net,self).__init__()self.conv1=nn.Conv2d(1,conv_depth,kernel_size=kernel_size)self.conv2=nn.Conv2d(conv_depth,20,kernel_size=kernel_size)self.conv2_drop=nn.Dropout2d()ifdropoutisTrueelselambdax:xself.fc1=nn.Linear(320,linear_depth)self.fc2=nn.Linear(linear_depth,10)self.activation=getattr(F,activation)defforward(self,x):x=self.activation(F.max_pool2d(self.conv1(x),2))x=self.activation(F.max_pool2d(self.conv2_drop(self.conv2(x)),2))x=x.view(-1,320)x=self.activation(self.fc1(x))x=F.dropout(x,training=self.training)x=self.fc2(x)returnF.log_softmax(x,dim=1)deftrain(model,device,train_loader,optimizer,epoch):model.train()forbatch_idx,(data,target)inenumerate(train_loader):data,target=data.to(device),target.to(device)optimizer.zero_grad()output=model(data)loss=F.nll_loss(output,target)loss.backward()optimizer.step()ifbatch_idx%10==0:print('Train Epoch:{}[{}/{}({:.0f}%)]\tLoss:{:.6f}'.format(epoch,batch_idx*len(data),len(train_loader.dataset),100.*batch_idx/len(train_loader),loss.item()))deftest(model,device,test_loader):model.eval()test_loss=0correct=0withtorch.no_grad():fordata,targetintest_loader:data,target=data.to(device),target.to(device)output=model(data)test_loss+=F.nll_loss(output,target,reduction='sum').item()pred=output.max(1,keepdim=True)[1]correct+=pred.eq(target.view_as(pred)).sum().item()test_loss/=len(test_loader.dataset)print('\nTest set: Average loss:{:.4f}, Accuracy:{}/{}({:.0f}%)\n'.format(test_loss,correct,len(test_loader.dataset),100.*correct/len(test_loader.dataset)))return(correct/len(test_loader.dataset))defrun(epochs=3,lr=0.01,momentum=0.5,dropout=True,activation="relu",kernel_size=5,conv_depth=10,linear_depth=50):torch.manual_seed(1)device=torch.device("cpu")train_loader=torch.utils.data.DataLoader(datasets.MNIST('../data',train=True,download=True,transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,),(0.3081,))])),batch_size=32,shuffle=True,)test_loader=torch.utils.data.DataLoader(datasets.MNIST('../data',train=False,transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,),(0.3081,))])),batch_size=1000,shuffle=True,)model=Net(dropout,activation).to(device)optimizer=optim.SGD(model.parameters(),lr=lr,momentum=momentum)accuracy=0forepochinrange(1,int(epochs)+1):train(model,device,train_loader,optimizer,epoch)accuracy=test(model,device,test_loader)returnaccuracyif__name__=='__main__':# HYPERPARAMETERS (That's what bender is interested in)# Here we select them on our own in an arbitrary wayhyperparameters={"kernel_size":5,"epochs":3,"lr":0.05,"momentum":0.2,"dropout":True,"activation":"relu","conv_depth":10,"linear_depth":50,}run(epochs=hyperparameters["epochs"],lr=hyperparameters["lr"],momentum=hyperparameters["momentum"],dropout=hyperparameters["dropout"],activation=hyperparameters["activation"],kernel_size=hyperparameters["kernel_size"],conv_depth=hyperparameters["conv_depth"],linear_depth=hyperparameters["linear_depth"],)Now let's plug Bender into It !Importing BenderfrombenderclientimportBenderbender=Bender()This will ask for your email and password. The client will use these to login and retrieve a TOKEN.
This TOKEN is personal, it should not be shared, it will be stored in your home folder as ".bender_token", and you will not be asked for your login/password again until it expires.
:warning: Again, your TOKEN is personal. You should not give it or add it to any public repository :warning:Create an ExperimentAn experiment is related to the problem you are trying to solve, here : MNIST classificationbender.new_experiment(name='MNIST Classification',description='Simple image classification on handwritten digits',metrics=[{"metric_name":"algorithm_accuracy",# It's just a name and there can be multiple watched metrics."type":"reward",# The type can either be "reward" or "loss" depending on if you want to maximize or minimize it.}],dataset='MNIST')Create an AlgoAn algo is simply corresponding to ONE solution to an Experiment problem : here it's as we saw a Neural Net with PyTorchbender.new_algo(name='PyTorch_NN',# The parameters below are actually the Hyper-Parameters of your algo described in a listparameters=[{"name":'kernel_size',"category":"categorical","search_space":{"values":[3,5,7],},},{"name":'conv_depth',"category":"uniform","search_space":{"low":1,"high":100,"step":1,},},{"name":'linear_depth',"category":"uniform","search_space":{"low":1,"high":100,"step":1,},},{"name":'epochs',"category":"uniform","search_space":{"low":1,"high":4,"step":1,},},{"name":'lr',"category":"loguniform","search_space":{"low":1e-5,"high":1e-1,"step":1e-6,},},{"name":'momentum',"category":"uniform","search_space":{"low":0,"high":1,"step":0.05,},},{"name":'dropout',"category":"categorical","search_space":{"values":[True,False],},},{"name":'activation',"category":"categorical","search_space":{"values":["relu","softmax","sigmoid","tanh"],},},])Get an Hyperparameters Set suggestion from BenderThe whole goal of what we did up there is to use Bender to get a new set of Hyperparameters to try according to the settings of your Experiment and Algo.suggestion=bender.suggest()# suggestion would for example contain something like :{"kernel_size":5,"epochs":3,"lr":0.05,"momentum":0.2,"dropout":True,"activation":"tanh","conv_depth":10,"linear_depth":50,}Feed a Trial to BenderA Trial is simply an attempt of you trying a Hyperparameters Set with your algorithm associated with the result metrics obtained. If you want bender to improve over time, feed him every trial you make.bender.new_trial(parameters={"kernel_size":5,"epochs":3,"lr":0.05,"momentum":0.2,"dropout":True,"activation":"tanh","conv_depth":10,"linear_depth":50,},results={"algorithm_accuracy":0.7,# We put an arbitrary value here just for the example.})The full code put togetherPsssssst... The magic starts at line 443... ;)To use this example, do not forget topip install numpy torch torchvision bender-client.from__future__importprint_functionimportargparseimporttorchimporttorch.nnasnnimporttorch.nn.functionalasFimporttorch.optimasoptimfromtorchvisionimportdatasets,transformsfrombenderclientimportBenderclassNet(nn.Module):def__init__(self,dropout=True,activation="relu",kernel_size=5,conv_depth=10,linear_depth=50):super(Net,self).__init__()self.conv1=nn.Conv2d(1,conv_depth,kernel_size=kernel_size)self.conv2=nn.Conv2d(conv_depth,20,kernel_size=kernel_size)self.conv2_drop=nn.Dropout2d()ifdropoutisTrueelselambdax:xself.fc1=nn.Linear(320,linear_depth)self.fc2=nn.Linear(linear_depth,10)self.activation=getattr(F,activation)defforward(self,x):x=self.activation(F.max_pool2d(self.conv1(x),2))x=self.activation(F.max_pool2d(self.conv2_drop(self.conv2(x)),2))x=x.view(-1,320)x=self.activation(self.fc1(x))x=F.dropout(x,training=self.training)x=self.fc2(x)returnF.log_softmax(x,dim=1)deftrain(model,device,train_loader,optimizer,epoch):model.train()forbatch_idx,(data,target)inenumerate(train_loader):data,target=data.to(device),target.to(device)optimizer.zero_grad()output=model(data)loss=F.nll_loss(output,target)loss.backward()optimizer.step()ifbatch_idx%10==0:print('Train Epoch:{}[{}/{}({:.0f}%)]\tLoss:{:.6f}'.format(epoch,batch_idx*len(data),len(train_loader.dataset),100.*batch_idx/len(train_loader),loss.item()))deftest(model,device,test_loader):model.eval()test_loss=0correct=0withtorch.no_grad():fordata,targetintest_loader:data,target=data.to(device),target.to(device)output=model(data)test_loss+=F.nll_loss(output,target,reduction='sum').item()pred=output.max(1,keepdim=True)[1]correct+=pred.eq(target.view_as(pred)).sum().item()test_loss/=len(test_loader.dataset)print('\nTest set: Average loss:{:.4f}, Accuracy:{}/{}({:.0f}%)\n'.format(test_loss,correct,len(test_loader.dataset),100.*correct/len(test_loader.dataset)))return(correct/len(test_loader.dataset))defrun(epochs=3,lr=0.01,momentum=0.5,dropout=True,activation="relu",kernel_size=5,conv_depth=10,linear_depth=50):torch.manual_seed(1)device=torch.device("cpu")train_loader=torch.utils.data.DataLoader(datasets.MNIST('../data',train=True,download=True,transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,),(0.3081,))])),batch_size=32,shuffle=True,)test_loader=torch.utils.data.DataLoader(datasets.MNIST('../data',train=False,transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,),(0.3081,))])),batch_size=1000,shuffle=True,)model=Net(dropout,activation).to(device)optimizer=optim.SGD(model.parameters(),lr=lr,momentum=momentum)accuracy=0forepochinrange(1,int(epochs)+1):train(model,device,train_loader,optimizer,epoch)accuracy=test(model,device,test_loader)returnaccuracydefinit_bender():bender=Bender()bender.create_experiment(name='MNIST Classification',description='Simple image classification on handwritten digits',metrics=[{"metric_name":"algorithm_accuracy","type":"reward"}],dataset='MNIST')bender.create_algo(name='PyTorch_NN',hyperparameters=[{"name":'kernel_size',"category":"categorical","search_space":{"values":[3,5,7],},},{"name":'conv_depth',"category":"uniform","search_space":{"low":1,"high":100,"step":1,},},{"name":'linear_depth',"category":"uniform","search_space":{"low":1,"high":100,"step":1,},},{"name":'epochs',"category":"uniform","search_space":{"low":1,"high":4,"step":1,},},{"name":'lr',"category":"loguniform","search_space":{"low":1e-5,"high":1e-1,"step":1e-6,},},{"name":'momentum',"category":"uniform","search_space":{"low":0,"high":1,"step":0.05,},},{"name":'dropout',"category":"categorical","search_space":{"values":[True,False],},},{"name":'activation',"category":"categorical","search_space":{"values":["relu","softmax","sigmoid","tanh"],},},])returnbenderif__name__=='__main__':# Create experiment and algo if they don't exist yet. Else, load them from the config file ./.benderconfbender=init_bender()whileTrue:# Get a set of Hyperparameters to testsuggestion=bender.suggest(metric="algorithm_accuracy")# Get algo result with themresult=run(epochs=suggestion["epochs"],lr=suggestion["lr"],momentum=suggestion["momentum"],dropout=suggestion["dropout"],activation=suggestion["activation"],kernel_size=suggestion["kernel_size"],conv_depth=suggestion["conv_depth"],linear_depth=suggestion["linear_depth"],)# Feed Bender a Trial, AKA => suggestion + resultbender.create_trial(hyperparameters=suggestion,results={"algorithm_accuracy":result})print('New trial sent -----------------------------------------------------\n\n')