Package: rlearn
Reinforcement learning AlgorithmsOn-PolicyA2C: Actor-CriticPPO: Proximal Policy OptimizationOff-PolicyDQN: Deep Q NetworksDuelingDQN: Dueling DQNDDPG: Deep Deterministic Policy GradientsTD3: Twin Delayed DDPGSAC: Soft Actor CriticUsageClassical wayimportgymnasiumfromtensorflowimportkerasimportrlearn# define an environmentenv=gymnasium.make('CartPole-v1',render_mode="human")# set reinforcement learning trainertrainer=rlearn.DQNTrainer()trainer.set_replay_buffer(max_size=1000)trainer.set_model_encoder(q=keras.Sequential([keras.layers.InputLayer(4),# state has dimension of 4keras.layers.Dense(32),keras.layers.ReLU(),]),action_num=env.action_space.n)# training loopfor_inrange(100):s,_=env.reset()for_inrange(200):a=trainer.predict(s)s_,r,done,_,_=env.step(a)trainer.store_transition(s,a,r,s_,done)trainer.train_batch()s=s_ifdone:breakset training hyper parametersimportrlearntrainer=rlearn.DQNTrainer()trainer.set_params(learning_rate=0.01,batch_size=32,gamma=0.9,replace_ratio=1.,replace_step=0,min_epsilon=0.1,epsilon_decay=1e-3,)Parallel trainingexperience parallelStart a remote buffer:fromrlearnimportdistributeddistributed.experience.start_replay_buffer_server(port=50051,)Start actors:fromrlearnimportdistributedimportgymnasiumclassCartPole(rlearn.EnvWrapper):def__init__(self,render_mode="human"):self.env=gymnasium.make('CartPole-v1',render_mode=render_mode)defreset(self):s,_=self.env.reset()returnsdefstep(self,a):s_,_,done,_,_=self.env.step(a)r=-1ifdoneelse0returns_,r,donedistributed.experience.start_actor_server(port=50052,remote_buffer_address="localhost:50051",env=CartPole(),)Start a learner:importrlearnfromtensorflowimportkerastrainer=rlearn.trainer.DQNTrainer()trainer.set_model_encoder(q=keras.Sequential([keras.layers.InputLayer(4),keras.layers.Dense(32),keras.layers.ReLU(),]),action_num=2)trainer.set_params(learning_rate=0.01,batch_size=32,replace_step=15,)trainer.set_action_transformer(rlearn.transformer.DiscreteAction([0,1]))learner=rlearn.distributed.experience.Learner(trainer=trainer,remote_buffer_address="localhost:50051",actors_address=["localhost:50052",],actor_buffer_size=10,remote_buffer_size=1000,remote_buffer_type="RandomReplayBuffer",)learner.run(max_train_time=100,max_ep_step=-1)gradient parallelStart a parameter serverimportrlearntrainer=rlearn.trainer.DQNTrainer()trainer.set_model_encoder(q=keras.Sequential([keras.layers.InputLayer(4),keras.layers.Dense(20),keras.layers.ReLU(),]),action_num=2)trainer.set_params(learning_rate=0.001,batch_size=32,replace_step=100,)trainer.set_action_transformer(rlearn.transformer.DiscreteAction([0,1]))rlearn.distributed.gradient.start_param_server(port=50051,trainer=trainer,sync_step=5,worker_buffer_type="RandomReplayBuffer",worker_buffer_size=3000,max_train_time=60,# debug=True,)Start workersimportgymnasiumimportrlearnclassCartPole(rlearn.EnvWrapper):def__init__(self,render_mode="human"):self.env=gymnasium.make('CartPole-v1',render_mode=render_mode)defreset(self):s,_=self.env.reset()returnsdefstep(self,a):s_,_,done,_,_=self.env.step(a)r=-1ifdoneelse0returns_,r,donerlearn.distributed.gradient.worker.run(env=CartPole(),params_server_address="localhost:50051",name="worker1",# debug=True,)Save and reloadSave entire modelimportrlearnfromtensorflowimportkerasimportnumpyasnp# define and save a modeltrainer=rlearn.DQNTrainer()trainer.set_model_encoder(keras.Sequential([keras.layers.InputLayer(2),keras.layers.Dense(32),]),action_num=3)path="tmp_model0"trainer.save_model(path)# reload directory from pathm=rlearn.load_model(path)action=m.predict(np.random.random((2,)))Save model parameters and reload to a new trainer or new model.importrlearnfromtensorflowimportkerasimportnumpyasnp# define and save a modeltrainer=rlearn.DQNTrainer()trainer.set_model_encoder(keras.Sequential([keras.layers.InputLayer(2),keras.layers.Dense(32),]),action_num=3)path="tmp_model_weights0"trainer.save_model_weights(path)# trainer load parameters from pathtrainer2=rlearn.DQNTrainer()trainer2.set_model_encoder(keras.Sequential([keras.layers.InputLayer(2),keras.layers.Dense(32),]),action_num=3)trainer2.load_model_weights(path)action=trainer2.predict(np.random.random((2,)))# model load parametersm=rlearn.DQN()m.set_encoder(encoder=keras.Sequential([keras.layers.InputLayer(2),keras.layers.Dense(32),]),action_num=3)action=m.predict(np.random.random((2,)))Installgitclonehttps://git.woa.com/TIPE/rlearn.gitcdrlearn# apple m1 silicon should use conda command:condainstall-cappletensorflow-deps########python3setup.pyinstall