Repository: robocin/rSoccer (language: python)

Issue:
Move reward clip is badly implemented
When implemented in the old environment, the intention was to clip rewards when something unexpected occurred and the value was too big, on the code as of now it is clipping normal rewards

https://github.com/robocin/envs/blob/5f422179ca24fdf9eca0bf53fe02f2af1a4c6f21/envs/rc_gym/vss/env_3v3/vss_gym_3v3.py#L194


Discussion / hints:
maybe the same thing occurs here 
https://github.com/robocin/envs/blob/b4e98159dc03a604e7413bb280e46f7f69199cd9/envs/rc_gym/vss/env_3v3/vss_gym_3v3.py#L179

Fix patch:
diff --git a/README.md b/README.md
index b7ae9488..a54ad490 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,8 @@
 - deepvss
     - environment to our team using FIRASim
 
+- agents
+    - RL agents used to train on the environments
+
 - envs
-    - environment for vss using RoboSim
-    - environment for SSL using GRSim
+    - environment for vss and SSL using RoboSim
diff --git a/envs/agents/deepvss/agents/agentDDPG.py b/agents/deepvss/agents/agentDDPG.py
similarity index 100%
rename from envs/agents/deepvss/agents/agentDDPG.py
rename to agents/deepvss/agents/agentDDPG.py
diff --git a/envs/agents/deepvss/agents/agentDDPGMA.py b/agents/deepvss/agents/agentDDPGMA.py
similarity index 100%
rename from envs/agents/deepvss/agents/agentDDPGMA.py
rename to agents/deepvss/agents/agentDDPGMA.py
diff --git a/envs/agents/deepvss/deepvss_ma.py b/agents/deepvss/deepvss_ma.py
similarity index 100%
rename from envs/agents/deepvss/deepvss_ma.py
rename to agents/deepvss/deepvss_ma.py
diff --git a/envs/agents/deepvss/deepvss_rcgym.py b/agents/deepvss/deepvss_rcgym.py
similarity index 100%
rename from envs/agents/deepvss/deepvss_rcgym.py
rename to agents/deepvss/deepvss_rcgym.py
diff --git a/envs/agents/deepvss/lib/common.py b/agents/deepvss/lib/common.py
similarity index 100%
rename from envs/agents/deepvss/lib/common.py
rename to agents/deepvss/lib/common.py
diff --git a/envs/agents/deepvss/lib/ddpg_model.py b/agents/deepvss/lib/ddpg_model.py
similarity index 100%
rename from envs/agents/deepvss/lib/ddpg_model.py
rename to agents/deepvss/lib/ddpg_model.py
diff --git a/envs/agents/deepvss/lib/dqn_model.py b/agents/deepvss/lib/dqn_model.py
similarity index 100%
rename from envs/agents/deepvss/lib/dqn_model.py
rename to agents/deepvss/lib/dqn_model.py
diff --git a/envs/agents/deepvss/lib/sac_model.py b/agents/deepvss/lib/sac_model.py
similarity index 100%
rename from envs/agents/deepvss/lib/sac_model.py
rename to agents/deepvss/lib/sac_model.py
diff --git a/envs/agents/deepvss/parameters_ddpg.py b/agents/deepvss/parameters_ddpg.py
similarity index 100%
rename from envs/agents/deepvss/parameters_ddpg.py
rename to agents/deepvss/parameters_ddpg.py
diff --git a/envs/agents/deepvss/parameters_dqn.py b/agents/deepvss/parameters_dqn.py
similarity index 100%
rename from envs/agents/deepvss/parameters_dqn.py
rename to agents/deepvss/parameters_dqn.py
diff --git a/envs/README.md b/envs/README.md
index 4dd46343..c21bb458 100644
--- a/envs/README.md
+++ b/envs/README.md
@@ -1,44 +1,14 @@
-# Robocup SSL OpenAi gym environments
+# RoboSim VSSS and SSL gym environments
 
-# Requirements
-## Compile protobuf files
-```bash
-$ sudo apt-get install libprotobuf-dev protobuf-compiler -y
-$ cd rc_gym/grsim_ssl/Communication/pb/proto
-$ protoc --python_out=../ *.proto
-```
-## Fix protobuf compiled files from relative reference to absolute
-On file **rc_gym/grsim_ssl/Communication/pb/messages_robocup_ssl_wrapper_pb2.py**:
-
-
-``` python
-'before:'
-15 - import messages_robocup_ssl_detection_pb2 as messages__robocup__ssl__detection__pb2
-16 - import messages_robocup_ssl_geometry_pb2 as messages__robocup__ssl__geometry__pb2
-
-'after:'
-15 + import rc_gym.grsim_ssl.Communication.pb.messages_robocup_ssl_detection_pb2 as messages__robocup__ssl__detection__pb2
-16 + import rc_gym.grsim_ssl.Communication.pb.messages_robocup_ssl_geometry_pb2 as messages__robocup__ssl__geometry__pb2
-```
-
-On file **rc_gym/grsim_ssl/Communication/pb/grSim_Packet_pb2.py**:
-
-``` python
-'before:'
-15 - import grSim_Commands_pb2 as grSim__Commands__pb2
-16 - import grSim_Replacement_pb2 as grSim__Replacement__pb2
-'after:'
-15 + import rc_gym.grsim_ssl.Communication.pb.grSim_Commands_pb2 as grSim__Commands__pb2
-16 + import rc_gym.grsim_ssl.Communication.pb.grSim_Replacement_pb2 as grSim__Replacement__pb2
-```
 ## Install environments
 
 ```bash
 $ pip install -e .
 ```
 # Available Envs
-- **grSimSSLPenalty-v0**
-- **grSimSSLShootGoalie-v0**
+- **VSS3v3-v0**
+- **VSSMA-v0**
+- **VSSMAOpp-v0**
 
 # Example code
 ```python
@@ -46,7 +16,7 @@ import gym
 import rc_gym
 
 # Using penalty env
-env = gym.make('grSimSSLPenalty-v0')
+env = gym.make('VSS3v3-v0')
 
 env.reset()
 # Run for 1 episode and print reward at the end
diff --git a/envs/agents/Utils/Buffers.py b/envs/agents/Utils/Buffers.py
deleted file mode 100644
index 1dd2e4b0..00000000
--- a/envs/agents/Utils/Buffers.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import random
-import numpy as np
-
-class ReplayBuffer:
-    def __init__(self, capacity):
-        self.capacity = capacity
-        self.buffer = []
-        self.position = 0
-    
-    def push(self, state, action, reward, next_state, done):
-        if len(self.buffer) < self.capacity:
-            self.buffer.append(None)
-        self.buffer[self.position] = (state, action, reward, next_state, done)
-        self.position = (self.position + 1) % self.capacity
-    
-    def sample(self, batch_size):
-        batch = random.sample(self.buffer, batch_size)
-        state, action, reward, next_state, done = map(np.stack, zip(*batch))
-        return state, action, reward, next_state, done
-    
-    def __len__(self):
-        return len(self.buffer)
-
-class AverageBuffer:
-    def __init__(self, capacity = 100):
-        self.capacity = capacity
-        self.buffer = []
-        self.index = 0
-    
-    def push(self, goal):
-        if len(self.buffer) < self.capacity:
-            self.buffer.append(None)
-         
-        self.buffer[self.index] = goal
-        self.index = (self.index + 1) % self.capacity
-    
-    def average(self):
-        return np.mean(self.buffer if len(self.buffer) > 0 else [0])
-
-    def state_dict(self):
-        buffer_dict = {}
-        buffer_dict['index'] = self.index
-        buffer_dict['bufferSize'] = len(self.buffer)
-
-        for i in range(len(self.buffer)):
-            buffer_dict[i] = self.buffer[i] 
-
-        return buffer_dict
-    
-    def load_state_dict(self, buffer_dict):
-        self.index = buffer_dict['index']
-        size = buffer_dict['bufferSize']
-
-        for i in range(size):
-            self.buffer.append(buffer_dict[i])
-    
-    def __len__(self):
-        return len(self.buffer)
\ No newline at end of file
diff --git a/envs/agents/Utils/Networks.py b/envs/agents/Utils/Networks.py
deleted file mode 100644
index c4d44b14..00000000
--- a/envs/agents/Utils/Networks.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class ValueNetwork(nn.Module):
-    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):
-        super(ValueNetwork, self).__init__()
-        
-        self.linear1 = nn.Linear(num_inputs + num_actions, hidden_size)
-        self.linear2 = nn.Linear(hidden_size, hidden_size)
-        self.linear3 = nn.Linear(hidden_size, 1)
-        
-        self.linear3.weight.data.uniform_(-init_w, init_w)
-        self.linear3.bias.data.uniform_(-init_w, init_w)
-        
-    def forward(self, state, action):
-        x = torch.cat([state, action], 1)
-        x = F.relu(self.linear1(x))
-        x = F.relu(self.linear2(x))
-        x = self.linear3(x)
-        return x
-    
-
-class PolicyNetwork(nn.Module):
-    def __init__(self, num_inputs, num_actions, hidden_size, device, init_w=3e-3):
-        super(PolicyNetwork, self).__init__()
-        
-        self.linear1 = nn.Linear(num_inputs, hidden_size)
-        self.linear2 = nn.Linear(hidden_size, hidden_size)
-        self.linear3 = nn.Linear(hidden_size, num_actions)
-        
-        self.linear3.weight.data.uniform_(-init_w, init_w)
-        self.linear3.bias.data.uniform_(-init_w, init_w)
-        self.device = device
-        
-    def forward(self, state):
-        x = F.relu(self.linear1(state))
-        x = F.relu(self.linear2(x))
-        x = torch.tanh(self.linear3(x))
-        return x
-    
-    def get_action(self, state):
-        state  = torch.FloatTensor(state).unsqueeze(0).to(self.device)
-        action = self.forward(state)
-        return action.detach().cpu().numpy()[0]
diff --git a/envs/agents/Utils/Normalization.py b/envs/agents/Utils/Normalization.py
deleted file mode 100644
index d34e8847..00000000
--- a/envs/agents/Utils/Normalization.py
+++ /dev/null
@@ -1,78 +0,0 @@
-import gym
-
-import numpy        as np
-
-class NormalizedWrapper(gym.Wrapper):
-    """
-    :param env: (gym.Env) Gym environment that will be wrapped
-    """
-
-    def __init__(self, env):
-        # Call the parent constructor, so we can access self.env later
-        super(NormalizedWrapper, self).__init__(env)
-
-        assert isinstance(self.env.action_space,
-                          gym.spaces.Box), "This wrapper only works with continuous action space (spaces.Box)"
-        assert isinstance(self.env.observation_space,
-                          gym.spaces.Box), "This wrapper only works with continuous observation space (spaces.Box)"
-
-        # We modify the wrapper action space, so all actions will lie in [-1, 1]
-        self.action_space = gym.spaces.Box(low=-1, high=1, shape=self.env.action_space.shape, dtype=np.float32)
-        self.observation_space = gym.spaces.Box(low=-1, high=1, shape=self.env.observation_space.shape, dtype=np.float32)
-
-
-
-    def rescale_action(self, scaled_action):
-        """
-        Rescale the action from [-1, 1] to [low, high]
-        (no need for symmetric action space)
-        :param scaled_action: (np.ndarray)
-        :return: (np.ndarray)
-        """
-        return self.env.action_space.low + (
-                0.5 * (scaled_action + 1.0) * (self.env.action_space.high - self.env.action_space.low))
-
-    def scale_observation(self, observation):
-        """
-        Scale the observation to bounds [-1, 1]
-        """
-        return (2 * ((observation - self.env.observation_space.low) /
-                     (self.env.observation_space.high - self.env.observation_space.low))) - 1
-
-    def reset(self):
-        """
-        Reset the environment
-        """
-        # Reset the counter
-        return self.scale_observation(self.env.reset())
-
-    def step(self, action):
-        """
-        :param action: ([float] or int) Action taken by the agent
-        :return: (np.ndarray, float, bool, dict) observation, reward, is the episode over?, additional informations
-        """
-        # Rescale action from [-1, 1] to original [low, high] interval
-        rescaled_action = self.rescale_action(action)
-        obs, reward, done, info = self.env.step(rescaled_action)
-        return self.scale_observation(obs), reward, done, info
-
-
-class NormalizedActions(gym.ActionWrapper):
-
-    def _action(self, action):
-        low_bound   = self.action_space.low
-        upper_bound = self.action_space.high
-        
-        action = low_bound + (action + 1.0) * 0.5 * (upper_bound - low_bound)
-        action = np.clip(action, low_bound, upper_bound)
-        
-        return action
-
-    def _reverse_action(self, action):
-        low_bound   = self.action_space.low
-        upper_bound = self.action_space.high
-        
-        action = 2 * (action - low_bound) / (upper_bound - low_bound) - 1
-        action = np.clip(action, low_bound, upper_bound)
-        
-        return action
\ No newline at end of file
diff --git a/envs/agents/Utils/OUNoise.py b/envs/agents/Utils/OUNoise.py
deleted file mode 100644
index 04a59cff..00000000
--- a/envs/agents/Utils/OUNoise.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import numpy as np
-
-# Ornstein-Uhlenbeck process
-# Adding time-correlated noise to the actions taken by the deterministic policy
-#https://github.com/vitchyr/rlkit/blob/master/rlkit/exploration_strategies/ou_strategy.py
-
-class OUNoise(object):
-    def __init__(self, action_space, mu=0.0, theta=0.15, max_sigma=0.1, min_sigma=0.1, decay_period=100000):
-        self.mu           = mu
-        self.theta        = theta
-        self.sigma        = max_sigma
-        self.max_sigma    = max_sigma
-        self.min_sigma    = min_sigma
-        self.decay_period = decay_period
-        self.action_dim   = action_space.shape[0]
-        self.low          = action_space.low
-        self.high         = action_space.high
-        self.reset()
-        
-    def reset(self):
-        self.state = np.ones(self.action_dim) * self.mu
-        
-    def evolve_state(self):
-        x  = self.state
-        dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim)
-        self.state = x + dx
-        return self.state
-    
-    def get_action(self, action, t=0):
-        ou_state = self.evolve_state()
-        self.sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(1.0, t / self.decay_period)
-        return np.clip(action + ou_state, self.low, self.high)
\ No newline at end of file
diff --git a/envs/agents/Utils/ReplayBuffer.py b/envs/agents/Utils/ReplayBuffer.py
deleted file mode 100644
index 1dd2e4b0..00000000
--- a/envs/agents/Utils/ReplayBuffer.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import random
-import numpy as np
-
-class ReplayBuffer:
-    def __init__(self, capacity):
-        self.capacity = capacity
-        self.buffer = []
-        self.position = 0
-    
-    def push(self, state, action, reward, next_state, done):
-        if len(self.buffer) < self.capacity:
-            self.buffer.append(None)
-        self.buffer[self.position] = (state, action, reward, next_state, done)
-        self.position = (self.position + 1) % self.capacity
-    
-    def sample(self, batch_size):
-        batch = random.sample(self.buffer, batch_size)
-        state, action, reward, next_state, done = map(np.stack, zip(*batch))
-        return state, action, reward, next_state, done
-    
-    def __len__(self):
-        return len(self.buffer)
-
-class AverageBuffer:
-    def __init__(self, capacity = 100):
-        self.capacity = capacity
-        self.buffer = []
-        self.index = 0
-    
-    def push(self, goal):
-        if len(self.buffer) < self.capacity:
-            self.buffer.append(None)
-         
-        self.buffer[self.index] = goal
-        self.index = (self.index + 1) % self.capacity
-    
-    def average(self):
-        return np.mean(self.buffer if len(self.buffer) > 0 else [0])
-
-    def state_dict(self):
-        buffer_dict = {}
-        buffer_dict['index'] = self.index
-        buffer_dict['bufferSize'] = len(self.buffer)
-
-        for i in range(len(self.buffer)):
-            buffer_dict[i] = self.buffer[i] 
-
-        return buffer_dict
-    
-    def load_state_dict(self, buffer_dict):
-        self.index = buffer_dict['index']
-        size = buffer_dict['bufferSize']
-
-        for i in range(size):
-            self.buffer.append(buffer_dict[i])
-    
-    def __len__(self):
-        return len(self.buffer)
\ No newline at end of file
diff --git a/envs/agents/agentGoToBall/rlAdventure2DDPG/agentDDPG.py b/envs/agents/agentGoToBall/rlAdventure2DDPG/agentDDPG.py
deleted file mode 100644
index b97bf3ea..00000000
--- a/envs/agents/agentGoToBall/rlAdventure2DDPG/agentDDPG.py
+++ /dev/null
@@ -1,235 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-import os
-import sys
-import time
-
-from torch.utils.tensorboard import SummaryWriter
-import torch.optim as optim
-import torch.nn as nn
-import torch
-
-from agents.Utils.Normalization import NormalizedWrapper
-from agents.Utils.Networks import ValueNetwork, PolicyNetwork
-from agents.Utils.OUNoise import OUNoise
-from agents.Utils.Buffers import ReplayBuffer, AverageBuffer
-
-
-class AgentDDPG:
-
-    def __init__(self, name='DDPG',
-                 maxEpisodes=30000, maxSteps=250, batchSize=256, replayBufferSize=100000, valueLR=1e-3, policyLR=1e-4,
-                 hiddenDim=256, nEpisodesPerCheckpoint=1000):
-        # Training Parameters
-        self.batchSize = batchSize
-        self.maxSteps = maxSteps
-        self.maxEpisodes = maxEpisodes
-        self.nEpisodesPerCheckpoint = nEpisodesPerCheckpoint
-        self.nEpisodes = 0
-
-        # Check if cuda gpu is available, and select it
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        print('Cuda Activated? ', torch.cuda.is_available())
-            
-        # Create Environment using a wrapper which scales actions and observations to [-1, 1]
-        self.env = NormalizedWrapper(gym.make("grSimSSLGoToBall-v0"))
-
-        # Init action noise object
-        self.ouNoise = OUNoise(self.env.action_space)
-
-        # Init networks
-        stateDim = self.env.observation_space.shape[0]
-        actionDim = self.env.action_space.shape[0]
-        self.valueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.policyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        self.targetValueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.targetPolicyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        # Same initial parameters for target networks
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(param.data)
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(param.data)
-
-        # Init optimizers
-        self.valueOptimizer = optim.Adam(self.valueNet.parameters(), lr=valueLR)
-        self.policyOptimizer = optim.Adam(self.policyNet.parameters(), lr=policyLR)
-
-        # Init replay buffer
-        self.replayBuffer = ReplayBuffer(replayBufferSize)
-
-         # Init goals buffer
-        self.goalsBuffer = AverageBuffer()
-
-        # Init rewars buffer
-        self.rewardsBuffer = AverageBuffer()
-
-        # Steps per Seconds Parameters
-        self.startTimeInEpisode  = 0.0
-
-        # Tensorboard Init
-        self.path = './runs/' + name
-        self.loadedModel = self._load()
-        self.writer = SummaryWriter(log_dir=self.path)
-        
-
-    def _update(self, batch_size,
-                gamma=0.99,
-                min_value=-np.inf,
-                max_value=np.inf,
-                soft_tau=1e-2):
-        state, action, reward, next_state, done = self.replayBuffer.sample(batch_size)
-
-        state = torch.FloatTensor(state).to(self.device)
-        next_state = torch.FloatTensor(next_state).to(self.device)
-        action = torch.FloatTensor(action).to(self.device)
-        reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device)
-        done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device)
-
-        policyLoss = self.valueNet(state, self.policyNet(state))
-        policyLoss = -policyLoss.mean()
-
-        next_action = self.targetPolicyNet(next_state)
-        target_value = self.targetValueNet(next_state, next_action.detach())
-        expected_value = reward + (1.0 - done) * gamma * target_value
-        expected_value = torch.clamp(expected_value, min_value, max_value)
-
-        value = self.valueNet(state, action)
-        value_criterion = nn.MSELoss()
-        value_loss = value_criterion(value, expected_value.detach())
-
-        self.policyOptimizer.zero_grad()
-        policyLoss.backward()
-        self.policyOptimizer.step()
-
-        self.valueOptimizer.zero_grad()
-        value_loss.backward()
-        self.valueOptimizer.step()
-
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-    # Training Loop
-    def train(self):
-        while self.nEpisodes < self.maxEpisodes:
-            state = self.env.reset()
-            self.ouNoise.reset()
-            episodeReward = 0
-            nStepsInEpisode = 0
-            stepSeg = -1
-            self.startTimeInEpisode = time.time()
-
-            while nStepsInEpisode < self.maxSteps:
-                action = self.policyNet.get_action(state)
-                action = self.ouNoise.get_action(action, nStepsInEpisode)
-                next_state, reward, done, _ = self.env.step(action)
-
-                self.replayBuffer.push(state, action, reward, next_state, done)
-                if len(self.replayBuffer) > self.batchSize:
-                    self._update(self.batchSize)
-
-                state = next_state
-                episodeReward += reward
-                nStepsInEpisode += 1
-
-                if done:
-                    self.goalsBuffer.push(1 if reward > 0 else 0)
-                    break   
-            
-
-            if nStepsInEpisode > 1:
-                stepSeg = nStepsInEpisode/(time.time() - self.startTimeInEpisode)
-
-            self.rewardsBuffer.push(episodeReward)
-            self.nEpisodes += 1
-
-            # TODO trocar por lista circular
-            # rewards.append(episodeReward)
-
-            self.writer.add_scalar('Train/Reward', episodeReward, self.nEpisodes)
-            self.writer.add_scalar('Train/Steps', nStepsInEpisode, self.nEpisodes)
-            #self.writer.add_scalar('Train/Goals_average_on_{}_previous_episodes'.format(self.goalsBuffer.capacity), self.goalsBuffer.average(), self.nEpisodes)
-            self.writer.add_scalar('Train/Steps_seconds',stepSeg, self.nEpisodes)
-            self.writer.add_scalar('Train/Reward_average_on_{}_previous_episodes'.format(self.rewardsBuffer.capacity), self.rewardsBuffer.average(), self.nEpisodes)
-
-            if (self.nEpisodes % self.nEpisodesPerCheckpoint) == 0:
-                self._save()
-
-        self.writer.flush()
-
-    # Playing loop
-    def play(self):
-        if self.loadedModel:
-            while True:
-                done = False
-                obs = self.env.reset()
-                steps = 0
-                while not done and steps < self.maxSteps:
-                    steps += 1
-                    action = self.policyNet.get_action(obs)
-                    obs, reward, done, _ = self.env.step(action)
-        else:
-            print("Correct usage: python train.py {name} (play | train) [-cs]")
-
-    def _load(self):
-        # Check if checkpoint file exists
-        if os.path.exists(self.path + '/checkpoint'):
-            checkpoint = torch.load(self.path + '/checkpoint')
-            # Load networks parameters checkpoint
-            self.valueNet.load_state_dict(checkpoint['valueNetDict'])
-            self.policyNet.load_state_dict(checkpoint['policyNetDict'])
-            self.targetValueNet.load_state_dict(checkpoint['targetValueNetDict'])
-            self.targetPolicyNet.load_state_dict(checkpoint['targetPolicyNetDict'])
-            # Load number of episodes on checkpoint
-            self.nEpisodes = checkpoint['nEpisodes']
-            print("Checkpoint with {} episodes successfully loaded".format(self.nEpisodes))
-            return True
-        else:
-            print("- No checkpoint " + self.path + '/checkpoint' + " loaded!")
-            return False
-
-    def _save(self):
-        print("Save network parameters in episode ", self.nEpisodes)
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint')
-
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint_' + str(self.nEpisodes))
-
-
-if __name__ == '__main__':
-    try:
-        if len(sys.argv) >= 3:
-            agent = AgentDDPG(name=sys.argv[1])
-            if sys.argv[2] == 'play':
-                agent.play()
-            elif sys.argv[2] == 'train':
-                agent.train()
-            else:
-                print("correct usage: python train.py {name} (play | train) [-cs]")
-        else:
-            print("correct usage: python train.py {name} (play | train) [-cs]")
-    except KeyboardInterrupt:
-        if len(sys.argv) >= 4 and sys.argv[3] == '-cs':
-            agent._save()
\ No newline at end of file
diff --git a/envs/agents/agentGoToBall/rlAdventure2DDPG/models/saved_networks b/envs/agents/agentGoToBall/rlAdventure2DDPG/models/saved_networks
deleted file mode 100644
index 368c1707..00000000
Binary files a/envs/agents/agentGoToBall/rlAdventure2DDPG/models/saved_networks and /dev/null differ
diff --git a/envs/agents/agentGoToBall/rlAdventure2DDPG/play.py b/envs/agents/agentGoToBall/rlAdventure2DDPG/play.py
deleted file mode 100644
index 66cab20a..00000000
--- a/envs/agents/agentGoToBall/rlAdventure2DDPG/play.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import gym
-import rc_gym
-import torch
-
-from agents.Utils.Networks             import PolicyNetwork
-
-# Using penalty env
-env = gym.make('grSimSSLGoToBall-v0')
-
-use_cuda = torch.cuda.is_available()
-print("use_cuda ->", use_cuda)
-device   = torch.device("cuda" if use_cuda else "cpu")
-
-state_dim  = env.observation_space.shape[0]
-action_dim = env.action_space.shape[0]
-hidden_dim = 256
-
-policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim, device=device).to(device)
-
-checkpoint = torch.load('./models/test_go_to_ball')
-policy_net.load_state_dict(checkpoint['target_policy_net_dict'])
-
-env.reset()
-# Run for 100 episode and print reward at the end
-for i in range(100):
-    done = False
-    obs = env.reset()
-    while not done:
-        action = policy_net.get_action(obs)
-        obs, reward, done, _ = env.step(action)
-    print(reward)
\ No newline at end of file
diff --git a/envs/agents/agentGoToBall/rlAdventure2DDPG/train.py b/envs/agents/agentGoToBall/rlAdventure2DDPG/train.py
deleted file mode 100644
index 011d8c28..00000000
--- a/envs/agents/agentGoToBall/rlAdventure2DDPG/train.py
+++ /dev/null
@@ -1,140 +0,0 @@
-import gym
-import rc_gym
-import numpy        as np
-
-from    torch.utils.tensorboard    import SummaryWriter
-import  torch.optim  as optim
-import  torch.nn     as nn
-import  torch
-
-from agents.Utils.Networks             import ValueNetwork, PolicyNetwork
-from agents.Utils.Normalization        import NormalizedWrapper
-from agents.Utils.ReplayBuffer         import ReplayBuffer
-from agents.Utils.OUNoise              import OUNoise
-
-use_cuda = torch.cuda.is_available()
-print("use_cuda ->", use_cuda)
-device   = torch.device("cuda" if use_cuda else "cpu")
-
-writer = SummaryWriter()
-
-max_episodes  = 2000000
-max_steps   = 250
-episode   = 0
-rewards     = []
-batch_size  = 256
-replay_buffer_size = 100000
-
-# Continuous control with deep reinforcement learning
-def ddpg_update(batch_size, 
-           gamma = 0.99,
-           min_value=-np.inf,
-           max_value=np.inf,
-           soft_tau=1e-2):
-    
-    state, action, reward, next_state, done = replay_buffer.sample(batch_size)
-    
-    state      = torch.FloatTensor(state).to(device)
-    next_state = torch.FloatTensor(next_state).to(device)
-    action     = torch.FloatTensor(action).to(device)
-    reward     = torch.FloatTensor(reward).unsqueeze(1).to(device)
-    done       = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(device)
-
-    policy_loss = value_net(state, policy_net(state))
-    policy_loss = -policy_loss.mean()
-
-    next_action    = target_policy_net(next_state)
-    target_value   = target_value_net(next_state, next_action.detach())
-    expected_value = reward + (1.0 - done) * gamma * target_value
-    expected_value = torch.clamp(expected_value, min_value, max_value)
-
-    value = value_net(state, action)
-    value_loss = value_criterion(value, expected_value.detach())
-
-
-    policy_optimizer.zero_grad()
-    policy_loss.backward()
-    policy_optimizer.step()
-
-    value_optimizer.zero_grad()
-    value_loss.backward()
-    value_optimizer.step()
-
-    for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-    for target_param, param in zip(target_policy_net.parameters(), policy_net.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-
-if __name__ == "__main__":
-        
-    env = NormalizedWrapper(gym.make('grSimSSLGoToBall-v0'))
-    #env = gym.make('grSimSSLGoToBall-v0')
-    ou_noise = OUNoise(env.action_space)
-
-    state_dim  = env.observation_space.shape[0]
-    action_dim = env.action_space.shape[0]
-    hidden_dim = 256
-
-    value_net  = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)
-    policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim, device=device).to(device)
-
-    target_value_net  = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)
-    target_policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim, device=device).to(device)
-
-    for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):
-        target_param.data.copy_(param.data)
-
-    for target_param, param in zip(target_policy_net.parameters(), policy_net.parameters()):
-        target_param.data.copy_(param.data)    
-        
-    value_lr  = 1e-3
-    policy_lr = 1e-4
-
-    value_optimizer  = optim.Adam(value_net.parameters(),  lr=value_lr)
-    policy_optimizer = optim.Adam(policy_net.parameters(), lr=policy_lr)
-
-    value_criterion = nn.MSELoss()
-
-    replay_buffer = ReplayBuffer(replay_buffer_size)
-
-    while episode < max_episodes:
-        state = env.reset()
-        ou_noise.reset()
-        episode_reward = 0
-        steps_episode = 0
-        
-        while steps_episode < max_steps:
-            action = policy_net.get_action(state)
-            action = ou_noise.get_action(action, steps_episode)
-            next_state, reward, done, _ = env.step(action)
-            
-            replay_buffer.push(state, action, reward, next_state, done)
-            if len(replay_buffer) > batch_size:
-                ddpg_update(batch_size)
-            
-            state = next_state
-            episode_reward += reward
-            steps_episode += 1
-
-            
-            if done:
-                break
-        
-        
-        episode += 1
-        rewards.append(episode_reward)
-
-        writer.add_scalar('Train/Reward', episode_reward, episode)
-        writer.add_scalar('Train/Steps', steps_episode, episode)
-
-        if (episode % 1000) == 0:
-            torch.save({
-                'target_value_net_dict': target_value_net.state_dict(),
-                'target_policy_net_dict': target_policy_net.state_dict()
-            }, './models/test_go_to_ball')
\ No newline at end of file
diff --git a/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/models/ddpg_shootgoalie2.zip b/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/models/ddpg_shootgoalie2.zip
deleted file mode 100644
index 8cafe238..00000000
Binary files a/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/models/ddpg_shootgoalie2.zip and /dev/null differ
diff --git a/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/play.py b/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/play.py
deleted file mode 100644
index 5c601161..00000000
--- a/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/play.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-
-from stable_baselines import DDPG
-
-env = gym.make('grSimSSLGoToBall-v0')
-model = DDPG.load("./models/ddpg_gotoball2")
-
-while True:
-    done = False
-    obs = env.reset()
-    while not done:
-        action, _states = model.predict(obs)
-        obs, rewards, done, info = env.step(action)
diff --git a/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/train.py b/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/train.py
deleted file mode 100644
index ce9c86bf..00000000
--- a/envs/agents/agentGoToBall/stableBaselinesV2_DDPG/train.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-
-from stable_baselines.ddpg.policies import MlpPolicy
-from stable_baselines.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise, AdaptiveParamNoiseSpec
-from stable_baselines import DDPG
-
-
-env = gym.make('grSimSSLGoToBall-v0')
-
-# The noise objects for DDPG
-n_actions = env.action_space.shape[-1]
-param_noise = None
-action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.1) * np.ones(n_actions), theta=float(0.15))
-
-model = DDPG(MlpPolicy, env, verbose=1, actor_lr = 0.001, critic_lr = 0.0001, normalize_observations=True, param_noise=param_noise, action_noise=action_noise, memory_limit=100000)
-model.learn(total_timesteps=100000, log_interval=10)
-
-model.save("./models/ddpg_gotoball2")
-env = model.get_env()
-
-print("Done!")
\ No newline at end of file
diff --git a/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/models/ddpg_shootGoalie_0.zip b/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/models/ddpg_shootGoalie_0.zip
deleted file mode 100644
index 3079a0d9..00000000
Binary files a/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/models/ddpg_shootGoalie_0.zip and /dev/null differ
diff --git a/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/play.py b/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/play.py
deleted file mode 100644
index 1e3b7cab..00000000
--- a/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/play.py
+++ /dev/null
@@ -1,16 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-
-from stable_baselines3 import DDPG
-
-env = gym.make('grSimSSLGoToBall-v0')
-
-model = DDPG.load("./models/gotoball3")
-
-while True:
-    done = False
-    obs = env.reset()
-    while not done:
-        action, _states = model.predict(obs)
-        obs, rewards, done, info = env.step(action)
diff --git a/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/train.py b/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/train.py
deleted file mode 100644
index 92b98023..00000000
--- a/envs/agents/agentGoToBall/stableBaselinesV3_DDPG/train.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-
-from stable_baselines3 import DDPG
-from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise
-
-env = gym.make('grSimSSLGoToBall-v0')
-
-# The noise objects for DDPG
-n_actions = env.action_space.shape[-1]
-action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
-
-model = DDPG('MlpPolicy', env, action_noise=action_noise, verbose=1, tensorboard_log='./ddpg_go_to_ball_tensorboard')
-model.learn(total_timesteps=100, log_interval=10)
-model.save("./models/gotoball3")
-env = model.get_env()
-
-print("Done!")
-
diff --git a/envs/agents/agentPenaltyEnv/rlAdventure2DDPG/README.md b/envs/agents/agentPenaltyEnv/rlAdventure2DDPG/README.md
deleted file mode 100644
index c0fa65ac..00000000
--- a/envs/agents/agentPenaltyEnv/rlAdventure2DDPG/README.md
+++ /dev/null
@@ -1,22 +0,0 @@
-# AgentDDPG
-Based on [RL-Adventure-2 Github Repository](https://github.com/higgsfield/RL-Adventure-2)
-## How to run
-### Training
-```bash
-../rlAdventure2DDPG $ conda activate env
-../rlAdventure2DDPG $ python agentDDPG.py {name} train
-```
-- Training checkpoints will be save in **./runs/_{name}_/**
-- If checkpoint already exists in **./runs/_{name}_/** it will resume training
-
-### Playing
-```bash
-../rlAdventure2DDPG $ conda activate env
-../rlAdventure2DDPG $ python agentDDPG.py {name} play
-```
-- Uses trained network if checkpoint exists in **./runs/_{name}_/**
-- Play with random network if no checkpoint is found
-
-
-# Requirements
-\#TODO
\ No newline at end of file
diff --git a/envs/agents/agentPenaltyEnv/rlAdventure2DDPG/agentDDPG.py b/envs/agents/agentPenaltyEnv/rlAdventure2DDPG/agentDDPG.py
deleted file mode 100644
index 73ec01f9..00000000
--- a/envs/agents/agentPenaltyEnv/rlAdventure2DDPG/agentDDPG.py
+++ /dev/null
@@ -1,222 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-import os
-import sys
-import time
-
-from torch.utils.tensorboard import SummaryWriter
-import torch.optim as optim
-import torch.nn as nn
-import torch
-
-from agents.Utils.Normalization import NormalizedWrapper
-from agents.Utils.Networks import ValueNetwork, PolicyNetwork
-from agents.Utils.OUNoise import OUNoise
-from agents.Utils.Buffers import ReplayBuffer, AverageBuffer
-
-
-class AgentDDPG:
-
-    def __init__(self, name='DDPG',
-                 maxEpisodes=10000, maxSteps=200, batchSize=256, replayBufferSize=200000, valueLR=1e-3, policyLR=1e-4,
-                 hiddenDim=256, nEpisodesPerCheckpoint=10000):
-        # Training Parameters
-        self.batchSize = batchSize
-        self.maxSteps = maxSteps
-        self.maxEpisodes = maxEpisodes
-        self.nEpisodesPerCheckpoint = nEpisodesPerCheckpoint
-        self.nEpisodes = 0
-
-        # Check if cuda gpu is available, and select it
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-        # Create Environment using a wrapper which scales actions and observations to [-1, 1]
-        self.env = NormalizedWrapper(gym.make("grSimSSLPenalty-v0"))
-
-        # Init action noise object
-        self.ouNoise = OUNoise(self.env.action_space)
-
-        # Init networks
-        stateDim = self.env.observation_space.shape[0]
-        actionDim = self.env.action_space.shape[0]
-        self.valueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.policyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        self.targetValueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.targetPolicyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        # Same initial parameters for target networks
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(param.data)
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(param.data)
-
-        # Init optimizers
-        self.valueOptimizer = optim.Adam(self.valueNet.parameters(), lr=valueLR)
-        self.policyOptimizer = optim.Adam(self.policyNet.parameters(), lr=policyLR)
-
-        # Init replay buffer
-        self.replayBuffer = ReplayBuffer(replayBufferSize)
-
-        # Init goals buffer
-        self.goalsBuffer = AverageBuffer()
-
-        # Init rewars buffer
-        self.rewardsBuffer = AverageBuffer()
-
-        # Tensorboard Init
-        self.path = './runs/' + name
-        self._load()
-        self.writer = SummaryWriter(log_dir=self.path)
-
-    def _update(self, batch_size,
-                gamma=0.99,
-                min_value=-np.inf,
-                max_value=np.inf,
-                soft_tau=1e-2):
-        state, action, reward, next_state, done = self.replayBuffer.sample(batch_size)
-
-        state = torch.FloatTensor(state).to(self.device)
-        next_state = torch.FloatTensor(next_state).to(self.device)
-        action = torch.FloatTensor(action).to(self.device)
-        reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device)
-        done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device)
-
-        policyLoss = self.valueNet(state, self.policyNet(state))
-        policyLoss = -policyLoss.mean()
-
-        next_action = self.targetPolicyNet(next_state)
-        target_value = self.targetValueNet(next_state, next_action.detach())
-        expected_value = reward + (1.0 - done) * gamma * target_value
-        expected_value = torch.clamp(expected_value, min_value, max_value)
-
-        value = self.valueNet(state, action)
-        value_criterion = nn.MSELoss()
-        value_loss = value_criterion(value, expected_value.detach())
-
-        self.policyOptimizer.zero_grad()
-        policyLoss.backward()
-        self.policyOptimizer.step()
-
-        self.valueOptimizer.zero_grad()
-        value_loss.backward()
-        self.valueOptimizer.step()
-
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-    # Training Loop
-    def train(self):
-        while self.nEpisodes < self.maxEpisodes:
-            state = self.env.reset()
-            self.ouNoise.reset()
-            episodeReward = 0
-            nStepsInEpisode = 0
-            stepSeg = -1
-            startTimeInEpisode = time.time()
-
-            while nStepsInEpisode < self.maxSteps:
-                action = self.policyNet.get_action(state)
-                action = self.ouNoise.get_action(action, nStepsInEpisode)
-                next_state, reward, done, _ = self.env.step(action)
-
-                self.replayBuffer.push(state, action, reward, next_state, done)
-                if len(self.replayBuffer) > self.batchSize:
-                    self._update(self.batchSize)
-
-                state = next_state
-                episodeReward += reward
-                nStepsInEpisode += 1
-
-                if done:
-                    self.goalsBuffer.push(1 if reward < 0 else 0)
-                    break
-            
-            if nStepsInEpisode > 1:
-                stepSeg = nStepsInEpisode/(time.time() - startTimeInEpisode)
-
-            self.rewardsBuffer.push(episodeReward)
-            self.nEpisodes += 1
-
-            self.writer.add_scalar('Train/Reward', episodeReward, self.nEpisodes)
-            self.writer.add_scalar('Train/Steps', nStepsInEpisode, self.nEpisodes)
-            self.writer.add_scalar('Train/Goals_average_on_{}_previous_episodes'.format(self.goalsBuffer.capacity), self.goalsBuffer.average(), self.nEpisodes)
-            self.writer.add_scalar('Train/Steps_seconds',stepSeg, self.nEpisodes)
-            self.writer.add_scalar('Train/Reward_average_on_{}_previous_episodes'.format(self.rewardsBuffer.capacity), self.rewardsBuffer.average(), self.nEpisodes)
-
-            if (self.nEpisodes % self.nEpisodesPerCheckpoint) == 0:
-                self._save()
-
-        self.writer.flush()
-
-    # Playing loop
-    def play(self):
-        while True:
-            done = False
-            obs = self.env.reset()
-            while not done:
-                action = self.policyNet.get_action(obs)
-                obs, reward, done, _ = self.env.step(action)
-
-    def _load(self):
-        # Check if checkpoint file exists
-        if os.path.exists(self.path + '/checkpoint'):
-            checkpoint = torch.load(self.path + '/checkpoint')
-            # Load networks parameters checkpoint
-            self.valueNet.load_state_dict(checkpoint['valueNetDict'])
-            self.policyNet.load_state_dict(checkpoint['policyNetDict'])
-            self.targetValueNet.load_state_dict(checkpoint['targetValueNetDict'])
-            self.targetPolicyNet.load_state_dict(checkpoint['targetPolicyNetDict'])
-            self.goalsBuffer.load_state_dict(checkpoint['goalsBuffer'])
-            self.rewardsBuffer.load_state_dict(checkpoint['rewardsBuffer'])
-            # Load number of episodes on checkpoint
-            self.nEpisodes = checkpoint['nEpisodes']
-            self.maxEpisodes += checkpoint['nEpisodes']
-            print("Checkpoint with {} episodes successfully loaded".format(self.nEpisodes))
-        else:
-            print("No checkpoint loaded")
-    
-    def _save(self):
-        print("Save network parameters in episode ", self.nEpisodes)
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint')
-
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint_' + str(self.nEpisodes))
-
-
-if __name__ == '__main__':
-    try:
-        if len(sys.argv) >= 3:
-            agent = AgentDDPG(name=sys.argv[1])
-            if sys.argv[2] == 'play':
-                agent.play()
-            elif sys.argv[2] == 'train':
-                agent.train()
-            else:
-                print("correct usage: python agentDDPG.py {name} (play or train) [-cs]")
-        else:
-            print("correct usage: python agentDDPG.py {name} (play or train) [-cs]")
-    except KeyboardInterrupt:
-        if len(sys.argv) >= 4 and sys.argv[3] == '-cs':
-            agent._save()
\ No newline at end of file
diff --git a/envs/agents/agentShootGoalie/rlAdventure2DDPG/README.md b/envs/agents/agentShootGoalie/rlAdventure2DDPG/README.md
deleted file mode 100644
index c0fa65ac..00000000
--- a/envs/agents/agentShootGoalie/rlAdventure2DDPG/README.md
+++ /dev/null
@@ -1,22 +0,0 @@
-# AgentDDPG
-Based on [RL-Adventure-2 Github Repository](https://github.com/higgsfield/RL-Adventure-2)
-## How to run
-### Training
-```bash
-../rlAdventure2DDPG $ conda activate env
-../rlAdventure2DDPG $ python agentDDPG.py {name} train
-```
-- Training checkpoints will be save in **./runs/_{name}_/**
-- If checkpoint already exists in **./runs/_{name}_/** it will resume training
-
-### Playing
-```bash
-../rlAdventure2DDPG $ conda activate env
-../rlAdventure2DDPG $ python agentDDPG.py {name} play
-```
-- Uses trained network if checkpoint exists in **./runs/_{name}_/**
-- Play with random network if no checkpoint is found
-
-
-# Requirements
-\#TODO
\ No newline at end of file
diff --git a/envs/agents/agentShootGoalie/rlAdventure2DDPG/agentDDPG.py b/envs/agents/agentShootGoalie/rlAdventure2DDPG/agentDDPG.py
deleted file mode 100644
index 026e8dae..00000000
--- a/envs/agents/agentShootGoalie/rlAdventure2DDPG/agentDDPG.py
+++ /dev/null
@@ -1,236 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-import os
-import sys
-import time
-
-from torch.utils.tensorboard import SummaryWriter
-import torch.optim as optim
-import torch.nn as nn
-import torch
-
-from agents.Utils.Normalization import NormalizedWrapper
-from agents.Utils.Networks import ValueNetwork, PolicyNetwork
-from agents.Utils.OUNoise import OUNoise
-from agents.Utils.Buffers import ReplayBuffer, AverageBuffer
-
-
-class AgentDDPG:
-
-    def __init__(self, name='DDPG',
-                 maxEpisodes=60000, maxSteps=150, batchSize=256, replayBufferSize=200000, valueLR=1e-3, policyLR=1e-4,
-                 hiddenDim=256, nEpisodesPerCheckpoint=5000):
-        # Training Parameters
-        self.batchSize = batchSize
-        self.maxSteps = maxSteps
-        self.maxEpisodes = maxEpisodes
-        self.nEpisodesPerCheckpoint = nEpisodesPerCheckpoint
-        self.nEpisodes = 0
-
-        # Check if cuda gpu is available, and select it
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        print('Cuda Activated? ', torch.cuda.is_available())
-            
-        # Create Environment using a wrapper which scales actions and observations to [-1, 1]
-        self.env = NormalizedWrapper(gym.make("grSimSSLShootGoalie-v0"))
-
-        # Init action noise object
-        self.ouNoise = OUNoise(self.env.action_space)
-
-        # Init networks
-        stateDim = self.env.observation_space.shape[0]
-        actionDim = self.env.action_space.shape[0]
-        self.valueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.policyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        self.targetValueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.targetPolicyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        # Same initial parameters for target networks
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(param.data)
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(param.data)
-
-        # Init optimizers
-        self.valueOptimizer = optim.Adam(self.valueNet.parameters(), lr=valueLR)
-        self.policyOptimizer = optim.Adam(self.policyNet.parameters(), lr=policyLR)
-
-        # Init replay buffer
-        self.replayBuffer = ReplayBuffer(replayBufferSize)
-
-         # Init goals buffer
-        self.goalsBuffer = AverageBuffer()
-
-        # Init rewars buffer
-        self.rewardsBuffer = AverageBuffer()
-
-        # Steps per Seconds Parameters
-        self.startTimeInEpisode  = 0.0
-
-        # Tensorboard Init
-        self.path = './runs/' + name
-        self.loadedModel = self._load()
-        self.writer = SummaryWriter(log_dir=self.path)
-        
-
-    def _update(self, batch_size,
-                gamma=0.99,
-                min_value=-np.inf,
-                max_value=np.inf,
-                soft_tau=1e-2):
-        state, action, reward, next_state, done = self.replayBuffer.sample(batch_size)
-
-        state = torch.FloatTensor(state).to(self.device)
-        next_state = torch.FloatTensor(next_state).to(self.device)
-        action = torch.FloatTensor(action).to(self.device)
-        reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device)
-        done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device)
-
-        policyLoss = self.valueNet(state, self.policyNet(state))
-        policyLoss = -policyLoss.mean()
-
-        next_action = self.targetPolicyNet(next_state)
-        target_value = self.targetValueNet(next_state, next_action.detach())
-        expected_value = reward + (1.0 - done) * gamma * target_value
-        expected_value = torch.clamp(expected_value, min_value, max_value)
-
-        value = self.valueNet(state, action)
-        value_criterion = nn.MSELoss()
-        value_loss = value_criterion(value, expected_value.detach())
-
-        self.policyOptimizer.zero_grad()
-        policyLoss.backward()
-        self.policyOptimizer.step()
-
-        self.valueOptimizer.zero_grad()
-        value_loss.backward()
-        self.valueOptimizer.step()
-
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-    # Training Loop
-    def train(self):
-        while self.nEpisodes < self.maxEpisodes:
-            state = self.env.reset()
-            self.ouNoise.reset()
-            episodeReward = 0
-            nStepsInEpisode = 0
-            stepSeg = -1
-            self.startTimeInEpisode = time.time()
-
-            while nStepsInEpisode < self.maxSteps:
-                action = self.policyNet.get_action(state)
-                action = self.ouNoise.get_action(action, nStepsInEpisode)
-                next_state, reward, done, _ = self.env.step(action)
-
-                self.replayBuffer.push(state, action, reward, next_state, done)
-                if len(self.replayBuffer) > self.batchSize:
-                    self._update(self.batchSize)
-
-                state = next_state
-                episodeReward += reward
-                nStepsInEpisode += 1
-
-                if done:
-                    self.goalsBuffer.push(1 if reward > 0 else 0)
-                    break   
-            
-
-            if nStepsInEpisode > 1:
-                stepSeg = nStepsInEpisode/(time.time() - self.startTimeInEpisode)
-
-            self.rewardsBuffer.push(episodeReward)
-            self.nEpisodes += 1
-
-            # TODO trocar por lista circular
-            # rewards.append(episodeReward)
-
-            self.writer.add_scalar('Train/Reward', episodeReward, self.nEpisodes)
-            self.writer.add_scalar('Train/Steps', nStepsInEpisode, self.nEpisodes)
-            self.writer.add_scalar('Train/Goals_average_on_{}_previous_episodes'.format(self.goalsBuffer.capacity), self.goalsBuffer.average(), self.nEpisodes)
-            self.writer.add_scalar('Train/Steps_seconds',stepSeg, self.nEpisodes)
-            self.writer.add_scalar('Train/Reward_average_on_{}_previous_episodes'.format(self.rewardsBuffer.capacity), self.rewardsBuffer.average(), self.nEpisodes)
-
-            if (self.nEpisodes % self.nEpisodesPerCheckpoint) == 0:
-                self._save()
-
-        self.writer.flush()
-
-    # Playing loop
-    def play(self):
-        if self.loadedModel:
-            while True:
-                done = False
-                obs = self.env.reset()
-                steps = 0
-                while not done and steps < self.maxSteps:
-                    steps += 1
-                    action = self.policyNet.get_action(obs)
-                    obs, reward, done, _ = self.env.step(action)
-                time.sleep(0.1)
-        else:
-            print("Correct usage: python train.py {name} (play | train) [-cs]")
-
-    def _load(self):
-        # Check if checkpoint file exists
-        if os.path.exists(self.path + '/checkpoint'):
-            checkpoint = torch.load(self.path + '/checkpoint', map_location=self.device)
-            # Load networks parameters checkpoint
-            self.valueNet.load_state_dict(checkpoint['valueNetDict'])
-            self.policyNet.load_state_dict(checkpoint['policyNetDict'])
-            self.targetValueNet.load_state_dict(checkpoint['targetValueNetDict'])
-            self.targetPolicyNet.load_state_dict(checkpoint['targetPolicyNetDict'])
-            # Load number of episodes on checkpoint
-            self.nEpisodes = checkpoint['nEpisodes']
-            print("Checkpoint with {} episodes successfully loaded".format(self.nEpisodes))
-            return True
-        else:
-            print("- No checkpoint " + self.path + '/checkpoint' + " loaded!")
-            return False
-
-    def _save(self):
-        print("Save network parameters in episode ", self.nEpisodes)
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint')
-
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint_' + str(self.nEpisodes))
-
-
-if __name__ == '__main__':
-    try:
-        if len(sys.argv) >= 3:
-            agent = AgentDDPG(name=sys.argv[1])
-            if sys.argv[2] == 'play':
-                agent.play()
-            elif sys.argv[2] == 'train':
-                agent.train()
-            else:
-                print("correct usage: python train.py {name} (play | train) [-cs]")
-        else:
-            print("correct usage: python train.py {name} (play | train) [-cs]")
-    except KeyboardInterrupt:
-        if len(sys.argv) >= 4 and sys.argv[3] == '-cs':
-            agent._save()
\ No newline at end of file
diff --git a/envs/agents/agentVSS/README.md b/envs/agents/agentVSS/README.md
deleted file mode 100644
index c0fa65ac..00000000
--- a/envs/agents/agentVSS/README.md
+++ /dev/null
@@ -1,22 +0,0 @@
-# AgentDDPG
-Based on [RL-Adventure-2 Github Repository](https://github.com/higgsfield/RL-Adventure-2)
-## How to run
-### Training
-```bash
-../rlAdventure2DDPG $ conda activate env
-../rlAdventure2DDPG $ python agentDDPG.py {name} train
-```
-- Training checkpoints will be save in **./runs/_{name}_/**
-- If checkpoint already exists in **./runs/_{name}_/** it will resume training
-
-### Playing
-```bash
-../rlAdventure2DDPG $ conda activate env
-../rlAdventure2DDPG $ python agentDDPG.py {name} play
-```
-- Uses trained network if checkpoint exists in **./runs/_{name}_/**
-- Play with random network if no checkpoint is found
-
-
-# Requirements
-\#TODO
\ No newline at end of file
diff --git a/envs/agents/agentVSS/agentDDPG.py b/envs/agents/agentVSS/agentDDPG.py
deleted file mode 100644
index 7d634b0a..00000000
--- a/envs/agents/agentVSS/agentDDPG.py
+++ /dev/null
@@ -1,227 +0,0 @@
-import gym
-import rc_gym
-import numpy as np
-import os
-import sys
-import time
-
-from torch.utils.tensorboard import SummaryWriter
-import torch.optim as optim
-import torch.nn as nn
-import torch
-
-from agents.Utils.Normalization import NormalizedWrapper
-from agents.Utils.Networks import ValueNetwork, PolicyNetwork
-from agents.Utils.OUNoise import OUNoise
-from agents.Utils.Buffers import ReplayBuffer, AverageBuffer
-
-
-class AgentDDPG:
-
-    def __init__(self, name='DDPG',
-                 maxEpisodes=10000, maxSteps=2000, batchSize=512, replayBufferSize=200000, valueLR=1e-3, policyLR=1e-4,
-                 hiddenDim=256, nEpisodesPerCheckpoint=50):
-        # Training Parameters
-        self.batchSize = batchSize
-        self.maxSteps = maxSteps
-        self.maxEpisodes = maxEpisodes
-        self.nEpisodesPerCheckpoint = nEpisodesPerCheckpoint
-        self.nEpisodes = 0
-
-        # Check if cuda gpu is available, and select it
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-        # Create Environment using a wrapper which scales actions and observations to [-1, 1]
-        self.env = NormalizedWrapper(gym.make("rSimVSS3v3-v0"))
-
-        # Init action noise object
-        self.ouNoise = OUNoise(self.env.action_space)
-
-        # Init networks
-        stateDim = self.env.observation_space.shape[0]
-        actionDim = self.env.action_space.shape[1] #TODO: esta uma gambiarra pois o action_shape tem duas dimensoes
-        self.valueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.policyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        self.targetValueNet = ValueNetwork(stateDim, actionDim, hiddenDim).to(self.device)
-        self.targetPolicyNet = PolicyNetwork(stateDim, actionDim, hiddenDim, device=self.device).to(self.device)
-        # Same initial parameters for target networks
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(param.data)
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(param.data)
-
-        # Init optimizers
-        self.valueOptimizer = optim.Adam(self.valueNet.parameters(), lr=valueLR)
-        self.policyOptimizer = optim.Adam(self.policyNet.parameters(), lr=policyLR)
-
-        # Init replay buffer
-        self.replayBuffer = ReplayBuffer(replayBufferSize)
-
-        # Init goals buffer
-        self.goalsBuffer = AverageBuffer()
-
-        # Init rewars buffer
-        self.rewardsBuffer = AverageBuffer()
-
-        # Tensorboard Init
-        self.path = './runs/' + name
-        self._load()
-        self.writer = SummaryWriter(log_dir=self.path)
-
-    def _update(self, batch_size,
-                gamma=0.99,
-                min_value=-np.inf,
-                max_value=np.inf,
-                soft_tau=1e-2):
-        state, action, reward, next_state, done = self.replayBuffer.sample(batch_size)
-        
-        state = torch.FloatTensor(state).to(self.device)
-        next_state = torch.FloatTensor(next_state).to(self.device)
-        action = torch.FloatTensor(action).to(self.device)
-        action = torch.flatten(action, start_dim=1)
-        reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device)
-        done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device)
-
-        policyLoss = self.valueNet(state, self.policyNet(state))
-        policyLoss = -policyLoss.mean()
-
-        next_action = self.targetPolicyNet(next_state)
-        target_value = self.targetValueNet(next_state, next_action.detach())
-        expected_value = reward + (1.0 - done) * gamma * target_value
-        expected_value = torch.clamp(expected_value, min_value, max_value)
-
-        value = self.valueNet(state, action)
-        value_criterion = nn.MSELoss()
-        value_loss = value_criterion(value, expected_value.detach())
-
-        self.policyOptimizer.zero_grad()
-        policyLoss.backward()
-        self.policyOptimizer.step()
-
-        self.valueOptimizer.zero_grad()
-        value_loss.backward()
-        self.valueOptimizer.step()
-
-        for target_param, param in zip(self.targetValueNet.parameters(), self.valueNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-        for target_param, param in zip(self.targetPolicyNet.parameters(), self.policyNet.parameters()):
-            target_param.data.copy_(
-                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
-            )
-
-    # Training Loop
-    def train(self):
-        while self.nEpisodes < self.maxEpisodes:
-            state = self.env.reset()
-            self.ouNoise.reset()
-            episodeReward = 0
-            nStepsInEpisode = 0
-            stepSeg = -1
-            startTimeInEpisode = time.time()
-
-            print("Episode number:", self.nEpisodes)
-            while nStepsInEpisode < self.maxSteps:
-                action = self.policyNet.get_action(state)
-                action = self.ouNoise.get_action(action, nStepsInEpisode)
-                next_state, reward, done, _ = self.env.step(action)
-                # if self.nEpisodes % 10 == 0:
-                # self.env.unwrapped.render()
-
-                self.replayBuffer.push(state, action, reward, next_state, done)
-                if len(self.replayBuffer) > self.batchSize:
-                    self._update(self.batchSize)
-
-                state = next_state
-                episodeReward += reward
-                nStepsInEpisode += 1
-
-                if done:
-                    # self.goalsBuffer.push(1 if reward < 0 else 0)
-                    break
-            
-            if nStepsInEpisode > 1:
-                stepSeg = nStepsInEpisode/(time.time() - startTimeInEpisode)
-
-            self.rewardsBuffer.push(episodeReward)
-            self.nEpisodes += 1
-
-            self.writer.add_scalar('Train/Reward', episodeReward, self.nEpisodes)
-            self.writer.add_scalar('Train/Steps', nStepsInEpisode, self.nEpisodes)
-            # self.writer.add_scalar('Train/Goals_average_on_{}_previous_episodes'.format(self.goalsBuffer.capacity), self.goalsBuffer.average(), self.nEpisodes)
-            self.writer.add_scalar('Train/Steps_seconds',stepSeg, self.nEpisodes)
-            self.writer.add_scalar('Train/Reward_average_on_{}_previous_episodes'.format(self.rewardsBuffer.capacity), self.rewardsBuffer.average(), self.nEpisodes)
-
-            if (self.nEpisodes % self.nEpisodesPerCheckpoint) == 0:
-                self._save()
-
-        self.writer.flush()
-
-    # Playing loop
-    def play(self):
-        while True:
-            done = False
-            obs = self.env.reset()
-            while not done:
-                action = self.policyNet.get_action(obs)
-                obs, reward, done, _ = self.env.step(action)
-                self.env.unwrapped.render()
-
-    def _load(self):
-        # Check if checkpoint file exists
-        if os.path.exists(self.path + '/checkpoint'):
-            checkpoint = torch.load(self.path + '/checkpoint')
-            # Load networks parameters checkpoint
-            self.valueNet.load_state_dict(checkpoint['valueNetDict'])
-            self.policyNet.load_state_dict(checkpoint['policyNetDict'])
-            self.targetValueNet.load_state_dict(checkpoint['targetValueNetDict'])
-            self.targetPolicyNet.load_state_dict(checkpoint['targetPolicyNetDict'])
-            self.goalsBuffer.load_state_dict(checkpoint['goalsBuffer'])
-            self.rewardsBuffer.load_state_dict(checkpoint['rewardsBuffer'])
-            # Load number of episodes on checkpoint
-            self.nEpisodes = checkpoint['nEpisodes']
-            self.maxEpisodes += checkpoint['nEpisodes']
-            print("Checkpoint with {} episodes successfully loaded".format(self.nEpisodes))
-        else:
-            print("No checkpoint loaded")
-    
-    def _save(self):
-        print("Save network parameters in episode ", self.nEpisodes)
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint')
-
-        torch.save({
-            'valueNetDict': self.valueNet.state_dict(),
-            'policyNetDict': self.targetPolicyNet.state_dict(),
-            'targetValueNetDict': self.targetValueNet.state_dict(),
-            'targetPolicyNetDict': self.targetPolicyNet.state_dict(),
-            'nEpisodes': self.nEpisodes,
-            'goalsBuffer': self.goalsBuffer.state_dict(),
-            'rewardsBuffer': self.rewardsBuffer.state_dict()
-        }, self.path + '/checkpoint_' + str(self.nEpisodes))
-
-
-if __name__ == '__main__':
-    try:
-        if len(sys.argv) >= 3:
-            agent = AgentDDPG(name=sys.argv[1])
-            if sys.argv[2] == 'play':
-                agent.play()
-            elif sys.argv[2] == 'train':
-                agent.train()
-            else:
-                print("correct usage: python agentDDPG.py {name} (play or train) [-cs]")
-        else:
-            print("correct usage: python agentDDPG.py {name} (play or train) [-cs]")
-    except KeyboardInterrupt:
-        if len(sys.argv) >= 4 and sys.argv[3] == '-cs':
-            agent._save()
\ No newline at end of file
diff --git a/envs/agents/sb3/train.py b/envs/agents/sb3/train.py
deleted file mode 100644
index b2c0f989..00000000
--- a/envs/agents/sb3/train.py
+++ /dev/null
@@ -1,92 +0,0 @@
-from stable_baselines3.common.vec_env import SubprocVecEnv
-from stable_baselines3.common.evaluation import evaluate_policy
-from stable_baselines3 import DDPG
-from stable_baselines3.common.noise import OrnsteinUhlenbeckActionNoise
-from stable_baselines3.common.utils import set_random_seed
-from stable_baselines3.common.env_util import make_vec_env
-import rc_gym
-import gym
-import numpy as np
-from tensorboardX import SummaryWriter
-import time
-env_name = 'VSS3v3-v0'
-n_procs = 1
-TRAIN_STEPS = 200000
-
-class NormalizedWrapper(gym.Wrapper):
-    """
-    :param env: (gym.Env) Gym environment that will be wrapped
-    """
-
-    def __init__(self, env):
-        # Call the parent constructor, so we can access self.env later
-        super(NormalizedWrapper, self).__init__(env)
-
-        assert isinstance(self.env.action_space,
-                          gym.spaces.Box), "This wrapper only works with continuous action space (spaces.Box)"
-        assert isinstance(self.env.observation_space,
-                          gym.spaces.Box), "This wrapper only works with continuous observation space (spaces.Box)"
-
-        # We modify the wrapper action space, so all actions will lie in [-1, 1]
-        self.action_space = gym.spaces.Box(low=-1, high=1, shape=self.env.action_space.shape, dtype=np.float32)
-        self.observation_space = gym.spaces.Box(low=-1, high=1, shape=self.env.observation_space.shape, dtype=np.float32)
-
-
-    def rescale_action(self, scaled_action):
-        """
-        Rescale the action from [-1, 1] to [low, high]
-        (no need for symmetric action space)
-        :param scaled_action: (np.ndarray)
-        :return: (np.ndarray)
-        """
-        return self.env.action_space.low + (
-                0.5 * (scaled_action + 1.0) * (self.env.action_space.high - self.env.action_space.low))
-
-    def scale_observation(self, observation):
-        """
-        Scale the observation to bounds [-1, 1]
-        """
-        return (2 * ((observation - self.env.observation_space.low) /
-                     (self.env.observation_space.high - self.env.observation_space.low))) - 1
-
-    def reset(self):
-        """
-        Reset the environment
-        """
-        # Reset the counter
-        return self.scale_observation(self.env.reset())
-
-    def step(self, action):
-        """
-        :param action: ([float] or int) Action taken by the agent
-        :return: (np.ndarray, float, bool, dict) observation, reward, is the episode over?, additional informations
-        """
-        # Rescale action from [-1, 1] to original [low, high] interval
-        rescaled_action = self.rescale_action(action)
-        obs, reward, done, info = self.env.step(rescaled_action)
-
-        return self.scale_observation(obs), reward, done, info
-
-writer = SummaryWriter(log_dir="log/ddpg", comment="-agent")
-eval_env = gym.make(env_name)
-
-n_actions = eval_env.action_space.shape[-1]
-action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
-
-train_env = make_vec_env(rc_gym.vss.env_3v3.vss_gym_3v3.VSS3v3Env, n_envs=n_procs, wrapper_class=NormalizedWrapper, vec_env_cls=SubprocVecEnv, vec_env_kwargs={'start_method': 'fork'})
-model = DDPG('MlpPolicy', train_env, verbose=1, action_noise=action_noise)
-try:
-    steps = 0
-    while True:
-        train_env.reset()
-        mean_reward, _  = evaluate_policy(model, eval_env, n_eval_episodes=10)
-        print("steps {}, evaluation mean_rewards: {:.6f}".format(steps, mean_reward))
-        writer.add_scalar("eval/mean_rewards", mean_reward, steps)
-        seconds_start = time.perf_counter()
-        model.learn(total_timesteps=TRAIN_STEPS, )
-        steps += TRAIN_STEPS
-        writer.add_scalar("learn/steps_s", TRAIN_STEPS / (time.perf_counter() - seconds_start), steps)
-        model.save("model/ddpg")
-except KeyboardInterrupt:
-    train_env.close()
-    eval_env.close()
diff --git a/envs/rc_gym/Entities/Frame.py b/envs/rc_gym/Entities/Frame.py
index 4c52f473..059240bf 100644
--- a/envs/rc_gym/Entities/Frame.py
+++ b/envs/rc_gym/Entities/Frame.py
@@ -2,16 +2,15 @@
 from typing import Dict
 from rc_gym.Entities.Ball import Ball
 from rc_gym.Entities.Robot import Robot
-from dataclasses import dataclass
 
 class Frame:
     """Units: seconds, m, m/s, degrees, degrees/s. Reference is field center."""
     
     def __init__(self):
         """Init Frame object."""
-        self.ball = Ball()
-        self.robots_blue = {}
-        self.robots_yellow = {}
+        self.ball: Ball = Ball()
+        self.robots_blue: Dict[int, Robot] = {}
+        self.robots_yellow: Dict[int, Robot] = {}
 
 class FrameVSS(Frame):
     def parse(self, state, n_robots_blue=3, n_robots_yellow=3):
diff --git a/envs/rc_gym/Simulators/rsim.py b/envs/rc_gym/Simulators/rsim.py
new file mode 100644
index 00000000..111a51fa
--- /dev/null
+++ b/envs/rc_gym/Simulators/rsim.py
@@ -0,0 +1,164 @@
+import robosim
+import numpy as np
+from typing import Dict, List
+from rc_gym.Entities import Frame, FrameVSS, FrameSSL
+
+
+class RSim:
+    def __init__(self, field_type: int, n_robots_blue: int,
+                 n_robots_yellow: int, time_step_ms: int):
+        self.n_robots_blue = n_robots_blue
+        self.n_robots_yellow = n_robots_yellow
+
+        # Positions needed just to initialize the simulator
+        ball_pos = [0, 0, 0, 0]
+        blue_robots_pos = [[-0.2 * i, 0, 0]
+                           for i in range(1, n_robots_blue + 1)]
+        yellow_robots_pos = [[0.2 * i, 0, 0]
+                             for i in range(1, n_robots_yellow + 1)]
+
+        self.simulator = self._init_simulator(field_type=field_type,
+                                              n_robots_blue=n_robots_blue,
+                                              n_robots_yellow=n_robots_yellow,
+                                              ball_pos=ball_pos,
+                                              blue_robots_pos=blue_robots_pos,
+                                              yellow_robots_pos=yellow_robots_pos,
+                                              time_step_ms=time_step_ms)
+
+    def reset(self, frame: Frame):
+        placement_pos = self._placement_dict_from_frame(frame)
+        self.simulator.reset(**placement_pos)
+
+    def stop(self):
+        del(self.simulator)
+
+    def send_commands(self, commands):
+        raise NotImplementedError
+
+    def get_frame(self) -> Frame:
+        raise NotImplementedError
+
+    def get_field_params(self):
+        return self.simulator.get_field_params()
+    
+    def _placement_dict_from_frame(self, frame: Frame):
+        replacement_pos: Dict[str, np.ndarray] = {}
+
+        ball_pos: List[float] = [frame.ball.x, frame.ball.y,
+                                 frame.ball.v_x, frame.ball.v_y]
+        replacement_pos['ball_pos'] = np.array(ball_pos)
+
+        blue_pos: List[List[float]] = []
+        for robot in frame.robots_blue.values():
+            robot_pos: List[float] = [robot.x, robot.y, robot.theta]
+            blue_pos.append(robot_pos)
+        replacement_pos['blue_robots_pos'] = np.array(blue_pos)
+
+        yellow_pos: List[List[float]] = []
+        for robot in frame.robots_yellow.values():
+            robot_pos: List[float] = [robot.x, robot.y, robot.theta]
+            yellow_pos.append(robot_pos)
+        replacement_pos['yellow_robots_pos'] = np.array(yellow_pos)
+
+        return replacement_pos
+
+    def _init_simulator(self, field_type, n_robots_blue, n_robots_yellow, 
+                        ball_pos, blue_robots_pos, yellow_robots_pos,
+                        time_step_ms):
+        raise NotImplementedError
+
+class RSimVSS(RSim):
+    def __init__(self, field_type: int, n_robots_blue: int,
+                 n_robots_yellow: int, time_step_ms: int):
+        super().__init__(field_type=field_type, n_robots_blue=n_robots_blue,
+                         n_robots_yellow=n_robots_yellow, time_step_ms=time_step_ms)
+
+        self.linear_speed_range = 1.2  # m/s
+        self.robot_wheel_radius = 0.026
+
+    def send_commands(self, commands):
+        sim_commands = np.zeros(
+            (self.n_robots_blue + self.n_robots_yellow, 2), dtype=np.float64)
+
+        for cmd in commands:
+            if cmd.yellow:
+                rbt_id = self.n_robots_blue + cmd.id
+            else:
+                rbt_id = cmd.id
+            # convert from linear speed to angular speed
+            sim_commands[rbt_id][0] = cmd.v_wheel1 / self.robot_wheel_radius
+            sim_commands[rbt_id][1] = cmd.v_wheel2 / self.robot_wheel_radius
+        self.simulator.step(sim_commands)
+
+    def get_frame(self) -> FrameVSS:
+        state = self.simulator.get_state()
+        # Update frame with new state
+        frame = FrameVSS()
+        frame.parse(state, self.n_robots_blue, self.n_robots_yellow)
+
+        return frame
+
+    def _init_simulator(self, field_type, n_robots_blue, n_robots_yellow,
+                        ball_pos, blue_robots_pos, yellow_robots_pos,
+                        time_step_ms):
+
+        return robosim.SimulatorVSS(
+            field_type=field_type,
+            n_robots_blue=n_robots_blue,
+            n_robots_yellow=n_robots_yellow,
+            ball_pos=ball_pos,
+            blue_robots_pos=blue_robots_pos,
+            yellow_robots_pos=yellow_robots_pos,
+            time_step_ms=time_step_ms
+            )
+
+class RSimSSL(RSim):
+    def __init__(self, field_type: int, n_robots_blue: int,
+                 n_robots_yellow: int, time_step_ms: int):
+        super().__init__(field_type=field_type, n_robots_blue=n_robots_blue,
+                         n_robots_yellow=n_robots_yellow, time_step_ms=time_step_ms)
+
+        self.linear_speed_range = 1  # m/s
+        self.angular_speed_range = 10  # rad/s
+
+    def send_commands(self, commands):
+        sim_commands = np.zeros(
+            (self.n_robots_blue + self.n_robots_yellow, 6), dtype=np.float64)
+
+        for cmd in commands:
+            if cmd.yellow:
+                rbt_id = self.n_robots_blue + cmd.id
+            else:
+                rbt_id = cmd.id
+            # convert from linear speed to angular speed
+            sim_commands[rbt_id][0] = cmd.v_x
+            sim_commands[rbt_id][1] = cmd.v_y
+            # TODO checar se a velocidade é rad ou deg/s
+            sim_commands[rbt_id][2] = cmd.v_theta * self.angular_speed_range
+            sim_commands[rbt_id][3] = cmd.kick_v_x
+            sim_commands[rbt_id][4] = cmd.kick_v_z
+            sim_commands[rbt_id][5] = cmd.dribbler
+            
+        self.simulator.step(sim_commands)
+
+    def get_frame(self) -> FrameSSL:
+        state = self.simulator.get_state()
+        # Update frame with new state
+        frame = FrameSSL()
+        frame.parse(state, self.n_robots_blue, self.n_robots_yellow)
+
+        return frame
+
+    def _init_simulator(self, field_type, n_robots_blue, n_robots_yellow,
+                        ball_pos, blue_robots_pos, yellow_robots_pos,
+                        time_step_ms):
+
+        return robosim.SimulatorSSL(
+            field_type=field_type,
+            n_robots_blue=n_robots_blue,
+            n_robots_yellow=n_robots_yellow,
+            ball_pos=ball_pos,
+            blue_robots_pos=blue_robots_pos,
+            yellow_robots_pos=yellow_robots_pos,
+            time_step_ms=time_step_ms
+            )
diff --git a/envs/rc_gym/Simulators/rsim_base.py b/envs/rc_gym/Simulators/rsim_base.py
deleted file mode 100644
index b5413a76..00000000
--- a/envs/rc_gym/Simulators/rsim_base.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import robosim
-import numpy as np
-from typing import Dict, List
-from rc_gym.Entities import Frame
-
-
-class RSim:
-    def __init__(self, field_type: int, n_robots_blue: int,
-                 n_robots_yellow: int, time_step_ms: int):
-        self.n_robots_blue = n_robots_blue
-        self.n_robots_yellow = n_robots_yellow
-
-        # Positions needed just to initialize the simulator
-        ball_pos = [0, 0, 0, 0]
-        blue_robots_pos = [[-0.2 * i, 0, 0]
-                           for i in range(1, n_robots_blue + 1)]
-        yellow_robots_pos = [[0.2 * i, 0, 0]
-                             for i in range(1, n_robots_yellow + 1)]
-
-        self.simulator = self._init_simulator(field_type=field_type,
-                                              n_robots_blue=n_robots_blue,
-                                              n_robots_yellow=n_robots_yellow,
-                                              ball_pos=ball_pos,
-                                              blue_robots_pos=blue_robots_pos,
-                                              yellow_robots_pos=yellow_robots_pos,
-                                              time_step_ms=time_step_ms)
-
-    def reset(self, frame: Frame):
-        placement_pos = self._placement_dict_from_frame(frame)
-        self.simulator.reset(**placement_pos)
-
-    def stop(self):
-        del(self.simulator)
-
-    def send_commands(self, commands):
-        raise NotImplementedError
-
-    def get_frame(self) -> Frame:
-        raise NotImplementedError
-
-    def get_field_params(self):
-        return self.simulator.get_field_params()
-    
-    def _placement_dict_from_frame(self, frame: Frame):
-        replacement_pos: Dict[str, np.ndarray] = {}
-
-        ball_pos: List[float] = [frame.ball.x, frame.ball.y,
-                                 frame.ball.v_x, frame.ball.v_y]
-        replacement_pos['ball_pos'] = np.array(ball_pos)
-
-        blue_pos: List[List[float]] = []
-        for robot in frame.robots_blue.values():
-            robot_pos: List[float] = [robot.x, robot.y, robot.theta]
-            blue_pos.append(robot_pos)
-        replacement_pos['blue_robots_pos'] = np.array(blue_pos)
-
-        yellow_pos: List[List[float]] = []
-        for robot in frame.robots_yellow.values():
-            robot_pos: List[float] = [robot.x, robot.y, robot.theta]
-            yellow_pos.append(robot_pos)
-        replacement_pos['yellow_robots_pos'] = np.array(yellow_pos)
-
-        return replacement_pos
-
-    def _init_simulator(self, field_type, n_robots_blue, n_robots_yellow, 
-                        ball_pos, blue_robots_pos, yellow_robots_pos,
-                        time_step_ms):
-        raise NotImplementedError
diff --git a/envs/rc_gym/Simulators/rsim_ssl.py b/envs/rc_gym/Simulators/rsim_ssl.py
deleted file mode 100644
index 49a0ca37..00000000
--- a/envs/rc_gym/Simulators/rsim_ssl.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import robosim
-import numpy as np
-from typing import Dict, List
-from rc_gym.Entities import FrameSSL
-from rc_gym.Simulators.rsim_base import RSim
-
-
-class RSimSSL(RSim):
-    def __init__(self, field_type: int, n_robots_blue: int,
-                 n_robots_yellow: int, time_step_ms: int):
-        super().__init__(field_type=field_type, n_robots_blue=n_robots_blue,
-                         n_robots_yellow=n_robots_yellow, time_step_ms=time_step_ms)
-
-        self.linear_speed_range = 1.15  # m/s
-
-    def send_commands(self, commands):
-        sim_commands = np.zeros(
-            (self.n_robots_blue + self.n_robots_yellow, 6), dtype=np.float64)
-
-        for cmd in commands:
-            if cmd.yellow:
-                rbt_id = self.n_robots_blue + cmd.id
-            else:
-                rbt_id = cmd.id
-            # convert from linear speed to angular speed
-            sim_commands[rbt_id][0] = cmd.v_x
-            sim_commands[rbt_id][1] = cmd.v_y
-            sim_commands[rbt_id][2] = cmd.v_theta
-            sim_commands[rbt_id][3] = cmd.kick_v_x
-            sim_commands[rbt_id][4] = cmd.kick_v_z
-            sim_commands[rbt_id][5] = cmd.dribbler
-            
-        self.simulator.step(sim_commands)
-
-    def get_frame(self) -> FrameSSL:
-        state = self.simulator.get_state()
-        # Update frame with new state
-        frame = FrameSSL()
-        frame.parse(state, self.n_robots_blue, self.n_robots_yellow)
-
-        return frame
-
-    def _init_simulator(self, field_type, n_robots_blue, n_robots_yellow,
-                        ball_pos, blue_robots_pos, yellow_robots_pos,
-                        time_step_ms):
-
-        return robosim.SimulatorSSL(
-            field_type=field_type,
-            n_robots_blue=n_robots_blue,
-            n_robots_yellow=n_robots_yellow,
-            ball_pos=ball_pos,
-            blue_robots_pos=blue_robots_pos,
-            yellow_robots_pos=yellow_robots_pos,
-            time_step_ms=time_step_ms
-            )
diff --git a/envs/rc_gym/Simulators/rsim_vss.py b/envs/rc_gym/Simulators/rsim_vss.py
deleted file mode 100644
index f85be9d4..00000000
--- a/envs/rc_gym/Simulators/rsim_vss.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import robosim
-import numpy as np
-from typing import Dict, List
-from rc_gym.Entities import FrameVSS
-from rc_gym.Simulators.rsim_base import RSim
-
-
-class RSimVSS(RSim):
-    def __init__(self, field_type: int, n_robots_blue: int,
-                 n_robots_yellow: int, time_step_ms: int):
-        super().__init__(field_type=field_type, n_robots_blue=n_robots_blue,
-                         n_robots_yellow=n_robots_yellow, time_step_ms=time_step_ms)
-
-        self.linear_speed_range = 1.15  # m/s
-        self.robot_wheel_radius = 0.026
-
-    def send_commands(self, commands):
-        sim_commands = np.zeros(
-            (self.n_robots_blue + self.n_robots_yellow, 2), dtype=np.float64)
-
-        for cmd in commands:
-            if cmd.yellow:
-                rbt_id = self.n_robots_blue + cmd.id
-            else:
-                rbt_id = cmd.id
-            # convert from linear speed to angular speed
-            sim_commands[rbt_id][0] = cmd.v_wheel1 / self.robot_wheel_radius
-            sim_commands[rbt_id][1] = cmd.v_wheel2 / self.robot_wheel_radius
-        self.simulator.step(sim_commands)
-
-    def get_frame(self) -> FrameVSS:
-        state = self.simulator.get_state()
-        # Update frame with new state
-        frame = FrameVSS()
-        frame.parse(state, self.n_robots_blue, self.n_robots_yellow)
-
-        return frame
-
-    def _init_simulator(self, field_type, n_robots_blue, n_robots_yellow,
-                        ball_pos, blue_robots_pos, yellow_robots_pos,
-                        time_step_ms):
-
-        return robosim.SimulatorVSS(
-            field_type=field_type,
-            n_robots_blue=n_robots_blue,
-            n_robots_yellow=n_robots_yellow,
-            ball_pos=ball_pos,
-            blue_robots_pos=blue_robots_pos,
-            yellow_robots_pos=yellow_robots_pos,
-            time_step_ms=time_step_ms
-            )
diff --git a/envs/rc_gym/Utils/Utils.py b/envs/rc_gym/Utils/Utils.py
index 1cd0f1bb..4736428d 100644
--- a/envs/rc_gym/Utils/Utils.py
+++ b/envs/rc_gym/Utils/Utils.py
@@ -1,75 +1,6 @@
-import math
 import numpy as np
 import gym
 
-
-# Math methods
-# ----------------------------
-def distance(pointA, pointB):
-
-    x1 = pointA[0]
-    y1 = pointA[1]
-
-    x2 = pointB[0]
-    y2 = pointB[1]
-
-    distance = np.sqrt((x1 - x2)**2 + (y1 - y2)**2)
-
-    return distance
-
-
-def mod(x, y):
-    return (math.sqrt(x*x + y*y))
-
-
-def angle(x, y):
-    return math.atan2(y, x)
-
-
-def toPositiveAngle(angle):
-    return math.fmod(angle + 2 * math.pi, 2 * math.pi)
-
-
-def smallestAngleDiff(target, source):
-    a = toPositiveAngle(target) - toPositiveAngle(source)
-
-    if a > math.pi:
-        a = a - 2 * math.pi
-    elif a < -math.pi:
-        a = a + 2 * math.pi
-
-    return a
-
-
-def to_pi_range(angle):
-    angle = math.fmod(angle, 2 * math.pi)
-    if angle < -math.pi:
-        angle = angle + 2 * math.pi
-    elif angle > math.pi:
-        angle = angle - 2 * math.pi
-
-    return angle
-
-
-def clip(val, vmin, vmax):
-    return min(max(val, vmin), vmax)
-
-
-def normX(x):
-    return clip(x / 0.85, -1.2, 1.2)
-
-
-def normVx(v_x):
-    return clip(v_x / 0.8, -1.25, 1.25)
-
-
-def normVt(vt):
-    return clip(vt / 573, -1.2, 1.2)
-
-
-def roundTo5(x, base=5):
-    return int(base * round(float(x) / base))
-
 # Base on baselines implementation
 class OrnsteinUhlenbeckAction(object):
     def __init__(self, action_space, theta=.17, dt=0.032, x0=None):
diff --git a/envs/rc_gym/__init__.py b/envs/rc_gym/__init__.py
index 9b37498c..704ce23b 100644
--- a/envs/rc_gym/__init__.py
+++ b/envs/rc_gym/__init__.py
@@ -1,25 +1,13 @@
 from gym.envs.registration import register
 
-register(id='grSimSSLPenalty-v0',
-         entry_point='rc_gym.grsim_ssl:GrSimSSLPenaltyEnv'
-         )
-
-register(id='grSimSSLShootGoalie-v0',
-         entry_point='rc_gym.grsim_ssl:shootGoalieEnv'
-         )
-
-register(id='grSimSSLGoToBall-v0',
-         entry_point='rc_gym.grsim_ssl:goToBallEnv'
-         )
-
 register(id='VSS3v3-v0',
-         entry_point='rc_gym.vss:VSS3v3Env'
+         entry_point='rc_gym.vss.env_3v3:VSS3v3Env'
          )
 
 register(id='VSSMA-v0',
-         entry_point='rc_gym.vss:VSSMAEnv',
+         entry_point='rc_gym.vss.env_ma:VSSMAEnv',
          )
 
 register(id='VSSMAOpp-v0',
-         entry_point='rc_gym.vss:VSSMAOpp',
+         entry_point='rc_gym.vss.env_ma:VSSMAOpp',
          )
diff --git a/envs/rc_gym/grsim_ssl/Communication/grSimClient.py b/envs/rc_gym/grsim_ssl/Communication/grSimClient.py
deleted file mode 100644
index 1dfcf349..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/grSimClient.py
+++ /dev/null
@@ -1,108 +0,0 @@
-'''
-#  Center all packets communication:
-#   - Vision (receives from grSim env) (receives ssl-vision packet + vx vy v_theta)
-
-'''
-
-
-import socket
-import rc_gym.grsim_ssl.Communication.pb.messages_robocup_ssl_wrapper_pb2 as wrapper_pb2
-import rc_gym.grsim_ssl.Communication.pb.grSim_Packet_pb2 as packet_pb2
-from rc_gym.Entities import Robot, Ball, FramePB
-
-class grSimClient:
-
-    def __init__(self, visionIp='224.0.0.1', commandIp='127.0.0.1', visionPort=10020, commandPort=20011):
-        """Init grSimClient object."""
-
-        self.visionIp = visionIp
-        self.commandIp = commandIp
-        self.visionPort = visionPort
-        self.commandPort = commandPort
-
-        # Connect vision and command sockets
-        self.visionSocket = socket.socket(
-            socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
-        self.visionSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        self.visionSocket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 128)
-        self.visionSocket.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)
-
-        self.commandSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        self.visionSocket.bind((self.visionIp, self.visionPort))
-        self.commandAddress = (self.commandIp, self.commandPort)
-
-        
-
-#-------------------------------------------------------------------------    
-
-    def receiveState(self):
-        """Receive SSL wrapper package and decode."""
-        data, _ = self.visionSocket.recvfrom(1024)
-        decoded_data = wrapper_pb2.SSL_WrapperPacket().FromString(data)
-        
-        frame = FramePB()
-        frame.parse(decoded_data)
-
-        return frame
-
-#-------------------------------------------------------------------------    
-    
-
-    def sendCommandsPacket(self, commands):
-        packet = self._fillCommandPacket(commands)
-        
-        """Sends packet to grSim"""
-        data = packet.SerializeToString()
-
-        self.commandSocket.sendto(data, self.commandAddress)
-
-
-    def _fillCommandPacket(self, commands):
-        packet = packet_pb2.grSim_Packet()
-        grSimCommands = packet.commands
-        grSimCommands.timestamp = 0.0
-        grSimRobotCommand = grSimCommands.robot_commands
-        for robotCommand in commands:
-            rbt = grSimRobotCommand.add()
-            rbt.isteamyellow = robotCommand.yellow
-            rbt.id = robotCommand.id
-            rbt.kickspeedx = robotCommand.kick_v_x
-            rbt.kickspeedz = robotCommand.kick_v_z
-            rbt.veltangent = robotCommand.v_x
-            rbt.velnormal = robotCommand.v_y
-            rbt.velangular = robotCommand.v_theta
-            rbt.spinner = robotCommand.dribbler
-            rbt.wheelsspeed = robotCommand.wheel_speed
-        return packet
-
-#-------------------------------------------------------------------------    
-
-    def sendReplacementPacket(self, robotPositions = None, ballPosition = None):
-        packet = self._fillReplacementPacket(robotPositions, ballPosition)
-        """Sends packet to grSim"""
-        data = packet.SerializeToString()
-
-        self.commandSocket.sendto(data, self.commandAddress)
-
-
-    def _fillReplacementPacket(self, robotPositions = None, ballPosition = None):
-        packet = packet_pb2.grSim_Packet()
-        grSimReplacement = packet.replacement
-        
-        if ballPosition != None:
-            grSimBall = grSimReplacement.ball
-            grSimBall.x = ballPosition.x
-            grSimBall.y = ballPosition.y
-            grSimBall.vx = ballPosition.v_x
-            grSimBall.vy = ballPosition.v_y
-        
-        if robotPositions != None:
-            grSimRobot = grSimReplacement.robots
-            for rbtPosition in robotPositions:
-                rbt = grSimRobot.add()
-                rbt.yellowteam = rbtPosition.yellow
-                rbt.id = rbtPosition.id
-                rbt.x = rbtPosition.x
-                rbt.y = rbtPosition.y
-                rbt.dir = rbtPosition.theta
-        return packet
\ No newline at end of file
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Commands.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Commands.proto
deleted file mode 100644
index 25fc1326..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Commands.proto
+++ /dev/null
@@ -1,23 +0,0 @@
-syntax = "proto2";
-
-message grSim_Robot_Command {
-    required bool isteamyellow = 1;
-    required uint32 id = 2;
-    required float kickspeedx = 3;
-    required float kickspeedz = 4;
-    required float veltangent = 5;
-    required float velnormal  = 6;
-    required float velangular = 7;
-    required bool spinner = 8;
-    required bool wheelsspeed = 9;
-    optional float wheel1 = 10;
-    optional float wheel2 = 11;
-    optional float wheel3 = 12;
-    optional float wheel4 = 13;
-}
-
-message grSim_Commands {
-    required double timestamp = 1;
-    repeated grSim_Robot_Command robot_commands = 2;
-}
-
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Packet.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Packet.proto
deleted file mode 100644
index 3eead2fb..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Packet.proto
+++ /dev/null
@@ -1,15 +0,0 @@
-
-// **********************************************
-//
-// Message sent to grSim (a robot command or a replacement)
-//
-// **********************************************
-
-syntax = "proto2";
-
-import "grSim_Commands.proto";
-import "grSim_Replacement.proto";
-message grSim_Packet {
-    optional grSim_Commands commands = 1;
-    optional grSim_Replacement replacement = 2;
-}
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Replacement.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Replacement.proto
deleted file mode 100644
index 0c7d0258..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/grSim_Replacement.proto
+++ /dev/null
@@ -1,22 +0,0 @@
-syntax = "proto2";
-
-message grSim_RobotReplacement {
-required double x=1;
-required double y=2;
-required double dir=3;
-required uint32 id=4;
-required bool yellowteam=5;
-optional bool turnon=6;
-}
-
-message grSim_BallReplacement {
-optional double x=1;
-optional double y=2;
-optional double vx=3;
-optional double vy=4;
-}
-
-message grSim_Replacement {
-optional grSim_BallReplacement ball=1;
-repeated grSim_RobotReplacement robots=2;
-}
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_detection.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_detection.proto
deleted file mode 100644
index acddeeb0..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_detection.proto
+++ /dev/null
@@ -1,38 +0,0 @@
-syntax = "proto2";
-
-message SSL_DetectionBall {
-  required float  confidence = 1;
-  optional uint32 area       = 2;
-  required float  x          = 3;
-  required float  y          = 4;
-  optional float  z          = 5;
-  required float  pixel_x    = 6;
-  required float  pixel_y    = 7;
-  required float vx          = 8;
-  required float vy          = 9;
-  required float vz          = 10;
-}
-
-message SSL_DetectionRobot {
-  required float  confidence  =  1;
-  optional uint32 robot_id    =  2;
-  required float  x           =  3;
-  required float  y           =  4;
-  optional float  orientation =  5;
-  required float  pixel_x     =  6;
-  required float  pixel_y     =  7;
-  optional float  height      =  8;
-  required float vx           = 9;
-  required float vy           = 10;
-  required float vorientation = 11;
-}
-
-message SSL_DetectionFrame {
-  required uint32             frame_number  = 1;
-  required double             t_capture     = 2;
-  required double             t_sent        = 3;
-  required uint32             camera_id     = 4;
-  repeated SSL_DetectionBall  balls         = 5;
-  repeated SSL_DetectionRobot robots_yellow = 6;
-  repeated SSL_DetectionRobot robots_blue   = 7;
-}
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_geometry.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_geometry.proto
deleted file mode 100644
index 6a0b9912..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_geometry.proto
+++ /dev/null
@@ -1,72 +0,0 @@
-syntax = "proto2";
-
-// A 2D float vector.
-message Vector2f {
-  required float x = 1;
-  required float y = 2;
-}
-
-// Represents a field marking as a line segment represented by a start point p1,
-// and end point p2, and a line thickness. The start and end points are along
-// the center of the line, so the thickness of the line extends by thickness / 2
-// on either side of the line.
-message SSL_FieldLineSegment {
-  // Name of this field marking.
-  required string name = 1;
-  // Start point of the line segment.
-  required Vector2f p1 = 2;
-  // End point of the line segment.
-  required Vector2f p2 = 3;
-  // Thickness of the line segment.
-  required float thickness = 4;
-}
-
-// Represents a field marking as a circular arc segment represented by center point, a
-// start angle, an end angle, and an arc thickness.
-message SSL_FieldCicularArc {
-  // Name of this field marking.
-  required string name = 1;
-  // Center point of the circular arc.
-  required Vector2f center = 2;
-  // Radius of the arc.
-  required float radius = 3;
-  // Start angle in counter-clockwise order.
-  required float a1 = 4;
-  // End angle in counter-clockwise order.
-  required float a2 = 5;
-  // Thickness of the arc.
-  required float thickness = 6;
-}
-
-message SSL_GeometryFieldSize {
-  required int32 field_length = 1;
-  required int32 field_width = 2;
-  required int32 goal_width = 3;
-  required int32 goal_depth = 4;
-  required int32 boundary_width = 5;
-  repeated SSL_FieldLineSegment field_lines = 6;
-  repeated SSL_FieldCicularArc field_arcs = 7;
-}
-
-message SSL_GeometryCameraCalibration {
-  required uint32 camera_id     = 1;
-  required float focal_length = 2;
-  required float principal_point_x = 3;
-  required float principal_point_y = 4;
-  required float distortion = 5;
-  required float q0 = 6;
-  required float q1 = 7;
-  required float q2 = 8;
-  required float q3 = 9;
-  required float tx = 10;
-  required float ty = 11;
-  required float tz = 12;
-  optional float derived_camera_world_tx = 13;
-  optional float derived_camera_world_ty = 14;
-  optional float derived_camera_world_tz = 15;
-}
-
-message SSL_GeometryData {
-  required SSL_GeometryFieldSize field = 1;
-  repeated SSL_GeometryCameraCalibration calib = 2;
-}
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_refbox_log.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_refbox_log.proto
deleted file mode 100644
index 8a437bce..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_refbox_log.proto
+++ /dev/null
@@ -1,14 +0,0 @@
-syntax = "proto2";
-
-import "messages_robocup_ssl_detection.proto";
-
-message Log_Frame
-{
-    required SSL_DetectionFrame frame = 1;
-    required string refbox_cmd = 2;
-}
-
-message Refbox_Log
-{
-    repeated Log_Frame log = 1;
-}
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_robot_status.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_robot_status.proto
deleted file mode 100644
index 328a0d90..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_robot_status.proto
+++ /dev/null
@@ -1,12 +0,0 @@
-syntax = "proto2";
-
-message Robots_Status{
-  repeated Robot_Status robots_status = 1;
-}
-
-message Robot_Status {
-  required int32 robot_id  = 1;
-  required bool infrared   = 2;
-  required bool flat_kick  = 3;
-  required bool chip_kick  = 4;
-}
\ No newline at end of file
diff --git a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_wrapper.proto b/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_wrapper.proto
deleted file mode 100644
index 17087593..00000000
--- a/envs/rc_gym/grsim_ssl/Communication/pb/proto/messages_robocup_ssl_wrapper.proto
+++ /dev/null
@@ -1,17 +0,0 @@
-
-// **********************************************
-//
-// Message from grSim with geometry and detection
-//
-// **********************************************
-
-
-syntax = "proto2";
-
-import "messages_robocup_ssl_detection.proto";
-import "messages_robocup_ssl_geometry.proto";
-
-message SSL_WrapperPacket {
-  optional SSL_DetectionFrame detection = 1;
-  optional SSL_GeometryData geometry = 2;
-}
diff --git a/envs/rc_gym/grsim_ssl/GoToBallEnv/__init__.py b/envs/rc_gym/grsim_ssl/GoToBallEnv/__init__.py
deleted file mode 100644
index a159fe67..00000000
--- a/envs/rc_gym/grsim_ssl/GoToBallEnv/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-from rc_gym.grsim_ssl.GoToBallEnv.goToBallState import *
diff --git a/envs/rc_gym/grsim_ssl/GoToBallEnv/goToBallEnv.py b/envs/rc_gym/grsim_ssl/GoToBallEnv/goToBallEnv.py
deleted file mode 100644
index 6944b12a..00000000
--- a/envs/rc_gym/grsim_ssl/GoToBallEnv/goToBallEnv.py
+++ /dev/null
@@ -1,153 +0,0 @@
-import gym
-from math import *
-import numpy as np
-import time
-import random
-
-from rc_gym.grsim_ssl.grSimSSL_env import GrSimSSLEnv
-from rc_gym.grsim_ssl.Communication.grSimClient import grSimClient
-from rc_gym.Entities import Robot, Ball, Frame
-from rc_gym.grsim_ssl.GoToBallEnv import goToBallState
-from rc_gym.Utils import *
-
-class goToBallEnv(GrSimSSLEnv):
-  """
-  Using cartpole env description as base example for our documentation
-  Description:
-      # TODO
-  Source:
-      # TODO
-
-  Observation:
-      Type: Box(14)
-      Num     Observation                                       Min                     Max
-      0       Ball X   (mm)                                   -7000                   7000
-      1       Ball Y   (mm)                                   -6000                   6000
-      2       Blue id 0 Vx  (mm/s)                            -10000                  10000
-      3       Blue id 0 Vy  (mm/s)                            -10000                  10000
-      4       Blue id 0 Robot Vw       (rad/s)                -math.pi * 3            math.pi * 3
-      5       Dist Blue id0 - ball (mm)                       -10000                  10000
-      6       Position Wall Top x                             -10000                  10000
-      7       Position Wall Top y                             -10000                  10000
-      8       Position Wall Bottom x                          -10000                  10000
-      9       Position Wall Bottom y                          -10000                  10000
-      10      Position Wall Left x                            -10000                  10000
-      11      Position Wall Left y                            -10000                  10000
-      12      Position Wall Right x                           -10000                  10000
-      13      Position Wall Right y                           -10000                  10000
-      
-
-  Actions:
-      Type: Box(2)
-      Num     Action                        Min                     Max
-      0       Blue id 0 Vx                  -10                      10
-      1       Blue id 0 Vy                  -10                      10
-      2       Blue id 0 Vw (rad/s)        -math.pi * 3            math.pi * 3
-  Reward:
-      Reward is 1 for success, -1 to fails. 0 otherwise.
-
-  Starting State:
-      All observations are assigned a uniform random value in [-0.05..0.05]
-      # TODO
-
-  Episode Termination:
-    # TODO
-  """
-  def __init__(self):
-    super().__init__()
-    ## Action Space
-    actSpaceThresholds = np.array([10, 10, math.pi * 3], dtype=np.float32)
-    self.action_space = gym.spaces.Box(low=-actSpaceThresholds, high=actSpaceThresholds)
-
-    # Observation Space thresholds
-    obsSpaceThresholds = np.array([7000, 6000, 10000, 10000, math.pi * 3, 10000, 10000, 10000,
-                                   10000, 10000, 10000, 10000, 10000, 10000], dtype=np.float32)
-    self.observation_space = gym.spaces.Box(low=-obsSpaceThresholds, high=obsSpaceThresholds)
-    self.goToballState = None
-    self.distAnt = 1000
-    self.angleAnt = 0
-    self.timestampAnt = 0
-    print('Environment initialized')
-  
-  def _getCommands(self, actions):
-    commands = []
-    cmdAttacker = Robot(id=0, yellow=False, v_x=actions[0], v_y=actions[1], v_theta=actions[2])
-    # cmdAttacker = Robot(id=0, yellow=False, v_theta=0, kick_v_x=0, dribbler=True)
-    
-    commands.append(cmdAttacker)
-
-    # TODO GOALIE
-    # cmdAttacker = self._getAttackerCommand()
-
-    # commands.append(cmdAttacker)
-
-    return commands
-
-  def _parseObservationFromState(self):
-    observation = []
-
-    self.goToBallState = goToBallState()
-    observation = self.goToBallState.getObservation(self.state)
-
-    return np.array(observation)
-
-  def _getFormation(self):
-    #To CHANGE: 
-    # ball penalty position
-    #ball = Ball(x=random.uniform(-4, 0), y=random.uniform(-4, 4), v_x=0, v_y=0)
-    ball = Ball(x=random.uniform(-4.5,-3.5), y=random.uniform(-1.5, 1.5), v_x=0, v_y=0)
-    
-    # Goalkeeper penalty position
-    #goalKeeper = Robot(id=0, x=-6, y=0, theta=0, yellow = True)
-    goalKeeper = Robot(id=0, x=6, y=1, theta=0, yellow = True)
-
-    # Kicker penalty position
-    #attacker = Robot(id=0, x=random.uniform(-3.5, 0), y=random.uniform(-4, 4), theta=180, yellow = False)
-    attacker = Robot(id=0, x=random.uniform(-3,-1), y=random.uniform(-2.5, 2.5), theta=180, yellow = False)
-
-    return [goalKeeper, attacker], ball
-    
-  def _calculateRewardsAndDoneFlag(self):
-    reward = 0
-    rewardContact = 0
-    rewardDistance = 0
-    rewardAngle =0
-    done = False
-    #print(self.state.timestamp)
-
-    
-    dt = self.state.timestamp - self.timestampAnt
-
-    if(dt==0):
-      dt =1
-
-    rewardDistance += (5 / pow(2 * math.pi, 1 / 2)) * math.exp(-((self.goToBallState.distance*0.001)**2 + self.goToBallState.angle_relative**2) / 2) - 2
-    #rewardDistance -= (0.1*(self.goToBallState.distance - self.distAnt))/dt
-    #rewardAngle -=(0.02*(self.goToBallState.angle_relative - self.angleAnt))/dt
-
-    self.distAnt = self.goToBallState.distance
-    self.timestampAnt = self.state.timestamp
-    self.angleAnt = self.goToBallState.angle_relative
-
-    if self.state.ball.x < -6000 or self.state.ball.y > 4500 or self.state.ball.y < -4500:
-      # the ball out the field limits
-      done = True
-      rewardContact += 0
-      
-    elif  self.steps > 250:
-      #finished the episode
-      done = True
-      if (self.goToBallState.distance*0.001 <= 0.130 and abs(self.goToBallState.angle_relative)<0.5):
-        #print("OI")
-        rewardContact += 100
-      #rewardDistance += (5 / pow(2 * math.pi, 1 / 2)) * math.exp(-((self.goToBallState.distance*0.001)**2 + self.goToBallState.angle_relative**2) / 2) - 2
-    else:
-      # the ball in the field limits
-      if (self.goToBallState.distance*0.001 <= 0.130 and abs(self.goToBallState.angle_relative)<0.5):
-        #print("OI")
-        rewardContact += 100
-      #rewardDistance += (5 / pow(2 * math.pi, 1 / 2)) * math.exp(-((self.goToBallState.distance*0.001)**2 + self.goToBallState.angle_relative**2) / 2) - 2 
-
-    reward = rewardContact + rewardDistance + rewardAngle
-    #print(reward)
-    return reward, done
\ No newline at end of file
diff --git a/envs/rc_gym/grsim_ssl/GoToBallEnv/goToBallState.py b/envs/rc_gym/grsim_ssl/GoToBallEnv/goToBallState.py
deleted file mode 100644
index 34fc52c2..00000000
--- a/envs/rc_gym/grsim_ssl/GoToBallEnv/goToBallState.py
+++ /dev/null
@@ -1,143 +0,0 @@
-from dataclasses import dataclass
-from rc_gym.Utils import *
-
-
-TOP_FIELD = 4.5
-BOTTOM_FIELD = -4.5
-LEFT_FIELD = -6
-RIGHT_FIELD = 6
-
-@dataclass
-class goToBallState:
-  """Init Frame object."""
-  ball_x: float = None
-  ball_y: float = None
-  robot_vx: float = None
-  robot_vy: float = None
-  robot_w: float = None
-  distance: float = None
-  wall_top_x: float = None
-  wall_top_y: float = None
-  wall_bottom_x: float = None
-  wall_bottom_y: float = None
-  wall_left_x: float = None
-  wall_left_y: float = None
-  wall_right_x: float = None
-  wall_right_y: float = None
-  angle_relative: float = None
-  
-
-  def getDistance(self, frame) -> float:
-    #print(float(mod(abs(frame.robots_blue[0].x-frame.ball.x), abs(frame.robots_blue[0].y-frame.ball.y))))
-    return float(mod(abs(frame.robots_blue[0].x-frame.ball.x), abs(frame.robots_blue[0].y-frame.ball.y)))
-
-  def getTopPosition(self, frame):
-    diff_y = TOP_FIELD - frame.robots_blue[0].y
-    pos_x = math.sin(frame.robots_blue[0].theta) * diff_y
-    pos_y = math.cos(frame.robots_blue[0].theta) * diff_y
-    #print(pos_x, pos_y)
-    return pos_x, pos_y
-
-  def getBottomPosition(self, frame):
-    diff_y = BOTTOM_FIELD - frame.robots_blue[0].y
-    pos_x = math.sin(frame.robots_blue[0].theta) * diff_y
-    pos_y = math.cos(frame.robots_blue[0].theta) * diff_y
-    #print(pos_x, pos_y)
-    return pos_x, pos_y
-
-  def getLeftPosition(self, frame):
-    diff_y = LEFT_FIELD - frame.robots_blue[0].x
-    pos_x = math.cos(frame.robots_blue[0].theta) * diff_y
-    pos_y = -math.sin(frame.robots_blue[0].theta) * diff_y
-    #print(pos_x, pos_y)
-    return pos_x, pos_y 
-
-  def getRightPosition(self, frame):
-    diff_y = RIGHT_FIELD - frame.robots_blue[0].x
-    pos_x = math.cos(frame.robots_blue[0].theta) * diff_y
-    pos_y = -math.sin(frame.robots_blue[0].theta) * diff_y
-    #print(pos_x, pos_y)
-    return pos_x, pos_y  
-
-  def getRelativeRobotToBallAngle(self, frame):
-    #dist_left = [abs(frame.ball.x - frame.robots_blue[0].x), abs(frame.ball.y - frame.robots_blue[0].y)]
-    #angle_ball = angle(dist_left[0], dist_left[1])
-    # print(angle_ball)
-    # if frame.robots_blue[0].theta * frame.ball.y >=0:
-    #   # the direction of the robot and the position of the ball are both in the same side of the fild (top or bottom)
-    #   angle_relative = 
-    # print(frame.robots_blue[0].theta)
-    #sign = lambda x: (1, -1)[x<0]
-    #if frame.ball.x <= frame.robots_blue[0].x:
-    #
-    #  if sign(frame.ball.y) ^ sign(frame.robots_blue[0].y) >= 0:
-    #    #if both values have the same signal
-    #    angle_relative = abs(angle_ball - (math.pi - abs(frame.robots_blue[0].theta)))
-    #  else:
-    #    angle_relative = abs(angle_ball + (math.pi - abs(frame.robots_blue[0].theta)))
-    #else:
-    #  if sign(frame.ball.y) ^ sign(frame.robots_blue[0].y) >= 0: 
-    #    angle_relative = abs(abs(frame.robots_blue[0].theta) - angle_ball)
-    #  else:
-    #    angle_relative = abs(abs(frame.robots_blue[0].theta) + angle_ball)
-    #
-    #print(angle_relative)
-    #
-    robot_ball = [frame.robots_blue[0].x - frame.ball.x, frame.robots_blue[0].y - frame.ball.y]
-    angle_to_ball = toPiRange(angle(robot_ball[0], robot_ball[1]) + (math.pi - frame.robots_blue[0].theta))
-    #print(angle_to_ball)
-    return angle_to_ball
-
-  def getBallLocalCoordinates(self, frame):
-    robot_ball = [frame.robots_blue[0].x - frame.ball.x, frame.robots_blue[0].y - frame.ball.y]
-    mod_to_ball = mod(robot_ball[0], robot_ball[1])
-    angle_to_ball = toPiRange(angle(robot_ball[0], robot_ball[1]) + (math.pi - frame.robots_blue[0].theta))
-    robot_ball_x = mod_to_ball* math.cos(angle_to_ball)
-    robot_ball_y = mod_to_ball* math.sin(angle_to_ball)
-    return robot_ball_x, robot_ball_y
-  
-  def getBallLocalSpeed(self, frame):
-    robot_ball = [frame.robots_blue[0].v_x - frame.ball.v_x, frame.robots_blue[0].v_y - frame.ball.v_y]
-    mod_to_ball = mod(robot_ball[0], robot_ball[1])
-    angle_to_ball = toPiRange(angle(robot_ball[0], robot_ball[1]) + (math.pi - frame.robots_blue[0].theta))
-    robot_ball_vx = mod_to_ball* math.cos(angle_to_ball)
-    robot_ball_vy = mod_to_ball* math.sin(angle_to_ball)
-    return robot_ball_vx, robot_ball_vy
-
-
-    
-  
-  def getObservation(self, frame):
-    self.ball_x, self.ball_y = self.getBallLocalCoordinates(frame)
-    self.ball_vx, self.ball_vy = self.getBallLocalSpeed(frame)
-    self.robot_vx = frame.robots_blue[0].v_x
-    self.robot_vy = frame.robots_blue[0].v_y
-    
-    self.distance = self.getDistance(frame)
-    self.robot_w = frame.robots_blue[0].v_theta
-    self.wall_top_x, self.wall_top_y = self.getTopPosition(frame)
-    self.wall_bottom_x, self.wall_bottom_y = self.getBottomPosition(frame)   
-    self.wall_left_x, self.wall_left_y = self.getLeftPosition(frame)
-    self.wall_right_x, self.wall_right_y = self.getRightPosition(frame)
-    self.angle_relative = self.getRelativeRobotToBallAngle(frame)
-
-    
-    
-    observation = []
-
-    observation.append(self.ball_x) 
-    observation.append(self.ball_y) 
-    observation.append(self.robot_vx) 
-    observation.append(self.robot_vy) 
-    observation.append(self.robot_w)
-    observation.append(self.distance)
-    observation.append(self.wall_top_x)
-    observation.append(self.wall_top_y)
-    observation.append(self.wall_bottom_x) 
-    observation.append(self.wall_bottom_y)
-    observation.append(self.wall_left_x)
-    observation.append(self.wall_left_y)
-    observation.append(self.wall_right_x)
-    observation.append(self.wall_right_y)
-    
-    return observation
\ No newline at end of file
diff --git a/envs/rc_gym/grsim_ssl/PenaltyEnv/grSimSSL_penalty_env.py b/envs/rc_gym/grsim_ssl/PenaltyEnv/grSimSSL_penalty_env.py
deleted file mode 100644
index 3850271e..00000000
--- a/envs/rc_gym/grsim_ssl/PenaltyEnv/grSimSSL_penalty_env.py
+++ /dev/null
@@ -1,212 +0,0 @@
-import gym
-import math
-import numpy as np
-
-from rc_gym.Utils import distance
-from rc_gym.Entities import Ball, FramePB, Robot
-from rc_gym.grsim_ssl.grSimSSL_env import GrSimSSLEnv
-from rc_gym.grsim_ssl.Communication.grSimClient import grSimClient
-
-class GrSimSSLPenaltyEnv(GrSimSSLEnv):
-    """
-    Description:
-        This environment controls a robot soccer goalkeeper in a Robocup 
-        Small Size League penalty situation
-    Observation:
-        Type: Box(11)
-        Num     Observation                         Min         Max
-        0       Ball X   (mm)                       -7000       7000
-        1       Ball Y   (mm)                       -6000       6000
-        2       Ball Vx  (mm/s)                     -5000       5000
-        3       Ball Vy  (mm/s)                     -5000       5000
-        6       id 0 Blue Robot X       (mm)        -7000       7000
-        4       id 0 Blue Robot Y       (mm)        -6000       6000
-        9       id 0 Blue Robot Vx      (mm/s)      -5000       5000
-        5       id 0 Blue Robot Vy      (mm/s)      -5000       5000
-        6       id 0 Yellow Robot X     (mm)        -7000       7000
-        7       id 0 Yellow Robot Y     (mm)        -6000       6000
-        8       id 0 Yellow Robot sin(Angle)        -1          1
-        9       id 0 Yellow Robot cos(Angle)        -1          1
-        9       id 0 Yellow Robot Vx    (mm/s)      -5000       5000
-        10      id 0 Yellow Robot Vy    (mm/s)      -5000       5000
-        11      id 0 Yellow Robot Vw    (rad/s)     -10         10
-    Actions:
-        Type: Box(1)
-        Num     Action                        Min       Max
-        0       id 0 Blue Team Robot Vy       -3        3
-        Note: Global reference
-    Reward:
-        # 1 if NOT GOAL
-        # -1 * Ball distance from goalkeeper (meters) if GOAL
-    Starting State:
-        Ball on penalty position, goalkeeper on goal line in center of goal,
-        and attacker behind ball facing goal
-    Episode Termination:
-        Ball crosses goal line, or ball has a velocity opposite to goal after attacker kick
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.action_space = gym.spaces.Box(low=-2, high=2, shape=(1,), dtype=np.float32)
-        # Observation Space thresholds
-        obsSpaceThresholds = np.array([7000, 6000, 5000, 5000, 7000, 6000, 5000, 5000, 7000, 6000,
-                                       1, 1, 5000, 5000, 10], dtype=np.float32)
-        self.observation_space = gym.spaces.Box(low=-obsSpaceThresholds, high=obsSpaceThresholds)
-        self.atkState = None
-        self.kickTargetAngle = None
-        self.AttackerVw = None
-        self.AtackerVKick = None
-        self.ball_x = None
-        self.ball_y = None
-
-        print('Environment initialized')
-
-    def reset(self):
-        self.atkState = 0
-        self.AttackerVw = np.random.uniform(0.4, 1.2)
-        self.AtackerVKick = np.random.uniform(4.5, 6.5)
-        self.ball_x = np.random.uniform(-0.015,0.05)
-        self.ball_y = np.random.uniform(-0.03,0.03)
-        # get a random target kick angle between -20 and 20 degrees
-        kickAngle = np.random.uniform(-0.4, 0.4)
-        if kickAngle < 0:
-            self.kickTargetAngle = -3.14 - kickAngle
-        else:
-            self.kickTargetAngle = 3.14 - kickAngle
-        return super().reset()
-
-    def _parseObservationFromState(self):
-        observation = []
-
-        observation.append(self.state.ball.x)
-        observation.append(self.state.ball.y)
-        observation.append(self.state.ball.v_x)
-        observation.append(self.state.ball.v_y)
-
-        observation.append(self.state.robots_blue[0].x)
-        observation.append(self.state.robots_blue[0].y)
-        observation.append(self.state.robots_blue[0].v_x)
-        observation.append(self.state.robots_blue[0].v_y)
-
-        observation.append(self.state.robots_yellow[0].x)
-        observation.append(self.state.robots_yellow[0].y)
-        observation.append(math.sin(self.state.robots_yellow[0].theta))
-        observation.append(math.cos(self.state.robots_yellow[0].theta))
-        observation.append(self.state.robots_yellow[0].v_x)
-        observation.append(self.state.robots_yellow[0].v_y)
-        observation.append(self.state.robots_yellow[0].v_theta)
-
-        return np.array(observation)
-
-    def _getFormation(self):
-        # returns robot's and ball initial position        
-        # ball penalty position
-        ball = Ball(x=-4.8 + self.ball_x, y=0.0 + self.ball_y)
-
-        robotPositions = []
-        # Goalkeeper penalty position
-        robotPositions.append(Robot(yellow=False, id=0, x=-6, y=0, theta=0))
-        # Kicker penalty position
-        robotPositions.append(Robot(yellow=True, id=0, x=-4, y=0, theta=180))
-
-        return robotPositions, ball
-
-    def _getAttackerCommand(self):
-        cmdAttacker = Robot(yellow=True, id=0, dribbler=True)
-
-        # attacker movement speed
-        v_x = np.clip(((-2) * (self.state.ball.x - self.state.robots_yellow[0].x) * (0.001)), 0.2, 1.5)
-
-        # If atkState == 0 -> move to ball
-        if self.atkState == 0:
-            if self.state.robots_yellow[0].x <= -4700:
-                cmdAttacker.v_x = 0.0
-                if self.kickTargetAngle < 0:
-                    self.atkState = 1
-                else:
-                    self.atkState = 2
-            else:
-                cmdAttacker.v_x = v_x
-        # If atkState == 1 -> rotate counterclockwise until kick angle        
-        if self.atkState == 1:
-            self.atkState = 0
-            if -(abs(self.state.robots_yellow[0].theta)) < self.kickTargetAngle:
-                cmdAttacker.v_theta = self.AttackerVw
-                cmdAttacker.v_x = 0.0
-            else:
-                cmdAttacker.dribbler = False
-                cmdAttacker.v_theta = 0.0
-                cmdAttacker.v_x = 0.0
-                cmdAttacker.kick_v_x = self.AtackerVKick
-        # If atkState == 2 -> rotate clockwise until kick angle        
-        if self.atkState == 2:
-            self.atkState = 0
-            if abs(self.state.robots_yellow[0].theta) > self.kickTargetAngle:
-                cmdAttacker.v_theta = -self.AttackerVw
-                cmdAttacker.v_x = 0.0
-            else:
-                cmdAttacker.dribbler = False
-                cmdAttacker.v_theta = 0.0
-                cmdAttacker.v_x = 0.0
-                cmdAttacker.kick_v_x = self.AtackerVKick
-
-        return cmdAttacker
-
-    def _getCommands(self, actions):
-        commands = []
-
-        cmdGoalKeeper = self._getCorrectGKCommand(actions)
-
-        commands.append(cmdGoalKeeper)
-
-        cmdAttacker = self._getAttackerCommand()
-
-        commands.append(cmdAttacker)
-
-        return commands
-
-    def _getCorrectGKCommand(self, v_y):
-        '''Control goalkeeper v_theta and vx to keep him at goal line'''
-        cmdGoalKeeper = Robot(yellow=False, id=0, v_y=v_y)
-
-        # Proportional Parameters for Vx and Vw
-        KpVx = 0.005
-        KpVw = 1.6
-        # Error between goal line and goalkeeper
-        errX = -6000 - self.state.robots_blue[0].x
-        # If the error is greater than 20mm, correct the goalkeeper
-        if abs(errX) > 20:
-            cmdGoalKeeper.v_x = KpVx * errX
-        else:
-            cmdGoalKeeper.v_x = 0.0
-        # Error between the desired angle and goalkeeper angle
-        errW = 0.0 - self.state.robots_blue[0].theta
-        # If the error is greater than 0.1 rad (5,73 deg), correct the goalkeeper
-        if abs(errW) > 0.1:
-            cmdGoalKeeper.v_theta = KpVw * errW
-        else:
-            cmdGoalKeeper.v_theta = 0.0
-
-        return cmdGoalKeeper
-
-    def _calculateRewardsAndDoneFlag(self):
-        reward = 0
-        done = False
-
-        # If ball crosses the goal line
-        if self.state.ball.x < -6025:
-            done = True
-            # If ball crosses goal line inside goal bounds GOAL
-            if self.state.ball.y < 600 and self.state.ball.y > -600:
-                # Reward based on distance from keeper to ball
-                reward = abs(self.state.ball.y - self.state.robots_blue[0].y) * (-0.001)
-            else:
-                # NOT GOAL
-                reward = 1
-        # If ball is moving away from goal after attacker kick NOT GOAL
-        if self.state.ball.x < -5000:
-            if self.state.ball.v_x > 0:
-                done = True
-                reward = 1
-
-        return reward, done
diff --git a/envs/rc_gym/grsim_ssl/ShootGoalieEnv/__init__.py b/envs/rc_gym/grsim_ssl/ShootGoalieEnv/__init__.py
deleted file mode 100644
index a18f449c..00000000
--- a/envs/rc_gym/grsim_ssl/ShootGoalieEnv/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-from rc_gym.grsim_ssl.ShootGoalieEnv.shootGoalieState import *
diff --git a/envs/rc_gym/grsim_ssl/ShootGoalieEnv/shootGoalieEnv.py b/envs/rc_gym/grsim_ssl/ShootGoalieEnv/shootGoalieEnv.py
deleted file mode 100644
index a2eb37c9..00000000
--- a/envs/rc_gym/grsim_ssl/ShootGoalieEnv/shootGoalieEnv.py
+++ /dev/null
@@ -1,201 +0,0 @@
-import gym
-import math
-import numpy as np
-import time
-import random
-
-from rc_gym.grsim_ssl.grSimSSL_env import GrSimSSLEnv
-from rc_gym.grsim_ssl.Communication.grSimClient import grSimClient
-from rc_gym.Entities import Ball, Frame, Robot
-from rc_gym.grsim_ssl.ShootGoalieEnv import shootGoalieState
-from rc_gym.Utils import mod
-
-
-class shootGoalieEnv(GrSimSSLEnv):
-
-  """
-  Description:
-    # TODO
-  Source:
-    # TODO
-
-  Observation:
-    Type: Box(16)
-    Num     Observation                                       Min                     Max
-    0       Ball X   (mm)                                   -7000                   7000
-    1       Ball Y   (mm)                                   -6000                   6000
-    2       Ball Vx  (mm/s)                                 -10000                  10000
-    3       Ball Vy  (mm/s)                                 -10000                  10000
-    4       Blue id 0 Robot Vw       (rad/s)                -math.pi * 3            math.pi * 3
-    5       Dist Blue id0 - goal center (mm)                -10000                  10000
-    6       Angle between blue id 0 and goal left (rad)     -math.pi                math.pi
-    7       Angle between blue id 0 and goal left (rad)     -math.pi                math.pi
-    8       Angle between blue id 0 and goal right (rad)    -math.pi                math.pi
-    9       Angle between blue id 0 and goal right (rad)    -math.pi                math.pi
-    10      Angle between blue id 0 and goalie center(rad)  -math.pi                math.pi
-    11      Angle between blue id 0 and goalie center(rad)  -math.pi                math.pi
-    12      Angle between blue id 0 and goalie left (rad)   -math.pi                math.pi
-    13      Angle between blue id 0 and goalie left (rad)   -math.pi                math.pi
-    14      Angle between blue id 0 and goalie right (rad)  -math.pi                math.pi
-    15      Angle between blue id 0 and goalie right (rad)  -math.pi                math.pi
-
-  Actions:
-    Type: Box(2)
-    Num     Action                        Min                     Max
-    0       Blue id 0 Vw (rad/s)        -math.pi * 3            math.pi * 3
-    1       Blue Kick Strength (m/s)        -6.5                   6.5
-  Reward:
-    Reward is 1 for success, -1 to fails. 0 otherwise.
-
-  Starting State:
-    All observations are assigned a uniform random value in [-0.05..0.05]
-    # TODO
-
-  Episode Termination:
-    # TODO
-  """
-  def __init__(self):
-    super().__init__()
-    ## Action Space
-    actSpaceThresholds = np.array([math.pi * 3, 6.5], dtype=np.float32)
-    self.action_space = gym.spaces.Box(low=-actSpaceThresholds, high=actSpaceThresholds)
-
-    # Observation Space thresholds
-    obsSpaceThresholds = np.array([7000, 6000, 10000, 10000, math.pi * 3, 10000, math.pi, math.pi,
-                                   math.pi, math.pi, math.pi, math.pi, math.pi, math.pi, math.pi, math.pi], dtype=np.float32)
-    self.observation_space = gym.spaces.Box(low=-obsSpaceThresholds, high=obsSpaceThresholds)
-    self.shootGoalieState = None
-    self.goalieState = 0
-
-    self.goalieState = 0
-
-    print('Environment initialized')
-  
-  def _getCommands(self, actions):
-    commands = []
-    cmdAttacker = Robot(id=0, yellow=False, v_theta=actions[0], kick_v_x=actions[1], dribbler=True)
-    
-    commands.append(cmdAttacker)
-
-
-    # Moving GOALIE
-    vy = (self.state.ball.y - self.state.robots_yellow[0].y)/1000
-    if abs(vy) > 0.4:
-      vy = 0.4*(vy/abs(vy))
-    if self.state.robots_yellow[0].y > 500 and vy > 0:
-      vy = 0
-    if self.state.robots_yellow[0].y < -500 and vy < 0:
-      vy = 0
-      
-    cmdGoalie = self._getCorrectGKCommand(vy)
-    
-    commands.append(cmdGoalie)
-
-    return commands
-
-  def reset(self):
-    # Remove ball from Robot
-    self.client.sendCommandsPacket([Robot(yellow=False, id = 0, kick_v_x=3), Robot(yellow=True, id = 0, kick_v_x=3)]) 
-    self.client.receiveState()
-    return super().reset()
-
-  def _parseObservationFromState(self):
-    observation = []
-
-    self.shootGoalieState = shootGoalieState()
-    observation = self.shootGoalieState.getObservation(self.state)
-
-    return np.array(observation)
-
-  def reset(self):
-    # Remove ball from Robot
-    self.client.sendCommandsPacket([Robot(yellow=False, id = 0, kick_v_x=0), Robot(yellow=True, id = 0, kick_v_x=3)]) 
-    self.client.receiveState()
-    return super().reset()
-
-  def _getFormation(self):
-    attacker_x = -4
-    attacker_y = random.randrange(-20, 20, 2)/10
-    robot_theta = random.randrange(0, 359, 5)
-    ball_x = 0.1*math.cos(math.radians(robot_theta)) + attacker_x
-    ball_y = 0.1*math.sin(math.radians(robot_theta)) + attacker_y
-    # ball position
-    ball = Ball(x=ball_x, y=ball_y, v_x=0, v_y=0)
-    # Goalkeeper position
-    goalkeeper_y = random.randrange(-5, 5, 1)/10
-    goalKeeper = Robot(id=0, x=-6, y=goalkeeper_y, theta=0, yellow = True)
-    # Kicker position
-    attacker = Robot(id=0, x=attacker_x, y=attacker_y, theta=robot_theta, yellow = False)
-
-    # For fixed positions!
-    # ball = Ball(x=-4.1, y=0, v_x=0, vy=0)
-    # # Goalkeeper position
-    # goalKeeper = Robot(id=0, x=-6, y=0, theta=0, yellow = True)
-    # # Kicker position
-    # attacker = Robot(id=0, x=-4, y=0, theta=180, yellow = False)
-
-    return [goalKeeper, attacker], ball
-    
-  def _calculateRewardsAndDoneFlag(self):
-    return self._penalizeRewardFunction()
-
-  def _firstRewardFunction(self):
-    reward = 0
-    done = False
-    if self.state.ball.x < -6000:
-      # the ball out the field limits
-      done = True
-      if self.state.ball.y < 600 and self.state.ball.y > -600:
-          # ball entered the goal
-          reward = 1
-      else:
-          # the ball went out the bottom line
-          reward = -1
-    elif self.state.ball.x < -5000 and self.state.ball.v_x > -1:
-        # goalkeeper caught the ball
-      done = True
-      reward = -1
-    elif mod(self.state.ball.v_x, self.state.ball.v_y) < 10 and self.steps > 15: # 1 cm/s
-      done = True
-      reward = -1
-    return reward, done
-
-  def _penalizeRewardFunction(self):
-    reward = -0.01
-    done = False
-    if self.state.ball.x < -6000:
-      # the ball out the field limits
-      done = True
-      if self.state.ball.y < 600 and self.state.ball.y > -600:
-          # ball entered the goal
-          reward = 2
-    elif self.state.ball.x < -5000 and self.state.ball.v_x > -1:
-      done = True
-      reward = -0.3
-       
-    return reward, done
-
-
-  def _getCorrectGKCommand(self,v_y):
-    '''Control goalkeeper v_theta and vx to keep him at goal line'''
-    cmdGoalKeeper = Robot(yellow=True, id=0, v_y=v_y)
-
-    # Proportional Parameters for Vx and Vw
-    KpVx = 0.0006
-    KpVw = 1
-    # Error between goal line and goalkeeper
-    errX = -6000 - self.state.robots_yellow[0].x
-    # If the error is greater than 20mm, correct the goalkeeper
-    if abs(errX) > 20:
-        cmdGoalKeeper.v_x = KpVx * errX
-    else:
-        cmdGoalKeeper.v_x = 0.0
-    # Error between the desired angle and goalkeeper angle
-    errW = 0.0 - self.state.robots_yellow[0].theta
-    # If the error is greater than 0.1 rad (5,73 deg), correct the goalkeeper
-    if abs(errW) > 0.1:
-        cmdGoalKeeper.v_theta = KpVw * errW
-    else:
-        cmdGoalKeeper.v_theta = 0.0
-
-    return cmdGoalKeeper
\ No newline at end of file
diff --git a/envs/rc_gym/grsim_ssl/ShootGoalieEnv/shootGoalieState.py b/envs/rc_gym/grsim_ssl/ShootGoalieEnv/shootGoalieState.py
deleted file mode 100644
index 18ad5ee7..00000000
--- a/envs/rc_gym/grsim_ssl/ShootGoalieEnv/shootGoalieState.py
+++ /dev/null
@@ -1,116 +0,0 @@
-from dataclasses import dataclass
-from rc_gym.Utils import *
-
-
-CENTER_GOAL_X = -6000
-CENTER_GOALY = 0
-
-LEFT_GOAL_X = -6000
-LEFT_GOALY = -600
-
-RIGHT_GOAL_X = -6000
-RIGHT_GOALY = 600
-
-ROBOT_RADIUS = 90
-
-@dataclass
-class shootGoalieState:
-  """Init Frame object."""
-  ball_x: float = None
-  ballY: float = None
-  ball_vx: float = None
-  ball_vy: float = None
-  robot_w: float = None
-  distance: float = None
-  theta_l_sen: float = None
-  theta_l_cos: float = None
-  theta_r_sen: float = None
-  theta_r_cos: float = None
-  theta_goalie_c_sen: float = None
-  theta_goalie_c_cos: float = None
-  theta_goalie_l_sen: float = None
-  theta_goalie_l_cos: float = None
-  theta_goalie_r_sen: float = None
-  theta_goalie_r_cos: float = None
-
-  def getDistance(self, frame) -> float:
-    return float(mod(abs(frame.robots_blue[0].x-CENTER_GOAL_X), abs(frame.robots_blue[0].y-CENTER_GOALY)))
-
-  def getLeftPoleAngle(self, frame):
-    dist_left = [frame.robots_blue[0].x - LEFT_GOAL_X, frame.robots_blue[0].y - LEFT_GOALY]
-    angle_left = toPiRange(angle(dist_left[0], dist_left[1]) + (math.pi - frame.robots_blue[0].theta))
-    return math.sin(angle_left), math.cos(angle_left)
-
-  def getRightPoleAngle(self, frame):
-    dist_right = [frame.robots_blue[0].x - RIGHT_GOAL_X, frame.robots_blue[0].y - RIGHT_GOALY]
-    angle_right = toPiRange(angle(dist_right[0], dist_right[1]) + (math.pi - frame.robots_blue[0].theta))
-    return math.sin(angle_right), math.cos(angle_right)
-
-  def getGoalieCenterUnifiedAngle(self, frame):
-    dist_g = [frame.robots_blue[0].x - frame.robots_yellow[0].x, frame.robots_blue[0].y - frame.robots_yellow[0].y]
-    angle_g = toPiRange(angle(dist_g[0], dist_g[1]) + (math.pi - frame.robots_blue[0].theta))
-    return angle_g
-  
-  def getGoalieCenterAngle(self, frame):
-    angle_c = self.getGoalieCenterUnifiedAngle(frame)
-    return math.sin(angle_c), math.cos(angle_c)
-
-  def getGoalieLeftAngle(self, frame):
-    dist_left = [frame.robots_blue[0].x - frame.robots_yellow[0].x, frame.robots_blue[0].y - (frame.robots_yellow[0].y - ROBOT_RADIUS)]
-    angle_left = toPiRange(angle(dist_left[0], dist_left[1]) + (math.pi - frame.robots_blue[0].theta))
-    return math.sin(angle_left), math.cos(angle_left)
-
-  def getGoalieRightAngle(self, frame):
-    dist_right = [frame.robots_blue[0].x - frame.robots_yellow[0].x, frame.robots_blue[0].y - (frame.robots_yellow[0].y + ROBOT_RADIUS)]
-    angle_right = toPiRange(angle(dist_right[0], dist_right[1]) + (math.pi - frame.robots_blue[0].theta))
-    return math.sin(angle_right), math.cos(angle_right)
-  
-  def getBallLocalCoordinates(self, frame):
-    robot_ball = [frame.robots_blue[0].x - frame.ball.x, frame.robots_blue[0].y - frame.ball.y]
-    mod_to_ball = mod(robot_ball[0], robot_ball[1])
-    angle_to_ball = toPiRange(angle(robot_ball[0], robot_ball[1]) + (math.pi - frame.robots_blue[0].theta))
-    robot_ball_x = mod_to_ball* math.cos(angle_to_ball)
-    robot_ball_y = mod_to_ball* math.sin(angle_to_ball)
-    return robot_ball_x, robot_ball_y
-  
-  def getBallLocalSpeed(self, frame):
-    robot_ball = [frame.robots_blue[0].v_x - frame.ball.v_x, frame.robots_blue[0].v_y - frame.ball.v_y]
-    mod_to_ball = mod(robot_ball[0], robot_ball[1])
-    angle_to_ball = toPiRange(angle(robot_ball[0], robot_ball[1]) + (math.pi - frame.robots_blue[0].theta))
-    robot_ball_vx = mod_to_ball* math.cos(angle_to_ball)
-    robot_ball_vy = mod_to_ball* math.sin(angle_to_ball)
-    return robot_ball_vx, robot_ball_vy
-
-  
-  def getObservation(self, frame):
-
-    self.ball_x, self.ballY = self.getBallLocalCoordinates(frame)
-    self.ball_vx, self.ball_vy = self.getBallLocalSpeed(frame)
-    
-    self.distance = self.getDistance(frame)
-    self.robot_w = frame.robots_blue[0].v_theta
-    self.theta_l_sen, self.theta_l_cos = self.getLeftPoleAngle(frame)
-    self.theta_r_sen, self.theta_r_cos = self.getRightPoleAngle(frame)
-    self.theta_goalie_c_sen, self.theta_goalie_c_cos = self.getGoalieCenterAngle(frame)
-    self.theta_goalie_l_sen, self.theta_l_cos = self.getGoalieLeftAngle(frame)
-    self.theta_goalie_r_sen, self.theta_r_cos = self.getGoalieRightAngle(frame)
-    
-    observation = []
-
-    observation.append(self.ball_x) 
-    observation.append(self.ballY) 
-    observation.append(self.ball_vx) 
-    observation.append(self.ball_vy) 
-    observation.append(self.robot_w)
-    observation.append(self.distance)
-    observation.append(self.theta_l_sen)
-    observation.append(self.theta_l_cos)
-    observation.append(self.theta_r_sen) 
-    observation.append(self.theta_r_cos)
-    observation.append(self.theta_goalie_c_sen)
-    observation.append(self.theta_goalie_c_cos)
-    observation.append(self.theta_goalie_l_sen)
-    observation.append(self.theta_l_cos)
-    observation.append(self.theta_goalie_r_sen)
-    observation.append(self.theta_r_cos)
-    return observation
\ No newline at end of file
diff --git a/envs/rc_gym/grsim_ssl/__init__.py b/envs/rc_gym/grsim_ssl/__init__.py
deleted file mode 100644
index 81342136..00000000
--- a/envs/rc_gym/grsim_ssl/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from rc_gym.grsim_ssl.PenaltyEnv.grSimSSL_penalty_env import GrSimSSLPenaltyEnv
-from rc_gym.grsim_ssl.ShootGoalieEnv.shootGoalieEnv import shootGoalieEnv
-from rc_gym.grsim_ssl.GoToBallEnv.goToBallEnv import goToBallEnv
diff --git a/envs/rc_gym/grsim_ssl/grSimSSL_env.py b/envs/rc_gym/grsim_ssl/grSimSSL_env.py
deleted file mode 100644
index 43ad1514..00000000
--- a/envs/rc_gym/grsim_ssl/grSimSSL_env.py
+++ /dev/null
@@ -1,61 +0,0 @@
-'''
-#   Environment Communication Structure
-#    - Father class that creates the structure to communicate with multples setups of enviroment
-#    - To create your wrapper from env to communcation, use inherit from this class! 
-'''
-
-
-import gym
-from rc_gym.grsim_ssl.Communication.grSimClient import grSimClient
-from rc_gym.Entities import Robot
-
-class GrSimSSLEnv(gym.Env):
-    def __init__(self):
-        self.client = grSimClient()
-        self.action_space = None
-        self.observation_space = None
-        self.state = None
-        self.steps = 0
-
-    def step(self, action):
-        self.steps += 1
-        # Sends actions
-        commands = self._getCommands(action)
-        self.client.sendCommandsPacket(commands) 
-
-        # Update state
-        self.state = self.client.receiveState()
-
-        # Calculate environment observation, reward and done condition
-        observation = self._parseObservationFromState()
-        reward, done = self._calculateRewardsAndDoneFlag()
-
-        return observation, reward, done, {}
-
-    def reset(self):
-        self.steps = 0
-        # Place robots on reset positions
-        resetRobotPositions, resetBallPosition = self._getFormation()
-        self.client.sendReplacementPacket(robotPositions=resetRobotPositions, ballPosition=resetBallPosition) 
-        # Update state and observation
-        self.state = self.client.receiveState()
-        observation = self._parseObservationFromState()
-
-        return observation
-
-    def _getCommands(self, action):
-        '''returns a list of commands of type List[Robot] from type action_space action'''
-        raise NotImplementedError
-
-    def _getFormation(self):
-        '''returns a positioning formation of type List[Robot]'''
-        raise NotImplementedError
-
-    def _parseObservationFromState(self):
-        '''returns a type observation_space observation from a type List[Robot] state'''
-        raise NotImplementedError
-
-    def _calculateRewardsAndDoneFlag(self):
-        '''returns reward value and done flag from type List[Robot] state'''
-        raise NotImplementedError
-
diff --git a/envs/rc_gym/ssl/__init__.py b/envs/rc_gym/ssl/__init__.py
deleted file mode 100644
index 2b18e698..00000000
--- a/envs/rc_gym/ssl/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-from rc_gym.ssl.env_11v11.ssl_gym_11v11 import SSL11v11Env
diff --git a/envs/rc_gym/ssl/ssl_gym_base.py b/envs/rc_gym/ssl/ssl_gym_base.py
index 940863e8..e30cc433 100644
--- a/envs/rc_gym/ssl/ssl_gym_base.py
+++ b/envs/rc_gym/ssl/ssl_gym_base.py
@@ -12,22 +12,35 @@
 import numpy as np
 from rc_gym.Entities import Frame, Robot
 from rc_gym.Utils import RCRender
-from rc_gym.Simulators.rsim_ssl import RSimSSL
+from rc_gym.Simulators.rsim import RSimSSL
 
 
 class SSLBaseEnv(gym.Env):
+    NORM_BOUNDS = 1.2
+
     def __init__(self, field_type: int,
                  n_robots_blue: int, n_robots_yellow: int, time_step: float):
+        # Initialize Simulator
         self.time_step = time_step
         self.rsim = RSimSSL(field_type=field_type,
                             n_robots_blue=n_robots_blue,
                             n_robots_yellow=n_robots_yellow,
                             time_step_ms=int(self.time_step*1000))
-        self.field_type: int = field_type
         self.n_robots_blue: int = n_robots_blue
         self.n_robots_yellow: int = n_robots_yellow
+        
+        # Get field dimensions
+        self.field_type: int = field_type
         self.field_params: Dict[str,
                                 np.float64] = self.rsim.get_field_params()
+        self.norm_max_pos = max(self.field_params['field_width'] / 2,
+                                (self.field_params['field_length'] / 2)
+                                 + self.field_params['penalty_length']
+                                )
+        self.norm_max_v = self.rsim.linear_speed_range
+        self.norm_max_w = np.rad2deg(self.rsim.angular_speed_range)
+
+        # Initiate 
         self.frame: Frame = None
         self.last_frame: Frame = None
         self.view = None
@@ -111,3 +124,24 @@ def _calculate_reward_and_done(self):
     def _get_initial_positions_frame(self) -> Frame:
         '''returns frame with robots initial positions'''
         raise NotImplementedError
+    
+    def norm_pos(self, pos):
+        return np.clip(
+            pos / self.norm_max_pos,
+            -self.NORM_BOUNDS,
+            self.NORM_BOUNDS
+        )
+
+    def norm_v(self, v):
+        return np.clip(
+            v / self.norm_max_v,
+            -self.NORM_BOUNDS,
+            self.NORM_BOUNDS
+        )
+
+    def norm_w(self, w):
+        return np.clip(
+            w / self.norm_max_w,
+            -self.NORM_BOUNDS,
+            self.NORM_BOUNDS
+        )
diff --git a/envs/rc_gym/vss/env_3v3/__init__.py b/envs/rc_gym/vss/env_3v3/__init__.py
new file mode 100644
index 00000000..d291fff2
--- /dev/null
+++ b/envs/rc_gym/vss/env_3v3/__init__.py
@@ -0,0 +1,1 @@
+from rc_gym.vss.env_3v3.vss_gym_3v3 import VSS3v3Env
diff --git a/envs/rc_gym/vss/env_3v3/vss_gym_3v3.py b/envs/rc_gym/vss/env_3v3/vss_gym_3v3.py
index e5574127..f3244fbe 100644
--- a/envs/rc_gym/vss/env_3v3/vss_gym_3v3.py
+++ b/envs/rc_gym/vss/env_3v3/vss_gym_3v3.py
@@ -6,7 +6,6 @@
 import gym
 import numpy as np
 from rc_gym.Entities import Frame, Robot
-from rc_gym.Utils import normVt, normVx, normX
 from rc_gym.vss.vss_gym_base import VSSBaseEnv
 
 
@@ -56,19 +55,10 @@ def __init__(self):
         super().__init__(field_type=0, n_robots_blue=3, n_robots_yellow=3,
                          time_step=0.032)
 
-        low_obs_bound = [-1.2, -1.2, -1.25, -1.25]
-        low_obs_bound += [-1.2, -1.2, -1, -1, -1.25, -1.25, -1.2]*3
-        low_obs_bound += [-1.2, -1.2, -1.25, -1.25, -1.2]*3
-        high_obs_bound = [1.2, 1.2, 1.25, 1.25]
-        high_obs_bound += [1.2, 1.2, 1, 1, 1.25, 1.25, 1.2]*3
-        high_obs_bound += [1.2, 1.2, 1.25, 1.25, 1.2]*3
-        low_obs_bound = np.array(low_obs_bound, dtype=np.float32)
-        high_obs_bound = np.array(high_obs_bound, dtype=np.float32)
-
         self.action_space = gym.spaces.Box(low=-1, high=1,
                                            shape=(2, ), dtype=np.float32)
-        self.observation_space = gym.spaces.Box(low=low_obs_bound,
-                                                high=high_obs_bound,
+        self.observation_space = gym.spaces.Box(low=-self.NORM_BOUNDS,
+                                                high=self.NORM_BOUNDS,
                                                 shape=(40, ), dtype=np.float32)
 
         # Initialize Class Atributes
@@ -88,6 +78,7 @@ def __init__(self):
     def reset(self):
         self.actions = None
         self.reward_shaping_total = None
+        self.previous_ball_potential = None
         for ou in self.ou_actions:
             ou.reset()
 
@@ -101,30 +92,32 @@ def _frame_to_observations(self):
 
         observation = []
 
-        observation.append(normX(self.frame.ball.x))
-        observation.append(normX(self.frame.ball.y))
-        observation.append(normVx(self.frame.ball.v_x))
-        observation.append(normVx(self.frame.ball.v_y))
+        observation.append(self.norm_pos(self.frame.ball.x))
+        observation.append(self.norm_pos(self.frame.ball.y))
+        observation.append(self.norm_v(self.frame.ball.v_x))
+        observation.append(self.norm_v(self.frame.ball.v_y))
 
         for i in range(self.n_robots_blue):
-            observation.append(normX(self.frame.robots_blue[i].x))
-            observation.append(normX(self.frame.robots_blue[i].y))
+            observation.append(self.norm_pos(self.frame.robots_blue[i].x))
+            observation.append(self.norm_pos(self.frame.robots_blue[i].y))
             observation.append(
                 np.sin(np.deg2rad(self.frame.robots_blue[i].theta))
             )
             observation.append(
                 np.cos(np.deg2rad(self.frame.robots_blue[i].theta))
             )
-            observation.append(normVx(self.frame.robots_blue[i].v_x))
-            observation.append(normVx(self.frame.robots_blue[i].v_y))
-            observation.append(normVt(self.frame.robots_blue[i].v_theta))
+            observation.append(self.norm_v(self.frame.robots_blue[i].v_x))
+            observation.append(self.norm_v(self.frame.robots_blue[i].v_y))
+            observation.append(self.norm_w(self.frame.robots_blue[i].v_theta))
 
         for i in range(self.n_robots_yellow):
-            observation.append(normX(self.frame.robots_yellow[i].x))
-            observation.append(normX(self.frame.robots_yellow[i].y))
-            observation.append(normVx(self.frame.robots_yellow[i].v_x))
-            observation.append(normVx(self.frame.robots_yellow[i].v_y))
-            observation.append(normVt(self.frame.robots_yellow[i].v_theta))
+            observation.append(self.norm_pos(self.frame.robots_yellow[i].x))
+            observation.append(self.norm_pos(self.frame.robots_yellow[i].y))
+            observation.append(self.norm_v(self.frame.robots_yellow[i].v_x))
+            observation.append(self.norm_v(self.frame.robots_yellow[i].v_y))
+            observation.append(
+                self.norm_w(self.frame.robots_yellow[i].v_theta)
+            )
 
         return np.array(observation)
 
@@ -177,7 +170,7 @@ def __ball_grad(self):
         if self.previous_ball_potential is not None:
             diff = ball_potential - self.previous_ball_potential
             grad_ball_potential = np.clip(diff * 3 / self.time_step,
-                                          -1.0, 1.0)
+                                          -5.0, 5.0)
 
         self.previous_ball_potential = ball_potential
 
@@ -200,7 +193,7 @@ def __move_reward(self):
 
         move_reward = np.dot(robot_ball, robot_vel)
 
-        move_reward = np.clip(move_reward / 0.4, -1.0, 1.0)
+        move_reward = np.clip(move_reward / 0.4, -5.0, 5.0)
         return move_reward
 
     def __energy_penalty(self):
@@ -258,6 +251,7 @@ def _calculate_reward_and_done(self):
             initial_pos_frame: Frame = self._get_initial_positions_frame()
             self.rsim.reset(initial_pos_frame)
             self.frame = self.rsim.get_frame()
+            self.previous_ball_potential = None
             self.last_frame = None
 
         done = self.steps * self.time_step >= 300
@@ -291,35 +285,29 @@ def theta(): return random.uniform(-180, 180)
 
         for i in range(self.n_robots_yellow):
             pos_frame.robots_yellow[i] = Robot(x=x(), y=y(), theta=theta())
-            agents.append(pos_frame.robots_blue[i])
+            agents.append(pos_frame.robots_yellow[i])
 
-        def same_position_ref(x, y, x_ref, y_ref, radius):
-            if x >= x_ref - radius and x <= x_ref + radius and \
-                    y >= y_ref - radius and y <= y_ref + radius:
+        def same_position_ref(obj, ref, radius):
+            if obj.x >= ref.x - radius and obj.x <= ref.x + radius and \
+                    obj.y >= ref.y - radius and obj.y <= ref.y + radius:
                 return True
             return False
 
-        radius_ball = 0.2
-        radius_robot = 0.2
-        same_pos = True
+        radius_ball = 0.04
+        radius_robot = 0.07
 
-        while same_pos:
-            for i in range(len(agents)):
-                same_pos = False
-                while same_position_ref(agents[i].x, agents[i].y, pos_frame.ball.x, pos_frame.ball.y, radius_ball):
+        for i in range(len(agents)):
+            while same_position_ref(agents[i], pos_frame.ball, radius_ball):
+                agents[i] = Robot(x=x(), y=y(), theta=theta())
+            for j in range(i):
+                while same_position_ref(agents[i], agents[j], radius_robot):
                     agents[i] = Robot(x=x(), y=y(), theta=theta())
-                    same_pos = True
-                for j in range(i + 1, len(agents)):
-                    while same_position_ref(agents[i].x, agents[i].y, agents[j].x, agents[j].y, radius_robot):
-                        agents[i] = Robot(x=x(), y=y(), theta=theta())
-                        same_pos = True
-
-        pos_frame.robots_blue[0] = agents[0]
-        pos_frame.robots_blue[1] = agents[1]
-        pos_frame.robots_blue[2] = agents[2]
-        pos_frame.robots_yellow[0] = agents[3]
-        pos_frame.robots_yellow[1] = agents[4]
-        pos_frame.robots_yellow[2] = agents[5]
+
+        for i in range(self.n_robots_blue):
+            pos_frame.robots_blue[i] = agents[i]
+
+        for i in range(self.n_robots_yellow):
+            pos_frame.robots_yellow[i] = agents[i+self.n_robots_blue]
 
         return pos_frame
 
diff --git a/envs/rc_gym/vss/__init__.py b/envs/rc_gym/vss/env_ma/__init__.py
similarity index 53%
rename from envs/rc_gym/vss/__init__.py
rename to envs/rc_gym/vss/env_ma/__init__.py
index fc67c519..36b3d98c 100644
--- a/envs/rc_gym/vss/__init__.py
+++ b/envs/rc_gym/vss/env_ma/__init__.py
@@ -1,2 +1,1 @@
-from rc_gym.vss.env_3v3.vss_gym_3v3 import VSS3v3Env
 from rc_gym.vss.env_ma.vss_gym_ma import VSSMAEnv, VSSMAOpp
diff --git a/envs/rc_gym/vss/env_ma/vss_gym_ma.py b/envs/rc_gym/vss/env_ma/vss_gym_ma.py
index 63fd364c..2cf8a6b3 100644
--- a/envs/rc_gym/vss/env_ma/vss_gym_ma.py
+++ b/envs/rc_gym/vss/env_ma/vss_gym_ma.py
@@ -7,7 +7,6 @@
 import numpy as np
 import torch
 from rc_gym.Entities import Frame, Robot
-from rc_gym.Utils import normVt, normVx, normX
 from rc_gym.vss.env_ma.opponent.model import DDPGActor
 from rc_gym.vss.vss_gym_base import VSSBaseEnv
 
@@ -61,14 +60,14 @@ def __init__(self, n_robots_control=3):
         super().__init__(field_type=0, n_robots_blue=3, n_robots_yellow=3,
                          time_step=0.032)
 
-        obs_bound = [1.2, 1.2, 1.25, 1.25]
-        obs_bound += [1.2, 1.2, 1, 1, 1.25, 1.25, 1.2]*3
-        obs_bound += [1.2, 1.2, 1.25, 1.25, 1.2]*3
-        obs_bound = np.array(obs_bound, dtype=np.float32)
         self.n_robots_control = n_robots_control
-        self.action_space = gym.spaces.Box(low=-1, high=1, shape=(2, ))
-        self.observation_space = gym.spaces.Box(low=-obs_bound,
-                                                high=obs_bound)
+        self.action_space = gym.spaces.Box(low=-1,
+                                           high=1,
+                                           shape=(n_robots_control, 2))
+        self.observation_space = gym.spaces.Box(low=-self.NORM_BOUNDS,
+                                                high=self.NORM_BOUNDS,
+                                                shape=(n_robots_control, 40),
+                                                dtype=np.float32)
 
         # Initialize Class Atributes
         self.previous_ball_potential = None
@@ -81,6 +80,7 @@ def __init__(self, n_robots_control=3):
     def reset(self):
         self.actions = None
         self.reward_shaping_total = None
+        self.previous_ball_potential = None
 
         return super().reset()
 
@@ -92,17 +92,17 @@ def get_rotated_obs(self):
         robots_dict = dict()
         for i in range(self.n_robots_blue):
             robots_dict[i] = list()
-            robots_dict[i].append(normX(self.frame.robots_blue[i].x))
-            robots_dict[i].append(normX(self.frame.robots_blue[i].y))
+            robots_dict[i].append(self.norm_pos(self.frame.robots_blue[i].x))
+            robots_dict[i].append(self.norm_pos(self.frame.robots_blue[i].y))
             robots_dict[i].append(
                 np.sin(np.deg2rad(self.frame.robots_blue[i].theta))
             )
             robots_dict[i].append(
                 np.cos(np.deg2rad(self.frame.robots_blue[i].theta))
             )
-            robots_dict[i].append(normVx(self.frame.robots_blue[i].v_x))
-            robots_dict[i].append(normVx(self.frame.robots_blue[i].v_y))
-            robots_dict[i].append(normVt(self.frame.robots_blue[i].v_theta))
+            robots_dict[i].append(self.norm_v(self.frame.robots_blue[i].v_x))
+            robots_dict[i].append(self.norm_v(self.frame.robots_blue[i].v_y))
+            robots_dict[i].append(self.norm_w(self.frame.robots_blue[i].v_theta))
 
         rotaded_obs = list()
         for i in range(self.n_robots_control):
@@ -124,19 +124,19 @@ def _frame_to_observations(self):
         for idx in range(self.n_robots_control):
             observation = []
 
-            observation.append(normX(self.frame.ball.x))
-            observation.append(normX(self.frame.ball.y))
-            observation.append(normVx(self.frame.ball.v_x))
-            observation.append(normVx(self.frame.ball.v_y))
+            observation.append(self.norm_pos(self.frame.ball.x))
+            observation.append(self.norm_pos(self.frame.ball.y))
+            observation.append(self.norm_v(self.frame.ball.v_x))
+            observation.append(self.norm_v(self.frame.ball.v_y))
 
             observation += robots[idx]
 
             for i in range(self.n_robots_yellow):
-                observation.append(normX(self.frame.robots_yellow[i].x))
-                observation.append(normX(self.frame.robots_yellow[i].y))
-                observation.append(normVx(self.frame.robots_yellow[i].v_x))
-                observation.append(normVx(self.frame.robots_yellow[i].v_y))
-                observation.append(normVt(self.frame.robots_yellow[i].v_theta))
+                observation.append(self.norm_pos(self.frame.robots_yellow[i].x))
+                observation.append(self.norm_pos(self.frame.robots_yellow[i].y))
+                observation.append(self.norm_v(self.frame.robots_yellow[i].v_x))
+                observation.append(self.norm_v(self.frame.robots_yellow[i].v_y))
+                observation.append(self.norm_w(self.frame.robots_yellow[i].v_theta))
 
             observations.append(np.array(observation, dtype=np.float))
 
@@ -156,13 +156,13 @@ def _get_commands(self, actions):
 
         for i in range(self.n_robots_control, self.n_robots_blue):
             actions = self.action_space.sample()
-            v_wheel1, v_wheel2 = self._actions_to_v_wheels(actions)
+            v_wheel1, v_wheel2 = self._actions_to_v_wheels(actions[0])
             commands.append(Robot(yellow=False, id=i, v_wheel1=v_wheel1,
                                   v_wheel2=v_wheel2))
 
         for i in range(self.n_robots_yellow):
             actions = self.action_space.sample()
-            v_wheel1, v_wheel2 = self._actions_to_v_wheels(actions)
+            v_wheel1, v_wheel2 = self._actions_to_v_wheels(actions[0])
             commands.append(Robot(yellow=True, id=i, v_wheel1=v_wheel1,
                                   v_wheel2=v_wheel2))
 
@@ -193,7 +193,7 @@ def _ball_grad(self):
         if self.previous_ball_potential is not None:
             diff = ball_potential - self.previous_ball_potential
             grad_ball_potential = np.clip(diff * 3 / self.time_step,
-                                          -1.0, 1.0)
+                                          -5.0, 5.0)
 
         self.previous_ball_potential = ball_potential
 
@@ -216,7 +216,7 @@ def _move_reward(self, robot_idx: int):
 
         move_reward = np.dot(robot_ball, robot_vel)
 
-        move_reward = np.clip(move_reward / 0.4, -1.0, 1.0)
+        move_reward = np.clip(move_reward / 0.4, -5.0, 5.0)
         return move_reward
 
     def _energy_penalty(self, robot_idx: int):
@@ -279,6 +279,7 @@ def _calculate_reward_and_done(self):
             self.rsim.reset(initial_pos_frame)
             self.frame = self.rsim.get_frame()
             self.last_frame = None
+            self.previous_ball_potential = None
 
         done = self.steps * self.time_step >= 300
 
@@ -348,16 +349,16 @@ def load_opp(self):
 
     def _opp_obs(self):
         observation = []
-        observation.append(normX(-self.frame.ball.x))
-        observation.append(normX(self.frame.ball.y))
-        observation.append(normVx(-self.frame.ball.v_x))
-        observation.append(normVx(self.frame.ball.v_y))
+        observation.append(self.norm_pos(-self.frame.ball.x))
+        observation.append(self.norm_pos(self.frame.ball.y))
+        observation.append(self.norm_v(-self.frame.ball.v_x))
+        observation.append(self.norm_v(self.frame.ball.v_y))
 
         #  we reflect the side that the opp is attacking,
         #  so that he will attack towards the goal where the goalkeeper is
         for i in range(self.n_robots_yellow):
-            observation.append(normX(-self.frame.robots_yellow[i].x))
-            observation.append(normX(self.frame.robots_yellow[i].y))
+            observation.append(self.norm_pos(-self.frame.robots_yellow[i].x))
+            observation.append(self.norm_pos(self.frame.robots_yellow[i].y))
 
             observation.append(
                 np.sin(np.deg2rad(self.frame.robots_yellow[i].theta))
@@ -365,17 +366,17 @@ def _opp_obs(self):
             observation.append(
                 -np.cos(np.deg2rad(self.frame.robots_yellow[i].theta))
             )
-            observation.append(normVx(-self.frame.robots_yellow[i].v_x))
-            observation.append(normVx(self.frame.robots_yellow[i].v_y))
+            observation.append(self.norm_v(-self.frame.robots_yellow[i].v_x))
+            observation.append(self.norm_v(self.frame.robots_yellow[i].v_y))
 
-            observation.append(normVt(-self.frame.robots_yellow[i].v_theta))
+            observation.append(self.norm_w(-self.frame.robots_yellow[i].v_theta))
 
         for i in range(self.n_robots_blue):
-            observation.append(normX(-self.frame.robots_blue[i].x))
-            observation.append(normX(self.frame.robots_blue[i].y))
-            observation.append(normVx(-self.frame.robots_blue[i].v_x))
-            observation.append(normVx(self.frame.robots_blue[i].v_y))
-            observation.append(normVt(-self.frame.robots_blue[i].v_theta))
+            observation.append(self.norm_pos(-self.frame.robots_blue[i].x))
+            observation.append(self.norm_pos(self.frame.robots_blue[i].y))
+            observation.append(self.norm_v(-self.frame.robots_blue[i].v_x))
+            observation.append(self.norm_v(self.frame.robots_blue[i].v_y))
+            observation.append(self.norm_w(-self.frame.robots_blue[i].v_theta))
 
         return np.array(observation)
 
diff --git a/envs/rc_gym/vss/vss_gym_base.py b/envs/rc_gym/vss/vss_gym_base.py
index b3157932..433117a9 100644
--- a/envs/rc_gym/vss/vss_gym_base.py
+++ b/envs/rc_gym/vss/vss_gym_base.py
@@ -12,22 +12,35 @@
 import numpy as np
 from rc_gym.Entities import Frame, Robot
 from rc_gym.Utils import RCRender
-from rc_gym.Simulators.rsim_vss import RSimVSS
+from rc_gym.Simulators.rsim import RSimVSS
 
 
 class VSSBaseEnv(gym.Env):
+    NORM_BOUNDS = 1.2
+    
     def __init__(self, field_type: int,
                  n_robots_blue: int, n_robots_yellow: int, time_step: float):
+        # Initialize Simulator
         self.time_step = time_step
         self.rsim = RSimVSS(field_type=field_type,
                                       n_robots_blue=n_robots_blue,
                                       n_robots_yellow=n_robots_yellow,
                                       time_step_ms=int(self.time_step*1000))
-        self.field_type: int = field_type
         self.n_robots_blue: int = n_robots_blue
         self.n_robots_yellow: int = n_robots_yellow
-        self.field_params: Dict[str,
-                                np.float64] = self.rsim.get_field_params()
+
+        # Get field dimensions
+        self.field_type: int = field_type
+        self.field_params: Dict[str, np.float64] = self.rsim.get_field_params()
+        self.norm_max_pos = max(self.field_params['field_width'] / 2,
+                                (self.field_params['field_length'] / 2)
+                                 + self.field_params['penalty_length']
+                                )
+        self.norm_max_v = self.rsim.linear_speed_range
+        # 0.0425 = robot radius (0.0375) + wheel thicknees (0.05)
+        self.norm_max_w = np.rad2deg(self.rsim.linear_speed_range / 0.0425)
+        
+        # Initiate 
         self.frame: Frame = None
         self.last_frame: Frame = None
         self.view = None
@@ -109,3 +122,24 @@ def _calculate_reward_and_done(self):
     def _get_initial_positions_frame(self) -> Frame:
         '''returns frame with robots initial positions'''
         raise NotImplementedError
+
+    def norm_pos(self, pos):
+        return np.clip(
+            pos / self.norm_max_pos,
+            -self.NORM_BOUNDS,
+            self.NORM_BOUNDS
+        )
+
+    def norm_v(self, v):
+        return np.clip(
+            v / self.norm_max_v,
+            -self.NORM_BOUNDS,
+            self.NORM_BOUNDS
+        )
+
+    def norm_w(self, w):
+        return np.clip(
+            w / self.norm_max_w,
+            -self.NORM_BOUNDS,
+            self.NORM_BOUNDS
+        )


Test patch:
diff --git a/agents/test.py b/agents/test.py
new file mode 100644
index 00000000..f40e0f60
--- /dev/null
+++ b/agents/test.py
@@ -0,0 +1,15 @@
+import gym
+import rc_gym
+
+# Using penalty env
+env = gym.make('VSS3v3-v0')
+
+env.reset()
+# Run for 1 episode and print reward at the end
+for i in range(1):
+    done = False
+    while not done:
+        action = env.action_space.sample()
+        next_state, reward, done, _ = env.step(action)
+        env.render()
+    print(reward)
\ No newline at end of file