Repository: WPI-MMR/gym_solo

Issue:
Create Episode Termination Factory
After running the first couple of simulations, the robot usually ends up on the ground and twitching. This is going to be an artifact of RL and we need to make a way to early terminate an episode so that the agent can start fresh.

I figure that the interface to this factory would be similar to our `ObservationFactory` and `RewardsFactory`. Create your env, register your stopping criteria, and you are off the races!

Discussion:

Patch:
diff --git a/gym_solo/core/termination.py b/gym_solo/core/termination.py
new file mode 100644
index 0000000..e6f2682
--- /dev/null
+++ b/gym_solo/core/termination.py
@@ -0,0 +1,83 @@
+from abc import ABC, abstractmethod
+
+
+class Termination(ABC):
+  @abstractmethod
+  def reset(self):
+    """Resets the state of the termination condition
+    """
+    pass
+
+  @abstractmethod
+  def is_terminated(self) -> bool:
+    """This should implement functionanlity that determines when an episode
+    should terminate
+    """
+    pass
+
+
+class TerminationFactory:
+  def __init__(self):
+    """Create a new Termination Factory. This way users can add multiple
+    different termination conditions and different ways to execute them to
+    signal the end of an episode
+    """
+    self._terminations = []
+    self._use_or = True
+
+  def register_termination(self, *terminations):
+    """Add termination conditions used to evaluate when an episode
+    has ended.
+
+    Args:
+      terminations (Termination): Termination condition(s) used to signal the 
+      end of an episode 
+    """
+    self._terminations.extend(terminations)
+
+  def is_terminated(self) -> bool:
+    """Tells whether an episode is terminated based on the registered
+    termination conditions and value of _is_or attribute
+    """
+    
+    if not self._use_or:
+      raise ValueError('No termination condition other than OR is defined')
+
+    for termination in self._terminations:
+      if termination.is_terminated():
+        return True
+    
+    return False
+
+  def reset(self):
+    """Resets all the termination conditions stored
+    """
+    for termination in self._terminations:
+      termination.reset()
+
+
+class TimeBasedTermination(Termination):
+  """The episode terminated when the step_delta becomes greater than
+  max_step_delta
+  """
+  def __init__(self, max_step_delta: int):
+    """Initializing TimeBasedTermination
+    Args:
+      max_step_delta (int): To specify the number of steps that a episode is 
+      made of
+    """
+    self.max_step_delta = max_step_delta
+    self.reset()
+  
+  def reset(self):
+    """Resetting the stepping counter to 0. Usually would happen at the instance
+    creation or when the termination condition has been met
+    """
+    self.step_delta = 0
+
+  def is_terminated(self) -> bool:
+    """Return true when step_delta becomes greater than max_step_delta.
+    Otherwise return false.
+    """
+    self.step_delta += 1
+    return self.step_delta > self.max_step_delta 
\ No newline at end of file
diff --git a/gym_solo/envs/solo8v2vanilla.py b/gym_solo/envs/solo8v2vanilla.py
index 4521e96..6632e80 100644
--- a/gym_solo/envs/solo8v2vanilla.py
+++ b/gym_solo/envs/solo8v2vanilla.py
@@ -78,6 +78,8 @@ class Solo8VanillaEnv(gym.Env):
     obs_values, obs_labels = self.obs_factory.get_obs()
     reward = self.reward_factory.get_reward()
 
+    # TODO: The termination is always being returned false.
+    # This would change with termination factory
     return obs_values, reward, False, {'labels': obs_labels}
 
   def reset(self) -> solo_types.obs:
@@ -89,6 +91,7 @@ class Solo8VanillaEnv(gym.Env):
     p.removeBody(self.robot)
     self.robot, _ = self._load_robot()
 
+    # TODO: We need to change this to have the robot always be in home position
     # Let gravity do it's thing and reset the environment deterministically
     for i in range(1000):
       p.setJointMotorControlArray(self.robot, 


Test patch:
diff --git a/.github/workflows/python-test-coverage.yml b/.github/workflows/python-test-coverage.yml
index 18fdb6a..3b83c28 100644
--- a/.github/workflows/python-test-coverage.yml
+++ b/.github/workflows/python-test-coverage.yml
@@ -10,8 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # Maintain compatibility with 3.7 due to compatibility with tf 1.
-        python-version: [3.7, 3.8]
+        # Maintain compatibility with 3.6 in case we need to use Pinocchio in the future.
+        # python-version: [3.6, 3.8]
+        python-version: [3.8]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/gym_solo/core/test_termination_conditions.py b/gym_solo/core/test_termination_conditions.py
new file mode 100644
index 0000000..bad6698
--- /dev/null
+++ b/gym_solo/core/test_termination_conditions.py
@@ -0,0 +1,30 @@
+import unittest
+from gym_solo.core import termination
+
+class TestTimeBasedTermination(unittest.TestCase):
+    def test_attributes(self):
+      max_step_delta = 3
+      term = termination.TimeBasedTermination(max_step_delta)
+      self.assertEqual(max_step_delta, term.max_step_delta)
+      self.assertEqual(0, term.step_delta)
+
+    def test_reset(self):
+      max_step_delta = 3
+      term = termination.TimeBasedTermination(max_step_delta)
+
+      for i in range(max_step_delta):
+        self.assertFalse(term.is_terminated())
+
+      term.reset()
+      self.assertEqual(0,term.step_delta)
+
+    def test_is_terminated(self):
+      max_step_delta = 3
+      term = termination.TimeBasedTermination(max_step_delta)
+      
+      for i in range(max_step_delta):
+        self.assertEqual(False, term.is_terminated())
+        self.assertEqual(i+1, term.step_delta)
+
+      self.assertEqual(True, term.is_terminated())
+      self.assertEqual(max_step_delta + 1, term.step_delta)
\ No newline at end of file
diff --git a/gym_solo/core/test_termination_factory.py b/gym_solo/core/test_termination_factory.py
new file mode 100644
index 0000000..6584f0e
--- /dev/null
+++ b/gym_solo/core/test_termination_factory.py
@@ -0,0 +1,56 @@
+import unittest
+from gym_solo.core import termination
+
+# TODO: Move this to gym_solo.testing
+class DummyTermination(termination.Termination):
+  def __init__(self, body_id: int, termination_var: bool):
+    self.body_id = body_id
+    self.termination_var = termination_var
+    self.reset_counter = 0
+    self.reset()
+    
+  def reset(self):
+    self.reset_counter += 1
+
+  def is_terminated(self) -> bool:
+    return self.termination_var
+
+
+class TestTerminationFactory(unittest.TestCase):
+  def test_initialization(self):
+    termination_factory = termination.TerminationFactory()
+    self.assertListEqual(termination_factory._terminations, [])
+    self.assertTrue(termination_factory._use_or)
+
+  def test_register_termination(self):
+    termination_factory = termination.TerminationFactory()
+    dummy_termination_1 = DummyTermination(0, True)
+    dummy_termination_2 = DummyTermination(0, False)
+    termination_factory.register_termination(dummy_termination_1, 
+      dummy_termination_2)
+    self.assertEqual(len(termination_factory._terminations), 2)
+
+  def test_is_terminated(self):
+    dummy_termination_true_1 = DummyTermination(0, True)
+    dummy_termination_false_1 = DummyTermination(0, False)    
+    dummy_termination_false_2 = DummyTermination(0, False)
+    
+    with self.subTest('True and False Termination'):
+      termination_factory = termination.TerminationFactory()
+      termination_factory.register_termination(dummy_termination_true_1, 
+        dummy_termination_false_1)
+      self.assertTrue(termination_factory.is_terminated())    
+
+    with self.subTest('False and False Termination'):
+      termination_factory = termination.TerminationFactory()
+      termination_factory.register_termination(dummy_termination_false_1, 
+        dummy_termination_false_2)
+      self.assertFalse(termination_factory.is_terminated())
+  
+  def test_reset(self):
+    dummy_termination = DummyTermination(0, True)
+    termination_factory = termination.TerminationFactory()
+    termination_factory.register_termination(dummy_termination)
+    self.assertEqual(1, dummy_termination.reset_counter)
+    termination_factory.reset()
+    self.assertEqual(2, dummy_termination.reset_counter)
\ No newline at end of file