Add QDQN Mountain Car agent trained for 600 episodes

README.md

@@ -0,0 +1,59 @@
+---
+tags:
+- MountainCar-v0
+- deep-reinforcement-learning
+- reinforcement-learning
+model-index:
+- name: QDQN
+  results:
+  - task:
+      type: reinforcement-learning
+      name: reinforcement-learning
+    dataset:
+      name: MountainCar-v0
+      type: MountainCar-v0
+    metrics:
+    - type: mean_reward
+      value: -200.0 +/- 0.0
+      name: mean_reward
+      verified: false
+---
+
+# **QDQN** Agent playing **MountainCar-v0**
+This is a trained model of a **QDQN** agent playing **MountainCar-v0**
+using the [qrl-dqn-gym](https://github.com/qdevpsi3/qrl-dqn-gym). 
+
+This agent has been trained for the [research project](https://github.com/agercas/QHack2023_QRL) during the QHack 2023 
+hackathon. The project explores the use of quantum algorithms in reinforcement learning. 
+More details about the project and the trained agent can be found in the [project repository](https://github.com/agercas/QHack2023_QRL).
+
+
+## Usage
+
+```python
+import gym
+import yaml
+import torch
+from model.qnn import QuantumNet
+from model.agent import Agent
+
+# Environment
+env_name = 'MountainCar-v0'
+env = gym.make(env_name)
+
+# Network
+with open('config.yaml', 'r') as f:
+  hparams = yaml.safe_load(f)
+
+net = QuantumNet(
+    n_layers=hparams['n_layers'], 
+    w_input=hparams['w_input'], 
+    w_output=hparams['w_output'],
+    data_reupload=hparams['data_reupload']
+)
+state_dict = torch.load('qdqn-MountainCar-v0.pt', map_location=torch.device('cpu'))
+net.load_state_dict(state_dict)
+
+# Agent
+agent = Agent(net)
+```

config.yaml

@@ -0,0 +1,24 @@
+batch_size: 16
+data_reupload: true
+device: auto
+eps_decay: 0.99
+eps_init: 1.0
+eps_min: 0.01
+gamma: 0.99
+log_ckp_freq: 50
+log_eval_freq: 20
+log_train_freq: 1
+logging: true
+loss: SmoothL1
+lr: 0.001
+lr_input: 0.001
+lr_output: 0.1
+memory: 10000
+n_eval_episodes: 5
+n_layers: 5
+optimizer: RMSprop
+target_freq: 30
+total_episodes: 5000
+train_freq: 10
+w_input: true
+w_output: true

example.py

@@ -0,0 +1,26 @@
+import gym
+import torch
+import yaml
+
+from model.agent import Agent
+from model.qnn import QuantumNet
+
+# Environment
+env_name = 'MountainCar-v0'
+env = gym.make(env_name)
+
+# Network
+with open('config.yaml', 'r') as f:
+    hparams = yaml.safe_load(f)
+
+net = QuantumNet(
+    n_layers=hparams['n_layers'],
+    w_input=hparams['w_input'],
+    w_output=hparams['w_output'],
+    data_reupload=hparams['data_reupload']
+)
+state_dict = torch.load('qdqn-MountainCar-v0.pt', map_location=torch.device('cpu'))
+net.load_state_dict(state_dict)
+
+# Agent
+agent = Agent(net)

model/agent.py

@@ -0,0 +1,48 @@
+import numpy as np
+import torch
+
+
+class Agent:
+    def __init__(self,
+                 net,
+                 action_space=None,
+                 exploration_initial_eps=None,
+                 exploration_decay=None,
+                 exploration_final_eps=None):
+
+        self.net = net
+        self.action_space = action_space
+        self.exploration_initial_eps = exploration_initial_eps
+        self.exploration_decay = exploration_decay
+        self.exploration_final_eps = exploration_final_eps
+        self.epsilon = 0.
+
+    def __call__(self, state, device=torch.device('cpu')):
+        if np.random.random() < self.epsilon:
+            action = self.get_random_action()
+        else:
+            action = self.get_action(state, device)
+
+        return action
+
+    def get_random_action(self):
+        action = self.action_space.sample()
+        return action
+
+    def get_action(self, state, device=torch.device('cpu')):
+        if not isinstance(state, torch.Tensor):
+            state = torch.tensor([state])
+
+        if device.type != 'cpu':
+            state = state.cuda(device)
+
+        q_values = self.net.eval()(state)
+        _, action = torch.max(q_values, dim=1)
+        return int(action.item())
+
+    def update_epsilon(self, step):
+        self.epsilon = max(
+            self.exploration_final_eps, self.exploration_final_eps +
+            (self.exploration_initial_eps - self.exploration_final_eps) *
+            self.exploration_decay**step)
+        return self.epsilon

model/qnn.py

@@ -0,0 +1,82 @@
+import pennylane as qml
+import torch
+import torch.nn as nn
+from torch.nn.parameter import Parameter
+
+
+def encode(n_qubits, inputs):
+    for wire in range(n_qubits):
+        qml.RX(inputs[wire], wires=wire)
+
+
+def layer(n_qubits, y_weight, z_weight):
+    for wire, y_weight in enumerate(y_weight):
+        qml.RY(y_weight, wires=wire)
+    for wire, z_weight in enumerate(z_weight):
+        qml.RZ(z_weight, wires=wire)
+    for wire in range(n_qubits):
+        qml.CZ(wires=[wire, (wire + 1) % n_qubits])
+
+
+def measure(n_qubits):
+    return [qml.expval(qml.PauliZ(wire)) for wire in range(n_qubits)]
+
+
+def get_model(n_qubits, n_layers, data_reupload):
+    # NOTE: need to select an appropriate device
+    # dev = qml.device('lightning.gpu', wires=n_qubits)
+    dev = qml.device("default.qubit", wires=n_qubits)
+    shapes = {
+        "y_weights": (n_layers, n_qubits),
+        "z_weights": (n_layers, n_qubits)
+    }
+
+    @qml.qnode(dev, interface='torch')
+    def circuit(inputs, y_weights, z_weights):
+        for layer_idx in range(n_layers):
+            if (layer_idx == 0) or data_reupload:
+                encode(n_qubits, inputs)
+            layer(n_qubits, y_weights[layer_idx], z_weights[layer_idx])
+        return measure(n_qubits)
+
+    model = qml.qnn.TorchLayer(circuit, shapes)
+
+    return model
+
+
+class QuantumNet(nn.Module):
+    def __init__(self, n_layers, w_input, w_output, data_reupload):
+        super(QuantumNet, self).__init__()
+        self.n_qubits = 2
+        self.n_actions = 3
+        self.data_reupload = data_reupload
+        self.q_layers = get_model(n_qubits=self.n_qubits, n_layers=n_layers, data_reupload=data_reupload)
+        # convert from 2 qubits to 3 actions
+        # not adding more complexity here because we want to learn through quantum circuit
+        self.layer1 = nn.Linear(2, 3)
+
+        if w_input:
+            self.w_input = Parameter(torch.Tensor(self.n_qubits))
+            nn.init.normal_(self.w_input)
+        else:
+            self.register_parameter("w_input", None)
+        if w_output:
+            self.w_output = Parameter(torch.Tensor(self.n_actions))
+            nn.init.normal_(self.w_output, mean=90.0)
+        else:
+            self.register_parameter("w_output", None)
+
+    def forward(self, inputs):
+        if self.w_input is not None:
+            inputs = inputs * self.w_input
+        inputs = torch.atan(inputs)
+        q_outputs = self.q_layers(inputs)
+        q_outputs = (1 + q_outputs) / 2
+
+        outputs = self.layer1(q_outputs)
+
+        if self.w_output is not None:
+            outputs = outputs * self.w_output
+        else:
+            outputs = 90 * outputs
+        return outputs

qdqn-MountainCar-v0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af4b523003e5cc160264642e07f71c9353648347fc93ca3f58504969bccecae2
+size 2231