DQN Snake not making great progress

username: Durotan97
cooked: <p>Hi everyone,</p>
<p>I am trying to train a DQN for a multiplayer snake version. I’ve tried multiple versions, but none of them seem to work well so I assume that there is an error in my code. Could somebody check please?</p>
<p>This is a dueling DQN. The network is a CNN which receives a 10(number of frames)<em>7</em>11(Board size) input. Frames 1-5 are the last board position (Myself, Opponent1, Opponent2,Opponent3, Food) and Frames 6-10 are the current board position. This is important because the opposite of the last move (e.g. North when South) is forbidden. Reward is defined as +1 at every step , -100 when dying and +10 when eating food. Is there an error somewhere?</p>
<p>from kaggle_environments import evaluate, make</p>
<p>from kaggle_environments.envs.hungry_geese.hungry_geese import Observation, Configuration, Action, row_col, random_agent, greedy_agent</p>
<p>import numpy as np<br>
import torch<br>
import torch.nn as nn<br>
import random<br>
import torch.nn.functional as F<br>
import matplotlib.pyplot as plt<br>
<span class="hashtag">#class</span> to handle the storage of past transitions</p>
<p>class Replay_buffer():<br>
def <strong>init</strong>(self,capacity):<br>
self.capacity = capacity<br>
self.pos = 0<br>
self.memory = list()<br>
self.action_size = 1<br>
self.reward_size = 1</p>
<pre><code>def add(self,element):
    element['state'] = torch.Tensor(element['state'])
    element['action'] = torch.LongTensor([element['action']])
    element['new_state'] = torch.Tensor(element['new_state'])
    element['reward'] = torch.Tensor([element['reward']])
    if len(self.memory) &lt; self.capacity:
        self.memory.append(element)
    else:
        self.memory[self.pos] = element
        self.pos = (self.pos+1) %self.capacity
def get_item(self,batch_size):
    return random.sample(self.memory,batch_size)
</code></pre>
<p>class Q_network(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(Q_network,self).<strong>init</strong>()<br>
self.input = nn.Sequential(<br>
nn.Conv2d(10, 24, kernel_size=2, stride = 2,padding = 1),<br>
nn.BatchNorm2d(24),<br>
nn.ReLU()</p>
<pre><code>    )

    

    self.hidden = nn.Sequential(
        nn.Conv2d(24, 32, kernel_size=2, stride=2 , padding = 1),
        nn.BatchNorm2d(32),
        nn.ReLU()

    )

    self.out_value = nn.Sequential(
        nn.Linear(384,128),
        nn.ReLU(),
        nn.Linear(128,1)

    )

    self.out_advantage = nn.Sequential(
        nn.Linear(384,128),
        nn.ReLU(),
        nn.Linear(128,4)

    )

def forward(self,x):
    x = self.input(x)
    x = self.hidden(x)
    x = torch.flatten(x,start_dim=1)
    x_v = self.out_value(x)
    x_a = self.out_advantage(x)
    x = x_v + (x_a-x_a.mean())
    return x
</code></pre>
<p>class DQN(nn.Module):</p>
<pre><code>def __init__(self):
    super(DQN,self).__init__()
    self.q_network = Q_network_linear()
    self.loss_fn = None
    self.optimizer = None

def forward(self,x):
    x = self.q_network(x)
    return x
</code></pre>
<p>def state_input_to_features(state):<br>
layers = np.zeros((5,77))<br>
for i,goose in enumerate(state.geese):<br>
if goose:<br>
layers[i][goose[0]] = 1<br>
if len(goose) &gt; 1:<br>
for pos in goose[1:]:<br>
layers[i][pos] = 0.5<br>
for food in state.food:<br>
layers[4][food] = 1<br>
layers = layers.reshape(5,7,11)<br>
return layers</p>
<p>def select_action(network,state,epsilon):<br>
if np.random.random() &lt; epsilon:<br>
return np.random.randint(0,4)<br>
else:<br>
with torch.no_grad():<br>
network.eval()<br>
tmp = network(state)<br>
tmp = tmp.max(1)<br>
return tmp[1].view(-1,1)</p>
<p>def training_step(network,target_network, replay_buffer):<br>
network.train()<br>
x = replay_buffer.get_item(network.batch_size)<br>
network.optimizer.zero_grad()<br>
s = torch.stack([tmp[‘state’] for tmp in x])<br>
a = torch.stack([tmp[‘action’] for tmp in x])<br>
s_new = torch.stack([tmp[‘new_state’] for tmp in x])<br>
r = torch.stack([tmp[‘reward’] for tmp in x])<br>
done = [tmp[‘done’] for tmp in x]<br>
output = network(s.view(network.batch_size,10,7,11))<br>
predicted_values = output.gather(1,a)<br>
with torch.no_grad():<br>
s_new = s_new.view(network.batch_size,10,7,11)<br>
output_target = target_network(s_new).gather(1,select_action(network,s_new,0))<br>
expected_values = output_target.view(network.batch_size,1)*0.99+r<br>
for i,d in enumerate(done):<br>
<span class="hashtag">#no</span> further state reward when done<br>
if d:<br>
expected_values[i] = r[i]<br>
loss = network.loss_fn(predicted_values,expected_values)<br>
loss.backward()<br>
for param in network.parameters():<br>
param.grad.data.clamp_(-1,1)<br>
network.optimizer.step()<br>
return loss.item()</p>
<p>def get_reward(last_observation,current_observation):<br>
if not current_observation.geese[0]:<br>
return -100<br>
elif not last_observation:<br>
if len(current_observation.geese[0]) &gt; 1:<br>
return 10<br>
else:<br>
return 1<br>
elif len(current_observation.geese[0]) &gt; len(last_observation.geese[0]):<br>
return 10<br>
else:<br>
return 1</p>
<p>def train_agent(path,epochs):<br>
env = make(‘hungry_geese’, debug = True)<br>
training = env.train([None,“greedy”,“greedy”,“greedy”])<br>
obs = training<br>
replay_buffer = Replay_buffer(100000)<br>
if path:<br>
network = load_network(path)<br>
target_network = load_network(path)<br>
else:<br>
network = DQN()<br>
target_network = DQN()<br>
network.loss_fn = torch.nn.SmoothL1Loss()<br>
network.optimizer = torch.optim.RMSprop(network.parameters(), lr  = 1e-2)<br>
network.batch_size = 32<br>
state = env.reset(num_agents = 4)[0].observation<br>
losses = list()<br>
episode_returns = list()<br>
episode_return = 0<br>
last_observation = None<br>
last_position = torch.zeros(5,7,11)<br>
for iteration in range(epochs):<br>
env.render()<br>
current_state = torch.Tensor(state_input_to_features(state))<br>
s = torch.cat((last_position,current_state),dim=0)<br>
action = select_action(network,s.view(10,7,11).unsqueeze(dim=0),0.2)<br>
if type(action) != int:<br>
action = action.item()<br>
obs,reward,done,info = training.step(action_dict[action])<br>
reward = get_reward(last_observation,obs)<br>
last_observation = obs<br>
last_position = current_state<br>
s_2 = torch.cat((current_state,torch.Tensor(state_input_to_features(obs))),dim=0)<br>
replay_buffer.add({‘state’: s, ‘action’: action, ‘new_state’: s_2 , ‘reward’: reward, ‘done’: done})<br>
if iteration &gt;= network.batch_size:<br>
losses.append(training_step(network,target_network,replay_buffer))<br>
if iteration % 1000 == 0:<br>
target_network.load_state_dict(network.state_dict())<br>
plt.plot(range(len(losses)),losses, label = ‘Loss’)<br>
plt.legend()<br>
plt.show()<br>
plt.plot(range(len(episode_returns)),episode_returns, label = ‘Episodes’)<br>
plt.legend()<br>
plt.show()<br>
state = obs<br>
episode_return += reward<br>
if done:<br>
state = env.reset(num_agents = 4)[0].observation<br>
episode_returns.append(episode_return)<br>
episode_return = 0<br>
last_observation = None<br>
last_position = torch.zeros(5,7,11)<br>
torch.save(network.state_dict(),‘soft_goose.pt’)<br>
plt.plot(range(len(losses)),losses, label = ‘Loss’)<br>
plt.legend()<br>
plt.show()</p>
<pre><code>plt.plot(range(len(episode_returns)),episode_returns, label = 'Episodes')
plt.legend()
plt.show()
</code></pre>
<p>def load_network(path):<br>
network = DQN()<br>
network.load_state_dict(torch.load(path, map_location=torch.device(‘cpu’)))<br>
network = network.eval()<br>
return network</p>