Some error of gym project

username: 111348
cooked: <p>import gym</p>
<p>import collections#</p>
<p>import random</p>
<p>import torch</p>
<p>import torch.nn as nn</p>
<p>import torch.nn.functional as F</p>
<p>import torch.optim as optim</p>
<p><span class="hashtag">#Hyperparameters</span></p>
<p>lr_mu = 0.0005</p>
<p>lr_q = 0.001</p>
<p>gamma         = 0.99</p>
<p>buffer_limit  = 50000</p>
<p>batch_size    = 32</p>
<p>tau = 0.005</p>
<p>class ReplayBuffer():</p>
<pre><code>def __init__(self):

    self.buffer = collections.deque(maxlen=buffer_limit)



def put(self, data):

    self.buffer.append(data)



def sample(self, n):

    mini_batch = random.sample(self.buffer, n)

    s_lst, a_lst, r_lst, s_prime_lst, done_mask_lst = [], [], [], [], []

    

    for transition in mini_batch:

        s, a, r, s_prime, done_mask = transition

        s_lst.append(s)

        a_lst.append([a])

        r_lst.append([r])

        s_prime_lst.append(s_prime)

        done_mask_lst.append([done_mask])

    return torch.tensor(s_lst, dtype=torch.float), torch.tensor(a_lst), \

           torch.tensor(r_lst), torch.tensor(s_prime_lst, dtype=torch.float), \

           torch.tensor(done_mask_lst)



def size(self):

    return len(self.buffer)
</code></pre>
<p>class Qnet(nn.Module):</p>
<pre><code>def __init__(self):

    super(Qnet, self).__init__()

    self.fc_s = nn.Linear(3, 128)

    self.fc_a = nn.Linear(128, 64)

    self.fc_q = nn.Linear(64,64)

    self.fc_3 = nn.Linear(64, 1)

def forward(self, a):

    h1 = F.relu(self.fc_s(a))

    h2 = F.relu(self.fc_a(a))

    cat = torch.cat([h1,h2],dim=1)

    q = F.relu(self.fc_q(cat))

    q = self.fc_3(q)

    return q
</code></pre>
<p>class Munet(nn.Module):</p>
<pre><code>def __init__(self):

    super(Munet, self).__init__()

    self.fc1 = nn.Linear(3, 128)

    self.fc2 = nn.Linear(128, 64)

    self.fc_mu = nn.Linear(64, 1)

def forward(self, x):

    x = F.relu(self.fc1(x))

    x = F.relu(self.fc2(x))

    mu = torch.tanh(self.fc_mu(x))*2

    return mu
</code></pre>
<p>def train(mu,mu_target,q,q_target,memory,q_optimizer,mu_optimizer):</p>
<pre><code>s,a,r,s_prime,done_mask = memory.sample(batch_size)

td_target = r+gamma*q_target(s_prime,mu_target(s_prime))

q_loss =  F.smooth_l1_loss(q(s,a), td_target.detach())

q_optimizer.zero_grad()

q_loss.backward()

q_optimizer.step()

mu_loss = -q(s,mu(s)).mean()

mu_optimizer.zero_grad()

mu_loss.backward()

mu_optimizer.step()
</code></pre>
<p>def main():</p>
<pre><code>env = gym.make('Pendulum-v0')

memory = ReplayBuffer()

q = Qnet()

q_target = Qnet()

q_target.load_state_dict(q.state_dict())

mu =Munet()

mu_target =  Munet()

mu_target.load_state_dict(mu.state_dict())

print_interval = 20

score = 0.0  

min_play_reward = 0

iterations = 10000

q_optimizer = optim.Adam(q.parameters(), lr=lr_q)

mu_optimizer = optim.Adam(mu.parameters(), lr=lr_mu)



for interation in range(iterations):

    s = env.reset()

    for t in range(300):

      a = mu(torch.from_numpy(s).float())

      a = a.item()

      s_prime, r, done, info = env.step([a])

      memory.put((s,a,r/100.0,s_prime,done))

      score +=r

      s = s_prime

    

    if memory.size()&gt;2000:

      for i in range(10):

        train(mu, mu_target, q, q_target, memory, q_optimizer,mu_optimizer)

        soft_update(mu,mu_target)

        soft_update(q,q_target)
</code></pre>
<p>def soft_update(net,net_target):</p>
<p>for param_target, param in zim(net_target.parametes(),\</p>
<pre><code>                              net.parameters()):

param_target.data.copy_(param_target.data*(1.0-tau)+\

                                param.data(tau))

    

env.close()
</code></pre>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:</p>
<pre><code>main()
</code></pre>
<p>It is my code of Pendulum project. it has some error<br>
"forward() takes 2 positional arguments but 3 were given "<br>
As far as i’m concerned, i think that cause of its is "class Qnet(nn.Module): ,class Munet(nn.Module): "<br>
How can i fix the code?</p>
<p>Thank you</p>

username: ptrblck
cooked: <p><code>Qnet</code> and <code>Munet</code> both accept one parameter, so these lines of code will raise the error:</p>
<pre><code class="lang-python">q_target(s_prime,mu_target(s_prime)) # q_target gets two inputs
q_loss =  F.smooth_l1_loss(q(s,a), td_target.detach()) # q gets two inputs
mu_loss = -q(s,mu(s)).mean() # q gets two inputs
</code></pre>
<p>You might want to add the second argument to <code>Qnet.forward</code> or just pass the expected <code>a</code> argument.</p>