Explosive gradient ,gradient fade TD3 DDPG

chamovalera: <p>I use pytorch 1.13.1+cu116 and I have a gradient problem I have tried to modify the learning rate, initialize the weights of the network, change the weights of the network, normalize the data as the weights, and I still have this problem of gradient explosive I have done everything but this problem does not disappear, I have written the model in a different way and the problem is not solved. I apply this in TD3 DDPG.</p>
<p>My code,</p>
<aside class="onebox githubblob" data-onebox-src="https://github.com/sfujim/TD3/blob/master/TD3.py">
  <header class="source">

      <a href="https://github.com/sfujim/TD3/blob/master/TD3.py" target="_blank" rel="noopener nofollow ugc">github.com</a>
  </header>

  <article class="onebox-body">
    <h4><a href="https://github.com/sfujim/TD3/blob/master/TD3.py" target="_blank" rel="noopener nofollow ugc">sfujim/TD3/blob/master/TD3.py</a></h4>


      <pre><code class="lang-py">import copy
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F


device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Implementation of Twin Delayed Deep Deterministic Policy Gradients (TD3)
# Paper: https://arxiv.org/abs/1802.09477


class Actor(nn.Module):
	def __init__(self, state_dim, action_dim, max_action):
		super(Actor, self).__init__()

		self.l1 = nn.Linear(state_dim, 256)
		self.l2 = nn.Linear(256, 256)
		self.l3 = nn.Linear(256, action_dim)
</code></pre>



  This file has been truncated. <a href="https://github.com/sfujim/TD3/blob/master/TD3.py" target="_blank" rel="noopener nofollow ugc">show original</a>

  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>

<p>import copy<br>
import numpy as np<br>
import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F</p>
<p>device = torch.device(“cuda” if torch.cuda.is_available() else “cpu”)</p>
<h1><a name="implementation-of-twin-delayed-deep-deterministic-policy-gradients-td3-1" class="anchor" href="#implementation-of-twin-delayed-deep-deterministic-policy-gradients-td3-1"></a>Implementation of Twin Delayed Deep Deterministic Policy Gradients (TD3)</h1>
<h1><a name="paper-httpsarxivorgabs180209477-2" class="anchor" href="#paper-httpsarxivorgabs180209477-2"></a>Paper: <a href="https://arxiv.org/abs/1802.09477" class="inline-onebox" rel="noopener nofollow ugc">[1802.09477] Addressing Function Approximation Error in Actor-Critic Methods</a></h1>
<p>class Actor(nn.Module):<br>
def <strong>init</strong>(self, state_dim, action_dim, max_action):<br>
super(Actor, self).<strong>init</strong>()</p>
<pre><code>	self.l1 = nn.Linear(state_dim, 256)
	self.l2 = nn.Linear(256, 256)
	self.l3 = nn.Linear(256, action_dim)
	
	self.max_action = max_action
	

def forward(self, state):
	a = F.relu(self.l1(state))
	a = F.relu(self.l2(a))
	return self.max_action * torch.tanh(self.l3(a))
</code></pre>
<p>class Critic(nn.Module):<br>
def <strong>init</strong>(self, state_dim, action_dim):<br>
super(Critic, self).<strong>init</strong>()</p>
<pre><code>	# Q1 architecture
	self.l1 = nn.Linear(state_dim + action_dim, 256)
	self.l2 = nn.Linear(256, 256)
	self.l3 = nn.Linear(256, 1)

	# Q2 architecture
	self.l4 = nn.Linear(state_dim + action_dim, 256)
	self.l5 = nn.Linear(256, 256)
	self.l6 = nn.Linear(256, 1)


def forward(self, state, action):
	sa = torch.cat([state, action], 1)

	q1 = F.relu(self.l1(sa))
	q1 = F.relu(self.l2(q1))
	q1 = self.l3(q1)

	q2 = F.relu(self.l4(sa))
	q2 = F.relu(self.l5(q2))
	q2 = self.l6(q2)
	return q1, q2


def Q1(self, state, action):
	sa = torch.cat([state, action], 1)

	q1 = F.relu(self.l1(sa))
	q1 = F.relu(self.l2(q1))
	q1 = self.l3(q1)
	return q1
</code></pre>
<p>class TD3(object):<br>
def <strong>init</strong>(<br>
self,<br>
state_dim,<br>
action_dim,<br>
max_action,<br>
discount=0.99,<br>
tau=0.005,<br>
policy_noise=0.2,<br>
noise_clip=0.5,<br>
policy_freq=2<br>
):</p>
<pre><code>	self.actor = Actor(state_dim, action_dim, max_action).to(device)
	self.actor_target = copy.deepcopy(self.actor)
	self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=3e-4)

	self.critic = Critic(state_dim, action_dim).to(device)
	self.critic_target = copy.deepcopy(self.critic)
	self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=3e-4)

	self.max_action = max_action
	self.discount = discount
	self.tau = tau
	self.policy_noise = policy_noise
	self.noise_clip = noise_clip
	self.policy_freq = policy_freq

	self.total_it = 0


def select_action(self, state):
	state = torch.FloatTensor(state.reshape(1, -1)).to(device)
	return self.actor(state).cpu().data.numpy().flatten()


def train(self, replay_buffer, batch_size=256):
	self.total_it += 1

	# Sample replay buffer 
	state, action, next_state, reward, not_done = replay_buffer.sample(batch_size)

	with torch.no_grad():
		# Select action according to policy and add clipped noise
		noise = (
			torch.randn_like(action) * self.policy_noise
		).clamp(-self.noise_clip, self.noise_clip)
		
		next_action = (
			self.actor_target(next_state) + noise
		).clamp(-self.max_action, self.max_action)

		# Compute the target Q value
		target_Q1, target_Q2 = self.critic_target(next_state, next_action)
		target_Q = torch.min(target_Q1, target_Q2)
		target_Q = reward + not_done * self.discount * target_Q

	# Get current Q estimates
	current_Q1, current_Q2 = self.critic(state, action)

	# Compute critic loss
	critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss(current_Q2, target_Q)

	# Optimize the critic
	self.critic_optimizer.zero_grad()
	critic_loss.backward()
	self.critic_optimizer.step()

	# Delayed policy updates
	if self.total_it % self.policy_freq == 0:

		# Compute actor losse
		actor_loss = -self.critic.Q1(state, self.actor(state)).mean()
		
		# Optimize the actor 
		self.actor_optimizer.zero_grad()
		actor_loss.backward()
		self.actor_optimizer.step()

		# Update the frozen target models
		for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
			target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)

		for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):
			target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)


def save(self, filename):
	torch.save(self.critic.state_dict(), filename + "_critic")
	torch.save(self.critic_optimizer.state_dict(), filename + "_critic_optimizer")
	
	torch.save(self.actor.state_dict(), filename + "_actor")
	torch.save(self.actor_optimizer.state_dict(), filename + "_actor_optimizer")


def load(self, filename):
	self.critic.load_state_dict(torch.load(filename + "_critic"))
	self.critic_optimizer.load_state_dict(torch.load(filename + "_critic_optimizer"))
	self.critic_target = copy.deepcopy(self.critic)

	self.actor.load_state_dict(torch.load(filename + "_actor"))
	self.actor_optimizer.load_state_dict(torch.load(filename + "_actor_optimizer"))
	self.actor_target = copy.deepcopy(self.actor)
</code></pre>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/e/6/e64d2b300fcb070b9f038186b439fe1ac34e3bf9.png" data-download-href="https://discuss.pytorch.org/uploads/default/e64d2b300fcb070b9f038186b439fe1ac34e3bf9" title="Captura de pantalla (60)"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/e/6/e64d2b300fcb070b9f038186b439fe1ac34e3bf9_2_690x388.png" alt="Captura de pantalla (60)" data-base62-sha1="wRl9nVinK6dtiPzT3bUg08wLbOF" width="690" height="388" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/e/6/e64d2b300fcb070b9f038186b439fe1ac34e3bf9_2_690x388.png, https://discuss.pytorch.org/uploads/default/optimized/3X/e/6/e64d2b300fcb070b9f038186b439fe1ac34e3bf9_2_1035x582.png 1.5x, https://discuss.pytorch.org/uploads/default/optimized/3X/e/6/e64d2b300fcb070b9f038186b439fe1ac34e3bf9_2_1380x776.png 2x" data-dominant-color="FBFBFB"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">Captura de pantalla (60)</span><span class="informations">1920×1080 125 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>