I need your help on how I can resolve this error:

amahoro12: <p>I need your help on how I can resolve this error:<br>
next_state, reward, done, info = env.step(action)<br>
TypeError: cannot unpack non-iterable int object</p>
<p>class QNetwork(nn.Module):<br>
def <strong>init</strong>(self, state_size, action_size, seed):<br>
super(QNetwork, self).<strong>init</strong>()<br>
self.seed = torch.manual_seed(seed)<br>
self.fc1 = nn.Linear(state_size, 32)<br>
self.fc2 = nn.Linear(32, 64)<br>
self.fc3 = nn.Linear(64, action_size)</p>
<pre><code>def forward(self, x):
    """Forward pass"""
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)

    return x
</code></pre>
<p>class ReplayBuffer:<br>
def <strong>init</strong>(self, buffer_size, batch_size, seed):<br>
self.batch_size = batch_size<br>
self.seed = random.seed(seed)<br>
self.memory = deque(maxlen=buffer_size)<br>
self.experience = namedtuple(“Experience”, field_names=[“state”, “action”, “reward”, “next_state”, “done”])</p>
<pre><code>def add(self, state, action, reward, next_state, done):
    """Add experience"""
    experience = self.experience(state, action, reward, next_state, done)
    self.memory.append(experience)

def sample(self):
    experiences = random.sample(self.memory, k=self.batch_size)

    # Convert to torch tensors
    states = torch.from_numpy(
        np.vstack([experience.state for experience in experiences if experience is not None])).float().to(device)
    actions = torch.from_numpy(
        np.vstack([experience.action for experience in experiences if experience is not None])).long().to(device)
    rewards = torch.from_numpy(
        np.vstack([experience.reward for experience in experiences if experience is not None])).float().to(device)
    next_states = torch.from_numpy(
        np.vstack([experience.next_state for experience in experiences if experience is not None])).float().to(
        device)
    # Convert done from boolean to int
    dones = torch.from_numpy(
        np.vstack([experience.done for experience in experiences if experience is not None]).astype(
            np.uint8)).float().to(device)

    return (states, actions, rewards, next_states, dones)

def __len__(self):
    return len(self.memory)
</code></pre>
<p>BUFFER_SIZE = int(1e5)  # Replay memory size<br>
BATCH_SIZE = 64  # Number of experiences to sample from memory<br>
GAMMA = 0.99  # Discount factor<br>
TAU = 1e-3  # Soft update parameter for updating fixed q network<br>
LR = 1e-4  # Q Network learning rate<br>
UPDATE_EVERY = 4  # How often to update Q network</p>
<p>class DQNAgent:<br>
def <strong>init</strong>(self, state_size, action_size, seed, learningRate, decayRate, numOfEpisodes, stepsPerEpisode, epsilon,<br>
annealingConstant, annealAfter, checkpoint=’’):<br>
self.state_size = state_size<br>
self.action_size = action_size<br>
random.seed(seed)<br>
# Initialize Q and Fixed Q networks<br>
self.q_network = QNetwork(state_size, action_size, seed).to(device)<br>
self.fixed_network = QNetwork(state_size, action_size, seed).to(device)<br>
self.optimizer = optim.Adam(self.q_network.parameters())<br>
# Initialise memory<br>
self.memory = ReplayBuffer(BUFFER_SIZE, BATCH_SIZE, seed)<br>
self.timestep = 0<br>
self.voltageRanges_2 = [’&lt;0.85’, ‘0.85-0.874’, ‘0.875-0.899’, ‘0.9-0.924’, ‘0.925-0-949’, ‘0.95-0.974’,<br>
‘0.975.0.999’, ‘1-1.024’, ‘1.025-1.049’, ‘1.05-1.074’, ‘1.075-1.1’, ‘&gt;=1.1’]<br>
self.loadingPercentRange = [‘0-9’, ‘10-19’, ‘20-29’, ‘30-39’, ‘40-49’, ‘50-59’, ‘60-69’, ‘70-79’, ‘80-89’,<br>
‘90-99’, ‘100-109’, ‘110-119’, ‘120-129’, ‘130-139’, ‘140-149’, ‘150 and above’]<br>
self.statesLev1 = [‘v_’ + x + ‘<em>l</em>’ + y for x in self.voltageRanges_2 for y in self.loadingPercentRange]<br>
self.states = [‘s1:’ + x + ‘;s2:’ + y + ‘;’ for x in self.statesLev1 for y in self.statesLev1]</p>
<pre><code>    # initialise environment
    self.env = case9()

    # Possible actions to take, combinations of v_ref into combinations of lp_ref
    self.actions = ['v_ref:' + str(x) + ';lp_ref:' + str(y) for x in self.env.actionSpace['v_ref_pu'] for y in
                    self.env.actionSpace['lp_ref']]

    # Check if pickle file with current hyperparams exist
    self.checkPointName = 'pickles_qlearning\pickled_q_table_lr' + str(learningRate) + 'dr' + str(
        decayRate) + 'noe' + str(numOfEpisodes) + 'spe' + str(stepsPerEpisode) + 'e' + str(epsilon) + 'ac' + str(
        annealingConstant) + 'aa' + str(annealAfter) + '.pkl'
    if os.path.isfile(self.checkPointName) or checkpoint != '':
        print('loading data from checkpoint')
        # Load Qtable from pickle file
        with open(self.checkPointName if os.path.isfile(self.checkPointName) else checkpoint, 'rb') as pickle_file:
            data = pickle.load(pickle_file)
            self.epsilon = data['e'] if os.path.isfile(self.checkPointName) else epsilon
            self.q_table = data['q_table']
            self.allRewards = data['allRewards'] if os.path.isfile(self.checkPointName) else []
    else:  # create new Q_table
        self.q_table = pd.DataFrame(0, index=np.arange(len(self.actions)), columns=self.states)
        self.epsilon = epsilon
        self.allRewards = []
    self.numOfEpisodes = numOfEpisodes
    self.annealingRate = annealingConstant
    self.numOfSteps = stepsPerEpisode
    self.learningRate = learningRate
    self.decayRate = decayRate
    self.annealAfter = annealAfter

def step(self, state, action, reward, next_state, done):

    self.memory.add(state, action, reward, next_state, done)
    self.timestep += 1
    if self.timestep % UPDATE_EVERY == 0:
        if len(self.memory) &gt; BATCH_SIZE:
            sampled_experiences = self.memory.sample()
            self.learn(sampled_experiences)

def learn(self, experiences):

    states, actions, rewards, next_states, dones = experiences
    action_values = self.fixed_network(next_states).detach()
    max_action_values = action_values.max(1)[0].unsqueeze(1)

    # If done just use reward, else update Q_target with discounted action values
    Q_target = rewards + (GAMMA * max_action_values * (1 - dones))
    Q_expected = self.q_network(states).gather(1, actions)

    # Calculate loss
    loss = F.mse_loss(Q_expected, Q_target)
    self.optimizer.zero_grad()
    # backward pass
    loss.backward()
    # update weights
    self.optimizer.step()

    # Update fixed weights
    self.update_fixed_network(self.q_network, self.fixed_network)

def update_fixed_network(self, q_network, fixed_network):
    for source_parameters, target_parameters in zip(q_network.parameters(), fixed_network.parameters()):
        target_parameters.data.copy_(TAU * source_parameters.data + (1.0 - TAU) * target_parameters.data)

def act(self, state, eps=0.0):

    rnd = random.random()
    if rnd &lt; eps:
        return np.random.randint(self.action_size)
    else:
        state = torch.from_numpy(state).float().unsqueeze(0).to(device)
        # set the network into evaluation mode
        self.q_network.eval()
        with torch.no_grad():
            action_values = self.q_network(state)
        # Back to training mode
        self.q_network.train()
        action = np.argmax(action_values.cpu().data.numpy())
        return action

def checkpoint(self, filename):
    torch.save(self.q_network.state_dict(), filename)
</code></pre>
<p>MAX_EPISODES = 2000<br>
MAX_STEPS = 1000<br>
ENV_SOLVED = 200<br>
PRINT_EVERY = 100</p>
<h1>
<a name="epsilon-schedule-1" class="anchor" href="#epsilon-schedule-1"></a>Epsilon schedule</h1>
<p>EPS_START = 1.0<br>
EPS_DECAY = 0.999<br>
EPS_MIN = 0.01</p>
<p>EPS_DECAY_RATES = [0.9, 0.99, 0.999, 0.9999]<br>
plt.figure(figsize=(10, 6))</p>
<p>for decay_rate in EPS_DECAY_RATES:<br>
test_eps = EPS_START<br>
eps_list = []<br>
for _ in range(MAX_EPISODES):<br>
test_eps = max(test_eps * decay_rate, EPS_MIN)<br>
eps_list.append(test_eps)</p>
<pre><code>plt.plot(eps_list, label='decay rate: {}'.format(decay_rate))
</code></pre>
<p>plt.title(‘Effect of various decay rates’)<br>
plt.legend(loc=‘best’)<br>
plt.xlabel(‘Number of episodes’)<br>
plt.ylabel(‘epsilon’)<br>
plt.show()</p>
<h1>
<a name="get-state-and-action-sizes-2" class="anchor" href="#get-state-and-action-sizes-2"></a>Get state and action sizes</h1>
<p>state_size = env.observation_space<br>
action_size = env.action_space<br>
print(‘State size: {}, action size: {}’.format(state_size, action_size))</p>
<p>State_size: 8<br>
action_size: 4</p>
<p>dqn_agent = DQNAgent(8, 4, seed=0, numOfEpisodes=50000, stepsPerEpisode=12, learningRate=0.1,<br>
decayRate=0.1, epsilon=1, annealingConstant=0.98, annealAfter=400, checkpoint=’’)</p>
<p>start = time()<br>
scores = []</p>
<h1>
<a name="maintain-a-list-of-last-100-scores-3" class="anchor" href="#maintain-a-list-of-last-100-scores-3"></a>Maintain a list of last 100 scores</h1>
<p>scores_window = deque(maxlen=100)<br>
eps = EPS_START<br>
for episode in range(1, MAX_EPISODES + 1):<br>
state = env.reset<br>
score = 0<br>
for t in range(MAX_STEPS):<br>
action = dqn_agent.act(state, eps)<br>
next_state, reward, done, info = env.step(action)<br>
dqn_agent.step(state, action, reward, next_state, done)<br>
state = next_state<br>
score += reward<br>
if done:<br>
break</p>
<pre><code>    eps = max(eps * EPS_DECAY, EPS_MIN)
    if episode % PRINT_EVERY == 0:
        mean_score = np.mean(scores_window)
        print('\r Progress {}/{}, average score:{:.2f}'.format(episode, MAX_EPISODES, mean_score), end="")
    if score &gt;= ENV_SOLVED:
        mean_score = np.mean(scores_window)
        print('\rEnvironment solved in {} episodes, average score: {:.2f}'.format(episode, mean_score), end="")
        sys.stdout.flush()
        dqn_agent.checkpoint('solved_200.pth')
        break

scores_window.append(score)
scores.append(score)
</code></pre>
<p>end = time()<br>
print(‘Took {} seconds’.format(end - start))</p>
<p>Progress: 2000 / 2000<br>
average_score: 205.24<br>
dqn_agent.checkpoint(‘solved_200.pth’)</p>
<p>plt.figure(figsize=(10, 6))<br>
plt.plot(scores)</p>
<h1>
<a name="a-bit-hard-to-see-the-above-plot-so-lets-smooth-it-red-4" class="anchor" href="#a-bit-hard-to-see-the-above-plot-so-lets-smooth-it-red-4"></a>A bit hard to see the above plot, so lets smooth it (red)</h1>
<p>plt.plot(pd.Series(scores).rolling(100).mean())<br>
plt.title(‘DQN Training’)<br>
plt.xlabel(’# of episodes’)<br>
plt.ylabel(‘score’)<br>
plt.show()</p>

ptrblck: <p>Based on the error message you are trying to unpack an <code>int</code>, which doesn’t work:</p>
<pre><code class="lang-python">def fun():
    return 1

a, b = fun()
&gt; TypeError: cannot unpack non-iterable int object
</code></pre>
<p>However, your code also doesn’t show any <code>return</code> statement in <code>step()</code>, so I’m unsure if you’ve changed the code already.<br>
In any case, a missing <code>return</code> statement would return a <code>None</code> object by default, so the error should say:</p>
<pre><code class="lang-python">TypeError: cannot unpack non-iterable NoneType object
</code></pre>

amahoro12: <p>thank you a lot for help. I will give you the feedback.</p>