python 强化学习报错：
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In [3], line 1
----> 1 returns = train(train_env, eval_env)

File /tf/notebooks/reinforcement/train.py:40, in train(train_env, eval_env, num_iterations, collect_steps_per_iteration, log_interval, eval_interval)
     30 def train(train_env, eval_env, num_iterations=20000, collect_steps_per_iteration=1,
     31           log_interval=100, eval_interval=100):
     32     # 主训练循环
   (...)
     35     # log_interval 每隔1000步打印日志
     36     # eval_interval 每隔1000步评估模型的性能
     38     print("开始训练")
---> 40     agent, iterator, replay_buffer = get_net(train_env)
     42     # 初始评估
     43     avg_return = compute_avg_return(eval_env, agent.policy, 10)

File /tf/notebooks/reinforcement/learn.py:110, in get_net(train_env)
    107 train_step_counter = tf.Variable(0)
    109 # 初始化DQN代理 (你可能需要修改参数，以适应你的具体环境)
--> 110 agent = dqn_agent.DqnAgent(
    111     time_step_spec,
    112     action_spec,
    113     q_network=lstm_q_net,
    114     optimizer=optimizer,
    115     td_errors_loss_fn=common.element_wise_squared_loss,
    116     train_step_counter=train_step_counter,
    117     name="dqn_agent"
    118 )
    120 agent.initialize()
    122 # 初始化Replay Buffer

File /usr/local/lib/python3.8/dist-packages/gin/config.py:1605, in _make_gin_wrapper.<locals>.gin_wrapper(*args, **kwargs)
   1603 scope_info = " in scope '{}'".format(scope_str) if scope_str else ''
   1604 err_str = err_str.format(name, fn_or_cls, scope_info)
-> 1605 utils.augment_exception_message_and_reraise(e, err_str)

File /usr/local/lib/python3.8/dist-packages/gin/utils.py:41, in augment_exception_message_and_reraise(exception, message)
     39 proxy = ExceptionProxy()
     40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None

File /usr/local/lib/python3.8/dist-packages/gin/config.py:

——————
完整代码：
def get_env(train_stock_pool_id, val_stock_pool_id):
    print(f"请求 train pool: {train_stock_pool_id}")
    stock_data_for_training = stock_pool_prices(train_stock_pool_id)
    print(f"请求 val pool: {val_stock_pool_id}")
    stock_data_for_eval = stock_pool_prices(val_stock_pool_id)
    print("数据请求完毕，准备对应股票的环境列表")
    # 准备对应股票的环境列表
    train_py_envs = [StockTradingEnv(data) for data in stock_data_for_training]
    eval_py_envs = [StockTradingEnv(data) for data in stock_data_for_eval]
    print("股票环境列表创建完毕，创建批量训练和评估环境")
    # 创建批量训练和评估环境
    train_env = tf_py_environment.TFPyEnvironment(batched_py_environment.BatchedPyEnvironment(train_py_envs))
    eval_env = tf_py_environment.TFPyEnvironment(batched_py_environment.BatchedPyEnvironment(eval_py_envs))
    print("确保所有环境是有效的")
    # validate_environments(train_py_envs)
    # validate_environments(eval_py_envs)
    print("环境创建完毕")
    return train_env, eval_env


class LstmQNetwork(network.Network):

    def __init__(self, input_tensor_spec, action_spec, lstm_units=(100,), fc_layer_params=(100,)):
        super(LstmQNetwork, self).__init__(
            input_tensor_spec=input_tensor_spec,
            state_spec=(),  # 你可以移除这一行，因为你不需要在这里定义 state_spec
        )

        # 可以保留一个输出Dense层的引用，但是需要将 LSTM 层分离出来
        self.dense_layers = [tf.keras.layers.Dense(num_units, activation='relu') for num_units in fc_layer_params]
        self.output_layer = tf.keras.layers.Dense(units=(action_spec.maximum - action_spec.minimum + 1))

        # 定义 LSTM 层，设置 return_sequences=False 来处理多个时间序列数据
        self.lstm_layer = tf.keras.layers.LSTM(units=lstm_units[0], return_sequences=False, return_state=True)

    def call(self, inputs, step_type, network_state=(), training=False):
        # 处理 LSTM 层和全连接层
        # LSTM 层返回的最终隐藏状态通常用于网络的连接决策，此外还返回了 cell 状态
        lstm_output, _state_h, _state_c = self.lstm_layer(inputs)

        x = lstm_output
        for dense_layer in self.dense_layers:
            x = dense_layer(x)
        output = self.output_layer(x)

        return output, network


class StockTradingEnv(py_environment.PyEnvironment):

    def __init__(self, stock_data, initial_cash=10000, start_step=400, history_length=50):
        # stock_data 应当为一个二维数组，每一行代表一个时间点（例如每日股票价格）
        # 假定收盘价在每行的第四列
        self._stock_data = stock_data
        self._history_length = history_length  # 添加了 history_length 来跟踪历史数据长度
        self._current_step = 0
        self._num_stocks_owned = 0
        self._cash_in_hand = initial_cash
        self._initial_cash = initial_cash
        self._start_step = start_step  # 添加一个新的参数来设定开始执行操作的时间步
        self._observation_spec = array_spec.ArraySpec(
            shape=(history_length, 4), dtype=np.float32, name='observation')

        # 动作空间：0-不操作，1-买入，2-卖出
        self._action_spec = array_spec.BoundedArraySpec(
            shape=(), dtype=np.int32, minimum=0, maximum=2, name='action')

        self._episode_ended = False

    def action_spec(self):
        return self._action_spec

    def observation_spec(self):
        return self._observation_spec

    def _step(self, action):
        # 检查是否是第一步
        if self._episode_ended:
            # 开始一个新的episode
            return self.reset()

        # 取得当日收盘价
        current_stock_price = self._stock_data[self._current_step][3]  # 假定第四列为收盘价

        if self._current_step > self._start_step:
            # 执行动作
            if action == 1:  # 买入
                # 只有当你手上的现金超过当前的股价时，你才能购买一股
                if self._cash_in_hand >= current_stock_price:
                    self._num_stocks_owned += 1
                    self._cash_in_hand -= current_stock_price
            elif action == 2 and self._num_stocks_owned > 0:  # 卖出
                self._num_stocks_owned -= 1
                self._cash_in_hand += current_stock_price

        # 更新当前步骤
        self._current_step += 1
        self._state = self._get_observation()

        if self._current_step >= len(self._stock_data) - 1:
            self._episode_ended = True

        # 你可以根据你的策略来设计奖励系统
        reward = self._num_stocks_owned * current_stock_price + self._cash_in_hand - self._initial_cash

        if self._episode_ended:
            return ts.termination(np.array(self._state, dtype=np.float32), reward)
        else:
            return ts.transition(
                np.array(self._state, dtype=np.float32), reward=reward, discount=1.0)

    def _reset(self):
        self._current_step = self._start_step
        self._num_stocks_owned = 0
        self._cash_in_hand = self._initial_cash
        self._episode_ended = False
        self._state = self._get_observation()
        return ts.restart(np.array(self._state, dtype=np.float32))  # 确保形状为 (history_length, 4)

    def _get_observation(self):
        # 如果当前步骤小于历史长度，我们用0填充前面的部分
        if self._current_step < self._history_length:
            # 创建一个形状为 (self._history_length, features) 的零填充数组
            history_padding = np.zeros((self._history_length, 4))
            # 从 stock_data 获取真实数据，可能还没有足够的历史数据
            real_data = self._stock_data[:self._current_step, :]
            # 合并零填充和真实数据
            observation = np.vstack((history_padding, real_data))
            # 确保我们只得到包含 history_length 步的观测
            observation = observation[-self._history_length:]
        else:
            # 当有足够的历史数据时，获取从 current_step-history_length 到 current_step 的切片
            observation = self._stock_data[self._current_step - self._history_length:self._current_step, :]

        return observation