我正在尝试使用TFAgents包学习自定义环境。我遵循Hands-on-ML书(Code in colab，见cell 129)。我的目标是在定制的网格世界环境中使用DQN代理。
Grid-World环境:
class MyEnvironment(tf_agents.environments.py_environment.PyEnvironment):
def __init__(self, discount=1.0):
super().__init__()
self.discount = discount
self._action_spec = tf_agents.specs.BoundedArraySpec(shape=(), dtype=np.int32, name="action", minimum=0, maximum=3)
self._observation_spec = tf_agents.specs.BoundedArraySpec(shape=(4, 4), dtype=np.int32, name="observation", minimum=0, maximum=1)
def action_spec(self):
return self._action_spec
def observation_spec(self):
return self._observation_spec
def _reset(self):
self._state = np.zeros(2, dtype=np.int32)
obs = np.zeros((4, 4), dtype=np.int32)
obs[self._state[0], self._state[1]] = 1
return tf_agents.trajectories.time_step.restart(obs)
def _step(self, action):
self._state += [(-1, 0), (+1, 0), (0, -1), (0, +1)][action]
reward = 0
obs = np.zeros((4, 4), dtype=np.int32)
done = (self._state.min() < 0 or self._state.max() > 3)
if not done:
obs[self._state[0], self._state[1]] = 1
if done or np.all(self._state == np.array([3, 3])):
reward = -1 if done else +10
return tf_agents.trajectories.time_step.termination(obs, reward)
else:
return tf_agents.trajectories.time_step.transition(obs, reward, self.discount)
， Q网络为:
tf_env = MyEnvironment()
preprocessing_layer = keras.layers.Lambda(lambda obs: tf.cast(obs, np.float32) / 255.)
conv_layer_params=[(32, (2, 2), 1)]
fc_layer_params=[512]
q_net = QNetwork(
tf_env.observation_spec(),
tf_env.action_spec(),
preprocessing_layers=preprocessing_layer,
conv_layer_params=conv_layer_params,
fc_layer_params=fc_layer_params)
最后，DQN代理
train_step = tf.Variable(0)
update_period = 4 # train the model every 4 steps
optimizer = keras.optimizers.RMSprop(lr=2.5e-4, rho=0.95, momentum=0.0, epsilon=0.00001, centered=True)
epsilon_fn = keras.optimizers.schedules.PolynomialDecay(initial_learning_rate=1.0, decay_steps=250000 // update_period, end_learning_rate=0.01)
agent = DqnAgent(tf_env.time_step_spec(),
tf_env.action_spec(),
q_network=q_net,
optimizer=optimizer,
target_update_period=2000, # <=> 32,000 ALE frames
td_errors_loss_fn=keras.losses.Huber(reduction="none"),
gamma=0.99, # discount factor
train_step_counter=train_step,
epsilon_greedy=lambda: epsilon_fn(train_step))
agent.initialize()
直接运行代码会得到以下错误跟踪:
/usr/local/lib/python3.6/dist-packages/gin/config.py in gin_wrapper(*args， **kwargs)1067 scope_info = "在作用域'{}'".format(scope_str)如果scope_str else "1068 err_str = err_str. 1068格式(name, fn_or_cls, scope_info)→1069跑龙套。augment_exception_message_and_reraise (e, err_str)10701071 return gin_wrapper
/usr/local/lib/python3.6/dist-packages/gin/utils.py in augment_exception_message_and_reraise(exception, message)
39 proxy = ExceptionProxy()
40 ExceptionProxy.__qualname__ = type(exception).__qualname__
---> 41 raise proxy.with_traceback(exception.__traceback__) from None
42
43
/usr/local/lib/python3.6/dist-packages/gin/config.py in gin_wrapper(*args, **kwargs)
1044
1045 try:
-> 1046 return fn(*new_args, **new_kwargs)
1047 except Exception as e: # pylint: disable=broad-except
1048 err_str = ''
/usr/local/lib/python3.6/dist-packages/tf_agents/agents/dqn/dqn_agent.py in __init__(self, time_step_spec, action_spec, q_network, optimizer, observation_and_action_constraint_splitter, epsilon_greedy, n_step_update, boltzmann_temperature, emit_log_probability, target_q_network, target_update_tau, target_update_period, td_errors_loss_fn, gamma, reward_scale_factor, gradient_clipping, debug_summaries, summarize_grads_and_vars, train_step_counter, name)
216 tf.Module.__init__(self, name=name)
217
--> 218 self._check_action_spec(action_spec)
219
220 if epsilon_greedy is not None and boltzmann_temperature is not None:
/usr/local/lib/python3.6/dist-packages/tf_agents/agents/dqn/dqn_agent.py in _check_action_spec(self, action_spec)
293
294 # TODO(oars): Get DQN working with more than one dim in the actions.
--> 295 if len(flat_action_spec) > 1 or flat_action_spec[0].shape.rank > 0:
296 raise ValueError(
297 'Only scalar actions are supported now, but action spec is: {}'
AttributeError: 'tuple' object has no attribute 'rank'
In call to configurable 'DqnAgent' (<class 'tf_agents.agents.dqn.dqn_agent.DqnAgent'>)
我试过了:以下是修改后的
self._action_spec = tf_agents.specs.BoundedArraySpec(shape=(), dtype=np.int32, name="action", minimum=0, maximum=3)
self._observation_spec = tf_agents.specs.BoundedArraySpec(shape=(4, 4), dtype=np.int32, name="observation", minimum=0, maximum=1)
:
self._action_spec = tf_agents.specs.BoundedTensorSpec(
shape=(), dtype=np.int32, name="action", minimum=0, maximum=3)
self._observation_spec = tf_agents.specs.BoundedTensorSpec(
shape=(4, 4), dtype=np.int32, name="observation", minimum=0, maximum=1)
然而，这导致了:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-53-ce737b2b13fd> in <module>()
21
22
---> 23 agent = DqnAgent(tf_env.time_step_spec(),
24 tf_env.action_spec(),
25 q_network=q_net,
1 frames
/usr/local/lib/python3.6/dist-packages/tf_agents/environments/py_environment.py in time_step_spec(self)
147 the step_type, reward, discount, and observation structure.
148 """
--> 149 return ts.time_step_spec(self.observation_spec(), self.reward_spec())
150
151 def current_time_step(self) -> ts.TimeStep:
/usr/local/lib/python3.6/dist-packages/tf_agents/trajectories/time_step.py in time_step_spec(observation_spec, reward_spec)
388 'Expected observation and reward specs to both be either tensor or '
389 'array specs, but saw spec values {} vs. {}'
--> 390 .format(first_observation_spec, first_reward_spec))
391 if isinstance(first_observation_spec, tf.TypeSpec):
392 return TimeStep(
TypeError: Expected observation and reward specs to both be either tensor or array specs, but saw spec values BoundedTensorSpec(shape=(4, 4), dtype=tf.int32, name='observation', minimum=array(0, dtype=int32), maximum=array(1, dtype=int32)) vs. ArraySpec(shape=(), dtype=dtype('float32'), name='reward')
我理解奖励是问题所在:所以，添加了额外的行
self._reward_spec = tf_agents.specs.TensorSpec((1,), np.dtype('float32'), 'reward')
，但仍然导致相同的错误。有什么办法可以解决这个问题吗?
您不能将TensorSpec与PyEnvironment类对象一起使用，这就是为什么您尝试的解决方案不起作用。一个简单的修复方法应该是使用原始代码
self._action_spec = tf_agents.specs.BoundedArraySpec(shape=(), dtype=np.int32, name="action", minimum=0, maximum=3)
self._observation_spec = tf_agents.specs.BoundedArraySpec(shape=(4, 4), dtype=np.int32, name="observation", minimum=0, maximum=1)
然后像这样包装环境:
env= MyEnvironment()
tf_env = tf_agents.environments.tf_py_environment.TFPyEnvironment(env)
这是最简单的事情。或者，您可以将环境定义为TFEnvironment类对象，使用TensorSpec并更改所有环境代码以操作张量。我不建议初学者这样做。