我正在尝试按照"开始使用rlib"(https://docs.ray.io/en/latest/rllib/rllib-training.html)以及我的自定义环境。
然而，我的运行不工作在第一个代码块显示在指南中。这实际上是我要运行的脚本:
import ray
from ray.rllib.algorithms.ppo import PPOConfig
from ray.tune.logger import pretty_print
from gym_sw_env.envs.Examplev2 import Example_v2 #this is my custom env
ray.init(ignore_reinit_error=True)
algo = (
PPOConfig()
.rollouts(num_rollout_workers=1)
.resources(num_gpus=0)
.environment(env=Example_v2)
.build()
)
虽然这是错误，我有:
(RolloutWorker pid=24420) 2022-12-17 11:36:34,235 ERROR worker.py:763 -- Exception raised in creation task: The actor died because of an error raised in its creation task, ray::RolloutWorker.__init__() (pid=24420, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x000001693BEB1C10>)
(RolloutWorker pid=24420) File "pythonray_raylet.pyx", line 859, in ray._raylet.execute_task
(RolloutWorker pid=24420) File "pythonray_raylet.pyx", line 863, in ray._raylet.execute_task
(RolloutWorker pid=24420) File "pythonray_raylet.pyx", line 810, in ray._raylet.execute_task.function_executor
(RolloutWorker pid=24420) File "C:Users**MYUSER**Anaconda3libsite-packagesray_privatefunction_manager.py", line 674, in actor_method_executor
(RolloutWorker pid=24420) return method(__ray_actor, *args, **kwargs)
(RolloutWorker pid=24420) File "C:Users**MYUSER**Anaconda3libsite-packagesrayutiltracingtracing_helper.py", line 466, in _resume_span
(RolloutWorker pid=24420) return method(self, *_args, **_kwargs)
(RolloutWorker pid=24420) File "C:Users**MYUSER**Anaconda3libsite-packagesrayrllibevaluationrollout_worker.py", line 492, in __init__
(RolloutWorker pid=24420) self.env = env_creator(copy.deepcopy(self.env_context))
(RolloutWorker pid=24420) File "C:Users**MYUSER**Anaconda3libsite-packagesrayrllibalgorithmsalgorithm.py", line 2139, in <lambda>
(RolloutWorker pid=24420) return env_id, lambda cfg: env_specifier(cfg)
(RolloutWorker pid=24420) TypeError: __init__() takes 1 positional argument but 2 were given
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In [3], line 7
2 from ray.tune.logger import pretty_print
3 from gym_sw_env.envs.Examplev2 import Example_v2
6 algo = (
----> 7 PPOConfig()
8 .rollouts(num_rollout_workers=1)
9 .resources(num_gpus=0)
10 .environment(env=Example_v2)
11 .build()
12 )
File ~Anaconda3libsite-packagesrayrllibalgorithmsalgorithm_config.py:311, in AlgorithmConfig.build(self, env, logger_creator)
308 if logger_creator is not None:
309 self.logger_creator = logger_creator
--> 311 return self.algo_class(
312 config=self.to_dict(),
313 env=self.env,
314 logger_creator=self.logger_creator,
315 )
File ~Anaconda3libsite-packagesrayrllibalgorithmsalgorithm.py:414, in Algorithm.__init__(self, config, env, logger_creator, **kwargs)
402 # Initialize common evaluation_metrics to nan, before they become
403 # available. We want to make sure the metrics are always present
404 # (although their values may be nan), so that Tune does not complain
405 # when we use these as stopping criteria.
406 self.evaluation_metrics = {
407 "evaluation": {
408 "episode_reward_max": np.nan,
(...)
411 }
412 }
--> 414 super().__init__(config=config, logger_creator=logger_creator, **kwargs)
416 # Check, whether `training_iteration` is still a tune.Trainable property
417 # and has not been overridden by the user in the attempt to implement the
418 # algos logic (this should be done now inside `training_step`).
419 try:
File ~Anaconda3libsite-packagesraytunetrainabletrainable.py:161, in Trainable.__init__(self, config, logger_creator, remote_checkpoint_dir, custom_syncer, sync_timeout)
159 start_time = time.time()
160 self._local_ip = ray.util.get_node_ip_address()
--> 161 self.setup(copy.deepcopy(self.config))
162 setup_time = time.time() - start_time
163 if setup_time > SETUP_TIME_THRESHOLD:
File ~Anaconda3libsite-packagesrayrllibalgorithmsalgorithm.py:549, in Algorithm.setup(self, config)
536 except RayActorError as e:
537 # In case of an actor (remote worker) init failure, the remote worker
538 # may still exist and will be accessible, however, e.g. calling
539 # its `sample.remote()` would result in strange "property not found"
540 # errors.
541 if e.actor_init_failed:
542 # Raise the original error here that the RolloutWorker raised
543 # during its construction process. This is to enforce transparency
(...)
547 # - e.args[0].args[2]: The original Exception (e.g. a ValueError due
548 # to a config mismatch) thrown inside the actor.
--> 549 raise e.args[0].args[2]
550 # In any other case, raise the RayActorError as-is.
551 else:
552 raise e
File pythonray_raylet.pyx:852, in ray._raylet.execute_task()
File pythonray_raylet.pyx:906, in ray._raylet.execute_task()
File pythonray_raylet.pyx:859, in ray._raylet.execute_task()
File pythonray_raylet.pyx:863, in ray._raylet.execute_task()
File pythonray_raylet.pyx:810, in ray._raylet.execute_task.function_executor()
File ~Anaconda3libsite-packagesray_privatefunction_manager.py:674, in actor_method_executor()
672 return method(*args, **kwargs)
673 else:
--> 674 return method(__ray_actor, *args, **kwargs)
File ~Anaconda3libsite-packagesrayutiltracingtracing_helper.py:466, in _resume_span()
464 # If tracing feature flag is not on, perform a no-op
465 if not _is_tracing_enabled() or _ray_trace_ctx is None:
--> 466 return method(self, *_args, **_kwargs)
468 tracer: _opentelemetry.trace.Tracer = _opentelemetry.trace.get_tracer(
469 __name__
470 )
472 # Retrieves the context from the _ray_trace_ctx dictionary we
473 # injected.
File ~Anaconda3libsite-packagesrayrllibevaluationrollout_worker.py:492, in __init__()
485 # Create a (single) env for this worker.
486 if not (
487 worker_index == 0
488 and num_workers > 0
489 and not policy_config.get("create_env_on_driver")
490 ):
491 # Run the `env_creator` function passing the EnvContext.
--> 492 self.env = env_creator(copy.deepcopy(self.env_context))
494 if self.env is not None:
495 # Validate environment (general validation function).
496 if not self._disable_env_checking:
File ~Anaconda3libsite-packagesrayrllibalgorithmsalgorithm.py:2139, in Algorithm._get_env_id_and_creator.<locals>.<lambda>()
2137 return env_id, lambda cfg: _wrapper.remote(cfg)
2138 else:
-> 2139 return env_id, lambda cfg: env_specifier(cfg)
2141 # No env -> Env creator always returns None.
2142 elif env_specifier is None:
TypeError: __init__() takes 1 positional argument but 2 were given
有人知道怎么解决吗?我只是按照指南的第一步来做。
我只是试图删除构建()方法的poconfigure，所以我有:
algo = (
PPOConfig()
.rollouts(num_rollout_workers=1)
.resources(num_gpus=0)
.environment(env=Example_v2)
)
这不会产生任何错误。所以问题可能是:build()方法是必要的吗?
我正在搜索其他类似的错误，但是没有找到。
您的自定义环境必须注册到Ray注册表中，之后您可以将名称作为字符串传递。
From the docs:
from ray.tune.registry import register_env
def env_creator(env_config):
return MyEnv(...) # return an env instance
register_env("my_env", env_creator)
algo = ppo.PPO(env="my_env")
或者，您可以传递您的自定义类名，但是它必须(仅)接受一个env_config参数进行初始化。同样，从文档:
import gym, ray
from ray.rllib.algorithms import ppo
class MyEnv(gym.Env):
def __init__(self, env_config):
self.action_space = <gym.Space>
self.observation_space = <gym.Space>
def reset(self):
return <obs>
def step(self, action):
return <obs>, <reward: float>, <done: bool>, <info: dict>
ray.init()
algo = ppo.PPO(env=MyEnv, config={
"env_config": {}, # config to pass to env class
})
while True:
print(algo.train())