raise TuneError("Trials did not complete", incomplete_trials)

[SExpert12]

Hi,
I have checked all the programs for training four_way and I got the same error like this:

2024-07-08 09:55:13.843116: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'libnvinfer.so.6'; dlerror: libnvinfer.so.6: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/cv2/../../lib64:
2024-07-08 09:55:13.843231: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'libnvinfer_plugin.so.6'; dlerror: libnvinfer_plugin.so.6: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/cv2/../../lib64:
2024-07-08 09:55:13.843246: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:30] Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
/home/ryzen/carla_out --------------------------------------------
/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/gym/utils/passive_env_checker.py:21: UserWarning: WARN: It seems a Box observation space is an image but the dtype is not np.uint8, actual type: float32. If the Box observation space is not an image, we recommend flattening the observation to have only a 1D vector.
f"It seems a Box observation space is an image but the dtype is not np.uint8, actual type: {observation_space.dtype}. "
2024-07-08 09:55:14,553 INFO resource_spec.py:212 -- Starting Ray with 16.31 GiB memory available for workers and up to 8.16 GiB for objects. You can adjust these settings with ray.init(memory=, object_store_memory=).
2024-07-08 09:55:14,907 INFO services.py:1148 -- View the Ray dashboard at localhost:8265
'/home/aizaz/ray_results/MA-Inde-TD3-SSUI3CCARLA/DDPG_UrbanSignalIntersection3CarTD3-v0_0_2022-04-23_14-54-070ev3hbyd/checkpoint_55/checkpoint-55'
False
2024-07-08 09:55:15,090 WARNING sample.py:27 -- DeprecationWarning: wrapping <function at 0x7f706efef6a8> with tune.function() is no longer needed
== Status ==
Memory usage on this node: 4.8/31.2 GiB
PopulationBasedTraining: 0 checkpoints, 0 perturbs
Resources requested: 2/16 CPUs, 0/1 GPUs, 0.0/16.31 GiB heap, 0.0/5.62 GiB objects
Result logdir: /home/ray_results/TD3_Four_way
Number of trials: 1 (1 RUNNING)
+----------------------------------+----------+-------+
| Trial name | status | loc |
|----------------------------------+----------+-------|
| DDPG_TD3_four_way_train-v0_00000 | RUNNING | |
+----------------------------------+----------+-------+

2024-07-08 09:55:16,671 ERROR trial_runner.py:521 -- Trial DDPG_TD3_four_way_train-v0_00000: Error processing event.
Traceback (most recent call last):
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 467, in _process_trial
result = self.trial_executor.fetch_result(trial)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 381, in fetch_result
result = ray.get(trial_future[0], DEFAULT_GET_TIMEOUT)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/worker.py", line 1513, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(ModuleNotFoundError): ray::DDPG.init() (pid=122566, ip=192.168.15.93)
File "python/ray/_raylet.pyx", line 414, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 449, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 450, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 452, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 407, in ray._raylet.execute_task.function_executor
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/rllib/agents/trainer_template.py", line 90, in init
Trainer.init(self, config, env, logger_creator)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/rllib/agents/trainer.py", line 455, in init
super().init(config, logger_creator)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/trainable.py", line 174, in init
self._setup(copy.deepcopy(self.config))
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/rllib/agents/trainer.py", line 554, in _setup
self.env_creator = _global_registry.get(ENV_CREATOR, env)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/registry.py", line 130, in get
return pickle.loads(value)
ModuleNotFoundError: No module named 'macad_agents'
== Status ==
Memory usage on this node: 5.3/31.2 GiB
PopulationBasedTraining: 0 checkpoints, 0 perturbs
Resources requested: 0/16 CPUs, 0/1 GPUs, 0.0/16.31 GiB heap, 0.0/5.62 GiB objects
Result logdir: /home/ryzen/ray_results/TD3_Four_way
Number of trials: 1 (1 ERROR)
+----------------------------------+----------+-------+
| Trial name | status | loc |
|----------------------------------+----------+-------|
| DDPG_TD3_four_way_train-v0_00000 | ERROR | |
+----------------------------------+----------+-------+
Number of errored trials: 1
+----------------------------------+--------------+---------------------------------------------------------------------------------------------------------+
| Trial name | # failures | error file |
|----------------------------------+--------------+---------------------------------------------------------------------------------------------------------|
| DDPG_TD3_four_way_train-v0_00000 | 1 | /home/ryzen/ray_results/TD3_Four_way/DDPG_TD3_four_way_train-v0_0_2024-07-08_09-55-15uy5vxgzc/error.txt |
+----------------------------------+--------------+---------------------------------------------------------------------------------------------------------+

Traceback (most recent call last):
File "Train_TD3_four_way.py", line 570, in
checkpoint_at_end = True,
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/tune.py", line 342, in run
raise TuneError("Trials did not complete", incomplete_trials)
ray.tune.error.TuneError: ('Trials did not complete', [DDPG_TD3_four_way_train-v0_00000])

How to resolve this now?

[SExpert12]

Hi,
I check the error file generated by all algorithm and the error is:

Failure # 1 (occurred at 2024-07-08_09-51-51)
Traceback (most recent call last):
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 467, in _process_trial
result = self.trial_executor.fetch_result(trial)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 381, in fetch_result
result = ray.get(trial_future[0], DEFAULT_GET_TIMEOUT)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/worker.py", line 1513, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(ModuleNotFoundError): �[36mray::DDPG.init()�[39m (pid=121283, ip=192.168.15.93)
File "python/ray/_raylet.pyx", line 414, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 449, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 450, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 452, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 407, in ray._raylet.execute_task.function_executor
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/rllib/agents/trainer_template.py", line 90, in init
Trainer.init(self, config, env, logger_creator)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/rllib/agents/trainer.py", line 455, in init
super().init(config, logger_creator)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/trainable.py", line 174, in init
self._setup(copy.deepcopy(self.config))
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/rllib/agents/trainer.py", line 554, in _setup
self.env_creator = _global_registry.get(ENV_CREATOR, env)
File "/home/miniconda3/envs/macad-gym-benchmarking/lib/python3.7/site-packages/ray/tune/registry.py", line 130, in get
return pickle.loads(value)
ModuleNotFoundError: No module named 'macad_agents'

What is the solution of this?